/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.pig.builtin;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import org.apache.pig.Accumulator;
import org.apache.pig.Algebraic;
import org.apache.pig.EvalFunc;
import org.apache.pig.FuncSpec;
import org.apache.pig.PigException;
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.data.DataBag;
import org.apache.pig.data.DataByteArray;
import org.apache.pig.data.DataType;
import org.apache.pig.data.Tuple;
import org.apache.pig.data.TupleFactory;
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.logicalLayer.schema.Schema;
/**
* Generates the sum of a set of values. This class implements
* {@link org.apache.pig.Algebraic}, so if possible the execution will
* performed in a distributed fashion.
* <p>
* SUM can operate on any numeric type. It can also operate on bytearrays,
* which it will cast to doubles. It expects a bag of
* tuples of one record each. If Pig knows from the schema that this function
* will be passed a bag of integers or longs, it will use a specially adapted version of
* SUM that uses integer arithmetic for summing the data. The return type
* of SUM is double for float, double, or bytearray arguments and long for int
* or long arguments.
* <p>
* SUM implements the {@link org.apache.pig.Accumulator} interface as well.
* While this will never be
* the preferred method of usage it is available in case the combiner can not be
* used for a given calculation.
*/
public class SUM extends EvalFunc<Double> implements Algebraic, Accumulator<Double> {
@Override
public Double exec(Tuple input) throws IOException {
try {
return sum(input);
} catch (ExecException ee) {
throw ee;
} catch (Exception e) {
int errCode = 2106;
String msg = "Error while computing sum in " + this.getClass().getSimpleName();
throw new ExecException(msg, errCode, PigException.BUG, e);
}
}
public String getInitial() {
return Initial.class.getName();
}
public String getIntermed() {
return Intermediate.class.getName();
}
public String getFinal() {
return Final.class.getName();
}
static public class Initial extends EvalFunc<Tuple> {
private static TupleFactory tfact = TupleFactory.getInstance();
@Override
public Tuple exec(Tuple input) throws IOException {
// Initial is called in the map - for SUM
// we just send the tuple down
try {
// input is a bag with one tuple containing
// the column we are trying to sum
DataBag bg = (DataBag) input.get(0);
DataByteArray dba = null;
if(bg.iterator().hasNext()) {
Tuple tp = bg.iterator().next();
dba = (DataByteArray)tp.get(0);
}
return tfact.newTuple(dba != null?
Double.valueOf(dba.toString()): null);
}catch(NumberFormatException nfe){
// treat this particular input as null
Tuple t = tfact.newTuple(1);
t.set(0, null);
return t;
} catch (ExecException e) {
throw e;
} catch (Exception e) {
int errCode = 2106;
String msg = "Error while computing sum in " + this.getClass().getSimpleName();
throw new ExecException(msg, errCode, PigException.BUG, e);
}
}
}
static public class Intermediate extends EvalFunc<Tuple> {
private static TupleFactory tfact = TupleFactory.getInstance();
@Override
public Tuple exec(Tuple input) throws IOException {
try {
return tfact.newTuple(sumDoubles(input));
} catch (ExecException ee) {
throw ee;
} catch (Exception e) {
int errCode = 2106;
String msg = "Error while computing sum in " + this.getClass().getSimpleName();
throw new ExecException(msg, errCode, PigException.BUG, e);
}
}
}
static public class Final extends EvalFunc<Double> {
@Override
public Double exec(Tuple input) throws IOException {
try {
return sumDoubles(input);
} catch (ExecException ee) {
throw ee;
} catch (Exception e) {
int errCode = 2106;
String msg = "Error while computing sum in " + this.getClass().getSimpleName();
throw new ExecException(msg, errCode, PigException.BUG, e);
}
}
}
static protected Double sum(Tuple input) throws ExecException {
DataBag values = (DataBag)input.get(0);
// if we were handed an empty bag, return NULL
// this is in compliance with SQL standard
if(values.size() == 0) {
return null;
}
double sum = 0;
boolean sawNonNull = false;
for (Iterator<Tuple> it = values.iterator(); it.hasNext();) {
Tuple t = it.next();
try {
DataByteArray dba = (DataByteArray)t.get(0);
Double d =
dba != null ? Double.valueOf(dba.toString()): null;
if (d == null) continue;
sawNonNull = true;
sum += d;
}catch(RuntimeException exp) {
int errCode = 2103;
String msg = "Problem while computing sum of doubles.";
throw new ExecException(msg, errCode, PigException.BUG, exp);
}
}
if(sawNonNull) {
return new Double(sum);
} else {
return null;
}
}
// same as above function except all its inputs are
// always Double - this should be used for better performance
// since we don't have to check the type of the object to
// decide it is a double. This should be used when the initial,
// intermediate and final versions are used.
static protected Double sumDoubles(Tuple input) throws ExecException {
DataBag values = (DataBag)input.get(0);
// if we were handed an empty bag, return NULL
// this is in compliance with SQL standard
if(values.size() == 0) {
return null;
}
double sum = 0;
boolean sawNonNull = false;
for (Iterator<Tuple> it = values.iterator(); it.hasNext();) {
Tuple t = it.next();
try {
// we can cast directly because we SHOULD
// only be getting Doubles here
Double d = (Double)(t.get(0));
if (d == null) continue;
sawNonNull = true;
sum += d;
}catch(RuntimeException exp) {
int errCode = 2103;
String msg = "Problem while computing sum of doubles.";
throw new ExecException(msg, errCode, PigException.BUG, exp);
}
}
if(sawNonNull) {
return new Double(sum);
} else {
return null;
}
}
@Override
public Schema outputSchema(Schema input) {
return new Schema(new Schema.FieldSchema(null, DataType.DOUBLE));
}
/* (non-Javadoc)
* @see org.apache.pig.EvalFunc#getArgToFuncMapping()
*/
@Override
public List<FuncSpec> getArgToFuncMapping() throws FrontendException {
List<FuncSpec> funcList = new ArrayList<FuncSpec>();
funcList.add(new FuncSpec(this.getClass().getName(), Schema.generateNestedSchema(DataType.BAG, DataType.BYTEARRAY)));
funcList.add(new FuncSpec(DoubleSum.class.getName(), Schema.generateNestedSchema(DataType.BAG, DataType.DOUBLE)));
funcList.add(new FuncSpec(FloatSum.class.getName(), Schema.generateNestedSchema(DataType.BAG, DataType.FLOAT)));
funcList.add(new FuncSpec(IntSum.class.getName(), Schema.generateNestedSchema(DataType.BAG, DataType.INTEGER)));
funcList.add(new FuncSpec(LongSum.class.getName(), Schema.generateNestedSchema(DataType.BAG, DataType.LONG)));
return funcList;
}
/* Accumulator interface implementation*/
private Double intermediateSum = null;
@Override
public void accumulate(Tuple b) throws IOException {
try {
Double curSum = sum(b);
if (curSum == null) {
return;
}
intermediateSum = (intermediateSum == null ? 0.0 : intermediateSum) + curSum;
} catch (ExecException ee) {
throw ee;
} catch (Exception e) {
int errCode = 2106;
String msg = "Error while computing sum in " + this.getClass().getSimpleName();
throw new ExecException(msg, errCode, PigException.BUG, e);
}
}
@Override
public void cleanup() {
intermediateSum = null;
}
@Override
public Double getValue() {
return intermediateSum;
}
}