/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.pig.builtin;
import java.io.IOException;
import java.util.Iterator;
import java.util.Vector;
import org.apache.pig.Algebraic;
import org.apache.pig.EvalFunc;
import org.apache.pig.data.DataBag;
import org.apache.pig.data.DataType;
import org.apache.pig.data.DefaultBagFactory;
import org.apache.pig.data.TupleFactory;
import org.apache.pig.data.Tuple;
import org.apache.pig.impl.logicalLayer.schema.Schema;
/**
* Computes the correlation between sets of data. The returned value
* will be a bag which will contain a tuple for each combination of input
* schema and inside tuple we will have two schema name and correlation between
* those two schemas.
*
* A = load 'input.xml' using PigStorage(':');<br/>
* B = group A all;<br/>
* D = foreach B generate group,COR(A.$0,A.$1,A.$2);<br/>
*/
public class COR extends EvalFunc<DataBag> implements Algebraic {
//name of the schemas. Initialize when user use define
protected Vector<String>schemaName = new Vector<String>();
//flag to indicate if define is called or not.
private boolean flag = false;
public COR(){}
public COR(String... schemaName){
for(int i=0;i<schemaName.length;i++){
this.schemaName.add(schemaName[i]);
flag = true;
}
}
/**
* Function to compute correlation between data sets.
* @param input input tuple which contains data sets.
* @return output output dataBag which contain correlation between each pair of data sets.
*/
@Override
public DataBag exec(Tuple input) throws IOException {
if (input == null || input.size() == 0)
return null;
DataBag output = DefaultBagFactory.getInstance().newDefaultBag();
for(int i=0;i<input.size();i++){
for(int j=i+1;j<input.size();j++){
Tuple temp = TupleFactory.getInstance().newTuple(3);
try{
if(flag){
temp.set(0, schemaName.elementAt(i));
temp.set(1, schemaName.elementAt(j));
}
else{
temp.set(0, "var"+i);
temp.set(1, "var"+j);
}
Tuple tempResult = computeAll((DataBag)input.get(i),(DataBag)input.get(j));
double size = ((DataBag)input.get(i)).size();
double sum_x_y = (Double)tempResult.get(0);
double sum_x = (Double)tempResult.get(1);
double sum_y = (Double)tempResult.get(2);
double sum_x_square = (Double)tempResult.get(3);
double sum_y_square = (Double)tempResult.get(4);
double result = (size*sum_x_y - sum_x*sum_y)/Math.sqrt((size*sum_x_square-sum_x*sum_x)*(size*sum_y_square-sum_y*sum_y));
temp.set(2, result);
}catch (Exception e){
System.err.println("Failed to process input record; error - " + e.getMessage());
return null;
}
output.add(temp);
}
}
return output;
}
//used to pass schema name to Final class constructor
/**
* Function to return argument of constructor as string. It append ( and ) at starting and end or argument respectively.
* If default constructor is called is returns empty string.
* @return argument of constructor
*/
@Override
public String toString() {
if(flag){
StringBuilder sb = new StringBuilder();
sb.append('(');
if(schemaName!=null){
for (String sch : schemaName) {
sb.append('\'' + sch + '\'' + ",");
}
sb.deleteCharAt(sb.length()-1);
}
sb.append(')');
return sb.toString();
}
else return "";
}
public String getInitial() {
return Initial.class.getName();
}
public String getIntermed() {
return Intermed.class.getName();
}
public String getFinal() {
return Final.class.getName() + toString();
}
static public class Initial extends EvalFunc<Tuple> {
@Override
public Tuple exec(Tuple input) throws IOException {
if (input == null || input.size() == 0)
return null;
Tuple output = TupleFactory.getInstance().newTuple(input.size()*(input.size()-1));
try {
int k = -1;
for(int i=0;i<input.size();i++){
for(int j=i+1;j<input.size();j++){
DataBag first = (DataBag)input.get(i);
DataBag second = (DataBag)input.get(j);
output.set(++k, computeAll(first, second));
output.set(++k, (Long)first.size());
}
}
} catch(Exception t) {
System.err.println("Failed to process input record; error - " + t.getMessage());
return null;
}
return output;
}
}
static public class Intermed extends EvalFunc<Tuple> {
@Override
public Tuple exec(Tuple input) throws IOException {
if (input == null || input.size() == 0)
return null;
try{
return combine((DataBag)input.get(0));
}catch(Exception e){
throw new IOException("Caught exception in COR.Intermed", e);
}
}
}
public static class Final extends EvalFunc<DataBag> {
protected Vector<String>schemaName = new Vector<String>();
boolean flag = false;
public Final(){}
public Final(String... schemaName){
for(int i=0;i<schemaName.length;i++){
this.schemaName.add(schemaName[i]);
flag = true;
}
}
@Override
public DataBag exec(Tuple input) throws IOException {
if (input == null || input.size() == 0)
return null;
Tuple combined;
try{
combined = combine((DataBag)input.get(0));
}catch(Exception e){
throw new IOException("Caught exception in COR.Final", e);
}
int count = 0;
//for each pair of input schema combined contain 2 member. first member
//is tuple containing sum_x,sum_y,sum_x_y, sum_x_square, sum_y_square
//and second is number of tuples from which it is calculated .
//So if arity of combined is n then total number of schemas would be
//root of x*x - x - n =0
int totalSchemas=2;
while(totalSchemas*(totalSchemas-1)<combined.size()){
totalSchemas++;
}
DataBag output = DefaultBagFactory.getInstance().newDefaultBag();
for(int i=0;i<totalSchemas;i++){
for(int j=i+1;j<totalSchemas;j++){
Tuple result = TupleFactory.getInstance().newTuple(3);
try{
if(flag){
result.set(0, schemaName.elementAt(i));
result.set(1, schemaName.elementAt(j));
}
else{
result.set(0, "var"+i);
result.set(1, "var"+j);
}
Tuple tup = (Tuple)combined.get(count);
double tempCount = (Long)combined.get(count+1);
double sum_x_y = (Double)tup.get(0);
double sum_x = (Double)tup.get(1);
double sum_y = (Double)tup.get(2);
double sum_x_square = (Double)tup.get(3);
double sum_y_square = (Double)tup.get(4);
double correl = (tempCount*sum_x_y - sum_x*sum_y)/Math.sqrt((tempCount*sum_x_square-sum_x*sum_x)*(tempCount*sum_y_square-sum_y*sum_y));
result.set(2, correl);
}catch(Exception e){
System.err.println("Failed to process input record; error - " + e.getMessage());
return null;
}
output.add(result);
count+=2;
}
}
return output;
}
}
/**
* combine results of different data chunk
* @param values DataBag containing partial results computed on different data chunks
* @return output Tuple containing combined data
* @throws IOException
*/
static protected Tuple combine(DataBag values) throws IOException {
Tuple output = TupleFactory.getInstance().newTuple();
Tuple tuple; // copy of DataBag values
tuple = TupleFactory.getInstance().newTuple();
try{
for (Iterator<Tuple> it = values.iterator(); it.hasNext();) {
Tuple t = it.next();
tuple.append(t);
}
}catch(Exception e){}
try{
int size = ((Tuple)tuple.get(0)).size();
for(int i=0;i<size;i=i+2){
long count = 0;
double sum_x_y = 0.0;
double sum_x = 0.0;
double sum_y = 0.0;
double sum_x_square = 0.0;
double sum_y_square = 0.0;
for(int j=0;j<tuple.size();j++){
Tuple temp = (Tuple)tuple.get(j);
Tuple tem = (Tuple)temp.get(i);
count += (Long)temp.get(i+1);
sum_x_y += (Double)tem.get(0);
sum_x += (Double)tem.get(1);
sum_y += (Double)tem.get(2);
sum_x_square += (Double)tem.get(3);
sum_y_square += (Double)tem.get(4);
}
Tuple result = TupleFactory.getInstance().newTuple(5);
result.set(0, sum_x_y);
result.set(1, sum_x);
result.set(2, sum_y);
result.set(3, sum_x_square);
result.set(4, sum_y_square);
output.append(result);
output.append(count);
}
}catch(Exception e){
throw new IOException("Caught exception in COR.combine", e);
}
return output;
}
/**
* compute sum(XY), sum(X), sum(Y), sum(XX), sum(YY) from given data sets
* @param first DataBag containing first data set
* @param second DataBag containing second data set
* @return tuple containing sum(XY), sum(X), sum(Y), sum(XX), sum(YY)
*/
protected static Tuple computeAll(DataBag first, DataBag second) throws IOException {
double sum_x_y = 0.0;
double sum_x = 0.0;
double sum_y = 0.0;
double sum_x_square = 0.0;
double sum_y_square = 0.0;
Iterator<Tuple> iterator_x = first.iterator();
Iterator<Tuple> iterator_y = second.iterator();
try{
while(iterator_x.hasNext()){
double x = (Double)iterator_x.next().get(0);
double y = (Double)iterator_y.next().get(0);
sum_x_y+=x*y;
sum_x+=x;
sum_y+=y;
sum_x_square+=x*x;
sum_y_square+=y*y;
}
}catch(Exception e){
throw new IOException("Caught exception processing input", e);
}
Tuple result = TupleFactory.getInstance().newTuple(5);
try{
result.set(0, sum_x_y);
result.set(1, sum_x);
result.set(2, sum_y);
result.set(3, sum_x_square);
result.set(4, sum_y_square);
}catch(Exception e){
throw new IOException("Caught exception processing result", e);
}
return result;
}
@Override
public Schema outputSchema(Schema input) {
return new Schema(new Schema.FieldSchema(getSchemaName(this.getClass().getName().toLowerCase(), input), DataType.BAG));
}
}