/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.pig.test;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Properties;
import java.util.Set;
import junit.framework.Assert;
import org.apache.pig.ExecType;
import org.apache.pig.FilterFunc;
import org.apache.pig.PigServer;
import org.apache.pig.data.Tuple;
import org.apache.pig.impl.PigContext;
import org.apache.pig.newplan.Operator;
import org.apache.pig.newplan.OperatorPlan;
import org.apache.pig.newplan.logical.optimizer.LogicalPlanOptimizer;
import org.apache.pig.newplan.logical.relational.LOCogroup;
import org.apache.pig.newplan.logical.relational.LOCross;
import org.apache.pig.newplan.logical.relational.LODistinct;
import org.apache.pig.newplan.logical.relational.LOFilter;
import org.apache.pig.newplan.logical.relational.LOForEach;
import org.apache.pig.newplan.logical.relational.LOJoin;
import org.apache.pig.newplan.logical.relational.LOLimit;
import org.apache.pig.newplan.logical.relational.LOLoad;
import org.apache.pig.newplan.logical.relational.LOSort;
import org.apache.pig.newplan.logical.relational.LOSplit;
import org.apache.pig.newplan.logical.relational.LOSplitOutput;
import org.apache.pig.newplan.logical.relational.LOStore;
import org.apache.pig.newplan.logical.relational.LOStream;
import org.apache.pig.newplan.logical.relational.LOUnion;
import org.apache.pig.newplan.logical.relational.LogicalPlan;
import org.apache.pig.newplan.logical.rules.LoadTypeCastInserter;
import org.apache.pig.newplan.logical.rules.PushUpFilter;
import org.apache.pig.newplan.optimizer.PlanOptimizer;
import org.apache.pig.newplan.optimizer.PlanTransformListener;
import org.apache.pig.newplan.optimizer.Rule;
import org.junit.Test;
import org.junit.Before;
/**
* Test the logical optimizer.
*/
public class TestNewPlanPushUpFilter {
PigContext pc = new PigContext(ExecType.LOCAL, new Properties());
@Before
public void tearDown() {
}
/**
* A simple filter UDF for testing
*/
static public class MyFilterFunc extends FilterFunc {
@Override
public Boolean exec(Tuple input) {
return false;
}
}
@Test
// Empty plan, nothing to update
public void testErrorEmptyInput() throws Exception {
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( "" );
Assert.assertTrue( !newLogicalPlan.getOperators().hasNext() );
}
@Test
//Test to ensure that the right exception is thrown when the input list is empty
public void testErrorNonFilterInput() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);store A into 'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
Operator op = newLogicalPlan.getSources().get(0);
Assert.assertTrue( op instanceof LOLoad );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOForEach );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOStore );
Assert.assertTrue( newLogicalPlan.getSuccessors(op) == null );
}
@Test
public void testFilterLoad() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);" +
"B = filter A by $1 < 18;" +
"store B into 'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
Operator op = newLogicalPlan.getSources().get(0);
Assert.assertTrue( op instanceof LOLoad );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOForEach );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOFilter );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOStore );
}
@Test
public void testFilterStreaming() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);" +
"B = stream A through `" + "ps -u" + "`;" +
"C = filter B by $1 < 18;" +
"D = STORE C into 'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
Operator op = newLogicalPlan.getSources().get( 0 );
Assert.assertTrue( op instanceof LOLoad );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOForEach );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOStream );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOFilter );
}
@Test
public void testFilterSort() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);" +
"B = order A by $1, $2;" +
"C = filter B by $1 < 18;" +
"D = STORE C into 'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
Operator op = newLogicalPlan.getSources().get( 0 );
Assert.assertTrue( op instanceof LOLoad );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOForEach );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOFilter );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOSort );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOStore );
}
@Test
public void testFilterConstantConditionSort() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);" +
"B = order A by $1, $2;" +
"C = filter B by 1 == 1;" +
"D = STORE C into 'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
Operator op = newLogicalPlan.getSources().get( 0 );
Assert.assertTrue( op instanceof LOLoad );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOForEach );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOFilter );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOSort );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOStore );
}
@Test
public void testFilterUDFSort() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);" +
"B = order A by $1, $2;" +
"C = filter B by " + MyFilterFunc.class.getName() + "($1) ;" +
"D = STORE C into 'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
Operator op = newLogicalPlan.getSources().get( 0 );
Assert.assertTrue( op instanceof LOLoad );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOForEach );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOFilter );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOSort );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOStore );
}
@Test
public void testFilterDistinct() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);" +
"B = distinct A;" +
"C = filter B by $1 < 18;" +
"D = STORE C into 'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
Operator op = newLogicalPlan.getSources().get( 0 );
Assert.assertTrue( op instanceof LOLoad );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOForEach );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOFilter );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LODistinct );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOStore );
}
@Test
public void testFilterConstantConditionDistinct() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);" +
"B = distinct A;" +
"C = filter B by 1 == 1;" +
"D = STORE C into 'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
Operator op = newLogicalPlan.getSources().get( 0 );
Assert.assertTrue( op instanceof LOLoad );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOForEach );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOFilter );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LODistinct );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOStore );
}
@Test
public void testFilterUDFDistinct() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);" +
"B = distinct A;" +
"C = filter B by " + MyFilterFunc.class.getName() + "($1) ;" +
"D = STORE C into 'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
Operator op = newLogicalPlan.getSources().get( 0 );
Assert.assertTrue( op instanceof LOLoad );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOForEach );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOFilter );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LODistinct );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOStore );
}
@Test
public void testFilterFilter() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);" +
"B = filter A by $0 != 'name';" +
"C = filter B by $1 < 18;" +
"D = STORE C into 'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
Operator op = newLogicalPlan.getSources().get( 0 );
Assert.assertTrue( op instanceof LOLoad );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOForEach );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOFilter );
Assert.assertTrue( ((LOFilter)op).getAlias().equals( "B" ) );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOFilter );
Assert.assertTrue( ((LOFilter)op).getAlias().equals( "C" ) );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOStore );
}
@Test
public void testFilterSplitOutput() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);" +
"split A into B if $1 < 18, C if $1 >= 18;" +
"C = filter B by $1 < 10;" +
"D = STORE C into 'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
Operator op = newLogicalPlan.getSources().get( 0 );
Assert.assertTrue( op instanceof LOLoad );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOForEach );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOSplit );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOSplitOutput );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOFilter );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOStore );
}
@Test
public void testFilterLimit() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);" +
"B = limit A 10;" +
"C = filter B by $1 < 18;" +
"D = STORE C into 'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
Operator op = newLogicalPlan.getSources().get( 0 );
Assert.assertTrue( op instanceof LOLoad );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOForEach );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOLimit );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOFilter );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOStore );
}
@Test
public void testFilterUnion() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);" +
"B = load 'anotherfile' as (name, age, preference);" +
"C = union A, B;" +
"D = filter C by $1 < 18;" +
"E = STORE D into'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
List<Operator> loads = newLogicalPlan.getSources();
Assert.assertTrue( loads.size() == 2 );
Assert.assertTrue( loads.get( 0 ) instanceof LOLoad );
Assert.assertTrue( loads.get( 1 ) instanceof LOLoad );
Operator loadA = null;
Operator loadB = null;
if( ((LOLoad)loads.get( 0 )).getAlias().equals( "A" ) ) {
loadA = loads.get( 0 );
loadB = loads.get( 1 );
} else {
loadA = loads.get( 1 );
loadB = loads.get( 0 );
}
Operator foreachA = newLogicalPlan.getSuccessors(loadA).get( 0 );
Assert.assertTrue( foreachA instanceof LOForEach );
Operator filterA = newLogicalPlan.getSuccessors(foreachA).get( 0 );
Assert.assertTrue( filterA instanceof LOFilter );
Operator foreachB = newLogicalPlan.getSuccessors(loadB).get( 0 );
Assert.assertTrue( foreachB instanceof LOForEach );
Operator filterB = newLogicalPlan.getSuccessors(foreachB).get( 0 );
Assert.assertTrue( filterB instanceof LOFilter );
Operator unionA = newLogicalPlan.getSuccessors( filterA ).get( 0 );
Assert.assertTrue( unionA instanceof LOUnion );
Operator unionB = newLogicalPlan.getSuccessors( filterB ).get( 0 );
Assert.assertTrue( unionB instanceof LOUnion );
Assert.assertTrue( unionB == unionA );
Operator store = newLogicalPlan.getSuccessors(unionA).get( 0 );
Assert.assertTrue( store instanceof LOStore );
}
@Test
public void testFilterConstantConditionUnion() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);" +
"B = load 'anotherfile' as (name, age, preference);" +
"C = union A, B;" +
"D = filter C by 1 == 1;" +
"E = STORE D into'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
List<Operator> loads = newLogicalPlan.getSources();
Assert.assertTrue( loads.size() == 2 );
Assert.assertTrue( loads.get( 0 ) instanceof LOLoad );
Assert.assertTrue( loads.get( 1 ) instanceof LOLoad );
Operator loadA = null;
Operator loadB = null;
if( ((LOLoad)loads.get( 0 )).getAlias().equals( "A" ) ) {
loadA = loads.get( 0 );
loadB = loads.get( 1 );
} else {
loadA = loads.get( 1 );
loadB = loads.get( 0 );
}
Operator foreachA = newLogicalPlan.getSuccessors(loadA).get( 0 );
Assert.assertTrue( foreachA instanceof LOForEach );
Operator filterA = newLogicalPlan.getSuccessors(foreachA).get( 0 );
Assert.assertTrue( filterA instanceof LOFilter );
Operator foreachB = newLogicalPlan.getSuccessors(loadB).get( 0 );
Assert.assertTrue( foreachB instanceof LOForEach );
Operator filterB = newLogicalPlan.getSuccessors(foreachB).get( 0 );
Assert.assertTrue( filterB instanceof LOFilter );
Operator unionA = newLogicalPlan.getSuccessors( filterA ).get( 0 );
Assert.assertTrue( unionA instanceof LOUnion );
Operator unionB = newLogicalPlan.getSuccessors( filterB ).get( 0 );
Assert.assertTrue( unionB instanceof LOUnion );
Assert.assertTrue( unionB == unionA );
Operator store = newLogicalPlan.getSuccessors(unionA).get( 0 );
Assert.assertTrue( store instanceof LOStore );
}
@Test
public void testFilterUDFUnion() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);" +
"B = load 'anotherfile' as (name, age, preference);" +
"C = union A, B;" +
"D = filter C by " + MyFilterFunc.class.getName() + "() ;" +
"E = STORE D into'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
List<Operator> loads = newLogicalPlan.getSources();
Assert.assertTrue( loads.size() == 2 );
Assert.assertTrue( loads.get( 0 ) instanceof LOLoad );
Assert.assertTrue( loads.get( 1 ) instanceof LOLoad );
Operator loadA = null;
Operator loadB = null;
if( ((LOLoad)loads.get( 0 )).getAlias().equals( "A" ) ) {
loadA = loads.get( 0 );
loadB = loads.get( 1 );
} else {
loadA = loads.get( 1 );
loadB = loads.get( 0 );
}
Operator foreachA = newLogicalPlan.getSuccessors(loadA).get( 0 );
Assert.assertTrue( foreachA instanceof LOForEach );
Operator filterA = newLogicalPlan.getSuccessors(foreachA).get( 0 );
Assert.assertTrue( filterA instanceof LOFilter );
Operator foreachB = newLogicalPlan.getSuccessors(loadB).get( 0 );
Assert.assertTrue( foreachB instanceof LOForEach );
Operator filterB = newLogicalPlan.getSuccessors(foreachB).get( 0 );
Assert.assertTrue( filterB instanceof LOFilter );
Operator unionA = newLogicalPlan.getSuccessors( filterA ).get( 0 );
Assert.assertTrue( unionA instanceof LOUnion );
Operator unionB = newLogicalPlan.getSuccessors( filterB ).get( 0 );
Assert.assertTrue( unionB instanceof LOUnion );
Assert.assertTrue( unionB == unionA );
Operator store = newLogicalPlan.getSuccessors(unionA).get( 0 );
Assert.assertTrue( store instanceof LOStore );
}
@Test
public void testFilterCross() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);" +
"B = load 'anotherfile' as (name, age, preference);" +
"C = cross A, B;" +
"D = filter C by $5 < 18;" +
"E = limit D 10;" +
"F = STORE E into 'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
List<Operator> loads = newLogicalPlan.getSources();
Assert.assertTrue( loads.size() == 2 );
Assert.assertTrue( loads.get( 0 ) instanceof LOLoad );
Assert.assertTrue( loads.get( 1 ) instanceof LOLoad );
Operator op = null;
if( ((LOLoad)loads.get( 0 )).getAlias().equals( "B" ) )
op = loads.get( 0 );
else
op = loads.get( 1 );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOForEach );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOFilter );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOCross );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOLimit );
}
@Test
public void testFilterCross1() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);" +
"B = load 'anotherfile' as (name, age, preference);" +
"C = cross A, B;" +
"D = filter C by $1 < 18;" +
"E = STORE D into 'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
List<Operator> loads = newLogicalPlan.getSources();
Assert.assertTrue( loads.size() == 2 );
Assert.assertTrue( loads.get( 0 ) instanceof LOLoad );
Assert.assertTrue( loads.get( 1 ) instanceof LOLoad );
Operator op = null;
if( ((LOLoad)loads.get( 0 )).getAlias().equals( "A" ) )
op = loads.get( 0 );
else
op = loads.get( 1 );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOForEach );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOFilter );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOCross );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOStore );
}
@Test
public void testFilterCross2() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);" +
"B = load 'anotherfile' as (name, age, preference);" +
"C = cross A, B;" +
"D = filter C by $1 < 18 and $5 < 18;" +
"E = STORE D into 'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
List<Operator> loads = newLogicalPlan.getSources();
Assert.assertTrue( loads.size() == 2 );
Assert.assertTrue( loads.get( 0 ) instanceof LOLoad );
Assert.assertTrue( loads.get( 1 ) instanceof LOLoad );
Operator loadA = null;
Operator loadB = null;
if( ((LOLoad)loads.get( 0 )).getAlias().equals( "A" ) ) {
loadA = loads.get( 0 );
loadB = loads.get( 1 );
} else {
loadA = loads.get( 1 );
loadB = loads.get( 0 );
}
Operator foreachA = newLogicalPlan.getSuccessors(loadA).get( 0 );
Assert.assertTrue( foreachA instanceof LOForEach );
Operator op = newLogicalPlan.getSuccessors(foreachA).get( 0 );
Assert.assertTrue( op instanceof LOCross );
Operator foreachB = newLogicalPlan.getSuccessors(loadB).get( 0 );
Assert.assertTrue( foreachB instanceof LOForEach );
op = newLogicalPlan.getSuccessors(foreachB).get( 0 );
Assert.assertTrue( op instanceof LOCross );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOFilter );
}
@Test
public void testFilterConstantConditionCross() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);" +
"B = load 'anotherfile' as (name, age, preference);" +
"C = cross A, B;" +
"D = filter C by 1 == 1;" +
"E = STORE D into 'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
List<Operator> loads = newLogicalPlan.getSources();
Assert.assertTrue( loads.size() == 2 );
Assert.assertTrue( loads.get( 0 ) instanceof LOLoad );
Assert.assertTrue( loads.get( 1 ) instanceof LOLoad );
Operator loadA = null;
Operator loadB = null;
if( ((LOLoad)loads.get( 0 )).getAlias().equals( "A" ) ) {
loadA = loads.get( 0 );
loadB = loads.get( 1 );
} else {
loadA = loads.get( 1 );
loadB = loads.get( 0 );
}
Operator op = newLogicalPlan.getSuccessors(loadA).get( 0 );
Assert.assertTrue( op instanceof LOForEach );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOFilter );
op = newLogicalPlan.getSuccessors(loadB).get( 0 );
Assert.assertTrue( op instanceof LOForEach );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOFilter );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOCross );
}
@Test
public void testFilterUDFCross() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);" +
"B = load 'anotherfile' as (name, age, preference);" +
"C = cross A, B;" +
"D = filter C by " + MyFilterFunc.class.getName() + "($0) ;" +
"E = STORE D into 'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
List<Operator> loads = newLogicalPlan.getSources();
Assert.assertTrue( loads.size() == 2 );
Assert.assertTrue( loads.get( 0 ) instanceof LOLoad );
Assert.assertTrue( loads.get( 1 ) instanceof LOLoad );
Operator op = null;
if( ((LOLoad)loads.get( 0 )).getAlias().equals( "A" ) )
op = loads.get( 0 );
else
op = loads.get( 1 );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOForEach );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOFilter );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOCross );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOStore );
}
@Test
public void testFilterCogroup() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);" +
"B = load 'anotherfile' as (name, age, preference);" +
"C = cogroup A by $0, B by $0;" +
"D = filter C by $0 < 'name';" +
"E = STORE D into 'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
List<Operator> loads = newLogicalPlan.getSources();
Assert.assertTrue( loads.size() == 2 );
Assert.assertTrue( loads.get( 0 ) instanceof LOLoad );
Assert.assertTrue( loads.get( 1 ) instanceof LOLoad );
Operator loadA = null;
Operator loadB = null;
if( ((LOLoad)loads.get( 0 )).getAlias().equals( "A" ) ) {
loadA = loads.get( 0 );
loadB = loads.get( 1 );
} else {
loadA = loads.get( 1 );
loadB = loads.get( 0 );
}
Operator foreachA = newLogicalPlan.getSuccessors(loadA).get( 0 );
Assert.assertTrue( foreachA instanceof LOForEach );
Operator filterA = newLogicalPlan.getSuccessors(foreachA).get( 0 );
Assert.assertTrue( filterA instanceof LOFilter );
Operator foreachB = newLogicalPlan.getSuccessors(loadB).get( 0 );
Assert.assertTrue( foreachB instanceof LOForEach );
Operator filterB = newLogicalPlan.getSuccessors(foreachB).get( 0 );
Assert.assertTrue( filterB instanceof LOFilter );
Operator cogrpA = newLogicalPlan.getSuccessors( filterA ).get( 0 );
Assert.assertTrue( cogrpA instanceof LOCogroup );
Operator cogrpB = newLogicalPlan.getSuccessors( filterB ).get( 0 );
Assert.assertTrue( cogrpB instanceof LOCogroup );
Assert.assertTrue( cogrpB == cogrpA );
Operator store = newLogicalPlan.getSuccessors(cogrpA).get( 0 );
Assert.assertTrue( store instanceof LOStore );
}
@Test
public void testFilterConstantConditionCogroup() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);" +
"B = load 'anotherfile' as (name, age, preference);" +
"C = cogroup A by $0, B by $0;" +
"D = filter C by 1 == 1;" +
"E = STORE D into 'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
List<Operator> loads = newLogicalPlan.getSources();
Assert.assertTrue( loads.size() == 2 );
Assert.assertTrue( loads.get( 0 ) instanceof LOLoad );
Assert.assertTrue( loads.get( 1 ) instanceof LOLoad );
Operator loadA = null;
Operator loadB = null;
if( ((LOLoad)loads.get( 0 )).getAlias().equals( "A" ) ) {
loadA = loads.get( 0 );
loadB = loads.get( 1 );
} else {
loadA = loads.get( 1 );
loadB = loads.get( 0 );
}
Operator foreachA = newLogicalPlan.getSuccessors(loadA).get( 0 );
Assert.assertTrue( foreachA instanceof LOForEach );
Operator filterA = newLogicalPlan.getSuccessors(foreachA).get( 0 );
Assert.assertTrue( filterA instanceof LOFilter );
Operator foreachB = newLogicalPlan.getSuccessors(loadB).get( 0 );
Assert.assertTrue( foreachB instanceof LOForEach );
Operator filterB = newLogicalPlan.getSuccessors(foreachB).get( 0 );
Assert.assertTrue( filterB instanceof LOFilter );
Operator cogrpA = newLogicalPlan.getSuccessors( filterA ).get( 0 );
Assert.assertTrue( cogrpA instanceof LOCogroup );
Operator cogrpB = newLogicalPlan.getSuccessors( filterB ).get( 0 );
Assert.assertTrue( cogrpB instanceof LOCogroup );
Assert.assertTrue( cogrpB == cogrpA );
Operator store = newLogicalPlan.getSuccessors(cogrpA).get( 0 );
Assert.assertTrue( store instanceof LOStore );
}
@Test
public void testFilterUDFCogroup() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);" +
"B = load 'anotherfile' as (name, age, preference);" +
"C = cogroup A by $0, B by $0;" +
"D = filter C by " + MyFilterFunc.class.getName() + "($1) ;" +
"E = STORE D into 'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
List<Operator> loads = newLogicalPlan.getSources();
Assert.assertTrue( loads.size() == 2 );
Assert.assertTrue( loads.get( 0 ) instanceof LOLoad );
Assert.assertTrue( loads.get( 1 ) instanceof LOLoad );
Operator loadA = null;
Operator loadB = null;
if( ((LOLoad)loads.get( 0 )).getAlias().equals( "A" ) ) {
loadA = loads.get( 0 );
loadB = loads.get( 1 );
} else {
loadA = loads.get( 1 );
loadB = loads.get( 0 );
}
Operator foreachA = newLogicalPlan.getSuccessors(loadA).get( 0 );
Assert.assertTrue( foreachA instanceof LOForEach );
Operator cogroupA = newLogicalPlan.getSuccessors(foreachA).get( 0 );
Assert.assertTrue( cogroupA instanceof LOCogroup );
Operator foreachB = newLogicalPlan.getSuccessors(loadB).get( 0 );
Assert.assertTrue( foreachB instanceof LOForEach );
Operator cogroupB = newLogicalPlan.getSuccessors(foreachB).get( 0 );
Assert.assertTrue( cogroupB instanceof LOCogroup );
Operator filter = newLogicalPlan.getSuccessors( cogroupA ).get( 0 );
Assert.assertTrue( filter instanceof LOFilter );
filter = newLogicalPlan.getSuccessors( cogroupB ).get( 0 );
Assert.assertTrue( cogroupB instanceof LOCogroup );
Assert.assertTrue( cogroupB == cogroupA );
Operator store = newLogicalPlan.getSuccessors(filter).get( 0 );
Assert.assertTrue( store instanceof LOStore );
}
@Test
public void testFilterCogroupOuter() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);" +
"B = load 'anotherfile' as (name, age, preference);" +
"C = cogroup A by $0, B by $0 outer;" +
"D = filter C by $0 < 'name';" +
"E = STORE D into 'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
List<Operator> loads = newLogicalPlan.getSources();
Assert.assertTrue( loads.size() == 2 );
Assert.assertTrue( loads.get( 0 ) instanceof LOLoad );
Assert.assertTrue( loads.get( 1 ) instanceof LOLoad );
Operator loadA = null;
Operator loadB = null;
if( ((LOLoad)loads.get( 0 )).getAlias().equals( "A" ) ) {
loadA = loads.get( 0 );
loadB = loads.get( 1 );
} else {
loadA = loads.get( 1 );
loadB = loads.get( 0 );
}
Operator foreachA = newLogicalPlan.getSuccessors(loadA).get( 0 );
Assert.assertTrue( foreachA instanceof LOForEach );
Operator filterA = newLogicalPlan.getSuccessors(foreachA).get( 0 );
Assert.assertTrue( filterA instanceof LOFilter );
Operator foreachB = newLogicalPlan.getSuccessors(loadB).get( 0 );
Assert.assertTrue( foreachB instanceof LOForEach );
Operator filterB = newLogicalPlan.getSuccessors(foreachB).get( 0 );
Assert.assertTrue( filterB instanceof LOFilter );
Operator cogrpA = newLogicalPlan.getSuccessors( filterA ).get( 0 );
Assert.assertTrue( cogrpA instanceof LOCogroup );
Operator cogrpB = newLogicalPlan.getSuccessors( filterB ).get( 0 );
Assert.assertTrue( cogrpB instanceof LOCogroup );
Assert.assertTrue( cogrpB == cogrpA );
Operator store = newLogicalPlan.getSuccessors(cogrpA).get( 0 );
Assert.assertTrue( store instanceof LOStore );
}
@Test
public void testFilterConstantConditionCogroupOuter() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);" +
"B = load 'anotherfile' as (name, age, preference);" +
"C = cogroup A by $0, B by $0 outer;" +
"D = filter C by 1 == 1;" +
"E = STORE D into 'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
List<Operator> loads = newLogicalPlan.getSources();
Assert.assertTrue( loads.size() == 2 );
Assert.assertTrue( loads.get( 0 ) instanceof LOLoad );
Assert.assertTrue( loads.get( 1 ) instanceof LOLoad );
Operator loadA = null;
Operator loadB = null;
if( ((LOLoad)loads.get( 0 )).getAlias().equals( "A" ) ) {
loadA = loads.get( 0 );
loadB = loads.get( 1 );
} else {
loadA = loads.get( 1 );
loadB = loads.get( 0 );
}
Operator foreachA = newLogicalPlan.getSuccessors(loadA).get( 0 );
Assert.assertTrue( foreachA instanceof LOForEach );
Operator filterA = newLogicalPlan.getSuccessors(foreachA).get( 0 );
Assert.assertTrue( filterA instanceof LOFilter );
Operator foreachB = newLogicalPlan.getSuccessors(loadB).get( 0 );
Assert.assertTrue( foreachB instanceof LOForEach );
Operator filterB = newLogicalPlan.getSuccessors(foreachB).get( 0 );
Assert.assertTrue( filterB instanceof LOFilter );
Operator cogrpA = newLogicalPlan.getSuccessors( filterA ).get( 0 );
Assert.assertTrue( cogrpA instanceof LOCogroup );
Operator cogrpB = newLogicalPlan.getSuccessors( filterB ).get( 0 );
Assert.assertTrue( cogrpB instanceof LOCogroup );
Assert.assertTrue( cogrpB == cogrpA );
Operator store = newLogicalPlan.getSuccessors(cogrpA).get( 0 );
Assert.assertTrue( store instanceof LOStore );
}
@Test
public void testFilterUDFCogroupOuter() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);" +
"B = load 'anotherfile' as (name, age, preference);" +
"C = cogroup A by $0, B by $0 outer;" +
"D = filter C by " + MyFilterFunc.class.getName() + "() ;" +
"E = STORE D into 'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
List<Operator> loads = newLogicalPlan.getSources();
Assert.assertTrue( loads.size() == 2 );
Assert.assertTrue( loads.get( 0 ) instanceof LOLoad );
Assert.assertTrue( loads.get( 1 ) instanceof LOLoad );
Operator loadA = null;
Operator loadB = null;
if( ((LOLoad)loads.get( 0 )).getAlias().equals( "A" ) ) {
loadA = loads.get( 0 );
loadB = loads.get( 1 );
} else {
loadA = loads.get( 1 );
loadB = loads.get( 0 );
}
Operator foreachA = newLogicalPlan.getSuccessors(loadA).get( 0 );
Assert.assertTrue( foreachA instanceof LOForEach );
Operator cogroupA = newLogicalPlan.getSuccessors(foreachA).get( 0 );
Assert.assertTrue( cogroupA instanceof LOCogroup );
Operator foreachB = newLogicalPlan.getSuccessors(loadB).get( 0 );
Assert.assertTrue( foreachB instanceof LOForEach );
Operator cogroupB = newLogicalPlan.getSuccessors(foreachB).get( 0 );
Assert.assertTrue( cogroupB instanceof LOCogroup );
Operator filter = newLogicalPlan.getSuccessors( cogroupA ).get( 0 );
Assert.assertTrue( filter instanceof LOFilter );
filter = newLogicalPlan.getSuccessors( cogroupB ).get( 0 );
Assert.assertTrue( filter instanceof LOFilter );
Assert.assertTrue( cogroupB == cogroupA );
Operator store = newLogicalPlan.getSuccessors(filter).get( 0 );
Assert.assertTrue( store instanceof LOStore );
}
@Test
public void testFilterGroupBy() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);" +
"B = group A by $0;" +
"C = filter B by $0 < 'name';" +
"D = STORE C into 'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
Operator op = newLogicalPlan.getSources().get( 0 );
Assert.assertTrue( op instanceof LOLoad );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOForEach );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOFilter );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOCogroup );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOStore );
}
@Test
public void testFilterConstantConditionGroupBy() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);" +
"B = group A by $0;" +
"C = filter B by 1 == 1;" +
"D = STORE C into 'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
Operator op = newLogicalPlan.getSources().get( 0 );
Assert.assertTrue( op instanceof LOLoad );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOForEach );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOFilter );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOCogroup );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOStore );
}
@Test
public void testFilterUDFGroupBy() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);" +
"B = group A by $0;" +
"C = filter B by " + MyFilterFunc.class.getName() + "($1) ;" +
"D = STORE C into 'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
Operator op = newLogicalPlan.getSources().get( 0 );
Assert.assertTrue( op instanceof LOLoad );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOForEach );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOCogroup );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOFilter );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOStore );
}
@Test
public void testFilterGroupByOuter() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);" +
"B = group A by $0 outer;" +
"C = filter B by $0 < 'name';" +
"D = STORE C into 'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
Operator op = newLogicalPlan.getSources().get( 0 );
Assert.assertTrue( op instanceof LOLoad );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOForEach );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOFilter );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOCogroup );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOStore );
}
@Test
public void testFilterConstantConditionGroupByOuter() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);" +
"B = group A by $0 outer;" +
"C = filter B by 1 == 1;" +
"D = STORE C into 'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
Operator op = newLogicalPlan.getSources().get( 0 );
Assert.assertTrue( op instanceof LOLoad );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOForEach );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOFilter );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOCogroup );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOStore );
}
@Test
public void testFilterUDFGroupByOuter() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);" +
"B = group A by $0 outer;" +
"C = filter B by " + MyFilterFunc.class.getName() + "($1) ;" +
"D = STORE C into 'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
Operator op = newLogicalPlan.getSources().get( 0 );
Assert.assertTrue( op instanceof LOLoad );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOForEach );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOCogroup );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOFilter );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOStore );
}
@Test
public void testFilterFRJoin() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);" +
"B = load 'anotherfile' as (name, age, preference);" +
"C = join A by $0, B by $0 using 'replicated';" +
"D = filter C by $0 < 'name';" +
"E = limit D 10;" +
"F = STORE E into 'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
List<Operator> loads = newLogicalPlan.getSources();
Assert.assertTrue( loads.size() == 2 );
Assert.assertTrue( loads.get( 0 ) instanceof LOLoad );
Assert.assertTrue( loads.get( 1 ) instanceof LOLoad );
Operator op = null;
if( ((LOLoad)loads.get( 0 )).getAlias().equals( "A" ) )
op = loads.get( 0 );
else
op = loads.get( 1 );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOForEach );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOFilter );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOJoin );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOLimit );
}
@Test
public void testFilterFRJoin1() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);" +
"B = load 'anotherfile' as (name, age, preference);" +
"C = join A by $0, B by $0 using 'replicated';" +
"D = filter C by $4 < 'name';" +
"E = limit D 10;" +
"F = STORE E into 'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
List<Operator> loads = newLogicalPlan.getSources();
Assert.assertTrue( loads.size() == 2 );
Assert.assertTrue( loads.get( 0 ) instanceof LOLoad );
Assert.assertTrue( loads.get( 1 ) instanceof LOLoad );
Operator op = null;
if( ((LOLoad)loads.get( 0 )).getAlias().equals( "B" ) )
op = loads.get( 0 );
else
op = loads.get( 1 );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOForEach );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOFilter );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOJoin );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOLimit );
}
@Test
// Constant filter condition, the filter will be pushed up to the first branch of join.
public void testFilterConstantConditionFRJoin() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);" +
"B = load 'anotherfile' as (name, age, preference);" +
"C = join A by $0, B by $0 using 'replicated';" +
"D = filter C by 1 == 1;" +
"E = STORE D into 'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
List<Operator> loads = newLogicalPlan.getSources();
Assert.assertTrue( loads.size() == 2 );
Assert.assertTrue( loads.get( 0 ) instanceof LOLoad );
Assert.assertTrue( loads.get( 1 ) instanceof LOLoad );
Operator op = null;
if( ((LOLoad)loads.get( 0 )).getAlias().equals( "A" ) )
op = loads.get( 0 );
else
op = loads.get( 1 );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOForEach );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOFilter );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOJoin );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOStore );
}
@Test
// UDF takes on argument, so it's constant. As a result, filter will pushed up to the first branch of the join.
public void testFilterUDFFRJoin() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);" +
"B = load 'anotherfile' as (name, age, preference);" +
"C = join A by $0, B by $0 using 'replicated';" +
"D = filter C by " + MyFilterFunc.class.getName() + "();" +
"E = STORE D into 'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
List<Operator> loads = newLogicalPlan.getSources();
Assert.assertTrue( loads.size() == 2 );
Assert.assertTrue( loads.get( 0 ) instanceof LOLoad );
Assert.assertTrue( loads.get( 1 ) instanceof LOLoad );
Operator op = null;
if( ((LOLoad)loads.get( 0 )).getAlias().equals( "A" ) )
op = loads.get( 0 );
else
op = loads.get( 1 );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOForEach );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOFilter );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOJoin );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOStore );
}
@Test
// UDF takes all input, so filter connot be pushed up.
public void testFilterUDFFRJoin1() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);" +
"B = load 'anotherfile' as (name, age, preference);" +
"C = join A by $0, B by $0 using 'replicated';" +
"D = filter C by TupleSize(*) > 5;" +
"E = STORE D into 'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
List<Operator> loads = newLogicalPlan.getSources();
Assert.assertTrue( loads.size() == 2 );
Assert.assertTrue( loads.get( 0 ) instanceof LOLoad );
Assert.assertTrue( loads.get( 1 ) instanceof LOLoad );
Operator op = newLogicalPlan.getSuccessors( loads.get( 0 ) ).get( 0 );
Assert.assertTrue( op instanceof LOForEach );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOJoin );
op = newLogicalPlan.getSuccessors( loads.get( 1 ) ).get( 0 );
Assert.assertTrue( op instanceof LOForEach );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOJoin );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOFilter );
}
@Test
public void testFilterInnerJoin() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);" +
"B = load 'anotherfile' as (name, age, preference);" +
"C = join A by $0, B by $0;" +
"D = filter C by $0 < 'name';" +
"E = limit D 10;" +
"F = STORE E into 'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
List<Operator> loads = newLogicalPlan.getSources();
Assert.assertTrue( loads.size() == 2 );
Assert.assertTrue( loads.get( 0 ) instanceof LOLoad );
Assert.assertTrue( loads.get( 1 ) instanceof LOLoad );
Operator op = null;
if( ((LOLoad)loads.get( 0 )).getAlias().equals( "A" ) )
op = loads.get( 0 );
else
op = loads.get( 1 );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOForEach );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOFilter );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOJoin );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOLimit );
}
@Test
public void testFilterInnerJoin1() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);" +
"B = load 'anotherfile' as (name, age, preference);" +
"C = join A by $0, B by $0;" +
"D = filter C by $4 < 'name';" +
"E = limit D 10;" +
"F = STORE E into 'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
List<Operator> loads = newLogicalPlan.getSources();
Assert.assertTrue( loads.size() == 2 );
Assert.assertTrue( loads.get( 0 ) instanceof LOLoad );
Assert.assertTrue( loads.get( 1 ) instanceof LOLoad );
Operator op = null;
if( ((LOLoad)loads.get( 0 )).getAlias().equals( "B" ) )
op = loads.get( 0 );
else
op = loads.get( 1 );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOForEach );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOFilter );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOJoin );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOLimit );
}
@Test
public void testFilterInnerJoin2() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);" +
"B = load 'anotherfile' as (name, age, preference);" +
"C = join A by $0, B by $0;" +
"D = filter C by $0 < 'jonh' OR $1 > 50;" +
"E = limit D 10;" +
"F = STORE E into 'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
List<Operator> loads = newLogicalPlan.getSources();
Assert.assertTrue( loads.size() == 2 );
Assert.assertTrue( loads.get( 0 ) instanceof LOLoad );
Assert.assertTrue( loads.get( 1 ) instanceof LOLoad );
Operator op = null;
if( ((LOLoad)loads.get( 0 )).getAlias().equals( "A" ) )
op = loads.get( 0 );
else
op = loads.get( 1 );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOForEach );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOFilter );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOJoin );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOLimit );
}
@Test
public void testFilterInnerJoinNegative() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);" +
"B = load 'anotherfile' as (name, age, preference);" +
"C = join A by $0, B by $0;" +
"D = filter C by $4 < 'name' AND $0 == 'joe';" +
"E = limit D 10;" +
"F = STORE E into 'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
List<Operator> loads = newLogicalPlan.getSources();
Assert.assertTrue( loads.size() == 2 );
Assert.assertTrue( loads.get( 0 ) instanceof LOLoad );
Assert.assertTrue( loads.get( 1 ) instanceof LOLoad );
Operator op = newLogicalPlan.getSuccessors( loads.get( 0 ) ).get( 0 );
Assert.assertTrue( op instanceof LOForEach );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOJoin );
op = newLogicalPlan.getSuccessors( loads.get( 1 ) ).get( 0 );
Assert.assertTrue( op instanceof LOForEach );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOJoin );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOFilter );
}
@Test
public void testFilterUDFInnerJoin() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);" +
"B = load 'anotherfile' as (name, age, preference);" +
"C = join A by $0, B by $0;" +
"D = filter C by " + MyFilterFunc.class.getName() + "() ;" +
"E = STORE D into 'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
List<Operator> loads = newLogicalPlan.getSources();
Assert.assertTrue( loads.size() == 2 );
Assert.assertTrue( loads.get( 0 ) instanceof LOLoad );
Assert.assertTrue( loads.get( 1 ) instanceof LOLoad );
Operator op = null;
if( ((LOLoad)loads.get( 0 )).getAlias().equals( "A" ) )
op = loads.get( 0 );
else
op = loads.get( 1 );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOForEach );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOFilter );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOJoin );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOStore );
}
// See PIG-1289
@Test
public void testOutJoin() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);" +
"B = load 'anotherfile' as (name);" +
"C = join A by name LEFT OUTER, B by name;" +
"D = filter C by B::name is null;" +
"store D into 'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan(query);
Operator op = newLogicalPlan.getSinks().get(0);
Assert.assertTrue( op instanceof LOStore );
op = newLogicalPlan.getPredecessors(op).get( 0 );
Assert.assertTrue( op instanceof LOFilter );
}
// See PIG-1507
@Test
public void testFullOutJoin() throws Exception {
String query = "A = load 'myfile' as (d1:int);" +
"B = load 'anotherfile' as (d2:int);" +
"c = join A by d1 full outer, B by d2;" +
"d = filter c by d2 is null;" +
"store d into 'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan(query);
Operator op = newLogicalPlan.getSinks().get(0);
Assert.assertTrue( op instanceof LOStore );
op = newLogicalPlan.getPredecessors(op).get( 0 );
Assert.assertTrue( op instanceof LOFilter );
}
/**
* Cascading joins. First one has outer, but filter should be able to push up above the second one, but below
* the first one.
*/
@Test
public void testFullOutJoin1() throws Exception {
String query = "A = load 'myfile' as (d1:int);" +
"B = load 'anotherfile' as (d2:int);" +
"C = join A by d1 full outer, B by d2;" +
"D = load 'xxx' as (d3:int);" +
"E = join C by d1, D by d3;" +
"F = filter E by d1 > 5;" +
"G = store F into 'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan(query);
List<Operator> ops = newLogicalPlan.getSinks();
Assert.assertTrue( ops.size() == 1 );
Operator op = ops.get( 0 );
Assert.assertTrue( op instanceof LOStore );
Operator join = newLogicalPlan.getPredecessors(op).get( 0 );
Assert.assertTrue( join instanceof LOJoin );
ops = newLogicalPlan.getPredecessors(join);
Assert.assertEquals( 2, ops.size() );
Assert.assertTrue( ops.get( 0 ) instanceof LOFilter || ops.get( 1 ) instanceof LOFilter );
}
private LogicalPlan migrateAndOptimizePlan(String query) throws Exception {
PigServer pigServer = new PigServer( pc );
LogicalPlan newLogicalPlan = Util.buildLp(pigServer, query);
PlanOptimizer optimizer = new MyPlanOptimizer( newLogicalPlan, 3 );
optimizer.optimize();
return newLogicalPlan;
}
public class MyPlanOptimizer extends LogicalPlanOptimizer {
protected MyPlanOptimizer(OperatorPlan p, int iterations) {
super(p, iterations, new HashSet<String>());
}
public void addPlanTransformListener(PlanTransformListener listener) {
super.addPlanTransformListener(listener);
}
protected List<Set<Rule>> buildRuleSets() {
List<Set<Rule>> ls = new ArrayList<Set<Rule>>();
Set<Rule> s = new HashSet<Rule>();
// add split filter rule
Rule r = new LoadTypeCastInserter( "TypeCastInserter" );
s.add(r);
ls.add(s);
s = new HashSet<Rule>();
r = new PushUpFilter( "PushUpFilter" );
s.add(r);
ls.add(s);
return ls;
}
}
}