LOStore.java example

Explorer
hadoop-pig-master
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.pig.impl.logicalLayer;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import org.apache.pig.FuncSpec;
import org.apache.pig.SortInfo;
import org.apache.pig.StoreFuncInterface;
import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.io.FileSpec;
import org.apache.pig.impl.plan.OperatorKey;
import org.apache.pig.impl.plan.ProjectionMap;
import org.apache.pig.impl.plan.RequiredFields;
import org.apache.pig.impl.plan.VisitorException;
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

public class LOStore extends RelationalOperator {
    private static final long serialVersionUID = 2L;

    private FileSpec mOutputFile;

    // If we know how to reload the store, here's how. The lFile
    // FileSpec is set in PigServer.postProcess. It can be used to
    // reload this store, if the optimizer has the need.
    private FileSpec mInputSpec;
    
    private String signature;

    transient private StoreFuncInterface mStoreFunc;
    private static Log log = LogFactory.getLog(LOStore.class);
    private boolean isTmpStore;
    
    public boolean isTmpStore() {
        return isTmpStore;
    }

    public void setTmpStore(boolean isTmpStore) {
        this.isTmpStore = isTmpStore;
    }

    private SortInfo sortInfo;

    public SortInfo getSortInfo() {
        return sortInfo;
    }

    public void setSortInfo(SortInfo sortInfo) {
        this.sortInfo = sortInfo;
    }

    /**
     * @param plan
     *            LogicalPlan this operator is a part of.
     * @param key
     *            OperatorKey for this operator
     * @param outputFileSpec
     *            the file to be stored
     */
    public LOStore(LogicalPlan plan, OperatorKey key,
            FileSpec outputFileSpec, String alias) throws IOException {
        super(plan, key);

        mOutputFile = outputFileSpec;
        isTmpStore = false;

        // TODO
        // The code below is commented out as PigContext pulls in
        // HExecutionEngine which in turn is completely commented out
        // Also remove the commented out import org.apache.pig.impl.PigContext

        try { 
             mStoreFunc = (StoreFuncInterface) PigContext.instantiateFuncFromSpec(outputFileSpec.getFuncSpec());
             this.mAlias = alias;
             this.signature = constructSignature(mAlias, outputFileSpec.getFileName(), mOutputFile.getFuncSpec());
             mStoreFunc.setStoreFuncUDFContextSignature(this.signature);
        } catch (Exception e) { 
            IOException ioe = new IOException(e.getMessage()); 
            ioe.setStackTrace(e.getStackTrace());
            throw ioe; 
        }
    }
    
    public static String constructSignature(String alias, String filename, FuncSpec funcSpec) {
        return alias+"_"+filename+"_"+funcSpec.toString();
    }

    public FileSpec getOutputFile() {
        return mOutputFile;
    }
    
    public void setOutputFile(FileSpec outputFileSpec) throws IOException {
        try { 
            mStoreFunc = (StoreFuncInterface) PigContext.instantiateFuncFromSpec(outputFileSpec.getFuncSpec()); 
       } catch (Exception e) { 
           IOException ioe = new IOException(e.getMessage()); 
           ioe.setStackTrace(e.getStackTrace());
           throw ioe; 
       }
       mOutputFile = outputFileSpec;
    }

    public StoreFuncInterface getStoreFunc() {
        return mStoreFunc;
    }

    @Override
    public String name() {
        return getAliasString() + "Store " + mKey.scope + "-" + mKey.id;
    }

    @Override
    public Schema getSchema() throws FrontendException {
        //throw new RuntimeException("Internal error: Requested schema of a "
         //       + "store operator.");
        return mPlan.getPredecessors(this).get(0).getSchema();
    }

    @Override
    public boolean supportsMultipleInputs() {
        return false;
    }

    @Override
    public boolean supportsMultipleOutputs() {
        return true;
    }

    @Override
    public void visit(LOVisitor v) throws VisitorException {
        v.visit(this);
    }

    public void setInputSpec(FileSpec in) {
        mInputSpec = in;
    }

    public FileSpec getInputSpec() {
        return mInputSpec;
    }
    
    @Override
    public ProjectionMap getProjectionMap() {
        
        if(mIsProjectionMapComputed) return mProjectionMap;
        mIsProjectionMapComputed = true;
        
        Schema outputSchema;
        try {
            outputSchema = getSchema();
        } catch (FrontendException fee) {
            mProjectionMap = null;
            return mProjectionMap;
        }
        
        Schema inputSchema = null;        
        
        List<LogicalOperator> predecessors = (ArrayList<LogicalOperator>)mPlan.getPredecessors(this);
        if(predecessors != null) {
            try {
                inputSchema = predecessors.get(0).getSchema();
            } catch (FrontendException fee) {
                mProjectionMap = null;
                return mProjectionMap;
            }
        }
        
        
        if(Schema.equals(inputSchema, outputSchema, false, true)) {
            //there is a one is to one mapping between input and output schemas
            mProjectionMap = new ProjectionMap(false);
            return mProjectionMap;
        } else {
            //problem - input and output schemas for a store have to match!
            mProjectionMap = null;
            return mProjectionMap;
        }
    }

    @Override
    public List<RequiredFields> getRequiredFields() {
        List<RequiredFields> requiredFields = new ArrayList<RequiredFields>();
        requiredFields.add(new RequiredFields(false, true));
        return requiredFields;
    }
    
    @Override
    public List<RequiredFields> getRelevantInputs(int output, int column) throws FrontendException {
        if (!mIsSchemaComputed)
            getSchema();
        
        if (output!=0)
            return null;
        
        if (column<0)
            return null;
        
        // if we have schema information, check if output column is valid
        if (mSchema!=null)
        {
            if (column >= mSchema.size())
                return null;
        }
        
        List<RequiredFields> result = new ArrayList<RequiredFields>();
        result.add(new RequiredFields(true));
        return result;
    }
    
    @Override
    public void setAlias(String newAlias) {
        super.setAlias(newAlias);
        signature = constructSignature(mAlias, mOutputFile.getFileName(), mOutputFile.getFuncSpec());
        mStoreFunc.setStoreFuncUDFContextSignature(signature);
    }
    
    public String getSignature() {
        return signature;
    }
}