CheckpointFilterBuilder.java

/*
 * Copyright (C) 2015 Stratio (http://stratio.com)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.stratio.morphlines.checkpointfilter.builder;

import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;

import org.kitesdk.morphline.api.Command;
import org.kitesdk.morphline.api.CommandBuilder;
import org.kitesdk.morphline.api.MorphlineContext;
import org.kitesdk.morphline.api.Record;
import org.kitesdk.morphline.base.AbstractCommand;

import com.google.common.collect.ImmutableMap;
import com.stratio.morphlines.checkpointfilter.handler.CheckpointFilterHandler;
import com.stratio.morphlines.checkpointfilter.type.CheckpointType;
import com.typesafe.config.Config;

//@formatter:off
/**
 * <p>The checkpointFilter command discard records that are less than specified checkpoint field value.</p>
 * <ul>
 * <li><em>handler</em></li>
 * <li><em>field</em></li>
 * <li><em>type</em></li>
 * <li><em>format</em></li>
 * <li><em>chunksize</em></li>
 * </ul>
 * <code>
 * Example:
 * {
 *     checkpointFilter {
 *         handler: com.stratio.morphlines.checkpointfilter.handler.MongoCheckpointHandler
 *         field: date
 *         type: com.stratio.morphlines.checkpointfilter.type.DateCheckpointType
 *         format: "dd/MM/yyyy"
 *         chunksize: 1000
 *         mongoUri: mongodb://127.0.0.1:27017/mydb.checkpoints
 *     }
 * }
 * </code>
 */
//@formatter:on
public class CheckpointFilterBuilder extends AbstractCheckpointFilter implements CommandBuilder {

    private static final String COMMAND_NAME = "checkpointFilter";
    private static final String CONF_HANDLER = "handler";
    private static final String CONF_FIELD = "field";
    private static final String CONF_TYPE = "type";
    private static final String CONF_FORMAT = "format";
    private static final String CONF_CHUNKSIZE = "chunksize";
    private static final String CONF_MONGO_URI = "mongoUri";

    public Collection<String> getNames() {
        return Collections.singleton(COMMAND_NAME);
    }

    public Command build(Config config, Command parent, Command child, MorphlineContext context) {
        return new CheckpointFilter(this, config, parent, child, context);
    }

    private static final class CheckpointFilter extends AbstractCommand {

        private final CheckpointFilterHandler handler;
        private final String field;
        private final CheckpointType type;
        private final String format;
        private final int chunksize;
        private final String mongoUri;
        private String lastCheckpoint;

        private Map<String, String> filterContext;
        private int processedRecordsCounter = 0;

        public CheckpointFilter(CommandBuilder builder, Config config, Command parent,
                Command child, MorphlineContext context) {
            super(builder, config, parent, child, context);

            field = getConfigs().getString(config, CONF_FIELD);
            format = getConfigs().getString(config, CONF_FORMAT);
            chunksize = Integer.valueOf(getConfigs().getString(config, CONF_CHUNKSIZE));
            mongoUri = getConfigs().getString(config, CONF_MONGO_URI);
            initContext(config);
            type = getTypeInstance(filterContext);
            handler = getHandlerInstance(type, filterContext);
            lastCheckpoint = handler.getLastCheckpoint(filterContext);
            filterContext.put("lastCheckpoint", lastCheckpoint);
        }

        @Override
        protected void doNotify(Record notification) {
            super.doNotify(notification);
        }

        @Override
        protected boolean doProcess(Record record) {
            if (type.isValidCurrentCheckpoint(getCheckpointField(record), filterContext)) {
                return processRecord(record);
            }
            return true;
        }

        private Object getCheckpointField(Record record) {
          return record.get(field).get(0);
        }

        private boolean processRecord(Record record) {
            processedRecordsCounter++;
            if (processedRecordsCounter >= chunksize) {
                processedRecordsCounter = 0;
                filterContext.put("lastCheckpoint", (String) type.getCheckpoint(record, filterContext));
                record.put("checkpoint", "true");
            }
            return super.doProcess(record);
        }

        private void initContext(Config config) {
            filterContext = new HashMap<String, String>(ImmutableMap.<String, String>builder()
                    .put("field", field)
                    .put("format", format)
                    .put("chunksize", String.valueOf(chunksize))
                    .put("mongoUri", mongoUri)
                    .put("type", getConfigs().getString(config, CONF_TYPE))
                    .put("handler", getConfigs().getString(config, CONF_HANDLER))
                    .build());
        }
    }
}