/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.addthis.hydra.task.output; import javax.annotation.Nonnull; import java.util.Arrays; import java.util.List; import java.nio.file.Path; import com.addthis.bundle.channel.DataChannelError; import com.addthis.bundle.core.Bundle; import com.addthis.bundle.core.Bundles; import com.addthis.codec.annotations.FieldConfig; import com.google.common.collect.ImmutableList; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * This output sink <span class="hydra-summary">writes to multiple output sinks</span>. * <p/> * <p>Each bundle will be emitted to the first sink, and then the second sink, etc. * The {@link #immutableCopy immutableCopy} parameter can be specified if one * of the children sinks will modify a bundle, and you do not wish those modifications * to appear in the remaining children sinks. * <p/> * <p>Example:</p> * <pre> * output: [ * {file { * path:[ "{{DATE_YMD}}", "/","{{PATH_TYPE}}", "/","{{SHARD}}"] * ... * }} * {tree { * root:{const:"ROOT"} * ... * }} * ] * </pre> * * @user-reference */ public class TaskDataOutputChain extends DataOutputTypeList { private static final Logger log = LoggerFactory.getLogger(TaskDataOutputChain.class); /** * Sequence of output sinks. Each bundle is emitted to the first sink, * then the second sink, etc. */ @FieldConfig(codable = true, required = true) private TaskDataOutput[] outputs; /** * If true then create copy of the bundle for each output. Default value is true. */ @FieldConfig(codable = true) private boolean copy = true; /** * If true then create a deep copy of a bundle when it is passed to a child * output sink. This may be useful when the child sink modifies * the bundle. Default value is false. */ @FieldConfig(codable = true) private boolean immutableCopy = false; @Override protected void open() { log.info("[init] beginning init chain"); for (int i = 0; i < outputs.length; i++) { outputs[i].open(); } log.info("[init] all outputs initialized"); } @Override public void send(Bundle row) throws DataChannelError { if (!copy && !immutableCopy) { Bundle withPreviousFormat = row; for (TaskDataOutput output : outputs) { // preserves all mutations, but maintains the typical (admittedly horrible) format behavior withPreviousFormat = Bundles.shallowCopyBundle(withPreviousFormat, output.createBundle()); output.send(withPreviousFormat); } } else if (immutableCopy) { for (TaskDataOutput output : outputs) { output.send(Bundles.deepCopyBundle(row, output.createBundle())); } } else { for (TaskDataOutput output : outputs) { output.send(Bundles.shallowCopyBundle(row, output.createBundle())); } } } @Override public void send(List<Bundle> bundles) { if (bundles != null && !bundles.isEmpty()) { for (Bundle bundle : bundles) { send(bundle); } } } @Override public void sendComplete() { log.info("[sendComplete] forwarding completion signal to all outputs"); for (TaskDataOutput output : outputs) { output.sendComplete(); } log.info("[sendComplete] forwarding complete"); } @Override public void sourceError(Throwable er) { log.error("[sourceError] forwarding to all outputs: ", er); for (TaskDataOutput output : outputs) { output.sourceError(er); } log.error("[sourceError] forwarding complete"); } @Nonnull @Override public ImmutableList<Path> writableRootPaths() { return ImmutableList.copyOf( Arrays.stream(outputs).flatMap( output -> output.writableRootPaths().stream()).iterator()); } }