/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.processors.standard;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.nifi.annotation.behavior.EventDriven;
import org.apache.nifi.annotation.behavior.InputRequirement;
import org.apache.nifi.annotation.behavior.SideEffectFree;
import org.apache.nifi.annotation.behavior.SupportsBatching;
import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.SeeAlso;
import org.apache.nifi.annotation.documentation.Tags;
import org.apache.nifi.annotation.lifecycle.OnScheduled;
import org.apache.nifi.components.AllowableValue;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.components.ValidationContext;
import org.apache.nifi.components.ValidationResult;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.processor.ProcessContext;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.record.path.FieldValue;
import org.apache.nifi.record.path.RecordPath;
import org.apache.nifi.record.path.RecordPathResult;
import org.apache.nifi.record.path.util.RecordPathCache;
import org.apache.nifi.record.path.validation.RecordPathPropertyNameValidator;
import org.apache.nifi.serialization.record.Record;
import org.apache.nifi.serialization.record.RecordSchema;
@EventDriven
@SideEffectFree
@SupportsBatching
@InputRequirement(Requirement.INPUT_REQUIRED)
@Tags({"update", "record", "generic", "schema", "json", "csv", "avro", "log", "logs", "freeform", "text"})
@CapabilityDescription("Updates the contents of a FlowFile that contains Record-oriented data (i.e., data that can be read via a RecordReader and written by a RecordWriter). "
+ "This Processor requires that at least one user-defined Property be added. The name of the Property should indicate a RecordPath that determines the field that should "
+ "be updated. The value of the Property is either a replacement value (optionally making use of the Expression Language) or is itself a RecordPath that extracts a value from "
+ "the Record. Whether the Property value is determined to be a RecordPath or a literal value depends on the configuration of the <Replacement Value Strategy> Property.")
@SeeAlso({ConvertRecord.class})
public class UpdateRecord extends AbstractRecordProcessor {
private volatile RecordPathCache recordPathCache;
private volatile List<String> recordPaths;
static final AllowableValue LITERAL_VALUES = new AllowableValue("literal-value", "Literal Value",
"The value entered for a Property (after Expression Language has been evaluated) is the desired value to update the Record Fields with.");
static final AllowableValue RECORD_PATH_VALUES = new AllowableValue("record-path-value", "Record Path Value",
"The value entered for a Property (after Expression Language has been evaluated) is not the literal value to use but rather is a Record Path "
+ "that should be evaluated against the Record, and the result of the RecordPath will be used to update the Record. Note that if this option is selected, "
+ "and the Record Path results in multiple values for a given Record, the input FlowFile will be routed to the 'failure' Relationship.");
static final PropertyDescriptor REPLACEMENT_VALUE_STRATEGY = new PropertyDescriptor.Builder()
.name("replacement-value-strategy")
.displayName("Replacement Value Strategy")
.description("Specifies how to interpret the configured replacement values")
.allowableValues(LITERAL_VALUES, RECORD_PATH_VALUES)
.defaultValue(LITERAL_VALUES.getValue())
.expressionLanguageSupported(false)
.required(true)
.build();
@Override
protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
final List<PropertyDescriptor> properties = new ArrayList<>(super.getSupportedPropertyDescriptors());
properties.add(REPLACEMENT_VALUE_STRATEGY);
return properties;
}
@Override
protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(final String propertyDescriptorName) {
return new PropertyDescriptor.Builder()
.name(propertyDescriptorName)
.description("Specifies the value to use to replace fields in the record that match the RecordPath: " + propertyDescriptorName)
.required(false)
.dynamic(true)
.expressionLanguageSupported(true)
.addValidator(new RecordPathPropertyNameValidator())
.build();
}
@Override
protected Collection<ValidationResult> customValidate(final ValidationContext validationContext) {
final boolean containsDynamic = validationContext.getProperties().keySet().stream()
.anyMatch(property -> property.isDynamic());
if (containsDynamic) {
return Collections.emptyList();
}
return Collections.singleton(new ValidationResult.Builder()
.subject("User-defined Properties")
.valid(false)
.explanation("At least one RecordPath must be specified")
.build());
}
@OnScheduled
public void createRecordPaths(final ProcessContext context) {
recordPathCache = new RecordPathCache(context.getProperties().size() * 2);
final List<String> recordPaths = new ArrayList<>(context.getProperties().size() - 2);
for (final PropertyDescriptor property : context.getProperties().keySet()) {
if (property.isDynamic()) {
recordPaths.add(property.getName());
}
}
this.recordPaths = recordPaths;
}
@Override
protected Record process(final Record record, final RecordSchema writeSchema, final FlowFile flowFile, final ProcessContext context) {
final boolean evaluateValueAsRecordPath = context.getProperty(REPLACEMENT_VALUE_STRATEGY).getValue().equals(RECORD_PATH_VALUES.getValue());
// Incorporate the RecordSchema that we will use for writing records into the Schema that we have
// for the record, because it's possible that the updates to the record will not be valid otherwise.
record.incorporateSchema(writeSchema);
for (final String recordPathText : recordPaths) {
final RecordPath recordPath = recordPathCache.getCompiled(recordPathText);
final RecordPathResult result = recordPath.evaluate(record);
final String replacementValue = context.getProperty(recordPathText).evaluateAttributeExpressions(flowFile).getValue();
if (evaluateValueAsRecordPath) {
final RecordPath replacementRecordPath = recordPathCache.getCompiled(replacementValue);
// If we have an Absolute RecordPath, we need to evaluate the RecordPath only once against the Record.
// If the RecordPath is a Relative Path, then we have to evaluate it against each FieldValue.
if (replacementRecordPath.isAbsolute()) {
processAbsolutePath(replacementRecordPath, result.getSelectedFields(), record, replacementValue);
} else {
processRelativePath(replacementRecordPath, result.getSelectedFields(), record, replacementValue);
}
} else {
result.getSelectedFields().forEach(fieldVal -> fieldVal.updateValue(replacementValue));
}
}
return record;
}
private void processAbsolutePath(final RecordPath replacementRecordPath, final Stream<FieldValue> destinationFields, final Record record, final String replacementValue) {
final RecordPathResult replacementResult = replacementRecordPath.evaluate(record);
final Object replacementObject = getReplacementObject(replacementResult, replacementValue);
destinationFields.forEach(fieldVal -> fieldVal.updateValue(replacementObject));
}
private void processRelativePath(final RecordPath replacementRecordPath, final Stream<FieldValue> destinationFields, final Record record, final String replacementValue) {
destinationFields.forEach(fieldVal -> {
final RecordPathResult replacementResult = replacementRecordPath.evaluate(fieldVal);
final Object replacementObject = getReplacementObject(replacementResult, replacementValue);
fieldVal.updateValue(replacementObject);
});
}
private Object getReplacementObject(final RecordPathResult recordPathResult, final String replacementValue) {
final List<FieldValue> selectedFields = recordPathResult.getSelectedFields().collect(Collectors.toList());
if (selectedFields.size() > 1) {
throw new ProcessException("Cannot update Record because the Replacement Record Path \"" + replacementValue + "\" yielded "
+ selectedFields.size() + " results but this Processor only supports a single result.");
}
if (selectedFields.isEmpty()) {
return null;
} else {
return selectedFields.get(0).getValue();
}
}
}