/* Copyright (c) 2011 Danish Maritime Authority.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package dk.dma.ais.lib;
import com.beust.jcommander.Parameter;
import com.google.inject.Injector;
import dk.dma.ais.packet.AisPacket;
import dk.dma.ais.packet.AisPacketOutputSinks;
import dk.dma.ais.packet.AisPacketReader;
import dk.dma.commons.app.AbstractCommandLineTool;
import dk.dma.commons.util.io.OutputStreamSink;
import dk.dma.enav.util.function.EConsumer;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.lang.reflect.Field;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
/**
* Converts a set of files into another sink format
*
* @author Jens Tuxen
* @author Thomas Borg Salling
*
*/
public class FileConvert extends AbstractCommandLineTool {
/** The logger. */
static final Logger LOG = LoggerFactory.getLogger(FileConvert.class);
@Parameter(required = true, description = "files to import...")
List<String> sources;
/** Where files should be moved to after having been processed. */
@Parameter(names = "-path", required = false, description = "base-path to put converted files")
String convertTo = "converted/";
/** Where files should be moved to after having been processed. */
@Parameter(names = "-keepStructure", required = false, description = "Whether to keep path structure")
boolean keepFileStructure;
@Parameter(names = "-fileEnding", required = false, description = "File ending")
String fileEnding;
@Parameter(names = "-outputFormat", required = false, description = "Output formats: [OUTPUT_TO_TEXT, OUTPUT_PREFIXED_SENTENCES, OUTPUT_TO_HTML, table, csv, csv_stateful, json, jsonobject, kml, kmz]")
String outputSinkFormat = "OUTPUT_PREFIXED_SENTENCES";
@Parameter(names = "-columns", required = false, description = "Optional columns, required with -outputFormat table. use ; as delimiter. Example: -columns mmsi;time;lat;lon")
String columns;
@Parameter(names = "-kmzPrimaryMmsi", required = false, description = "For use with '-outputFormat kmz': Primary mmsi no. Affects color of vessel.")
int kmzPrimaryMmsi = -1;
@Parameter(names = "-kmzSecondaryMmsi", required = false, description = "For use with '-outputFormat kmz': Secondary mmsi no. Affects color of vessel.")
int kmzSecondaryMmsi = -1;
@Parameter(names = "-kmzSnapshotAt", required = false, description = "For use with '-outputFormat kmz': Date and time in datastream for creation of snapshot folder. [YYYY-MM-DD-hh:mm]")
String kmzSnapshotAt;
DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd-HH:mm");
public FileConvert() {
}
/**
* getOutputSinks
*
* @return
*/
static final List<String> getOutputSinks() {
LinkedList<String> methods = new LinkedList<String>();
Field[] declaredMethods = AisPacketOutputSinks.class.getDeclaredFields();
for (Field m : declaredMethods) {
if (m.getName().startsWith("OUTPUT")) {
methods.add(m.getName());
}
}
return methods;
}
/** {@inheritDoc} */
@Override
protected void run(Injector injector) throws Exception {
configureFileEnding();
final EConsumer<String> consumer = new EConsumer<String>() {
@Override
public void accept(String s) throws IllegalArgumentException, IllegalAccessException, NoSuchFieldException,
SecurityException, IOException, InterruptedException {
Path path = Paths.get(s);
LOG.debug("Started processing file " + path);
Path endPath;
if (keepFileStructure) {
Path relative;
relative = path;
endPath = Paths.get(Paths.get(convertTo).toString(), relative.toString());
new File(endPath.toString()).mkdirs();
} else {
endPath = Paths.get("");
}
String filename = path.getFileName().toString();
if (! filename.endsWith(fileEnding))
filename = FilenameUtils.removeExtension(filename) + fileEnding;
Path filePath = Paths.get(endPath.toString(), filename);
LOG.debug("Output File: " + filePath.toString());
final OutputStream fos = new FileOutputStream(filePath.toString()); // 2
final boolean createSituationFolder = !StringUtils.isBlank(kmzSnapshotAt);
final long snapshotAtEpochMillis = createSituationFolder ? LocalDateTime.parse(kmzSnapshotAt, formatter).toInstant(ZoneOffset.UTC).toEpochMilli() : -1;
OutputStreamSink<AisPacket> sink;
if ("kmz".equals(outputSinkFormat)) {
//AisPacketKMZOutputSink(filter, createSituationFolder, createMovementsFolder, createTracksFolder, isPrimaryTarget, isSecondaryTarget, triggerSnapshot, snapshotDescriptionSupplier, movementInterpolationStep, supplyTitle, supplyDescription, iconHrefSupplier);
sink = AisPacketOutputSinks.newKmzSink(
e -> true, // this.filter = e -> true;
createSituationFolder, // this.createSituationFolder = true;
true, // createMovementsFolder = true;
true, // this.createTracksFolder = true;
e -> kmzPrimaryMmsi <= 0 ? false : e.tryGetAisMessage().getUserId() == kmzPrimaryMmsi, // this.isPrimaryTarget = e -> false;
e -> kmzSecondaryMmsi <= 0 ? false : e.tryGetAisMessage().getUserId() == kmzSecondaryMmsi, // this.isSecondaryTarget = e -> false;
e -> e.getBestTimestamp() >= snapshotAtEpochMillis, // this.triggerSnapshot = e -> false;
() -> "Situation at " + kmzSnapshotAt, // this.snapshotDescriptionSupplier = null;
() -> 10, // this.title = defaultTitleSupplier;
() -> "description", // this.description = defaultDescriptionSupplier;
() -> "10", //this.movementInterpolationStep = defaultMovementInterpolationStepSupplier;
(shipTypeCargo, navigationalStatus) -> "" // this.iconHrefSupplier = defaultIconHrefSupplier;
);
} else
sink = AisPacketOutputSinks.getOutputSink(outputSinkFormat, columns);
sink.closeWhenFooterWritten();
AisPacketReader apis = AisPacketReader.createFromFile(path, false);
apis.writeTo(fos, sink);
apis.close();
fos.close();
}
};
/*
* Creates a pool of executors, 4 threads. Each thread will open a file using an aispacket reader, 10000 files can be
* submitted to the queue, afterwards the calling thread will execute the job instead.
*/
ThreadPoolExecutor threadpoolexecutor = new ThreadPoolExecutor(4, 4, 1, TimeUnit.SECONDS, new ArrayBlockingQueue<>(
10000), new ThreadPoolExecutor.CallerRunsPolicy());
for (final String s : sources) {
threadpoolexecutor.execute(() -> {
try {
consumer.accept(s);
} catch (Exception e) {
e.printStackTrace();
}
});
}
threadpoolexecutor.shutdown();
threadpoolexecutor.awaitTermination(999, TimeUnit.DAYS);
}
private void configureFileEnding() {
if (StringUtils.isBlank(fileEnding)) {
if (outputSinkFormat.startsWith("csv")) {
fileEnding = ".csv";
} else if (outputSinkFormat.startsWith("kmz")) {
fileEnding = ".kmz";
} else if (outputSinkFormat.startsWith("json")) {
fileEnding = ".json";
} else if (outputSinkFormat.startsWith("kml")) {
fileEnding = ".kml";
} else {
fileEnding = ".txt";
}
}
}
public static void main(String[] args) throws Exception {
FileConvert fc = new FileConvert();
fc.execute(args);
}
}