package org.yamcs.web.rest.archive;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.yamcs.api.MediaType;
import org.yamcs.archive.CommandHistoryRecorder;
import org.yamcs.archive.EventRecorder;
import org.yamcs.archive.GPBHelper;
import org.yamcs.archive.XtceTmRecorder;
import org.yamcs.protobuf.Archive.TableData.TableRecord;
import org.yamcs.protobuf.Commanding.CommandHistoryEntry;
import org.yamcs.protobuf.Rest.BulkDownloadParameterValueRequest;
import org.yamcs.protobuf.SchemaArchive;
import org.yamcs.protobuf.SchemaCommanding;
import org.yamcs.protobuf.SchemaRest;
import org.yamcs.protobuf.SchemaYamcs;
import org.yamcs.protobuf.Yamcs.EndAction;
import org.yamcs.protobuf.Yamcs.Event;
import org.yamcs.protobuf.Yamcs.NamedObjectId;
import org.yamcs.protobuf.Yamcs.ParameterReplayRequest;
import org.yamcs.protobuf.Yamcs.ReplayRequest;
import org.yamcs.protobuf.Yamcs.ReplaySpeed;
import org.yamcs.protobuf.Yamcs.ReplaySpeed.ReplaySpeedType;
import org.yamcs.protobuf.Yamcs.TmPacketData;
import org.yamcs.security.Privilege;
import org.yamcs.security.Privilege.Type;
import org.yamcs.tctm.TmDataLinkInitialiser;
import org.yamcs.web.BadRequestException;
import org.yamcs.web.HttpException;
import org.yamcs.web.rest.ParameterReplayToChunkedCSVEncoder;
import org.yamcs.web.rest.ParameterReplayToChunkedProtobufEncoder;
import org.yamcs.web.rest.RestHandler;
import org.yamcs.web.rest.RestParameterReplayListener;
import org.yamcs.web.rest.RestRequest;
import org.yamcs.web.rest.RestRequest.IntervalResult;
import org.yamcs.web.rest.RestStreams;
import org.yamcs.web.rest.Route;
import org.yamcs.web.rest.SqlBuilder;
import org.yamcs.web.rest.StreamToChunkedCSVEncoder;
import org.yamcs.web.rest.StreamToChunkedProtobufEncoder;
import org.yamcs.web.rest.StreamToChunkedTransferEncoder;
import org.yamcs.xtce.Parameter;
import org.yamcs.xtce.XtceDb;
import org.yamcs.xtceproc.XtceDbFactory;
import org.yamcs.yarch.TableDefinition;
import org.yamcs.yarch.Tuple;
import org.yamcs.yarch.YarchDatabase;
import com.csvreader.CsvWriter;
import io.netty.buffer.ByteBufOutputStream;
/**
* Serves archived data through a web api.
*
* <p>Archive requests use chunked encoding with an unspecified content length, which enables
* us to send large dumps without needing to determine a content length on the server.
*/
public class ArchiveDownloadRestHandler extends RestHandler {
@Route(path = "/api/archive/:instance/downloads/parameters", method = {"GET", "POST"})
public void downloadParameters(RestRequest req) throws HttpException {
String instance = verifyInstance(req, req.getRouteParam("instance"));
ReplayRequest.Builder rr = ReplayRequest.newBuilder().setEndAction(EndAction.QUIT);
rr.setSpeed(ReplaySpeed.newBuilder().setType(ReplaySpeedType.AFAP));
// First try from body
BulkDownloadParameterValueRequest request = req.bodyAsMessage(SchemaRest.BulkDownloadParameterValueRequest.MERGE).build();
if (request.hasStart()) rr.setStart(RestRequest.parseTime(request.getStart()));
if (request.hasStop()) rr.setStop(RestRequest.parseTime(request.getStop()));
// Next, try query param (potentially overriding previous)
IntervalResult ir = req.scanForInterval();
if (ir.hasStart()) rr.setStart(req.getQueryParameterAsDate("start"));
if (ir.hasStop()) rr.setStop(req.getQueryParameterAsDate("stop"));
XtceDb mdb = XtceDbFactory.getInstance(instance);
List<NamedObjectId> ids = new ArrayList<>();
for (NamedObjectId id : request.getIdList()) {
Parameter p = mdb.getParameter(id);
if (p == null) {
throw new BadRequestException("Invalid parameter name specified "+id);
}
if (!Privilege.getInstance().hasPrivilege1(req.getAuthToken(), Type.TM_PARAMETER, p.getQualifiedName())) {
throw new BadRequestException("Insufficient privileges for parameter " + p.getQualifiedName());
}
ids.add(id);
}
if (ids.isEmpty()) {
for (Parameter p : mdb.getParameters()) {
if (!Privilege.getInstance().hasPrivilege1(req.getAuthToken(), Type.TM_PARAMETER, p.getQualifiedName())) {
continue;
}
if (request.hasNamespace()) {
String alias = p.getAlias(request.getNamespace());
if (alias != null) {
ids.add(NamedObjectId.newBuilder().setNamespace(request.getNamespace()).setName(alias).build());
}
} else {
ids.add(NamedObjectId.newBuilder().setName(p.getQualifiedName()).build());
}
}
}
rr.setParameterRequest(ParameterReplayRequest.newBuilder().addAllNameFilter(ids));
if (req.asksFor(MediaType.CSV)) {
RestParameterReplayListener l = new ParameterReplayToChunkedCSVEncoder(req, ids);
RestReplays.replay(instance, req.getAuthToken(), rr.build(), l);
} else {
RestParameterReplayListener l = new ParameterReplayToChunkedProtobufEncoder(req);
RestReplays.replay(instance, req.getAuthToken(), rr.build(), l);
}
}
@Route(path = "/api/archive/:instance/downloads/parameters/:name*", method = "GET")
public void downloadParameter(RestRequest req) throws HttpException {
String instance = verifyInstance(req, req.getRouteParam("instance"));
XtceDb mdb = XtceDbFactory.getInstance(instance);
NamedObjectId requestedId = verifyParameterId(req, mdb, req.getRouteParam("name"));
Parameter p = mdb.getParameter(requestedId);
ReplayRequest rr = ArchiveHelper.toParameterReplayRequest(req, p, false);
boolean noRepeat = req.getQueryParameterAsBoolean("norepeat", false);
if (req.asksFor(MediaType.CSV)) {
List<NamedObjectId> idList = Arrays.asList(requestedId);
RestParameterReplayListener l = new ParameterReplayToChunkedCSVEncoder(req, idList);
l.setNoRepeat(noRepeat);
RestReplays.replay(instance, req.getAuthToken(), rr, l);
} else {
RestParameterReplayListener l = new ParameterReplayToChunkedProtobufEncoder(req);
l.setNoRepeat(noRepeat);
RestReplays.replay(instance, req.getAuthToken(), rr, l);
}
}
@Route(path = "/api/archive/:instance/downloads/packets", method = "GET")
public void downloadPackets(RestRequest req) throws HttpException {
String instance = verifyInstance(req, req.getRouteParam("instance"));
Set<String> nameSet = new HashSet<>();
for (String names : req.getQueryParameterList("name", Collections.emptyList())) {
for (String name : names.split(",")) {
nameSet.add(name.trim());
}
}
SqlBuilder sqlb = new SqlBuilder(XtceTmRecorder.TABLE_NAME);
IntervalResult ir = req.scanForInterval();
if (ir.hasInterval()) {
sqlb.where(ir.asSqlCondition("gentime"));
}
if (!nameSet.isEmpty()) {
sqlb.where("pname in ('" + String.join("','", nameSet) + "')");
}
sqlb.descend(req.asksDescending(false));
String sql = sqlb.toString();
if (req.asksFor(MediaType.OCTET_STREAM)) {
RestStreams.stream(instance, sql, new StreamToChunkedTransferEncoder(req, MediaType.OCTET_STREAM) {
@Override
public void processTuple(Tuple tuple, ByteBufOutputStream bufOut) throws IOException {
byte[] raw = (byte[]) tuple.getColumn(TmDataLinkInitialiser.PACKET_COLUMN);
bufOut.write(raw);
}
});
} else {
RestStreams.stream(instance, sql, new StreamToChunkedProtobufEncoder<TmPacketData>(req, SchemaYamcs.TmPacketData.WRITE) {
@Override
public TmPacketData mapTuple(Tuple tuple) {
return GPBHelper.tupleToTmPacketData(tuple);
}
});
}
}
@Route(path = "/api/archive/:instance/downloads/commands", method = "GET")
public void downloadCommands(RestRequest req) throws HttpException {
String instance = verifyInstance(req, req.getRouteParam("instance"));
Set<String> nameSet = new HashSet<>();
for (String names : req.getQueryParameterList("name", Collections.emptyList())) {
for (String name : names.split(",")) {
nameSet.add(name.trim());
}
}
SqlBuilder sqlb = new SqlBuilder(CommandHistoryRecorder.TABLE_NAME);
IntervalResult ir = req.scanForInterval();
if (ir.hasInterval()) {
sqlb.where(ir.asSqlCondition("gentime"));
}
if (!nameSet.isEmpty()) {
sqlb.where("cmdName in ('" + String.join("','", nameSet) + "')");
}
sqlb.descend(req.asksDescending(false));
String sql = sqlb.toString();
RestStreams.stream(instance, sql, new StreamToChunkedProtobufEncoder<CommandHistoryEntry>(req, SchemaCommanding.CommandHistoryEntry.WRITE) {
@Override
public CommandHistoryEntry mapTuple(Tuple tuple) {
return GPBHelper.tupleToCommandHistoryEntry(tuple);
}
});
}
@Route(path = "/api/archive/:instance/downloads/tables/:name", method = "GET")
public void downloadTableData(RestRequest req) throws HttpException {
String instance = verifyInstance(req, req.getRouteParam("instance"));
YarchDatabase ydb = YarchDatabase.getInstance(instance);
TableDefinition table = verifyTable(req, ydb, req.getRouteParam("name"));
boolean dumpFormat = req.hasQueryParameter("format") && "dump".equalsIgnoreCase(req.getQueryParameter("format"));
List<String> cols = null;
if (req.hasQueryParameter("cols")) {
cols = new ArrayList<>(); // Order, and non-unique
for (String para : req.getQueryParameterList("cols")) {
for (String col : para.split(",")) {
cols.add(col.trim());
}
}
}
SqlBuilder sqlb = new SqlBuilder(table.getName());
if (cols != null) {
if (cols.isEmpty()) {
throw new BadRequestException("No columns were specified");
} else {
cols.forEach(col -> sqlb.select(col));
}
}
sqlb.descend(req.asksDescending(false));
String sql = sqlb.toString();
if (dumpFormat) {
RestStreams.stream(instance, sql, new TableDumpEncoder(req));
} else {
RestStreams.stream(instance, sql, new StreamToChunkedProtobufEncoder<TableRecord>(req, SchemaArchive.TableData.TableRecord.WRITE) {
@Override
public TableRecord mapTuple(Tuple tuple) {
TableRecord.Builder rec = TableRecord.newBuilder();
rec.addAllColumn(ArchiveHelper.toColumnDataList(tuple));
return rec.build();
}
});
}
}
@Route(path = "/api/archive/:instance/downloads/events", method = "GET")
public void downloadEvents(RestRequest req) throws HttpException {
String instance = verifyInstance(req, req.getRouteParam("instance"));
ArchiveEventRestHandler.verifyEventArchiveSupport(instance);
Set<String> sourceSet = new HashSet<>();
for (String names : req.getQueryParameterList("source", Collections.emptyList())) {
for (String name : names.split(",")) {
sourceSet.add(name);
}
}
SqlBuilder sqlb = new SqlBuilder(EventRecorder.TABLE_NAME);
IntervalResult ir = req.scanForInterval();
if (ir.hasInterval()) {
sqlb.where(ir.asSqlCondition("gentime"));
}
if (!sourceSet.isEmpty()) {
sqlb.where("source in ('" + String.join("','", sourceSet) + "')");
}
sqlb.descend(req.asksDescending(false));
String sql = sqlb.toString();
if (req.asksFor(MediaType.CSV)) {
transferChunkedCSVEvents(req, instance, sql);
} else {
transferChunkedProtobufEvents(req, instance, sql);
}
}
private void transferChunkedCSVEvents(RestRequest req, String instance, String sql) throws HttpException {
RestStreams.stream(instance, sql, new StreamToChunkedCSVEncoder(req) {
@Override
public String[] getCSVHeader() {
return ArchiveHelper.EVENT_CSV_HEADER;
}
@Override
public void processTuple(Tuple tuple, CsvWriter csvWriter) throws IOException {
String[] record = ArchiveHelper.tupleToCSVEvent(tuple);
csvWriter.writeRecord(record);
}
});
}
private void transferChunkedProtobufEvents(RestRequest req, String instance, String sql) throws HttpException {
RestStreams.stream(instance, sql, new StreamToChunkedProtobufEncoder<Event>(req, SchemaYamcs.Event.WRITE) {
@Override
public Event mapTuple(Tuple tuple) {
return ArchiveHelper.tupleToEvent(tuple);
}
});
}
}