/* * Data Hub Service (DHuS) - For Space data distribution. * Copyright (C) 2013,2014,2015 GAEL Systems * * This file is part of DHuS software sources. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as * published by the Free Software Foundation, either version 3 of the * License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Affero General Public License for more details. * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. */ package fr.gael.dhus.datastore.scanner; import java.io.IOException; import java.net.HttpURLConnection; import java.net.URISyntaxException; import java.net.URL; import java.util.Date; import java.util.HashMap; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.commons.codec.binary.Base64; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; import org.apache.http.client.utils.URIBuilder; import org.apache.olingo.odata2.api.edm.EdmLiteralKind; import org.apache.olingo.odata2.api.edm.EdmSimpleType; import org.apache.olingo.odata2.api.edm.provider.Facets; import org.apache.olingo.odata2.api.ep.entry.ODataEntry; import org.apache.olingo.odata2.api.ep.feed.ODataFeed; import org.apache.olingo.odata2.api.exception.ODataException; import org.apache.olingo.odata2.api.rt.RuntimeDelegate; import fr.gael.dhus.olingo.ODataClient; import static org.apache.olingo.odata2.api.edm.EdmSimpleTypeKind.DateTime; /** * This class scans another DHuS OData interface. * It will retrieve every unknown product to this DHuS. */ public class ODataScanner extends AbstractScanner { private static final Logger LOGGER = LogManager.getLogger(ODataScanner.class); private final ODataClient client; private final String uri, username, password; private long lastScanTime = 0L; public ODataScanner (String uri, boolean store_scan_list, String username, String password) throws URISyntaxException, IOException, ODataException { super (store_scan_list); this.uri = uri; this.username = username; this.password = password; // Workaround: if we are using this scanner to transfer a product. // see ProcessProductTransfer.upload() if (!uri.endsWith ("$value")) { // Creates an ODataClient for `uri`. this.client = new ODataClient (uri, username, password); LOGGER.info ( "ODataScanner on " + client.getServiceRoot () + " created."); } else { this.client = null; } } @Override public int scan () throws InterruptedException { // Workaround: transferring 1 product. see l.50 if (this.client == null) { try { URL url = new URL(this.uri); HttpURLConnection co = (HttpURLConnection) url.openConnection (); // HTTP Basic Authentication. String userpass = this.username + ":" + this.password; String basicAuth = "Basic " + new String(new Base64 ().encode (userpass.getBytes ())); co.setRequestProperty ("Authorization", basicAuth); co.connect (); String cd = co.getHeaderField ("Content-Disposition"); co.disconnect (); Matcher m = Pattern.compile (".*?filename=\"(.*?)\\.zip\".*?") .matcher ((cd)); if (!m.matches ()) throw new Exception ("filename not in header"); String filename = m.group (1); //url = new URIBuilder (url.toURI ()) // .setUserInfo (this.username, this.password).build ().toURL (); this.getScanList ().add (new URLExt (url, false, filename)); } catch (Exception e) { LOGGER.error ("failed to transfer " + this.uri, e); return 0; } return 1; } // The actual scan() code. int res = 0; try { // Prepare OData request, we want Products. String resource_path = new String ("/Products"); Facets facets = (new Facets ()).setNullable (false); Map<String, String> query_params = new HashMap<> (); long now = System.currentTimeMillis (); EdmSimpleType type = RuntimeDelegate.getEdmSimpleType (DateTime); // Filtering by ingestionDate. if (this.lastScanTime != 0L) { StringBuilder sb = new StringBuilder ("IngestionDate ge "); Date l_time = new Date (this.lastScanTime); sb.append (type.valueToString (l_time, EdmLiteralKind.URI, facets)); sb.append (" and IngestionDate lt "); l_time = new Date (now); sb.append (type.valueToString (l_time, EdmLiteralKind.URI, facets)); query_params.put ("$filter", sb.toString ()); } else { Date l_time = new Date (now); String value = "IngestionDate lt " + type.valueToString (l_time, EdmLiteralKind.URI, facets); query_params.put ("$filter", value); } // Ordering by ingestionDate. query_params.put ("$orderby", "IngestionDate"); // Pagination. int page_len = 50; // TODO get this value from config. query_params.put ("$top", String.valueOf (page_len)); try { for (long i=0; ; i++) { if (i!=0) query_params.put ("$skip", String.valueOf (page_len * i)); ODataFeed of = this.client.readFeed (resource_path, query_params); for (ODataEntry entry: of.getEntries ()) { String pdt_name = (String) entry.getProperties ().get("Name"); if (this.getUserPattern () == null || this.getUserPattern ().matcher (pdt_name).matches ()) { String key = (String) entry.getProperties ().get("Id"); URL url = new URIBuilder ( this.client.getServiceRoot () + "/Products" + "('" + key + "')/$value") .setUserInfo (this.username, this.password) .build ().toURL (); this.getScanList ().add ( new URLExt (url, false, pdt_name)); res += 1; } } if (of.getEntries ().size () != page_len) // End of pagination. break; } } catch (ODataException | IOException | URISyntaxException e) { LOGGER.error ("Product retrieval failed", e); } this.lastScanTime = now; } catch (ODataException e) { LOGGER.error (e); } return res; } }