/* * Data Hub Service (DHuS) - For Space data distribution. * Copyright (C) 2013,2014,2015,2016 GAEL Systems * * This file is part of DHuS software sources. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as * published by the Free Software Foundation, either version 3 of the * License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Affero General Public License for more details. * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. */ package fr.gael.dhus.sync.impl; import fr.gael.dhus.database.object.MetadataIndex; import fr.gael.dhus.database.object.Product; import fr.gael.dhus.database.object.SynchronizerConf; import fr.gael.dhus.datastore.IncomingManager; import fr.gael.dhus.olingo.ODataClient; import fr.gael.dhus.olingo.v1.Model; import fr.gael.dhus.service.CollectionService; import fr.gael.dhus.service.ISynchronizerService; import fr.gael.dhus.service.MetadataTypeService; import fr.gael.dhus.service.ProductService; import fr.gael.dhus.service.SearchService; import fr.gael.dhus.service.metadata.MetadataType; import fr.gael.dhus.spring.context.ApplicationContextProvider; import fr.gael.dhus.sync.Synchronizer; import fr.gael.dhus.util.http.HttpAsyncClientProducer; import fr.gael.dhus.util.http.ParallelizedDownloadManager; import fr.gael.dhus.util.http.ParallelizedDownloadManager.DownloadResult; import fr.gael.dhus.util.http.Timeouts; import java.io.File; import java.io.IOException; import java.net.URISyntaxException; import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import java.security.DigestException; import java.util.ArrayList; import java.util.Collections; import java.util.Date; import java.util.GregorianCalendar; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.CancellationException; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import org.apache.http.auth.AuthScope; import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.client.CredentialsProvider; import org.apache.http.client.config.CookieSpecs; import org.apache.http.client.config.RequestConfig; import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.impl.nio.client.CloseableHttpAsyncClient; import org.apache.http.impl.nio.client.HttpAsyncClients; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.olingo.odata2.api.edm.EdmLiteralKind; import org.apache.olingo.odata2.api.edm.EdmSimpleTypeKind; import org.apache.olingo.odata2.api.ep.entry.ODataEntry; import org.apache.olingo.odata2.api.ep.feed.ODataDeltaFeed; import org.apache.olingo.odata2.api.ep.feed.ODataFeed; import org.apache.olingo.odata2.api.exception.ODataException; import org.hibernate.HibernateException; import org.hibernate.exception.LockAcquisitionException; import org.springframework.dao.CannotAcquireLockException; import org.springframework.security.crypto.codec.Hex; /** * A synchronizer using the OData API of another DHuS. */ public class ODataProductSynchronizer extends Synchronizer { /** Log. */ private static final Logger LOGGER = LogManager.getLogger(ODataProductSynchronizer.class); /** Synchronizer Service, to save the */ private static final ISynchronizerService SYNC_SERVICE = ApplicationContextProvider.getBean (ISynchronizerService.class); /** Product service, to store Products in the database. */ private static final ProductService PRODUCT_SERVICE = ApplicationContextProvider.getBean (ProductService.class); /** Metadata Type Service, MetadataIndex name to Queryable. */ private static final MetadataTypeService METADATA_TYPE_SERVICE = ApplicationContextProvider.getBean (MetadataTypeService.class); /** Search Service, to add a new product in the index. */ private static final SearchService SEARCH_SERVICE = ApplicationContextProvider.getBean (SearchService.class); /** Collection Service, for to add a Product in the configured targetCollection. */ private static final CollectionService COLLECTION_SERVICE = ApplicationContextProvider.getBean (CollectionService.class); /** Incoming manager, tells where to download products. */ private static final IncomingManager INCOMING_MANAGER = ApplicationContextProvider.getBean (IncomingManager.class); /** An {@link ODataClient} configured to query another DHuS OData service. */ private final ODataClient client; /** Credentials: username. */ private final String serviceUser; /** Credentials: password. */ private final String servicePass; /** Path to the remote DHuS incoming directory (if accessible). */ private final String remoteIncoming; /** Adds every new product in this collection. */ private final String targetCollectionUUID; /** OData resource path to a remote source collection: "Collections('a')/.../Collections('z')" */ private final String sourceCollection; /** True if this synchronizer must download a local copy of the product. */ private final boolean copyProduct; /** Custom $filter parameter, to be added to the query URI. */ private final String filterParam; /** Last created product's updated time. */ private Date lastCreated; /** Last updated product's updated time. */ private Date lastUpdated; /** Last deleted product's deletion time. */ private Date lastDeleted; /** Set to true whenever one of the three date fields is modified. */ private boolean dateChanged = false; /** Size of a Page (count of products to retrieve at once). */ private int pageSize; /** * Creates a new ODataSynchronizer. * * @param sc configuration for this synchronizer. * * @throws IllegalStateException if the configuration doe not contains the * required fields, or those fields are malformed. * @throws IOException when the OdataClient fails to contact the server * at {@code url}. * @throws ODataException when no OData service have been found at the * given url. * @throws NumberFormatException if the value of the `target_collection` * configuration field is not a number. */ public ODataProductSynchronizer (SynchronizerConf sc) throws IOException, ODataException { super (sc); // Checks if required configuration is set String urilit = sc.getConfig ("service_uri"); serviceUser = sc.getConfig ("service_username"); servicePass = sc.getConfig ("service_password"); if (urilit == null || urilit.isEmpty ()) { throw new IllegalStateException ("`service_uri` is not set"); } try { client = new ODataClient (urilit, serviceUser, servicePass); } catch (URISyntaxException e) { throw new IllegalStateException ("`service_uri` is malformed"); } String dec_name = client.getSchema ().getDefaultEntityContainer ().getName (); if (!dec_name.equals(Model.ENTITY_CONTAINER)) { throw new IllegalStateException ("`service_uri` does not reference a DHuS odata service"); } String last_cr = sc.getConfig ("last_created"); if (last_cr != null && !last_cr.isEmpty ()) { lastCreated = new Date (Long.decode (last_cr)); } else { lastCreated = new Date (0L); } String last_up = sc.getConfig ("last_updated"); if (last_up != null && !last_up.isEmpty ()) { lastUpdated = new Date (Long.decode (last_up)); } else { lastUpdated = new Date (0L); } String last_del = sc.getConfig ("last_deleted"); if (last_del != null && !last_del.isEmpty ()) { lastDeleted = new Date (Long.decode (last_del)); } else { lastDeleted = new Date (0L); } String page_size = sc.getConfig ("page_size"); if (page_size != null && !page_size.isEmpty ()) { pageSize = Integer.decode (page_size); } else { pageSize = 30; // FIXME get that value from the config? } String remote_incoming = sc.getConfig ("remote_incoming_path"); if (remote_incoming != null && !remote_incoming.isEmpty ()) { File ri = new File (remote_incoming); if (!ri.exists () || !ri.isDirectory () || !ri.canRead ()) { throw new IOException ("Cannot access remote incoming " + remote_incoming); } this.remoteIncoming = remote_incoming; } else { this.remoteIncoming = null; } String target_collection = sc.getConfig ("target_collection"); if (target_collection != null && !target_collection.isEmpty ()) { this.targetCollectionUUID = target_collection; } else { this.targetCollectionUUID = null; } String filter_param = sc.getConfig ("filter_param"); if (filter_param != null && !filter_param.isEmpty ()) { filterParam = filter_param; } else { filterParam = null; } String source_collection = sc.getConfig("source_collection"); if (source_collection != null && !source_collection.isEmpty()) { sourceCollection = source_collection; } else { sourceCollection = ""; } String copy_product = sc.getConfig ("copy_product"); if (copy_product != null && !copy_product.isEmpty ()) { this.copyProduct = Boolean.parseBoolean (copy_product); } else { this.copyProduct = false; } } /** Logs how much time an OData command consumed. */ private void logODataPerf(String query, long delta_time) { LOGGER.debug ("Synchronizer#" + getId () + " query(" + query + ") done in " + delta_time + "ms"); } /** * Downloads a product, * returns 3 Futures, 1st is the product, 2nd is the quicklook and 3rd is the thumbnail. * 2nd and 3rd Futures may be null! */ private Future<DownloadResult>[] download(ParallelizedDownloadManager downloader, Product p) throws IOException, InterruptedException { @SuppressWarnings("unchecked") Future<DownloadResult>[] res = new Future[3]; res[0] = downloader.download(p.getOrigin()); // Downloads and sets the quicklook and thumbnail (if any) if (p.getQuicklookFlag()) { res[1] = downloader.download(p.getQuicklookPath()); } if (p.getThumbnailFlag()) { res[2] = downloader.download(p.getThumbnailPath()); } return res; } /** * Gets `pageSize` products from the data source. * @param optional_skip an optional $skip parameter, may be null. * @param expand_navlinks if `true`, the query will contain: `$expand=Class,Attributes,Products`. */ private ODataFeed getPage(Integer optional_skip, boolean expand_navlinks) throws ODataException, IOException, InterruptedException { // Makes the query parameters Map<String, String> query_param = new HashMap<>(); String lup_s = EdmSimpleTypeKind.DateTime.getEdmSimpleTypeInstance() .valueToString(lastCreated, EdmLiteralKind.URI, null); // 'GreaterEqual' because of products with the same IngestionDate String filter = "IngestionDate ge " + lup_s; // Appends custom $filter parameter if (filterParam != null) { filter += " and (" + filterParam + ")"; } query_param.put("$filter", filter); query_param.put("$top", String.valueOf(pageSize)); query_param.put("$orderby", "IngestionDate"); if (optional_skip != null && optional_skip > 0) { query_param.put("$skip", optional_skip.toString()); } if (expand_navlinks) { query_param.put("$expand", "Class,Attributes,Products"); } // Executes the query long delta = System.currentTimeMillis(); ODataFeed pdf = client.readFeed(sourceCollection + "/Products", query_param); logODataPerf("Products", System.currentTimeMillis() - delta); return pdf; } /** Returns the IngestionDate of the given product entry. */ private Date getIngestionDate(ODataEntry entry) { return ((GregorianCalendar) entry.getProperties().get("IngestionDate")).getTime(); } /** Returns `true` if the given product entry already exists in the database. */ private boolean exists(ODataEntry entry) { String uuid = (String) entry.getProperties().get("Id"); // FIXME: might not be the same product return PRODUCT_SERVICE.systemGetProduct(uuid) != null; } /** Creates and returns a new Product from the given entry. */ private Product entryToProducts(ODataEntry entry) throws ODataException, IOException, InterruptedException { long delta; Map<String, Object> props = entry.getProperties(); // (`UUID` and `PATH` have unique constraint), PATH references the UUID String uuid = (String) props.get("Id"); // Makes the product resource path String pdt_p = "/Products('" + uuid + "')"; Product product = new Product(); product.setUuid(uuid); // Reads the properties product.setIdentifier((String) props.get("Name")); product.setIngestionDate(((GregorianCalendar) props.get("IngestionDate")).getTime()); product.setCreated(((GregorianCalendar) props.get("CreationDate")).getTime()); product.setFootPrint((String) props.get("ContentGeometry")); product.setProcessed(Boolean.TRUE); product.setSize((Long) props.get("ContentLength")); // Reads the ContentDate complex type Map contentDate = (Map) props.get("ContentDate"); product.setContentStart(((GregorianCalendar) contentDate.get("Start")).getTime()); product.setContentEnd(((GregorianCalendar) contentDate.get("End")).getTime()); // Sets the origin to the remote URI product.setOrigin(client.getServiceRoot() + pdt_p + "/$value"); product.setPath(new URL(entry.getMetadata().getId() + "/$value")); // Sets the size, ContentType and Checksum of product Product.Download d = new Product.Download(); Map<String, String> checksum = (Map) props.get("Checksum"); d.setSize(product.getSize()); d.setType((String) props.get("ContentType")); d.setChecksums( Collections.singletonMap( checksum.get(Model.ALGORITHM), checksum.get(Model.VALUE))); product.setDownload(d); // Sets the download path to LocalPath (if LocalPaths are exposed) if (this.remoteIncoming != null && !this.copyProduct) { String path = (String) props.get("LocalPath"); if (path != null && !path.isEmpty()) { d.setPath(Paths.get(this.remoteIncoming, path).toString()); File f = new File(d.getPath()); if (!f.exists()) { // The incoming path is probably false // Throws an exception to notify the admin about this issue throw new RuntimeException("ODataSynchronizer: Local file '" + path + "' not found in remote incoming '" + this.remoteIncoming + '\''); } product.setPath(new URL("file://" + d.getPath())); } else { throw new RuntimeException("RemoteIncoming is set" + " but the LocalPath property is missing in remote products"); } } // Retrieves the Product Class if not inlined ODataEntry pdt_class_e; if (entry.containsInlineEntry() && props.get("Class") != null) { pdt_class_e = ODataEntry.class.cast(props.get("Class")); } else { delta = System.currentTimeMillis(); pdt_class_e = client.readEntry(pdt_p + "/Class", null); logODataPerf(pdt_p + "/Class", System.currentTimeMillis() - delta); } Map<String, Object> pdt_class_pm = pdt_class_e.getProperties(); String pdt_class = String.class.cast(pdt_class_pm.get("Uri")); product.setItemClass(pdt_class); // Retrieves Metadata Indexes (aka Attributes on odata) if not inlined ODataFeed mif; if (entry.containsInlineEntry() && props.get("Attributes") != null) { mif = ODataDeltaFeed.class.cast(props.get("Attributes")); } else { delta = System.currentTimeMillis(); mif = client.readFeed(pdt_p + "/Attributes", null); logODataPerf(pdt_p + "/Attributes", System.currentTimeMillis() - delta); } List<MetadataIndex> mi_l = new ArrayList<>(mif.getEntries().size()); for (ODataEntry mie: mif.getEntries()) { Map<String, Object> mi_pm = mie.getProperties(); MetadataIndex mi = new MetadataIndex(); String mi_name = (String) mi_pm.get("Name"); mi.setName(mi_name); mi.setType((String) mi_pm.get("ContentType")); mi.setValue((String) mi_pm.get("Value")); MetadataType mt = METADATA_TYPE_SERVICE.getMetadataTypeByName(pdt_class, mi_name); if (mt != null) { mi.setCategory(mt.getCategory()); if (mt.getSolrField() != null) { mi.setQueryable(mt.getSolrField().getName()); } } else if (mi_name.equals("Identifier")) { mi.setCategory(""); mi.setQueryable("identifier"); } else if (mi_name.equals("Ingestion Date")) { mi.setCategory("product"); mi.setQueryable("ingestionDate"); } else { mi.setCategory(""); } mi_l.add(mi); } product.setIndexes(mi_l); // Retrieves subProducts if not inlined ODataFeed subp; if (entry.containsInlineEntry() && props.get("Products") != null) { subp = ODataDeltaFeed.class.cast(props.get("Products")); } else { delta = System.currentTimeMillis(); subp = client.readFeed(pdt_p + "/Products", null); logODataPerf(pdt_p + "/Products", System.currentTimeMillis() - delta); } for (ODataEntry subpe: subp.getEntries()) { String id = (String) subpe.getProperties().get("Id"); Long content_len = (Long) subpe.getProperties().get("ContentLength"); String path = (String) subpe.getProperties().get("LocalPath"); if (this.remoteIncoming != null && !this.copyProduct && path != null && !path.isEmpty()) { path = Paths.get(this.remoteIncoming, path).toString(); } else { path = client.getServiceRoot() + pdt_p + "/Products('" + subpe.getProperties().get("Id") + "')/$value"; } // Retrieves the Quicklook if (id.equals("Quicklook")) { product.setQuicklookSize(content_len); product.setQuicklookPath(path); } // Retrieves the Thumbnail else if (id.equals("Thumbnail")) { product.setThumbnailSize(content_len); product.setThumbnailPath(path); } } // `processed` must be set to TRUE product.setProcessed(Boolean.TRUE); return product; } private void save(Product product) { List<MetadataIndex> metadatas = product.getIndexes(); // Stores `product` in the database product = PRODUCT_SERVICE.addProduct(product); product.setIndexes(metadatas); // DELME lazy loading not working atm ... // Stores `product` in the index try { long delta = System.currentTimeMillis(); SEARCH_SERVICE.index(product); LOGGER.debug("Synchronizer#" + getId() + " indexed product " + product.getIdentifier() + " in " + (System.currentTimeMillis() - delta) + "ms"); } catch (Exception e) { // Solr errors are not considered fatal LOGGER.error("Synchronizer#" + getId() + " Failed to index product " + product.getIdentifier() + " in Solr's index", e); } // Sets the target collection both in the DB and Solr if (this.targetCollectionUUID != null) { try { COLLECTION_SERVICE.systemAddProduct(this.targetCollectionUUID, product.getId(), false); } catch (HibernateException e) { LOGGER.error("Synchronizer#" + getId() + " Failed to set collection#" + this.targetCollectionUUID + " for product " + product.getIdentifier(), e); // Reverting ... PRODUCT_SERVICE.systemDeleteProduct(product.getId()); throw e; } catch (Exception e) { LOGGER.error("Synchronizer#" + getId() + " Failed to update product " + product.getIdentifier() + " in Solr's index", e); } } } /** move file at `path_to_file` to `path_to_dir`. Returns the resulting Path. */ private Path chDir(Path path_to_file, Path path_to_dir) throws IOException { Path res = path_to_dir.resolve(path_to_file.getFileName()); Files.move(path_to_file, res, StandardCopyOption.ATOMIC_MOVE); return res; } /** Retrieves and download new products, downloads are parallelized. */ private int getAndCopyNewProduct() throws InterruptedException { int res = 0; int count = this.pageSize; int skip = 0; ParallelizedDownloadManager downloader = new ParallelizedDownloadManager( this.pageSize, this.pageSize, 0, TimeUnit.SECONDS, new BasicAuthHttpClientProducer(), INCOMING_MANAGER.getTempDir().toPath()); // Downloads are done asynchronously in another threads List<Product> products = new ArrayList<>(this.pageSize); List<Future<DownloadResult>[]> futures = new ArrayList<>(this.pageSize); try { // Downloads at least `pageSize` products while (count > 0) { ODataFeed pdf = getPage(skip, false); if (pdf.getEntries().isEmpty()) // No more products { break; } skip += this.pageSize; for (ODataEntry pdt: pdf.getEntries ()) { if (exists(pdt)) { continue; } count--; Product product = entryToProducts(pdt); Future<DownloadResult>[] future = download(downloader, product); products.add(product); futures.add(future); } } // Get download results from Futures, and create product entries in DB, Solr boolean update_lid = true; // Controls whether we are updating LastIngestionDate or not for (int it=0; it<products.size(); it++) { Product product = products.get(it); Future<DownloadResult>[] future = futures.get(it); try { DownloadResult prod_res = future[0].get(); String data_md5 = String.valueOf(Hex.encode(prod_res.md5sum)).toUpperCase(); String sync_md5 = product.getDownload().getChecksums().get("MD5").toUpperCase(); if (!data_md5.equals(sync_md5)) { throw new DigestException(data_md5 + " != " + sync_md5); } // Asks the Incoming manager for dest directory Path dir = Paths.get(INCOMING_MANAGER.getNewIncomingPath().toURI()); // Sets download info in the product Path prod_path = chDir(prod_res.data, dir); product.setPath(prod_path.toUri().toURL()); product.setDownloadablePath(prod_path.toString()); product.setDownloadableType(prod_res.dataType); product.setDownloadableSize(prod_res.dataSize); // Sets its QuickLook image (if any) if (product.getQuicklookFlag() && future[1] != null) { DownloadResult ql_res = future[1].get(); Path ql_path = chDir(ql_res.data, dir); product.setQuicklookPath(ql_path.toString()); product.setQuicklookSize(ql_res.dataSize); } // Sets its Thumbnail image (if any) if (product.getThumbnailFlag() && future[2] != null) { DownloadResult tn_res = future[2].get(); Path tn_path = chDir(tn_res.data, dir); product.setThumbnailPath(tn_path.toString()); product.setThumbnailSize(tn_res.dataSize); } save(product); if (update_lid) { this.lastCreated = product.getIngestionDate(); this.dateChanged = true; } res++; LOGGER.info(String.format( "Synchronizer#%d Product %s (%d bytes compressed) successfully synchronized from %s", getId(), product.getIdentifier(), product.getSize(), this.client.getServiceRoot())); } catch (DigestException | ExecutionException ex) { if (ex instanceof DigestException) { LOGGER.error(String.format("Synchronizer#%d Product %s md5sum comparison failed", getId(), product.getIdentifier()), ex); } else { LOGGER.error(String.format("Synchronizer#%d Product %s failed to download", getId(), product.getIdentifier()), ex); } // Remove temp files (cleaning up downloaded files) for (int ju=0; ju<3; ju++) { if (future[ju] != null) { try { Files.delete(future[ju].get().data); } catch (ExecutionException | IOException none) {} } } if (update_lid) { this.lastCreated = product.getIngestionDate(); this.dateChanged = true; // Only update the lastIngestionDate to the first consecutive successful downloads update_lid = false; } } } } catch (IOException | ODataException ex) { LOGGER.error ("OData failure", ex); } catch (InterruptedException ex) { // Interruption required, stopping downloads and cleaning temp files for (Future<DownloadResult>[] future: futures) { for (Future<DownloadResult> f: future) { if (f != null) { f.cancel(true); try { Files.delete(f.get().data); } catch (CancellationException | ExecutionException | IOException none) {} } } } } finally { // Save the ingestionDate of the last created Product this.syncConf.setConfig ("last_created", String.valueOf (this.lastCreated.getTime ())); downloader.shutdownNow(); } return res; } /** * Retrieve new/updated products. * @return how many products have been retrieved. */ private int getNewProducts () throws InterruptedException { int res = 0; try { ODataFeed pdf = getPage(null, true); // For each entry, creates a DataBase Object for (ODataEntry pdt: pdf.getEntries ()) { if (exists(pdt)) { this.lastCreated = getIngestionDate(pdt); this.dateChanged = true; continue; } Product product = entryToProducts(pdt); save(product); this.lastCreated = product.getIngestionDate (); this.dateChanged = true; LOGGER.info("Synchronizer#" + getId () + " Product " + product.getIdentifier () + " ("+ product.getDownloadableSize() + " bytes compressed) " + "successfully synchronized from " + this.client.getServiceRoot ()); res++; // Checks if we have to abandon the current pass if (Thread.interrupted ()) { throw new InterruptedException (); } } } catch (IOException | ODataException ex) { LOGGER.error ("OData failure", ex); } finally { // Save the ingestionDate of the last created Product this.syncConf.setConfig ("last_created", String.valueOf (this.lastCreated.getTime ())); } return res; } /** * Retrieves updated products. * Not Yet Implemented. * @return how many products have been retrieved. */ private int getUpdatedProducts () { // NYI return 0; } /** * Retrieves deleted products. * Not Yet Implemented. * @return how many products have been retrieved. */ private int getDeletedProducts () { // NYI return 0; } @Override public boolean synchronize () throws InterruptedException { int retrieved = 0, updated = 0, deleted = 0; LOGGER.info("Synchronizer#" + getId () + " started"); try { if (this.copyProduct) { retrieved = getAndCopyNewProduct(); } else { retrieved = getNewProducts(); } if (Thread.interrupted ()) { throw new InterruptedException (); } updated = getUpdatedProducts (); if (Thread.interrupted ()) { throw new InterruptedException (); } deleted = getDeletedProducts (); } catch (LockAcquisitionException | CannotAcquireLockException e) { throw new InterruptedException (e.getMessage ()); } finally { // Logs a summary of what it has done this session StringBuilder sb = new StringBuilder ("Synchronizer#"); sb.append (getId ()).append (" done: "); sb.append (retrieved).append (" new Products, "); sb.append (updated).append (" updated Products, "); sb.append (deleted).append (" deleted Products"); sb.append (" from ").append (this.client.getServiceRoot ()); LOGGER.info(sb.toString()); // Writes the database only if there is a modification if (this.dateChanged) { SYNC_SERVICE.saveSynchronizer (this); this.dateChanged = false; } } return retrieved < pageSize && updated < pageSize && deleted < pageSize; } @Override public String toString () { return "OData Product Synchronizer on " + syncConf.getConfig("service_uri"); } /** Creates a client producer that produces HTTP Basic auth aware clients. */ private class BasicAuthHttpClientProducer implements HttpAsyncClientProducer { @Override public CloseableHttpAsyncClient generateClient () { CredentialsProvider credsProvider = new BasicCredentialsProvider(); credsProvider.setCredentials(new AuthScope (AuthScope.ANY), new UsernamePasswordCredentials(serviceUser, servicePass)); RequestConfig rqconf = RequestConfig.custom() .setCookieSpec(CookieSpecs.DEFAULT) .setSocketTimeout(Timeouts.SOCKET_TIMEOUT) .setConnectTimeout(Timeouts.CONNECTION_TIMEOUT) .setConnectionRequestTimeout(Timeouts.CONNECTION_REQUEST_TIMEOUT) .build(); CloseableHttpAsyncClient res = HttpAsyncClients.custom () .setDefaultCredentialsProvider (credsProvider) .setDefaultRequestConfig(rqconf) .build (); res.start (); return res; } } }