/* * ToroDB * Copyright © 2014 8Kdata Technology (www.8kdata.com) * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Affero General Public License for more details. * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. */ package com.torodb.torod.pipeline.impl; import akka.actor.ActorSystem; import akka.dispatch.ExecutionContexts; import akka.stream.ActorMaterializer; import akka.stream.ActorMaterializerSettings; import akka.stream.Materializer; import akka.stream.javadsl.Keep; import akka.stream.javadsl.Sink; import akka.stream.javadsl.Source; import com.google.common.base.Throwables; import com.torodb.core.backend.WriteBackendTransaction; import com.torodb.core.concurrent.ConcurrentToolsFactory; import com.torodb.core.d2r.D2RTranslatorFactory; import com.torodb.core.dsl.backend.BackendTransactionJobFactory; import com.torodb.core.exceptions.SystemException; import com.torodb.core.exceptions.SystemInterruptedException; import com.torodb.core.exceptions.user.UserException; import com.torodb.core.services.IdleTorodbService; import com.torodb.core.transaction.RollbackException; import com.torodb.core.transaction.metainf.MetaDatabase; import com.torodb.core.transaction.metainf.MutableMetaCollection; import com.torodb.kvdocument.values.KvDocument; import com.torodb.torod.pipeline.D2RTranslationBatchFunction; import com.torodb.torod.pipeline.DefaultToBackendFunction; import com.torodb.torod.pipeline.InsertPipeline; import com.torodb.torod.pipeline.InsertPipelineFactory; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import scala.concurrent.Await; import scala.concurrent.duration.Duration; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.ThreadFactory; import java.util.stream.Stream; import javax.inject.Inject; /** * */ public class AkkaInsertPipelineFactory extends IdleTorodbService implements InsertPipelineFactory { private static final Logger LOGGER = LogManager.getLogger(AkkaInsertPipelineFactory.class); private final ConcurrentToolsFactory concurrentToolsFactory; private ActorSystem actorSystem; private Materializer materializer; private final BackendTransactionJobFactory factory; private final int docBatch; private ExecutorService executorService; @Inject public AkkaInsertPipelineFactory(ThreadFactory threadFactory, ConcurrentToolsFactory concurrentToolsFactory, BackendTransactionJobFactory factory, int docBatch) { super(threadFactory); this.concurrentToolsFactory = concurrentToolsFactory; this.factory = factory; this.docBatch = docBatch; } @Override protected void startUp() throws Exception { executorService = concurrentToolsFactory.createExecutorService( "insert-pipeline", true ); actorSystem = ActorSystem.create("insert-pipeline", null, null, ExecutionContexts.fromExecutor(executorService) ); this.materializer = ActorMaterializer.create( ActorMaterializerSettings.create(actorSystem), actorSystem, "insert" ); } @Override protected void shutDown() throws Exception { if (actorSystem != null) { try { Await.result(actorSystem.terminate(), Duration.Inf()); } catch (Exception ex) { throw new RuntimeException("It was impossible to shutdown the " + "insert-pipeline actor system", ex); } } if (executorService != null) { executorService.shutdown(); } LOGGER.debug("Insert pipeline actor system terminated"); } @Override public InsertPipeline createInsertPipeline( D2RTranslatorFactory translatorFactory, MetaDatabase metaDb, MutableMetaCollection mutableMetaCollection, WriteBackendTransaction backendConnection, boolean concurrent) { if (!concurrent) { LOGGER.debug("Akka insert pipeline has been used when concurrent " + "hint is marked as false. It will be ignored"); } return new AkkaInsertPipeline(translatorFactory, metaDb, mutableMetaCollection, backendConnection); } private class AkkaInsertPipeline implements InsertPipeline { private final D2RTranslatorFactory translatorFactory; private final MetaDatabase metaDb; private final MutableMetaCollection mutableMetaCollection; private final WriteBackendTransaction backendConnection; public AkkaInsertPipeline(D2RTranslatorFactory translatorFactory, MetaDatabase metaDb, MutableMetaCollection mutableMetaCollection, WriteBackendTransaction backendConnection) { this.translatorFactory = translatorFactory; this.metaDb = metaDb; this.mutableMetaCollection = mutableMetaCollection; this.backendConnection = backendConnection; } @Override public void insert(Stream<KvDocument> docs) throws UserException { D2RTranslationBatchFunction d2rFun = new D2RTranslationBatchFunction(translatorFactory, metaDb, mutableMetaCollection); DefaultToBackendFunction r2BackendFun = new DefaultToBackendFunction(factory, metaDb, mutableMetaCollection); try { Source.fromIterator(() -> docs.iterator()) .grouped(docBatch) .map(d2rFun::apply) .mapConcat(r2BackendFun::apply) .async() .toMat( Sink.foreach((job) -> job.execute(backendConnection)), Keep.right()) .run(materializer) .toCompletableFuture() .get(); } catch (InterruptedException ex) { throw new SystemInterruptedException("insertion interrupted", ex); } catch (ExecutionException ex) { Throwable t; if (ex.getCause() != null) { t = ex.getCause(); } else { t = ex; } Throwables.propagateIfPossible(t, UserException.class, RollbackException.class); throw new SystemException("Execution exception while trying to insert", t); } } @Override public int getDocsBatchLength() { return docBatch; } @Override public void setDocsBatchLength(int newBatchLength) { throw new UnsupportedOperationException( "Doc batch lenght is not supported on Akka factory yet."); } } }