package net.ion.bleujin.crawl; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.util.concurrent.atomic.AtomicInteger; import junit.framework.TestCase; import net.ion.craken.node.ReadSession; import net.ion.craken.node.TransactionJob; import net.ion.craken.node.WriteSession; import net.ion.craken.node.crud.Craken; import net.ion.craken.node.crud.store.OldFileConfigBuilder; import net.ion.framework.util.Debug; import net.ion.framework.util.IOUtil; import net.ion.framework.util.InfinityThread; import org.infinispan.configuration.cache.CacheMode; import org.infinispan.configuration.global.GlobalConfiguration; import org.infinispan.configuration.global.GlobalConfigurationBuilder; import org.infinispan.manager.DefaultCacheManager; public class TestIndexCrawlData extends TestCase { private ReadSession session; @Override protected void setUp() throws Exception { super.setUp(); GlobalConfiguration gconfig = new GlobalConfigurationBuilder() .transport().defaultTransport() .clusterName("storetest") .nodeName("external") .addProperty("configurationFile", "./resource/config/jgroups-udp.xml") .transport().addProperty("maxThreads", "100").addProperty("threadNamePrefix", "mytransport-thread") .build(); DefaultCacheManager dcm = new DefaultCacheManager(gconfig) ; Craken r = Craken.create(dcm, "bleujin"); r.createWorkspace("enha", OldFileConfigBuilder.directory("./resource/enha").distMode(CacheMode.REPL_SYNC)); this.session = r.login("enha"); } @Override protected void tearDown() throws Exception { session.workspace().repository().shutdown(); super.tearDown(); } public void testRespository() throws Exception { session.tran(new TransactionJob<Void>() { private AtomicInteger count = new AtomicInteger(0); private long start = System.currentTimeMillis(); @Override public Void handle(WriteSession wsession) throws Exception { File homeDir = new File("C:/crawl/enha/wiki"); saveProperty(wsession, homeDir, ""); return null; } private void saveProperty(WriteSession wsession, File file, String path) throws IOException { int icount = count.incrementAndGet(); if (icount >= 200) return; if (file.isDirectory()) { for (File sfile : file.listFiles()) { saveProperty(wsession, sfile, path + "/" + sfile.getName()); } } else { String content = IOUtil.toStringWithClose(new FileInputStream(file), "UTF-8"); wsession.pathBy(path).property("content", content); if ((icount % 50) == 0) { System.out.println(count.get() + " committed. elapsed time for unit : " + (System.currentTimeMillis() - start)); this.start = System.currentTimeMillis(); wsession.continueUnit(); } } } }); // session.pathBy(parentName).children().debugPrint(); } public void testCount() throws Exception { Debug.line("loaded"); int count = session.root().childQuery("").find().totalCount() ; Debug.line(count); new InfinityThread().startNJoin(); } }