package proj.zoie.example.service;
import org.apache.log4j.Logger;
import proj.zoie.api.DataConsumer;
import proj.zoie.impl.indexing.StreamDataProvider;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.ReadableByteChannel;
public class SingleFileDataProvider<T> extends StreamDataProvider<T> {
private static final Logger log = Logger.getLogger(SingleFileDataProvider.class);
protected Deserializer<T> deserializer;
protected final int datumByteSize;
protected ReadableByteChannel inputChannel;
protected ByteBuffer buf;
protected byte[] bytes;
protected long position;
public SingleFileDataProvider(int size)
{
datumByteSize = size;
}
public SingleFileDataProvider(String fileName,
int datumByteSize,
Deserializer<T> deserializer) throws IOException {
this.deserializer = deserializer;
this.datumByteSize = datumByteSize;
bytes = new byte[datumByteSize];
File f = new File(fileName);
if(!f.exists() || !f.canRead()) throw new IOException("Unreadable file: " + fileName);
inputChannel = new FileInputStream(new File(fileName)).getChannel();
buf = ByteBuffer.wrap(bytes);
position = 0;
new Thread(new Starter()).start();
}
private final class Starter implements Runnable {
@Override
public void run() {
try {
Thread.sleep(10000);
SingleFileDataProvider.this.start();
} catch(Exception e) {
System.out.println(e);
}
}
}
public static interface Deserializer<T> {
public T deserialize(byte[] bytes) throws IOException;
}
protected long time = 0;
@Override
public DataConsumer.DataEvent<T> next() {
if((position % 10000) == 0) {
if(time > 0) System.out.println(((float)(System.nanoTime() - time)) / 1e9 + "s per 10k docs");
time = System.nanoTime();
}
try {
long version = position;
buf.clear();
boolean full = false;
int bytesRead = 0;
T t = null;
while(!full && t == null) {
bytesRead += Math.max(0, inputChannel.read(buf));
full = (bytesRead >= datumByteSize);
if(!full) {
try { sleep(); } catch (InterruptedException ie) {
return null;
}
} else {
byte[] bufArray = buf.array(); // latest buf bits.
try {
t = deserializer.deserialize(bufArray);
position += bytesRead / datumByteSize;
} catch (IOException ioe) {
char[] out = new char[bufArray.length];
for(int j=0;j<out.length;j++) out[j] = (char)bufArray[j];
System.out.println(new String(out));
flushInputToNextEvent();
return next();
}
}
}
return new DataConsumer.DataEvent<T>(version, t);
} catch (IOException ioe) {
log.error("Cannot read from file!", ioe);
try { inputChannel.close(); } catch(Exception e) {
log.warn("Could not close FileChannel: ", e);
}
throw new RuntimeException(ioe);
}
}
protected void flushInputToNextEvent() throws IOException {
ByteBuffer smallBuf = ByteBuffer.wrap(new byte[1]);
while(true) {
smallBuf.clear();
int read = inputChannel.read(smallBuf);
if(read < 1) {
try { sleep(); } catch (InterruptedException ie) {
return;
}
} else {
char c = (char)smallBuf.array()[0];
if(c == '\n') {
return;
}
}
}
}
static int sleepCt = 0;
public void sleep() throws InterruptedException {
Thread.sleep(25);
// if((sleepCt++ % 120) == 0) System.out.println("\n.");
}
@Override
public void reset() {
// noop!
}
}