ListWithDiskBuffer.java

package com.zendesk.maxwell.util;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.*;
import java.util.*;

/*
   a wrapper class for a linked list that will keep N tail elements
   in memory, spilling its head onto disk as needed.
 */
public class ListWithDiskBuffer<T> {
	static final Logger LOGGER = LoggerFactory.getLogger(ListWithDiskBuffer.class);
	private final long maxInMemoryElements;
	private final LinkedList<T> list;
	private long elementsInFile = 0;
	private File file;
	private ObjectInputStream is;
	private ObjectOutputStream os;

	public ListWithDiskBuffer(long maxInMemoryElements) {
		this.maxInMemoryElements = maxInMemoryElements;
		list = new LinkedList<>();
	}

	public void add(T element) throws IOException {
		list.add(element);

		while ( shouldBuffer() )
			evict();
	}

	protected boolean shouldBuffer() {
		return this.list.size() > maxInMemoryElements;
	}

	protected void resetOutputStreamCaches() throws IOException {
		os.reset();
	}

	public void flushToDisk() throws IOException {
		if ( os != null )
			os.flush();
	}

	public boolean isEmpty() {
		return this.size() == 0;
	}

	public T getLast() {
		return list.getLast();
	}

	public T removeFirst(Class<T> clazz) throws IOException, ClassNotFoundException {
		if ( elementsInFile > 0 ) {
			if ( is == null ) {
				os.flush();
				is = new ObjectInputStream(new BufferedInputStream(new FileInputStream(file)));
			}

			Object object = is.readObject();
			T element = clazz.cast(object);
			elementsInFile--;

			return element;
		} else {
			return list.removeFirst();
		}
	}

	public Long size() {
		return list.size() + elementsInFile;
	}

	public Long inMemorySize() {
		return Long.valueOf(list.size());
	}

	@Override
	protected void finalize() throws Throwable {
		try {
			file.delete();
		} finally {
			super.finalize();
		}
	}

	protected T evict() throws IOException {
		if ( file == null ) {
			file = File.createTempFile("maxwell", "events");
			file.deleteOnExit();
			os = new ObjectOutputStream(new BufferedOutputStream(new FileOutputStream(file)));
		}

		if ( elementsInFile == 0 )
			LOGGER.info("Overflowed in-memory buffer, spilling over into " + file);

		T evicted = this.list.removeFirst();
		os.writeObject(evicted);

		elementsInFile++;

		if ( elementsInFile % maxInMemoryElements == 0 )
			resetOutputStreamCaches();

		return evicted;
	}

}