/* * Data Hub Service (DHuS) - For Space data distribution. * Copyright (C) 2013,2014,2015 GAEL Systems * * This file is part of DHuS software sources. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as * published by the Free Software Foundation, either version 3 of the * License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Affero General Public License for more details. * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. */ package fr.gael.dhus.network; import java.io.FileInputStream; import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; import org.apache.commons.net.io.CopyStreamAdapter; import fr.gael.dhus.database.object.User; import fr.gael.dhus.service.NetworkUsageService; import fr.gael.dhus.spring.context.ApplicationContextProvider; public class RegulatedInputStream extends FilterInputStream { /** * Default buffer size in bytes. */ public static final int DEFAULT_BUFFER_SIZE = 8192; /** * Atomic updater to provide compareAndSet for buf. This is necessary because * closes can be asynchronous. We use nullness of buf[] as primary indicator * that this stream is closed. (The "in" field is also nulled out on close.) */ private static final AtomicReferenceFieldUpdater <RegulatedInputStream, byte[]> BUF_UPDATER = AtomicReferenceFieldUpdater.newUpdater (RegulatedInputStream.class, byte[].class, "buf"); /** * A logger for this class. */ private static final Logger LOGGER = LogManager.getLogger(RegulatedInputStream.class); /** * The internal buffer array where the data is stored. When necessary, it may * be replaced by another array of a different size. */ protected volatile byte buf[]; /** * The index one greater than the index of the last valid byte in the buffer. * This value is always in the range <code>0</code> through * <code>buf.length</code>; elements <code>buf[0]</code> through * <code>buf[count-1] * </code>contain buffered input data obtained from the underlying input * stream. */ protected int count; /** * The current position in the buffer. This is the index of the next * character to be read from the <code>buf</code> array. * <p> * This value is always in the range <code>0</code> through * <code>count</code>. If it is less than <code>count</code>, then * <code>buf[pos]</code> is the next byte to be supplied as input; if it is * equal to <code>count</code>, then the next <code>read</code> or * <code>skip</code> operation will require more bytes to be read from the * contained input stream. * * @see java.io.BufferedInputStream#buf */ protected int pos; /** * The value of the <code>pos</code> field at the time the last * <code>mark</code> method was called. * <p> * This value is always in the range <code>-1</code> through <code>pos</code> * . If there is no marked position in the input stream, this field is * <code>-1</code>. If there is a marked position in the input stream, then * <code>buf[markpos]</code> is the first byte to be supplied as input after * a <code>reset</code> operation. If <code>markpos</code> is not * <code>-1</code>, then all bytes from positions <code>buf[markpos]</code> * through <code>buf[pos-1]</code> must remain in the buffer array (though * they may be moved to another place in the buffer array, with suitable * adjustments to the values of <code>count</code>, <code>pos</code>, and * <code>markpos</code>); they may not be discarded unless and until the * difference between <code>pos</code> and <code>markpos</code> exceeds * <code>marklimit</code>. * * @see java.io.BufferedInputStream#mark(int) * @see java.io.BufferedInputStream#pos */ protected int markpos = -1; /** * The maximum read ahead allowed after a call to the <code>mark</code> * method before subsequent calls to the <code>reset</code> method fail. * Whenever the difference between <code>pos</code> and <code>markpos</code> * exceeds <code>marklimit</code>, then the mark may be dropped by setting * <code>markpos</code> to <code>-1</code>. * * @see java.io.BufferedInputStream#mark(int) * @see java.io.BufferedInputStream#reset() */ protected int marklimit; /** * The network regulator to be used. TODO Should be auto-wired by default */ private final Regulator regulator; /** * Connection parameters */ private final ConnectionParameters connectionParameters; /** * Controlled flow */ private final ChannelFlow flow; /** * Copy listener (optional) */ private CopyStreamAdapter listener; /** * This stream size if known. */ private Long streamSize = null; /** * Builds a regulated stream from a builder. * * @param builder the builder wrapping all parameters. * @throws IllegalArgumentException if {@link Builder#bufferSize} <= 0 * @throws RegulationException if a regulation rule prevent the creation of * this stream with this regulator e.g. maximum connections * reached, invalid user, etc. */ private RegulatedInputStream(final Builder builder) throws IllegalArgumentException, RegulationException { // Build the buffered input stream super class super(builder.wrappedStream); // Check buffer size if (builder.bufferSize <= 0) { throw new IllegalArgumentException( "Invalid negative or null buffer size: " + builder.bufferSize); } if (builder.listener!=null) this.listener = builder.listener; if (builder.streamSize!=null) this.streamSize=builder.streamSize; // Allocate buffer array buf = new byte[builder.bufferSize]; // Set regulator (if any provided) if (builder.regulator != null) { this.regulator = builder.regulator; } else { this.regulator = Regulator.getDefaultRegulator(); } long stream_size = 0L; try { stream_size= getStreamSize (builder.wrappedStream); if (stream_size == 0) throw new IOException ("Unable to retrieve stream size."); } catch (IOException e) { LOGGER.warn(e); } // Build connection parameters this.connectionParameters = new ConnectionParameters.Builder(builder.direction).user(builder.user) .userName(builder.userName).streamSize (stream_size).build(); // Get regulated flow try { this.flow = (ChannelFlow)this.regulator.getChannel( this.connectionParameters); } catch (RegulationException exception) { LOGGER.error(exception); throw exception; } // Report opened flow LOGGER.debug("OPEN - " + this.flow); } /** * This method aims to retrieve stream size. Of course, stream size is * not always available, but in the case of file, the size is known, * overwise stream is probably aware of possible block size can be served... * @param is the input stream to ge the size * @return the possible size of the stream * @throws IOException when stream access fails. */ private long getStreamSize (InputStream is) throws IOException { if (streamSize == null) { if (is instanceof FileInputStream) { return ((FileInputStream)is).getChannel ().size (); } streamSize = (long)is.available (); } return streamSize; } /** * Check to make sure that underlying input stream has not been nulled out * due to close; if not return it; */ private InputStream getInIfOpen() throws IOException { InputStream input = in; if (input == null) throw new IOException("Stream closed"); return input; } /** * Check to make sure that buffer has not been nulled out due to close; if * not return it; */ private byte[] getBufIfOpen() throws IOException { byte[] buffer = buf; if (buffer == null) throw new IOException("Stream closed"); return buffer; } /** * Fills the buffer with more data, taking into account shuffling and other * tricks for dealing with marks. Assumes that it is being called by a * synchronized method. This method also assumes that all data has already * been read in, hence pos > count. */ private void fill() throws IOException { byte[] buffer = getBufIfOpen(); if (markpos < 0) pos = 0; /* no mark: throw away the buffer */ else if (pos >= buffer.length) /* no room left in buffer */ if (markpos > 0) { /* can throw away early part of the buffer */ int sz = pos - markpos; System.arraycopy(buffer, markpos, buffer, 0, sz); pos = sz; markpos = 0; } else if (buffer.length >= marklimit) { markpos = -1; /* buffer got too big, invalidate mark */ pos = 0; /* drop buffer contents */ } else { /* grow buffer */ int nsz = pos * 2; if (nsz > marklimit) nsz = marklimit; byte nbuf[] = new byte[nsz]; System.arraycopy(buffer, 0, nbuf, 0, pos); if (!BUF_UPDATER.compareAndSet(this, buffer, nbuf)) { // Can't replace buf if there was an async close. // Note: This would need to be changed if fill() // is ever made accessible to multiple threads. // But for now, the only way CAS can fail is via close. // assert buf == null; throw new IOException("Stream closed"); } buffer = nbuf; } count = pos; int n = getInIfOpen().read(buffer, pos, buffer.length - pos); if (n > 0) count = n + pos; } /** * See the general contract of the <code>read</code> method of * <code>InputStream</code>. * * @return the next byte of data, or <code>-1</code> if the end of the stream * is reached. * @exception IOException if this input stream has been closed by invoking * its {@link #close()} method, or an I/O error occurs. * @see java.io.FilterInputStream#in */ public synchronized int read() throws IOException { // Acquire from regulated flow if (this.flow != null) { try { this.flow.acquire(1); } catch (InterruptedException exception) { LOGGER.error(exception); this.close(); throw new IOException(exception); } catch (RegulationException exception) { LOGGER.error(exception); this.close(); throw exception; } } // Continue if (pos >= count) { fill(); if (pos >= count) return -1; } if (listener != null) listener.bytesTransferred(pos, 1, this.connectionParameters.getStreamSize ()); return getBufIfOpen()[pos++] & 0xff; } /** * Read characters into a portion of an array, reading from the underlying * stream at most once if necessary. */ private int read1(byte[] b, int off, int len) throws IOException { int avail = count - pos; if (avail <= 0) { /* * If the requested length is at least as large as the buffer, and if * there is no mark/reset activity, do not bother to copy the bytes * into the local buffer. In this way buffered streams will cascade * harmlessly. */ if (len >= getBufIfOpen().length && markpos < 0) { return getInIfOpen().read(b, off, len); } fill(); avail = count - pos; if (avail <= 0) return -1; } int cnt = (avail < len) ? avail : len; System.arraycopy(getBufIfOpen(), pos, b, off, cnt); pos += cnt; return cnt; } /** * Reads bytes from this byte-input stream into the specified byte array, * starting at the given offset. * <p> * This method implements the general contract of the corresponding * <code>{@link InputStream#read(byte[], int, int) read}</code> method of the * <code>{@link InputStream}</code> class. As an additional convenience, it * attempts to read as many bytes as possible by repeatedly invoking the * <code>read</code> method of the underlying stream. This iterated * <code>read</code> continues until one of the following conditions becomes * true: * <ul> * <li>The specified number of bytes have been read, * <li>The <code>read</code> method of the underlying stream returns * <code>-1</code>, indicating end-of-file, or * <li>The <code>available</code> method of the underlying stream returns * zero, indicating that further input requests would block. * </ul> * If the first <code>read</code> on the underlying stream returns * <code>-1</code> to indicate end-of-file then this method returns * <code>-1</code>. Otherwise this method returns the number of bytes * actually read. * <p> * Subclasses of this class are encouraged, but not required, to attempt to * read as many bytes as possible in the same fashion. * * @param b destination buffer. * @param off offset at which to start storing bytes. * @param len maximum number of bytes to read. * @return the number of bytes read, or <code>-1</code> if the end of the * stream has been reached. * @exception IOException if this input stream has been closed by invoking * its {@link #close()} method, or an I/O error occurs. */ public synchronized int read(byte b[], int off, int len) throws IOException { getBufIfOpen(); // Check for closed stream if ((off | len | (off + len) | (b.length - (off + len))) < 0) { throw new IndexOutOfBoundsException(); } else if (len == 0) { return 0; } int n = 0; int total_nread = 0; for (;;) { int nread = read1(b, off + n, len - n); if (nread <= 0) { total_nread = (n == 0) ? nread : n; break; } n += nread; if (n >= len) { total_nread = n; break; } // if not closed but no bytes available, return InputStream input = in; if (input != null && input.available() <= 0) { total_nread = n; break; } } // Acquire from regulated flow if ((this.flow != null) && (total_nread > 0)) { try { this.flow.acquire(total_nread); } catch (InterruptedException exception) { LOGGER.error(exception); this.close(); throw new IOException(exception); } catch (RegulationException exception) { LOGGER.error(exception); this.close(); throw exception; } } if (listener != null) listener.bytesTransferred(this.flow.getTransferedSize (), total_nread, this.connectionParameters.getStreamSize ()); // Return total read number return total_nread; } /** * See the general contract of the <code>skip</code> method of * <code>InputStream</code>. * * @exception IOException if the stream does not support seek, or if this * input stream has been closed by invoking its * {@link #close()} method, or an I/O error occurs. */ public synchronized long skip(long n) throws IOException { getBufIfOpen(); // Check for closed stream if (n <= 0) { return 0; } long avail = count - pos; if (avail <= 0) { // If no mark position set then don't keep in buffer if (markpos < 0) return getInIfOpen().skip(n); // Fill in buffer to save bytes for reset fill(); avail = count - pos; if (avail <= 0) return 0; } long skipped = (avail < n) ? avail : n; pos += skipped; return skipped; } /** * Returns an estimate of the number of bytes that can be read (or skipped * over) from this input stream without blocking by the next invocation of a * method for this input stream. The next invocation might be the same thread * or another thread. A single read or skip of this many bytes will not * block, but may read or skip fewer bytes. * <p> * This method returns the sum of the number of bytes remaining to be read in * the buffer (<code>count - pos</code>) and the result of calling the * {@link java.io.FilterInputStream#in in}.available(). * * @return an estimate of the number of bytes that can be read (or skipped * over) from this input stream without blocking. * @exception IOException if this input stream has been closed by invoking * its {@link #close()} method, or an I/O error occurs. */ public synchronized int available() throws IOException { int n = count - pos; int avail = getInIfOpen().available(); return n > (Integer.MAX_VALUE - avail) ? Integer.MAX_VALUE : n + avail; } /** * See the general contract of the <code>mark</code> method of * <code>InputStream</code>. * * @param readlimit the maximum limit of bytes that can be read before the * mark position becomes invalid. * @see java.io.BufferedInputStream#reset() */ public synchronized void mark(int readlimit) { marklimit = readlimit; markpos = pos; } /** * See the general contract of the <code>reset</code> method of * <code>InputStream</code>. * <p> * If <code>markpos</code> is <code>-1</code> (no mark has been set or the * mark has been invalidated), an <code>IOException</code> is thrown. * Otherwise, <code>pos</code> is set equal to <code>markpos</code>. * * @exception IOException if this stream has not been marked or, if the mark * has been invalidated, or the stream has been closed by * invoking its {@link #close()} method, or an I/O error * occurs. * @see java.io.BufferedInputStream#mark(int) */ public synchronized void reset() throws IOException { getBufIfOpen(); // Cause exception if closed if (markpos < 0) throw new IOException("Resetting to invalid mark"); pos = markpos; } /** * Tests if this input stream supports the <code>mark</code> and * <code>reset</code> methods. The <code>markSupported</code> method of * <code>BufferedInputStream</code> returns <code>true</code>. * * @return a <code>boolean</code> indicating if this stream type supports the * <code>mark</code> and <code>reset</code> methods. * @see java.io.InputStream#mark(int) * @see java.io.InputStream#reset() */ public boolean markSupported() { return true; } /** * Closes this input stream and releases any system resources associated with * the stream. Once the stream has been closed, further read(), available(), * reset(), or skip() invocations will throw an IOException. Closing a * previously closed stream has no effect. * * @exception IOException if an I/O error occurs. */ public void close() throws IOException { boolean error = !(flow.getTransferedSize () == this.connectionParameters.getStreamSize ()); // Store transfer information into database if transfer completed. if (!error) { NetworkUsageService network_service = ApplicationContextProvider. getBean (NetworkUsageService.class); // Write database only if service exists and // if quota configuration requires persistent informations to // be saved. if ((network_service != null) && (flow.getUserQuotas() != null) && ((flow.getUserQuotas().getMaxCount()!=null) || (flow.getUserQuotas().getMaxCumulativeSize()!=null))) { network_service.createDownloadUsage (flow.getTransferedSize (), flow.getStartDate (), connectionParameters.getUser ()); } } // Notification of the error if (error && (this.listener != null)) { this.listener.bytesTransferred (flow.getTransferedSize (), -1, this.connectionParameters.getStreamSize ()); } // Release flow this.regulator.releaseChannel(this.flow); LOGGER.debug("CLOSED - " + this.flow); // Close stream byte[] buffer; while ((buffer = buf) != null) { if (BUF_UPDATER.compareAndSet(this, buffer, null)) { InputStream input = in; in = null; if (input != null) input.close(); return; } // Else retry in case a new buf was CASed in fill() } } /** * A builder class stemming from multiple constructors, multiple optional * parameters and overuse of setters while building a * {@link RegulatedInputStream}. */ public static class Builder { /** * Wrapped input stream. */ private final InputStream wrappedStream; /** * Traffic direction. */ private final TrafficDirection direction; /** * Regulator that will register the stream to be created. */ private Regulator regulator = null; /** * Buffer size in bytes. */ private int bufferSize = DEFAULT_BUFFER_SIZE; /** * User (optional). */ private User user = null; /** * User name (optional and used only is user class not provided). */ private String userName = null; /** * Listener to monitor stream copy (optional). */ private CopyStreamAdapter listener; /** * The size of the passed stream if known */ private Long streamSize; /** * Build a RegulatedInputStream builder. * * @param input_stream the input_stream to be regulated. This parameter * shall not be null. */ public Builder(final InputStream input_stream, TrafficDirection direction) throws IllegalArgumentException { // Check input stream if (input_stream == null) { throw new IllegalArgumentException("Null input stream."); } // Assign input stream this.wrappedStream = input_stream; // Check direction if (direction == null) { throw new IllegalArgumentException("Null traffic direction."); } // Assign traffic direction this.direction = direction; } // End Builder(Regulator, InputStream) /** * Set network regulator. */ public Builder regulator(final Regulator regulator) { this.regulator = regulator; return this; } /** * Set buffer size. */ public Builder bufferSize(final int buffer_size) { this.bufferSize = buffer_size; return this; } /** * Set stream size. */ public Builder streamSize(final long stream_size) { this.streamSize = stream_size; return this; } /** * Set user. */ public Builder user(final User user) { this.user = user; return this; } /** * Set user name. This parameter will only be used if the user ( * {@link User}) class has not been provided. */ public Builder userName(final String user_name) { this.userName = user_name; return this; } /** * Set the copy stream listener to listen copy progress. */ public Builder copyStreamListener (CopyStreamAdapter listener) { this.listener = listener; return this; } /** * Builds a RegulatedInputStream from this class members. * * @return a regulated input stream. * @throws IllegalArgumentException if {@link Builder#bufferSize} <= 0 * @throws RegulationException if a regulation rule prevent the creation * of this stream with this regulator e.g. maximum connections * reached, invalid user, etc. */ public RegulatedInputStream build() throws IllegalArgumentException, RegulationException { return new RegulatedInputStream(this); } } // End Builder class } // End RegulatedInputStream class