// ========================================================================
// Copyright (c) 2004-2009 Mort Bay Consulting Pty. Ltd.
// ------------------------------------------------------------------------
// All rights reserved. This program and the accompanying materials
// are made available under the terms of the Eclipse Public License v1.0
// and Apache License v2.0 which accompanies this distribution.
// The Eclipse Public License is available at
// http://www.eclipse.org/legal/epl-v10.html
// The Apache License v2.0 is available at
// http://www.opensource.org/licenses/apache2.0.php
// You may elect to redistribute this code under either of these licenses.
// ========================================================================
package org.eclipse.jetty.io.nio;
import java.io.IOException;
import java.nio.channels.CancelledKeyException;
import java.nio.channels.Channel;
import java.nio.channels.ClosedSelectorException;
import java.nio.channels.SelectableChannel;
import java.nio.channels.SelectionKey;
import java.nio.channels.Selector;
import java.nio.channels.ServerSocketChannel;
import java.nio.channels.SocketChannel;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import org.eclipse.jetty.io.AsyncEndPoint;
import org.eclipse.jetty.io.ConnectedEndPoint;
import org.eclipse.jetty.io.Connection;
import org.eclipse.jetty.io.EndPoint;
import org.eclipse.jetty.util.TypeUtil;
import org.eclipse.jetty.util.component.AbstractLifeCycle;
import org.eclipse.jetty.util.component.AggregateLifeCycle;
import org.eclipse.jetty.util.component.Dumpable;
import org.eclipse.jetty.util.log.Log;
import org.eclipse.jetty.util.log.Logger;
import org.eclipse.jetty.util.thread.Timeout;
import org.eclipse.jetty.util.thread.Timeout.Task;
/* ------------------------------------------------------------ */
/**
* The Selector Manager manages and number of SelectSets to allow
* NIO scheduling to scale to large numbers of connections.
* <p>
*/
public abstract class SelectorManager extends AbstractLifeCycle implements Dumpable
{
public static final Logger LOG=Log.getLogger("org.eclipse.jetty.io.nio");
private static final int __MONITOR_PERIOD=Integer.getInteger("org.eclipse.jetty.io.nio.MONITOR_PERIOD",1000).intValue();
private static final int __MAX_SELECTS=Integer.getInteger("org.eclipse.jetty.io.nio.MAX_SELECTS",100000).intValue();
private static final int __BUSY_PAUSE=Integer.getInteger("org.eclipse.jetty.io.nio.BUSY_PAUSE",50).intValue();
private static final int __IDLE_TICK=Integer.getInteger("org.eclipse.jetty.io.nio.IDLE_TICK",400).intValue();
private int _maxIdleTime;
private int _lowResourcesMaxIdleTime;
private long _lowResourcesConnections;
private SelectSet[] _selectSet;
private int _selectSets=1;
private volatile int _set=0;
private boolean _deferringInterestedOps0=true;
private int _selectorPriorityDelta=0;
/* ------------------------------------------------------------ */
/**
* @param maxIdleTime The maximum period in milli seconds that a connection may be idle before it is closed.
* @see #setLowResourcesMaxIdleTime(long)
*/
public void setMaxIdleTime(long maxIdleTime)
{
_maxIdleTime=(int)maxIdleTime;
}
/* ------------------------------------------------------------ */
/**
* @param selectSets number of select sets to create
*/
public void setSelectSets(int selectSets)
{
long lrc = _lowResourcesConnections * _selectSets;
_selectSets=selectSets;
_lowResourcesConnections=lrc/_selectSets;
}
/* ------------------------------------------------------------ */
/**
* @return the max idle time
*/
public long getMaxIdleTime()
{
return _maxIdleTime;
}
/* ------------------------------------------------------------ */
/**
* @return the number of select sets in use
*/
public int getSelectSets()
{
return _selectSets;
}
/* ------------------------------------------------------------ */
/**
* @param i
* @return The select set
*/
public SelectSet getSelectSet(int i)
{
return _selectSet[i];
}
/* ------------------------------------------------------------ */
/** Register a channel
* @param channel
* @param att Attached Object
*/
public void register(SocketChannel channel, Object att)
{
// The ++ increment here is not atomic, but it does not matter.
// so long as the value changes sometimes, then connections will
// be distributed over the available sets.
int s=_set++;
if (s<0)
s=-s;
s=s%_selectSets;
SelectSet[] sets=_selectSet;
if (sets!=null)
{
SelectSet set=sets[s];
set.addChange(channel,att);
set.wakeup();
}
}
/* ------------------------------------------------------------ */
/** Register a channel
* @param channel
*/
public void register(SocketChannel channel)
{
// The ++ increment here is not atomic, but it does not matter.
// so long as the value changes sometimes, then connections will
// be distributed over the available sets.
int s=_set++;
if (s<0)
s=-s;
s=s%_selectSets;
SelectSet[] sets=_selectSet;
if (sets!=null)
{
SelectSet set=sets[s];
set.addChange(channel);
set.wakeup();
}
}
/* ------------------------------------------------------------ */
/** Register a {@link ServerSocketChannel}
* @param acceptChannel
*/
public void register(ServerSocketChannel acceptChannel)
{
int s=_set++;
if (s<0)
s=-s;
s=s%_selectSets;
SelectSet set=_selectSet[s];
set.addChange(acceptChannel);
set.wakeup();
}
/* ------------------------------------------------------------ */
/**
* @return delta The value to add to the selector thread priority.
*/
public int getSelectorPriorityDelta()
{
return _selectorPriorityDelta;
}
/* ------------------------------------------------------------ */
/** Set the selector thread priorty delta.
* @param delta The value to add to the selector thread priority.
*/
public void setSelectorPriorityDelta(int delta)
{
_selectorPriorityDelta=delta;
}
/* ------------------------------------------------------------ */
/**
* @return the lowResourcesConnections
*/
public long getLowResourcesConnections()
{
return _lowResourcesConnections*_selectSets;
}
/* ------------------------------------------------------------ */
/**
* Set the number of connections, which if exceeded places this manager in low resources state.
* This is not an exact measure as the connection count is averaged over the select sets.
* @param lowResourcesConnections the number of connections
* @see #setLowResourcesMaxIdleTime(long)
*/
public void setLowResourcesConnections(long lowResourcesConnections)
{
_lowResourcesConnections=(lowResourcesConnections+_selectSets-1)/_selectSets;
}
/* ------------------------------------------------------------ */
/**
* @return the lowResourcesMaxIdleTime
*/
public long getLowResourcesMaxIdleTime()
{
return _lowResourcesMaxIdleTime;
}
/* ------------------------------------------------------------ */
/**
* @param lowResourcesMaxIdleTime the period in ms that a connection is allowed to be idle when this SelectSet has more connections than {@link #getLowResourcesConnections()}
* @see #setMaxIdleTime(long)
*/
public void setLowResourcesMaxIdleTime(long lowResourcesMaxIdleTime)
{
_lowResourcesMaxIdleTime=(int)lowResourcesMaxIdleTime;
}
/* ------------------------------------------------------------------------------- */
public abstract boolean dispatch(Runnable task);
/* ------------------------------------------------------------ */
/* (non-Javadoc)
* @see org.eclipse.component.AbstractLifeCycle#doStart()
*/
@Override
protected void doStart() throws Exception
{
_selectSet = new SelectSet[_selectSets];
for (int i=0;i<_selectSet.length;i++)
_selectSet[i]= new SelectSet(i);
super.doStart();
// start a thread to Select
for (int i=0;i<getSelectSets();i++)
{
final int id=i;
boolean selecting=dispatch(new Runnable()
{
public void run()
{
String name=Thread.currentThread().getName();
int priority=Thread.currentThread().getPriority();
try
{
SelectSet[] sets=_selectSet;
if (sets==null)
return;
SelectSet set=sets[id];
Thread.currentThread().setName(name+" Selector"+id);
if (getSelectorPriorityDelta()!=0)
Thread.currentThread().setPriority(Thread.currentThread().getPriority()+getSelectorPriorityDelta());
LOG.debug("Starting {} on {}",Thread.currentThread(),this);
while (isRunning())
{
try
{
set.doSelect();
}
catch(IOException e)
{
LOG.ignore(e);
}
catch(Exception e)
{
LOG.warn(e);
}
}
}
finally
{
LOG.debug("Stopped {} on {}",Thread.currentThread(),this);
Thread.currentThread().setName(name);
if (getSelectorPriorityDelta()!=0)
Thread.currentThread().setPriority(priority);
}
}
});
if (!selecting)
throw new IllegalStateException("!Selecting");
}
}
/* ------------------------------------------------------------------------------- */
@Override
protected void doStop() throws Exception
{
SelectSet[] sets= _selectSet;
_selectSet=null;
if (sets!=null)
{
for (SelectSet set : sets)
{
if (set!=null)
set.stop();
}
}
super.doStop();
}
/* ------------------------------------------------------------ */
/**
* @param endpoint
*/
protected abstract void endPointClosed(SelectChannelEndPoint endpoint);
/* ------------------------------------------------------------ */
/**
* @param endpoint
*/
protected abstract void endPointOpened(SelectChannelEndPoint endpoint);
/* ------------------------------------------------------------ */
protected abstract void endPointUpgraded(ConnectedEndPoint endpoint,Connection oldConnection);
/* ------------------------------------------------------------------------------- */
public abstract AsyncConnection newConnection(SocketChannel channel, AsyncEndPoint endpoint, Object attachment);
/* ------------------------------------------------------------ */
/**
* Create a new end point
* @param channel
* @param selectSet
* @param sKey the selection key
* @return the new endpoint {@link SelectChannelEndPoint}
* @throws IOException
*/
protected abstract SelectChannelEndPoint newEndPoint(SocketChannel channel, SelectorManager.SelectSet selectSet, SelectionKey sKey) throws IOException;
/* ------------------------------------------------------------------------------- */
protected void connectionFailed(SocketChannel channel,Throwable ex,Object attachment)
{
LOG.warn(ex+","+channel+","+attachment);
LOG.debug(ex);
}
/* ------------------------------------------------------------ */
public String dump()
{
return AggregateLifeCycle.dump(this);
}
/* ------------------------------------------------------------ */
public void dump(Appendable out, String indent) throws IOException
{
AggregateLifeCycle.dumpObject(out,this);
AggregateLifeCycle.dump(out,indent,TypeUtil.asList(_selectSet));
}
/* ------------------------------------------------------------------------------- */
/* ------------------------------------------------------------------------------- */
/* ------------------------------------------------------------------------------- */
public class SelectSet implements Dumpable
{
private final int _setID;
private final Timeout _timeout;
private final ConcurrentLinkedQueue<Object> _changes = new ConcurrentLinkedQueue<Object>();
private volatile Selector _selector;
private volatile Thread _selecting;
private int _busySelects;
private long _monitorNext;
private boolean _pausing;
private boolean _paused;
private volatile long _idleTick;
private ConcurrentMap<SelectChannelEndPoint,Object> _endPoints = new ConcurrentHashMap<SelectChannelEndPoint, Object>();
/* ------------------------------------------------------------ */
SelectSet(int acceptorID) throws Exception
{
_setID=acceptorID;
_idleTick = System.currentTimeMillis();
_timeout = new Timeout(this);
_timeout.setDuration(0L);
// create a selector;
_selector = Selector.open();
_monitorNext=System.currentTimeMillis()+__MONITOR_PERIOD;
}
/* ------------------------------------------------------------ */
public void addChange(Object change)
{
_changes.add(change);
}
/* ------------------------------------------------------------ */
public void addChange(SelectableChannel channel, Object att)
{
if (att==null)
addChange(channel);
else if (att instanceof EndPoint)
addChange(att);
else
addChange(new ChannelAndAttachment(channel,att));
}
/* ------------------------------------------------------------ */
/**
* Select and dispatch tasks found from changes and the selector.
*
* @throws IOException
*/
public void doSelect() throws IOException
{
try
{
_selecting=Thread.currentThread();
final Selector selector=_selector;
// Stopped concurrently ?
if (selector == null)
return;
// Make any key changes required
Object change;
int changes=_changes.size();
while (changes-->0 && (change=_changes.poll())!=null)
{
Channel ch=null;
SelectionKey key=null;
try
{
if (change instanceof EndPoint)
{
// Update the operations for a key.
SelectChannelEndPoint endpoint = (SelectChannelEndPoint)change;
ch=endpoint.getChannel();
endpoint.doUpdateKey();
}
else if (change instanceof ChannelAndAttachment)
{
// finish accepting/connecting this connection
final ChannelAndAttachment asc = (ChannelAndAttachment)change;
final SelectableChannel channel=asc._channel;
ch=channel;
final Object att = asc._attachment;
if ((channel instanceof SocketChannel) && ((SocketChannel)channel).isConnected())
{
key = channel.register(selector,SelectionKey.OP_READ,att);
SelectChannelEndPoint endpoint = createEndPoint((SocketChannel)channel,key);
key.attach(endpoint);
endpoint.schedule();
}
else if (channel.isOpen())
{
key = channel.register(selector,SelectionKey.OP_CONNECT,att);
}
}
else if (change instanceof SocketChannel)
{
// Newly registered channel
final SocketChannel channel=(SocketChannel)change;
ch=channel;
key = channel.register(selector,SelectionKey.OP_READ,null);
SelectChannelEndPoint endpoint = createEndPoint(channel,key);
key.attach(endpoint);
endpoint.schedule();
}
else if (change instanceof ChangeTask)
{
((Runnable)change).run();
}
else if (change instanceof Runnable)
{
dispatch((Runnable)change);
}
else
throw new IllegalArgumentException(change.toString());
}
catch (CancelledKeyException e)
{
LOG.ignore(e);
}
catch (Throwable e)
{
if (isRunning())
LOG.warn(e);
else
LOG.debug(e);
try
{
if (ch!=null)
ch.close();
}
catch(IOException e2)
{
LOG.debug(e2);
}
}
}
// Do and instant select to see if any connections can be handled.
int selected=selector.selectNow();
long now=System.currentTimeMillis();
// if no immediate things to do
if (selected==0 && selector.selectedKeys().isEmpty())
{
// If we are in pausing mode
if (_pausing)
{
try
{
Thread.sleep(__BUSY_PAUSE); // pause to reduce impact of busy loop
}
catch(InterruptedException e)
{
LOG.ignore(e);
}
now=System.currentTimeMillis();
}
// workout how long to wait in select
_timeout.setNow(now);
long to_next_timeout=_timeout.getTimeToNext();
long wait = _changes.size()==0?__IDLE_TICK:0L;
if (wait > 0 && to_next_timeout >= 0 && wait > to_next_timeout)
wait = to_next_timeout;
// If we should wait with a select
if (wait>0)
{
long before=now;
selected=selector.select(wait);
now = System.currentTimeMillis();
_timeout.setNow(now);
// If we are monitoring for busy selector
// and this select did not wait more than 1ms
if (__MONITOR_PERIOD>0 && now-before <=1)
{
// count this as a busy select and if there have been too many this monitor cycle
if (++_busySelects>__MAX_SELECTS)
{
// Start injecting pauses
_pausing=true;
// if this is the first pause
if (!_paused)
{
// Log and dump some status
_paused=true;
LOG.warn("Selector {} is too busy, pausing!",this);
}
}
}
}
}
// have we been destroyed while sleeping
if (_selector==null || !selector.isOpen())
return;
// Look for things to do
for (SelectionKey key: selector.selectedKeys())
{
SocketChannel channel=null;
try
{
if (!key.isValid())
{
key.cancel();
SelectChannelEndPoint endpoint = (SelectChannelEndPoint)key.attachment();
if (endpoint != null)
endpoint.doUpdateKey();
continue;
}
Object att = key.attachment();
if (att instanceof SelectChannelEndPoint)
{
if (key.isReadable()||key.isWritable())
((SelectChannelEndPoint)att).schedule();
}
else if (key.isConnectable())
{
// Complete a connection of a registered channel
channel = (SocketChannel)key.channel();
boolean connected=false;
try
{
connected=channel.finishConnect();
}
catch(Exception e)
{
connectionFailed(channel,e,att);
}
finally
{
if (connected)
{
key.interestOps(SelectionKey.OP_READ);
SelectChannelEndPoint endpoint = createEndPoint(channel,key);
key.attach(endpoint);
endpoint.schedule();
}
else
{
key.cancel();
}
}
}
else
{
// Wrap readable registered channel in an endpoint
channel = (SocketChannel)key.channel();
SelectChannelEndPoint endpoint = createEndPoint(channel,key);
key.attach(endpoint);
if (key.isReadable())
endpoint.schedule();
}
key = null;
}
catch (CancelledKeyException e)
{
LOG.ignore(e);
}
catch (Exception e)
{
if (isRunning())
LOG.warn(e);
else
LOG.ignore(e);
try
{
if (channel!=null)
channel.close();
}
catch(IOException e2)
{
LOG.debug(e2);
}
if (key != null && !(key.channel() instanceof ServerSocketChannel) && key.isValid())
key.cancel();
}
}
// Everything always handled
selector.selectedKeys().clear();
now=System.currentTimeMillis();
_timeout.setNow(now);
Task task = _timeout.expired();
while (task!=null)
{
if (task instanceof Runnable)
dispatch((Runnable)task);
task = _timeout.expired();
}
// Idle tick
if (now-_idleTick>__IDLE_TICK)
{
_idleTick=now;
final long idle_now=((_lowResourcesConnections>0 && selector.keys().size()>_lowResourcesConnections))
?(now+_maxIdleTime-_lowResourcesMaxIdleTime)
:now;
dispatch(new Runnable()
{
public void run()
{
for (SelectChannelEndPoint endp:_endPoints.keySet())
{
endp.checkIdleTimestamp(idle_now);
}
}
public String toString() {return "Idle-"+super.toString();}
});
}
// Reset busy select monitor counts
if (__MONITOR_PERIOD>0 && now>_monitorNext)
{
_busySelects=0;
_pausing=false;
_monitorNext=now+__MONITOR_PERIOD;
}
}
catch (ClosedSelectorException e)
{
if (isRunning())
LOG.warn(e);
else
LOG.ignore(e);
}
catch (CancelledKeyException e)
{
LOG.ignore(e);
}
finally
{
_selecting=null;
}
}
/* ------------------------------------------------------------ */
private void renewSelector()
{
try
{
synchronized (this)
{
Selector selector=_selector;
if (selector==null)
return;
final Selector new_selector = Selector.open();
for (SelectionKey k: selector.keys())
{
if (!k.isValid() || k.interestOps()==0)
continue;
final SelectableChannel channel = k.channel();
final Object attachment = k.attachment();
if (attachment==null)
addChange(channel);
else
addChange(channel,attachment);
}
_selector.close();
_selector=new_selector;
}
}
catch(IOException e)
{
throw new RuntimeException("recreating selector",e);
}
}
/* ------------------------------------------------------------ */
public SelectorManager getManager()
{
return SelectorManager.this;
}
/* ------------------------------------------------------------ */
public long getNow()
{
return _timeout.getNow();
}
/* ------------------------------------------------------------ */
/**
* @param task The task to timeout. If it implements Runnable, then
* expired will be called from a dispatched thread.
*
* @param timeoutMs
*/
public void scheduleTimeout(Timeout.Task task, long timeoutMs)
{
if (!(task instanceof Runnable))
throw new IllegalArgumentException("!Runnable");
_timeout.schedule(task, timeoutMs);
}
/* ------------------------------------------------------------ */
public void cancelTimeout(Timeout.Task task)
{
task.cancel();
}
/* ------------------------------------------------------------ */
public void wakeup()
{
try
{
Selector selector = _selector;
if (selector!=null)
selector.wakeup();
}
catch(Exception e)
{
addChange(new ChangeTask()
{
public void run()
{
renewSelector();
}
});
renewSelector();
}
}
/* ------------------------------------------------------------ */
private SelectChannelEndPoint createEndPoint(SocketChannel channel, SelectionKey sKey) throws IOException
{
SelectChannelEndPoint endp = newEndPoint(channel,this,sKey);
LOG.debug("created {}",endp);
endPointOpened(endp);
_endPoints.put(endp,this);
return endp;
}
/* ------------------------------------------------------------ */
public void destroyEndPoint(SelectChannelEndPoint endp)
{
LOG.debug("destroyEndPoint {}",endp);
_endPoints.remove(endp);
endPointClosed(endp);
}
/* ------------------------------------------------------------ */
Selector getSelector()
{
return _selector;
}
/* ------------------------------------------------------------ */
void stop() throws Exception
{
// Spin for a while waiting for selector to complete
// to avoid unneccessary closed channel exceptions
try
{
for (int i=0;i<100 && _selecting!=null;i++)
{
wakeup();
Thread.sleep(10);
}
}
catch(Exception e)
{
LOG.ignore(e);
}
// close endpoints and selector
synchronized (this)
{
Selector selector=_selector;
for (SelectionKey key:selector.keys())
{
if (key==null)
continue;
Object att=key.attachment();
if (att instanceof EndPoint)
{
EndPoint endpoint = (EndPoint)att;
try
{
endpoint.close();
}
catch(IOException e)
{
LOG.ignore(e);
}
}
}
_timeout.cancelAll();
try
{
selector=_selector;
if (selector != null)
selector.close();
}
catch (IOException e)
{
LOG.ignore(e);
}
_selector=null;
}
}
/* ------------------------------------------------------------ */
public String dump()
{
return AggregateLifeCycle.dump(this);
}
/* ------------------------------------------------------------ */
public void dump(Appendable out, String indent) throws IOException
{
out.append(String.valueOf(this)).append(" id=").append(String.valueOf(_setID)).append("\n");
Thread selecting = _selecting;
Object where = "not selecting";
StackTraceElement[] trace =selecting==null?null:selecting.getStackTrace();
if (trace!=null)
{
for (StackTraceElement t:trace)
if (t.getClassName().startsWith("org.eclipse.jetty."))
{
where=t;
break;
}
}
Selector selector=_selector;
if (selector!=null)
{
final ArrayList<Object> dump = new ArrayList<Object>(selector.keys().size()*2);
dump.add(where);
final CountDownLatch latch = new CountDownLatch(1);
addChange(new ChangeTask()
{
public void run()
{
dumpKeyState(dump);
latch.countDown();
}
});
try
{
latch.await(5,TimeUnit.SECONDS);
}
catch(InterruptedException e)
{
LOG.ignore(e);
}
AggregateLifeCycle.dump(out,indent,dump);
}
}
/* ------------------------------------------------------------ */
public void dumpKeyState(List<Object> dumpto)
{
Selector selector=_selector;
Set<SelectionKey> keys = selector.keys();
dumpto.add(selector + " keys=" + keys.size());
for (SelectionKey key: keys)
{
if (key.isValid())
dumpto.add(key.attachment()+" iOps="+key.interestOps()+" rOps="+key.readyOps());
else
dumpto.add(key.attachment()+" iOps=-1 rOps=-1");
}
}
/* ------------------------------------------------------------ */
public String toString()
{
Selector selector=_selector;
return String.format("%s keys=%d selected=%d",
super.toString(),
selector != null && selector.isOpen() ? selector.keys().size() : -1,
selector != null && selector.isOpen() ? selector.selectedKeys().size() : -1);
}
}
/* ------------------------------------------------------------ */
private static class ChannelAndAttachment
{
final SelectableChannel _channel;
final Object _attachment;
public ChannelAndAttachment(SelectableChannel channel, Object attachment)
{
super();
_channel = channel;
_attachment = attachment;
}
}
/* ------------------------------------------------------------ */
public boolean isDeferringInterestedOps0()
{
return _deferringInterestedOps0;
}
/* ------------------------------------------------------------ */
public void setDeferringInterestedOps0(boolean deferringInterestedOps0)
{
_deferringInterestedOps0 = deferringInterestedOps0;
}
/* ------------------------------------------------------------ */
/* ------------------------------------------------------------ */
/* ------------------------------------------------------------ */
private interface ChangeTask extends Runnable
{}
}