package org.apache.maven.index.updater;
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import javax.inject.Inject;
import javax.inject.Named;
import javax.inject.Singleton;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Properties;
import java.util.Set;
import java.util.TimeZone;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.MultiFields;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.maven.index.context.DocumentFilter;
import org.apache.maven.index.context.IndexUtils;
import org.apache.maven.index.context.IndexingContext;
import org.apache.maven.index.context.NexusAnalyzer;
import org.apache.maven.index.context.NexusIndexWriter;
import org.apache.maven.index.fs.Lock;
import org.apache.maven.index.fs.Locker;
import org.apache.maven.index.incremental.IncrementalHandler;
import org.apache.maven.index.updater.IndexDataReader.IndexDataReadResult;
import org.codehaus.plexus.util.FileUtils;
import org.codehaus.plexus.util.io.RawInputStreamFacade;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* A default index updater implementation
*
* @author Jason van Zyl
* @author Eugene Kuleshov
*/
@Singleton
@Named
public class DefaultIndexUpdater
implements IndexUpdater
{
private final Logger logger = LoggerFactory.getLogger( getClass() );
protected Logger getLogger()
{
return logger;
}
private final IncrementalHandler incrementalHandler;
private final List<IndexUpdateSideEffect> sideEffects;
@Inject
public DefaultIndexUpdater( final IncrementalHandler incrementalHandler, final List<IndexUpdateSideEffect> sideEffects )
{
this.incrementalHandler = incrementalHandler;
this.sideEffects = sideEffects;
}
public IndexUpdateResult fetchAndUpdateIndex( final IndexUpdateRequest updateRequest )
throws IOException
{
IndexUpdateResult result = new IndexUpdateResult();
IndexingContext context = updateRequest.getIndexingContext();
ResourceFetcher fetcher = null;
if ( !updateRequest.isOffline() )
{
fetcher = updateRequest.getResourceFetcher();
// If no resource fetcher passed in, use the wagon fetcher by default
// and put back in request for future use
if ( fetcher == null )
{
throw new IOException( "Update of the index without provided ResourceFetcher is impossible." );
}
fetcher.connect( context.getId(), context.getIndexUpdateUrl() );
}
File cacheDir = updateRequest.getLocalIndexCacheDir();
Locker locker = updateRequest.getLocker();
Lock lock = locker != null && cacheDir != null ? locker.lock( cacheDir ) : null;
try
{
if ( cacheDir != null )
{
LocalCacheIndexAdaptor cache = new LocalCacheIndexAdaptor( cacheDir, result );
if ( !updateRequest.isOffline() )
{
cacheDir.mkdirs();
try
{
if( fetchAndUpdateIndex( updateRequest, fetcher, cache ).isSuccessful() )
{
cache.commit();
}
}
finally
{
fetcher.disconnect();
}
}
fetcher = cache.getFetcher();
}
else if ( updateRequest.isOffline() )
{
throw new IllegalArgumentException( "LocalIndexCacheDir can not be null in offline mode" );
}
try
{
if ( !updateRequest.isCacheOnly() )
{
LuceneIndexAdaptor target = new LuceneIndexAdaptor( updateRequest );
result = fetchAndUpdateIndex( updateRequest, fetcher, target );
if(result.isSuccessful())
{
target.commit();
}
}
}
finally
{
fetcher.disconnect();
}
}
finally
{
if ( lock != null )
{
lock.release();
}
}
return result;
}
private Date loadIndexDirectory( final IndexUpdateRequest updateRequest, final ResourceFetcher fetcher,
final boolean merge, final String remoteIndexFile )
throws IOException
{
File indexDir = File.createTempFile( remoteIndexFile, ".dir" );
indexDir.delete();
indexDir.mkdirs();
try(BufferedInputStream is = new BufferedInputStream( fetcher.retrieve( remoteIndexFile ) ); //
Directory directory = updateRequest.getFSDirectoryFactory().open( indexDir ))
{
Date timestamp = null;
Set<String> rootGroups = null;
Set<String> allGroups = null;
if ( remoteIndexFile.endsWith( ".gz" ) )
{
IndexDataReadResult result = unpackIndexData( is, directory, updateRequest.getIndexingContext() );
timestamp = result.getTimestamp();
rootGroups = result.getRootGroups();
allGroups = result.getAllGroups();
}
else
{
// legacy transfer format
throw new IllegalArgumentException("The legacy format is no longer supported by this version of maven-indexer.");
}
if ( updateRequest.getDocumentFilter() != null )
{
filterDirectory( directory, updateRequest.getDocumentFilter() );
}
if ( merge )
{
updateRequest.getIndexingContext().merge( directory );
}
else
{
updateRequest.getIndexingContext().replace( directory, rootGroups, allGroups );
}
if ( sideEffects != null && sideEffects.size() > 0 )
{
getLogger().info( IndexUpdateSideEffect.class.getName() + " extensions found: " + sideEffects.size() );
for ( IndexUpdateSideEffect sideeffect : sideEffects )
{
sideeffect.updateIndex( directory, updateRequest.getIndexingContext(), merge );
}
}
return timestamp;
}
finally
{
try
{
FileUtils.deleteDirectory( indexDir );
}
catch ( IOException ex )
{
// ignore
}
}
}
private static void filterDirectory( final Directory directory, final DocumentFilter filter )
throws IOException
{
IndexReader r = null;
IndexWriter w = null;
try
{
r = DirectoryReader.open( directory );
w = new NexusIndexWriter( directory, new NexusAnalyzer(), false );
Bits liveDocs = MultiFields.getLiveDocs(r);
int numDocs = r.maxDoc();
for ( int i = 0; i < numDocs; i++ )
{
if (liveDocs != null && ! liveDocs.get(i) )
{
continue;
}
Document d = r.document( i );
if ( !filter.accept( d ) )
{
boolean success = w.tryDeleteDocument(r, i);
//FIXME handle deletion failure
}
}
w.commit();
}
finally
{
IndexUtils.close( r );
IndexUtils.close( w );
}
w = null;
try
{
// analyzer is unimportant, since we are not adding/searching to/on index, only reading/deleting
w = new NexusIndexWriter( directory, new NexusAnalyzer(), false );
w.commit();
}
finally
{
IndexUtils.close( w );
}
}
private Properties loadIndexProperties( final File indexDirectoryFile, final String remoteIndexPropertiesName )
{
File indexProperties = new File( indexDirectoryFile, remoteIndexPropertiesName );
try ( FileInputStream fis = new FileInputStream( indexProperties ))
{
Properties properties = new Properties();
properties.load( fis );
return properties;
}
catch ( IOException e )
{
getLogger().debug( "Unable to read remote properties stored locally", e );
}
return null;
}
private void storeIndexProperties( final File dir, final String indexPropertiesName, final Properties properties )
throws IOException
{
File file = new File( dir, indexPropertiesName );
if ( properties != null )
{
try (OutputStream os = new BufferedOutputStream( new FileOutputStream( file ) ))
{
properties.store( os, null );
}
}
else
{
file.delete();
}
}
private Properties downloadIndexProperties( final ResourceFetcher fetcher )
throws IOException
{
try (InputStream fis = fetcher.retrieve( IndexingContext.INDEX_REMOTE_PROPERTIES_FILE ))
{
Properties properties = new Properties();
properties.load( fis );
return properties;
}
}
public Date getTimestamp( final Properties properties, final String key )
{
String indexTimestamp = properties.getProperty( key );
if ( indexTimestamp != null )
{
try
{
SimpleDateFormat df = new SimpleDateFormat( IndexingContext.INDEX_TIME_FORMAT );
df.setTimeZone( TimeZone.getTimeZone( "GMT" ) );
return df.parse( indexTimestamp );
}
catch ( ParseException ex )
{
}
}
return null;
}
/**
* Unpack index data using specified Lucene Index writer
*
* @param is an input stream to unpack index data from
* @param w a writer to save index data
* @param ics a collection of index creators for updating unpacked documents.
*/
public static IndexDataReadResult unpackIndexData( final InputStream is, final Directory d, final IndexingContext context )
throws IOException
{
NexusIndexWriter w = new NexusIndexWriter( d, new NexusAnalyzer(), true );
try
{
IndexDataReader dr = new IndexDataReader( is );
return dr.readIndex( w, context );
}
finally
{
IndexUtils.close( w );
}
}
/**
* Filesystem-based ResourceFetcher implementation
*/
public static class FileFetcher
implements ResourceFetcher
{
private final File basedir;
public FileFetcher( File basedir )
{
this.basedir = basedir;
}
public void connect( String id, String url )
throws IOException
{
// don't need to do anything
}
public void disconnect()
throws IOException
{
// don't need to do anything
}
public void retrieve( String name, File targetFile )
throws IOException, FileNotFoundException
{
FileUtils.copyFile( getFile( name ), targetFile );
}
public InputStream retrieve( String name )
throws IOException, FileNotFoundException
{
return new FileInputStream( getFile( name ) );
}
private File getFile( String name )
{
return new File( basedir, name );
}
}
private abstract class IndexAdaptor
{
protected final File dir;
protected Properties properties;
protected IndexAdaptor( File dir )
{
this.dir = dir;
}
public abstract Properties getProperties();
public abstract void storeProperties()
throws IOException;
public abstract void addIndexChunk( ResourceFetcher source, String filename )
throws IOException;
public abstract Date setIndexFile( ResourceFetcher source, String string )
throws IOException;
public Properties setProperties( ResourceFetcher source )
throws IOException
{
this.properties = downloadIndexProperties( source );
return properties;
}
public abstract Date getTimestamp();
public void commit()
throws IOException
{
storeProperties();
}
}
private class LuceneIndexAdaptor
extends IndexAdaptor
{
private final IndexUpdateRequest updateRequest;
public LuceneIndexAdaptor( IndexUpdateRequest updateRequest )
{
super( updateRequest.getIndexingContext().getIndexDirectoryFile() );
this.updateRequest = updateRequest;
}
public Properties getProperties()
{
if ( properties == null )
{
properties = loadIndexProperties( dir, IndexingContext.INDEX_UPDATER_PROPERTIES_FILE );
}
return properties;
}
public void storeProperties()
throws IOException
{
storeIndexProperties( dir, IndexingContext.INDEX_UPDATER_PROPERTIES_FILE, properties );
}
public Date getTimestamp()
{
return updateRequest.getIndexingContext().getTimestamp();
}
public void addIndexChunk( ResourceFetcher source, String filename )
throws IOException
{
loadIndexDirectory( updateRequest, source, true, filename );
}
public Date setIndexFile( ResourceFetcher source, String filename )
throws IOException
{
return loadIndexDirectory( updateRequest, source, false, filename );
}
public void commit()
throws IOException
{
super.commit();
updateRequest.getIndexingContext().commit();
}
}
private class LocalCacheIndexAdaptor
extends IndexAdaptor
{
private static final String CHUNKS_FILENAME = "chunks.lst";
private static final String CHUNKS_FILE_ENCODING = "UTF-8";
private final IndexUpdateResult result;
private final ArrayList<String> newChunks = new ArrayList<String>();
public LocalCacheIndexAdaptor( File dir, IndexUpdateResult result )
{
super( dir );
this.result = result;
}
public Properties getProperties()
{
if ( properties == null )
{
properties = loadIndexProperties( dir, IndexingContext.INDEX_REMOTE_PROPERTIES_FILE );
}
return properties;
}
public void storeProperties()
throws IOException
{
storeIndexProperties( dir, IndexingContext.INDEX_REMOTE_PROPERTIES_FILE, properties );
}
public Date getTimestamp()
{
Properties properties = getProperties();
if ( properties == null )
{
return null;
}
Date timestamp = DefaultIndexUpdater.this.getTimestamp( properties, IndexingContext.INDEX_TIMESTAMP );
if ( timestamp == null )
{
timestamp = DefaultIndexUpdater.this.getTimestamp( properties, IndexingContext.INDEX_LEGACY_TIMESTAMP );
}
return timestamp;
}
public void addIndexChunk( ResourceFetcher source, String filename )
throws IOException
{
File chunk = new File( dir, filename );
FileUtils.copyStreamToFile( new RawInputStreamFacade( source.retrieve( filename ) ), chunk );
newChunks.add( filename );
}
public Date setIndexFile( ResourceFetcher source, String filename )
throws IOException
{
cleanCacheDirectory( dir );
result.setFullUpdate( true );
File target = new File( dir, filename );
FileUtils.copyStreamToFile( new RawInputStreamFacade( source.retrieve( filename ) ), target );
return null;
}
@Override
public void commit()
throws IOException
{
File chunksFile = new File( dir, CHUNKS_FILENAME );
try (BufferedOutputStream os = new BufferedOutputStream( new FileOutputStream( chunksFile, true ) ); //
Writer w = new OutputStreamWriter( os, CHUNKS_FILE_ENCODING ))
{
for ( String filename : newChunks )
{
w.write( filename + "\n" );
}
w.flush();
}
super.commit();
}
public List<String> getChunks()
throws IOException
{
ArrayList<String> chunks = new ArrayList<String>();
File chunksFile = new File( dir, CHUNKS_FILENAME );
try (BufferedReader r =
new BufferedReader( new InputStreamReader( new FileInputStream( chunksFile ), CHUNKS_FILE_ENCODING ) ))
{
String str;
while ( ( str = r.readLine() ) != null )
{
chunks.add( str );
}
}
return chunks;
}
public ResourceFetcher getFetcher()
{
return new LocalIndexCacheFetcher( dir )
{
@Override
public List<String> getChunks()
throws IOException
{
return LocalCacheIndexAdaptor.this.getChunks();
}
};
}
}
abstract static class LocalIndexCacheFetcher
extends FileFetcher
{
public LocalIndexCacheFetcher( File basedir )
{
super( basedir );
}
public abstract List<String> getChunks()
throws IOException;
}
private IndexUpdateResult fetchAndUpdateIndex( final IndexUpdateRequest updateRequest, ResourceFetcher source,
IndexAdaptor target )
throws IOException
{
IndexUpdateResult result = new IndexUpdateResult();
if ( !updateRequest.isForceFullUpdate() )
{
Properties localProperties = target.getProperties();
Date localTimestamp = null;
if ( localProperties != null )
{
localTimestamp = getTimestamp( localProperties, IndexingContext.INDEX_TIMESTAMP );
}
// this will download and store properties in the target, so next run
// target.getProperties() will retrieve it
Properties remoteProperties = target.setProperties( source );
Date updateTimestamp = getTimestamp( remoteProperties, IndexingContext.INDEX_TIMESTAMP );
// If new timestamp is missing, dont bother checking incremental, we have an old file
if ( updateTimestamp != null )
{
List<String> filenames =
incrementalHandler.loadRemoteIncrementalUpdates( updateRequest, localProperties, remoteProperties );
// if we have some incremental files, merge them in
if ( filenames != null )
{
for ( String filename : filenames )
{
target.addIndexChunk( source, filename );
}
result.setTimestamp(updateTimestamp);
result.setSuccessful(true);
return result;
}
}
else
{
updateTimestamp = getTimestamp( remoteProperties, IndexingContext.INDEX_LEGACY_TIMESTAMP );
}
// fallback to timestamp comparison, but try with one coming from local properties, and if not possible (is
// null)
// fallback to context timestamp
if ( localTimestamp != null )
{
// if we have localTimestamp
// if incremental can't be done for whatever reason, simply use old logic of
// checking the timestamp, if the same, nothing to do
if ( updateTimestamp != null && localTimestamp != null && !updateTimestamp.after( localTimestamp ) )
{
//Index is up to date
result.setSuccessful(true);
return result;
}
}
}
else
{
// create index properties during forced full index download
target.setProperties( source );
}
if( !updateRequest.isIncrementalOnly() )
{
Date timestamp = null;
try
{
timestamp = target.setIndexFile( source, IndexingContext.INDEX_FILE_PREFIX + ".gz" );
if ( source instanceof LocalIndexCacheFetcher )
{
// local cache has inverse organization compared to remote indexes,
// i.e. initial index file and delta chunks to apply on top of it
for ( String filename : ( (LocalIndexCacheFetcher) source ).getChunks() )
{
target.addIndexChunk( source, filename );
}
}
}
catch ( IOException ex )
{
// try to look for legacy index transfer format
try
{
timestamp = target.setIndexFile( source, IndexingContext.INDEX_FILE_PREFIX + ".zip" );
}
catch ( IOException ex2 )
{
getLogger().error( "Fallback to *.zip also failed: " + ex2 ); // do not bother with stack trace
throw ex; // original exception more likely to be interesting
}
}
result.setTimestamp(timestamp);
result.setSuccessful(true);
result.setFullUpdate(true);
}
return result;
}
/**
* Cleans specified cache directory. If present, Locker.LOCK_FILE will not be deleted.
*/
protected void cleanCacheDirectory( File dir )
throws IOException
{
File[] members = dir.listFiles();
if ( members == null )
{
return;
}
for ( File member : members )
{
if ( !Locker.LOCK_FILE.equals( member.getName() ) )
{
FileUtils.forceDelete( member );
}
}
}
}