package org.apache.maven.index.context;
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import java.io.File;
import java.io.IOException;
import java.nio.channels.FileChannel;
import java.nio.channels.FileLock;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.MultiFields;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.SearcherManager;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.TopScoreDocCollector;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.FSLockFactory;
import org.apache.lucene.store.Lock;
import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.util.Bits;
import org.apache.maven.index.ArtifactInfo;
import org.apache.maven.index.artifact.GavCalculator;
import org.apache.maven.index.artifact.M2GavCalculator;
import org.codehaus.plexus.util.StringUtils;
/**
* The default {@link IndexingContext} implementation.
*
* @author Jason van Zyl
* @author Tamas Cservenak
*/
public class DefaultIndexingContext
extends AbstractIndexingContext
{
/**
* A standard location for indices served up by a webserver.
*/
private static final String INDEX_DIRECTORY = ".index";
public static final String FLD_DESCRIPTOR = "DESCRIPTOR";
public static final String FLD_DESCRIPTOR_CONTENTS = "NexusIndex";
public static final String FLD_IDXINFO = "IDXINFO";
public static final String VERSION = "1.0";
private static final Term DESCRIPTOR_TERM = new Term( FLD_DESCRIPTOR, FLD_DESCRIPTOR_CONTENTS );
private Directory indexDirectory;
private TrackingLockFactory lockFactory;
private File indexDirectoryFile;
private String id;
private boolean searchable;
private String repositoryId;
private File repository;
private String repositoryUrl;
private String indexUpdateUrl;
private NexusIndexWriter indexWriter;
private SearcherManager searcherManager;
private Date timestamp;
private List<? extends IndexCreator> indexCreators;
/**
* Currently nexus-indexer knows only M2 reposes
* <p>
* XXX move this into a concrete Scanner implementation
*/
private GavCalculator gavCalculator;
private DefaultIndexingContext( String id,
String repositoryId,
File repository, //
String repositoryUrl, String indexUpdateUrl,
List<? extends IndexCreator> indexCreators, Directory indexDirectory,
TrackingLockFactory lockFactory,
boolean reclaimIndex )
throws ExistingLuceneIndexMismatchException, IOException
{
this.id = id;
this.searchable = true;
this.repositoryId = repositoryId;
this.repository = repository;
this.repositoryUrl = repositoryUrl;
this.indexUpdateUrl = indexUpdateUrl;
this.indexWriter = null;
this.searcherManager = null;
this.indexCreators = indexCreators;
this.indexDirectory = indexDirectory;
this.lockFactory = lockFactory;
// eh?
// Guice does NOT initialize these, and we have to do manually?
// While in Plexus, all is well, but when in guice-shim,
// these objects are still LazyHintedBeans or what not and IndexerFields are NOT registered!
for ( IndexCreator indexCreator : indexCreators )
{
indexCreator.getIndexerFields();
}
this.gavCalculator = new M2GavCalculator();
prepareIndex( reclaimIndex );
setIndexDirectoryFile( null );
}
private DefaultIndexingContext( String id, String repositoryId, File repository, File indexDirectoryFile,
TrackingLockFactory lockFactory, String repositoryUrl, String indexUpdateUrl,
List<? extends IndexCreator> indexCreators, boolean reclaimIndex )
throws IOException, ExistingLuceneIndexMismatchException
{
this( id, repositoryId, repository, repositoryUrl, indexUpdateUrl, indexCreators,
FSDirectory.open( indexDirectoryFile.toPath(), lockFactory ), lockFactory, reclaimIndex );
setIndexDirectoryFile( indexDirectoryFile );
}
public DefaultIndexingContext( String id, String repositoryId, File repository, File indexDirectoryFile,
String repositoryUrl, String indexUpdateUrl,
List<? extends IndexCreator> indexCreators, boolean reclaimIndex )
throws IOException, ExistingLuceneIndexMismatchException
{
this( id, repositoryId, repository, indexDirectoryFile, new TrackingLockFactory( FSLockFactory.getDefault() ),
repositoryUrl, indexUpdateUrl, indexCreators, reclaimIndex );
}
@Deprecated
public DefaultIndexingContext( String id, String repositoryId, File repository, Directory indexDirectory,
String repositoryUrl, String indexUpdateUrl,
List<? extends IndexCreator> indexCreators, boolean reclaimIndex )
throws IOException, ExistingLuceneIndexMismatchException
{
this( id, repositoryId, repository, repositoryUrl, indexUpdateUrl, indexCreators, indexDirectory, null, reclaimIndex ); //Lock factory already installed - pass null
if ( indexDirectory instanceof FSDirectory )
{
setIndexDirectoryFile(( (FSDirectory) indexDirectory ).getDirectory().toFile() );
}
}
public Directory getIndexDirectory()
{
return indexDirectory;
}
/**
* Sets index location. As usually index is persistent (is on disk), this will point to that value, but in
* some circumstances (ie, using RAMDisk for index), this will point to an existing tmp directory.
*/
protected void setIndexDirectoryFile(File dir) throws IOException
{
if ( dir == null )
{
// best effort, to have a directory thru the life of a ctx
File tmpFile = File.createTempFile( "mindexer-ctx" + id, "tmp" );
tmpFile.delete();
tmpFile.mkdirs();
this.indexDirectoryFile = tmpFile;
}
else
{
this.indexDirectoryFile = dir;
}
}
public File getIndexDirectoryFile()
{
return indexDirectoryFile;
}
private void prepareIndex( boolean reclaimIndex )
throws IOException, ExistingLuceneIndexMismatchException
{
if ( DirectoryReader.indexExists( indexDirectory ) )
{
try
{
// unlock the dir forcibly
if ( IndexWriter.isLocked( indexDirectory ) )
{
unlockForcibly( lockFactory, indexDirectory );
}
openAndWarmup();
checkAndUpdateIndexDescriptor( reclaimIndex );
}
catch ( IOException e )
{
if ( reclaimIndex )
{
prepareCleanIndex( true );
}
else
{
throw e;
}
}
}
else
{
prepareCleanIndex( false );
}
timestamp = IndexUtils.getTimestamp( indexDirectory );
}
private void prepareCleanIndex( boolean deleteExisting )
throws IOException
{
if ( deleteExisting )
{
closeReaders();
// unlock the dir forcibly
if ( IndexWriter.isLocked( indexDirectory ) )
{
unlockForcibly( lockFactory, indexDirectory );
}
deleteIndexFiles( true );
}
openAndWarmup();
if ( StringUtils.isEmpty( getRepositoryId() ) )
{
throw new IllegalArgumentException( "The repositoryId cannot be null when creating new repository!" );
}
storeDescriptor();
}
private void checkAndUpdateIndexDescriptor( boolean reclaimIndex )
throws IOException, ExistingLuceneIndexMismatchException
{
if ( reclaimIndex )
{
// forcefully "reclaiming" the ownership of the index as ours
storeDescriptor();
return;
}
// check for descriptor if this is not a "virgin" index
if ( getSize() > 0 )
{
final TopScoreDocCollector collector = TopScoreDocCollector.create( 1 );
final IndexSearcher indexSearcher = acquireIndexSearcher();
try
{
indexSearcher.search( new TermQuery( DESCRIPTOR_TERM ), collector );
if ( collector.getTotalHits() == 0 )
{
throw new ExistingLuceneIndexMismatchException(
"The existing index has no NexusIndexer descriptor" );
}
if ( collector.getTotalHits() > 1 )
{
// eh? this is buggy index it seems, just iron it out then
storeDescriptor();
return;
}
else
{
// good, we have one descriptor as should
Document descriptor = indexSearcher.doc( collector.topDocs().scoreDocs[0].doc );
String[] h = StringUtils.split( descriptor.get( FLD_IDXINFO ), ArtifactInfo.FS );
// String version = h[0];
String repoId = h[1];
// // compare version
// if ( !VERSION.equals( version ) )
// {
// throw new UnsupportedExistingLuceneIndexException(
// "The existing index has version [" + version + "] and not [" + VERSION + "] version!" );
// }
if ( getRepositoryId() == null )
{
repositoryId = repoId;
}
else if ( !getRepositoryId().equals( repoId ) )
{
throw new ExistingLuceneIndexMismatchException( "The existing index is for repository " //
+ "[" + repoId + "] and not for repository [" + getRepositoryId() + "]" );
}
}
}
finally
{
releaseIndexSearcher( indexSearcher );
}
}
}
private void storeDescriptor()
throws IOException
{
Document hdr = new Document();
hdr.add( new Field( FLD_DESCRIPTOR, FLD_DESCRIPTOR_CONTENTS, Field.Store.YES, Field.Index.NOT_ANALYZED ) );
hdr.add( new Field( FLD_IDXINFO, VERSION + ArtifactInfo.FS + getRepositoryId(), Field.Store.YES, Field.Index.NO ) );
IndexWriter w = getIndexWriter();
w.updateDocument( DESCRIPTOR_TERM, hdr );
w.commit();
}
private void deleteIndexFiles( boolean full )
throws IOException
{
if ( indexDirectory != null )
{
String[] names = indexDirectory.listAll();
if ( names != null )
{
for (String name : names)
{
if (! (name.equals(INDEX_PACKER_PROPERTIES_FILE) || name.equals(INDEX_UPDATER_PROPERTIES_FILE) ))
{
indexDirectory.deleteFile(name);
}
}
}
if ( full )
{
try
{
indexDirectory.deleteFile( INDEX_PACKER_PROPERTIES_FILE );
}
catch ( IOException ioe )
{
//Does not exist
}
try
{
indexDirectory.deleteFile( INDEX_UPDATER_PROPERTIES_FILE );
}
catch ( IOException ioe )
{
//Does not exist
}
}
IndexUtils.deleteTimestamp( indexDirectory );
}
}
// ==
public boolean isSearchable()
{
return searchable;
}
public void setSearchable( boolean searchable )
{
this.searchable = searchable;
}
public String getId()
{
return id;
}
public void updateTimestamp()
throws IOException
{
updateTimestamp( false );
}
public void updateTimestamp( boolean save )
throws IOException
{
updateTimestamp( save, new Date() );
}
public void updateTimestamp( boolean save, Date timestamp )
throws IOException
{
this.timestamp = timestamp;
if ( save )
{
IndexUtils.updateTimestamp( indexDirectory, getTimestamp() );
}
}
public Date getTimestamp()
{
return timestamp;
}
public int getSize()
throws IOException
{
final IndexSearcher is = acquireIndexSearcher();
try
{
return is.getIndexReader().numDocs();
}
finally
{
releaseIndexSearcher( is );
}
}
public String getRepositoryId()
{
return repositoryId;
}
public File getRepository()
{
return repository;
}
public String getRepositoryUrl()
{
return repositoryUrl;
}
public String getIndexUpdateUrl()
{
if ( repositoryUrl != null )
{
if ( indexUpdateUrl == null || indexUpdateUrl.trim().length() == 0 )
{
return repositoryUrl + ( repositoryUrl.endsWith( "/" ) ? "" : "/" ) + INDEX_DIRECTORY;
}
}
return indexUpdateUrl;
}
public Analyzer getAnalyzer()
{
return new NexusAnalyzer();
}
protected void openAndWarmup()
throws IOException
{
// IndexWriter (close)
if ( indexWriter != null )
{
indexWriter.close();
indexWriter = null;
}
if ( searcherManager != null )
{
searcherManager.close();
searcherManager = null;
}
this.indexWriter = new NexusIndexWriter( getIndexDirectory(), getWriterConfig() );
this.indexWriter.commit(); // LUCENE-2386
this.searcherManager = new SearcherManager( indexWriter, false, new NexusIndexSearcherFactory( this ) );
}
/**
* Returns new IndexWriterConfig instance
*
* @since 5.1
*/
protected IndexWriterConfig getWriterConfig()
{
return NexusIndexWriter.defaultConfig();
}
public IndexWriter getIndexWriter()
throws IOException
{
return indexWriter;
}
public IndexSearcher acquireIndexSearcher()
throws IOException
{
// TODO: move this to separate thread to not penalty next incoming searcher
searcherManager.maybeRefresh();
return searcherManager.acquire();
}
public void releaseIndexSearcher( final IndexSearcher is )
throws IOException
{
if ( is == null )
{
return;
}
searcherManager.release( is );
}
public void commit()
throws IOException
{
getIndexWriter().commit();
}
public void rollback()
throws IOException
{
getIndexWriter().rollback();
}
public synchronized void optimize()
throws CorruptIndexException, IOException
{
commit();
}
public synchronized void close( boolean deleteFiles )
throws IOException
{
if ( indexDirectory != null )
{
IndexUtils.updateTimestamp( indexDirectory, getTimestamp() );
closeReaders();
if ( deleteFiles )
{
deleteIndexFiles( true );
}
indexDirectory.close();
}
indexDirectory = null;
}
public synchronized void purge()
throws IOException
{
closeReaders();
deleteIndexFiles( true );
openAndWarmup();
try
{
prepareIndex( true );
}
catch ( ExistingLuceneIndexMismatchException e )
{
// just deleted it
}
rebuildGroups();
updateTimestamp( true, null );
}
public synchronized void replace( Directory directory )
throws IOException
{
replace( directory, null, null );
}
public synchronized void replace( Directory directory, Set<String> allGroups, Set<String> rootGroups )
throws IOException
{
final Date ts = IndexUtils.getTimestamp( directory );
closeReaders();
deleteIndexFiles( false );
IndexUtils.copyDirectory( directory, indexDirectory );
openAndWarmup();
// reclaim the index as mine
storeDescriptor();
if ( allGroups == null && rootGroups == null )
{
rebuildGroups();
}
else
{
if ( allGroups != null )
{
setAllGroups( allGroups );
}
if ( rootGroups != null )
{
setRootGroups( rootGroups );
}
}
updateTimestamp( true, ts );
optimize();
}
public synchronized void merge( Directory directory )
throws IOException
{
merge( directory, null );
}
public synchronized void merge( Directory directory, DocumentFilter filter )
throws IOException
{
final IndexSearcher s = acquireIndexSearcher();
try
{
final IndexWriter w = getIndexWriter();
final IndexReader directoryReader = DirectoryReader.open( directory );
TopScoreDocCollector collector = null;
try
{
int numDocs = directoryReader.maxDoc();
Bits liveDocs = MultiFields.getLiveDocs(directoryReader);
for ( int i = 0; i < numDocs; i++ )
{
if (liveDocs != null && ! liveDocs.get(i) )
{
continue;
}
Document d = directoryReader.document( i );
if ( filter != null && !filter.accept( d ) )
{
continue;
}
String uinfo = d.get( ArtifactInfo.UINFO );
if ( uinfo != null )
{
collector = TopScoreDocCollector.create( 1 );
s.search( new TermQuery( new Term( ArtifactInfo.UINFO, uinfo ) ), collector );
if ( collector.getTotalHits() == 0 )
{
w.addDocument( IndexUtils.updateDocument( d, this, false ) );
}
}
else
{
String deleted = d.get( ArtifactInfo.DELETED );
if ( deleted != null )
{
// Deleting the document loses history that it was delete,
// so incrementals wont work. Therefore, put the delete
// document in as well
w.deleteDocuments( new Term( ArtifactInfo.UINFO, deleted ) );
w.addDocument( d );
}
}
}
}
finally
{
directoryReader.close();
commit();
}
rebuildGroups();
Date mergedTimestamp = IndexUtils.getTimestamp( directory );
if ( getTimestamp() != null && mergedTimestamp != null && mergedTimestamp.after( getTimestamp() ) )
{
// we have both, keep the newest
updateTimestamp( true, mergedTimestamp );
}
else
{
updateTimestamp( true );
}
optimize();
}
finally
{
releaseIndexSearcher( s );
}
}
private void closeReaders()
throws CorruptIndexException, IOException
{
if ( searcherManager != null )
{
searcherManager.close();
searcherManager = null;
}
if ( indexWriter != null )
{
indexWriter.close();
indexWriter = null;
}
}
public GavCalculator getGavCalculator()
{
return gavCalculator;
}
public List<IndexCreator> getIndexCreators()
{
return Collections.<IndexCreator>unmodifiableList( indexCreators );
}
// groups
public synchronized void rebuildGroups()
throws IOException
{
final IndexSearcher is = acquireIndexSearcher();
try
{
final IndexReader r = is.getIndexReader();
Set<String> rootGroups = new LinkedHashSet<String>();
Set<String> allGroups = new LinkedHashSet<String>();
int numDocs = r.maxDoc();
Bits liveDocs = MultiFields.getLiveDocs(r);
for ( int i = 0; i < numDocs; i++ )
{
if (liveDocs != null && !liveDocs.get(i) )
{
continue;
}
Document d = r.document( i );
String uinfo = d.get( ArtifactInfo.UINFO );
if ( uinfo != null )
{
ArtifactInfo info = IndexUtils.constructArtifactInfo( d, this );
rootGroups.add( info.getRootGroup() );
allGroups.add( info.getGroupId() );
}
}
setRootGroups( rootGroups );
setAllGroups( allGroups );
optimize();
}
finally
{
releaseIndexSearcher( is );
}
}
public Set<String> getAllGroups()
throws IOException
{
return getGroups( ArtifactInfo.ALL_GROUPS, ArtifactInfo.ALL_GROUPS_VALUE, ArtifactInfo.ALL_GROUPS_LIST );
}
public synchronized void setAllGroups( Collection<String> groups )
throws IOException
{
setGroups( groups, ArtifactInfo.ALL_GROUPS, ArtifactInfo.ALL_GROUPS_VALUE, ArtifactInfo.ALL_GROUPS_LIST );
commit();
}
public Set<String> getRootGroups()
throws IOException
{
return getGroups( ArtifactInfo.ROOT_GROUPS, ArtifactInfo.ROOT_GROUPS_VALUE, ArtifactInfo.ROOT_GROUPS_LIST );
}
public synchronized void setRootGroups( Collection<String> groups )
throws IOException
{
setGroups( groups, ArtifactInfo.ROOT_GROUPS, ArtifactInfo.ROOT_GROUPS_VALUE, ArtifactInfo.ROOT_GROUPS_LIST );
commit();
}
protected Set<String> getGroups( String field, String filedValue, String listField )
throws IOException, CorruptIndexException
{
final TopScoreDocCollector collector = TopScoreDocCollector.create( 1 );
final IndexSearcher indexSearcher = acquireIndexSearcher();
try
{
indexSearcher.search( new TermQuery( new Term( field, filedValue ) ), collector );
TopDocs topDocs = collector.topDocs();
Set<String> groups = new LinkedHashSet<String>( Math.max( 10, topDocs.totalHits ) );
if ( topDocs.totalHits > 0 )
{
Document doc = indexSearcher.doc( topDocs.scoreDocs[0].doc );
String groupList = doc.get( listField );
if ( groupList != null )
{
groups.addAll( Arrays.asList( groupList.split( "\\|" ) ) );
}
}
return groups;
}
finally
{
releaseIndexSearcher( indexSearcher );
}
}
protected void setGroups( Collection<String> groups, String groupField, String groupFieldValue,
String groupListField )
throws IOException, CorruptIndexException
{
final IndexWriter w = getIndexWriter();
w.updateDocument( new Term( groupField, groupFieldValue ),
createGroupsDocument( groups, groupField, groupFieldValue, groupListField ) );
}
protected Document createGroupsDocument( Collection<String> groups, String field, String fieldValue,
String listField )
{
final Document groupDoc = new Document();
groupDoc.add( new Field( field, //
fieldValue, Field.Store.YES, Field.Index.NOT_ANALYZED ) );
groupDoc.add( new Field( listField, //
ArtifactInfo.lst2str( groups ), Field.Store.YES, Field.Index.NO ) );
return groupDoc;
}
@Override
public String toString()
{
return id + " : " + timestamp;
}
private static void unlockForcibly( final TrackingLockFactory lockFactory, final Directory dir ) throws IOException
{
//Warning: Not doable in lucene >= 5.3 consider to remove it as IndexWriter.unlock
//was always strongly non recommended by Lucene.
//For now try to do the best to simulate the IndexWriter.unlock at least on FSDirectory
//using FSLockFactory, the RAMDirectory uses SingleInstanceLockFactory.
//custom lock factory?
if (lockFactory != null) {
final Set<? extends Lock> emittedLocks = lockFactory.getEmittedLocks(IndexWriter.WRITE_LOCK_NAME);
for (Lock emittedLock : emittedLocks) {
emittedLock.close();
}
}
if (dir instanceof FSDirectory) {
final FSDirectory fsdir = (FSDirectory) dir;
final Path dirPath = fsdir.getDirectory();
if (Files.isDirectory(dirPath)) {
Path lockPath = dirPath.resolve(IndexWriter.WRITE_LOCK_NAME);
try {
lockPath = lockPath.toRealPath();
} catch (IOException ioe) {
//Not locked
return;
}
try (final FileChannel fc = FileChannel.open(lockPath, StandardOpenOption.CREATE, StandardOpenOption.WRITE)) {
final FileLock lck = fc.tryLock();
if (lck == null) {
//Still active
throw new LockObtainFailedException("Lock held by another process: " + lockPath);
} else {
//Not held fine to release
lck.close();
}
}
Files.delete(lockPath);
}
}
}
}