package org.apache.archiva.proxy;
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import org.apache.archiva.admin.model.RepositoryAdminException;
import org.apache.archiva.admin.model.beans.NetworkProxy;
import org.apache.archiva.admin.model.beans.ProxyConnectorRuleType;
import org.apache.archiva.admin.model.beans.RemoteRepository;
import org.apache.archiva.admin.model.networkproxy.NetworkProxyAdmin;
import org.apache.archiva.common.filelock.FileLockException;
import org.apache.archiva.common.filelock.FileLockManager;
import org.apache.archiva.common.filelock.FileLockTimeoutException;
import org.apache.archiva.common.filelock.Lock;
import org.apache.archiva.configuration.ArchivaConfiguration;
import org.apache.archiva.configuration.Configuration;
import org.apache.archiva.configuration.ConfigurationNames;
import org.apache.archiva.configuration.NetworkProxyConfiguration;
import org.apache.archiva.configuration.ProxyConnectorConfiguration;
import org.apache.archiva.configuration.ProxyConnectorRuleConfiguration;
import org.apache.archiva.model.ArtifactReference;
import org.apache.archiva.model.Keys;
import org.apache.archiva.model.RepositoryURL;
import org.apache.archiva.policies.DownloadErrorPolicy;
import org.apache.archiva.policies.DownloadPolicy;
import org.apache.archiva.policies.PolicyConfigurationException;
import org.apache.archiva.policies.PolicyViolationException;
import org.apache.archiva.policies.PostDownloadPolicy;
import org.apache.archiva.policies.PreDownloadPolicy;
import org.apache.archiva.policies.ProxyDownloadException;
import org.apache.archiva.policies.urlcache.UrlFailureCache;
import org.apache.archiva.proxy.common.WagonFactory;
import org.apache.archiva.proxy.common.WagonFactoryException;
import org.apache.archiva.proxy.common.WagonFactoryRequest;
import org.apache.archiva.proxy.model.ProxyConnector;
import org.apache.archiva.proxy.model.ProxyFetchResult;
import org.apache.archiva.proxy.model.RepositoryProxyConnectors;
import org.apache.archiva.redback.components.registry.Registry;
import org.apache.archiva.redback.components.registry.RegistryListener;
import org.apache.archiva.redback.components.taskqueue.TaskQueueException;
import org.apache.archiva.repository.ManagedRepositoryContent;
import org.apache.archiva.repository.RemoteRepositoryContent;
import org.apache.archiva.repository.RepositoryContentFactory;
import org.apache.archiva.repository.RepositoryException;
import org.apache.archiva.repository.RepositoryNotFoundException;
import org.apache.archiva.repository.metadata.MetadataTools;
import org.apache.archiva.repository.metadata.RepositoryMetadataException;
import org.apache.archiva.scheduler.ArchivaTaskScheduler;
import org.apache.archiva.scheduler.repository.model.RepositoryTask;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.SystemUtils;
import org.apache.maven.wagon.ConnectionException;
import org.apache.maven.wagon.ResourceDoesNotExistException;
import org.apache.maven.wagon.Wagon;
import org.apache.maven.wagon.WagonException;
import org.apache.maven.wagon.authentication.AuthenticationException;
import org.apache.maven.wagon.authentication.AuthenticationInfo;
import org.apache.maven.wagon.proxy.ProxyInfo;
import org.apache.maven.wagon.repository.Repository;
import org.apache.tools.ant.types.selectors.SelectorUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.MarkerFactory;
import org.springframework.stereotype.Service;
import javax.annotation.PostConstruct;
import javax.inject.Inject;
import javax.inject.Named;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.TimeUnit;
/**
* DefaultRepositoryProxyConnectors
* TODO exception handling needs work - "not modified" is not really an exceptional case, and it has more layers than
* your average brown onion
*/
@Service("repositoryProxyConnectors#default")
public class DefaultRepositoryProxyConnectors
implements RepositoryProxyConnectors, RegistryListener
{
private Logger log = LoggerFactory.getLogger( DefaultRepositoryProxyConnectors.class );
@Inject
@Named(value = "archivaConfiguration#default")
private ArchivaConfiguration archivaConfiguration;
@Inject
@Named(value = "repositoryContentFactory#default")
private RepositoryContentFactory repositoryFactory;
@Inject
@Named(value = "metadataTools#default")
private MetadataTools metadataTools;
@Inject
private Map<String, PreDownloadPolicy> preDownloadPolicies;
@Inject
private Map<String, PostDownloadPolicy> postDownloadPolicies;
@Inject
private Map<String, DownloadErrorPolicy> downloadErrorPolicies;
@Inject
private UrlFailureCache urlFailureCache;
private ConcurrentMap<String, List<ProxyConnector>> proxyConnectorMap = new ConcurrentHashMap<>();
private ConcurrentMap<String, ProxyInfo> networkProxyMap = new ConcurrentHashMap<>();
@Inject
private WagonFactory wagonFactory;
@Inject
@Named(value = "archivaTaskScheduler#repository")
private ArchivaTaskScheduler scheduler;
@Inject
private NetworkProxyAdmin networkProxyAdmin;
@Inject
@Named(value = "fileLockManager#default")
private FileLockManager fileLockManager;
@PostConstruct
public void initialize()
{
initConnectorsAndNetworkProxies();
archivaConfiguration.addChangeListener( this );
}
@SuppressWarnings("unchecked")
private void initConnectorsAndNetworkProxies()
{
ProxyConnectorOrderComparator proxyOrderSorter = new ProxyConnectorOrderComparator();
this.proxyConnectorMap.clear();
Configuration configuration = archivaConfiguration.getConfiguration();
List<ProxyConnectorRuleConfiguration> allProxyConnectorRuleConfigurations =
configuration.getProxyConnectorRuleConfigurations();
List<ProxyConnectorConfiguration> proxyConfigs = configuration.getProxyConnectors();
for ( ProxyConnectorConfiguration proxyConfig : proxyConfigs )
{
String key = proxyConfig.getSourceRepoId();
try
{
// Create connector object.
ProxyConnector connector = new ProxyConnector();
connector.setSourceRepository(
repositoryFactory.getManagedRepositoryContent( proxyConfig.getSourceRepoId() ) );
connector.setTargetRepository(
repositoryFactory.getRemoteRepositoryContent( proxyConfig.getTargetRepoId() ) );
connector.setProxyId( proxyConfig.getProxyId() );
connector.setPolicies( proxyConfig.getPolicies() );
connector.setOrder( proxyConfig.getOrder() );
connector.setDisabled( proxyConfig.isDisabled() );
// Copy any blacklist patterns.
List<String> blacklist = new ArrayList<>( 0 );
if ( CollectionUtils.isNotEmpty( proxyConfig.getBlackListPatterns() ) )
{
blacklist.addAll( proxyConfig.getBlackListPatterns() );
}
connector.setBlacklist( blacklist );
// Copy any whitelist patterns.
List<String> whitelist = new ArrayList<>( 0 );
if ( CollectionUtils.isNotEmpty( proxyConfig.getWhiteListPatterns() ) )
{
whitelist.addAll( proxyConfig.getWhiteListPatterns() );
}
connector.setWhitelist( whitelist );
List<ProxyConnectorRuleConfiguration> proxyConnectorRuleConfigurations =
findProxyConnectorRules( connector.getSourceRepository().getId(),
connector.getTargetRepository().getId(),
allProxyConnectorRuleConfigurations );
if ( !proxyConnectorRuleConfigurations.isEmpty() )
{
for ( ProxyConnectorRuleConfiguration proxyConnectorRuleConfiguration : proxyConnectorRuleConfigurations )
{
if ( StringUtils.equals( proxyConnectorRuleConfiguration.getRuleType(),
ProxyConnectorRuleType.BLACK_LIST.getRuleType() ) )
{
connector.getBlacklist().add( proxyConnectorRuleConfiguration.getPattern() );
}
if ( StringUtils.equals( proxyConnectorRuleConfiguration.getRuleType(),
ProxyConnectorRuleType.WHITE_LIST.getRuleType() ) )
{
connector.getWhitelist().add( proxyConnectorRuleConfiguration.getPattern() );
}
}
}
// Get other connectors
List<ProxyConnector> connectors = this.proxyConnectorMap.get( key );
if ( connectors == null )
{
// Create if we are the first.
connectors = new ArrayList<>( 1 );
}
// Add the connector.
connectors.add( connector );
// Ensure the list is sorted.
Collections.sort( connectors, proxyOrderSorter );
// Set the key to the list of connectors.
this.proxyConnectorMap.put( key, connectors );
}
catch ( RepositoryNotFoundException e )
{
log.warn( "Unable to use proxy connector: {}", e.getMessage(), e );
}
catch ( RepositoryException e )
{
log.warn( "Unable to use proxy connector: {}", e.getMessage(), e );
}
}
this.networkProxyMap.clear();
List<NetworkProxyConfiguration> networkProxies = archivaConfiguration.getConfiguration().getNetworkProxies();
for ( NetworkProxyConfiguration networkProxyConfig : networkProxies )
{
String key = networkProxyConfig.getId();
ProxyInfo proxy = new ProxyInfo();
proxy.setType( networkProxyConfig.getProtocol() );
proxy.setHost( networkProxyConfig.getHost() );
proxy.setPort( networkProxyConfig.getPort() );
proxy.setUserName( networkProxyConfig.getUsername() );
proxy.setPassword( networkProxyConfig.getPassword() );
this.networkProxyMap.put( key, proxy );
}
}
private List<ProxyConnectorRuleConfiguration> findProxyConnectorRules( String sourceRepository,
String targetRepository,
List<ProxyConnectorRuleConfiguration> all )
{
List<ProxyConnectorRuleConfiguration> proxyConnectorRuleConfigurations = new ArrayList<>();
for ( ProxyConnectorRuleConfiguration proxyConnectorRuleConfiguration : all )
{
for ( ProxyConnectorConfiguration proxyConnector : proxyConnectorRuleConfiguration.getProxyConnectors() )
{
if ( StringUtils.equals( sourceRepository, proxyConnector.getSourceRepoId() ) && StringUtils.equals(
targetRepository, proxyConnector.getTargetRepoId() ) )
{
proxyConnectorRuleConfigurations.add( proxyConnectorRuleConfiguration );
}
}
}
return proxyConnectorRuleConfigurations;
}
@Override
public File fetchFromProxies( ManagedRepositoryContent repository, ArtifactReference artifact )
throws ProxyDownloadException
{
File localFile = toLocalFile( repository, artifact );
Properties requestProperties = new Properties();
requestProperties.setProperty( "filetype", "artifact" );
requestProperties.setProperty( "version", artifact.getVersion() );
requestProperties.setProperty( "managedRepositoryId", repository.getId() );
List<ProxyConnector> connectors = getProxyConnectors( repository );
Map<String, Exception> previousExceptions = new LinkedHashMap<>();
for ( ProxyConnector connector : connectors )
{
if ( connector.isDisabled() )
{
continue;
}
RemoteRepositoryContent targetRepository = connector.getTargetRepository();
requestProperties.setProperty( "remoteRepositoryId", targetRepository.getId() );
String targetPath = targetRepository.toPath( artifact );
if ( SystemUtils.IS_OS_WINDOWS )
{
// toPath use system PATH_SEPARATOR so on windows url are \ which doesn't work very well :-)
targetPath = FilenameUtils.separatorsToUnix( targetPath );
}
try
{
File downloadedFile =
transferFile( connector, targetRepository, targetPath, repository, localFile, requestProperties,
true );
if ( fileExists( downloadedFile ) )
{
log.debug( "Successfully transferred: {}", downloadedFile.getAbsolutePath() );
return downloadedFile;
}
}
catch ( NotFoundException e )
{
log.debug( "Artifact {} not found on repository \"{}\".", Keys.toKey( artifact ),
targetRepository.getRepository().getId() );
}
catch ( NotModifiedException e )
{
log.debug( "Artifact {} not updated on repository \"{}\".", Keys.toKey( artifact ),
targetRepository.getRepository().getId() );
}
catch ( ProxyException | RepositoryAdminException e )
{
validatePolicies( this.downloadErrorPolicies, connector.getPolicies(), requestProperties, artifact,
targetRepository, localFile, e, previousExceptions );
}
}
if ( !previousExceptions.isEmpty() )
{
throw new ProxyDownloadException( "Failures occurred downloading from some remote repositories",
previousExceptions );
}
log.debug( "Exhausted all target repositories, artifact {} not found.", Keys.toKey( artifact ) );
return null;
}
@Override
public File fetchFromProxies( ManagedRepositoryContent repository, String path )
{
File localFile = new File( repository.getRepoRoot(), path );
// no update policies for these paths
if ( localFile.exists() )
{
return null;
}
Properties requestProperties = new Properties();
requestProperties.setProperty( "filetype", "resource" );
requestProperties.setProperty( "managedRepositoryId", repository.getId() );
List<ProxyConnector> connectors = getProxyConnectors( repository );
for ( ProxyConnector connector : connectors )
{
if ( connector.isDisabled() )
{
continue;
}
RemoteRepositoryContent targetRepository = connector.getTargetRepository();
requestProperties.setProperty( "remoteRepositoryId", targetRepository.getId() );
String targetPath = path;
try
{
File downloadedFile =
transferFile( connector, targetRepository, targetPath, repository, localFile, requestProperties,
false );
if ( fileExists( downloadedFile ) )
{
log.debug( "Successfully transferred: {}", downloadedFile.getAbsolutePath() );
return downloadedFile;
}
}
catch ( NotFoundException e )
{
log.debug( "Resource {} not found on repository \"{}\".", path,
targetRepository.getRepository().getId() );
}
catch ( NotModifiedException e )
{
log.debug( "Resource {} not updated on repository \"{}\".", path,
targetRepository.getRepository().getId() );
}
catch ( ProxyException e )
{
log.warn(
"Transfer error from repository {} for resource {}, continuing to next repository. Error message: {}",
targetRepository.getRepository().getId(), path, e.getMessage() );
log.debug( MarkerFactory.getDetachedMarker( "transfer.error" ),
"Transfer error from repository \"" + targetRepository.getRepository().getId()
+ "\" for resource " + path + ", continuing to next repository. Error message: {}",
e.getMessage(), e
);
}
catch ( RepositoryAdminException e )
{
log.debug( MarkerFactory.getDetachedMarker( "transfer.error" ),
"Transfer error from repository {} for resource {}, continuing to next repository. Error message: {}",
targetRepository.getRepository().getId(), path, e.getMessage(), e );
log.debug( MarkerFactory.getDetachedMarker( "transfer.error" ), "Full stack trace", e );
}
}
log.debug( "Exhausted all target repositories, resource {} not found.", path );
return null;
}
@Override
public ProxyFetchResult fetchMetadataFromProxies( ManagedRepositoryContent repository, String logicalPath )
{
File localFile = new File( repository.getRepoRoot(), logicalPath );
Properties requestProperties = new Properties();
requestProperties.setProperty( "filetype", "metadata" );
boolean metadataNeedsUpdating = false;
long originalTimestamp = getLastModified( localFile );
List<ProxyConnector> connectors = new ArrayList<>( getProxyConnectors( repository ) );
for ( ProxyConnector connector : connectors )
{
if ( connector.isDisabled() )
{
continue;
}
RemoteRepositoryContent targetRepository = connector.getTargetRepository();
File localRepoFile = toLocalRepoFile( repository, targetRepository, logicalPath );
long originalMetadataTimestamp = getLastModified( localRepoFile );
try
{
transferFile( connector, targetRepository, logicalPath, repository, localRepoFile, requestProperties,
true );
if ( hasBeenUpdated( localRepoFile, originalMetadataTimestamp ) )
{
metadataNeedsUpdating = true;
}
}
catch ( NotFoundException e )
{
log.debug( "Metadata {} not found on remote repository '{}'.", logicalPath,
targetRepository.getRepository().getId(), e );
}
catch ( NotModifiedException e )
{
log.debug( "Metadata {} not updated on remote repository '{}'.", logicalPath,
targetRepository.getRepository().getId(), e );
}
catch ( ProxyException | RepositoryAdminException e )
{
log.warn(
"Transfer error from repository {} for versioned Metadata {}, continuing to next repository. Error message: {}",
targetRepository.getRepository().getId(), logicalPath, e.getMessage() );
log.debug( "Full stack trace", e );
}
}
if ( hasBeenUpdated( localFile, originalTimestamp ) )
{
metadataNeedsUpdating = true;
}
if ( metadataNeedsUpdating || !localFile.exists() )
{
try
{
metadataTools.updateMetadata( repository, logicalPath );
}
catch ( RepositoryMetadataException e )
{
log.warn( "Unable to update metadata {}:{}", localFile.getAbsolutePath(), e.getMessage(), e );
}
}
if ( fileExists( localFile ) )
{
return new ProxyFetchResult( localFile, metadataNeedsUpdating );
}
return new ProxyFetchResult( null, false );
}
/**
* @param connector
* @param remoteRepository
* @param tmpMd5
* @param tmpSha1
* @param tmpResource
* @param url
* @param remotePath
* @param resource
* @param workingDirectory
* @param repository
* @throws ProxyException
* @throws NotModifiedException
* @throws org.apache.archiva.admin.model.RepositoryAdminException
*/
protected void transferResources( ProxyConnector connector, RemoteRepositoryContent remoteRepository, File tmpMd5,
File tmpSha1, File tmpResource, String url, String remotePath, File resource,
File workingDirectory, ManagedRepositoryContent repository )
throws ProxyException, NotModifiedException, RepositoryAdminException
{
Wagon wagon = null;
try
{
RepositoryURL repoUrl = remoteRepository.getURL();
String protocol = repoUrl.getProtocol();
NetworkProxy networkProxy = null;
if ( StringUtils.isNotBlank( connector.getProxyId() ) )
{
networkProxy = networkProxyAdmin.getNetworkProxy( connector.getProxyId() );
}
WagonFactoryRequest wagonFactoryRequest = new WagonFactoryRequest( "wagon#" + protocol,
remoteRepository.getRepository().getExtraHeaders() ).networkProxy(
networkProxy );
wagon = wagonFactory.getWagon( wagonFactoryRequest );
if ( wagon == null )
{
throw new ProxyException( "Unsupported target repository protocol: " + protocol );
}
if ( wagon == null )
{
throw new ProxyException( "Unsupported target repository protocol: " + protocol );
}
boolean connected = connectToRepository( connector, wagon, remoteRepository );
if ( connected )
{
transferArtifact( wagon, remoteRepository, remotePath, repository, resource, workingDirectory,
tmpResource );
// TODO: these should be used to validate the download based on the policies, not always downloaded
// to
// save on connections since md5 is rarely used
transferChecksum( wagon, remoteRepository, remotePath, repository, resource, workingDirectory, ".sha1",
tmpSha1 );
transferChecksum( wagon, remoteRepository, remotePath, repository, resource, workingDirectory, ".md5",
tmpMd5 );
}
}
catch ( NotFoundException e )
{
urlFailureCache.cacheFailure( url );
throw e;
}
catch ( NotModifiedException e )
{
// Do not cache url here.
throw e;
}
catch ( ProxyException e )
{
urlFailureCache.cacheFailure( url );
throw e;
}
catch ( WagonFactoryException e )
{
throw new ProxyException( e.getMessage(), e );
}
finally
{
if ( wagon != null )
{
try
{
wagon.disconnect();
}
catch ( ConnectionException e )
{
log.warn( "Unable to disconnect wagon.", e );
}
}
}
}
private void transferArtifact( Wagon wagon, RemoteRepositoryContent remoteRepository, String remotePath,
ManagedRepositoryContent repository, File resource, File tmpDirectory,
File destFile )
throws ProxyException
{
transferSimpleFile( wagon, remoteRepository, remotePath, repository, resource, destFile );
}
private long getLastModified( File file )
{
if ( !file.exists() || !file.isFile() )
{
return 0;
}
return file.lastModified();
}
private boolean hasBeenUpdated( File file, long originalLastModified )
{
if ( !file.exists() || !file.isFile() )
{
return false;
}
long currentLastModified = getLastModified( file );
return ( currentLastModified > originalLastModified );
}
private File toLocalRepoFile( ManagedRepositoryContent repository, RemoteRepositoryContent targetRepository,
String targetPath )
{
String repoPath = metadataTools.getRepositorySpecificName( targetRepository, targetPath );
return new File( repository.getRepoRoot(), repoPath );
}
/**
* Test if the provided ManagedRepositoryContent has any proxies configured for it.
*/
@Override
public boolean hasProxies( ManagedRepositoryContent repository )
{
synchronized ( this.proxyConnectorMap )
{
return this.proxyConnectorMap.containsKey( repository.getId() );
}
}
private File toLocalFile( ManagedRepositoryContent repository, ArtifactReference artifact )
{
return repository.toFile( artifact );
}
/**
* Simple method to test if the file exists on the local disk.
*
* @param file the file to test. (may be null)
* @return true if file exists. false if the file param is null, doesn't exist, or is not of type File.
*/
private boolean fileExists( File file )
{
if ( file == null )
{
return false;
}
if ( !file.exists() )
{
return false;
}
return file.isFile();
}
/**
* Perform the transfer of the file.
*
* @param connector the connector configuration to use.
* @param remoteRepository the remote repository get the resource from.
* @param remotePath the path in the remote repository to the resource to get.
* @param repository the managed repository that will hold the file
* @param resource the local file to place the downloaded resource into
* @param requestProperties the request properties to utilize for policy handling.
* @param executeConsumers whether to execute the consumers after proxying
* @return the local file that was downloaded, or null if not downloaded.
* @throws NotFoundException if the file was not found on the remote repository.
* @throws NotModifiedException if the localFile was present, and the resource was present on remote repository, but
* the remote resource is not newer than the local File.
* @throws ProxyException if transfer was unsuccessful.
*/
private File transferFile( ProxyConnector connector, RemoteRepositoryContent remoteRepository, String remotePath,
ManagedRepositoryContent repository, File resource, Properties requestProperties,
boolean executeConsumers )
throws ProxyException, NotModifiedException, RepositoryAdminException
{
String url = remoteRepository.getURL().getUrl();
if ( !url.endsWith( "/" ) )
{
url = url + "/";
}
url = url + remotePath;
requestProperties.setProperty( "url", url );
// Is a whitelist defined?
if ( CollectionUtils.isNotEmpty( connector.getWhitelist() ) )
{
// Path must belong to whitelist.
if ( !matchesPattern( remotePath, connector.getWhitelist() ) )
{
log.debug( "Path [{}] is not part of defined whitelist (skipping transfer from repository [{}]).",
remotePath, remoteRepository.getRepository().getName() );
return null;
}
}
// Is target path part of blacklist?
if ( matchesPattern( remotePath, connector.getBlacklist() ) )
{
log.debug( "Path [{}] is part of blacklist (skipping transfer from repository [{}]).", remotePath,
remoteRepository.getRepository().getName() );
return null;
}
// Handle pre-download policy
try
{
validatePolicies( this.preDownloadPolicies, connector.getPolicies(), requestProperties, resource );
}
catch ( PolicyViolationException e )
{
String emsg = "Transfer not attempted on " + url + " : " + e.getMessage();
if ( fileExists( resource ) )
{
log.debug( "{} : using already present local file.", emsg );
return resource;
}
log.debug( emsg );
return null;
}
File workingDirectory = createWorkingDirectory( repository );
File tmpResource = new File( workingDirectory, resource.getName() );
File tmpMd5 = new File( workingDirectory, resource.getName() + ".md5" );
File tmpSha1 = new File( workingDirectory, resource.getName() + ".sha1" );
try
{
transferResources( connector, remoteRepository, tmpMd5, tmpSha1, tmpResource, url, remotePath, resource,
workingDirectory, repository );
// Handle post-download policies.
try
{
validatePolicies( this.postDownloadPolicies, connector.getPolicies(), requestProperties, tmpResource );
}
catch ( PolicyViolationException e )
{
log.warn( "Transfer invalidated from {} : {}", url, e.getMessage() );
executeConsumers = false;
if ( !fileExists( tmpResource ) )
{
resource = null;
}
}
if ( resource != null )
{
synchronized ( resource.getAbsolutePath().intern() )
{
File directory = resource.getParentFile();
moveFileIfExists( tmpMd5, directory );
moveFileIfExists( tmpSha1, directory );
moveFileIfExists( tmpResource, directory );
}
}
}
finally
{
FileUtils.deleteQuietly( workingDirectory );
}
if ( executeConsumers )
{
// Just-in-time update of the index and database by executing the consumers for this artifact
//consumers.executeConsumers( connector.getSourceRepository().getRepository(), resource );
queueRepositoryTask( connector.getSourceRepository().getRepository().getId(), resource );
}
return resource;
}
private void queueRepositoryTask( String repositoryId, File localFile )
{
RepositoryTask task = new RepositoryTask();
task.setRepositoryId( repositoryId );
task.setResourceFile( localFile );
task.setUpdateRelatedArtifacts( true );
task.setScanAll( true );
try
{
scheduler.queueTask( task );
}
catch ( TaskQueueException e )
{
log.error( "Unable to queue repository task to execute consumers on resource file ['" + localFile.getName()
+ "']." );
}
}
/**
* Moves the file into repository location if it exists
*
* @param fileToMove this could be either the main artifact, sha1 or md5 checksum file.
* @param directory directory to write files to
*/
private void moveFileIfExists( File fileToMove, File directory )
throws ProxyException
{
if ( fileToMove != null && fileToMove.exists() )
{
File newLocation = new File( directory, fileToMove.getName() );
moveTempToTarget( fileToMove, newLocation );
}
}
/**
* <p>
* Quietly transfer the checksum file from the remote repository to the local file.
* </p>
*
* @param wagon the wagon instance (should already be connected) to use.
* @param remoteRepository the remote repository to transfer from.
* @param remotePath the remote path to the resource to get.
* @param repository the managed repository that will hold the file
* @param resource the local file that should contain the downloaded contents
* @param tmpDirectory the temporary directory to download to
* @param ext the type of checksum to transfer (example: ".md5" or ".sha1")
* @throws ProxyException if copying the downloaded file into place did not succeed.
*/
private void transferChecksum( Wagon wagon, RemoteRepositoryContent remoteRepository, String remotePath,
ManagedRepositoryContent repository, File resource, File tmpDirectory, String ext,
File destFile )
throws ProxyException
{
String url = remoteRepository.getURL().getUrl() + remotePath + ext;
// Transfer checksum does not use the policy.
if ( urlFailureCache.hasFailedBefore( url ) )
{
return;
}
try
{
transferSimpleFile( wagon, remoteRepository, remotePath + ext, repository, resource, destFile );
log.debug( "Checksum {} Downloaded: {} to move to {}", url, destFile, resource );
}
catch ( NotFoundException e )
{
urlFailureCache.cacheFailure( url );
log.debug( "Transfer failed, checksum not found: {}", url );
// Consume it, do not pass this on.
}
catch ( NotModifiedException e )
{
log.debug( "Transfer skipped, checksum not modified: {}", url );
// Consume it, do not pass this on.
}
catch ( ProxyException e )
{
urlFailureCache.cacheFailure( url );
log.warn( "Transfer failed on checksum: {} : {}", url, e.getMessage(), e );
// Critical issue, pass it on.
throw e;
}
}
/**
* Perform the transfer of the remote file to the local file specified.
*
* @param wagon the wagon instance to use.
* @param remoteRepository the remote repository to use
* @param remotePath the remote path to attempt to get
* @param repository the managed repository that will hold the file
* @param origFile the local file to save to
* @throws ProxyException if there was a problem moving the downloaded file into place.
*/
private void transferSimpleFile( Wagon wagon, RemoteRepositoryContent remoteRepository, String remotePath,
ManagedRepositoryContent repository, File origFile, File destFile )
throws ProxyException
{
assert ( remotePath != null );
// Transfer the file.
try
{
boolean success = false;
if ( !origFile.exists() )
{
log.debug( "Retrieving {} from {}", remotePath, remoteRepository.getRepository().getName() );
wagon.get( addParameters( remotePath, remoteRepository.getRepository() ), destFile );
success = true;
// You wouldn't get here on failure, a WagonException would have been thrown.
log.debug( "Downloaded successfully." );
}
else
{
log.debug( "Retrieving {} from {} if updated", remotePath, remoteRepository.getRepository().getName() );
success = wagon.getIfNewer( addParameters( remotePath, remoteRepository.getRepository() ), destFile,
origFile.lastModified() );
if ( !success )
{
throw new NotModifiedException(
"Not downloaded, as local file is newer than remote side: " + origFile.getAbsolutePath() );
}
if ( destFile.exists() )
{
log.debug( "Downloaded successfully." );
}
}
}
catch ( ResourceDoesNotExistException e )
{
throw new NotFoundException(
"Resource [" + remoteRepository.getURL() + "/" + remotePath + "] does not exist: " + e.getMessage(),
e );
}
catch ( WagonException e )
{
// TODO: shouldn't have to drill into the cause, but TransferFailedException is often not descriptive enough
String msg =
"Download failure on resource [" + remoteRepository.getURL() + "/" + remotePath + "]:" + e.getMessage();
if ( e.getCause() != null )
{
msg += " (cause: " + e.getCause() + ")";
}
throw new ProxyException( msg, e );
}
}
/**
* Apply the policies.
*
* @param policies the map of policies to execute. (Map of String policy keys, to {@link DownloadPolicy} objects)
* @param settings the map of settings for the policies to execute. (Map of String policy keys, to String policy
* setting)
* @param request the request properties (utilized by the {@link DownloadPolicy#applyPolicy(String, Properties, File)}
* )
* @param localFile the local file (utilized by the {@link DownloadPolicy#applyPolicy(String, Properties, File)})
* @throws PolicyViolationException
*/
private void validatePolicies( Map<String, ? extends DownloadPolicy> policies, Map<String, String> settings,
Properties request, File localFile )
throws PolicyViolationException
{
for ( Entry<String, ? extends DownloadPolicy> entry : policies.entrySet() )
{
// olamy with spring rolehint is now downloadPolicy#hint
// so substring after last # to get the hint as with plexus
String key = StringUtils.substringAfterLast( entry.getKey(), "#" );
DownloadPolicy policy = entry.getValue();
String defaultSetting = policy.getDefaultOption();
String setting = StringUtils.defaultString( settings.get( key ), defaultSetting );
log.debug( "Applying [{}] policy with [{}]", key, setting );
try
{
policy.applyPolicy( setting, request, localFile );
}
catch ( PolicyConfigurationException e )
{
log.error( e.getMessage(), e );
}
}
}
private void validatePolicies( Map<String, DownloadErrorPolicy> policies, Map<String, String> settings,
Properties request, ArtifactReference artifact, RemoteRepositoryContent content,
File localFile, Exception exception, Map<String, Exception> previousExceptions )
throws ProxyDownloadException
{
boolean process = true;
for ( Entry<String, ? extends DownloadErrorPolicy> entry : policies.entrySet() )
{
// olamy with spring rolehint is now downloadPolicy#hint
// so substring after last # to get the hint as with plexus
String key = StringUtils.substringAfterLast( entry.getKey(), "#" );
DownloadErrorPolicy policy = entry.getValue();
String defaultSetting = policy.getDefaultOption();
String setting = StringUtils.defaultString( settings.get( key ), defaultSetting );
log.debug( "Applying [{}] policy with [{}]", key, setting );
try
{
// all policies must approve the exception, any can cancel
process = policy.applyPolicy( setting, request, localFile, exception, previousExceptions );
if ( !process )
{
break;
}
}
catch ( PolicyConfigurationException e )
{
log.error( e.getMessage(), e );
}
}
if ( process )
{
// if the exception was queued, don't throw it
if ( !previousExceptions.containsKey( content.getId() ) )
{
throw new ProxyDownloadException(
"An error occurred in downloading from the remote repository, and the policy is to fail immediately",
content.getId(), exception );
}
}
else
{
// if the exception was queued, but cancelled, remove it
previousExceptions.remove( content.getId() );
}
log.warn(
"Transfer error from repository {} for artifact {} , continuing to next repository. Error message: {}",
content.getRepository().getId(), Keys.toKey( artifact ), exception.getMessage() );
log.debug( "Full stack trace", exception );
}
/**
* Creates a working directory
*
* @param repository
* @return file location of working directory
*/
private File createWorkingDirectory( ManagedRepositoryContent repository )
{
try
{
return Files.createTempDirectory( "temp" ).toFile();
}
catch ( IOException e )
{
throw new RuntimeException( e.getMessage(), e );
}
}
/**
* Used to move the temporary file to its real destination. This is patterned from the way WagonManager handles its
* downloaded files.
*
* @param temp The completed download file
* @param target The final location of the downloaded file
* @throws ProxyException when the temp file cannot replace the target file
*/
private void moveTempToTarget( File temp, File target )
throws ProxyException
{
Lock lock;
try
{
lock = fileLockManager.writeFileLock( target );
if ( lock.getFile().exists() && !lock.getFile().delete() )
{
throw new ProxyException( "Unable to overwrite existing target file: " + target.getAbsolutePath() );
}
lock.getFile().getParentFile().mkdirs();
if ( !temp.renameTo( lock.getFile() ) )
{
log.warn( "Unable to rename tmp file to its final name... resorting to copy command." );
try
{
FileUtils.copyFile( temp, lock.getFile() );
}
catch ( IOException e )
{
if ( lock.getFile().exists() )
{
log.debug( "Tried to copy file {} to {} but file with this name already exists.",
temp.getName(), lock.getFile().getAbsolutePath() );
}
else
{
throw new ProxyException(
"Cannot copy tmp file " + temp.getAbsolutePath() + " to its final location", e );
}
}
finally
{
FileUtils.deleteQuietly( temp );
}
}
}
catch ( FileLockException | FileLockTimeoutException e )
{
throw new ProxyException( e.getMessage(), e );
}
}
/**
* Using wagon, connect to the remote repository.
*
* @param connector the connector configuration to utilize (for obtaining network proxy configuration from)
* @param wagon the wagon instance to establish the connection on.
* @param remoteRepository the remote repository to connect to.
* @return true if the connection was successful. false if not connected.
*/
private boolean connectToRepository( ProxyConnector connector, Wagon wagon,
RemoteRepositoryContent remoteRepository )
{
boolean connected = false;
final ProxyInfo networkProxy =
connector.getProxyId() == null ? null : this.networkProxyMap.get( connector.getProxyId() );
if ( log.isDebugEnabled() )
{
if ( networkProxy != null )
{
// TODO: move to proxyInfo.toString()
String msg = "Using network proxy " + networkProxy.getHost() + ":" + networkProxy.getPort()
+ " to connect to remote repository " + remoteRepository.getURL();
if ( networkProxy.getNonProxyHosts() != null )
{
msg += "; excluding hosts: " + networkProxy.getNonProxyHosts();
}
if ( StringUtils.isNotBlank( networkProxy.getUserName() ) )
{
msg += "; as user: " + networkProxy.getUserName();
}
log.debug( msg );
}
}
AuthenticationInfo authInfo = null;
String username = remoteRepository.getRepository().getUserName();
String password = remoteRepository.getRepository().getPassword();
if ( StringUtils.isNotBlank( username ) && StringUtils.isNotBlank( password ) )
{
log.debug( "Using username {} to connect to remote repository {}", username, remoteRepository.getURL() );
authInfo = new AuthenticationInfo();
authInfo.setUserName( username );
authInfo.setPassword( password );
}
// Convert seconds to milliseconds
long timeoutInMilliseconds = TimeUnit.MILLISECONDS.convert( remoteRepository.getRepository().getTimeout(), //
TimeUnit.SECONDS );
// Set timeout read and connect
// FIXME olamy having 2 config values
wagon.setReadTimeout( (int) timeoutInMilliseconds );
wagon.setTimeout( (int) timeoutInMilliseconds );
try
{
Repository wagonRepository =
new Repository( remoteRepository.getId(), remoteRepository.getURL().toString() );
wagon.connect( wagonRepository, authInfo, networkProxy );
connected = true;
}
catch ( ConnectionException | AuthenticationException e )
{
log.warn( "Could not connect to {}: {}", remoteRepository.getRepository().getName(), e.getMessage() );
connected = false;
}
return connected;
}
/**
* Tests whitelist and blacklist patterns against path.
*
* @param path the path to test.
* @param patterns the list of patterns to check.
* @return true if the path matches at least 1 pattern in the provided patterns list.
*/
private boolean matchesPattern( String path, List<String> patterns )
{
if ( CollectionUtils.isEmpty( patterns ) )
{
return false;
}
if ( !path.startsWith( "/" ) )
{
path = "/" + path;
}
for ( String pattern : patterns )
{
if ( !pattern.startsWith( "/" ) )
{
pattern = "/" + pattern;
}
if ( SelectorUtils.matchPath( pattern, path, false ) )
{
return true;
}
}
return false;
}
/**
* TODO: Ensure that list is correctly ordered based on configuration. See MRM-477
*/
@Override
public List<ProxyConnector> getProxyConnectors( ManagedRepositoryContent repository )
{
if ( !this.proxyConnectorMap.containsKey( repository.getId() ) )
{
return Collections.emptyList();
}
List<ProxyConnector> ret = new ArrayList<>( this.proxyConnectorMap.get( repository.getId() ) );
Collections.sort( ret, ProxyConnectorOrderComparator.getInstance() );
return ret;
}
@Override
public void afterConfigurationChange( Registry registry, String propertyName, Object propertyValue )
{
if ( ConfigurationNames.isNetworkProxy( propertyName ) //
|| ConfigurationNames.isManagedRepositories( propertyName ) //
|| ConfigurationNames.isRemoteRepositories( propertyName ) //
|| ConfigurationNames.isProxyConnector( propertyName ) ) //
{
initConnectorsAndNetworkProxies();
}
}
protected String addParameters( String path, RemoteRepository remoteRepository )
{
if ( remoteRepository.getExtraParameters().isEmpty() )
{
return path;
}
boolean question = false;
StringBuilder res = new StringBuilder( path == null ? "" : path );
for ( Entry<String, String> entry : remoteRepository.getExtraParameters().entrySet() )
{
if ( !question )
{
res.append( '?' ).append( entry.getKey() ).append( '=' ).append( entry.getValue() );
}
}
return res.toString();
}
@Override
public void beforeConfigurationChange( Registry registry, String propertyName, Object propertyValue )
{
/* do nothing */
}
public ArchivaConfiguration getArchivaConfiguration()
{
return archivaConfiguration;
}
public void setArchivaConfiguration( ArchivaConfiguration archivaConfiguration )
{
this.archivaConfiguration = archivaConfiguration;
}
public RepositoryContentFactory getRepositoryFactory()
{
return repositoryFactory;
}
public void setRepositoryFactory( RepositoryContentFactory repositoryFactory )
{
this.repositoryFactory = repositoryFactory;
}
public MetadataTools getMetadataTools()
{
return metadataTools;
}
public void setMetadataTools( MetadataTools metadataTools )
{
this.metadataTools = metadataTools;
}
public UrlFailureCache getUrlFailureCache()
{
return urlFailureCache;
}
public void setUrlFailureCache( UrlFailureCache urlFailureCache )
{
this.urlFailureCache = urlFailureCache;
}
public WagonFactory getWagonFactory()
{
return wagonFactory;
}
public void setWagonFactory( WagonFactory wagonFactory )
{
this.wagonFactory = wagonFactory;
}
public Map<String, PreDownloadPolicy> getPreDownloadPolicies()
{
return preDownloadPolicies;
}
public void setPreDownloadPolicies( Map<String, PreDownloadPolicy> preDownloadPolicies )
{
this.preDownloadPolicies = preDownloadPolicies;
}
public Map<String, PostDownloadPolicy> getPostDownloadPolicies()
{
return postDownloadPolicies;
}
public void setPostDownloadPolicies( Map<String, PostDownloadPolicy> postDownloadPolicies )
{
this.postDownloadPolicies = postDownloadPolicies;
}
public Map<String, DownloadErrorPolicy> getDownloadErrorPolicies()
{
return downloadErrorPolicies;
}
public void setDownloadErrorPolicies( Map<String, DownloadErrorPolicy> downloadErrorPolicies )
{
this.downloadErrorPolicies = downloadErrorPolicies;
}
}