/*
* Seldon -- open source prediction engine
* =======================================
*
* Copyright 2011-2015 Seldon Technologies Ltd and Rummble Ltd (http://www.seldon.io/)
*
* ********************************************************************************************
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* ********************************************************************************************
*/
package io.seldon.resources.external;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectInputStream;
import org.apache.log4j.Logger;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import java.io.FilterInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.zip.GZIPInputStream;
/**
*
* Makes S3 files available as streams
*
* @author firemanphil
* Date: 06/10/2014
* Time: 14:30
*/
@Component
public class S3FileStreamer {
private static Logger logger = Logger.getLogger(S3FileStreamer.class.getName());
private final AWSCredentials creds;
@Autowired
public S3FileStreamer(@Value("${aws.key:}") String awsKey, @Value("${aws.secret:}") String awsSecret) {
if (awsKey == null || awsKey.equals("") || awsSecret == null || awsSecret.equals("")) {
creds = null;
} else {
this.creds = new BasicAWSCredentials(awsKey, awsSecret);
}
}
public InputStream getResourceStream(String reference) throws IOException {
logger.info("Reading file from s3://"+reference);
AmazonS3Client client;
if(creds != null) {
client = new AmazonS3Client(creds);
} else {
client = new AmazonS3Client();
}
String[] bucketAndFile = reference.split("/", 2);
if(bucketAndFile.length!=2){
return null;
}
S3Object object = client.getObject(new GetObjectRequest(bucketAndFile[0], bucketAndFile[1]));
if(reference.endsWith(".gz")){
return new S3ObjectInputStreamWrapper(new GZIPInputStream(object.getObjectContent()),client);
} else {
return new S3ObjectInputStreamWrapper(object.getObjectContent(), client);
}
}
// to fix a bug in the S3 lib where if the client gets GCed before the data is read, everything breaks
public class S3ObjectInputStreamWrapper extends FilterInputStream {
@SuppressWarnings("unused")
private AmazonS3Client client;
// don't call this
protected S3ObjectInputStreamWrapper(InputStream inputStream) {
super(inputStream);
}
public S3ObjectInputStreamWrapper(InputStream inputStream, AmazonS3Client client) {
super(inputStream);
this.client = client;
}
// S3ObjectInputStream also implements abort() and getHttpRequest(). Override and delegate if needed.
}
}