/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.notebook.repo;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.util.LinkedList;
import java.util.List;
import org.apache.commons.io.IOUtils;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
import org.apache.zeppelin.notebook.Note;
import org.apache.zeppelin.notebook.NoteInfo;
import org.apache.zeppelin.notebook.Paragraph;
import org.apache.zeppelin.scheduler.Job.Status;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.amazonaws.AmazonClientException;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.ListObjectsRequest;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
/**
* Backend for storing Notebooks on S3
*/
public class S3NotebookRepo implements NotebookRepo {
private static final Logger LOG = LoggerFactory.getLogger(S3NotebookRepo.class);
// Use a credential provider chain so that instance profiles can be utilized
// on an EC2 instance. The order of locations where credentials are searched
// is documented here
//
// http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/
// auth/DefaultAWSCredentialsProviderChain.html
//
// In summary, the order is:
//
// 1. Environment Variables - AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
// 2. Java System Properties - aws.accessKeyId and aws.secretKey
// 3. Credential profiles file at the default location (~/.aws/credentials)
// shared by all AWS SDKs and the AWS CLI
// 4. Instance profile credentials delivered through the Amazon EC2 metadata service
private AmazonS3 s3client = new AmazonS3Client(new DefaultAWSCredentialsProviderChain());
private static String bucketName = "";
private String user = "";
private ZeppelinConfiguration conf;
public S3NotebookRepo(ZeppelinConfiguration conf) throws IOException {
this.conf = conf;
user = conf.getUser();
bucketName = conf.getBucketName();
}
@Override
public List<NoteInfo> list() throws IOException {
List<NoteInfo> infos = new LinkedList<NoteInfo>();
NoteInfo info = null;
try {
ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
.withBucketName(bucketName)
.withPrefix(user + "/" + "notebook");
ObjectListing objectListing;
do {
objectListing = s3client.listObjects(listObjectsRequest);
for (S3ObjectSummary objectSummary :
objectListing.getObjectSummaries()) {
if (objectSummary.getKey().contains("note.json")) {
try {
info = getNoteInfo(objectSummary.getKey());
if (info != null) {
infos.add(info);
}
} catch (IOException e) {
LOG.error("Can't read note ", e);
}
}
}
listObjectsRequest.setMarker(objectListing.getNextMarker());
} while (objectListing.isTruncated());
} catch (AmazonServiceException ase) {
} catch (AmazonClientException ace) {
LOG.info("Caught an AmazonClientException, " +
"which means the client encountered " +
"an internal error while trying to communicate" +
" with S3, " +
"such as not being able to access the network.");
LOG.info("Error Message: " + ace.getMessage());
}
return infos;
}
private Note getNote(String key) throws IOException {
GsonBuilder gsonBuilder = new GsonBuilder();
gsonBuilder.setPrettyPrinting();
Gson gson = gsonBuilder.create();
S3Object s3object = s3client.getObject(new GetObjectRequest(
bucketName, key));
InputStream ins = s3object.getObjectContent();
String json = IOUtils.toString(ins, conf.getString(ConfVars.ZEPPELIN_ENCODING));
ins.close();
Note note = gson.fromJson(json, Note.class);
for (Paragraph p : note.getParagraphs()) {
if (p.getStatus() == Status.PENDING || p.getStatus() == Status.RUNNING) {
p.setStatus(Status.ABORT);
}
}
return note;
}
private NoteInfo getNoteInfo(String key) throws IOException {
Note note = getNote(key);
return new NoteInfo(note);
}
@Override
public Note get(String noteId) throws IOException {
return getNote(user + "/" + "notebook" + "/" + noteId + "/" + "note.json");
}
@Override
public void save(Note note) throws IOException {
GsonBuilder gsonBuilder = new GsonBuilder();
gsonBuilder.setPrettyPrinting();
Gson gson = gsonBuilder.create();
String json = gson.toJson(note);
String key = user + "/" + "notebook" + "/" + note.id() + "/" + "note.json";
File file = File.createTempFile("note", "json");
file.deleteOnExit();
Writer writer = new OutputStreamWriter(new FileOutputStream(file));
writer.write(json);
writer.close();
s3client.putObject(new PutObjectRequest(bucketName, key, file));
}
@Override
public void remove(String noteId) throws IOException {
String key = user + "/" + "notebook" + "/" + noteId;
final ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
.withBucketName(bucketName).withPrefix(key);
ObjectListing objects = s3client.listObjects(listObjectsRequest);
do {
for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
s3client.deleteObject(bucketName, objectSummary.getKey());
}
objects = s3client.listNextBatchOfObjects(objects);
} while (objects.isTruncated());
}
@Override
public void close() {
//no-op
}
}