/**************************************************************************************************
* Copyright (c) 2014 Dennis Fischer. *
* All rights reserved. This program and the accompanying materials *
* are made available under the terms of the GNU Public License v3.0+ *
* which accompanies this distribution, and is available at *
* http://www.gnu.org/licenses/gpl.html *
* *
* Contributors: Dennis Fischer *
**************************************************************************************************/
package de.chaosfisch.google.youtube.upload;
import com.blogspot.nurkiewicz.asyncretry.AsyncRetryExecutor;
import com.blogspot.nurkiewicz.asyncretry.RetryContext;
import com.blogspot.nurkiewicz.asyncretry.RetryExecutor;
import com.blogspot.nurkiewicz.asyncretry.function.RetryRunnable;
import com.google.api.client.auth.oauth2.Credential;
import com.google.common.base.Charsets;
import com.google.common.base.Predicate;
import com.google.common.eventbus.EventBus;
import com.google.common.io.CharStreams;
import com.google.common.io.InputSupplier;
import com.google.common.util.concurrent.RateLimiter;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
import com.mashape.unirest.http.HttpResponse;
import com.mashape.unirest.http.Unirest;
import com.mashape.unirest.http.exceptions.UnirestException;
import de.chaosfisch.google.GDATAConfig;
import de.chaosfisch.google.YouTubeProvider;
import de.chaosfisch.google.youtube.upload.events.UploadJobProgressEvent;
import de.chaosfisch.google.youtube.upload.metadata.IMetadataService;
import de.chaosfisch.google.youtube.upload.metadata.MetaBadRequestException;
import org.joda.time.DateTime;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.annotation.Nullable;
import java.io.*;
import java.net.HttpURLConnection;
import java.net.SocketException;
import java.net.URI;
import java.net.URL;
import java.util.Calendar;
import java.util.Set;
import java.util.concurrent.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class UploadJob implements Callable<Upload> {
private static final int SC_OK = 200;
private static final int SC_CREATED = 201;
private static final int SC_MULTIPLE_CHOICES = 300;
private static final int SC_RESUME_INCOMPLETE = 308;
private static final int SC_BAD_REQUEST = 400;
private static final long chunkSize = 10485760;
private static final int DEFAULT_BUFFER_SIZE = 65536;
private static final Logger LOGGER = LoggerFactory.getLogger(UploadJob.class);
private static final String METADATA_CREATE_RESUMEABLE_URL = "http://uploads.gdata.youtube.com/resumable/feeds/api/users/default/uploads";
private static final Pattern RANGE_HEADER_PATTERN = Pattern.compile("-");
private static final int SC_500 = 500;
private static final Pattern RAW_FILE_PATTERN = Pattern.compile("[^a-zA-Z0-9.]+");
/**
* File that is uploaded
*/
private File fileToUpload;
private long start;
private long bytesToUpload;
private long totalBytesUploaded;
private long fileSize;
private final Set<UploadPreProcessor> uploadPreProcessors;
private final Set<UploadPostProcessor> uploadPostProcessors;
private final EventBus eventBus;
private final IUploadService uploadService;
private final RateLimiter rateLimiter;
private UploadJobProgressEvent uploadProgress;
private Upload upload;
private final YouTubeProvider youTubeProvider;
private final IMetadataService metadataService;
private Credential credential;
@Inject
private UploadJob(@Assisted final Upload upload, @Assisted final RateLimiter rateLimiter, final Set<UploadPreProcessor> uploadPreProcessors,
final Set<UploadPostProcessor> uploadPostProcessors, final EventBus eventBus, final IUploadService uploadService,
final YouTubeProvider youTubeProvider, final IMetadataService metadataService) {
this.upload = upload;
this.rateLimiter = rateLimiter;
this.uploadPreProcessors = uploadPreProcessors;
this.uploadPostProcessors = uploadPostProcessors;
this.eventBus = eventBus;
this.uploadService = uploadService;
this.youTubeProvider = youTubeProvider;
this.metadataService = metadataService;
this.eventBus.register(this);
}
@Override
public Upload call() throws Exception {
if (null == upload.getUploadurl()) {
for (final UploadPreProcessor preProcessor : uploadPreProcessors) {
try {
upload = preProcessor.process(upload);
} catch (final Exception e) {
LOGGER.error("Preprocessor error", e);
}
}
}
final ScheduledExecutorService schedueler = Executors.newSingleThreadScheduledExecutor();
final RetryExecutor executor = new AsyncRetryExecutor(schedueler).withExponentialBackoff(TimeUnit.SECONDS.toMillis(3), 2)
.withMaxDelay(TimeUnit.MINUTES.toMillis(1))
.retryOn(IOException.class)
.retryOn(RuntimeException.class)
.retryOn(UploadResponseException.class)
.retryOn(SocketException.class)
.abortIf(new Predicate<Throwable>() {
@Override
public boolean apply(@Nullable final Throwable input) {
return input instanceof UploadResponseException && SC_500 >= (
(UploadResponseException) input)
.getStatus();
}
})
.abortOn(MetaBadRequestException.class)
.abortOn(FileNotFoundException.class)
.abortOn(UploadFinishedException.class);
try {
// Schritt 1: Initialize
initialize();
// Schritt 2: MetadataUpload + UrlFetch
executor.doWithRetry(metadata()).get();
// Schritt 3: Chunkupload
executor.doWithRetry(upload()).get();
} catch (final InterruptedException ignored) {
upload.getStatus().setAborted(true);
} catch (final Exception e) {
if (!upload.getStatus().isArchived()) {
LOGGER.error("Upload error", e);
upload.getStatus().setFailed(true);
}
} finally {
schedueler.shutdownNow();
eventBus.unregister(this);
}
if (upload.getStatus().isArchived()) {
LOGGER.info("Starting postprocessing");
for (final UploadPostProcessor postProcessor : uploadPostProcessors) {
try {
upload = postProcessor.process(upload);
} catch (final Exception e) {
LOGGER.error("Postprocessor error", e);
}
}
}
upload.getStatus().setRunning(false);
uploadService.update(upload);
return upload;
}
private void initialize() throws FileNotFoundException {
// Set the time uploaded started
upload.setDateTimeOfStart(DateTime.now());
uploadService.update(upload);
// Get File and Check if existing
fileToUpload = upload.getFile();
if (!fileToUpload.exists()) {
throw new FileNotFoundException("Datei existiert nicht.");
}
}
private RetryRunnable metadata() {
return new RetryRunnable() {
@Override
public void run(final RetryContext retryContext) throws IOException, MetaBadRequestException, UnirestException {
fileSize = fileToUpload.length();
totalBytesUploaded = 0;
start = 0;
bytesToUpload = fileSize;
if (null != upload.getUploadurl() && !upload.getUploadurl().isEmpty()) {
LOGGER.info("Uploadurl existing: {}", upload.getUploadurl());
return;
}
upload.setUploadurl(fetchUploadUrl(upload));
uploadService.update(upload);
// Log operation
LOGGER.info("Uploadurl received: {}", upload.getUploadurl());
}
};
}
private String fetchUploadUrl(final Upload upload) throws MetaBadRequestException, UnirestException, IOException {
// Upload atomData and fetch uploadUrl
final String atomData = metadataService.atomBuilder(upload);
final HttpResponse<String> response = Unirest.post(METADATA_CREATE_RESUMEABLE_URL)
.header("GData-Version", GDATAConfig.GDATA_V2)
.header("X-GData-Key", "key=" + GDATAConfig.DEVELOPER_KEY)
.header("Content-Type", "application/atom+xml; charset=UTF-8;")
.header("Slug", RAW_FILE_PATTERN.matcher(fileToUpload.getName()).replaceAll(""))
.header("Authorization", getAuthHeader())
.body(atomData)
.asString();
LOGGER.info("fetchUploadUrl response code: {}", response.getCode());
LOGGER.info("fetchUploadUrl response headers: {}", response.getHeaders());
LOGGER.info("fetchUploadUrl response: {}", response.getBody());
// Check the response code for any problematic codes.
if (SC_BAD_REQUEST == response.getCode()) {
throw new MetaBadRequestException(atomData, response.getCode());
}
// Check if uploadurl is available
if (response.getHeaders().containsKey("location")) {
return response.getHeaders().get("location");
} else {
throw new MetaBadRequestException("Location missing", response.getCode());
}
}
private String getAuthHeader() throws IOException {
if (null == credential) {
credential = youTubeProvider.getCredential(upload.getAccount());
}
if (null == credential.getAccessToken() || null != credential.getExpiresInSeconds() && 60 >= credential.getExpiresInSeconds()) {
credential.refreshToken();
}
return String.format("Bearer %s", credential.getAccessToken());
}
private RetryRunnable upload() {
return new RetryRunnable() {
@Override
public void run(final RetryContext retryContext) throws IOException, UploadResponseException, UploadFinishedException, UnirestException {
if (null != upload.getUploadurl() || null != retryContext.getLastThrowable()) {
if (0 < retryContext.getRetryCount()) {
LOGGER.info("############ RETRY " + retryContext.getRetryCount() + " ############");
}
resumeinfo();
}
uploadChunks();
}
};
}
private void uploadChunks() throws IOException, UploadResponseException, UploadFinishedException {
while (!Thread.currentThread().isInterrupted() && totalBytesUploaded != fileSize) {
uploadChunk();
}
}
private void uploadChunk() throws IOException, UploadResponseException, UploadFinishedException {
// GET END SIZE
final long end = generateEndBytes(start, bytesToUpload);
// Log operation
LOGGER.debug("start={} end={} filesize={}", start, end, fileSize);
// Log operation
LOGGER.debug("Uploaded {} bytes so far, using PUT method.", totalBytesUploaded);
if (null == uploadProgress) {
uploadProgress = new UploadJobProgressEvent(upload, upload.getFile().length());
uploadProgress.setTime(Calendar.getInstance().getTimeInMillis());
}
// Calculating the chunk size
final int chunk = (int) (end - start + 1);
// Building PUT RequestImpl for chunk data
final URL url = URI.create(upload.getUploadurl()).toURL();
final HttpURLConnection request = (HttpURLConnection) url.openConnection();
request.setRequestMethod("POST");
request.setDoOutput(true);
request.setFixedLengthStreamingMode(chunk);
//Properties
request.setRequestProperty("Content-Type", upload.getMimetype());
request.setRequestProperty("Content-Range", String.format("bytes %d-%d/%d", start, end, fileToUpload.length()));
request.setRequestProperty("Authorization", getAuthHeader());
request.setRequestProperty("GData-Version", GDATAConfig.GDATA_V2);
request.setRequestProperty("X-GData-Key", String.format("key=%s", GDATAConfig.DEVELOPER_KEY));
request.connect();
try (final TokenInputStream tokenInputStream = new TokenInputStream(new FileInputStream(upload.getFile()));
final BufferedOutputStream throttledOutputStream = new BufferedOutputStream(request.getOutputStream())) {
tokenInputStream.skip(start);
flowChunk(tokenInputStream, throttledOutputStream, start, end);
switch (request.getResponseCode()) {
case SC_OK:
case SC_CREATED:
//FILE UPLOADED
final InputSupplier<InputStream> supplier = new InputSupplier<InputStream>() {
@Override
public InputStream getInput() throws IOException {
return request.getInputStream();
}
};
handleSuccessfulUpload(CharStreams.toString(CharStreams.newReaderSupplier(supplier, Charsets.UTF_8)));
break;
case SC_RESUME_INCOMPLETE:
// OK, the chunk completed succesfully
LOGGER.debug("responseMessage={}", request.getResponseMessage());
break;
default:
throw new UploadResponseException(request.getResponseCode());
}
bytesToUpload -= chunkSize;
start = end + 1;
}
}
private void resumeinfo() throws UploadFinishedException, UploadResponseException, UnirestException, IOException {
final HttpResponse<String> response = Unirest.put(upload.getUploadurl())
.header("GData-Version", GDATAConfig.GDATA_V2)
.header("X-GData-Key", "key=" + GDATAConfig.DEVELOPER_KEY)
.header("Content-Type", "application/atom+xml; charset=UTF-8;")
.header("Authorization", getAuthHeader())
.header("Content-Range", "bytes */*")
.asString();
if (SC_OK <= response.getCode() && SC_MULTIPLE_CHOICES > response.getCode()) {
handleSuccessfulUpload(response.getBody());
} else if (SC_RESUME_INCOMPLETE != response.getCode()) {
throw new UploadResponseException(response.getCode());
}
if (!response.getHeaders().containsKey("range")) {
LOGGER.info("PUT to {} did not return Range-header.", upload.getUploadurl());
totalBytesUploaded = 0;
} else {
LOGGER.info("Range header is: {}", response.getHeaders().get("range"));
final String[] parts = RANGE_HEADER_PATTERN.split(response.getHeaders().get("range"));
if (1 < parts.length) {
totalBytesUploaded = Long.parseLong(parts[1]) + 1;
} else {
totalBytesUploaded = 0;
}
bytesToUpload = fileSize - totalBytesUploaded;
start = totalBytesUploaded;
LOGGER.info("Next byte to upload is {}.", start);
}
if (response.getHeaders().containsKey("location")) {
upload.setUploadurl(response.getHeaders().get("location"));
uploadService.update(upload);
}
}
private void handleSuccessfulUpload(final String body) throws UploadFinishedException {
upload.setVideoid(parseVideoId(body));
upload.getStatus().setArchived(true);
uploadService.update(upload);
throw new UploadFinishedException();
}
String parseVideoId(final String atomData) {
LOGGER.info(atomData);
final Pattern pattern = Pattern.compile("<yt:videoid>(.*)</yt:videoid>");
final Matcher matcher = pattern.matcher(atomData);
if (matcher.find()) {
return matcher.group(1);
} else {
return "missed";
}
}
private long generateEndBytes(final long start, final double bytesToUpload) {
final long end;
if (0 < bytesToUpload - chunkSize) {
end = start + chunkSize - 1;
} else {
end = start + (int) bytesToUpload - 1;
}
return end;
}
private void flowChunk(final InputStream inputStream, final OutputStream outputStream, final long startByte, final long endByte) throws IOException {
// Write Chunk
final byte[] buffer = new byte[DEFAULT_BUFFER_SIZE];
long totalRead = 0;
while (!Thread.currentThread().isInterrupted() && totalRead != endByte - startByte + 1) {
// Upload bytes in buffer
final int bytesRead = flowChunk(inputStream, outputStream, buffer, 0, DEFAULT_BUFFER_SIZE);
// Calculate all uploadinformation
totalRead += bytesRead;
}
}
int flowChunk(final InputStream is, final OutputStream os, final byte[] buf, final int off, final int len) throws IOException {
final int numRead;
if (0 <= (numRead = is.read(buf, off, len))) {
os.write(buf, 0, numRead);
}
os.flush();
return numRead;
}
private class TokenInputStream extends BufferedInputStream {
public TokenInputStream(final InputStream inputStream) {
super(inputStream, DEFAULT_BUFFER_SIZE);
}
@Override
public synchronized int read(final byte[] b, final int off, final int len) throws IOException {
if (0 < rateLimiter.getRate()) {
rateLimiter.acquire(b.length);
}
if (Thread.currentThread().isInterrupted()) {
LOGGER.error("Upload aborted / stopped.");
upload.getStatus().setAborted(true);
throw new CancellationException("Thread cancled");
}
final int bytes = super.read(b, off, len);
// Event Upload Progress
// Calculate all uploadinformation
totalBytesUploaded += b.length;
final long diffTime = Calendar.getInstance().getTimeInMillis() - uploadProgress.getTime();
if (1000 < diffTime) {
uploadProgress.setBytes(totalBytesUploaded);
uploadProgress.setTime(diffTime);
eventBus.post(uploadProgress);
}
return bytes;
}
}
private static class UploadResponseException extends Exception {
private static final long serialVersionUID = 9064482080311824304L;
private final int status;
public UploadResponseException(final int status) {
super(String.format("Upload response exception: %d", status));
this.status = status;
}
private int getStatus() {
return status;
}
}
private static class UploadFinishedException extends Exception {
private static final long serialVersionUID = -5907578118391546810L;
public UploadFinishedException() {
super("Upload finished!");
}
}
}