JobsRestController.java

package com.thinkbiganalytics.jobrepo.rest.controller;

/*-
 * #%L
 * thinkbig-job-repository-controller
 * %%
 * Copyright (C) 2017 ThinkBig Analytics
 * %%
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *     http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * #L%
 */

import com.thinkbiganalytics.DateTimeUtil;
import com.thinkbiganalytics.jobrepo.query.model.ExecutedJob;
import com.thinkbiganalytics.jobrepo.query.model.ExecutedStep;
import com.thinkbiganalytics.jobrepo.query.model.FeedHealth;
import com.thinkbiganalytics.jobrepo.query.model.JobStatusCount;
import com.thinkbiganalytics.jobrepo.query.model.SearchResult;
import com.thinkbiganalytics.jobrepo.query.model.transform.JobModelTransform;
import com.thinkbiganalytics.jobrepo.query.model.transform.JobStatusTransform;
import com.thinkbiganalytics.jobrepo.query.model.transform.ModelUtils;
import com.thinkbiganalytics.jobrepo.repository.rest.model.JobAction;
import com.thinkbiganalytics.jobrepo.security.OperationsAccessControl;
import com.thinkbiganalytics.jobrepo.service.JobExecutionException;
import com.thinkbiganalytics.jobrepo.service.JobService;
import com.thinkbiganalytics.metadata.api.MetadataAccess;
import com.thinkbiganalytics.metadata.api.feed.OpsManagerFeedProvider;
import com.thinkbiganalytics.metadata.api.jobrepo.job.BatchJobExecution;
import com.thinkbiganalytics.metadata.api.jobrepo.job.BatchJobExecutionProvider;
import com.thinkbiganalytics.metadata.api.jobrepo.step.BatchStepExecution;
import com.thinkbiganalytics.metadata.api.jobrepo.step.BatchStepExecutionProvider;
import com.thinkbiganalytics.rest.model.RestResponseStatus;
import com.thinkbiganalytics.security.AccessController;

import org.apache.commons.lang3.StringUtils;
import org.joda.time.Period;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Sort;

import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;

import javax.inject.Inject;
import javax.servlet.http.HttpServletRequest;
import javax.ws.rs.Consumes;
import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;

import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiResponse;
import io.swagger.annotations.ApiResponses;

/**
 * Provides rest endpoints for control and monitoring of the pipeline
 */
@Api(tags = "Operations Manager - Jobs", produces = "application/json")
@Path(JobsRestController.BASE)
public class JobsRestController {

    public static final String BASE = "/v1/jobs";

    @Inject
    OpsManagerFeedProvider opsFeedManagerFeedProvider;

    @Inject
    BatchJobExecutionProvider jobExecutionProvider;

    @Inject
    BatchStepExecutionProvider stepExecutionProvider;

    @Inject
    private MetadataAccess metadataAccess;

    @Inject
    private JobService jobService;

    @Inject
    private AccessController accessController;

    @GET
    @Path("/{executionId}")
    @Produces(MediaType.APPLICATION_JSON)
    @ApiOperation("Gets the specified job.")
    @ApiResponses({
                      @ApiResponse(code = 200, message = "Returns the job.", response = ExecutedJob.class),
                      @ApiResponse(code = 400, message = "The executionId is not a valid integer.", response = RestResponseStatus.class)
                  })
    public ExecutedJob getJob(@PathParam("executionId") String executionId,
                              @QueryParam(value = "includeSteps") @DefaultValue("false") boolean includeSteps) {
        this.accessController.checkPermission(AccessController.SERVICES, OperationsAccessControl.ACCESS_OPS);

        return metadataAccess.read(() -> {
            ExecutedJob executedJob = null;
            BatchJobExecution jobExecution = jobExecutionProvider.findByJobExecutionId(Long.parseLong(executionId));
            if (jobExecution != null) {
                if (includeSteps) {
                    executedJob = JobModelTransform.executedJob(jobExecution);
                } else {
                    executedJob = JobModelTransform.executedJobSimple(jobExecution);
                }
            }
            return executedJob;
        });

    }

    /**
     * Get the progress of each of the steps of the job execution for the given job instance id
     *
     * @return A list of each step and its progress, or an HTTP error code on failure
     */
    @GET
    @Path("/{executionId}/steps")
    @Produces(MediaType.APPLICATION_JSON)
    @ApiOperation("Gets the steps of the specified job.")
    @ApiResponses({
                      @ApiResponse(code = 200, message = "Returns the steps.", response = ExecutedStep.class, responseContainer = "List"),
                      @ApiResponse(code = 400, message = "The executionId is not a valid integer.", response = RestResponseStatus.class)
                  })
    public List<ExecutedStep> getJobSteps(@PathParam("executionId") String executionId) {
        this.accessController.checkPermission(AccessController.SERVICES, OperationsAccessControl.ACCESS_OPS);
        return metadataAccess.read(() -> {
            List<? extends BatchStepExecution> steps = stepExecutionProvider.getSteps(Long.parseLong(executionId));
            return JobModelTransform.executedSteps(steps);
        });
    }

    /**
     * Restart the job associated with the given instance id
     *
     * @return A status message and the appropriate http status code
     */
    @POST
    @Path("/{executionId}/restart")
    @Produces(MediaType.APPLICATION_JSON)
    @ApiOperation(value = "Restarts the specified job.", hidden = true)
    @ApiResponses({
                      @ApiResponse(code = 200, message = "Returns the job.", response = ExecutedJob.class),
                      @ApiResponse(code = 404, message = "The executionId is not a valid integer.", response = RestResponseStatus.class)
                  })
    public ExecutedJob restartJob(@PathParam("executionId") Long executionId) throws JobExecutionException {

        this.accessController.checkPermission(AccessController.SERVICES, OperationsAccessControl.ADMIN_OPS);
        ExecutedJob job = metadataAccess.commit(() -> {
            Long newJobExecutionId = this.jobService.restartJobExecution(executionId);
            if (newJobExecutionId != null) {
                BatchJobExecution jobExecution = jobExecutionProvider.findByJobExecutionId(newJobExecutionId);
                if (jobExecution != null) {
                    return JobModelTransform.executedJob(jobExecution);
                } else {
                    return null;
                }
            } else {
                return null;
            }
        });
        if (job == null) {
            throw new JobExecutionException("Could not restart the job with execution Id of " + executionId);
        }
        return job;
    }

    /**
     * Stop the job associated with the given instance id
     *
     * @param executionId The job instance id
     * @return A status message and the appropriate http status code
     */
    @POST
    @Path("/{executionId}/stop")
    @Consumes({MediaType.APPLICATION_JSON, MediaType.APPLICATION_FORM_URLENCODED})
    @Produces(MediaType.APPLICATION_JSON)
    @ApiOperation(value = "Stops the specified job.", hidden = true)
    @ApiResponses({
                      @ApiResponse(code = 200, message = "Returns the job.", response = ExecutedJob.class),
                      @ApiResponse(code = 404, message = "The executionId is not a valid integer.", response = RestResponseStatus.class)
                  })
    public ExecutedJob stopJob(@PathParam("executionId") Long executionId, JobAction jobAction) {

        this.accessController.checkPermission(AccessController.SERVICES, OperationsAccessControl.ADMIN_OPS);
        metadataAccess.commit(() -> {
            boolean stopped = this.jobService.stopJobExecution(executionId);
            return stopped;
        });
        return getJob(executionId.toString(), jobAction.isIncludeSteps());
    }

    /**
     * Abandon the job associated with the given instance id
     *
     * @param executionId The job instance id
     * @return A status message and the appropriate http status code
     */
    @POST
    @Path("/{executionId}/abandon")
    @Consumes({MediaType.APPLICATION_JSON, MediaType.APPLICATION_FORM_URLENCODED})
    @Produces(MediaType.APPLICATION_JSON)
    @ApiOperation("Abandons the specified job.")
    @ApiResponses({
                      @ApiResponse(code = 200, message = "Returns the abandoned job.", response = ExecutedJob.class),
                      @ApiResponse(code = 204, message = "The job could not be found."),
                      @ApiResponse(code = 404, message = "The executionId is not a valid integer.", response = RestResponseStatus.class)
                  })
    public ExecutedJob abandonJob(@PathParam("executionId") Long executionId, JobAction jobAction) {

        this.accessController.checkPermission(AccessController.SERVICES, OperationsAccessControl.ADMIN_OPS);
        metadataAccess.commit(() -> {
            this.jobService.abandonJobExecution(executionId);
            return null;
        });
        return getJob(executionId.toString(), jobAction.isIncludeSteps());
    }

    /**
     * Abandon the job associated with the given instance id
     *
     * @param feedName Full feed name (including category) for which all jobs are to be abandoned
     * @return Feed Health status
     */
    @POST
    @Path("/abandon-all/{feedName}")
    @Consumes({MediaType.APPLICATION_JSON, MediaType.APPLICATION_FORM_URLENCODED})
    @Produces(MediaType.APPLICATION_JSON)
    @ApiOperation("Abandons all jobs for the specified feed.")
    @ApiResponses(
        @ApiResponse(code = 200, message = "Returns the feed health.", response = FeedHealth.class)
    )
    public FeedHealth abandonAllJobs(@Context HttpServletRequest request, @PathParam("feedName") String feedName) {

        this.accessController.checkPermission(AccessController.SERVICES, OperationsAccessControl.ADMIN_OPS);

        return metadataAccess.commit(() -> {
            opsFeedManagerFeedProvider.abandonFeedJobs(feedName);
            return null;
        });
    }

    /**
     * Fail the job associated with the given instance id
     *
     * @param executionId The job instance id
     * @return A status message and the appropriate http status code
     */
    @POST
    @Path("/{executionId}/fail")
    @Consumes({MediaType.APPLICATION_JSON, MediaType.APPLICATION_FORM_URLENCODED})
    @Produces(MediaType.APPLICATION_JSON)
    @ApiOperation("Fails the specified job.")
    @ApiResponses({
                      @ApiResponse(code = 200, message = "Returns the job.", response = ExecutedJob.class),
                      @ApiResponse(code = 404, message = "The executionId is not a valid integer.", response = RestResponseStatus.class)
                  })
    public ExecutedJob failJob(@PathParam("executionId") Long executionId, JobAction jobAction) {

        this.accessController.checkPermission(AccessController.SERVICES, OperationsAccessControl.ADMIN_OPS);
        metadataAccess.commit(() -> {
            this.jobService.failJobExecution(executionId);
            return null;
        });
        return getJob(executionId.toString(), jobAction.isIncludeSteps());
    }


    @GET
    @Path("/")
    @Produces(MediaType.APPLICATION_JSON)
    @ApiOperation("Lists all jobs.")
    @ApiResponses({
                      @ApiResponse(code = 200, message = "Returns the jobs.", response = SearchResult.class),
                      @ApiResponse(code = 400, message = "The sort cannot be empty.", response = RestResponseStatus.class),
                      @ApiResponse(code = 404, message = "The start or limit is not a valid integer.", response = RestResponseStatus.class),
                      @ApiResponse(code = 500, message = "The sort contains an invalid value.", response = RestResponseStatus.class)
                  })
    public SearchResult findJobs(@QueryParam("sort") @DefaultValue("") String sort,
                                 @QueryParam("limit") @DefaultValue("10") Integer limit,
                                 @QueryParam("start") @DefaultValue("1") Integer start,
                                 @QueryParam("filter") String filter,
                                 @Context HttpServletRequest request) {
        return metadataAccess.read(() -> {
            Page<ExecutedJob> page = jobExecutionProvider.findAll(filter, pageRequest(start, limit, sort)).map(jobExecution -> JobModelTransform.executedJobSimple(jobExecution));
            return ModelUtils.toSearchResult(page);
        });


    }

    @GET
    @Path("/list")
    @ApiOperation("Lists all jobs.")
    @ApiResponses({
                      @ApiResponse(code = 200, message = "Returns the jobs.", response = SearchResult.class),
                      @ApiResponse(code = 400, message = "The sort cannot be empty.", response = RestResponseStatus.class),
                      @ApiResponse(code = 404, message = "The start or limit is not a valid integer.", response = RestResponseStatus.class),
                      @ApiResponse(code = 500, message = "The sort contains an invalid value.", response = RestResponseStatus.class)
                  })
    public List<ExecutedJob> findJobsList(@QueryParam("sort") @DefaultValue("") String sort,
                                          @QueryParam("limit") @DefaultValue("10") Integer limit,
                                          @QueryParam("start") @DefaultValue("1") Integer start,
                                          @QueryParam("filter") String filter,
                                          @Context HttpServletRequest request) {
        this.accessController.checkPermission(AccessController.SERVICES, OperationsAccessControl.ACCESS_OPS);

        return metadataAccess.read(() -> {
            Page<ExecutedJob> page = jobExecutionProvider.findAll(filter, pageRequest(start, limit, sort)).map(jobExecution -> JobModelTransform.executedJobSimple(jobExecution));
            return page != null ? page.getContent() : Collections.emptyList();
        });
    }


    @GET
    @Path("/running")
    @Produces(MediaType.APPLICATION_JSON)
    @ApiOperation("Gets the list of running jobs.")
    @ApiResponses({
                      @ApiResponse(code = 200, message = "Returns the jobs.", response = SearchResult.class),
                      @ApiResponse(code = 400, message = "The sort cannot be empty.", response = RestResponseStatus.class),
                      @ApiResponse(code = 404, message = "The start or limit is not a valid integer.", response = RestResponseStatus.class),
                      @ApiResponse(code = 500, message = "The sort contains an invalid value.", response = RestResponseStatus.class)
                  })
    public SearchResult findRunningJobs(@QueryParam("sort") @DefaultValue("") String sort,
                                        @QueryParam("limit") @DefaultValue("10") Integer limit,
                                        @QueryParam("start") @DefaultValue("1") Integer start,
                                        @QueryParam("filter") String filter,
                                        @Context HttpServletRequest request) {

        this.accessController.checkPermission(AccessController.SERVICES, OperationsAccessControl.ACCESS_OPS);

        return metadataAccess.read(() -> {
            String defaultFilter = ensureDefaultFilter(filter, jobExecutionProvider.RUNNING_FILTER);
            Page<ExecutedJob> page = jobExecutionProvider.findAll(defaultFilter, pageRequest(start, limit, sort)).map(jobExecution -> JobModelTransform.executedJobSimple(jobExecution));
            return ModelUtils.toSearchResult(page);
        });

    }


    @GET
    @Path("/failed")
    @Produces(MediaType.APPLICATION_JSON)
    @ApiOperation("Gets the list of failed jobs.")
    @ApiResponses({
                      @ApiResponse(code = 200, message = "Returns the jobs.", response = SearchResult.class),
                      @ApiResponse(code = 400, message = "The sort cannot be empty.", response = RestResponseStatus.class),
                      @ApiResponse(code = 404, message = "The start or limit is not a valid integer.", response = RestResponseStatus.class),
                      @ApiResponse(code = 500, message = "The sort contains an invalid value.", response = RestResponseStatus.class)
                  })
    public SearchResult findFailedJobs(@QueryParam("sort") @DefaultValue("") String sort,
                                       @QueryParam("limit") @DefaultValue("10") Integer limit,
                                       @QueryParam("start") @DefaultValue("1") Integer start,
                                       @QueryParam("filter") String filter,
                                       @Context HttpServletRequest request) {

        return metadataAccess.read(() -> {
            String defaultFilter = ensureDefaultFilter(filter, jobExecutionProvider.FAILED_FILTER);
            Page<ExecutedJob> page = jobExecutionProvider.findAll(defaultFilter, pageRequest(start, limit, sort)).map(jobExecution -> JobModelTransform.executedJobSimple(jobExecution));
            return ModelUtils.toSearchResult(page);
        });
    }


    @GET
    @Path("/stopped")
    @Produces(MediaType.APPLICATION_JSON)
    @ApiOperation("Gets the list of failed jobs.")
    @ApiResponses({
                      @ApiResponse(code = 200, message = "Returns the jobs.", response = SearchResult.class),
                      @ApiResponse(code = 400, message = "The sort cannot be empty.", response = RestResponseStatus.class),
                      @ApiResponse(code = 404, message = "The start or limit is not a valid integer.", response = RestResponseStatus.class),
                      @ApiResponse(code = 500, message = "The sort contains an invalid value.", response = RestResponseStatus.class)
                  })
    public SearchResult findStoppedJobs(@QueryParam("sort") @DefaultValue("") String sort,
                                        @QueryParam("limit") @DefaultValue("10") Integer limit,
                                        @QueryParam("start") @DefaultValue("1") Integer start,
                                        @QueryParam("filter") String filter,
                                        @Context HttpServletRequest request) {

        this.accessController.checkPermission(AccessController.SERVICES, OperationsAccessControl.ACCESS_OPS);

        return metadataAccess.read(() -> {
            String defaultFilter = ensureDefaultFilter(filter, jobExecutionProvider.STOPPED_FILTER);
            Page<ExecutedJob> page = jobExecutionProvider.findAll(defaultFilter, pageRequest(start, limit, sort)).map(jobExecution -> JobModelTransform.executedJobSimple(jobExecution));
            return ModelUtils.toSearchResult(page);
        });

    }

    @GET
    @Path("/completed")
    @Produces(MediaType.APPLICATION_JSON)
    @ApiOperation("Gets the list of completed jobs.")
    @ApiResponses({
                      @ApiResponse(code = 200, message = "Returns the jobs.", response = SearchResult.class),
                      @ApiResponse(code = 400, message = "The sort cannot be empty.", response = RestResponseStatus.class),
                      @ApiResponse(code = 404, message = "The start or limit is not a valid integer.", response = RestResponseStatus.class),
                      @ApiResponse(code = 500, message = "The sort contains an invalid value.", response = RestResponseStatus.class)
                  })
    public SearchResult findCompletedJobs(@QueryParam("sort") @DefaultValue("") String sort,
                                          @QueryParam("limit") @DefaultValue("10") Integer limit,
                                          @QueryParam("start") @DefaultValue("1") Integer start,
                                          @QueryParam("filter") String filter,
                                          @Context HttpServletRequest request) {

        this.accessController.checkPermission(AccessController.SERVICES, OperationsAccessControl.ACCESS_OPS);

        return metadataAccess.read(() -> {
            String defaultFilter = ensureDefaultFilter(filter, jobExecutionProvider.COMPLETED_FILTER);
            Page<ExecutedJob> page = jobExecutionProvider.findAll(defaultFilter, pageRequest(start, limit, sort)).map(jobExecution -> JobModelTransform.executedJobSimple(jobExecution));
            return ModelUtils.toSearchResult(page);
        });

    }


    @GET
    @Path("/abandoned")
    @Produces(MediaType.APPLICATION_JSON)
    @ApiOperation("Gets the list of abandoned jobs.")
    @ApiResponses({
                      @ApiResponse(code = 200, message = "Returns the jobs.", response = SearchResult.class),
                      @ApiResponse(code = 400, message = "The sort cannot be empty.", response = RestResponseStatus.class),
                      @ApiResponse(code = 404, message = "The start or limit is not a valid integer.", response = RestResponseStatus.class),
                      @ApiResponse(code = 500, message = "The sort contains an invalid value.", response = RestResponseStatus.class)
                  })
    public SearchResult findAbandonedJobs(@QueryParam("sort") @DefaultValue("") String sort,
                                          @QueryParam("limit") @DefaultValue("10") Integer limit,
                                          @QueryParam("start") @DefaultValue("1") Integer start,
                                          @QueryParam("filter") String filter,
                                          @Context HttpServletRequest request) {

        this.accessController.checkPermission(AccessController.SERVICES, OperationsAccessControl.ACCESS_OPS);

        return metadataAccess.read(() -> {
            String defaultFilter = ensureDefaultFilter(filter, jobExecutionProvider.ABANDONED_FILTER);
            Page<ExecutedJob> page = jobExecutionProvider.findAll(defaultFilter, pageRequest(start, limit, sort)).map(jobExecution -> JobModelTransform.executedJobSimple(jobExecution));
            return ModelUtils.toSearchResult(page);
        });
    }


    @GET
    @Path("/daily-status-count")
    @Produces(MediaType.APPLICATION_JSON)
    @ApiOperation("Gets the daily statistics.")
    @ApiResponses(
        @ApiResponse(code = 200, message = "Returns the daily stats.", response = JobStatusCount.class, responseContainer = "List")
    )
    public List<JobStatusCount> findDailyStatusCount(@QueryParam("period") String periodString) {

        this.accessController.checkPermission(AccessController.SERVICES, OperationsAccessControl.ACCESS_OPS);

        Period period = DateTimeUtil.period(periodString);
        return metadataAccess.read(() -> {
            List<com.thinkbiganalytics.metadata.api.jobrepo.job.JobStatusCount> counts = jobExecutionProvider.getJobStatusCountByDateFromNow(period, null);
            if (counts != null) {
                List<JobStatusCount> jobStatusCounts = counts.stream().map(c -> JobStatusTransform.jobStatusCount(c)).collect(Collectors.toList());
                JobStatusTransform.ensureDateFromPeriodExists(jobStatusCounts, period);
                return jobStatusCounts;
            }
            return Collections.emptyList();
        });
    }


    @GET
    @Path("/running-failed-counts")
    @Produces(MediaType.APPLICATION_JSON)
    @ApiOperation("Gets the daily statistics.")
    @ApiResponses(
        @ApiResponse(code = 200, message = "Returns the daily stats.", response = JobStatusCount.class, responseContainer = "List")
    )
    public List<JobStatusCount> getRunningOrFailedJobCounts() {
        this.accessController.checkPermission(AccessController.SERVICES, OperationsAccessControl.ACCESS_OPS);

        return metadataAccess.read(() -> {
            List<com.thinkbiganalytics.metadata.api.jobrepo.job.JobStatusCount> counts = jobExecutionProvider.getJobStatusCount(jobExecutionProvider.RUNNING_OR_FAILED_FILTER);
            if (counts != null) {
                return counts.stream().map(c -> JobStatusTransform.jobStatusCount(c)).collect(Collectors.toList());
            }
            return Collections.emptyList();
        });

    }

    /**
     * This will evaluate the {@code incomingFilter} and append/set the value including the {@code defaultFilter} and return a new String with the updated filter
     */
    private String ensureDefaultFilter(String incomingFilter, String defaultFilter) {
        String filter = incomingFilter;
        if (StringUtils.isBlank(filter) || !StringUtils.containsIgnoreCase(filter, defaultFilter)) {
            if (StringUtils.isNotBlank(filter)) {
                if (StringUtils.endsWith(filter, ",")) {
                    filter += defaultFilter;
                } else {
                    filter += "," + defaultFilter;
                }
            } else {
                filter = defaultFilter;
            }
        }
        return filter;
    }

    private PageRequest pageRequest(Integer start, Integer limit, String sort) {
        if (StringUtils.isNotBlank(sort)) {
            Sort.Direction dir = Sort.Direction.ASC;
            if (sort.startsWith("-")) {
                dir = Sort.Direction.DESC;
                sort = sort.substring(1);
            }
            return new PageRequest((start / limit), limit, dir, sort);
        } else {
            return new PageRequest((start / limit), limit);
        }
    }

}