GatewayFuncTestDriver.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.gateway;

import com.jayway.restassured.RestAssured;
import com.jayway.restassured.path.json.JsonPath;
import com.jayway.restassured.response.Response;
import com.mycila.xmltool.XMLTag;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.directory.server.protocol.shared.transport.TcpTransport;
import org.apache.hadoop.gateway.config.GatewayConfig;
import org.apache.hadoop.gateway.security.ldap.SimpleLdapDirectoryServer;
import org.apache.hadoop.gateway.services.DefaultGatewayServices;
import org.apache.hadoop.gateway.services.ServiceLifecycleException;
import org.apache.hadoop.test.mock.MockServer;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.AuthCache;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.protocol.ClientContext;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.auth.BasicScheme;
import org.apache.http.impl.client.BasicAuthCache;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.protocol.BasicHttpContext;
import org.apache.http.util.EntityUtils;
import org.hamcrest.CoreMatchers;
import org.hamcrest.MatcherAssert;
import org.hamcrest.Matchers;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.InetAddress;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.net.UnknownHostException;
import java.nio.charset.Charset;
import java.util.HashMap;
import java.util.Map;
import java.util.UUID;

import static org.junit.Assert.assertThat;
import static org.junit.Assert.fail;

/**
 * This class was created to reduce much of the duplication and boiler plate that was ending up in the GatewayBasicFuncTest class.
 * It basically does a number of different things.
 * 1) Creates a GATEWAY_HOME starts a gateway instance and deployes a test topology.
 * 2) Provides a registry of mock Hadoop services.
 * 3) Provides "bundled" methods for common Hadoop operations to avoid duplication in tests.
 * 4) Provides methods to access test resources.
 */
public class GatewayFuncTestDriver {

  private static Logger log = LoggerFactory.getLogger( GatewayFuncTestDriver.class );

  public Class<?> resourceBaseClass;
  public Map<String,Service> services = new HashMap<String,Service>();
  public SimpleLdapDirectoryServer ldap;
  public TcpTransport ldapTransport;
  public boolean useGateway;
  public GatewayServer gateway;
  public GatewayConfig config;
  public String clusterName;

  /**
   * Sets the class from which relative test resource names should be resolved.
   * @param resourceBaseClass The class from which relative test resource names should be resolved.
   */
  public void setResourceBase( Class<?> resourceBaseClass ) {
    this.resourceBaseClass = resourceBaseClass;
  }

  /**
   * Starts an embedded LDAP server of the specified port.
   * @param port The desired port the LDAP server should listen on.
   * @return The actual port the LDAP server is listening on.
   * @throws Exception Thrown if a failure occurs.
   */
  public int setupLdap( int port ) throws Exception {
    URL usersUrl = getResourceUrl( "users.ldif" );
    ldapTransport = new TcpTransport( port );
    ldap = new SimpleLdapDirectoryServer( "dc=hadoop,dc=apache,dc=org", new File( usersUrl.toURI() ), ldapTransport );
    ldap.start();
    log.info( "LDAP port = " + ldapTransport.getAcceptor().getLocalAddress().getPort() );
    return port;
  }

  /**
   * Adds a mock service to the registry.
   */
  public void setupService( String role, String realUrl, String gatewayPath, boolean mock ) throws Exception {
    Service service = new Service( role, realUrl, gatewayPath, mock );
    services.put( role, service );
    log.info( role + " port = " + service.server.getPort() );
  }

  /**
   * Creates a GATEWAY_HOME, starts a gateway instance and deploys a test topology.
   */
  public void setupGateway( GatewayTestConfig config, String cluster, XMLTag topology, boolean use ) throws Exception {
    this.useGateway = use;
    this.config = config;
    this.clusterName = cluster;

    File targetDir = new File( System.getProperty( "user.dir" ), "target" );
    File gatewayDir = new File( targetDir, "gateway-home-" + UUID.randomUUID() );
    gatewayDir.mkdirs();

    config.setGatewayHomeDir( gatewayDir.getAbsolutePath() );

    File topoDir = new File( config.getGatewayTopologyDir() );
    topoDir.mkdirs();

    File deployDir = new File( config.getGatewayDeploymentDir() );
    deployDir.mkdirs();

    File descriptor = new File( topoDir, cluster + ".xml" );
    FileOutputStream stream = new FileOutputStream( descriptor );
    topology.toStream( stream );
    stream.close();

    DefaultGatewayServices srvcs = new DefaultGatewayServices();
    Map<String,String> options = new HashMap<String,String>();
    options.put("persist-master", "false");
    options.put("master", "password");
    try {
      srvcs.init(config, options);
    } catch (ServiceLifecycleException e) {
      e.printStackTrace(); // I18N not required.
    }
    File stacksDir = new File( config.getGatewayServicesDir() );
    stacksDir.mkdirs();
    //TODO: [sumit] This is a hack for now, need to find a better way to locate the source resources for 'stacks' to be tested
    String pathToStacksSource = "gateway-service-definitions/src/main/resources/services";
    File stacksSourceDir = new File( targetDir.getParent(), pathToStacksSource);
    if (!stacksSourceDir.exists()) {
      stacksSourceDir = new File( targetDir.getParentFile().getParent(), pathToStacksSource);
    }
    if (stacksSourceDir.exists()) {
      FileUtils.copyDirectoryToDirectory(stacksSourceDir, stacksDir);
    }

    gateway = GatewayServer.startGateway( config, srvcs );
    MatcherAssert.assertThat( "Failed to start gateway.", gateway, CoreMatchers.notNullValue() );

    log.info( "Gateway port = " + gateway.getAddresses()[ 0 ].getPort() );
  }

  public void cleanup() throws Exception {
    gateway.stop();
    FileUtils.deleteQuietly( new File( config.getGatewayTopologyDir() ) );
    FileUtils.deleteQuietly( new File( config.getGatewayConfDir() ) );
    FileUtils.deleteQuietly( new File( config.getGatewaySecurityDir() ) );
    FileUtils.deleteQuietly( new File( config.getGatewayDeploymentDir() ) );
    FileUtils.deleteQuietly( new File( config.getGatewayDataDir() ) );
    FileUtils.deleteQuietly( new File( config.getGatewayServicesDir() ) );

    for( Service service : services.values() ) {
      service.server.stop();
    }
    services.clear();

    ldap.stop( true );
  }

  public boolean isUseGateway() {
    return useGateway;
  }

  public MockServer getMock( String serviceRole ) {
    Service service = services.get( serviceRole );
    return service.server;
  }

  public String getRealUrl( String serviceRole ) {
    return getUrl( serviceRole, true );
  }

  public String getUrl( String serviceRole ) {
    return getUrl( serviceRole, false );
  }

  private String getLocalHostName() {
    String hostName = "localhost";
    try {
      hostName = InetAddress.getByName( "127.0.0.1" ).getHostName();
    } catch( UnknownHostException e ) {
      // Ignore and use the default.
    }
    return hostName;
  }

  public String getUrl( String serviceRole, boolean real ) {
    String url;
    String localHostName = getLocalHostName();
    Service service = services.get( serviceRole );
    if( useGateway && !real ) {
      url = "http://" + localHostName + ":" + gateway.getAddresses()[0].getPort() + "/" + config.getGatewayPath() + service.gatewayPath;
    } else if( service.mock ) {
      url = "http://" + localHostName + ":" + service.server.getPort();
    } else {
      url = service.realUrl.toASCIIString();
    }
    return url;
  }

  public String getClusterUrl() {
    String url;
    String localHostName = getLocalHostName();
    url = "http://" + localHostName + ":" + gateway.getAddresses()[0].getPort() + "/" + config.getGatewayPath() + "/" + clusterName;
    return url;
  }

  public int getGatewayPort() {
    return gateway.getAddresses()[0].getPort();
  }

  public String getRealAddr( String role ) {
    String addr;
    String localHostName = getLocalHostName();
    Service service = services.get( role );
    if( service.mock ) {
      addr = localHostName + ":" + service.server.getPort();
    } else {
      addr = service.realUrl.getHost() + ":" + service.realUrl.getPort();
    }
    return addr;
  }

  public String getLdapUrl() {
    return "ldap://localhost:" + ldapTransport.getAcceptor().getLocalAddress().getPort();
  }

  private static class Service {
    String role;
    URI realUrl;
    String gatewayPath;
    boolean mock;
    MockServer server;
    private Service( String role, String realUrl, String gatewayPath, boolean mock ) throws Exception {
      this.role = role;
      this.realUrl = new URI( realUrl );
      this.gatewayPath = gatewayPath;
      this.mock = mock;
      this.server = new MockServer( role, true );
    }
  }

  public String getResourceBaseName() {
    return resourceBaseClass.getName().replaceAll( "\\.", "/" ) + "/";
  }

  public String getResourceName( String resource ) {
    return getResourceBaseName() + resource;
  }

  public URL getResourceUrl( String resource ) {
    URL url = ClassLoader.getSystemResource( getResourceName( resource ) );
    assertThat( "Failed to find test resource " + resource, url, Matchers.notNullValue() );
    return url;
  }

  public InputStream getResourceStream( String resource ) throws IOException {
    InputStream stream = null;
    if( resource.startsWith( "file:/" ) ) {
      try {
        stream = FileUtils.openInputStream( new File( new URI( resource ) ) );
      } catch( URISyntaxException e ) {
        throw new IOException( e  );
      }
    } else {
      stream = ClassLoader.getSystemResourceAsStream( getResourceName( resource ) );
    }
    assertThat( "Failed to find test resource " + resource, stream, Matchers.notNullValue() );
    return stream;
  }

  public byte[] getResourceBytes( String resource ) throws IOException {
    return IOUtils.toByteArray( getResourceStream( resource ) );
  }

  public String getResourceString( String resource, Charset charset ) throws IOException {
    return IOUtils.toString( getResourceBytes( resource ), "UTF-8" );
  }

  public void assertComplete() {
    // Check to make sure that all interaction were satisfied if for mocked services.
    // Otherwise just clear the mock interaction queue.
    for( Service service : services.values() ) {
      if( service.mock ) {
        assertThat(
            "Service " + service.role + " has remaining expected interactions.",
            service.server.getCount(), Matchers.is(0) );
      }
      service.server.reset();
    }
  }


  public void assertNotComplete(String serviceName) {
    // Check to make sure that all interaction were satisfied if for mocked services.
    // Otherwise just clear the mock interaction queue.

    Service service = services.get(serviceName);

    if(service != null) {
      if(service.mock) {
        assertThat(
            "Service " + service.role + " has remaining expected interactions.",
            service.server.getCount(), Matchers.not(0));
      }
      service.server.reset();
    } else {
      fail();
    }
  }

  public void reset() {
    for( Service service : services.values() ) {
      service.server.reset();
    }
  }

  public String createFileNN( String user, String password, String file, String permsOctal, int status ) throws IOException {
    if( status == HttpStatus.SC_TEMPORARY_REDIRECT ) {
      getMock( "WEBHDFS" )
          .expect()
          .method( "PUT" )
          .pathInfo( "/v1" + file )
          .queryParam( "user.name", user )
          .queryParam( "op", "CREATE" )
          .respond()
          .status( status )
          .header( "Location", getRealUrl("DATANODE") + file + "?op=CREATE&user.name="+user );
    } else {
      getMock( "WEBHDFS" )
          .expect()
          .method( "PUT" )
          .pathInfo( "/v1" + file )
          .queryParam( "user.name", user )
          .queryParam( "op", "CREATE" )
          .respond()
          .status( status );
    }
    Response response = RestAssured.given()
        //.log().headers()
        //.log().parameters()
        .auth().preemptive().basic( user, password )
        .header( "X-XSRF-Header", "jksdhfkhdsf" )
        .queryParam( "op", "CREATE" )
        .queryParam( "permission", permsOctal )
        .expect()
        //.log().all()
        .statusCode( status )
        .when().put( getUrl( "WEBHDFS" ) + "/v1" + file + ( isUseGateway() ? "" : "?user.name=" + user ) );
    String location = response.getHeader( "Location" );
    log.trace( "Redirect location: " + response.getHeader( "Location" ) );
    return location;
  }

  public int createFileDN( String user, String password, String path, String location, String contentType, String resource, int status ) throws IOException {
    if( status == HttpStatus.SC_CREATED ) {
      getMock( "DATANODE" )
          .expect()
          .method( "PUT" )
          .pathInfo( path )
          .queryParam( "user.name", user )
          .queryParam( "op", "CREATE" )
          .contentType( contentType )
          .content( getResourceBytes( resource ) )
          .respond()
          .status( status )
          .header( "Location", "webhdfs://" + getRealAddr( "DATANODE" ) + "/v1" + path );
    } else {
      getMock( "DATANODE" )
          .expect()
          .method( "PUT" )
          .pathInfo( path )
          .queryParam( "user.name", user )
          .queryParam( "op", "CREATE" )
          .contentType( contentType )
          .content( getResourceStream( resource ) )
          .respond()
          .status( status );
    }
    Response response = RestAssured.given()
        //.log().all()
        .auth().preemptive().basic( user, password )
        .header( "X-XSRF-Header", "jksdhfkhdsf" )
        .contentType( contentType )
        .content( getResourceBytes( resource ) )
        .expect()
        //.log().all()
        .statusCode( status )
        .when().put( location );
    return response.getStatusCode();
  }

  public String createFile(
      String user, String password, String group, String file, String permsOctal, String contentType, String resource,
      int nnStatus, int dnStatus, int chownStatus ) throws IOException {
    String location = createFileNN( user, password, file, permsOctal, nnStatus );
    if( location != null ) {
      int status = createFileDN( user, password, file, location, contentType, resource, dnStatus );
      if( status < 300 && permsOctal != null ) {
        chmodFile( user, password, file, permsOctal, chownStatus );
        if( group != null ) {
          chownFile( user, password, file, user, group, chownStatus );
        }
      }
    }
    assertComplete();
    return location;
  }

  public void readFile( String user, String password, String file, String contentType, String resource, int status ) throws IOException {
    getMock( "WEBHDFS" )
        .expect()
        .method( "GET" )
        .pathInfo( "/v1" + file )
        .queryParam( "user.name", user )
        .queryParam( "op", "OPEN" )
        .respond()
        .status( HttpStatus.SC_TEMPORARY_REDIRECT )
        .header( "Location", getRealUrl( "DATANODE" ) + file + "?op=OPEN&user.name="+user );
    if( status == HttpStatus.SC_OK ) {
      getMock( "DATANODE" )
          .expect()
          .method( "GET" )
          .pathInfo( file )
          .queryParam( "user.name", user )
          .queryParam( "op", "OPEN" )
          .respond()
          .status( status )
          .contentType( contentType )
          .content( getResourceBytes( resource ) );
    } else {
      getMock( "DATANODE" )
          .expect()
          .method( "GET" )
          .pathInfo( file )
          .queryParam( "user.name", user )
          .queryParam( "op", "OPEN" )
          .respond()
          .status( status );
    }
    Response response = RestAssured.given()
        //.log().all()
        .auth().preemptive().basic( user, password )
        .header( "X-XSRF-Header", "jksdhfkhdsf" )
        .queryParam( "op", "OPEN" )
        .expect()
        //.log().all()
        .statusCode( status )
        .when().get( getUrl("WEBHDFS") + "/v1" + file + ( isUseGateway() ? "" : "?user.name=" + user ) );
    if( response.getStatusCode() == HttpStatus.SC_OK ) {
      String actualContent = response.asString();
      String expectedContent = getResourceString( resource, Charset.forName("UTF-8") );
      assertThat( actualContent, Matchers.is(expectedContent) );
    }
    assertComplete();
  }

  public void chownFile( String user, String password, String file, String owner, String group, int status ) {
    getMock( "WEBHDFS" )
        .expect()
        .method( "PUT" )
        .pathInfo( "/v1" + file )
        .queryParam( "op", "SETOWNER" )
        .queryParam( "user.name", user )
        .queryParam( "owner", owner )
        .queryParam( "group", group )
        .respond()
        .status( HttpStatus.SC_OK );
    RestAssured.given()
        //.log().all()
        .auth().preemptive().basic( user, password )
        .header( "X-XSRF-Header", "jksdhfkhdsf" )
        .queryParam( "op", "SETOWNER" )
        .queryParam( "owner", owner )
        .queryParam( "group", group )
        .expect()
        //.log().all()
        .statusCode( status )
        .when().put( getUrl("WEBHDFS") + "/v1" + file + ( isUseGateway() ? "" : "?user.name=" + user ) );
    assertComplete();
  }

  public void chmodFile( String user, String password, String file, String permsOctal, int status ) {
    getMock( "WEBHDFS" )
        .expect()
        .method( "PUT" )
        .pathInfo( "/v1" + file )
        .queryParam( "op", "SETPERMISSION" )
        .queryParam( "user.name", user )
        .queryParam( "permission", permsOctal )
        .respond()
        .status( HttpStatus.SC_OK );
    RestAssured.given()
        //.log().all()
        .auth().preemptive().basic( user, password )
        .header( "X-XSRF-Header", "jksdhfkhdsf" )
        .queryParam( "op", "SETPERMISSION" )
        .queryParam( "permission", permsOctal )
        .expect()
        //.log().all()
        .statusCode( status )
        .when().put( getUrl("WEBHDFS") + "/v1" + file + ( isUseGateway() ? "" : "?user.name=" + user ) );
    assertComplete();
  }

  public String updateFile( String user, String password, String file, String contentType, String resource, int nnStatus, int dnStatus ) throws IOException {
    String location;
    location = updateFileNN( user, password, file, resource, nnStatus );
    if( location != null ) {
      updateFileDN( user, password, file, location, contentType, resource, dnStatus );
    }
    assertComplete();
    return location;
  }

  public String updateFileNN( String user, String password, String file, String resource, int status ) throws IOException {
    if( status == HttpStatus.SC_TEMPORARY_REDIRECT ) {
      getMock( "WEBHDFS" )
          .expect()
          .method( "PUT" )
          .pathInfo( "/v1" + file )
          .queryParam( "op", "CREATE" )
          .queryParam( "user.name", user )
          .queryParam( "overwrite", "true" )
          .respond()
          .status( status )
          .header( "Location", getRealUrl("DATANODE") + file + "?op=CREATE&user.name="+user );
    } else {
      getMock( "WEBHDFS" )
          .expect()
          .method( "PUT" )
          .pathInfo( "v1" + file )
          .queryParam( "user.name", user )
          .queryParam( "op", "CREATE" )
          .respond()
          .status( status );
    }
    Response response = RestAssured.given()
        //.log().all()
        .auth().preemptive().basic( user, password )
        .header( "X-XSRF-Header", "jksdhfkhdsf" )
        .queryParam( "op", "CREATE" )
        .queryParam( "overwrite", "true" )
        .content( getResourceBytes( resource ) )
        .expect()
        //.log().all()
        .statusCode( status )
        .when().put( getUrl("WEBHDFS") + "/v1" + file + ( isUseGateway() ? "" : "?user.name=" + user ) );
    String location = response.getHeader( "Location" );
    log.trace( "Redirect location: " + response.getHeader( "Location" ) );
    return location;
  }

  public void updateFileDN( String user, String password, String path, String location, String contentType, String resource, int status ) throws IOException {
    if( status == HttpStatus.SC_CREATED ) {
      getMock( "DATANODE" )
          .expect()
          .method( "PUT" )
          .pathInfo( path )
          .queryParam( "user.name", user )
          .queryParam( "op", "CREATE" )
          .contentType( contentType )
          .content( getResourceBytes( resource ) )
          .respond()
          .status( status )
          .header( "Location", "webhdfs://" + getRealAddr( "DATANODE" ) + "/v1" + path );
    } else {
      getMock( "DATANODE" )
          .expect()
          .method( "PUT" )
          .pathInfo( path )
          .queryParam( "user.name", user )
          .queryParam( "op", "CREATE" )
          .contentType( contentType )
          .content( getResourceBytes( resource ) )
          .respond()
          .status( status );
    }
    Response response = RestAssured.given()
        //.log().all()
        .auth().preemptive().basic( user, password )
        .header( "X-XSRF-Header", "jksdhfkhdsf" )
        .queryParam( "op", "CREATE" )
        .queryParam( "overwrite", "true" )
        .contentType( contentType )
        .content( getResourceBytes( resource ) )
        .expect()
        //.log().all()
        .statusCode( status )
        .when().put( location );
  }

  public void deleteFile( String user, String password, String file, String recursive, int... status ) {
    getMock( "WEBHDFS" )
        .expect()
        .method( "DELETE" )
        .pathInfo( "/v1" + file )
        .queryParam( "user.name", user )
        .queryParam( "op", "DELETE" )
        .queryParam( "recursive", recursive )
        .respond().status( status[0] );
    RestAssured.given()
        //.log().all()
        .auth().preemptive().basic( user, password )
        .header( "X-XSRF-Header", "jksdhfkhdsf" )
        .queryParam( "op", "DELETE" )
        .queryParam( "recursive", recursive )
        .expect()
        //.log().all()
        .statusCode( Matchers.isIn(ArrayUtils.toObject(status)) )
        .when()
        .delete( getUrl( "WEBHDFS" ) + "/v1" + file + ( isUseGateway() ? "" : "?user.name=" + user ) );
    assertComplete();
  }

  public String createDir( String user, String password, String dir, String permsOctal, int status ) {
    getMock( "WEBHDFS" )
        .expect()
        .method( "PUT" )
        .pathInfo( "/v1" + dir )
        .queryParam( "op", "MKDIRS" )
        .queryParam( "user.name", user )
        .queryParam( "permission", permsOctal )
        .respond()
        .status( HttpStatus.SC_OK )
        .contentType( "application/json" )
        .content( "{\"boolean\": true}".getBytes() );
    Response response = RestAssured.given()
        //.log().all()
        .auth().preemptive().basic( user, password )
        .header( "X-XSRF-Header", "jksdhfkhdsf" )
        .queryParam( "op", "MKDIRS" )
        .queryParam( "permission", permsOctal )
        .expect()
        //.log().all()
        .statusCode( status )
        .contentType( "application/json" )
        .content( "boolean", CoreMatchers.equalTo(true) )
        .when()
        .put( getUrl("WEBHDFS") + "/v1" + dir + ( isUseGateway() ? "" : "?user.name=" + user ) );
    String location = response.getHeader( "Location" );
    return location;
  }

  public String createDir( String user, String password, String group, String dir, String permsOctal, int nnStatus, int chownStatus ) {
    String location = createDir( user, password, dir, permsOctal, nnStatus );
    if( location != null ) {
      chownFile( user, password, dir, user, group, chownStatus );
    }
    return location;
  }

  public void readDir( String user, String password, String dir, String resource, int status ) {
    RestAssured.given()
        //.log().all()
        .auth().preemptive().basic( user, password )
        .queryParam( "op", "LISTSTATUS" )
        .expect()
        //.log().all()
        .statusCode( status )
        .content( CoreMatchers.equalTo("TODO") )
        .when()
        .get( getUrl( "WEBHDFS" ) + "/v1" + dir );
  }

  public String submitJava( String user, String password, String jar, String main, String input, String output, int status ) {
    getMock( "WEBHCAT" )
        .expect()
        .method( "POST" )
        .pathInfo( "/v1/mapreduce/jar" )
        .formParam( "user.name", user )
        .formParam( "jar", jar )
        .formParam( "class", main )
        .formParam( "arg", input, output )
        .respond()
        .status( status )
        .contentType( "application/json" )
        .content( "{\"id\":\"job_201210301335_0086\"}".getBytes() );
    String json = RestAssured.given()
        //.log().all()
        .auth().preemptive().basic( user, password )
        .header( "X-XSRF-Header", "jksdhfkhdsf" )
        .formParam( "user.name", user )
        .formParam( "jar", jar )    //"/user/hdfs/test/hadoop-examples.jar" )
        .formParam( "class", main ) //"org.apache.org.apache.hadoop.examples.WordCount" )
        .formParam( "arg", input, output ) //.formParam( "arg", "/user/hdfs/test/input", "/user/hdfs/test/output" )
        .expect()
        //.log().all()
        .statusCode( status )
        .when().post( getUrl( "WEBHCAT" ) + "/v1/mapreduce/jar" + ( isUseGateway() ? "" : "?user.name=" + user ) ).asString();
    log.trace( "JSON=" + json );
    String job = JsonPath.from(json).getString( "id" );
    log.debug( "JOB=" + job );
    assertComplete();
    return job;
  }

  public String submitPig( String user, String password, String group, String file, String arg, String statusDir, int... status ) {
    getMock( "WEBHCAT" )
        .expect()
        .method( "POST" )
        .pathInfo( "/v1/pig" )
        .respond()
        .status( status[0] )
        .contentType( "application/json" )
        .content( "{\"id\":\"job_201210301335_0086\"}".getBytes() );
    String json = RestAssured.given()
        //.log().all()
        .auth().preemptive().basic( user, password )
        .header( "X-XSRF-Header", "jksdhfkhdsf" )
//BUG: The identity asserter needs to check for this too.
        .formParam( "user.name", user )
        .formParam( "group", group )
        .formParam( "file", file )
        .formParam( "arg", arg )
        .formParam( "statusdir", statusDir )
        .expect()
        //.log().all();
        .statusCode( Matchers.isIn(ArrayUtils.toObject(status)) )
        .contentType( "application/json" )
        //.content( "boolean", equalTo( true ) )
        .when()
        .post( getUrl( "WEBHCAT" ) + "/v1/pig" + ( isUseGateway() ? "" : "?user.name=" + user ) )
        .asString();
    log.trace( "JSON=" + json );
    String job = JsonPath.from(json).getString( "id" );
    log.debug( "JOB=" + job );
    assertComplete();
    return job;
  }

  public String submitHive( String user, String password, String group, String file, String statusDir, int... status ) {
    getMock( "WEBHCAT" )
        .expect()
        .method( "POST" )
        .pathInfo( "/v1/hive" )
        .respond()
        .status( status[ 0 ] )
        .contentType( "application/json" )
        .content( "{\"id\":\"job_201210301335_0086\"}".getBytes() );
    String json = RestAssured.given()
        //.log().all()
        .auth().preemptive().basic( user, password )
        .header( "X-XSRF-Header", "jksdhfkhdsf" )
        .formParam( "user.name", user )
        .formParam( "group", group )
        .formParam( "group", group )
        .formParam( "file", file )
        .formParam( "statusdir", statusDir )
        .expect()
        //.log().all()
        .statusCode( Matchers.isIn(ArrayUtils.toObject(status)) )
        .contentType( "application/json" )
        //.content( "boolean", equalTo( true ) )
        .when()
        .post( getUrl( "WEBHCAT" ) + "/v1/hive" + ( isUseGateway() ? "" : "?user.name=" + user ) )
        .asString();
    log.trace( "JSON=" + json );
    String job = JsonPath.from(json).getString( "id" );
    log.debug( "JOB=" + job );
    assertComplete();
    return job;
  }

  public void queryQueue( String user, String password, String job ) throws IOException {
    getMock( "WEBHCAT" )
          .expect()
          .method( "GET" )
          .pathInfo( "/v1/jobs/" + job )
          .respond()
          .status( HttpStatus.SC_OK )
          .content( getResourceBytes( "webhcat-job-status.json" ) )
          .contentType( "application/json" );
    String status = RestAssured.given()
        //.log().all()
        .auth().preemptive().basic( user, password )
        .header( "X-XSRF-Header", "jksdhfkhdsf" )
        .pathParam( "job", job )
        .expect()
        //.log().all()
        .content( "status.jobId", CoreMatchers.equalTo(job) )
        .statusCode( HttpStatus.SC_OK )
        .when().get( getUrl( "WEBHCAT" ) + "/v1/jobs/{job}" + ( isUseGateway() ? "" : "?user.name=" + user ) ).asString();
    log.debug( "STATUS=" + status );
    assertComplete();
  }

  /* GET /oozie/versions
  HTTP/1.1 200 OK
  Content-Type: application/json;charset=UTF-8
  Content-Length: 5
  Server: Apache-Coyote/1.1
  Date: Thu, 14 Feb 2013 15:47:51 GMT
  See: oozie-versions.json
  */
  public void oozieGetVersions( String user, String password ) throws IOException {
    RestAssured.given()
        .auth().preemptive().basic( user, password )
        .header( "X-XSRF-Header", "jksdhfkhdsf" )
        .expect()
        .statusCode( 200 )
        .body( "", Matchers.hasItems(0, 1) )
        .when().get( getUrl( "OOZIE" ) + "/versions" + ( isUseGateway() ? "" : "?user.name=" + user ) ).asString();
  }

  /* GET /oozie/v1/admin/status
  HTTP/1.1 200 OK
  Content-Type: application/json;charset=UTF-8
  Content-Length: 23
  Server: Apache-Coyote/1.1
  Date: Thu, 14 Feb 2013 15:49:16 GMT
  See: oozie-admin-status.json
  */

  /* PUT /oozie/v1/admin/status?safemode=true
TODO
  */

  /* GET /oozie/v1/admin/os-env
  HTTP/1.1 200 OK
  Content-Type: application/json;charset=UTF-8
  Content-Length: 2039
  Server: Apache-Coyote/1.1
  Date: Thu, 14 Feb 2013 15:51:56 GMT
  See: oozie-admin-os-env.json
  */

  /* GET /oozie/v1/admin/java-sys-properties
  HTTP/1.1 200 OK
  Content-Type: application/json;charset=UTF-8
  Content-Length: 3673
  Server: Apache-Coyote/1.1
  Date: Thu, 14 Feb 2013 15:53:00 GMT
  See: oozie-admin-java-sys-properties.json
  */

  /* GET /oozie/v1/admin/configuration
  HTTP/1.1 200 OK
  Transfer-Encoding: Identity
  Content-Type: application/json;charset=UTF-8
  Server: Apache-Coyote/1.1
  Date: Thu, 14 Feb 2013 15:53:31 GMT
  See: oozie-admin-configuration.json
  */

  /* GET /oozie/v1/admin/instrumentation
  HTTP/1.1 200 OK
  Transfer-Encoding: Identity
  Content-Type: application/json;charset=UTF-8
  Server: Apache-Coyote/1.1
  Date: Thu, 14 Feb 2013 15:55:43 GMT
  See: oozie-admin-instrumentation.json
  */

  /* GET /oozie/v1/admin/build-version
  HTTP/1.1 200 OK
  Content-Type: application/json;charset=UTF-8
  Content-Length: 27
  Server: Apache-Coyote/1.1
  Date: Thu, 14 Feb 2013 16:08:31 GMT
  See: oozie-admin-build-version.json
  */

  /* POST /oozie/v1/jobs (request XML; contains URL, response JSON)
  Content-Type: application/json;charset=UTF-8
  Content-Length: 45
  Server: Apache-Coyote/1.1
  Date: Thu, 14 Feb 2013 18:10:52 GMT
  */
  public String oozieSubmitJob( String user, String password, String request, int status ) throws IOException, URISyntaxException {
    getMock( "OOZIE" )
        .expect()
        .method( "POST" )
        .pathInfo( "/v1/jobs" )
        .respond()
        .status( HttpStatus.SC_CREATED )
        .content( getResourceBytes( "oozie-jobs-submit-response.json" ) )
        .contentType( "application/json" );
    //System.out.println( "REQUEST LENGTH = " + request.length() );

    URL url = new URL( getUrl( "OOZIE" ) + "/v1/jobs?action=start" + ( isUseGateway() ? "" : "&user.name=" + user ) );
    HttpHost targetHost = new HttpHost( url.getHost(), url.getPort(), url.getProtocol() );
    DefaultHttpClient client = new DefaultHttpClient();
    client.getCredentialsProvider().setCredentials(
        new AuthScope( targetHost ),
        new UsernamePasswordCredentials( user, password ) );

    // Create AuthCache instance
    AuthCache authCache = new BasicAuthCache();
    // Generate BASIC scheme object and add it to the local auth cache
    BasicScheme basicAuth = new BasicScheme();
    authCache.put( targetHost, basicAuth );
    // Add AuthCache to the execution context
    BasicHttpContext localContext = new BasicHttpContext();
    localContext.setAttribute( ClientContext.AUTH_CACHE, authCache );

    HttpPost post = new HttpPost( url.toURI() );
//    post.getParams().setParameter( "action", "start" );
    StringEntity entity = new StringEntity( request, ContentType.create( "application/xml", "UTF-8" ) );
    post.setEntity( entity );
    post.setHeader( "X-XSRF-Header", "ksdjfhdsjkfhds" );
    HttpResponse response = client.execute( targetHost, post, localContext );
    assertThat( response.getStatusLine().getStatusCode(), Matchers.is(status) );
    String json = EntityUtils.toString( response.getEntity() );

//    String json = given()
//        .log().all()
//        .auth().preemptive().basic( user, password )
//        .queryParam( "action", "start" )
//        .contentType( "application/xml;charset=UTF-8" )
//        .content( request )
//        .expect()
//        .log().all()
//        .statusCode( status )
//        .when().post( getUrl( "OOZIE" ) + "/v1/jobs" + ( isUseGateway() ? "" : "?user.name=" + user ) ).asString();
    //System.out.println( "JSON=" + json );
    String id = JsonPath.from(json).getString( "id" );
    return id;
  }

  /* GET /oozie/v1/jobs?filter=user%3Dbansalm&offset=1&len=50 (body JSON; contains URL)
  HTTP/1.1 200 OK
  Content-Type: application/json;charset=UTF-8
  Content-Length: 46
  Server: Apache-Coyote/1.1
  Date: Thu, 14 Feb 2013 16:10:25 GMT
  */

  /* GET /oozie/v1/job/0000000-130214094519989-oozie-oozi-W
  HTTP/1.1 200 OK
  Content-Type: application/json;charset=UTF-8
  Content-Length: 2611
  Server: Apache-Coyote/1.1
  Date: Thu, 14 Feb 2013 17:39:36 GMT
  */

  /* http://192.168.56.101:11000/oozie/v1/job/0000000-130214094519989-oozie-oozi-W?action=start&user.name=sandbox
  HTTP/1.1 200 OK
  Date: Thu, 14 Feb 2013 17:52:13 GMT
  Content-Length: 0
  Server: Apache-Coyote/1.1
  Set-Cookie: hadoop.auth="u=sandbox&p=sandbox&t=simple&e=1360900333149&s=AU/GeHDNBuK9RBRaBJfrqatjfz8="; Version=1; Path=/
  */

  /* PUT /oozie/v1/job/job-3?action=rerun (request body XML, contains URL)
  HTTP/1.1 200 OK
  Date: Thu, 14 Feb 2013 18:07:45 GMT
  Content-Length: 0
  Server: Apache-Coyote/1.1
  Set-Cookie: hadoop.auth="u=sandbox&p=sandbox&t=simple&e=1360901264892&s=DCOczPqn9mcisCeOb5x2C7LIRc8="; Version=1; Path=/
  */

  /* GET /oozie/v1/job/0000000-130214094519989-oozie-oozi-W?show=info (body JSON, contains URL)
  HTTP/1.1 200 OK
  Content-Type: application/json;charset=UTF-8
  Content-Length: 2611
  Server: Apache-Coyote/1.1
  Date: Thu, 14 Feb 2013 17:45:23 GMT
  */
  public String oozieQueryJobStatus( String user, String password, String id, int status ) throws Exception {
    getMock( "OOZIE" )
        .expect()
        .method( "GET" )
        .pathInfo( "/v1/job/" + id )
        .respond()
        .status( HttpStatus.SC_OK )
        .content( getResourceBytes( "oozie-job-show-info.json" ) )
        .contentType( "application/json" );

    //NOTE:  For some reason REST-assured doesn't like this and ends up failing with Content-Length issues.
    URL url = new URL( getUrl( "OOZIE" ) + "/v1/job/" + id + ( isUseGateway() ? "" : "?user.name=" + user ) );
    HttpHost targetHost = new HttpHost( url.getHost(), url.getPort(), url.getProtocol() );
    DefaultHttpClient client = new DefaultHttpClient();
    client.getCredentialsProvider().setCredentials(
        new AuthScope( targetHost ),
        new UsernamePasswordCredentials( user, password ) );

    // Create AuthCache instance
    AuthCache authCache = new BasicAuthCache();
    // Generate BASIC scheme object and add it to the local auth cache
    BasicScheme basicAuth = new BasicScheme();
    authCache.put( targetHost, basicAuth );
    // Add AuthCache to the execution context
    BasicHttpContext localContext = new BasicHttpContext();
    localContext.setAttribute( ClientContext.AUTH_CACHE, authCache );

    HttpGet request = new HttpGet( url.toURI() );
    request.setHeader("X-XSRF-Header", "ksdhfjkhdsjkf");
    HttpResponse response = client.execute( targetHost, request, localContext );
    assertThat( response.getStatusLine().getStatusCode(), Matchers.is(status) );
    String json = EntityUtils.toString( response.getEntity() );
    String jobStatus = JsonPath.from(json).getString( "status" );
    return jobStatus;
  }

  /* GET /oozie/v1/job/0000000-130214094519989-oozie-oozi-W?show=definition
  HTTP/1.1 200 OK
  Content-Type: application/xml;charset=UTF-8
  Content-Length: 1494
  Server: Apache-Coyote/1.1
  Date: Thu, 14 Feb 2013 17:43:30 GMT
  */

  /* GET GET /oozie/v1/job/0000000-130214094519989-oozie-oozi-W?show=log
  HTTP/1.1 200 OK
  Transfer-Encoding: Identity
  Content-Type: text/plain;charset=UTF-8
  Server: Apache-Coyote/1.1
  Date: Thu, 14 Feb 2013 17:41:43 GMT
  */

}