/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.webproxy;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.HttpCookie;
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URL;
import java.util.List;
import java.util.Map;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationReportPBImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import org.mortbay.jetty.Server;
import org.mortbay.jetty.servlet.Context;
import org.mortbay.jetty.servlet.ServletHolder;
/**
* Test the WebAppProxyServlet and WebAppProxy. For back end use simple web
* server.
*/
public class TestWebAppProxyServlet {
private static final Log LOG = LogFactory
.getLog(TestWebAppProxyServlet.class);
private static Server server;
private static int originalPort = 0;
/**
* Simple http server. Server should send answer with status 200
*/
@BeforeClass
public static void start() throws Exception {
server = new Server(0);
Context context = new Context();
context.setContextPath("/foo");
server.setHandler(context);
context.addServlet(new ServletHolder(TestServlet.class), "/bar/");
server.getConnectors()[0].setHost("localhost");
server.start();
originalPort = server.getConnectors()[0].getLocalPort();
LOG.info("Running embedded servlet container at: http://localhost:"
+ originalPort);
}
@SuppressWarnings("serial")
public static class TestServlet extends HttpServlet {
@Override
protected void doGet(HttpServletRequest req, HttpServletResponse resp)
throws ServletException, IOException {
resp.setStatus(HttpServletResponse.SC_OK);
}
@Override
protected void doPost(HttpServletRequest req, HttpServletResponse resp)
throws ServletException, IOException {
InputStream is = req.getInputStream();
OutputStream os = resp.getOutputStream();
int c = is.read();
while (c > -1) {
os.write(c);
c = is.read();
}
is.close();
os.close();
resp.setStatus(HttpServletResponse.SC_OK);
}
}
@Test(timeout=5000)
public void testWebAppProxyServlet() throws Exception {
Configuration configuration = new Configuration();
configuration.set(YarnConfiguration.PROXY_ADDRESS, "localhost:9090");
// overriding num of web server threads, see HttpServer.HTTP_MAXTHREADS
configuration.setInt("hadoop.http.max.threads", 5);
WebAppProxyServerForTest proxy = new WebAppProxyServerForTest();
proxy.init(configuration);
proxy.start();
int proxyPort = proxy.proxy.proxyServer.getConnectorAddress(0).getPort();
AppReportFetcherForTest appReportFetcher = proxy.proxy.appReportFetcher;
// wrong url
try {
// wrong url. Set wrong app ID
URL wrongUrl = new URL("http://localhost:" + proxyPort + "/proxy/app");
HttpURLConnection proxyConn = (HttpURLConnection) wrongUrl
.openConnection();
proxyConn.connect();
assertEquals(HttpURLConnection.HTTP_INTERNAL_ERROR,
proxyConn.getResponseCode());
// set true Application ID in url
URL url = new URL("http://localhost:" + proxyPort + "/proxy/application_00_0");
proxyConn = (HttpURLConnection) url.openConnection();
// set cookie
proxyConn.setRequestProperty("Cookie", "checked_application_0_0000=true");
proxyConn.connect();
assertEquals(HttpURLConnection.HTTP_OK, proxyConn.getResponseCode());
assertTrue(isResponseCookiePresent(
proxyConn, "checked_application_0_0000", "true"));
// cannot found application 1: null
appReportFetcher.answer = 1;
proxyConn = (HttpURLConnection) url.openConnection();
proxyConn.setRequestProperty("Cookie", "checked_application_0_0000=true");
proxyConn.connect();
assertEquals(HttpURLConnection.HTTP_NOT_FOUND,
proxyConn.getResponseCode());
assertFalse(isResponseCookiePresent(
proxyConn, "checked_application_0_0000", "true"));
// cannot found application 2: ApplicationNotFoundException
appReportFetcher.answer = 4;
proxyConn = (HttpURLConnection) url.openConnection();
proxyConn.setRequestProperty("Cookie", "checked_application_0_0000=true");
proxyConn.connect();
assertEquals(HttpURLConnection.HTTP_NOT_FOUND,
proxyConn.getResponseCode());
assertFalse(isResponseCookiePresent(
proxyConn, "checked_application_0_0000", "true"));
// wrong user
appReportFetcher.answer = 2;
proxyConn = (HttpURLConnection) url.openConnection();
proxyConn.connect();
assertEquals(HttpURLConnection.HTTP_OK, proxyConn.getResponseCode());
String s = readInputStream(proxyConn.getInputStream());
assertTrue(s
.contains("to continue to an Application Master web interface owned by"));
assertTrue(s.contains("WARNING: The following page may not be safe!"));
//case if task has a not running status
appReportFetcher.answer = 3;
proxyConn = (HttpURLConnection) url.openConnection();
proxyConn.setRequestProperty("Cookie", "checked_application_0_0000=true");
proxyConn.connect();
assertEquals(HttpURLConnection.HTTP_OK, proxyConn.getResponseCode());
} finally {
proxy.close();
}
}
/**
* Test main method of WebAppProxyServer
*/
@Test(timeout=5000)
public void testWebAppProxyServerMainMethod() throws Exception {
WebAppProxyServer mainServer = null;
Configuration conf = new YarnConfiguration();
conf.set(YarnConfiguration.PROXY_ADDRESS, "localhost:9099");
try {
mainServer = WebAppProxyServer.startServer(conf);
int counter = 20;
URL wrongUrl = new URL("http://localhost:9099/proxy/app");
HttpURLConnection proxyConn = null;
while (counter > 0) {
counter--;
try {
proxyConn = (HttpURLConnection) wrongUrl.openConnection();
proxyConn.connect();
proxyConn.getResponseCode();
// server started ok
counter = 0;
} catch (Exception e) {
Thread.sleep(100);
}
}
assertNotNull(proxyConn);
// wrong application Id
assertEquals(HttpURLConnection.HTTP_INTERNAL_ERROR,
proxyConn.getResponseCode());
} finally {
if (mainServer != null) {
mainServer.stop();
}
}
}
private String readInputStream(InputStream input) throws Exception {
ByteArrayOutputStream data = new ByteArrayOutputStream();
byte[] buffer = new byte[512];
int read;
while ((read = input.read(buffer)) >= 0) {
data.write(buffer, 0, read);
}
return new String(data.toByteArray(), "UTF-8");
}
private boolean isResponseCookiePresent(HttpURLConnection proxyConn,
String expectedName, String expectedValue) {
Map<String, List<String>> headerFields = proxyConn.getHeaderFields();
List<String> cookiesHeader = headerFields.get("Set-Cookie");
if (cookiesHeader != null) {
for (String cookie : cookiesHeader) {
HttpCookie c = HttpCookie.parse(cookie).get(0);
if (c.getName().equals(expectedName)
&& c.getValue().equals(expectedValue)) {
return true;
}
}
}
return false;
}
@AfterClass
public static void stop() throws Exception {
try {
server.stop();
} catch (Exception e) {
}
try {
server.destroy();
} catch (Exception e) {
}
}
private class WebAppProxyServerForTest extends CompositeService {
private WebAppProxyForTest proxy = null;
public WebAppProxyServerForTest() {
super(WebAppProxyServer.class.getName());
}
@Override
public synchronized void init(Configuration conf) {
Configuration config = new YarnConfiguration(conf);
proxy = new WebAppProxyForTest();
addService(proxy);
super.init(config);
}
}
private class WebAppProxyForTest extends WebAppProxy {
HttpServer2 proxyServer;
AppReportFetcherForTest appReportFetcher;
@Override
public void start() {
try {
Configuration conf = getConfig();
String bindAddress = conf.get(YarnConfiguration.PROXY_ADDRESS);
bindAddress = StringUtils.split(bindAddress, ':')[0];
AccessControlList acl = new AccessControlList(
conf.get(YarnConfiguration.YARN_ADMIN_ACL,
YarnConfiguration.DEFAULT_YARN_ADMIN_ACL));
proxyServer = new HttpServer2.Builder()
.setName("proxy")
.addEndpoint(
URI.create(WebAppUtils.getHttpSchemePrefix(conf) + bindAddress
+ ":0")).setFindPort(true)
.setConf(conf)
.setACL(acl)
.build();
proxyServer.addServlet(ProxyUriUtils.PROXY_SERVLET_NAME,
ProxyUriUtils.PROXY_PATH_SPEC, WebAppProxyServlet.class);
appReportFetcher = new AppReportFetcherForTest(conf);
proxyServer.setAttribute(FETCHER_ATTRIBUTE,
appReportFetcher );
proxyServer.setAttribute(IS_SECURITY_ENABLED_ATTRIBUTE, Boolean.TRUE);
String proxy = WebAppUtils.getProxyHostAndPort(conf);
String[] proxyParts = proxy.split(":");
String proxyHost = proxyParts[0];
proxyServer.setAttribute(PROXY_HOST_ATTRIBUTE, proxyHost);
proxyServer.start();
System.out.println("Proxy server is started at port " +
proxyServer.getConnectorAddress(0).getPort());
} catch (Exception e) {
LOG.fatal("Could not start proxy web server", e);
throw new YarnRuntimeException("Could not start proxy web server", e);
}
}
}
private class AppReportFetcherForTest extends AppReportFetcher {
int answer = 0;
public AppReportFetcherForTest(Configuration conf) {
super(conf);
}
public ApplicationReport getApplicationReport(ApplicationId appId)
throws YarnException {
if (answer == 0) {
return getDefaultApplicationReport(appId);
} else if (answer == 1) {
return null;
} else if (answer == 2) {
ApplicationReport result = getDefaultApplicationReport(appId);
result.setUser("user");
return result;
} else if (answer == 3) {
ApplicationReport result = getDefaultApplicationReport(appId);
result.setYarnApplicationState(YarnApplicationState.KILLED);
return result;
} else if (answer == 4) {
throw new ApplicationNotFoundException("Application is not found");
}
return null;
}
private ApplicationReport getDefaultApplicationReport(ApplicationId appId) {
ApplicationReport result = new ApplicationReportPBImpl();
result.setApplicationId(appId);
result.setOriginalTrackingUrl("localhost:" + originalPort + "/foo/bar");
result.setYarnApplicationState(YarnApplicationState.RUNNING);
result.setUser(CommonConfigurationKeys.DEFAULT_HADOOP_HTTP_STATIC_USER);
return result;
}
}
}