/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.flink.streaming.connectors.fs;
import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.CoreOptions;
import org.apache.flink.configuration.HighAvailabilityOptions;
import org.apache.flink.configuration.SecurityOptions;
import org.apache.flink.runtime.security.SecurityUtils;
import org.apache.flink.streaming.util.TestStreamEnvironment;
import org.apache.flink.test.util.SecureTestEnvironment;
import org.apache.flink.test.util.TestingSecurityContext;
import org.apache.flink.test.util.TestBaseUtils;
import org.apache.flink.util.NetUtils;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.VersionInfo;
import org.junit.AfterClass;
import org.junit.Assume;
import org.junit.BeforeClass;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HTTP_ADDRESS_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_USER_NAME_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_KEYTAB_FILE_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HTTP_POLICY_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ENCRYPT_DATA_TRANSFER_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_ADDRESS_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HOST_NAME_KEY;
/**
* Tests for running {@link RollingSinkSecuredITCase} which is an extension of {@link RollingSink} in secure environment
* Note: only executed for Hadoop version > 3.x.x
*/
public class RollingSinkSecuredITCase extends RollingSinkITCase {
protected static final Logger LOG = LoggerFactory.getLogger(RollingSinkSecuredITCase.class);
/**
* Skips all tests if the Hadoop version doesn't match.
* We can't run this test class until HDFS-9213 is fixed which allows a secure DataNode
* to bind to non-privileged ports for testing.
* For now, we skip this test class until Hadoop version 3.x.x.
*/
private static void skipIfHadoopVersionIsNotAppropriate() {
// Skips all tests if the Hadoop version doesn't match
String hadoopVersionString = VersionInfo.getVersion();
String[] split = hadoopVersionString.split("\\.");
if (split.length != 3) {
throw new IllegalStateException("Hadoop version was not of format 'X.X.X': " + hadoopVersionString);
}
Assume.assumeTrue(
// check whether we're running Hadoop version >= 3.x.x
Integer.parseInt(split[0]) >= 3
);
}
/*
* override super class static methods to avoid creating MiniDFS and MiniFlink with wrong configurations
* and out-of-order sequence for secure cluster
*/
@BeforeClass
public static void setup() throws Exception {}
@AfterClass
public static void teardown() throws Exception {}
@BeforeClass
public static void createHDFS() throws IOException {}
@AfterClass
public static void destroyHDFS() {}
@BeforeClass
public static void startSecureCluster() throws Exception {
skipIfHadoopVersionIsNotAppropriate();
LOG.info("starting secure cluster environment for testing");
dataDir = tempFolder.newFolder();
conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, dataDir.getAbsolutePath());
SecureTestEnvironment.prepare(tempFolder);
populateSecureConfigurations();
Configuration flinkConfig = new Configuration();
flinkConfig.setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB,
SecureTestEnvironment.getTestKeytab());
flinkConfig.setString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL,
SecureTestEnvironment.getHadoopServicePrincipal());
SecurityUtils.SecurityConfiguration ctx = new SecurityUtils.SecurityConfiguration(flinkConfig, conf);
try {
TestingSecurityContext.install(ctx, SecureTestEnvironment.getClientSecurityConfigurationMap());
} catch (Exception e) {
throw new RuntimeException("Exception occurred while setting up secure test context. Reason: {}", e);
}
File hdfsSiteXML = new File(dataDir.getAbsolutePath() + "/hdfs-site.xml");
FileWriter writer = new FileWriter(hdfsSiteXML);
conf.writeXml(writer);
writer.flush();
writer.close();
Map<String, String> map = new HashMap<String, String>(System.getenv());
map.put("HADOOP_CONF_DIR", hdfsSiteXML.getParentFile().getAbsolutePath());
TestBaseUtils.setEnv(map);
MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf);
builder.checkDataNodeAddrConfig(true);
builder.checkDataNodeHostConfig(true);
hdfsCluster = builder.build();
dfs = hdfsCluster.getFileSystem();
hdfsURI = "hdfs://"
+ NetUtils.hostAndPortToUrlString(hdfsCluster.getURI().getHost(), hdfsCluster.getNameNodePort())
+ "/";
startSecureFlinkClusterWithRecoveryModeEnabled();
}
@AfterClass
public static void teardownSecureCluster() throws Exception {
LOG.info("tearing down secure cluster environment");
TestStreamEnvironment.unsetAsContext();
stopCluster(cluster, TestBaseUtils.DEFAULT_TIMEOUT);
if (hdfsCluster != null) {
hdfsCluster.shutdown();
}
SecureTestEnvironment.cleanup();
}
private static void populateSecureConfigurations() {
String dataTransferProtection = "authentication";
SecurityUtil.setAuthenticationMethod(UserGroupInformation.AuthenticationMethod.KERBEROS, conf);
conf.set(DFS_NAMENODE_USER_NAME_KEY, SecureTestEnvironment.getHadoopServicePrincipal());
conf.set(DFS_NAMENODE_KEYTAB_FILE_KEY, SecureTestEnvironment.getTestKeytab());
conf.set(DFS_DATANODE_USER_NAME_KEY, SecureTestEnvironment.getHadoopServicePrincipal());
conf.set(DFS_DATANODE_KEYTAB_FILE_KEY, SecureTestEnvironment.getTestKeytab());
conf.set(DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY, SecureTestEnvironment.getHadoopServicePrincipal());
conf.setBoolean(DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, true);
conf.set("dfs.data.transfer.protection", dataTransferProtection);
conf.set(DFS_HTTP_POLICY_KEY, HttpConfig.Policy.HTTP_ONLY.name());
conf.set(DFS_ENCRYPT_DATA_TRANSFER_KEY, "false");
conf.setInt("dfs.datanode.socket.write.timeout", 0);
/*
* We ae setting the port number to privileged port - see HDFS-9213
* This requires the user to have root privilege to bind to the port
* Use below command (ubuntu) to set privilege to java process for the
* bind() to work if the java process is not running as root.
* setcap 'cap_net_bind_service=+ep' /path/to/java
*/
conf.set(DFS_DATANODE_ADDRESS_KEY, "localhost:1002");
conf.set(DFS_DATANODE_HOST_NAME_KEY, "localhost");
conf.set(DFS_DATANODE_HTTP_ADDRESS_KEY, "localhost:1003");
}
private static void startSecureFlinkClusterWithRecoveryModeEnabled() {
try {
LOG.info("Starting Flink and ZK in secure mode");
dfs.mkdirs(new Path("/flink/checkpoints"));
dfs.mkdirs(new Path("/flink/recovery"));
org.apache.flink.configuration.Configuration config = new org.apache.flink.configuration.Configuration();
config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 1);
config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, DEFAULT_PARALLELISM);
config.setBoolean(ConfigConstants.LOCAL_START_WEBSERVER, false);
config.setInteger(ConfigConstants.LOCAL_NUMBER_JOB_MANAGER, 3);
config.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");
config.setString(CoreOptions.STATE_BACKEND, "filesystem");
config.setString(ConfigConstants.ZOOKEEPER_CHECKPOINTS_PATH, hdfsURI + "/flink/checkpoints");
config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, hdfsURI + "/flink/recovery");
config.setString("state.backend.fs.checkpointdir", hdfsURI + "/flink/checkpoints");
SecureTestEnvironment.populateFlinkSecureConfigurations(config);
cluster = TestBaseUtils.startCluster(config, false);
TestStreamEnvironment.setAsContext(cluster, DEFAULT_PARALLELISM);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
/* For secure cluster testing, it is enough to run only one test and override below test methods
* to keep the overall build time minimal
*/
@Override
public void testNonRollingSequenceFileWithoutCompressionWriter() throws Exception {}
@Override
public void testNonRollingSequenceFileWithCompressionWriter() throws Exception {}
@Override
public void testNonRollingAvroKeyValueWithoutCompressionWriter() throws Exception {}
@Override
public void testNonRollingAvroKeyValueWithCompressionWriter() throws Exception {}
@Override
public void testDateTimeRollingStringWriter() throws Exception {}
}