/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.zeppelin.file; import java.text.SimpleDateFormat; import java.util.*; import com.google.gson.Gson; import org.apache.commons.lang.StringUtils; import org.apache.zeppelin.completer.CompletionType; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; /** * HDFS implementation of File interpreter for Zeppelin. * */ public class HDFSFileInterpreter extends FileInterpreter { static final String HDFS_URL = "hdfs.url"; static final String HDFS_USER = "hdfs.user"; static final String HDFS_MAXLENGTH = "hdfs.maxlength"; Exception exceptionOnConnect = null; HDFSCommand cmd = null; Gson gson = null; public void prepare() { String userName = getProperty(HDFS_USER); String hdfsUrl = getProperty(HDFS_URL); int i = Integer.parseInt(getProperty(HDFS_MAXLENGTH)); cmd = new HDFSCommand(hdfsUrl, userName, logger, i); gson = new Gson(); } public HDFSFileInterpreter(Properties property){ super(property); prepare(); } /** * Status of one file * * matches returned JSON */ public class OneFileStatus { public long accessTime; public int blockSize; public int childrenNum; public int fileId; public String group; public long length; public long modificationTime; public String owner; public String pathSuffix; public String permission; public int replication; public int storagePolicy; public String type; public String toString() { StringBuilder sb = new StringBuilder(); sb.append("\nAccessTime = ").append(accessTime); sb.append("\nBlockSize = ").append(blockSize); sb.append("\nChildrenNum = ").append(childrenNum); sb.append("\nFileId = ").append(fileId); sb.append("\nGroup = ").append(group); sb.append("\nLength = ").append(length); sb.append("\nModificationTime = ").append(modificationTime); sb.append("\nOwner = ").append(owner); sb.append("\nPathSuffix = ").append(pathSuffix); sb.append("\nPermission = ").append(permission); sb.append("\nReplication = ").append(replication); sb.append("\nStoragePolicy = ").append(storagePolicy); sb.append("\nType = ").append(type); return sb.toString(); } } /** * Status of one file * * matches returned JSON */ public class SingleFileStatus { public OneFileStatus FileStatus; } /** * Status of all files in a directory * * matches returned JSON */ public class MultiFileStatus { public OneFileStatus[] FileStatus; } /** * Status of all files in a directory * * matches returned JSON */ public class AllFileStatus { public MultiFileStatus FileStatuses; } // tests whether we're able to connect to HDFS private void testConnection() { try { if (isDirectory("/")) logger.info("Successfully created WebHDFS connection"); } catch (Exception e) { logger.error("testConnection: Cannot open WebHDFS connection. Bad URL: " + "/", e); exceptionOnConnect = e; } } @Override public void open() { testConnection(); } @Override public void close() { } private String listDir(String path) throws Exception { return cmd.runCommand(cmd.listStatus, path, null); } private String listPermission(OneFileStatus fs){ StringBuilder sb = new StringBuilder(); sb.append(fs.type.equalsIgnoreCase("Directory") ? 'd' : '-'); int p = Integer.parseInt(fs.permission, 16); sb.append(((p & 0x400) == 0) ? '-' : 'r'); sb.append(((p & 0x200) == 0) ? '-' : 'w'); sb.append(((p & 0x100) == 0) ? '-' : 'x'); sb.append(((p & 0x40) == 0) ? '-' : 'r'); sb.append(((p & 0x20) == 0) ? '-' : 'w'); sb.append(((p & 0x10) == 0) ? '-' : 'x'); sb.append(((p & 0x4) == 0) ? '-' : 'r'); sb.append(((p & 0x2) == 0) ? '-' : 'w'); sb.append(((p & 0x1) == 0) ? '-' : 'x'); return sb.toString(); } private String listDate(OneFileStatus fs) { return new SimpleDateFormat("yyyy-MM-dd HH:mm").format(new Date(fs.modificationTime)); } private String listOne(String path, OneFileStatus fs) { if (args.flags.contains(new Character('l'))) { StringBuilder sb = new StringBuilder(); sb.append(listPermission(fs) + "\t"); sb.append(((fs.replication == 0) ? "-" : fs.replication) + "\t "); sb.append(fs.owner + "\t"); sb.append(fs.group + "\t"); if (args.flags.contains(new Character('h'))){ //human readable sb.append(humanReadableByteCount(fs.length) + "\t\t"); } else { sb.append(fs.length + "\t"); } sb.append(listDate(fs) + "GMT\t"); sb.append((path.length() == 1) ? path + fs.pathSuffix : path + '/' + fs.pathSuffix); return sb.toString(); } return fs.pathSuffix; } private String humanReadableByteCount(long bytes) { int unit = 1024; if (bytes < unit) return bytes + " B"; int exp = (int) (Math.log(bytes) / Math.log(unit)); String pre = "KMGTPE".charAt(exp - 1) + ""; return String.format("%.1f %sB", bytes / Math.pow(unit, exp), pre); } public String listFile(String filePath) { try { String str = cmd.runCommand(cmd.getFileStatus, filePath, null); SingleFileStatus sfs = gson.fromJson(str, SingleFileStatus.class); if (sfs != null) { return listOne(filePath, sfs.FileStatus); } } catch (Exception e) { logger.error("listFile: " + filePath, e); } return "No such File or directory"; } public String listAll(String path) { String all = ""; if (exceptionOnConnect != null) return "Error connecting to provided endpoint."; try { //see if directory. if (isDirectory(path)) { String sfs = listDir(path); if (sfs != null) { AllFileStatus allFiles = gson.fromJson(sfs, AllFileStatus.class); if (allFiles != null && allFiles.FileStatuses != null && allFiles.FileStatuses.FileStatus != null) { for (OneFileStatus fs : allFiles.FileStatuses.FileStatus) all = all + listOne(path, fs) + '\n'; } } return all; } else { return listFile(path); } } catch (Exception e) { logger.error("listall: listDir " + path, e); throw new InterpreterException("Could not find file or directory:\t" + path); } } public boolean isDirectory(String path) { boolean ret = false; if (exceptionOnConnect != null) return ret; try { String str = cmd.runCommand(cmd.getFileStatus, path, null); SingleFileStatus sfs = gson.fromJson(str, SingleFileStatus.class); if (sfs != null) return sfs.FileStatus.type.equals("DIRECTORY"); } catch (Exception e) { logger.error("IsDirectory: " + path, e); return false; } return ret; } @Override public List<InterpreterCompletion> completion(String buf, int cursor, InterpreterContext interpreterContext) { logger.info("Completion request at position\t" + cursor + " in string " + buf); final List<InterpreterCompletion> suggestions = new ArrayList<>(); if (StringUtils.isEmpty(buf)) { suggestions.add(new InterpreterCompletion("ls", "ls", CompletionType.command.name())); suggestions.add(new InterpreterCompletion("cd", "cd", CompletionType.command.name())); suggestions.add(new InterpreterCompletion("pwd", "pwd", CompletionType.command.name())); return suggestions; } //part of a command == no spaces if (buf.split(" ").length == 1){ if ("cd".contains(buf)) suggestions.add(new InterpreterCompletion("cd", "cd", CompletionType.command.name())); if ("ls".contains(buf)) suggestions.add(new InterpreterCompletion("ls", "ls", CompletionType.command.name())); if ("pwd".contains(buf)) suggestions.add(new InterpreterCompletion("pwd", "pwd", CompletionType.command.name())); return suggestions; } // last word will contain the path we're working with. String lastToken = buf.substring(buf.lastIndexOf(" ") + 1); if (lastToken.startsWith("-")) { //flag not path return null; } String localPath = ""; //all things before the last '/' String unfinished = lastToken; //unfished filenames or directories if (lastToken.contains("/")) { localPath = lastToken.substring(0, lastToken.lastIndexOf('/') + 1); unfinished = lastToken.substring(lastToken.lastIndexOf('/') + 1); } String globalPath = getNewPath(localPath); //adjust for cwd if (isDirectory(globalPath)){ try { String fileStatusString = listDir(globalPath); if (fileStatusString != null) { AllFileStatus allFiles = gson.fromJson(fileStatusString, AllFileStatus.class); if (allFiles != null && allFiles.FileStatuses != null && allFiles.FileStatuses.FileStatus != null) { for (OneFileStatus fs : allFiles.FileStatuses.FileStatus) { if (fs.pathSuffix.contains(unfinished)) { //only suggest the text after the last . String beforeLastPeriod = unfinished.substring(0, unfinished.lastIndexOf('.') + 1); //beforeLastPeriod should be the start of fs.pathSuffix, so take the end of it. String suggestedFinish = fs.pathSuffix.substring(beforeLastPeriod.length()); suggestions.add(new InterpreterCompletion(suggestedFinish, suggestedFinish, CompletionType.path.name())); } } return suggestions; } } } catch (Exception e) { logger.error("listall: listDir " + globalPath, e); return null; } } else { logger.info("path is not a directory. No values suggested."); } //Error in string. return null; } }