/**
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.cpd;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import java.util.Properties;
import org.apache.commons.io.IOUtils;
import net.sourceforge.pmd.PMD;
import net.sourceforge.pmd.lang.LanguageRegistry;
import net.sourceforge.pmd.lang.LanguageVersionHandler;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
import net.sourceforge.pmd.lang.cpp.CppLanguageModule;
import net.sourceforge.pmd.lang.cpp.ast.Token;
import net.sourceforge.pmd.util.IOUtil;
/**
* The C++ tokenizer.
*/
public class CPPTokenizer implements Tokenizer {
private boolean skipBlocks = true;
private String skipBlocksStart;
private String skipBlocksEnd;
/**
* Sets the possible options for the C++ tokenizer.
*
* @param properties
* the properties
* @see #OPTION_SKIP_BLOCKS
* @see #OPTION_SKIP_BLOCKS_PATTERN
*/
public void setProperties(Properties properties) {
skipBlocks = Boolean.parseBoolean(properties.getProperty(OPTION_SKIP_BLOCKS, Boolean.TRUE.toString()));
if (skipBlocks) {
String skipBlocksPattern = properties.getProperty(OPTION_SKIP_BLOCKS_PATTERN, DEFAULT_SKIP_BLOCKS_PATTERN);
String[] split = skipBlocksPattern.split("\\|", 2);
skipBlocksStart = split[0];
if (split.length == 1) {
skipBlocksEnd = split[0];
} else {
skipBlocksEnd = split[1];
}
}
}
@Override
public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
StringBuilder buffer = sourceCode.getCodeBuffer();
Reader reader = null;
try {
LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(CppLanguageModule.NAME)
.getDefaultVersion().getLanguageVersionHandler();
reader = new StringReader(maybeSkipBlocks(buffer.toString()));
reader = IOUtil.skipBOM(reader);
TokenManager tokenManager = languageVersionHandler
.getParser(languageVersionHandler.getDefaultParserOptions())
.getTokenManager(sourceCode.getFileName(), reader);
Token currentToken = (Token) tokenManager.getNextToken();
while (currentToken.image.length() > 0) {
tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine));
currentToken = (Token) tokenManager.getNextToken();
}
tokenEntries.add(TokenEntry.getEOF());
System.err.println("Added " + sourceCode.getFileName());
} catch (TokenMgrError err) {
err.printStackTrace();
System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error");
tokenEntries.add(TokenEntry.getEOF());
} catch (IOException e) {
e.printStackTrace();
System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error");
tokenEntries.add(TokenEntry.getEOF());
} finally {
IOUtils.closeQuietly(reader);
}
}
private String maybeSkipBlocks(String test) throws IOException {
if (!skipBlocks) {
return test;
}
BufferedReader reader = new BufferedReader(new StringReader(test));
StringBuilder filtered = new StringBuilder(test.length());
String line;
boolean skip = false;
while ((line = reader.readLine()) != null) {
if (skipBlocksStart.equalsIgnoreCase(line.trim())) {
skip = true;
} else if (skip && skipBlocksEnd.equalsIgnoreCase(line.trim())) {
skip = false;
}
if (!skip) {
filtered.append(line);
}
// always add a new line to keep the line-numbering
filtered.append(PMD.EOL);
}
return filtered.toString();
}
}