JavaTokensTokenizerTest.java example

Explorer
pmd-master
/**
 * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
 */

package net.sourceforge.pmd.cpd;

import static org.junit.Assert.assertEquals;

import java.io.IOException;
import java.util.List;

import org.junit.Test;

import net.sourceforge.pmd.PMD;
import net.sourceforge.pmd.lang.java.ast.JavaParserConstants;

public class JavaTokensTokenizerTest {

    @Test
    public void test1() throws IOException {
        Tokenizer tokenizer = new JavaTokenizer();
        SourceCode sourceCode = new SourceCode(new SourceCode.StringCodeLoader("public class Foo {}"));
        Tokens tokens = new Tokens();
        tokenizer.tokenize(sourceCode, tokens);
        assertEquals(6, tokens.size());
        assertEquals("public class Foo {}", sourceCode.getSlice(1, 1));
    }

    @Test
    public void testCommentsIgnored() throws IOException {
        Tokenizer tokenizer = new JavaTokenizer();
        SourceCode sourceCode = new SourceCode(
                new SourceCode.StringCodeLoader("public class Foo { // class Bar */ \n }"));
        Tokens tokens = new Tokens();
        tokenizer.tokenize(sourceCode, tokens);
        assertEquals(6, tokens.size());
    }

    @Test
    public void test2() throws IOException {
        Tokenizer t = new JavaTokenizer();
        String data = "public class Foo {" + PMD.EOL + "public void bar() {}" + PMD.EOL + "public void buz() {}"
                + PMD.EOL + "}";
        SourceCode sourceCode = new SourceCode(new SourceCode.StringCodeLoader(data));
        Tokens tokens = new Tokens();
        t.tokenize(sourceCode, tokens);
        assertEquals("public class Foo {" + PMD.EOL + "public void bar() {}", sourceCode.getSlice(1, 2));
    }

    @Test
    public void testDiscardSemicolons() throws IOException {
        Tokenizer t = new JavaTokenizer();
        SourceCode sourceCode = new SourceCode(new SourceCode.StringCodeLoader("public class Foo {private int x;}"));
        Tokens tokens = new Tokens();
        t.tokenize(sourceCode, tokens);
        assertEquals(9, tokens.size());
    }

    @Test
    public void testDiscardImports() throws IOException {
        Tokenizer t = new JavaTokenizer();
        SourceCode sourceCode = new SourceCode(
                new SourceCode.StringCodeLoader("import java.io.File;" + PMD.EOL + "public class Foo {}"));
        Tokens tokens = new Tokens();
        t.tokenize(sourceCode, tokens);
        assertEquals(6, tokens.size());
    }

    @Test
    public void testDiscardPkgStmts() throws IOException {
        Tokenizer t = new JavaTokenizer();
        SourceCode sourceCode = new SourceCode(
                new SourceCode.StringCodeLoader("package foo.bar.baz;" + PMD.EOL + "public class Foo {}"));
        Tokens tokens = new Tokens();
        t.tokenize(sourceCode, tokens);
        assertEquals(6, tokens.size());
    }

    @Test
    public void testDiscardSimpleOneLineAnnotation() throws IOException {
        JavaTokenizer t = new JavaTokenizer();
        t.setIgnoreAnnotations(true);
        SourceCode sourceCode = new SourceCode(new SourceCode.StringCodeLoader(
                "package foo.bar.baz;" + PMD.EOL + "@MyAnnotation" + PMD.EOL + "public class Foo {}"));
        Tokens tokens = new Tokens();
        t.tokenize(sourceCode, tokens);
        assertEquals(6, tokens.size());
    }

    @Test
    public void testIgnoreComments() {
        JavaTokenizer t = new JavaTokenizer();
        t.setIgnoreAnnotations(false);
        SourceCode sourceCode = new SourceCode(new SourceCode.StringCodeLoader("package foo.bar.baz;" + PMD.EOL
                + "/*****" + PMD.EOL + " * ugh" + PMD.EOL + " *****/" + PMD.EOL + "public class Foo {}"));
        Tokens tokens = new Tokens();
        t.tokenize(sourceCode, tokens);
        assertEquals(6, tokens.size());
    }

    @Test
    public void testDiscardOneLineAnnotationWithParams() throws IOException {
        JavaTokenizer t = new JavaTokenizer();
        t.setIgnoreAnnotations(true);

        SourceCode sourceCode = new SourceCode(new SourceCode.StringCodeLoader(
                "package foo.bar.baz;" + PMD.EOL + "@ MyAnnotation (\"ugh\")" + PMD.EOL + "@NamedQueries({" + PMD.EOL
                        + "@NamedQuery(" + PMD.EOL + ")})" + PMD.EOL + "public class Foo {" + PMD.EOL + "}"));
        Tokens tokens = new Tokens();
        t.tokenize(sourceCode, tokens);
        TokenEntry.getEOF();
        assertEquals(6, tokens.size());
    }

    @Test
    public void testIgnoreBetweenSpecialComments() throws IOException {
        JavaTokenizer t = new JavaTokenizer();
        t.setIgnoreAnnotations(false);
        SourceCode sourceCode = new SourceCode(new SourceCode.StringCodeLoader("package foo.bar.baz;" + PMD.EOL
                + "// CPD-OFF" + PMD.EOL + "// CPD-OFF" + PMD.EOL
                + "@ MyAnnotation (\"ugh\")" + PMD.EOL + "@NamedQueries({" + PMD.EOL + "@NamedQuery(" + PMD.EOL + ")})"
                + PMD.EOL + "public class Foo {" + "// CPD-ON" + PMD.EOL
                + "}"
        ));
        Tokens tokens = new Tokens();
        t.tokenize(sourceCode, tokens);
        TokenEntry.getEOF();
        assertEquals(2, tokens.size()); // 2 tokens: "}" + EOF
    }

    @Test
    public void testIgnoreBetweenSpecialCommentsMultiple() throws IOException {
        JavaTokenizer t = new JavaTokenizer();
        t.setIgnoreAnnotations(false);
        SourceCode sourceCode = new SourceCode(new SourceCode.StringCodeLoader("package foo.bar.baz;" + PMD.EOL
                + "// CPD-OFF" + PMD.EOL + "// another irrelevant comment" + PMD.EOL
                + "@ MyAnnotation (\"ugh\")" + PMD.EOL + "@NamedQueries({" + PMD.EOL + "@NamedQuery(" + PMD.EOL + ")})"
                + PMD.EOL + "public class Foo {" + "// CPD-ON" + PMD.EOL
                + "}"
        ));
        Tokens tokens = new Tokens();
        t.tokenize(sourceCode, tokens);
        TokenEntry.getEOF();
        assertEquals(2, tokens.size()); // 2 tokens: "}" + EOF
    }

    @Test
    public void testIgnoreBetweenSpecialCommentsMultiline() throws IOException {
        JavaTokenizer t = new JavaTokenizer();
        t.setIgnoreAnnotations(false);
        SourceCode sourceCode = new SourceCode(new SourceCode.StringCodeLoader("package foo.bar.baz;" + PMD.EOL
                + "/* " + PMD.EOL + " * CPD-OFF" + PMD.EOL + "*/" + PMD.EOL
                + "@ MyAnnotation (\"ugh\")" + PMD.EOL + "@NamedQueries({" + PMD.EOL + "@NamedQuery(" + PMD.EOL + ")})"
                + PMD.EOL + "public class Foo {" + PMD.EOL
                + "/* " + PMD.EOL + " * CPD-ON" + PMD.EOL + "*/" + PMD.EOL
                + "}"
        ));
        Tokens tokens = new Tokens();
        t.tokenize(sourceCode, tokens);
        TokenEntry.getEOF();
        assertEquals(2, tokens.size()); // 2 tokens: "}" + EOF
    }

    @Test
    public void testIgnoreBetweenSpecialAnnotation() throws IOException {
        JavaTokenizer t = new JavaTokenizer();
        t.setIgnoreAnnotations(false);
        SourceCode sourceCode = new SourceCode(new SourceCode.StringCodeLoader("package foo.bar.baz;" + PMD.EOL
                + "@SuppressWarnings({\"woof\",\"CPD-START\"})" + PMD.EOL + "@SuppressWarnings(\"CPD-START\")" + PMD.EOL
                + "@ MyAnnotation (\"ugh\")" + PMD.EOL + "@NamedQueries({" + PMD.EOL + "@NamedQuery(" + PMD.EOL + ")})"
                + PMD.EOL + "public class Foo {}" + "@SuppressWarnings({\"ugh\",\"CPD-END\"})" + PMD.EOL

        ));
        Tokens tokens = new Tokens();
        t.tokenize(sourceCode, tokens);
        TokenEntry.getEOF();
        assertEquals(10, tokens.size());
    }

    @Test
    public void testIgnoreBetweenSpecialAnnotationAndIgnoreAnnotations() throws IOException {
        JavaTokenizer t = new JavaTokenizer();
        t.setIgnoreAnnotations(true);
        SourceCode sourceCode = new SourceCode(new SourceCode.StringCodeLoader("package foo.bar.baz;" + PMD.EOL
                + "@SuppressWarnings({\"woof\",\"CPD-START\"})" + PMD.EOL + "@SuppressWarnings(\"CPD-START\")" + PMD.EOL
                + "@ MyAnnotation (\"ugh\")" + PMD.EOL + "@NamedQueries({" + PMD.EOL + "@NamedQuery(" + PMD.EOL + ")})"
                + PMD.EOL + "public class Foo {}" + PMD.EOL + "@SuppressWarnings({\"ugh\",\"CPD-END\"})" + PMD.EOL

        ));
        Tokens tokens = new Tokens();
        t.tokenize(sourceCode, tokens);
        TokenEntry.getEOF();
        assertEquals(1, tokens.size());
    }

    @Test
    public void testIgnoreIdentifiersDontAffectConstructors() throws IOException {
        JavaTokenizer t = new JavaTokenizer();
        t.setIgnoreAnnotations(false);
        t.setIgnoreIdentifiers(true);

        SourceCode sourceCode = new SourceCode(new SourceCode.StringCodeLoader("package foo.bar.baz;" + PMD.EOL
                + "public class Foo extends Bar {" + PMD.EOL + "private Foo notAConstructor;" + PMD.EOL
                + "public Foo(int i) { super(i); }" + PMD.EOL + "private Foo(int i, String s) { super(i, s); }"
                + PMD.EOL + "/* default */ Foo(int i, String s, Object o) { super(i, s, o); }" + PMD.EOL
                + "private static class Inner {" + PMD.EOL + "Inner() { System.out.println(\"Guess who?\"); }" + PMD.EOL
                + "}" + PMD.EOL + "}" + PMD.EOL

        ));
        Tokens tokens = new Tokens();
        t.tokenize(sourceCode, tokens);
        TokenEntry.getEOF();
        List<TokenEntry> tokenList = tokens.getTokens();

        // Member variable of type Foo
        assertEquals(String.valueOf(JavaParserConstants.IDENTIFIER), tokenList.get(7).toString());
        // Public constructor
        assertEquals("Foo", tokenList.get(10).toString());
        // Private constructor
        assertEquals("Foo", tokenList.get(22).toString());
        // Package-private constructor
        assertEquals("Foo", tokenList.get(38).toString());
        // Inner class constructor
        assertEquals("Inner", tokenList.get(64).toString());
    }

    @Test
    public void testIgnoreIdentifiersHandlesEnums() throws IOException {
        JavaTokenizer t = new JavaTokenizer();
        t.setIgnoreAnnotations(false);
        t.setIgnoreIdentifiers(true);

        SourceCode sourceCode = new SourceCode(new SourceCode.StringCodeLoader(
                "package foo.bar.baz;" + PMD.EOL + "public enum Foo {" + PMD.EOL + "BAR(1)," + PMD.EOL + "BAZ(2);"
                        + PMD.EOL + "Foo(int val) {" + PMD.EOL + "}" + PMD.EOL + "}" + PMD.EOL

        ));
        Tokens tokens = new Tokens();
        t.tokenize(sourceCode, tokens);
        TokenEntry.getEOF();
        List<TokenEntry> tokenList = tokens.getTokens();

        // Enum member
        assertEquals(String.valueOf(JavaParserConstants.IDENTIFIER), tokenList.get(4).toString());
        assertEquals(String.valueOf(JavaParserConstants.IDENTIFIER), tokenList.get(9).toString());
        // Enum constructor
        assertEquals("Foo", tokenList.get(13).toString());
    }

    @Test
    public void testIgnoreIdentifiersWithClassKeyword() throws IOException {
        JavaTokenizer t = new JavaTokenizer();
        t.setIgnoreAnnotations(false);
        t.setIgnoreIdentifiers(true);

        SourceCode sourceCode = new SourceCode(new SourceCode.StringCodeLoader(
                "package foo.bar.baz;" + PMD.EOL + "public class Foo {" + PMD.EOL + "Foo() {" + PMD.EOL
                        + "}" + PMD.EOL + "public void bar() {" + PMD.EOL + "Bar.baz(Foo.class, () -> {});"
                        + PMD.EOL + "}" + PMD.EOL + "}" + PMD.EOL
        ));
        Tokens tokens = new Tokens();
        t.tokenize(sourceCode, tokens);
        TokenEntry.getEOF();
        List<TokenEntry> tokenList = tokens.getTokens();

        // Class constructor
        assertEquals("Foo", tokenList.get(4).toString());
        assertEquals(String.valueOf(JavaParserConstants.IDENTIFIER), tokenList.get(11).toString());
    }
}