ScotchLayoutLexerTest.java

package scotch.compiler.parser;

import static org.antlr.v4.runtime.Recognizer.EOF;
import static org.hamcrest.Matchers.is;
import static org.junit.Assert.assertThat;
import static scotch.compiler.parser.ScotchLexer.CLOSE_CURLY;
import static scotch.compiler.parser.ScotchLexer.ID_VAR;
import static scotch.compiler.parser.ScotchLexer.IN;
import static scotch.compiler.parser.ScotchLexer.OPEN_CURLY;
import static scotch.compiler.parser.ScotchLexer.OPERATOR;
import static scotch.compiler.parser.ScotchLexer.SEMICOLON;
import static scotch.compiler.parser.ScotchLexer.VOCABULARY;

import java.util.ArrayList;
import java.util.List;
import org.antlr.v4.runtime.ANTLRInputStream;
import org.antlr.v4.runtime.Token;
import org.junit.Test;

public class ScotchLayoutLexerTest {

    @Test
    public void shouldRemoveNewlineBeforeIndent() {
        List<Token> tokens = lexAll(
            "fn a = a",
            "  + b"
        );
        shouldHaveToken(tokens, 4, OPERATOR, "+");
    }

    @Test
    public void sequentialLet_shouldCollapseIntoSingleLet() {
        List<Token> tokens = lexAll(
            "let one = 1",
            "let two = 2",
            "three one two"
        );
        shouldHaveToken(tokens, 1, OPEN_CURLY, "{");
        shouldHaveToken(tokens, 5, SEMICOLON, ";");
        shouldHaveToken(tokens, 6, ID_VAR, "two");
        shouldHaveToken(tokens, 9, SEMICOLON, ";");
        shouldHaveToken(tokens, 10, CLOSE_CURLY, "}");
        shouldHaveToken(tokens, 11, IN, "in");
        shouldHaveToken(tokens, 12, ID_VAR, "three");
    }

    @Test
    public void shouldAcceptLetWithoutIn_whenScopeOnSeparateLine() {
        List<Token> tokens = lexAll(
            "let x = 1",
            "    y = 2",
            "x + y"
        );
        shouldHaveToken(tokens, 1, OPEN_CURLY, "{");
        shouldHaveToken(tokens, 5, SEMICOLON, ";");
        shouldHaveToken(tokens, 6, ID_VAR, "y");
        shouldHaveToken(tokens, 9, SEMICOLON, ";");
        shouldHaveToken(tokens, 10, CLOSE_CURLY, "}");
        shouldHaveToken(tokens, 11, IN, "in");
    }

    @Test
    public void shouldGetTokensOnSameLine() {
        List<Token> tokens = lexAll("one two three");
        shouldHaveToken(tokens, 0, ID_VAR, "one");
        shouldHaveToken(tokens, 1, ID_VAR, "two");
        shouldHaveToken(tokens, 2, ID_VAR, "three");
    }

    @Test
    public void shouldIndentAccordingToFirstTokenAfterLet() {
        List<Token> tokens = lexAll(
            "let",
            "  x = 1",
            "  y = 2",
            "x + y"
        );
        shouldHaveToken(tokens, 1, OPEN_CURLY, "{");
        shouldHaveToken(tokens, 5, SEMICOLON, ";");
        shouldHaveToken(tokens, 6, ID_VAR, "y");
        shouldHaveToken(tokens, 9, SEMICOLON, ";");
        shouldHaveToken(tokens, 10, CLOSE_CURLY, "}");
        shouldHaveToken(tokens, 11, IN, "in");
    }

    @Test
    public void shouldInsertCurliesAroundLet() {
        List<Token> tokens = lexAll(
            "let x = y",
            "    z = a",
            "in x + z"
        );
        shouldHaveToken(tokens, 1, OPEN_CURLY, "{");
        shouldHaveToken(tokens, 5, SEMICOLON, ";");
        shouldHaveToken(tokens, 6, ID_VAR, "z");
        shouldHaveToken(tokens, 9, SEMICOLON, ";");
        shouldHaveToken(tokens, 10, CLOSE_CURLY, "}");
        shouldHaveToken(tokens, 11, IN, "in");
    }

    @Test
    public void shouldInsertCurliesAroundLetOnOneLine() {
        List<Token> tokens = lexAll("let z = \\x -> y in z");
        shouldHaveToken(tokens, 1, OPEN_CURLY, "{");
        shouldHaveToken(tokens, 8, SEMICOLON, ";");
        shouldHaveToken(tokens, 9, CLOSE_CURLY, "}");
        shouldHaveToken(tokens, 10, IN, "in");
    }

    @Test
    public void shouldRemoveNewLinesBetweenAllIndentedLines() {
        List<Token> tokens = lexAll(
            "these are",
            "  all",
            "  together",
            "this is not"
        );
        shouldHaveToken(tokens, 2, ID_VAR, "all");
        shouldHaveToken(tokens, 3, ID_VAR, "together");
        shouldHaveToken(tokens, 4, SEMICOLON, ";");
    }

    @Test
    public void shouldInsertSemicolonBetweenLinesWithSameIndent() {
        List<Token> tokens = lexAll(
            "these are",
            "separate"
        );
        shouldHaveToken(tokens, 2, SEMICOLON, ";");
    }

    @Test
    public void shouldLayoutMatch() {
        List<Token> tokens = lexAll(
            "fib n = match n on",
            "    0 = 0",
            "    1 = 1",
            "    n = fib (n - 1) + fib (n - 2)",
            "main = ..."
        );
        shouldHaveToken(tokens, 6, OPEN_CURLY, "{");
        shouldHaveToken(tokens, 10, SEMICOLON, ";");
        shouldHaveToken(tokens, 14, SEMICOLON, ";");
        shouldHaveToken(tokens, 30, SEMICOLON, ";");
        shouldHaveToken(tokens, 31, CLOSE_CURLY, "}");
        shouldHaveToken(tokens, 32, SEMICOLON, ";");
        shouldHaveToken(tokens, 33, ID_VAR, "main");
    }

    @Test
    public void shouldLayoutWhere_whenEndingInEof() {
        List<Token> tokens = lexAll(
            "where",
            "    (+) :: a",
            "    (-) :: a",
            "    abs :: a"
        );
        shouldHaveToken(tokens, 1, OPEN_CURLY, "{");
        shouldHaveToken(tokens, 7, SEMICOLON, ";");
        shouldHaveToken(tokens, 13, SEMICOLON, ";");
        shouldHaveToken(tokens, 17, SEMICOLON, ";");
        shouldHaveToken(tokens, 19, CLOSE_CURLY, "}");
        shouldHaveToken(tokens, 20, SEMICOLON, ";");
    }

    @Test
    public void shouldLayoutWhere_whenEndingWithDedent() {
        List<Token> tokens = lexAll(
            "where",
            "    (+) :: a",
            "    (-) :: b",
            "    abs :: c",
            "sub :: d"
        );
        shouldHaveToken(tokens, 1, OPEN_CURLY, "{");
        shouldHaveToken(tokens, 7, SEMICOLON, ";");
        shouldHaveToken(tokens, 13, SEMICOLON, ";");
        shouldHaveToken(tokens, 17, SEMICOLON, ";");
        shouldHaveToken(tokens, 18, CLOSE_CURLY, "}");
        shouldHaveToken(tokens, 19, SEMICOLON, ";");
        shouldHaveToken(tokens, 20, ID_VAR, "sub");
    }

    @Test
    public void shouldLayoutWhere_withMemberOnSameLine() {
        List<Token> tokens = lexAll(
            "where (+) :: a",
            "      (-) :: b",
            "      abs :: c",
            "sub :: d"
        );
        shouldHaveToken(tokens, 1, OPEN_CURLY, "{");
        shouldHaveToken(tokens, 7, SEMICOLON, ";");
        shouldHaveToken(tokens, 13, SEMICOLON, ";");
        shouldHaveToken(tokens, 17, SEMICOLON, ";");
        shouldHaveToken(tokens, 18, CLOSE_CURLY, "}");
        shouldHaveToken(tokens, 19, SEMICOLON, ";");
        shouldHaveToken(tokens, 20, ID_VAR, "sub");
    }

    @Test
    public void shouldNotLayoutWhenCurliesProvided() {
        List<Token> tokens = lexAll(
            "   let { x = 1;",
            "y = 2;",
            " } in x + y"
        );
        shouldHaveToken(tokens, 1, OPEN_CURLY, "{");
        shouldHaveToken(tokens, 5, SEMICOLON, ";");
        shouldHaveToken(tokens, 6, ID_VAR, "y");
        shouldHaveToken(tokens, 9, SEMICOLON, ";");
        shouldHaveToken(tokens, 10, CLOSE_CURLY, "}");
        shouldHaveToken(tokens, 11, IN, "in");
    }

    private ScotchLayoutLexer lex(String... input) {
        return new ScotchLayoutLexer(new ScotchLexer(new ANTLRInputStream(String.join("\n", input))));
    }

    private List<Token> lexAll(String... input) {
        ScotchLayoutLexer lexer = lex(input);
        List<Token> tokens = new ArrayList<>();
        while (true) {
            tokens.add(lexer.nextToken());
            if (tokens.get(tokens.size() - 1).getType() == EOF) {
                break;
            }
        }
        return tokens;
    }

    private void shouldHaveToken(List<Token> tokens, int offset, int type, String text) {
        shouldHaveToken(tokens.get(offset), type, text);
    }

    private void shouldHaveToken(Token token, int type, String text) {
        assertThat(VOCABULARY.getSymbolicName(token.getType()), is(VOCABULARY.getSymbolicName(type)));
        assertThat(token.getText(), is(text));
    }
}