/****************************************************************************
**
** Copyright (C) 2017 The Qt Company Ltd.
** Contact: https://www.qt.io/licensing/
**
** This file is part of Qt Creator.
**
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3 as published by the Free Software
** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-3.0.html.
**
****************************************************************************/

#include <haskelltokenizer.h>

#include <QObject>
#include <QtTest>

using namespace Haskell::Internal;

const QSet<char> escapes{'a', 'b', 'f', 'n', 'r', 't', 'v', '\\', '"', '\'', '&'};

struct TokenInfo
{
    TokenType type;
    int column;
    QString text;
};

Q_DECLARE_METATYPE(TokenInfo)

bool operator==(const TokenInfo &info, const Token &token)
{
    return info.type == token.type
            && info.column == token.startCol
            && info.text.length() == token.length
            && info.text == token.text.toString();
}

bool operator==(const Token &token, const TokenInfo &info)
{
    return info == token;
}

class tst_Tokenizer : public QObject
{
    Q_OBJECT

private slots:
    void singleLineComment_data();
    void singleLineComment();

    void multiLineComment_data();
    void multiLineComment();

    void string_data();
    void string();

    void character_data();
    void character();

    void number_data();
    void number();

    void keyword_data();
    void keyword();

    void variable_data();
    void variable();

    void constructor_data();
    void constructor();

    void op_data();
    void op();

private:
    void setupData();
    void addRow(const char *name,
                const QString &input,
                const QList<TokenInfo> &tokens,
                Tokens::State startState = Tokens::State::None,
                Tokens::State endState = Tokens::State::None);
    void checkData();
};

void tst_Tokenizer::setupData()
{
    QTest::addColumn<QString>("input");
    QTest::addColumn<QList<TokenInfo>>("output");
    QTest::addColumn<int>("startState");
    QTest::addColumn<int>("endState");
}

void tst_Tokenizer::addRow(const char *name,
                           const QString &input,
                           const QList<TokenInfo> &tokens,
                           Tokens::State startState,
                           Tokens::State endState)
{
    QTest::newRow(name) << input << tokens << int(startState) << int(endState);
}

void tst_Tokenizer::checkData()
{
    QFETCH(QString, input);
    QFETCH(QList<TokenInfo>, output);
    QFETCH(int, startState);
    QFETCH(int, endState);
    const Tokens tokens = HaskellTokenizer::tokenize(input, startState);
    QCOMPARE(tokens.length(), output.length());
    QCOMPARE(tokens.state, endState);
    for (int i = 0; i < tokens.length(); ++i) {
        const Token t = tokens.at(i);
        const TokenInfo ti = output.at(i);
        QVERIFY2(t == ti, QString("Token at index %1 does not match, {%2, %3, \"%4\"} != {%5, %6, \"%7\"}")
                 .arg(i)
                 .arg(int(t.type)).arg(t.startCol).arg(t.text.toString())
                 .arg(int(ti.type)).arg(ti.column).arg(ti.text)
                 .toUtf8().constData());
    }
}

void tst_Tokenizer::singleLineComment_data()
{
    setupData();

    addRow("simple", " -- foo", {
               {TokenType::Whitespace, 0, " "},
               {TokenType::SingleLineComment, 1, "-- foo"}
           });
    addRow("dash, id", "--foo", {
               {TokenType::SingleLineComment, 0, "--foo"}
           });
    addRow("dash, space, op", "-- |foo", {
               {TokenType::SingleLineComment, 0, "-- |foo"}
           });
    addRow("multi-dash, space", "---- foo", {
               {TokenType::SingleLineComment, 0, "---- foo"}
           });
    addRow("dash, op", "--| foo", {
               {TokenType::Operator, 0, "--|"},
               {TokenType::Whitespace, 3, " "},
               {TokenType::Variable, 4, "foo"}
           });
    addRow("dash, special", "--(foo", {
               {TokenType::SingleLineComment, 0, "--(foo"}
           });
    addRow("not a qualified varsym", "F.-- foo", {
               {TokenType::Constructor, 0, "F"},
               {TokenType::Operator, 1, "."},
               {TokenType::SingleLineComment, 2, "-- foo"}
           });
}

void tst_Tokenizer::singleLineComment()
{
    checkData();
}

void tst_Tokenizer::multiLineComment_data()
{
    setupData();

    addRow("trailing dashes", "{---foo -}", {
               {TokenType::MultiLineComment, 0, "{---foo -}"}
           });
    addRow("multiline", "{- foo", {
               {TokenType::MultiLineComment, 0, "{- foo"}
           },
           Tokens::State::None,
           Tokens::State(int(Tokens::State::MultiLineCommentGuard) + 1));
    addRow("multiline2", "bar -}", {
               {TokenType::MultiLineComment, 0, "bar -}"}
           },
           Tokens::State(int(Tokens::State::MultiLineCommentGuard) + 1),
           Tokens::State::None);
    addRow("nested", "{- fo{-o", {
               {TokenType::MultiLineComment, 0, "{- fo{-o"}
           },
           Tokens::State::None,
           Tokens::State(int(Tokens::State::MultiLineCommentGuard) + 2));
    addRow("nested2", "bar -}", {
               {TokenType::MultiLineComment, 0, "bar -}"}
           },
           Tokens::State(int(Tokens::State::MultiLineCommentGuard) + 2),
           Tokens::State(int(Tokens::State::MultiLineCommentGuard) + 1));
    addRow("nested3", "bar -}", {
               {TokenType::MultiLineComment, 0, "bar -}"}
           },
           Tokens::State(int(Tokens::State::MultiLineCommentGuard) + 1),
           Tokens::State::None);
}

void tst_Tokenizer::multiLineComment()
{
    checkData();
}

void tst_Tokenizer::string_data()
{
    setupData();

    addRow("simple", "\"foo\"", {
               {TokenType::String, 0, "\"foo\""}
           });

    addRow("unterminated", "\"", {
               {TokenType::StringError, 0, "\""}
           });
    addRow("unterminated2", "\"foo", {
               {TokenType::String, 0, "\"fo"},
               {TokenType::StringError, 3, "o"}
           });
    addRow("unterminated with escape", "\"\\\\", {
               {TokenType::String, 0, "\""},
               {TokenType::EscapeSequence, 1, "\\"},
               {TokenType::StringError, 2, "\\"}
           });

    // gaps
    addRow("gap", "\" \\   \\\"", {
               {TokenType::String, 0, "\" \\   \\\""}
           });
    addRow("gap over endline", "\"foo\\", {
               {TokenType::String, 0, "\"foo\\"}
           },
           Tokens::State::None, Tokens::State::StringGap);
    addRow("gap over endline2", "\\foo\"", {
               {TokenType::String, 0, "\\foo\""}
           },
           Tokens::State::StringGap, Tokens::State::None);
    addRow("gap error", "\"\\ ab \\\"", {
               {TokenType::String, 0, "\"\\ "},
               {TokenType::StringError, 3, "ab"},
               {TokenType::String, 5, " \\\""}
           });
    addRow("gap error with quote", "\"\\ \"", {
               {TokenType::String, 0, "\"\\ "},
               {TokenType::StringError, 3, "\""}
           },
           Tokens::State::None, Tokens::State::StringGap);

    // char escapes (including wrong ones)
    for (char c = '!'; c <= '~'; ++c) {
        // skip uppercase and '^', since these can be part of ascii escapes
        // and 'o' and 'x' since they start octal and hex escapes
        // and digits as part of decimal escape
        if ((c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '^' || c == 'o' || c == 'x')
            continue;
        const QChar qc(c);
        const QByteArray name = QString("charesc '%1'").arg(qc).toUtf8();
        const QString input = QString("\"\\%1\"").arg(qc);
        if (escapes.contains(c)) {
            addRow(name.constData(), input, {
                       {TokenType::String, 0, "\""},
                       {TokenType::EscapeSequence, 1, QLatin1String("\\") + qc},
                       {TokenType::String, 3, "\""}
                   });
        } else {
            addRow(name.constData(), input, {
                       {TokenType::String, 0, "\"\\"},
                       {TokenType::StringError, 2, qc},
                       {TokenType::String, 3, "\""}
                   });
        }
    }

    addRow("decimal escape", "\"\\1234a\"", {
               {TokenType::String, 0, "\""},
               {TokenType::EscapeSequence, 1, "\\1234"},
               {TokenType::String, 6, "a\""}
           });

    addRow("octal escape", "\"\\o0678a\"", {
               {TokenType::String, 0, "\""},
               {TokenType::EscapeSequence, 1, "\\o067"},
               {TokenType::String, 6, "8a\""}
           });
    addRow("octal escape error", "\"\\o8a\"", {
               {TokenType::String, 0, "\"\\"},
               {TokenType::StringError, 2, "o"},
               {TokenType::String, 3, "8a\""}
           });

    addRow("hexadecimal escape", "\"\\x0678Abg\"", {
               {TokenType::String, 0, "\""},
               {TokenType::EscapeSequence, 1, "\\x0678Ab"},
               {TokenType::String, 9, "g\""}
           });
    addRow("hexadecimal escape error", "\"\\xg\"", {
               {TokenType::String, 0, "\"\\"},
               {TokenType::StringError, 2, "x"},
               {TokenType::String, 3, "g\""}
           });

    // ascii cntrl escapes (including wrong ones)
    for (char c = '!'; c <= '~'; ++c) {
        if (c == '"') // is special because it also ends the string
            continue;
        const QChar qc(c);
        const QByteArray name = QString("ascii cntrl '^%1'").arg(qc).toUtf8();
        const QString input = QString("\"\\^%1\"").arg(qc);
        if ((qc >= 'A' && qc <= 'Z') || qc == '@' || qc == '[' || qc == '\\' || qc == ']'
                || qc == '^' || qc == '_') {
            addRow(name.constData(), input, {
                       {TokenType::String, 0, "\""},
                       {TokenType::EscapeSequence, 1, QLatin1String("\\^") + qc},
                       {TokenType::String, 4, "\""}
                   });
        } else {
            addRow(name.constData(), input, {
                       {TokenType::String, 0, "\"\\"},
                       {TokenType::StringError, 2, "^"},
                       {TokenType::String, 3, QString(qc) + "\""}
                   });
        }
    }

    addRow("ascii escape SOH", "\"\\SOHN\"", {
               {TokenType::String, 0, "\""},
               {TokenType::EscapeSequence, 1, "\\SOH"},
               {TokenType::String, 5, "N\""}
           });
    addRow("ascii escape SO", "\"\\SON\"", {
               {TokenType::String, 0, "\""},
               {TokenType::EscapeSequence, 1, "\\SO"},
               {TokenType::String, 4, "N\""}
           });
    addRow("ascii escape error", "\"\\TON\"", {
               {TokenType::String, 0, "\"\\"},
               {TokenType::StringError, 2, "T"},
               {TokenType::String, 3, "ON\""}
           });
    addRow("ascii escape error 2", "\"\\STO\"", {
               {TokenType::String, 0, "\"\\"},
               {TokenType::StringError, 2, "S"},
               {TokenType::String, 3, "TO\""}
           });
}

void tst_Tokenizer::string()
{
    checkData();
}

void tst_Tokenizer::character_data()
{
    setupData();

    addRow("simple", "'a'", {
               {TokenType::Char, 0, "'a'"}
           });
    addRow("too many", "'abc'", {
               {TokenType::Char, 0, "'a"},
               {TokenType::CharError, 2, "bc"},
               {TokenType::Char, 4, "'"}
           });
    addRow("too few", "''", {
               {TokenType::Char, 0, "'"},
               {TokenType::CharError, 1, "'"}
           });
    addRow("only quote", "'", {
               {TokenType::CharError, 0, "'"}
           });
    addRow("unterminated", "'a", {
               {TokenType::Char, 0, "'"},
               {TokenType::CharError, 1, "a"}
           });
    addRow("unterminated too many", "'abc", {
               {TokenType::Char, 0, "'a"},
               {TokenType::CharError, 2, "bc"}
           });
    addRow("unterminated backslash", "'\\", {
               {TokenType::Char, 0, "'"},
               {TokenType::CharError, 1, "\\"}
           });

    // char escapes (including wrong ones)
    for (char c = '!'; c <= '~'; ++c) {
        // skip uppercase and '^', since these can be part of ascii escapes
        // and 'o' and 'x' since they start octal and hex escapes
        // and digits as part of decimal escape
        if ((c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '^' || c == 'o' || c == 'x')
            continue;
        const QChar qc(c);
        const QByteArray name = QString("charesc '%1'").arg(qc).toUtf8();
        const QString input = QString("'\\%1'").arg(qc);
        if (c != '&' && escapes.contains(c)) {
            addRow(name.constData(), input, {
                       {TokenType::Char, 0, "'"},
                       {TokenType::EscapeSequence, 1, QLatin1String("\\") + qc},
                       {TokenType::Char, 3, "'"}
                   });
        } else {
            addRow(name.constData(), input, {
                       {TokenType::Char, 0, "'\\"},
                       {TokenType::CharError, 2, qc},
                       {TokenType::Char, 3, "'"}
                   });
        }
    }

    addRow("decimal escape", "'\\1234'", {
               {TokenType::Char, 0, "'"},
               {TokenType::EscapeSequence, 1, "\\1234"},
               {TokenType::Char, 6, "'"}
           });
    addRow("decimal escape too long", "'\\1234a'", {
               {TokenType::Char, 0, "'"},
               {TokenType::EscapeSequence, 1, "\\1234"},
               {TokenType::CharError, 6, "a"},
               {TokenType::Char, 7, "'"}
           });

    addRow("octal escape", "'\\o067'", {
               {TokenType::Char, 0, "'"},
               {TokenType::EscapeSequence, 1, "\\o067"},
               {TokenType::Char, 6, "'"}
           });
    addRow("octal escape error", "'\\o8'", {
               {TokenType::Char, 0, "'\\"},
               {TokenType::CharError, 2, "o"},
               {TokenType::CharError, 3, "8"},
               {TokenType::Char, 4, "'"}
           });

    addRow("hexadecimal escape", "'\\x0678Ab'", {
               {TokenType::Char, 0, "'"},
               {TokenType::EscapeSequence, 1, "\\x0678Ab"},
               {TokenType::Char, 9, "'"}
           });
    addRow("hexadecimal escape error", "'\\xg'", {
               {TokenType::Char, 0, "'\\"},
               {TokenType::CharError, 2, "x"},
               {TokenType::CharError, 3, "g"},
               {TokenType::Char, 4, "'"}
           });

    // ascii cntrl escapes (including wrong ones)
    for (char c = '!'; c <= '~'; ++c) {
        if (c == '\'') // is special because it also ends the string
            continue;
        const QChar qc(c);
        const QByteArray name = QString("ascii cntrl '^%1'").arg(qc).toUtf8();
        const QString input = QString("'\\^%1'").arg(qc);
        if ((qc >= 'A' && qc <= 'Z') || qc == '@' || qc == '[' || qc == '\\' || qc == ']'
                || qc == '^' || qc == '_') {
            addRow(name.constData(), input, {
                       {TokenType::Char, 0, "'"},
                       {TokenType::EscapeSequence, 1, QLatin1String("\\^") + qc},
                       {TokenType::Char, 4, "'"}
                   });
        } else {
            addRow(name.constData(), input, {
                       {TokenType::Char, 0, "'\\"},
                       {TokenType::CharError, 2, "^"},
                       {TokenType::CharError, 3, qc},
                       {TokenType::Char, 4, "'"}
                   });
        }
    }

    addRow("ascii escape SOH", "'\\SOH'", {
               {TokenType::Char, 0, "'"},
               {TokenType::EscapeSequence, 1, "\\SOH"},
               {TokenType::Char, 5, "'"}
           });
    addRow("ascii escape SO, too long", "'\\SON'", {
               {TokenType::Char, 0, "'"},
               {TokenType::EscapeSequence, 1, "\\SO"},
               {TokenType::CharError, 4, "N"},
               {TokenType::Char, 5, "'"}
           });
    addRow("ascii escape error", "'\\TON'", {
               {TokenType::Char, 0, "'\\"},
               {TokenType::CharError, 2, "T"},
               {TokenType::CharError, 3, "ON"},
               {TokenType::Char, 5, "'"}
           });
}

void tst_Tokenizer::character()
{
    checkData();
}

void tst_Tokenizer::number_data()
{
    setupData();

    addRow("decimal", "012345", {
               {TokenType::Integer, 0, "012345"}
           });
    addRow("single digit decimal", "0", {
               {TokenType::Integer, 0, "0"}
           });
    addRow("octal", "0o1234", {
               {TokenType::Integer, 0, "0o1234"}
           });
    // this is a bit weird, but correct: octal 1 followed by decimal 8
    addRow("number after octal", "0O18", {
               {TokenType::Integer, 0, "0O1"},
               {TokenType::Integer, 3, "8"}
           });
    addRow("not octal", "0o9", {
               {TokenType::Integer, 0, "0"},
               {TokenType::Variable, 1, "o9"},
           });
    addRow("hexadecimal", "0x9fA", {
               {TokenType::Integer, 0, "0x9fA"}
           });
    // hex number followed by identifier 'g'
    addRow("hexadecimal", "0X9fg", {
               {TokenType::Integer, 0, "0X9f"},
               {TokenType::Variable, 4, "g"}
           });

    // 0 followed by identifier
    addRow("decimal followed by identifier", "0z6", {
               {TokenType::Integer, 0, "0"},
               {TokenType::Variable, 1, "z6"}
           });

    addRow("float", "0123.45", {
               {TokenType::Float, 0, "0123.45"}
           });
    addRow("decimal + operator '.'", "0123.", {
               {TokenType::Integer, 0, "0123"},
               {TokenType::Operator, 4, "."}
           });
    addRow("operator '.' + decimal", ".0123", {
               {TokenType::Operator, 0, "."},
               {TokenType::Integer, 1, "0123"}
           });
    addRow("without '.', with exp 'e'", "0123e45", {
               {TokenType::Float, 0, "0123e45"}
           });
    addRow("without '.', with exp 'E'", "0123E45", {
               {TokenType::Float, 0, "0123E45"}
           });
    addRow("without '.', with '+'", "0123e+45", {
               {TokenType::Float, 0, "0123e+45"}
           });
    addRow("without '.', with '-'", "0123e-45", {
               {TokenType::Float, 0, "0123e-45"}
           });
    addRow("without '.', with '+', missing decimal", "0123e+", {
               {TokenType::Integer, 0, "0123"},
               {TokenType::Variable, 4, "e"},
               {TokenType::Operator, 5, "+"}
           });
    addRow("without '.', missing decimal", "0123e", {
               {TokenType::Integer, 0, "0123"},
               {TokenType::Variable, 4, "e"}
           });
    addRow("exp 'e'", "01.23e45", {
               {TokenType::Float, 0, "01.23e45"}
           });
    addRow("exp 'E'", "01.23E45", {
               {TokenType::Float, 0, "01.23E45"}
           });
    addRow("with '+'", "01.23e+45", {
               {TokenType::Float, 0, "01.23e+45"}
           });
    addRow("with '-'", "01.23e-45", {
               {TokenType::Float, 0, "01.23e-45"}
           });
    addRow("with '+', missing decimal", "01.23e+", {
               {TokenType::Float, 0, "01.23"},
               {TokenType::Variable, 5, "e"},
               {TokenType::Operator, 6, "+"}
           });
    addRow("missing decimal", "01.23e", {
               {TokenType::Float, 0, "01.23"},
               {TokenType::Variable, 5, "e"}
           });
}

void tst_Tokenizer::number()
{
    checkData();
}

void tst_Tokenizer::keyword_data()
{
    setupData();

    addRow("data", "data", {
               {TokenType::Keyword, 0, "data"}
           });
    addRow("not a qualified varid", "Foo.case", {
               {TokenType::Constructor, 0, "Foo"},
               {TokenType::Operator, 3, "."},
               {TokenType::Keyword, 4, "case"}
           });
    addRow(":", ":", {
               {TokenType::Keyword, 0, ":"}
           });
    addRow("->", "->", {
               {TokenType::Keyword, 0, "->"}
           });
    addRow("not a qualified varsym", "Foo...", {
               {TokenType::Constructor, 0, "Foo"},
               {TokenType::Operator, 3, "..."}
           });
}

void tst_Tokenizer::keyword()
{
    checkData();
}

void tst_Tokenizer::variable_data()
{
    setupData();

    addRow("simple", "fOo_1'", {
               {TokenType::Variable, 0, "fOo_1'"}
           });
    addRow("start with '_'", "_1", {
               {TokenType::Variable, 0, "_1"}
           });
    addRow("not a keyword", "cases", {
               {TokenType::Variable, 0, "cases"}
           });
    addRow("not a keyword 2", "qualified", {
               {TokenType::Variable, 0, "qualified"}
           });
    addRow("not a keyword 3", "as", {
               {TokenType::Variable, 0, "as"}
           });
    addRow("not a keyword 4", "hiding", {
               {TokenType::Variable, 0, "hiding"}
           });
    addRow(".variable", ".foo", {
               {TokenType::Operator, 0, "."},
               {TokenType::Variable, 1, "foo"}
           });
    addRow("variable.", "foo.", {
               {TokenType::Variable, 0, "foo"},
               {TokenType::Operator, 3, "."}
           });
    addRow("variable.variable", "blah.foo", {
               {TokenType::Variable, 0, "blah"},
               {TokenType::Operator, 4, "."},
               {TokenType::Variable, 5, "foo"}
           });
    addRow("qualified", "Blah.foo", {
               {TokenType::Variable, 0, "Blah.foo"}
           });
    addRow("qualified2", "Goo.Blah.foo", {
               {TokenType::Variable, 0, "Goo.Blah.foo"}
           });
    addRow("variable + op '..'", "foo..", {
               {TokenType::Variable, 0, "foo"},
               {TokenType::Keyword, 3, ".."}
           });
    addRow("variable + op '...'", "foo...", {
               {TokenType::Variable, 0, "foo"},
               {TokenType::Operator, 3, "..."}
           });
}

void tst_Tokenizer::variable()
{
    checkData();
}

void tst_Tokenizer::constructor_data()
{
    setupData();

    addRow("simple", "Foo", {
               {TokenType::Constructor, 0, "Foo"}
           });
    addRow("qualified", "Foo.Bar", {
               {TokenType::Constructor, 0, "Foo.Bar"}
           });
    addRow("followed by op '.'", "Foo.Bar.", {
               {TokenType::Constructor, 0, "Foo.Bar"},
               {TokenType::Operator, 7, "."}
           });

}

void tst_Tokenizer::constructor()
{
    checkData();
}

void tst_Tokenizer::op_data()
{
    setupData();

    addRow("simple", "+-=", {
               {TokenType::Operator, 0, "+-="}
           });
    addRow("qualified", "Foo.+-=", {
               {TokenType::Operator, 0, "Foo.+-="}
           });
    addRow("qualified '.'", "Foo..", {
               {TokenType::Operator, 0, "Foo.."}
           });
    addRow("constructor plus op", "Foo+", {
               {TokenType::Constructor, 0, "Foo"},
               {TokenType::Operator, 3, "+"}
           });
}

void tst_Tokenizer::op()
{
    checkData();
}

QTEST_MAIN(tst_Tokenizer)

#include "tst_tokenizer.moc"