← index
MXVM/src/scanner/scanner.cpp
Source: MXVM/src/scanner/scanner.cpp
/*
    coded by Jared Bruni (jaredbruni@protonmail.com)
    https://lostsidedead.biz
*/
#include "scanner/scanner.hpp"

#include <set>


namespace scan {

    Scanner::Scanner(const TString &b) : string_buffer{b} {}

    void Scanner::removeEOL() {
        for (int64_t i = tokens.size() - 1; i >= 0; --i) {
            if (tokens[i].getTokenValue() == "\n" && tokens[i].getTokenType() == types::TokenType::TT_SYM) {
                tokens.erase(tokens.begin() + i);
            }
        }
    }

    uint64_t Scanner::parseLineNumber() {
        std::string lineNumberStr;
        while (true) {
            auto ch = string_buffer.getch();
            if (!ch.has_value()) {
                return 1;
            }
            if (isdigit(*ch)) {
                lineNumberStr += *ch;
                break;
            }
        }
        while (true) {
            auto ch = string_buffer.getch();
            if (!ch.has_value() || !isdigit(*ch)) {
                break;
            }
            lineNumberStr += *ch;
        }
        return !lineNumberStr.empty() ? std::stoull(lineNumberStr) : 1;
    }

    std::string Scanner::parseFileName() {
        std::string fileName;
        while (true) {
            auto ch = string_buffer.getch();
            if (!ch.has_value() || !isspace(*ch)) {
                if (ch.has_value() && *ch == '\"')
                    break;
            }
        }
        while (true) {
            auto ch = string_buffer.getch();
            if (!ch.has_value() || *ch == '\"')
                break;
            fileName += *ch;
        }
        return fileName.empty() ? "unknown" : fileName;
    }

    uint64_t Scanner::scan() {
        tokens.clear();
        while (!string_buffer.eof(0)) {
            auto ch = string_buffer.getch();
            if (!ch.has_value())
                break;

            if (*ch == '\n') {
                TToken token;
                auto pos = string_buffer.cur_line();
                auto filename = string_buffer.cur_file();

                token.set_pos(pos);
                token.set_filename(filename);
                token.setToken(types::TokenType::TT_SYM, "\n");

                tokens.push_back(token);
                continue;
            }

            if (*ch == '\r') {
                auto next = string_buffer.getch();
                if (next.has_value() && *next == '\n') {
                    continue;
                }
            }

            if (*ch == '"') {
                auto token = grabString();
                if (token.has_value()) {
                    tokens.push_back(*token);
                }
                continue;
            }

            if (ch.has_value() && *ch == '#') {
                do {
                    ch = string_buffer.getch();
                } while (ch.has_value() && *ch != '\n');
                string_buffer.backward_step(1);
                continue;
            } else if (ch.has_value() && *ch == '/') {
                auto ch2 = string_buffer.curch();
                if (ch2.has_value() && *ch2 == '/') {
                    auto ch_ln = string_buffer.curch();
                    do {
                        ch_ln = string_buffer.getch();
                    } while (ch_ln.has_value() && *ch_ln != '\n');
                    continue;
                } else if (ch2.has_value() && *ch2 == '*') {
                    string_buffer.getch();
                    while (true) {
                        auto ch_in = string_buffer.getch();
                        if (!ch_in.has_value())
                            break;
                        if (*ch_in == '*') {
                            auto ch_ex = string_buffer.curch();
                            if (ch_ex.has_value() && *ch_ex == '/') {
                                string_buffer.getch();
                                break;
                            }
                        }
                    }
                    continue;
                }
            }
            if (ch.has_value()) {
                auto t_ch = token_map.lookup_int8(*ch);
                if (t_ch.has_value()) {
                    switch (*t_ch) {
                    case types::CharType::TT_CHAR: {
                        auto tok = grabId();
                        if (tok.has_value()) {
                            tokens.push_back(*tok);
                        }
                    } break;
                    case types::CharType::TT_DIGIT: {
                        auto tok = grabDigits();
                        if (tok.has_value()) {
                            tokens.push_back(*tok);
                        }
                    } break;
                    case types::CharType::TT_SYMBOL: {
                        auto tok = grabSymbols();
                        if (tok.has_value()) {
                            tokens.push_back(*tok);
                        }
                    } break;
                    case types::CharType::TT_STRING: {
                        auto tok = grabString();
                        if (tok.has_value()) {
                            tokens.push_back(*tok);
                        }
                    } break;
                    case types::CharType::TT_SINGLE: {
                        auto tok = grabSingle();
                        if (tok.has_value()) {
                            tokens.push_back(*tok);
                        }
                    } break;
                    case types::CharType::TT_SPACE:
                        while (true) {
                            auto next = string_buffer.peekch(0);
                            if (next.has_value() && (*next == ' ' || *next == '\t' || *next == '\r')) {
                                string_buffer.getch();
                                continue;
                            }
                            if (!next.has_value() || token_map.lookup_int8(*next) != types::CharType::TT_SPACE) {
                                break;
                            }
                            string_buffer.getch();
                        }
                        continue;
                    case types::CharType::TT_NULL:
                        continue;
                    }
                }
            } else
                break;
        }

        return tokens.size();
    }
    std::optional<TToken> Scanner::grabId() {

        auto startPos = string_buffer.cur_line();
        auto filename = string_buffer.cur_file();

        string_buffer.backward_step(1);

        std::string val;
        while (true) {
            auto c = string_buffer.curch();
            if (!c.has_value())
                break;
            if (!std::isalnum(*c) && *c != '_')
                break;
            val += *string_buffer.getch();
        }

        if (val.empty())
            return std::nullopt;

        TToken tok;
        tok.set_pos(startPos);
        tok.set_filename(filename);
        tok.setToken(types::TokenType::TT_ID, val);
        return tok;
    }
    std::optional<TToken> Scanner::grabDigits() {
        auto pos = string_buffer.cur_line();
        auto filename = string_buffer.cur_file();
        auto is_op = [](char ch) {
            return ch == '+' || ch == '-' || ch == '*' || ch == '/' ||
                   ch == '%' || ch == '^' || ch == '|' || ch == '&' ||
                   ch == '(' || ch == ')';
        };
        auto is_hex_digit = [](char ch) {
            return std::isdigit(ch) ||
                   (ch >= 'a' && ch <= 'f') ||
                   (ch >= 'A' && ch <= 'F');
        };

        string_buffer.backward_step(1);
        TToken token;
        std::string tok_value;

        auto first_opt = string_buffer.getch();
        if (!first_opt.has_value())
            return std::nullopt;
        char first = *first_opt;
        tok_value += first;

        if (first == '0') {
            auto nx = string_buffer.peekch(0);
            if (nx.has_value() && (*nx == 'x' || *nx == 'X')) {
                tok_value += *string_buffer.getch();
                while (true) {
                    auto p = string_buffer.peekch(0);
                    if (!p.has_value() || !is_hex_digit(*p))
                        break;
                    tok_value += *string_buffer.getch();
                }
                token.set_pos(pos);
                token.set_filename(filename);
                token.setToken(types::TokenType::TT_HEX, tok_value);
                return token;
            }
        }

        int decimal_dots = 0;
        while (true) {
            auto p = string_buffer.peekch(0);
            if (!p.has_value())
                break;
            char c = *p;
            if (std::isspace(c) || is_op(c))
                break;
            if (c == '.' && string_buffer.peekch(1).has_value() && *string_buffer.peekch(1) == '.')
                break;
            if (!std::isdigit(c) && c != '.')
                break;

            string_buffer.getch();
            if (c == '.') {
                if (++decimal_dots > 1) {
                    std::ostringstream err;
                    auto lp = string_buffer.cur_line();
                    err << "File: " << filename << " Line: " << lp.first << " Col: " << lp.second << " -> Too many decimal points in number.";
                    throw ScanExcept(err.str());
                }
            }
            tok_value += c;
        }

        token.set_pos(pos);
        token.set_filename(filename);
        token.setToken(types::TokenType::TT_NUM, tok_value);
        return token;
    }

    bool Scanner::is_c_sym(const StringType &str) {
        static const std::set<std::string> c_op = {
            "++", "--", ">>=", "<<=", "+=", "-=", "*=", "/=", "%=", "&=", "|=", "^=",
            "==", "!=", ">=", "<=", ">>", "<<", "&&", "||", "->", ".", "&", "*", "+",
            "-", "~", "!", "/", "%", ">", "<", "=", "^", "|", "?", ":", ";", ",",
            "(", ")", "[", "]", "{", "}", "->", ".", "::", "$", ":=", "<>", ".."};
        return c_op.find(str) != c_op.end();
    }

    std::optional<TToken> Scanner::grabSymbols() {
        TToken token;
        std::string tok_value;
        string_buffer.backward_step(1);
        auto filename = string_buffer.cur_file();
        auto pos = string_buffer.cur_line();

        auto ch = string_buffer.getch();
        if (!ch.has_value())
            return std::nullopt;

        if (*ch == '-') {
            auto next = string_buffer.peekch(0);
            if (next.has_value() && *next == '-') {
                string_buffer.getch();
                std::string arg_value = "--";
                while (true) {
                    auto peeked = string_buffer.peekch(0);
                    if (peeked.has_value() && (isalnum(*peeked) || *peeked == '_')) {
                        arg_value += string_buffer.getch().value();
                    } else {
                        break;
                    }
                }
                token.set_pos(pos);
                token.set_filename(filename);
                token.setToken(types::TokenType::TT_ARG, arg_value);
                return token;
            } else {
                token.set_pos(pos);
                token.set_filename(filename);
                token.setToken(types::TokenType::TT_SYM, "-");
                return token;
            }
        }

        if (*ch == '.' && string_buffer.peekch(0).has_value() && *string_buffer.peekch(0) == '.') {
            string_buffer.getch();
            token.set_pos(pos);
            token.set_filename(filename);
            token.setToken(types::TokenType::TT_SYM, "..");
            return token;
        }

        tok_value = *ch;
        std::string temp_value = tok_value;
        int look = 0;
        bool found = false;

        while (true) {
            auto temp = string_buffer.peekch(look);
            if (temp.has_value() && token_map.lookup_int8(*temp) == types::CharType::TT_SYMBOL) {
                temp_value += *temp;
                if (is_c_sym(temp_value)) {
                    tok_value = temp_value;
                    look++;
                    found = true;
                } else {
                    break;
                }
            } else {
                break;
            }
        }

        if (found && look != 0) {
            string_buffer.forward_step(look);
        }

        token.set_pos(pos);
        token.set_filename(filename);
        token.setToken(types::TokenType::TT_SYM, tok_value);
        return token;
    }

    std::optional<TToken> Scanner::grabString() {
        TToken token;
        std::string tok_value;
        auto pos = string_buffer.cur_line();
        auto filename = string_buffer.cur_file();
        while (true) {
            auto ch = string_buffer.getch();
            if (!ch.has_value()) {
                std::ostringstream stream;
                auto cpos = string_buffer.cur_line();
                stream << "File: " << string_buffer.cur_file() << " Line: " << cpos.first << " Col:" << cpos.second << " Double quote not terminated.";
                throw ScanExcept(stream.str());
            }
            if (*ch == '\\') {
                auto next_ch = string_buffer.getch();
                if (!next_ch.has_value())
                    break;
                switch (*next_ch) {
                case 'n':
                    tok_value += '\n';
                    break;
                case 't':
                    tok_value += '\t';
                    break;
                case 'r':
                    tok_value += '\r';
                    break;
                case '\\':
                    tok_value += '\\';
                    break;
                case '\"':
                    tok_value += "\\\"";
                    break;
                case '\'':
                    tok_value += '\'';
                    break;
                default:
                    tok_value += '\\';
                    tok_value += *next_ch;
                    break;
                }
            } else if (*ch == '\"') {
                break;
            } else {
                tok_value += *ch;
            }
        }
        token.set_pos(pos);
        token.set_filename(filename);
        token.setToken(types::TokenType::TT_STR, tok_value);
        return token;
    }

    std::optional<TToken> Scanner::grabSingle() {
        TToken token;
        std::string tok_value;
        auto filename = string_buffer.cur_file();
        auto pos = string_buffer.cur_line();
        while (true) {
            auto ch = string_buffer.getch();
            if (!ch.has_value()) {
                std::ostringstream stream;
                auto cpos = string_buffer.cur_line();
                stream << "File: " << string_buffer.cur_file() << " Line: " << cpos.first << " Col:" << cpos.second << " String quote not terminated.";
                throw ScanExcept(stream.str());
            }
            if (*ch == '\\') {
                auto next_ch = string_buffer.getch();
                if (!next_ch.has_value())
                    break;
                switch (*next_ch) {
                case 'n':
                    tok_value += '\n';
                    break;
                case 't':
                    tok_value += '\t';
                    break;
                case 'r':
                    tok_value += '\r';
                    break;
                case '\\':
                    tok_value += '\\';
                    break;
                case '\"':
                    tok_value += '\"';
                    break;
                case '\'':
                    tok_value += '\'';
                    break;
                default:
                    tok_value += '\\';
                    tok_value += *next_ch;
                    break;
                }
            } else if (*ch == '\'') {
                break;
            } else {
                tok_value += *ch;
            }
        }

        if (string_buffer.peekch().has_value() && *string_buffer.peekch() == '\'') {
            string_buffer.getch();
        }

        token.set_pos(pos);
        token.set_filename(filename);
        token.setToken(types::TokenType::TT_STR, tok_value);
        return token;
    }

    TToken &Scanner::operator[](size_t index) {
        if (index < size())
            return tokens[index];
        std::ostringstream stream;
        stream << "Token index: index out of bounds, Scanner Exception: " << index << "\n";
        throw ScanExcept(stream.str());
    }
    size_t Scanner::size() const { return tokens.size(); }
} // namespace scan