← index
ats/system/include/lexer.hpp
Source: ats/system/include/lexer.hpp
// Created by Jared Bruni
// http://lostsidedead.com



#ifndef lexer_lexer_h
#define lexer_lexer_h

#include<iostream>
#include<vector>
#include<cstdlib>
#include<unordered_map>

namespace lex {
    
    extern const char *keywords[];
    
    enum Keyword_type {
          KEY_ALIGNOF, KEY_AND, KEY_AND_EQ, KEY_ASM, KEY_AUTO, KEY_BITAND, KEY_BITOR, KEY_BOOL, KEY_BREAK, KEY_CASE, KEY_CATCH, KEY_CHAR, KEY_CHAR16_T, KEY_CHAR32_T, KEY_CLASS, KEY_COMPL, KEY_CONST, KEY_CONSTEXPR, KEY_CONST_CAST, KEY_CONTINUE, KEY_DECLTYPE, KEY_DEFAULT, KEY_DELETE, KEY_DO, KEY_DOUBLE, KEY_DYNAMIC_CAST, KEY_ELSE, KEY_ENUM, KEY_EXPLICIT, KEY_EXPORT, KEY_EXTERN, KEY_FALSE, KEY_FLOAT, KEY_FOR, KEY_FRIEND, KEY_GOTO, KEY_IF, KEY_INLINE, KEY_INT, KEY_LONG, KEY_MUTABLE, KEY_NAMESPACE, KEY_NEW, KEY_NOEXCEPT, KEY_NOT, KEY_NOT_EQ, KEY_NULLPTR, KEY_OPERATOR, KEY_OR, KEY_OR_EQ, KEY_PRIVATE, KEY_PROTECTED, KEY_PUBLIC, KEY_REGISTER, KEY_REINTERPRET_CAST, KEY_RETURN, KEY_SHORT, KEY_SIGNED, KEY_SIZEOF, KEY_STATIC, KEY_STATIC_ASSERT, KEY_STATIC_CAST, KEY_STRUCT, KEY_SWITCH, KEY_TEMPLATE, KEY_THIS, KEY_THREAD_LOCAL, KEY_THROW, KEY_TRUE, KEY_TRY, KEY_TYPEDEF, KEY_TYPEID, KEY_TYPENAME, KEY_UNION, KEY_UNSIGNED, KEY_USING, KEY_VIRTUAL, KEY_VOID, KEY_VOLATILE, KEY_WCHAR_T, KEY_WHILE, KEY_XOR, KEY_XOR_EQ, KEY_NULL
    };

    extern const char *operators[];
    
    enum Symbol_type { SYM_EEQUAL, SYM_NOTEQUAL, SYM_STREAM_LEFT, SYM_STREAM_RIGHT, SYM_PLUS_EQUAL, SYM_MINUS_EQUAL,SYM_MUL_EQUAL, SYM_DIV_EQUAL, SYM_AND_EUAL, SYM_OR_EQUAL, SYM_XOR_EQUAL,SYM_SEMICOLON, SYM_COLON, SYM_DOT, SYM_MINUS, SYM_POPEN, SYM_PCLOSE, SYM_BOPEN, SYM_BCLOSE, SYM_COPEN, SYM_CCLOSE, SYM_EXCLAMATION, SYM_AT, SYM_POUND, SYM_DOLLAR, SYM_MOD, SYM_XOR, SYM_AND, SYM_MUL, SYM_EQUAL, SYM_PLUS, SYM_TIDLE, SYM_QUOTE, SYM_ANGLE, SYM_CLASS_POINTER, SYM_NULL };
    
    enum Token_type { TOKEN_NOTHING=0, TOKEN_CHAR, TOKEN_WHITESPACE, TOKEN_STRING, TOKEN_SINGLE,TOKEN_DIGIT, TOKEN_OPERATOR, TOKEN_IDENTIFIER, TOKEN_PRINT, TOKEN_HEX, /*TOKEN_MINUS,*/ TOKEN_EOF };
    
    class Scanner_EOF {};
    
    class Scanner_Error {
    public:
        std::string error_text;
        Scanner_Error() = default;
        Scanner_Error(const std::string &text) : error_text(text) {};
    };
    
    class Exit_Exception {};

    class Token {
    public:
        Token() : text(""), line(0), offset(0), type(TOKEN_NOTHING), keyword_type(-1), symbol_type(-1) {}
        Token(std::string txt, Token_type t) : text(txt), type(t), line(0), offset(0) {}
        std::string getToken() const { return text; }
        Token_type getType() const { return type; }
        
        Token &operator=(const Token &t) {
            setToken(t);
            return *this;
        }
        
        bool operator==(const Token &t) {
            if(text == t.text && type == t.type)
                return true;
            return false;
        }
        
        bool isKeyword() {
            
            for(unsigned int i = 0; keywords[i] != 0; ++i) {
                if(keywords[i] == text) return true;
            }
            return false;
        }
        
        int keywordToType(const std::string &text) {
            for(unsigned int i = 0; keywords[i] != 0; ++i) {
                if(text == keywords[i]) return i;
            }
            return -1;
        }
        
        int symbolToType(const std::string &text) {
            for(unsigned int i = 0; operators[i] != 0; ++i) {
                if(text == operators[i]) return i;
            }
            return -1;
        }
        
        Token_type getTokenType() const { return type; }
        int getKeywordType() const { return keyword_type; }
        int getSymbolType() const { return symbol_type; }
        
        void setToken(const std::string &txt, Token_type t, unsigned int coffset, unsigned int cline) {
            text = txt;
            keyword_type = keywordToType(text);
            symbol_type = symbolToType(text);
            type = t;
            offset = coffset;
            line = cline;
            
        }
        void setToken(const Token &t) {
            text = t.text;
            keyword_type = t.keyword_type;
            symbol_type = t.symbol_type;
            type = t.type;
            offset = t.offset;
            line = t.line;
        }
        
        friend std::ostream &operator<<(std::ostream &, const Token &);
        
        unsigned int line, offset;
        std::string text;
    private:
        
        Token_type type;
        int keyword_type;
        int symbol_type;
      
    };
    
    class Token_map {
    public:
        unsigned char token_map[256] = { TOKEN_NOTHING };
        
        Token_map() {
            unsigned char i;
            for(i = 0; i < 0xFF; ++i)
                if(isOperator(i) == true)
                    token_map[i] = TOKEN_OPERATOR;
                else
                    token_map[i] = TOKEN_NOTHING;
            
            for(i = 'a'; i <= 'z'; ++i)
                token_map[i] = TOKEN_CHAR;
            for(i = 'A'; i <= 'Z'; ++i)
                token_map[i] = TOKEN_CHAR;
            
            setToken('_', TOKEN_CHAR);
            
            for(i = '0'; i <= '9'; ++i)
                token_map[i] = TOKEN_DIGIT;
            
            //setToken('-', TOKEN_MINUS);
            setToken('$', TOKEN_HEX);
            setToken('\'', TOKEN_SINGLE);
            setToken('"', TOKEN_STRING);
            setToken(' ', TOKEN_WHITESPACE);
            setToken('\t', TOKEN_WHITESPACE);
        }

        void setToken(signed int pos, Token_type token) {
            token_map[pos] = token;
        }
        
        
        
        unsigned char &operator[](char pos) {
            return token_map[static_cast<int>(pos)];
        }
        
        bool isOperator(char c) {
            static const char ch[] = ";:.-()[]{}!@#%^&*=+~\"`,\\/<>";
            
            for(unsigned int i = 0; ch[i] != 0; ++i)
                if(ch[i] == c)
                    return true;
            
            return false;
        }
    
    };

    class LexInput {
    private:
        bool owns;
        static Token_map token_map;
    public:
         LexInput(std::istream *s) : owns(true), input(s), offset(0), line(1) {
            initMap();
        }
        LexInput(std::istream &s) : owns(false), input(&s), offset(0), line(1) {
            initMap();
        }
        
        ~LexInput() {
            if(owns)
                delete input;
        }
        
     
        
        void initMap() {

        }
        
        Token_type characterToType(const unsigned char &c) {
            return static_cast<Token_type>(token_map[c]);
        }
        
        unsigned char getChar() {
            ++offset;
            current = input->get();
            if(current == '\n') { ++line; offset = 0; return getChar(); }
            if(*input) return current;
            return 0;
        }
        
        void removeLineComment() {
            current = input->get();
            ++offset;
            if(current == '\n') { ++line; offset = 0; return; }
            while(current != '\n' && *input) {
                current = input->get();
                ++offset;
                if(!*input) break;
            }
            if(current == '\n') { ++line; offset = 0; return; }
        }
        
        void removeMultiLineComment() {
            current = input->get();
            while(current != '*' && *input) {
                current = input->get();
                ++offset;
                if(current == '*') {
                    current = input->get();
                    ++offset;
                    if(current == '/') {
                        break;
                    }
                } else if(current == '\n') { ++line; offset = 0; }
            }
        }
        
        const unsigned char &curChar() const { return current; }

        void putBack(const unsigned char &c) {
            input->putback(c);
        }
        
        Token GetToken() {
            Token tok;
            unsigned char c;
            if(input && input->eof()) {
              //  throw Scanner_EOF();
                tok = Token("", TOKEN_EOF);
                return tok;
            }
            c = getChar();
            if(c == 0) return Token("", TOKEN_EOF);
            Token_type type = characterToType(c);
            
            switch(type) {
                case TOKEN_WHITESPACE: {
                    unsigned char cc = c;
                    while(cc == ' ' || cc =='\r' || cc == '\t') {
                        cc = getChar();
                    }
                    
                    putBack(cc);
                    return GetToken();
                }
                    break;
               /* case TOKEN_MINUS: {
                    char ch = getChar();
                    if(characterToType(ch) == TOKEN_DIGIT) {
                        GetDigitToken(tok);
                        tok.text="-"+tok.text;
                    } else {
                        if(ch == '-') {
                            tok.setToken("--", TOKEN_OPERATOR, line, offset);
                        } else {
                            tok.setToken("-", TOKEN_OPERATOR, line, offset);
                            putBack(ch);
                        }
                    }
                }
                    break; */
                case TOKEN_HEX:
                case TOKEN_DIGIT:
                    GetDigitToken(tok);
                    break;
                case TOKEN_STRING:
                    GetStringToken(tok);
                    break;
                case TOKEN_SINGLE:
                    GetCharacterToken(tok);
                    break;
                case TOKEN_CHAR:
                    GetIdToken(tok);
                    break;
                case TOKEN_OPERATOR:
                    GetOperatorToken(tok);
                    
                    if(tok.getToken() == "//") {
                        removeLineComment();
                        return GetToken();
                    }
                    
                    if(tok.getToken() == "/*") {
                        removeMultiLineComment();
                        return GetToken();
                    }
                    
                    break;
                case TOKEN_NOTHING:
                    std::cout << "Warning unrecognized character: " << c << "\n";
                    return GetToken();
                    break;
                default:
                    break;
            }
            ctoken = tok;
            return tok;
        }
        
        bool valid() {
            if(input != nullptr && *input)
                return true;
            return false;
        }
        
        Token &GetCurrent() {
            return ctoken;
        }
  
    private:
        std::istream *input;
        unsigned int offset, line;
        unsigned char current;
        Token ctoken;
        
        void GetDigitToken(Token &token) {
            std::string tok;
            unsigned char cc = curChar();
            unsigned int sline = line, soffset = offset;
            signed int count = 0;
            bool trunc = false;

            if(characterToType(cc) == TOKEN_HEX) {
                cc = getChar();
                if(characterToType(cc) != TOKEN_DIGIT && !(toupper(cc) >= 'A' && toupper(cc) <= 'F')) {
                    tok = "$";
                    token.setToken(tok, TOKEN_OPERATOR, soffset, sline);
                    putBack(cc);
                    return;
                }
                while((characterToType(cc) == TOKEN_DIGIT || (toupper(cc) >= 'A' && toupper(cc) <= 'Z'))) {
                    if(characterToType(cc) != TOKEN_DIGIT && !(toupper(cc) >= 'A' && toupper(cc) <= 'F')) {
                        std::string ch;
                        ch += cc;
                        throw Scanner_Error(std::string("Scanner Error:  Token [") + tok + std::string("] contains Invalid Hex value: ") +ch+ std::string("\n"));
                    }
                    tok += cc;
                    cc = getChar();
                }
                
                if(characterToType(cc) != TOKEN_CHAR)
                    putBack(cc);
                
                token.setToken(tok,TOKEN_HEX, soffset, sline);
            } else {
                while(((characterToType(cc) == TOKEN_DIGIT) || cc == '.') && *input) {
                    if(cc == '.') ++count;
                    if(count < 2)
                        tok += cc;
                    else { trunc = true; }
                    cc = getChar();
                }
                if(characterToType(cc) !=  TOKEN_CHAR)
                    putBack(cc);
                if(trunc == true) std::cout << tok << " truncated.\n";
                token.setToken(tok, TOKEN_DIGIT, soffset, sline);
            }
        }
        
        void GetIdToken(Token &token) {
            std::string tok;
            unsigned char cc = curChar();
            unsigned int sline = line, soffset = offset;
            Token_type ccType = characterToType(cc);
            while(((ccType == TOKEN_CHAR)||(ccType == TOKEN_DIGIT))&&*input) {
                tok += cc;
                cc = getChar();
                ccType = characterToType(cc);
            }
            putBack(cc);
            token.setToken(tok, TOKEN_CHAR, soffset, sline);
        }
        
        void GetStringToken(Token &token) {
            std::string tok;
            unsigned int sline = line, soffset = offset-1;
            unsigned char prev_cc = curChar(), cc = getChar();
            Token_type ccType = characterToType(cc);
            
            while(ccType != TOKEN_STRING && *input) {
                tok += cc;
                prev_cc = cc;
                cc = getChar();
                Token_type cur_Type = characterToType(cc);
                if(prev_cc == '\\') {
                    cc = getChar();
                    ccType = characterToType(cc);
                    continue;
                }
                ccType = characterToType(cc);
            }
            token.setToken(tok, TOKEN_STRING, soffset, sline);
        }
        
        
        void GetCharacterToken(Token &token) {
            std::string tok;
            unsigned int sline = line, soffset = offset;
            unsigned char cc = getChar();
            tok += cc;
            unsigned char cx = getChar();
            if(cc == '\\') {
                do {
                    tok += cx;
                    cx = getChar();
                } while(characterToType(cx) != TOKEN_SINGLE && *input);
            }
            token.setToken(tok, TOKEN_SINGLE, soffset, sline);
        }
        
        void GetOperatorToken(Token &token) {
            unsigned char cc = curChar();
            unsigned int sline = line, soffset = offset;
            std::string tok;
            tok += cc;
            unsigned char oc = cc;
            cc = getChar();
            if(oc == cc && (cc != '{' && cc != '[' && cc != '(' && cc != ']' && cc != ')' && cc != '}')) {
                tok += cc;
            } else if(cc == '=' || cc == '>') {
                tok += cc;
            } else {
                putBack(cc);
            }
            token.setToken(tok, TOKEN_OPERATOR, soffset, sline);
        }
    };

    std::ostream &operator<<(std::ostream &out, const Token_type &t);
    std::ostream &operator<<(std::ostream &out, const Token &t);

    class Scanner {
        
    public:
        Scanner(std::istream *str) : input{str} { }
        Scanner(std::istream &str) : input(str) { }
        
        // will throw Scanner_EOF at end of file
        Scanner &operator>>(Token &token) {
            if(valid()) token = input.GetToken();
            else throw Scanner_EOF();
            return *this;
        }
        
        void collect() {
            Token t;
            while(input.valid()) {
                t = input.GetToken();
                std::cout << t << "\n";
            }
        }
        
        void err(std::string s) {
            std::cerr << s << "\n";
            throw Scanner_Error();
        }

    
        bool valid() {
            return input.valid();
        }
        
    protected:
        LexInput input;
        Token cur_token;
    };
}

#endif