#include"scanner.hpp" #include<set> namespace scan { Scanner::Scanner(const TString &b) : string_buffer{b} {} uint64_t Scanner::parseLineNumber() { std::string lineNumberStr; while (true) { auto ch = string_buffer.getch(); if (!ch.has_value()) { return 1; } if (isdigit(*ch)) { lineNumberStr += *ch; break; } } while (true) { auto ch = string_buffer.getch(); if (!ch.has_value() || !isdigit(*ch)) { break; } lineNumberStr += *ch; } return !lineNumberStr.empty() ? std::stoull(lineNumberStr) : 1; } std::string Scanner::parseFileName() { std::string fileName; while (true) { auto ch = string_buffer.getch(); if (!ch.has_value() || !isspace(*ch)) { if (ch.has_value() && *ch == '\"') break; } } while (true) { auto ch = string_buffer.getch(); if (!ch.has_value() || *ch == '\"') break; fileName += *ch; } return fileName.empty() ? "unknown" : fileName; } uint64_t Scanner::scan() { tokens.clear(); while(!string_buffer.eof(0)) { auto ch = string_buffer.getch(); if(!ch.has_value()) break; if (*ch == '\n') { TToken token; auto pos = string_buffer.cur_line(); auto filename = string_buffer.cur_file(); token.set_pos(pos); token.set_filename(filename); token.setToken(types::TokenType::TT_SYM, "\n"); tokens.push_back(token); continue; } if (*ch == '\r') { auto next = string_buffer.getch(); if (next.has_value() && *next == '\n') { continue; } } if(*ch == '"') { auto token = grabString(); if(token.has_value()) { tokens.push_back(*token); } continue; } if(ch.has_value() && *ch == '#') { do { ch = string_buffer.getch(); } while(ch.has_value() && *ch != '\n'); string_buffer.backward_step(1); continue; } else if(ch.has_value() && *ch == '/') { auto ch2 = string_buffer.curch(); if(ch2.has_value() && *ch2 == '/') { auto ch_ln = string_buffer.curch(); do { ch_ln = string_buffer.getch(); } while(ch_ln.has_value() && *ch_ln != '\n'); continue; } else if(ch2.has_value() && *ch2 == '*') { string_buffer.getch(); while(true) { auto ch_in = string_buffer.getch(); if(!ch_in.has_value()) break; if(*ch_in == '*') { auto ch_ex = string_buffer.curch(); if(ch_ex.has_value() && *ch_ex == '/') { string_buffer.getch(); break; } } } continue; } } if(ch.has_value()) { auto t_ch = token_map.lookup_int8(*ch); if(t_ch.has_value()) { switch(*t_ch) { case types::CharType::TT_CHAR: { auto tok = grabId(); if(tok.has_value()) { tokens.push_back(*tok); } } break; case types::CharType::TT_DIGIT: { auto tok = grabDigits(); if(tok.has_value()) { tokens.push_back(*tok); } } break; case types::CharType::TT_SYMBOL: { auto tok = grabSymbols(); if(tok.has_value()) { tokens.push_back(*tok); } } break; case types::CharType::TT_STRING: { auto tok = grabString(); if(tok.has_value()) { tokens.push_back(*tok); } } break; case types::CharType::TT_SINGLE: { auto tok = grabSingle(); if(tok.has_value()) { tokens.push_back(*tok); } } break; case types::CharType::TT_SPACE: while (true) { auto next = string_buffer.peekch(0); if(next.has_value() && (*next == ' ' || *next == '\t' || *next == '\r')) { string_buffer.getch(); continue; } if (!next.has_value() || token_map.lookup_int8(*next) != types::CharType::TT_SPACE) { break; } string_buffer.getch(); } continue; case types::CharType::TT_NULL: continue; } } } else break; } return tokens.size(); } std::optional<TToken> Scanner::grabId() { auto ch = string_buffer.backward_step(1); TToken token; auto filename = string_buffer.cur_file(); auto pos = string_buffer.cur_line(); if(ch.has_value()) { auto ch_t = token_map.lookup_int8(*ch); decltype(token.getTokenValue()) tok_value; while(true) { ch = string_buffer.getch(); if(!ch.has_value()) break; ch_t = token_map.lookup_int8(*ch); if(!ch_t.has_value() || (*ch_t != types::CharType::TT_CHAR && *ch_t != types::CharType::TT_DIGIT)) break; tok_value += *ch; } token.set_pos(pos); token.set_filename(filename); token.setToken(types::TokenType::TT_ID, tok_value); string_buffer.backward_step(1); return token; } return std::nullopt; } std::optional<TToken> Scanner::grabDigits() { auto is_op =[](char ch) -> bool { return ch == '+' || ch == '-' || ch == '*' || ch == '/' || ch == '%' || ch == '^' || ch == '|' || ch == '&' || ch == '(' || ch == ')'; }; auto ch = string_buffer.backward_step(1); TToken token; auto pos = string_buffer.cur_line(); auto filename = string_buffer.cur_file(); if(ch.has_value()) { auto ch_t = token_map.lookup_int8(*ch); decltype(token.getTokenValue()) tok_value; int decimal = 0; while(true) { ch = string_buffer.getch(); if(!ch.has_value()) break; if(isspace(*ch) || is_op(*ch)) break; ch_t = token_map.lookup_int8(*ch); if(!ch_t.has_value() || (*ch_t != types::CharType::TT_DIGIT && *ch != '.')) break; if(*ch == '.') { ++decimal; if(decimal > 1) { std::ostringstream stream; auto lp = string_buffer.cur_line(); stream << "File: " << string_buffer.cur_file() << " Line: " << lp.first << " Col: " << lp.second << " -> Too may precision points for floating point number only one is allowed."; throw ScanExcept(stream.str()); } } tok_value += *ch; } string_buffer.backward_step(1); token.set_pos(pos); token.set_filename(filename); token.setToken(types::TokenType::TT_NUM, tok_value); return token; } return std::nullopt; } bool Scanner::is_c_sym(const StringType &str) { static const std::set<std::string> c_op = { "++", "--", ">>=", "<<=", "+=", "-=", "*=", "/=", "%=", "&=", "|=", "^=", "==", "!=", ">=", "<=", ">>", "<<", "&&", "||", "->", ".", "&", "*", "+", "-", "~", "!", "/", "%", ">", "<", "=", "^", "|", "?", ":", ";", ",", "(", ")", "[", "]", "{", "}", "->", ".","::", "$" }; return c_op.find(str) != c_op.end(); } std::optional<TToken> Scanner::grabSymbols() { TToken token; std::string tok_value; string_buffer.backward_step(1); auto filename = string_buffer.cur_file(); auto pos = string_buffer.cur_line(); auto ch = string_buffer.getch(); if (!ch.has_value()) return std::nullopt; if (*ch == '-') { auto next = string_buffer.peekch(0); if (next.has_value() && *next == '-') { string_buffer.getch(); std::string arg_value = "--"; while (true) { auto peeked = string_buffer.peekch(0); if (peeked.has_value() && (isalnum(*peeked) || *peeked == '_')) { arg_value += string_buffer.getch().value(); } else { break; } } token.set_pos(pos); token.set_filename(filename); token.setToken(types::TokenType::TT_ARG, arg_value); return token; } else { token.set_pos(pos); token.set_filename(filename); token.setToken(types::TokenType::TT_SYM, "-"); return token; } } tok_value = *ch; std::string temp_value = tok_value; int look = 0; bool found = false; while (true) { auto temp = string_buffer.peekch(look); if (temp.has_value() && token_map.lookup_int8(*temp) == types::CharType::TT_SYMBOL) { temp_value += *temp; if (is_c_sym(temp_value)) { tok_value = temp_value; look++; found = true; } else { break; } } else { break; } } if (found && look != 0) { string_buffer.forward_step(look); } token.set_pos(pos); token.set_filename(filename); token.setToken(types::TokenType::TT_SYM, tok_value); return token; } std::optional<TToken> Scanner::grabString() { TToken token; std::string tok_value; auto pos = string_buffer.cur_line(); auto filename = string_buffer.cur_file(); while (true) { auto ch = string_buffer.getch(); if (!ch.has_value()) { std::ostringstream stream; auto cpos = string_buffer.cur_line(); stream << "File: " << string_buffer.cur_file() << " Line: " << cpos.first << " Col:"<< cpos.second << " Double quote not terminated."; throw ScanExcept(stream.str()); } if (*ch == '\\') { auto next_ch = string_buffer.getch(); if (!next_ch.has_value()) break; switch (*next_ch) { case 'n': tok_value += '\n'; break; case 't': tok_value += '\t'; break; case 'r': tok_value += '\r'; break; case '\\': tok_value += '\\'; break; case '\"': tok_value += "\\\""; break; case '\'': tok_value += '\''; break; default: tok_value += '\\'; tok_value += *next_ch; break; } } else if (*ch == '\"') { break; } else { tok_value += *ch; } } token.set_pos(pos); token.set_filename(filename); token.setToken(types::TokenType::TT_STR, tok_value); return token; } std::optional<TToken> Scanner::grabSingle() { TToken token; std::string tok_value; auto filename = string_buffer.cur_file(); auto pos = string_buffer.cur_line(); while (true) { auto ch = string_buffer.getch(); if (!ch.has_value()) { std::ostringstream stream; auto cpos = string_buffer.cur_line(); stream << "File: " << string_buffer.cur_file() << " Line: " << cpos.first << " Col:"<< cpos.second << " String quote not terminated."; throw ScanExcept(stream.str()); } if (*ch == '\\') { auto next_ch = string_buffer.getch(); if (!next_ch.has_value()) break; switch (*next_ch) { case 'n': tok_value += '\n'; break; case 't': tok_value += '\t'; break; case 'r': tok_value += '\r'; break; case '\\': tok_value += '\\'; break; case '\"': tok_value += '\"'; break; case '\'': tok_value += '\''; break; default: tok_value += '\\'; tok_value += *next_ch; break; } } else if (*ch == '\'') { break; } else { tok_value += *ch; } } if(string_buffer.peekch().has_value() && *string_buffer.peekch() == '\'') { string_buffer.getch(); } token.set_pos(pos); token.set_filename(filename); token.setToken(types::TokenType::TT_STR, tok_value); return token; } TToken &Scanner::operator[](size_t index) { if(index < size()) return tokens[index]; std::ostringstream stream; stream << "Token index: index out of bounds, Scanner Exception: " << index << "\n"; throw ScanExcept(stream.str()); } size_t Scanner::size() const { return tokens.size(); } }