MXVM 1.8.1
Virtual Machine, Compiler, and Pascal Frontend
Loading...
Searching...
No Matches
scanner.cpp
Go to the documentation of this file.
1
6#include "scanner/scanner.hpp"
7#include <set>
8#include <sstream>
9
10namespace scan {
11
12 Scanner::Scanner(const TString &b) : string_buffer{b} {}
13
14 void Scanner::removeEOL() {
15 for (int64_t i = tokens.size() - 1; i >= 0; --i) {
16 if (tokens[i].getTokenValue() == "\n" && tokens[i].getTokenType() == types::TokenType::TT_SYM) {
17 tokens.erase(tokens.begin() + i);
18 }
19 }
20 }
21
22 uint64_t Scanner::parseLineNumber() {
23 std::string lineNumberStr;
24 while (true) {
25 auto ch = string_buffer.getch();
26 if (!ch.has_value()) {
27 return 1;
28 }
29 if (isdigit(*ch)) {
30 lineNumberStr += *ch;
31 break;
32 }
33 }
34 while (true) {
35 auto ch = string_buffer.getch();
36 if (!ch.has_value() || !isdigit(*ch)) {
37 break;
38 }
39 lineNumberStr += *ch;
40 }
41 return !lineNumberStr.empty() ? std::stoull(lineNumberStr) : 1;
42 }
43
44 std::string Scanner::parseFileName() {
45 std::string fileName;
46 while (true) {
47 auto ch = string_buffer.getch();
48 if (!ch.has_value() || !isspace(*ch)) {
49 if (ch.has_value() && *ch == '\"')
50 break;
51 }
52 }
53 while (true) {
54 auto ch = string_buffer.getch();
55 if (!ch.has_value() || *ch == '\"')
56 break;
57 fileName += *ch;
58 }
59 return fileName.empty() ? "unknown" : fileName;
60 }
61
62 uint64_t Scanner::scan() {
63 tokens.clear();
64 while (!string_buffer.eof(0)) {
65 auto ch = string_buffer.getch();
66 if (!ch.has_value())
67 break;
68
69 if (*ch == '\n') {
70 TToken token;
71 auto pos = string_buffer.cur_line();
72 auto filename = string_buffer.cur_file();
73
74 token.set_pos(pos);
75 token.set_filename(filename);
76 token.setToken(types::TokenType::TT_SYM, "\n");
77
78 tokens.push_back(token);
79 continue;
80 }
81
82 if (*ch == '\r') {
83 auto next = string_buffer.getch();
84 if (next.has_value() && *next == '\n') {
85 continue;
86 }
87 }
88
89 if (*ch == '"') {
90 auto token = grabString();
91 if (token.has_value()) {
92 tokens.push_back(*token);
93 }
94 continue;
95 }
96
97 if (ch.has_value() && *ch == '#') {
98 do {
99 ch = string_buffer.getch();
100 } while (ch.has_value() && *ch != '\n');
101 string_buffer.backward_step(1);
102 continue;
103 } else if (ch.has_value() && *ch == '/') {
104 auto ch2 = string_buffer.curch();
105 if (ch2.has_value() && *ch2 == '/') {
106 auto ch_ln = string_buffer.curch();
107 do {
108 ch_ln = string_buffer.getch();
109 } while (ch_ln.has_value() && *ch_ln != '\n');
110 continue;
111 } else if (ch2.has_value() && *ch2 == '*') {
112 string_buffer.getch();
113 while (true) {
114 auto ch_in = string_buffer.getch();
115 if (!ch_in.has_value())
116 break;
117 if (*ch_in == '*') {
118 auto ch_ex = string_buffer.curch();
119 if (ch_ex.has_value() && *ch_ex == '/') {
120 string_buffer.getch();
121 break;
122 }
123 }
124 }
125 continue;
126 }
127 }
128 if (ch.has_value()) {
129 auto t_ch = token_map.lookup_int8(*ch);
130 if (t_ch.has_value()) {
131 switch (*t_ch) {
132 case types::CharType::TT_CHAR: {
133 auto tok = grabId();
134 if (tok.has_value()) {
135 tokens.push_back(*tok);
136 }
137 } break;
138 case types::CharType::TT_DIGIT: {
139 auto tok = grabDigits();
140 if (tok.has_value()) {
141 tokens.push_back(*tok);
142 }
143 } break;
144 case types::CharType::TT_SYMBOL: {
145 auto tok = grabSymbols();
146 if (tok.has_value()) {
147 tokens.push_back(*tok);
148 }
149 } break;
150 case types::CharType::TT_STRING: {
151 auto tok = grabString();
152 if (tok.has_value()) {
153 tokens.push_back(*tok);
154 }
155 } break;
156 case types::CharType::TT_SINGLE: {
157 auto tok = grabSingle();
158 if (tok.has_value()) {
159 tokens.push_back(*tok);
160 }
161 } break;
162 case types::CharType::TT_SPACE:
163 while (true) {
164 auto next = string_buffer.peekch(0);
165 if (next.has_value() && (*next == ' ' || *next == '\t' || *next == '\r')) {
166 string_buffer.getch();
167 continue;
168 }
169 if (!next.has_value() || token_map.lookup_int8(*next) != types::CharType::TT_SPACE) {
170 break;
171 }
172 string_buffer.getch();
173 }
174 continue;
175 case types::CharType::TT_NULL:
176 continue;
177 }
178 }
179 } else
180 break;
181 }
182
183 return tokens.size();
184 }
185 std::optional<TToken> Scanner::grabId() {
186
187 auto startPos = string_buffer.cur_line();
188 auto filename = string_buffer.cur_file();
189
190 string_buffer.backward_step(1);
191
192 std::string val;
193 while (true) {
194 auto c = string_buffer.curch();
195 if (!c.has_value())
196 break;
197 if (!std::isalnum(*c) && *c != '_')
198 break;
199 val += *string_buffer.getch();
200 }
201
202 if (val.empty())
203 return std::nullopt;
204
205 TToken tok;
206 tok.set_pos(startPos);
207 tok.set_filename(filename);
208 tok.setToken(types::TokenType::TT_ID, val);
209 return tok;
210 }
211 std::optional<TToken> Scanner::grabDigits() {
212 auto pos = string_buffer.cur_line();
213 auto filename = string_buffer.cur_file();
214 auto is_op = [](char ch) {
215 return ch == '+' || ch == '-' || ch == '*' || ch == '/' ||
216 ch == '%' || ch == '^' || ch == '|' || ch == '&' ||
217 ch == '(' || ch == ')';
218 };
219 auto is_hex_digit = [](char ch) {
220 return std::isdigit(ch) ||
221 (ch >= 'a' && ch <= 'f') ||
222 (ch >= 'A' && ch <= 'F');
223 };
224
225 string_buffer.backward_step(1);
226 TToken token;
227 std::string tok_value;
228
229 auto first_opt = string_buffer.getch();
230 if (!first_opt.has_value())
231 return std::nullopt;
232 char first = *first_opt;
233 tok_value += first;
234
235 if (first == '0') {
236 auto nx = string_buffer.peekch(0);
237 if (nx.has_value() && (*nx == 'x' || *nx == 'X')) {
238 tok_value += *string_buffer.getch();
239 while (true) {
240 auto p = string_buffer.peekch(0);
241 if (!p.has_value() || !is_hex_digit(*p))
242 break;
243 tok_value += *string_buffer.getch();
244 }
245 token.set_pos(pos);
246 token.set_filename(filename);
247 token.setToken(types::TokenType::TT_HEX, tok_value);
248 return token;
249 }
250 }
251
252 int decimal_dots = 0;
253 while (true) {
254 auto p = string_buffer.peekch(0);
255 if (!p.has_value())
256 break;
257 char c = *p;
258 if (std::isspace(c) || is_op(c))
259 break;
260 if (c == '.' && string_buffer.peekch(1).has_value() && *string_buffer.peekch(1) == '.')
261 break;
262 if (!std::isdigit(c) && c != '.')
263 break;
264
265 string_buffer.getch();
266 if (c == '.') {
267 if (++decimal_dots > 1) {
268 std::ostringstream err;
269 auto lp = string_buffer.cur_line();
270 err << "File: " << filename << " Line: " << lp.first << " Col: " << lp.second << " -> Too many decimal points in number.";
271 throw ScanExcept(err.str());
272 }
273 }
274 tok_value += c;
275 }
276
277 token.set_pos(pos);
278 token.set_filename(filename);
279 token.setToken(types::TokenType::TT_NUM, tok_value);
280 return token;
281 }
282
283 bool Scanner::is_c_sym(const StringType &str) {
284 static const std::set<std::string> c_op = {
285 "++", "--", ">>=", "<<=", "+=", "-=", "*=", "/=", "%=", "&=", "|=", "^=",
286 "==", "!=", ">=", "<=", ">>", "<<", "&&", "||", "->", ".", "&", "*", "+",
287 "-", "~", "!", "/", "%", ">", "<", "=", "^", "|", "?", ":", ";", ",",
288 "(", ")", "[", "]", "{", "}", "->", ".", "::", "$", ":=", "<>", ".."};
289 return c_op.find(str) != c_op.end();
290 }
291
292 std::optional<TToken> Scanner::grabSymbols() {
293 TToken token;
294 std::string tok_value;
295 string_buffer.backward_step(1);
296 auto filename = string_buffer.cur_file();
297 auto pos = string_buffer.cur_line();
298
299 auto ch = string_buffer.getch();
300 if (!ch.has_value())
301 return std::nullopt;
302
303 if (*ch == '-') {
304 auto next = string_buffer.peekch(0);
305 if (next.has_value() && *next == '-') {
306 string_buffer.getch();
307 std::string arg_value = "--";
308 while (true) {
309 auto peeked = string_buffer.peekch(0);
310 if (peeked.has_value() && (isalnum(*peeked) || *peeked == '_')) {
311 arg_value += string_buffer.getch().value();
312 } else {
313 break;
314 }
315 }
316 token.set_pos(pos);
317 token.set_filename(filename);
318 token.setToken(types::TokenType::TT_ARG, arg_value);
319 return token;
320 } else {
321 token.set_pos(pos);
322 token.set_filename(filename);
323 token.setToken(types::TokenType::TT_SYM, "-");
324 return token;
325 }
326 }
327
328 if (*ch == '.' && string_buffer.peekch(0).has_value() && *string_buffer.peekch(0) == '.') {
329 string_buffer.getch();
330 token.set_pos(pos);
331 token.set_filename(filename);
332 token.setToken(types::TokenType::TT_SYM, "..");
333 return token;
334 }
335
336 tok_value = *ch;
337 std::string temp_value = tok_value;
338 int look = 0;
339 bool found = false;
340
341 while (true) {
342 auto temp = string_buffer.peekch(look);
343 if (temp.has_value() && token_map.lookup_int8(*temp) == types::CharType::TT_SYMBOL) {
344 temp_value += *temp;
345 if (is_c_sym(temp_value)) {
346 tok_value = temp_value;
347 look++;
348 found = true;
349 } else {
350 break;
351 }
352 } else {
353 break;
354 }
355 }
356
357 if (found && look != 0) {
358 string_buffer.forward_step(look);
359 }
360
361 token.set_pos(pos);
362 token.set_filename(filename);
363 token.setToken(types::TokenType::TT_SYM, tok_value);
364 return token;
365 }
366
367 std::optional<TToken> Scanner::grabString() {
368 TToken token;
369 std::string tok_value;
370 auto pos = string_buffer.cur_line();
371 auto filename = string_buffer.cur_file();
372 while (true) {
373 auto ch = string_buffer.getch();
374 if (!ch.has_value()) {
375 std::ostringstream stream;
376 auto cpos = string_buffer.cur_line();
377 stream << "File: " << string_buffer.cur_file() << " Line: " << cpos.first << " Col:" << cpos.second << " Double quote not terminated.";
378 throw ScanExcept(stream.str());
379 }
380 if (*ch == '\\') {
381 auto next_ch = string_buffer.getch();
382 if (!next_ch.has_value())
383 break;
384 switch (*next_ch) {
385 case 'n':
386 tok_value += '\n';
387 break;
388 case 't':
389 tok_value += '\t';
390 break;
391 case 'r':
392 tok_value += '\r';
393 break;
394 case '\\':
395 tok_value += '\\';
396 break;
397 case '\"':
398 tok_value += "\\\"";
399 break;
400 case '\'':
401 tok_value += '\'';
402 break;
403 default:
404 tok_value += '\\';
405 tok_value += *next_ch;
406 break;
407 }
408 } else if (*ch == '\"') {
409 break;
410 } else {
411 tok_value += *ch;
412 }
413 }
414 token.set_pos(pos);
415 token.set_filename(filename);
416 token.setToken(types::TokenType::TT_STR, tok_value);
417 return token;
418 }
419
420 std::optional<TToken> Scanner::grabSingle() {
421 TToken token;
422 std::string tok_value;
423 auto filename = string_buffer.cur_file();
424 auto pos = string_buffer.cur_line();
425 while (true) {
426 auto ch = string_buffer.getch();
427 if (!ch.has_value()) {
428 std::ostringstream stream;
429 auto cpos = string_buffer.cur_line();
430 stream << "File: " << string_buffer.cur_file() << " Line: " << cpos.first << " Col:" << cpos.second << " String quote not terminated.";
431 throw ScanExcept(stream.str());
432 }
433 if (*ch == '\\') {
434 auto next_ch = string_buffer.getch();
435 if (!next_ch.has_value())
436 break;
437 switch (*next_ch) {
438 case 'n':
439 tok_value += '\n';
440 break;
441 case 't':
442 tok_value += '\t';
443 break;
444 case 'r':
445 tok_value += '\r';
446 break;
447 case '\\':
448 tok_value += '\\';
449 break;
450 case '\"':
451 tok_value += '\"';
452 break;
453 case '\'':
454 tok_value += '\'';
455 break;
456 default:
457 tok_value += '\\';
458 tok_value += *next_ch;
459 break;
460 }
461 } else if (*ch == '\'') {
462 // Pascal doubled-quote convention: '' becomes a literal '
463 if (string_buffer.peekch(0).has_value() && *string_buffer.peekch(0) == '\'') {
464 string_buffer.getch();
465 tok_value += '\'';
466 } else {
467 break;
468 }
469 } else {
470 tok_value += *ch;
471 }
472 }
473
474 token.set_pos(pos);
475 token.set_filename(filename);
476 token.setToken(types::TokenType::TT_STR, tok_value);
477 return token;
478 }
479
480 TToken &Scanner::operator[](size_t index) {
481 if (index < size())
482 return tokens[index];
483 std::ostringstream stream;
484 stream << "Token index: index out of bounds, Scanner Exception: " << index << "\n";
485 throw ScanExcept(stream.str());
486 }
487 size_t Scanner::size() const { return tokens.size(); }
488} // namespace scan
scan::ScanExcept
Exception thrown during scanning errors.
Definition scanner.hpp:80
scan::Scanner::grabSingle
std::optional< TToken > grabSingle()
Try to grab a single-quoted character literal token.
Definition scanner.cpp:420
scan::Scanner::grabDigits
std::optional< TToken > grabDigits()
Try to grab a numeric literal token.
Definition scanner.cpp:211
scan::Scanner::parseLineNumber
uint64_t parseLineNumber()
Parse a line directive line number.
Definition scanner.cpp:22
scan::Scanner::size
size_t size() const
Return the total number of scanned tokens.
Definition scanner.cpp:487
scan::Scanner::operator[]
TToken & operator[](size_t index)
Access a token by index.
Definition scanner.cpp:480
scan::Scanner::grabId
std::optional< TToken > grabId()
Try to grab an identifier or keyword token.
Definition scanner.cpp:185
scan::Scanner::parseFileName
std::string parseFileName()
Parse a line directive filename.
Definition scanner.cpp:44
scan::Scanner::string_buffer
TString string_buffer
source text buffer
Definition scanner.hpp:65
scan::Scanner::grabString
std::optional< TToken > grabString()
Try to grab a double-quoted string literal token.
Definition scanner.cpp:367
scan::Scanner::tokens
std::vector< TToken > tokens
scanned token list
Definition scanner.hpp:67
scan::Scanner::Scanner
Scanner(const TString &b)
Construct a scanner from a string buffer.
Definition scanner.cpp:12
scan::Scanner::scan
uint64_t scan()
Tokenize the entire source buffer.
Definition scanner.cpp:62
scan::Scanner::removeEOL
void removeEOL()
Remove end-of-line tokens from the token list.
Definition scanner.cpp:14
scan::Scanner::is_c_sym
bool is_c_sym(const StringType &str)
Check if a string is a recognized C-style symbol.
Definition scanner.cpp:283
scan::Scanner::grabSymbols
std::optional< TToken > grabSymbols()
Try to grab one or more symbol characters as a token.
Definition scanner.cpp:292
scan::Scanner::token_map
TMap token_map
character classification map
Definition scanner.hpp:66
scan::token::Token::set_filename
void set_filename(const std::string &filename)
Set the source filename for this token.
Definition string_buffer.hpp:310
scan::token::Token::set_pos
void set_pos(const std::pair< uint64_t, uint64_t > &p)
Set the source position from a (line, column) pair.
Definition string_buffer.hpp:355
scan::token::Token::setToken
void setToken(const types::TokenType &type, const String &value)
Set the token's type and value.
Definition string_buffer.hpp:326
pos
int64_t pos(const char *substr, const char *s)
Definition cstring.c:51
scan::StringType
buffer::StringBuffer< char >::string_type StringType
Underlying string type.
Definition scanner.hpp:18
scan::TToken
token::Token< char > TToken
Default token type.
Definition scanner.hpp:19
scan::TString
buffer::StringBuffer< char > TString
Default character string buffer.
Definition scanner.hpp:17
types::TokenType::TT_SYM
@ TT_SYM
symbol / operator token
Definition types.hpp:19
types::TokenType::TT_HEX
@ TT_HEX
hexadecimal numeric literal
Definition types.hpp:22
types::TokenType::TT_STR
@ TT_STR
string literal
Definition types.hpp:20
types::TokenType::TT_ID
@ TT_ID
identifier or keyword
Definition types.hpp:17
types::TokenType::TT_NUM
@ TT_NUM
decimal numeric literal
Definition types.hpp:21
types::TokenType::TT_ARG
@ TT_ARG
argument token (e.g. $1)
Definition types.hpp:18
types::CharType::TT_CHAR
@ TT_CHAR
alphabetic character
Definition types.hpp:27
types::CharType::TT_SINGLE
@ TT_SINGLE
single-char operator
Definition types.hpp:31
types::CharType::TT_SYMBOL
@ TT_SYMBOL
multi-char symbol start
Definition types.hpp:29
types::CharType::TT_SPACE
@ TT_SPACE
whitespace
Definition types.hpp:32
types::CharType::TT_NULL
@ TT_NULL
unmapped / null
Definition types.hpp:33
types::CharType::TT_DIGIT
@ TT_DIGIT
digit character
Definition types.hpp:28
types::CharType::TT_STRING
@ TT_STRING
string delimiter (quote)
Definition types.hpp:30
scanner.hpp
Lexical scanner that tokenizes source text into typed tokens.