krzosa/corelang
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix lexer end of file not unwinding scopes, Trying to run a program

Browse Source
This commit is contained in:
Krzosa Karol
2022-06-01 14:00:42 +02:00
parent 02b6a1c85b
commit 82bb3ae493
4 changed files with 58 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
new_lex.cpp
View File

@@ -300,12 +300,38 @@ lex_is_scope(Token *t){
return result;
}
function void
lex_unwind_indent_stack(Token *t, Lex_Stream *s, Array<Token> *array){
for(S64 i = s->indent_stack.len-1; i >= 0; i-=1){
auto it = s->indent_stack.data[i];
assert(lex_is_scope(it));
if(it->indent == t->indent){
t->kind = SAME_SCOPE;
array->add(*t);
break;
}
else if(it->indent < t->indent){
token_error(t, "Bad indentation"_s);
array->add(*t);
break;
}
else{
s->indent_stack.pop();
t->kind = CLOSE_SCOPE;
array->add(*t);
}
}
}
function void
lex__stream(Intern_Table *table, Array<Token> *array, Lex_Stream *s){
B32 beginning = true;
while(lexc(s)){
if(s->iter >= s->stream.len) // End of stream
for(;;){
if(lexc(s) == 0 || s->iter >= s->stream.len){
Token t = token_make(lexcp(s), s->file, s->line, s->line_begin);
lex_unwind_indent_stack(&t, s, array);
break;
}
// @note: the lexer is going to be a 2 stage process
// first we tokenize the indentation and then proceed to tokenize
@@ -388,7 +414,6 @@ lex__stream(Intern_Table *table, Array<Token> *array, Lex_Stream *s){
// s->inside_brace_paren--;
// t.kind = CLOSE_SCOPE;
// } break;
default:{
if(s->inside_brace_paren) should_emit = false;
if(should_emit){
@@ -400,25 +425,7 @@ lex__stream(Intern_Table *table, Array<Token> *array, Lex_Stream *s){
}
else if(t.indent < last->indent){
for(S64 i = s->indent_stack.len-1; i >= 0; i-=1){
auto it = s->indent_stack.data[i];
assert(lex_is_scope(it));
if(it->indent == t.indent){
t.kind = SAME_SCOPE;
array->add(t);
break;
}
else if(it->indent < t.indent){
token_error(&t, "Bad indentation"_s);
array->add(t);
break;
}
else{
s->indent_stack.pop();
t.kind = CLOSE_SCOPE;
array->add(t);
}
}
lex_unwind_indent_stack(&t, s, array);
}
else {
t.kind = SAME_SCOPE;