krzosa/corelang
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Using arena as token array, remove arenas idea

Browse Source
This commit is contained in:
Krzosa Karol
2023-02-09 12:36:36 +01:00
parent 5138ba0097
commit 7370e8b716
16 changed files with 295 additions and 407 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
core_lexing.cpp
View File

@@ -205,24 +205,32 @@ lex_is_scope(Token *t){
}
CORE_Static void
lex_unwind_indent_stack(Token *t, Lex_Stream *s, Array<Token> *array){
lex_add_token(Core_Ctx *ctx, Token *token) {
Token *top = (Token *)arena_push_size(ctx->stage_arena, sizeof(Token));
*top = *token;
ctx->tokens.len += 1;
ctx->tokens.data = (Token *)ctx->stage_arena->memory.data;
}
CORE_Static void
lex_unwind_indent_stack(Core_Ctx *ctx, Token *t, Lex_Stream *s){
for(S64 i = s->indent_stack.len-1; i >= 0; i-=1){
auto it = s->indent_stack.data[i];
assert(lex_is_scope(it));
if(it->indent == t->indent){
t->kind = SAME_SCOPE;
array->add(*t);
lex_add_token(ctx, t);
break;
}
else if(it->indent < t->indent){
token_error(t, "Bad indentation"_s);
array->add(*t);
lex_add_token(ctx, t);
break;
}
else{
s->indent_stack.pop();
t->kind = CLOSE_SCOPE;
array->add(*t);
lex_add_token(ctx, t);
}
}
}
@@ -230,7 +238,6 @@ lex_unwind_indent_stack(Token *t, Lex_Stream *s, Array<Token> *array){
CORE_Static void
lex__stream(Core_Ctx *lexer){
Intern_Table *table = &lexer->interns;
Array<Token> *array = &lexer->tokens;
Lex_Stream *s = &lexer->stream;
B32 beginning = true;
@@ -238,7 +245,7 @@ lex__stream(Core_Ctx *lexer){
if(lexc(s) == 0 || s->iter >= s->stream.len){
end_of_stream:
Token t = token_make(lexer);
lex_unwind_indent_stack(&t, s, array);
lex_unwind_indent_stack(lexer, &t, s);
break;
}
@@ -307,11 +314,11 @@ lex__stream(Core_Ctx *lexer){
lex_advance(s);
semi.kind = OPEN_SCOPE;
semi.indent = last->indent + 2; // @todo: proper detection of indentation
array->add(semi);
s->indent_stack.add(array->last());
lex_add_token(lexer, &semi);
s->indent_stack.add(lexer->tokens.last());
} else{
semi.kind = SAME_SCOPE;
array->add(semi);
lex_add_token(lexer, &semi);
}
} break;
@@ -327,16 +334,16 @@ lex__stream(Core_Ctx *lexer){
Token *last = lex_last_indent_token(s);
if(t.indent > last->indent){
t.kind = OPEN_SCOPE;
array->add(t);
s->indent_stack.add(array->last());
lex_add_token(lexer, &t);
s->indent_stack.add(lexer->tokens.last());
}
else if(t.indent < last->indent){
lex_unwind_indent_stack(&t, s, array);
lex_unwind_indent_stack(lexer, &t, s);
}
else {
t.kind = SAME_SCOPE;
array->add(t);
lex_add_token(lexer, &t);
}
}
@@ -579,7 +586,7 @@ lex__stream(Core_Ctx *lexer){
if(t.len==0)
lex_set_len(s,&t);
array->add(t);
lex_add_token(lexer, &t);
}
#undef CASE2
#undef CASE3