From c88b38cc44a561aa0d3f1f2fa1e1bef478ec3912 Mon Sep 17 00:00:00 2001 From: Krzosa Karol Date: Thu, 26 May 2022 14:49:50 +0200 Subject: [PATCH] Successful compile with new parser! --- lambdas.kl | 2 +- new_lex.cpp | 66 +++++++++++++++++++++++++-------------------------- new_parse.cpp | 55 ++++++++++-------------------------------- 3 files changed, 46 insertions(+), 77 deletions(-) diff --git a/lambdas.kl b/lambdas.kl index cdfcc47..a8a2e45 100644 --- a/lambdas.kl +++ b/lambdas.kl @@ -1,5 +1,5 @@ /* -/*begin of file*/ thing // indent 2 == error + thing // indent 2 == error add_10 :: (size: int): int // scope 0 diff --git a/new_lex.cpp b/new_lex.cpp index 92d1ff6..caf234f 100644 --- a/new_lex.cpp +++ b/new_lex.cpp @@ -326,6 +326,31 @@ lex__stream(Intern_Table *table, Array *array, Lex_Stream *s){ switch(lexc(s)){ case '\t': case ' ': lex_advance(s); t.indent++; break; case '\r': lex_advance(s); break; + case '/': { + if(lexci(s,1) == '/'){ + lex_advance(s); lex_advance(s); + t.kind = TK_Comment; + for(;;){ + if(lexc(s) == '\n' || lexc(s) == 0) break; + lex_advance(s); + } + } + else if(lexci(s,1) == '*'){ + lex_advance(s); lex_advance(s); + t.kind = TK_Comment; + for(;;){ + if(lexc(s) == '*' && lexci(s,1) == '/'){ + lex_advance(s); lex_advance(s); + break; + } + else if(lexc(s) == 0){ + token_error(&t, "Unterminated block comment"_s); + break; + } + lex_advance(s); + } + } + } break; case '\n':{ lex_advance(s); should_emit = true; @@ -341,14 +366,15 @@ lex__stream(Intern_Table *table, Array *array, Lex_Stream *s){ s->indent_stack.add(array->last()); } else if(t.indent < last->indent){ - For_Reverse(s->indent_stack){ - assert(token_is_scope(*it)); - if(it[0]->indent == t.indent){ + for(S64 i = s->indent_stack.len-1; i >= 0; i-=1){ + auto it = s->indent_stack.data[i]; + assert(token_is_scope(it)); + if(it->indent == t.indent){ t.kind = SAME_SCOPE; array->add(t); break; } - else if(it[0]->indent < t.indent){ + else if(it->indent < t.indent){ token_error(&t, "Bad indentation"_s); array->add(t); break; @@ -362,7 +388,7 @@ lex__stream(Intern_Table *table, Array *array, Lex_Stream *s){ } else { t.kind = SAME_SCOPE; - array->add(t); // else SAME_SCOPE + array->add(t); } } @@ -372,10 +398,9 @@ lex__stream(Intern_Table *table, Array *array, Lex_Stream *s){ } indent_loop_break: beginning = false; + // @note: handle the indented token t = token_make(lexcp(s), s->file, s->line, s->line_begin); lex_advance(s); - - // @note: handle the indented token switch(*t.str){ case 0 : break; case '@': t.kind = TK_At; break; @@ -497,33 +522,6 @@ lex__stream(Intern_Table *table, Array *array, Lex_Stream *s){ t.kind = TK_DivAssign; lex_advance(s); } - else if(lexc(s) == '/'){ - lex_advance(s); - t.kind = TK_Comment; - for(;;){ - if(lexc(s) == '\n' || lexc(s) == 0) break; - lex_advance(s); - } - continue; - } - else if(lexc(s) == '*'){ - lex_advance(s); - t.kind = TK_Comment; - for(;;){ - if(lexc(s) == '*' && lexci(s,1) == '/'){ - lex_advance(s); - lex_advance(s); - break; - } - else if(lexc(s) == 0){ - token_error(&t, "Unterminated block comment"_s); - goto skip_continue; - } - lex_advance(s); - } - continue; - skip_continue:; - } else { t.kind = TK_Div; } diff --git a/new_parse.cpp b/new_parse.cpp index efaced9..de8506d 100644 --- a/new_parse.cpp +++ b/new_parse.cpp @@ -38,7 +38,7 @@ token_get(S64 i = 0){ function Token * token_next(){ Token *token = token_get(); - if(token->kind == TK_NewLine) pctx->indent = token->indent; + if(token_is_scope(token)) pctx->indent = token->indent; pctx->token_iter++; return token; } @@ -81,42 +81,6 @@ token_expect(Token_Kind kind){ return 0; } -// @note: right now we check if on downscope there is a end of file -// not sure if this is the right approach codewise but the fact -// that end of file is treated as end of scope feels intuitive -function Token * -token_is_scope(Token_Kind scope){ - assert(scope == OPEN_SCOPE || scope == CLOSE_SCOPE || scope == SAME_SCOPE); - Token *token = token_get(); - if(token->kind == TK_NewLine || token->kind == TK_End){ - if (scope == OPEN_SCOPE && token->indent > pctx->indent) return token; - else if(scope == SAME_SCOPE && token->indent == pctx->indent) return token; - else if((scope == CLOSE_SCOPE) && ((token->indent < pctx->indent) || (token->kind == TK_End))) return token; - } - return 0; -} - -function Token * -token_match_scope(Token_Kind scope){ - Token *token = token_is_scope(scope); - if(token) return token_next(); - return 0; -} - -function Token * -token_expect_scope(Token_Kind scope){ - assert(scope == OPEN_SCOPE || scope == CLOSE_SCOPE || scope == SAME_SCOPE); - Token *token = token_get(); - if(token->kind == TK_NewLine || token->kind == TK_End){ - if (scope == OPEN_SCOPE && token->indent > pctx->indent) return token_next(); - else if(scope == SAME_SCOPE && token->indent == pctx->indent) return token_next(); - else if((scope == CLOSE_SCOPE) && ((token->indent < pctx->indent) || (token->kind == TK_End))) return token_next(); - else parsing_error(token, "Expected a scope of kind [%s]", token_kind_string(scope)); - } - parsing_error(token, "Expected Scope[%s] got instead: [%s]", token_kind_string(scope).str, token_kind_string(token->kind).str); - return 0; -} - //----------------------------------------------------------------------------- // Expression parsing //----------------------------------------------------------------------------- @@ -199,7 +163,7 @@ function Ast_Decl *parse_decl(B32); function Ast_Block * parse_block(){ Ast_Block *block = 0; - if(token_match_scope(OPEN_SCOPE)){ + if(token_match(OPEN_SCOPE)){ Token *token_block = token_get(); Scratch scratch; @@ -208,7 +172,7 @@ parse_block(){ Token *token = token_get(); if(token_match_keyword(keyword_return)){ AST_NEW(Return, AST_RETURN, token); - if(!token_is(TK_NewLine)) result->expr = parse_expr(); + if(!token_is_scope(token_get())) result->expr = parse_expr(); stmts.add(result); } else{ @@ -216,8 +180,8 @@ parse_block(){ if(result) stmts.add(result); else parsing_error(token, "Unexpected token while parsing statement"); } - } while(token_match_scope(SAME_SCOPE)); - token_expect_scope(CLOSE_SCOPE); + } while(token_match(SAME_SCOPE)); + token_expect(CLOSE_SCOPE); block = ast_block(token_block, stmts); } return block; @@ -391,6 +355,7 @@ parse_assign_expr(){ function Ast_Decl * parse_decl(B32 is_global){ Ast_Decl *result = 0; + if(is_global) token_match(SAME_SCOPE); if(token_is(TK_Identifier)){ if(is_global && pctx->indent != 0) parsing_error(token_get(), "Top level declarations shouldn't be indented"); Token *name = token_next(); @@ -424,10 +389,16 @@ parse_decl(B32 is_global){ function Ast_Package * parse_file(){ Scratch scratch; + + // + // @note: pop the first token which which always should be an indentation token, + // it updates the indent info on the parser, making sure that indentation on + // the first line is properly updated + // Token *token = token_get(); Arraydecls = {scratch}; while(!token_is(TK_End)){ - while(token_match(TK_NewLine)); + token_expect(SAME_SCOPE); Ast_Decl *decl = parse_decl(true); if(!decl) break; decls.add(decl);