diff --git a/ast.c b/ast.c index 26bb434..a63ccb9 100644 --- a/ast.c +++ b/ast.c @@ -353,7 +353,7 @@ expr_new(Arena *p, Expr_Kind kind, Token *token){ function Expr * expr_int(Arena *p, Token *token){ - assert(token->kind == TK_Int); + assert(token->kind == TK_Integer); Expr *expr = expr_new(p, EK_Int, token); expr->int_val = token->int_val; return expr; diff --git a/lex.c b/lex.c index cb334aa..9562807 100644 --- a/lex.c +++ b/lex.c @@ -140,7 +140,7 @@ typedef enum Token_Kind{ TK_Character, TK_Error, TK_Float, - TK_Int, + TK_Integer, TK_Keyword, }Token_Kind; @@ -467,7 +467,7 @@ lex__stream(Token_Array *array, Lex_Stream *s){ case '0':case '1':case '2':case '3':case '4': case '5':case '6':case '7':case '8':case '9':{ - t.kind = TK_Int; + t.kind = TK_Integer; while(lex_is_numeric(lexc(s))) lex_advance(s); lex_set_len(s, &t); @@ -544,7 +544,7 @@ lex_test(){ TK_At,TK_Question,TK_BitAnd,TK_Add,TK_Sub,TK_Semicolon, TK_ThreeDots, TK_Dot, TK_Arrow, TK_Comma, TK_DoubleColon, TK_Colon, TK_StringLit, TK_Identifier, TK_StringLit, TK_AddAssign, TK_SubAssign, - TK_Equals, TK_Int, TK_Int, TK_Int, TK_Keyword, TK_Keyword, + TK_Equals, TK_Integer, TK_Integer, TK_Integer, TK_Keyword, TK_Keyword, TK_Keyword, TK_Keyword, TK_ColonAssign, TK_End }; String strs[] = { @@ -564,7 +564,7 @@ lex_test(){ for(Token *t = token_array_iter_begin(&array); t->kind != TK_End; t = token_array_iter_next(&array)){ assert(t->kind == kind[i]); assert(string_compare(t->string, strs[i++])); - if(t->kind == TK_Int){ + if(t->kind == TK_Integer){ assert(t->int_val == vals[ui++]); } } @@ -637,6 +637,6 @@ global const char *token_kind_string[] = { [TK_Character] = "Character", [TK_Error] = "Error", [TK_Float] = "Float", - [TK_Int] = "Int", + [TK_Integer] = "Int", [TK_Keyword] = "Keyword", }; diff --git a/main.cpp b/main.cpp index e0e766a..e77fd3d 100644 --- a/main.cpp +++ b/main.cpp @@ -154,7 +154,7 @@ wrap_around_pow2(U64 x, U64 power_of_2) { //----------------------------------------------------------------------------- // OS Memory //----------------------------------------------------------------------------- -constexpr SizeU os_page_size = 4096; +const SizeU os_page_size = 4096; struct OS_Memory{ SizeU commit, reserve; U8 *data; @@ -395,6 +395,7 @@ struct Thread_Ctx{ Arena scratch[2]; Allocator *implicit_allocator; void *ctx; + U64 ctx_id; Log_Proc *log_proc; }; thread_local Thread_Ctx thread_ctx; @@ -433,15 +434,26 @@ struct Scoped_Allocator{ } }; -#define Get_Ctx(T) T *ctx = (T *)thread_ctx.ctx -#define Set_Ctx(ctx) Scoped_Ctx scoped_ctx_##__LINE__((void *)ctx) +function void * +thread_ctx_get_user_ctx(U64 id){ + assert(id == thread_ctx.ctx_id); + assert(id != 0); + assert(thread_ctx.ctx_id != 0); + assert(thread_ctx.ctx != 0); + return thread_ctx.ctx; +} +#define Get_Ctx(T) T *ctx = (T *)thread_ctx_get_user_ctx(T##_ID) +#define Set_Ctx(ctx, id) Scoped_Ctx scoped_ctx_##__LINE__((void *)ctx, id) struct Scoped_Ctx{ void *prev_ctx; - Scoped_Ctx(void *in_ctx){ + U64 prev_id; + Scoped_Ctx(void *in_ctx, U64 id){ prev_ctx = thread_ctx.ctx; + prev_id = thread_ctx.ctx_id; thread_ctx.ctx = in_ctx; + thread_ctx.ctx_id = id; } - ~Scoped_Ctx(){thread_ctx.ctx = prev_ctx;} + ~Scoped_Ctx(){thread_ctx.ctx = prev_ctx; thread_ctx.ctx_id = prev_id;} }; enum Alloc_Flag{AF_None,AF_ZeroMemory}; @@ -579,6 +591,7 @@ test_heap_allocator(){ assert(thread_ctx.implicit_allocator == &heap); } +const U64 Test_Context_ID = 14242; struct Test_Context{ int value; }; @@ -592,7 +605,7 @@ test_custom_context_2(){ function void test_custom_context_1(){ Test_Context context = {}; - Set_Ctx(&context); + Set_Ctx(&context, Test_Context_ID); Get_Ctx(Test_Context); ctx->value = 10; test_custom_context_2(); @@ -640,78 +653,60 @@ struct Array{ S64 len; Allocator *allocator; + void init(S64 size){ + if(!allocator) allocator = imp_get(); + data = exp_alloc_array(allocator, T, size); + cap = size; + } + + void grow(S64 required_size){ + if(cap == 0){ + S64 cap = max(required_size*2, (S64)16); + init(cap); + } + else if(len + required_size > cap){ + S64 cap = (len + required_size)*2; + data = exp_resize_array(allocator, data, T, cap); + cap = cap; + } + } + + void add(T item){ + grow(1); + data[len++] = item; + } + + void add(T *item){ + grow(1); + data[len++] = *item; + } + + void clear(){ + len = 0; + } + T *begin(){ return data; } T *end (){ return data + len; } T &operator[](S64 i){ return data[i]; } }; #define For(array,it,i) for(SizeU i = 0; i < array.len; i++) for(auto *it = &array[i]; it; it = 0) +#define IFor(array) for(auto *it = array.begin(); it != array.end(); it++) #define IterList(list,it) for(auto *it = list->first; it; it=it->next) -template -void array_init(Array *a, S64 size){ - if(!a->allocator) a->allocator = thread_ctx.implicit_allocator; - a->data = exp_alloc_array(a->allocator, T, size); - a->cap = size; -} - -template -void array_grow(Array *a, S64 required_size){ - if(a->cap == 0){ - S64 cap = max(required_size*2, (S64)16); - array_init(a, cap); - } - else if(a->len + required_size > a->cap){ - S64 cap = (a->len + required_size)*2; - a->data = exp_resize_array(a->allocator, a->data, T, cap); - a->cap = cap; - } -} - template Array array_make(S64 size){ Array result = {}; - array_init(&result, size); + result.init(size); return result; } -template -T *array_alloc(Array *a, S64 count){ - array_grow(a, count); - T *result = a->data + a->len; - a->len += count; - return result; -} - -template -void array_push(Array *a, T &item){ - array_grow(a, 1); - a->data[a->len++] = item; -} - -template -T array_pop_get(Array *a){ - assert(a->len > 0); - return a->data[--a->len]; -} - -template -void array_pop(Array *a){ - assert(a->len > 0); - --a->len; -} - -template -void array_clear(Array *array){ - array->len = 0; -} - function void test_array(){ Set_Scratch(); Array array = {}; int size = 1000; for(int i = 0; i < size; i++){ - array_push(&array, i); + array.add(i); } For(array, it, i){ assert(*it == i); @@ -721,17 +716,11 @@ test_array(){ Array array2 = {}; array2.allocator = &arena; for(int i = 0; i < size; i++){ - array_push(&array2, i); + array2.add(i); } For(array2, iterator, count){ assert(*iterator == count); } - for(int i = 999; i > 950; i--){ - assert(array_pop_get(&array) == i); - } - for(int i = 0; i < 10; i++){ - array_pop(&array2); - } exp_destroy(&arena); assert(arena.memory.data == 0); assert(thread_ctx.scratch->memory.data != 0); @@ -775,6 +764,13 @@ map_grow(Map *map, S64 new_size){ *map = new_map; } +function Map +map_make(S64 size){ + Map result = {}; + map_grow(&result, size); + return result; +} + function void map_insert_u64(Map *map, U64 key, void *val){ assert(val); @@ -849,30 +845,6 @@ map_test(){ } } -//----------------------------------------------------------------------------- -// Bucket Array -//----------------------------------------------------------------------------- -template -struct Bucket_Array{ - struct Bucket{ - Bucket *next; - S64 len, cap; - T data[0]; - }; - - Allocator *allocator; - Bucket *first; - Bucket *last; - - Bucket *iter; - S64 iter_len; -}; - -function void -test_bucket_array(){ - Bucket_Array arr = {}; -} - //----------------------------------------------------------------------------- // Linked lists //----------------------------------------------------------------------------- @@ -1027,8 +999,18 @@ test_string_builder(){ struct Intern_Table{ Allocator *string_allocator; Map map; + U8 *first_keyword; + U8 *last_keyword; }; +function Intern_Table +intern_table_make(S64 initial_size){ + Intern_Table result = {}; + result.map = map_make(initial_size); + result.string_allocator = imp_get(); + return result; +} + function Intern_String intern_string(Intern_Table *t, String string){ if(!t->string_allocator) t->string_allocator = imp_get(); @@ -1063,11 +1045,18 @@ test_intern_table(){ Set_Scratch(); } #include "new_lex.cpp" +#include "new_ast.cpp" +#include "new_parse.cpp" int main(){ test_custom_context(); test_heap_allocator(); test_os_memory(); + thread_ctx_init(); + + test_parse_decl(); + test_parse_expr(); + test_array(); map_test(); test_string_builder(); diff --git a/new_ast.cpp b/new_ast.cpp new file mode 100644 index 0000000..21cf8de --- /dev/null +++ b/new_ast.cpp @@ -0,0 +1,186 @@ + +// Lexer::interns::map::allocator - array allocator, resizing +// Lexer::tokens - array allocator, resizing +// +// Parser::ast_arena - arena for asts +// Lexer::interns::string_allocator - arena for interns +// +Intern_String keyword_const; +Intern_String keyword_struct; +Intern_String keyword_union; +Intern_String keyword_enum; + +const U64 Parse_Ctx_ID = 115151; +struct Parse_Ctx:Lexer{ + Arena ast_arena; + Token empty_token; + S64 pt[256]; // precedence table + + void init(){ + const S64 addp = 1; + const S64 mulp = 2; + pt[TK_Add] = addp; + pt[TK_Sub] = addp; + pt[TK_Div] = mulp; + pt[TK_Mul] = mulp; + + arena_init(&ast_arena); + lex_init(this); + keyword_const = intern_string(&interns, "const"_s); + keyword_struct= intern_string(&interns, "struct"_s); + keyword_union = intern_string(&interns, "union"_s); + keyword_enum = intern_string(&interns, "enum"_s); + interns.first_keyword = keyword_const.str; + interns.last_keyword = keyword_enum.str; + } +}; + +//----------------------------------------------------------------------------- +// AST +//----------------------------------------------------------------------------- +enum Ast_Kind{ + AK_None, + + AK_Expr_Str, + AK_Expr_Int, + AK_Expr_Ident, + AK_Expr_Binary, + + AK_Decl_Func, + AK_Decl_Func_Arg, + AK_Decl_Const, + + AK_Typespec_Ident, + AK_Typespec_Pointer, + AK_Typespec_Array, + AK_Typespec_Func +}; + +struct Ast{ + Ast_Kind kind; + Token *pos; +}; + +struct Ast_Expr:Ast{ + union{ + Intern_String intern_val; + U64 int_val; + struct{ + Token_Kind op; + Ast_Expr *left; + Ast_Expr *right; + } binary; + }; +}; + +struct Ast_Typespec:Ast{ + union{ + Ast_Typespec *base; + Intern_String name; + struct{ + Ast_Typespec *base; + Ast_Expr *expr; + }arr; + struct{ + Ast_Typespec *ret; + Array args; + }func; + }; +}; + +struct Ast_Decl:Ast{ + Intern_String name; + union{ + struct{ + Ast_Typespec *typespec; + Intern_String name; + Ast_Expr *expr; + }var; + struct{ + Array args; + Ast_Typespec *ret; + }func; + }; +}; + +//----------------------------------------------------------------------------- +// AST Constructors beginning with expressions +//----------------------------------------------------------------------------- +#define AST_NEW(T,ikind,ipos) \ +Get_Ctx(Parse_Ctx); \ +Ast_##T *result = exp_alloc_type(&ctx->ast_arena, Ast_##T); \ +result->kind = ikind; \ +result->pos = ipos + +function Ast_Expr * +ast_expr_string(Token *pos, Intern_String string){ + AST_NEW(Expr, AK_Expr_Str, pos); + result->intern_val = string; + return result; +} + +function Ast_Expr * +ast_expr_identifier(Token *pos, Intern_String string){ + AST_NEW(Expr, AK_Expr_Ident, pos); + result->intern_val = string; + return result; +} + +function Ast_Expr * +ast_expr_integer(Token *pos, S64 integer){ + AST_NEW(Expr, AK_Expr_Int, pos); + result->int_val = integer; + return result; +} + +function Ast_Expr * +ast_expr_binary(Ast_Expr *left, Ast_Expr *right, Token *op){ + AST_NEW(Expr, AK_Expr_Binary, op); + result->binary.op = op->kind; + result->binary.left = left; + result->binary.right = right; + return result; +} + +//----------------------------------------------------------------------------- +// Typespecs +//----------------------------------------------------------------------------- +function Ast_Typespec * +ast_typespec_name(Token *pos, Intern_String name){ + AST_NEW(Typespec, AK_Typespec_Ident, pos); + result->name = name; + return result; +} + +function Ast_Typespec * +ast_typespec_pointer(Token *pos, Ast_Typespec *base){ + AST_NEW(Typespec, AK_Typespec_Pointer, pos); + result->base = base; + return result; +} + +function Ast_Typespec * +ast_typespec_array(Token *pos, Ast_Typespec *base, Ast_Expr *expr){ + AST_NEW(Typespec, AK_Typespec_Array, pos); + result->arr.base = base; + result->arr.expr = expr; + return result; +} + +//----------------------------------------------------------------------------- +// Declarations +//----------------------------------------------------------------------------- +function Ast_Decl * +ast_decl_func(Token *pos, Intern_String name){ + AST_NEW(Decl, AK_Decl_Func, pos); + result->name = name; + return result; +} + +function Ast_Decl * +ast_decl_const(Token *pos, Intern_String name, Ast_Expr *expr){ + AST_NEW(Decl, AK_Decl_Const, pos); + result->var.expr = expr; + result->name = name; + return result; +} diff --git a/new_lex.cpp b/new_lex.cpp index 7c7dac2..a71e976 100644 --- a/new_lex.cpp +++ b/new_lex.cpp @@ -81,7 +81,7 @@ enum Token_Kind{ TK_Character, TK_Error, TK_Float, - TK_Int, + TK_Integer, TK_Keyword, }; @@ -114,6 +114,13 @@ struct Lex_Stream{ S32 line; }; +struct Lexer{ + Lex_Stream stream; + Array tokens; + Intern_Table interns; + S64 token_iter; +}; + function U8 lexc(Lex_Stream *s){ return s->stream.str[s->iter]; @@ -159,6 +166,23 @@ lex_set_len(Lex_Stream *s, Token *token){ token->len = lexcp(s) - token->str; } +function void +lex_set_keywords(Lexer *lexer, Array keywords){ + Intern_String keyword = {}; + IFor(keywords){ + keyword = intern_string(&lexer->interns, *it); + if(it == keywords.begin()) + lexer->interns.first_keyword = keyword.str; + } + lexer->interns.last_keyword = keyword.str; +} + +function B32 +lex_is_keyword(Intern_Table *lexer, Intern_String keyword){ + B32 result = keyword.str >= lexer->first_keyword && keyword.str <= lexer->last_keyword; + return result; +} + function void token_error(Token *t, String error_val){ t->kind = TK_Error; @@ -237,7 +261,7 @@ t.kind = OpName; break function void -lex__stream(Array *array, Lex_Stream *s){ +lex__stream(Intern_Table *table, Array *array, Lex_Stream *s){ while(lexc(s)){ while(lex_is_whitespace(lexc(s))) lex_advance(s); @@ -261,7 +285,6 @@ lex__stream(Array *array, Lex_Stream *s){ case ',': t.kind = TK_Comma; break; case '~': t.kind = TK_Neg; break; case '?': t.kind = TK_Question; break; - case ';': t.kind = TK_Semicolon; break; case '#': t.kind = TK_Pound; break; CASE2('!', TK_Not, TK_NotEquals); CASE2('^', TK_BitXor, TK_XorAssign); @@ -273,6 +296,10 @@ lex__stream(Array *array, Lex_Stream *s){ CASE3('|', TK_BitOr, TK_OrAssign, TK_Or); #undef CASE2 #undef CASE3 + case ';': { + t.kind = TK_Semicolon; + }break; + case '\n': { t.kind = TK_NewLine; if(lexc(s) == '\r') @@ -378,7 +405,7 @@ lex__stream(Array *array, Lex_Stream *s){ t.str += 1; t.len -= 2; } - //t.intern_val = intern_string(&array->interns, t.string); + t.intern_val = intern_string(table, t.string); } break; case '/': { @@ -420,7 +447,7 @@ lex__stream(Array *array, Lex_Stream *s){ case '0':case '1':case '2':case '3':case '4': case '5':case '6':case '7':case '8':case '9':{ - t.kind = TK_Int; + t.kind = TK_Integer; while(lex_is_numeric(lexc(s))) lex_advance(s); lex_set_len(s, &t); @@ -440,10 +467,10 @@ lex__stream(Array *array, Lex_Stream *s){ while(lex_is_alphanumeric(lexc(s)) || lexc(s) == '_') lex_advance(s); lex_set_len(s,&t); - //t.intern_val = intern_string(&array->interns, t.string); - //if(lex_is_keyword(t.intern_val)){ - //t.kind = TK_Keyword; - //} + t.intern_val = intern_string(table, t.string); + if(lex_is_keyword(table, t.intern_val)){ + t.kind = TK_Keyword; + } } break; default: { @@ -454,38 +481,67 @@ lex__stream(Array *array, Lex_Stream *s){ if(t.len==0) lex_set_len(s,&t); - array_push(array, t); + array->add(t); } } -function Array -lex_stream(String istream, String file){ - Lex_Stream stream = {istream, 0, istream.str, file, 0}; - Array tokens = array_make(1024); - lex__stream(&tokens, &stream); - return tokens; +function void +lex_init(Lexer *l){ + l->tokens = array_make(1024*2); + l->interns= intern_table_make(1024); +} + +function Lexer +lex_make(){ + Lexer result = {}; + lex_init(&result); + return result; } function void -lex_test(){ - Set_Scratch(); - String test = "//R\n 18446744073709551616{})(@?&+-;....->,:::/**/\"Thing\" Thingy" +lex_restream(Lexer *lexer, String istream, String file){ + lexer->stream = {istream, 0, istream.str, file, 0}; + lexer->tokens.clear(); + lexer->token_iter = 0; + lex__stream(&lexer->interns, &lexer->tokens, &lexer->stream); +} + +function Lexer +lex_stream(String istream, String file){ + Lexer result = lex_make(); + lex_restream(&result, istream, file); + return result; +} + +function void +lex_test(){ Set_Scratch(); + String test = "Keyword //R\n 18446744073709551616{})(@?&+-;....->,:::/**/\"Thing\" Thingy" "\"Test_Meme\"+=-===42524 4294967295 18446744073709551615" "for if while switch :="_s; - Array array = lex_stream(test, "Test1"_s); + + Array keywords = {}; + keywords.add("Keyword"_s); + keywords.add("for"_s); + keywords.add("if"_s); + keywords.add("while"_s); + keywords.add("switch"_s); + + Lexer lexer = lex_make(); + lex_set_keywords(&lexer, keywords); + lex_restream(&lexer, test, "Test1"_s); + Array array = lexer.tokens; Token_Kind kind[] = { - TK_NewLine, TK_Error,TK_OpenBrace,TK_CloseBrace,TK_CloseParen,TK_OpenParen, + TK_Keyword, TK_NewLine, TK_Error,TK_OpenBrace,TK_CloseBrace,TK_CloseParen,TK_OpenParen, TK_At,TK_Question,TK_BitAnd,TK_Add,TK_Sub,TK_Semicolon, TK_ThreeDots, TK_Dot, TK_Arrow, TK_Comma, TK_DoubleColon, TK_Colon, TK_StringLit, TK_Identifier, TK_StringLit, TK_AddAssign, TK_SubAssign, - TK_Equals, TK_Int, TK_Int, TK_Int, - TK_Identifier, TK_Identifier, TK_Identifier, TK_Identifier, - // TK_Keyword, TK_Keyword, TK_Keyword, TK_Keyword, + TK_Equals, TK_Integer, TK_Integer, TK_Integer, + TK_Keyword, TK_Keyword, TK_Keyword, TK_Keyword, TK_ColonAssign, TK_End }; String strs[] = { - "\n "_s, "18446744073709551616"_s,"{"_s,"}"_s,")"_s,"("_s, + "Keyword"_s, "\n "_s, "18446744073709551616"_s,"{"_s,"}"_s,")"_s,"("_s, "@"_s,"?"_s,"&"_s,"+"_s,"-"_s,";"_s, "..."_s,"."_s,"->"_s,","_s,"::"_s,":"_s, "Thing"_s,"Thingy"_s,"Test_Meme"_s, "+="_s,"-="_s, @@ -500,11 +556,10 @@ lex_test(){ For(array, t, i){ assert(t->kind == kind[i]); assert(string_compare(t->string, strs[i])); - if(t->kind == TK_Int){ + if(t->kind == TK_Integer){ assert(t->int_val == vals[ui++]); } } - } //----------------------------------------------------------------------------- @@ -574,7 +629,7 @@ token_kind_string(Token_Kind kind){ case TK_Character: return "Character"_s; case TK_Error: return "Error"_s; case TK_Float: return "Float"_s; - case TK_Int: return "Int"_s; + case TK_Integer: return "Int"_s; case TK_Keyword: return "Keyword"_s; default: invalid_codepath; return ""_s; } diff --git a/new_parse.cpp b/new_parse.cpp new file mode 100644 index 0000000..8a7b517 --- /dev/null +++ b/new_parse.cpp @@ -0,0 +1,231 @@ + +function Token * +token_get(S64 i = 0){ Get_Ctx(Parse_Ctx); + i += ctx->token_iter; + if(i >= ctx->tokens.len){ + return &ctx->empty_token; + } + + Token *result = &ctx->tokens[i]; + return result; +} + +function Token * +token_next(){ Get_Ctx(Parse_Ctx); + Token *token = token_get(); + ctx->token_iter++; + return token; +} + +function Token * +token_is(Token_Kind kind){ + Token *token = token_get(); + if(token->kind == kind){ + return token; + } + return 0; +} + +function Token * +token_match(Token_Kind kind){ + Token *token = token_get(); + if(token->kind == kind){ + return token_next(); + } + return 0; +} + +function Token * +token_match_keyword(Intern_String string){ + Token *token = token_get(); + if(token->kind == TK_Keyword){ + if(string.str == token->intern_val.str){ + token = token_next(); + return token; + } + } + return 0; +} + +function void +parsing_error(Token *token, const char *str, ...){ + Set_Scratch(); + STRING_FMT(imp_get(), str, string); + + // @Note(Krzosa): Print nice error message + printf("\nError: %s", string.str); + if(token){ + printf(" %s:%d\n", token->file.str, (S32)token->line); + + // @Note(Krzosa): Print error line + { + int i = 0; + while(token->line_begin[i]!='\n' && token->line_begin[i]!=0) i++; + printf("%.*s\n", i, token->line_begin); + + // @Note(Krzosa): Print error marker + int token_i = token->str - token->line_begin; + for(int i = 0; i < token_i-2; i++) printf(" "); + printf("^^^^^^\n"); + } + } + + __debugbreak(); +} + +function Token * +token_expect(Token_Kind kind){ + Token *token = token_get(); + if(token->kind == kind){ + token = token_next(); + return token; + } + parsing_error(token, "Expected token of kind: [%s], got instead token of kind: [%s]", token_kind_string(kind).str, token_kind_string(token->kind).str); + return 0; +} + +//----------------------------------------------------------------------------- +// Expression parsing +//----------------------------------------------------------------------------- +/* +add = [+-] +mul = [/%*] +compare = == | != | >= | > | <= | < +logical = [&|^] | && | || +unary = [&*-!~+] | ++ | -- + +atom_expr = Int +| Float +| String +| Identifier +| 'cast' '(' typespec ',' expr ')' +| 'size_type' '(' typespec ')' +| 'size_expr' '(' expr ')' +| '{' compound_expr '}' +| '(' expr ')' +| '(' ':' typespec ')' '{' compound_expr '}' +postfix_expr = atom_expr ('[' expr ']' | '.' Identifier | ++ | -- | '(' expr_list ')')* +unary_expr = unary ? unary_expr : atom_expr +mul_expr = atom_expr (mul atom_expr)* +add_expr = mul_expr (add mul_expr)* +logical_expr = add_expr (logical add_expr)* +compare_expr = logical_expr (compare logical_expr)* +ternary_expr = compare_expr ('?' ternary_expr ':' ternary_expr)? +expr = logical_expr + +Compound literals +- (:[23]*Type){} + - Type{} +- { } +*/ +function Ast_Expr *parse_expr(S64 rbp = 0); + +function Ast_Expr * +parse_expr_nud(Token *token){ + switch(token->kind){ + case TK_StringLit: return ast_expr_string(token, token->intern_val); + case TK_Identifier: return ast_expr_identifier(token, token->intern_val); + case TK_Integer: return ast_expr_integer(token, token->int_val); + case TK_OpenParen: {Ast_Expr *result = parse_expr(); token_expect(TK_CloseParen); return result;} + default: parsing_error(token, "Unexpected token of kind: [%s] in expression", token_kind_string(token->kind).str); return 0; + } +} + +function S64 +op_precedence(Token_Kind kind){ Get_Ctx(Parse_Ctx); + S64 result = ctx->pt[kind]; + return result; +} + +function Ast_Expr * +parse_expr_led(Token *op, Ast_Expr *left){ + enum{ Left_Associative, Right_Associative }; + S64 assoc = Left_Associative; + Ast_Expr *right = parse_expr(op_precedence(op->kind) - assoc); + switch(op->kind){ + case TK_Add: case TK_Mul: case TK_Sub: case TK_Div: return ast_expr_binary(left, right, op); + default: parsing_error(op, "Unexpected token of kind: [%s] in expression", token_kind_string(op->kind).str); return 0; + } +} + +function Ast_Expr * +parse_expr(S64 rbp){ + Token *token = token_next(); + Ast_Expr *left = parse_expr_nud(token); + while(rbp < op_precedence(token_get()->kind)){ + token = token_next(); + left = parse_expr_led(token, left); + } + return left; +} + +function S64 +expr_eval(Ast_Expr *expr){ + switch(expr->kind){ + case AK_Expr_Int: return expr->int_val; + case AK_Expr_Binary : { + S64 left = expr_eval(expr->binary.left); + S64 right = expr_eval(expr->binary.right); + switch(expr->binary.op){ + case TK_Add: return left + right; + case TK_Sub: return left - right; + case TK_Mul: return left * right; + case TK_Div: return left / right; + default: invalid_codepath; + } + }break; + default: invalid_codepath; + } + return 0; +} + +#define TEST_PARSER() \ +Set_Scratch(); \ +Parse_Ctx ctx = {}; \ +ctx.init(); \ +Set_Ctx(&ctx, Parse_Ctx_ID) + +function void +test_parse_expr(){ + TEST_PARSER(); + struct Test{String str;S64 val;}; + Array exprs = {}; + exprs.add({"(4+5)*2"_s, (4+5)*2}); + exprs.add({"4+5*2"_s, 4+5*2}); + exprs.add({"4*5+5"_s, 4*5+5}); + exprs.add({"4+5+5+3"_s, 4+5+5+3}); + + For(exprs,it,i){ + lex_restream(&ctx, it->str, "test_expr"_s); + Ast_Expr *result = parse_expr(); + S64 val = expr_eval(result); + assert(val == it->val); + } +} + +//----------------------------------------------------------------------------- +// Parsing declarations +//----------------------------------------------------------------------------- +function Ast_Decl * +parse_decl(){ + Ast_Decl *result = 0; + Token *token = token_match(TK_Identifier); + if(token){ + if(token_match(TK_DoubleColon)){ + if(token_match_keyword(keyword_const)){ + Ast_Expr *expr = parse_expr(); + result = ast_decl_const(token, token->intern_val, expr); + } + } + else parsing_error(token, "Encountered unexpected token while parsing a top level declarations"); + } + return result; +} + +function void +test_parse_decl(){ + TEST_PARSER(); + lex_restream(&ctx, "thing :: const 24252\n"_s, "test_parse_decl"_s); + Ast_Decl *result = parse_decl(); +} + diff --git a/parse.c b/parse.c index cb1c233..4653a6d 100644 --- a/parse.c +++ b/parse.c @@ -245,7 +245,7 @@ parse_expr_atom(Parser *p){ result = expr_identifier(p->arena, token); } } - else if(token_match(p, TK_Int)){ + else if(token_match(p, TK_Integer)){ result = expr_int(p->arena, token); } else if(token_is(p, TK_OpenBrace)){