diff --git a/build.bat b/build.bat index a3b2358..9b51886 100644 --- a/build.bat +++ b/build.bat @@ -1,8 +1,3 @@ @echo off -rem clang generate.c -fdiagnostics-absolute-paths -std=c99 -g -o generate.exe -Wl,user32.lib -rem generate.exe - clang main.cpp -Wall -Wno-unused-function -fno-exceptions -fdiagnostics-absolute-paths -g -o main.exe -Wl,user32.lib -rem gcc main.cpp -rem cl main.c -std:c17 diff --git a/main.cpp b/main.cpp index e77fd3d..bd5dc11 100644 --- a/main.cpp +++ b/main.cpp @@ -151,6 +151,36 @@ wrap_around_pow2(U64 x, U64 power_of_2) { return r; } +function B32 +string_compare(String a, String b){ + if(a.len != b.len) + return false; + for(S64 i = 0; i < a.len; i++){ + if(a.str[i] != b.str[i]) + return false; + } + return true; +} + +function U8 +char_to_lower(U8 c){ + if(c >= 'A' && c <= 'Z') + c += 32; + return c; +} + +function U8 +char_to_upper(U8 c){ + if(c >= 'a' && c <= 'z') + c -= 32; + return c; +} + +force_inline String +operator""_s(const char *str, size_t size){ + return String{(U8 *)str, (S64)size}; +} + //----------------------------------------------------------------------------- // OS Memory //----------------------------------------------------------------------------- @@ -245,7 +275,7 @@ test_os_memory(){ enum Allocation_Kind{Allocation_Alloc,Allocation_Resize,Allocation_FreeAll,Allocation_Free,Allocation_Destroy}; struct Allocator; typedef void *Allocator_Proc(Allocator*, Allocation_Kind, void *, SizeU); -struct Allocator{Allocator_Proc *proc;}; +struct Allocator{Allocator_Proc *proc; String debug_name;}; //----------------------------------------------------------------------------- // Memory arenas @@ -259,7 +289,7 @@ struct Arena:Allocator{ SizeU len; }; -function void arena_init(Arena *arena); +function void arena_init(Arena *arena, String debug_name); function void arena_pop_pos(Arena *arena, SizeU pos){ @@ -282,7 +312,7 @@ arena_push_size(Arena *a, SizeU size){ SizeU generous_size = size + a->alignment; if(a->len+generous_size>a->memory.commit){ if(a->memory.reserve == 0){ - arena_init(a); + arena_init(a, "Zero initialized arena"_s); } B32 result = os_commit(&a->memory, generous_size+additional_commit_size); assert(result); @@ -305,7 +335,7 @@ arena_allocator_proc(Allocator *a, Allocation_Kind kind, void *old_pointer, Size memory_copy(result, old_pointer, size); return result; } - case Allocation_Free : invalid_codepath; return 0; + case Allocation_Free : return 0; case Allocation_FreeAll: arena_clear(arena); return 0; case Allocation_Destroy: arena_release(arena); return 0; } @@ -321,17 +351,18 @@ personal_arena_allocator_proc(Allocator *a, Allocation_Kind kind, void *old_poin } function void -arena_init(Arena *a){ - a->memory = os_reserve(default_reserve_size); - a->alignment = default_alignment; +arena_init(Arena *a, String debug_name){ + a->memory = os_reserve(default_reserve_size); + a->alignment = default_alignment; + a->debug_name = debug_name; if(!a->proc) a->proc = arena_allocator_proc; } function Arena -arena_make_personal(){ +arena_make_personal(String debug_name){ Arena arena = {}; arena.proc = personal_arena_allocator_proc; - arena_init(&arena); + arena_init(&arena, debug_name); return arena; } @@ -397,11 +428,25 @@ struct Thread_Ctx{ void *ctx; U64 ctx_id; Log_Proc *log_proc; + + int line; + char *file; }; thread_local Thread_Ctx thread_ctx; global Arena pernament_arena; global OS_Heap os_process_heap; +#define report_file_and_line() report__file_and_line(__FILE__, __LINE__) +#define DEBUG_TRACE(x) (report_file_and_line(), (x)) +function void +report__file_and_line(const char *file, int line){ + thread_ctx.file = (char *)file; + thread_ctx.line = line; +} + +//----------------------------------------------------------------------------- +// Implicit scratch stack +//----------------------------------------------------------------------------- #define Set_Scratch() Scoped_Scratch scratch_##__LINE__ #define Set_Backup_Scratch() Scoped_Scratch scratch_##__LINE__(true) struct Scoped_Scratch{ @@ -422,6 +467,9 @@ struct Scoped_Scratch{ } }; +//----------------------------------------------------------------------------- +// Implicit allocator stack +//----------------------------------------------------------------------------- #define Set_Allocator(a) Scoped_Allocator scoped_##__LINE__(a) struct Scoped_Allocator{ Allocator *allocator; @@ -442,6 +490,7 @@ thread_ctx_get_user_ctx(U64 id){ assert(thread_ctx.ctx != 0); return thread_ctx.ctx; } + #define Get_Ctx(T) T *ctx = (T *)thread_ctx_get_user_ctx(T##_ID) #define Set_Ctx(ctx, id) Scoped_Ctx scoped_ctx_##__LINE__((void *)ctx, id) struct Scoped_Ctx{ @@ -456,55 +505,80 @@ struct Scoped_Ctx{ ~Scoped_Ctx(){thread_ctx.ctx = prev_ctx; thread_ctx.ctx_id = prev_id;} }; + enum Alloc_Flag{AF_None,AF_ZeroMemory}; +//----------------------------------------------------------------------------- +// Explicit allocator +//----------------------------------------------------------------------------- #define exp_alloc_array(a, T, size,...) (T *)exp_alloc(a, sizeof(T)*(size), ## __VA_ARGS__) #define exp_alloc_type(a, T, ...) exp_alloc_array(a, T, 1, ## __VA_ARGS__) -#define exp_resize_array(a, p, T, size, ...) (T *)exp_resize(a, p, sizeof(T)*(size),## __VA_ARGS__) +#define exp_alloc(a, size, ...) DEBUG_TRACE(exp__alloc(a, size, ## __VA_ARGS__)) +#define exp_resize(a,p,size) DEBUG_TRACE(exp__resize(a, p, size)) +#define exp_resize_array(a, p, T, size) DEBUG_TRACE((T *)exp_resize(a, p, sizeof(T)*(size))) +#define exp_free(a, p) DEBUG_TRACE(exp__free(a, p)) +#define exp_free_all(a) DEBUG_TRACE(exp__free_all(a)) +#define exp_destroy(a) DEBUG_TRACE(exp__destroy(a)) + +#include force_inline void * -exp_alloc(Allocator *a, SizeU size, Alloc_Flag flag = AF_None){ +exp__alloc(Allocator *a, SizeU size, Alloc_Flag flag = AF_None){ + printf("Alloc(%s) %s:%d %u\n", a->debug_name.str, thread_ctx.file, thread_ctx.line, (U32)size); void *result = a->proc(a, Allocation_Alloc, 0, size); if(flag & AF_ZeroMemory) memory_zero(result, size); return result; } force_inline void * -exp_resize(Allocator *a, void *pointer, SizeU size){ +exp__resize(Allocator *a, void *pointer, SizeU size){ + printf("Resize(%s) %s:%d %u\n", a->debug_name.str, thread_ctx.file, thread_ctx.line, (U32)size); return a->proc(a, Allocation_Resize, pointer, size); } force_inline void -exp_free(Allocator *a, void *pointer){ +exp__free(Allocator *a, void *pointer){ + printf("Free(%s) %s:%d\n", a->debug_name.str, thread_ctx.file, thread_ctx.line); a->proc(a, Allocation_Free, pointer, 0); } force_inline void -exp_free_all(Allocator *a){ +exp__free_all(Allocator *a){ + printf("FreeAll(%s) %s:%d\n", a->debug_name.str, thread_ctx.file, thread_ctx.line); a->proc(a, Allocation_FreeAll, 0, 0); } force_inline void -exp_destroy(Allocator *a){ +exp__destroy(Allocator *a){ + printf("Destroy(%s) %s:%d\n", a->debug_name.str, thread_ctx.file, thread_ctx.line); a->proc(a, Allocation_Destroy, 0, 0); } -#define imp_alloc_array(T,size,...) (T *)imp_alloc(sizeof(T) * (size),##__VA_ARGS__) -#define imp_alloc_type (T,...) imp_alloc_array(T,1,## __VA_ARGS__) -#define imp_resize_array(p, T,size, ...) (T *)imp_resize(p, sizeof(T) * (size),##__VA_ARGS__) +//----------------------------------------------------------------------------- +// Implicit allocator +//----------------------------------------------------------------------------- +#define imp_alloc_array(T,size,...) (T *)imp_alloc(sizeof(T) * (size), ##__VA_ARGS__) +#define imp_alloc_type (T,...) imp_alloc_array(T,1, ## __VA_ARGS__) +#define imp_alloc(size,...) DEBUG_TRACE(imp__alloc(size, ##__VA_ARGS__)) +#define imp_resize_array(p, T,size, ...) (T *)imp_resize(p, sizeof(T) * (size), ##__VA_ARGS__) +#define imp_resize(p,size) DEBUG_TRACE(imp_resize(p, size)) +#define imp_free(p) DEBUG_TRACE(imp__free(p)) +#define imp_free_all() DEBUG_TRACE(imp__free_all()) +#define imp_destroy() DEBUG_TRACE(imp__destroy()) + force_inline void * -imp_alloc(SizeU size, Alloc_Flag flag=AF_None){ - return exp_alloc(thread_ctx.implicit_allocator, size, flag); +imp__alloc(SizeU size, Alloc_Flag flag=AF_None){ + return exp__alloc(thread_ctx.implicit_allocator, size, flag); } force_inline void * -imp_resize(void *pointer, SizeU size){ - return exp_resize(thread_ctx.implicit_allocator, pointer, size); +imp__resize(void *pointer, SizeU size){ + return exp__resize(thread_ctx.implicit_allocator, pointer, size); } force_inline void -imp_free(void *pointer){ - exp_free(thread_ctx.implicit_allocator, pointer); +imp__free(void *pointer){ + exp__free(thread_ctx.implicit_allocator, pointer); } force_inline void -imp_free_all(){ - exp_free_all(thread_ctx.implicit_allocator); +imp__free_all(){ + exp__free_all(thread_ctx.implicit_allocator); } force_inline void -imp_destroy(){ - exp_destroy(thread_ctx.implicit_allocator); +imp__destroy(){ + exp__destroy(thread_ctx.implicit_allocator); } force_inline Allocator * @@ -515,12 +589,14 @@ imp_get(){ function void thread_ctx_init(){ - arena_init(thread_ctx.scratch); - arena_init(thread_ctx.scratch+1); - arena_init(&pernament_arena); - os_process_heap.proc = os_heap_allocator_proc; - os_process_heap.handle = GetProcessHeap(); - thread_ctx.implicit_allocator = &os_process_heap; + arena_init(thread_ctx.scratch, "Scratch1"_s); + arena_init(thread_ctx.scratch+1, "Scratch2"_s); + arena_init(&pernament_arena, "Pernament Arena"_s); + os_process_heap.proc = os_heap_allocator_proc; + os_process_heap.handle = GetProcessHeap(); + os_process_heap.debug_name = "Win32 Process Heap"_s; + + thread_ctx.implicit_allocator = &os_process_heap; } function String @@ -546,10 +622,10 @@ string_fmtv(Allocator *a, const char *str, va_list args1) { return res; } -#define STRING_FMT(alloc, str, result) \ -va_list args1; \ -va_start(args1, str); \ -String result = string_fmtv(alloc, str, args1); \ +#define STRING_FMT(alloc, str, result) \ +va_list args1; \ +va_start(args1, str); \ +String result = string_fmtv(alloc, str, args1); \ va_end(args1) function String @@ -573,6 +649,7 @@ handle_log_message(Log_Kind kind, int line, const char *file, const char *str, . function void test_heap_allocator(){ OS_Heap heap = win32_os_heap_create(false, mib(1), 0); + heap.debug_name = "Test heap"_s; Set_Allocator(&heap); assert(thread_ctx.implicit_allocator == &heap); @@ -665,9 +742,9 @@ struct Array{ init(cap); } else if(len + required_size > cap){ - S64 cap = (len + required_size)*2; - data = exp_resize_array(allocator, data, T, cap); - cap = cap; + U64 new_cap = max(cap * 2, len+required_size+1); + data = exp_resize_array(allocator, data, T, new_cap); + cap = new_cap; } } @@ -676,11 +753,6 @@ struct Array{ data[len++] = item; } - void add(T *item){ - grow(1); - data[len++] = *item; - } - void clear(){ len = 0; } @@ -712,7 +784,7 @@ test_array(){ assert(*it == i); } - Arena arena = arena_make_personal(); + Arena arena = arena_make_personal("Test personal arena"_s); Array array2 = {}; array2.allocator = &arena; for(int i = 0; i < size; i++){ @@ -760,7 +832,7 @@ map_grow(Map *map, S64 new_size){ map_insert_u64(&new_map, map->data[i].key, map->data[i].value); } } - if(map->data) free(map->data); + if(map->data) exp_free(map->allocator, map->data); *map = new_map; } @@ -834,6 +906,7 @@ map_insert(Map *map, void *key, void *value){ function void map_test(){ + Set_Scratch(); Map map = {0}; const SizeU size = 1025; for(SizeU i = 1; i < size; i++){ @@ -874,7 +947,6 @@ if(l){\ //----------------------------------------------------------------------------- // String builder //----------------------------------------------------------------------------- -#include struct String_Builder_Block{ String_Builder_Block *next; S64 cap; @@ -951,36 +1023,6 @@ string_flatten(String_Builder *b){ return result; } -function B32 -string_compare(String a, String b){ - if(a.len != b.len) - return false; - for(S64 i = 0; i < a.len; i++){ - if(a.str[i] != b.str[i]) - return false; - } - return true; -} - -function U8 -char_to_lower(U8 c){ - if(c >= 'A' && c <= 'Z') - c += 32; - return c; -} - -function U8 -char_to_upper(U8 c){ - if(c >= 'a' && c <= 'z') - c -= 32; - return c; -} - -force_inline String -operator""_s(const char *str, size_t size){ - return String{(U8 *)str, (S64)size}; -} - function void test_string_builder(){ Set_Scratch(); @@ -1053,12 +1095,12 @@ int main(){ test_os_memory(); thread_ctx_init(); + map_test(); test_parse_decl(); test_parse_expr(); test_array(); - map_test(); test_string_builder(); test_intern_table(); lex_test(); diff --git a/new_ast.cpp b/new_ast.cpp index 21cf8de..f84d96d 100644 --- a/new_ast.cpp +++ b/new_ast.cpp @@ -14,6 +14,7 @@ const U64 Parse_Ctx_ID = 115151; struct Parse_Ctx:Lexer{ Arena ast_arena; Token empty_token; + S64 indent; S64 pt[256]; // precedence table void init(){ @@ -24,7 +25,7 @@ struct Parse_Ctx:Lexer{ pt[TK_Div] = mulp; pt[TK_Mul] = mulp; - arena_init(&ast_arena); + arena_init(&ast_arena, "AST Arena"_s); lex_init(this); keyword_const = intern_string(&interns, "const"_s); keyword_struct= intern_string(&interns, "struct"_s); @@ -41,6 +42,8 @@ struct Parse_Ctx:Lexer{ enum Ast_Kind{ AK_None, + AK_Package, + AK_Expr_Str, AK_Expr_Int, AK_Expr_Ident, @@ -49,6 +52,7 @@ enum Ast_Kind{ AK_Decl_Func, AK_Decl_Func_Arg, AK_Decl_Const, + AK_Decl_Var, AK_Typespec_Ident, AK_Typespec_Pointer, @@ -103,6 +107,11 @@ struct Ast_Decl:Ast{ }; }; +struct Ast_Package:Ast{ + Intern_String name; + Array decls; +}; + //----------------------------------------------------------------------------- // AST Constructors beginning with expressions //----------------------------------------------------------------------------- @@ -178,9 +187,17 @@ ast_decl_func(Token *pos, Intern_String name){ } function Ast_Decl * -ast_decl_const(Token *pos, Intern_String name, Ast_Expr *expr){ - AST_NEW(Decl, AK_Decl_Const, pos); +ast_decl_var(Token *pos, Ast_Typespec *typespec, Intern_String name, Ast_Expr *expr){ + AST_NEW(Decl, AK_Decl_Var, pos); result->var.expr = expr; + result->var.typespec = typespec; result->name = name; return result; } + +function Ast_Package * +ast_package(Token *pos, String name){ + AST_NEW(Package, AK_Package, pos); + result->name = intern_string(&ctx->interns, name); + return result; +} \ No newline at end of file diff --git a/new_lex.cpp b/new_lex.cpp index a71e976..21cef24 100644 --- a/new_lex.cpp +++ b/new_lex.cpp @@ -138,7 +138,7 @@ lexcp(Lex_Stream *s){ function B32 lex_is_whitespace(U8 c){ - B32 result = c == '\r' || c == ' ' || c == '\r'; + B32 result = c == ' ' || c == '\r'; return result; } @@ -263,8 +263,6 @@ break function void lex__stream(Intern_Table *table, Array *array, Lex_Stream *s){ while(lexc(s)){ - while(lex_is_whitespace(lexc(s))) - lex_advance(s); Token t = {}; t.str = lexcp(s); @@ -300,14 +298,18 @@ lex__stream(Intern_Table *table, Array *array, Lex_Stream *s){ t.kind = TK_Semicolon; }break; + case ' ' : s->stream.str -= 1; case '\n': { t.kind = TK_NewLine; if(lexc(s) == '\r') lex_advance(s); for(;;){ - if(lexc(s) == ' ') t.indent++; - else if(lexc(s) == '\t') t.indent += 2; + if(lexc(s) == ' ') { + t.indent++; + // @Todo(Krzosa): Detect indentation method, file an error while methods are mixed + } + else if(lexc(s) == '\t') t.indent++; else break; lex_advance(s); } @@ -482,6 +484,12 @@ lex__stream(Intern_Table *table, Array *array, Lex_Stream *s){ lex_set_len(s,&t); array->add(t); + + while(lex_is_whitespace(lexc(s))) + lex_advance(s); + + if(s->iter >= s->stream.len) // End of stream + break; } } diff --git a/new_parse.cpp b/new_parse.cpp index 8a7b517..67f5a91 100644 --- a/new_parse.cpp +++ b/new_parse.cpp @@ -5,8 +5,11 @@ token_get(S64 i = 0){ Get_Ctx(Parse_Ctx); if(i >= ctx->tokens.len){ return &ctx->empty_token; } - Token *result = &ctx->tokens[i]; + if(result->kind == TK_NewLine){ + ctx->indent = result->indent; + } + return result; } @@ -207,25 +210,38 @@ test_parse_expr(){ // Parsing declarations //----------------------------------------------------------------------------- function Ast_Decl * -parse_decl(){ +parse_decl(){Get_Ctx(Parse_Ctx); Ast_Decl *result = 0; Token *token = token_match(TK_Identifier); if(token){ - if(token_match(TK_DoubleColon)){ - if(token_match_keyword(keyword_const)){ - Ast_Expr *expr = parse_expr(); - result = ast_decl_const(token, token->intern_val, expr); - } + if(token_match(TK_ColonAssign)){ + if(ctx->indent != 0) + parsing_error(token, "Top level declarations shouldn't be indented"); + + Ast_Expr *expr = parse_expr(); + result = ast_decl_var(token, 0, token->intern_val, expr); } else parsing_error(token, "Encountered unexpected token while parsing a top level declarations"); } return result; } +function Ast_Package * +parse_file(){ + Ast_Package *result = ast_package(token_get(), token_get()->file); + while(!token_is(TK_End)){ + while(token_match(TK_NewLine)); + Ast_Decl *decl = parse_decl(); + if(!decl) break; + result->decls.add(decl); + } + return result; +} + function void test_parse_decl(){ TEST_PARSER(); - lex_restream(&ctx, "thing :: const 24252\n"_s, "test_parse_decl"_s); - Ast_Decl *result = parse_decl(); + lex_restream(&ctx, "thing := 24252\nanother_thing := \"string\"\n\nref := thing"_s, "test_parse_decl"_s); + Ast_Package *result = parse_file(); }