diff --git a/.gitignore b/.gitignore index 8fc490d..2d2e60c 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ *.c *.lib *.o +*.10x *.sublime-* tests/ diff --git a/base.cpp b/base.cpp index 0b26fce..c4b4998 100644 --- a/base.cpp +++ b/base.cpp @@ -496,6 +496,18 @@ arena_push_size(Arena *a, size_t size){ return result; } +CORE_Static Arena +push_arena(Allocator *allocator, size_t size, String debug_name) { + Arena result = {}; + result.memory.data = (U8 *)allocate_size(allocator, size); + result.memory.reserve = size; + result.alignment = default_alignment; + result.debug_string = debug_name; + result.allocate = (Allocator::Allocate *)arena_push_size; + result.deallocate = (Allocator::Deallocate *)deallocate_stub; + return result; +} + CORE_Static void arena_init(Arena *a, String debug_name){ a->memory = os_reserve(default_reserve_size); @@ -634,7 +646,7 @@ struct Array{ *item = data[--len]; } - void init(Arena *a, S64 size = 16){ + void init(Allocator *a, S64 size = 16){ allocator = a; data = allocate_array(a, T, size); cap = size; @@ -686,7 +698,7 @@ struct Array{ template CORE_Static Array -array_make(Arena *a, S64 size = 16){ +array_make(Allocator *a, S64 size = 16){ Array result = {}; result.init(a, size); return result; @@ -774,7 +786,7 @@ map_grow(Map *map, S64 new_size){ } CORE_Static Map -map_make(Arena *a, S64 size){ +map_make(Allocator *a, S64 size){ Map result = {a}; map_grow(&result, size); return result; @@ -891,14 +903,14 @@ map_test(){ // String intern //----------------------------------------------------------------------------- struct Intern_Table{ - Arena *string_allocator; + Allocator *string_allocator; Map map; U8 *first_keyword; U8 *last_keyword; }; CORE_Static Intern_Table -intern_table_make(Arena *string_allocator, Arena *map_allocator, S64 initial_size = 32){ +intern_table_make(Allocator *string_allocator, Allocator *map_allocator, S64 initial_size = 32){ Intern_Table result = {}; result.map = map_make(map_allocator, initial_size); result.string_allocator = string_allocator; @@ -916,7 +928,7 @@ intern_string(Intern_Table *t, String string){ return result; } - S64 *len_address = (S64 *)arena_push_size(t->string_allocator, string.len+1+sizeof(S64)); + S64 *len_address = (S64 *)allocate_size(t->string_allocator, string.len+1+sizeof(S64), false); *len_address = string.len; U8 *string_address = (U8 *)(len_address + 1); diff --git a/build.bat b/build.bat index d4e2460..8de3cfc 100644 --- a/build.bat +++ b/build.bat @@ -5,7 +5,7 @@ rem cl main.cpp -I.. user32.lib clang core_main.cpp -O0 -Wall -Wno-unused-function -fno-exceptions -fdiagnostics-absolute-paths -g -o main.exe -Wl,user32.lib rem ubuntu run clang core_main.cpp -O0 -Wall -Wno-unused-function -fno-exceptions -fdiagnostics-absolute-paths -g -o core.out -clang test.cpp +rem clang test.cpp rem main.exe -testing rem echo Building arms race diff --git a/core_codegen_c_language.cpp b/core_codegen_c_language.cpp index 596fd48..3b5306b 100644 --- a/core_codegen_c_language.cpp +++ b/core_codegen_c_language.cpp @@ -771,7 +771,7 @@ gen_ast(Ast *ast){ CORE_Static String compile_to_c_code(){ - generating_time_begin = os_time(); + pctx->generating_time_begin = os_time(); prefixed_string_type = string_fmt(pctx->perm, "%QString", symbol_prefix); if(single_header_library_mode){ @@ -977,6 +977,6 @@ typedef struct String{ String string_result = string_flatten(pctx->perm, &pctx->gen); - generating_time_end = os_time(); + pctx->generating_time_end = os_time(); return string_result; } diff --git a/core_compiler.cpp b/core_compiler.cpp index 3519dbb..8fc314c 100644 --- a/core_compiler.cpp +++ b/core_compiler.cpp @@ -1,90 +1,87 @@ CORE_Static void -lex_init(Arena *token_string_arena, Arena *map_allocator, Lexer *l) { - l->arena = token_string_arena; - l->tokens = array_make(token_string_arena, 4096 * 4); - l->interns = intern_table_make(token_string_arena, map_allocator, 2048); - - /*#import meta -for i in meta.keywords: - print(f'keyword_{i.lower()} = l->intern("{i}"_s);') -print(f'l->interns.first_keyword = keyword_{meta.keywords[0].lower()}.str;') -print(f'l->interns.last_keyword = keyword_{meta.keywords[-1].lower()}.str;') - -for i in meta.interns: - print(f'intern_{i.lower()} = l->intern("{i}"_s);') - -index = 0 -for i in meta.token_simple_expr: - if i[1] != "SPECIAL": - print(f'op_info_table[{index}].op = l->intern("{i[1]}"_s);') - index += 1 - -*/ -keyword_struct = l->intern("struct"_s); -keyword_union = l->intern("union"_s); -keyword_true = l->intern("true"_s); -keyword_default = l->intern("default"_s); -keyword_break = l->intern("break"_s); -keyword_false = l->intern("false"_s); -keyword_return = l->intern("return"_s); -keyword_switch = l->intern("switch"_s); -keyword_assert = l->intern("Assert"_s); -keyword_if = l->intern("if"_s); -keyword_elif = l->intern("elif"_s); -keyword_pass = l->intern("pass"_s); -keyword_else = l->intern("else"_s); -keyword_for = l->intern("for"_s); -keyword_enum = l->intern("enum"_s); -l->interns.first_keyword = keyword_struct.str; -l->interns.last_keyword = keyword_enum.str; -intern_typeof = l->intern("TypeOf"_s); -intern_sizeof = l->intern("SizeOf"_s); -intern_len = l->intern("Len"_s); -intern_alignof = l->intern("AlignOf"_s); -intern_foreign = l->intern("foreign"_s); -intern_strict = l->intern("strict"_s); -intern_void = l->intern("void"_s); -intern_flag = l->intern("flag"_s); -intern_it = l->intern("it"_s); -intern_load = l->intern("load"_s); -intern_import = l->intern("import"_s); -intern_link = l->intern("link"_s); -op_info_table[0].op = l->intern("*"_s); -op_info_table[1].op = l->intern("/"_s); -op_info_table[2].op = l->intern("%"_s); -op_info_table[3].op = l->intern("<<"_s); -op_info_table[4].op = l->intern(">>"_s); -op_info_table[5].op = l->intern("+"_s); -op_info_table[6].op = l->intern("-"_s); -op_info_table[7].op = l->intern("=="_s); -op_info_table[8].op = l->intern("<="_s); -op_info_table[9].op = l->intern(">="_s); -op_info_table[10].op = l->intern("<"_s); -op_info_table[11].op = l->intern(">"_s); -op_info_table[12].op = l->intern("!="_s); -op_info_table[13].op = l->intern("&"_s); -op_info_table[14].op = l->intern("|"_s); -op_info_table[15].op = l->intern("^"_s); -op_info_table[16].op = l->intern("&&"_s); -op_info_table[17].op = l->intern("||"_s); -op_info_table[18].op = l->intern("~"_s); -op_info_table[19].op = l->intern("!"_s); -/*END*/ -} - -CORE_Static void -parse_init(Parse_Ctx *ctx, Arena *perm_allocator) { +core_init_compiler(Parse_Ctx *ctx, Allocator *allocator) { + ctx->init_ctx_time_begin = os_time(); pctx = ctx; - ctx->perm = perm_allocator; - ctx->type_map = map_make(ctx->perm, 2048); + Allocator *heap_allocator = allocator; + ctx->perm = (Arena *)allocator; + ctx->type_map = map_make(heap_allocator, 2048); ctx->gen = {ctx->perm}; ctx->helper_builder = {ctx->perm}; ctx->scope_ids = 1; bigint_allocator = ctx->perm; arena_init(&ctx->stage_arena, "Compiler stage arena"_s); - lex_init(ctx->perm, ctx->perm, ctx); + ctx->tokens = array_make(heap_allocator, 4096 * 4); + ctx->interns = intern_table_make(ctx->perm, heap_allocator, 2048); + + /*#import meta + for i in meta.keywords: + print(f'keyword_{i.lower()} = ctx->intern("{i}"_s);') + print(f'ctx->interns.first_keyword = keyword_{meta.keywords[0].lower()}.str;') + print(f'ctx->interns.last_keyword = keyword_{meta.keywords[-1].lower()}.str;') + + for i in meta.interns: + print(f'intern_{i.lower()} = ctx->intern("{i}"_s);') + + index = 0 + for i in meta.token_simple_expr: + if i[1] != "SPECIAL": + print(f'op_info_table[{index}].op = ctx->intern("{i[1]}"_s);') + index += 1 + + */ + keyword_struct = ctx->intern("struct"_s); + keyword_union = ctx->intern("union"_s); + keyword_true = ctx->intern("true"_s); + keyword_default = ctx->intern("default"_s); + keyword_break = ctx->intern("break"_s); + keyword_false = ctx->intern("false"_s); + keyword_return = ctx->intern("return"_s); + keyword_switch = ctx->intern("switch"_s); + keyword_assert = ctx->intern("Assert"_s); + keyword_if = ctx->intern("if"_s); + keyword_elif = ctx->intern("elif"_s); + keyword_pass = ctx->intern("pass"_s); + keyword_else = ctx->intern("else"_s); + keyword_for = ctx->intern("for"_s); + keyword_enum = ctx->intern("enum"_s); + ctx->interns.first_keyword = keyword_struct.str; + ctx->interns.last_keyword = keyword_enum.str; + intern_typeof = ctx->intern("TypeOf"_s); + intern_sizeof = ctx->intern("SizeOf"_s); + intern_len = ctx->intern("Len"_s); + intern_alignof = ctx->intern("AlignOf"_s); + intern_foreign = ctx->intern("foreign"_s); + intern_strict = ctx->intern("strict"_s); + intern_void = ctx->intern("void"_s); + intern_flag = ctx->intern("flag"_s); + intern_it = ctx->intern("it"_s); + intern_load = ctx->intern("load"_s); + intern_import = ctx->intern("import"_s); + intern_link = ctx->intern("link"_s); + op_info_table[0].op = ctx->intern("*"_s); + op_info_table[1].op = ctx->intern("/"_s); + op_info_table[2].op = ctx->intern("%"_s); + op_info_table[3].op = ctx->intern("<<"_s); + op_info_table[4].op = ctx->intern(">>"_s); + op_info_table[5].op = ctx->intern("+"_s); + op_info_table[6].op = ctx->intern("-"_s); + op_info_table[7].op = ctx->intern("=="_s); + op_info_table[8].op = ctx->intern("<="_s); + op_info_table[9].op = ctx->intern(">="_s); + op_info_table[10].op = ctx->intern("<"_s); + op_info_table[11].op = ctx->intern(">"_s); + op_info_table[12].op = ctx->intern("!="_s); + op_info_table[13].op = ctx->intern("&"_s); + op_info_table[14].op = ctx->intern("|"_s); + op_info_table[15].op = ctx->intern("^"_s); + op_info_table[16].op = ctx->intern("&&"_s); + op_info_table[17].op = ctx->intern("||"_s); + op_info_table[18].op = ctx->intern("~"_s); + op_info_table[19].op = ctx->intern("!"_s); + /*END*/ + init_type(); // Init paths @@ -93,20 +90,13 @@ parse_init(Parse_Ctx *ctx, Arena *perm_allocator) { String main_module = string_fmt(ctx->perm, "%Q/modules", ctx->exe_folder); add(ctx->perm, &ctx->module_folders, main_module); + ctx->init_ctx_time_end = os_time(); } CORE_Static void -begin_compilation(Arena *allocator) { - init_ctx_time_begin = os_time(); - Parse_Ctx *ctx = allocate_struct(allocator, Parse_Ctx); - parse_init(ctx, allocator); - init_ctx_time_end = os_time(); -} - -CORE_Static void -destroy_compiler() { - arena_clear(pctx->perm); - arena_clear(&pctx->stage_arena); +core_bootstrap_compiler(Allocator *allocator) { + Parse_Ctx *ctx = allocate_struct(allocator, Parse_Ctx); + core_init_compiler(ctx, allocator); } CORE_Static void @@ -120,11 +110,9 @@ insert_builtin_type_into_scope(Ast_Scope *p, String name, Ast_Type *type) { add(pctx->perm, &pctx->all_types, type); } -global F64 parsing_time_begin; -global F64 parsing_time_end; CORE_Static void parse_all_modules() { - parsing_time_begin = os_time(); + pctx->parsing_time_begin = os_time(); Iter_Named(&pctx->modules, mod_it) { Ast_Module *module = mod_it.item[0]; @@ -142,7 +130,7 @@ parse_all_modules() { module->state = MODULE_PARSED; } - parsing_time_end = os_time(); + pctx->parsing_time_end = os_time(); } CORE_Static Ast_Module * @@ -207,7 +195,7 @@ CORE_Static void resolve_everything_in_module(Ast_Module *module) { if (module->state == MODULE_RESOLVED) return; - resolving_time_begin = os_time(); + pctx->resolving_time_begin = os_time(); Iter_Named(&module->all_loaded_files, file) { Iter(&file.item[0]->decls) { Ast_Decl *decl = it.item[0]; @@ -219,13 +207,13 @@ resolve_everything_in_module(Ast_Module *module) { } } module->state = MODULE_RESOLVED; - resolving_time_end = os_time(); + pctx->resolving_time_end = os_time(); } CORE_Static String compile_file_to_string(Arena *arena, String filename) { - total_time = os_time(); - begin_compilation(arena); + pctx->total_time = os_time(); + core_bootstrap_compiler(arena); { Ast_Module *module = add_module(0, pctx->intern("Language.core"_s)); { @@ -321,11 +309,11 @@ compile_file(Arena *arena, String filename, U32 compile_flags = COMPILE_NULL) { F64 end = os_time(); if (is_flag_set(compile_flags, COMPILE_PRINT_STATS)) { - log_info("total = %f", os_time() - total_time); + log_info("total = %f", os_time() - pctx->total_time); log_info("clang = %f", end - begin); - log_info("parsing = %f", parsing_time_end - parsing_time_begin); - log_info("resolving = %f", resolving_time_end - resolving_time_begin); - log_info("generatin = %f", generating_time_end - generating_time_begin); + log_info("parsing = %f", pctx->parsing_time_end - pctx->parsing_time_begin); + log_info("resolving = %f", pctx->resolving_time_end - pctx->resolving_time_begin); + log_info("generatin = %f", pctx->generating_time_end - pctx->generating_time_begin); } if (is_flag_set(compile_flags, COMPILE_PRINT_ALLOCATOR_STATS_BEFORE_DESTROY)) { @@ -355,5 +343,4 @@ compile_file(Arena *arena, String filename, U32 compile_flags = COMPILE_NULL) { log_info(""); } - destroy_compiler(); } diff --git a/core_compiler.h b/core_compiler.h index 7f0be47..b46192a 100644 --- a/core_compiler.h +++ b/core_compiler.h @@ -1,4 +1,13 @@ +/* + * Get rid of Scratch, move that concept onto the main context, create scratch memory from user allocator + * Simplify allocators, only should be derived from the user allocator + * Cleanup big int allocator + * Replace iterator interface + + * Reset compiler + + */ struct Lex_Stream{ String stream; @@ -11,31 +20,19 @@ struct Lex_Stream{ Array indent_stack; }; -struct Lexer{ - Arena *arena; +struct Parse_Ctx{ + Arena *perm; // Stores: AST, tokens, interns + Arena stage_arena; + Arena scratch; + + // Lexer stuff Lex_Stream stream; Array tokens; Intern_Table interns; S64 token_iter; U32 token_debug_ids; - Intern_String intern(String string){ - assert(string.len > 0); - return intern_string(&interns, string); - } -}; - -// Lexer::interns::map::allocator - array allocator, resizing -// Lexer::tokens - array allocator, resizing -// -// Parser::ast_arena - arena for asts -// Lexer::interns::string_allocator - arena for interns -// - -struct Parse_Ctx:Lexer{ - Arena *perm; // Stores: AST, tokens, interns - Arena stage_arena; - + // Types List all_types; S32 type_ids; int lambda_ids; @@ -63,10 +60,24 @@ struct Parse_Ctx:Lexer{ S64 indent; String_Builder gen; String_Builder helper_builder; + + F64 generating_time_begin; + F64 generating_time_end; + F64 resolving_time_begin; + F64 resolving_time_end; + F64 total_time; + F64 init_ctx_time_begin; + F64 init_ctx_time_end; + F64 parsing_time_begin; + F64 parsing_time_end; + + Intern_String intern(String string){ + assert(string.len > 0); + return intern_string(&interns, string); + } }; CORE_Static void init_type(); -CORE_Static void lex_init(Arena *token_string_arena, Arena *map_allocator, Lexer *l); CORE_Static String compile_to_c_code(); CORE_Static Ast_Module *ast_module(Token *pos, Intern_String filename); CORE_Static void insert_builtin_types_into_scope(Ast_Scope *p); diff --git a/core_compiler_interface.hpp b/core_compiler_interface.hpp index 5578a36..7ce24e7 100644 --- a/core_compiler_interface.hpp +++ b/core_compiler_interface.hpp @@ -65,7 +65,6 @@ struct List{ enum Token_Kind{ TK_End, - /*# import meta for i in meta.token_kinds: @@ -630,4 +629,4 @@ struct Ast_Decl: Ast{ }; }; /*END*/ -}; +}; \ No newline at end of file diff --git a/core_globals.cpp b/core_globals.cpp index 092c463..bb31b54 100644 --- a/core_globals.cpp +++ b/core_globals.cpp @@ -3,10 +3,10 @@ global B32 emit_type_info = true; global String symbol_prefix = ""_s; global B32 single_header_library_mode = false; global String single_header_library_name = ""_s; +global bool color_codes_enabled; thread_local Parse_Ctx *pctx; -global bool color_codes_enabled; Arena *bigint_allocator; global S64 bigint_allocation_count; @@ -108,13 +108,4 @@ global Ast_Type *untyped_bool = &type__untyped_bool; global Ast_Type *untyped_int = &type__untyped_int; global Ast_Type *untyped_float = &type__untyped_float; -//----------------------------------------------------------------------------- -// Time records -//----------------------------------------------------------------------------- -global F64 generating_time_begin; -global F64 generating_time_end; -global F64 resolving_time_begin; -global F64 resolving_time_end; -global F64 total_time; -global F64 init_ctx_time_begin; -global F64 init_ctx_time_end; + diff --git a/core_lexing.cpp b/core_lexing.cpp index 20d458d..189ee40 100644 --- a/core_lexing.cpp +++ b/core_lexing.cpp @@ -56,7 +56,7 @@ lex_set_len(Lex_Stream *s, Token *token){ } CORE_Static void -lex_set_keywords(Lexer *lexer, Array keywords){ +lex_set_keywords(Parse_Ctx *lexer, Array keywords){ Intern_String keyword = {}; For(keywords){ keyword = intern_string(&lexer->interns, it); @@ -79,7 +79,7 @@ token_error(Token *t, String error_val){ } CORE_Static void -lex_parse_u64(Lexer *lexer, Token *t, S64 base){ +lex_parse_u64(Parse_Ctx *lexer, Token *t, S64 base){ Scratch scratch; Set_BigInt_Arena(scratch); @@ -100,7 +100,7 @@ lex_parse_u64(Lexer *lexer, Token *t, S64 base){ m = bigint_mul(&m, &base_mul); } - t->int_val = bigint_copy(lexer->arena, &result); + t->int_val = bigint_copy(lexer->perm, &result); } CORE_Static void @@ -175,7 +175,7 @@ lex_parse_ident(Intern_Table *table, Lex_Stream *s, Token *t){ break CORE_Static Token -token_make(Lexer *lexer, U8 *str, Intern_String file, int line, U8 *line_begin){ +token_make(Parse_Ctx *lexer, U8 *str, Intern_String file, int line, U8 *line_begin){ Token t = {}; t.str = str; t.file = file; @@ -186,7 +186,7 @@ token_make(Lexer *lexer, U8 *str, Intern_String file, int line, U8 *line_begin){ } CORE_Static Token -token_make(Lexer *lexer){ +token_make(Parse_Ctx *lexer){ return token_make(lexer, lexcp(&lexer->stream), lexer->stream.file, lexer->stream.line, lexer->stream.line_begin); } @@ -228,7 +228,7 @@ lex_unwind_indent_stack(Token *t, Lex_Stream *s, Array *array){ } CORE_Static void -lex__stream(Lexer *lexer){ +lex__stream(Parse_Ctx *lexer){ Intern_Table *table = &lexer->interns; Array *array = &lexer->tokens; Lex_Stream *s = &lexer->stream; @@ -585,15 +585,8 @@ lex__stream(Lexer *lexer){ #undef CASE3 } -CORE_Static Lexer -lex_make(Arena *token_string_arena, Arena *map_allocator){ - Lexer result = {}; - lex_init(token_string_arena, map_allocator, &result); - return result; -} - CORE_Static void -lex_restream(Lexer *lexer, String istream, String file){ +lex_restream(Parse_Ctx *lexer, String istream, String file){ lexer->stream = {}; lexer->stream.stream = istream; lexer->stream.line_begin = istream.str; @@ -605,13 +598,6 @@ lex_restream(Lexer *lexer, String istream, String file){ lex__stream(lexer); } -CORE_Static Lexer -lex_stream(Arena *token_string_arena, Arena *map_allocator, String istream, String file){ - Lexer result = lex_make(token_string_arena, map_allocator); - lex_restream(&result, istream, file); - return result; -} - //----------------------------------------------------------------------------- // Token metadata //----------------------------------------------------------------------------- diff --git a/core_typechecking.cpp b/core_typechecking.cpp index 6ef4567..7cdafd0 100644 --- a/core_typechecking.cpp +++ b/core_typechecking.cpp @@ -1510,7 +1510,6 @@ resolve_expr(Ast_Expr *ast, Resolve_Flag flags, Ast_Type *compound_and_const_str Scratch scratch; Array items = {scratch}; - S64 was_name_indexed = false; S64 default_iter = 0; /* @@ -1530,7 +1529,6 @@ resolve_expr(Ast_Expr *ast, Resolve_Flag flags, Ast_Type *compound_and_const_str if(is_flag_set(it->call_flags, CALL_NAME)){ Ast_Atom *name = it->name; assert(name->kind == AST_IDENT); - was_name_indexed = true; if(name->intern_val.str == lambda_arg->name.str) item = it; } diff --git a/core_typechecking.h b/core_typechecking.h index 0cda3c2..5e67f36 100644 --- a/core_typechecking.h +++ b/core_typechecking.h @@ -11,7 +11,7 @@ union { Ast_Decl *resolved_decl; union { bool bool_val; - F64 f64_val; + double f64_val; Intern_String intern_val; BigInt big_int_val; Ast_Type *type_val;