From b0553c38cf0dee7333e79ad91d0d89f06f24438c Mon Sep 17 00:00:00 2001 From: Krzosa Karol Date: Mon, 13 Jun 2022 10:49:10 +0200 Subject: [PATCH] New module scheme --- G.globals.kl | 76 ---------------------------- Windows.kl | 2 + ast.cpp | 35 +++++++------ ccodegen.cpp | 128 +++++++++++++++++++++++++++++++++++++++++++++-- compiler.h | 26 ++++++++-- enums.kl | 2 - lambdas.kl | 6 +-- main.cpp | 32 ++++++++---- order1.kl | 2 - parsing.cpp | 33 ++++++++++++ typechecking.cpp | 68 +++++++------------------ typechecking.h | 2 +- 12 files changed, 247 insertions(+), 165 deletions(-) delete mode 100644 G.globals.kl diff --git a/G.globals.kl b/G.globals.kl deleted file mode 100644 index a16786f..0000000 --- a/G.globals.kl +++ /dev/null @@ -1,76 +0,0 @@ -//----------------------------------------------------------------------------- -// Function types -//----------------------------------------------------------------------------- -test_function :: (thing: S64): *S64 -function_type: test_function -const_function_alias :: test_function -// null_function: (t: S64): *S64 = null - -//----------------------------------------------------------------------------- -// Booleans -//----------------------------------------------------------------------------- -Boolean: Bool = true - -//----------------------------------------------------------------------------- -// Nulls -//----------------------------------------------------------------------------- -// int_null: S64 = null -// str_null: String = null -// Bool_null: Bool = null - -//----------------------------------------------------------------------------- -// Compound expressions -//----------------------------------------------------------------------------- -array1: [4]S64 = {1,2,3,4} -imp_array := [5]S64{1,2} -// imp_array_a := [5]S64{1,2,3,4,5,6} -// imp_array_b: [5]S64 = {1,2,3,4,5,6} -imp_array_c: [5]S64 = {[0] = 1, [2] = 2, [1] = 0} // @todo this should be illegal -// without_size: []S64 = {} // @todo: this should be slice, converting from array should be implicit - -string: *char = "string" -first_letter := string[0] -decl_char: char = 55 -thing: *void - -//----------------------------------------------------------------------------- -// Pointers -//----------------------------------------------------------------------------- -pointer_decl : *S64 -variable_from_deref: S64 = *pointer_decl -pointer_from_var : *S64 = &variable_from_deref -Boolean_pointer := &Boolean - -//----------------------------------------------------------------------------- -// Implicit type -//----------------------------------------------------------------------------- -implicit_int :: 10 -implicit_str :: "Hello world" - -//----------------------------------------------------------------------------- -// Pointers -//----------------------------------------------------------------------------- -// pointer1: *S64 = 0 -// pointer2: *S64 = pointer1 -// pointer3: **S64 = 0 - -//----------------------------------------------------------------------------- -// String types -//----------------------------------------------------------------------------- -string1 :: "Test" -string2 :: string1 - -//----------------------------------------------------------------------------- -// Constant S64 variables -//----------------------------------------------------------------------------- -thing0 :: 10 -thing1 :: thing0 + 11 -thing2 :: thing1 + 20 -combin :: thing0 + thing1 + thing2 - -Some :: struct - len: S64 - cap: S64 - -some := Some{3, 2} -other := Some{len = 1, cap = 3} \ No newline at end of file diff --git a/Windows.kl b/Windows.kl index 278dfef..9058e37 100644 --- a/Windows.kl +++ b/Windows.kl @@ -1,3 +1,5 @@ +#load "globals.kl" +#load "lambdas.kl" DWORD :: U32 LPCSTR :: *char diff --git a/ast.cpp b/ast.cpp index c4ecc12..0abe778 100644 --- a/ast.cpp +++ b/ast.cpp @@ -5,7 +5,7 @@ enum Ast_Kind: U32{ AST_NONE, - AST_PACKAGE, + AST_FILE_NAMESPACE, AST_SCOPE, AST_VALUE, @@ -48,7 +48,7 @@ enum{ AST_ATOM = bit_flag(7), AST_FOREIGN = bit_flag(8), AST_DECL = bit_flag(9), - AST_PACKAGE_LEVEL = bit_flag(10), + AST_FILE_NAMESPACE_LEVEL = bit_flag(10), AST_FLAG = bit_flag(11), }; @@ -104,6 +104,10 @@ struct Ast_Unary: Ast_Expr{ U64 padding[2]; // For folding constants into atoms }; +struct Ast_Load: Ast{ + String string; +}; + struct Ast_Cast: Ast_Expr{ Ast_Expr *expr; Ast_Expr *typespec; @@ -196,6 +200,7 @@ struct Ast_Decl; struct Ast_Scope: Ast{ Array decls; Array stmts; + Ast_File *file; }; struct Ast_Decl: Ast{ @@ -213,16 +218,10 @@ struct Ast_Decl: Ast{ INLINE_VALUE_FIELDS; }; -struct Ast_File{ - String filename; - String filecontent; - - Intern_String name; - Array decls; +struct Ast_File_Namespace : Ast_Decl{ + Ast_File *file; }; -struct Ast_Package : Ast_Decl{}; - //----------------------------------------------------------------------------- // AST Constructors beginning with expressions //----------------------------------------------------------------------------- @@ -399,6 +398,8 @@ function Ast_Scope * begin_decl_scope(Allocator *scratch, Token *pos){ AST_NEW(Scope, SCOPE, pos, AST_DECL); result->decls = {scratch}; + result->file = pctx->currently_parsed_file; + assert(result->file); pctx->currently_parsed_scope = result; return result; } @@ -414,6 +415,8 @@ begin_stmt_scope(Allocator *scratch, Token *pos){ AST_NEW(Scope, SCOPE, pos, AST_STMT); result->stmts = {scratch}; result->decls = {pctx->heap}; + result->file = pctx->currently_parsed_file; + assert(result->file); pctx->currently_parsed_scope = result; return result; } @@ -477,14 +480,16 @@ function Ast_Scope * ast_decl_scope(Token *pos, Allocator *allocator){ AST_NEW(Scope, SCOPE, pos, AST_DECL); result->decls = {allocator}; + result->file = pctx->currently_parsed_file; + assert(result->file); return result; } -function Ast_Package * -ast_package(Token *pos, Allocator *allocator, Intern_String name){ - AST_NEW(Package, PACKAGE, pos, 0); - result->kind = AST_PACKAGE; - result->scope = ast_decl_scope(pos, allocator); +function Ast_File_Namespace * +ast_file_namespace(Token *pos, Ast_File *file, Intern_String name){ + AST_NEW(File_Namespace, FILE_NAMESPACE, pos, 0); + result->kind = AST_FILE_NAMESPACE; + result->file = file; result->name = name; return result; } diff --git a/ccodegen.cpp b/ccodegen.cpp index c7587cb..dc1634b 100644 --- a/ccodegen.cpp +++ b/ccodegen.cpp @@ -400,6 +400,8 @@ gen_ast(Ast *ast){ BREAK(); } + CASE(FILE_NAMESPACE, Decl){unused(node); BREAK();} + default: { assert(is_flag_set(ast->flags, AST_EXPR)); gen_expr((Ast_Expr *)ast); @@ -408,6 +410,125 @@ gen_ast(Ast *ast){ } } +function void +parse_file(Ast_File *file){ + Scratch scratch; + file->filecontent = os_read_file(pctx->perm, file->filename.s); + assert(file); + assert(file->filecontent.len); + assert(file->filename.len); + + + pctx->currently_parsed_file = file; + String name = string_chop_last_period(file->filename.s); + file->name = pctx->intern(name); + file->scope = ast_decl_scope(0, pctx->heap); + + pctx->currently_parsed_scope = file->scope; + lex_restream(pctx, file->filecontent, file->filename.s); + while(token_expect(SAME_SCOPE)){ + if(token_match_pound(pctx->intern("load"_s))){ + parse_load(true); + continue; + } + + Ast_Decl *decl = parse_decl(true); + if(!decl) break; + + set_flag(decl->flags, AST_FILE_NAMESPACE_LEVEL); + if(decl->kind == AST_STRUCT){ + decl->type = type_type; + decl->type_val = type_incomplete(decl); + decl->state = DECL_RESOLVED; + } + + insert_into_scope(file->scope, decl); + } + pctx->currently_parsed_scope = 0; + pctx->currently_parsed_file = 0; +} + +function void +parse_files(Ast_Module *module){ + for(S64 i = 0; i < module->files.len; i++){ + auto it = module->files.data[i]; + parse_file(it); + } +} + +function void +insert_type_into_file(Ast_File *p, String name, Ast_Type *type){ + Intern_String string = pctx->intern(name); + Ast_Decl *decl = ast_type(&null_token, string, type); + decl->parent_scope = p->scope; + decl->state = DECL_RESOLVED; + insert_into_scope(p->scope, decl); +} + +function void +insert_builtin_types_into_file(Ast_File *p){ + insert_type_into_file(p, "void"_s , type_void); + insert_type_into_file(p, "Bool"_s , type_bool); + insert_type_into_file(p, "String"_s, type_string); + insert_type_into_file(p, "char"_s, type_char); + insert_type_into_file(p, "int"_s, type_int); + insert_type_into_file(p, "S8"_s, type_s8); + insert_type_into_file(p, "S16"_s, type_s16); + insert_type_into_file(p, "S32"_s, type_s32); + insert_type_into_file(p, "S64"_s, type_s64); + insert_type_into_file(p, "U8"_s, type_u8); + insert_type_into_file(p, "U16"_s, type_u16); + insert_type_into_file(p, "U32"_s, type_u32); + insert_type_into_file(p, "U64"_s, type_u64); + insert_type_into_file(p, "F32"_s, type_f32); + insert_type_into_file(p, "F64"_s, type_f64); +} + +function Ast_Module * +parse_module(String filename){ + Ast_Module *result = exp_alloc_type(pctx->perm, Ast_Module); + result->name = pctx->intern(filename); + result->files = {pctx->heap}; + Ast_File *file = register_ast_file(result->name, result, true); + parse_files(result); + insert_builtin_types_into_file(file); + pctx->modules.add(result); + return result; +} + +function void +resolve_everything_in_module(Ast_Module *module){ + For(module->files){ + For_It(it->scope->decls, jt){ + resolve_name(it->scope, jt->pos, jt->name); + if(jt->kind == AST_STRUCT){ + type_complete(jt->type); + } + } + } +} + +function void +begin_compilation(){ + OS_Heap *heap = exp_alloc_type(&pernament_arena, OS_Heap); + *heap = win32_os_heap_create(false, mib(16), 0); + Parse_Ctx *ctx = exp_alloc_type(&pernament_arena, Parse_Ctx); + parse_init(ctx, &pernament_arena, heap); +} + +function String +end_compilation(){ + For(pctx->ordered_decls){ + genln(""); + gen_ast(it); + } + + exp_destroy(pctx->heap); + String string_result = string_flatten(pctx->perm, &pctx->gen); + return string_result; +} + +#if 0 function String compile_files(Array filename){ Scratch scratch(thread_ctx.scratch); @@ -444,11 +565,11 @@ compile_files(Array filename){ it.name = package_token->intern_val; } - Ast_Package *package = find_package(it.name, &pctx->packages); + Ast_File_Namespace *package = find_package(it.name, &pctx->packages); if(package){ package->scope->decls.add(it.decls); } else { - package = ast_package(token, &heap, it.name); + package = ast_file_namespace(token, &heap, it.name); insert_builtin_types_into_package(package); pctx->packages.add(package); } @@ -458,7 +579,7 @@ compile_files(Array filename){ Ast_Decl *decl = parse_decl(true); if(!decl) break; - set_flag(decl->flags, AST_PACKAGE_LEVEL); + set_flag(decl->flags, AST_FILE_NAMESPACE_LEVEL); if(decl->kind == AST_STRUCT){ decl->type = type_type; decl->type_val = type_incomplete(decl); @@ -537,3 +658,4 @@ int main(){ return string_result; } +#endif \ No newline at end of file diff --git a/compiler.h b/compiler.h index 21ebde0..a56a2e9 100644 --- a/compiler.h +++ b/compiler.h @@ -153,7 +153,6 @@ Intern_String keyword_if; Intern_String keyword_else; Intern_String keyword_true; Intern_String keyword_false; -Intern_String keyword_package; Intern_String keyword_for; Intern_String keyword_pass; Intern_String keyword_cast; @@ -164,10 +163,25 @@ Intern_String intern_foreign; Intern_String intern_strict; Intern_String intern_flag; - struct Ast_Scope; struct Ast_Decl; -struct Ast_Package; +struct Ast_File_Namespace; +struct Ast_File; +struct Ast_Module{ + Intern_String name; + Array files; +}; + +struct Ast_File{ + Intern_String filename; + Ast_Module *module; + + B32 global_implicit_load; + String filecontent; + Intern_String name; + Ast_Scope *scope; +}; + struct Parse_Ctx:Lexer{ Allocator *perm; // Stores: AST, tokens, interns Allocator *heap; @@ -175,8 +189,10 @@ struct Parse_Ctx:Lexer{ U64 unique_ids; Map type_map; - Array packages; + Array modules; + Array packages; Ast_Scope *currently_parsed_scope; + Ast_File *currently_parsed_file; Array ordered_decls; S64 indent; @@ -200,7 +216,6 @@ lex_init(Allocator *token_string_arena, Allocator *map_allocator, Lexer *l){ keyword_cast = l->intern("cast"_s); keyword_true = l->intern("true"_s); keyword_false = l->intern("false"_s); - keyword_package = l->intern("package"_s); keyword_return = l->intern("return"_s); keyword_if = l->intern("if"_s); keyword_pass = l->intern("pass"_s); @@ -223,6 +238,7 @@ parse_init(Parse_Ctx *ctx, Allocator *perm_allocator, Allocator *heap_allocator) ctx->gen = {ctx->perm}; ctx->ordered_decls = {ctx->heap}; ctx->type_map = {ctx->heap}; + ctx->modules = {ctx->heap}; bigint_allocator = ctx->perm; lex_init(ctx->perm, ctx->heap, ctx); diff --git a/enums.kl b/enums.kl index 2aaae6d..2edf04b 100644 --- a/enums.kl +++ b/enums.kl @@ -1,5 +1,3 @@ -package Memory - Allocator_Kind :: enum #flag Null Arena diff --git a/lambdas.kl b/lambdas.kl index 3f3e09a..61a94ea 100644 --- a/lambdas.kl +++ b/lambdas.kl @@ -1,11 +1,11 @@ -package lambdas - Test :: struct len: S64 test: Test member := test.len +Memory :: #load "enums.kl" +imp_val: Allocator_Kind = Allocator_Kind.Heap enum_val: Memory.Allocator_Kind = Memory.Allocator_Kind.Heap other_enum_val: S64 = cast(enum_val: S64) @@ -14,7 +14,7 @@ pointer_type :: *S64 // null_pointer: pointer_type = null if_stmt :: (cond: S64): type - new_types.basic_type_assignment() + //new_types.basic_type_assignment() CONSTANT :: 10 thing := 10 if i := thing + cond, cond > CONSTANT diff --git a/main.cpp b/main.cpp index fbd90c1..bf6da40 100644 --- a/main.cpp +++ b/main.cpp @@ -31,6 +31,14 @@ For now I don't thing it should be overloadable. ------------------------------------------------------------------------------- +We compile lot's of files, we keep track of them in Parse_Ctx, making sure we don't +parse same thing twice. Files belong to a module, files can be loaded #load "file". +All the files see all the decls from all the files in that module. We can import +other modules using a different directive #import. #import perhaps should be lazily +evaluated, making sure we don't resolve stuff we don't require. Currently probably +want to export all the symbols, we can namespace them optionally. + +------------------------------------------------------------------------------- @todo [ ] - Should compound resolution use an algorithm to reorder compounds to initialize all fields in order @@ -40,6 +48,7 @@ For now I don't thing it should be overloadable. [ ] - slices should be properly displayed in debugger [ ] - Rewrite where # happen, [ ] - cast -> +[ ] - elif [ ] - #assert that handles constants at compile time and vars at runtime [ ] - Comma notation when declaring variables thing1, thing2: S32 @@ -163,16 +172,21 @@ int main(int argument_count, char **arguments){ system((const char *)run_program.str); } #else - Scratch scratch; - Array files = {scratch}; - files.add("lambdas.kl"_s); - files.add("order1.kl"_s); - files.add("order2.kl"_s); - files.add("new_types.kl"_s); - files.add("enums.kl"_s); - files.add("Windows.kl"_s); + // Scratch scratch; + // Array files = {scratch}; + // files.add("lambdas.kl"_s); + // files.add("order1.kl"_s); + // files.add("order2.kl"_s); + // files.add("new_types.kl"_s); + // files.add("enums.kl"_s); + // files.add("Windows.kl"_s); // files.add("euler.kl"_s); - String result = compile_files(files); + // String result = compile_files(files); + begin_compilation(); + Ast_Module *module = parse_module("Windows.kl"_s); + assert(module); + resolve_everything_in_module(module); + String result = end_compilation(); printf("%s", result.str); #endif __debugbreak(); diff --git a/order1.kl b/order1.kl index e173965..e70a4d5 100644 --- a/order1.kl +++ b/order1.kl @@ -1,5 +1,3 @@ -package lambdas - recursive_lambda :: (thing: S64) in_val := recursive_lambda some_value := thing + const_in_lambda diff --git a/parsing.cpp b/parsing.cpp index 8a1d8ef..00996d2 100644 --- a/parsing.cpp +++ b/parsing.cpp @@ -582,6 +582,34 @@ parse_enum(Token *pos){ return result; } +function Ast_File * +register_ast_file(Intern_String filename, Ast_Module *module, B32 global_implicit_load){ + Ast_File *file = 0; + For(module->files){ + if(it->filename == filename){ + file = it; + break; + } + } + if(!file){ + file = exp_alloc_type(pctx->perm, Ast_File, AF_ZeroMemory); + file->filename = filename; + file->module = module; + file->module->files.add(file); + } + + if(global_implicit_load) file->global_implicit_load = true; + + return file; +} + +function Ast_File * +parse_load(B32 global_implicit_load){ + Token *file = token_expect(TK_StringLit); + Ast_File *result = register_ast_file(file->intern_val, pctx->currently_parsed_file->module, global_implicit_load); + return result; +} + /* Needs peeking only because I didn't want to duplicate code for parsing statements and it makes code nicer. @@ -614,6 +642,11 @@ parse_decl(B32 is_global){ result = parse_enum(tname); } + else if(token_match_pound(pctx->intern("load"_s))){ + Ast_File *file = parse_load(false); + result = ast_file_namespace(tname, file, tname->intern_val); + } + else{ Ast_Expr *expr = parse_expr(); result = ast_const(tname, tname->intern_val, expr); diff --git a/typechecking.cpp b/typechecking.cpp index 41bab88..4c701c5 100644 --- a/typechecking.cpp +++ b/typechecking.cpp @@ -322,16 +322,6 @@ _search_for_decl(Ast_Scope *scope, Intern_String name){ return 0; } -function Ast_Package * -search_for_package(Intern_String name){ - For(pctx->packages){ - if(name == it->name){ - return it; - } - } - return 0; -} - function Ast_Decl * search_for_decl(Ast_Scope *scope, Intern_String name, Search_Flag flags = 0){ Ast_Decl *result = 0; @@ -340,10 +330,18 @@ search_for_decl(Ast_Scope *scope, Intern_String name, Search_Flag flags = 0){ if(result) break; if(is_flag_set(flags, SEARCH_ONLY_CURRENT_SCOPE)) break; } - if(!result && is_flag_set(flags, SEARCH_ALSO_FOR_PACKAGE)){ - result = (Ast_Decl *)search_for_package(name); - } + if(!result && !is_flag_set(flags, SEARCH_ONLY_CURRENT_SCOPE)){ + For(scope->file->module->files){ + if(scope->file != it){ + // If parsing the scope can be invalid + if(it->scope == 0 && pctx->currently_parsed_file != 0) continue; + if(it->global_implicit_load == false) continue; + result = _search_for_decl(it->scope, name); + if(result) break; + } + } + } return result; } @@ -359,7 +357,7 @@ insert_into_scope(Ast_Scope *scope, Ast_Decl *decl){ } function void -insert_type_into_package(Ast_Package *p, String name, Ast_Type *type){ +insert_type_into_package(Ast_File_Namespace *p, String name, Ast_Type *type){ Intern_String string = pctx->intern(name); Ast_Decl *decl = ast_type(&null_token, string, type); decl->parent_scope = p->scope; @@ -368,7 +366,7 @@ insert_type_into_package(Ast_Package *p, String name, Ast_Type *type){ } function void -insert_builtin_types_into_package(Ast_Package *p){ +insert_builtin_types_into_package(Ast_File_Namespace *p){ insert_type_into_package(p, "void"_s , type_void); insert_type_into_package(p, "Bool"_s , type_bool); insert_type_into_package(p, "String"_s, type_string); @@ -437,7 +435,7 @@ require_const_int(Ast_Expr *expr, Resolve_Flag flags, Ast_Type *compound_context function void resolve_stmt(Ast *ast, Ast_Type *ret){ if(!ast) return; - assert(ast->parent_scope->kind == AST_SCOPE || ast->parent_scope->kind == AST_PACKAGE); + assert(ast->parent_scope->kind == AST_SCOPE || ast->parent_scope->kind == AST_FILE_NAMESPACE); switch(ast->kind){ @@ -666,9 +664,9 @@ resolve_field_access(Ast_Expr *node, Ast_Scope *context){ Ast_Atom *ident = (Ast_Atom *)node; Ast_Scope *scope = context ? context : node->parent_scope; - Search_Flag flag = context ? SEARCH_ONLY_CURRENT_SCOPE : SEARCH_ALSO_FOR_PACKAGE; + Search_Flag flag = context ? SEARCH_ONLY_CURRENT_SCOPE : SEARCH_ALSO_FOR_FILE_NAMESPACE; Ast_Decl *decl = resolve_name(scope, node->pos, ident->intern_val, flag); - if(decl->kind == AST_PACKAGE){ + if(decl->kind == AST_FILE_NAMESPACE){ assert(next); return resolve_field_access(next, decl->scope); } @@ -696,7 +694,7 @@ function Operand resolve_expr(Ast_Expr *ast, Resolve_Flag flags, Ast_Type *compound_context){ if(!ast && is_flag_set(flags, AST_CAN_BE_NULL)) return {}; assert(is_flag_set(ast->flags, AST_EXPR)); - assert(ast->parent_scope->kind == AST_SCOPE || ast->parent_scope->kind == AST_PACKAGE); + assert(ast->parent_scope->kind == AST_SCOPE || ast->parent_scope->kind == AST_FILE_NAMESPACE); switch(ast->kind){ @@ -983,7 +981,7 @@ resolve_decl(Ast_Decl *ast){ BREAK(); } - CASE(PACKAGE, Package){ + CASE(FILE_NAMESPACE, File_Namespace){ unused(node); BREAK(); } @@ -1017,7 +1015,7 @@ resolve_decl(Ast_Decl *ast){ } ast->state = DECL_RESOLVED; - if(is_flag_set(ast->flags, AST_PACKAGE_LEVEL)) + if(is_flag_set(ast->flags, AST_FILE_NAMESPACE_LEVEL)) pctx->ordered_decls.add(ast); } @@ -1028,31 +1026,3 @@ resolve_name(Ast_Scope *scope, Token *pos, Intern_String name, Search_Flag searc resolve_decl(decl); return decl; } - -function void -resolve_package(Ast_Package *package){ - For(package->scope->decls){ - resolve_name(package->scope, it->pos, it->name); - if(it->kind == AST_STRUCT){ - type_complete(it->type); - } - } -} - -function Ast_File -ast_file(String filename, String filecontent){ - Ast_File result = {}; - result.filecontent = filecontent; - result.filename = filename; - return result; -} - -function Ast_Package * -find_package(Intern_String name, Array *packages){ - For(*packages){ - if(it->name == name){ - return it; - } - } - return 0; -} diff --git a/typechecking.h b/typechecking.h index 98d21bf..9633edc 100644 --- a/typechecking.h +++ b/typechecking.h @@ -14,7 +14,7 @@ FLAG32(Resolve_Flag){ FLAG32(Search_Flag){ SEARCH_ONLY_CURRENT_SCOPE = bit_flag(1), - SEARCH_ALSO_FOR_PACKAGE = bit_flag(2), + SEARCH_ALSO_FOR_FILE_NAMESPACE = bit_flag(2), }; function Operand resolve_expr(Ast_Expr *ast, Resolve_Flag flags, Ast_Type *compound_context = 0);