New module scheme

This commit is contained in:
Krzosa Karol
2022-06-13 10:49:10 +02:00
parent cdaf85438e
commit b0553c38cf
12 changed files with 247 additions and 165 deletions

View File

@@ -1,76 +0,0 @@
//-----------------------------------------------------------------------------
// Function types
//-----------------------------------------------------------------------------
test_function :: (thing: S64): *S64
function_type: test_function
const_function_alias :: test_function
// null_function: (t: S64): *S64 = null
//-----------------------------------------------------------------------------
// Booleans
//-----------------------------------------------------------------------------
Boolean: Bool = true
//-----------------------------------------------------------------------------
// Nulls
//-----------------------------------------------------------------------------
// int_null: S64 = null
// str_null: String = null
// Bool_null: Bool = null
//-----------------------------------------------------------------------------
// Compound expressions
//-----------------------------------------------------------------------------
array1: [4]S64 = {1,2,3,4}
imp_array := [5]S64{1,2}
// imp_array_a := [5]S64{1,2,3,4,5,6}
// imp_array_b: [5]S64 = {1,2,3,4,5,6}
imp_array_c: [5]S64 = {[0] = 1, [2] = 2, [1] = 0} // @todo this should be illegal
// without_size: []S64 = {} // @todo: this should be slice, converting from array should be implicit
string: *char = "string"
first_letter := string[0]
decl_char: char = 55
thing: *void
//-----------------------------------------------------------------------------
// Pointers
//-----------------------------------------------------------------------------
pointer_decl : *S64
variable_from_deref: S64 = *pointer_decl
pointer_from_var : *S64 = &variable_from_deref
Boolean_pointer := &Boolean
//-----------------------------------------------------------------------------
// Implicit type
//-----------------------------------------------------------------------------
implicit_int :: 10
implicit_str :: "Hello world"
//-----------------------------------------------------------------------------
// Pointers
//-----------------------------------------------------------------------------
// pointer1: *S64 = 0
// pointer2: *S64 = pointer1
// pointer3: **S64 = 0
//-----------------------------------------------------------------------------
// String types
//-----------------------------------------------------------------------------
string1 :: "Test"
string2 :: string1
//-----------------------------------------------------------------------------
// Constant S64 variables
//-----------------------------------------------------------------------------
thing0 :: 10
thing1 :: thing0 + 11
thing2 :: thing1 + 20
combin :: thing0 + thing1 + thing2
Some :: struct
len: S64
cap: S64
some := Some{3, 2}
other := Some{len = 1, cap = 3}

View File

@@ -1,3 +1,5 @@
#load "globals.kl"
#load "lambdas.kl"
DWORD :: U32
LPCSTR :: *char

35
ast.cpp
View File

@@ -5,7 +5,7 @@
enum Ast_Kind: U32{
AST_NONE,
AST_PACKAGE,
AST_FILE_NAMESPACE,
AST_SCOPE,
AST_VALUE,
@@ -48,7 +48,7 @@ enum{
AST_ATOM = bit_flag(7),
AST_FOREIGN = bit_flag(8),
AST_DECL = bit_flag(9),
AST_PACKAGE_LEVEL = bit_flag(10),
AST_FILE_NAMESPACE_LEVEL = bit_flag(10),
AST_FLAG = bit_flag(11),
};
@@ -104,6 +104,10 @@ struct Ast_Unary: Ast_Expr{
U64 padding[2]; // For folding constants into atoms
};
struct Ast_Load: Ast{
String string;
};
struct Ast_Cast: Ast_Expr{
Ast_Expr *expr;
Ast_Expr *typespec;
@@ -196,6 +200,7 @@ struct Ast_Decl;
struct Ast_Scope: Ast{
Array<Ast_Decl *> decls;
Array<Ast *> stmts;
Ast_File *file;
};
struct Ast_Decl: Ast{
@@ -213,16 +218,10 @@ struct Ast_Decl: Ast{
INLINE_VALUE_FIELDS;
};
struct Ast_File{
String filename;
String filecontent;
Intern_String name;
Array<Ast_Decl *> decls;
struct Ast_File_Namespace : Ast_Decl{
Ast_File *file;
};
struct Ast_Package : Ast_Decl{};
//-----------------------------------------------------------------------------
// AST Constructors beginning with expressions
//-----------------------------------------------------------------------------
@@ -399,6 +398,8 @@ function Ast_Scope *
begin_decl_scope(Allocator *scratch, Token *pos){
AST_NEW(Scope, SCOPE, pos, AST_DECL);
result->decls = {scratch};
result->file = pctx->currently_parsed_file;
assert(result->file);
pctx->currently_parsed_scope = result;
return result;
}
@@ -414,6 +415,8 @@ begin_stmt_scope(Allocator *scratch, Token *pos){
AST_NEW(Scope, SCOPE, pos, AST_STMT);
result->stmts = {scratch};
result->decls = {pctx->heap};
result->file = pctx->currently_parsed_file;
assert(result->file);
pctx->currently_parsed_scope = result;
return result;
}
@@ -477,14 +480,16 @@ function Ast_Scope *
ast_decl_scope(Token *pos, Allocator *allocator){
AST_NEW(Scope, SCOPE, pos, AST_DECL);
result->decls = {allocator};
result->file = pctx->currently_parsed_file;
assert(result->file);
return result;
}
function Ast_Package *
ast_package(Token *pos, Allocator *allocator, Intern_String name){
AST_NEW(Package, PACKAGE, pos, 0);
result->kind = AST_PACKAGE;
result->scope = ast_decl_scope(pos, allocator);
function Ast_File_Namespace *
ast_file_namespace(Token *pos, Ast_File *file, Intern_String name){
AST_NEW(File_Namespace, FILE_NAMESPACE, pos, 0);
result->kind = AST_FILE_NAMESPACE;
result->file = file;
result->name = name;
return result;
}

View File

@@ -400,6 +400,8 @@ gen_ast(Ast *ast){
BREAK();
}
CASE(FILE_NAMESPACE, Decl){unused(node); BREAK();}
default: {
assert(is_flag_set(ast->flags, AST_EXPR));
gen_expr((Ast_Expr *)ast);
@@ -408,6 +410,125 @@ gen_ast(Ast *ast){
}
}
function void
parse_file(Ast_File *file){
Scratch scratch;
file->filecontent = os_read_file(pctx->perm, file->filename.s);
assert(file);
assert(file->filecontent.len);
assert(file->filename.len);
pctx->currently_parsed_file = file;
String name = string_chop_last_period(file->filename.s);
file->name = pctx->intern(name);
file->scope = ast_decl_scope(0, pctx->heap);
pctx->currently_parsed_scope = file->scope;
lex_restream(pctx, file->filecontent, file->filename.s);
while(token_expect(SAME_SCOPE)){
if(token_match_pound(pctx->intern("load"_s))){
parse_load(true);
continue;
}
Ast_Decl *decl = parse_decl(true);
if(!decl) break;
set_flag(decl->flags, AST_FILE_NAMESPACE_LEVEL);
if(decl->kind == AST_STRUCT){
decl->type = type_type;
decl->type_val = type_incomplete(decl);
decl->state = DECL_RESOLVED;
}
insert_into_scope(file->scope, decl);
}
pctx->currently_parsed_scope = 0;
pctx->currently_parsed_file = 0;
}
function void
parse_files(Ast_Module *module){
for(S64 i = 0; i < module->files.len; i++){
auto it = module->files.data[i];
parse_file(it);
}
}
function void
insert_type_into_file(Ast_File *p, String name, Ast_Type *type){
Intern_String string = pctx->intern(name);
Ast_Decl *decl = ast_type(&null_token, string, type);
decl->parent_scope = p->scope;
decl->state = DECL_RESOLVED;
insert_into_scope(p->scope, decl);
}
function void
insert_builtin_types_into_file(Ast_File *p){
insert_type_into_file(p, "void"_s , type_void);
insert_type_into_file(p, "Bool"_s , type_bool);
insert_type_into_file(p, "String"_s, type_string);
insert_type_into_file(p, "char"_s, type_char);
insert_type_into_file(p, "int"_s, type_int);
insert_type_into_file(p, "S8"_s, type_s8);
insert_type_into_file(p, "S16"_s, type_s16);
insert_type_into_file(p, "S32"_s, type_s32);
insert_type_into_file(p, "S64"_s, type_s64);
insert_type_into_file(p, "U8"_s, type_u8);
insert_type_into_file(p, "U16"_s, type_u16);
insert_type_into_file(p, "U32"_s, type_u32);
insert_type_into_file(p, "U64"_s, type_u64);
insert_type_into_file(p, "F32"_s, type_f32);
insert_type_into_file(p, "F64"_s, type_f64);
}
function Ast_Module *
parse_module(String filename){
Ast_Module *result = exp_alloc_type(pctx->perm, Ast_Module);
result->name = pctx->intern(filename);
result->files = {pctx->heap};
Ast_File *file = register_ast_file(result->name, result, true);
parse_files(result);
insert_builtin_types_into_file(file);
pctx->modules.add(result);
return result;
}
function void
resolve_everything_in_module(Ast_Module *module){
For(module->files){
For_It(it->scope->decls, jt){
resolve_name(it->scope, jt->pos, jt->name);
if(jt->kind == AST_STRUCT){
type_complete(jt->type);
}
}
}
}
function void
begin_compilation(){
OS_Heap *heap = exp_alloc_type(&pernament_arena, OS_Heap);
*heap = win32_os_heap_create(false, mib(16), 0);
Parse_Ctx *ctx = exp_alloc_type(&pernament_arena, Parse_Ctx);
parse_init(ctx, &pernament_arena, heap);
}
function String
end_compilation(){
For(pctx->ordered_decls){
genln("");
gen_ast(it);
}
exp_destroy(pctx->heap);
String string_result = string_flatten(pctx->perm, &pctx->gen);
return string_result;
}
#if 0
function String
compile_files(Array<String> filename){
Scratch scratch(thread_ctx.scratch);
@@ -444,11 +565,11 @@ compile_files(Array<String> filename){
it.name = package_token->intern_val;
}
Ast_Package *package = find_package(it.name, &pctx->packages);
Ast_File_Namespace *package = find_package(it.name, &pctx->packages);
if(package){
package->scope->decls.add(it.decls);
} else {
package = ast_package(token, &heap, it.name);
package = ast_file_namespace(token, &heap, it.name);
insert_builtin_types_into_package(package);
pctx->packages.add(package);
}
@@ -458,7 +579,7 @@ compile_files(Array<String> filename){
Ast_Decl *decl = parse_decl(true);
if(!decl) break;
set_flag(decl->flags, AST_PACKAGE_LEVEL);
set_flag(decl->flags, AST_FILE_NAMESPACE_LEVEL);
if(decl->kind == AST_STRUCT){
decl->type = type_type;
decl->type_val = type_incomplete(decl);
@@ -537,3 +658,4 @@ int main(){
return string_result;
}
#endif

View File

@@ -153,7 +153,6 @@ Intern_String keyword_if;
Intern_String keyword_else;
Intern_String keyword_true;
Intern_String keyword_false;
Intern_String keyword_package;
Intern_String keyword_for;
Intern_String keyword_pass;
Intern_String keyword_cast;
@@ -164,10 +163,25 @@ Intern_String intern_foreign;
Intern_String intern_strict;
Intern_String intern_flag;
struct Ast_Scope;
struct Ast_Decl;
struct Ast_Package;
struct Ast_File_Namespace;
struct Ast_File;
struct Ast_Module{
Intern_String name;
Array<Ast_File *> files;
};
struct Ast_File{
Intern_String filename;
Ast_Module *module;
B32 global_implicit_load;
String filecontent;
Intern_String name;
Ast_Scope *scope;
};
struct Parse_Ctx:Lexer{
Allocator *perm; // Stores: AST, tokens, interns
Allocator *heap;
@@ -175,8 +189,10 @@ struct Parse_Ctx:Lexer{
U64 unique_ids;
Map type_map;
Array<Ast_Package *> packages;
Array<Ast_Module *> modules;
Array<Ast_File_Namespace *> packages;
Ast_Scope *currently_parsed_scope;
Ast_File *currently_parsed_file;
Array<Ast_Decl *> ordered_decls;
S64 indent;
@@ -200,7 +216,6 @@ lex_init(Allocator *token_string_arena, Allocator *map_allocator, Lexer *l){
keyword_cast = l->intern("cast"_s);
keyword_true = l->intern("true"_s);
keyword_false = l->intern("false"_s);
keyword_package = l->intern("package"_s);
keyword_return = l->intern("return"_s);
keyword_if = l->intern("if"_s);
keyword_pass = l->intern("pass"_s);
@@ -223,6 +238,7 @@ parse_init(Parse_Ctx *ctx, Allocator *perm_allocator, Allocator *heap_allocator)
ctx->gen = {ctx->perm};
ctx->ordered_decls = {ctx->heap};
ctx->type_map = {ctx->heap};
ctx->modules = {ctx->heap};
bigint_allocator = ctx->perm;
lex_init(ctx->perm, ctx->heap, ctx);

View File

@@ -1,5 +1,3 @@
package Memory
Allocator_Kind :: enum #flag
Null
Arena

View File

@@ -1,11 +1,11 @@
package lambdas
Test :: struct
len: S64
test: Test
member := test.len
Memory :: #load "enums.kl"
imp_val: Allocator_Kind = Allocator_Kind.Heap
enum_val: Memory.Allocator_Kind = Memory.Allocator_Kind.Heap
other_enum_val: S64 = cast(enum_val: S64)
@@ -14,7 +14,7 @@ pointer_type :: *S64
// null_pointer: pointer_type = null
if_stmt :: (cond: S64): type
new_types.basic_type_assignment()
//new_types.basic_type_assignment()
CONSTANT :: 10
thing := 10
if i := thing + cond, cond > CONSTANT

View File

@@ -31,6 +31,14 @@ For now I don't thing it should be overloadable.
-------------------------------------------------------------------------------
We compile lot's of files, we keep track of them in Parse_Ctx, making sure we don't
parse same thing twice. Files belong to a module, files can be loaded #load "file".
All the files see all the decls from all the files in that module. We can import
other modules using a different directive #import. #import perhaps should be lazily
evaluated, making sure we don't resolve stuff we don't require. Currently probably
want to export all the symbols, we can namespace them optionally.
-------------------------------------------------------------------------------
@todo
[ ] - Should compound resolution use an algorithm to reorder compounds to initialize all fields in order
@@ -40,6 +48,7 @@ For now I don't thing it should be overloadable.
[ ] - slices should be properly displayed in debugger
[ ] - Rewrite where # happen,
[ ] - cast ->
[ ] - elif
[ ] - #assert that handles constants at compile time and vars at runtime
[ ] - Comma notation when declaring variables thing1, thing2: S32
@@ -163,16 +172,21 @@ int main(int argument_count, char **arguments){
system((const char *)run_program.str);
}
#else
Scratch scratch;
Array<String> files = {scratch};
files.add("lambdas.kl"_s);
files.add("order1.kl"_s);
files.add("order2.kl"_s);
files.add("new_types.kl"_s);
files.add("enums.kl"_s);
files.add("Windows.kl"_s);
// Scratch scratch;
// Array<String> files = {scratch};
// files.add("lambdas.kl"_s);
// files.add("order1.kl"_s);
// files.add("order2.kl"_s);
// files.add("new_types.kl"_s);
// files.add("enums.kl"_s);
// files.add("Windows.kl"_s);
// files.add("euler.kl"_s);
String result = compile_files(files);
// String result = compile_files(files);
begin_compilation();
Ast_Module *module = parse_module("Windows.kl"_s);
assert(module);
resolve_everything_in_module(module);
String result = end_compilation();
printf("%s", result.str);
#endif
__debugbreak();

View File

@@ -1,5 +1,3 @@
package lambdas
recursive_lambda :: (thing: S64)
in_val := recursive_lambda
some_value := thing + const_in_lambda

View File

@@ -582,6 +582,34 @@ parse_enum(Token *pos){
return result;
}
function Ast_File *
register_ast_file(Intern_String filename, Ast_Module *module, B32 global_implicit_load){
Ast_File *file = 0;
For(module->files){
if(it->filename == filename){
file = it;
break;
}
}
if(!file){
file = exp_alloc_type(pctx->perm, Ast_File, AF_ZeroMemory);
file->filename = filename;
file->module = module;
file->module->files.add(file);
}
if(global_implicit_load) file->global_implicit_load = true;
return file;
}
function Ast_File *
parse_load(B32 global_implicit_load){
Token *file = token_expect(TK_StringLit);
Ast_File *result = register_ast_file(file->intern_val, pctx->currently_parsed_file->module, global_implicit_load);
return result;
}
/*
Needs peeking only because I didn't want to duplicate code
for parsing statements and it makes code nicer.
@@ -614,6 +642,11 @@ parse_decl(B32 is_global){
result = parse_enum(tname);
}
else if(token_match_pound(pctx->intern("load"_s))){
Ast_File *file = parse_load(false);
result = ast_file_namespace(tname, file, tname->intern_val);
}
else{
Ast_Expr *expr = parse_expr();
result = ast_const(tname, tname->intern_val, expr);

View File

@@ -322,16 +322,6 @@ _search_for_decl(Ast_Scope *scope, Intern_String name){
return 0;
}
function Ast_Package *
search_for_package(Intern_String name){
For(pctx->packages){
if(name == it->name){
return it;
}
}
return 0;
}
function Ast_Decl *
search_for_decl(Ast_Scope *scope, Intern_String name, Search_Flag flags = 0){
Ast_Decl *result = 0;
@@ -340,10 +330,18 @@ search_for_decl(Ast_Scope *scope, Intern_String name, Search_Flag flags = 0){
if(result) break;
if(is_flag_set(flags, SEARCH_ONLY_CURRENT_SCOPE)) break;
}
if(!result && is_flag_set(flags, SEARCH_ALSO_FOR_PACKAGE)){
result = (Ast_Decl *)search_for_package(name);
}
if(!result && !is_flag_set(flags, SEARCH_ONLY_CURRENT_SCOPE)){
For(scope->file->module->files){
if(scope->file != it){
// If parsing the scope can be invalid
if(it->scope == 0 && pctx->currently_parsed_file != 0) continue;
if(it->global_implicit_load == false) continue;
result = _search_for_decl(it->scope, name);
if(result) break;
}
}
}
return result;
}
@@ -359,7 +357,7 @@ insert_into_scope(Ast_Scope *scope, Ast_Decl *decl){
}
function void
insert_type_into_package(Ast_Package *p, String name, Ast_Type *type){
insert_type_into_package(Ast_File_Namespace *p, String name, Ast_Type *type){
Intern_String string = pctx->intern(name);
Ast_Decl *decl = ast_type(&null_token, string, type);
decl->parent_scope = p->scope;
@@ -368,7 +366,7 @@ insert_type_into_package(Ast_Package *p, String name, Ast_Type *type){
}
function void
insert_builtin_types_into_package(Ast_Package *p){
insert_builtin_types_into_package(Ast_File_Namespace *p){
insert_type_into_package(p, "void"_s , type_void);
insert_type_into_package(p, "Bool"_s , type_bool);
insert_type_into_package(p, "String"_s, type_string);
@@ -437,7 +435,7 @@ require_const_int(Ast_Expr *expr, Resolve_Flag flags, Ast_Type *compound_context
function void
resolve_stmt(Ast *ast, Ast_Type *ret){
if(!ast) return;
assert(ast->parent_scope->kind == AST_SCOPE || ast->parent_scope->kind == AST_PACKAGE);
assert(ast->parent_scope->kind == AST_SCOPE || ast->parent_scope->kind == AST_FILE_NAMESPACE);
switch(ast->kind){
@@ -666,9 +664,9 @@ resolve_field_access(Ast_Expr *node, Ast_Scope *context){
Ast_Atom *ident = (Ast_Atom *)node;
Ast_Scope *scope = context ? context : node->parent_scope;
Search_Flag flag = context ? SEARCH_ONLY_CURRENT_SCOPE : SEARCH_ALSO_FOR_PACKAGE;
Search_Flag flag = context ? SEARCH_ONLY_CURRENT_SCOPE : SEARCH_ALSO_FOR_FILE_NAMESPACE;
Ast_Decl *decl = resolve_name(scope, node->pos, ident->intern_val, flag);
if(decl->kind == AST_PACKAGE){
if(decl->kind == AST_FILE_NAMESPACE){
assert(next);
return resolve_field_access(next, decl->scope);
}
@@ -696,7 +694,7 @@ function Operand
resolve_expr(Ast_Expr *ast, Resolve_Flag flags, Ast_Type *compound_context){
if(!ast && is_flag_set(flags, AST_CAN_BE_NULL)) return {};
assert(is_flag_set(ast->flags, AST_EXPR));
assert(ast->parent_scope->kind == AST_SCOPE || ast->parent_scope->kind == AST_PACKAGE);
assert(ast->parent_scope->kind == AST_SCOPE || ast->parent_scope->kind == AST_FILE_NAMESPACE);
switch(ast->kind){
@@ -983,7 +981,7 @@ resolve_decl(Ast_Decl *ast){
BREAK();
}
CASE(PACKAGE, Package){
CASE(FILE_NAMESPACE, File_Namespace){
unused(node);
BREAK();
}
@@ -1017,7 +1015,7 @@ resolve_decl(Ast_Decl *ast){
}
ast->state = DECL_RESOLVED;
if(is_flag_set(ast->flags, AST_PACKAGE_LEVEL))
if(is_flag_set(ast->flags, AST_FILE_NAMESPACE_LEVEL))
pctx->ordered_decls.add(ast);
}
@@ -1028,31 +1026,3 @@ resolve_name(Ast_Scope *scope, Token *pos, Intern_String name, Search_Flag searc
resolve_decl(decl);
return decl;
}
function void
resolve_package(Ast_Package *package){
For(package->scope->decls){
resolve_name(package->scope, it->pos, it->name);
if(it->kind == AST_STRUCT){
type_complete(it->type);
}
}
}
function Ast_File
ast_file(String filename, String filecontent){
Ast_File result = {};
result.filecontent = filecontent;
result.filename = filename;
return result;
}
function Ast_Package *
find_package(Intern_String name, Array<Ast_Package *> *packages){
For(*packages){
if(it->name == name){
return it;
}
}
return 0;
}

View File

@@ -14,7 +14,7 @@ FLAG32(Resolve_Flag){
FLAG32(Search_Flag){
SEARCH_ONLY_CURRENT_SCOPE = bit_flag(1),
SEARCH_ALSO_FOR_PACKAGE = bit_flag(2),
SEARCH_ALSO_FOR_FILE_NAMESPACE = bit_flag(2),
};
function Operand resolve_expr(Ast_Expr *ast, Resolve_Flag flags, Ast_Type *compound_context = 0);