From e3b5e9b33a349320a7965f4379ff0c722f675f43 Mon Sep 17 00:00:00 2001 From: Krzosa Karol Date: Fri, 6 May 2022 10:13:16 +0200 Subject: [PATCH] New approach, new lexer --- ast.c | 455 ++++++++++++++++++++++++++++++++++++++- ast.h | 309 +++++++++++++++------------ build.bat | 5 +- codegen_c.c | 531 ++++++++++++++++++++++++++++++++++++++-------- common.c | 112 +++++++++- file.c | 21 ++ generate.c | 36 ++++ generated.c | 69 ++++++ generated_ast.cpp | 0 generated_ast.h | 225 ++++++++++++++++++++ generated_lex.cpp | 53 +++++ generated_lex.h | 136 ++++++++++++ ideas.cpp | 13 +- lang.h | 20 +- lex.c | 33 ++- lex.h | 200 ++++++++--------- main.c | 113 +--------- memory.c | 30 ++- memory.h | 5 +- new_ast.c | 295 ++++++++++++++++++++++++++ new_lex.c | 516 ++++++++++++++++++++++++++++++++++++++++++++ new_parse.c | 53 +++++ old_main.c | 66 ++++++ os.h | 4 +- os_win32.c | 28 +-- output.cc | 229 ++++++++++++-------- parse_decl.c | 371 ++++++++++++++------------------ parse_expr.c | 7 + parser.c | 12 +- parser.h | 1 + scratch.c | 14 ++ test.cc | 53 ++++- token_array.c | 100 +++++++++ 33 files changed, 3331 insertions(+), 784 deletions(-) create mode 100644 file.c create mode 100644 generate.c create mode 100644 generated.c create mode 100644 generated_ast.cpp create mode 100644 generated_ast.h create mode 100644 generated_lex.cpp create mode 100644 generated_lex.h create mode 100644 new_ast.c create mode 100644 new_lex.c create mode 100644 new_parse.c create mode 100644 old_main.c create mode 100644 scratch.c create mode 100644 token_array.c diff --git a/ast.c b/ast.c index 6c7f63c..9da9d4a 100644 --- a/ast.c +++ b/ast.c @@ -81,6 +81,19 @@ decl_list_push(Decl *parent, Decl *child){ SLLQueuePush(parent->list.first, parent->list.last, child); } +//----------------------------------------------------------------------------- +// Ops +//----------------------------------------------------------------------------- +function Decl * +decl_struct_find_node(Decl *decl, String string){ + for(Decl *n = decl->struct_decl.first; n; n=n->next){ + if(string_compare(string, n->name.s)){ + return n; + } + } + return 0; +} + //----------------------------------------------------------------------------- // Notes //----------------------------------------------------------------------------- @@ -172,4 +185,444 @@ typespec_function(Parser *p, Token *pos, Typespec *ret){ function void typespec_function_push(Typespec *func, Typespec *arg){ SLLQueuePush(func->function_spec.first, func->function_spec.last, arg); -} \ No newline at end of file +} + +//----------------------------------------------------------------------------- +// +//----------------------------------------------------------------------------- +function Typespec * +typespec_get_func(Typespec *type){ + switch(type->kind){ + case TS_Name:{return 0;} break; + case TS_Pointer:{return typespec_get_func(type->base);} break; + case TS_Array:{return typespec_get_func(type->array_spec.base);} break; + case TS_Function:{return type;} break; + default: {invalid_codepath;} break; + } + return 0; +} + +function Typespec * +typespec_get_name(Typespec *type){ + switch(type->kind){ + case TS_Name:{return type;} break; + case TS_Pointer:{return typespec_get_name(type->base);} break; + case TS_Array:{return typespec_get_name(type->array_spec.base);} break; + case TS_Function:{return 0;} break; + default: {invalid_codepath;} break; + + } + return 0; +} + +function String +typespec_require_name_string(Typespec *type){ + Typespec *result = typespec_get_name(type); + if(result){ + if(result->kind == TS_Name){ + return result->name.s; + } + } + invalid_codepath; + return string_empty; +} + +//----------------------------------------------------------------------------- +// Statements +//----------------------------------------------------------------------------- +function Stmt * +stmt_new(Parser *p, Stmt_Kind kind, Token *pos){ + Stmt *result = arena_push_struct(&p->main_arena, Stmt); + result->kind = kind; + result->pos = pos; + return result; +} + +function Stmt * +stmt_decl(Parser *p, Token *pos, Decl *decl){ + Stmt *result = stmt_new(p, STMT_Decl, pos); + result->decl = decl; + return result; +} + +function Stmt * +stmt_expr(Parser *p, Token *pos, Expr *expr){ + Stmt *result = stmt_new(p, STMT_Expr, pos); + result->expr = expr; + return result; +} + +function Stmt * +stmt_list(Parser *p, Token *pos){ + Stmt *result = stmt_new(p, STMT_List, pos); + return result; +} + +function Stmt * +stmt_return(Parser *p, Token *pos, Expr *expr){ + Stmt *result = stmt_new(p, STMT_Return, pos); + result->ret.expr = expr; + return result; +} + +function Stmt * +stmt_if(Parser *p, Token *pos, Stmt *body, Expr *cond){ + Stmt *result = stmt_new(p, STMT_If, pos); + result->stmt_if.cond = cond; + result->stmt_if.body = body; + return result; +} + +function void +stmt_push(Stmt *stmt, Stmt *child){ + SLLQueuePush(stmt->list.first, stmt->list.last, child); +} + +//----------------------------------------------------------------------------- +// Pointer Array +//----------------------------------------------------------------------------- +function Pointer_Array +pointer_array_make(Arena *arena){ + Pointer_Array result = { + .last = &result.first, + .arena = arena, + }; + return result; +} + +#define bucket_size (buff_cap(array->first.data)) +function void +pointer_array_push(Pointer_Array *array, Pointer p){ + if(array->len >= bucket_size){ + Pointer_Bucket *bucket = arena_push_struct(array->arena, Pointer_Bucket); + array->last = array->last->next = bucket; + array->len = 0; + array->block += 1; + } + array->last->data[array->len++] = p; +} + +function B32 +pointer_array_iter_is_end(Pointer_Array *array){ + B32 result = array->iter_len == array->len && array->iter_block == array->block; + return result; +} + +function Pointer +pointer_array_iter_next(Pointer_Array *array){ + if(pointer_array_iter_is_end(array)){ + return (Pointer){0}; + } + if(array->iter_len >= bucket_size){ + array->iter_len = 0; + array->iter_block += 1; + array->iter_bucket = array->iter_bucket->next; + } + Pointer result = array->iter_bucket->data[array->iter_len++]; + return result; +} + +function Pointer +pointer_array_iter_begin(Pointer_Array *array){ + array->iter_len = 0; + array->iter_block = 0; + array->iter_bucket = &array->first; + Pointer result = pointer_array_iter_next(array); + return result; +} + +function Pointer +pointer_typespec(Typespec *t){ + return (Pointer){.kind = PK_Typespec, .typespec = t}; +} +function Pointer +pointer_expr(Expr *t){ + return (Pointer){.kind = PK_Expr, .expr = t}; +} +function Pointer +pointer_decl(Decl *t){ + return (Pointer){.kind = PK_Decl, .decl = t}; +} +function Pointer +pointer_stmt(Stmt *t){ + return (Pointer){.kind = PK_Stmt, .stmt = t}; +} +function Pointer +pointer_enum_child(Decl_Enum_Child *t){ + return (Pointer){.kind = PK_Enum_Child, .enum_child = t}; +} +function Pointer +pointer_func_arg(Decl_Function_Arg *t){ + return (Pointer){.kind = PK_Func_Arg, .func_arg=t}; +} + +function void +pointer_array_push_typespec(Pointer_Array *array, Typespec *typespec){ + pointer_array_push(array, pointer_typespec(typespec)); +} +function void +pointer_array_push_decl(Pointer_Array *array, Decl *decl){ + pointer_array_push(array, pointer_decl(decl)); +} +function void +pointer_array_push_stmt(Pointer_Array *array, Stmt *stmt){ + pointer_array_push(array, pointer_stmt(stmt)); +} +function void +pointer_array_push_expr(Pointer_Array *array, Expr *expr){ + pointer_array_push(array, pointer_expr(expr)); +} +function void +pointer_array_push_enum_child(Pointer_Array *array, Decl_Enum_Child *enum_child){ + pointer_array_push(array, pointer_enum_child(enum_child)); +} +function void +pointer_array_push_func_arg(Pointer_Array *array, Decl_Function_Arg *func_arg){ + pointer_array_push(array, pointer_func_arg(func_arg)); +} + +//----------------------------------------------------------------------------- +// Gather +//----------------------------------------------------------------------------- + +function void +gather_try_recurse_typespec(Pointer_Array *array, Typespec *typespec, Gather_Flag gflag, Traversal_Flag tflag); +function void +gather_recurse_expr(Pointer_Array *array, Expr *expr, Gather_Flag gflag, Traversal_Flag tflag){ + if(gflag & GATHER_Expr) + pointer_array_push_expr(array, expr); + + switch(expr->kind) { + case EK_Atom: {} break; + case EK_Sizeof:{ + if(expr->size_of.kind == SIZEOF_Expr){ + gather_recurse_expr(array, expr->size_of.expr, gflag, tflag); + } + else{ + assert(expr->size_of.kind == SIZEOF_Type); + gather_try_recurse_typespec(array, expr->size_of.type, gflag, tflag); + } + }break; + + case EK_Binary:{ + gather_recurse_expr(array, expr->binary.left, gflag, tflag); + gather_recurse_expr(array, expr->binary.right, gflag, tflag); + } break; + case EK_Unary:{ + gather_recurse_expr(array, expr->unary.expr, gflag, tflag); + } break; + + case EK_Ternary:{ + gather_recurse_expr(array, expr->ternary.cond, gflag, tflag); + gather_recurse_expr(array, expr->ternary.on_true, gflag, tflag); + gather_recurse_expr(array, expr->ternary.on_false, gflag, tflag); + } break; + case EK_List:{ + for(Expr *n = expr->list.first; n; n=n->next){ + gather_recurse_expr(array, n, gflag, tflag); + } + }break; + + case EK_Cast:{ + gather_try_recurse_typespec(array, expr->cast.type, gflag, tflag); + gather_recurse_expr(array, expr->cast.expr, gflag, tflag); + } break; + + case EK_Index:{ + gather_recurse_expr(array, expr->index.atom, gflag, tflag); + gather_recurse_expr(array, expr->index.index, gflag, tflag); + }break; + + case EK_Call:{ + gather_recurse_expr(array, expr->call.atom, gflag, tflag); + gather_recurse_expr(array, expr->call.list, gflag, tflag); + }break; + default: {invalid_codepath;} break; + } +} + +function void +gather_try_recurse_expr(Pointer_Array *array, Expr *expr, Gather_Flag gflag, Traversal_Flag tflag){ + if(expr){ + if(tflag & TRAVERS_Expr){ + gather_recurse_expr(array, expr, gflag, tflag); + } + else if(gflag & GATHER_Expr){ + pointer_array_push_expr(array, expr); + } + } +} + +function void +gather_recurse_decl(Pointer_Array *array, Decl *decl, Gather_Flag gflag, Traversal_Flag tflag); +function void +gather_try_recurse_stmt(Pointer_Array *array, Stmt *stmt, Gather_Flag gflag, Traversal_Flag tflag); +function void +gather_recurse_stmt(Pointer_Array *array, Stmt *stmt, Gather_Flag gflag, Traversal_Flag tflag){ + switch(stmt->kind) { + case STMT_List: { + for(Stmt *s = stmt->list.first; s; s=s->next){ + gather_recurse_stmt(array, s, gflag, tflag); + } + } break; + case STMT_Return:{ + gather_try_recurse_expr(array, stmt->ret.expr, gflag, tflag); + } break; + case STMT_If:{ + gather_recurse_expr(array, stmt->stmt_if.cond, gflag, tflag); + gather_recurse_stmt(array, stmt->stmt_if.body, gflag, tflag); + for(Stmt_If *s = stmt->stmt_if.next; s; s=s->next){ + gather_recurse_stmt(array, s->body, gflag, tflag); + gather_recurse_expr(array, s->cond, gflag, tflag); + } + } break; + case STMT_Expr: { + gather_recurse_expr(array, stmt->expr, gflag, tflag); + } break; + case STMT_Decl: { + gather_recurse_decl(array, stmt->decl, gflag, tflag); + } break; + default: {invalid_codepath;} break; + } +} + +function void +gather_try_recurse_stmt(Pointer_Array *array, Stmt *stmt, Gather_Flag gflag, Traversal_Flag tflag){ + if(stmt){ + if(tflag & TRAVERS_Stmt){ + gather_recurse_stmt(array, stmt, gflag, tflag); + } + else if(gflag & GATHER_Stmt){ + pointer_array_push_stmt(array, stmt); + } + } +} + +function void +gather_recurse_typespec(Pointer_Array *array, Typespec *typespec, Gather_Flag gflag, Traversal_Flag tflag){ + if(gflag & GATHER_Typespec) + pointer_array_push_typespec(array, typespec); + switch(typespec->kind) { + case TS_Name: { + } break; + case TS_Pointer: { + gather_recurse_typespec(array, typespec->base, gflag, tflag); + } break; + case TS_Array: { + gather_recurse_typespec(array, typespec->array_spec.base, gflag, tflag); + gather_try_recurse_expr(array, typespec->array_spec.size, gflag, tflag); + } break; + case TS_Function: { + gather_recurse_typespec(array, typespec->function_spec.ret, gflag, tflag); + for(Typespec *n = typespec->function_spec.first; n; n=n->next){ + gather_recurse_typespec(array, n, gflag, tflag); + } + } break; + default: {invalid_codepath;} break; + } +} + +function void +gather_try_recurse_typespec(Pointer_Array *array, Typespec *typespec, Gather_Flag gflag, Traversal_Flag tflag){ + if(typespec){ + if(tflag & TRAVERS_Typespec){ + gather_recurse_typespec(array, typespec, gflag, tflag); + } + else if(gflag & GATHER_Typespec){ + pointer_array_push_typespec(array, typespec); + } + } +} + +function void +gather_recurse_decl(Pointer_Array *array, Decl *decl, Gather_Flag gflag, Traversal_Flag tflag){ + if(gflag & GATHER_Decl) pointer_array_push_decl(array, decl); + + switch(decl->kind){ + case DECL_Struct: + case DECL_Union:{ + for(Decl *n = decl->struct_decl.first; n; n=n->next){ + gather_recurse_decl(array, n, gflag, tflag); + } + } break; + case DECL_Enum:{ + gather_try_recurse_typespec(array, decl->enum_decl.typespec, gflag, tflag); + for(Decl_Enum_Child *child = decl->enum_decl.first; child; child=child->next){ + if(gflag & GATHER_Enum_Child) pointer_array_push_enum_child(array, child); + gather_try_recurse_expr(array, child->expr, gflag, tflag); + } + }break; + case DECL_Variable:{ + gather_try_recurse_typespec(array, decl->variable_decl.type, gflag, tflag); + gather_try_recurse_expr(array, decl->variable_decl.expr, gflag, tflag); + }break; + case DECL_Typedef:{ + gather_try_recurse_typespec(array, decl->typedef_decl.type, gflag, tflag); + }break; + case DECL_Function:{ + gather_try_recurse_typespec(array, decl->function_decl.ret, gflag, tflag); + for(Decl_Function_Arg *arg = decl->function_decl.first; arg; arg=arg->next){ + if(gflag & GATHER_Func_Arg) pointer_array_push_func_arg(array, arg); + gather_try_recurse_typespec(array, arg->typespec, gflag, tflag); + } + gather_try_recurse_stmt(array, decl->function_decl.body, gflag, tflag); + }break; + case DECL_List:{ + for(Decl *n = decl->list.first; n; n=n->next){ + gather_recurse_decl(array, n, gflag, tflag); + } + }break; + default:invalid_codepath; + } + +} + +function Pointer_Array +gather(Arena *arena, Decl *decl, Gather_Flag gflag, Traversal_Flag tflag){ + Pointer_Array array = pointer_array_make(arena); + gather_recurse_decl(&array, decl, gflag, tflag); + return array; +} + +//----------------------------------------------------------------------------- +// +//----------------------------------------------------------------------------- + +function Decl * +decl_deep_copy_recurse(Arena *arena, Decl *decl){ + Decl *result = arena_push_struct(arena, Decl); + memory_copy(result, decl, sizeof(*decl)); + switch(decl->kind) { + case DECL_Struct:{ + + }break; + case DECL_Union:{ + + }break; + case DECL_Enum:{ + + }break; + case DECL_Variable:{ + + }break; + case DECL_Typedef:{ + + }break; + case DECL_Function:{ + + }break; + case DECL_List:{ + + }break; + default: {invalid_codepath;} break; + } + return result; +} + +function Decl * +decl_deep_copy(Arena *arena){ + Decl *result = decl_deep_copy_recurse(arena, decl); + return result; +} + + diff --git a/ast.h b/ast.h index dd6f13a..05bf437 100644 --- a/ast.h +++ b/ast.h @@ -1,164 +1,201 @@ -typedef struct Decl_Enum_Child Decl_Enum_Child; -typedef struct Decl_Function_Arg Decl_Function_Arg; -typedef struct Typespec Typespec; -typedef struct Decl Decl; -typedef struct Note Note; -typedef struct Stmt Stmt; - +#if 0 //----------------------------------------------------------------------------- -// Type specifier +// Type specifiers //----------------------------------------------------------------------------- -typedef enum Typespec_Kind{ - TS_None, - TS_Name, - TS_Pointer, - TS_Array, - TS_Function, -}Typespec_Kind; - -struct Typespec{ - Typespec_Kind kind; - Typespec *next; - Token *pos; +@prefix="TS_" Typespec_Kind :: enum{ None, Name, Pointer, Array, Function } +Typespec :: struct{ + kind: Typespec_Kind; + next: Typespec*; + pos : Token*; + union{ - Intern_String name; - struct{ - Typespec *first; - Typespec *last; - Typespec *ret; - }function_spec; - struct{ - Typespec *base; - Expr *size; - }array_spec; - Typespec *base; - }; -}; + name: Intern_String; + base: Typespec*; + function_spec: struct{ + first: Typespec*; + last : Typespec*; + ret : Typespec*; + } + array_spec: struct{ + base: Typespec*; + size: Expr*; + } + } +} //----------------------------------------------------------------------------- // Notes //----------------------------------------------------------------------------- -struct Note{ - Token *pos; - Intern_String name; - Expr *expr; +Note :: struct{ + pos : Token*; + name: Intern_String; + expr: Expr*; - Note *next; - Note *first; - Note *last; + next : Note*; + first: Note*; + last : Note*; }; //----------------------------------------------------------------------------- // Declarations //----------------------------------------------------------------------------- -typedef enum Decl_Kind{ - DECL_None, - DECL_Struct, - DECL_Union, - DECL_Enum, - DECL_Variable, - DECL_Typedef, - DECL_Function, - DECL_List, -}Decl_Kind; +@prefix = "DECL_" Decl_Kind :: enum { None, Struct, Union, Enum, Variable, Typedef, Function, List } +@prefix="STRUCT_" Decl_Struct_Kind :: enum { Nested, Base } -typedef enum Decl_Struct_Kind{ - STRUCT_Base , - STRUCT_Nested, -}Decl_Struct_Kind; +Decl_Function_Arg :: struct{ + next: Decl_Function_Arg *; + name: Intern_String; + pos : Token *; + typespec: Typespec *; +} -struct Decl_Function_Arg{ - Decl_Function_Arg *next; - Intern_String name; - Typespec *typespec; - Token *pos; +Decl_Enum_Child :: struct{ + next: Decl_Enum_Child *; + name: Intern_String; + pos : Token *; + expr: Expr *; + + first_note: Note *; + last_note : Note *; }; -struct Decl_Enum_Child{ - Decl_Enum_Child *next; - Intern_String name; - Token *pos; - Expr *expr; +Decl :: struct{ + kind: Decl_Kind; + next: Decl *; - Note *first_note; - Note *last_note; -}; - -struct Decl{ - Decl_Kind kind; - Decl *next; + name: Intern_String; + pos : Token *; - Intern_String name; - Token *pos; - - Note *first_note; - Note *last_note; + first_note: Note *; + last_note : Note *; union{ - struct{ - Decl_Enum_Child *first; - Decl_Enum_Child *last; - Typespec *typespec; - }enum_decl; - struct{ - Decl_Struct_Kind kind; - Decl *first; - Decl *last; - } struct_decl; - struct{ - Typespec *type; - Expr *expr; - }variable_decl; - struct{ - Typespec *type; - }typedef_decl; - struct{ - Decl_Function_Arg *first; - Decl_Function_Arg *last ; - Typespec *ret; - }function_decl; - struct{ - Decl *first; - Decl *last; - }list; - }; -}; + enum_decl: struct{ + first: Decl_Enum_Child *; + last: Decl_Enum_Child *; + typespec: Typespec *; + } + struct_decl: struct{ + first: Decl *; + last: Decl *; + kind: Decl_Struct_Kind ; + } + variable_decl: struct{ + type: Typespec *; + expr: Expr *; + } + typedef_decl: struct{ + type: Typespec *; + } + function_decl: struct{ + first: Decl_Function_Arg *; + last : Decl_Function_Arg *; + ret: Typespec *; + body : Stmt*; + } + list: struct{ + first: Decl *; + last: Decl *; + } + } +} //----------------------------------------------------------------------------- // Statements //----------------------------------------------------------------------------- -typedef enum Stmt_Kind{ - STMT_None, - STMT_Decl, - STMT_Expr, - STMT_Return, - STMT_If, - STMT_For, -}Stmt_Kind; +@prefix="STMT_" Stmt_Kind :: enum{ None, Decl, Expr, List, Return, If, For} -typedef struct Stmt_List Stmt_List; -struct Stmt_List{ - Stmt *first; - Stmt *last; -}; +Stmt_If :: struct { + next: Stmt_If*; + cond: Expr*; + body: Stmt*; +} -typedef struct Stmt_If Stmt_If; -struct Stmt_If{ - Stmt_If *next; - Expr *expr; - union{ - struct{Stmt *first; Stmt *last;}; - Stmt_List *list; - }; -}; - -struct Stmt{ - Stmt_Kind kind; - Stmt *next; +Stmt :: struct { + kind: Stmt_Kind; + next: Stmt*; + pos : Token*; + union{ - Decl *decl; - Expr *expr; - Stmt_If *if_stmt; - Stmt_List *list; - }; -}; + stmt_if: Stmt_If; + decl: Decl*; + expr: Expr*; + list: struct{ + first: Stmt*; + last : Stmt*; + } + ret: struct{ + expr: Expr*; + } + } +} + +//----------------------------------------------------------------------------- +// Gather +//----------------------------------------------------------------------------- + +@prefix="PK_" +Pointer_Kind::enum{ + None, + Typespec, + Expr, + Decl, + Stmt, + Enum_Child, + Func_Arg, + Intern_String +} + +Pointer::struct{ + kind: Pointer_Kind; + union { + typespec: Typespec *; + decl: Decl *; + expr: Expr *; + stmt: Stmt *; + func_arg: Decl_Function_Arg*; + enum_child: Decl_Enum_Child*; + string: Intern_String *; + } +} + +Pointer_Bucket::struct{ + next: Pointer_Bucket*; + data: Pointer[4096]; +} + +Pointer_Array::struct{ + first: Pointer_Bucket; + last : Pointer_Bucket*; + len : S64; + block: S64; + arena: Arena*; + + iter_bucket: Pointer_Bucket*; + iter_len: S64; + iter_block: S64; +} + +@prefix="GATHER_" +Gather_Flag::enum{ + None = 0, + Typespec = 1, + Expr = 2, + Decl = 4, + Stmt = 8, + Enum_Child = 16, + Func_Arg = 32, +} + +@prefix="TRAVERS_" +Traversal_Flag :: enum{ + None, + Typespec = 1, + Expr = 2, + //Decl = 4, + Stmt = 8, + All = TRAVERS_Typespec | TRAVERS_Expr | TRAVERS_Stmt, +} + +#endif +#include "generated_ast.h" diff --git a/build.bat b/build.bat index 8ed2f5f..a498933 100644 --- a/build.bat +++ b/build.bat @@ -1,3 +1,6 @@ @echo off -clang main.c -fdiagnostics-absolute-paths -std=c99 -g -o main.exe -Wl,-subsystem:windows -Wl,user32.lib +rem clang generate.c -fdiagnostics-absolute-paths -std=c99 -g -o generate.exe -Wl,user32.lib +rem generate.exe +clang main.c -fdiagnostics-absolute-paths -std=c99 -g -o main.exe -Wl,user32.lib +rem cl main.c -std:c17 diff --git a/codegen_c.c b/codegen_c.c index 928754d..ecb21e7 100644 --- a/codegen_c.c +++ b/codegen_c.c @@ -1,126 +1,475 @@ -function void gen_decl(Parser *p, Decl *node); +function void gen_decl(Decl *node); +function void gen_cdecl(Typespec *type, String str); +function void gen_stmt_list(Stmt *stmt); + +global String_List *glist; +global Arena *gscratch; +global Parser *genp; +#define strf(...) string_listf(gscratch, glist, __VA_ARGS__) + +//----------------------------------------------------------------------------- +// String replacing util +//----------------------------------------------------------------------------- +typedef struct String_Map String_Map; +struct String_Map{ + String replace; + String with; +}; function void -gen_typespec(Parser *p, Typespec *spec, B32 is_left){ - switch(spec->kind) { - - case TS_Name: { - if(is_left) lex_print("%s", spec->name.s.str); - } break; - - case TS_Pointer: { - gen_typespec(p, spec->base,is_left); - if(is_left) lex_print("*"); - } break; - - case TS_Array: { - gen_typespec(p, spec->array_spec.base,is_left); - if(!is_left){ - lex_print("["); - expr_print(p, spec->array_spec.size); - lex_print("]"); +string_mapped_print(String string, String_Map *map, SizeU count){ + Tokens tokens = lex_stream(string, lit("string_mapped_print")); + for(Token *t = tokens.tokens; t != tokens.tokens + tokens.len; t++){ + String string = t->string; + for(SizeU i = 0; i < count; i++){ + if(string_compare(string, map[i].replace)){ + string = map[i].with; + break; } - } break; + } - case TS_Function: { - invalid_codepath; - } break; - default: {invalid_codepath;} break; + if(t->kind == TK_At) continue; + if(string_compare(t->string,keyword_function.s)) strf("\n"); + strf("%.*s", (int)string.len, string.str); + if(string_compare(t->string,keyword_function.s)) strf("\n"); + if(t->kind == TK_OpenBrace) strf("\n"); + if(t->kind == TK_CloseBrace && t[1].kind != TK_Semicolon) strf("\n"); + if(t->kind == TK_Semicolon) strf("\n"); + if(t->kind == TK_Identifier && (t[1].kind == TK_Keyword || t[1].kind == TK_Identifier)) strf(" "); + if(t->kind == TK_Keyword) strf(" "); + if(t->kind == TK_Comma) strf(" "); } + free(tokens.tokens); } +#define STR(X) lit(#X) -function Typespec * -typespec_get_func(Typespec *type){ - switch(type->kind){ - - case TS_Name:{return 0;} break; - case TS_Pointer:{return typespec_get_func(type->base);} break; - case TS_Array:{return typespec_get_func(type->array_spec.base);} break; - case TS_Function:{return type;} break; - default: {invalid_codepath;} break; - - } - return 0; +//----------------------------------------------------------------------------- +// Codegen +//----------------------------------------------------------------------------- + +function void +token_strf(Token *token){ + strf("%.*s", (S32)token->len, token->str); } function void -gen_decl(Parser *p, Decl *node){ - +gen_expr(Expr *expr){ + switch(expr->kind) { + case EK_Atom: { + token_strf(expr->token); + } break; + + case EK_Sizeof:{ + strf("sizeof("); + if(expr->size_of.kind == SIZEOF_Expr){ + gen_expr(expr->size_of.expr); + } + else{ + assert(expr->size_of.kind == SIZEOF_Type); + gen_cdecl(expr->size_of.type, string_empty); + } + strf(")"); + }break; + + case EK_Binary:{ + strf("("); + gen_expr(expr->binary.left); + token_strf(expr->token); + gen_expr(expr->binary.right); + strf(")"); + } break; + case EK_Unary:{ + strf("("); + token_strf(expr->token); + gen_expr(expr->unary.expr); + strf(")"); + } break; + + case EK_Ternary:{ + strf("("); + gen_expr(expr->ternary.cond); + strf("?"); + gen_expr(expr->ternary.on_true); + strf(":"); + gen_expr(expr->ternary.on_false); + strf(")"); + } break; + case EK_List:{ + strf("("); + for(Expr *n = expr->list.first; n; n=n->next){ + gen_expr(n); + if(n!=expr->list.last) strf(","); + } + strf(")"); + }break; + + case EK_Cast:{ + strf("("); + strf("("); + gen_cdecl(expr->cast.type, string_empty); + strf(")"); + gen_expr(expr->cast.expr); + strf(")"); + } break; + + case EK_Index:{ + gen_expr(expr->index.atom); + strf("["); + gen_expr(expr->index.index); + strf("]"); + }break; + + case EK_Call:{ + gen_expr(expr->call.atom); + strf("("); + gen_expr(expr->call.list); + strf(")"); + }break; + default: {invalid_codepath;} break; + } +} + +function String +gen_cdecl_paren(String str, String original){ + if(string_is_empty(original) || original.str[0] == '['){ + return str; + } + else{ + String result = string_fmt(gscratch, "(%.*s)", (int)str.len, str.str); + return result; + } +} + +function String +gen__cdecl(Typespec *type, String str){ + switch(type->kind) { + case TS_Name: { + String space = string_is_empty(str) ? lit(""):lit(" "); + String result = string_fmt(gscratch, "%s%s%.*s", type->name.s.str, space.str, (int)str.len, str.str); + return result; + } break; + case TS_Pointer: { + String pointer = string_fmt(gscratch, "*%.*s", (int)str.len, str.str); + String add_paren = gen_cdecl_paren(pointer, str); + String result = gen__cdecl(type->base, add_paren); + return result; + } break; + case TS_Array: { + String left = string_fmt(gscratch, "%s[", str.str); + + String_List *save = glist; + String_List list = {0}; + glist = &list; + gen_expr(type->array_spec.size); + String expr_string = string_list_flatten(gscratch, glist); + glist = save; + + + String right = string_fmt(gscratch, "%s%s]", left.str, expr_string.str); + String paren = gen_cdecl_paren(right, str); + String result = gen__cdecl(type->array_spec.base, paren); + return result; + } break; + case TS_Function: { + String result = string_fmt(gscratch, "(*%s)(", str.str); + if (type->function_spec.first == 0) { + result= string_fmt(gscratch, "%svoid", result.str); + } + else { + for(Typespec *n = type->function_spec.first; n; n=n->next){ + String arg = gen__cdecl(n, string_empty); + result = string_fmt(gscratch, "%s%s", result.str, arg.str); + if(n != type->function_spec.last) + result = string_fmt(gscratch, "%s, ", result.str); + } + } + result = string_fmt(gscratch, "%s)", result.str); + result = gen__cdecl(type->function_spec.ret, result); + return result; + } break; + default: {invalid_codepath;} break; + } + return string_empty; +} + +function void +gen_cdecl(Typespec *type, String str){ + String string = gen__cdecl(type, str); + strf("%.*s", (int)string.len, string.str); +} + +function void +gen_assign_expr(Expr *expr){ + if(expr){ + strf(" = "); + gen_expr(expr); + } +} + +function void +gen_function_decl(Decl *node){ + strf("function "); + gen_cdecl(node->function_decl.ret, string_empty); + strf(" %s(", node->name.s.str); + for(Decl_Function_Arg *arg = node->function_decl.first; arg; arg=arg->next){ + gen_cdecl(arg->typespec, arg->name.s); + if(arg!=node->function_decl.last) strf(", "); + } + strf(")"); +} + +typedef struct Macro{ + Intern_String name; + Note *param_type; + Note *param_expr; + Decl *decl; +}Macro; +global Macro macros[32]; +global S32 macros_i; + +function void +gen_forward_decl(Decl *node){ + U8 *name = node->name.s.str; switch(node->kind) { case DECL_List: { for(Decl *n = node->list.first; n; n=n->next){ - gen_decl(p,n); - lex_new_line(); + gen_forward_decl(n); } } break; case DECL_Variable:{ - Typespec *func = typespec_get_func(node->variable_decl.type); - if(!func){ - gen_typespec(p, node->variable_decl.type, true); - lex_print(" %s", node->name.s.str); - gen_typespec(p, node->variable_decl.type, false); - } - else{ - gen_typespec(p, func->function_spec.ret, true); - lex_print("(*"); - lex_print("%s", node->name.s.str); - lex_print(")"); - lex_print("("); - for(Typespec *t = func->function_spec.first; t; t=t->next){ - gen_typespec(p, t, true); - gen_typespec(p, t, false); - if(t != func->function_spec.last) lex_print(", "); - } - lex_print(")"); - } - print_assign_expr(p,node->variable_decl.expr); - lex_print(";"); } break; case DECL_Typedef:{ - lex_print("typedef %s ", node->name.s.str); - gen_typespec(p, node->typedef_decl.type, true); - gen_typespec(p, node->typedef_decl.type, false); - lex_print(";"); } break; + case DECL_Function:{ + gen_function_decl(node); + strf(";\n"); + }break; + case DECL_Struct: case DECL_Union :{ - U8 *struct_name = node->kind==DECL_Struct ? (U8*)"struct" : (U8*)"union"; - U8 *name = node->name.s.str; - if(node->struct_decl.kind == STRUCT_Base) - lex_print("typedef %s %s %s;\n", struct_name, name, name); - lex_print("%s %s{\n", struct_name, name?name:(U8*)""); - for(Decl *n = node->struct_decl.first; n; n=n->next){ - gen_decl(p, n); - lex_print("\n"); + Note *note = decl_find_note(node, lit("register")); + if(note){ + Note *param_expr = decl_find_note(node, lit("param_expr")); + Note *param_type = decl_find_note(node, lit("param_type")); + macros[macros_i++] = (Macro){ + .name=node->name, + .param_type=param_type, + .param_expr=param_expr, + .decl=node + }; + } + else{ + U8 *struct_name = node->kind==DECL_Struct ? (U8*)"struct" : (U8*)"union"; + if(node->struct_decl.kind == STRUCT_Base) + strf("typedef %s %s %s;\n", struct_name, name, name); } - lex_print("};"); - if(node->struct_decl.kind == STRUCT_Base) - lex_print("\n"); } break; case DECL_Enum:{ - lex_print("enum %s", node->name.s.str); - if(!intern_compare(node->enum_decl.typespec->name, intern_s64)){ - lex_print(" : "); - lex_print("%s", node->enum_decl.typespec->name.s.str); - } - lex_print("{\n"); - - String prefix = decl_find_string_note(node, lit("prefix"), string_empty); - for(Decl_Enum_Child *n = node->enum_decl.first; n; n=n->next){ - lex_print("%.*s%s", (int)prefix.len, prefix.str, n->name.s.str); - print_assign_expr(p, n->expr); - lex_print(",\n"); - } - - lex_print("};\n"); } break; default: {invalid_codepath;} break; } } +function void +gen_decl(Decl *node){ + U8 *name = node->name.s.str; + + switch(node->kind) { + case DECL_List: { + for(Decl *n = node->list.first; n; n=n->next){ + gen_decl(n); + strf("\n"); + } + } break; + + case DECL_Variable:{ + gen_cdecl(node->variable_decl.type, node->name.s); + gen_assign_expr(node->variable_decl.expr); + strf(";"); + } break; + + case DECL_Typedef:{ + strf("typedef "); + gen_cdecl(node->typedef_decl.type, node->name.s); + strf(";"); + } break; + + case DECL_Struct: + case DECL_Union :{ + Note *note = decl_find_note(node, lit("register")); + if(note){ + Arena_Checkpoint checkpoint = arena_checkpoint(gscratch); + Pointer_Array array = gather(checkpoint.arena, node, GATHER_Decl|GATHER_Typespec, TRAVERS_All); + Intern_String based = intern_string(genp, lit("Based_Type_Represent")); + Intern_String type = intern_string(genp, lit("Type")); + for(Pointer it = pointer_array_iter_begin(&array); it.typespec; it = pointer_array_iter_next(&array)){ + if(it.kind == PK_Typespec){ + Typespec *typespec = it.typespec; + assert(typespec->kind == TS_Name || typespec->kind == TS_Pointer || typespec->kind == TS_Array || typespec->kind == TS_Function); + if(typespec->kind == TS_Name){ + if(intern_compare(typespec->name, type)){ + typespec->name = based; + } + } + } + else if(it.kind == PK_Decl){ + Decl *decl = it.decl; + if(decl->kind == DECL_Struct || decl->kind == DECL_Function){ + if(intern_compare(decl->name, type)){ + decl->name = based; + } + } + } + } + arena_restore(checkpoint); + } + U8 *struct_name = node->kind==DECL_Struct ? (U8*)"struct" : (U8*)"union"; + strf("%s", struct_name); + if(node->struct_decl.kind == STRUCT_Base) + strf(" %s", node->name.s.str); + strf("{\n"); + for(Decl *n = node->struct_decl.first; n; n=n->next){ + gen_decl(n); + strf("\n"); + } + strf("}"); + if(node->struct_decl.kind == STRUCT_Nested) + strf("%.*s", (int)node->name.s.len, node->name.s.str); + + strf(";"); + if(node->struct_decl.kind == STRUCT_Base) + strf("\n"); + + } break; + + case DECL_Enum:{ + strf("typedef enum %s", name); + if(!intern_compare(node->enum_decl.typespec->name, intern_s64)){ + strf(" : "); + strf("%s", node->enum_decl.typespec->name.s.str); + } + strf("{\n"); + + String prefix = decl_find_string_note(node, lit("prefix"), string_empty); + for(Decl_Enum_Child *n = node->enum_decl.first; n; n=n->next){ + strf("%.*s%s", (int)prefix.len, prefix.str, n->name.s.str); + gen_assign_expr(n->expr); + strf(",\n"); + } + + strf("}%s;\n", name); + } break; + + case DECL_Function:{ + gen_function_decl(node); + gen_stmt_list(node->function_decl.body); + } break; + + default: {invalid_codepath;} break; + } +} +function void +gen_stmt_list(Stmt *stmt){ + strf("{\n"); + for(Stmt *s = stmt->list.first; s; s=s->next){ + gen_stmt(s); + strf("\n"); + } + strf("}"); +} + +function void +gen_stmt(Stmt *stmt){ + switch(stmt->kind) { + case STMT_List: { + gen_stmt_list(stmt); + } break; + case STMT_Return:{ + strf("return "); + gen_expr(stmt->ret.expr); + strf(";"); + } break; + case STMT_If:{ + strf("if "); + gen_expr(stmt->stmt_if.cond); + gen_stmt_list(stmt->stmt_if.body); + for(Stmt_If *s = stmt->stmt_if.next; s; s=s->next){ + if(s->next){ + strf("else if "); + gen_expr(s->cond); + gen_stmt_list(s->body); + } + else{ + strf("else"); + gen_stmt_list(s->body); + } + } + } break; + case STMT_Expr: { + gen_expr(stmt->expr); + strf(";"); + } break; + case STMT_Decl: { + gen_decl(stmt->decl); + } break; + default: {invalid_codepath;} break; + } +} + +//----------------------------------------------------------------------------- +// +//----------------------------------------------------------------------------- +global Decl *gen_parent; +function void +gen_code(Decl *node){ + U8 *name = node->name.s.str; + + switch(node->kind) { + case DECL_List: { + for(Decl *n = node->list.first; n; n=n->next){ + gen_code(n); + } + } break; + + case DECL_Variable:{ + } break; + + case DECL_Typedef:{ + } break; + + case DECL_Struct: + case DECL_Union :{ + gen_parent = node; + for(Decl *n = node->struct_decl.first; n; n=n->next){ + gen_code(n); + } + + } break; + + case DECL_Enum:{ + } break; + + default: {invalid_codepath;} break; + } +} + +String_List list; +function void +gen_begin(Arena *scratch, Parser *p){ + list = (String_List){0}; + gscratch = scratch; + glist = &list; + genp = p; +} + +function void +gen_end(){ + String result = string_list_flatten(gscratch, glist); + lex_print("%s", result.str); +} diff --git a/common.c b/common.c index 16dbb97..0d914b6 100644 --- a/common.c +++ b/common.c @@ -1,4 +1,3 @@ -global String string_empty; function S64 clamp_top_s64(S64 val, S64 max){ @@ -52,6 +51,11 @@ wrap_around_pow2(U64 x, U64 power_of_2) { return r; } +//----------------------------------------------------------------------------- +// Strings +//----------------------------------------------------------------------------- +global String string_empty; + function B32 string_compare(String a, String b){ if(a.len != b.len) @@ -77,6 +81,12 @@ char_to_upper(U8 c){ return c; } +function B32 +string_is_empty(String a){ + B32 result = a.len == 0; + return result; +} + function String string_to_lower(Arena *arena, String string){ String result = arena_push_string_copy(arena, string); @@ -85,3 +95,103 @@ string_to_lower(Arena *arena, String string){ } return result; } + +//----------------------------------------------------------------------------- +// String interning +//----------------------------------------------------------------------------- +typedef struct Table_Index{ + U64 hash; + U64 index; + U64 iter; + U64 max_size; +}Table_Index; + +typedef struct Intern_String{ + String s; +}Intern_String; + +typedef struct Intern_Table{ + S64 interns_in_bytes; + S64 interns_inserted; + S64 interns_max; + Intern_String *interns; + Arena *arena; +}Intern_Table; + +function Intern_Table +intern_table(Arena *arena, SizeU size){ + Intern_Table result = {0}; + result.arena = arena; + result.interns = arena_push_array(arena, Intern_String, size); + result.interns_max = size; + return result; +} + +function Table_Index +table_index_from_hash(U64 hash, U64 max_size){ + Table_Index result = {0}; + result.hash = hash; + result.index = result.hash % max_size; + result.iter = result.index; + result.max_size = max_size; + return result; +} + +function Table_Index +table_index_from_string(String string, U64 max_size){ + U64 hash = hash_fnv(string); + Table_Index result = table_index_from_hash(hash, max_size); + return result; +} + +function B32 +table_index_advance(Table_Index *index){ + index->iter = wrap_around_pow2(index->iter + 1, index->max_size); + B32 result = index->iter == index->index; + return result; +} + +function Intern_String +intern_string(Intern_Table *p, String string){ + Intern_String result = {0}; + Table_Index index = table_index_from_string(string, p->interns_max); + for(;;){ + Intern_String *intern = p->interns + index.iter; + if(intern->s.str == 0){ + result.s = arena_push_string_copy(p->arena, string); + p->interns_in_bytes += string.len; + p->interns_inserted += 1; + *intern = result; + break; + } + else if(string_compare(intern->s, string)){ + result = *intern; + break; + } + + if (table_index_advance(&index)) + break; + + } + + return result; +} + +function B32 +intern_compare(Intern_String a, Intern_String b){ + B32 result = a.s.str == b.s.str; + return result; +} + +function void +intern_test(){ + Arena *scratch = arena_begin_scratch(); + Intern_Table table = intern_table(scratch, 512); + assert(intern_compare(intern_string(&table, lit("Thing")), + intern_string(&table, lit("Thing")))); + assert(!intern_compare(intern_string(&table, lit("Thing")), + intern_string(&table, lit("No_Thing")))); + assert(intern_compare(intern_string(&table, lit("No_Thing")), + intern_string(&table, lit("No_Thing")))); + +} diff --git a/file.c b/file.c new file mode 100644 index 0000000..2fd4fd8 --- /dev/null +++ b/file.c @@ -0,0 +1,21 @@ + +global FILE *global_output_file; +global FILE *global_all_files[32]; +global U32 global_all_files_count; + +#define lex_print(...) fprintf(global_output_file, __VA_ARGS__) +#define lex_new_line() lex_print("\n") + +function void +use_write_file(const char *file){ + global_output_file = fopen(file, "w"); + global_all_files[global_all_files_count++] = global_output_file; + assert_msg(global_output_file, "Failed to open file:%s", file); +} + +function void +close_all_files(){ + for(U32 i = 0; i < global_all_files_count; i++){ + fclose(global_all_files[i]); + } +} \ No newline at end of file diff --git a/generate.c b/generate.c new file mode 100644 index 0000000..ef384a6 --- /dev/null +++ b/generate.c @@ -0,0 +1,36 @@ +#define _CRT_SECURE_NO_WARNINGS +#include +#include + +#include "lang.h" +#include "os.h" +#include "memory.h" + +#include "common.c" +#include "memory.c" +#include "scratch.c" +#include "os_win32.c" + +#include "new_lex.c" + +int main(){ + + FILE *f = fopen("generated.c", "w"); + assert(f); + + Arena *scratch = arena_begin_scratch(); + String file = os_read_file(scratch, lit("token_array.c")); + Token_Array array = lex_stream(scratch, file, lit("token_array.c")); + for(Token *t = token_array_iter_begin(&array); t; t = token_array_iter_next(&array)){ + + fprintf(f, "%.*s", (int)t->len, t->str); + if(t->kind == TK_OpenBrace) fprintf(f, "\n"); + if(t->kind == TK_CloseBrace && token_array_iter_peek(&array, 0)->kind != TK_Semicolon) fprintf(f, "\n"); + if(t->kind == TK_Semicolon) fprintf(f, "\n"); + if(t[0].kind == TK_Identifier && token_array_iter_peek(&array, 0)->kind == TK_Identifier) fprintf(f, " "); + token_array_iter_peek(&array, 1); + } + arena_end_scratch(); + + fclose(f); +} \ No newline at end of file diff --git a/generated.c b/generated.c new file mode 100644 index 0000000..49de99d --- /dev/null +++ b/generated.c @@ -0,0 +1,69 @@ +typedef struct Token_Bucket Token_Bucket; +typedef struct Token_Array Token_Array; +struct Token_Bucket{ +Token_Bucket*next; +Token data[4096]; +}; +struct Token_Array{ +Intern_Table interns; +Token_Bucket first; +Token_Bucket*last; +S64 len; +S64 block; +Arena*arena; +Token_Bucket*iter_bucket; +S64 iter_len; +S64 iter_block; +}; +function Token_Array token_array_make(Arena*arena){ +Token_Array result={ +.last=&result.first,.arena=arena,.interns=intern_table(arena,4096*4)}; +return result; +} +function void token_array_push(Token_Array*array,Token*p){ +if(array->len>=buff_cap(array->first.data)){ +Token_Bucket*bucket=arena_push_struct(array->arena,Token_Bucket); +array->last=array->last->next=bucket; +array->len=0; +array->block+=1; +} +array->last->data[array->len++]=*p; +} +function B32 token_array_iter_is_end(Token_Array*array){ +B32 result=array->iter_len==array->len&&array->iter_block==array->block; +return result; +} +function Token*token_array_iter_next(Token_Array*array){ +if(token_array_iter_is_end(array)){ +return0; +} +if(array->iter_len>=buff_cap(array->first.data)){ +array->iter_len=0; +array->iter_block+=1; +array->iter_bucket=array->iter_bucket->next; +} +Token*result=array->iter_bucket->data+array->iter_len++; +return result; +} +function Token*token_array_iter_peek(Token_Array*array,S64 i){ +S64 save_len=array->iter_len; +Token_Bucket*save_bucket=array->iter_bucket; +assert(ifirst.data)); +S64 over=i; +if(array->iter_len+i>=buff_cap(array->first.data)){ +over=buff_cap(array->first.data)-(array->iter_len+i); +array->iter_len=0; +array->iter_bucket=array->iter_bucket->next; +} +Token*result=array->iter_bucket->data+array->iter_len+over; +array->iter_len=save_len; +array->iter_bucket=save_bucket; +return result; +} +function Token*token_array_iter_begin(Token_Array*array){ +array->iter_len=0; +array->iter_block=0; +array->iter_bucket=&array->first; +Token*result=token_array_iter_next(array); +return result; +} diff --git a/generated_ast.cpp b/generated_ast.cpp new file mode 100644 index 0000000..e69de29 diff --git a/generated_ast.h b/generated_ast.h new file mode 100644 index 0000000..ce84b2b --- /dev/null +++ b/generated_ast.h @@ -0,0 +1,225 @@ + +typedef struct Typespec Typespec; + +typedef struct Note Note; + + + +typedef struct Decl_Function_Arg Decl_Function_Arg; + +typedef struct Decl_Enum_Child Decl_Enum_Child; + +typedef struct Decl Decl; + + +typedef struct Stmt_If Stmt_If; + +typedef struct Stmt Stmt; + + +typedef struct Pointer Pointer; + +typedef struct Pointer_Bucket Pointer_Bucket; + +typedef struct Pointer_Array Pointer_Array; + + + +typedef enum Typespec_Kind{ +TS_None, +TS_Name, +TS_Pointer, +TS_Array, +TS_Function, +}Typespec_Kind; + +struct Typespec{ +Typespec_Kind kind; +Typespec (*next); +Token (*pos); +union{ +Intern_String name; +Typespec (*base); +struct{ +Typespec (*first); +Typespec (*last); +Typespec (*ret); +}function_spec; +struct{ +Typespec (*base); +Expr (*size); +}array_spec; +}; +}; + +struct Note{ +Token (*pos); +Intern_String name; +Expr (*expr); +Note (*next); +Note (*first); +Note (*last); +}; + +typedef enum Decl_Kind{ +DECL_None, +DECL_Struct, +DECL_Union, +DECL_Enum, +DECL_Variable, +DECL_Typedef, +DECL_Function, +DECL_List, +}Decl_Kind; + +typedef enum Decl_Struct_Kind{ +STRUCT_Nested, +STRUCT_Base, +}Decl_Struct_Kind; + +struct Decl_Function_Arg{ +Decl_Function_Arg (*next); +Intern_String name; +Token (*pos); +Typespec (*typespec); +}; + +struct Decl_Enum_Child{ +Decl_Enum_Child (*next); +Intern_String name; +Token (*pos); +Expr (*expr); +Note (*first_note); +Note (*last_note); +}; + +struct Decl{ +Decl_Kind kind; +Decl (*next); +Intern_String name; +Token (*pos); +Note (*first_note); +Note (*last_note); +union{ +struct{ +Decl_Enum_Child (*first); +Decl_Enum_Child (*last); +Typespec (*typespec); +}enum_decl; +struct{ +Decl (*first); +Decl (*last); +Decl_Struct_Kind kind; +}struct_decl; +struct{ +Typespec (*type); +Expr (*expr); +}variable_decl; +struct{ +Typespec (*type); +}typedef_decl; +struct{ +Decl_Function_Arg (*first); +Decl_Function_Arg (*last); +Typespec (*ret); +Stmt (*body); +}function_decl; +struct{ +Decl (*first); +Decl (*last); +}list; +}; +}; + +typedef enum Stmt_Kind{ +STMT_None, +STMT_Decl, +STMT_Expr, +STMT_List, +STMT_Return, +STMT_If, +STMT_For, +}Stmt_Kind; + +struct Stmt_If{ +Stmt_If (*next); +Expr (*cond); +Stmt (*body); +}; + +struct Stmt{ +Stmt_Kind kind; +Stmt (*next); +Token (*pos); +union{ +Stmt_If stmt_if; +Decl (*decl); +Expr (*expr); +struct{ +Stmt (*first); +Stmt (*last); +}list; +struct{ +Expr (*expr); +}ret; +}; +}; + +typedef enum Pointer_Kind{ +PK_None, +PK_Typespec, +PK_Expr, +PK_Decl, +PK_Stmt, +PK_Enum_Child, +PK_Func_Arg, +PK_Intern_String, +}Pointer_Kind; + +struct Pointer{ +Pointer_Kind kind; +union{ +Typespec (*typespec); +Decl (*decl); +Expr (*expr); +Stmt (*stmt); +Decl_Function_Arg (*func_arg); +Decl_Enum_Child (*enum_child); +Intern_String (*string); +}; +}; + +struct Pointer_Bucket{ +Pointer_Bucket (*next); +Pointer (data[4096]); +}; + +struct Pointer_Array{ +Pointer_Bucket first; +Pointer_Bucket (*last); +S64 len; +S64 block; +Arena (*arena); +Pointer_Bucket (*iter_bucket); +S64 iter_len; +S64 iter_block; +}; + +typedef enum Gather_Flag{ +GATHER_None = 0, +GATHER_Typespec = 1, +GATHER_Expr = 2, +GATHER_Decl = 4, +GATHER_Stmt = 8, +GATHER_Enum_Child = 16, +GATHER_Func_Arg = 32, +}Gather_Flag; + +typedef enum Traversal_Flag{ +TRAVERS_None, +TRAVERS_Typespec = 1, +TRAVERS_Expr = 2, +TRAVERS_Stmt = 8, +TRAVERS_All = ((TRAVERS_Typespec|TRAVERS_Expr)|TRAVERS_Stmt), +}Traversal_Flag; + diff --git a/generated_lex.cpp b/generated_lex.cpp new file mode 100644 index 0000000..9a440a0 --- /dev/null +++ b/generated_lex.cpp @@ -0,0 +1,53 @@ + +// Generated Array block + +function + Token_Array token_array_make(Arena*arena){ +Token_Array result={ +0}; +result.arena=arena; +result.last=&result.first; +return result; +} + +function + void token_array_push(Token_Array*array, Token*item){ +if(array->len+1>4096){ +assert(array->arena); +Token_Array_Block*block=arena_push_struct(array->arena, Token_Array_Block); +array->last=array->last->next=block; +array->len=0; +array->block+=1; +} +array->last->data[array->len++]=*item; +} + +function + Token_Array_Iter token_array_iter(Token_Array*array){ +return((Token_Array_Iter){ +.total_len=array->len, .total_block=array->block, .block=&array->first, } +); +} + +function + Token*token_array_iter_next(Token_Array_Iter*it){ +if(it->iter_len+1>4096){ +it->iter_len=0; +it->iter_block+=1; +it->block=it->block->next; +} +return it->block->data+it->iter_len++; +} + +function + B32 token_array_iter_is_end(Token_Array_Iter*it){ +B32 result=it->iter_len==it->total_len&&it->iter_block==it->total_block; +return result; +} + +function + Token*token_array_iter_begin(Token_Array*array){ +array->it=token_array_iter(array); +Token*result=token_array_iter_next(&array->it); +return result; +} diff --git a/generated_lex.h b/generated_lex.h new file mode 100644 index 0000000..778798f --- /dev/null +++ b/generated_lex.h @@ -0,0 +1,136 @@ +typedef struct Intern_String Intern_String; + + +typedef struct Token Token; + +typedef struct Lex_Stream Lex_Stream; + +typedef struct Tokens Tokens; + +struct Intern_String{ +String s; +}; + +typedef enum Token_Kind{ +TK_End, +TK_Mul, +TK_Div, +TK_Add, +TK_Sub, +TK_Mod, +TK_BitAnd, +TK_BitOr, +TK_BitXor, +TK_Neg, +TK_Not, +TK_OpenParen, +TK_CloseParen, +TK_OpenBrace, +TK_CloseBrace, +TK_OpenBracket, +TK_CloseBracket, +TK_Comma, +TK_Pound, +TK_Question, +TK_ThreeDots, +TK_Semicolon, +TK_Dot, +TK_LesserThen, +TK_GreaterThen, +TK_Colon, +TK_Assign, +TK_DivAssign, +TK_MulAssign, +TK_ModAssign, +TK_SubAssign, +TK_AddAssign, +TK_AndAssign, +TK_OrAssign, +TK_XorAssign, +TK_LeftShiftAssign, +TK_RightShiftAssign, +TK_DoubleColon, +TK_At, +TK_Decrement, +TK_Increment, +TK_PostDecrement, +TK_PostIncrement, +TK_LesserThenOrEqual, +TK_GreaterThenOrEqual, +TK_Equals, +TK_And, +TK_Or, +TK_NotEquals, +TK_LeftShift, +TK_RightShift, +TK_Arrow, +TK_ExprSizeof, +TK_DocComment, +TK_Comment, +TK_Identifier, +TK_StringLit, +TK_U8Lit, +TK_Character, +TK_Error, +TK_Float, +TK_Int, +TK_Keyword, +}Token_Kind; + +struct Token{ +Token_Kind kind; +union{ +struct{ +U8 (*str); +S64 len; +}; +String string; +}; +union{ +S64 int_val; +String error_val; +Intern_String intern_val; +}; +String file; +S64 line; +U8 (*line_begin); +}; + +// Generated Array block +typedef struct Token_Array Token_Array; +typedef struct Token_Array_Block Token_Array_Block; +typedef struct Token_Array_Iter Token_Array_Iter; +struct Token_Array_Block{ +Token_Array_Block*next; +Token data[4096]; +}; +struct Token_Array_Iter{ +Token_Array_Block*block; +U32 iter_len; +U32 iter_block; +U32 total_len; +U32 total_block; +}; +struct Token_Array{ +Token_Array_Block first; +Token_Array_Block*last; +Token_Array_Iter it; +Arena*arena; +S64 block; +S64 len; +}; + +struct Lex_Stream{ +U8 (*stream); +U8 (*line_begin); +String filename; +S64 line; +}; + +struct Tokens{ +Token (*tokens); +S64 len; +S64 cap; +S64 iter; +}; + diff --git a/ideas.cpp b/ideas.cpp index 3e4076a..6359bc1 100644 --- a/ideas.cpp +++ b/ideas.cpp @@ -32,14 +32,15 @@ Thing :: struct{ } - - - -Thingy::enum:U32{ - @str=10 Value = 1, - +Scope :: scope{ + Thingy::enum:U32{ + @str=10 Value = 1, + + } } +//Scope.Thingy + Thing::union{ union:{ } diff --git a/lang.h b/lang.h index b0c66df..1452fa4 100644 --- a/lang.h +++ b/lang.h @@ -4,7 +4,7 @@ #define function static #define assert(x) do{if(!(x)) __debugbreak();}while(0) -#define assert_msg(x,msg) assert(x) +#define assert_msg(x,...) assert(x) #define not_implemented assert_msg(0, "Not implemented") #define invalid_codepath assert_msg(0, "Invalid codepath") @@ -35,12 +35,30 @@ const B32 false = 0; #define kib(x) ((x)*1024llu) #define mib(x) (kib(x)*1024llu) #define gib(x) (mib(x)*1024llu) +#define string_expand(x) (int)x.len, x.str +typedef struct String_Node String_Node; +typedef struct String_List String_List; typedef struct String{ U8 *str; S64 len; }String; +struct String_Node{ + String_Node *next; + union{ + String string; + struct{U8*str; S64 len;}; + }; +}; + +struct String_List{ + String_Node *first; + String_Node *last; + S64 char_count; + S64 node_count; +}; + #define SLLQueuePushMod(f,l,n,next) do{\ if((f)==0){\ (f)=(l)=(n);\ diff --git a/lex.c b/lex.c index 1ad524b..a2a1c9d 100644 --- a/lex.c +++ b/lex.c @@ -1,4 +1,4 @@ -global Token token_end_of_stream = {}; +global Token token_end_of_stream = {0}; function Token * token_alloc(Tokens *t){ @@ -363,7 +363,7 @@ break function Tokens lex_stream(String in_stream, String filename){ Lex_Stream stream = {in_stream.str, in_stream.str, filename, 0}; - Tokens tokens = {}; + Tokens tokens = {0}; lex_base(&stream, &tokens); return tokens; } @@ -498,3 +498,32 @@ token_peek_is_keyword(Parser *p, S64 count, Intern_String keyword){ } return 0; } + +function void +lex_test(){ + Tokens t; + t = lex_stream(lit("3252342510 42524 \"U8Literal\""), lit("test")); + //tokens_print(t); + assert(t.len == 3); + assert(t.tokens[0].int_val == 3252342510); + assert(t.tokens[1].int_val == 42524); + assert(t.tokens[2].kind == TK_U8Lit); + assert(token_compare(t.tokens + 2, lit("U8Literal"))); + + t = lex_stream(lit("_identifier Thing Thing2 lit(\"String_Test\")"), lit("test")); + //tokens_print(t); + assert(t.tokens[0].kind == TK_Identifier); + assert(t.tokens[1].kind == TK_Identifier); + assert(t.tokens[2].kind == TK_Identifier); + assert(t.tokens[3].kind == TK_StringLit); + assert(token_compare(t.tokens, lit("_identifier"))); + assert(token_compare(t.tokens+1, lit("Thing"))); + assert(token_compare(t.tokens+2, lit("Thing2"))); + assert(token_compare(t.tokens+3, lit("String_Test"))); + + t = lex_stream(lit("lit(\"String_Test\"{})(324*=+=-/ *% // Comment \n" + "Thing /*Thing*/ += -= =- +/%^&*&&|| |>> << <<= >>=/*Error"), + lit("test")); + assert(t.tokens[0].kind == TK_Error); + //tokens_print(t); +} diff --git a/lex.h b/lex.h index 5d8796b..3e49f46 100644 --- a/lex.h +++ b/lex.h @@ -1,107 +1,115 @@ #pragma once -typedef struct Intern_String{ - String s; -}Intern_String; +#pragma once +#if 0 +Intern_String :: struct { + s: String; +} -typedef enum Token_Kind{ - meta("End of stream")TK_End, - meta("*")TK_Mul, - meta("/")TK_Div, - meta("+")TK_Add, - meta("-")TK_Sub, - meta("%")TK_Mod, - meta("&")TK_BitAnd, - meta("|")TK_BitOr, - meta("^")TK_BitXor, - meta("~")TK_Neg, - meta("!")TK_Not, - meta("(")TK_OpenParen, - meta(")")TK_CloseParen, - meta("{")TK_OpenBrace, - meta("}")TK_CloseBrace, - meta("[")TK_OpenBracket, - meta("]")TK_CloseBracket, - meta(",")TK_Comma, - meta("#")TK_Pound, - meta("?")TK_Question, - meta("...")TK_ThreeDots, - meta(";")TK_Semicolon, - meta(".")TK_Dot, - meta("<")TK_LesserThen, - meta(">")TK_GreaterThen, - meta(":")TK_Colon, - meta("=")TK_Assign, - meta("/=")TK_DivAssign, - meta("*=")TK_MulAssign, - meta("%=")TK_ModAssign, - meta("-=")TK_SubAssign, - meta("+=")TK_AddAssign, - meta("&=")TK_AndAssign, - meta("|=")TK_OrAssign, - meta("^=")TK_XorAssign, - meta("<<=")TK_LeftShiftAssign, - meta(">>=")TK_RightShiftAssign, - meta("::")TK_DoubleColon, - meta("@")TK_At, - meta("--")TK_Decrement, - meta("++")TK_Increment, - meta("--")TK_PostDecrement, - meta("++")TK_PostIncrement, - meta("<=")TK_LesserThenOrEqual, - meta(">=")TK_GreaterThenOrEqual, - meta("==")TK_Equals, - meta("&&")TK_And, - meta("||")TK_Or, - meta("!=")TK_NotEquals, - meta("<<")TK_LeftShift, - meta(">>")TK_RightShift, - meta("->")TK_Arrow, - meta("sizeof")TK_ExprSizeof, - TK_DocComment, - TK_Comment, - TK_Identifier, - TK_StringLit, - TK_U8Lit, - TK_Character, - TK_Error, - TK_Float, - TK_Int, - TK_Keyword, +@prefix = "TK_" +Token_Kind :: enum { + @str = "End of stream" End, + @str = "*" Mul, + @str = "/" Div, + @str = "+" Add, + @str = "-" Sub, + @str = "%" Mod, + @str = "&" BitAnd, + @str = "|" BitOr, + @str = "^" BitXor, + @str = "~" Neg, + @str = "!" Not, + @str = "(" OpenParen, + @str = " " CloseParen, + @str = "{" OpenBrace, + @str = "}" CloseBrace, + @str = "[" OpenBracket, + @str = "]" CloseBracket, + @str = "," Comma, + @str = "#" Pound, + @str = "?" Question, + @str = "..." ThreeDots, + @str = ";" Semicolon, + @str = "." Dot, + @str = "<" LesserThen, + @str = ">" GreaterThen, + @str = ":" Colon, + @str = "=" Assign, + @str = "/=" DivAssign, + @str = "*=" MulAssign, + @str = "%=" ModAssign, + @str = "-=" SubAssign, + @str = "+=" AddAssign, + @str = "&=" AndAssign, + @str = "|=" OrAssign, + @str = "^=" XorAssign, + @str = "<<=" LeftShiftAssign, + @str = ">>=" RightShiftAssign, + @str = "::" DoubleColon, + @str = "@" At, + @str = "--" Decrement, + @str = "++" Increment, + @str = "--" PostDecrement, + @str = "++" PostIncrement, + @str = "<=" LesserThenOrEqual, + @str = ">=" GreaterThenOrEqual, + @str = "==" Equals, + @str = "&&" And, + @str = "||" Or, + @str = "!=" NotEquals, + @str = "<<" LeftShift, + @str = ">>" RightShift, + @str = "->" Arrow, + @str = "sizeof" ExprSizeof, + DocComment, + Comment, + Identifier, + StringLit, + U8Lit, + Character, + Error, + Float, + Int, + Keyword, }Token_Kind; -typedef struct Token{ - Token_Kind kind; - union{ - struct{ - U8 *str; - S64 len; - }; - String string; - }; +@array(type=block) +Token :: struct{ + kind: Token_Kind; union { - S64 int_val; - String error_val; - Intern_String intern_val; - }; + struct{ + str: U8*; + len: S64; + } + string: String; + } + union { + int_val: S64; + error_val: String; + intern_val: Intern_String; + } - String file; - S64 line; - U8 *line_begin; -} Token; + file: String; + line: S64; + line_begin: U8*; +} -typedef struct Tokens{ - Token *tokens; - S64 len; - S64 cap; - S64 iter; -}Tokens; +Lex_Stream :: struct{ + stream: U8 *; + line_begin: U8 *; + filename: String; + line: S64; +} -typedef struct Lex_Stream{ - U8 *stream; - U8 *line_begin; - String filename; - S64 line; -}Lex_Stream; +Tokens :: struct{ + tokens: Token*; + len: S64; + cap: S64; + iter: S64; +} + +#endif +#include "generated_lex.h" +#include "generated_lex.cpp" //----------------------------------------------------------------------------- // diff --git a/main.c b/main.c index 2b5f5eb..6e7a0fe 100644 --- a/main.c +++ b/main.c @@ -6,117 +6,16 @@ #include "os.h" #include "memory.h" -#include "lex.h" -#include "parser.h" -#include "expr.h" -#include "ast.h" - -global FILE *global_output_file; -#define lex_print(...) fprintf(global_output_file, __VA_ARGS__) -#define lex_new_line() lex_print("\n") - #include "common.c" #include "memory.c" -#include "parser.c" +#include "scratch.c" #include "os_win32.c" -#include "lex.c" -#include "expr.c" -#include "ast.c" -#include "parse_expr.c" -#include "parse_decl.c" -#include "print.c" -#include "codegen_c.c" +#include "new_lex.c" +#include "new_ast.c" +#include "new_parse.c" -function void -lex_test(){ - Tokens t; - t = lex_stream(lit("3252342510 42524 \"U8Literal\""), lit("test")); - //tokens_print(t); - assert(t.len == 3); - assert(t.tokens[0].int_val == 3252342510); - assert(t.tokens[1].int_val == 42524); - assert(t.tokens[2].kind == TK_U8Lit); - assert(token_compare(t.tokens + 2, lit("U8Literal"))); - - t = lex_stream(lit("_identifier Thing Thing2 lit(\"String_Test\")"), lit("test")); - //tokens_print(t); - assert(t.tokens[0].kind == TK_Identifier); - assert(t.tokens[1].kind == TK_Identifier); - assert(t.tokens[2].kind == TK_Identifier); - assert(t.tokens[3].kind == TK_StringLit); - assert(token_compare(t.tokens, lit("_identifier"))); - assert(token_compare(t.tokens+1, lit("Thing"))); - assert(token_compare(t.tokens+2, lit("Thing2"))); - assert(token_compare(t.tokens+3, lit("String_Test"))); - - t = lex_stream(lit("lit(\"String_Test\"{})(324*=+=-/ *% // Comment \n" - "Thing /*Thing*/ += -= =- +/%^&*&&|| |>> << <<= >>=/*Error"), - lit("test")); - assert(t.tokens[0].kind == TK_Error); - //tokens_print(t); -} - -function void -parser_test(){ - Parser p = {}; - { - parser_init(&p); - Intern_String a = intern_string(&p, lit("Thing")); - Intern_String b = intern_string(&p, lit("Thing")); - assert(a.s.str == b.s.str); - } - - parser_lex_stream(&p, lit("S64 thing; S64 second_var = 10;"), lit("File")); - assert(token_match(&p, TK_Identifier)); - assert(token_match(&p, TK_Identifier)); - assert(token_match(&p, TK_Semicolon)); - assert(token_match(&p, TK_Identifier)); - assert(token_match(&p, TK_Identifier)); - assert(token_match(&p, TK_Assign)); - assert(token_match(&p, TK_Int)); - assert(token_match(&p, TK_Semicolon)); - assert(token_match(&p, TK_End)); - assert(token_match(&p, TK_End)); - assert(token_match(&p, TK_End)); - - String exprs[] = { - lit("(534>43?435:42,234,cast(S64)32/*todo cast*/,Thing[10][2],Thing(1,2))"), - lit("(4+2*53)"), - lit("((4+2)*53)"), - lit("++5"), - lit("5--"), // @Todo(Krzosa): - lit("-5"), - lit("(+5)"), - lit("sizeof(32) + sizeof(:S32*)"), - lit("cast(S64**)5"), - lit("cast(S64)5+3"), - lit("534>43?435:42"), - }; - for(S64 i = 0; i < buff_cap(exprs); i++){ - parser_lex_stream(&p, exprs[i], lit("File")); - Expr *expr = parse_expr(&p); - assert(expr); - //expr_print(&p, expr); - //lex_print("\n"); - } - - String string = os_read_file(lit("test.cc")); - parser_lex_stream(&p, string, lit("Parse")); - Decl *decls = parse(&p); - assert(decls->list.first); - gen_decl(&p, decls); - -} - -function S32 -os_main(){ - global_output_file = fopen("output.cc", "w"); - assert_msg(global_output_file, "Failed to open output.txt"); +int main(){ lex_test(); - parser_test(); - - fclose(global_output_file); - - return 0; + test_ast(); } \ No newline at end of file diff --git a/memory.c b/memory.c index 7a08304..8e5055f 100644 --- a/memory.c +++ b/memory.c @@ -80,12 +80,34 @@ string_fmtv(Arena *arena, const char *str, va_list args1) { return (String){(U8 *)result, len}; } +#define STRING_FMT(arena, str, result) \ +va_list args1; \ +va_start(args1, str); \ +String result = string_fmtv(arena, str, args1); \ +va_end(args1) + function String string_fmt(Arena *arena, const char *str, ...) { - va_list args1; - va_start(args1, str); - String result = string_fmtv(arena, str, args1); - va_end(args1); + STRING_FMT(arena, str, result); return result; } +function void +string_listf(Arena *arena, String_List *list, const char *str, ...){ + STRING_FMT(arena, str, string); + String_Node *node = arena_push_struct(arena, String_Node); + node->string = string; + SLLQueuePush(list->first, list->last, node); + list->char_count += node->string.len; + list->node_count += 1; +} + +function String +string_list_flatten(Arena *arena, String_List *list){ + String result = {arena_push_size(arena, list->char_count + 1)}; + for(String_Node *node = list->first; node; node=node->next){ + memory_copy(result.str+result.len, node->str, node->len); + result.len += node->len; + } + return result; +} diff --git a/memory.h b/memory.h index a163284..4c55c21 100644 --- a/memory.h +++ b/memory.h @@ -14,4 +14,7 @@ function B32 string_compare(String a, String b); function void *arena_push_size(Arena *a, SizeU size); function String arena_push_string_copy(Arena *arena, String string); #define arena_push_array(a,T,c) arena_push_size(a,sizeof(T)*(c)) -#define arena_push_struct(a,T) arena_push_array(a,T,1) \ No newline at end of file +#define arena_push_struct(a,T) arena_push_array(a,T,1) + +function Arena *arena_begin_scratch(); +function void arena_end_scratch(); \ No newline at end of file diff --git a/new_ast.c b/new_ast.c new file mode 100644 index 0000000..eade645 --- /dev/null +++ b/new_ast.c @@ -0,0 +1,295 @@ + +typedef struct Expr Expr; +typedef struct Typespec Typespec; +typedef enum Typespec_Kind{ + TS_None, + TS_Name, + TS_Pointer, + TS_Array, + TS_Function, +}Typespec_Kind; + +struct Typespec{ + Typespec_Kind kind; + Typespec *next; + Token *pos; + union{ + Intern_String name; + Typespec *base; + struct{ + Typespec *first; + Typespec *last; + Typespec *ret; + }func; + struct{ + Typespec *base; + Expr *size; + }arr; + }; +}; + +//----------------------------------------------------------------------------- +// Expressions +//----------------------------------------------------------------------------- +typedef enum Expr_Sizeof_Kind{ + SIZEOF_Expr, + SIZEOF_Type, +}Expr_Sizeof_Kind; + +typedef enum Expr_Kind{ + EK_None, + EK_Int, + EK_String, + EK_Unary, + EK_Binary, + EK_Ternary, + EK_Cast, + EK_List, + EK_Call, + EK_Index, + EK_Sizeof, +} Expr_Kind; + +struct Expr { + Expr_Kind kind; + Token *token; + Expr *next; + union { + U64 int_val; + Intern_String intern_val; + double float_val; + + struct { + Typespec *type; + Expr* expr; + } cast; + struct { + Expr *first; + Expr *last; + } list; + struct { + Expr *atom; + Expr *list; + } call; + struct { + Expr *atom; + Expr *index; + } index; + struct { + Token_Kind op; + Expr* expr; + } unary; + struct { + Token_Kind op; + Expr* left; + Expr* right; + } binary; + struct { + Expr* cond; + Expr* on_true; + Expr* on_false; + } ternary; + struct{ + Expr_Sizeof_Kind kind; + union{ + Typespec *type; + Expr *expr; + }; + } size_of; + }; +}; + +//----------------------------------------------------------------------------- +// Expr +//----------------------------------------------------------------------------- +function Expr * +expr_new(Arena *p, Expr_Kind kind, Token *token){ + Expr *expr = arena_push_struct(p, Expr); + expr->kind = kind; + expr->token = token; + return expr; +} + +function Expr * +expr_int(Arena *p, Token *token){ + assert(token->kind == TK_Int); + Expr *expr = expr_new(p, EK_Int, token); + expr->int_val = token->int_val; + return expr; +} + +function Expr * +expr_str(Arena *p, Token *token){ + assert(token->kind == TK_StringLit); + Expr *expr = expr_new(p, EK_String, token); + expr->intern_val = token->intern_val; + return expr; +} + +function Expr * +expr_unary(Arena *p, Token *op, Expr *exp){ + Expr *expr = expr_new(p, EK_Unary, op); + expr->unary.op = op->kind; + expr->unary.expr = exp; + return expr; +} + +function Expr * +expr_binary(Arena *p, Token *op, Expr *left, Expr *right){ + Expr *expr = expr_new(p, EK_Binary, op); + expr->binary.op = op->kind; + expr->binary.left = left; + expr->binary.right = right; + return expr; +} + +function Expr * +expr_ternary(Arena *p, Token *op, Expr *cond, Expr *on_true, Expr *on_false){ + Expr *expr = expr_new(p, EK_Ternary, op); + expr->ternary.cond = cond; + expr->ternary.on_true = on_true; + expr->ternary.on_false = on_false; + return expr; +} + +function Expr * +expr_call(Arena *p, Token *token, Expr *atom, Expr *list){ + Expr *expr = expr_new(p, EK_Call, token); + expr->call.atom = atom; + expr->call.list = list; + return expr; +} + +function Expr * +expr_index(Arena *p, Token *token, Expr *atom, Expr *index){ + Expr *expr = expr_new(p, EK_Index, token); + expr->index.atom = atom; + expr->index.index = index; + return expr; +} + +function Expr * +expr_cast(Arena *p, Token *token, Typespec *type, Expr *exp){ + Expr *expr = expr_new(p, EK_Cast, token); + expr->cast.type = type; + expr->cast.expr = exp; + return expr; +} + +function Expr * +expr_sizeof_type(Arena *p, Token *token, Typespec *type){ + Expr *expr = expr_new(p, EK_Sizeof, token); + expr->size_of.kind = SIZEOF_Type; + expr->size_of.type = type; + return expr; +} + +function Expr * +expr_sizeof_expr(Arena *p, Token *token, Expr *in_expr){ + Expr *expr = expr_new(p, EK_Sizeof, token); + expr->size_of.kind = SIZEOF_Expr; + expr->size_of.expr = in_expr; + return expr; +} + +function Expr * +expr_list(Arena *p, Token *token){ + Expr *expr = expr_new(p, EK_List, token); + return expr; +} + +function void +expr_list_push(Expr *list, Expr *expr){ + SLLQueuePush(list->list.first, list->list.last, expr); +} + +//----------------------------------------------------------------------------- +// +//----------------------------------------------------------------------------- +function void +token_print(Token *token){ + printf("%.*s", (S32)token->len, token->str); +} + +function void +expr_print(Expr *expr){ + switch(expr->kind) { + case EK_Int: + case EK_String: { + token_print(expr->token); + } break; + + case EK_Sizeof:{ + printf("sizeof("); + if(expr->size_of.kind == SIZEOF_Expr){ + expr_print(expr->size_of.expr); + } + else{ + assert(expr->size_of.kind == SIZEOF_Type); + //print_typespec(expr->size_of.type); + } + printf(")"); + }break; + + case EK_Binary:{ + printf("("); + expr_print(expr->binary.left); + token_print(expr->token); + expr_print(expr->binary.right); + printf(")"); + } break; + case EK_Unary:{ + printf("("); + token_print(expr->token); + expr_print(expr->unary.expr); + printf(")"); + } break; + + case EK_Ternary:{ + printf("("); + expr_print(expr->ternary.cond); + printf("?"); + expr_print(expr->ternary.on_true); + printf(":"); + expr_print(expr->ternary.on_false); + printf(")"); + } break; + case EK_List:{ + printf("("); + for(Expr *n = expr->list.first; n; n=n->next){ + expr_print(n); + if(n!=expr->list.last) printf(","); + } + printf(")"); + }break; + + case EK_Cast:{ + printf("("); + printf("("); + //print_typespec(expr->cast.type); + printf(")"); + expr_print(expr->cast.expr); + printf(")"); + } break; + + case EK_Index:{ + expr_print(expr->index.atom); + printf("["); + expr_print(expr->index.index); + printf("]"); + }break; + + case EK_Call:{ + expr_print(expr->call.atom); + printf("("); + expr_print(expr->call.list); + printf(")"); + }break; + default: {invalid_codepath;} break; + } +} + +function void +test_ast(){ + +} \ No newline at end of file diff --git a/new_lex.c b/new_lex.c new file mode 100644 index 0000000..89a3a2a --- /dev/null +++ b/new_lex.c @@ -0,0 +1,516 @@ +global Intern_String keyword_if; +global Intern_String keyword_for; +global Intern_String keyword_else; +global Intern_String keyword_sizeof; +global Intern_String keyword_typeof; +global Intern_String keyword_while; +global Intern_String keyword_switch; +global Intern_String keyword_case; +global Intern_String keyword_struct; +global Intern_String keyword_enum; +global Intern_String keyword_union; +global U8 *first_keyword; +global U8 *last_keyword; + +function void +init_default_keywords(Intern_Table *t){ + keyword_if = intern_string(t, lit("if")); + first_keyword = keyword_if.s.str; + + keyword_for = intern_string(t, lit("for")); + keyword_else = intern_string(t, lit("else")); + keyword_sizeof = intern_string(t, lit("sizeof")); + keyword_typeof = intern_string(t, lit("typeof")); + keyword_while = intern_string(t, lit("while")); + keyword_switch = intern_string(t, lit("switch")); + keyword_case = intern_string(t, lit("case")); + keyword_struct = intern_string(t, lit("struct")); + keyword_enum = intern_string(t, lit("enum")); + + keyword_union = intern_string(t, lit("union")); + last_keyword = keyword_union.s.str; +} + +function B32 +lex_is_keyword(Intern_String str){ + B32 result = str.s.str >= first_keyword && str.s.str <= last_keyword; + return result; +} + +typedef enum Token_Kind{ + TK_End, + TK_Mul, + TK_Div, + TK_Add, + TK_Sub, + TK_Mod, + TK_BitAnd, + TK_BitOr, + TK_BitXor, + TK_Neg, + TK_Not, + TK_OpenParen, + TK_CloseParen, + TK_OpenBrace, + TK_CloseBrace, + TK_OpenBracket, + TK_CloseBracket, + TK_Comma, + TK_Pound, + TK_Question, + TK_ThreeDots, + TK_Semicolon, + TK_Dot, + TK_LesserThen, + TK_GreaterThen, + TK_Colon, + TK_Assign, + TK_DivAssign, + TK_MulAssign, + TK_ModAssign, + TK_SubAssign, + TK_AddAssign, + TK_AndAssign, + TK_OrAssign, + TK_XorAssign, + TK_LeftShiftAssign, + TK_RightShiftAssign, + TK_DoubleColon, + TK_At, + TK_Decrement, + TK_Increment, + TK_PostDecrement, + TK_PostIncrement, + TK_LesserThenOrEqual, + TK_GreaterThenOrEqual, + TK_Equals, + TK_And, + TK_Or, + TK_NotEquals, + TK_LeftShift, + TK_RightShift, + TK_Arrow, + TK_ExprSizeof, + TK_DocComment, + TK_Comment, + TK_Identifier, + TK_StringLit, + TK_Character, + TK_Error, + TK_Float, + TK_Int, + TK_Keyword, +}Token_Kind; + +typedef struct Token{ + Token_Kind kind; + union{ + String string; + struct{U8 *str; S64 len;}; + }; + + union { + U64 int_val; + F64 float_val; + String error_val; + Intern_String intern_val; + }; + + String file; + S32 line; + U8 *line_begin; +}Token; +#include "token_array.c" + +typedef struct Lex_Stream{ + String stream; + S64 iter; + + U8 *line_begin; + String file; + S32 line; +}Lex_Stream; + + +function U8 +lexc(Lex_Stream *s){ + return s->stream.str[s->iter]; +} + +function U8 +lexci(Lex_Stream *s, S32 i){ + return s->stream.str[s->iter+i]; +} + +function U8 * +lexcp(Lex_Stream *s){ + return s->stream.str + s->iter; +} + +function B32 +lex_is_whitespace(U8 c){ + B32 result = c == '\n' || c == '\r' || c == ' ' || c == '\r'; + return result; +} + +function B32 +lex_is_alphabetic(U8 c){ + B32 result = (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z'); + return result; +} + +function B32 +lex_is_numeric(U8 c){ + B32 result = c >= '0' && c <= '9'; + return result; +} + +function B32 +lex_is_alphanumeric(U8 c){ + B32 result = lex_is_numeric(c) || lex_is_alphabetic(c); + return result; +} + +function void +lex_set_len(Lex_Stream *s, Token *token){ + assert(lexcp(s) >= token->str); + token->len = lexcp(s) - token->str; +} + +function void +token_error(Token *t, String error_val){ + t->kind = TK_Error; + t->error_val = error_val; +} + +function void +lex_parse_u64(Token *t){ + U64 result = 0; + U64 m = 1; + for(S64 i = t->len - 1; i >= 0; --i){ + U64 val = t->str[i] - '0'; + U64 new_val = val * m; + if((result + new_val) < result){ + token_error(t, lit("Integer overflow")); + return; + } + result+=new_val; + m *= 10; + } + t->int_val = result; +} + +function void +lex_advance(Lex_Stream *s){ + if(s->iter >= s->stream.len){ + return; + } + else if(lexc(s) == '\n'){ + s->iter++; + s->line++; + s->line_begin = lexcp(s); + } + else{ + s->iter++; + } +} + +function Token +token_int(U64 val){ + Token result = {.kind = TK_Int, .int_val=val}; + return result; +} + +function void +lex_parse_string(Lex_Stream *s, Token *t, U8 c){ + for(;;){ + if(lexc(s) == '\\') lex_advance(s); + else if(lexc(s) == c) break; + else if(lexc(s) == 0){ + token_error(t, lit("Unterminated string, reached end of file")); + break; + } + lex_advance(s); + } + if(t->kind != TK_Error){ + lex_advance(s); + lex_set_len(s,t); + } +} + +#define CASE2(op, OpName, Assign) \ +case op: \ +if (lexc(s) == '=') { \ +lex_advance(s); \ +t.kind = Assign; \ +} else { \ +t.kind = OpName; \ +} \ +break +#define CASE3(op, OpName, Assign, Incr) \ +case op: \ +if (lexc(s) == '=') { \ +lex_advance(s); \ +t.kind = Assign; \ +} else if (lexc(s) == op) { \ +lex_advance(s); \ +t.kind = Incr; \ +} else { \ +t.kind = OpName; \ +} \ +break + +function void +lex__stream(Token_Array *array, Lex_Stream *s){ + while(lexc(s)){ + while(lex_is_whitespace(lexc(s))) + lex_advance(s); + + Token t = {0}; + t.str = lexcp(s); + t.file = s->file; + t.line = s->line; + t.line_begin = s->line_begin; + lex_advance(s); + + switch(*t.str){ + case 0: break; + case '@': t.kind = TK_At; break; + case '(': t.kind = TK_OpenParen; break; + case ')': t.kind = TK_CloseParen; break; + case '{': t.kind = TK_OpenBrace; break; + case '}': t.kind = TK_CloseBrace; break; + case '[': t.kind = TK_OpenBracket; break; + case ']': t.kind = TK_CloseBracket; break; + case ',': t.kind = TK_Comma; break; + case '~': t.kind = TK_Neg; break; + case '?': t.kind = TK_Question; break; + case ';': t.kind = TK_Semicolon; break; + case '#': t.kind = TK_Pound; break; + CASE2('!', TK_Not, TK_NotEquals); + CASE2('^', TK_BitXor, TK_XorAssign); + CASE2('=', TK_Assign, TK_Equals); + CASE2('*', TK_Mul, TK_MulAssign); + CASE2('%', TK_Mod, TK_ModAssign); + CASE3('+', TK_Add, TK_AddAssign, TK_Increment); + CASE3('&', TK_BitAnd, TK_AndAssign, TK_And); + CASE3('|', TK_BitOr, TK_OrAssign, TK_Or); +#undef CASE2 +#undef CASE3 + case '.': { + if(lexc(s) == '.' && lexci(s,1) == '.') { + lex_advance(s); lex_advance(s); + t.kind = TK_ThreeDots; + } + else { + t.kind = TK_Dot; + } + } break; + + + case '<': { + if (lexc(s) == '<') { + lex_advance(s); + if (lexc(s) == '=') { + lex_advance(s); + t.kind = TK_LeftShiftAssign; + } + else { + t.kind = TK_LeftShift; + } + } + else if (lexc(s) == '=') { + lex_advance(s); + t.kind = TK_LesserThenOrEqual; + } + else { + t.kind = TK_LesserThen; + } + } break; + + case '>': { + if (lexc(s) == '>') { + lex_advance(s); + if (lexc(s) == '=') { + lex_advance(s); + t.kind = TK_RightShiftAssign; + } + else { + t.kind = TK_RightShift; + } + } + else if (lexc(s) == '=') { + lex_advance(s); + t.kind = TK_GreaterThenOrEqual; + } + else { + t.kind = TK_GreaterThen; + } + } break; + + case ':': { + if (lexc(s) == ':') { + lex_advance(s); + t.kind = TK_DoubleColon; + } + else { + t.kind = TK_Colon; + } + } break; + + case '-':{ + if (lexc(s) == '=') { + lex_advance(s); + t.kind = TK_SubAssign; + } + else if (lexc(s) == '-') { + lex_advance(s); + t.kind = TK_Decrement; + } + else if (lexc(s) == '>') { + lex_advance(s); + t.kind = TK_Arrow; + } + else { + t.kind = TK_Sub; + } + } break; + + + case '\'':{not_implemented;} break; + case '"': { + t.kind = TK_StringLit; + lex_parse_string(s,&t,'"'); + if(t.kind != TK_Error){ + t.str += 1; + t.len -= 2; + } + t.intern_val = intern_string(&array->interns, t.string); + } break; + + case '/': { + if(lexc(s) == '='){ + t.kind = TK_DivAssign; + lex_advance(s); + } + else if(lexc(s) == '/'){ + lex_advance(s); + t.kind = TK_Comment; + for(;;){ + if(lexc(s) == '\n' || lexc(s) == 0) break; + lex_advance(s); + } + continue; + } + else if(lexc(s) == '*'){ + lex_advance(s); + t.kind = TK_Comment; + for(;;){ + if(lexc(s) == '*' && lexci(s,1) == '/'){ + lex_advance(s); + lex_advance(s); + break; + } + else if(lexc(s) == 0){ + token_error(&t, lit("Unterminated block comment")); + goto skip_continue; + } + lex_advance(s); + } + continue; + skip_continue:; + } + else { + t.kind = TK_Div; + } + } break; + + case '0':case '1':case '2':case '3':case '4': + case '5':case '6':case '7':case '8':case '9':{ + t.kind = TK_Int; + while(lex_is_numeric(lexc(s))) + lex_advance(s); + lex_set_len(s, &t); + lex_parse_u64(&t); + } break; + + case 'A':case 'a':case 'M':case 'm':case 'B': + case 'b':case 'N':case 'n':case 'C':case 'c':case 'O': + case 'o':case 'D':case 'd':case 'P':case 'p':case 'E': + case 'e':case 'Q':case 'q':case 'F':case 'f':case 'R': + case 'r':case 'G':case 'g':case 'S':case 's':case 'H': + case 'h':case 'T':case 't':case 'I':case 'i':case 'U': + case 'u':case 'J':case 'j':case 'V':case 'v':case 'K': + case 'k':case 'W':case 'w':case 'L':case 'X':case 'l': + case 'x':case 'Z':case 'z':case 'Y':case 'y':case '_': { + t.kind = TK_Identifier; + while(lex_is_alphanumeric(lexc(s)) || lexc(s) == '_') + lex_advance(s); + lex_set_len(s,&t); + t.intern_val = intern_string(&array->interns, t.string); + if(lex_is_keyword(t.intern_val)){ + t.kind = TK_Keyword; + } + } break; + + default: { + token_error(&t, lit("Unknown token")); + } + } + + if(t.len==0) + lex_set_len(s,&t); + + token_array_push(array, &t); + } +} + +function Token_Array +lex_stream(Arena *arena, String stream, String file){ + Lex_Stream s = {stream, 0, stream.str, file, 0}; + Token_Array array = token_array_make(arena); + init_default_keywords(&array.interns); + lex__stream(&array, &s); + return array; +} + +function void +lex_test(){ + Arena *scratch = arena_begin_scratch(); + String test = lit("18446744073709551616{})(@?&+-;....->,:::/**/\"Thing\"//R\n Thingy" + "\"Test_Meme\"+=-===42524 4294967295 18446744073709551615" + "for if while switch"); + Token_Array array = lex_stream(scratch, test, lit("Test1")); + + Token_Kind kind[] = { + TK_Error,TK_OpenBrace,TK_CloseBrace,TK_CloseParen,TK_OpenParen, + TK_At,TK_Question,TK_BitAnd,TK_Add,TK_Sub,TK_Semicolon, + TK_ThreeDots, TK_Dot, TK_Arrow, TK_Comma, TK_DoubleColon, TK_Colon, + TK_StringLit, TK_Identifier, TK_StringLit, TK_AddAssign, TK_SubAssign, + TK_Equals, TK_Int, TK_Int, TK_Int, TK_Keyword, TK_Keyword, + TK_Keyword, TK_Keyword + }; + String strs[] = { + lit("18446744073709551616"),lit("{"),lit("}"),lit(")"),lit("("), + lit("@"),lit("?"),lit("&"),lit("+"),lit("-"),lit(";"), + lit("..."),lit("."),lit("->"),lit(","),lit("::"),lit(":"), + lit("Thing"),lit("Thingy"),lit("Test_Meme"), lit("+="),lit("-="), + lit("=="),lit("42524"),lit("4294967295"),lit("18446744073709551615"), + lit("for"), lit("if"), lit("while"), lit("switch"), + }; + U64 vals[] = { + 42524, 4294967295, 18446744073709551615llu + }; + + int i = 0; + int ui = 0; + for(Token *t = token_array_iter_begin(&array); t; t = token_array_iter_next(&array)){ + assert(t->kind == kind[i]); + assert(string_compare(t->string, strs[i++])); + if(t->kind == TK_Int){ + assert(t->int_val == vals[ui++]); + } + } + arena_end_scratch(); + +} diff --git a/new_parse.c b/new_parse.c new file mode 100644 index 0000000..e2b00ea --- /dev/null +++ b/new_parse.c @@ -0,0 +1,53 @@ + +typedef struct Parser{ + Token_Array array; + Arena *arena; +}Parser; + +function Token * +token_get(Parser *p){ + Token *result = p->array.iter_bucket->data + p->array.iter_len; + return result; +} + +function Token * +token_next(Parser *p){ + Token *result = token_array_iter_next(&p->array); + return result; +} + +function Token * +token_match(Parser *p, Token_Kind kind){ + Token *token = token_get(p); + if(token->kind == kind){ + token = token_next(p); + return token; + } + return 0; +} + +/* + + +*/ + +function Expr * +parse_expr_atom(Parser *p){ + Token *token = 0; + if((token = token_match(p, TK_StringLit))){ + Expr *result = expr_str(p->arena, token); + return result; + } + else if((token = token_match(p, TK_Int))){ + Expr *result = expr_int(p->arena, token); + return result; + } + + invalid_codepath; + return 0; +} + +function Expr * +parse_expr(Parser *p){ + return 0; +} \ No newline at end of file diff --git a/old_main.c b/old_main.c new file mode 100644 index 0000000..cdb5355 --- /dev/null +++ b/old_main.c @@ -0,0 +1,66 @@ +#define _CRT_SECURE_NO_WARNINGS +#include +#include + +#include "lang.h" +#include "os.h" +#include "memory.h" + +#include "lex.h" +#include "parser.h" +#include "expr.h" +#include "ast.h" + +#include "common.c" +#include "file.c" +#include "memory.c" +#include "parser.c" +#include "os_win32.c" + +#include "lex.c" +#include "expr.c" +#include "ast.c" +#include "parse_expr.c" +#include "parse_decl.c" +#include "print.c" +#include "codegen_c.c" + +function void +full_test(){ + Parser p = {0}; + +#if 1 + parser_init(&p); + String string = os_read_file(lit("test.cc")); + parser_lex_stream(&p, string, lit("Parse")); + Decl *decls = parse(&p); + assert(decls->list.first); + { + gen_begin(&p.scratch, &p); + gen_forward_decl(decls); + gen_decl(decls); + //gen_code(decls); + gen_end(); + } + +#endif + + +} + +int +main(){ + use_write_file("output.cc"); + lex_test(); + parser_test(); + full_test(); + + Arena arena = {}; + Token_Array array = token_array_make(&arena); + token_array_push(&array, &(Token){}); + close_all_files(); + + + + return 0; +} \ No newline at end of file diff --git a/os.h b/os.h index 7c37253..f14e644 100644 --- a/os.h +++ b/os.h @@ -4,6 +4,8 @@ typedef struct OS_Memory{ SizeU commit; SizeU reserve; }OS_Memory; +typedef struct Arena Arena; function OS_Memory os_reserve(SizeU size); -function void os_commit(OS_Memory *m, SizeU size); \ No newline at end of file +function void os_commit(OS_Memory *m, SizeU size); +function String os_read_file(Arena *arena, String file); \ No newline at end of file diff --git a/os_win32.c b/os_win32.c index 76165bc..9e91a79 100644 --- a/os_win32.c +++ b/os_win32.c @@ -1,36 +1,16 @@ function S32 os_main(); const SizeU page_size = 4096; -LRESULT CALLBACK -WindowProc(HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam){ - return DefWindowProcW(hwnd, uMsg, wParam, lParam);; -} - -int -WinMain(HINSTANCE hInstance, HINSTANCE a, LPSTR b, int nShowCmd){ - wchar_t *CLASS_NAME = L"Cool window class"; - wchar_t *WINDOW_NAME = L"Have a good day!"; - - WNDCLASSW wc = { }; - wc.lpfnWndProc = WindowProc; - wc.hInstance = hInstance; - wc.lpszClassName = CLASS_NAME; - RegisterClassW(&wc); - - HWND window_handle = CreateWindowExW(0, CLASS_NAME, WINDOW_NAME, WS_OVERLAPPEDWINDOW, CW_USEDEFAULT, CW_USEDEFAULT, CW_USEDEFAULT, CW_USEDEFAULT, 0, 0, hInstance, 0); - ShowWindow(window_handle, nShowCmd); - return os_main(); -} - function String -os_read_file(String name){ - String result = {}; +os_read_file(Arena *arena, String name){ + String result = {0}; FILE *f = fopen((char *)name.str, "rb"); + assert(f); fseek(f, 0, SEEK_END); result.len = ftell(f); fseek(f, 0, SEEK_SET); /* same as rewind(f); */ - result.str = malloc(result.len + 1); + result.str = arena_push_size(arena, result.len + 1); fread(result.str, result.len, 1, f); fclose(f); diff --git a/output.cc b/output.cc index 464ff81..11ef992 100644 --- a/output.cc +++ b/output.cc @@ -1,32 +1,97 @@ -typedef struct OpenGL OpenGL; -struct OpenGL{ -void(*glVertexAttribPointer)(GLuint, GLint, GLenum, GLboolean, GLsizei, GLvoid*); -void(*glBindTexture)(GLenum, GLuint); -void(*glDrawArrays)(GLenum, GLint, GLsizei); +(((534>43)?435:42),234,((S64)32),Thing[10][2],Thing((1,2))) +(4+(2*53)) +((4+2)*53) +(++5) +(--5) +(-5) +(+5) +(sizeof(32)+sizeof(S32*)) +((S64**)5) +(((S64)5)+3) +((534>43)?435:42) +struct Thing{ +int test; }; +S32 thing = 100; +(thing=(thing+10)); +(++thing); +{ +S32 thing_scoped = 10; +(thing_scoped+=10); +} +typedef struct OpenGL OpenGL; typedef struct OS_Memory OS_Memory; +typedef struct Arena Arena; +typedef struct String String; +typedef struct Token Token; +typedef struct Lex_Stream Lex_Stream; +typedef struct Expr Expr; +typedef struct AST_Node AST_Node; +typedef struct Parser_Error Parser_Error; +typedef struct Scope Scope; +typedef struct Parser Parser; +struct test{ +struct Array_Block{ +Array_Block (*next); +Based_Type_Represent (data[size]); +}; + +struct Array{ +Array_Block first; +Array_Block (*last); +S64 block; +S64 len; +}; + +function Based_Type_Represent array_make(Arena (*arena)){ +Test_Type thing; +Based_Type_Represent result; +((result.arena)=arena); +((result.last)=(&(result.first))); +return result; +} +function void array_push(Array (*array), Based_Type_Represent (*item)){ +if (((array->len)+1)>size){ +assert((array->len)); +Array_Block (*block) = ((Thing)arena_push_struct(((array->arena),sizeof(Array_Block)))); +((array->last)=(((array->last).next)=block)); +((array->len)=0); +((array->block)+=1); +} +(((array->last).data)[(++(array->len))]=(*item)); +} +}; + +struct OpenGL{ +void (*glVertexAttribPointer)(GLuint, GLint, GLenumGLenum, GLboolean, GLsizei, GLvoid *); +void (*glBindTexture)(GLenum, GLuint); +void (*glDrawArrays)(GLenumGLenum, GLint, GLsizei); +int (*(test_array[10])); +Things (*thing); +S64 thing_cap; +S64 thing_len; +}; + struct OS_Memory{ -void* data; +void (*data); SizeU commit; SizeU reserve; }; -typedef struct Arena Arena; struct Arena{ OS_Memory memory; U64 len; U64 alignment; }; -typedef struct String String; struct String{ -U8* str; +U8 (*str); S64 len; }; -typedef Intern_String String; -enum Token_Kind{ +typedef String Intern_String; +typedef enum Token_Kind{ TK_End, TK_Mul, TK_Div, @@ -90,37 +155,29 @@ TK_Error, TK_Float, TK_Int, TK_Keyword, -}; +}Token_Kind; -typedef struct Token Token; struct Token{ Token_Kind kind; String string; -union { +union{ S64 integer_val; String error_val; Intern_String intern_val; }; String file; S64 line; -U8* line_begin; +U8 (*line_begin); }; -typedef struct Tokens Tokens; -struct Tokens{ -Token* tokens; -S64 iter; -}; - -typedef struct Lex_Stream Lex_Stream; struct Lex_Stream{ -U8* stream; -U8* line_begin; +U8 (*stream); +U8 (*line_begin); String filename; S64 line; }; -enum Expr_Kind{ +typedef enum Expr_Kind{ EK_None, EK_Atom, EK_Unary, @@ -130,46 +187,45 @@ EK_Cast, EK_List, EK_Call, EK_Index, -}; +}Expr_Kind; -typedef struct Expr Expr; struct Expr{ Expr_Kind kind; -Token* token; -Expr* next; -union { -struct cast_val{ -AST_Node* type; -Expr* expr; -}; -struct list{ -Expr* first; -Expr* last; -}; -struct call{ -Expr* atom; -Expr* list; -}; -struct index{ -Expr* atom; -Expr* index; -}; -struct unary{ -Expr* expr; -}; -struct binary{ -Expr* left; -Expr* right; -}; -struct ternary{ -Expr* cond; -Expr* on_true; -Expr* on_false; -}; +Token (*token); +Expr (*next); +union{ +struct{ +AST_Node (*type); +Expr (*expr); +}cast_val; +struct{ +Expr (*first); +Expr (*last); +}list; +struct{ +Expr (*atom); +Expr (*list); +}call; +struct{ +Expr (*atom); +Expr (*index); +}index; +struct{ +Expr (*expr); +}unary; +struct{ +Expr (*left); +Expr (*right); +}binary; +struct{ +Expr (*cond); +Expr (*on_true); +Expr (*on_false); +}ternary; }; }; -enum AST_Kind{ +typedef enum AST_Kind{ AK_None, AK_BaseType, AK_Typedef, @@ -183,61 +239,56 @@ AK_Array, AK_Function, AK_Variable, AK_EnumChild, -}; +}AST_Kind; -typedef struct AST_Node AST_Node; struct AST_Node{ AST_Kind kind; -Token* pos; +Token (*pos); Intern_String name; -AST_Node* next; -AST_Node* next_scope; -AST_Node* first_note; -AST_Node* last_note; -AST_Node* first_child; -AST_Node* last_child; -union { +AST_Node (*next); +AST_Node (*next_scope); +AST_Node (*first_note); +AST_Node (*last_note); +AST_Node (*first_child); +AST_Node (*last_child); +union{ SizeU base_type_size; -AST_Node* pointer; -AST_Node* typedef_type; -AST_Node* variable_type; -AST_Node* func_return_type; +AST_Node (*pointer); +AST_Node (*typedef_type); +AST_Node (*variable_type); +AST_Node (*func_return_type); }; }; -typedef struct Parser_Error Parser_Error; struct Parser_Error{ -Parser_Error* next; +Parser_Error (*next); String message; -Token* token; +Token (*token); }; -typedef struct Scope Scope; struct Scope{ -Scope* next; -AST_Node* first; -AST_Node* last; +Scope (*next); +AST_Node (*first); +AST_Node (*last); }; -typedef struct Parser Parser; struct Parser{ Arena main_arena; Arena intern_table_arena; Arena symbol_table_arena; -Scope* scope_free_list; -Scope* scope_stack; -Scope* global_scope; +Scope (*scope_free_list); +Scope (*scope_stack); +Scope (*global_scope); S64 symbols_inserted; S64 symbols_count; -AST_Node* symbols; -Intern_String* interns; +AST_Node (*symbols); +Intern_String (*interns); S64 interns_in_bytes; S64 interns_inserted; S64 interns_count; -U8* first_keyword; -U8* last_keyword; +U8 (*first_keyword); +U8 (*last_keyword); Parser_Error default; Parser_Error error; -Tokens token_array; }; diff --git a/parse_decl.c b/parse_decl.c index fc53442..44a1052 100644 --- a/parse_decl.c +++ b/parse_decl.c @@ -7,6 +7,15 @@ parse_type_function(Parser *p, Token *token){ Typespec *result = typespec_function(p, token, 0); if(!token_is(p, TK_CloseParen)) for(;;) { + // Optional name + if(token_is(p, TK_Identifier)){ + if(token_peek_is(p, 1, TK_Colon)){ + token_next(p); + token_next(p); + } + } + + // Parse type if(token_is(p, TK_Identifier)){ Typespec *arg = parse_type(p); typespec_function_push(result, arg); @@ -16,18 +25,16 @@ parse_type_function(Parser *p, Token *token){ } } token_expect(p, TK_CloseParen); - if(token_is(p, TK_Identifier)){ + if(token_is(p, TK_Identifier)) result->function_spec.ret = parse_type(p); - } - else{ + else result->function_spec.ret = typespec_name(p, token_get(p), intern_void); - } + return result; } function Typespec * parse_type(Parser *p){ - // Parse as function type or normal Token *token = 0; Typespec *result = 0; @@ -58,7 +65,6 @@ parse_type(Parser *p){ else break; } - return result; } @@ -72,35 +78,39 @@ parse_expr_assignment(Parser *p){ } function void -parse_note_list(Parser *ctx, Note *parent) { - if(token_match(ctx, TK_OpenParen)) { +parse_note_list(Parser *p, Note *parent) { + if(token_match(p, TK_OpenParen)) { + if(token_match(p, TK_CloseParen)){ + return; + } do { - Token *name = token_expect(ctx, TK_Identifier); - Note *current = note_push_new(ctx, parent, name, name->intern_val, 0); - parse_note_list(ctx, current); - if(token_match(ctx, TK_Assign)) { - current->expr = parse_expr(ctx); + Token *name = token_expect(p, TK_Identifier); + Note *current = note_push_new(p, parent, name, name->intern_val, 0); + parse_note_list(p, current); + if(token_match(p, TK_Assign)) { + current->expr = parse_expr(p); } - } while(token_match(ctx, TK_Comma)); - token_expect(ctx, TK_CloseParen); + } while(token_match(p, TK_Comma)); + token_expect(p, TK_CloseParen); } } function void -parse__notes(Parser *ctx, Note *result) { - while(token_match(ctx, TK_At)) { - Token *name = token_expect(ctx, TK_Identifier); - Note *current = note_push_new(ctx, result, name, name->intern_val, 0); - parse_note_list(ctx, current); - if(token_match(ctx, TK_Assign)) { - current->expr = parse_expr(ctx); +parse__notes(Parser *p, Note *result) { + while(token_match(p, TK_At)) { + Token *name = token_expect(p, TK_Identifier); + Note *current = note_push_new(p, result, name, name->intern_val, 0); + parse_note_list(p, current); + if(token_match(p, TK_Assign)) { + current->expr = parse_expr(p); } + token_match(p, TK_Semicolon); } } function Note parse_notes(Parser *p){ - Note result = {}; + Note result = {0}; parse__notes(p, &result); return result; } @@ -172,7 +182,6 @@ function Decl * parse_variable(Parser *p, Token *name){ Typespec *type = parse_type(p); Expr *expr = parse_expr_assignment(p); - token_expect(p, TK_Semicolon); return decl_variable(p, name, name->intern_val, type, expr); } @@ -182,14 +191,77 @@ parse_typedef(Parser *p, Token *name){ return decl_typedef(p, name, name->intern_val, type); } -/* +function Stmt * +parse_stmt(Parser *p); +function Stmt * +parse_stmt_list(Parser *p){ + Token *token = token_expect(p, TK_OpenBrace); + Stmt *result = stmt_list(p, token); + while(!token_match(p, TK_CloseBrace)) { + Stmt *stmt = parse_stmt(p); + stmt_push(result, stmt); + } + return result; +} + +function Stmt * +parse_stmt(Parser *p){ + Token *token = token_get(p); + Decl *decl = parse_decl(p); + if(decl){ + Stmt *result = stmt_decl(p, token, decl); + return result; + } + else if(token_match_keyword(p, keyword_return)){ + Expr *expr = parse_expr(p); + Stmt *result = stmt_return(p, token, expr); + token_expect(p, TK_Semicolon); + return result; + } + else if(token_match_keyword(p, keyword_if)){ + Expr *expr = parse_expr(p); + Stmt *if_body = parse_stmt_list(p); + Stmt *result = stmt_if(p, token, if_body, expr); + Stmt *head = result; + while(token_match_keyword(p, keyword_else)){ + if(token_match_keyword(p, keyword_if)){ + expr = parse_expr(p); + if_body = parse_stmt_list(p); + head = head->next = stmt_if(p, token, if_body, expr); + } + else{ + if_body = parse_stmt_list(p); + head = head->next = stmt_if(p, token, if_body, 0); + break; + } + } + return result; + + } + else if((token_is(p, TK_OpenBrace))){ + Stmt *result = parse_stmt_list(p); + return result; + } + else{ + Expr *expr = parse_expr(p); + token_expect(p, TK_Semicolon); + return stmt_expr(p, token, expr); + } +} + function Decl * parse_function(Parser *p, Token *name){ Decl *result = decl_function(p, name, name->intern_val, 0); if(!token_is(p, TK_CloseParen)){ for(;;) { - if((name = token_match(p, TK_Identifier))) - decl_function_push(p, result, name, name->intern_val, parse_type(p)); + if(token_peek_is(p, 1, TK_Colon)){ + if(token_peek_is(p, 2, TK_Identifier) || + token_peek_is(p, 2, TK_OpenParen)){ + Token *name = parse_get_token_name(p,1); + Typespec *type = parse_type(p); + decl_function_push(p, result, name, name->intern_val, type); + } + } else if(!token_match(p, TK_Comma)) break; } @@ -198,13 +270,12 @@ parse_function(Parser *p, Token *name){ if(token_is(p, TK_Identifier)) result->function_decl.ret = parse_type(p); + else + result->function_decl.ret = typespec_name(p, token_get(p), intern_void); - token_expect(p, TK_OpenBrace); - parse_stmt_list(p, result); - token_expect(p, TK_CloseBrace); + result->function_decl.body = parse_stmt_list(p); return result; } - */ function Decl * parse_decl(Parser *p){ @@ -225,18 +296,21 @@ parse_decl(Parser *p){ else if(token_peek_is_keyword(p, 2, keyword_typedef)){ result = parse_typedef(p, parse_get_token_name(p,2)); } + else if(token_peek_is(p, 2, TK_OpenParen)){ + result = parse_function(p, parse_get_token_name(p,2)); + } } else if(token_peek_is(p, 1, TK_Colon)){ if(token_peek_is(p, 2, TK_Identifier) || token_peek_is(p, 2, TK_OpenParen)){ result = parse_variable(p, parse_get_token_name(p,1)); + token_expect(p, TK_Semicolon); } } } if(result){ - result->first_note = notes.first; - result->last_note = notes.last; + decl_pass_notes(result, ¬es); } else if(notes.first != 0){ parser_push_error(p, token_get(p), "Detected notes that are not attached to anything"); @@ -247,7 +321,7 @@ parse_decl(Parser *p){ function Decl * parse(Parser *p){ - Decl *result = decl_new(p, DECL_List, token_get(p), (Intern_String){}); + Decl *result = decl_new(p, DECL_List, token_get(p), (Intern_String){0}); for(;;){ Decl *decl = 0; if(token_is(p, TK_End)) { @@ -265,188 +339,55 @@ parse(Parser *p){ return result; } - -/* -function Token * -parse__get_name(Parser *p){ -Token *name = token_next(p); -token_next(p); -token_next(p); -return name; -} - -function Note * -parse_enum(Parser *p, Token *name){ -Note *result = 0; -if(token_expect(p, TK_OpenBrace)){ - result = ast_enum(p, name, name->intern_val); - do{ - Note notes = parse_notes(p); - Token *token = token_match(p, TK_Identifier); - if(token){ - Expr *expr = 0; - if(token_match(p, TK_Assign)){ - expr = parse_expr(p); - } - - Note *child = ast_enum_child(p, token, token->intern_val, expr); - ast_node_pass_note_list(child,¬es); - ast_node_push_child(result, child); - } - else{ - break; - } - } while(token_match(p, TK_Comma)); - token_expect(p, TK_CloseBrace); -} -return result; -} - -function Note * -parse_variable(Parser *p, Token *name){ -Note *result = 0; -Token *type_token = token_expect(p, TK_Identifier); -if(type_token){ - Note *type = symbol_require_type(p, type_token); - Token *star; - while((star = token_match(p, TK_Mul))){ - type = ast_type_pointer(p, star, type); - } - while((star = token_match(p, TK_OpenBracket))){ - Expr *expr = parse_expr(p); - type = ast_type_array(p, star, type, expr); - token_expect(p, TK_CloseBracket); - } - Expr *expr = 0; - if(token_match(p, TK_Assign)){ - expr = parse_expr(p); - } - result = ast_variable(p, name, name->intern_val, type, expr); -} -return result; -} - -function Note * -parse_struct(Parser *p, Token *name, AST_Kind kind, B32 is_global){ -Note *result = 0; -if(token_expect(p, TK_OpenBrace)){ - result = ast_struct(p, name, name?name->intern_val:(Intern_String){}, kind); - if(is_global) symbol_register(p, result); - - for(;;){ - Note notes = parse_notes(p); - Note *mem = 0; - Token *mem_name = token_match(p, TK_Identifier); - if(mem_name){ - if(token_expect(p, TK_Colon)){ - - if(token_match_keyword(p, keyword_struct)){ - mem = parse_struct(p, mem_name, AK_Struct, false); - } - else if(token_match_keyword(p, keyword_union)){ - mem = parse_struct(p, mem_name, AK_Union, false); - } - //else if(token_match_keyword(p, keyword_enum)){ - //mem = parse_enum(p, mem_name); - //} - else if(token_is(p, TK_Identifier)){ - mem = parse_variable(p, mem_name); - token_expect(p, TK_Semicolon); - } - else parser_push_error(p, mem_name, "Unrecognized token while parsing struct"); - - if(mem){ - ast_node_pass_note_list(mem, ¬es); - ast_node_push_child(result, mem); - } - - } - else{ - break; - } - } - else if(token_match_keyword(p, keyword_union) && - token_match(p, TK_Colon)){ - mem = parse_struct(p, 0, AK_Union, false); - ast_node_pass_note_list(mem, ¬es); - ast_node_push_child(result, mem); - } - else if(token_match_keyword(p, keyword_struct) && - token_match(p, TK_Colon)){ - mem = parse_struct(p, 0, AK_Struct, false); - ast_node_pass_note_list(mem, ¬es); - ast_node_push_child(result, mem); - } - else if(token_expect(p, TK_CloseBrace)){ - break; - } - else{ - break; - } - } -} -return result; -} - -function Note * -parse_typedef(Parser *p, Token *name){ -Token *type_token = token_expect(p, TK_Identifier); -Note *type = symbol_require_type(p, type_token); -Note *result = ast_typedef(p, name, name->intern_val, type); -token_expect(p, TK_Semicolon); -return result; -} - -function Note_List * -parse(Parser *p){ -Note_List *result = ast_node_new(p, AK_List, token_get(p), intern_empty); - -for(;;){ - Note *node = 0; - Note notes = parse_notes(p); - - if(token_is(p, TK_End)){ - break; - } - else if(token_is(p, TK_Error)){ - break; - } - else if(token_is(p, TK_Identifier) && // Peeking to be more error averse - token_peek_is(p, 1, TK_Colon)){ - if(token_peek_is_keyword(p, 2, keyword_struct)){ - node = parse_struct(p, parse__get_name(p), AK_Struct, true); - symbol_register(p, node); - } - else if(token_peek_is_keyword(p, 2, keyword_union)){ - node = parse_struct(p, parse__get_name(p), AK_Union, true); - symbol_register(p, node); - } - else if(token_peek_is_keyword(p, 2, keyword_enum)){ - node = parse_enum(p, parse__get_name(p)); - symbol_register(p, node); - } - else if(token_peek_is_keyword(p, 2, keyword_typedef)){ - node = parse_typedef(p, parse__get_name(p)); - symbol_register(p, node); - } - else{ - token_next(p); - } - } - else{ - token_next(p); +function void gen_stmt(Stmt *stmt); +function void gen_end(); +function void gen_begin(Arena *arena, Parser *p); +function void expr_print(Parser *p, Expr *expr); +function void +parser_test(){ + Parser p = {0}; + { + parser_init(&p); + Intern_String a = intern_string(&p, lit("Thing")); + Intern_String b = intern_string(&p, lit("Thing")); + assert(a.s.str == b.s.str); } + String exprs[] = { + lit("(534>43?435:42,234,cast(S64)32/*todo cast*/,Thing[10][2],Thing(1,2))"), + lit("(4+2*53)"), + lit("((4+2)*53)"), + lit("++5"), + lit("5--"), // @Todo(Krzosa): + lit("-5"), + lit("(+5)"), + lit("sizeof(32) + sizeof(:S32*)"), + lit("cast(S64**)5"), + lit("cast(S64)5+3"), + lit("534>43?435:42"), + }; + for(S64 i = 0; i < buff_cap(exprs); i++){ + parser_lex_stream(&p, exprs[i], lit("File")); + Expr *expr = parse_expr(&p); + assert(expr); + expr_print(&p, expr); + lex_print("\n"); + } - if(node){ - ast_node_pass_note_list(node, ¬es); - ast_node_push_child(result, node); + String stmts[] = { + lit("Thing :: struct { test: int; } "), + lit("thing: S32 = 100; "), + lit("thing = thing + 10; "), + lit("thing++; "), + lit("{ thing_scoped: S32 = 10; thing_scoped += 10; } "), + }; + for(S64 i = 0; i < buff_cap(stmts); i++){ + parser_lex_stream(&p, stmts[i], lit("File")); + Stmt *stmt = parse_stmt(&p); + assert(stmt); + gen_begin(&p.scratch, &p); + gen_stmt(stmt); + gen_end(); + lex_print("\n"); } - else if(notes.first_note != 0){ - parser_push_error(p, token_get(p), "Warning: notes got lost"); - } -} - -return result; -} -*/ \ No newline at end of file +} \ No newline at end of file diff --git a/parse_expr.c b/parse_expr.c index c54cf69..5ccbf29 100644 --- a/parse_expr.c +++ b/parse_expr.c @@ -1,3 +1,4 @@ + function Expr* parse_expr(Parser* p); function Expr* parse_list_expr(Parser* p); function Typespec *parse_type(Parser *p); @@ -40,6 +41,7 @@ function Expr* parse_postfix_expr(Parser* p){ Expr* result = parse_atom_expr(p); while (token_is(p, TK_Dot) + || token_is(p, TK_Arrow) || token_is(p, TK_OpenParen) || token_is(p, TK_OpenBracket) || token_is(p, TK_Decrement) @@ -51,6 +53,11 @@ parse_postfix_expr(Parser* p){ result = expr_binary(p, op, result, r); } + else if (token_match(p, TK_Arrow)){ + Expr* r = parse_atom_expr(p); + result = expr_binary(p, op, result, r); + } + else if (token_match(p, TK_OpenParen)){ Expr* list = 0; if (!token_match(p, TK_CloseParen)){ diff --git a/parser.c b/parser.c index 6894f51..cb26747 100644 --- a/parser.c +++ b/parser.c @@ -16,6 +16,10 @@ global Intern_String intern_sizeu; global Intern_String keyword_sizeof; global Intern_String keyword_cast; global Intern_String keyword_enum; +global Intern_String keyword_for; +global Intern_String keyword_if; +global Intern_String keyword_else; +global Intern_String keyword_return; global Intern_String keyword_typedef; global Intern_String keyword_struct; global Intern_String keyword_union; @@ -28,6 +32,10 @@ parser_init(Parser *p){ p->interns = arena_push_array(&p->intern_table_arena, Intern_String, p->interns_count); keyword_sizeof = intern_string(p, lit("sizeof")); + keyword_if = intern_string(p, lit("if")); + keyword_else = intern_string(p, lit("else")); + keyword_return = intern_string(p, lit("return")); + keyword_for = intern_string(p, lit("for")); keyword_cast = intern_string(p, lit("cast")); keyword_struct = intern_string(p, lit("struct")); keyword_enum = intern_string(p, lit("enum")); @@ -74,7 +82,7 @@ parser_push_error(Parser *p, Token *token, char *str, ...){ va_end(args1); } - lex_print("Error: %s\n", string.str);// @Todo(Krzosa): + printf("Error: %s %s:%d\n", string.str, token->file.str, (S32)token->line); Parser_Error *error = arena_push_struct(&p->main_arena, Parser_Error); error->message = string; error->next = 0; @@ -140,7 +148,7 @@ table_index_advance(Table_Index *index){ function Intern_String intern_string(Parser *p, String string){ - Intern_String result = {}; + Intern_String result = {0}; Table_Index index = table_index_from_string(string, p->interns_count); for(;;){ Intern_String *intern = p->interns + index.iter; diff --git a/parser.h b/parser.h index 99494d3..abdc345 100644 --- a/parser.h +++ b/parser.h @@ -12,6 +12,7 @@ typedef struct Parser{ Arena main_arena; Arena intern_table_arena; Arena symbol_table_arena; + Arena scratch; S64 interns_in_bytes; S64 interns_inserted; diff --git a/scratch.c b/scratch.c new file mode 100644 index 0000000..fee7de8 --- /dev/null +++ b/scratch.c @@ -0,0 +1,14 @@ +global Arena global_scratch; +global Arena_Checkpoint global_scratch_checkpoint; + +function Arena * +arena_begin_scratch(){ + global_scratch_checkpoint = arena_checkpoint(&global_scratch); + return &global_scratch; +} + +function void +arena_end_scratch(){ + arena_restore(global_scratch_checkpoint); +} + diff --git a/test.cc b/test.cc index 446a843..511eedb 100644 --- a/test.cc +++ b/test.cc @@ -1,9 +1,53 @@ #if 0 + +@register +@param_expr(size) +@param_type(Type) +test::struct{ + Array_Block::struct{ + next: Array_Block*; + data: Type[size]; + } + + @inline Array::struct{ + first: Array_Block; + last : Array_Block*; + block: S64; + len : S64; + } + + array_make::(arena: Arena*) Type { + thing: Test_Type; + result: Type; + result.arena = arena; + result.last = &result.first; + return result; + } + + array_push::(array: Array*, item: Type*){ + if array->len+1 > size{ + assert(array->len); + block: Array_Block* = cast(Thing)arena_push_struct(array->arena, sizeof(:Array_Block)); + array->last = array->last.next = block; + array->len = 0; + array->block += 1; + } + array->last.data[array->len++] = *item; + } +} + +@test(size = 4096) OpenGL :: struct{ glVertexAttribPointer: (GLuint, GLint, GLenum, GLboolean, GLsizei, GLvoid*); glBindTexture: (GLenum, GLuint); glDrawArrays: (GLenum, GLint, GLsizei); + + test_array: int*[10]; + + thing : Things*; + thing_cap: S64; + thing_len: S64; } OS_Memory::struct{ @@ -92,6 +136,7 @@ Token_Kind::enum{ Keyword, } +@array(type=block) Token::struct{ kind:Token_Kind; @using string:String; @@ -105,11 +150,6 @@ Token::struct{ line_begin:U8*; } -Tokens::struct{ - @array tokens: Token*; - iter : S64; -} - Lex_Stream::struct{ stream: U8*; line_begin: U8*; @@ -217,6 +257,7 @@ Scope :: struct{ last : AST_Node*; } + Parser :: struct{ main_arena: Arena; intern_table_arena: Arena; @@ -247,7 +288,7 @@ Parser :: struct{ // This works ok @sllqueue error: Parser_Error; - @using token_array: Tokens; + //@using token_array: Tokens; } /* diff --git a/token_array.c b/token_array.c new file mode 100644 index 0000000..82e5776 --- /dev/null +++ b/token_array.c @@ -0,0 +1,100 @@ + +typedef struct Token_Bucket Token_Bucket; +typedef struct Token_Array Token_Array; + +struct Token_Bucket{ + Token_Bucket *next; + Token data[4096]; +}; + +// @Note(Krzosa): +// Interns has an arena - this arena points to arena in Token_Array +// Token_Array arena is used to allocate the token buckets, Intern table and Intern_Strings +// +// If we want to resize the table in the future we need a second arena or use the heap +// For now lets leave it this way +// or +// We could potentially also use the bucket array and allocate additional buckets when resizing +// the table. In that case we would need to traverse the bucket list when inserting +struct Token_Array{ + Intern_Table interns; + + Token_Bucket first; + Token_Bucket *last; + S64 len; + S64 block; + Arena *arena; + + // @Note(Krzosa): Iterator + Token_Bucket *iter_bucket; + S64 iter_len; + S64 iter_block; +}; + +function Token_Array +token_array_make(Arena *arena){ + Token_Array result = { + .last = &result.first, + .arena = arena, + .interns = intern_table(arena, 4096*4) + }; + return result; +} + +function void +token_array_push(Token_Array *array, Token *p){ + if(array->len >= buff_cap(array->first.data)){ + Token_Bucket *bucket = arena_push_struct(array->arena, Token_Bucket); + array->last = array->last->next = bucket; + array->len = 0; + array->block += 1; + } + array->last->data[array->len++] = *p; +} + +function B32 +token_array_iter_is_end(Token_Array *array){ + B32 result = array->iter_len == array->len && array->iter_block == array->block; + return result; +} + +function Token * +token_array_iter_next(Token_Array *array){ + if(token_array_iter_is_end(array)){ + return 0; + } + if(array->iter_len >= buff_cap(array->first.data)){ + array->iter_len = 0; + array->iter_block += 1; + array->iter_bucket = array->iter_bucket->next; + } + Token *result = array->iter_bucket->data + array->iter_len++; + return result; +} + +function Token * +token_array_iter_peek(Token_Array *array, S64 i){ + S64 save_len = array->iter_len; + Token_Bucket *save_bucket = array->iter_bucket; + assert(i < buff_cap(array->first.data)); + + S64 over = i; + if(array->iter_len + i >= buff_cap(array->first.data)){ + over = buff_cap(array->first.data) - (array->iter_len + i); + array->iter_len = 0; + array->iter_bucket = array->iter_bucket->next; + } + Token *result = array->iter_bucket->data + array->iter_len + over; + array->iter_len = save_len; + array->iter_bucket = save_bucket; + return result; +} + +function Token * +token_array_iter_begin(Token_Array *array){ + array->iter_len = 0; + array->iter_block = 0; + array->iter_bucket = &array->first; + Token *result = token_array_iter_next(array); + return result; +}