krzosa/corelang
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

New approach, new lexer

Browse Source
This commit is contained in:
Krzosa Karol
2022-05-06 10:13:16 +02:00
parent 557dde1936
commit e3b5e9b33a
33 changed files with 3331 additions and 784 deletions
Download Patch File Download Diff File Expand all files Collapse all files

455
ast.c
View File

@@ -81,6 +81,19 @@ decl_list_push(Decl *parent, Decl *child){
SLLQueuePush(parent->list.first, parent->list.last, child);
}
//-----------------------------------------------------------------------------
// Ops
//-----------------------------------------------------------------------------
function Decl *
decl_struct_find_node(Decl *decl, String string){
for(Decl *n = decl->struct_decl.first; n; n=n->next){
if(string_compare(string, n->name.s)){
return n;
}
}
return 0;
}
//-----------------------------------------------------------------------------
// Notes
//-----------------------------------------------------------------------------
@@ -172,4 +185,444 @@ typespec_function(Parser *p, Token *pos, Typespec *ret){
function void
typespec_function_push(Typespec *func, Typespec *arg){
SLLQueuePush(func->function_spec.first, func->function_spec.last, arg);
}
}
//-----------------------------------------------------------------------------
//
//-----------------------------------------------------------------------------
function Typespec *
typespec_get_func(Typespec *type){
switch(type->kind){
case TS_Name:{return 0;} break;
case TS_Pointer:{return typespec_get_func(type->base);} break;
case TS_Array:{return typespec_get_func(type->array_spec.base);} break;
case TS_Function:{return type;} break;
default: {invalid_codepath;} break;
}
return 0;
}
function Typespec *
typespec_get_name(Typespec *type){
switch(type->kind){
case TS_Name:{return type;} break;
case TS_Pointer:{return typespec_get_name(type->base);} break;
case TS_Array:{return typespec_get_name(type->array_spec.base);} break;
case TS_Function:{return 0;} break;
default: {invalid_codepath;} break;
}
return 0;
}
function String
typespec_require_name_string(Typespec *type){
Typespec *result = typespec_get_name(type);
if(result){
if(result->kind == TS_Name){
return result->name.s;
}
}
invalid_codepath;
return string_empty;
}
//-----------------------------------------------------------------------------
// Statements
//-----------------------------------------------------------------------------
function Stmt *
stmt_new(Parser *p, Stmt_Kind kind, Token *pos){
Stmt *result = arena_push_struct(&p->main_arena, Stmt);
result->kind = kind;
result->pos = pos;
return result;
}
function Stmt *
stmt_decl(Parser *p, Token *pos, Decl *decl){
Stmt *result = stmt_new(p, STMT_Decl, pos);
result->decl = decl;
return result;
}
function Stmt *
stmt_expr(Parser *p, Token *pos, Expr *expr){
Stmt *result = stmt_new(p, STMT_Expr, pos);
result->expr = expr;
return result;
}
function Stmt *
stmt_list(Parser *p, Token *pos){
Stmt *result = stmt_new(p, STMT_List, pos);
return result;
}
function Stmt *
stmt_return(Parser *p, Token *pos, Expr *expr){
Stmt *result = stmt_new(p, STMT_Return, pos);
result->ret.expr = expr;
return result;
}
function Stmt *
stmt_if(Parser *p, Token *pos, Stmt *body, Expr *cond){
Stmt *result = stmt_new(p, STMT_If, pos);
result->stmt_if.cond = cond;
result->stmt_if.body = body;
return result;
}
function void
stmt_push(Stmt *stmt, Stmt *child){
SLLQueuePush(stmt->list.first, stmt->list.last, child);
}
//-----------------------------------------------------------------------------
// Pointer Array
//-----------------------------------------------------------------------------
function Pointer_Array
pointer_array_make(Arena *arena){
Pointer_Array result = {
.last = &result.first,
.arena = arena,
};
return result;
}
#define bucket_size (buff_cap(array->first.data))
function void
pointer_array_push(Pointer_Array *array, Pointer p){
if(array->len >= bucket_size){
Pointer_Bucket *bucket = arena_push_struct(array->arena, Pointer_Bucket);
array->last = array->last->next = bucket;
array->len = 0;
array->block += 1;
}
array->last->data[array->len++] = p;
}
function B32
pointer_array_iter_is_end(Pointer_Array *array){
B32 result = array->iter_len == array->len && array->iter_block == array->block;
return result;
}
function Pointer
pointer_array_iter_next(Pointer_Array *array){
if(pointer_array_iter_is_end(array)){
return (Pointer){0};
}
if(array->iter_len >= bucket_size){
array->iter_len = 0;
array->iter_block += 1;
array->iter_bucket = array->iter_bucket->next;
}
Pointer result = array->iter_bucket->data[array->iter_len++];
return result;
}
function Pointer
pointer_array_iter_begin(Pointer_Array *array){
array->iter_len = 0;
array->iter_block = 0;
array->iter_bucket = &array->first;
Pointer result = pointer_array_iter_next(array);
return result;
}
function Pointer
pointer_typespec(Typespec *t){
return (Pointer){.kind = PK_Typespec, .typespec = t};
}
function Pointer
pointer_expr(Expr *t){
return (Pointer){.kind = PK_Expr, .expr = t};
}
function Pointer
pointer_decl(Decl *t){
return (Pointer){.kind = PK_Decl, .decl = t};
}
function Pointer
pointer_stmt(Stmt *t){
return (Pointer){.kind = PK_Stmt, .stmt = t};
}
function Pointer
pointer_enum_child(Decl_Enum_Child *t){
return (Pointer){.kind = PK_Enum_Child, .enum_child = t};
}
function Pointer
pointer_func_arg(Decl_Function_Arg *t){
return (Pointer){.kind = PK_Func_Arg, .func_arg=t};
}
function void
pointer_array_push_typespec(Pointer_Array *array, Typespec *typespec){
pointer_array_push(array, pointer_typespec(typespec));
}
function void
pointer_array_push_decl(Pointer_Array *array, Decl *decl){
pointer_array_push(array, pointer_decl(decl));
}
function void
pointer_array_push_stmt(Pointer_Array *array, Stmt *stmt){
pointer_array_push(array, pointer_stmt(stmt));
}
function void
pointer_array_push_expr(Pointer_Array *array, Expr *expr){
pointer_array_push(array, pointer_expr(expr));
}
function void
pointer_array_push_enum_child(Pointer_Array *array, Decl_Enum_Child *enum_child){
pointer_array_push(array, pointer_enum_child(enum_child));
}
function void
pointer_array_push_func_arg(Pointer_Array *array, Decl_Function_Arg *func_arg){
pointer_array_push(array, pointer_func_arg(func_arg));
}
//-----------------------------------------------------------------------------
// Gather
//-----------------------------------------------------------------------------
function void
gather_try_recurse_typespec(Pointer_Array *array, Typespec *typespec, Gather_Flag gflag, Traversal_Flag tflag);
function void
gather_recurse_expr(Pointer_Array *array, Expr *expr, Gather_Flag gflag, Traversal_Flag tflag){
if(gflag & GATHER_Expr)
pointer_array_push_expr(array, expr);
switch(expr->kind) {
case EK_Atom: {} break;
case EK_Sizeof:{
if(expr->size_of.kind == SIZEOF_Expr){
gather_recurse_expr(array, expr->size_of.expr, gflag, tflag);
}
else{
assert(expr->size_of.kind == SIZEOF_Type);
gather_try_recurse_typespec(array, expr->size_of.type, gflag, tflag);
}
}break;
case EK_Binary:{
gather_recurse_expr(array, expr->binary.left, gflag, tflag);
gather_recurse_expr(array, expr->binary.right, gflag, tflag);
} break;
case EK_Unary:{
gather_recurse_expr(array, expr->unary.expr, gflag, tflag);
} break;
case EK_Ternary:{
gather_recurse_expr(array, expr->ternary.cond, gflag, tflag);
gather_recurse_expr(array, expr->ternary.on_true, gflag, tflag);
gather_recurse_expr(array, expr->ternary.on_false, gflag, tflag);
} break;
case EK_List:{
for(Expr *n = expr->list.first; n; n=n->next){
gather_recurse_expr(array, n, gflag, tflag);
}
}break;
case EK_Cast:{
gather_try_recurse_typespec(array, expr->cast.type, gflag, tflag);
gather_recurse_expr(array, expr->cast.expr, gflag, tflag);
} break;
case EK_Index:{
gather_recurse_expr(array, expr->index.atom, gflag, tflag);
gather_recurse_expr(array, expr->index.index, gflag, tflag);
}break;
case EK_Call:{
gather_recurse_expr(array, expr->call.atom, gflag, tflag);
gather_recurse_expr(array, expr->call.list, gflag, tflag);
}break;
default: {invalid_codepath;} break;
}
}
function void
gather_try_recurse_expr(Pointer_Array *array, Expr *expr, Gather_Flag gflag, Traversal_Flag tflag){
if(expr){
if(tflag & TRAVERS_Expr){
gather_recurse_expr(array, expr, gflag, tflag);
}
else if(gflag & GATHER_Expr){
pointer_array_push_expr(array, expr);
}
}
}
function void
gather_recurse_decl(Pointer_Array *array, Decl *decl, Gather_Flag gflag, Traversal_Flag tflag);
function void
gather_try_recurse_stmt(Pointer_Array *array, Stmt *stmt, Gather_Flag gflag, Traversal_Flag tflag);
function void
gather_recurse_stmt(Pointer_Array *array, Stmt *stmt, Gather_Flag gflag, Traversal_Flag tflag){
switch(stmt->kind) {
case STMT_List: {
for(Stmt *s = stmt->list.first; s; s=s->next){
gather_recurse_stmt(array, s, gflag, tflag);
}
} break;
case STMT_Return:{
gather_try_recurse_expr(array, stmt->ret.expr, gflag, tflag);
} break;
case STMT_If:{
gather_recurse_expr(array, stmt->stmt_if.cond, gflag, tflag);
gather_recurse_stmt(array, stmt->stmt_if.body, gflag, tflag);
for(Stmt_If *s = stmt->stmt_if.next; s; s=s->next){
gather_recurse_stmt(array, s->body, gflag, tflag);
gather_recurse_expr(array, s->cond, gflag, tflag);
}
} break;
case STMT_Expr: {
gather_recurse_expr(array, stmt->expr, gflag, tflag);
} break;
case STMT_Decl: {
gather_recurse_decl(array, stmt->decl, gflag, tflag);
} break;
default: {invalid_codepath;} break;
}
}
function void
gather_try_recurse_stmt(Pointer_Array *array, Stmt *stmt, Gather_Flag gflag, Traversal_Flag tflag){
if(stmt){
if(tflag & TRAVERS_Stmt){
gather_recurse_stmt(array, stmt, gflag, tflag);
}
else if(gflag & GATHER_Stmt){
pointer_array_push_stmt(array, stmt);
}
}
}
function void
gather_recurse_typespec(Pointer_Array *array, Typespec *typespec, Gather_Flag gflag, Traversal_Flag tflag){
if(gflag & GATHER_Typespec)
pointer_array_push_typespec(array, typespec);
switch(typespec->kind) {
case TS_Name: {
} break;
case TS_Pointer: {
gather_recurse_typespec(array, typespec->base, gflag, tflag);
} break;
case TS_Array: {
gather_recurse_typespec(array, typespec->array_spec.base, gflag, tflag);
gather_try_recurse_expr(array, typespec->array_spec.size, gflag, tflag);
} break;
case TS_Function: {
gather_recurse_typespec(array, typespec->function_spec.ret, gflag, tflag);
for(Typespec *n = typespec->function_spec.first; n; n=n->next){
gather_recurse_typespec(array, n, gflag, tflag);
}
} break;
default: {invalid_codepath;} break;
}
}
function void
gather_try_recurse_typespec(Pointer_Array *array, Typespec *typespec, Gather_Flag gflag, Traversal_Flag tflag){
if(typespec){
if(tflag & TRAVERS_Typespec){
gather_recurse_typespec(array, typespec, gflag, tflag);
}
else if(gflag & GATHER_Typespec){
pointer_array_push_typespec(array, typespec);
}
}
}
function void
gather_recurse_decl(Pointer_Array *array, Decl *decl, Gather_Flag gflag, Traversal_Flag tflag){
if(gflag & GATHER_Decl) pointer_array_push_decl(array, decl);
switch(decl->kind){
case DECL_Struct:
case DECL_Union:{
for(Decl *n = decl->struct_decl.first; n; n=n->next){
gather_recurse_decl(array, n, gflag, tflag);
}
} break;
case DECL_Enum:{
gather_try_recurse_typespec(array, decl->enum_decl.typespec, gflag, tflag);
for(Decl_Enum_Child *child = decl->enum_decl.first; child; child=child->next){
if(gflag & GATHER_Enum_Child) pointer_array_push_enum_child(array, child);
gather_try_recurse_expr(array, child->expr, gflag, tflag);
}
}break;
case DECL_Variable:{
gather_try_recurse_typespec(array, decl->variable_decl.type, gflag, tflag);
gather_try_recurse_expr(array, decl->variable_decl.expr, gflag, tflag);
}break;
case DECL_Typedef:{
gather_try_recurse_typespec(array, decl->typedef_decl.type, gflag, tflag);
}break;
case DECL_Function:{
gather_try_recurse_typespec(array, decl->function_decl.ret, gflag, tflag);
for(Decl_Function_Arg *arg = decl->function_decl.first; arg; arg=arg->next){
if(gflag & GATHER_Func_Arg) pointer_array_push_func_arg(array, arg);
gather_try_recurse_typespec(array, arg->typespec, gflag, tflag);
}
gather_try_recurse_stmt(array, decl->function_decl.body, gflag, tflag);
}break;
case DECL_List:{
for(Decl *n = decl->list.first; n; n=n->next){
gather_recurse_decl(array, n, gflag, tflag);
}
}break;
default:invalid_codepath;
}
}
function Pointer_Array
gather(Arena *arena, Decl *decl, Gather_Flag gflag, Traversal_Flag tflag){
Pointer_Array array = pointer_array_make(arena);
gather_recurse_decl(&array, decl, gflag, tflag);
return array;
}
//-----------------------------------------------------------------------------
//
//-----------------------------------------------------------------------------
function Decl *
decl_deep_copy_recurse(Arena *arena, Decl *decl){
Decl *result = arena_push_struct(arena, Decl);
memory_copy(result, decl, sizeof(*decl));
switch(decl->kind) {
case DECL_Struct:{
}break;
case DECL_Union:{
}break;
case DECL_Enum:{
}break;
case DECL_Variable:{
}break;
case DECL_Typedef:{
}break;
case DECL_Function:{
}break;
case DECL_List:{
}break;
default: {invalid_codepath;} break;
}
return result;
}
function Decl *
decl_deep_copy(Arena *arena){
Decl *result = decl_deep_copy_recurse(arena, decl);
return result;
}

309
ast.h
View File

@@ -1,164 +1,201 @@
typedef struct Decl_Enum_Child Decl_Enum_Child;
typedef struct Decl_Function_Arg Decl_Function_Arg;
typedef struct Typespec Typespec;
typedef struct Decl Decl;
typedef struct Note Note;
typedef struct Stmt Stmt;
#if 0
//-----------------------------------------------------------------------------
// Type specifier
// Type specifiers
//-----------------------------------------------------------------------------
typedef enum Typespec_Kind{
TS_None,
TS_Name,
TS_Pointer,
TS_Array,
TS_Function,
}Typespec_Kind;
struct Typespec{
Typespec_Kind kind;
Typespec *next;
Token *pos;
@prefix="TS_" Typespec_Kind :: enum{ None, Name, Pointer, Array, Function }
Typespec :: struct{
kind: Typespec_Kind;
next: Typespec*;
pos : Token*;
union{
Intern_String name;
struct{
Typespec *first;
Typespec *last;
Typespec *ret;
}function_spec;
struct{
Typespec *base;
Expr *size;
}array_spec;
Typespec *base;
};
};
name: Intern_String;
base: Typespec*;
function_spec: struct{
first: Typespec*;
last : Typespec*;
ret : Typespec*;
}
array_spec: struct{
base: Typespec*;
size: Expr*;
}
}
}
//-----------------------------------------------------------------------------
// Notes
//-----------------------------------------------------------------------------
struct Note{
Token *pos;
Intern_String name;
Expr *expr;
Note :: struct{
pos : Token*;
name: Intern_String;
expr: Expr*;
Note *next;
Note *first;
Note *last;
next : Note*;
first: Note*;
last : Note*;
};
//-----------------------------------------------------------------------------
// Declarations
//-----------------------------------------------------------------------------
typedef enum Decl_Kind{
DECL_None,
DECL_Struct,
DECL_Union,
DECL_Enum,
DECL_Variable,
DECL_Typedef,
DECL_Function,
DECL_List,
}Decl_Kind;
@prefix = "DECL_" Decl_Kind :: enum { None, Struct, Union, Enum, Variable, Typedef, Function, List }
@prefix="STRUCT_" Decl_Struct_Kind :: enum { Nested, Base }
typedef enum Decl_Struct_Kind{
STRUCT_Base ,
STRUCT_Nested,
}Decl_Struct_Kind;
Decl_Function_Arg :: struct{
next: Decl_Function_Arg *;
name: Intern_String;
pos : Token *;
typespec: Typespec *;
}
struct Decl_Function_Arg{
Decl_Function_Arg *next;
Intern_String name;
Typespec *typespec;
Token *pos;
Decl_Enum_Child :: struct{
next: Decl_Enum_Child *;
name: Intern_String;
pos : Token *;
expr: Expr *;
first_note: Note *;
last_note : Note *;
};
struct Decl_Enum_Child{
Decl_Enum_Child *next;
Intern_String name;
Token *pos;
Expr *expr;
Decl :: struct{
kind: Decl_Kind;
next: Decl *;
Note *first_note;
Note *last_note;
};
struct Decl{
Decl_Kind kind;
Decl *next;
name: Intern_String;
pos : Token *;
Intern_String name;
Token *pos;
Note *first_note;
Note *last_note;
first_note: Note *;
last_note : Note *;
union{
struct{
Decl_Enum_Child *first;
Decl_Enum_Child *last;
Typespec *typespec;
}enum_decl;
struct{
Decl_Struct_Kind kind;
Decl *first;
Decl *last;
} struct_decl;
struct{
Typespec *type;
Expr *expr;
}variable_decl;
struct{
Typespec *type;
}typedef_decl;
struct{
Decl_Function_Arg *first;
Decl_Function_Arg *last ;
Typespec *ret;
}function_decl;
struct{
Decl *first;
Decl *last;
}list;
};
};
enum_decl: struct{
first: Decl_Enum_Child *;
last: Decl_Enum_Child *;
typespec: Typespec *;
}
struct_decl: struct{
first: Decl *;
last: Decl *;
kind: Decl_Struct_Kind ;
}
variable_decl: struct{
type: Typespec *;
expr: Expr *;
}
typedef_decl: struct{
type: Typespec *;
}
function_decl: struct{
first: Decl_Function_Arg *;
last : Decl_Function_Arg *;
ret: Typespec *;
body : Stmt*;
}
list: struct{
first: Decl *;
last: Decl *;
}
}
}
//-----------------------------------------------------------------------------
// Statements
//-----------------------------------------------------------------------------
typedef enum Stmt_Kind{
STMT_None,
STMT_Decl,
STMT_Expr,
STMT_Return,
STMT_If,
STMT_For,
}Stmt_Kind;
@prefix="STMT_" Stmt_Kind :: enum{ None, Decl, Expr, List, Return, If, For}
typedef struct Stmt_List Stmt_List;
struct Stmt_List{
Stmt *first;
Stmt *last;
};
Stmt_If :: struct {
next: Stmt_If*;
cond: Expr*;
body: Stmt*;
}
typedef struct Stmt_If Stmt_If;
struct Stmt_If{
Stmt_If *next;
Expr *expr;
union{
struct{Stmt *first; Stmt *last;};
Stmt_List *list;
};
};
struct Stmt{
Stmt_Kind kind;
Stmt *next;
Stmt :: struct {
kind: Stmt_Kind;
next: Stmt*;
pos : Token*;
union{
Decl *decl;
Expr *expr;
Stmt_If *if_stmt;
Stmt_List *list;
};
};
stmt_if: Stmt_If;
decl: Decl*;
expr: Expr*;
list: struct{
first: Stmt*;
last : Stmt*;
}
ret: struct{
expr: Expr*;
}
}
}
//-----------------------------------------------------------------------------
// Gather
//-----------------------------------------------------------------------------
@prefix="PK_"
Pointer_Kind::enum{
None,
Typespec,
Expr,
Decl,
Stmt,
Enum_Child,
Func_Arg,
Intern_String
}
Pointer::struct{
kind: Pointer_Kind;
union {
typespec: Typespec *;
decl: Decl *;
expr: Expr *;
stmt: Stmt *;
func_arg: Decl_Function_Arg*;
enum_child: Decl_Enum_Child*;
string: Intern_String *;
}
}
Pointer_Bucket::struct{
next: Pointer_Bucket*;
data: Pointer[4096];
}
Pointer_Array::struct{
first: Pointer_Bucket;
last : Pointer_Bucket*;
len : S64;
block: S64;
arena: Arena*;
iter_bucket: Pointer_Bucket*;
iter_len: S64;
iter_block: S64;
}
@prefix="GATHER_"
Gather_Flag::enum{
None = 0,
Typespec = 1,
Expr = 2,
Decl = 4,
Stmt = 8,
Enum_Child = 16,
Func_Arg = 32,
}
@prefix="TRAVERS_"
Traversal_Flag :: enum{
None,
Typespec = 1,
Expr = 2,
//Decl = 4,
Stmt = 8,
All = TRAVERS_Typespec | TRAVERS_Expr | TRAVERS_Stmt,
}
#endif
#include "generated_ast.h"

5
build.bat
View File

@@ -1,3 +1,6 @@
@echo off
clang main.c -fdiagnostics-absolute-paths -std=c99 -g -o main.exe -Wl,-subsystem:windows -Wl,user32.lib
rem clang generate.c -fdiagnostics-absolute-paths -std=c99 -g -o generate.exe -Wl,user32.lib
rem generate.exe
clang main.c -fdiagnostics-absolute-paths -std=c99 -g -o main.exe -Wl,user32.lib
rem cl main.c -std:c17

531
codegen_c.c
View File

@@ -1,126 +1,475 @@
function void gen_decl(Parser *p, Decl *node);
function void gen_decl(Decl *node);
function void gen_cdecl(Typespec *type, String str);
function void gen_stmt_list(Stmt *stmt);
global String_List *glist;
global Arena *gscratch;
global Parser *genp;
#define strf(...) string_listf(gscratch, glist, __VA_ARGS__)
//-----------------------------------------------------------------------------
// String replacing util
//-----------------------------------------------------------------------------
typedef struct String_Map String_Map;
struct String_Map{
String replace;
String with;
};
function void
gen_typespec(Parser *p, Typespec *spec, B32 is_left){
switch(spec->kind) {
case TS_Name: {
if(is_left) lex_print("%s", spec->name.s.str);
} break;
case TS_Pointer: {
gen_typespec(p, spec->base,is_left);
if(is_left) lex_print("*");
} break;
case TS_Array: {
gen_typespec(p, spec->array_spec.base,is_left);
if(!is_left){
lex_print("[");
expr_print(p, spec->array_spec.size);
lex_print("]");
string_mapped_print(String string, String_Map *map, SizeU count){
Tokens tokens = lex_stream(string, lit("string_mapped_print"));
for(Token *t = tokens.tokens; t != tokens.tokens + tokens.len; t++){
String string = t->string;
for(SizeU i = 0; i < count; i++){
if(string_compare(string, map[i].replace)){
string = map[i].with;
break;
}
} break;
}
case TS_Function: {
invalid_codepath;
} break;
default: {invalid_codepath;} break;
if(t->kind == TK_At) continue;
if(string_compare(t->string,keyword_function.s)) strf("\n");
strf("%.*s", (int)string.len, string.str);
if(string_compare(t->string,keyword_function.s)) strf("\n");
if(t->kind == TK_OpenBrace) strf("\n");
if(t->kind == TK_CloseBrace && t[1].kind != TK_Semicolon) strf("\n");
if(t->kind == TK_Semicolon) strf("\n");
if(t->kind == TK_Identifier && (t[1].kind == TK_Keyword || t[1].kind == TK_Identifier)) strf(" ");
if(t->kind == TK_Keyword) strf(" ");
if(t->kind == TK_Comma) strf(" ");
}
free(tokens.tokens);
}
#define STR(X) lit(#X)
function Typespec *
typespec_get_func(Typespec *type){
switch(type->kind){
case TS_Name:{return 0;} break;
case TS_Pointer:{return typespec_get_func(type->base);} break;
case TS_Array:{return typespec_get_func(type->array_spec.base);} break;
case TS_Function:{return type;} break;
default: {invalid_codepath;} break;
}
return 0;
//-----------------------------------------------------------------------------
// Codegen
//-----------------------------------------------------------------------------
function void
token_strf(Token *token){
strf("%.*s", (S32)token->len, token->str);
}
function void
gen_decl(Parser *p, Decl *node){
gen_expr(Expr *expr){
switch(expr->kind) {
case EK_Atom: {
token_strf(expr->token);
} break;
case EK_Sizeof:{
strf("sizeof(");
if(expr->size_of.kind == SIZEOF_Expr){
gen_expr(expr->size_of.expr);
}
else{
assert(expr->size_of.kind == SIZEOF_Type);
gen_cdecl(expr->size_of.type, string_empty);
}
strf(")");
}break;
case EK_Binary:{
strf("(");
gen_expr(expr->binary.left);
token_strf(expr->token);
gen_expr(expr->binary.right);
strf(")");
} break;
case EK_Unary:{
strf("(");
token_strf(expr->token);
gen_expr(expr->unary.expr);
strf(")");
} break;
case EK_Ternary:{
strf("(");
gen_expr(expr->ternary.cond);
strf("?");
gen_expr(expr->ternary.on_true);
strf(":");
gen_expr(expr->ternary.on_false);
strf(")");
} break;
case EK_List:{
strf("(");
for(Expr *n = expr->list.first; n; n=n->next){
gen_expr(n);
if(n!=expr->list.last) strf(",");
}
strf(")");
}break;
case EK_Cast:{
strf("(");
strf("(");
gen_cdecl(expr->cast.type, string_empty);
strf(")");
gen_expr(expr->cast.expr);
strf(")");
} break;
case EK_Index:{
gen_expr(expr->index.atom);
strf("[");
gen_expr(expr->index.index);
strf("]");
}break;
case EK_Call:{
gen_expr(expr->call.atom);
strf("(");
gen_expr(expr->call.list);
strf(")");
}break;
default: {invalid_codepath;} break;
}
}
function String
gen_cdecl_paren(String str, String original){
if(string_is_empty(original) || original.str[0] == '['){
return str;
}
else{
String result = string_fmt(gscratch, "(%.*s)", (int)str.len, str.str);
return result;
}
}
function String
gen__cdecl(Typespec *type, String str){
switch(type->kind) {
case TS_Name: {
String space = string_is_empty(str) ? lit(""):lit(" ");
String result = string_fmt(gscratch, "%s%s%.*s", type->name.s.str, space.str, (int)str.len, str.str);
return result;
} break;
case TS_Pointer: {
String pointer = string_fmt(gscratch, "*%.*s", (int)str.len, str.str);
String add_paren = gen_cdecl_paren(pointer, str);
String result = gen__cdecl(type->base, add_paren);
return result;
} break;
case TS_Array: {
String left = string_fmt(gscratch, "%s[", str.str);
String_List *save = glist;
String_List list = {0};
glist = &list;
gen_expr(type->array_spec.size);
String expr_string = string_list_flatten(gscratch, glist);
glist = save;
String right = string_fmt(gscratch, "%s%s]", left.str, expr_string.str);
String paren = gen_cdecl_paren(right, str);
String result = gen__cdecl(type->array_spec.base, paren);
return result;
} break;
case TS_Function: {
String result = string_fmt(gscratch, "(*%s)(", str.str);
if (type->function_spec.first == 0) {
result= string_fmt(gscratch, "%svoid", result.str);
}
else {
for(Typespec *n = type->function_spec.first; n; n=n->next){
String arg = gen__cdecl(n, string_empty);
result = string_fmt(gscratch, "%s%s", result.str, arg.str);
if(n != type->function_spec.last)
result = string_fmt(gscratch, "%s, ", result.str);
}
}
result = string_fmt(gscratch, "%s)", result.str);
result = gen__cdecl(type->function_spec.ret, result);
return result;
} break;
default: {invalid_codepath;} break;
}
return string_empty;
}
function void
gen_cdecl(Typespec *type, String str){
String string = gen__cdecl(type, str);
strf("%.*s", (int)string.len, string.str);
}
function void
gen_assign_expr(Expr *expr){
if(expr){
strf(" = ");
gen_expr(expr);
}
}
function void
gen_function_decl(Decl *node){
strf("function ");
gen_cdecl(node->function_decl.ret, string_empty);
strf(" %s(", node->name.s.str);
for(Decl_Function_Arg *arg = node->function_decl.first; arg; arg=arg->next){
gen_cdecl(arg->typespec, arg->name.s);
if(arg!=node->function_decl.last) strf(", ");
}
strf(")");
}
typedef struct Macro{
Intern_String name;
Note *param_type;
Note *param_expr;
Decl *decl;
}Macro;
global Macro macros[32];
global S32 macros_i;
function void
gen_forward_decl(Decl *node){
U8 *name = node->name.s.str;
switch(node->kind) {
case DECL_List: {
for(Decl *n = node->list.first; n; n=n->next){
gen_decl(p,n);
lex_new_line();
gen_forward_decl(n);
}
} break;
case DECL_Variable:{
Typespec *func = typespec_get_func(node->variable_decl.type);
if(!func){
gen_typespec(p, node->variable_decl.type, true);
lex_print(" %s", node->name.s.str);
gen_typespec(p, node->variable_decl.type, false);
}
else{
gen_typespec(p, func->function_spec.ret, true);
lex_print("(*");
lex_print("%s", node->name.s.str);
lex_print(")");
lex_print("(");
for(Typespec *t = func->function_spec.first; t; t=t->next){
gen_typespec(p, t, true);
gen_typespec(p, t, false);
if(t != func->function_spec.last) lex_print(", ");
}
lex_print(")");
}
print_assign_expr(p,node->variable_decl.expr);
lex_print(";");
} break;
case DECL_Typedef:{
lex_print("typedef %s ", node->name.s.str);
gen_typespec(p, node->typedef_decl.type, true);
gen_typespec(p, node->typedef_decl.type, false);
lex_print(";");
} break;
case DECL_Function:{
gen_function_decl(node);
strf(";\n");
}break;
case DECL_Struct:
case DECL_Union :{
U8 *struct_name = node->kind==DECL_Struct ? (U8*)"struct" : (U8*)"union";
U8 *name = node->name.s.str;
if(node->struct_decl.kind == STRUCT_Base)
lex_print("typedef %s %s %s;\n", struct_name, name, name);
lex_print("%s %s{\n", struct_name, name?name:(U8*)"");
for(Decl *n = node->struct_decl.first; n; n=n->next){
gen_decl(p, n);
lex_print("\n");
Note *note = decl_find_note(node, lit("register"));
if(note){
Note *param_expr = decl_find_note(node, lit("param_expr"));
Note *param_type = decl_find_note(node, lit("param_type"));
macros[macros_i++] = (Macro){
.name=node->name,
.param_type=param_type,
.param_expr=param_expr,
.decl=node
};
}
else{
U8 *struct_name = node->kind==DECL_Struct ? (U8*)"struct" : (U8*)"union";
if(node->struct_decl.kind == STRUCT_Base)
strf("typedef %s %s %s;\n", struct_name, name, name);
}
lex_print("};");
if(node->struct_decl.kind == STRUCT_Base)
lex_print("\n");
} break;
case DECL_Enum:{
lex_print("enum %s", node->name.s.str);
if(!intern_compare(node->enum_decl.typespec->name, intern_s64)){
lex_print(" : ");
lex_print("%s", node->enum_decl.typespec->name.s.str);
}
lex_print("{\n");
String prefix = decl_find_string_note(node, lit("prefix"), string_empty);
for(Decl_Enum_Child *n = node->enum_decl.first; n; n=n->next){
lex_print("%.*s%s", (int)prefix.len, prefix.str, n->name.s.str);
print_assign_expr(p, n->expr);
lex_print(",\n");
}
lex_print("};\n");
} break;
default: {invalid_codepath;} break;
}
}
function void
gen_decl(Decl *node){
U8 *name = node->name.s.str;
switch(node->kind) {
case DECL_List: {
for(Decl *n = node->list.first; n; n=n->next){
gen_decl(n);
strf("\n");
}
} break;
case DECL_Variable:{
gen_cdecl(node->variable_decl.type, node->name.s);
gen_assign_expr(node->variable_decl.expr);
strf(";");
} break;
case DECL_Typedef:{
strf("typedef ");
gen_cdecl(node->typedef_decl.type, node->name.s);
strf(";");
} break;
case DECL_Struct:
case DECL_Union :{
Note *note = decl_find_note(node, lit("register"));
if(note){
Arena_Checkpoint checkpoint = arena_checkpoint(gscratch);
Pointer_Array array = gather(checkpoint.arena, node, GATHER_Decl|GATHER_Typespec, TRAVERS_All);
Intern_String based = intern_string(genp, lit("Based_Type_Represent"));
Intern_String type = intern_string(genp, lit("Type"));
for(Pointer it = pointer_array_iter_begin(&array); it.typespec; it = pointer_array_iter_next(&array)){
if(it.kind == PK_Typespec){
Typespec *typespec = it.typespec;
assert(typespec->kind == TS_Name || typespec->kind == TS_Pointer || typespec->kind == TS_Array || typespec->kind == TS_Function);
if(typespec->kind == TS_Name){
if(intern_compare(typespec->name, type)){
typespec->name = based;
}
}
}
else if(it.kind == PK_Decl){
Decl *decl = it.decl;
if(decl->kind == DECL_Struct || decl->kind == DECL_Function){
if(intern_compare(decl->name, type)){
decl->name = based;
}
}
}
}
arena_restore(checkpoint);
}
U8 *struct_name = node->kind==DECL_Struct ? (U8*)"struct" : (U8*)"union";
strf("%s", struct_name);
if(node->struct_decl.kind == STRUCT_Base)
strf(" %s", node->name.s.str);
strf("{\n");
for(Decl *n = node->struct_decl.first; n; n=n->next){
gen_decl(n);
strf("\n");
}
strf("}");
if(node->struct_decl.kind == STRUCT_Nested)
strf("%.*s", (int)node->name.s.len, node->name.s.str);
strf(";");
if(node->struct_decl.kind == STRUCT_Base)
strf("\n");
} break;
case DECL_Enum:{
strf("typedef enum %s", name);
if(!intern_compare(node->enum_decl.typespec->name, intern_s64)){
strf(" : ");
strf("%s", node->enum_decl.typespec->name.s.str);
}
strf("{\n");
String prefix = decl_find_string_note(node, lit("prefix"), string_empty);
for(Decl_Enum_Child *n = node->enum_decl.first; n; n=n->next){
strf("%.*s%s", (int)prefix.len, prefix.str, n->name.s.str);
gen_assign_expr(n->expr);
strf(",\n");
}
strf("}%s;\n", name);
} break;
case DECL_Function:{
gen_function_decl(node);
gen_stmt_list(node->function_decl.body);
} break;
default: {invalid_codepath;} break;
}
}
function void
gen_stmt_list(Stmt *stmt){
strf("{\n");
for(Stmt *s = stmt->list.first; s; s=s->next){
gen_stmt(s);
strf("\n");
}
strf("}");
}
function void
gen_stmt(Stmt *stmt){
switch(stmt->kind) {
case STMT_List: {
gen_stmt_list(stmt);
} break;
case STMT_Return:{
strf("return ");
gen_expr(stmt->ret.expr);
strf(";");
} break;
case STMT_If:{
strf("if ");
gen_expr(stmt->stmt_if.cond);
gen_stmt_list(stmt->stmt_if.body);
for(Stmt_If *s = stmt->stmt_if.next; s; s=s->next){
if(s->next){
strf("else if ");
gen_expr(s->cond);
gen_stmt_list(s->body);
}
else{
strf("else");
gen_stmt_list(s->body);
}
}
} break;
case STMT_Expr: {
gen_expr(stmt->expr);
strf(";");
} break;
case STMT_Decl: {
gen_decl(stmt->decl);
} break;
default: {invalid_codepath;} break;
}
}
//-----------------------------------------------------------------------------
//
//-----------------------------------------------------------------------------
global Decl *gen_parent;
function void
gen_code(Decl *node){
U8 *name = node->name.s.str;
switch(node->kind) {
case DECL_List: {
for(Decl *n = node->list.first; n; n=n->next){
gen_code(n);
}
} break;
case DECL_Variable:{
} break;
case DECL_Typedef:{
} break;
case DECL_Struct:
case DECL_Union :{
gen_parent = node;
for(Decl *n = node->struct_decl.first; n; n=n->next){
gen_code(n);
}
} break;
case DECL_Enum:{
} break;
default: {invalid_codepath;} break;
}
}
String_List list;
function void
gen_begin(Arena *scratch, Parser *p){
list = (String_List){0};
gscratch = scratch;
glist = &list;
genp = p;
}
function void
gen_end(){
String result = string_list_flatten(gscratch, glist);
lex_print("%s", result.str);
}

112
common.c
View File

@@ -1,4 +1,3 @@
global String string_empty;
function S64
clamp_top_s64(S64 val, S64 max){
@@ -52,6 +51,11 @@ wrap_around_pow2(U64 x, U64 power_of_2) {
return r;
}
//-----------------------------------------------------------------------------
// Strings
//-----------------------------------------------------------------------------
global String string_empty;
function B32
string_compare(String a, String b){
if(a.len != b.len)
@@ -77,6 +81,12 @@ char_to_upper(U8 c){
return c;
}
function B32
string_is_empty(String a){
B32 result = a.len == 0;
return result;
}
function String
string_to_lower(Arena *arena, String string){
String result = arena_push_string_copy(arena, string);
@@ -85,3 +95,103 @@ string_to_lower(Arena *arena, String string){
}
return result;
}
//-----------------------------------------------------------------------------
// String interning
//-----------------------------------------------------------------------------
typedef struct Table_Index{
U64 hash;
U64 index;
U64 iter;
U64 max_size;
}Table_Index;
typedef struct Intern_String{
String s;
}Intern_String;
typedef struct Intern_Table{
S64 interns_in_bytes;
S64 interns_inserted;
S64 interns_max;
Intern_String *interns;
Arena *arena;
}Intern_Table;
function Intern_Table
intern_table(Arena *arena, SizeU size){
Intern_Table result = {0};
result.arena = arena;
result.interns = arena_push_array(arena, Intern_String, size);
result.interns_max = size;
return result;
}
function Table_Index
table_index_from_hash(U64 hash, U64 max_size){
Table_Index result = {0};
result.hash = hash;
result.index = result.hash % max_size;
result.iter = result.index;
result.max_size = max_size;
return result;
}
function Table_Index
table_index_from_string(String string, U64 max_size){
U64 hash = hash_fnv(string);
Table_Index result = table_index_from_hash(hash, max_size);
return result;
}
function B32
table_index_advance(Table_Index *index){
index->iter = wrap_around_pow2(index->iter + 1, index->max_size);
B32 result = index->iter == index->index;
return result;
}
function Intern_String
intern_string(Intern_Table *p, String string){
Intern_String result = {0};
Table_Index index = table_index_from_string(string, p->interns_max);
for(;;){
Intern_String *intern = p->interns + index.iter;
if(intern->s.str == 0){
result.s = arena_push_string_copy(p->arena, string);
p->interns_in_bytes += string.len;
p->interns_inserted += 1;
*intern = result;
break;
}
else if(string_compare(intern->s, string)){
result = *intern;
break;
}
if (table_index_advance(&index))
break;
}
return result;
}
function B32
intern_compare(Intern_String a, Intern_String b){
B32 result = a.s.str == b.s.str;
return result;
}
function void
intern_test(){
Arena *scratch = arena_begin_scratch();
Intern_Table table = intern_table(scratch, 512);
assert(intern_compare(intern_string(&table, lit("Thing")),
intern_string(&table, lit("Thing"))));
assert(!intern_compare(intern_string(&table, lit("Thing")),
intern_string(&table, lit("No_Thing"))));
assert(intern_compare(intern_string(&table, lit("No_Thing")),
intern_string(&table, lit("No_Thing"))));
}

21
file.c Normal file
View File

@@ -0,0 +1,21 @@
global FILE *global_output_file;
global FILE *global_all_files[32];
global U32 global_all_files_count;
#define lex_print(...) fprintf(global_output_file, __VA_ARGS__)
#define lex_new_line() lex_print("\n")
function void
use_write_file(const char *file){
global_output_file = fopen(file, "w");
global_all_files[global_all_files_count++] = global_output_file;
assert_msg(global_output_file, "Failed to open file:%s", file);
}
function void
close_all_files(){
for(U32 i = 0; i < global_all_files_count; i++){
fclose(global_all_files[i]);
}
}

36
generate.c Normal file
View File

@@ -0,0 +1,36 @@
#define _CRT_SECURE_NO_WARNINGS
#include <stdio.h>
#include <windows.h>
#include "lang.h"
#include "os.h"
#include "memory.h"
#include "common.c"
#include "memory.c"
#include "scratch.c"
#include "os_win32.c"
#include "new_lex.c"
int main(){
FILE *f = fopen("generated.c", "w");
assert(f);
Arena *scratch = arena_begin_scratch();
String file = os_read_file(scratch, lit("token_array.c"));
Token_Array array = lex_stream(scratch, file, lit("token_array.c"));
for(Token *t = token_array_iter_begin(&array); t; t = token_array_iter_next(&array)){
fprintf(f, "%.*s", (int)t->len, t->str);
if(t->kind == TK_OpenBrace) fprintf(f, "\n");
if(t->kind == TK_CloseBrace && token_array_iter_peek(&array, 0)->kind != TK_Semicolon) fprintf(f, "\n");
if(t->kind == TK_Semicolon) fprintf(f, "\n");
if(t[0].kind == TK_Identifier && token_array_iter_peek(&array, 0)->kind == TK_Identifier) fprintf(f, " ");
token_array_iter_peek(&array, 1);
}
arena_end_scratch();
fclose(f);
}

69
generated.c Normal file
View File

@@ -0,0 +1,69 @@
typedef struct Token_Bucket Token_Bucket;
typedef struct Token_Array Token_Array;
struct Token_Bucket{
Token_Bucket*next;
Token data[4096];
};
struct Token_Array{
Intern_Table interns;
Token_Bucket first;
Token_Bucket*last;
S64 len;
S64 block;
Arena*arena;
Token_Bucket*iter_bucket;
S64 iter_len;
S64 iter_block;
};
function Token_Array token_array_make(Arena*arena){
Token_Array result={
.last=&result.first,.arena=arena,.interns=intern_table(arena,4096*4)};
return result;
}
function void token_array_push(Token_Array*array,Token*p){
if(array->len>=buff_cap(array->first.data)){
Token_Bucket*bucket=arena_push_struct(array->arena,Token_Bucket);
array->last=array->last->next=bucket;
array->len=0;
array->block+=1;
}
array->last->data[array->len++]=*p;
}
function B32 token_array_iter_is_end(Token_Array*array){
B32 result=array->iter_len==array->len&&array->iter_block==array->block;
return result;
}
function Token*token_array_iter_next(Token_Array*array){
if(token_array_iter_is_end(array)){
return0;
}
if(array->iter_len>=buff_cap(array->first.data)){
array->iter_len=0;
array->iter_block+=1;
array->iter_bucket=array->iter_bucket->next;
}
Token*result=array->iter_bucket->data+array->iter_len++;
return result;
}
function Token*token_array_iter_peek(Token_Array*array,S64 i){
S64 save_len=array->iter_len;
Token_Bucket*save_bucket=array->iter_bucket;
assert(i<buff_cap(array->first.data));
S64 over=i;
if(array->iter_len+i>=buff_cap(array->first.data)){
over=buff_cap(array->first.data)-(array->iter_len+i);
array->iter_len=0;
array->iter_bucket=array->iter_bucket->next;
}
Token*result=array->iter_bucket->data+array->iter_len+over;
array->iter_len=save_len;
array->iter_bucket=save_bucket;
return result;
}
function Token*token_array_iter_begin(Token_Array*array){
array->iter_len=0;
array->iter_block=0;
array->iter_bucket=&array->first;
Token*result=token_array_iter_next(array);
return result;
}

0
generated_ast.cpp Normal file
View File

225
generated_ast.h Normal file
View File

@@ -0,0 +1,225 @@
typedef struct Typespec Typespec;
typedef struct Note Note;
typedef struct Decl_Function_Arg Decl_Function_Arg;
typedef struct Decl_Enum_Child Decl_Enum_Child;
typedef struct Decl Decl;
typedef struct Stmt_If Stmt_If;
typedef struct Stmt Stmt;
typedef struct Pointer Pointer;
typedef struct Pointer_Bucket Pointer_Bucket;
typedef struct Pointer_Array Pointer_Array;
typedef enum Typespec_Kind{
TS_None,
TS_Name,
TS_Pointer,
TS_Array,
TS_Function,
}Typespec_Kind;
struct Typespec{
Typespec_Kind kind;
Typespec (*next);
Token (*pos);
union{
Intern_String name;
Typespec (*base);
struct{
Typespec (*first);
Typespec (*last);
Typespec (*ret);
}function_spec;
struct{
Typespec (*base);
Expr (*size);
}array_spec;
};
};
struct Note{
Token (*pos);
Intern_String name;
Expr (*expr);
Note (*next);
Note (*first);
Note (*last);
};
typedef enum Decl_Kind{
DECL_None,
DECL_Struct,
DECL_Union,
DECL_Enum,
DECL_Variable,
DECL_Typedef,
DECL_Function,
DECL_List,
}Decl_Kind;
typedef enum Decl_Struct_Kind{
STRUCT_Nested,
STRUCT_Base,
}Decl_Struct_Kind;
struct Decl_Function_Arg{
Decl_Function_Arg (*next);
Intern_String name;
Token (*pos);
Typespec (*typespec);
};
struct Decl_Enum_Child{
Decl_Enum_Child (*next);
Intern_String name;
Token (*pos);
Expr (*expr);
Note (*first_note);
Note (*last_note);
};
struct Decl{
Decl_Kind kind;
Decl (*next);
Intern_String name;
Token (*pos);
Note (*first_note);
Note (*last_note);
union{
struct{
Decl_Enum_Child (*first);
Decl_Enum_Child (*last);
Typespec (*typespec);
}enum_decl;
struct{
Decl (*first);
Decl (*last);
Decl_Struct_Kind kind;
}struct_decl;
struct{
Typespec (*type);
Expr (*expr);
}variable_decl;
struct{
Typespec (*type);
}typedef_decl;
struct{
Decl_Function_Arg (*first);
Decl_Function_Arg (*last);
Typespec (*ret);
Stmt (*body);
}function_decl;
struct{
Decl (*first);
Decl (*last);
}list;
};
};
typedef enum Stmt_Kind{
STMT_None,
STMT_Decl,
STMT_Expr,
STMT_List,
STMT_Return,
STMT_If,
STMT_For,
}Stmt_Kind;
struct Stmt_If{
Stmt_If (*next);
Expr (*cond);
Stmt (*body);
};
struct Stmt{
Stmt_Kind kind;
Stmt (*next);
Token (*pos);
union{
Stmt_If stmt_if;
Decl (*decl);
Expr (*expr);
struct{
Stmt (*first);
Stmt (*last);
}list;
struct{
Expr (*expr);
}ret;
};
};
typedef enum Pointer_Kind{
PK_None,
PK_Typespec,
PK_Expr,
PK_Decl,
PK_Stmt,
PK_Enum_Child,
PK_Func_Arg,
PK_Intern_String,
}Pointer_Kind;
struct Pointer{
Pointer_Kind kind;
union{
Typespec (*typespec);
Decl (*decl);
Expr (*expr);
Stmt (*stmt);
Decl_Function_Arg (*func_arg);
Decl_Enum_Child (*enum_child);
Intern_String (*string);
};
};
struct Pointer_Bucket{
Pointer_Bucket (*next);
Pointer (data[4096]);
};
struct Pointer_Array{
Pointer_Bucket first;
Pointer_Bucket (*last);
S64 len;
S64 block;
Arena (*arena);
Pointer_Bucket (*iter_bucket);
S64 iter_len;
S64 iter_block;
};
typedef enum Gather_Flag{
GATHER_None = 0,
GATHER_Typespec = 1,
GATHER_Expr = 2,
GATHER_Decl = 4,
GATHER_Stmt = 8,
GATHER_Enum_Child = 16,
GATHER_Func_Arg = 32,
}Gather_Flag;
typedef enum Traversal_Flag{
TRAVERS_None,
TRAVERS_Typespec = 1,
TRAVERS_Expr = 2,
TRAVERS_Stmt = 8,
TRAVERS_All = ((TRAVERS_Typespec|TRAVERS_Expr)|TRAVERS_Stmt),
}Traversal_Flag;

53
generated_lex.cpp Normal file
View File

@@ -0,0 +1,53 @@
// Generated Array block
function
Token_Array token_array_make(Arena*arena){
Token_Array result={
0};
result.arena=arena;
result.last=&result.first;
return result;
}
function
void token_array_push(Token_Array*array, Token*item){
if(array->len+1>4096){
assert(array->arena);
Token_Array_Block*block=arena_push_struct(array->arena, Token_Array_Block);
array->last=array->last->next=block;
array->len=0;
array->block+=1;
}
array->last->data[array->len++]=*item;
}
function
Token_Array_Iter token_array_iter(Token_Array*array){
return((Token_Array_Iter){
.total_len=array->len, .total_block=array->block, .block=&array->first, }
);
}
function
Token*token_array_iter_next(Token_Array_Iter*it){
if(it->iter_len+1>4096){
it->iter_len=0;
it->iter_block+=1;
it->block=it->block->next;
}
return it->block->data+it->iter_len++;
}
function
B32 token_array_iter_is_end(Token_Array_Iter*it){
B32 result=it->iter_len==it->total_len&&it->iter_block==it->total_block;
return result;
}
function
Token*token_array_iter_begin(Token_Array*array){
array->it=token_array_iter(array);
Token*result=token_array_iter_next(&array->it);
return result;
}

136
generated_lex.h Normal file
View File

@@ -0,0 +1,136 @@
typedef struct Intern_String Intern_String;
typedef struct Token Token;
typedef struct Lex_Stream Lex_Stream;
typedef struct Tokens Tokens;
struct Intern_String{
String s;
};
typedef enum Token_Kind{
TK_End,
TK_Mul,
TK_Div,
TK_Add,
TK_Sub,
TK_Mod,
TK_BitAnd,
TK_BitOr,
TK_BitXor,
TK_Neg,
TK_Not,
TK_OpenParen,
TK_CloseParen,
TK_OpenBrace,
TK_CloseBrace,
TK_OpenBracket,
TK_CloseBracket,
TK_Comma,
TK_Pound,
TK_Question,
TK_ThreeDots,
TK_Semicolon,
TK_Dot,
TK_LesserThen,
TK_GreaterThen,
TK_Colon,
TK_Assign,
TK_DivAssign,
TK_MulAssign,
TK_ModAssign,
TK_SubAssign,
TK_AddAssign,
TK_AndAssign,
TK_OrAssign,
TK_XorAssign,
TK_LeftShiftAssign,
TK_RightShiftAssign,
TK_DoubleColon,
TK_At,
TK_Decrement,
TK_Increment,
TK_PostDecrement,
TK_PostIncrement,
TK_LesserThenOrEqual,
TK_GreaterThenOrEqual,
TK_Equals,
TK_And,
TK_Or,
TK_NotEquals,
TK_LeftShift,
TK_RightShift,
TK_Arrow,
TK_ExprSizeof,
TK_DocComment,
TK_Comment,
TK_Identifier,
TK_StringLit,
TK_U8Lit,
TK_Character,
TK_Error,
TK_Float,
TK_Int,
TK_Keyword,
}Token_Kind;
struct Token{
Token_Kind kind;
union{
struct{
U8 (*str);
S64 len;
};
String string;
};
union{
S64 int_val;
String error_val;
Intern_String intern_val;
};
String file;
S64 line;
U8 (*line_begin);
};
// Generated Array block
typedef struct Token_Array Token_Array;
typedef struct Token_Array_Block Token_Array_Block;
typedef struct Token_Array_Iter Token_Array_Iter;
struct Token_Array_Block{
Token_Array_Block*next;
Token data[4096];
};
struct Token_Array_Iter{
Token_Array_Block*block;
U32 iter_len;
U32 iter_block;
U32 total_len;
U32 total_block;
};
struct Token_Array{
Token_Array_Block first;
Token_Array_Block*last;
Token_Array_Iter it;
Arena*arena;
S64 block;
S64 len;
};
struct Lex_Stream{
U8 (*stream);
U8 (*line_begin);
String filename;
S64 line;
};
struct Tokens{
Token (*tokens);
S64 len;
S64 cap;
S64 iter;
};

13
ideas.cpp
View File

@@ -32,14 +32,15 @@ Thing :: struct{
}
Thingy::enum:U32{
@str=10 Value = 1,
Scope :: scope{
Thingy::enum:U32{
@str=10 Value = 1,
}
}
//Scope.Thingy
Thing::union{
union:{
}

20
lang.h
View File

@@ -4,7 +4,7 @@
#define function static
#define assert(x) do{if(!(x)) __debugbreak();}while(0)
#define assert_msg(x,msg) assert(x)
#define assert_msg(x,...) assert(x)
#define not_implemented assert_msg(0, "Not implemented")
#define invalid_codepath assert_msg(0, "Invalid codepath")
@@ -35,12 +35,30 @@ const B32 false = 0;
#define kib(x) ((x)*1024llu)
#define mib(x) (kib(x)*1024llu)
#define gib(x) (mib(x)*1024llu)
#define string_expand(x) (int)x.len, x.str
typedef struct String_Node String_Node;
typedef struct String_List String_List;
typedef struct String{
U8 *str;
S64 len;
}String;
struct String_Node{
String_Node *next;
union{
String string;
struct{U8*str; S64 len;};
};
};
struct String_List{
String_Node *first;
String_Node *last;
S64 char_count;
S64 node_count;
};
#define SLLQueuePushMod(f,l,n,next) do{\
if((f)==0){\
(f)=(l)=(n);\

33
lex.c
View File

@@ -1,4 +1,4 @@
global Token token_end_of_stream = {};
global Token token_end_of_stream = {0};
function Token *
token_alloc(Tokens *t){
@@ -363,7 +363,7 @@ break
function Tokens
lex_stream(String in_stream, String filename){
Lex_Stream stream = {in_stream.str, in_stream.str, filename, 0};
Tokens tokens = {};
Tokens tokens = {0};
lex_base(&stream, &tokens);
return tokens;
}
@@ -498,3 +498,32 @@ token_peek_is_keyword(Parser *p, S64 count, Intern_String keyword){
}
return 0;
}
function void
lex_test(){
Tokens t;
t = lex_stream(lit("3252342510 42524 \"U8Literal\""), lit("test"));
//tokens_print(t);
assert(t.len == 3);
assert(t.tokens[0].int_val == 3252342510);
assert(t.tokens[1].int_val == 42524);
assert(t.tokens[2].kind == TK_U8Lit);
assert(token_compare(t.tokens + 2, lit("U8Literal")));
t = lex_stream(lit("_identifier Thing Thing2 lit(\"String_Test\")"), lit("test"));
//tokens_print(t);
assert(t.tokens[0].kind == TK_Identifier);
assert(t.tokens[1].kind == TK_Identifier);
assert(t.tokens[2].kind == TK_Identifier);
assert(t.tokens[3].kind == TK_StringLit);
assert(token_compare(t.tokens, lit("_identifier")));
assert(token_compare(t.tokens+1, lit("Thing")));
assert(token_compare(t.tokens+2, lit("Thing2")));
assert(token_compare(t.tokens+3, lit("String_Test")));
t = lex_stream(lit("lit(\"String_Test\"{})(324*=+=-/ *% // Comment \n"
"Thing /*Thing*/ += -= =- +/%^&*&&|| |>> << <<= >>=/*Error"),
lit("test"));
assert(t.tokens[0].kind == TK_Error);
//tokens_print(t);
}

200
lex.h
View File

@@ -1,107 +1,115 @@
#pragma once
typedef struct Intern_String{
String s;
}Intern_String;
#pragma once
#if 0
Intern_String :: struct {
s: String;
}
typedef enum Token_Kind{
meta("End of stream")TK_End,
meta("*")TK_Mul,
meta("/")TK_Div,
meta("+")TK_Add,
meta("-")TK_Sub,
meta("%")TK_Mod,
meta("&")TK_BitAnd,
meta("|")TK_BitOr,
meta("^")TK_BitXor,
meta("~")TK_Neg,
meta("!")TK_Not,
meta("(")TK_OpenParen,
meta(")")TK_CloseParen,
meta("{")TK_OpenBrace,
meta("}")TK_CloseBrace,
meta("[")TK_OpenBracket,
meta("]")TK_CloseBracket,
meta(",")TK_Comma,
meta("#")TK_Pound,
meta("?")TK_Question,
meta("...")TK_ThreeDots,
meta(";")TK_Semicolon,
meta(".")TK_Dot,
meta("<")TK_LesserThen,
meta(">")TK_GreaterThen,
meta(":")TK_Colon,
meta("=")TK_Assign,
meta("/=")TK_DivAssign,
meta("*=")TK_MulAssign,
meta("%=")TK_ModAssign,
meta("-=")TK_SubAssign,
meta("+=")TK_AddAssign,
meta("&=")TK_AndAssign,
meta("|=")TK_OrAssign,
meta("^=")TK_XorAssign,
meta("<<=")TK_LeftShiftAssign,
meta(">>=")TK_RightShiftAssign,
meta("::")TK_DoubleColon,
meta("@")TK_At,
meta("--")TK_Decrement,
meta("++")TK_Increment,
meta("--")TK_PostDecrement,
meta("++")TK_PostIncrement,
meta("<=")TK_LesserThenOrEqual,
meta(">=")TK_GreaterThenOrEqual,
meta("==")TK_Equals,
meta("&&")TK_And,
meta("||")TK_Or,
meta("!=")TK_NotEquals,
meta("<<")TK_LeftShift,
meta(">>")TK_RightShift,
meta("->")TK_Arrow,
meta("sizeof")TK_ExprSizeof,
TK_DocComment,
TK_Comment,
TK_Identifier,
TK_StringLit,
TK_U8Lit,
TK_Character,
TK_Error,
TK_Float,
TK_Int,
TK_Keyword,
@prefix = "TK_"
Token_Kind :: enum {
@str = "End of stream" End,
@str = "*" Mul,
@str = "/" Div,
@str = "+" Add,
@str = "-" Sub,
@str = "%" Mod,
@str = "&" BitAnd,
@str = "|" BitOr,
@str = "^" BitXor,
@str = "~" Neg,
@str = "!" Not,
@str = "(" OpenParen,
@str = " " CloseParen,
@str = "{" OpenBrace,
@str = "}" CloseBrace,
@str = "[" OpenBracket,
@str = "]" CloseBracket,
@str = "," Comma,
@str = "#" Pound,
@str = "?" Question,
@str = "..." ThreeDots,
@str = ";" Semicolon,
@str = "." Dot,
@str = "<" LesserThen,
@str = ">" GreaterThen,
@str = ":" Colon,
@str = "=" Assign,
@str = "/=" DivAssign,
@str = "*=" MulAssign,
@str = "%=" ModAssign,
@str = "-=" SubAssign,
@str = "+=" AddAssign,
@str = "&=" AndAssign,
@str = "|=" OrAssign,
@str = "^=" XorAssign,
@str = "<<=" LeftShiftAssign,
@str = ">>=" RightShiftAssign,
@str = "::" DoubleColon,
@str = "@" At,
@str = "--" Decrement,
@str = "++" Increment,
@str = "--" PostDecrement,
@str = "++" PostIncrement,
@str = "<=" LesserThenOrEqual,
@str = ">=" GreaterThenOrEqual,
@str = "==" Equals,
@str = "&&" And,
@str = "||" Or,
@str = "!=" NotEquals,
@str = "<<" LeftShift,
@str = ">>" RightShift,
@str = "->" Arrow,
@str = "sizeof" ExprSizeof,
DocComment,
Comment,
Identifier,
StringLit,
U8Lit,
Character,
Error,
Float,
Int,
Keyword,
}Token_Kind;
typedef struct Token{
Token_Kind kind;
union{
struct{
U8 *str;
S64 len;
};
String string;
};
@array(type=block)
Token :: struct{
kind: Token_Kind;
union {
S64 int_val;
String error_val;
Intern_String intern_val;
};
struct{
str: U8*;
len: S64;
}
string: String;
}
union {
int_val: S64;
error_val: String;
intern_val: Intern_String;
}
String file;
S64 line;
U8 *line_begin;
} Token;
file: String;
line: S64;
line_begin: U8*;
}
typedef struct Tokens{
Token *tokens;
S64 len;
S64 cap;
S64 iter;
}Tokens;
Lex_Stream :: struct{
stream: U8 *;
line_begin: U8 *;
filename: String;
line: S64;
}
typedef struct Lex_Stream{
U8 *stream;
U8 *line_begin;
String filename;
S64 line;
}Lex_Stream;
Tokens :: struct{
tokens: Token*;
len: S64;
cap: S64;
iter: S64;
}
#endif
#include "generated_lex.h"
#include "generated_lex.cpp"
//-----------------------------------------------------------------------------
//

113
main.c
View File

@@ -6,117 +6,16 @@
#include "os.h"
#include "memory.h"
#include "lex.h"
#include "parser.h"
#include "expr.h"
#include "ast.h"
global FILE *global_output_file;
#define lex_print(...) fprintf(global_output_file, __VA_ARGS__)
#define lex_new_line() lex_print("\n")
#include "common.c"
#include "memory.c"
#include "parser.c"
#include "scratch.c"
#include "os_win32.c"
#include "lex.c"
#include "expr.c"
#include "ast.c"
#include "parse_expr.c"
#include "parse_decl.c"
#include "print.c"
#include "codegen_c.c"
#include "new_lex.c"
#include "new_ast.c"
#include "new_parse.c"
function void
lex_test(){
Tokens t;
t = lex_stream(lit("3252342510 42524 \"U8Literal\""), lit("test"));
//tokens_print(t);
assert(t.len == 3);
assert(t.tokens[0].int_val == 3252342510);
assert(t.tokens[1].int_val == 42524);
assert(t.tokens[2].kind == TK_U8Lit);
assert(token_compare(t.tokens + 2, lit("U8Literal")));
t = lex_stream(lit("_identifier Thing Thing2 lit(\"String_Test\")"), lit("test"));
//tokens_print(t);
assert(t.tokens[0].kind == TK_Identifier);
assert(t.tokens[1].kind == TK_Identifier);
assert(t.tokens[2].kind == TK_Identifier);
assert(t.tokens[3].kind == TK_StringLit);
assert(token_compare(t.tokens, lit("_identifier")));
assert(token_compare(t.tokens+1, lit("Thing")));
assert(token_compare(t.tokens+2, lit("Thing2")));
assert(token_compare(t.tokens+3, lit("String_Test")));
t = lex_stream(lit("lit(\"String_Test\"{})(324*=+=-/ *% // Comment \n"
"Thing /*Thing*/ += -= =- +/%^&*&&|| |>> << <<= >>=/*Error"),
lit("test"));
assert(t.tokens[0].kind == TK_Error);
//tokens_print(t);
}
function void
parser_test(){
Parser p = {};
{
parser_init(&p);
Intern_String a = intern_string(&p, lit("Thing"));
Intern_String b = intern_string(&p, lit("Thing"));
assert(a.s.str == b.s.str);
}
parser_lex_stream(&p, lit("S64 thing; S64 second_var = 10;"), lit("File"));
assert(token_match(&p, TK_Identifier));
assert(token_match(&p, TK_Identifier));
assert(token_match(&p, TK_Semicolon));
assert(token_match(&p, TK_Identifier));
assert(token_match(&p, TK_Identifier));
assert(token_match(&p, TK_Assign));
assert(token_match(&p, TK_Int));
assert(token_match(&p, TK_Semicolon));
assert(token_match(&p, TK_End));
assert(token_match(&p, TK_End));
assert(token_match(&p, TK_End));
String exprs[] = {
lit("(534>43?435:42,234,cast(S64)32/*todo cast*/,Thing[10][2],Thing(1,2))"),
lit("(4+2*53)"),
lit("((4+2)*53)"),
lit("++5"),
lit("5--"), // @Todo(Krzosa):
lit("-5"),
lit("(+5)"),
lit("sizeof(32) + sizeof(:S32*)"),
lit("cast(S64**)5"),
lit("cast(S64)5+3"),
lit("534>43?435:42"),
};
for(S64 i = 0; i < buff_cap(exprs); i++){
parser_lex_stream(&p, exprs[i], lit("File"));
Expr *expr = parse_expr(&p);
assert(expr);
//expr_print(&p, expr);
//lex_print("\n");
}
String string = os_read_file(lit("test.cc"));
parser_lex_stream(&p, string, lit("Parse"));
Decl *decls = parse(&p);
assert(decls->list.first);
gen_decl(&p, decls);
}
function S32
os_main(){
global_output_file = fopen("output.cc", "w");
assert_msg(global_output_file, "Failed to open output.txt");
int main(){
lex_test();
parser_test();
fclose(global_output_file);
return 0;
test_ast();
}

30
memory.c
View File

@@ -80,12 +80,34 @@ string_fmtv(Arena *arena, const char *str, va_list args1) {
return (String){(U8 *)result, len};
}
#define STRING_FMT(arena, str, result) \
va_list args1; \
va_start(args1, str); \
String result = string_fmtv(arena, str, args1); \
va_end(args1)
function String
string_fmt(Arena *arena, const char *str, ...) {
va_list args1;
va_start(args1, str);
String result = string_fmtv(arena, str, args1);
va_end(args1);
STRING_FMT(arena, str, result);
return result;
}
function void
string_listf(Arena *arena, String_List *list, const char *str, ...){
STRING_FMT(arena, str, string);
String_Node *node = arena_push_struct(arena, String_Node);
node->string = string;
SLLQueuePush(list->first, list->last, node);
list->char_count += node->string.len;
list->node_count += 1;
}
function String
string_list_flatten(Arena *arena, String_List *list){
String result = {arena_push_size(arena, list->char_count + 1)};
for(String_Node *node = list->first; node; node=node->next){
memory_copy(result.str+result.len, node->str, node->len);
result.len += node->len;
}
return result;
}

5
memory.h
View File

@@ -14,4 +14,7 @@ function B32 string_compare(String a, String b);
function void *arena_push_size(Arena *a, SizeU size);
function String arena_push_string_copy(Arena *arena, String string);
#define arena_push_array(a,T,c) arena_push_size(a,sizeof(T)*(c))
#define arena_push_struct(a,T) arena_push_array(a,T,1)
#define arena_push_struct(a,T) arena_push_array(a,T,1)
function Arena *arena_begin_scratch();
function void arena_end_scratch();

295
new_ast.c Normal file
View File

@@ -0,0 +1,295 @@
typedef struct Expr Expr;
typedef struct Typespec Typespec;
typedef enum Typespec_Kind{
TS_None,
TS_Name,
TS_Pointer,
TS_Array,
TS_Function,
}Typespec_Kind;
struct Typespec{
Typespec_Kind kind;
Typespec *next;
Token *pos;
union{
Intern_String name;
Typespec *base;
struct{
Typespec *first;
Typespec *last;
Typespec *ret;
}func;
struct{
Typespec *base;
Expr *size;
}arr;
};
};
//-----------------------------------------------------------------------------
// Expressions
//-----------------------------------------------------------------------------
typedef enum Expr_Sizeof_Kind{
SIZEOF_Expr,
SIZEOF_Type,
}Expr_Sizeof_Kind;
typedef enum Expr_Kind{
EK_None,
EK_Int,
EK_String,
EK_Unary,
EK_Binary,
EK_Ternary,
EK_Cast,
EK_List,
EK_Call,
EK_Index,
EK_Sizeof,
} Expr_Kind;
struct Expr {
Expr_Kind kind;
Token *token;
Expr *next;
union {
U64 int_val;
Intern_String intern_val;
double float_val;
struct {
Typespec *type;
Expr* expr;
} cast;
struct {
Expr *first;
Expr *last;
} list;
struct {
Expr *atom;
Expr *list;
} call;
struct {
Expr *atom;
Expr *index;
} index;
struct {
Token_Kind op;
Expr* expr;
} unary;
struct {
Token_Kind op;
Expr* left;
Expr* right;
} binary;
struct {
Expr* cond;
Expr* on_true;
Expr* on_false;
} ternary;
struct{
Expr_Sizeof_Kind kind;
union{
Typespec *type;
Expr *expr;
};
} size_of;
};
};
//-----------------------------------------------------------------------------
// Expr
//-----------------------------------------------------------------------------
function Expr *
expr_new(Arena *p, Expr_Kind kind, Token *token){
Expr *expr = arena_push_struct(p, Expr);
expr->kind = kind;
expr->token = token;
return expr;
}
function Expr *
expr_int(Arena *p, Token *token){
assert(token->kind == TK_Int);
Expr *expr = expr_new(p, EK_Int, token);
expr->int_val = token->int_val;
return expr;
}
function Expr *
expr_str(Arena *p, Token *token){
assert(token->kind == TK_StringLit);
Expr *expr = expr_new(p, EK_String, token);
expr->intern_val = token->intern_val;
return expr;
}
function Expr *
expr_unary(Arena *p, Token *op, Expr *exp){
Expr *expr = expr_new(p, EK_Unary, op);
expr->unary.op = op->kind;
expr->unary.expr = exp;
return expr;
}
function Expr *
expr_binary(Arena *p, Token *op, Expr *left, Expr *right){
Expr *expr = expr_new(p, EK_Binary, op);
expr->binary.op = op->kind;
expr->binary.left = left;
expr->binary.right = right;
return expr;
}
function Expr *
expr_ternary(Arena *p, Token *op, Expr *cond, Expr *on_true, Expr *on_false){
Expr *expr = expr_new(p, EK_Ternary, op);
expr->ternary.cond = cond;
expr->ternary.on_true = on_true;
expr->ternary.on_false = on_false;
return expr;
}
function Expr *
expr_call(Arena *p, Token *token, Expr *atom, Expr *list){
Expr *expr = expr_new(p, EK_Call, token);
expr->call.atom = atom;
expr->call.list = list;
return expr;
}
function Expr *
expr_index(Arena *p, Token *token, Expr *atom, Expr *index){
Expr *expr = expr_new(p, EK_Index, token);
expr->index.atom = atom;
expr->index.index = index;
return expr;
}
function Expr *
expr_cast(Arena *p, Token *token, Typespec *type, Expr *exp){
Expr *expr = expr_new(p, EK_Cast, token);
expr->cast.type = type;
expr->cast.expr = exp;
return expr;
}
function Expr *
expr_sizeof_type(Arena *p, Token *token, Typespec *type){
Expr *expr = expr_new(p, EK_Sizeof, token);
expr->size_of.kind = SIZEOF_Type;
expr->size_of.type = type;
return expr;
}
function Expr *
expr_sizeof_expr(Arena *p, Token *token, Expr *in_expr){
Expr *expr = expr_new(p, EK_Sizeof, token);
expr->size_of.kind = SIZEOF_Expr;
expr->size_of.expr = in_expr;
return expr;
}
function Expr *
expr_list(Arena *p, Token *token){
Expr *expr = expr_new(p, EK_List, token);
return expr;
}
function void
expr_list_push(Expr *list, Expr *expr){
SLLQueuePush(list->list.first, list->list.last, expr);
}
//-----------------------------------------------------------------------------
//
//-----------------------------------------------------------------------------
function void
token_print(Token *token){
printf("%.*s", (S32)token->len, token->str);
}
function void
expr_print(Expr *expr){
switch(expr->kind) {
case EK_Int:
case EK_String: {
token_print(expr->token);
} break;
case EK_Sizeof:{
printf("sizeof(");
if(expr->size_of.kind == SIZEOF_Expr){
expr_print(expr->size_of.expr);
}
else{
assert(expr->size_of.kind == SIZEOF_Type);
//print_typespec(expr->size_of.type);
}
printf(")");
}break;
case EK_Binary:{
printf("(");
expr_print(expr->binary.left);
token_print(expr->token);
expr_print(expr->binary.right);
printf(")");
} break;
case EK_Unary:{
printf("(");
token_print(expr->token);
expr_print(expr->unary.expr);
printf(")");
} break;
case EK_Ternary:{
printf("(");
expr_print(expr->ternary.cond);
printf("?");
expr_print(expr->ternary.on_true);
printf(":");
expr_print(expr->ternary.on_false);
printf(")");
} break;
case EK_List:{
printf("(");
for(Expr *n = expr->list.first; n; n=n->next){
expr_print(n);
if(n!=expr->list.last) printf(",");
}
printf(")");
}break;
case EK_Cast:{
printf("(");
printf("(");
//print_typespec(expr->cast.type);
printf(")");
expr_print(expr->cast.expr);
printf(")");
} break;
case EK_Index:{
expr_print(expr->index.atom);
printf("[");
expr_print(expr->index.index);
printf("]");
}break;
case EK_Call:{
expr_print(expr->call.atom);
printf("(");
expr_print(expr->call.list);
printf(")");
}break;
default: {invalid_codepath;} break;
}
}
function void
test_ast(){
}

516
new_lex.c Normal file
View File

@@ -0,0 +1,516 @@
global Intern_String keyword_if;
global Intern_String keyword_for;
global Intern_String keyword_else;
global Intern_String keyword_sizeof;
global Intern_String keyword_typeof;
global Intern_String keyword_while;
global Intern_String keyword_switch;
global Intern_String keyword_case;
global Intern_String keyword_struct;
global Intern_String keyword_enum;
global Intern_String keyword_union;
global U8 *first_keyword;
global U8 *last_keyword;
function void
init_default_keywords(Intern_Table *t){
keyword_if = intern_string(t, lit("if"));
first_keyword = keyword_if.s.str;
keyword_for = intern_string(t, lit("for"));
keyword_else = intern_string(t, lit("else"));
keyword_sizeof = intern_string(t, lit("sizeof"));
keyword_typeof = intern_string(t, lit("typeof"));
keyword_while = intern_string(t, lit("while"));
keyword_switch = intern_string(t, lit("switch"));
keyword_case = intern_string(t, lit("case"));
keyword_struct = intern_string(t, lit("struct"));
keyword_enum = intern_string(t, lit("enum"));
keyword_union = intern_string(t, lit("union"));
last_keyword = keyword_union.s.str;
}
function B32
lex_is_keyword(Intern_String str){
B32 result = str.s.str >= first_keyword && str.s.str <= last_keyword;
return result;
}
typedef enum Token_Kind{
TK_End,
TK_Mul,
TK_Div,
TK_Add,
TK_Sub,
TK_Mod,
TK_BitAnd,
TK_BitOr,
TK_BitXor,
TK_Neg,
TK_Not,
TK_OpenParen,
TK_CloseParen,
TK_OpenBrace,
TK_CloseBrace,
TK_OpenBracket,
TK_CloseBracket,
TK_Comma,
TK_Pound,
TK_Question,
TK_ThreeDots,
TK_Semicolon,
TK_Dot,
TK_LesserThen,
TK_GreaterThen,
TK_Colon,
TK_Assign,
TK_DivAssign,
TK_MulAssign,
TK_ModAssign,
TK_SubAssign,
TK_AddAssign,
TK_AndAssign,
TK_OrAssign,
TK_XorAssign,
TK_LeftShiftAssign,
TK_RightShiftAssign,
TK_DoubleColon,
TK_At,
TK_Decrement,
TK_Increment,
TK_PostDecrement,
TK_PostIncrement,
TK_LesserThenOrEqual,
TK_GreaterThenOrEqual,
TK_Equals,
TK_And,
TK_Or,
TK_NotEquals,
TK_LeftShift,
TK_RightShift,
TK_Arrow,
TK_ExprSizeof,
TK_DocComment,
TK_Comment,
TK_Identifier,
TK_StringLit,
TK_Character,
TK_Error,
TK_Float,
TK_Int,
TK_Keyword,
}Token_Kind;
typedef struct Token{
Token_Kind kind;
union{
String string;
struct{U8 *str; S64 len;};
};
union {
U64 int_val;
F64 float_val;
String error_val;
Intern_String intern_val;
};
String file;
S32 line;
U8 *line_begin;
}Token;
#include "token_array.c"
typedef struct Lex_Stream{
String stream;
S64 iter;
U8 *line_begin;
String file;
S32 line;
}Lex_Stream;
function U8
lexc(Lex_Stream *s){
return s->stream.str[s->iter];
}
function U8
lexci(Lex_Stream *s, S32 i){
return s->stream.str[s->iter+i];
}
function U8 *
lexcp(Lex_Stream *s){
return s->stream.str + s->iter;
}
function B32
lex_is_whitespace(U8 c){
B32 result = c == '\n' || c == '\r' || c == ' ' || c == '\r';
return result;
}
function B32
lex_is_alphabetic(U8 c){
B32 result = (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z');
return result;
}
function B32
lex_is_numeric(U8 c){
B32 result = c >= '0' && c <= '9';
return result;
}
function B32
lex_is_alphanumeric(U8 c){
B32 result = lex_is_numeric(c) || lex_is_alphabetic(c);
return result;
}
function void
lex_set_len(Lex_Stream *s, Token *token){
assert(lexcp(s) >= token->str);
token->len = lexcp(s) - token->str;
}
function void
token_error(Token *t, String error_val){
t->kind = TK_Error;
t->error_val = error_val;
}
function void
lex_parse_u64(Token *t){
U64 result = 0;
U64 m = 1;
for(S64 i = t->len - 1; i >= 0; --i){
U64 val = t->str[i] - '0';
U64 new_val = val * m;
if((result + new_val) < result){
token_error(t, lit("Integer overflow"));
return;
}
result+=new_val;
m *= 10;
}
t->int_val = result;
}
function void
lex_advance(Lex_Stream *s){
if(s->iter >= s->stream.len){
return;
}
else if(lexc(s) == '\n'){
s->iter++;
s->line++;
s->line_begin = lexcp(s);
}
else{
s->iter++;
}
}
function Token
token_int(U64 val){
Token result = {.kind = TK_Int, .int_val=val};
return result;
}
function void
lex_parse_string(Lex_Stream *s, Token *t, U8 c){
for(;;){
if(lexc(s) == '\\') lex_advance(s);
else if(lexc(s) == c) break;
else if(lexc(s) == 0){
token_error(t, lit("Unterminated string, reached end of file"));
break;
}
lex_advance(s);
}
if(t->kind != TK_Error){
lex_advance(s);
lex_set_len(s,t);
}
}
#define CASE2(op, OpName, Assign) \
case op: \
if (lexc(s) == '=') { \
lex_advance(s); \
t.kind = Assign; \
} else { \
t.kind = OpName; \
} \
break
#define CASE3(op, OpName, Assign, Incr) \
case op: \
if (lexc(s) == '=') { \
lex_advance(s); \
t.kind = Assign; \
} else if (lexc(s) == op) { \
lex_advance(s); \
t.kind = Incr; \
} else { \
t.kind = OpName; \
} \
break
function void
lex__stream(Token_Array *array, Lex_Stream *s){
while(lexc(s)){
while(lex_is_whitespace(lexc(s)))
lex_advance(s);
Token t = {0};
t.str = lexcp(s);
t.file = s->file;
t.line = s->line;
t.line_begin = s->line_begin;
lex_advance(s);
switch(*t.str){
case 0: break;
case '@': t.kind = TK_At; break;
case '(': t.kind = TK_OpenParen; break;
case ')': t.kind = TK_CloseParen; break;
case '{': t.kind = TK_OpenBrace; break;
case '}': t.kind = TK_CloseBrace; break;
case '[': t.kind = TK_OpenBracket; break;
case ']': t.kind = TK_CloseBracket; break;
case ',': t.kind = TK_Comma; break;
case '~': t.kind = TK_Neg; break;
case '?': t.kind = TK_Question; break;
case ';': t.kind = TK_Semicolon; break;
case '#': t.kind = TK_Pound; break;
CASE2('!', TK_Not, TK_NotEquals);
CASE2('^', TK_BitXor, TK_XorAssign);
CASE2('=', TK_Assign, TK_Equals);
CASE2('*', TK_Mul, TK_MulAssign);
CASE2('%', TK_Mod, TK_ModAssign);
CASE3('+', TK_Add, TK_AddAssign, TK_Increment);
CASE3('&', TK_BitAnd, TK_AndAssign, TK_And);
CASE3('|', TK_BitOr, TK_OrAssign, TK_Or);
#undef CASE2
#undef CASE3
case '.': {
if(lexc(s) == '.' && lexci(s,1) == '.') {
lex_advance(s); lex_advance(s);
t.kind = TK_ThreeDots;
}
else {
t.kind = TK_Dot;
}
} break;
case '<': {
if (lexc(s) == '<') {
lex_advance(s);
if (lexc(s) == '=') {
lex_advance(s);
t.kind = TK_LeftShiftAssign;
}
else {
t.kind = TK_LeftShift;
}
}
else if (lexc(s) == '=') {
lex_advance(s);
t.kind = TK_LesserThenOrEqual;
}
else {
t.kind = TK_LesserThen;
}
} break;
case '>': {
if (lexc(s) == '>') {
lex_advance(s);
if (lexc(s) == '=') {
lex_advance(s);
t.kind = TK_RightShiftAssign;
}
else {
t.kind = TK_RightShift;
}
}
else if (lexc(s) == '=') {
lex_advance(s);
t.kind = TK_GreaterThenOrEqual;
}
else {
t.kind = TK_GreaterThen;
}
} break;
case ':': {
if (lexc(s) == ':') {
lex_advance(s);
t.kind = TK_DoubleColon;
}
else {
t.kind = TK_Colon;
}
} break;
case '-':{
if (lexc(s) == '=') {
lex_advance(s);
t.kind = TK_SubAssign;
}
else if (lexc(s) == '-') {
lex_advance(s);
t.kind = TK_Decrement;
}
else if (lexc(s) == '>') {
lex_advance(s);
t.kind = TK_Arrow;
}
else {
t.kind = TK_Sub;
}
} break;
case '\'':{not_implemented;} break;
case '"': {
t.kind = TK_StringLit;
lex_parse_string(s,&t,'"');
if(t.kind != TK_Error){
t.str += 1;
t.len -= 2;
}
t.intern_val = intern_string(&array->interns, t.string);
} break;
case '/': {
if(lexc(s) == '='){
t.kind = TK_DivAssign;
lex_advance(s);
}
else if(lexc(s) == '/'){
lex_advance(s);
t.kind = TK_Comment;
for(;;){
if(lexc(s) == '\n' || lexc(s) == 0) break;
lex_advance(s);
}
continue;
}
else if(lexc(s) == '*'){
lex_advance(s);
t.kind = TK_Comment;
for(;;){
if(lexc(s) == '*' && lexci(s,1) == '/'){
lex_advance(s);
lex_advance(s);
break;
}
else if(lexc(s) == 0){
token_error(&t, lit("Unterminated block comment"));
goto skip_continue;
}
lex_advance(s);
}
continue;
skip_continue:;
}
else {
t.kind = TK_Div;
}
} break;
case '0':case '1':case '2':case '3':case '4':
case '5':case '6':case '7':case '8':case '9':{
t.kind = TK_Int;
while(lex_is_numeric(lexc(s)))
lex_advance(s);
lex_set_len(s, &t);
lex_parse_u64(&t);
} break;
case 'A':case 'a':case 'M':case 'm':case 'B':
case 'b':case 'N':case 'n':case 'C':case 'c':case 'O':
case 'o':case 'D':case 'd':case 'P':case 'p':case 'E':
case 'e':case 'Q':case 'q':case 'F':case 'f':case 'R':
case 'r':case 'G':case 'g':case 'S':case 's':case 'H':
case 'h':case 'T':case 't':case 'I':case 'i':case 'U':
case 'u':case 'J':case 'j':case 'V':case 'v':case 'K':
case 'k':case 'W':case 'w':case 'L':case 'X':case 'l':
case 'x':case 'Z':case 'z':case 'Y':case 'y':case '_': {
t.kind = TK_Identifier;
while(lex_is_alphanumeric(lexc(s)) || lexc(s) == '_')
lex_advance(s);
lex_set_len(s,&t);
t.intern_val = intern_string(&array->interns, t.string);
if(lex_is_keyword(t.intern_val)){
t.kind = TK_Keyword;
}
} break;
default: {
token_error(&t, lit("Unknown token"));
}
}
if(t.len==0)
lex_set_len(s,&t);
token_array_push(array, &t);
}
}
function Token_Array
lex_stream(Arena *arena, String stream, String file){
Lex_Stream s = {stream, 0, stream.str, file, 0};
Token_Array array = token_array_make(arena);
init_default_keywords(&array.interns);
lex__stream(&array, &s);
return array;
}
function void
lex_test(){
Arena *scratch = arena_begin_scratch();
String test = lit("18446744073709551616{})(@?&+-;....->,:::/**/\"Thing\"//R\n Thingy"
"\"Test_Meme\"+=-===42524 4294967295 18446744073709551615"
"for if while switch");
Token_Array array = lex_stream(scratch, test, lit("Test1"));
Token_Kind kind[] = {
TK_Error,TK_OpenBrace,TK_CloseBrace,TK_CloseParen,TK_OpenParen,
TK_At,TK_Question,TK_BitAnd,TK_Add,TK_Sub,TK_Semicolon,
TK_ThreeDots, TK_Dot, TK_Arrow, TK_Comma, TK_DoubleColon, TK_Colon,
TK_StringLit, TK_Identifier, TK_StringLit, TK_AddAssign, TK_SubAssign,
TK_Equals, TK_Int, TK_Int, TK_Int, TK_Keyword, TK_Keyword,
TK_Keyword, TK_Keyword
};
String strs[] = {
lit("18446744073709551616"),lit("{"),lit("}"),lit(")"),lit("("),
lit("@"),lit("?"),lit("&"),lit("+"),lit("-"),lit(";"),
lit("..."),lit("."),lit("->"),lit(","),lit("::"),lit(":"),
lit("Thing"),lit("Thingy"),lit("Test_Meme"), lit("+="),lit("-="),
lit("=="),lit("42524"),lit("4294967295"),lit("18446744073709551615"),
lit("for"), lit("if"), lit("while"), lit("switch"),
};
U64 vals[] = {
42524, 4294967295, 18446744073709551615llu
};
int i = 0;
int ui = 0;
for(Token *t = token_array_iter_begin(&array); t; t = token_array_iter_next(&array)){
assert(t->kind == kind[i]);
assert(string_compare(t->string, strs[i++]));
if(t->kind == TK_Int){
assert(t->int_val == vals[ui++]);
}
}
arena_end_scratch();
}

53
new_parse.c Normal file
View File

@@ -0,0 +1,53 @@
typedef struct Parser{
Token_Array array;
Arena *arena;
}Parser;
function Token *
token_get(Parser *p){
Token *result = p->array.iter_bucket->data + p->array.iter_len;
return result;
}
function Token *
token_next(Parser *p){
Token *result = token_array_iter_next(&p->array);
return result;
}
function Token *
token_match(Parser *p, Token_Kind kind){
Token *token = token_get(p);
if(token->kind == kind){
token = token_next(p);
return token;
}
return 0;
}
/*
*/
function Expr *
parse_expr_atom(Parser *p){
Token *token = 0;
if((token = token_match(p, TK_StringLit))){
Expr *result = expr_str(p->arena, token);
return result;
}
else if((token = token_match(p, TK_Int))){
Expr *result = expr_int(p->arena, token);
return result;
}
invalid_codepath;
return 0;
}
function Expr *
parse_expr(Parser *p){
return 0;
}

66
old_main.c Normal file
View File

@@ -0,0 +1,66 @@
#define _CRT_SECURE_NO_WARNINGS
#include <stdio.h>
#include <windows.h>
#include "lang.h"
#include "os.h"
#include "memory.h"
#include "lex.h"
#include "parser.h"
#include "expr.h"
#include "ast.h"
#include "common.c"
#include "file.c"
#include "memory.c"
#include "parser.c"
#include "os_win32.c"
#include "lex.c"
#include "expr.c"
#include "ast.c"
#include "parse_expr.c"
#include "parse_decl.c"
#include "print.c"
#include "codegen_c.c"
function void
full_test(){
Parser p = {0};
#if 1
parser_init(&p);
String string = os_read_file(lit("test.cc"));
parser_lex_stream(&p, string, lit("Parse"));
Decl *decls = parse(&p);
assert(decls->list.first);
{
gen_begin(&p.scratch, &p);
gen_forward_decl(decls);
gen_decl(decls);
//gen_code(decls);
gen_end();
}
#endif
}
int
main(){
use_write_file("output.cc");
lex_test();
parser_test();
full_test();
Arena arena = {};
Token_Array array = token_array_make(&arena);
token_array_push(&array, &(Token){});
close_all_files();
return 0;
}

4
os.h
View File

@@ -4,6 +4,8 @@ typedef struct OS_Memory{
SizeU commit;
SizeU reserve;
}OS_Memory;
typedef struct Arena Arena;
function OS_Memory os_reserve(SizeU size);
function void os_commit(OS_Memory *m, SizeU size);
function void os_commit(OS_Memory *m, SizeU size);
function String os_read_file(Arena *arena, String file);

28
os_win32.c
View File

@@ -1,36 +1,16 @@
function S32 os_main();
const SizeU page_size = 4096;
LRESULT CALLBACK
WindowProc(HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam){
return DefWindowProcW(hwnd, uMsg, wParam, lParam);;
}
int
WinMain(HINSTANCE hInstance, HINSTANCE a, LPSTR b, int nShowCmd){
wchar_t *CLASS_NAME = L"Cool window class";
wchar_t *WINDOW_NAME = L"Have a good day!";
WNDCLASSW wc = { };
wc.lpfnWndProc = WindowProc;
wc.hInstance = hInstance;
wc.lpszClassName = CLASS_NAME;
RegisterClassW(&wc);
HWND window_handle = CreateWindowExW(0, CLASS_NAME, WINDOW_NAME, WS_OVERLAPPEDWINDOW, CW_USEDEFAULT, CW_USEDEFAULT, CW_USEDEFAULT, CW_USEDEFAULT, 0, 0, hInstance, 0);
ShowWindow(window_handle, nShowCmd);
return os_main();
}
function String
os_read_file(String name){
String result = {};
os_read_file(Arena *arena, String name){
String result = {0};
FILE *f = fopen((char *)name.str, "rb");
assert(f);
fseek(f, 0, SEEK_END);
result.len = ftell(f);
fseek(f, 0, SEEK_SET); /* same as rewind(f); */
result.str = malloc(result.len + 1);
result.str = arena_push_size(arena, result.len + 1);
fread(result.str, result.len, 1, f);
fclose(f);

229
output.cc
View File

@@ -1,32 +1,97 @@
typedef struct OpenGL OpenGL;
struct OpenGL{
void(*glVertexAttribPointer)(GLuint, GLint, GLenum, GLboolean, GLsizei, GLvoid*);
void(*glBindTexture)(GLenum, GLuint);
void(*glDrawArrays)(GLenum, GLint, GLsizei);
(((534>43)?435:42),234,((S64)32),Thing[10][2],Thing((1,2)))
(4+(2*53))
((4+2)*53)
(++5)
(--5)
(-5)
(+5)
(sizeof(32)+sizeof(S32*))
((S64**)5)
(((S64)5)+3)
((534>43)?435:42)
struct Thing{
int test;
};
S32 thing = 100;
(thing=(thing+10));
(++thing);
{
S32 thing_scoped = 10;
(thing_scoped+=10);
}
typedef struct OpenGL OpenGL;
typedef struct OS_Memory OS_Memory;
typedef struct Arena Arena;
typedef struct String String;
typedef struct Token Token;
typedef struct Lex_Stream Lex_Stream;
typedef struct Expr Expr;
typedef struct AST_Node AST_Node;
typedef struct Parser_Error Parser_Error;
typedef struct Scope Scope;
typedef struct Parser Parser;
struct test{
struct Array_Block{
Array_Block (*next);
Based_Type_Represent (data[size]);
};
struct Array{
Array_Block first;
Array_Block (*last);
S64 block;
S64 len;
};
function Based_Type_Represent array_make(Arena (*arena)){
Test_Type thing;
Based_Type_Represent result;
((result.arena)=arena);
((result.last)=(&(result.first)));
return result;
}
function void array_push(Array (*array), Based_Type_Represent (*item)){
if (((array->len)+1)>size){
assert((array->len));
Array_Block (*block) = ((Thing)arena_push_struct(((array->arena),sizeof(Array_Block))));
((array->last)=(((array->last).next)=block));
((array->len)=0);
((array->block)+=1);
}
(((array->last).data)[(++(array->len))]=(*item));
}
};
struct OpenGL{
void (*glVertexAttribPointer)(GLuint, GLint, GLenumGLenum, GLboolean, GLsizei, GLvoid *);
void (*glBindTexture)(GLenum, GLuint);
void (*glDrawArrays)(GLenumGLenum, GLint, GLsizei);
int (*(test_array[10]));
Things (*thing);
S64 thing_cap;
S64 thing_len;
};
struct OS_Memory{
void* data;
void (*data);
SizeU commit;
SizeU reserve;
};
typedef struct Arena Arena;
struct Arena{
OS_Memory memory;
U64 len;
U64 alignment;
};
typedef struct String String;
struct String{
U8* str;
U8 (*str);
S64 len;
};
typedef Intern_String String;
enum Token_Kind{
typedef String Intern_String;
typedef enum Token_Kind{
TK_End,
TK_Mul,
TK_Div,
@@ -90,37 +155,29 @@ TK_Error,
TK_Float,
TK_Int,
TK_Keyword,
};
}Token_Kind;
typedef struct Token Token;
struct Token{
Token_Kind kind;
String string;
union {
union{
S64 integer_val;
String error_val;
Intern_String intern_val;
};
String file;
S64 line;
U8* line_begin;
U8 (*line_begin);
};
typedef struct Tokens Tokens;
struct Tokens{
Token* tokens;
S64 iter;
};
typedef struct Lex_Stream Lex_Stream;
struct Lex_Stream{
U8* stream;
U8* line_begin;
U8 (*stream);
U8 (*line_begin);
String filename;
S64 line;
};
enum Expr_Kind{
typedef enum Expr_Kind{
EK_None,
EK_Atom,
EK_Unary,
@@ -130,46 +187,45 @@ EK_Cast,
EK_List,
EK_Call,
EK_Index,
};
}Expr_Kind;
typedef struct Expr Expr;
struct Expr{
Expr_Kind kind;
Token* token;
Expr* next;
union {
struct cast_val{
AST_Node* type;
Expr* expr;
};
struct list{
Expr* first;
Expr* last;
};
struct call{
Expr* atom;
Expr* list;
};
struct index{
Expr* atom;
Expr* index;
};
struct unary{
Expr* expr;
};
struct binary{
Expr* left;
Expr* right;
};
struct ternary{
Expr* cond;
Expr* on_true;
Expr* on_false;
};
Token (*token);
Expr (*next);
union{
struct{
AST_Node (*type);
Expr (*expr);
}cast_val;
struct{
Expr (*first);
Expr (*last);
}list;
struct{
Expr (*atom);
Expr (*list);
}call;
struct{
Expr (*atom);
Expr (*index);
}index;
struct{
Expr (*expr);
}unary;
struct{
Expr (*left);
Expr (*right);
}binary;
struct{
Expr (*cond);
Expr (*on_true);
Expr (*on_false);
}ternary;
};
};
enum AST_Kind{
typedef enum AST_Kind{
AK_None,
AK_BaseType,
AK_Typedef,
@@ -183,61 +239,56 @@ AK_Array,
AK_Function,
AK_Variable,
AK_EnumChild,
};
}AST_Kind;
typedef struct AST_Node AST_Node;
struct AST_Node{
AST_Kind kind;
Token* pos;
Token (*pos);
Intern_String name;
AST_Node* next;
AST_Node* next_scope;
AST_Node* first_note;
AST_Node* last_note;
AST_Node* first_child;
AST_Node* last_child;
union {
AST_Node (*next);
AST_Node (*next_scope);
AST_Node (*first_note);
AST_Node (*last_note);
AST_Node (*first_child);
AST_Node (*last_child);
union{
SizeU base_type_size;
AST_Node* pointer;
AST_Node* typedef_type;
AST_Node* variable_type;
AST_Node* func_return_type;
AST_Node (*pointer);
AST_Node (*typedef_type);
AST_Node (*variable_type);
AST_Node (*func_return_type);
};
};
typedef struct Parser_Error Parser_Error;
struct Parser_Error{
Parser_Error* next;
Parser_Error (*next);
String message;
Token* token;
Token (*token);
};
typedef struct Scope Scope;
struct Scope{
Scope* next;
AST_Node* first;
AST_Node* last;
Scope (*next);
AST_Node (*first);
AST_Node (*last);
};
typedef struct Parser Parser;
struct Parser{
Arena main_arena;
Arena intern_table_arena;
Arena symbol_table_arena;
Scope* scope_free_list;
Scope* scope_stack;
Scope* global_scope;
Scope (*scope_free_list);
Scope (*scope_stack);
Scope (*global_scope);
S64 symbols_inserted;
S64 symbols_count;
AST_Node* symbols;
Intern_String* interns;
AST_Node (*symbols);
Intern_String (*interns);
S64 interns_in_bytes;
S64 interns_inserted;
S64 interns_count;
U8* first_keyword;
U8* last_keyword;
U8 (*first_keyword);
U8 (*last_keyword);
Parser_Error default;
Parser_Error error;
Tokens token_array;
};

371
parse_decl.c
View File

@@ -7,6 +7,15 @@ parse_type_function(Parser *p, Token *token){
Typespec *result = typespec_function(p, token, 0);
if(!token_is(p, TK_CloseParen))
for(;;) {
// Optional name
if(token_is(p, TK_Identifier)){
if(token_peek_is(p, 1, TK_Colon)){
token_next(p);
token_next(p);
}
}
// Parse type
if(token_is(p, TK_Identifier)){
Typespec *arg = parse_type(p);
typespec_function_push(result, arg);
@@ -16,18 +25,16 @@ parse_type_function(Parser *p, Token *token){
}
}
token_expect(p, TK_CloseParen);
if(token_is(p, TK_Identifier)){
if(token_is(p, TK_Identifier))
result->function_spec.ret = parse_type(p);
}
else{
else
result->function_spec.ret = typespec_name(p, token_get(p), intern_void);
}
return result;
}
function Typespec *
parse_type(Parser *p){
// Parse as function type or normal
Token *token = 0;
Typespec *result = 0;
@@ -58,7 +65,6 @@ parse_type(Parser *p){
else break;
}
return result;
}
@@ -72,35 +78,39 @@ parse_expr_assignment(Parser *p){
}
function void
parse_note_list(Parser *ctx, Note *parent) {
if(token_match(ctx, TK_OpenParen)) {
parse_note_list(Parser *p, Note *parent) {
if(token_match(p, TK_OpenParen)) {
if(token_match(p, TK_CloseParen)){
return;
}
do {
Token *name = token_expect(ctx, TK_Identifier);
Note *current = note_push_new(ctx, parent, name, name->intern_val, 0);
parse_note_list(ctx, current);
if(token_match(ctx, TK_Assign)) {
current->expr = parse_expr(ctx);
Token *name = token_expect(p, TK_Identifier);
Note *current = note_push_new(p, parent, name, name->intern_val, 0);
parse_note_list(p, current);
if(token_match(p, TK_Assign)) {
current->expr = parse_expr(p);
}
} while(token_match(ctx, TK_Comma));
token_expect(ctx, TK_CloseParen);
} while(token_match(p, TK_Comma));
token_expect(p, TK_CloseParen);
}
}
function void
parse__notes(Parser *ctx, Note *result) {
while(token_match(ctx, TK_At)) {
Token *name = token_expect(ctx, TK_Identifier);
Note *current = note_push_new(ctx, result, name, name->intern_val, 0);
parse_note_list(ctx, current);
if(token_match(ctx, TK_Assign)) {
current->expr = parse_expr(ctx);
parse__notes(Parser *p, Note *result) {
while(token_match(p, TK_At)) {
Token *name = token_expect(p, TK_Identifier);
Note *current = note_push_new(p, result, name, name->intern_val, 0);
parse_note_list(p, current);
if(token_match(p, TK_Assign)) {
current->expr = parse_expr(p);
}
token_match(p, TK_Semicolon);
}
}
function Note
parse_notes(Parser *p){
Note result = {};
Note result = {0};
parse__notes(p, &result);
return result;
}
@@ -172,7 +182,6 @@ function Decl *
parse_variable(Parser *p, Token *name){
Typespec *type = parse_type(p);
Expr *expr = parse_expr_assignment(p);
token_expect(p, TK_Semicolon);
return decl_variable(p, name, name->intern_val, type, expr);
}
@@ -182,14 +191,77 @@ parse_typedef(Parser *p, Token *name){
return decl_typedef(p, name, name->intern_val, type);
}
/*
function Stmt *
parse_stmt(Parser *p);
function Stmt *
parse_stmt_list(Parser *p){
Token *token = token_expect(p, TK_OpenBrace);
Stmt *result = stmt_list(p, token);
while(!token_match(p, TK_CloseBrace)) {
Stmt *stmt = parse_stmt(p);
stmt_push(result, stmt);
}
return result;
}
function Stmt *
parse_stmt(Parser *p){
Token *token = token_get(p);
Decl *decl = parse_decl(p);
if(decl){
Stmt *result = stmt_decl(p, token, decl);
return result;
}
else if(token_match_keyword(p, keyword_return)){
Expr *expr = parse_expr(p);
Stmt *result = stmt_return(p, token, expr);
token_expect(p, TK_Semicolon);
return result;
}
else if(token_match_keyword(p, keyword_if)){
Expr *expr = parse_expr(p);
Stmt *if_body = parse_stmt_list(p);
Stmt *result = stmt_if(p, token, if_body, expr);
Stmt *head = result;
while(token_match_keyword(p, keyword_else)){
if(token_match_keyword(p, keyword_if)){
expr = parse_expr(p);
if_body = parse_stmt_list(p);
head = head->next = stmt_if(p, token, if_body, expr);
}
else{
if_body = parse_stmt_list(p);
head = head->next = stmt_if(p, token, if_body, 0);
break;
}
}
return result;
}
else if((token_is(p, TK_OpenBrace))){
Stmt *result = parse_stmt_list(p);
return result;
}
else{
Expr *expr = parse_expr(p);
token_expect(p, TK_Semicolon);
return stmt_expr(p, token, expr);
}
}
function Decl *
parse_function(Parser *p, Token *name){
Decl *result = decl_function(p, name, name->intern_val, 0);
if(!token_is(p, TK_CloseParen)){
for(;;) {
if((name = token_match(p, TK_Identifier)))
decl_function_push(p, result, name, name->intern_val, parse_type(p));
if(token_peek_is(p, 1, TK_Colon)){
if(token_peek_is(p, 2, TK_Identifier) ||
token_peek_is(p, 2, TK_OpenParen)){
Token *name = parse_get_token_name(p,1);
Typespec *type = parse_type(p);
decl_function_push(p, result, name, name->intern_val, type);
}
}
else if(!token_match(p, TK_Comma))
break;
}
@@ -198,13 +270,12 @@ parse_function(Parser *p, Token *name){
if(token_is(p, TK_Identifier))
result->function_decl.ret = parse_type(p);
else
result->function_decl.ret = typespec_name(p, token_get(p), intern_void);
token_expect(p, TK_OpenBrace);
parse_stmt_list(p, result);
token_expect(p, TK_CloseBrace);
result->function_decl.body = parse_stmt_list(p);
return result;
}
*/
function Decl *
parse_decl(Parser *p){
@@ -225,18 +296,21 @@ parse_decl(Parser *p){
else if(token_peek_is_keyword(p, 2, keyword_typedef)){
result = parse_typedef(p, parse_get_token_name(p,2));
}
else if(token_peek_is(p, 2, TK_OpenParen)){
result = parse_function(p, parse_get_token_name(p,2));
}
}
else if(token_peek_is(p, 1, TK_Colon)){
if(token_peek_is(p, 2, TK_Identifier) ||
token_peek_is(p, 2, TK_OpenParen)){
result = parse_variable(p, parse_get_token_name(p,1));
token_expect(p, TK_Semicolon);
}
}
}
if(result){
result->first_note = notes.first;
result->last_note = notes.last;
decl_pass_notes(result, &notes);
}
else if(notes.first != 0){
parser_push_error(p, token_get(p), "Detected notes that are not attached to anything");
@@ -247,7 +321,7 @@ parse_decl(Parser *p){
function Decl *
parse(Parser *p){
Decl *result = decl_new(p, DECL_List, token_get(p), (Intern_String){});
Decl *result = decl_new(p, DECL_List, token_get(p), (Intern_String){0});
for(;;){
Decl *decl = 0;
if(token_is(p, TK_End)) {
@@ -265,188 +339,55 @@ parse(Parser *p){
return result;
}
/*
function Token *
parse__get_name(Parser *p){
Token *name = token_next(p);
token_next(p);
token_next(p);
return name;
}
function Note *
parse_enum(Parser *p, Token *name){
Note *result = 0;
if(token_expect(p, TK_OpenBrace)){
result = ast_enum(p, name, name->intern_val);
do{
Note notes = parse_notes(p);
Token *token = token_match(p, TK_Identifier);
if(token){
Expr *expr = 0;
if(token_match(p, TK_Assign)){
expr = parse_expr(p);
}
Note *child = ast_enum_child(p, token, token->intern_val, expr);
ast_node_pass_note_list(child,&notes);
ast_node_push_child(result, child);
}
else{
break;
}
} while(token_match(p, TK_Comma));
token_expect(p, TK_CloseBrace);
}
return result;
}
function Note *
parse_variable(Parser *p, Token *name){
Note *result = 0;
Token *type_token = token_expect(p, TK_Identifier);
if(type_token){
Note *type = symbol_require_type(p, type_token);
Token *star;
while((star = token_match(p, TK_Mul))){
type = ast_type_pointer(p, star, type);
}
while((star = token_match(p, TK_OpenBracket))){
Expr *expr = parse_expr(p);
type = ast_type_array(p, star, type, expr);
token_expect(p, TK_CloseBracket);
}
Expr *expr = 0;
if(token_match(p, TK_Assign)){
expr = parse_expr(p);
}
result = ast_variable(p, name, name->intern_val, type, expr);
}
return result;
}
function Note *
parse_struct(Parser *p, Token *name, AST_Kind kind, B32 is_global){
Note *result = 0;
if(token_expect(p, TK_OpenBrace)){
result = ast_struct(p, name, name?name->intern_val:(Intern_String){}, kind);
if(is_global) symbol_register(p, result);
for(;;){
Note notes = parse_notes(p);
Note *mem = 0;
Token *mem_name = token_match(p, TK_Identifier);
if(mem_name){
if(token_expect(p, TK_Colon)){
if(token_match_keyword(p, keyword_struct)){
mem = parse_struct(p, mem_name, AK_Struct, false);
}
else if(token_match_keyword(p, keyword_union)){
mem = parse_struct(p, mem_name, AK_Union, false);
}
//else if(token_match_keyword(p, keyword_enum)){
//mem = parse_enum(p, mem_name);
//}
else if(token_is(p, TK_Identifier)){
mem = parse_variable(p, mem_name);
token_expect(p, TK_Semicolon);
}
else parser_push_error(p, mem_name, "Unrecognized token while parsing struct");
if(mem){
ast_node_pass_note_list(mem, &notes);
ast_node_push_child(result, mem);
}
}
else{
break;
}
}
else if(token_match_keyword(p, keyword_union) &&
token_match(p, TK_Colon)){
mem = parse_struct(p, 0, AK_Union, false);
ast_node_pass_note_list(mem, &notes);
ast_node_push_child(result, mem);
}
else if(token_match_keyword(p, keyword_struct) &&
token_match(p, TK_Colon)){
mem = parse_struct(p, 0, AK_Struct, false);
ast_node_pass_note_list(mem, &notes);
ast_node_push_child(result, mem);
}
else if(token_expect(p, TK_CloseBrace)){
break;
}
else{
break;
}
}
}
return result;
}
function Note *
parse_typedef(Parser *p, Token *name){
Token *type_token = token_expect(p, TK_Identifier);
Note *type = symbol_require_type(p, type_token);
Note *result = ast_typedef(p, name, name->intern_val, type);
token_expect(p, TK_Semicolon);
return result;
}
function Note_List *
parse(Parser *p){
Note_List *result = ast_node_new(p, AK_List, token_get(p), intern_empty);
for(;;){
Note *node = 0;
Note notes = parse_notes(p);
if(token_is(p, TK_End)){
break;
}
else if(token_is(p, TK_Error)){
break;
}
else if(token_is(p, TK_Identifier) && // Peeking to be more error averse
token_peek_is(p, 1, TK_Colon)){
if(token_peek_is_keyword(p, 2, keyword_struct)){
node = parse_struct(p, parse__get_name(p), AK_Struct, true);
symbol_register(p, node);
}
else if(token_peek_is_keyword(p, 2, keyword_union)){
node = parse_struct(p, parse__get_name(p), AK_Union, true);
symbol_register(p, node);
}
else if(token_peek_is_keyword(p, 2, keyword_enum)){
node = parse_enum(p, parse__get_name(p));
symbol_register(p, node);
}
else if(token_peek_is_keyword(p, 2, keyword_typedef)){
node = parse_typedef(p, parse__get_name(p));
symbol_register(p, node);
}
else{
token_next(p);
}
}
else{
token_next(p);
function void gen_stmt(Stmt *stmt);
function void gen_end();
function void gen_begin(Arena *arena, Parser *p);
function void expr_print(Parser *p, Expr *expr);
function void
parser_test(){
Parser p = {0};
{
parser_init(&p);
Intern_String a = intern_string(&p, lit("Thing"));
Intern_String b = intern_string(&p, lit("Thing"));
assert(a.s.str == b.s.str);
}
String exprs[] = {
lit("(534>43?435:42,234,cast(S64)32/*todo cast*/,Thing[10][2],Thing(1,2))"),
lit("(4+2*53)"),
lit("((4+2)*53)"),
lit("++5"),
lit("5--"), // @Todo(Krzosa):
lit("-5"),
lit("(+5)"),
lit("sizeof(32) + sizeof(:S32*)"),
lit("cast(S64**)5"),
lit("cast(S64)5+3"),
lit("534>43?435:42"),
};
for(S64 i = 0; i < buff_cap(exprs); i++){
parser_lex_stream(&p, exprs[i], lit("File"));
Expr *expr = parse_expr(&p);
assert(expr);
expr_print(&p, expr);
lex_print("\n");
}
if(node){
ast_node_pass_note_list(node, &notes);
ast_node_push_child(result, node);
String stmts[] = {
lit("Thing :: struct { test: int; } "),
lit("thing: S32 = 100; "),
lit("thing = thing + 10; "),
lit("thing++; "),
lit("{ thing_scoped: S32 = 10; thing_scoped += 10; } "),
};
for(S64 i = 0; i < buff_cap(stmts); i++){
parser_lex_stream(&p, stmts[i], lit("File"));
Stmt *stmt = parse_stmt(&p);
assert(stmt);
gen_begin(&p.scratch, &p);
gen_stmt(stmt);
gen_end();
lex_print("\n");
}
else if(notes.first_note != 0){
parser_push_error(p, token_get(p), "Warning: notes got lost");
}
}
return result;
}
*/
}

7
parse_expr.c
View File

@@ -1,3 +1,4 @@
function Expr* parse_expr(Parser* p);
function Expr* parse_list_expr(Parser* p);
function Typespec *parse_type(Parser *p);
@@ -40,6 +41,7 @@ function Expr*
parse_postfix_expr(Parser* p){
Expr* result = parse_atom_expr(p);
while (token_is(p, TK_Dot)
|| token_is(p, TK_Arrow)
|| token_is(p, TK_OpenParen)
|| token_is(p, TK_OpenBracket)
|| token_is(p, TK_Decrement)
@@ -51,6 +53,11 @@ parse_postfix_expr(Parser* p){
result = expr_binary(p, op, result, r);
}
else if (token_match(p, TK_Arrow)){
Expr* r = parse_atom_expr(p);
result = expr_binary(p, op, result, r);
}
else if (token_match(p, TK_OpenParen)){
Expr* list = 0;
if (!token_match(p, TK_CloseParen)){

12
parser.c
View File

@@ -16,6 +16,10 @@ global Intern_String intern_sizeu;
global Intern_String keyword_sizeof;
global Intern_String keyword_cast;
global Intern_String keyword_enum;
global Intern_String keyword_for;
global Intern_String keyword_if;
global Intern_String keyword_else;
global Intern_String keyword_return;
global Intern_String keyword_typedef;
global Intern_String keyword_struct;
global Intern_String keyword_union;
@@ -28,6 +32,10 @@ parser_init(Parser *p){
p->interns = arena_push_array(&p->intern_table_arena, Intern_String, p->interns_count);
keyword_sizeof = intern_string(p, lit("sizeof"));
keyword_if = intern_string(p, lit("if"));
keyword_else = intern_string(p, lit("else"));
keyword_return = intern_string(p, lit("return"));
keyword_for = intern_string(p, lit("for"));
keyword_cast = intern_string(p, lit("cast"));
keyword_struct = intern_string(p, lit("struct"));
keyword_enum = intern_string(p, lit("enum"));
@@ -74,7 +82,7 @@ parser_push_error(Parser *p, Token *token, char *str, ...){
va_end(args1);
}
lex_print("Error: %s\n", string.str);// @Todo(Krzosa):
printf("Error: %s %s:%d\n", string.str, token->file.str, (S32)token->line);
Parser_Error *error = arena_push_struct(&p->main_arena, Parser_Error);
error->message = string;
error->next = 0;
@@ -140,7 +148,7 @@ table_index_advance(Table_Index *index){
function Intern_String
intern_string(Parser *p, String string){
Intern_String result = {};
Intern_String result = {0};
Table_Index index = table_index_from_string(string, p->interns_count);
for(;;){
Intern_String *intern = p->interns + index.iter;

1
parser.h
View File

@@ -12,6 +12,7 @@ typedef struct Parser{
Arena main_arena;
Arena intern_table_arena;
Arena symbol_table_arena;
Arena scratch;
S64 interns_in_bytes;
S64 interns_inserted;

14
scratch.c Normal file
View File

@@ -0,0 +1,14 @@
global Arena global_scratch;
global Arena_Checkpoint global_scratch_checkpoint;
function Arena *
arena_begin_scratch(){
global_scratch_checkpoint = arena_checkpoint(&global_scratch);
return &global_scratch;
}
function void
arena_end_scratch(){
arena_restore(global_scratch_checkpoint);
}

53
test.cc
View File

@@ -1,9 +1,53 @@
#if 0
@register
@param_expr(size)
@param_type(Type)
test::struct{
Array_Block::struct{
next: Array_Block*;
data: Type[size];
}
@inline Array::struct{
first: Array_Block;
last : Array_Block*;
block: S64;
len : S64;
}
array_make::(arena: Arena*) Type {
thing: Test_Type;
result: Type;
result.arena = arena;
result.last = &result.first;
return result;
}
array_push::(array: Array*, item: Type*){
if array->len+1 > size{
assert(array->len);
block: Array_Block* = cast(Thing)arena_push_struct(array->arena, sizeof(:Array_Block));
array->last = array->last.next = block;
array->len = 0;
array->block += 1;
}
array->last.data[array->len++] = *item;
}
}
@test(size = 4096)
OpenGL :: struct{
glVertexAttribPointer: (GLuint, GLint, GLenum, GLboolean, GLsizei, GLvoid*);
glBindTexture: (GLenum, GLuint);
glDrawArrays: (GLenum, GLint, GLsizei);
test_array: int*[10];
thing : Things*;
thing_cap: S64;
thing_len: S64;
}
OS_Memory::struct{
@@ -92,6 +136,7 @@ Token_Kind::enum{
Keyword,
}
@array(type=block)
Token::struct{
kind:Token_Kind;
@using string:String;
@@ -105,11 +150,6 @@ Token::struct{
line_begin:U8*;
}
Tokens::struct{
@array tokens: Token*;
iter : S64;
}
Lex_Stream::struct{
stream: U8*;
line_begin: U8*;
@@ -217,6 +257,7 @@ Scope :: struct{
last : AST_Node*;
}
Parser :: struct{
main_arena: Arena;
intern_table_arena: Arena;
@@ -247,7 +288,7 @@ Parser :: struct{
// This works ok
@sllqueue error: Parser_Error;
@using token_array: Tokens;
//@using token_array: Tokens;
}
/*

100
token_array.c Normal file
View File

@@ -0,0 +1,100 @@
typedef struct Token_Bucket Token_Bucket;
typedef struct Token_Array Token_Array;
struct Token_Bucket{
Token_Bucket *next;
Token data[4096];
};
// @Note(Krzosa):
// Interns has an arena - this arena points to arena in Token_Array
// Token_Array arena is used to allocate the token buckets, Intern table and Intern_Strings
//
// If we want to resize the table in the future we need a second arena or use the heap
// For now lets leave it this way
// or
// We could potentially also use the bucket array and allocate additional buckets when resizing
// the table. In that case we would need to traverse the bucket list when inserting
struct Token_Array{
Intern_Table interns;
Token_Bucket first;
Token_Bucket *last;
S64 len;
S64 block;
Arena *arena;
// @Note(Krzosa): Iterator
Token_Bucket *iter_bucket;
S64 iter_len;
S64 iter_block;
};
function Token_Array
token_array_make(Arena *arena){
Token_Array result = {
.last = &result.first,
.arena = arena,
.interns = intern_table(arena, 4096*4)
};
return result;
}
function void
token_array_push(Token_Array *array, Token *p){
if(array->len >= buff_cap(array->first.data)){
Token_Bucket *bucket = arena_push_struct(array->arena, Token_Bucket);
array->last = array->last->next = bucket;
array->len = 0;
array->block += 1;
}
array->last->data[array->len++] = *p;
}
function B32
token_array_iter_is_end(Token_Array *array){
B32 result = array->iter_len == array->len && array->iter_block == array->block;
return result;
}
function Token *
token_array_iter_next(Token_Array *array){
if(token_array_iter_is_end(array)){
return 0;
}
if(array->iter_len >= buff_cap(array->first.data)){
array->iter_len = 0;
array->iter_block += 1;
array->iter_bucket = array->iter_bucket->next;
}
Token *result = array->iter_bucket->data + array->iter_len++;
return result;
}
function Token *
token_array_iter_peek(Token_Array *array, S64 i){
S64 save_len = array->iter_len;
Token_Bucket *save_bucket = array->iter_bucket;
assert(i < buff_cap(array->first.data));
S64 over = i;
if(array->iter_len + i >= buff_cap(array->first.data)){
over = buff_cap(array->first.data) - (array->iter_len + i);
array->iter_len = 0;
array->iter_bucket = array->iter_bucket->next;
}
Token *result = array->iter_bucket->data + array->iter_len + over;
array->iter_len = save_len;
array->iter_bucket = save_bucket;
return result;
}
function Token *
token_array_iter_begin(Token_Array *array){
array->iter_len = 0;
array->iter_block = 0;
array->iter_bucket = &array->first;
Token *result = token_array_iter_next(array);
return result;
}