krzosa/corelang
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Parsing complete mostly, ordering, resolving, C++ tests

Browse Source
This commit is contained in:
Krzosa Karol
2022-05-12 17:10:24 +02:00
parent feae74b0b9
commit 9d54ed8195
37 changed files with 2748 additions and 5341 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1380
ast.c
View File

File diff suppressed because it is too large Load Diff

201
ast.h
View File

@@ -1,201 +0,0 @@
#if 0
//-----------------------------------------------------------------------------
// Type specifiers
//-----------------------------------------------------------------------------
@prefix="TS_" Typespec_Kind :: enum{ None, Name, Pointer, Array, Function }
Typespec :: struct{
kind: Typespec_Kind;
next: Typespec*;
pos : Token*;
union{
name: Intern_String;
base: Typespec*;
function_spec: struct{
first: Typespec*;
last : Typespec*;
ret : Typespec*;
}
array_spec: struct{
base: Typespec*;
size: Expr*;
}
}
}
//-----------------------------------------------------------------------------
// Notes
//-----------------------------------------------------------------------------
Note :: struct{
pos : Token*;
name: Intern_String;
expr: Expr*;
next : Note*;
first: Note*;
last : Note*;
};
//-----------------------------------------------------------------------------
// Declarations
//-----------------------------------------------------------------------------
@prefix = "DECL_" Decl_Kind :: enum { None, Struct, Union, Enum, Variable, Typedef, Function, List }
@prefix="STRUCT_" Decl_Struct_Kind :: enum { Nested, Base }
Decl_Function_Arg :: struct{
next: Decl_Function_Arg *;
name: Intern_String;
pos : Token *;
typespec: Typespec *;
}
Decl_Enum_Child :: struct{
next: Decl_Enum_Child *;
name: Intern_String;
pos : Token *;
expr: Expr *;
first_note: Note *;
last_note : Note *;
};
Decl :: struct{
kind: Decl_Kind;
next: Decl *;
name: Intern_String;
pos : Token *;
first_note: Note *;
last_note : Note *;
union{
enum_decl: struct{
first: Decl_Enum_Child *;
last: Decl_Enum_Child *;
typespec: Typespec *;
}
struct_decl: struct{
first: Decl *;
last: Decl *;
kind: Decl_Struct_Kind ;
}
variable_decl: struct{
type: Typespec *;
expr: Expr *;
}
typedef_decl: struct{
type: Typespec *;
}
function_decl: struct{
first: Decl_Function_Arg *;
last : Decl_Function_Arg *;
ret: Typespec *;
body : Stmt*;
}
list: struct{
first: Decl *;
last: Decl *;
}
}
}
//-----------------------------------------------------------------------------
// Statements
//-----------------------------------------------------------------------------
@prefix="STMT_" Stmt_Kind :: enum{ None, Decl, Expr, List, Return, If, For}
Stmt_If :: struct {
next: Stmt_If*;
cond: Expr*;
body: Stmt*;
}
Stmt :: struct {
kind: Stmt_Kind;
next: Stmt*;
pos : Token*;
union{
stmt_if: Stmt_If;
decl: Decl*;
expr: Expr*;
list: struct{
first: Stmt*;
last : Stmt*;
}
ret: struct{
expr: Expr*;
}
}
}
//-----------------------------------------------------------------------------
// Gather
//-----------------------------------------------------------------------------
@prefix="PK_"
Pointer_Kind::enum{
None,
Typespec,
Expr,
Decl,
Stmt,
Enum_Child,
Func_Arg,
Intern_String
}
Pointer::struct{
kind: Pointer_Kind;
union {
typespec: Typespec *;
decl: Decl *;
expr: Expr *;
stmt: Stmt *;
func_arg: Decl_Function_Arg*;
enum_child: Decl_Enum_Child*;
string: Intern_String *;
}
}
Pointer_Bucket::struct{
next: Pointer_Bucket*;
data: Pointer[4096];
}
Pointer_Array::struct{
first: Pointer_Bucket;
last : Pointer_Bucket*;
len : S64;
block: S64;
arena: Arena*;
iter_bucket: Pointer_Bucket*;
iter_len: S64;
iter_block: S64;
}
@prefix="GATHER_"
Gather_Flag::enum{
None = 0,
Typespec = 1,
Expr = 2,
Decl = 4,
Stmt = 8,
Enum_Child = 16,
Func_Arg = 32,
}
@prefix="TRAVERS_"
Traversal_Flag :: enum{
None,
Typespec = 1,
Expr = 2,
//Decl = 4,
Stmt = 8,
All = TRAVERS_Typespec | TRAVERS_Expr | TRAVERS_Stmt,
}
#endif
#include "generated_ast.h"

2
build.bat
View File

@@ -3,5 +3,5 @@
rem clang generate.c -fdiagnostics-absolute-paths -std=c99 -g -o generate.exe -Wl,user32.lib
rem generate.exe
clang main.c -Wall -Wno-unused-function -fdiagnostics-absolute-paths -std=c17 -g -o main.exe -Wl,user32.lib
clang main.cpp -Wall -Wno-unused-function -fdiagnostics-absolute-paths -g -o main.exe -Wl,user32.lib
rem cl main.c -std:c17

475
codegen_c.c
View File

@@ -1,475 +0,0 @@
function void gen_decl(Decl *node);
function void gen_cdecl(Typespec *type, String str);
function void gen_stmt_list(Stmt *stmt);
global String_List *glist;
global Arena *gscratch;
global Parser *genp;
#define strf(...) string_listf(gscratch, glist, __VA_ARGS__)
//-----------------------------------------------------------------------------
// String replacing util
//-----------------------------------------------------------------------------
typedef struct String_Map String_Map;
struct String_Map{
String replace;
String with;
};
function void
string_mapped_print(String string, String_Map *map, SizeU count){
Tokens tokens = lex_stream(string, lit("string_mapped_print"));
for(Token *t = tokens.tokens; t != tokens.tokens + tokens.len; t++){
String string = t->string;
for(SizeU i = 0; i < count; i++){
if(string_compare(string, map[i].replace)){
string = map[i].with;
break;
}
}
if(t->kind == TK_At) continue;
if(string_compare(t->string,keyword_function.s)) strf("\n");
strf("%.*s", (int)string.len, string.str);
if(string_compare(t->string,keyword_function.s)) strf("\n");
if(t->kind == TK_OpenBrace) strf("\n");
if(t->kind == TK_CloseBrace && t[1].kind != TK_Semicolon) strf("\n");
if(t->kind == TK_Semicolon) strf("\n");
if(t->kind == TK_Identifier && (t[1].kind == TK_Keyword || t[1].kind == TK_Identifier)) strf(" ");
if(t->kind == TK_Keyword) strf(" ");
if(t->kind == TK_Comma) strf(" ");
}
free(tokens.tokens);
}
#define STR(X) lit(#X)
//-----------------------------------------------------------------------------
// Codegen
//-----------------------------------------------------------------------------
function void
token_strf(Token *token){
strf("%.*s", (S32)token->len, token->str);
}
function void
gen_expr(Expr *expr){
switch(expr->kind) {
case EK_Atom: {
token_strf(expr->token);
} break;
case EK_Sizeof:{
strf("sizeof(");
if(expr->size_of.kind == SIZEOF_Expr){
gen_expr(expr->size_of.expr);
}
else{
assert(expr->size_of.kind == SIZEOF_Type);
gen_cdecl(expr->size_of.type, string_empty);
}
strf(")");
}break;
case EK_Binary:{
strf("(");
gen_expr(expr->binary.left);
token_strf(expr->token);
gen_expr(expr->binary.right);
strf(")");
} break;
case EK_Unary:{
strf("(");
token_strf(expr->token);
gen_expr(expr->unary.expr);
strf(")");
} break;
case EK_Ternary:{
strf("(");
gen_expr(expr->ternary.cond);
strf("?");
gen_expr(expr->ternary.on_true);
strf(":");
gen_expr(expr->ternary.on_false);
strf(")");
} break;
case EK_List:{
strf("(");
for(Expr *n = expr->list.first; n; n=n->next){
gen_expr(n);
if(n!=expr->list.last) strf(",");
}
strf(")");
}break;
case EK_Cast:{
strf("(");
strf("(");
gen_cdecl(expr->cast.type, string_empty);
strf(")");
gen_expr(expr->cast.expr);
strf(")");
} break;
case EK_Index:{
gen_expr(expr->index.atom);
strf("[");
gen_expr(expr->index.index);
strf("]");
}break;
case EK_Call:{
gen_expr(expr->call.atom);
strf("(");
gen_expr(expr->call.list);
strf(")");
}break;
default: {invalid_codepath;} break;
}
}
function String
gen_cdecl_paren(String str, String original){
if(string_is_empty(original) || original.str[0] == '['){
return str;
}
else{
String result = string_fmt(gscratch, "(%.*s)", (int)str.len, str.str);
return result;
}
}
function String
gen__cdecl(Typespec *type, String str){
switch(type->kind) {
case TS_Name: {
String space = string_is_empty(str) ? lit(""):lit(" ");
String result = string_fmt(gscratch, "%s%s%.*s", type->name.s.str, space.str, (int)str.len, str.str);
return result;
} break;
case TS_Pointer: {
String pointer = string_fmt(gscratch, "*%.*s", (int)str.len, str.str);
String add_paren = gen_cdecl_paren(pointer, str);
String result = gen__cdecl(type->base, add_paren);
return result;
} break;
case TS_Array: {
String left = string_fmt(gscratch, "%s[", str.str);
String_List *save = glist;
String_List list = {0};
glist = &list;
gen_expr(type->array_spec.size);
String expr_string = string_list_flatten(gscratch, glist);
glist = save;
String right = string_fmt(gscratch, "%s%s]", left.str, expr_string.str);
String paren = gen_cdecl_paren(right, str);
String result = gen__cdecl(type->array_spec.base, paren);
return result;
} break;
case TS_Function: {
String result = string_fmt(gscratch, "(*%s)(", str.str);
if (type->function_spec.first == 0) {
result= string_fmt(gscratch, "%svoid", result.str);
}
else {
for(Typespec *n = type->function_spec.first; n; n=n->next){
String arg = gen__cdecl(n, string_empty);
result = string_fmt(gscratch, "%s%s", result.str, arg.str);
if(n != type->function_spec.last)
result = string_fmt(gscratch, "%s, ", result.str);
}
}
result = string_fmt(gscratch, "%s)", result.str);
result = gen__cdecl(type->function_spec.ret, result);
return result;
} break;
default: {invalid_codepath;} break;
}
return string_empty;
}
function void
gen_cdecl(Typespec *type, String str){
String string = gen__cdecl(type, str);
strf("%.*s", (int)string.len, string.str);
}
function void
gen_assign_expr(Expr *expr){
if(expr){
strf(" = ");
gen_expr(expr);
}
}
function void
gen_function_decl(Decl *node){
strf("function ");
gen_cdecl(node->function_decl.ret, string_empty);
strf(" %s(", node->name.s.str);
for(Decl_Function_Arg *arg = node->function_decl.first; arg; arg=arg->next){
gen_cdecl(arg->typespec, arg->name.s);
if(arg!=node->function_decl.last) strf(", ");
}
strf(")");
}
typedef struct Macro{
Intern_String name;
Note *param_type;
Note *param_expr;
Decl *decl;
}Macro;
global Macro macros[32];
global S32 macros_i;
function void
gen_forward_decl(Decl *node){
U8 *name = node->name.s.str;
switch(node->kind) {
case DECL_List: {
for(Decl *n = node->list.first; n; n=n->next){
gen_forward_decl(n);
}
} break;
case DECL_Variable:{
} break;
case DECL_Typedef:{
} break;
case DECL_Function:{
gen_function_decl(node);
strf(";\n");
}break;
case DECL_Struct:
case DECL_Union :{
Note *note = decl_find_note(node, lit("register"));
if(note){
Note *param_expr = decl_find_note(node, lit("param_expr"));
Note *param_type = decl_find_note(node, lit("param_type"));
macros[macros_i++] = (Macro){
.name=node->name,
.param_type=param_type,
.param_expr=param_expr,
.decl=node
};
}
else{
U8 *struct_name = node->kind==DECL_Struct ? (U8*)"struct" : (U8*)"union";
if(node->struct_decl.kind == STRUCT_Base)
strf("typedef %s %s %s;\n", struct_name, name, name);
}
} break;
case DECL_Enum:{
} break;
default: {invalid_codepath;} break;
}
}
function void
gen_decl(Decl *node){
U8 *name = node->name.s.str;
switch(node->kind) {
case DECL_List: {
for(Decl *n = node->list.first; n; n=n->next){
gen_decl(n);
strf("\n");
}
} break;
case DECL_Variable:{
gen_cdecl(node->variable_decl.type, node->name.s);
gen_assign_expr(node->variable_decl.expr);
strf(";");
} break;
case DECL_Typedef:{
strf("typedef ");
gen_cdecl(node->typedef_decl.type, node->name.s);
strf(";");
} break;
case DECL_Struct:
case DECL_Union :{
Note *note = decl_find_note(node, lit("register"));
if(note){
Arena_Checkpoint checkpoint = arena_checkpoint(gscratch);
Pointer_Array array = gather(checkpoint.arena, node, GATHER_Decl|GATHER_Typespec, TRAVERS_All);
Intern_String based = intern_string(genp, lit("Based_Type_Represent"));
Intern_String type = intern_string(genp, lit("Type"));
for(Pointer it = pointer_array_iter_begin(&array); it.typespec; it = pointer_array_iter_next(&array)){
if(it.kind == PK_Typespec){
Typespec *typespec = it.typespec;
assert(typespec->kind == TS_Name || typespec->kind == TS_Pointer || typespec->kind == TS_Array || typespec->kind == TS_Function);
if(typespec->kind == TS_Name){
if(intern_compare(typespec->name, type)){
typespec->name = based;
}
}
}
else if(it.kind == PK_Decl){
Decl *decl = it.decl;
if(decl->kind == DECL_Struct || decl->kind == DECL_Function){
if(intern_compare(decl->name, type)){
decl->name = based;
}
}
}
}
arena_restore(checkpoint);
}
U8 *struct_name = node->kind==DECL_Struct ? (U8*)"struct" : (U8*)"union";
strf("%s", struct_name);
if(node->struct_decl.kind == STRUCT_Base)
strf(" %s", node->name.s.str);
strf("{\n");
for(Decl *n = node->struct_decl.first; n; n=n->next){
gen_decl(n);
strf("\n");
}
strf("}");
if(node->struct_decl.kind == STRUCT_Nested)
strf("%.*s", (int)node->name.s.len, node->name.s.str);
strf(";");
if(node->struct_decl.kind == STRUCT_Base)
strf("\n");
} break;
case DECL_Enum:{
strf("typedef enum %s", name);
if(!intern_compare(node->enum_decl.typespec->name, intern_s64)){
strf(" : ");
strf("%s", node->enum_decl.typespec->name.s.str);
}
strf("{\n");
String prefix = decl_find_string_note(node, lit("prefix"), string_empty);
for(Decl_Enum_Child *n = node->enum_decl.first; n; n=n->next){
strf("%.*s%s", (int)prefix.len, prefix.str, n->name.s.str);
gen_assign_expr(n->expr);
strf(",\n");
}
strf("}%s;\n", name);
} break;
case DECL_Function:{
gen_function_decl(node);
gen_stmt_list(node->function_decl.body);
} break;
default: {invalid_codepath;} break;
}
}
function void
gen_stmt_list(Stmt *stmt){
strf("{\n");
for(Stmt *s = stmt->list.first; s; s=s->next){
gen_stmt(s);
strf("\n");
}
strf("}");
}
function void
gen_stmt(Stmt *stmt){
switch(stmt->kind) {
case STMT_List: {
gen_stmt_list(stmt);
} break;
case STMT_Return:{
strf("return ");
gen_expr(stmt->ret.expr);
strf(";");
} break;
case STMT_If:{
strf("if ");
gen_expr(stmt->stmt_if.cond);
gen_stmt_list(stmt->stmt_if.body);
for(Stmt_If *s = stmt->stmt_if.next; s; s=s->next){
if(s->next){
strf("else if ");
gen_expr(s->cond);
gen_stmt_list(s->body);
}
else{
strf("else");
gen_stmt_list(s->body);
}
}
} break;
case STMT_Expr: {
gen_expr(stmt->expr);
strf(";");
} break;
case STMT_Decl: {
gen_decl(stmt->decl);
} break;
default: {invalid_codepath;} break;
}
}
//-----------------------------------------------------------------------------
//
//-----------------------------------------------------------------------------
global Decl *gen_parent;
function void
gen_code(Decl *node){
U8 *name = node->name.s.str;
switch(node->kind) {
case DECL_List: {
for(Decl *n = node->list.first; n; n=n->next){
gen_code(n);
}
} break;
case DECL_Variable:{
} break;
case DECL_Typedef:{
} break;
case DECL_Struct:
case DECL_Union :{
gen_parent = node;
for(Decl *n = node->struct_decl.first; n; n=n->next){
gen_code(n);
}
} break;
case DECL_Enum:{
} break;
default: {invalid_codepath;} break;
}
}
String_List list;
function void
gen_begin(Arena *scratch, Parser *p){
list = (String_List){0};
gscratch = scratch;
glist = &list;
genp = p;
}
function void
gen_end(){
String result = string_list_flatten(gscratch, glist);
lex_print("%s", result.str);
}

206
common.c
View File

@@ -6,25 +6,9 @@ clamp_top_s64(S64 val, S64 max){
}
function SizeU
clamp_top_sizeu(SizeU val, SizeU max){
if(val>max)return max;
return val;
}
function SizeU
get_align_offset(SizeU size, SizeU align){
SizeU mask = align - 1;
SizeU val = size & mask;
if(val){
val = align - val;
}
return val;
}
function SizeU
align_up(SizeU size, SizeU align){
SizeU result = size + get_align_offset(size, align);
return result;
max_sizeu(SizeU a, SizeU b){
if(a>b) return a;
return b;
}
function U64
@@ -37,6 +21,26 @@ hash_fnv(String string) {
return hash;
}
function U64
hash_u64(U64 x) {
x *= 0xff51afd7ed558ccd;
x ^= x >> 32;
return x;
}
function U64
hash_ptr(const void *ptr) {
return hash_u64((uintptr_t)ptr);
}
function U64
hash_mix(U64 x, U64 y) {
x ^= y;
x *= 0xff51afd7ed558ccd;
x ^= x >> 32;
return x;
}
function U64
is_pow2(U64 x) {
assert(x != 0);
@@ -96,6 +100,166 @@ string_to_lower(Arena *arena, String string){
return result;
}
//-----------------------------------------------------------------------------
// Per Vognsen's like Map to pointers
//-----------------------------------------------------------------------------
typedef struct Map_Key_Val{
U64 key;
void *value;
}Map_Key_Val;
typedef struct Map{
Map_Key_Val *data;
int len;
int cap;
}Map;
function void map_insert_u64(Map *map, U64 key, void *val);
function int max_int(int a, int b);
#include <stdlib.h>
function void
map_grow(Map *map, int new_size){
new_size = max_int(16, new_size);
assert(new_size > map->cap);
assert(is_pow2(new_size));
Map new_map = {
.data = calloc(new_size, sizeof(Map_Key_Val)),
.cap = new_size,
};
for(int i = 0; i < map->cap; i++){
if(map->data[i].key){
map_insert_u64(&new_map, map->data[i].key, map->data[i].value);
}
}
if(map->data) free(map->data);
*map = new_map;
}
function void
map_insert_u64(Map *map, U64 key, void *val){
assert(val);
if(key == 0) key++;
if((2*map->len) + 1 > map->cap){
map_grow(map, 2*map->cap);
}
U64 hash = hash_u64(key);
U64 index = wrap_around_pow2(hash, map->cap);
U64 i = index;
for(;;){
if(map->data[i].key == 0){
map->len++;
map->data[i].key = key;
map->data[i].value = val;
return;
}
else if(map->data[i].key == key){
map->data[i].value = val;
return;
}
i = wrap_around_pow2(i+1, map->cap);
if(i == map->cap){
return;
}
}
}
function void *
map_get_u64(Map *map, U64 key){
if(map->len == 0) return 0;
if(key == 0) key++;
U64 hash = hash_u64(key);
U64 index = wrap_around_pow2(hash, map->cap);
U64 i = index;
for(;;){
if(map->data[i].key == key){
return map->data[i].value;
}
else if(map->data[i].key == 0){
return 0;
}
i = wrap_around_pow2(i+1, map->cap);
if(i == map->cap){
return 0;
}
}
}
function void *
map_get(Map *map, void *pointer){
return map_get_u64(map, (U64)pointer);
}
function void
map_insert(Map *map, void *key, void *value){
map_insert_u64(map, (U64)key, value);
}
function void
map_test(){
Map map = {0};
const SizeU size = 1025;
for(SizeU i = 1; i < size; i++){
map_insert_u64(&map, i, (void *)i);
}
for(SizeU i = 1; i < size; i++){
SizeU val = (SizeU)map_get_u64(&map, i);
assert(val == i);
}
}
//-----------------------------------------------------------------------------
// Array
//-----------------------------------------------------------------------------
typedef struct Array_Head{
int cap, len;
}Array_Head;
#define array_get_head(x) (((Array_Head *)(x)) - 1)
#define array_cap(x) array_get_head(x)->cap
#define array_len(x) array_get_head(x)->len
#define array_push(arr,i) (array_grow((void **)&arr, sizeof(arr[0])), (arr)[array_len(arr)++] = (i))
#define array_init(arr,cap) array__init((void **)&arr,sizeof(arr[0]), cap)
function void
array__init(void **array, SizeU sizeof_item, int cap){
Array_Head *head = malloc(sizeof_item*cap + sizeof(Array_Head));
head->cap = cap;
head->len = 0;
*array = head + 1;
}
function void
array_grow(void **array, SizeU sizeof_item){
if(*array == 0){
array__init(array, sizeof_item, 16);
}
else if(array_len(*array) + 1 > array_cap(*array)){
Array_Head *head = array_get_head(*array);
SizeU len = head->len;
SizeU cap = head->cap * 2;
head = realloc(head, sizeof_item*cap + sizeof(Array_Head));
head->cap = cap;
head->len = len;
*array = head + 1;
}
}
function void
array_test(){
int *array = 0;
for(int i = 0; i < 100; i++){
array_push(array, i);
}
for(int i = 0; i < 100; i++){
assert(array[i] == i);
}
}
//-----------------------------------------------------------------------------
// String interning
//-----------------------------------------------------------------------------
@@ -195,3 +359,7 @@ intern_test(){
intern_string(&table, lit("No_Thing"))));
}

55
dllqueue.c
View File

@@ -1,55 +0,0 @@
function void
push(Type *l, Type *node){
if(l->first == 0){
l->first = l->last = node;
node->prev = 0;
node->next = 0;
}
else{
l->last->next = node;
node->prev = l->last;
node->next = 0;
l->last = node;
}
node->parent = l;
}
function void
push_front(Type *l, Type *node){
if(l->first == 0){
l->first = l->last = node;
node->prev = 0;
node->next = 0;
}
else{
node->next = l->first;
l->first->prev = node;
node->prev = 0;
l->first = node;
}
node->parent = l;
}
function void
remove(Type *l, Type *node){
if(l->first == l->last){
assert(node == l->last);
l->first = l->last = 0;
}
else if(l->last == node){
l->last = l->last->prev;
l->last->next = 0;
}
else if(l->first == node){
l->first = l->first->next;
l->first->prev = 0;
}
else{
node->prev->next = node->next;
node->next->prev = node->prev;
}
node->parent = 0;
node->prev = 0;
node->next = 0;
}

89
expr.c
View File

@@ -1,89 +0,0 @@
function Expr *
expr_new(Parser *p, Expr_Kind kind, Token *token){
Expr *expr = arena_push_struct(&p->main_arena, Expr);
expr->kind = kind;
expr->token = token;
return expr;
}
function Expr *
expr_atom(Parser *p, Token *token){
Expr *expr = expr_new(p, EK_Atom, token);
return expr;
}
function Expr *
expr_unary(Parser *p, Token *op, Expr *exp){
Expr *expr = expr_new(p, EK_Unary, op);
expr->unary.expr = exp;
return expr;
}
function Expr *
expr_binary(Parser *p, Token *op, Expr *left, Expr *right){
Expr *expr = expr_new(p, EK_Binary, op);
expr->binary.left = left;
expr->binary.right = right;
return expr;
}
function Expr *
expr_ternary(Parser *p, Token *op, Expr *cond, Expr *on_true, Expr *on_false){
Expr *expr = expr_new(p, EK_Ternary, op);
expr->ternary.cond = cond;
expr->ternary.on_true = on_true;
expr->ternary.on_false = on_false;
return expr;
}
function Expr *
expr_call(Parser *p, Token *token, Expr *atom, Expr *list){
Expr *expr = expr_new(p, EK_Call, token);
expr->call.atom = atom;
expr->call.list = list;
return expr;
}
function Expr *
expr_index(Parser *p, Token *token, Expr *atom, Expr *index){
Expr *expr = expr_new(p, EK_Index, token);
expr->index.atom = atom;
expr->index.index = index;
return expr;
}
function Expr *
expr_cast(Parser *p, Token *token, Typespec *type, Expr *exp){
Expr *expr = expr_new(p, EK_Cast, token);
expr->cast.type = type;
expr->cast.expr = exp;
return expr;
}
function Expr *
expr_sizeof_type(Parser *p, Token *token, Typespec *type){
Expr *expr = expr_new(p, EK_Sizeof, token);
expr->size_of.kind = SIZEOF_Type;
expr->size_of.type = type;
return expr;
}
function Expr *
expr_sizeof_expr(Parser *p, Token *token, Expr *in_expr){
Expr *expr = expr_new(p, EK_Sizeof, token);
expr->size_of.kind = SIZEOF_Expr;
expr->size_of.expr = in_expr;
return expr;
}
function Expr *
expr_list(Parser *p, Token *token){
Expr *expr = expr_new(p, EK_List, token);
return expr;
}
function void
expr_list_push(Expr *list, Expr *expr){
SLLQueuePush(list->list.first, list->list.last, expr);
}

64
expr.h
View File

@@ -1,64 +0,0 @@
#pragma once
typedef struct Expr Expr;
typedef struct Typespec Typespec;
typedef enum Expr_Sizeof_Kind{
SIZEOF_Expr,
SIZEOF_Type,
}Expr_Sizeof_Kind;
typedef enum Expr_Kind{
EK_None,
EK_Atom,
EK_Unary,
EK_Binary,
EK_Ternary,
EK_Cast,
EK_List,
EK_Call,
EK_Index,
EK_Sizeof,
} Expr_Kind;
struct Expr {
Expr_Kind kind;
Token *token;
Expr *next;
union {
struct {
Typespec *type;
Expr* expr;
} cast;
struct {
Expr *first;
Expr *last;
} list;
struct {
Expr *atom;
Expr *list;
} call;
struct {
Expr *atom;
Expr *index;
} index;
struct {
Expr* expr;
} unary;
struct {
Expr* left;
Expr* right;
} binary;
struct {
Expr* cond;
Expr* on_true;
Expr* on_false;
} ternary;
struct{
Expr_Sizeof_Kind kind;
union{
Typespec *type;
Expr *expr;
};
} size_of;
};
};

21
file.c
View File

@@ -1,21 +0,0 @@
global FILE *global_output_file;
global FILE *global_all_files[32];
global U32 global_all_files_count;
#define lex_print(...) fprintf(global_output_file, __VA_ARGS__)
#define lex_new_line() lex_print("\n")
function void
use_write_file(const char *file){
global_output_file = fopen(file, "w");
global_all_files[global_all_files_count++] = global_output_file;
assert_msg(global_output_file, "Failed to open file:%s", file);
}
function void
close_all_files(){
for(U32 i = 0; i < global_all_files_count; i++){
fclose(global_all_files[i]);
}
}

49
generated.c
View File

@@ -1,49 +0,0 @@
function void decl_dll_list_push(Decl*l,Decl*node){
if (l->list_decl.first==0){
l->list_decl.first=l->list_decl.last=node;
node->prev=0;
node->next=0;
}
else {
l->list_decl.last->next=node;
node->prev=l->list_decl.last;
node->next=0;
l->list_decl.last=node;
}
node->parent=l;
}
function void decl_dll_list_push_front(Decl*l,Decl*node){
if (l->list_decl.first==0){
l->list_decl.first=l->list_decl.last=node;
node->prev=0;
node->next=0;
}
else {
node->next=l->list_decl.first;
l->list_decl.first->prev=node;
node->prev=0;
l->list_decl.first=node;
}
node->parent=l;
}
function void decl_list_remove(Decl*l,Decl*node){
if (l->list_decl.first==l->list_decl.last){
assert(node==l->list_decl.last);
l->list_decl.first=l->list_decl.last=0;
}
else if (l->list_decl.last==node){
l->list_decl.last=l->list_decl.last->prev;
l->list_decl.last->next=0;
}
else if (l->list_decl.first==node){
l->list_decl.first=l->list_decl.first->next;
l->list_decl.first->prev=0;
}
else {
node->prev->next=node->next;
node->next->prev=node->prev;
}
node->parent=0;
node->prev=0;
node->next=0;
}

0
generated_ast.cpp
View File

53
generated_lex.cpp
View File

@@ -1,53 +0,0 @@
// Generated Array block
function
Token_Array token_array_make(Arena*arena){
Token_Array result={
0};
result.arena=arena;
result.last=&result.first;
return result;
}
function
void token_array_push(Token_Array*array, Token*item){
if(array->len+1>4096){
assert(array->arena);
Token_Array_Block*block=arena_push_struct(array->arena, Token_Array_Block);
array->last=array->last->next=block;
array->len=0;
array->block+=1;
}
array->last->data[array->len++]=*item;
}
function
Token_Array_Iter token_array_iter(Token_Array*array){
return((Token_Array_Iter){
.total_len=array->len, .total_block=array->block, .block=&array->first, }
);
}
function
Token*token_array_iter_next(Token_Array_Iter*it){
if(it->iter_len+1>4096){
it->iter_len=0;
it->iter_block+=1;
it->block=it->block->next;
}
return it->block->data+it->iter_len++;
}
function
B32 token_array_iter_is_end(Token_Array_Iter*it){
B32 result=it->iter_len==it->total_len&&it->iter_block==it->total_block;
return result;
}
function
Token*token_array_iter_begin(Token_Array*array){
array->it=token_array_iter(array);
Token*result=token_array_iter_next(&array->it);
return result;
}

98
generated_lex.h
View File

@@ -1,98 +0,0 @@
typedef struct Intern_String Intern_String;
typedef struct Token Token;
typedef struct Lex_Stream Lex_Stream;
typedef struct Tokens Tokens;
struct Intern_String{
String s;
};
typedef enum Token_Kind{
TK_End,
TK_Mul,
TK_Div,
TK_Add,
TK_Sub,
TK_Mod,
TK_BitAnd,
TK_BitOr,
TK_BitXor,
TK_Neg,
TK_Not,
TK_OpenParen,
TK_CloseParen,
TK_OpenBrace,
TK_CloseBrace,
TK_OpenBracket,
TK_CloseBracket,
TK_Comma,
TK_Pound,
TK_Question,
TK_ThreeDots,
TK_Semicolon,
TK_Dot,
TK_LesserThen,
TK_GreaterThen,
TK_Colon,
TK_Assign,
TK_DivAssign,
TK_MulAssign,
TK_ModAssign,
TK_SubAssign,
TK_AddAssign,
TK_AndAssign,
TK_OrAssign,
TK_XorAssign,
TK_LeftShiftAssign,
TK_RightShiftAssign,
TK_DoubleColon,
TK_At,
TK_Decrement,
TK_Increment,
TK_PostDecrement,
TK_PostIncrement,
TK_LesserThenOrEqual,
TK_GreaterThenOrEqual,
TK_Equals,
TK_And,
TK_Or,
TK_NotEquals,
TK_LeftShift,
TK_RightShift,
TK_Arrow,
TK_ExprSizeof,
TK_DocComment,
TK_Comment,
TK_Identifier,
TK_StringLit,
TK_U8Lit,
TK_Character,
TK_Error,
TK_Float,
TK_Int,
TK_Keyword,
}Token_Kind;
struct Token{
Token_Kind kind;
union{
struct{
U8 (*str);
S64 len;
};
String string;
};
union{
S64 int_val;
String error_val;
Intern_String intern_val;
};
String file;
S64 line;
U8 (*line_begin);
};

5
lang.h
View File

@@ -30,8 +30,9 @@ typedef int64_t SizeI;
typedef float F32;
typedef double F64;
const B32 true = 1;
const B32 false = 0;
#include <stdbool.h>
//const B32 true = 1;
//const B32 false = 0;
#define kib(x) ((x)*1024llu)
#define mib(x) (kib(x)*1024llu)
#define gib(x) (mib(x)*1024llu)

741
lex.c
View File

@@ -1,34 +1,192 @@
global Token token_end_of_stream = {0};
global Intern_String keyword_if;
global Intern_String keyword_for;
global Intern_String keyword_cast;
global Intern_String keyword_else;
global Intern_String keyword_defer;
global Intern_String keyword_do;
global Intern_String keyword_size_type;
global Intern_String keyword_size_expr;
global Intern_String keyword_const;
global Intern_String keyword_typedef;
global Intern_String keyword_return;
global Intern_String keyword_typeof;
global Intern_String keyword_while;
global Intern_String keyword_switch;
global Intern_String keyword_case;
global Intern_String keyword_struct;
global Intern_String keyword_enum;
global Intern_String keyword_union;
global U8 *first_keyword;
global U8 *last_keyword;
function Token *
token_alloc(Tokens *t){
if(t->cap == 0){
t->cap = 1024;
t->tokens = malloc(sizeof(Token)*t->cap);
}
else if(t->len+1 > t->cap){
t->cap *= 2;
t->tokens = realloc(t->tokens, sizeof(Token)*t->cap);
global Intern_String intern_char;
global Intern_String intern_void;
global Intern_String intern_int;
function void
init_default_keywords(Intern_Table *t){
keyword_if = intern_string(t, lit("if"));
first_keyword = keyword_if.s.str;
keyword_cast = intern_string(t, lit("cast"));
keyword_for = intern_string(t, lit("for"));
keyword_else = intern_string(t, lit("else"));
keyword_defer = intern_string(t, lit("defer"));
keyword_do = intern_string(t, lit("do"));
keyword_size_type = intern_string(t, lit("size_type"));
keyword_size_expr = intern_string(t, lit("size_expr"));
keyword_typeof = intern_string(t, lit("typeof"));
keyword_const = intern_string(t, lit("const"));
keyword_while = intern_string(t, lit("while"));
keyword_return = intern_string(t, lit("return"));
keyword_switch = intern_string(t, lit("switch"));
keyword_typedef = intern_string(t, lit("typedef"));
keyword_case = intern_string(t, lit("case"));
keyword_struct = intern_string(t, lit("struct"));
keyword_enum = intern_string(t, lit("enum"));
keyword_union = intern_string(t, lit("union"));
last_keyword = keyword_union.s.str;
intern_char = intern_string(t, lit("char"));
intern_void = intern_string(t, lit("void"));
intern_int = intern_string(t, lit("int"));
}
Token *result = t->tokens + t->len++;
memory_zero(result, sizeof(*result));
function B32
lex_is_keyword(Intern_String str){
B32 result = str.s.str >= first_keyword && str.s.str <= last_keyword;
return result;
}
function void
lex_advance(Lex_Stream *s){
if(*s->stream == '\n'){
s->stream++;
s->line++;
s->line_begin = s->stream;
typedef enum Token_Kind{
TK_End,
TK_Mul,
TK_Div,
TK_Mod,
TK_LeftShift,
TK_RightShift,
TK_FirstMul = TK_Mul,
TK_LastMul = TK_RightShift,
TK_Add,
TK_Sub,
TK_FirstAdd = TK_Add,
TK_LastAdd = TK_Sub,
TK_Equals,
TK_LesserThenOrEqual,
TK_GreaterThenOrEqual,
TK_LesserThen,
TK_GreaterThen,
TK_NotEquals,
TK_FirstCompare = TK_Equals,
TK_LastCompare = TK_NotEquals,
TK_BitAnd,
TK_BitOr,
TK_BitXor,
TK_And,
TK_Or,
TK_FirstLogical = TK_BitAnd,
TK_LastLogical = TK_Or,
TK_Neg,
TK_Not,
TK_OpenParen,
TK_CloseParen,
TK_OpenBrace,
TK_CloseBrace,
TK_OpenBracket,
TK_CloseBracket,
TK_Comma,
TK_Pound,
TK_Question,
TK_ThreeDots,
TK_Semicolon,
TK_Dot,
TK_Colon,
TK_Assign,
TK_ColonAssign,
TK_DivAssign,
TK_MulAssign,
TK_ModAssign,
TK_SubAssign,
TK_AddAssign,
TK_AndAssign,
TK_OrAssign,
TK_XorAssign,
TK_LeftShiftAssign,
TK_RightShiftAssign,
TK_FirstAssign = TK_Assign,
TK_LastAssign = TK_RightShiftAssign,
TK_DoubleColon,
TK_At,
TK_Decrement,
TK_Increment,
TK_PostDecrement,
TK_PostIncrement,
TK_Arrow,
TK_ExprSizeof,
TK_DocComment,
TK_Comment,
TK_Identifier,
TK_StringLit,
TK_Character,
TK_Error,
TK_Float,
TK_Int,
TK_Keyword,
}Token_Kind;
typedef struct Token{
Token_Kind kind;
union{
String string;
struct{U8 *str; S64 len;};
};
union {
U64 int_val;
F64 float_val;
String error_val;
Intern_String intern_val;
};
String file;
S32 line;
U8 *line_begin;
}Token;
#include "token_array.c"
typedef struct Lex_Stream{
String stream;
S64 iter;
U8 *line_begin;
String file;
S32 line;
}Lex_Stream;
function U8
lexc(Lex_Stream *s){
return s->stream.str[s->iter];
}
else if(*s->stream == 0){
// Don't advance, end of stream
}
else{
s->stream++;
function U8
lexci(Lex_Stream *s, S32 i){
return s->stream.str[s->iter+i];
}
function U8 *
lexcp(Lex_Stream *s){
return s->stream.str + s->iter;
}
function B32
@@ -57,13 +215,8 @@ lex_is_alphanumeric(U8 c){
function void
lex_set_len(Lex_Stream *s, Token *token){
assert(s->stream >= token->str);
token->len = s->stream - token->str;
}
function U8
lexc(Lex_Stream *s){
return *s->stream;
assert(lexcp(s) >= token->str);
token->len = lexcp(s) - token->str;
}
function void
@@ -72,6 +225,44 @@ token_error(Token *t, String error_val){
t->error_val = error_val;
}
function void
lex_parse_u64(Token *t){
U64 result = 0;
U64 m = 1;
for(S64 i = t->len - 1; i >= 0; --i){
U64 val = t->str[i] - '0';
U64 new_val = val * m;
if((result + new_val) < result){
token_error(t, lit("Integer overflow"));
return;
}
result+=new_val;
m *= 10;
}
t->int_val = result;
}
function void
lex_advance(Lex_Stream *s){
if(s->iter >= s->stream.len){
return;
}
else if(lexc(s) == '\n'){
s->iter++;
s->line++;
s->line_begin = lexcp(s);
}
else{
s->iter++;
}
}
function Token
token_int(U64 val){
Token result = {.kind = TK_Int, .int_val=val};
return result;
}
function void
lex_parse_string(Lex_Stream *s, Token *t, U8 c){
for(;;){
@@ -89,73 +280,55 @@ lex_parse_string(Lex_Stream *s, Token *t, U8 c){
}
}
function void
lex_token_seed(Lex_Stream *s, Token *t){
t->str = s->stream;
t->file = s->filename;
t->line = s->line;
t->line_begin = s->line_begin;
}
function U64
parse_u64(U8 *str, S64 len){
U64 result = 0;
U64 m = 1;
for(S64 i = len - 1; i >= 0; --i){
U64 val = str[i] - '0';
U64 new_val = val * m;
assert_msg(result+new_val >= result, "Integer overflow!");
result+=new_val;
m *= 10;
}
return result;
}
function void
lex_base(Lex_Stream *s, Tokens *tokens){
while(*s->stream){
#define CASE2(op, OpName, Assign) \
case op: \
if (lexc(s) == '=') { \
lex_advance(s); \
t->kind = Assign; \
t.kind = Assign; \
} else { \
t->kind = OpName; \
t.kind = OpName; \
} \
break
#define CASE3(op, OpName, Assign, Incr) \
case op: \
if (lexc(s) == '=') { \
lex_advance(s); \
t->kind = Assign; \
t.kind = Assign; \
} else if (lexc(s) == op) { \
lex_advance(s); \
t->kind = Incr; \
t.kind = Incr; \
} else { \
t->kind = OpName; \
t.kind = OpName; \
} \
break
Token *t = token_alloc(tokens);
top:
while(lex_is_whitespace(*s->stream))
function void
lex__stream(Token_Array *array, Lex_Stream *s){
while(lexc(s)){
while(lex_is_whitespace(lexc(s)))
lex_advance(s);
lex_token_seed(s, t);
Token t = {0};
t.str = lexcp(s);
t.file = s->file;
t.line = s->line;
t.line_begin = s->line_begin;
lex_advance(s);
switch(*t->str) {
switch(*t.str){
case 0: break;
case '@': t->kind = TK_At; break;
case '(': t->kind = TK_OpenParen; break;
case ')': t->kind = TK_CloseParen; break;
case '{': t->kind = TK_OpenBrace; break;
case '}': t->kind = TK_CloseBrace; break;
case '[': t->kind = TK_OpenBracket; break;
case ']': t->kind = TK_CloseBracket; break;
case ',': t->kind = TK_Comma; break;
case '~': t->kind = TK_Neg; break;
case '?': t->kind = TK_Question; break;
case ';': t->kind = TK_Semicolon; break;
case '@': t.kind = TK_At; break;
case '(': t.kind = TK_OpenParen; break;
case ')': t.kind = TK_CloseParen; break;
case '{': t.kind = TK_OpenBrace; break;
case '}': t.kind = TK_CloseBrace; break;
case '[': t.kind = TK_OpenBracket; break;
case ']': t.kind = TK_CloseBracket; break;
case ',': t.kind = TK_Comma; break;
case '~': t.kind = TK_Neg; break;
case '?': t.kind = TK_Question; break;
case ';': t.kind = TK_Semicolon; break;
case '#': t.kind = TK_Pound; break;
CASE2('!', TK_Not, TK_NotEquals);
CASE2('^', TK_BitXor, TK_XorAssign);
CASE2('=', TK_Assign, TK_Equals);
@@ -166,40 +339,34 @@ break
CASE3('|', TK_BitOr, TK_OrAssign, TK_Or);
#undef CASE2
#undef CASE3
case '#': {
t->kind = TK_Pound;
// @Todo(Krzosa): Some convenient way to recognize macros
} break;
case '.': {
if(s->stream[0] == '.' && s->stream[1] == '.') {
lex_advance(s);
lex_advance(s);
t->kind = TK_ThreeDots;
if(lexc(s) == '.' && lexci(s,1) == '.') {
lex_advance(s); lex_advance(s);
t.kind = TK_ThreeDots;
}
else {
t->kind = TK_Dot;
t.kind = TK_Dot;
}
} break;
case '<': {
if (lexc(s) == '<') {
lex_advance(s);
if (lexc(s) == '=') {
lex_advance(s);
t->kind = TK_LeftShiftAssign;
t.kind = TK_LeftShiftAssign;
}
else {
t->kind = TK_LeftShift;
t.kind = TK_LeftShift;
}
}
else if (lexc(s) == '=') {
lex_advance(s);
t->kind = TK_LesserThenOrEqual;
t.kind = TK_LesserThenOrEqual;
}
else {
t->kind = TK_LesserThen;
t.kind = TK_LesserThen;
}
} break;
@@ -208,129 +375,111 @@ break
lex_advance(s);
if (lexc(s) == '=') {
lex_advance(s);
t->kind = TK_RightShiftAssign;
t.kind = TK_RightShiftAssign;
}
else {
t->kind = TK_RightShift;
t.kind = TK_RightShift;
}
}
else if (lexc(s) == '=') {
lex_advance(s);
t->kind = TK_GreaterThenOrEqual;
t.kind = TK_GreaterThenOrEqual;
}
else {
t->kind = TK_GreaterThen;
t.kind = TK_GreaterThen;
}
} break;
case ':': {
if (lexc(s) == ':') {
lex_advance(s);
t->kind = TK_DoubleColon;
t.kind = TK_DoubleColon;
}
else if(lexc(s) == '='){
lex_advance(s);
t.kind = TK_ColonAssign;
}
else {
t->kind = TK_Colon;
t.kind = TK_Colon;
}
} break;
case '-':{
if (lexc(s) == '=') {
lex_advance(s);
t->kind = TK_SubAssign;
t.kind = TK_SubAssign;
}
else if (lexc(s) == '-') {
lex_advance(s);
t->kind = TK_Decrement;
t.kind = TK_Decrement;
}
else if (lexc(s) == '>') {
lex_advance(s);
t->kind = TK_Arrow;
t.kind = TK_Arrow;
}
else {
t->kind = TK_Sub;
t.kind = TK_Sub;
}
} break;
case '\'':{not_implemented;} break;
case '"': {
t->kind = TK_U8Lit;
lex_parse_string(s,t,'"');
if(t->kind != TK_Error){
t->str += 1;
t->len -= 2;
t.kind = TK_StringLit;
lex_parse_string(s,&t,'"');
if(t.kind != TK_Error){
t.str += 1;
t.len -= 2;
}
t.intern_val = intern_string(&array->interns, t.string);
} break;
case '/': {
if(lexc(s) == '='){
t->kind = TK_DivAssign;
t.kind = TK_DivAssign;
lex_advance(s);
}
else if(lexc(s) == '/'){
lex_advance(s);
if(lexc(s) == '/'){
lex_advance(s);
//t->kind = TK_DocComment;
}
else {
//t->kind = TK_Comment;
}
t.kind = TK_Comment;
for(;;){
if(lexc(s) == '\n' || lexc(s) == 0) break;
lex_advance(s);
}
goto top;
//lex_set_len(s,t);
continue;
}
else if(lexc(s) == '*'){
lex_advance(s);
//t->kind = TK_Comment;
t.kind = TK_Comment;
for(;;){
if(s->stream[0] == '*' && s->stream[1] == '/'){
if(lexc(s) == '*' && lexci(s,1) == '/'){
lex_advance(s);
lex_advance(s);
break;
}
else if(lexc(s) == 0){
token_error(t, lit("Unterminated block comment"));
break;
token_error(&t, lit("Unterminated block comment"));
goto skip_continue;
}
lex_advance(s);
}
goto top;
//lex_set_len(s,t);
continue;
skip_continue:;
}
else {
t.kind = TK_Div;
}
else t->kind = TK_Div;
} break;
case '0':
case '1':case '2':case '3':
case '4':case '5':case '6':
case '7':case '8':case '9': {
t->kind = TK_Int;
case '0':case '1':case '2':case '3':case '4':
case '5':case '6':case '7':case '8':case '9':{
t.kind = TK_Int;
while(lex_is_numeric(lexc(s)))
lex_advance(s);
lex_set_len(s, t);
t->int_val = parse_u64(t->str, t->len);
lex_set_len(s, &t);
lex_parse_u64(&t);
} break;
case 'l':{
if(s->stream[0] == 'i' && s->stream[1] == 't' && s->stream[2] == '(' && s->stream[3] == '"'){
t->kind = TK_StringLit;
lex_advance(s);lex_advance(s);lex_advance(s);lex_advance(s);
lex_parse_string(s,t,'"');
if(s->stream[0] == ')') {
t->str += 5;
t->len -= 6;
lex_advance(s);
}
else token_error(t, lit("Unterminated string literal, missing closing parenthesis"));
break;
}
};
case 'A':case 'a':case 'M':case 'm':case 'B':
case 'b':case 'N':case 'n':case 'C':case 'c':case 'O':
case 'o':case 'D':case 'd':case 'P':case 'p':case 'E':
@@ -338,192 +487,162 @@ break
case 'r':case 'G':case 'g':case 'S':case 's':case 'H':
case 'h':case 'T':case 't':case 'I':case 'i':case 'U':
case 'u':case 'J':case 'j':case 'V':case 'v':case 'K':
case 'k':case 'W':case 'w':case 'L':case 'X':
case 'k':case 'W':case 'w':case 'L':case 'X':case 'l':
case 'x':case 'Z':case 'z':case 'Y':case 'y':case '_': {
t->kind = TK_Identifier;
t.kind = TK_Identifier;
while(lex_is_alphanumeric(lexc(s)) || lexc(s) == '_')
lex_advance(s);
lex_set_len(s,t);
lex_set_len(s,&t);
t.intern_val = intern_string(&array->interns, t.string);
if(lex_is_keyword(t.intern_val)){
t.kind = TK_Keyword;
}
} break;
default: {
token_error(t, lit("Unknown token"));
} break;
}
if(t->len==0){
lex_set_len(s,t);
token_error(&t, lit("Unknown token"));
}
}
// Token end of stream
Token *t = token_alloc(tokens);
*t = token_end_of_stream;
tokens->len -= 1;
}
if(t.len==0)
lex_set_len(s,&t);
function Tokens
lex_stream(String in_stream, String filename){
Lex_Stream stream = {in_stream.str, in_stream.str, filename, 0};
Tokens tokens = {0};
lex_base(&stream, &tokens);
return tokens;
token_array_push(array, &t);
}
}
function void
parser_lex_stream(Parser *p, String in_stream, String filename){
Lex_Stream stream = {in_stream.str, in_stream.str, filename, 0};
p->tokens.len = 0;
p->tokens.iter = 0;
lex_base(&stream, &p->tokens);
intern_tokens(p);
lex_add_stream(Token_Array *array, String stream, String file){
Lex_Stream s = {stream, 0, stream.str, file, 0};
lex__stream(array, &s);
}
//-----------------------------------------------------------------------------
//
//-----------------------------------------------------------------------------
function B32
token_compare(Token *t, String str){
B32 result = string_compare(t->string, str);
return result;
function Token_Array
lex_make_token_array(Arena *arena){
Token_Array array = token_array_make(arena);
init_default_keywords(&array.interns);
return array;
}
function B32
token_is_comment(Token *token){
B32 result = token->kind == TK_Comment || token->kind == TK_DocComment;
return result;
}
function Token *
token_get(Parser *p){
Token *token = p->tokens.tokens + p->tokens.iter;
return token;
}
function B32
intern_compare(Intern_String a, Intern_String b){
B32 result = a.s.str == b.s.str;
return result;
}
function Token *
token_is_keyword(Parser *p, Intern_String keyword){
assert(intern_is_keyword(p, keyword));
Token *t = token_get(p);
if(t->kind == TK_Keyword && intern_compare(t->intern_val, keyword)){
return t;
}
return 0;
function Token_Array
lex_stream(Arena *arena, String stream, String file){
Token_Array array = lex_make_token_array(arena);
lex_add_stream(&array, stream, file);
return array;
}
function void
token_advance(Parser *p){
p->tokens.iter = clamp_top_s64(p->tokens.iter + 1, p->tokens.len);
}
function Token *
token_next(Parser *p){
Token *token = token_get(p);
token_advance(p);
return token;
}
function Token *
token_match(Parser *p, Token_Kind kind){
Token *token = token_get(p);
if(token->kind == kind){
return token_next(p);
}
return 0;
}
function Token *
token_match_keyword(Parser *p, Intern_String keyword){
assert(intern_is_keyword(p, keyword));
Token *token = token_get(p);
if(token->kind == TK_Keyword && intern_compare(keyword, token->intern_val)){
return token_next(p);
}
return 0;
}
function Token *
token_expect(Parser *p, Token_Kind kind){
Token *token = token_get(p);
if(token->kind == kind){
return token_next(p);
}
parser_push_error(p, token,
"Expected token of kind: %s, got instead token of kind: %s",
token_kind_string[kind].str, token_kind_string[token->kind].str);
return 0;
}
function B32
token_is(Parser *p, Token_Kind kind){
B32 result = token_get(p)->kind == kind;
return result;
}
function Token *
token_is_assignment(Parser *p){
Token *t = token_get(p);
if(t->kind >= TK_Assign && t->kind <= TK_RightShiftAssign)
return t;
return 0;
}
function Token *
token_peek(Parser *p, S64 count){
S64 index = clamp_top_s64(p->tokens.iter + count, p->tokens.len);
Token *result = p->tokens.tokens + index;
return result;
}
function Token *
token_peek_is(Parser *p, S64 count, Token_Kind kind){
Token *token = token_peek(p, count);
if(token->kind == kind)
return token;
return 0;
}
function Token *
token_peek_is_keyword(Parser *p, S64 count, Intern_String keyword){
Token *token = token_peek(p, count);
if(token->kind == TK_Keyword){
if(intern_compare(keyword, token->intern_val)){
return token;
}
}
return 0;
lex_restream(Token_Array *array, String stream, String file){
token_array_reset(array);
lex_add_stream(array, stream, file);
}
function void
lex_test(){
Tokens t;
t = lex_stream(lit("3252342510 42524 \"U8Literal\""), lit("test"));
//tokens_print(t);
assert(t.len == 3);
assert(t.tokens[0].int_val == 3252342510);
assert(t.tokens[1].int_val == 42524);
assert(t.tokens[2].kind == TK_U8Lit);
assert(token_compare(t.tokens + 2, lit("U8Literal")));
Arena *scratch = arena_begin_scratch();
String test = lit("18446744073709551616{})(@?&+-;....->,:::/**/\"Thing\"//R\n Thingy"
"\"Test_Meme\"+=-===42524 4294967295 18446744073709551615"
"for if while switch :=");
Token_Array array = lex_stream(scratch, test, lit("Test1"));
t = lex_stream(lit("_identifier Thing Thing2 lit(\"String_Test\")"), lit("test"));
//tokens_print(t);
assert(t.tokens[0].kind == TK_Identifier);
assert(t.tokens[1].kind == TK_Identifier);
assert(t.tokens[2].kind == TK_Identifier);
assert(t.tokens[3].kind == TK_StringLit);
assert(token_compare(t.tokens, lit("_identifier")));
assert(token_compare(t.tokens+1, lit("Thing")));
assert(token_compare(t.tokens+2, lit("Thing2")));
assert(token_compare(t.tokens+3, lit("String_Test")));
Token_Kind kind[] = {
TK_Error,TK_OpenBrace,TK_CloseBrace,TK_CloseParen,TK_OpenParen,
TK_At,TK_Question,TK_BitAnd,TK_Add,TK_Sub,TK_Semicolon,
TK_ThreeDots, TK_Dot, TK_Arrow, TK_Comma, TK_DoubleColon, TK_Colon,
TK_StringLit, TK_Identifier, TK_StringLit, TK_AddAssign, TK_SubAssign,
TK_Equals, TK_Int, TK_Int, TK_Int, TK_Keyword, TK_Keyword,
TK_Keyword, TK_Keyword, TK_ColonAssign, TK_End
};
String strs[] = {
lit("18446744073709551616"),lit("{"),lit("}"),lit(")"),lit("("),
lit("@"),lit("?"),lit("&"),lit("+"),lit("-"),lit(";"),
lit("..."),lit("."),lit("->"),lit(","),lit("::"),lit(":"),
lit("Thing"),lit("Thingy"),lit("Test_Meme"), lit("+="),lit("-="),
lit("=="),lit("42524"),lit("4294967295"),lit("18446744073709551615"),
lit("for"), lit("if"), lit("while"), lit("switch"), lit(":="), lit(""),
};
U64 vals[] = {
42524, 4294967295, 18446744073709551615llu
};
t = lex_stream(lit("lit(\"String_Test\"{})(324*=+=-/ *% // Comment \n"
"Thing /*Thing*/ += -= =- +/%^&*&&|| |>> << <<= >>=/*Error"),
lit("test"));
assert(t.tokens[0].kind == TK_Error);
//tokens_print(t);
int i = 0;
int ui = 0;
for(Token *t = token_array_iter_begin(&array); t->kind != TK_End; t = token_array_iter_next(&array)){
assert(t->kind == kind[i]);
assert(string_compare(t->string, strs[i++]));
if(t->kind == TK_Int){
assert(t->int_val == vals[ui++]);
}
}
arena_end_scratch();
}
//-----------------------------------------------------------------------------
// Token metadata
//-----------------------------------------------------------------------------
global const char *token_kind_string[] = {
[TK_End] = "End of stream",
[TK_Mul] = "*",
[TK_Div] = "/",
[TK_Add] = "+",
[TK_Sub] = "-",
[TK_Mod] = "%",
[TK_BitAnd] = "&",
[TK_BitOr] = "|",
[TK_BitXor] = "^",
[TK_Neg] = "~",
[TK_Not] = "!",
[TK_OpenParen] = "(",
[TK_CloseParen] = " ",
[TK_OpenBrace] = "{",
[TK_CloseBrace] = "}",
[TK_OpenBracket] = "[",
[TK_CloseBracket] = "]",
[TK_Comma] = ",",
[TK_Pound] = "#",
[TK_Question] = "?",
[TK_ThreeDots] = "...",
[TK_Semicolon] = ";",
[TK_Dot] = ".",
[TK_LesserThen] = "<",
[TK_GreaterThen] = ">",
[TK_Colon] = ":",
[TK_Assign] = "=",
[TK_ColonAssign] = ":=",
[TK_DivAssign] = "/=",
[TK_MulAssign] = "*=",
[TK_ModAssign] = "%=",
[TK_SubAssign] = "-=",
[TK_AddAssign] = "+=",
[TK_AndAssign] = "&=",
[TK_OrAssign] = "|=",
[TK_XorAssign] = "^=",
[TK_LeftShiftAssign] = "<<=",
[TK_RightShiftAssign] = ">>=",
[TK_DoubleColon] = "::",
[TK_At] = "@",
[TK_Decrement] = "--",
[TK_Increment] = "++",
[TK_PostDecrement] = "--",
[TK_PostIncrement] = "++",
[TK_LesserThenOrEqual] = "<=",
[TK_GreaterThenOrEqual] = ">=",
[TK_Equals] = "==",
[TK_And] = "&&",
[TK_Or] = "||",
[TK_NotEquals] = "!=",
[TK_LeftShift] = "<<",
[TK_RightShift] = ">>",
[TK_Arrow] = "->",
[TK_ExprSizeof] = "sizeof",
[TK_DocComment] = "DocComment",
[TK_Comment] = "Comment",
[TK_Identifier] = "Identifier",
[TK_StringLit] = "StringLit",
[TK_Character] = "Character",
[TK_Error] = "Error",
[TK_Float] = "Float",
[TK_Int] = "Int",
[TK_Keyword] = "Keyword",
};

247
lex.h
View File

@@ -1,247 +0,0 @@
#pragma once
#pragma once
#if 0
Intern_String :: struct {
s: String;
}
@prefix = "TK_"
Token_Kind :: enum {
@str = "End of stream" End,
@str = "*" Mul,
@str = "/" Div,
@str = "+" Add,
@str = "-" Sub,
@str = "%" Mod,
@str = "&" BitAnd,
@str = "|" BitOr,
@str = "^" BitXor,
@str = "~" Neg,
@str = "!" Not,
@str = "(" OpenParen,
@str = " " CloseParen,
@str = "{" OpenBrace,
@str = "}" CloseBrace,
@str = "[" OpenBracket,
@str = "]" CloseBracket,
@str = "," Comma,
@str = "#" Pound,
@str = "?" Question,
@str = "..." ThreeDots,
@str = ";" Semicolon,
@str = "." Dot,
@str = "<" LesserThen,
@str = ">" GreaterThen,
@str = ":" Colon,
@str = "=" Assign,
@str = "/=" DivAssign,
@str = "*=" MulAssign,
@str = "%=" ModAssign,
@str = "-=" SubAssign,
@str = "+=" AddAssign,
@str = "&=" AndAssign,
@str = "|=" OrAssign,
@str = "^=" XorAssign,
@str = "<<=" LeftShiftAssign,
@str = ">>=" RightShiftAssign,
@str = "::" DoubleColon,
@str = "@" At,
@str = "--" Decrement,
@str = "++" Increment,
@str = "--" PostDecrement,
@str = "++" PostIncrement,
@str = "<=" LesserThenOrEqual,
@str = ">=" GreaterThenOrEqual,
@str = "==" Equals,
@str = "&&" And,
@str = "||" Or,
@str = "!=" NotEquals,
@str = "<<" LeftShift,
@str = ">>" RightShift,
@str = "->" Arrow,
@str = "sizeof" ExprSizeof,
DocComment,
Comment,
Identifier,
StringLit,
U8Lit,
Character,
Error,
Float,
Int,
Keyword,
}Token_Kind;
@array(type=block)
Token :: struct{
kind: Token_Kind;
union {
struct{
str: U8*;
len: S64;
}
string: String;
}
union {
int_val: S64;
error_val: String;
intern_val: Intern_String;
}
file: String;
line: S64;
line_begin: U8*;
}
Lex_Stream :: struct{
stream: U8 *;
line_begin: U8 *;
filename: String;
line: S64;
}
Tokens :: struct{
tokens: Token*;
len: S64;
cap: S64;
iter: S64;
}
#endif
#include "generated_lex.h"
#include "generated_lex.cpp"
//-----------------------------------------------------------------------------
//
//-----------------------------------------------------------------------------
global String token_kind_string[] = {
[TK_End] = lit("End of stream"),
[TK_Error] = lit("Error"),
[TK_Comment] = lit("Comment"),
[TK_Identifier] = lit("Identifier"),
[TK_Keyword] = lit("Keyword"),
[TK_Character] = lit("Character"),
[TK_StringLit] = lit("StringLiteral"),
[TK_U8Lit] = lit("U8Literal"),
[TK_Float] = lit("Float"),
[TK_Int] = lit("Integer"),
[TK_Mul] = lit("*"),
[TK_Div] = lit("/"),
[TK_Add] = lit("+"),
[TK_Sub] = lit("-"),
[TK_Mod] = lit("%"),
[TK_BitAnd] = lit("&"),
[TK_BitOr] = lit("|"),
[TK_BitXor] = lit("^"),
[TK_Neg] = lit("~"),
[TK_Not] = lit("!"),
[TK_OpenParen] = lit("("),
[TK_CloseParen] = lit(")"),
[TK_OpenBrace] = lit("{"),
[TK_CloseBrace] = lit("}"),
[TK_OpenBracket] = lit("["),
[TK_CloseBracket] = lit("]"),
[TK_Comma] = lit(","),
[TK_Pound] = lit("#"),
[TK_Question] = lit("?"),
[TK_ThreeDots] = lit("..."),
[TK_Semicolon] = lit(";"),
[TK_Dot] = lit("."),
[TK_LesserThen] = lit("<"),
[TK_GreaterThen] = lit(">"),
[TK_Colon] = lit(":"),
[TK_Assign] = lit("="),
[TK_DivAssign] = lit("/="),
[TK_MulAssign] = lit("*="),
[TK_ModAssign] = lit("%="),
[TK_SubAssign] = lit("-="),
[TK_AddAssign] = lit("+="),
[TK_AndAssign] = lit("&="),
[TK_OrAssign] = lit("|="),
[TK_XorAssign] = lit("^="),
[TK_LeftShiftAssign] = lit("<<="),
[TK_RightShiftAssign] = lit(">>="),
[TK_DoubleColon] = lit("::"),
[TK_At] = lit("@"),
[TK_Decrement] = lit("--"),
[TK_Increment] = lit("++"),
[TK_PostDecrement] = lit("--"),
[TK_PostIncrement] = lit("++"),
[TK_LesserThenOrEqual] = lit("<="),
[TK_GreaterThenOrEqual] = lit(">="),
[TK_Equals] = lit("=="),
[TK_And] = lit("&&"),
[TK_Or] = lit("||"),
[TK_NotEquals] = lit("!="),
[TK_LeftShift] = lit("<<"),
[TK_RightShift] = lit(">>"),
[TK_Arrow] = lit("->"),
[TK_ExprSizeof] = lit("sizeof"),
};
global String Token_Kind_metadata[] = {
[TK_End] = lit("End of stream"),
[TK_Mul] = lit("*"),
[TK_Div] = lit("/"),
[TK_Add] = lit("+"),
[TK_Sub] = lit("-"),
[TK_Mod] = lit("%"),
[TK_BitAnd] = lit("&"),
[TK_BitOr] = lit("|"),
[TK_BitXor] = lit("^"),
[TK_Neg] = lit("~"),
[TK_Not] = lit("!"),
[TK_OpenParen] = lit("("),
[TK_CloseParen] = lit(" "),
[TK_OpenBrace] = lit("{"),
[TK_CloseBrace] = lit("}"),
[TK_OpenBracket] = lit("["),
[TK_CloseBracket] = lit("]"),
[TK_Comma] = lit(","),
[TK_Pound] = lit("#"),
[TK_Question] = lit("?"),
[TK_ThreeDots] = lit("..."),
[TK_Semicolon] = lit(";"),
[TK_Dot] = lit("."),
[TK_LesserThen] = lit("<"),
[TK_GreaterThen] = lit(">"),
[TK_Colon] = lit(":"),
[TK_Assign] = lit("="),
[TK_DivAssign] = lit("/="),
[TK_MulAssign] = lit("*="),
[TK_ModAssign] = lit("%="),
[TK_SubAssign] = lit("-="),
[TK_AddAssign] = lit("+="),
[TK_AndAssign] = lit("&="),
[TK_OrAssign] = lit("|="),
[TK_XorAssign] = lit("^="),
[TK_LeftShiftAssign] = lit("<<="),
[TK_RightShiftAssign] = lit(">>="),
[TK_DoubleColon] = lit("::"),
[TK_At] = lit("@"),
[TK_Decrement] = lit("--"),
[TK_Increment] = lit("++"),
[TK_PostDecrement] = lit("--"),
[TK_PostIncrement] = lit("++"),
[TK_LesserThenOrEqual] = lit("<="),
[TK_GreaterThenOrEqual] = lit(">="),
[TK_Equals] = lit("=="),
[TK_And] = lit("&&"),
[TK_Or] = lit("||"),
[TK_NotEquals] = lit("!="),
[TK_LeftShift] = lit("<<"),
[TK_RightShift] = lit(">>"),
[TK_Arrow] = lit("->"),
[TK_ExprSizeof] = lit("sizeof"),
[TK_DocComment] = lit("DocComment"),
[TK_Comment] = lit("Comment"),
[TK_Identifier] = lit("Identifier"),
[TK_StringLit] = lit("StringLit"),
[TK_U8Lit] = lit("U8Lit"),
[TK_Character] = lit("Character"),
[TK_Error] = lit("Error"),
[TK_Float] = lit("Float"),
[TK_Int] = lit("Int"),
[TK_Keyword] = lit("Keyword"),
};

16
main.c
View File

@@ -8,16 +8,22 @@
#include "common.c"
#include "memory.c"
#include "scratch.c"
#include "os_win32.c"
#include "new_lex.c"
#include "new_ast.c"
#include "new_print.c"
#include "new_parse.c"
#include "lex.c"
#include "ast.c"
#include "print.c"
#include "parse.c"
#include "type.c"
#include "resolve.c"
int main(){
lex_test();
parse_test();
ast_test();
intern_test();
map_test();
array_test();
test_types();
resolve_test();
}

400
main.cpp Normal file
View File

@@ -0,0 +1,400 @@
#define NOMINMAX
#include <windows.h>
#include <stdint.h>
typedef int8_t S8;
typedef int16_t S16;
typedef int32_t S32;
typedef int64_t S64;
typedef uint8_t U8;
typedef uint16_t U16;
typedef uint32_t U32;
typedef uint64_t U64;
typedef S8 B8;
typedef S16 B16;
typedef S32 B32;
typedef S64 B64;
typedef U64 SizeU;
typedef S64 SizeI;
typedef float F32;
typedef double F64;
#define function static
#define global static
#define force_inline __forceinline
#define assert(x) do{if(!(x))__debugbreak();}while(0)
#define assert_msg(x,...) assert(x)
#define invalid_codepath assert_msg(0, "Invalid codepath")
#define not_implemented assert_msg(0, "Not implemented")
#define buff_cap(x) (sizeof(x)/sizeof((x)[0]))
#define kib(x) ((x)*1024llu)
#define mib(x) (kib(x)*1024llu)
#define gib(x) (mib(x)*1024llu)
struct String{U8 *str;S64 len;};
function SizeU
get_align_offset(SizeU size, SizeU align){
SizeU mask = align - 1;
SizeU val = size & mask;
if(val){
val = align - val;
}
return val;
}
function SizeU
align_up(SizeU size, SizeU align){
SizeU result = size + get_align_offset(size, align);
return result;
}
function SizeU
align_down(SizeU size, SizeU align){
size += 1; // Make sure 8 when align is 8 doesn't get rounded down to 0
SizeU result = size - (align - get_align_offset(size, align));
return result;
}
function void
memory_copy(void *dst, void *src, SizeU size){
U8 *d = (U8*)dst;
U8 *s = (U8*)src;
for(SizeU i = 0; i < size; i++){
d[i] = s[i];
}
}
function void
memory_zero(void *p, SizeU size){
U8 *pp = (U8 *)p;
for(SizeU i = 0; i < size; i++)
pp[i] = 0;
}
template<class T>
T clamp_top(T val, T max){
if(val > max) val = max;
return val;
}
template<class T>
T clamp_bot(T bot, T val){
if(val < bot) val = bot;
return val;
}
template<class T>
T clamp(T min, T val, T max){
if(val > max) val = max;
if(val < min) val = min;
return val;
}
constexpr SizeU os_page_size = 4096;
struct OS_Memory{
SizeU commit, reserve;
U8 *data;
};
function OS_Memory
os_reserve(SizeU size){
OS_Memory result = {};
SizeU adjusted_size = align_up(size, os_page_size);
result.data = (U8*)VirtualAlloc(0, adjusted_size, MEM_RESERVE, PAGE_READWRITE);
assert_msg(result.data, "Failed to reserve virtual memory");
result.reserve = adjusted_size;
return result;
}
function B32
os_commit(OS_Memory *m, SizeU size){
SizeU commit = align_up(size, os_page_size);
SizeU total_commit = m->commit + commit;
total_commit = clamp_top(total_commit, m->reserve);
SizeU adjusted_commit = total_commit - m->commit;
if(adjusted_commit != 0){
void *result = VirtualAlloc((U8*)m->data + m->commit, adjusted_commit, MEM_COMMIT, PAGE_READWRITE);
assert_msg(result, "Failed to commit more memory");
m->commit += adjusted_commit;
return true;
}
return false;
}
function void
os_release(OS_Memory *m){
BOOL result = VirtualFree(m->data, 0, MEM_RELEASE);
assert_msg(result != 0, "Failed to release OS_Memory");
if(result){
m->data = 0;
m->commit = 0;
m->reserve = 0;
}
}
function B32
os_decommit_pos(OS_Memory *m, SizeU pos){
SizeU aligned = align_down(pos, os_page_size);
SizeU adjusted_pos = clamp_top(aligned, m->commit);
SizeU size_to_decommit = m->commit - adjusted_pos;
if(size_to_decommit){
U8 *imp_address = m->data + adjusted_pos;
BOOL result = VirtualFree(imp_address, size_to_decommit, MEM_DECOMMIT);
if(result){
m->commit -= size_to_decommit;
return true;
}
}
return false;
}
function void
test_os_memory(){
assert(align_down(4096, 4096) == 4096);
assert(align_down(4095, 4096) == 0);
OS_Memory memory = os_reserve(9000);
assert(memory.reserve == 4096*3 && memory.data && memory.commit == 0);
os_commit(&memory, 100);
assert(memory.commit == 4096);
os_commit(&memory, 100);
assert(memory.commit == 4096*2);
os_commit(&memory, 9000);
assert(memory.commit == 4096*3);
os_commit(&memory, 9000);
assert(memory.commit == 4096*3);
os_decommit_pos(&memory, 4096);
assert(memory.commit == 4096);
os_decommit_pos(&memory, 4096);
assert(memory.commit == 4096);
os_decommit_pos(&memory, 0);
assert(memory.commit == 0);
os_release(&memory);
assert(memory.data == 0);
}
enum Allocation_Kind{Allocation_Alloc,Allocation_Resize,Allocation_FreeAll,Allocation_Free};
struct Allocator;
typedef void *Allocator_Proc(Allocator*, Allocation_Kind, void *, SizeU);
struct Allocator{Allocator_Proc *proc;};
global const SizeU default_reserve_size = gib(4);
global const SizeU default_alignment = 8;
global const SizeU additional_commit_size = mib(1);
struct Arena:Allocator{
OS_Memory memory;
SizeU alignment;
SizeU len;
};
function void arena_init(Arena *arena);
function void
arena_pop_pos(Arena *arena, SizeU pos){
pos = clamp_top(pos, arena->len);
arena->len = pos;
}
function void
arena_clear(Arena *arena){
arena_pop_pos(arena, 0);
}
function void *
arena_push_size(Arena *a, SizeU size){
SizeU generous_size = size;
if(a->memory.commit+generous_size>a->memory.commit){
if(a->memory.reserve == 0){
arena_init(a);
}
B32 result = os_commit(&a->memory, generous_size+additional_commit_size);
assert(result);
}
a->len = align_up(a->len, a->alignment);
assert(a->memory.reserve > a->len + a->memory.commit);
void *result = (U8*)a->memory.data + a->len;
a->len += size;
return result;
}
force_inline void *
arena_allocator_proc(Allocator *a, Allocation_Kind kind, void *old_pointer, SizeU size){
Arena *arena = (Arena *)a;
switch(kind){
case Allocation_Alloc: return arena_push_size(arena, size);
case Allocation_Resize: return arena_push_size(arena, size);
case Allocation_Free : invalid_codepath; return 0;
case Allocation_FreeAll: arena_clear(arena); return 0;
}
}
force_inline void *
big_personal_arena_allocator_proc(Allocator *a, Allocation_Kind kind, void *old_pointer, SizeU size){
Arena *arena = (Arena *)a;
arena->alignment = 1;
return arena_allocator_proc(a, kind, old_pointer, size);
}
function void
arena_init(Arena *a){
a->memory = os_reserve(default_reserve_size);
a->alignment = default_alignment;
if(!a->proc) a->proc = arena_allocator_proc;
}
struct OS_Heap:Allocator{
HANDLE handle;
};
function void *
os_heap_allocator_proc(Allocator *a, Allocation_Kind kind, void *old_pointer, SizeU size){
OS_Heap *heap = (OS_Heap *)a;
switch(kind){
case Allocation_FreeAll:{
BOOL result = HeapDestroy(heap->handle);
assert(result != 0);
return 0;
}
case Allocation_Free:{
BOOL result = HeapFree(heap->handle, 0, old_pointer);
assert(result != 0);
return 0;
}
case Allocation_Alloc:{
void *result = HeapAlloc(heap->handle, 0, size);
assert(result);
return result;
}
case Allocation_Resize:{
void *result = HeapReAlloc(heap->handle, 0, old_pointer, size);
assert(result);
return result;
}
}
}
function OS_Heap // max_size == 0 == growing heap
win32_os_heap_create(B32 multithreaded, SizeU initial_size, SizeU max_size){
OS_Heap result = {};
result.proc = os_heap_allocator_proc;
result.handle = HeapCreate(multithreaded ? 0 : HEAP_NO_SERIALIZE, initial_size, max_size);
assert(result.handle);
return result;
}
struct Thread_Ctx{
Arena scratch[2];
Allocator *implicit_allocator;
};
thread_local Thread_Ctx thread_ctx;
global Arena pernament_arena;
global OS_Heap os_process_heap;
#define Set_Scratch() Scoped_Scratch scratch_##__LINE__
#define Set_Backup_Scratch() Scoped_Scratch scratch_##__LINE__(true)
struct Scoped_Scratch{
SizeU saved_pos;
Allocator *saved_allocator;
Arena *arena;
Scoped_Scratch(B32 backup_scratch=false){
if(!backup_scratch) arena = thread_ctx.scratch;
else arena = thread_ctx.scratch + 1;
saved_allocator = thread_ctx.implicit_allocator;
saved_pos = arena->len;
thread_ctx.implicit_allocator = arena;
}
~Scoped_Scratch(){
arena_pop_pos(arena, saved_pos);
thread_ctx.implicit_allocator = saved_allocator;
}
};
#define Set_Allocator(a) Scoped_Allocator scoped_##__LINE__(a)
struct Scoped_Allocator{
Allocator *allocator;
Scoped_Allocator(Allocator *a){
allocator = thread_ctx.implicit_allocator;
thread_ctx.implicit_allocator = a;
}
~Scoped_Allocator(){
thread_ctx.implicit_allocator = allocator;
}
};
#define exp_alloc_array(a, T, size) (T *)exp_alloc(a, sizeof(T)*(size))
#define exp_alloc_type(a, T) exp_alloc_array(a, T, 1)
#define exp_resize_array(a, p, T, size) expr_resize(a, p, sizeof(T)*(size))
force_inline void *
exp_alloc(Allocator *a, SizeU size){
return a->proc(a, Allocation_Alloc, 0, size);
}
force_inline void *
exp_resize(Allocator *a, void *pointer, SizeU size){
return a->proc(a, Allocation_Resize, pointer, size);
}
force_inline void
exp_free(Allocator *a, void *pointer){
a->proc(a, Allocation_Free, pointer, 0);
}
force_inline void
exp_free_all(Allocator *a){
a->proc(a, Allocation_FreeAll, 0, 0);
}
#define imp_alloc_array(T,size) (T *)imp_alloc(sizeof(T) * (size))
#define imp_alloc_type (T) imp_alloc_array(T,1)
#define imp_resize_array(p, T,size) (T *)imp_resize(p, sizeof(T) * (size))
force_inline void *
imp_alloc(SizeU size){
return exp_alloc(thread_ctx.implicit_allocator, size);
}
force_inline void *
imp_resize(void *pointer, SizeU size){
return exp_resize(thread_ctx.implicit_allocator, pointer, size);
}
force_inline void
imp_free(void *pointer){
exp_free(thread_ctx.implicit_allocator, pointer);
}
force_inline void
imp_free_all(){
exp_free_all(thread_ctx.implicit_allocator);
}
function void
thread_ctx_init(){
arena_init(thread_ctx.scratch);
arena_init(thread_ctx.scratch+1);
arena_init(&pernament_arena);
os_process_heap.proc = os_heap_allocator_proc;
os_process_heap.handle = GetProcessHeap();
}
function void
test_heap_allocator(){
OS_Heap heap = win32_os_heap_create(false, mib(1), 0);
Set_Allocator(&heap);
assert(thread_ctx.implicit_allocator == &heap);
U8 *result = imp_alloc_array(U8,1024);
result[1023] = 1;
result = exp_alloc_type(&heap, U8);
*result = 0;
imp_free_all();
assert(thread_ctx.implicit_allocator == &heap);
{
Set_Scratch();
assert(thread_ctx.implicit_allocator != &heap);
assert(thread_ctx.implicit_allocator == thread_ctx.scratch);
}
assert(thread_ctx.implicit_allocator == &heap);
}
int main(){
test_heap_allocator();
thread_ctx_init();
}

35
memory.c
View File

@@ -1,21 +1,30 @@
global const SizeU default_reserve_size = gib(4);
global const SizeU default_alignment = 8;
global const SizeU additional_commit_size = mib(1);
function SizeU align_up(SizeU size, SizeU align);
function void
memory_copy(U8 *dst, U8 *src, SizeU size){
memory_copy(void *dst, void *src, SizeU size){
U8 *d = (U8*)dst;
U8 *s = (U8*)src;
for(SizeU i = 0; i < size; i++){
dst[i] = src[i];
d[i] = s[i];
}
}
function void
memory_zero(void *p, SizeU size){
U8 *pp = p;
U8 *pp = (U8 *)p;
for(SizeU i = 0; i < size; i++)
pp[i] = 0;
}
function int
max_int(int a, int b){
if(a>b) return a;
return b;
}
function void
arena_init(Arena *a){
a->memory = os_reserve(default_reserve_size);
@@ -40,6 +49,19 @@ arena_push_size(Arena *a, SizeU size){
return result;
}
function void *
arena_push_copy(Arena *a, void *pointer, SizeU size){
void *result = arena_push_size(a, size);
memory_copy(result, pointer, size);
return result;
}
function SizeU
clamp_top_sizeu(SizeU val, SizeU max){
if(val>max)return max;
return val;
}
function void
arena_pop_pos(Arena *arena, SizeU pos){
pos = clamp_top_sizeu(pos, arena->len);
@@ -69,7 +91,7 @@ function String
string_fmtv(Arena *arena, const char *str, va_list args1) {
va_list args2;
va_copy(args2, args1);
U64 len = vsnprintf(0, 0, str, args2);
S64 len = vsnprintf(0, 0, str, args2);
va_end(args2);
char *result = (char *)arena_push_size(arena, len + 1);
@@ -77,7 +99,8 @@ string_fmtv(Arena *arena, const char *str, va_list args1) {
if (arena->len > 0)
arena->len -= 1;
return (String){(U8 *)result, len};
String res = {(U8 *)result, len};
return res;
}
#define STRING_FMT(arena, str, result) \
@@ -104,7 +127,7 @@ string_listf(Arena *arena, String_List *list, const char *str, ...){
function String
string_list_flatten(Arena *arena, String_List *list){
String result = {arena_push_size(arena, list->char_count + 1)};
String result = {(U8 *)arena_push_size(arena, list->char_count + 1)};
for(String_Node *node = list->first; node; node=node->next){
memory_copy(result.str+result.len, node->str, node->len);
result.len += node->len;

2
memory.h
View File

@@ -13,7 +13,7 @@ typedef struct Arena_Checkpoint{
function B32 string_compare(String a, String b);
function void *arena_push_size(Arena *a, SizeU size);
function String arena_push_string_copy(Arena *arena, String string);
#define arena_push_array(a,T,c) arena_push_size(a,sizeof(T)*(c))
#define arena_push_array(a,T,c) (T *)arena_push_size(a,sizeof(T)*(c))
#define arena_push_struct(a,T) arena_push_array(a,T,1)
function Arena *arena_begin_scratch();

1013
new_ast.c
View File

File diff suppressed because it is too large Load Diff

638
new_lex.c
View File

@@ -1,638 +0,0 @@
global Intern_String keyword_if;
global Intern_String keyword_for;
global Intern_String keyword_cast;
global Intern_String keyword_else;
global Intern_String keyword_size_type;
global Intern_String keyword_size_expr;
global Intern_String keyword_const;
global Intern_String keyword_typedef;
global Intern_String keyword_return;
global Intern_String keyword_typeof;
global Intern_String keyword_while;
global Intern_String keyword_switch;
global Intern_String keyword_case;
global Intern_String keyword_struct;
global Intern_String keyword_enum;
global Intern_String keyword_union;
global U8 *first_keyword;
global U8 *last_keyword;
global Intern_String intern_void;
global Intern_String intern_int;
function void
init_default_keywords(Intern_Table *t){
keyword_if = intern_string(t, lit("if"));
first_keyword = keyword_if.s.str;
keyword_cast = intern_string(t, lit("cast"));
keyword_for = intern_string(t, lit("for"));
keyword_else = intern_string(t, lit("else"));
keyword_size_type = intern_string(t, lit("size_type"));
keyword_size_expr = intern_string(t, lit("size_expr"));
keyword_typeof = intern_string(t, lit("typeof"));
keyword_const = intern_string(t, lit("const"));
keyword_while = intern_string(t, lit("while"));
keyword_return = intern_string(t, lit("return"));
keyword_switch = intern_string(t, lit("switch"));
keyword_typedef = intern_string(t, lit("typedef"));
keyword_case = intern_string(t, lit("case"));
keyword_struct = intern_string(t, lit("struct"));
keyword_enum = intern_string(t, lit("enum"));
keyword_union = intern_string(t, lit("union"));
last_keyword = keyword_union.s.str;
intern_void = intern_string(t, lit("void"));
intern_int = intern_string(t, lit("int"));
}
function B32
lex_is_keyword(Intern_String str){
B32 result = str.s.str >= first_keyword && str.s.str <= last_keyword;
return result;
}
typedef enum Token_Kind{
TK_End,
TK_Mul,
TK_Div,
TK_Mod,
TK_LeftShift,
TK_RightShift,
TK_FirstMul = TK_Mul,
TK_LastMul = TK_RightShift,
TK_Add,
TK_Sub,
TK_FirstAdd = TK_Add,
TK_LastAdd = TK_Sub,
TK_Equals,
TK_LesserThenOrEqual,
TK_GreaterThenOrEqual,
TK_LesserThen,
TK_GreaterThen,
TK_NotEquals,
TK_FirstCompare = TK_Equals,
TK_LastCompare = TK_NotEquals,
TK_BitAnd,
TK_BitOr,
TK_BitXor,
TK_And,
TK_Or,
TK_FirstLogical = TK_BitAnd,
TK_LastLogical = TK_Or,
TK_Neg,
TK_Not,
TK_OpenParen,
TK_CloseParen,
TK_OpenBrace,
TK_CloseBrace,
TK_OpenBracket,
TK_CloseBracket,
TK_Comma,
TK_Pound,
TK_Question,
TK_ThreeDots,
TK_Semicolon,
TK_Dot,
TK_Colon,
TK_Assign,
TK_ColonAssign,
TK_DivAssign,
TK_MulAssign,
TK_ModAssign,
TK_SubAssign,
TK_AddAssign,
TK_AndAssign,
TK_OrAssign,
TK_XorAssign,
TK_LeftShiftAssign,
TK_RightShiftAssign,
TK_DoubleColon,
TK_At,
TK_Decrement,
TK_Increment,
TK_PostDecrement,
TK_PostIncrement,
TK_Arrow,
TK_ExprSizeof,
TK_DocComment,
TK_Comment,
TK_Identifier,
TK_StringLit,
TK_Character,
TK_Error,
TK_Float,
TK_Int,
TK_Keyword,
}Token_Kind;
typedef struct Token{
Token_Kind kind;
union{
String string;
struct{U8 *str; S64 len;};
};
union {
U64 int_val;
F64 float_val;
String error_val;
Intern_String intern_val;
};
String file;
S32 line;
U8 *line_begin;
}Token;
#include "token_array.c"
typedef struct Lex_Stream{
String stream;
S64 iter;
U8 *line_begin;
String file;
S32 line;
}Lex_Stream;
function U8
lexc(Lex_Stream *s){
return s->stream.str[s->iter];
}
function U8
lexci(Lex_Stream *s, S32 i){
return s->stream.str[s->iter+i];
}
function U8 *
lexcp(Lex_Stream *s){
return s->stream.str + s->iter;
}
function B32
lex_is_whitespace(U8 c){
B32 result = c == '\n' || c == '\r' || c == ' ' || c == '\r';
return result;
}
function B32
lex_is_alphabetic(U8 c){
B32 result = (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z');
return result;
}
function B32
lex_is_numeric(U8 c){
B32 result = c >= '0' && c <= '9';
return result;
}
function B32
lex_is_alphanumeric(U8 c){
B32 result = lex_is_numeric(c) || lex_is_alphabetic(c);
return result;
}
function void
lex_set_len(Lex_Stream *s, Token *token){
assert(lexcp(s) >= token->str);
token->len = lexcp(s) - token->str;
}
function void
token_error(Token *t, String error_val){
t->kind = TK_Error;
t->error_val = error_val;
}
function void
lex_parse_u64(Token *t){
U64 result = 0;
U64 m = 1;
for(S64 i = t->len - 1; i >= 0; --i){
U64 val = t->str[i] - '0';
U64 new_val = val * m;
if((result + new_val) < result){
token_error(t, lit("Integer overflow"));
return;
}
result+=new_val;
m *= 10;
}
t->int_val = result;
}
function void
lex_advance(Lex_Stream *s){
if(s->iter >= s->stream.len){
return;
}
else if(lexc(s) == '\n'){
s->iter++;
s->line++;
s->line_begin = lexcp(s);
}
else{
s->iter++;
}
}
function Token
token_int(U64 val){
Token result = {.kind = TK_Int, .int_val=val};
return result;
}
function void
lex_parse_string(Lex_Stream *s, Token *t, U8 c){
for(;;){
if(lexc(s) == '\\') lex_advance(s);
else if(lexc(s) == c) break;
else if(lexc(s) == 0){
token_error(t, lit("Unterminated string, reached end of file"));
break;
}
lex_advance(s);
}
if(t->kind != TK_Error){
lex_advance(s);
lex_set_len(s,t);
}
}
#define CASE2(op, OpName, Assign) \
case op: \
if (lexc(s) == '=') { \
lex_advance(s); \
t.kind = Assign; \
} else { \
t.kind = OpName; \
} \
break
#define CASE3(op, OpName, Assign, Incr) \
case op: \
if (lexc(s) == '=') { \
lex_advance(s); \
t.kind = Assign; \
} else if (lexc(s) == op) { \
lex_advance(s); \
t.kind = Incr; \
} else { \
t.kind = OpName; \
} \
break
function void
lex__stream(Token_Array *array, Lex_Stream *s){
while(lexc(s)){
while(lex_is_whitespace(lexc(s)))
lex_advance(s);
Token t = {0};
t.str = lexcp(s);
t.file = s->file;
t.line = s->line;
t.line_begin = s->line_begin;
lex_advance(s);
switch(*t.str){
case 0: break;
case '@': t.kind = TK_At; break;
case '(': t.kind = TK_OpenParen; break;
case ')': t.kind = TK_CloseParen; break;
case '{': t.kind = TK_OpenBrace; break;
case '}': t.kind = TK_CloseBrace; break;
case '[': t.kind = TK_OpenBracket; break;
case ']': t.kind = TK_CloseBracket; break;
case ',': t.kind = TK_Comma; break;
case '~': t.kind = TK_Neg; break;
case '?': t.kind = TK_Question; break;
case ';': t.kind = TK_Semicolon; break;
case '#': t.kind = TK_Pound; break;
CASE2('!', TK_Not, TK_NotEquals);
CASE2('^', TK_BitXor, TK_XorAssign);
CASE2('=', TK_Assign, TK_Equals);
CASE2('*', TK_Mul, TK_MulAssign);
CASE2('%', TK_Mod, TK_ModAssign);
CASE3('+', TK_Add, TK_AddAssign, TK_Increment);
CASE3('&', TK_BitAnd, TK_AndAssign, TK_And);
CASE3('|', TK_BitOr, TK_OrAssign, TK_Or);
#undef CASE2
#undef CASE3
case '.': {
if(lexc(s) == '.' && lexci(s,1) == '.') {
lex_advance(s); lex_advance(s);
t.kind = TK_ThreeDots;
}
else {
t.kind = TK_Dot;
}
} break;
case '<': {
if (lexc(s) == '<') {
lex_advance(s);
if (lexc(s) == '=') {
lex_advance(s);
t.kind = TK_LeftShiftAssign;
}
else {
t.kind = TK_LeftShift;
}
}
else if (lexc(s) == '=') {
lex_advance(s);
t.kind = TK_LesserThenOrEqual;
}
else {
t.kind = TK_LesserThen;
}
} break;
case '>': {
if (lexc(s) == '>') {
lex_advance(s);
if (lexc(s) == '=') {
lex_advance(s);
t.kind = TK_RightShiftAssign;
}
else {
t.kind = TK_RightShift;
}
}
else if (lexc(s) == '=') {
lex_advance(s);
t.kind = TK_GreaterThenOrEqual;
}
else {
t.kind = TK_GreaterThen;
}
} break;
case ':': {
if (lexc(s) == ':') {
lex_advance(s);
t.kind = TK_DoubleColon;
}
else if(lexc(s) == '='){
lex_advance(s);
t.kind = TK_ColonAssign;
}
else {
t.kind = TK_Colon;
}
} break;
case '-':{
if (lexc(s) == '=') {
lex_advance(s);
t.kind = TK_SubAssign;
}
else if (lexc(s) == '-') {
lex_advance(s);
t.kind = TK_Decrement;
}
else if (lexc(s) == '>') {
lex_advance(s);
t.kind = TK_Arrow;
}
else {
t.kind = TK_Sub;
}
} break;
case '\'':{not_implemented;} break;
case '"': {
t.kind = TK_StringLit;
lex_parse_string(s,&t,'"');
if(t.kind != TK_Error){
t.str += 1;
t.len -= 2;
}
t.intern_val = intern_string(&array->interns, t.string);
} break;
case '/': {
if(lexc(s) == '='){
t.kind = TK_DivAssign;
lex_advance(s);
}
else if(lexc(s) == '/'){
lex_advance(s);
t.kind = TK_Comment;
for(;;){
if(lexc(s) == '\n' || lexc(s) == 0) break;
lex_advance(s);
}
continue;
}
else if(lexc(s) == '*'){
lex_advance(s);
t.kind = TK_Comment;
for(;;){
if(lexc(s) == '*' && lexci(s,1) == '/'){
lex_advance(s);
lex_advance(s);
break;
}
else if(lexc(s) == 0){
token_error(&t, lit("Unterminated block comment"));
goto skip_continue;
}
lex_advance(s);
}
continue;
skip_continue:;
}
else {
t.kind = TK_Div;
}
} break;
case '0':case '1':case '2':case '3':case '4':
case '5':case '6':case '7':case '8':case '9':{
t.kind = TK_Int;
while(lex_is_numeric(lexc(s)))
lex_advance(s);
lex_set_len(s, &t);
lex_parse_u64(&t);
} break;
case 'A':case 'a':case 'M':case 'm':case 'B':
case 'b':case 'N':case 'n':case 'C':case 'c':case 'O':
case 'o':case 'D':case 'd':case 'P':case 'p':case 'E':
case 'e':case 'Q':case 'q':case 'F':case 'f':case 'R':
case 'r':case 'G':case 'g':case 'S':case 's':case 'H':
case 'h':case 'T':case 't':case 'I':case 'i':case 'U':
case 'u':case 'J':case 'j':case 'V':case 'v':case 'K':
case 'k':case 'W':case 'w':case 'L':case 'X':case 'l':
case 'x':case 'Z':case 'z':case 'Y':case 'y':case '_': {
t.kind = TK_Identifier;
while(lex_is_alphanumeric(lexc(s)) || lexc(s) == '_')
lex_advance(s);
lex_set_len(s,&t);
t.intern_val = intern_string(&array->interns, t.string);
if(lex_is_keyword(t.intern_val)){
t.kind = TK_Keyword;
}
} break;
default: {
token_error(&t, lit("Unknown token"));
}
}
if(t.len==0)
lex_set_len(s,&t);
token_array_push(array, &t);
}
}
function void
lex_add_stream(Token_Array *array, String stream, String file){
Lex_Stream s = {stream, 0, stream.str, file, 0};
lex__stream(array, &s);
}
function Token_Array
lex_make_token_array(Arena *arena){
Token_Array array = token_array_make(arena);
init_default_keywords(&array.interns);
return array;
}
function Token_Array
lex_stream(Arena *arena, String stream, String file){
Token_Array array = lex_make_token_array(arena);
lex_add_stream(&array, stream, file);
return array;
}
function void
lex_restream(Token_Array *array, String stream, String file){
token_array_reset(array);
lex_add_stream(array, stream, file);
}
function void
lex_test(){
Arena *scratch = arena_begin_scratch();
String test = lit("18446744073709551616{})(@?&+-;....->,:::/**/\"Thing\"//R\n Thingy"
"\"Test_Meme\"+=-===42524 4294967295 18446744073709551615"
"for if while switch :=");
Token_Array array = lex_stream(scratch, test, lit("Test1"));
Token_Kind kind[] = {
TK_Error,TK_OpenBrace,TK_CloseBrace,TK_CloseParen,TK_OpenParen,
TK_At,TK_Question,TK_BitAnd,TK_Add,TK_Sub,TK_Semicolon,
TK_ThreeDots, TK_Dot, TK_Arrow, TK_Comma, TK_DoubleColon, TK_Colon,
TK_StringLit, TK_Identifier, TK_StringLit, TK_AddAssign, TK_SubAssign,
TK_Equals, TK_Int, TK_Int, TK_Int, TK_Keyword, TK_Keyword,
TK_Keyword, TK_Keyword, TK_ColonAssign, TK_End
};
String strs[] = {
lit("18446744073709551616"),lit("{"),lit("}"),lit(")"),lit("("),
lit("@"),lit("?"),lit("&"),lit("+"),lit("-"),lit(";"),
lit("..."),lit("."),lit("->"),lit(","),lit("::"),lit(":"),
lit("Thing"),lit("Thingy"),lit("Test_Meme"), lit("+="),lit("-="),
lit("=="),lit("42524"),lit("4294967295"),lit("18446744073709551615"),
lit("for"), lit("if"), lit("while"), lit("switch"), lit(":="), lit(""),
};
U64 vals[] = {
42524, 4294967295, 18446744073709551615llu
};
int i = 0;
int ui = 0;
for(Token *t = token_array_iter_begin(&array); t->kind != TK_End; t = token_array_iter_next(&array)){
assert(t->kind == kind[i]);
assert(string_compare(t->string, strs[i++]));
if(t->kind == TK_Int){
assert(t->int_val == vals[ui++]);
}
}
arena_end_scratch();
}
//-----------------------------------------------------------------------------
// Token metadata
//-----------------------------------------------------------------------------
global const char *token_kind_string[] = {
[TK_End] = "End of stream",
[TK_Mul] = "*",
[TK_Div] = "/",
[TK_Add] = "+",
[TK_Sub] = "-",
[TK_Mod] = "%",
[TK_BitAnd] = "&",
[TK_BitOr] = "|",
[TK_BitXor] = "^",
[TK_Neg] = "~",
[TK_Not] = "!",
[TK_OpenParen] = "(",
[TK_CloseParen] = " ",
[TK_OpenBrace] = "{",
[TK_CloseBrace] = "}",
[TK_OpenBracket] = "[",
[TK_CloseBracket] = "]",
[TK_Comma] = ",",
[TK_Pound] = "#",
[TK_Question] = "?",
[TK_ThreeDots] = "...",
[TK_Semicolon] = ";",
[TK_Dot] = ".",
[TK_LesserThen] = "<",
[TK_GreaterThen] = ">",
[TK_Colon] = ":",
[TK_Assign] = "=",
[TK_ColonAssign] = ":=",
[TK_DivAssign] = "/=",
[TK_MulAssign] = "*=",
[TK_ModAssign] = "%=",
[TK_SubAssign] = "-=",
[TK_AddAssign] = "+=",
[TK_AndAssign] = "&=",
[TK_OrAssign] = "|=",
[TK_XorAssign] = "^=",
[TK_LeftShiftAssign] = "<<=",
[TK_RightShiftAssign] = ">>=",
[TK_DoubleColon] = "::",
[TK_At] = "@",
[TK_Decrement] = "--",
[TK_Increment] = "++",
[TK_PostDecrement] = "--",
[TK_PostIncrement] = "++",
[TK_LesserThenOrEqual] = "<=",
[TK_GreaterThenOrEqual] = ">=",
[TK_Equals] = "==",
[TK_And] = "&&",
[TK_Or] = "||",
[TK_NotEquals] = "!=",
[TK_LeftShift] = "<<",
[TK_RightShift] = ">>",
[TK_Arrow] = "->",
[TK_ExprSizeof] = "sizeof",
[TK_DocComment] = "DocComment",
[TK_Comment] = "Comment",
[TK_Identifier] = "Identifier",
[TK_StringLit] = "StringLit",
[TK_Character] = "Character",
[TK_Error] = "Error",
[TK_Float] = "Float",
[TK_Int] = "Int",
[TK_Keyword] = "Keyword",
};

284
new_print.c
View File

@@ -1,284 +0,0 @@
function void expr_print(Expr *expr);
function B32 typespec_print(Typespec *spec);
global S64 indent;
#define println(...) do{ printf("\n"); print_indent(); printf(__VA_ARGS__); }while(0)
#define AST_CAST(item, T) T *item = (T *)item; assert(item->kind == AST_##Kind
#define AST_Iter(parent, name, T) for(T *name = (T *)(parent)->first; name; name=(T *)name->next) // , assert(name->kind == AST_##type
function void
print_indent(){
for(S64 i = 0; i < indent*2; i++)
printf(" ");
}
function void
token_print(Token *token){
printf("%.*s", (S32)token->len, token->str);
}
function void
expr_compound_print(Expr_Compound_Field *field){
switch(field->kind){
case COMPOUND_Default: {
expr_print(field->init);
}break;
case COMPOUND_Named: {
printf("[%s] = ", field->name.s.str);
expr_print(field->init);
}break;
case COMPOUND_Index: {
printf("[");
expr_print(field->index);
printf("] = ");
expr_print(field->init);
}break;
default: invalid_codepath;
}
}
function void
expr_print(Expr *expr){
switch(expr->kind) {
case EK_Int:case EK_String:case EK_Identifier: {
token_print(expr->token);
} break;
case EK_SizeExpr:{
printf("size_expr(");
expr_print(expr->size_expr.expr);
printf(")");
}break;
case EK_Compound:{
if(expr->compound.typespec){
printf("(");
typespec_print(expr->compound.typespec);
printf(")");
}
printf("{");
for(Expr_Compound_Field *n = expr->compound.first; n; n=n->next){
expr_compound_print(n);
if(n!=expr->compound.last) printf(",");
}
printf("}");
} break;
case EK_SizeType:{
printf("size_type(");
printf(")");
}break;
case EK_Paren:{
printf("(");
expr_print(expr->paren.expr);
printf(")");
} break;
case EK_Field:{
expr_print(expr->field.expr);
printf(".%s", expr->field.name.s.str);
} break;
case EK_Binary:{
printf("(");
expr_print(expr->binary.left);
token_print(expr->token);
expr_print(expr->binary.right);
printf(")");
} break;
case EK_PostfixUnary:{
printf("(");
expr_print(expr->unary.expr);
token_print(expr->token);
printf(")");
} break;
case EK_Unary:{
printf("(");
token_print(expr->token);
expr_print(expr->unary.expr);
printf(")");
} break;
case EK_Ternary:{
printf("(");
expr_print(expr->ternary.cond);
printf("?");
expr_print(expr->ternary.on_true);
printf(":");
expr_print(expr->ternary.on_false);
printf(")");
} break;
case EK_Cast:{
printf("(");
printf("(");
typespec_print(expr->cast.typespec);
printf(")");
expr_print(expr->cast.expr);
printf(")");
} break;
case EK_Index:{
expr_print(expr->index.atom);
printf("[");
expr_print(expr->index.index);
printf("]");
}break;
case EK_Call:{
expr_print(expr->call.atom);
printf("(");
for(Expr_Compound_Field *n = expr->call.first; n; n=n->next){
expr_compound_print(n);
if(n!=expr->call.last) printf(",");
}
printf(")");
}break;
default: {invalid_codepath;} break;
}
}
function B32
typespec_print(Typespec *spec){
switch(spec->kind) {
case TS_Name: {
printf("%s", spec->name.s.str);
} break;
case TS_NamedArgument: {
printf("%s: ", spec->named.name.s.str);
typespec_print(spec->named.base);
}break;
case TS_Pointer: {
typespec_print(spec->base);
printf("*");
} break;
case TS_Array: {
typespec_print(spec->arr.base);
printf("[");
expr_print(spec->arr.size);
printf("]");
} break;
case TS_Function: {
printf("(");
for(Typespec *n = spec->func.first; n; n=n->next){
typespec_print(n);
if(n!=spec->func.last)
printf(", ");
}
printf(")");
typespec_print(spec->func.ret);
} break;
default: {invalid_codepath;} break;
}
return true;
}
function void
print_assign_expr(Expr *expr){
if(expr){
printf(" = ");
expr_print(expr);
}
}
function void
ast_print(AST *in){
//print_notes(p, node->first_note);
switch(in->kind) {
case AST_Program: {
AST_Parent *node = (AST_Parent *)in;
for(AST *n = node->first; n; n=n->next){
ast_print(n);
printf("\n");
}
} break;
case AST_Decl_Const:
case AST_Decl_Var:{
Decl_Var *node = (Decl_Var *)in;
println("%s: ", node->name.s.str);
if(node->typespec) typespec_print(node->typespec);
print_assign_expr(node->expr);
} break;
case AST_Decl_Typedef:{
Decl_Typedef *node = (Decl_Typedef *)in;
println("typedef %s ", node->name.s.str);
typespec_print(node->typespec);
printf(";");
} break;
case AST_Decl_Func:{
Decl_Func *node = (Decl_Func *)in;
println("");
typespec_print(node->ret);
printf(" %s", node->name.s.str);
printf("(");
AST_Iter(node, arg, Decl_Func_Arg){
printf("%s: ", arg->name.s.str);
typespec_print(arg->typespec);
if((AST *)arg != node->last)
printf(", ");
}
printf(");");
} break;
case AST_Decl_Struct:
case AST_Decl_Union :{
Decl_Struct *node = (Decl_Struct *)in;
const char *struct_name = node->kind==AST_Decl_Struct ? "struct" : "union";
println("%s %s{", struct_name, node->name.s.str?(char*)node->name.s.str:"");
indent++;
for(AST *n = node->first; n; n=n->next){
ast_print(n);
}
indent--;
println("};");
} break;
case AST_Decl_SubStruct:
case AST_Decl_SubUnion :{
Decl_Struct *node = (Decl_Struct *)in;
const char *struct_name = node->kind==AST_Decl_Struct ? "struct" : "union";
println("%s %s{", struct_name, node->name.s.str?(char*)node->name.s.str:"");
indent++;
for(AST *n = node->first; n; n=n->next){
ast_print(n);
}
indent--;
println("};");
} break;
case AST_Decl_Enum:{
Decl_Enum *node = (Decl_Enum *)in;
println("enum %s : ", node->name.s.str);
typespec_print(node->typespec);
printf("{");
indent++;
AST_Iter(node, n, Decl_Enum_Child){
//print_notes(p, n->first_note);
println("%s", n->name.s.str);
print_assign_expr(n->expr);
printf(",");
}
indent--;
println("};");
} break;
default: {invalid_codepath;} break;
}
}

66
old_main.c
View File

@@ -1,66 +0,0 @@
#define _CRT_SECURE_NO_WARNINGS
#include <stdio.h>
#include <windows.h>
#include "lang.h"
#include "os.h"
#include "memory.h"
#include "lex.h"
#include "parser.h"
#include "expr.h"
#include "ast.h"
#include "common.c"
#include "file.c"
#include "memory.c"
#include "parser.c"
#include "os_win32.c"
#include "lex.c"
#include "expr.c"
#include "ast.c"
#include "parse_expr.c"
#include "parse_decl.c"
#include "print.c"
#include "codegen_c.c"
function void
full_test(){
Parser p = {0};
#if 1
parser_init(&p);
String string = os_read_file(lit("test.cc"));
parser_lex_stream(&p, string, lit("Parse"));
Decl *decls = parse(&p);
assert(decls->list.first);
{
gen_begin(&p.scratch, &p);
gen_forward_decl(decls);
gen_decl(decls);
//gen_code(decls);
gen_end();
}
#endif
}
int
main(){
use_write_file("output.cc");
lex_test();
parser_test();
full_test();
Arena arena = {};
Token_Array array = token_array_make(&arena);
token_array_push(&array, &(Token){});
close_all_files();
return 0;
}

18
os_win32.c
View File

@@ -10,7 +10,7 @@ os_read_file(Arena *arena, String name){
result.len = ftell(f);
fseek(f, 0, SEEK_SET);
result.str = arena_push_size(arena, result.len + 1);
result.str = (U8 *)arena_push_size(arena, result.len + 1);
fread(result.str, result.len, 1, f);
fclose(f);
@@ -18,6 +18,22 @@ os_read_file(Arena *arena, String name){
return result;
}
function SizeU
get_align_offset(SizeU size, SizeU align){
SizeU mask = align - 1;
SizeU val = size & mask;
if(val){
val = align - val;
}
return val;
}
function SizeU
align_up(SizeU size, SizeU align){
SizeU result = size + get_align_offset(size, align);
return result;
}
function OS_Memory
os_reserve(SizeU size){
OS_Memory result = {0};

293
new_parse.c → parse.c
View File

@@ -14,6 +14,21 @@ typedef struct Parser{
Parser_Error *last_error;
}Parser;
__thread Parser *global_parser;
__thread Arena global_scratch;
__thread Arena_Checkpoint global_scratch_checkpoint;
function Arena *
arena_begin_scratch(){
global_scratch_checkpoint = arena_checkpoint(&global_scratch);
return &global_scratch;
}
function void
arena_end_scratch(){
arena_restore(global_scratch_checkpoint);
}
function void
parser_push_error(Parser *p, Token *token, char *str, ...){
String string;
@@ -30,8 +45,9 @@ parser_push_error(Parser *p, Token *token, char *str, ...){
}
// @Note(Krzosa): Print nice error message
{
printf("\nError: %s %s:%d\n", string.str, token->file.str, (S32)token->line);
printf("\nError: %s", string.str);
if(token){
printf(" %s:%d\n", token->file.str, (S32)token->line);
// @Note(Krzosa): Print error line
{
@@ -458,6 +474,19 @@ Examples:
(CoolType: optional, S32) - Implicit void return value
*/
function Typespec *
parse_optional_type(Parser *p, Intern_String type){
Typespec *result = 0;
if(token_match(p, TK_Colon)){
result = parse_typespec(p);
}
else{
result = typespec_name(p->arena, token_get(p), type);
}
return result;
}
function Typespec *
parse_typespec_function(Parser *p, Token *token){
Typespec *result = typespec_function(p->arena, token, 0);
@@ -485,13 +514,8 @@ parse_typespec_function(Parser *p, Token *token){
}
}
token_expect(p, TK_CloseParen);
if(token_is(p, TK_Identifier)
|| token_is(p, TK_OpenParen)
|| token_is(p, TK_Mul)
|| token_is(p, TK_OpenBracket))
result->func.ret = parse_typespec(p);
else
result->func.ret = typespec_name(p->arena, token_get(p), intern_void);
result->func.ret = parse_optional_type(p, intern_void);
return result;
}
@@ -587,9 +611,13 @@ parse__notes(Parser *p, AST_Parent *result) {
function Note_List *
parse_notes(Parser *p){
Note_List *result = note_list(p->arena, token_get(p));
parse__notes(p, result);
return result;
Note_List result = (Note_List){.pos=token_get(p), .kind=AST_Note_List};
parse__notes(p, &result);
if(result.first){
Note_List *ast = (Note_List *)ast_shallow_copy(p->arena, (AST *)&result);
return ast;
}
return 0;
}
function Expr *
@@ -599,18 +627,6 @@ parse_assign_expr(Parser *p){
return 0;
}
function Typespec *
parse_optional_type(Parser *p, Intern_String type){
Typespec *result = 0;
if(token_match(p, TK_Colon)){
result = parse_typespec(p);
}
else{
result = typespec_name(p->arena, token_get(p), type);
}
return result;
}
function Decl_Enum *
parse_enum(Parser *p, Token *name){
Typespec *typespec = parse_optional_type(p, intern_int);
@@ -714,6 +730,7 @@ parse_const(Parser *p, Token *name){
return result;
}
function void parse_stmt_block(Parser *p, AST_Parent *parent);
function Decl_Func *
parse_function(Parser *p, Token *name){
Decl_Func *result = decl_function(p->arena, name, name->intern_val, 0);
@@ -727,11 +744,17 @@ parse_function(Parser *p, Token *name){
}
token_expect(p, TK_CloseParen);
result->ret = parse_optional_type(p, intern_void);
token_expect(p, TK_Semicolon);
if(token_is(p, TK_OpenBrace)){
result->body = stmt_block(p->arena, token_get(p));
parse_stmt_block(p, (AST_Parent *)result->body);
}
else if(token_match(p, TK_Semicolon)){
result->is_incomplete = true;
}
return result;
}
function B32
function AST *
parse_decl(Parser *p, AST_Parent *parent){
Token *name = 0;
AST *result = 0;
@@ -772,26 +795,15 @@ parse_decl(Parser *p, AST_Parent *parent){
if(result){
result->notes = note;
ast_push_last(parent, result);
return true;
return result;
}
return false;
}
function Program *
parse_decls(Parser *p){
Program *decl_list = ast_program(p->arena, token_get(p));
while(!token_is(p, TK_End)){
B32 success = parse_decl(p, decl_list);
if(!success){
parser_push_error(p, token_get(p), "Failed to parse decls, unexpected token!");
}
}
return decl_list;
return 0;
}
//-----------------------------------------------------------------------------
// Statement parsing
//-----------------------------------------------------------------------------
function AST *parse_stmt(Parser *p);
/*
stmt_list = '{' stmt* '}'
stmt =
@@ -805,28 +817,163 @@ stmt =
*/
function AST_Parent *
parse_stmt(Parser *p){
Note_List *notes = parse_notes(p);
AST_Parent *result = 0;
if(token_match_keyword(p, keyword_if)){
function void
parse_stmt_block(Parser *p, AST_Parent *parent){
token_expect(p, TK_OpenBrace);
while(!token_is(p, TK_End) && !token_is(p, TK_CloseBrace)){
AST *stmt = parse_stmt(p);
ast_push_last(parent, stmt);
}
else if(token_match_keyword(p, keyword_for)){
token_expect(p, TK_CloseBrace);
}
else if(token_match_keyword(p, keyword_while)){
function AST *
parse_stmt_if(Parser *p){
Expr *expr = parse_expr(p);
Stmt_If *result = stmt_if(p->arena, token_get(p), expr);
parse_stmt_block(p, (AST_Parent *)result->body);
Token *token = 0;
while((token = token_match_keyword(p, keyword_else))){
if(!token_match_keyword(p, keyword_if)){
Stmt_Else *stmt_else = stmt_else_push(p->arena, result, token);
parse_stmt_block(p, (AST_Parent *)stmt_else->body);
break;
}
Expr *expr = parse_expr(p);
Stmt_ElseIf *stmt_else_if = stmt_else_if_push(p->arena, result, token, expr);
parse_stmt_block(p, (AST_Parent *)stmt_else_if->body);
}
return (AST *)result;
}
else if(token_match_keyword(p, keyword_return)){
function Stmt_Init *
parse_stmt_init(Parser *p, Expr *left){
Stmt_Init *result = 0;
if(token_match(p, TK_ColonAssign)){
if(left->kind != EK_Identifier){
parser_push_error(p, token_get(p), "Expected an identifier before ':='");
}
else if(token_match(p, TK_OpenBrace)){
// Scope
Expr *expr = parse_expr(p);
result = stmt_init(p->arena, left->token, left->token->intern_val, 0, expr);
}
else if(token_match(p, TK_Colon)){
if(left->kind != EK_Identifier){
parser_push_error(p, token_get(p), "Expected an identifier before ':'");
}
Typespec *typespec = parse_typespec(p);
Expr *expr = parse_assign_expr(p);
result = stmt_init(p->arena, left->token, left->token->intern_val, typespec, expr);
}
return result;
}
function Token *
token_is_assignment(Parser *p){
Token *t = token_get(p);
if(t->kind >= TK_FirstAssign && t->kind <= TK_LastAssign)
return t;
return 0;
}
function AST *
parse_simple_stmt(Parser *p){
Token *token = token_get(p);
Expr *left = parse_expr(p);
Stmt_Init *init = parse_stmt_init(p, left);
if(init){
return (AST *)init;
}
if(token_is_assignment(p)){
token = token_next(p);
Expr *right = parse_expr(p);
Stmt_Assign *result = stmt_assign(p->arena, token, token->kind, left, right);
return (AST *)result;
}
else{
// Expr
Stmt_Expr *result = stmt_expr(p->arena, token, left);
return (AST *)result;
}
}
function AST *
parse_stmt_return(Parser *p){
Token *token = token_get(p);
Expr *expr = parse_expr(p);
token_expect(p, TK_Semicolon);
Stmt_Return *result = stmt_return(p->arena, token, expr);
return (AST *)result;
}
function AST *
parse_stmt_defer(Parser *p){
Token *token = token_get(p);
Stmt_Defer *result = stmt_defer(p->arena, token);
parse_stmt_block(p, result->body);
return (AST *)result;
}
function AST *
parse_stmt_for(Parser *p){
Token *token = token_get(p);
AST *on_begin = 0;
AST *on_iter = 0;
Expr*condition = 0;
if(!token_is(p, TK_OpenBrace)){
if(!token_is(p, TK_Semicolon)){
on_begin = parse_simple_stmt(p);
}
if(token_match(p, TK_Semicolon)){
if(!token_is(p, TK_Semicolon)){
condition = parse_expr(p);
}
if(token_match(p, TK_Semicolon)){
if(!token_is(p, TK_OpenBrace)){
on_iter = parse_simple_stmt(p);
if(on_iter->kind == AST_Stmt_Init){
parser_push_error(p, token_get(p), "Init statements are not allowed in for on_iter");
}
}
}
}
}
Stmt_For *result = stmt_for(p->arena, token, token_get(p), on_begin, condition, on_iter);
parse_stmt_block(p, (AST_Parent *)result->body);
return (AST*)result;
}
function AST *
parse_stmt(Parser *p){
Note_List *notes = parse_notes(p);
AST *result = 0;
if(token_match_keyword(p, keyword_if)){
result = (AST *)parse_stmt_if(p);
}
else if(token_match_keyword(p, keyword_for)){
result = parse_stmt_for(p);
}
else if(token_match_keyword(p, keyword_while)){
not_implemented;
}
else if(token_match_keyword(p, keyword_defer)){
result = parse_stmt_defer(p);
}
else if(token_match_keyword(p, keyword_return)){
result = parse_stmt_return(p);
}
else if(token_is(p, TK_OpenBrace)){
Stmt_Block *block = stmt_block(p->arena, token_get(p));
parse_stmt_block(p, block);
result = (AST *)block;
}
else{
result = parse_simple_stmt(p);
token_expect(p, TK_Semicolon);
}
if(result){
result->notes = notes;
}
return result;
}
@@ -889,6 +1036,7 @@ parser_make(Arena *arena){
.tokens = lex_make_token_array(arena),
.arena = arena,
};
global_parser = &result;
return result;
}
@@ -909,6 +1057,19 @@ parser_add_stream(Parser *p, String string, String file){
lex_add_stream(&p->tokens, string, file);
}
function Program *
test_parse_decls(Parser *p){
Program *decl_list = ast_program(p->arena, token_get(p));
while(!token_is(p, TK_End)){
AST *success = parse_decl(p, decl_list);
if(!success){
parser_push_error(p, token_get(p), "Failed to parse decls, unexpected token!");
break;
}
}
return decl_list;
}
function void
parse_test_expr(){
Arena *scratch = arena_begin_scratch();
@@ -947,7 +1108,7 @@ parse_test_expr(){
}
String exprs[] = {
lit("cast([12](thing: U32, qwe: *U32) [32]Result, (123+234))"),
lit("cast([12](thing: U32, qwe: *U32): [32]Result, (123+234))"),
lit("cast((thing: U32, qwe: *U32), (123+234))"),
lit("(:(U32,U32)){Thing=10}"),
lit("--Not_Thing[156](Thing) + test_func(asd=func1, af=func2, gg=func3)"),
@@ -977,7 +1138,7 @@ parse_test_decls(){
};
for(SizeU i = 0; i < buff_cap(decls); i++){
parser_restream(&p, decls[i], lit("Decl_Test"));
Program *decl = parse_decls(&p);
Program *decl = test_parse_decls(&p);
assert(decl->first);
ast_print((AST *)decl);
}
@@ -990,14 +1151,34 @@ parse_test_from_file(){
Arena *scratch = arena_begin_scratch();
String file = os_read_file(scratch, FILENAME);
Parser p = parser_make_stream(scratch, file, FILENAME);
Program *d = parse_decls(&p);
Program *d = test_parse_decls(&p);
ast_print((AST *)d);
arena_end_scratch();
}
function void
parse_test_stmt(){
Arena *scratch = arena_begin_scratch();
Parser p = parser_make(scratch);
String stmts[] = {
lit("if thing > 10 { thing++; }"),
lit("thing := 23;"),
lit("thing+=245;"),
lit("thing++;"),
lit("return thing;"),
};
for(SizeU i = 0; i < buff_cap(stmts); i++){
parser_restream(&p, stmts[i], lit("Stmt_Test"));
AST *stmt = parse_stmt(&p);
ast_print(stmt);
}
arena_end_scratch();
}
function void
parse_test(){
parse_test_expr();
parse_test_decls();
parse_test_from_file();
parse_test_stmt();
}

393
parse_decl.c
View File

@@ -1,393 +0,0 @@
function Decl *parse_decl(Parser *p);
function Decl *parse_struct(Parser *p, Token *name, Decl_Kind kind, Decl_Struct_Kind);
function Typespec *parse_type(Parser *p);
function Typespec *
parse_type_function(Parser *p, Token *token){
Typespec *result = typespec_function(p, token, 0);
if(!token_is(p, TK_CloseParen))
for(;;) {
// Optional name
if(token_is(p, TK_Identifier)){
if(token_peek_is(p, 1, TK_Colon)){
token_next(p);
token_next(p);
}
}
// Parse type
if(token_is(p, TK_Identifier)){
Typespec *arg = parse_type(p);
typespec_function_push(result, arg);
}
else if(!token_match(p, TK_Comma)){
break;
}
}
token_expect(p, TK_CloseParen);
if(token_is(p, TK_Identifier))
result->function_spec.ret = parse_type(p);
else
result->function_spec.ret = typespec_name(p, token_get(p), intern_void);
return result;
}
function Typespec *
parse_type(Parser *p){
// Parse as function type or normal
Token *token = 0;
Typespec *result = 0;
if((token = token_match(p, TK_Identifier))){
result = typespec_name(p, token, token->intern_val);
}
else if((token = token_match(p, TK_OpenParen))){
result = parse_type_function(p, token);
}
else{
parser_push_error(p, token, "Failed to parse type, unexpected token");
return 0;
}
// Parse Pointer/Array
for(;;){
if((token = token_match(p, TK_Mul))){
result = typespec_pointer(p, token, result);
}
else if((token = token_match(p, TK_OpenBracket))){
Expr *expr = parse_expr(p);
result = typespec_array(p, token, result, expr);
token_expect(p, TK_CloseBracket);
}
else if(token_match(p, TK_At)){
break;
}
else break;
}
return result;
}
function Expr *
parse_expr_assignment(Parser *p){
Expr *result = 0;
if(token_match(p, TK_Assign)){
result = parse_expr(p);
}
return result;
}
function void
parse_note_list(Parser *p, Note *parent) {
if(token_match(p, TK_OpenParen)) {
if(token_match(p, TK_CloseParen)){
return;
}
do {
Token *name = token_expect(p, TK_Identifier);
Note *current = note_push_new(p, parent, name, name->intern_val, 0);
parse_note_list(p, current);
if(token_match(p, TK_Assign)) {
current->expr = parse_expr(p);
}
} while(token_match(p, TK_Comma));
token_expect(p, TK_CloseParen);
}
}
function void
parse__notes(Parser *p, Note *result) {
while(token_match(p, TK_At)) {
Token *name = token_expect(p, TK_Identifier);
Note *current = note_push_new(p, result, name, name->intern_val, 0);
parse_note_list(p, current);
if(token_match(p, TK_Assign)) {
current->expr = parse_expr(p);
}
token_match(p, TK_Semicolon);
}
}
function Note
parse_notes(Parser *p){
Note result = {0};
parse__notes(p, &result);
return result;
}
function Token *
parse_get_token_name(Parser *p, S32 count){
Token *result = token_next(p);
for(S32 i = 0; i < count; i++)
token_next(p);
return result;
}
function Decl *
parse_enum(Parser *p, Token *name){
Typespec *type = 0;
if(token_match(p, TK_Colon)) type = parse_type(p);
else type = typespec_name(p, token_get(p), intern_s64);
Decl *result = decl_enum(p, name, name->intern_val, type);
token_expect(p, TK_OpenBrace);
do{
Note notes = parse_notes(p);
Token *token = token_match(p, TK_Identifier);
if(token){
Expr *expr = parse_expr_assignment(p);
decl_enum_push(p, result, token, token->intern_val, expr, &notes);
} else break;
} while(token_match(p, TK_Comma));
token_expect(p, TK_CloseBrace);
return result;
}
function Decl *
parse_struct(Parser *p, Token *name, Decl_Kind kind, Decl_Struct_Kind struct_kind){
Decl *result = decl_struct(p, kind, name, name->intern_val, struct_kind);
token_expect(p, TK_OpenBrace);
while(!token_is(p, TK_CloseBrace)){
Decl *decl = 0;
if((decl = parse_decl(p))){
}
else if(token_match_keyword(p, keyword_union)){
decl = parse_struct(p, token_get(p), DECL_Union, STRUCT_Nested);
}
else if(token_match_keyword(p, keyword_struct)){
decl = parse_struct(p, token_get(p), DECL_Struct, STRUCT_Nested);
}
else if(token_is(p, TK_Identifier) &&
token_peek_is(p, 1, TK_Colon) &&
token_peek_is_keyword(p, 2, keyword_union)){
decl = parse_struct(p, parse_get_token_name(p, 2), DECL_Union, STRUCT_Nested);
}
else if(token_is(p, TK_Identifier) &&
token_peek_is(p, 1, TK_Colon) &&
token_peek_is_keyword(p, 2, keyword_struct)){
decl = parse_struct(p, parse_get_token_name(p, 2), DECL_Struct, STRUCT_Nested);
}
else {
parser_push_error(p, token_get(p), "Unexpected token while parsing struct");
break;
}
if(decl) decl_struct_push(result, decl);
}
token_expect(p, TK_CloseBrace);
return result;
}
function Decl *
parse_variable(Parser *p, Token *name){
Typespec *type = parse_type(p);
Expr *expr = parse_expr_assignment(p);
return decl_variable(p, name, name->intern_val, type, expr);
}
function Decl *
parse_typedef(Parser *p, Token *name){
Typespec *type = parse_type(p);
return decl_typedef(p, name, name->intern_val, type);
}
function Stmt *
parse_stmt(Parser *p);
function Stmt *
parse_stmt_list(Parser *p){
Token *token = token_expect(p, TK_OpenBrace);
Stmt *result = stmt_list(p, token);
while(!token_match(p, TK_CloseBrace)) {
Stmt *stmt = parse_stmt(p);
stmt_push(result, stmt);
}
return result;
}
function Stmt *
parse_stmt(Parser *p){
Token *token = token_get(p);
Decl *decl = parse_decl(p);
if(decl){
Stmt *result = stmt_decl(p, token, decl);
return result;
}
else if(token_match_keyword(p, keyword_return)){
Expr *expr = parse_expr(p);
Stmt *result = stmt_return(p, token, expr);
token_expect(p, TK_Semicolon);
return result;
}
else if(token_match_keyword(p, keyword_if)){
Expr *expr = parse_expr(p);
Stmt *if_body = parse_stmt_list(p);
Stmt *result = stmt_if(p, token, if_body, expr);
Stmt *head = result;
while(token_match_keyword(p, keyword_else)){
if(token_match_keyword(p, keyword_if)){
expr = parse_expr(p);
if_body = parse_stmt_list(p);
head = head->next = stmt_if(p, token, if_body, expr);
}
else{
if_body = parse_stmt_list(p);
head = head->next = stmt_if(p, token, if_body, 0);
break;
}
}
return result;
}
else if((token_is(p, TK_OpenBrace))){
Stmt *result = parse_stmt_list(p);
return result;
}
else{
Expr *expr = parse_expr(p);
token_expect(p, TK_Semicolon);
return stmt_expr(p, token, expr);
}
}
function Decl *
parse_function(Parser *p, Token *name){
Decl *result = decl_function(p, name, name->intern_val, 0);
if(!token_is(p, TK_CloseParen)){
for(;;) {
if(token_peek_is(p, 1, TK_Colon)){
if(token_peek_is(p, 2, TK_Identifier) ||
token_peek_is(p, 2, TK_OpenParen)){
Token *name = parse_get_token_name(p,1);
Typespec *type = parse_type(p);
decl_function_push(p, result, name, name->intern_val, type);
}
}
else if(!token_match(p, TK_Comma))
break;
}
}
token_expect(p, TK_CloseParen);
if(token_is(p, TK_Identifier))
result->function_decl.ret = parse_type(p);
else
result->function_decl.ret = typespec_name(p, token_get(p), intern_void);
result->function_decl.body = parse_stmt_list(p);
return result;
}
function Decl *
parse_decl(Parser *p){
Decl *result = 0;
Note notes = parse_notes(p);
if(token_is(p, TK_Identifier)){
if(token_peek_is(p, 1, TK_DoubleColon)){
if(token_peek_is_keyword(p, 2, keyword_struct)){
result = parse_struct(p, parse_get_token_name(p,2), DECL_Struct, STRUCT_Base);
}
else if(token_peek_is_keyword(p, 2, keyword_union)){
result = parse_struct(p, parse_get_token_name(p,2), DECL_Union, STRUCT_Base);
}
else if(token_peek_is_keyword(p, 2, keyword_enum)){
result = parse_enum(p, parse_get_token_name(p,2));
}
else if(token_peek_is_keyword(p, 2, keyword_typedef)){
result = parse_typedef(p, parse_get_token_name(p,2));
}
else if(token_peek_is(p, 2, TK_OpenParen)){
result = parse_function(p, parse_get_token_name(p,2));
}
}
else if(token_peek_is(p, 1, TK_Colon)){
if(token_peek_is(p, 2, TK_Identifier) ||
token_peek_is(p, 2, TK_OpenParen)){
result = parse_variable(p, parse_get_token_name(p,1));
token_expect(p, TK_Semicolon);
}
}
}
if(result){
decl_pass_notes(result, &notes);
}
else if(notes.first != 0){
parser_push_error(p, token_get(p), "Detected notes that are not attached to anything");
}
return result;
}
function Decl *
parse(Parser *p){
Decl *result = decl_new(p, DECL_List, token_get(p), (Intern_String){0});
for(;;){
Decl *decl = 0;
if(token_is(p, TK_End)) {
break;
}
else if((decl = parse_decl(p))){
// Noop
}
else token_next(p);
if(decl){
decl_list_push(result, decl);
}
}
return result;
}
function void gen_stmt(Stmt *stmt);
function void gen_end();
function void gen_begin(Arena *arena, Parser *p);
function void expr_print(Parser *p, Expr *expr);
function void
parser_test(){
Parser p = {0};
{
parser_init(&p);
Intern_String a = intern_string(&p, lit("Thing"));
Intern_String b = intern_string(&p, lit("Thing"));
assert(a.s.str == b.s.str);
}
String exprs[] = {
lit("(534>43?435:42,234,cast(S64)32/*todo cast*/,Thing[10][2],Thing(1,2))"),
lit("(4+2*53)"),
lit("((4+2)*53)"),
lit("++5"),
lit("5--"), // @Todo(Krzosa):
lit("-5"),
lit("(+5)"),
lit("sizeof(32) + sizeof(:S32*)"),
lit("cast(S64**)5"),
lit("cast(S64)5+3"),
lit("534>43?435:42"),
};
for(S64 i = 0; i < buff_cap(exprs); i++){
parser_lex_stream(&p, exprs[i], lit("File"));
Expr *expr = parse_expr(&p);
assert(expr);
expr_print(&p, expr);
lex_print("\n");
}
String stmts[] = {
lit("Thing :: struct { test: int; } "),
lit("thing: S32 = 100; "),
lit("thing = thing + 10; "),
lit("thing++; "),
lit("{ thing_scoped: S32 = 10; thing_scoped += 10; } "),
};
for(S64 i = 0; i < buff_cap(stmts); i++){
parser_lex_stream(&p, stmts[i], lit("File"));
Stmt *stmt = parse_stmt(&p);
assert(stmt);
gen_begin(&p.scratch, &p);
gen_stmt(stmt);
gen_end();
lex_print("\n");
}
}

285
parse_expr.c
View File

@@ -1,285 +0,0 @@
function Expr* parse_expr(Parser* p);
function Expr* parse_list_expr(Parser* p);
function Typespec *parse_type(Parser *p);
function Expr*
parse_atom_expr(Parser* p){
Expr* result = 0;
if (token_is(p, TK_Identifier) ||
token_is(p, TK_StringLit) ||
token_is(p, TK_U8Lit) ||
token_is(p, TK_Int)){
result = expr_atom(p, token_next(p));
}
else if (token_is_keyword(p, keyword_sizeof)) {
Token *token = token_next(p);
token_expect(p, TK_OpenParen);
Typespec *type = 0;
Expr *expr = 0;
if(token_match(p, TK_Colon)) {
result = expr_sizeof_type(p, token, parse_type(p));
}
else {
result = expr_sizeof_expr(p, token, parse_expr(p));
}
token_expect(p, TK_CloseParen);
}
else if (token_match(p, TK_OpenParen)){
result = parse_list_expr(p);
token_expect(p, TK_CloseParen);
}
else {
parser_push_error(p, token_next(p), "Invalid expression token");
}
return result;
}
function Expr*
parse_postfix_expr(Parser* p){
Expr* result = parse_atom_expr(p);
while (token_is(p, TK_Dot)
|| token_is(p, TK_Arrow)
|| token_is(p, TK_OpenParen)
|| token_is(p, TK_OpenBracket)
|| token_is(p, TK_Decrement)
|| token_is(p, TK_Increment)){
Token *op = token_get(p);
if (token_match(p, TK_Dot)){
Expr* r = parse_atom_expr(p);
result = expr_binary(p, op, result, r);
}
else if (token_match(p, TK_Arrow)){
Expr* r = parse_atom_expr(p);
result = expr_binary(p, op, result, r);
}
else if (token_match(p, TK_OpenParen)){
Expr* list = 0;
if (!token_match(p, TK_CloseParen)){
list = parse_list_expr(p);
token_expect(p, TK_CloseParen);
}
result = expr_call(p, op, result, list);
}
else if (token_match(p, TK_OpenBracket)){
Expr* list = 0;
if (!token_match(p, TK_CloseBracket)){
list = parse_list_expr(p);
token_match(p, TK_CloseBracket);
}
result = expr_index(p, op, result, list);
}
else {
assert(op->kind == TK_Increment || op->kind == TK_Decrement);
token_next(p);
if (op->kind == TK_Increment) op->kind = TK_PostIncrement;
else if (op->kind == TK_Decrement) op->kind = TK_PostDecrement;
result = expr_unary(p, op, result);
}
}
return result;
}
function
Expr* parse_unary_expr(Parser* p) {
Expr* result = 0;
if (token_is(p, TK_Sub)
|| token_is(p, TK_Add)
|| token_is(p, TK_Mul)
|| token_is(p, TK_BitAnd)
|| token_is(p, TK_Not)
|| token_is(p, TK_Neg)
|| token_is(p, TK_Increment)
|| token_is(p, TK_Decrement)) {
Token *op = token_next(p);
result = parse_unary_expr(p);
result = expr_unary(p, op, result);
}
else if (token_is_keyword(p, keyword_cast)) {
Token *token = token_next(p);
token_expect(p, TK_OpenParen);
Typespec *type = parse_type(p);
token_expect(p, TK_CloseParen);
result = parse_unary_expr(p);
result = expr_cast(p, token, type, result);
}
else {
result = parse_postfix_expr(p);
}
return result;
}
function
Expr* parse_mul_expr(Parser* p) {
Expr* result = parse_unary_expr(p);
while (token_is(p, TK_Mul)
|| token_is(p, TK_Div)
|| token_is(p, TK_Mod)) {
Token *op = token_next(p);
Expr* r = parse_unary_expr(p);
result = expr_binary(p, op, result, r);
}
return result;
}
function
Expr* parse_add_expr(Parser* p) {
Expr* result = parse_mul_expr(p);
while (token_is(p, TK_Add)
|| token_is(p, TK_Sub)) {
Token *op = token_next(p);
Expr* right = parse_mul_expr(p);
result = expr_binary(p, op, result, right);
}
return result;
}
function
Expr* parse_shift_expr(Parser* p) {
Expr* result = parse_add_expr(p);
while (token_is(p, TK_RightShift)
|| token_is(p, TK_LeftShift)) {
Token *op = token_next(p);
Expr* r = parse_add_expr(p);
result = expr_binary(p, op, result, r);
}
return result;
}
function
Expr* parse_compare_expr(Parser* p) {
Expr* result = parse_shift_expr(p);
while (token_is(p, TK_LesserThen)
|| token_is(p, TK_GreaterThen)
|| token_is(p, TK_LesserThenOrEqual)
|| token_is(p, TK_GreaterThenOrEqual)) {
Token *op = token_next(p);
Expr* r = parse_shift_expr(p);
result = expr_binary(p, op, result, r);
}
return result;
}
function
Expr* parse_equality_expr(Parser* p) {
Expr* result = parse_compare_expr(p);
while (token_is(p, TK_Equals)
|| token_is(p, TK_NotEquals)) {
Token *op = token_next(p);
Expr* r = parse_compare_expr(p);
result = expr_binary(p, op, result, r);
}
return result;
}
function
Expr* parse_bit_and_expr(Parser* p) {
Expr* result = parse_equality_expr(p);
while (token_is(p, TK_BitAnd)) {
Token *op = token_next(p);
Expr* r = parse_equality_expr(p);
result = expr_binary(p, op, result, r);
}
return result;
}
function
Expr* parse_bit_xor_expr(Parser* p) {
Expr* result = parse_bit_and_expr(p);
while (token_is(p, TK_BitXor)) {
Token *op = token_next(p);
Expr* r = parse_bit_and_expr(p);
result = expr_binary(p, op, result, r);
}
return result;
}
function
Expr* parse_bit_or_expr(Parser* p) {
Expr* result = parse_bit_xor_expr(p);
while (token_is(p, TK_BitOr)) {
Token *op = token_next(p);
Expr* r = parse_bit_xor_expr(p);
result = expr_binary(p, op, result, r);
}
return result;
}
function
Expr* parse_and_expr(Parser* p) {
Expr* result = parse_bit_or_expr(p);
while (token_is(p, TK_And)) {
Token *op = token_next(p);
Expr* r = parse_bit_or_expr(p);
result = expr_binary(p, op, result, r);
}
return result;
}
function
Expr* parse_or_expr(Parser* p) {
Expr* result = parse_and_expr(p);
while (token_is(p, TK_Or)) {
Token *op = token_next(p);
Expr* r = parse_and_expr(p);
result = expr_binary(p, op, result, r);
}
return result;
}
function
Expr* parse_ternary_expr(Parser* p) {
Expr* result = parse_or_expr(p);
if (token_is(p, TK_Question)) {
Token *token = token_next(p);
Expr* on_true = parse_ternary_expr(p);
token_expect(p, TK_Colon);
Expr* on_false = parse_ternary_expr(p);
result = expr_ternary(p, token, result, on_true, on_false);
}
return result;
}
function
Expr* parse_assign_expr(Parser* p) {
Expr* result = parse_ternary_expr(p);
if (token_is_assignment(p)) {
Token *op = token_next(p);
Expr* right = parse_assign_expr(p);
result = expr_binary(p, op, result, right);
}
return result;
}
function
Expr* parse_list_expr(Parser* p) {
Expr* result = parse_assign_expr(p);
if (token_is(p, TK_Comma)) {
Expr *list = expr_list(p, token_get(p));
expr_list_push(list, result);
result = list;
}
while (token_is(p, TK_Comma)) {
Token *token = token_next(p);
Expr* expr = parse_assign_expr(p);
expr_list_push(result, expr);
}
return result;
}
function Expr*
parse_expr(Parser* p) {
return parse_assign_expr(p);
}

173
parser.c
View File

@@ -1,173 +0,0 @@
global Intern_String intern_empty;
global Intern_String intern_s64;
global Intern_String intern_s32;
global Intern_String intern_s16;
global Intern_String intern_s8;
global Intern_String intern_u64;
global Intern_String intern_u32;
global Intern_String intern_u16;
global Intern_String intern_u8;
global Intern_String intern_void;
global Intern_String intern_sizeu;
global Intern_String keyword_sizeof;
global Intern_String keyword_cast;
global Intern_String keyword_enum;
global Intern_String keyword_for;
global Intern_String keyword_if;
global Intern_String keyword_else;
global Intern_String keyword_return;
global Intern_String keyword_typedef;
global Intern_String keyword_struct;
global Intern_String keyword_union;
global Intern_String keyword_function;
global Intern_String keyword_global;
function void
parser_init(Parser *p){
p->interns_count = 4096*2;
p->interns = arena_push_array(&p->intern_table_arena, Intern_String, p->interns_count);
keyword_sizeof = intern_string(p, lit("sizeof"));
keyword_if = intern_string(p, lit("if"));
keyword_else = intern_string(p, lit("else"));
keyword_return = intern_string(p, lit("return"));
keyword_for = intern_string(p, lit("for"));
keyword_cast = intern_string(p, lit("cast"));
keyword_struct = intern_string(p, lit("struct"));
keyword_enum = intern_string(p, lit("enum"));
keyword_typedef = intern_string(p, lit("typedef"));
keyword_union = intern_string(p, lit("union"));
keyword_function = intern_string(p, lit("function"));
keyword_global = intern_string(p, lit("global"));
p->first_keyword = keyword_sizeof.s.str;
p->last_keyword = keyword_global.s.str;
intern_s64 = intern_string(p, lit("S64"));
intern_s32 = intern_string(p, lit("S32"));
intern_s16 = intern_string(p, lit("S16"));
intern_s8 = intern_string(p, lit("S8"));
intern_u64 = intern_string(p, lit("U64"));
intern_u32 = intern_string(p, lit("U32"));
intern_u16 = intern_string(p, lit("U16"));
intern_u8 = intern_string(p, lit("U8"));
intern_void = intern_string(p, lit("void"));
intern_sizeu = intern_string(p, lit("SizeU"));
}
function B32
intern_is_keyword(Parser *p, Intern_String intern){
if(intern.s.str >= p->first_keyword && intern.s.str <= p->last_keyword)
return true;
return false;
}
function void
parser_push_error(Parser *p, Token *token, char *str, ...){
String string;
{
va_list args1, args2;
va_start(args1, str);
va_copy(args2, args1);
string.len = vsnprintf(0, 0, str, args2);
va_end(args2);
string.str = arena_push_size(&p->main_arena, string.len + 1);
vsnprintf((char*)string.str, string.len + 1, str, args1);
va_end(args1);
}
printf("Error: %s %s:%d\n", string.str, token->file.str, (S32)token->line);
Parser_Error *error = arena_push_struct(&p->main_arena, Parser_Error);
error->message = string;
error->next = 0;
error->token = token;
SLLQueuePush(p->first_error, p->last_error, error);
__debugbreak();
exit(-1);
}
function void
intern_tokens(Parser *p){
for(S64 i = 0; i < p->tokens.len; i++){
Token *t = p->tokens.tokens + i;
if(t->kind == TK_Identifier){
t->intern_val = intern_string(p, t->string);
if(intern_is_keyword(p, t->intern_val)){
t->kind = TK_Keyword;
}
}
else if(t->kind == TK_U8Lit){
t->intern_val = intern_string(p, t->string);
}
else if(t->kind == TK_Error){
parser_push_error(p, t, (char *)t->error_val.str);
}
}
}
//-----------------------------------------------------------------------------
//
//-----------------------------------------------------------------------------
typedef struct Table_Index{
U64 hash;
U64 index;
U64 iter;
U64 max_size;
}Table_Index;
function Table_Index
table_index_from_hash(U64 hash, U64 max_size){
Table_Index result = {0};
result.hash = hash;
result.index = result.hash % max_size;
result.iter = result.index;
result.max_size = max_size;
return result;
}
function Table_Index
table_index_from_string(String string, U64 max_size){
U64 hash = hash_fnv(string);
Table_Index result = table_index_from_hash(hash, max_size);
return result;
}
function B32
table_index_advance(Table_Index *index){
index->iter = wrap_around_pow2(index->iter + 1, index->max_size);
B32 result = index->iter == index->index;
return result;
}
function Intern_String
intern_string(Parser *p, String string){
Intern_String result = {0};
Table_Index index = table_index_from_string(string, p->interns_count);
for(;;){
Intern_String *intern = p->interns + index.iter;
if(intern->s.str == 0){
result.s = arena_push_string_copy(&p->main_arena, string);
p->interns_in_bytes += string.len;
p->interns_inserted += 1;
*intern = result;
break;
}
else if(string_compare(intern->s, string)){
result = *intern;
break;
}
if (table_index_advance(&index))
break;
}
return result;
}

33
parser.h
View File

@@ -1,33 +0,0 @@
#pragma once
typedef struct Expr Expr;
typedef struct Parser_Error Parser_Error;
struct Parser_Error{
Parser_Error *next;
String message;
Token *token;
};
typedef struct Parser{
Arena main_arena;
Arena intern_table_arena;
Arena symbol_table_arena;
Arena scratch;
S64 interns_in_bytes;
S64 interns_inserted;
Intern_String *interns;
S64 interns_count;
U8 *first_keyword;
U8 *last_keyword;
Parser_Error *first_error;
Parser_Error *last_error;
Tokens tokens;
}Parser;
function Intern_String intern_string(Parser *p, String string);
function Token *token_get(Parser *p);
function B32 intern_compare(Intern_String a, Intern_String b);

428
print.c
View File

@@ -1,212 +1,368 @@
function void print_decl(Parser *p, Decl *node);
function B32 print_typespec(Parser *p, Typespec *spec);
function void expr_print(Expr *expr);
function void typespec_print(Typespec *spec);
global S64 indent;
#define println(...) do{ printf("\n"); print_indent(); printf(__VA_ARGS__); }while(0)
#define AST_CAST(item, T) T *item = (T *)item
#define AST_IterT(parent, name, T) for(T *name = (T *)(parent)->first; name; name=(T *)name->next) // , assert(name->kind == AST_##type
#define AST_Iter(parent,name) AST_IterT(parent,name,AST)
function void
tokens_print(Tokens tokens){
lex_print("\n== Token count = %d\n", (S32)tokens.len);
for(Token *t = tokens.tokens; t != tokens.tokens + tokens.len; t++){
lex_print("%s \"%.*s\"\n", token_kind_string[t->kind].str, (S32)t->len, t->str);
}
print_indent(){
for(S64 i = 0; i < indent*2; i++)
printf(" ");
}
function void
token_print(Token *token){
lex_print("%.*s", (S32)token->len, token->str);
printf("%.*s", (S32)token->len, token->str);
}
function void
expr_print(Parser *p, Expr *expr){
expr_compound_print(Expr_Compound_Field *field){
switch(field->kind){
case COMPOUND_Default: {
expr_print(field->init);
}break;
case COMPOUND_Named: {
printf(".%s = ", field->name.s.str);
expr_print(field->init);
}break;
case COMPOUND_Index: {
printf("[");
expr_print(field->index);
printf("] = ");
expr_print(field->init);
}break;
default: invalid_codepath;
}
}
function void
expr_print(Expr *expr){
switch(expr->kind) {
case EK_Atom: {
case EK_Int:case EK_String:case EK_Identifier: {
token_print(expr->token);
} break;
case EK_Sizeof:{
lex_print("sizeof(");
if(expr->size_of.kind == SIZEOF_Expr){
expr_print(p, expr->size_of.expr);
case EK_SizeExpr:{
printf("sizeof(");
expr_print(expr->size_expr.expr);
printf(")");
}break;
case EK_Compound:{
if(expr->compound.typespec){
printf("(");
typespec_print(expr->compound.typespec);
printf(")");
}
else{
assert(expr->size_of.kind == SIZEOF_Type);
print_typespec(p, expr->size_of.type);
printf("{");
for(Expr_Compound_Field *n = expr->compound.first; n; n=n->next){
expr_compound_print(n);
if(n!=expr->compound.last) printf(",");
}
lex_print(")");
printf("}");
} break;
case EK_SizeType:{
printf("sizeof(");
typespec_print(expr->size_type.typespec);
printf(")");
}break;
case EK_Paren:{
printf("(");
expr_print(expr->paren.expr);
printf(")");
} break;
case EK_Field:{
expr_print(expr->field.expr);
printf(".%s", expr->field.name.s.str);
} break;
case EK_Binary:{
lex_print("(");
expr_print(p, expr->binary.left);
token_print(expr->token);
expr_print(p, expr->binary.right);
lex_print(")");
printf("(");
expr_print(expr->binary.left);
printf("%s", token_kind_string[expr->binary.op]);
expr_print(expr->binary.right);
printf(")");
} break;
case EK_Unary:{
lex_print("(");
case EK_PostfixUnary:{
printf("(");
expr_print(expr->unary.expr);
token_print(expr->token);
expr_print(p, expr->unary.expr);
lex_print(")");
printf(")");
} break;
case EK_Unary:{
printf("(");
token_print(expr->token);
expr_print(expr->unary.expr);
printf(")");
} break;
case EK_Ternary:{
lex_print("(");
expr_print(p, expr->ternary.cond);
lex_print("?");
expr_print(p, expr->ternary.on_true);
lex_print(":");
expr_print(p, expr->ternary.on_false);
lex_print(")");
} break;
case EK_List:{
lex_print("(");
for(Expr *n = expr->list.first; n; n=n->next){
expr_print(p, n);
if(n!=expr->list.last) lex_print(",");
}
lex_print(")");
printf("(");
expr_print(expr->ternary.cond);
printf("?");
expr_print(expr->ternary.on_true);
printf(":");
expr_print(expr->ternary.on_false);
printf(")");
} break;
case EK_Cast:{
lex_print("(");
lex_print("(");
print_typespec(p, expr->cast.type);
lex_print(")");
expr_print(p, expr->cast.expr);
lex_print(")");
printf("(");
printf("(");
typespec_print(expr->cast.typespec);
printf(")");
expr_print(expr->cast.expr);
printf(")");
} break;
case EK_Index:{
expr_print(p, expr->index.atom);
lex_print("[");
expr_print(p, expr->index.index);
lex_print("]");
expr_print(expr->index.atom);
printf("[");
expr_print(expr->index.index);
printf("]");
}break;
case EK_Call:{
expr_print(p, expr->call.atom);
lex_print("(");
expr_print(p, expr->call.list);
lex_print(")");
expr_print(expr->call.atom);
printf("(");
for(Expr_Compound_Field *n = expr->call.first; n; n=n->next){
expr_compound_print(n);
if(n!=expr->call.last) printf(",");
}
printf(")");
}break;
default: {invalid_codepath;} break;
}
}
function B32
print_typespec(Parser *p, Typespec *spec){
function void
typespec_print(Typespec *spec){
switch(spec->kind) {
case TS_Name: {
lex_print("%s", spec->name.s.str);
printf("%s", spec->name.s.str);
} break;
case TS_NamedArgument: {
printf("%s: ", spec->named.name.s.str);
typespec_print(spec->named.base);
}break;
case TS_Pointer: {
print_typespec(p, spec->base);
lex_print("*");
typespec_print(spec->base);
printf("*");
} break;
case TS_Array: {
print_typespec(p, spec->array_spec.base);
lex_print("[");
expr_print(p, spec->array_spec.size);
lex_print("]");
typespec_print(spec->arr.base);
printf("[");
expr_print(spec->arr.size);
printf("]");
} break;
case TS_Function: {
lex_print("(");
for(Typespec *n = spec->function_spec.first; n; n=n->next){
print_typespec(p,n);
if(n!=spec->function_spec.last)
lex_print(", ");
printf("(");
for(Typespec *n = spec->func.first; n; n=n->next){
typespec_print(n);
if(n!=spec->func.last)
printf(", ");
}
lex_print(")");
print_typespec(p,spec->function_spec.ret);
printf(")");
typespec_print(spec->func.ret);
} break;
default: {invalid_codepath;} break;
}
return true;
}
function void
print_assign_expr(Parser *p, Expr *expr){
print_assign_expr(Expr *expr){
if(expr){
lex_print(" = ");
expr_print(p, expr);
printf(" = ");
expr_print(expr);
}
}
function void
print_note_list(Parser *p, Note *note){
if(note){
lex_print("(");
for(Note *n = note; n; n=n->next){
lex_print("%s", n->name.s.str);
print_note_list(p,n->first);
print_assign_expr(p, n->expr);
}
lex_print(")");
}
}
function void
print_notes(Parser *p, Note *note){
for(Note *n = note; n; n=n->next){
lex_print("@%s", n->name.s.str);
print_note_list(p,n->first);
print_assign_expr(p,n->expr);
lex_print(" ");
}
}
function void
print_decl(Parser *p, Decl *node){
print_notes(p, node->first_note);
switch(node->kind) {
case DECL_List: {
for(Decl *n = node->list.first; n; n=n->next){
print_decl(p,n);
lex_new_line();
ast_print(AST *in){
switch(in->kind) {
case AST_Program: {
Program *node = (Program *)in;
for(AST *n = node->first; n; n=n->next){
ast_print(n);
printf("\n");
}
} break;
case DECL_Variable:{
lex_print("%s: ", node->name.s.str);
B32 r = print_typespec(p, node->variable_decl.type);
print_assign_expr(p,node->variable_decl.expr);
if(r) lex_print(";");
case AST_Decl_Const:
case AST_Decl_Var:{
Decl_Var *node = (Decl_Var *)in;
println("");
if(node->typespec) typespec_print(node->typespec);
printf(" %s", node->name.s.str);
print_assign_expr(node->expr);
} break;
case DECL_Typedef:{
lex_print("typedef %s ", node->name.s.str);
print_typespec(p, node->typedef_decl.type);
lex_print(";");
case AST_Decl_Typedef:{
Decl_Typedef *node = (Decl_Typedef *)in;
println("typedef %s ", node->name.s.str);
typespec_print(node->typespec);
printf(";");
} break;
case DECL_Struct:
case DECL_Union :{
const char *struct_name = node->kind==DECL_Struct ? "struct" : "union";
lex_print("%s %s{\n", struct_name, node->name.s.str?(char*)node->name.s.str:"");
for(Decl *n = node->struct_decl.first; n; n=n->next){
print_decl(p, n);
lex_print("\n");
case AST_Decl_Func:{
Decl_Func *node = (Decl_Func *)in;
println("function ");
typespec_print(node->ret);
printf("\n%s", node->name.s.str);
printf("(");
AST_IterT(node, arg, Decl_Func_Arg){
typespec_print(arg->typespec);
printf(" %s", arg->name.s.str);
if((AST *)arg != node->last)
printf(", ");
}
lex_print("};\n");
} break;
case DECL_Enum:{
lex_print("enum %s : ", node->name.s.str);
print_typespec(p, node->enum_decl.typespec);
lex_print("{\n");
for(Decl_Enum_Child *n = node->enum_decl.first; n; n=n->next){
print_notes(p, n->first_note);
lex_print("%s", n->name.s.str);
print_assign_expr(p, n->expr);
lex_print(",\n");
printf(")");
if(!node->is_incomplete){
ast_print((AST *)node->body);
}
lex_print("};\n");
} break;
case AST_Decl_Struct:
case AST_Decl_Union :{
Decl_Struct *node = (Decl_Struct *)in;
const char *struct_name = node->kind==AST_Decl_Struct ? "struct" : "union";
println("%s %s{", struct_name, node->name.s.str?(char*)node->name.s.str:"");
indent++;
for(AST *n = node->first; n; n=n->next){
ast_print(n);
printf(";");
}
indent--;
println("};");
} break;
case AST_Decl_SubStruct:
case AST_Decl_SubUnion :{
Decl_Struct *node = (Decl_Struct *)in;
const char *struct_name = node->kind==AST_Decl_Struct ? "struct" : "union";
println("%s %s{", struct_name, node->name.s.str?(char*)node->name.s.str:"");
indent++;
for(AST *n = node->first; n; n=n->next){
ast_print(n);
}
indent--;
println("};");
} break;
case AST_Decl_Enum:{
Decl_Enum *node = (Decl_Enum *)in;
println("enum %s : ", node->name.s.str);
typespec_print(node->typespec);
printf("{");
indent++;
AST_IterT(node, n, Decl_Enum_Child){
//print_notes(p, n->first_note);
println("%s", n->name.s.str);
print_assign_expr(n->expr);
printf(",");
}
indent--;
println("};");
} break;
case AST_Stmt_Init:{
Stmt_Init *node = (Stmt_Init *)in;
if(node->typespec) typespec_print(node->typespec);
printf(" %s", node->name.s.str);
print_assign_expr(node->expr);
}break;
case AST_Stmt_Assign:{
Stmt_Assign *node = (Stmt_Assign *)in;
expr_print(node->left);
token_print(node->pos);
expr_print(node->right);
}break;
case AST_Stmt_Expr:{
Stmt_Expr *node = (Stmt_Expr *)in;
expr_print(node->expr);
}break;
case AST_Stmt_Defer:{
Stmt_Defer *node = (Stmt_Defer *)in;
printf("defer ");
ast_print((AST *)node->body);
}break;
case AST_Stmt_Return:{
Stmt_Return *node = (Stmt_Return *)in;
printf("return ");
expr_print(node->expr);
}break;
case AST_Stmt_Block:{
Stmt_Block *node = (Stmt_Block *)in;
printf(" {");
indent++;
AST_IterT(node, n, AST){
println("");
ast_print(n);
if(n->kind != AST_Stmt_If && n->kind != AST_Stmt_For && n->kind != AST_Stmt_Else &&
n->kind != AST_Stmt_ElseIf)
printf(";");
}
indent--;
println("}");
}break;
case AST_Stmt_Else:{
Stmt_Else *node = (Stmt_Else *)in;
printf("else ");
ast_print((AST *)node->body);
}break;
case AST_Stmt_ElseIf:{
Stmt_ElseIf *node = (Stmt_ElseIf *)in;
printf("else if ");
expr_print(node->condition);
ast_print((AST *)node->body);
}break;
case AST_Stmt_If:{
Stmt_If *node = (Stmt_If *)in;
printf("if ");
expr_print(node->condition);
ast_print((AST *)node->body);
AST_IterT(node, n, AST){
ast_print(n);
}
}break;
case AST_Stmt_For:{
Stmt_For *node = (Stmt_For *)in;
printf("for(");
if(node->on_begin) ast_print(node->on_begin);
printf(";");
if(node->condition) expr_print(node->condition);
printf(";");
if(node->on_iter) ast_print(node->on_iter);
printf(")");
ast_print((AST *)node->body);
}break;
default: {invalid_codepath;} break;
}
}

28
resolve.c Normal file
View File

@@ -0,0 +1,28 @@
typedef struct Sym Sym;
typedef enum Sym_Kind{
Sym_Kind_None,
Sym_Kind_Type,
Sym_Kind_Const,
Sym_Kind_Func,
}Sym_Kind;
typedef enum Sym_State{
Sym_State_NotVisited,
Sym_State_Visited,
};
struct Sym{
Sym_Kind kind;
Sym_State state;
AST *ast;
Type *type;
};
global Sym *global_syms;
function void
resolve_test(){
}

14
scratch.c
View File

@@ -1,14 +0,0 @@
global Arena global_scratch;
global Arena_Checkpoint global_scratch_checkpoint;
function Arena *
arena_begin_scratch(){
global_scratch_checkpoint = arena_checkpoint(&global_scratch);
return &global_scratch;
}
function void
arena_end_scratch(){
arena_restore(global_scratch_checkpoint);
}

34
sym.h
View File

@@ -1,34 +0,0 @@
typedef enum Symbol_Kind{
SYM_None,
SYM_Decl,
SYM_Type,
}Symbol_Kind;
typedef enum Symbol_State{
SYM_STATE_Used,
SYM_STATE_Declared,
}Symbol_State;
typedef struct Symbol{
Symbol_Kind kind;
Symbol_State state;
Symbol *next;
union{
Decl *decl;
struct{
Typespec *spec;
} type;
};
} Symbol;
// First stage would add all symbols as underspecified to the table
// Every encountered typespec would be added to the table as either specified or
// just as encountered name
//
// Then second stage would loop over all of them and report errors on all symbols that were
// found in the wild but not declared
//
//
//
//

32
test.cc
View File

@@ -1,6 +1,5 @@
Thing :: const: *U32 = 0;
CONST_VAL::const = 185210591;
new_function::(Thing:[32]U32): U32;
@test(size = 4096)
@@ -258,17 +257,30 @@ Parser :: struct{
//@using token_array: Tokens;
}
/*
@register_tag(sllqueue)
@params(next=next,last=last,first=first)
function void
struct_type_lower_var_name_lower_push(struct_type *parent, var_type *child){
if(parent->first == 0){
- parent->first = parent->last = child;
function_test::(a:U32, b:U32):*U32{
scratch := get_scratch();
defer{release_scratch(scratch);}
if a > 10 {
c := b + 10;
b = c;
}
else if a == 20 {
d := 1241215;
b = d;
}
else {
- parent->last = parent->last->next = child;
c := b+20;
b = c;
}
for {
c: U32 = b + 1;
}
for i:=0; i < 10; i++ {
print(i);
}
return *b;
}
*/

31
test2.cc Normal file
View File

@@ -0,0 +1,31 @@
Thing :: struct {
arena: *Arena;
}
main :: (argc: int, argv: **char): int {
thing: int = proc(&argc);
arena: Arena;
for i:=0; i < 10; i++ {
}
}
proc :: (thing: *int): *int {
thing = SCRATCH_COUNT + 10;
return thing;
}
Arena :: struct {
cap: int;
len: int;
data: *void;
}
Thread_Ctx :: struct {
arenas: [SCRATCH_COUNT]Arena;
}
SCRATCH_COUNT::const = 3;

157
type.c Normal file
View File

@@ -0,0 +1,157 @@
typedef struct Type Type;
typedef enum Type_Kind{
TYPE_None,
TYPE_Incomplete,
TYPE_Int,
TYPE_Char,
TYPE_Void,
TYPE_Pointer,
TYPE_Array,
TYPE_Func,
TYPE_Struct,
TYPE_Union,
TYPE_Enum,
}Type_Kind;
struct Type{
Type_Kind kind;
SizeU size;
SizeU align;
struct{
Type *base;
}pointer;
struct{
Type *base;
SizeU size;
}array;
struct{
Type **params;
S32 count;
Type *ret;
}func;
};
global Type global_type[] = {
{TYPE_Void},
{TYPE_Int, sizeof(int), __alignof(int)},
{TYPE_Char, sizeof(char), __alignof(char)},
};
const SizeU pointer_size = sizeof(SizeU);
const SizeU pointer_align = __alignof(SizeU);
global Type *type_void = global_type + 0;
global Type *type_int = global_type + 1;
global Type *type_char = global_type + 2;
global Map map_pointer_types;
global Map map_array_types;
global Map map_func_types;
function Type *
type_new(Arena *arena, Type_Kind kind, SizeU size, SizeU align){
Type *result = arena_push_struct(arena, Type);
result->kind = kind;
result->size = size;
result->align = align;
return result;
}
function Type *
type_copy(Arena *arena, Type *type){
Type *result = arena_push_struct(arena, Type);
memory_copy(result, type, sizeof(Type));
return result;
}
function Type *
type_incomplete(Arena *arena){
Type *result = type_new(arena, TYPE_Incomplete, 0, 0);
return result;
}
function Type *
type_pointer(Arena *arena, Type *base){
Type *result = map_get(&map_pointer_types, (void *)base);
if(!result){
result = type_new(arena, TYPE_Pointer, pointer_size, pointer_align);
result->pointer.base = base;
map_insert(&map_pointer_types, base, result);
}
return result;
}
function Type *
type_array(Arena *arena, Type *base, SizeU size){
U64 hash = hash_mix(hash_ptr(base), hash_u64(size));
Type *result = map_get_u64(&map_array_types, hash);
if(result){
assert(result->array.size == size);
assert(result->array.base == base);
return result;
}
result = type_new(arena, TYPE_Array, pointer_size, pointer_align);
result->array.base = base;
result->array.size = size;
map_insert_u64(&map_array_types, hash, result);
return result;
}
function Type *
type_function(Arena *arena, Type *ret, Type **params, S32 count){
U64 hash = hash_ptr(ret);
for(S32 i = 0; i < count; i++)
hash = hash_mix(hash, hash_ptr(params[i]));
Type *result = map_get_u64(&map_func_types, hash);
if(result){
assert(result->func.ret == ret);
assert(result->func.count == count);
return result;
}
result = type_new(arena, TYPE_Func, pointer_size, pointer_align);
result->func.ret = ret;
result->func.count = count;
result->func.params = arena_push_array(arena, Type*, count);
memory_copy(result->func.params, params, count);
map_insert_u64(&map_func_types, hash, result);
return result;
}
function void
test_types(){
Arena *arena = arena_begin_scratch();
Type *array_type1 = type_array(arena, type_int, 32);
Type *array_type2 = type_array(arena, type_int, 32);
Type *array_type3 = type_array(arena, type_int, 48);
assert(array_type1 == array_type2);
assert(array_type2 != array_type3);
Type *pointer_type1 = type_pointer(arena, type_int);
Type *pointer_type2 = type_pointer(arena, type_int);
assert(pointer_type2 == pointer_type1);
Type *pointer_type3 = type_pointer(arena, pointer_type1);
Type *pointer_type4 = type_pointer(arena, pointer_type2);
assert(pointer_type3 != pointer_type1);
assert(pointer_type3 == pointer_type4);
Type **types = 0;
array_push(types, type_array(arena, type_int, 32));
Type *func_type1 = type_function(arena, types[0], types, array_len(types));
Type *func_type2 = type_function(arena, types[0], types, array_len(types));
assert(func_type1 == func_type2);
Type **types2 = 0;
{
array_push(types2, type_array(arena, type_int, 32));
array_push(types2, type_int);
}
array_push(types, type_int);
Type *func_type3 = type_function(arena, types[0], types, array_len(types));
Type *func_type4 = type_function(arena, types[0], types2, array_len(types));
assert(func_type1 != func_type3);
assert(func_type3 == func_type4);
arena_end_scratch();
}