New approach, new lexer

ast.c

@@ -81,6 +81,19 @@ decl_list_push(Decl *parent, Decl *child){
   SLLQueuePush(parent->list.first, parent->list.last, child);
 }
 
+//-----------------------------------------------------------------------------
+// Ops
+//-----------------------------------------------------------------------------
+function Decl *
+decl_struct_find_node(Decl *decl, String string){
+  for(Decl *n = decl->struct_decl.first; n; n=n->next){
+    if(string_compare(string, n->name.s)){
+      return n;
+    }
+  }
+  return 0;
+}
+
 //-----------------------------------------------------------------------------
 // Notes
 //-----------------------------------------------------------------------------
@@ -173,3 +186,443 @@ function void
 typespec_function_push(Typespec *func, Typespec *arg){
   SLLQueuePush(func->function_spec.first, func->function_spec.last, arg);
 }
+
+//-----------------------------------------------------------------------------
+// 
+//-----------------------------------------------------------------------------
+function Typespec *
+typespec_get_func(Typespec *type){
+  switch(type->kind){
+    case TS_Name:{return 0;} break;
+    case TS_Pointer:{return typespec_get_func(type->base);} break;
+    case TS_Array:{return typespec_get_func(type->array_spec.base);} break;
+    case TS_Function:{return type;} break;
+    default: {invalid_codepath;} break;
+  }
+  return 0;
+}
+
+function Typespec *
+typespec_get_name(Typespec *type){
+  switch(type->kind){
+    case TS_Name:{return type;} break;
+    case TS_Pointer:{return typespec_get_name(type->base);} break;
+    case TS_Array:{return typespec_get_name(type->array_spec.base);} break;
+    case TS_Function:{return 0;} break;
+    default: {invalid_codepath;} break;
+    
+  }
+  return 0;
+}
+
+function String
+typespec_require_name_string(Typespec *type){
+  Typespec *result = typespec_get_name(type);
+  if(result){
+    if(result->kind == TS_Name){
+      return result->name.s;
+    }
+  }
+  invalid_codepath;
+  return string_empty;
+}
+
+//-----------------------------------------------------------------------------
+// Statements
+//-----------------------------------------------------------------------------
+function Stmt *
+stmt_new(Parser *p, Stmt_Kind kind, Token *pos){
+  Stmt *result = arena_push_struct(&p->main_arena, Stmt);
+  result->kind = kind;
+  result->pos  = pos;
+  return result;
+}
+
+function Stmt *
+stmt_decl(Parser *p, Token *pos, Decl *decl){
+  Stmt *result = stmt_new(p, STMT_Decl, pos);
+  result->decl = decl;
+  return result;
+}
+
+function Stmt *
+stmt_expr(Parser *p, Token *pos, Expr *expr){
+  Stmt *result = stmt_new(p, STMT_Expr, pos);
+  result->expr = expr;
+  return result;
+}
+
+function Stmt *
+stmt_list(Parser *p, Token *pos){
+  Stmt *result = stmt_new(p, STMT_List, pos);
+  return result;
+}
+
+function Stmt *
+stmt_return(Parser *p, Token *pos, Expr *expr){
+  Stmt *result = stmt_new(p, STMT_Return, pos);
+  result->ret.expr = expr;
+  return result;
+}
+
+function Stmt *
+stmt_if(Parser *p, Token *pos, Stmt *body, Expr *cond){
+  Stmt *result = stmt_new(p, STMT_If, pos);
+  result->stmt_if.cond = cond;
+  result->stmt_if.body = body;
+  return result;
+}
+
+function void
+stmt_push(Stmt *stmt, Stmt *child){
+  SLLQueuePush(stmt->list.first, stmt->list.last, child);
+}
+
+//-----------------------------------------------------------------------------
+// Pointer Array
+//-----------------------------------------------------------------------------
+function Pointer_Array
+pointer_array_make(Arena *arena){
+  Pointer_Array result = {
+    .last  = &result.first,
+    .arena = arena,
+  };
+  return result;
+}
+
+#define bucket_size (buff_cap(array->first.data))
+function void
+pointer_array_push(Pointer_Array *array, Pointer p){
+  if(array->len >= bucket_size){
+    Pointer_Bucket *bucket = arena_push_struct(array->arena, Pointer_Bucket);
+    array->last = array->last->next = bucket;
+    array->len = 0;
+    array->block += 1;
+  }
+  array->last->data[array->len++] = p;
+}
+
+function B32
+pointer_array_iter_is_end(Pointer_Array *array){
+  B32 result = array->iter_len == array->len && array->iter_block == array->block;
+  return result;
+}
+
+function Pointer
+pointer_array_iter_next(Pointer_Array *array){
+  if(pointer_array_iter_is_end(array)){
+    return (Pointer){0};
+  }
+  if(array->iter_len >= bucket_size){
+    array->iter_len = 0;
+    array->iter_block += 1;
+    array->iter_bucket = array->iter_bucket->next;
+  }
+  Pointer result = array->iter_bucket->data[array->iter_len++];
+  return result;
+}
+
+function Pointer
+pointer_array_iter_begin(Pointer_Array *array){
+  array->iter_len = 0;
+  array->iter_block = 0;
+  array->iter_bucket = &array->first;
+  Pointer result = pointer_array_iter_next(array);
+  return result;
+}
+
+function Pointer
+pointer_typespec(Typespec *t){
+  return (Pointer){.kind = PK_Typespec, .typespec = t};
+}
+function Pointer
+pointer_expr(Expr *t){
+  return (Pointer){.kind = PK_Expr, .expr = t};
+}
+function Pointer
+pointer_decl(Decl *t){
+  return (Pointer){.kind = PK_Decl, .decl = t};
+}
+function Pointer
+pointer_stmt(Stmt *t){
+  return (Pointer){.kind = PK_Stmt, .stmt = t};
+}
+function Pointer
+pointer_enum_child(Decl_Enum_Child *t){
+  return (Pointer){.kind = PK_Enum_Child, .enum_child = t};
+}
+function Pointer
+pointer_func_arg(Decl_Function_Arg *t){
+  return (Pointer){.kind = PK_Func_Arg, .func_arg=t};
+}
+
+function void
+pointer_array_push_typespec(Pointer_Array *array, Typespec *typespec){
+  pointer_array_push(array, pointer_typespec(typespec));
+}
+function void
+pointer_array_push_decl(Pointer_Array *array, Decl *decl){
+  pointer_array_push(array, pointer_decl(decl));
+}
+function void
+pointer_array_push_stmt(Pointer_Array *array, Stmt *stmt){
+  pointer_array_push(array, pointer_stmt(stmt));
+}
+function void
+pointer_array_push_expr(Pointer_Array *array, Expr *expr){
+  pointer_array_push(array, pointer_expr(expr));
+}
+function void
+pointer_array_push_enum_child(Pointer_Array *array, Decl_Enum_Child *enum_child){
+  pointer_array_push(array, pointer_enum_child(enum_child));
+}
+function void
+pointer_array_push_func_arg(Pointer_Array *array, Decl_Function_Arg *func_arg){
+  pointer_array_push(array, pointer_func_arg(func_arg));
+}
+
+//-----------------------------------------------------------------------------
+// Gather
+//-----------------------------------------------------------------------------
+
+function void
+gather_try_recurse_typespec(Pointer_Array *array, Typespec *typespec, Gather_Flag gflag, Traversal_Flag tflag);
+function void
+gather_recurse_expr(Pointer_Array *array, Expr *expr, Gather_Flag gflag, Traversal_Flag tflag){
+  if(gflag & GATHER_Expr)
+    pointer_array_push_expr(array, expr);
+  
+  switch(expr->kind) {
+    case EK_Atom: {} break;
+    case EK_Sizeof:{
+      if(expr->size_of.kind == SIZEOF_Expr){
+        gather_recurse_expr(array, expr->size_of.expr, gflag, tflag);
+      }
+      else{
+        assert(expr->size_of.kind == SIZEOF_Type);
+        gather_try_recurse_typespec(array, expr->size_of.type, gflag, tflag);
+      }
+    }break;
+    
+    case EK_Binary:{
+      gather_recurse_expr(array, expr->binary.left, gflag, tflag);
+      gather_recurse_expr(array, expr->binary.right, gflag, tflag);
+    } break;
+    case EK_Unary:{
+      gather_recurse_expr(array, expr->unary.expr, gflag, tflag);
+    } break;
+    
+    case EK_Ternary:{
+      gather_recurse_expr(array, expr->ternary.cond, gflag, tflag);
+      gather_recurse_expr(array, expr->ternary.on_true, gflag, tflag);
+      gather_recurse_expr(array, expr->ternary.on_false, gflag, tflag);
+    } break;
+    case EK_List:{
+      for(Expr *n = expr->list.first; n; n=n->next){
+        gather_recurse_expr(array,  n, gflag, tflag);
+      }
+    }break;
+    
+    case EK_Cast:{
+      gather_try_recurse_typespec(array, expr->cast.type, gflag, tflag);
+      gather_recurse_expr(array,  expr->cast.expr, gflag, tflag);
+    } break;
+    
+    case EK_Index:{
+      gather_recurse_expr(array,  expr->index.atom, gflag, tflag);
+      gather_recurse_expr(array,  expr->index.index, gflag, tflag);
+    }break;
+    
+    case EK_Call:{
+      gather_recurse_expr(array,  expr->call.atom, gflag, tflag);
+      gather_recurse_expr(array,  expr->call.list, gflag, tflag);
+    }break;
+    default: {invalid_codepath;} break;
+  }
+}
+
+function void
+gather_try_recurse_expr(Pointer_Array *array, Expr *expr, Gather_Flag gflag, Traversal_Flag tflag){
+  if(expr){
+    if(tflag & TRAVERS_Expr){
+      gather_recurse_expr(array, expr, gflag, tflag);
+    }
+    else if(gflag & GATHER_Expr){
+      pointer_array_push_expr(array, expr);
+    }
+  }
+}
+
+function void
+gather_recurse_decl(Pointer_Array *array, Decl *decl, Gather_Flag gflag, Traversal_Flag tflag);
+function void
+gather_try_recurse_stmt(Pointer_Array *array, Stmt *stmt, Gather_Flag gflag, Traversal_Flag tflag);
+function void
+gather_recurse_stmt(Pointer_Array *array, Stmt *stmt, Gather_Flag gflag, Traversal_Flag tflag){
+  switch(stmt->kind) {
+    case STMT_List: {
+      for(Stmt *s = stmt->list.first; s; s=s->next){
+        gather_recurse_stmt(array, s, gflag, tflag);
+      }
+    } break;
+    case STMT_Return:{
+      gather_try_recurse_expr(array, stmt->ret.expr, gflag, tflag);
+    } break;
+    case STMT_If:{
+      gather_recurse_expr(array, stmt->stmt_if.cond, gflag, tflag);
+      gather_recurse_stmt(array, stmt->stmt_if.body, gflag, tflag);
+      for(Stmt_If *s = stmt->stmt_if.next; s; s=s->next){
+        gather_recurse_stmt(array, s->body, gflag, tflag);
+        gather_recurse_expr(array, s->cond, gflag, tflag);
+      }
+    } break;
+    case STMT_Expr: {
+      gather_recurse_expr(array, stmt->expr, gflag, tflag);
+    } break;
+    case STMT_Decl: {
+      gather_recurse_decl(array, stmt->decl, gflag, tflag);
+    } break;
+    default: {invalid_codepath;} break;
+  }
+}
+
+function void
+gather_try_recurse_stmt(Pointer_Array *array, Stmt *stmt, Gather_Flag gflag, Traversal_Flag tflag){
+  if(stmt){
+    if(tflag & TRAVERS_Stmt){
+      gather_recurse_stmt(array, stmt, gflag, tflag);
+    }
+    else if(gflag & GATHER_Stmt){
+      pointer_array_push_stmt(array, stmt);
+    }
+  }
+}
+
+function void
+gather_recurse_typespec(Pointer_Array *array, Typespec *typespec, Gather_Flag gflag, Traversal_Flag tflag){
+  if(gflag & GATHER_Typespec)
+    pointer_array_push_typespec(array, typespec);
+  switch(typespec->kind) {
+    case TS_Name: {
+    } break;
+    case TS_Pointer: {
+      gather_recurse_typespec(array, typespec->base, gflag, tflag);
+    } break;
+    case TS_Array: {
+      gather_recurse_typespec(array, typespec->array_spec.base, gflag, tflag);
+      gather_try_recurse_expr(array, typespec->array_spec.size, gflag, tflag);
+    } break;
+    case TS_Function: {
+      gather_recurse_typespec(array, typespec->function_spec.ret, gflag, tflag);
+      for(Typespec *n = typespec->function_spec.first; n; n=n->next){
+        gather_recurse_typespec(array, n, gflag, tflag);
+      }
+    } break;
+    default: {invalid_codepath;} break;
+  }
+}
+
+function void
+gather_try_recurse_typespec(Pointer_Array *array, Typespec *typespec, Gather_Flag gflag, Traversal_Flag tflag){
+  if(typespec){
+    if(tflag & TRAVERS_Typespec){
+      gather_recurse_typespec(array, typespec, gflag, tflag);
+    }
+    else if(gflag & GATHER_Typespec){
+      pointer_array_push_typespec(array, typespec);
+    }
+  }
+}
+
+function void
+gather_recurse_decl(Pointer_Array *array, Decl *decl, Gather_Flag gflag, Traversal_Flag tflag){
+  if(gflag & GATHER_Decl) pointer_array_push_decl(array, decl);
+  
+  switch(decl->kind){
+    case DECL_Struct:
+    case DECL_Union:{
+      for(Decl *n = decl->struct_decl.first; n; n=n->next){
+        gather_recurse_decl(array, n, gflag, tflag);
+      }
+    } break;
+    case DECL_Enum:{
+      gather_try_recurse_typespec(array, decl->enum_decl.typespec, gflag, tflag);
+      for(Decl_Enum_Child *child = decl->enum_decl.first; child; child=child->next){
+        if(gflag & GATHER_Enum_Child) pointer_array_push_enum_child(array, child);
+        gather_try_recurse_expr(array, child->expr, gflag, tflag);
+      }
+    }break;
+    case DECL_Variable:{
+      gather_try_recurse_typespec(array, decl->variable_decl.type, gflag, tflag);
+      gather_try_recurse_expr(array, decl->variable_decl.expr, gflag, tflag);
+    }break;
+    case DECL_Typedef:{
+      gather_try_recurse_typespec(array, decl->typedef_decl.type, gflag, tflag);
+    }break;
+    case DECL_Function:{
+      gather_try_recurse_typespec(array, decl->function_decl.ret, gflag, tflag);
+      for(Decl_Function_Arg *arg = decl->function_decl.first; arg; arg=arg->next){
+        if(gflag & GATHER_Func_Arg) pointer_array_push_func_arg(array, arg);
+        gather_try_recurse_typespec(array, arg->typespec, gflag, tflag);
+      }
+      gather_try_recurse_stmt(array, decl->function_decl.body, gflag, tflag);
+    }break;
+    case DECL_List:{
+      for(Decl *n = decl->list.first; n; n=n->next){
+        gather_recurse_decl(array, n, gflag, tflag);
+      }
+    }break;
+    default:invalid_codepath;
+  }
+  
+}
+
+function Pointer_Array
+gather(Arena *arena, Decl *decl, Gather_Flag gflag, Traversal_Flag tflag){
+  Pointer_Array array = pointer_array_make(arena);
+  gather_recurse_decl(&array, decl, gflag, tflag);
+  return array;
+}
+
+//-----------------------------------------------------------------------------
+// 
+//-----------------------------------------------------------------------------
+
+function Decl *
+decl_deep_copy_recurse(Arena *arena, Decl *decl){
+  Decl *result = arena_push_struct(arena, Decl);
+  memory_copy(result, decl, sizeof(*decl));
+  switch(decl->kind) {
+    case DECL_Struct:{
+      
+    }break;
+    case DECL_Union:{
+      
+    }break;
+    case DECL_Enum:{
+      
+    }break;
+    case DECL_Variable:{
+      
+    }break;
+    case DECL_Typedef:{
+      
+    }break;
+    case DECL_Function:{
+      
+    }break;
+    case DECL_List:{
+      
+    }break;
+    default: {invalid_codepath;} break;
+  }
+  return result;
+}
+
+function Decl *
+decl_deep_copy(Arena *arena){
+  Decl *result = decl_deep_copy_recurse(arena, decl);
+  return result;
+}
+
+

ast.h

@@ -1,164 +1,201 @@
-typedef struct Decl_Enum_Child Decl_Enum_Child;
+#if 0
-typedef struct Decl_Function_Arg Decl_Function_Arg;
-typedef struct Typespec Typespec;
-typedef struct Decl Decl;
-typedef struct Note Note;
-typedef struct Stmt Stmt;
-
 //-----------------------------------------------------------------------------
-// Type specifier
+// Type specifiers
 //-----------------------------------------------------------------------------
-typedef enum Typespec_Kind{
+@prefix="TS_" Typespec_Kind :: enum{ None, Name, Pointer, Array, Function }
-  TS_None,
+Typespec :: struct{
-  TS_Name,
+  kind: Typespec_Kind;
-  TS_Pointer,
+  next: Typespec*;
-  TS_Array,
+  pos : Token*;
-  TS_Function,
-}Typespec_Kind;
   
-struct Typespec{
-  Typespec_Kind kind;
-  Typespec *next;
-  Token *pos;
   union{
-    Intern_String name;
+    name: Intern_String;
-    struct{
+    base: Typespec*;
-      Typespec *first;
+    function_spec: struct{
-      Typespec *last;
+      first: Typespec*;
-      Typespec *ret;
+      last : Typespec*;
-    }function_spec;
+      ret  : Typespec*;
-    struct{
+    }
-      Typespec *base;
+    array_spec: struct{
-      Expr *size;
+      base: Typespec*;
-    }array_spec;
+      size: Expr*;
-    Typespec *base;
+    }
-  };
+  }
-};
+}
 
 //-----------------------------------------------------------------------------
 // Notes
 //-----------------------------------------------------------------------------
-struct Note{
+Note :: struct{
-  Token *pos;
+  pos : Token*;
-  Intern_String name;
+  name: Intern_String;
-  Expr *expr;
+  expr: Expr*;
   
-  Note *next;
+  next : Note*;
-  Note *first;
+  first: Note*;
-  Note *last;
+  last : Note*;
 };
 
 //-----------------------------------------------------------------------------
 // Declarations
 //-----------------------------------------------------------------------------
-typedef enum Decl_Kind{
+@prefix = "DECL_" Decl_Kind :: enum { None, Struct, Union, Enum, Variable, Typedef, Function, List }
-  DECL_None,
+@prefix="STRUCT_" Decl_Struct_Kind :: enum { Nested, Base }
-  DECL_Struct,
-  DECL_Union,
-  DECL_Enum,
-  DECL_Variable,
-  DECL_Typedef,
-  DECL_Function,
-  DECL_List,
-}Decl_Kind;
 
-typedef enum Decl_Struct_Kind{
+Decl_Function_Arg :: struct{
-  STRUCT_Base  ,
+  next: Decl_Function_Arg *;
-  STRUCT_Nested,
+  name: Intern_String;
-}Decl_Struct_Kind;
+  pos : Token *;
+  typespec: Typespec *;
+}
 
-struct Decl_Function_Arg{
+Decl_Enum_Child :: struct{
-  Decl_Function_Arg *next;
+  next: Decl_Enum_Child *;
-  Intern_String name;
+  name: Intern_String;
-  Typespec *typespec;
+  pos : Token *;
-  Token *pos;
+  expr: Expr *;
+  
+  first_note: Note *;
+  last_note : Note *;
 };
 
-struct Decl_Enum_Child{
+Decl :: struct{
-  Decl_Enum_Child *next;
+  kind: Decl_Kind;
-  Intern_String name;
+  next: Decl *;
-  Token *pos;
-  Expr *expr;
   
-  Note *first_note;
+  name: Intern_String;
-  Note *last_note;
+  pos : Token *;
-};
   
-struct Decl{
+  first_note: Note *;
-  Decl_Kind kind;
+  last_note : Note *;
-  Decl *next;
-  
-  Intern_String name;
-  Token *pos;
-  
-  Note *first_note;
-  Note *last_note;
   
   union{
-    struct{
+    enum_decl: struct{
-      Decl_Enum_Child *first;
+      first: Decl_Enum_Child *;
-      Decl_Enum_Child *last;
+      last: Decl_Enum_Child *;
-      Typespec *typespec;
+      typespec: Typespec *;
-    }enum_decl;
+    }
-    struct{
+    struct_decl: struct{
-      Decl_Struct_Kind kind;
+      first: Decl *;
-      Decl *first;
+      last: Decl *;
-      Decl *last;
+      kind: Decl_Struct_Kind ;
-    } struct_decl;
+    }
-    struct{
+    variable_decl: struct{
-      Typespec *type;
+      type: Typespec *;
-      Expr *expr;
+      expr: Expr *;
-    }variable_decl;
+    }
-    struct{
+    typedef_decl: struct{
-      Typespec *type;
+      type: Typespec *;
-    }typedef_decl;
+    }
-    struct{
+    function_decl: struct{
-      Decl_Function_Arg *first;
+      first: Decl_Function_Arg *;
-      Decl_Function_Arg *last ;
+      last : Decl_Function_Arg *;
-      Typespec *ret;
+      ret: Typespec *;
-    }function_decl;
+      body : Stmt*;
-    struct{
+    }
-      Decl *first;
+    list: struct{
-      Decl *last;
+      first: Decl *;
-    }list;
+      last: Decl *;
-  };
+    }
-};
+  }
+}
 
 //-----------------------------------------------------------------------------
 // Statements
 //-----------------------------------------------------------------------------
-typedef enum Stmt_Kind{
+@prefix="STMT_" Stmt_Kind :: enum{ None, Decl, Expr, List, Return, If, For}
-  STMT_None,
-  STMT_Decl,
-  STMT_Expr,
-  STMT_Return,
-  STMT_If,
-  STMT_For,
-}Stmt_Kind;
 
-typedef struct Stmt_List Stmt_List;
+Stmt_If :: struct {
-struct Stmt_List{
+  next: Stmt_If*;
-  Stmt *first;
+  cond: Expr*;
-  Stmt *last;
+  body: Stmt*;
-};
+}
+
+Stmt :: struct {
+  kind: Stmt_Kind;
+  next: Stmt*;
+  pos : Token*;
   
-typedef struct Stmt_If Stmt_If;
-struct Stmt_If{
-  Stmt_If *next;
-  Expr *expr;
   union{
-    struct{Stmt *first; Stmt *last;};
+    stmt_if: Stmt_If;
-    Stmt_List *list;
+    decl: Decl*;
-  };
+    expr: Expr*;
-};
+    list: struct{
+      first: Stmt*;
+      last : Stmt*;
+    }
+    ret: struct{
+      expr: Expr*;
+    }
+  }
+}
 
-struct Stmt{
+//-----------------------------------------------------------------------------
-  Stmt_Kind kind;
+// Gather
-  Stmt *next;
+//-----------------------------------------------------------------------------
-  union{
+
-    Decl *decl;
+@prefix="PK_"
-    Expr *expr;
+Pointer_Kind::enum{
-    Stmt_If *if_stmt;
+  None,
-    Stmt_List *list;
+  Typespec,
-  };
+  Expr,
-};
+  Decl,
+  Stmt,
+  Enum_Child,
+  Func_Arg,
+  Intern_String
+}
+
+Pointer::struct{
+  kind: Pointer_Kind;
+  union {
+    typespec: Typespec *;
+    decl: Decl *;
+    expr: Expr *;
+    stmt: Stmt *;
+    func_arg: Decl_Function_Arg*;
+    enum_child: Decl_Enum_Child*;
+    string: Intern_String *;
+  }
+}
+
+Pointer_Bucket::struct{
+  next: Pointer_Bucket*;
+  data: Pointer[4096];
+}
+
+Pointer_Array::struct{
+  first: Pointer_Bucket;
+  last : Pointer_Bucket*;
+  len  : S64;
+  block: S64;
+  arena: Arena*;
+  
+  iter_bucket: Pointer_Bucket*;
+  iter_len: S64;
+  iter_block: S64;
+}
+
+@prefix="GATHER_"
+Gather_Flag::enum{
+  None = 0, 
+  Typespec = 1,
+  Expr = 2,
+  Decl = 4,
+  Stmt = 8,
+  Enum_Child = 16,
+  Func_Arg = 32,
+}
+
+@prefix="TRAVERS_"
+Traversal_Flag :: enum{
+  None,
+  Typespec = 1,
+  Expr = 2,
+  //Decl = 4,
+  Stmt = 8,
+  All = TRAVERS_Typespec | TRAVERS_Expr | TRAVERS_Stmt,
+}
+
+#endif
+#include "generated_ast.h"

build.bat

@@ -1,3 +1,6 @@
 @echo off
 
-clang main.c -fdiagnostics-absolute-paths -std=c99 -g -o main.exe -Wl,-subsystem:windows -Wl,user32.lib
+rem clang generate.c  -fdiagnostics-absolute-paths -std=c99 -g -o generate.exe -Wl,user32.lib
+rem generate.exe
+clang main.c -fdiagnostics-absolute-paths -std=c99 -g -o main.exe -Wl,user32.lib
+rem cl main.c -std:c17

codegen_c.c

@@ -1,126 +1,475 @@
-function void gen_decl(Parser *p, Decl *node);
+function void gen_decl(Decl *node);
+function void gen_cdecl(Typespec *type, String str);
+function void gen_stmt_list(Stmt *stmt);
+
+global String_List *glist;
+global Arena       *gscratch;
+global Parser      *genp;
+#define strf(...) string_listf(gscratch, glist, __VA_ARGS__)
+
+//-----------------------------------------------------------------------------
+// String replacing util
+//-----------------------------------------------------------------------------
+typedef struct String_Map String_Map;
+struct String_Map{
+  String replace;
+  String with;
+};
 
 function void
-gen_typespec(Parser *p, Typespec *spec, B32 is_left){
+string_mapped_print(String string, String_Map *map, SizeU count){
-  switch(spec->kind) {
+  Tokens tokens = lex_stream(string, lit("string_mapped_print"));
+  for(Token *t = tokens.tokens; t != tokens.tokens + tokens.len; t++){
+    String string = t->string;
+    for(SizeU i = 0; i < count; i++){
+      if(string_compare(string, map[i].replace)){
+        string = map[i].with;
+        break;
+      }
+    }
     
+    if(t->kind == TK_At) continue;
+    if(string_compare(t->string,keyword_function.s)) strf("\n");
+    strf("%.*s", (int)string.len, string.str);
+    if(string_compare(t->string,keyword_function.s)) strf("\n");
+    if(t->kind == TK_OpenBrace) strf("\n");
+    if(t->kind == TK_CloseBrace && t[1].kind != TK_Semicolon) strf("\n");
+    if(t->kind == TK_Semicolon) strf("\n");
+    if(t->kind == TK_Identifier && (t[1].kind == TK_Keyword || t[1].kind == TK_Identifier)) strf(" ");
+    if(t->kind == TK_Keyword) strf(" ");
+    if(t->kind == TK_Comma) strf(" ");
+  }
+  free(tokens.tokens);
+}
+#define STR(X) lit(#X)
+
+//-----------------------------------------------------------------------------
+// Codegen
+//-----------------------------------------------------------------------------
+
+function void
+token_strf(Token *token){
+  strf("%.*s", (S32)token->len, token->str);
+}
+
+function void
+gen_expr(Expr *expr){
+  switch(expr->kind) {
+    case EK_Atom: {
+      token_strf(expr->token);
+    } break;
+    
+    case EK_Sizeof:{
+      strf("sizeof(");
+      if(expr->size_of.kind == SIZEOF_Expr){
+        gen_expr(expr->size_of.expr);
+      }
+      else{
+        assert(expr->size_of.kind == SIZEOF_Type);
+        gen_cdecl(expr->size_of.type, string_empty);
+      }
+      strf(")");
+    }break;
+    
+    case EK_Binary:{
+      strf("(");
+      gen_expr(expr->binary.left);
+      token_strf(expr->token);
+      gen_expr(expr->binary.right);
+      strf(")");
+    } break;
+    case EK_Unary:{
+      strf("(");
+      token_strf(expr->token);
+      gen_expr(expr->unary.expr);
+      strf(")");
+    } break;
+    
+    case EK_Ternary:{
+      strf("(");
+      gen_expr(expr->ternary.cond);
+      strf("?");
+      gen_expr(expr->ternary.on_true);
+      strf(":");
+      gen_expr(expr->ternary.on_false);
+      strf(")");
+    } break;
+    case EK_List:{
+      strf("(");
+      for(Expr *n = expr->list.first; n; n=n->next){
+        gen_expr(n);
+        if(n!=expr->list.last) strf(",");
+      }
+      strf(")");
+    }break;
+    
+    case EK_Cast:{
+      strf("(");
+      strf("(");
+      gen_cdecl(expr->cast.type, string_empty);
+      strf(")");
+      gen_expr(expr->cast.expr);
+      strf(")");
+    } break;
+    
+    case EK_Index:{
+      gen_expr(expr->index.atom);
+      strf("[");
+      gen_expr(expr->index.index);
+      strf("]");
+    }break;
+    
+    case EK_Call:{
+      gen_expr(expr->call.atom);
+      strf("(");
+      gen_expr(expr->call.list);
+      strf(")");
+    }break;
+    default: {invalid_codepath;} break;
+  }
+}
+
+function String
+gen_cdecl_paren(String str, String original){
+  if(string_is_empty(original) || original.str[0] == '['){
+    return str;
+  }
+  else{
+    String result = string_fmt(gscratch, "(%.*s)", (int)str.len, str.str);
+    return result;
+  }
+}
+
+function String
+gen__cdecl(Typespec *type, String str){
+  switch(type->kind) {
     case TS_Name: {
-      if(is_left) lex_print("%s", spec->name.s.str);
+      String space = string_is_empty(str) ? lit(""):lit(" ");
+      String result = string_fmt(gscratch, "%s%s%.*s", type->name.s.str, space.str, (int)str.len, str.str);
+      return result;
     } break;
-    
     case TS_Pointer: {
-      gen_typespec(p, spec->base,is_left);
+      String pointer   = string_fmt(gscratch, "*%.*s", (int)str.len, str.str);
-      if(is_left) lex_print("*");
+      String add_paren = gen_cdecl_paren(pointer, str);
+      String result    = gen__cdecl(type->base, add_paren);
+      return result;
     } break;
-    
     case TS_Array: {
-      gen_typespec(p, spec->array_spec.base,is_left);
+      String left  = string_fmt(gscratch, "%s[", str.str);
-      if(!is_left){
-        lex_print("[");
-        expr_print(p, spec->array_spec.size);
-        lex_print("]");
-      }
-    } break;
       
+      String_List *save = glist;
+      String_List list = {0};
+      glist = &list;
+      gen_expr(type->array_spec.size);
+      String expr_string = string_list_flatten(gscratch, glist);
+      glist = save;
+      
+      
+      String right = string_fmt(gscratch, "%s%s]", left.str, expr_string.str);
+      String paren = gen_cdecl_paren(right, str);
+      String result = gen__cdecl(type->array_spec.base, paren);
+      return result;
+    } break;
     case TS_Function: {
-      invalid_codepath;
+      String result = string_fmt(gscratch, "(*%s)(", str.str);
+      if (type->function_spec.first == 0) {
+        result= string_fmt(gscratch, "%svoid", result.str);
+      } 
+      else {
+        for(Typespec *n = type->function_spec.first; n; n=n->next){
+          String arg = gen__cdecl(n, string_empty);
+          result = string_fmt(gscratch, "%s%s", result.str, arg.str);
+          if(n != type->function_spec.last)
+            result = string_fmt(gscratch, "%s, ", result.str);
+        }
+      }
+      result = string_fmt(gscratch, "%s)", result.str);
+      result = gen__cdecl(type->function_spec.ret, result);
+      return result;
     } break;
     default: {invalid_codepath;} break;
   }
-}
+  return string_empty;
-
-function Typespec *
-typespec_get_func(Typespec *type){
-  switch(type->kind){
-    
-    case TS_Name:{return 0;} break;
-    case TS_Pointer:{return typespec_get_func(type->base);} break;
-    case TS_Array:{return typespec_get_func(type->array_spec.base);} break;
-    case TS_Function:{return type;} break;
-    default: {invalid_codepath;} break;
-    
-  }
-  return 0;
 }
 
 function void
-gen_decl(Parser *p, Decl *node){
+gen_cdecl(Typespec *type, String str){
+  String string = gen__cdecl(type, str);
+  strf("%.*s", (int)string.len, string.str);
+}
 
+function void
+gen_assign_expr(Expr *expr){
+  if(expr){
+    strf(" = ");
+    gen_expr(expr);
+  }
+}
+
+function void
+gen_function_decl(Decl *node){
+  strf("function ");
+  gen_cdecl(node->function_decl.ret, string_empty);
+  strf(" %s(", node->name.s.str);
+  for(Decl_Function_Arg *arg = node->function_decl.first; arg; arg=arg->next){
+    gen_cdecl(arg->typespec, arg->name.s);
+    if(arg!=node->function_decl.last) strf(", ");
+  }
+  strf(")");
+}
+
+typedef struct Macro{
+  Intern_String name;
+  Note *param_type;
+  Note *param_expr;
+  Decl *decl;
+}Macro;
+global Macro macros[32];
+global S32 macros_i;
+
+function void
+gen_forward_decl(Decl *node){
+  U8 *name = node->name.s.str;
   switch(node->kind) {
     case DECL_List: {
       for(Decl *n = node->list.first; n; n=n->next){
-        gen_decl(p,n);
+        gen_forward_decl(n);
-        lex_new_line();
       }
     } break;
     
     case DECL_Variable:{
-      Typespec *func = typespec_get_func(node->variable_decl.type);
-      if(!func){
-        gen_typespec(p, node->variable_decl.type, true);
-        lex_print(" %s", node->name.s.str);
-        gen_typespec(p, node->variable_decl.type, false);
-      }
-      else{
-        gen_typespec(p, func->function_spec.ret, true);
-        lex_print("(*");
-        lex_print("%s", node->name.s.str);
-        lex_print(")");
-        lex_print("(");
-        for(Typespec *t = func->function_spec.first; t; t=t->next){
-          gen_typespec(p, t, true);
-          gen_typespec(p, t, false);
-          if(t != func->function_spec.last) lex_print(", ");
-        }
-        lex_print(")");
-      }
-      print_assign_expr(p,node->variable_decl.expr);
-      lex_print(";");
     } break;
     
     case DECL_Typedef:{
-      lex_print("typedef %s ", node->name.s.str);
-      gen_typespec(p, node->typedef_decl.type, true);
-      gen_typespec(p, node->typedef_decl.type, false);
-      lex_print(";");
     } break;
     
+    case DECL_Function:{
+      gen_function_decl(node);
+      strf(";\n");
+    }break;
+    
     case DECL_Struct:
     case DECL_Union :{
-      U8 *struct_name = node->kind==DECL_Struct ? (U8*)"struct" : (U8*)"union";
+      Note *note = decl_find_note(node, lit("register"));
-      U8 *name = node->name.s.str;
+      if(note){
-      if(node->struct_decl.kind == STRUCT_Base) 
+        Note *param_expr = decl_find_note(node, lit("param_expr"));
-        lex_print("typedef %s %s %s;\n", struct_name, name, name);
+        Note *param_type = decl_find_note(node, lit("param_type"));
-      lex_print("%s %s{\n", struct_name, name?name:(U8*)"");
+        macros[macros_i++] = (Macro){
-      for(Decl *n = node->struct_decl.first; n; n=n->next){
+          .name=node->name, 
-        gen_decl(p, n);
+          .param_type=param_type, 
-        lex_print("\n");
+          .param_expr=param_expr,
+          .decl=node
+        };
       }
-      lex_print("};");
+      else{
+        U8 *struct_name = node->kind==DECL_Struct ? (U8*)"struct" : (U8*)"union";
         if(node->struct_decl.kind == STRUCT_Base) 
-        lex_print("\n");
+          strf("typedef %s %s %s;\n", struct_name, name, name);
+      }
     } break;
     
     case DECL_Enum:{
-      lex_print("enum %s", node->name.s.str);
-      if(!intern_compare(node->enum_decl.typespec->name, intern_s64)){
-        lex_print(" : ");
-        lex_print("%s", node->enum_decl.typespec->name.s.str);
-      }
-      lex_print("{\n");
-      
-      String prefix = decl_find_string_note(node, lit("prefix"), string_empty);
-      for(Decl_Enum_Child *n = node->enum_decl.first; n; n=n->next){
-        lex_print("%.*s%s", (int)prefix.len, prefix.str, n->name.s.str);
-        print_assign_expr(p, n->expr);
-        lex_print(",\n");
-      }
-      
-      lex_print("};\n");
     } break;
     
     default: {invalid_codepath;} break;
   }
 }
 
+function void
+gen_decl(Decl *node){
+  U8 *name = node->name.s.str;
   
+  switch(node->kind) {
+    case DECL_List: {
+      for(Decl *n = node->list.first; n; n=n->next){
+        gen_decl(n);
+        strf("\n");
+      }
+    } break;
+    
+    case DECL_Variable:{
+      gen_cdecl(node->variable_decl.type, node->name.s);
+      gen_assign_expr(node->variable_decl.expr);
+      strf(";");
+    } break;
+    
+    case DECL_Typedef:{
+      strf("typedef ");
+      gen_cdecl(node->typedef_decl.type, node->name.s);
+      strf(";");
+    } break;
+    
+    case DECL_Struct:
+    case DECL_Union :{
+      Note *note = decl_find_note(node, lit("register"));
+      if(note){
+        Arena_Checkpoint checkpoint = arena_checkpoint(gscratch);
+        Pointer_Array array = gather(checkpoint.arena, node, GATHER_Decl|GATHER_Typespec, TRAVERS_All);
+        Intern_String based = intern_string(genp, lit("Based_Type_Represent"));
+        Intern_String type = intern_string(genp, lit("Type"));
+        for(Pointer it = pointer_array_iter_begin(&array); it.typespec; it = pointer_array_iter_next(&array)){
+          if(it.kind == PK_Typespec){
+            Typespec *typespec = it.typespec;
+            assert(typespec->kind == TS_Name || typespec->kind == TS_Pointer || typespec->kind == TS_Array || typespec->kind == TS_Function);
+            if(typespec->kind == TS_Name){
+              if(intern_compare(typespec->name, type)){
+                typespec->name = based;
+              }
+            }
+          }
+          else if(it.kind == PK_Decl){
+            Decl *decl = it.decl;
+            if(decl->kind == DECL_Struct || decl->kind == DECL_Function){
+              if(intern_compare(decl->name, type)){
+                decl->name = based;
+              }
+            }
+          }
+        }
+        arena_restore(checkpoint);
+      }
+      U8 *struct_name = node->kind==DECL_Struct ? (U8*)"struct" : (U8*)"union";
+      strf("%s", struct_name);
+      if(node->struct_decl.kind == STRUCT_Base) 
+        strf(" %s", node->name.s.str);
+      strf("{\n");
+      for(Decl *n = node->struct_decl.first; n; n=n->next){
+        gen_decl(n);
+        strf("\n");
+      }
+      strf("}");
+      if(node->struct_decl.kind == STRUCT_Nested)
+        strf("%.*s", (int)node->name.s.len, node->name.s.str);
+      
+      strf(";");
+      if(node->struct_decl.kind == STRUCT_Base)
+        strf("\n");
+      
+    } break;
+    
+    case DECL_Enum:{
+      strf("typedef enum %s", name);
+      if(!intern_compare(node->enum_decl.typespec->name, intern_s64)){
+        strf(" : ");
+        strf("%s", node->enum_decl.typespec->name.s.str);
+      }
+      strf("{\n");
+      
+      String prefix = decl_find_string_note(node, lit("prefix"), string_empty);
+      for(Decl_Enum_Child *n = node->enum_decl.first; n; n=n->next){
+        strf("%.*s%s", (int)prefix.len, prefix.str, n->name.s.str);
+        gen_assign_expr(n->expr);
+        strf(",\n");
+      }
+      
+      strf("}%s;\n", name);
+    } break;
+    
+    case DECL_Function:{
+      gen_function_decl(node);
+      gen_stmt_list(node->function_decl.body);
+    } break;
+    
+    default: {invalid_codepath;} break;
+  }
+}
+
+function void
+gen_stmt_list(Stmt *stmt){
+  strf("{\n");
+  for(Stmt *s = stmt->list.first; s; s=s->next){
+    gen_stmt(s);
+    strf("\n");
+  }
+  strf("}");
+}
+
+function void
+gen_stmt(Stmt *stmt){
+  switch(stmt->kind) {
+    case STMT_List: {
+      gen_stmt_list(stmt);
+    } break;
+    case STMT_Return:{
+      strf("return ");
+      gen_expr(stmt->ret.expr);
+      strf(";");
+    } break;
+    case STMT_If:{
+      strf("if ");
+      gen_expr(stmt->stmt_if.cond);
+      gen_stmt_list(stmt->stmt_if.body);
+      for(Stmt_If *s = stmt->stmt_if.next; s; s=s->next){
+        if(s->next){
+          strf("else if ");
+          gen_expr(s->cond);
+          gen_stmt_list(s->body);
+        }
+        else{
+          strf("else");
+          gen_stmt_list(s->body);
+        }
+      }
+    } break;
+    case STMT_Expr: {
+      gen_expr(stmt->expr);
+      strf(";");
+    } break;
+    case STMT_Decl: {
+      gen_decl(stmt->decl);
+    } break;
+    default: {invalid_codepath;} break;
+  }
+}
+
+//-----------------------------------------------------------------------------
+// 
+//-----------------------------------------------------------------------------
+global Decl *gen_parent;
+function void
+gen_code(Decl *node){
+  U8 *name = node->name.s.str;
+  
+  switch(node->kind) {
+    case DECL_List: {
+      for(Decl *n = node->list.first; n; n=n->next){
+        gen_code(n);
+      }
+    } break;
+    
+    case DECL_Variable:{
+    } break;
+    
+    case DECL_Typedef:{
+    } break;
+    
+    case DECL_Struct:
+    case DECL_Union :{
+      gen_parent = node;
+      for(Decl *n = node->struct_decl.first; n; n=n->next){
+        gen_code(n);
+      }
+      
+    } break;
+    
+    case DECL_Enum:{
+    } break;
+    
+    default: {invalid_codepath;} break;
+  }
+}
+
+String_List list;
+function void
+gen_begin(Arena *scratch, Parser *p){
+  list     = (String_List){0};
+  gscratch = scratch;
+  glist    = &list;
+  genp     = p;
+}
+
+function void
+gen_end(){
+  String result = string_list_flatten(gscratch, glist);
+  lex_print("%s", result.str);
+}

common.c

@@ -1,4 +1,3 @@
-global String string_empty;
 
 function S64
 clamp_top_s64(S64 val, S64 max){
@@ -52,6 +51,11 @@ wrap_around_pow2(U64 x, U64 power_of_2) {
   return r;
 }
 
+//-----------------------------------------------------------------------------
+// Strings
+//-----------------------------------------------------------------------------
+global String string_empty;
+
 function B32
 string_compare(String a, String b){
   if(a.len != b.len)
@@ -77,6 +81,12 @@ char_to_upper(U8 c){
   return c;
 }
 
+function B32 
+string_is_empty(String a){
+  B32 result = a.len == 0;
+  return result;
+}
+
 function String
 string_to_lower(Arena *arena, String string){
   String result = arena_push_string_copy(arena, string);
@@ -85,3 +95,103 @@ string_to_lower(Arena *arena, String string){
   }
   return result;
 }
+
+//-----------------------------------------------------------------------------
+// String interning
+//-----------------------------------------------------------------------------
+typedef struct Table_Index{
+  U64 hash;
+  U64 index;
+  U64 iter;
+  U64 max_size;
+}Table_Index;
+
+typedef struct Intern_String{
+  String s;
+}Intern_String;
+
+typedef struct Intern_Table{
+  S64 interns_in_bytes;
+  S64 interns_inserted;
+  S64 interns_max;
+  Intern_String *interns;
+  Arena *arena;
+}Intern_Table;
+
+function Intern_Table
+intern_table(Arena *arena, SizeU size){
+  Intern_Table result = {0};
+  result.arena = arena;
+  result.interns = arena_push_array(arena, Intern_String, size);
+  result.interns_max = size;
+  return result;
+}
+
+function Table_Index
+table_index_from_hash(U64 hash, U64 max_size){
+  Table_Index result = {0};
+  result.hash = hash;
+  result.index = result.hash % max_size;
+  result.iter = result.index;
+  result.max_size = max_size;
+  return result;
+}
+
+function Table_Index
+table_index_from_string(String string, U64 max_size){
+  U64 hash = hash_fnv(string);
+  Table_Index result = table_index_from_hash(hash, max_size);
+  return result;
+}
+
+function B32
+table_index_advance(Table_Index *index){
+  index->iter = wrap_around_pow2(index->iter + 1, index->max_size);
+  B32 result = index->iter == index->index;
+  return result;
+}
+
+function Intern_String
+intern_string(Intern_Table *p, String string){
+  Intern_String result = {0};
+  Table_Index index = table_index_from_string(string, p->interns_max);
+  for(;;){
+    Intern_String *intern = p->interns + index.iter;
+    if(intern->s.str == 0){
+      result.s = arena_push_string_copy(p->arena, string);
+      p->interns_in_bytes += string.len;
+      p->interns_inserted += 1;
+      *intern = result;
+      break;
+    }
+    else if(string_compare(intern->s, string)){
+      result = *intern;
+      break;
+    }
+    
+    if (table_index_advance(&index)) 
+      break;
+    
+  }
+  
+  return result;
+}
+
+function B32
+intern_compare(Intern_String a, Intern_String b){
+  B32 result = a.s.str == b.s.str;
+  return result;
+}
+
+function void
+intern_test(){
+  Arena *scratch = arena_begin_scratch();
+  Intern_Table table = intern_table(scratch, 512);
+  assert(intern_compare(intern_string(&table, lit("Thing")), 
+                        intern_string(&table, lit("Thing"))));
+  assert(!intern_compare(intern_string(&table, lit("Thing")), 
+                         intern_string(&table, lit("No_Thing"))));
+  assert(intern_compare(intern_string(&table, lit("No_Thing")), 
+                        intern_string(&table, lit("No_Thing"))));
+  
+}

file.c

@@ -0,0 +1,21 @@
+
+global FILE *global_output_file;
+global FILE *global_all_files[32];
+global U32   global_all_files_count;
+
+#define lex_print(...) fprintf(global_output_file, __VA_ARGS__)
+#define lex_new_line() lex_print("\n")
+
+function void
+use_write_file(const char *file){
+  global_output_file = fopen(file, "w");
+  global_all_files[global_all_files_count++] = global_output_file;
+  assert_msg(global_output_file, "Failed to open file:%s", file);
+}
+
+function void
+close_all_files(){
+  for(U32 i = 0; i < global_all_files_count; i++){
+    fclose(global_all_files[i]);
+  }
+}

generate.c

@@ -0,0 +1,36 @@
+#define _CRT_SECURE_NO_WARNINGS
+#include <stdio.h>
+#include <windows.h>
+
+#include "lang.h"
+#include "os.h"
+#include "memory.h"
+
+#include "common.c"
+#include "memory.c"
+#include "scratch.c"
+#include "os_win32.c"
+
+#include "new_lex.c"
+
+int main(){
+  
+  FILE *f = fopen("generated.c", "w");
+  assert(f);
+  
+  Arena *scratch = arena_begin_scratch();
+  String file = os_read_file(scratch, lit("token_array.c"));
+  Token_Array array = lex_stream(scratch, file, lit("token_array.c"));
+  for(Token *t = token_array_iter_begin(&array); t; t = token_array_iter_next(&array)){
+    
+    fprintf(f, "%.*s", (int)t->len, t->str);
+    if(t->kind == TK_OpenBrace) fprintf(f, "\n");
+    if(t->kind == TK_CloseBrace && token_array_iter_peek(&array, 0)->kind != TK_Semicolon) fprintf(f, "\n");
+    if(t->kind == TK_Semicolon) fprintf(f, "\n");
+    if(t[0].kind == TK_Identifier && token_array_iter_peek(&array, 0)->kind == TK_Identifier) fprintf(f, " ");
+    token_array_iter_peek(&array, 1);
+  }
+  arena_end_scratch();
+  
+  fclose(f);
+}

generated.c

@@ -0,0 +1,69 @@
+typedef struct Token_Bucket Token_Bucket;
+typedef struct Token_Array Token_Array;
+struct Token_Bucket{
+Token_Bucket*next;
+Token data[4096];
+};
+struct Token_Array{
+Intern_Table interns;
+Token_Bucket first;
+Token_Bucket*last;
+S64 len;
+S64 block;
+Arena*arena;
+Token_Bucket*iter_bucket;
+S64 iter_len;
+S64 iter_block;
+};
+function Token_Array token_array_make(Arena*arena){
+Token_Array result={
+.last=&result.first,.arena=arena,.interns=intern_table(arena,4096*4)};
+return result;
+}
+function void token_array_push(Token_Array*array,Token*p){
+if(array->len>=buff_cap(array->first.data)){
+Token_Bucket*bucket=arena_push_struct(array->arena,Token_Bucket);
+array->last=array->last->next=bucket;
+array->len=0;
+array->block+=1;
+}
+array->last->data[array->len++]=*p;
+}
+function B32 token_array_iter_is_end(Token_Array*array){
+B32 result=array->iter_len==array->len&&array->iter_block==array->block;
+return result;
+}
+function Token*token_array_iter_next(Token_Array*array){
+if(token_array_iter_is_end(array)){
+return0;
+}
+if(array->iter_len>=buff_cap(array->first.data)){
+array->iter_len=0;
+array->iter_block+=1;
+array->iter_bucket=array->iter_bucket->next;
+}
+Token*result=array->iter_bucket->data+array->iter_len++;
+return result;
+}
+function Token*token_array_iter_peek(Token_Array*array,S64 i){
+S64 save_len=array->iter_len;
+Token_Bucket*save_bucket=array->iter_bucket;
+assert(i<buff_cap(array->first.data));
+S64 over=i;
+if(array->iter_len+i>=buff_cap(array->first.data)){
+over=buff_cap(array->first.data)-(array->iter_len+i);
+array->iter_len=0;
+array->iter_bucket=array->iter_bucket->next;
+}
+Token*result=array->iter_bucket->data+array->iter_len+over;
+array->iter_len=save_len;
+array->iter_bucket=save_bucket;
+return result;
+}
+function Token*token_array_iter_begin(Token_Array*array){
+array->iter_len=0;
+array->iter_block=0;
+array->iter_bucket=&array->first;
+Token*result=token_array_iter_next(array);
+return result;
+}

generated_ast.h

@@ -0,0 +1,225 @@
+
+typedef struct Typespec Typespec;
+
+typedef struct Note Note;
+
+
+
+typedef struct Decl_Function_Arg Decl_Function_Arg;
+
+typedef struct Decl_Enum_Child Decl_Enum_Child;
+
+typedef struct Decl Decl;
+
+
+typedef struct Stmt_If Stmt_If;
+
+typedef struct Stmt Stmt;
+
+
+typedef struct Pointer Pointer;
+
+typedef struct Pointer_Bucket Pointer_Bucket;
+
+typedef struct Pointer_Array Pointer_Array;
+
+
+
+typedef enum Typespec_Kind{
+TS_None,
+TS_Name,
+TS_Pointer,
+TS_Array,
+TS_Function,
+}Typespec_Kind;
+
+struct Typespec{
+Typespec_Kind kind;
+Typespec (*next);
+Token (*pos);
+union{
+Intern_String name;
+Typespec (*base);
+struct{
+Typespec (*first);
+Typespec (*last);
+Typespec (*ret);
+}function_spec;
+struct{
+Typespec (*base);
+Expr (*size);
+}array_spec;
+};
+};
+
+struct Note{
+Token (*pos);
+Intern_String name;
+Expr (*expr);
+Note (*next);
+Note (*first);
+Note (*last);
+};
+
+typedef enum Decl_Kind{
+DECL_None,
+DECL_Struct,
+DECL_Union,
+DECL_Enum,
+DECL_Variable,
+DECL_Typedef,
+DECL_Function,
+DECL_List,
+}Decl_Kind;
+
+typedef enum Decl_Struct_Kind{
+STRUCT_Nested,
+STRUCT_Base,
+}Decl_Struct_Kind;
+
+struct Decl_Function_Arg{
+Decl_Function_Arg (*next);
+Intern_String name;
+Token (*pos);
+Typespec (*typespec);
+};
+
+struct Decl_Enum_Child{
+Decl_Enum_Child (*next);
+Intern_String name;
+Token (*pos);
+Expr (*expr);
+Note (*first_note);
+Note (*last_note);
+};
+
+struct Decl{
+Decl_Kind kind;
+Decl (*next);
+Intern_String name;
+Token (*pos);
+Note (*first_note);
+Note (*last_note);
+union{
+struct{
+Decl_Enum_Child (*first);
+Decl_Enum_Child (*last);
+Typespec (*typespec);
+}enum_decl;
+struct{
+Decl (*first);
+Decl (*last);
+Decl_Struct_Kind kind;
+}struct_decl;
+struct{
+Typespec (*type);
+Expr (*expr);
+}variable_decl;
+struct{
+Typespec (*type);
+}typedef_decl;
+struct{
+Decl_Function_Arg (*first);
+Decl_Function_Arg (*last);
+Typespec (*ret);
+Stmt (*body);
+}function_decl;
+struct{
+Decl (*first);
+Decl (*last);
+}list;
+};
+};
+
+typedef enum Stmt_Kind{
+STMT_None,
+STMT_Decl,
+STMT_Expr,
+STMT_List,
+STMT_Return,
+STMT_If,
+STMT_For,
+}Stmt_Kind;
+
+struct Stmt_If{
+Stmt_If (*next);
+Expr (*cond);
+Stmt (*body);
+};
+
+struct Stmt{
+Stmt_Kind kind;
+Stmt (*next);
+Token (*pos);
+union{
+Stmt_If stmt_if;
+Decl (*decl);
+Expr (*expr);
+struct{
+Stmt (*first);
+Stmt (*last);
+}list;
+struct{
+Expr (*expr);
+}ret;
+};
+};
+
+typedef enum Pointer_Kind{
+PK_None,
+PK_Typespec,
+PK_Expr,
+PK_Decl,
+PK_Stmt,
+PK_Enum_Child,
+PK_Func_Arg,
+PK_Intern_String,
+}Pointer_Kind;
+
+struct Pointer{
+Pointer_Kind kind;
+union{
+Typespec (*typespec);
+Decl (*decl);
+Expr (*expr);
+Stmt (*stmt);
+Decl_Function_Arg (*func_arg);
+Decl_Enum_Child (*enum_child);
+Intern_String (*string);
+};
+};
+
+struct Pointer_Bucket{
+Pointer_Bucket (*next);
+Pointer (data[4096]);
+};
+
+struct Pointer_Array{
+Pointer_Bucket first;
+Pointer_Bucket (*last);
+S64 len;
+S64 block;
+Arena (*arena);
+Pointer_Bucket (*iter_bucket);
+S64 iter_len;
+S64 iter_block;
+};
+
+typedef enum Gather_Flag{
+GATHER_None = 0,
+GATHER_Typespec = 1,
+GATHER_Expr = 2,
+GATHER_Decl = 4,
+GATHER_Stmt = 8,
+GATHER_Enum_Child = 16,
+GATHER_Func_Arg = 32,
+}Gather_Flag;
+
+typedef enum Traversal_Flag{
+TRAVERS_None,
+TRAVERS_Typespec = 1,
+TRAVERS_Expr = 2,
+TRAVERS_Stmt = 8,
+TRAVERS_All = ((TRAVERS_Typespec|TRAVERS_Expr)|TRAVERS_Stmt),
+}Traversal_Flag;
+

generated_lex.cpp

@@ -0,0 +1,53 @@
+
+// Generated Array block
+
+function
+ Token_Array token_array_make(Arena*arena){
+Token_Array result={
+0};
+result.arena=arena;
+result.last=&result.first;
+return result;
+}
+
+function
+ void token_array_push(Token_Array*array, Token*item){
+if(array->len+1>4096){
+assert(array->arena);
+Token_Array_Block*block=arena_push_struct(array->arena, Token_Array_Block);
+array->last=array->last->next=block;
+array->len=0;
+array->block+=1;
+}
+array->last->data[array->len++]=*item;
+}
+
+function
+ Token_Array_Iter token_array_iter(Token_Array*array){
+return((Token_Array_Iter){
+.total_len=array->len, .total_block=array->block, .block=&array->first, }
+);
+}
+
+function
+ Token*token_array_iter_next(Token_Array_Iter*it){
+if(it->iter_len+1>4096){
+it->iter_len=0;
+it->iter_block+=1;
+it->block=it->block->next;
+}
+return it->block->data+it->iter_len++;
+}
+
+function
+ B32 token_array_iter_is_end(Token_Array_Iter*it){
+B32 result=it->iter_len==it->total_len&&it->iter_block==it->total_block;
+return result;
+}
+
+function
+ Token*token_array_iter_begin(Token_Array*array){
+array->it=token_array_iter(array);
+Token*result=token_array_iter_next(&array->it);
+return result;
+}

generated_lex.h

@@ -0,0 +1,136 @@
+typedef struct Intern_String Intern_String;
+
+
+typedef struct Token Token;
+
+typedef struct Lex_Stream Lex_Stream;
+
+typedef struct Tokens Tokens;
+
+struct Intern_String{
+String s;
+};
+
+typedef enum Token_Kind{
+TK_End,
+TK_Mul,
+TK_Div,
+TK_Add,
+TK_Sub,
+TK_Mod,
+TK_BitAnd,
+TK_BitOr,
+TK_BitXor,
+TK_Neg,
+TK_Not,
+TK_OpenParen,
+TK_CloseParen,
+TK_OpenBrace,
+TK_CloseBrace,
+TK_OpenBracket,
+TK_CloseBracket,
+TK_Comma,
+TK_Pound,
+TK_Question,
+TK_ThreeDots,
+TK_Semicolon,
+TK_Dot,
+TK_LesserThen,
+TK_GreaterThen,
+TK_Colon,
+TK_Assign,
+TK_DivAssign,
+TK_MulAssign,
+TK_ModAssign,
+TK_SubAssign,
+TK_AddAssign,
+TK_AndAssign,
+TK_OrAssign,
+TK_XorAssign,
+TK_LeftShiftAssign,
+TK_RightShiftAssign,
+TK_DoubleColon,
+TK_At,
+TK_Decrement,
+TK_Increment,
+TK_PostDecrement,
+TK_PostIncrement,
+TK_LesserThenOrEqual,
+TK_GreaterThenOrEqual,
+TK_Equals,
+TK_And,
+TK_Or,
+TK_NotEquals,
+TK_LeftShift,
+TK_RightShift,
+TK_Arrow,
+TK_ExprSizeof,
+TK_DocComment,
+TK_Comment,
+TK_Identifier,
+TK_StringLit,
+TK_U8Lit,
+TK_Character,
+TK_Error,
+TK_Float,
+TK_Int,
+TK_Keyword,
+}Token_Kind;
+
+struct Token{
+Token_Kind kind;
+union{
+struct{
+U8 (*str);
+S64 len;
+};
+String string;
+};
+union{
+S64 int_val;
+String error_val;
+Intern_String intern_val;
+};
+String file;
+S64 line;
+U8 (*line_begin);
+};
+
+// Generated Array block
+typedef struct Token_Array Token_Array;
+typedef struct Token_Array_Block Token_Array_Block;
+typedef struct Token_Array_Iter Token_Array_Iter;
+struct Token_Array_Block{
+Token_Array_Block*next;
+Token data[4096];
+};
+struct Token_Array_Iter{
+Token_Array_Block*block;
+U32 iter_len;
+U32 iter_block;
+U32 total_len;
+U32 total_block;
+};
+struct Token_Array{
+Token_Array_Block first;
+Token_Array_Block*last;
+Token_Array_Iter it;
+Arena*arena;
+S64 block;
+S64 len;
+};
+
+struct Lex_Stream{
+U8 (*stream);
+U8 (*line_begin);
+String filename;
+S64 line;
+};
+
+struct Tokens{
+Token (*tokens);
+S64 len;
+S64 cap;
+S64 iter;
+};
+

ideas.cpp

@@ -32,14 +32,15 @@ Thing :: struct{
   
 }
 
-
+Scope :: scope{
-
+  Thingy::enum:U32{
-
-Thingy::enum:U32{
     @str=10 Value = 1,
     
+  }
 }
 
+//Scope.Thingy
+
 Thing::union{
   union:{
   }

lang.h

@@ -4,7 +4,7 @@
 #define function static
 
 #define assert(x) do{if(!(x)) __debugbreak();}while(0)
-#define assert_msg(x,msg) assert(x)
+#define assert_msg(x,...) assert(x)
 #define not_implemented assert_msg(0, "Not implemented")
 #define invalid_codepath assert_msg(0, "Invalid codepath")
 
@@ -35,12 +35,30 @@ const B32 false = 0;
 #define kib(x) ((x)*1024llu)
 #define mib(x) (kib(x)*1024llu)
 #define gib(x) (mib(x)*1024llu)
+#define string_expand(x) (int)x.len, x.str
 
+typedef struct String_Node String_Node;
+typedef struct String_List String_List;
 typedef struct String{
   U8 *str;
   S64 len;
 }String;
 
+struct String_Node{
+  String_Node *next;
+  union{
+    String string;
+    struct{U8*str; S64 len;};
+  };
+};
+
+struct String_List{
+  String_Node *first;
+  String_Node *last;
+  S64 char_count;
+  S64 node_count;
+};
+
 #define SLLQueuePushMod(f,l,n,next) do{\
 if((f)==0){\
 (f)=(l)=(n);\

lex.c

@@ -1,4 +1,4 @@
-global Token token_end_of_stream = {};
+global Token token_end_of_stream = {0};
 
 function Token *
 token_alloc(Tokens *t){
@@ -363,7 +363,7 @@ break
 function Tokens
 lex_stream(String in_stream, String filename){
   Lex_Stream stream = {in_stream.str, in_stream.str, filename, 0};
-  Tokens tokens = {};
+  Tokens tokens = {0};
   lex_base(&stream, &tokens);
   return tokens;
 }
@@ -498,3 +498,32 @@ token_peek_is_keyword(Parser *p, S64 count, Intern_String keyword){
   }
   return 0;
 }
+
+function void
+lex_test(){
+  Tokens t;
+  t = lex_stream(lit("3252342510 42524 \"U8Literal\""), lit("test"));
+  //tokens_print(t);
+  assert(t.len == 3);
+  assert(t.tokens[0].int_val == 3252342510);
+  assert(t.tokens[1].int_val == 42524);
+  assert(t.tokens[2].kind == TK_U8Lit);
+  assert(token_compare(t.tokens + 2, lit("U8Literal")));
+  
+  t = lex_stream(lit("_identifier Thing Thing2 lit(\"String_Test\")"), lit("test"));
+  //tokens_print(t);
+  assert(t.tokens[0].kind == TK_Identifier);
+  assert(t.tokens[1].kind == TK_Identifier);
+  assert(t.tokens[2].kind == TK_Identifier);
+  assert(t.tokens[3].kind == TK_StringLit);
+  assert(token_compare(t.tokens, lit("_identifier")));
+  assert(token_compare(t.tokens+1, lit("Thing")));
+  assert(token_compare(t.tokens+2, lit("Thing2")));
+  assert(token_compare(t.tokens+3, lit("String_Test")));
+  
+  t = lex_stream(lit("lit(\"String_Test\"{})(324*=+=-/ *% // Comment \n"
+                     "Thing /*Thing*/ += -= =- +/%^&*&&|| |>> << <<= >>=/*Error"), 
+                 lit("test"));
+  assert(t.tokens[0].kind == TK_Error);
+  //tokens_print(t);
+}

lex.h

@@ -1,107 +1,115 @@
 #pragma once
-typedef struct Intern_String{
+#pragma once
-  String s;
+#if 0
-}Intern_String;
+Intern_String :: struct {
+  s: String;
+}
 
-typedef enum Token_Kind{
+@prefix = "TK_"
-  meta("End of stream")TK_End,
+Token_Kind :: enum {
-  meta("*")TK_Mul,
+  @str = "End of stream" End,
-  meta("/")TK_Div,
+  @str = "*" Mul,
-  meta("+")TK_Add,
+  @str = "/" Div,
-  meta("-")TK_Sub,
+  @str = "+" Add,
-  meta("%")TK_Mod,
+  @str = "-" Sub,
-  meta("&")TK_BitAnd,
+  @str = "%" Mod,
-  meta("|")TK_BitOr,
+  @str = "&" BitAnd,
-  meta("^")TK_BitXor,
+  @str = "|" BitOr,
-  meta("~")TK_Neg,
+  @str = "^" BitXor,
-  meta("!")TK_Not,
+  @str = "~" Neg,
-  meta("(")TK_OpenParen,
+  @str = "!" Not,
-  meta(")")TK_CloseParen,
+  @str = "(" OpenParen,
-  meta("{")TK_OpenBrace,
+  @str = " " CloseParen,
-  meta("}")TK_CloseBrace,
+  @str = "{" OpenBrace,
-  meta("[")TK_OpenBracket,
+  @str = "}" CloseBrace,
-  meta("]")TK_CloseBracket,
+  @str = "[" OpenBracket,
-  meta(",")TK_Comma,
+  @str = "]" CloseBracket,
-  meta("#")TK_Pound,
+  @str = "," Comma,
-  meta("?")TK_Question,
+  @str = "#" Pound,
-  meta("...")TK_ThreeDots,
+  @str = "?" Question,
-  meta(";")TK_Semicolon,
+  @str = "..." ThreeDots,
-  meta(".")TK_Dot,
+  @str = ";" Semicolon,
-  meta("<")TK_LesserThen,
+  @str = "." Dot,
-  meta(">")TK_GreaterThen,
+  @str = "<" LesserThen,
-  meta(":")TK_Colon,
+  @str = ">" GreaterThen,
-  meta("=")TK_Assign,
+  @str = ":" Colon,
-  meta("/=")TK_DivAssign,
+  @str = "=" Assign,
-  meta("*=")TK_MulAssign,
+  @str = "/=" DivAssign,
-  meta("%=")TK_ModAssign,
+  @str = "*=" MulAssign,
-  meta("-=")TK_SubAssign,
+  @str = "%=" ModAssign,
-  meta("+=")TK_AddAssign,
+  @str = "-=" SubAssign,
-  meta("&=")TK_AndAssign,
+  @str = "+=" AddAssign,
-  meta("|=")TK_OrAssign,
+  @str = "&=" AndAssign,
-  meta("^=")TK_XorAssign,
+  @str = "|=" OrAssign,
-  meta("<<=")TK_LeftShiftAssign,
+  @str = "^=" XorAssign,
-  meta(">>=")TK_RightShiftAssign,
+  @str = "<<=" LeftShiftAssign,
-  meta("::")TK_DoubleColon,
+  @str = ">>=" RightShiftAssign,
-  meta("@")TK_At,
+  @str = "::" DoubleColon,
-  meta("--")TK_Decrement,
+  @str = "@" At,
-  meta("++")TK_Increment,
+  @str = "--" Decrement,
-  meta("--")TK_PostDecrement,
+  @str = "++" Increment,
-  meta("++")TK_PostIncrement,
+  @str = "--" PostDecrement,
-  meta("<=")TK_LesserThenOrEqual,
+  @str = "++" PostIncrement,
-  meta(">=")TK_GreaterThenOrEqual,
+  @str = "<=" LesserThenOrEqual,
-  meta("==")TK_Equals,
+  @str = ">=" GreaterThenOrEqual,
-  meta("&&")TK_And,
+  @str = "==" Equals,
-  meta("||")TK_Or,
+  @str = "&&" And,
-  meta("!=")TK_NotEquals,
+  @str = "||" Or,
-  meta("<<")TK_LeftShift,
+  @str = "!=" NotEquals,
-  meta(">>")TK_RightShift,
+  @str = "<<" LeftShift,
-  meta("->")TK_Arrow,
+  @str = ">>" RightShift,
-  meta("sizeof")TK_ExprSizeof,
+  @str = "->" Arrow,
-  TK_DocComment,
+  @str = "sizeof" ExprSizeof,
-  TK_Comment,
+  DocComment,
-  TK_Identifier,
+  Comment,
-  TK_StringLit,
+  Identifier,
-  TK_U8Lit,
+  StringLit,
-  TK_Character,
+  U8Lit,
-  TK_Error,
+  Character,
-  TK_Float,
+  Error,
-  TK_Int,
+  Float,
-  TK_Keyword,
+  Int,
+  Keyword,
 }Token_Kind;
 
-typedef struct Token{
+@array(type=block)
-  Token_Kind kind;
+Token :: struct{
-  union{
+  kind: Token_Kind;
-    struct{
-      U8 *str;
-      S64 len;
-    };
-    String string;
-  };
   union {
-    S64 int_val;
+    struct{
-    String error_val;
+      str: U8*;
-    Intern_String intern_val;
+      len: S64;
-  };
+    }
+    string: String;
+  }
+  union {
+    int_val: S64;
+    error_val: String;
+    intern_val: Intern_String;
+  }
   
-  String file;
+  file: String;
-  S64 line;
+  line: S64;
-  U8 *line_begin;
+  line_begin: U8*;
-} Token;
+}
 
-typedef struct Tokens{
+Lex_Stream :: struct{
-  Token *tokens;
+  stream: U8 *;
-  S64 len;
+  line_begin: U8 *;
-  S64 cap;
+  filename: String;
-  S64 iter;
+  line: S64;
-}Tokens;
+}
 
-typedef struct Lex_Stream{
+Tokens :: struct{
-  U8 *stream;
+  tokens: Token*;
-  U8 *line_begin;
+  len:  S64;
-  String filename;
+  cap:  S64;
-  S64 line;
+  iter: S64;
-}Lex_Stream;
+}
+
+#endif
+#include "generated_lex.h"
+#include "generated_lex.cpp"
 
 //-----------------------------------------------------------------------------
 // 

main.c

@@ -6,117 +6,16 @@
 #include "os.h"
 #include "memory.h"
 
-#include "lex.h"
-#include "parser.h"
-#include "expr.h"
-#include "ast.h"
-
-global FILE *global_output_file;
-#define lex_print(...) fprintf(global_output_file, __VA_ARGS__)
-#define lex_new_line() lex_print("\n")
-
 #include "common.c"
 #include "memory.c"
-#include "parser.c"
+#include "scratch.c"
 #include "os_win32.c"
 
-#include "lex.c"
+#include "new_lex.c"
-#include "expr.c"
+#include "new_ast.c"
-#include "ast.c"
+#include "new_parse.c"
-#include "parse_expr.c"
-#include "parse_decl.c"
-#include "print.c"
-#include "codegen_c.c"
 
-function void
+int main(){
-lex_test(){
-  Tokens t;
-  t = lex_stream(lit("3252342510 42524 \"U8Literal\""), lit("test"));
-  //tokens_print(t);
-  assert(t.len == 3);
-  assert(t.tokens[0].int_val == 3252342510);
-  assert(t.tokens[1].int_val == 42524);
-  assert(t.tokens[2].kind == TK_U8Lit);
-  assert(token_compare(t.tokens + 2, lit("U8Literal")));
-  
-  t = lex_stream(lit("_identifier Thing Thing2 lit(\"String_Test\")"), lit("test"));
-  //tokens_print(t);
-  assert(t.tokens[0].kind == TK_Identifier);
-  assert(t.tokens[1].kind == TK_Identifier);
-  assert(t.tokens[2].kind == TK_Identifier);
-  assert(t.tokens[3].kind == TK_StringLit);
-  assert(token_compare(t.tokens, lit("_identifier")));
-  assert(token_compare(t.tokens+1, lit("Thing")));
-  assert(token_compare(t.tokens+2, lit("Thing2")));
-  assert(token_compare(t.tokens+3, lit("String_Test")));
-  
-  t = lex_stream(lit("lit(\"String_Test\"{})(324*=+=-/ *% // Comment \n"
-                     "Thing /*Thing*/ += -= =- +/%^&*&&|| |>> << <<= >>=/*Error"), 
-                 lit("test"));
-  assert(t.tokens[0].kind == TK_Error);
-  //tokens_print(t);
-}
-
-function void
-parser_test(){
-  Parser p = {};
-  {
-    parser_init(&p);
-    Intern_String a = intern_string(&p, lit("Thing"));
-    Intern_String b = intern_string(&p, lit("Thing"));
-    assert(a.s.str == b.s.str);
-  }
-  
-  parser_lex_stream(&p, lit("S64 thing; S64 second_var = 10;"), lit("File"));
-  assert(token_match(&p, TK_Identifier));
-  assert(token_match(&p, TK_Identifier));
-  assert(token_match(&p, TK_Semicolon));
-  assert(token_match(&p, TK_Identifier));
-  assert(token_match(&p, TK_Identifier));
-  assert(token_match(&p, TK_Assign));
-  assert(token_match(&p, TK_Int));
-  assert(token_match(&p, TK_Semicolon));
-  assert(token_match(&p, TK_End));
-  assert(token_match(&p, TK_End));
-  assert(token_match(&p, TK_End));
-  
-  String exprs[] = {
-    lit("(534>43?435:42,234,cast(S64)32/*todo cast*/,Thing[10][2],Thing(1,2))"), 
-    lit("(4+2*53)"),
-    lit("((4+2)*53)"),
-    lit("++5"),
-    lit("5--"), // @Todo(Krzosa): 
-    lit("-5"), 
-    lit("(+5)"), 
-    lit("sizeof(32) + sizeof(:S32*)"), 
-    lit("cast(S64**)5"), 
-    lit("cast(S64)5+3"), 
-    lit("534>43?435:42"), 
-  };
-  for(S64 i = 0; i < buff_cap(exprs); i++){
-    parser_lex_stream(&p, exprs[i], lit("File"));
-    Expr *expr = parse_expr(&p);
-    assert(expr);
-    //expr_print(&p, expr);
-    //lex_print("\n");
-  }
-  
-  String string = os_read_file(lit("test.cc"));
-  parser_lex_stream(&p, string, lit("Parse"));
-  Decl *decls = parse(&p);
-  assert(decls->list.first);
-  gen_decl(&p, decls);
-  
-}
-
-function S32
-os_main(){
-  global_output_file = fopen("output.cc", "w");
-  assert_msg(global_output_file, "Failed to open output.txt");
   lex_test();
-  parser_test();
+  test_ast();
-  
-  fclose(global_output_file);
-  
-  return 0;
 }

memory.c

@@ -80,12 +80,34 @@ string_fmtv(Arena *arena, const char *str, va_list args1) {
   return (String){(U8 *)result, len};
 }
 
+#define STRING_FMT(arena, str, result) \
+va_list args1; \
+va_start(args1, str); \
+String result = string_fmtv(arena, str, args1); \
+va_end(args1)
+
 function String 
 string_fmt(Arena *arena, const char *str, ...) {
-  va_list args1;
+  STRING_FMT(arena, str, result);
-  va_start(args1, str);
-  String result = string_fmtv(arena, str, args1);
-  va_end(args1);
   return result;
 }
 
+function void
+string_listf(Arena *arena, String_List *list, const char *str, ...){
+  STRING_FMT(arena, str, string);
+  String_Node *node = arena_push_struct(arena, String_Node);
+  node->string = string;
+  SLLQueuePush(list->first, list->last, node);
+  list->char_count += node->string.len;
+  list->node_count += 1;
+}
+
+function String
+string_list_flatten(Arena *arena, String_List *list){
+  String result = {arena_push_size(arena, list->char_count + 1)};
+  for(String_Node *node = list->first; node; node=node->next){
+    memory_copy(result.str+result.len, node->str, node->len);
+    result.len += node->len;
+  }
+  return result;
+}

memory.h

@@ -15,3 +15,6 @@ function void *arena_push_size(Arena *a, SizeU size);
 function String arena_push_string_copy(Arena *arena, String string);
 #define arena_push_array(a,T,c) arena_push_size(a,sizeof(T)*(c))
 #define arena_push_struct(a,T) arena_push_array(a,T,1)
+
+function Arena *arena_begin_scratch();
+function void arena_end_scratch();

new_ast.c

@@ -0,0 +1,295 @@
+
+typedef struct Expr Expr;
+typedef struct Typespec Typespec;
+typedef enum Typespec_Kind{
+  TS_None,
+  TS_Name,
+  TS_Pointer,
+  TS_Array,
+  TS_Function,
+}Typespec_Kind;
+
+struct Typespec{
+  Typespec_Kind kind;
+  Typespec *next;
+  Token *pos;
+  union{
+    Intern_String name;
+    Typespec *base;
+    struct{
+      Typespec *first;
+      Typespec *last;
+      Typespec *ret;
+    }func;
+    struct{
+      Typespec *base;
+      Expr     *size;
+    }arr;
+  };
+};
+
+//-----------------------------------------------------------------------------
+// Expressions
+//-----------------------------------------------------------------------------
+typedef enum Expr_Sizeof_Kind{
+  SIZEOF_Expr,
+  SIZEOF_Type,
+}Expr_Sizeof_Kind;
+
+typedef enum Expr_Kind{
+  EK_None,
+  EK_Int,
+  EK_String,
+  EK_Unary, 
+  EK_Binary, 
+  EK_Ternary,
+  EK_Cast,
+  EK_List, 
+  EK_Call, 
+  EK_Index,
+  EK_Sizeof,
+} Expr_Kind;
+
+struct Expr {
+  Expr_Kind kind;
+  Token *token;
+  Expr  *next;
+  union {
+    U64 int_val;
+    Intern_String intern_val;
+    double float_val;
+    
+    struct {
+      Typespec *type;
+      Expr* expr;
+    } cast;
+    struct {
+      Expr *first;
+      Expr *last;
+    } list;
+    struct {
+      Expr *atom;
+      Expr *list;
+    } call;
+    struct {
+      Expr *atom;
+      Expr *index;
+    } index;
+    struct {
+      Token_Kind op;
+      Expr* expr;
+    } unary;
+    struct {
+      Token_Kind op;
+      Expr* left;
+      Expr* right;
+    } binary;
+    struct {
+      Expr* cond;
+      Expr* on_true;
+      Expr* on_false;
+    } ternary;
+    struct{
+      Expr_Sizeof_Kind kind;
+      union{
+        Typespec *type;
+        Expr *expr;
+      };
+    } size_of;
+  };
+};
+
+//-----------------------------------------------------------------------------
+// Expr
+//-----------------------------------------------------------------------------
+function Expr *
+expr_new(Arena *p, Expr_Kind kind, Token *token){
+  Expr *expr = arena_push_struct(p, Expr);
+  expr->kind = kind;
+  expr->token = token;
+  return expr;
+}
+
+function Expr *
+expr_int(Arena *p, Token *token){
+  assert(token->kind == TK_Int);
+  Expr *expr = expr_new(p, EK_Int, token);
+  expr->int_val = token->int_val;
+  return expr;
+}
+
+function Expr *
+expr_str(Arena *p, Token *token){
+  assert(token->kind == TK_StringLit);
+  Expr *expr = expr_new(p, EK_String, token);
+  expr->intern_val = token->intern_val;
+  return expr;
+}
+
+function Expr *
+expr_unary(Arena *p, Token *op, Expr *exp){
+  Expr *expr = expr_new(p, EK_Unary, op);
+  expr->unary.op = op->kind;
+  expr->unary.expr = exp;
+  return expr;
+}
+
+function Expr *
+expr_binary(Arena *p, Token *op, Expr *left, Expr *right){
+  Expr *expr = expr_new(p, EK_Binary, op);
+  expr->binary.op = op->kind;
+  expr->binary.left = left;
+  expr->binary.right = right;
+  return expr;
+}
+
+function Expr *
+expr_ternary(Arena *p, Token *op, Expr *cond, Expr *on_true, Expr *on_false){
+  Expr *expr = expr_new(p, EK_Ternary, op);
+  expr->ternary.cond = cond;
+  expr->ternary.on_true = on_true;
+  expr->ternary.on_false = on_false;
+  return expr;
+}
+
+function Expr *
+expr_call(Arena *p, Token *token, Expr *atom, Expr *list){
+  Expr *expr = expr_new(p, EK_Call, token);
+  expr->call.atom = atom;
+  expr->call.list = list;
+  return expr;
+}
+
+function Expr *
+expr_index(Arena *p, Token *token, Expr *atom, Expr *index){
+  Expr *expr = expr_new(p, EK_Index, token);
+  expr->index.atom = atom;
+  expr->index.index = index;
+  return expr;
+}
+
+function Expr *
+expr_cast(Arena *p, Token *token, Typespec *type, Expr *exp){
+  Expr *expr = expr_new(p, EK_Cast, token);
+  expr->cast.type = type;
+  expr->cast.expr = exp;
+  return expr;
+}
+
+function Expr *
+expr_sizeof_type(Arena *p, Token *token, Typespec *type){
+  Expr *expr = expr_new(p, EK_Sizeof, token);
+  expr->size_of.kind = SIZEOF_Type;
+  expr->size_of.type = type;
+  return expr;
+}
+
+function Expr *
+expr_sizeof_expr(Arena *p, Token *token, Expr *in_expr){
+  Expr *expr = expr_new(p, EK_Sizeof, token);
+  expr->size_of.kind = SIZEOF_Expr;
+  expr->size_of.expr = in_expr;
+  return expr;
+}
+
+function Expr *
+expr_list(Arena *p, Token *token){
+  Expr *expr = expr_new(p, EK_List, token);
+  return expr;
+}
+
+function void
+expr_list_push(Expr *list, Expr *expr){
+  SLLQueuePush(list->list.first, list->list.last, expr);
+}
+
+//-----------------------------------------------------------------------------
+// 
+//-----------------------------------------------------------------------------
+function void
+token_print(Token *token){
+  printf("%.*s", (S32)token->len, token->str);
+}
+
+function void
+expr_print(Expr *expr){
+  switch(expr->kind) {
+    case EK_Int: 
+    case EK_String: {
+      token_print(expr->token);
+    } break;
+    
+    case EK_Sizeof:{
+      printf("sizeof(");
+      if(expr->size_of.kind == SIZEOF_Expr){
+        expr_print(expr->size_of.expr);
+      }
+      else{
+        assert(expr->size_of.kind == SIZEOF_Type);
+        //print_typespec(expr->size_of.type);
+      }
+      printf(")");
+    }break;
+    
+    case EK_Binary:{
+      printf("(");
+      expr_print(expr->binary.left);
+      token_print(expr->token);
+      expr_print(expr->binary.right);
+      printf(")");
+    } break;
+    case EK_Unary:{
+      printf("(");
+      token_print(expr->token);
+      expr_print(expr->unary.expr);
+      printf(")");
+    } break;
+    
+    case EK_Ternary:{
+      printf("(");
+      expr_print(expr->ternary.cond);
+      printf("?");
+      expr_print(expr->ternary.on_true);
+      printf(":");
+      expr_print(expr->ternary.on_false);
+      printf(")");
+    } break;
+    case EK_List:{
+      printf("(");
+      for(Expr *n = expr->list.first; n; n=n->next){
+        expr_print(n);
+        if(n!=expr->list.last) printf(",");
+      }
+      printf(")");
+    }break;
+    
+    case EK_Cast:{
+      printf("(");
+      printf("(");
+      //print_typespec(expr->cast.type);
+      printf(")");
+      expr_print(expr->cast.expr);
+      printf(")");
+    } break;
+    
+    case EK_Index:{
+      expr_print(expr->index.atom);
+      printf("[");
+      expr_print(expr->index.index);
+      printf("]");
+    }break;
+    
+    case EK_Call:{
+      expr_print(expr->call.atom);
+      printf("(");
+      expr_print(expr->call.list);
+      printf(")");
+    }break;
+    default: {invalid_codepath;} break;
+  }
+}
+
+function void
+test_ast(){
+  
+}

new_lex.c

@@ -0,0 +1,516 @@
+global Intern_String keyword_if;
+global Intern_String keyword_for;
+global Intern_String keyword_else;
+global Intern_String keyword_sizeof;
+global Intern_String keyword_typeof;
+global Intern_String keyword_while;
+global Intern_String keyword_switch;
+global Intern_String keyword_case;
+global Intern_String keyword_struct;
+global Intern_String keyword_enum;
+global Intern_String keyword_union;
+global U8 *first_keyword;
+global U8 *last_keyword;
+
+function void
+init_default_keywords(Intern_Table *t){
+  keyword_if = intern_string(t, lit("if"));
+  first_keyword = keyword_if.s.str;
+  
+  keyword_for = intern_string(t, lit("for"));
+  keyword_else = intern_string(t, lit("else"));
+  keyword_sizeof = intern_string(t, lit("sizeof"));
+  keyword_typeof = intern_string(t, lit("typeof"));
+  keyword_while = intern_string(t, lit("while"));
+  keyword_switch = intern_string(t, lit("switch"));
+  keyword_case = intern_string(t, lit("case"));
+  keyword_struct = intern_string(t, lit("struct"));
+  keyword_enum = intern_string(t, lit("enum"));
+  
+  keyword_union = intern_string(t, lit("union"));
+  last_keyword = keyword_union.s.str;
+}
+
+function B32 
+lex_is_keyword(Intern_String str){
+  B32 result = str.s.str >= first_keyword && str.s.str <= last_keyword;
+  return result;
+}
+
+typedef enum Token_Kind{
+  TK_End,
+  TK_Mul,
+  TK_Div,
+  TK_Add,
+  TK_Sub,
+  TK_Mod,
+  TK_BitAnd,
+  TK_BitOr,
+  TK_BitXor,
+  TK_Neg,
+  TK_Not,
+  TK_OpenParen,
+  TK_CloseParen,
+  TK_OpenBrace,
+  TK_CloseBrace,
+  TK_OpenBracket,
+  TK_CloseBracket,
+  TK_Comma,
+  TK_Pound,
+  TK_Question,
+  TK_ThreeDots,
+  TK_Semicolon,
+  TK_Dot,
+  TK_LesserThen,
+  TK_GreaterThen,
+  TK_Colon,
+  TK_Assign,
+  TK_DivAssign,
+  TK_MulAssign,
+  TK_ModAssign,
+  TK_SubAssign,
+  TK_AddAssign,
+  TK_AndAssign,
+  TK_OrAssign,
+  TK_XorAssign,
+  TK_LeftShiftAssign,
+  TK_RightShiftAssign,
+  TK_DoubleColon,
+  TK_At,
+  TK_Decrement,
+  TK_Increment,
+  TK_PostDecrement,
+  TK_PostIncrement,
+  TK_LesserThenOrEqual,
+  TK_GreaterThenOrEqual,
+  TK_Equals,
+  TK_And,
+  TK_Or,
+  TK_NotEquals,
+  TK_LeftShift,
+  TK_RightShift,
+  TK_Arrow,
+  TK_ExprSizeof,
+  TK_DocComment,
+  TK_Comment,
+  TK_Identifier,
+  TK_StringLit,
+  TK_Character,
+  TK_Error,
+  TK_Float,
+  TK_Int,
+  TK_Keyword,
+}Token_Kind;
+
+typedef struct Token{
+  Token_Kind kind;
+  union{
+    String string;
+    struct{U8 *str; S64 len;};
+  };
+  
+  union {
+    U64 int_val;
+    F64 float_val;
+    String error_val;
+    Intern_String intern_val;
+  };
+  
+  String file;
+  S32    line;
+  U8    *line_begin;
+}Token;
+#include "token_array.c"
+
+typedef struct Lex_Stream{
+  String stream;
+  S64    iter;
+  
+  U8    *line_begin;
+  String file;
+  S32    line;
+}Lex_Stream;
+
+
+function U8
+lexc(Lex_Stream *s){
+  return s->stream.str[s->iter];
+}
+
+function U8
+lexci(Lex_Stream *s, S32 i){
+  return s->stream.str[s->iter+i];
+}
+
+function U8 *
+lexcp(Lex_Stream *s){
+  return s->stream.str + s->iter;
+}
+
+function B32
+lex_is_whitespace(U8 c){
+  B32 result = c == '\n' || c == '\r' || c == ' ' || c == '\r';
+  return result;
+}
+
+function B32
+lex_is_alphabetic(U8 c){
+  B32 result = (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z');
+  return result;
+}
+
+function B32
+lex_is_numeric(U8 c){
+  B32 result = c >= '0' && c <= '9';
+  return result;
+}
+
+function B32 
+lex_is_alphanumeric(U8 c){
+  B32 result = lex_is_numeric(c) || lex_is_alphabetic(c);
+  return result;
+}
+
+function void
+lex_set_len(Lex_Stream *s, Token *token){
+  assert(lexcp(s) >= token->str);
+  token->len = lexcp(s) - token->str;
+}
+
+function void
+token_error(Token *t, String error_val){
+  t->kind = TK_Error;
+  t->error_val = error_val;
+}
+
+function void
+lex_parse_u64(Token *t){ 
+  U64 result = 0;
+  U64 m = 1;
+  for(S64 i = t->len - 1; i >= 0; --i){
+    U64 val = t->str[i] - '0';
+    U64 new_val = val * m;
+    if((result + new_val) < result){
+      token_error(t, lit("Integer overflow"));
+      return;
+    }
+    result+=new_val;
+    m *= 10;
+  }
+  t->int_val = result;
+}
+
+function void
+lex_advance(Lex_Stream *s){
+  if(s->iter >= s->stream.len){
+    return;
+  }
+  else if(lexc(s) == '\n'){
+    s->iter++;
+    s->line++;
+    s->line_begin = lexcp(s);
+  }
+  else{
+    s->iter++;
+  }
+}
+
+function Token
+token_int(U64 val){
+  Token result = {.kind = TK_Int, .int_val=val};
+  return result;
+}
+
+function void 
+lex_parse_string(Lex_Stream *s, Token *t, U8 c){
+  for(;;){
+    if(lexc(s) == '\\') lex_advance(s);
+    else if(lexc(s) == c) break;
+    else if(lexc(s) == 0){
+      token_error(t, lit("Unterminated string, reached end of file"));
+      break;
+    }
+    lex_advance(s);
+  }
+  if(t->kind != TK_Error){
+    lex_advance(s);
+    lex_set_len(s,t);
+  }
+}
+
+#define CASE2(op, OpName, Assign)                                                      \
+case op:                                                                             \
+if (lexc(s) == '=') {                                                                \
+lex_advance(s);                                                                    \
+t.kind = Assign;                                                                  \
+} else {                                                                             \
+t.kind = OpName;                                                                  \
+}                                                                                    \
+break
+#define CASE3(op, OpName, Assign, Incr)                                                \
+case op:                                                                             \
+if (lexc(s) == '=') {                                                                \
+lex_advance(s);                                                                    \
+t.kind = Assign;                                                                  \
+} else if (lexc(s) == op) {                                                          \
+lex_advance(s);                                                                    \
+t.kind = Incr;                                                                    \
+} else {                                                                             \
+t.kind = OpName;                                                                  \
+}                                                                                    \
+break
+
+function void
+lex__stream(Token_Array *array, Lex_Stream *s){
+  while(lexc(s)){
+    while(lex_is_whitespace(lexc(s)))
+      lex_advance(s);
+    
+    Token t = {0};
+    t.str = lexcp(s);
+    t.file = s->file;
+    t.line = s->line;
+    t.line_begin = s->line_begin;
+    lex_advance(s);
+    
+    switch(*t.str){
+      case 0: break;
+      case '@': t.kind = TK_At; break;
+      case '(': t.kind = TK_OpenParen; break;
+      case ')': t.kind = TK_CloseParen; break;
+      case '{': t.kind = TK_OpenBrace; break;
+      case '}': t.kind = TK_CloseBrace; break;
+      case '[': t.kind = TK_OpenBracket; break;
+      case ']': t.kind = TK_CloseBracket; break;
+      case ',': t.kind = TK_Comma; break;
+      case '~': t.kind = TK_Neg; break;
+      case '?': t.kind = TK_Question; break;
+      case ';': t.kind = TK_Semicolon; break;
+      case '#': t.kind = TK_Pound; break;
+      CASE2('!', TK_Not, TK_NotEquals);
+      CASE2('^', TK_BitXor, TK_XorAssign);
+      CASE2('=', TK_Assign, TK_Equals);
+      CASE2('*', TK_Mul, TK_MulAssign);
+      CASE2('%', TK_Mod, TK_ModAssign);
+      CASE3('+', TK_Add, TK_AddAssign, TK_Increment);
+      CASE3('&', TK_BitAnd, TK_AndAssign, TK_And);
+      CASE3('|', TK_BitOr, TK_OrAssign, TK_Or);
+#undef CASE2
+#undef CASE3
+      case '.': {
+        if(lexc(s) == '.' && lexci(s,1) == '.') {
+          lex_advance(s); lex_advance(s);
+          t.kind = TK_ThreeDots;
+        }
+        else {
+          t.kind = TK_Dot;
+        }
+      } break;
+      
+      
+      case '<': {
+        if (lexc(s) == '<') {
+          lex_advance(s);
+          if (lexc(s) == '=') {
+            lex_advance(s);
+            t.kind = TK_LeftShiftAssign;
+          }
+          else {
+            t.kind = TK_LeftShift;
+          }
+        }
+        else if (lexc(s) == '=') {
+          lex_advance(s);
+          t.kind = TK_LesserThenOrEqual;
+        }
+        else {
+          t.kind = TK_LesserThen;
+        }
+      } break;
+      
+      case '>': {
+        if (lexc(s) == '>') {
+          lex_advance(s);
+          if (lexc(s) == '=') {
+            lex_advance(s);
+            t.kind = TK_RightShiftAssign;
+          }
+          else {
+            t.kind = TK_RightShift;
+          }
+        }
+        else if (lexc(s) == '=') {
+          lex_advance(s);
+          t.kind = TK_GreaterThenOrEqual;
+        }
+        else {
+          t.kind = TK_GreaterThen;
+        }
+      } break;
+      
+      case ':': {
+        if (lexc(s) == ':') {
+          lex_advance(s);
+          t.kind = TK_DoubleColon;
+        }
+        else {
+          t.kind = TK_Colon;
+        }
+      } break;
+      
+      case '-':{
+        if (lexc(s) == '=') {
+          lex_advance(s);
+          t.kind = TK_SubAssign;
+        }
+        else if (lexc(s) == '-') {
+          lex_advance(s);
+          t.kind = TK_Decrement;
+        }
+        else if (lexc(s) == '>') {
+          lex_advance(s);
+          t.kind = TK_Arrow;
+        }
+        else {
+          t.kind = TK_Sub;
+        }
+      } break;
+      
+      
+      case '\'':{not_implemented;} break;
+      case '"': {
+        t.kind = TK_StringLit; 
+        lex_parse_string(s,&t,'"');
+        if(t.kind != TK_Error){
+          t.str += 1;
+          t.len -= 2;
+        }
+        t.intern_val = intern_string(&array->interns, t.string);
+      } break;
+      
+      case '/': {
+        if(lexc(s) == '='){
+          t.kind = TK_DivAssign;
+          lex_advance(s);
+        }
+        else if(lexc(s) == '/'){
+          lex_advance(s);
+          t.kind = TK_Comment;
+          for(;;){
+            if(lexc(s) == '\n' || lexc(s) == 0) break;
+            lex_advance(s);
+          }
+          continue;
+        }
+        else if(lexc(s) == '*'){
+          lex_advance(s);
+          t.kind = TK_Comment;
+          for(;;){
+            if(lexc(s) == '*' && lexci(s,1) == '/'){
+              lex_advance(s);
+              lex_advance(s);
+              break;
+            }
+            else if(lexc(s) == 0){
+              token_error(&t, lit("Unterminated block comment"));
+              goto skip_continue;
+            }
+            lex_advance(s);
+          }
+          continue;
+          skip_continue:;
+        }
+        else {
+          t.kind = TK_Div;
+        }
+      } break;
+      
+      case '0':case '1':case '2':case '3':case '4':
+      case '5':case '6':case '7':case '8':case '9':{
+        t.kind = TK_Int;
+        while(lex_is_numeric(lexc(s)))
+          lex_advance(s);
+        lex_set_len(s, &t);
+        lex_parse_u64(&t);
+      } break;
+      
+      case 'A':case 'a':case 'M':case 'm':case 'B':
+      case 'b':case 'N':case 'n':case 'C':case 'c':case 'O':
+      case 'o':case 'D':case 'd':case 'P':case 'p':case 'E':
+      case 'e':case 'Q':case 'q':case 'F':case 'f':case 'R':
+      case 'r':case 'G':case 'g':case 'S':case 's':case 'H':
+      case 'h':case 'T':case 't':case 'I':case 'i':case 'U':
+      case 'u':case 'J':case 'j':case 'V':case 'v':case 'K':
+      case 'k':case 'W':case 'w':case 'L':case 'X':case 'l':
+      case 'x':case 'Z':case 'z':case 'Y':case 'y':case '_': {
+        t.kind = TK_Identifier;
+        while(lex_is_alphanumeric(lexc(s)) || lexc(s) == '_')
+          lex_advance(s);
+        lex_set_len(s,&t);
+        t.intern_val = intern_string(&array->interns, t.string);
+        if(lex_is_keyword(t.intern_val)){
+          t.kind = TK_Keyword;
+        }
+      } break;
+      
+      default: {
+        token_error(&t, lit("Unknown token"));
+      }
+    }
+    
+    if(t.len==0)
+      lex_set_len(s,&t);
+    
+    token_array_push(array, &t);
+  }
+}
+
+function Token_Array
+lex_stream(Arena *arena, String stream, String file){
+  Lex_Stream s = {stream, 0, stream.str, file, 0};
+  Token_Array array = token_array_make(arena);
+  init_default_keywords(&array.interns);
+  lex__stream(&array, &s);
+  return array;
+}
+
+function void
+lex_test(){
+  Arena *scratch = arena_begin_scratch();
+  String test = lit("18446744073709551616{})(@?&+-;....->,:::/**/\"Thing\"//R\n Thingy"
+                    "\"Test_Meme\"+=-===42524 4294967295 18446744073709551615"
+                    "for if while switch");
+  Token_Array array = lex_stream(scratch, test, lit("Test1"));
+  
+  Token_Kind kind[] = {
+    TK_Error,TK_OpenBrace,TK_CloseBrace,TK_CloseParen,TK_OpenParen,
+    TK_At,TK_Question,TK_BitAnd,TK_Add,TK_Sub,TK_Semicolon,
+    TK_ThreeDots, TK_Dot, TK_Arrow, TK_Comma, TK_DoubleColon, TK_Colon,
+    TK_StringLit, TK_Identifier, TK_StringLit, TK_AddAssign, TK_SubAssign,
+    TK_Equals, TK_Int, TK_Int, TK_Int, TK_Keyword, TK_Keyword, 
+    TK_Keyword, TK_Keyword
+  };
+  String strs[] = {
+    lit("18446744073709551616"),lit("{"),lit("}"),lit(")"),lit("("),
+    lit("@"),lit("?"),lit("&"),lit("+"),lit("-"),lit(";"),
+    lit("..."),lit("."),lit("->"),lit(","),lit("::"),lit(":"),
+    lit("Thing"),lit("Thingy"),lit("Test_Meme"), lit("+="),lit("-="),
+    lit("=="),lit("42524"),lit("4294967295"),lit("18446744073709551615"),
+    lit("for"), lit("if"), lit("while"), lit("switch"),
+  };
+  U64 vals[] = {
+    42524, 4294967295, 18446744073709551615llu
+  };
+  
+  int i = 0;
+  int ui = 0;
+  for(Token *t = token_array_iter_begin(&array); t; t = token_array_iter_next(&array)){
+    assert(t->kind == kind[i]);
+    assert(string_compare(t->string, strs[i++]));
+    if(t->kind == TK_Int){
+      assert(t->int_val == vals[ui++]);
+    }
+  }
+  arena_end_scratch();
+  
+}

new_parse.c

@@ -0,0 +1,53 @@
+
+typedef struct Parser{
+  Token_Array array;
+  Arena *arena;
+}Parser;
+
+function Token *
+token_get(Parser *p){
+  Token *result = p->array.iter_bucket->data + p->array.iter_len;
+  return result;
+}
+
+function Token *
+token_next(Parser *p){
+  Token *result = token_array_iter_next(&p->array);
+  return result;
+}
+
+function Token *
+token_match(Parser *p, Token_Kind kind){
+  Token *token = token_get(p);
+  if(token->kind == kind){
+    token = token_next(p);
+    return token;
+  }
+  return 0;
+}
+
+/*
+
+
+*/
+
+function Expr *
+parse_expr_atom(Parser *p){
+  Token *token = 0;
+  if((token = token_match(p, TK_StringLit))){
+    Expr *result = expr_str(p->arena, token);
+    return result;
+  }
+  else if((token = token_match(p, TK_Int))){
+    Expr *result = expr_int(p->arena, token);
+    return result;
+  }
+  
+  invalid_codepath;
+  return 0;
+}
+
+function Expr *
+parse_expr(Parser *p){
+  return 0;
+}

old_main.c

@@ -0,0 +1,66 @@
+#define _CRT_SECURE_NO_WARNINGS
+#include <stdio.h>
+#include <windows.h>
+
+#include "lang.h"
+#include "os.h"
+#include "memory.h"
+
+#include "lex.h"
+#include "parser.h"
+#include "expr.h"
+#include "ast.h"
+
+#include "common.c"
+#include "file.c"
+#include "memory.c"
+#include "parser.c"
+#include "os_win32.c"
+
+#include "lex.c"
+#include "expr.c"
+#include "ast.c"
+#include "parse_expr.c"
+#include "parse_decl.c"
+#include "print.c"
+#include "codegen_c.c"
+
+function void
+full_test(){
+  Parser p = {0};
+  
+#if 1
+  parser_init(&p);
+  String string = os_read_file(lit("test.cc"));
+  parser_lex_stream(&p, string, lit("Parse"));
+  Decl *decls = parse(&p);
+  assert(decls->list.first);
+  {
+    gen_begin(&p.scratch, &p);
+    gen_forward_decl(decls);
+    gen_decl(decls);
+    //gen_code(decls);
+    gen_end();
+  }
+  
+#endif
+  
+  
+}
+
+int
+main(){
+  use_write_file("output.cc");
+  lex_test();
+  parser_test();
+  full_test();
+  
+  Arena arena = {};
+  Token_Array array = token_array_make(&arena);
+  token_array_push(&array, &(Token){});
+  close_all_files();
+  
+  
+  
+  return 0;
+}

os.h

@@ -4,6 +4,8 @@ typedef struct OS_Memory{
   SizeU commit;
   SizeU reserve;
 }OS_Memory;
+typedef struct Arena Arena;
 
 function OS_Memory os_reserve(SizeU size);
 function void os_commit(OS_Memory *m, SizeU size);
+function String os_read_file(Arena *arena, String file);

os_win32.c

@@ -1,36 +1,16 @@
 function S32 os_main();
 const SizeU page_size = 4096;
 
-LRESULT CALLBACK 
-WindowProc(HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam){
-  return DefWindowProcW(hwnd, uMsg, wParam, lParam);;
-}
-
-int
-WinMain(HINSTANCE hInstance, HINSTANCE a, LPSTR b, int nShowCmd){
-  wchar_t *CLASS_NAME  = L"Cool window class";
-  wchar_t *WINDOW_NAME = L"Have a good day!";
-  
-  WNDCLASSW wc = { };
-  wc.lpfnWndProc   = WindowProc;
-  wc.hInstance     = hInstance;
-  wc.lpszClassName = CLASS_NAME;
-  RegisterClassW(&wc);
-  
-  HWND window_handle = CreateWindowExW(0, CLASS_NAME, WINDOW_NAME, WS_OVERLAPPEDWINDOW, CW_USEDEFAULT, CW_USEDEFAULT, CW_USEDEFAULT, CW_USEDEFAULT, 0, 0, hInstance, 0);
-  ShowWindow(window_handle, nShowCmd);
-  return os_main();
-}
-
 function String
-os_read_file(String name){
+os_read_file(Arena *arena, String name){
-  String result = {};
+  String result = {0};
   FILE *f = fopen((char *)name.str, "rb");
+  assert(f);
   fseek(f, 0, SEEK_END);
   result.len = ftell(f);
   fseek(f, 0, SEEK_SET);  /* same as rewind(f); */
   
-  result.str = malloc(result.len + 1);
+  result.str = arena_push_size(arena, result.len + 1);
   fread(result.str, result.len, 1, f);
   fclose(f);
   

output.cc

@@ -1,32 +1,97 @@
-typedef struct OpenGL OpenGL;
+(((534>43)?435:42),234,((S64)32),Thing[10][2],Thing((1,2)))
-struct OpenGL{
+(4+(2*53))
-void(*glVertexAttribPointer)(GLuint, GLint, GLenum, GLboolean, GLsizei, GLvoid*);
+((4+2)*53)
-void(*glBindTexture)(GLenum, GLuint);
+(++5)
-void(*glDrawArrays)(GLenum, GLint, GLsizei);
+(--5)
+(-5)
+(+5)
+(sizeof(32)+sizeof(S32*))
+((S64**)5)
+(((S64)5)+3)
+((534>43)?435:42)
+struct Thing{
+int test;
 };
 
+S32 thing = 100;
+(thing=(thing+10));
+(++thing);
+{
+S32 thing_scoped = 10;
+(thing_scoped+=10);
+}
+typedef struct OpenGL OpenGL;
 typedef struct OS_Memory OS_Memory;
+typedef struct Arena Arena;
+typedef struct String String;
+typedef struct Token Token;
+typedef struct Lex_Stream Lex_Stream;
+typedef struct Expr Expr;
+typedef struct AST_Node AST_Node;
+typedef struct Parser_Error Parser_Error;
+typedef struct Scope Scope;
+typedef struct Parser Parser;
+struct test{
+struct Array_Block{
+Array_Block (*next);
+Based_Type_Represent (data[size]);
+};
+
+struct Array{
+Array_Block first;
+Array_Block (*last);
+S64 block;
+S64 len;
+};
+
+function Based_Type_Represent array_make(Arena (*arena)){
+Test_Type thing;
+Based_Type_Represent result;
+((result.arena)=arena);
+((result.last)=(&(result.first)));
+return result;
+}
+function void array_push(Array (*array), Based_Type_Represent (*item)){
+if (((array->len)+1)>size){
+assert((array->len));
+Array_Block (*block) = ((Thing)arena_push_struct(((array->arena),sizeof(Array_Block))));
+((array->last)=(((array->last).next)=block));
+((array->len)=0);
+((array->block)+=1);
+}
+(((array->last).data)[(++(array->len))]=(*item));
+}
+};
+
+struct OpenGL{
+void (*glVertexAttribPointer)(GLuint, GLint, GLenumGLenum, GLboolean, GLsizei, GLvoid *);
+void (*glBindTexture)(GLenum, GLuint);
+void (*glDrawArrays)(GLenumGLenum, GLint, GLsizei);
+int (*(test_array[10]));
+Things (*thing);
+S64 thing_cap;
+S64 thing_len;
+};
+
 struct OS_Memory{
-void* data;
+void (*data);
 SizeU commit;
 SizeU reserve;
 };
 
-typedef struct Arena Arena;
 struct Arena{
 OS_Memory memory;
 U64 len;
 U64 alignment;
 };
 
-typedef struct String String;
 struct String{
-U8* str;
+U8 (*str);
 S64 len;
 };
 
-typedef Intern_String String;
+typedef String Intern_String;
-enum Token_Kind{
+typedef enum Token_Kind{
 TK_End,
 TK_Mul,
 TK_Div,
@@ -90,37 +155,29 @@ TK_Error,
 TK_Float,
 TK_Int,
 TK_Keyword,
-};
+}Token_Kind;
 
-typedef struct Token Token;
 struct Token{
 Token_Kind kind;
 String string;
-union {
+union{
 S64 integer_val;
 String error_val;
 Intern_String intern_val;
 };
 String file;
 S64 line;
-U8* line_begin;
+U8 (*line_begin);
 };
 
-typedef struct Tokens Tokens;
-struct Tokens{
-Token* tokens;
-S64 iter;
-};
-
-typedef struct Lex_Stream Lex_Stream;
 struct Lex_Stream{
-U8* stream;
+U8 (*stream);
-U8* line_begin;
+U8 (*line_begin);
 String filename;
 S64 line;
 };
 
-enum Expr_Kind{
+typedef enum Expr_Kind{
 EK_None,
 EK_Atom,
 EK_Unary,
@@ -130,46 +187,45 @@ EK_Cast,
 EK_List,
 EK_Call,
 EK_Index,
-};
+}Expr_Kind;
 
-typedef struct Expr Expr;
 struct Expr{
 Expr_Kind kind;
-Token* token;
+Token (*token);
-Expr* next;
+Expr (*next);
-union {
+union{
-struct cast_val{
+struct{
-AST_Node* type;
+AST_Node (*type);
-Expr* expr;
+Expr (*expr);
-};
+}cast_val;
-struct list{
+struct{
-Expr* first;
+Expr (*first);
-Expr* last;
+Expr (*last);
-};
+}list;
-struct call{
+struct{
-Expr* atom;
+Expr (*atom);
-Expr* list;
+Expr (*list);
-};
+}call;
-struct index{
+struct{
-Expr* atom;
+Expr (*atom);
-Expr* index;
+Expr (*index);
-};
+}index;
-struct unary{
+struct{
-Expr* expr;
+Expr (*expr);
-};
+}unary;
-struct binary{
+struct{
-Expr* left;
+Expr (*left);
-Expr* right;
+Expr (*right);
-};
+}binary;
-struct ternary{
+struct{
-Expr* cond;
+Expr (*cond);
-Expr* on_true;
+Expr (*on_true);
-Expr* on_false;
+Expr (*on_false);
-};
+}ternary;
 };
 };
 
-enum AST_Kind{
+typedef enum AST_Kind{
 AK_None,
 AK_BaseType,
 AK_Typedef,
@@ -183,61 +239,56 @@ AK_Array,
 AK_Function,
 AK_Variable,
 AK_EnumChild,
-};
+}AST_Kind;
 
-typedef struct AST_Node AST_Node;
 struct AST_Node{
 AST_Kind kind;
-Token* pos;
+Token (*pos);
 Intern_String name;
-AST_Node* next;
+AST_Node (*next);
-AST_Node* next_scope;
+AST_Node (*next_scope);
-AST_Node* first_note;
+AST_Node (*first_note);
-AST_Node* last_note;
+AST_Node (*last_note);
-AST_Node* first_child;
+AST_Node (*first_child);
-AST_Node* last_child;
+AST_Node (*last_child);
-union {
+union{
 SizeU base_type_size;
-AST_Node* pointer;
+AST_Node (*pointer);
-AST_Node* typedef_type;
+AST_Node (*typedef_type);
-AST_Node* variable_type;
+AST_Node (*variable_type);
-AST_Node* func_return_type;
+AST_Node (*func_return_type);
 };
 };
 
-typedef struct Parser_Error Parser_Error;
 struct Parser_Error{
-Parser_Error* next;
+Parser_Error (*next);
 String message;
-Token* token;
+Token (*token);
 };
 
-typedef struct Scope Scope;
 struct Scope{
-Scope* next;
+Scope (*next);
-AST_Node* first;
+AST_Node (*first);
-AST_Node* last;
+AST_Node (*last);
 };
 
-typedef struct Parser Parser;
 struct Parser{
 Arena main_arena;
 Arena intern_table_arena;
 Arena symbol_table_arena;
-Scope* scope_free_list;
+Scope (*scope_free_list);
-Scope* scope_stack;
+Scope (*scope_stack);
-Scope* global_scope;
+Scope (*global_scope);
 S64 symbols_inserted;
 S64 symbols_count;
-AST_Node* symbols;
+AST_Node (*symbols);
-Intern_String* interns;
+Intern_String (*interns);
 S64 interns_in_bytes;
 S64 interns_inserted;
 S64 interns_count;
-U8* first_keyword;
+U8 (*first_keyword);
-U8* last_keyword;
+U8 (*last_keyword);
 Parser_Error default;
 Parser_Error error;
-Tokens token_array;
 };
 

parse_decl.c

@@ -7,6 +7,15 @@ parse_type_function(Parser *p, Token *token){
   Typespec *result = typespec_function(p, token, 0);
   if(!token_is(p, TK_CloseParen)) 
     for(;;) {
+    // Optional name
+    if(token_is(p, TK_Identifier)){
+      if(token_peek_is(p, 1, TK_Colon)){
+        token_next(p);
+        token_next(p);
+      }
+    }
+    
+    // Parse type
     if(token_is(p, TK_Identifier)){
       Typespec *arg = parse_type(p);
       typespec_function_push(result, arg);
@@ -16,18 +25,16 @@ parse_type_function(Parser *p, Token *token){
     }
   }
   token_expect(p, TK_CloseParen);
-  if(token_is(p, TK_Identifier)){
+  if(token_is(p, TK_Identifier))
     result->function_spec.ret = parse_type(p);
-  }
+  else
-  else{
     result->function_spec.ret = typespec_name(p, token_get(p), intern_void);
-  }
+  
   return result;
 }
 
 function Typespec *
 parse_type(Parser *p){
-  
   // Parse as function type or normal
   Token *token = 0;
   Typespec *result = 0;
@@ -58,7 +65,6 @@ parse_type(Parser *p){
     else break;
   }
   
-  
   return result;
 }
 
@@ -72,35 +78,39 @@ parse_expr_assignment(Parser *p){
 }
 
 function void 
-parse_note_list(Parser *ctx, Note *parent) {
+parse_note_list(Parser *p, Note *parent) {
-  if(token_match(ctx, TK_OpenParen)) {
+  if(token_match(p, TK_OpenParen)) {
-    do {
+    if(token_match(p, TK_CloseParen)){
-      Token *name = token_expect(ctx, TK_Identifier);
+      return;
-      Note *current = note_push_new(ctx, parent, name, name->intern_val, 0);
-      parse_note_list(ctx, current);
-      if(token_match(ctx, TK_Assign)) {
-        current->expr = parse_expr(ctx);
     }
-    } while(token_match(ctx, TK_Comma));
+    do {
-    token_expect(ctx, TK_CloseParen);
+      Token *name = token_expect(p, TK_Identifier);
+      Note *current = note_push_new(p, parent, name, name->intern_val, 0);
+      parse_note_list(p, current);
+      if(token_match(p, TK_Assign)) {
+        current->expr = parse_expr(p);
+      }
+    } while(token_match(p, TK_Comma));
+    token_expect(p, TK_CloseParen);
   }
 }
 
 function void 
-parse__notes(Parser *ctx, Note *result) {
+parse__notes(Parser *p, Note *result) {
-  while(token_match(ctx, TK_At)) {
+  while(token_match(p, TK_At)) {
-    Token *name = token_expect(ctx, TK_Identifier);
+    Token *name = token_expect(p, TK_Identifier);
-    Note *current = note_push_new(ctx, result, name, name->intern_val, 0);
+    Note *current = note_push_new(p, result, name, name->intern_val, 0);
-    parse_note_list(ctx, current);
+    parse_note_list(p, current);
-    if(token_match(ctx, TK_Assign)) {
+    if(token_match(p, TK_Assign)) {
-      current->expr = parse_expr(ctx);
+      current->expr = parse_expr(p);
     }
+    token_match(p, TK_Semicolon);
   }
 }
 
 function Note
 parse_notes(Parser *p){
-  Note result = {};
+  Note result = {0};
   parse__notes(p, &result);
   return result;
 }
@@ -172,7 +182,6 @@ function Decl *
 parse_variable(Parser *p, Token *name){
   Typespec *type = parse_type(p);
   Expr *expr = parse_expr_assignment(p);
-  token_expect(p, TK_Semicolon);
   return decl_variable(p, name, name->intern_val, type, expr);
 }
 
@@ -182,14 +191,77 @@ parse_typedef(Parser *p, Token *name){
   return decl_typedef(p, name, name->intern_val, type);
 }
 
-/* 
+function Stmt *
+parse_stmt(Parser *p);
+function Stmt *
+parse_stmt_list(Parser *p){
+  Token *token = token_expect(p, TK_OpenBrace);
+  Stmt *result = stmt_list(p, token);
+  while(!token_match(p, TK_CloseBrace)) {
+    Stmt *stmt = parse_stmt(p);
+    stmt_push(result, stmt);
+  }
+  return result;
+}
+
+function Stmt *
+parse_stmt(Parser *p){
+  Token *token = token_get(p);
+  Decl *decl = parse_decl(p);
+  if(decl){
+    Stmt *result = stmt_decl(p, token, decl);
+    return result;
+  }
+  else if(token_match_keyword(p, keyword_return)){
+    Expr *expr = parse_expr(p);
+    Stmt *result = stmt_return(p, token, expr);
+    token_expect(p, TK_Semicolon);
+    return result;
+  }
+  else if(token_match_keyword(p, keyword_if)){
+    Expr *expr = parse_expr(p);
+    Stmt *if_body = parse_stmt_list(p);
+    Stmt *result = stmt_if(p, token, if_body, expr);
+    Stmt *head = result;
+    while(token_match_keyword(p, keyword_else)){
+      if(token_match_keyword(p, keyword_if)){
+        expr = parse_expr(p);
+        if_body = parse_stmt_list(p);
+        head = head->next = stmt_if(p, token, if_body, expr);
+      }
+      else{
+        if_body = parse_stmt_list(p);
+        head = head->next = stmt_if(p, token, if_body, 0);
+        break;
+      }
+    }
+    return result;
+    
+  }
+  else if((token_is(p, TK_OpenBrace))){
+    Stmt *result = parse_stmt_list(p);
+    return result;
+  }
+  else{
+    Expr *expr = parse_expr(p);
+    token_expect(p, TK_Semicolon);
+    return stmt_expr(p, token, expr);
+  }
+}
+
 function Decl *
 parse_function(Parser *p, Token *name){
   Decl *result = decl_function(p, name, name->intern_val, 0);
   if(!token_is(p, TK_CloseParen)){
     for(;;) {
-      if((name = token_match(p, TK_Identifier)))
+      if(token_peek_is(p, 1, TK_Colon)){
-        decl_function_push(p, result, name, name->intern_val, parse_type(p));
+        if(token_peek_is(p, 2, TK_Identifier) || 
+           token_peek_is(p, 2, TK_OpenParen)){
+          Token *name = parse_get_token_name(p,1);
+          Typespec *type = parse_type(p);
+          decl_function_push(p, result, name, name->intern_val, type);
+        }
+      }
       else if(!token_match(p, TK_Comma))
         break;
     }
@@ -198,13 +270,12 @@ parse_function(Parser *p, Token *name){
   
   if(token_is(p, TK_Identifier))
     result->function_decl.ret = parse_type(p);
+  else
+    result->function_decl.ret = typespec_name(p, token_get(p), intern_void);
   
-  token_expect(p, TK_OpenBrace);
+  result->function_decl.body = parse_stmt_list(p);
-  parse_stmt_list(p, result);
-  token_expect(p, TK_CloseBrace);
   return result;
 }
- */
 
 function Decl *
 parse_decl(Parser *p){
@@ -225,18 +296,21 @@ parse_decl(Parser *p){
       else if(token_peek_is_keyword(p, 2, keyword_typedef)){
         result = parse_typedef(p, parse_get_token_name(p,2));
       }
+      else if(token_peek_is(p, 2, TK_OpenParen)){
+        result = parse_function(p, parse_get_token_name(p,2));
+      }
     }
     else if(token_peek_is(p, 1, TK_Colon)){
       if(token_peek_is(p, 2, TK_Identifier) || 
          token_peek_is(p, 2, TK_OpenParen)){
         result = parse_variable(p, parse_get_token_name(p,1));
+        token_expect(p, TK_Semicolon);
       }
     }
   }
   
   if(result){
-    result->first_note = notes.first;
+    decl_pass_notes(result, &notes);
-    result->last_note = notes.last;
   }
   else if(notes.first != 0){
     parser_push_error(p, token_get(p), "Detected notes that are not attached to anything");
@@ -247,7 +321,7 @@ parse_decl(Parser *p){
 
 function Decl *
 parse(Parser *p){
-  Decl *result = decl_new(p, DECL_List, token_get(p), (Intern_String){});
+  Decl *result = decl_new(p, DECL_List, token_get(p), (Intern_String){0});
   for(;;){
     Decl *decl = 0;
     if(token_is(p, TK_End)) {
@@ -265,188 +339,55 @@ parse(Parser *p){
   return result;
 }
 
-
+function void gen_stmt(Stmt *stmt);
-/*
+function void gen_end();
-function Token *
+function void gen_begin(Arena *arena, Parser *p);
-parse__get_name(Parser *p){
+function void expr_print(Parser *p, Expr *expr);
-Token *name = token_next(p);
+function void
-token_next(p);
+parser_test(){
-token_next(p);
+  Parser p = {0};
-return name;
+  {
-}
+    parser_init(&p);
-
+    Intern_String a = intern_string(&p, lit("Thing"));
-function Note *
+    Intern_String b = intern_string(&p, lit("Thing"));
-parse_enum(Parser *p, Token *name){
+    assert(a.s.str == b.s.str);
-Note *result = 0;
-if(token_expect(p, TK_OpenBrace)){
-  result = ast_enum(p, name, name->intern_val);
-  do{
-    Note notes = parse_notes(p);
-    Token *token = token_match(p, TK_Identifier);
-    if(token){
-      Expr *expr = 0;
-      if(token_match(p, TK_Assign)){
-        expr = parse_expr(p);
   }
   
-      Note *child = ast_enum_child(p, token, token->intern_val, expr);
+  String exprs[] = {
-      ast_node_pass_note_list(child,&notes);
+    lit("(534>43?435:42,234,cast(S64)32/*todo cast*/,Thing[10][2],Thing(1,2))"), 
-      ast_node_push_child(result, child);
+    lit("(4+2*53)"),
-    }
+    lit("((4+2)*53)"),
-    else{
+    lit("++5"),
-      break;
+    lit("5--"), // @Todo(Krzosa): 
-    }
+    lit("-5"), 
-  } while(token_match(p, TK_Comma));
+    lit("(+5)"), 
-  token_expect(p, TK_CloseBrace);
+    lit("sizeof(32) + sizeof(:S32*)"), 
-}
+    lit("cast(S64**)5"), 
-return result;
+    lit("cast(S64)5+3"), 
-}
+    lit("534>43?435:42"), 
-
+  };
-function Note *
+  for(S64 i = 0; i < buff_cap(exprs); i++){
-parse_variable(Parser *p, Token *name){
+    parser_lex_stream(&p, exprs[i], lit("File"));
-Note *result = 0;
+    Expr *expr = parse_expr(&p);
-Token *type_token = token_expect(p, TK_Identifier);
+    assert(expr);
-if(type_token){
+    expr_print(&p, expr);
-  Note *type = symbol_require_type(p, type_token);
+    lex_print("\n");
-  Token *star;
-  while((star  = token_match(p, TK_Mul))){
-    type = ast_type_pointer(p, star, type);
-  }
-  while((star  = token_match(p, TK_OpenBracket))){
-    Expr *expr = parse_expr(p);
-    type = ast_type_array(p, star, type, expr);
-    token_expect(p, TK_CloseBracket);
-  }
-  Expr *expr = 0;
-  if(token_match(p, TK_Assign)){
-    expr = parse_expr(p);
-  }
-  result = ast_variable(p, name, name->intern_val, type, expr);
-}
-return result;
-}
-
-function Note *
-parse_struct(Parser *p, Token *name, AST_Kind kind, B32 is_global){
-Note *result = 0;
-if(token_expect(p, TK_OpenBrace)){
-  result = ast_struct(p, name, name?name->intern_val:(Intern_String){}, kind);
-  if(is_global) symbol_register(p, result);
-  
-  for(;;){
-    Note notes = parse_notes(p);
-    Note *mem = 0;
-    Token *mem_name = token_match(p, TK_Identifier);
-    if(mem_name){
-      if(token_expect(p, TK_Colon)){
-        
-        if(token_match_keyword(p, keyword_struct)){
-          mem = parse_struct(p, mem_name, AK_Struct, false);
-        }
-        else if(token_match_keyword(p, keyword_union)){
-          mem = parse_struct(p, mem_name, AK_Union, false);
-        }
-        //else if(token_match_keyword(p, keyword_enum)){
-        //mem = parse_enum(p, mem_name);
-        //}
-        else if(token_is(p, TK_Identifier)){
-          mem = parse_variable(p, mem_name);
-          token_expect(p, TK_Semicolon);
-        }
-        else parser_push_error(p, mem_name, "Unrecognized token while parsing struct");
-        
-        if(mem){
-          ast_node_pass_note_list(mem, &notes);
-          ast_node_push_child(result, mem);
   }
   
-      }
+  String stmts[] = {
-      else{
+    lit("Thing :: struct { test: int; } "),
-        break;
+    lit("thing: S32 = 100; "),
-      }
+    lit("thing = thing + 10; "),
-    }
+    lit("thing++; "),
-    else if(token_match_keyword(p, keyword_union) &&
+    lit("{ thing_scoped: S32 = 10; thing_scoped += 10; } "),
-            token_match(p, TK_Colon)){
+  };
-      mem = parse_struct(p, 0, AK_Union, false);
+  for(S64 i = 0; i < buff_cap(stmts); i++){
-      ast_node_pass_note_list(mem, &notes);
+    parser_lex_stream(&p, stmts[i], lit("File"));
-      ast_node_push_child(result, mem);
+    Stmt *stmt = parse_stmt(&p);
-    }
+    assert(stmt);
-    else if(token_match_keyword(p, keyword_struct) &&
+    gen_begin(&p.scratch, &p);
-            token_match(p, TK_Colon)){
+    gen_stmt(stmt);
-      mem = parse_struct(p, 0, AK_Struct, false);
+    gen_end();
-      ast_node_pass_note_list(mem, &notes);
+    lex_print("\n");
-      ast_node_push_child(result, mem);
-    }
-    else if(token_expect(p, TK_CloseBrace)){
-      break;
-    }
-    else{
-      break;
-    }
   }
 }
-return result;
-}
-
-function Note *
-parse_typedef(Parser *p, Token *name){
-Token *type_token = token_expect(p, TK_Identifier);
-Note *type = symbol_require_type(p, type_token);
-Note *result = ast_typedef(p, name, name->intern_val, type);
-token_expect(p, TK_Semicolon);
-return result;
-}
-
-function Note_List *
-parse(Parser *p){
-Note_List *result = ast_node_new(p, AK_List, token_get(p), intern_empty);
-
-for(;;){
-  Note *node = 0;
-  Note notes = parse_notes(p);
-  
-  if(token_is(p, TK_End)){
-    break;
-  }
-  else if(token_is(p, TK_Error)){
-    break;
-  }
-  else if(token_is(p, TK_Identifier) &&  // Peeking to be more error averse
-          token_peek_is(p, 1, TK_Colon)){
-    if(token_peek_is_keyword(p, 2, keyword_struct)){
-      node = parse_struct(p, parse__get_name(p), AK_Struct, true);
-      symbol_register(p, node);
-    }
-    else if(token_peek_is_keyword(p, 2, keyword_union)){
-      node = parse_struct(p, parse__get_name(p), AK_Union, true);
-      symbol_register(p, node);
-    }
-    else if(token_peek_is_keyword(p, 2, keyword_enum)){
-      node = parse_enum(p, parse__get_name(p));
-      symbol_register(p, node);
-    }
-    else if(token_peek_is_keyword(p, 2, keyword_typedef)){
-      node = parse_typedef(p, parse__get_name(p));
-      symbol_register(p, node);
-    }
-    else{
-      token_next(p);
-    }
-  }
-  else{
-    token_next(p);
-  }
-  
-  
-  if(node){
-    ast_node_pass_note_list(node, &notes);
-    ast_node_push_child(result, node);
-  }
-  else if(notes.first_note != 0){
-    parser_push_error(p, token_get(p), "Warning: notes got lost");
-  }
-}
-
-return result;
-}
-*/

parse_expr.c

@@ -1,3 +1,4 @@
+
 function Expr* parse_expr(Parser* p);
 function Expr* parse_list_expr(Parser* p);
 function Typespec *parse_type(Parser *p);
@@ -40,6 +41,7 @@ function Expr*
 parse_postfix_expr(Parser* p){
   Expr* result = parse_atom_expr(p);
   while (token_is(p, TK_Dot)
+         || token_is(p, TK_Arrow)
          || token_is(p, TK_OpenParen)
          || token_is(p, TK_OpenBracket)
          || token_is(p, TK_Decrement)
@@ -51,6 +53,11 @@ parse_postfix_expr(Parser* p){
       result = expr_binary(p, op, result, r);
     }
     
+    else if (token_match(p, TK_Arrow)){
+      Expr* r = parse_atom_expr(p);
+      result = expr_binary(p, op, result, r);
+    }
+    
     else if (token_match(p, TK_OpenParen)){
       Expr* list = 0;
       if (!token_match(p, TK_CloseParen)){

parser.c

@@ -16,6 +16,10 @@ global Intern_String intern_sizeu;
 global Intern_String keyword_sizeof;
 global Intern_String keyword_cast;
 global Intern_String keyword_enum;
+global Intern_String keyword_for;
+global Intern_String keyword_if;
+global Intern_String keyword_else;
+global Intern_String keyword_return;
 global Intern_String keyword_typedef;
 global Intern_String keyword_struct;
 global Intern_String keyword_union;
@@ -28,6 +32,10 @@ parser_init(Parser *p){
   p->interns = arena_push_array(&p->intern_table_arena, Intern_String, p->interns_count);
   
   keyword_sizeof = intern_string(p, lit("sizeof"));
+  keyword_if = intern_string(p, lit("if"));
+  keyword_else = intern_string(p, lit("else"));
+  keyword_return = intern_string(p, lit("return"));
+  keyword_for = intern_string(p, lit("for"));
   keyword_cast = intern_string(p, lit("cast"));
   keyword_struct = intern_string(p, lit("struct"));
   keyword_enum = intern_string(p, lit("enum"));
@@ -74,7 +82,7 @@ parser_push_error(Parser *p, Token *token, char *str, ...){
     va_end(args1);
   }
   
-  lex_print("Error: %s\n", string.str);// @Todo(Krzosa): 
+  printf("Error: %s %s:%d\n", string.str, token->file.str, (S32)token->line);
   Parser_Error *error = arena_push_struct(&p->main_arena, Parser_Error);
   error->message = string;
   error->next = 0;
@@ -140,7 +148,7 @@ table_index_advance(Table_Index *index){
 
 function Intern_String
 intern_string(Parser *p, String string){
-  Intern_String result = {};
+  Intern_String result = {0};
   Table_Index index = table_index_from_string(string, p->interns_count);
   for(;;){
     Intern_String *intern = p->interns + index.iter;

parser.h

@@ -12,6 +12,7 @@ typedef struct Parser{
   Arena main_arena;
   Arena intern_table_arena;
   Arena symbol_table_arena;
+  Arena scratch;
   
   S64 interns_in_bytes;
   S64 interns_inserted;

scratch.c

@@ -0,0 +1,14 @@
+global Arena global_scratch;
+global Arena_Checkpoint global_scratch_checkpoint;
+
+function Arena *
+arena_begin_scratch(){
+  global_scratch_checkpoint = arena_checkpoint(&global_scratch);
+  return &global_scratch;
+}
+
+function void
+arena_end_scratch(){
+  arena_restore(global_scratch_checkpoint);
+}
+

test.cc

@@ -1,9 +1,53 @@
 #if 0
 
+
+@register
+@param_expr(size)
+@param_type(Type)
+test::struct{
+  Array_Block::struct{
+    next: Array_Block*;
+    data: Type[size];
+  }
+  
+  @inline Array::struct{
+    first: Array_Block;
+    last : Array_Block*;
+    block: S64;
+    len  : S64;
+  }
+  
+  array_make::(arena: Arena*) Type {
+    thing: Test_Type;
+    result: Type;
+    result.arena = arena;
+    result.last = &result.first;
+    return result;
+  }
+  
+  array_push::(array: Array*, item: Type*){
+    if array->len+1 > size{
+      assert(array->len);
+      block: Array_Block* = cast(Thing)arena_push_struct(array->arena, sizeof(:Array_Block));
+      array->last = array->last.next = block;
+      array->len = 0;
+      array->block += 1;
+    }
+    array->last.data[array->len++] = *item;
+  }
+}
+
+@test(size = 4096)
 OpenGL :: struct{
   glVertexAttribPointer: (GLuint, GLint, GLenum, GLboolean, GLsizei, GLvoid*);
   glBindTexture: (GLenum, GLuint);
   glDrawArrays: (GLenum, GLint, GLsizei);
+  
+  test_array: int*[10];
+  
+  thing    : Things*; 
+  thing_cap: S64;
+  thing_len: S64;
 }
 
 OS_Memory::struct{
@@ -92,6 +136,7 @@ Token_Kind::enum{
   Keyword,
 }
 
+@array(type=block)
 Token::struct{
   kind:Token_Kind;
   @using string:String;
@@ -105,11 +150,6 @@ Token::struct{
   line_begin:U8*;
 }
 
-Tokens::struct{
-  @array tokens: Token*;
-  iter  : S64;
-}
-
 Lex_Stream::struct{
   stream:     U8*;
   line_begin: U8*;
@@ -217,6 +257,7 @@ Scope :: struct{
   last : AST_Node*;
 }
 
+
 Parser :: struct{
   main_arena: Arena;
   intern_table_arena: Arena;
@@ -247,7 +288,7 @@ Parser :: struct{
   
   // This works ok
   @sllqueue error: Parser_Error;
-  @using token_array: Tokens;
+  //@using token_array: Tokens;
 }
 
 /*

token_array.c

@@ -0,0 +1,100 @@
+
+typedef struct Token_Bucket Token_Bucket;
+typedef struct Token_Array Token_Array;
+
+struct Token_Bucket{
+  Token_Bucket *next;
+  Token data[4096];
+};
+
+// @Note(Krzosa): 
+// Interns has an arena - this arena points to arena in Token_Array
+// Token_Array arena is used to allocate the token buckets, Intern table and Intern_Strings
+// 
+// If we want to resize the table in the future we need a second arena or use the heap
+// For now lets leave it this way
+// or
+// We could potentially also use the bucket array and allocate additional buckets when resizing
+// the table. In that case we would need to traverse the bucket list when inserting
+struct Token_Array{
+  Intern_Table interns;
+  
+  Token_Bucket first;
+  Token_Bucket *last;
+  S64 len;
+  S64 block;
+  Arena *arena; 
+  
+  // @Note(Krzosa): Iterator
+  Token_Bucket *iter_bucket;
+  S64 iter_len;
+  S64 iter_block;
+};
+
+function Token_Array
+token_array_make(Arena *arena){
+  Token_Array result = {
+    .last  = &result.first,
+    .arena = arena,
+    .interns = intern_table(arena, 4096*4)
+  };
+  return result;
+}
+
+function void
+token_array_push(Token_Array *array, Token *p){
+  if(array->len >= buff_cap(array->first.data)){
+    Token_Bucket *bucket = arena_push_struct(array->arena, Token_Bucket);
+    array->last = array->last->next = bucket;
+    array->len = 0;
+    array->block += 1;
+  }
+  array->last->data[array->len++] = *p;
+}
+
+function B32
+token_array_iter_is_end(Token_Array *array){
+  B32 result = array->iter_len == array->len && array->iter_block == array->block;
+  return result;
+}
+
+function Token *
+token_array_iter_next(Token_Array *array){
+  if(token_array_iter_is_end(array)){
+    return 0;
+  }
+  if(array->iter_len >= buff_cap(array->first.data)){
+    array->iter_len = 0;
+    array->iter_block += 1;
+    array->iter_bucket = array->iter_bucket->next;
+  }
+  Token *result = array->iter_bucket->data + array->iter_len++;
+  return result;
+}
+
+function Token *
+token_array_iter_peek(Token_Array *array, S64 i){
+  S64 save_len = array->iter_len;
+  Token_Bucket *save_bucket = array->iter_bucket;
+  assert(i < buff_cap(array->first.data));
+  
+  S64 over = i;
+  if(array->iter_len + i >= buff_cap(array->first.data)){
+    over = buff_cap(array->first.data) - (array->iter_len + i);
+    array->iter_len = 0;
+    array->iter_bucket = array->iter_bucket->next;
+  }
+  Token *result = array->iter_bucket->data + array->iter_len + over;
+  array->iter_len = save_len;
+  array->iter_bucket = save_bucket;
+  return result;
+}
+
+function Token *
+token_array_iter_begin(Token_Array *array){
+  array->iter_len = 0;
+  array->iter_block = 0;
+  array->iter_bucket = &array->first;
+  Token *result = token_array_iter_next(array);
+  return result;
+}