Allocator logging

build.bat

@@ -1,8 +1,3 @@
 @echo off
 
-rem clang generate.c  -fdiagnostics-absolute-paths -std=c99 -g -o generate.exe -Wl,user32.lib
-rem generate.exe
-
 clang main.cpp -Wall -Wno-unused-function -fno-exceptions -fdiagnostics-absolute-paths -g -o main.exe -Wl,user32.lib
-rem gcc main.cpp
-rem cl main.c -std:c17

main.cpp

@@ -151,6 +151,36 @@ wrap_around_pow2(U64 x, U64 power_of_2) {
   return r;
 }
 
+function B32
+string_compare(String a, String b){
+  if(a.len != b.len)
+    return false;
+  for(S64 i = 0; i < a.len; i++){
+    if(a.str[i] != b.str[i])
+      return false;
+  }
+  return true;
+}
+
+function U8
+char_to_lower(U8 c){
+  if(c >= 'A' && c <= 'Z')
+    c += 32;
+  return c;
+}
+
+function U8
+char_to_upper(U8 c){
+  if(c >= 'a' && c <= 'z')
+    c -= 32;
+  return c;
+}
+
+force_inline String
+operator""_s(const char *str, size_t size){
+  return String{(U8 *)str, (S64)size};
+}
+
 //-----------------------------------------------------------------------------
 // OS Memory
 //-----------------------------------------------------------------------------
@@ -245,7 +275,7 @@ test_os_memory(){
 enum Allocation_Kind{Allocation_Alloc,Allocation_Resize,Allocation_FreeAll,Allocation_Free,Allocation_Destroy};
 struct Allocator;
 typedef void *Allocator_Proc(Allocator*, Allocation_Kind, void *, SizeU);
-struct Allocator{Allocator_Proc *proc;};
+struct Allocator{Allocator_Proc *proc; String debug_name;};
 
 //-----------------------------------------------------------------------------
 // Memory arenas
@@ -259,7 +289,7 @@ struct Arena:Allocator{
   SizeU len;
 };
 
-function void arena_init(Arena *arena);
+function void arena_init(Arena *arena, String debug_name);
 
 function void
 arena_pop_pos(Arena *arena, SizeU pos){
@@ -282,7 +312,7 @@ arena_push_size(Arena *a, SizeU size){
   SizeU generous_size = size + a->alignment;
   if(a->len+generous_size>a->memory.commit){
     if(a->memory.reserve == 0){
-      arena_init(a);
+      arena_init(a, "Zero initialized arena"_s);
     }
     B32 result = os_commit(&a->memory, generous_size+additional_commit_size);
     assert(result);
@@ -305,7 +335,7 @@ arena_allocator_proc(Allocator *a, Allocation_Kind kind, void *old_pointer, Size
       memory_copy(result, old_pointer, size);
       return result;
     } 
-    case Allocation_Free : invalid_codepath; return 0;
+    case Allocation_Free : return 0;
     case Allocation_FreeAll: arena_clear(arena); return 0;
     case Allocation_Destroy: arena_release(arena); return 0;
   }
@@ -321,17 +351,18 @@ personal_arena_allocator_proc(Allocator *a, Allocation_Kind kind, void *old_poin
 }
 
 function void
-arena_init(Arena *a){
+arena_init(Arena *a, String debug_name){
   a->memory     = os_reserve(default_reserve_size);
   a->alignment  = default_alignment;
+  a->debug_name = debug_name;
   if(!a->proc) a->proc = arena_allocator_proc;
 }
 
 function Arena
-arena_make_personal(){
+arena_make_personal(String debug_name){
   Arena arena = {};
   arena.proc = personal_arena_allocator_proc;
-  arena_init(&arena);
+  arena_init(&arena, debug_name);
   return arena;
 }
 
@@ -397,11 +428,25 @@ struct Thread_Ctx{
   void      *ctx;
   U64        ctx_id;
   Log_Proc  *log_proc;
+  
+  int line;
+  char *file;
 };
 thread_local Thread_Ctx thread_ctx;
 global       Arena pernament_arena;
 global       OS_Heap os_process_heap;
 
+#define report_file_and_line() report__file_and_line(__FILE__, __LINE__)
+#define DEBUG_TRACE(x) (report_file_and_line(), (x))
+function void
+report__file_and_line(const char *file, int line){
+  thread_ctx.file = (char *)file;
+  thread_ctx.line =         line;
+}
+
+//-----------------------------------------------------------------------------
+// Implicit scratch stack
+//-----------------------------------------------------------------------------
 #define Set_Scratch() Scoped_Scratch scratch_##__LINE__
 #define Set_Backup_Scratch() Scoped_Scratch scratch_##__LINE__(true)
 struct Scoped_Scratch{
@@ -422,6 +467,9 @@ struct Scoped_Scratch{
   }
 };
 
+//-----------------------------------------------------------------------------
+// Implicit allocator stack
+//-----------------------------------------------------------------------------
 #define Set_Allocator(a) Scoped_Allocator scoped_##__LINE__(a)
 struct Scoped_Allocator{
   Allocator *allocator;
@@ -442,6 +490,7 @@ thread_ctx_get_user_ctx(U64 id){
   assert(thread_ctx.ctx != 0);
   return thread_ctx.ctx;
 }
+
 #define Get_Ctx(T)   T *ctx = (T *)thread_ctx_get_user_ctx(T##_ID)
 #define Set_Ctx(ctx, id) Scoped_Ctx scoped_ctx_##__LINE__((void *)ctx, id)
 struct Scoped_Ctx{
@@ -456,55 +505,80 @@ struct Scoped_Ctx{
   ~Scoped_Ctx(){thread_ctx.ctx = prev_ctx; thread_ctx.ctx_id = prev_id;}
 };
 
+
 enum Alloc_Flag{AF_None,AF_ZeroMemory};
+//-----------------------------------------------------------------------------
+// Explicit allocator
+//-----------------------------------------------------------------------------
 #define exp_alloc_array(a, T, size,...) (T *)exp_alloc(a, sizeof(T)*(size), ## __VA_ARGS__)
 #define exp_alloc_type(a, T, ...) exp_alloc_array(a, T, 1, ## __VA_ARGS__)
-#define exp_resize_array(a, p, T, size, ...) (T *)exp_resize(a, p, sizeof(T)*(size),## __VA_ARGS__)
+#define exp_alloc(a, size, ...) DEBUG_TRACE(exp__alloc(a, size, ## __VA_ARGS__))
+#define exp_resize(a,p,size) DEBUG_TRACE(exp__resize(a, p, size))
+#define exp_resize_array(a, p, T, size) DEBUG_TRACE((T *)exp_resize(a, p, sizeof(T)*(size)))
+#define exp_free(a, p)       DEBUG_TRACE(exp__free(a, p))
+#define exp_free_all(a)      DEBUG_TRACE(exp__free_all(a))
+#define exp_destroy(a)       DEBUG_TRACE(exp__destroy(a))
+
+#include <stdio.h>
 force_inline void *
-exp_alloc(Allocator *a, SizeU size, Alloc_Flag flag = AF_None){
+exp__alloc(Allocator *a, SizeU size, Alloc_Flag flag = AF_None){
+  printf("Alloc(%s) %s:%d %u\n", a->debug_name.str, thread_ctx.file, thread_ctx.line, (U32)size);
   void *result = a->proc(a, Allocation_Alloc, 0, size);
   if(flag & AF_ZeroMemory) memory_zero(result, size);
   return result;
 }
 force_inline void *
-exp_resize(Allocator *a, void *pointer, SizeU size){
+exp__resize(Allocator *a, void *pointer, SizeU size){
+  printf("Resize(%s) %s:%d %u\n", a->debug_name.str, thread_ctx.file, thread_ctx.line, (U32)size);
   return a->proc(a, Allocation_Resize, pointer, size);
 }
 force_inline void
-exp_free(Allocator *a, void *pointer){
+exp__free(Allocator *a, void *pointer){
+  printf("Free(%s) %s:%d\n", a->debug_name.str, thread_ctx.file, thread_ctx.line);
   a->proc(a, Allocation_Free, pointer, 0);
 }
 force_inline void
-exp_free_all(Allocator *a){
+exp__free_all(Allocator *a){
+  printf("FreeAll(%s) %s:%d\n", a->debug_name.str, thread_ctx.file, thread_ctx.line);
   a->proc(a, Allocation_FreeAll, 0, 0);
 }
 force_inline void
-exp_destroy(Allocator *a){
+exp__destroy(Allocator *a){
+  printf("Destroy(%s) %s:%d\n", a->debug_name.str, thread_ctx.file, thread_ctx.line);
   a->proc(a, Allocation_Destroy, 0, 0);
 }
 
+//-----------------------------------------------------------------------------
+// Implicit allocator
+//-----------------------------------------------------------------------------
 #define imp_alloc_array(T,size,...) (T *)imp_alloc(sizeof(T) * (size), ##__VA_ARGS__)
 #define imp_alloc_type (T,...) imp_alloc_array(T,1, ## __VA_ARGS__)
+#define imp_alloc(size,...) DEBUG_TRACE(imp__alloc(size, ##__VA_ARGS__))
 #define imp_resize_array(p, T,size, ...) (T *)imp_resize(p, sizeof(T) * (size), ##__VA_ARGS__)
+#define imp_resize(p,size) DEBUG_TRACE(imp_resize(p, size))
+#define imp_free(p)        DEBUG_TRACE(imp__free(p))
+#define imp_free_all()     DEBUG_TRACE(imp__free_all())
+#define imp_destroy()      DEBUG_TRACE(imp__destroy())
+
 force_inline void *
-imp_alloc(SizeU size, Alloc_Flag flag=AF_None){
+imp__alloc(SizeU size, Alloc_Flag flag=AF_None){
-  return exp_alloc(thread_ctx.implicit_allocator, size, flag);
+  return exp__alloc(thread_ctx.implicit_allocator, size, flag);
 }
 force_inline void *
-imp_resize(void *pointer, SizeU size){
+imp__resize(void *pointer, SizeU size){
-  return exp_resize(thread_ctx.implicit_allocator, pointer, size);
+  return exp__resize(thread_ctx.implicit_allocator, pointer, size);
 }
 force_inline void
-imp_free(void *pointer){
+imp__free(void *pointer){
-  exp_free(thread_ctx.implicit_allocator, pointer);
+  exp__free(thread_ctx.implicit_allocator, pointer);
 }
 force_inline void
-imp_free_all(){
+imp__free_all(){
-  exp_free_all(thread_ctx.implicit_allocator);
+  exp__free_all(thread_ctx.implicit_allocator);
 }
 force_inline void
-imp_destroy(){
+imp__destroy(){
-  exp_destroy(thread_ctx.implicit_allocator);
+  exp__destroy(thread_ctx.implicit_allocator);
 }
 
 force_inline Allocator *
@@ -515,11 +589,13 @@ imp_get(){
 
 function void 
 thread_ctx_init(){
-  arena_init(thread_ctx.scratch);
+  arena_init(thread_ctx.scratch, "Scratch1"_s);
-  arena_init(thread_ctx.scratch+1);
+  arena_init(thread_ctx.scratch+1, "Scratch2"_s);
-  arena_init(&pernament_arena);
+  arena_init(&pernament_arena, "Pernament Arena"_s);
   os_process_heap.proc             = os_heap_allocator_proc;
   os_process_heap.handle           = GetProcessHeap();
+  os_process_heap.debug_name       = "Win32 Process Heap"_s;
+  
   thread_ctx.implicit_allocator    = &os_process_heap;
 }
 
@@ -573,6 +649,7 @@ handle_log_message(Log_Kind kind, int line, const char *file, const char *str, .
 function void
 test_heap_allocator(){
   OS_Heap heap = win32_os_heap_create(false, mib(1), 0);
+  heap.debug_name = "Test heap"_s;
   Set_Allocator(&heap);
   assert(thread_ctx.implicit_allocator == &heap);
   
@@ -665,9 +742,9 @@ struct Array{
       init(cap);
     }
     else if(len + required_size > cap){
-      S64 cap = (len + required_size)*2;
+      U64 new_cap = max(cap * 2, len+required_size+1);
-      data = exp_resize_array(allocator, data, T, cap);
+      data = exp_resize_array(allocator, data, T, new_cap);
-      cap = cap;
+      cap = new_cap;
     }
   }
   
@@ -676,11 +753,6 @@ struct Array{
     data[len++] = item;
   }
   
-  void add(T *item){
-    grow(1);
-    data[len++] = *item;
-  }
-  
   void clear(){
     len = 0;
   }
@@ -712,7 +784,7 @@ test_array(){
     assert(*it == i);
   }
   
-  Arena arena = arena_make_personal();
+  Arena arena = arena_make_personal("Test personal arena"_s);
   Array<int> array2 = {};
   array2.allocator = &arena;
   for(int i = 0; i < size; i++){
@@ -760,7 +832,7 @@ map_grow(Map *map, S64 new_size){
       map_insert_u64(&new_map, map->data[i].key, map->data[i].value);
     }
   }
-  if(map->data) free(map->data);
+  if(map->data) exp_free(map->allocator, map->data);
   *map = new_map;
 }
 
@@ -834,6 +906,7 @@ map_insert(Map *map, void *key, void *value){
 
 function void
 map_test(){
+  Set_Scratch();
   Map map = {0};
   const SizeU size = 1025;
   for(SizeU i = 1; i < size; i++){
@@ -874,7 +947,6 @@ if(l){\
 //-----------------------------------------------------------------------------
 // String builder
 //-----------------------------------------------------------------------------
-#include <stdio.h>
 struct String_Builder_Block{
   String_Builder_Block *next;
   S64 cap;
@@ -951,36 +1023,6 @@ string_flatten(String_Builder *b){
   return result;
 }
 
-function B32
-string_compare(String a, String b){
-  if(a.len != b.len)
-    return false;
-  for(S64 i = 0; i < a.len; i++){
-    if(a.str[i] != b.str[i])
-      return false;
-  }
-  return true;
-}
-
-function U8
-char_to_lower(U8 c){
-  if(c >= 'A' && c <= 'Z')
-    c += 32;
-  return c;
-}
-
-function U8
-char_to_upper(U8 c){
-  if(c >= 'a' && c <= 'z')
-    c -= 32;
-  return c;
-}
-
-force_inline String
-operator""_s(const char *str, size_t size){
-  return String{(U8 *)str, (S64)size};
-}
-
 function void
 test_string_builder(){
   Set_Scratch();
@@ -1053,12 +1095,12 @@ int main(){
   test_os_memory();
   
   thread_ctx_init();
+  map_test();
   
   test_parse_decl();
   test_parse_expr();
   
   test_array();
-  map_test();
   test_string_builder();
   test_intern_table();
   lex_test();

new_ast.cpp

@@ -14,6 +14,7 @@ const U64 Parse_Ctx_ID = 115151;
 struct Parse_Ctx:Lexer{
   Arena ast_arena;
   Token empty_token;
+  S64 indent;
   S64 pt[256]; // precedence table
   
   void init(){
@@ -24,7 +25,7 @@ struct Parse_Ctx:Lexer{
     pt[TK_Div] = mulp;
     pt[TK_Mul] = mulp;
     
-    arena_init(&ast_arena);
+    arena_init(&ast_arena, "AST Arena"_s);
     lex_init(this);
     keyword_const = intern_string(&interns, "const"_s);
     keyword_struct= intern_string(&interns, "struct"_s);
@@ -41,6 +42,8 @@ struct Parse_Ctx:Lexer{
 enum Ast_Kind{
   AK_None,
   
+  AK_Package,
+  
   AK_Expr_Str,
   AK_Expr_Int,
   AK_Expr_Ident,
@@ -49,6 +52,7 @@ enum Ast_Kind{
   AK_Decl_Func,
   AK_Decl_Func_Arg,
   AK_Decl_Const,
+  AK_Decl_Var,
   
   AK_Typespec_Ident,
   AK_Typespec_Pointer,
@@ -103,6 +107,11 @@ struct Ast_Decl:Ast{
   };
 };
 
+struct Ast_Package:Ast{
+  Intern_String name;
+  Array<Ast_Decl *> decls;
+};
+
 //-----------------------------------------------------------------------------
 // AST Constructors beginning with expressions
 //-----------------------------------------------------------------------------
@@ -178,9 +187,17 @@ ast_decl_func(Token *pos, Intern_String name){
 }
 
 function Ast_Decl *
-ast_decl_const(Token *pos, Intern_String name, Ast_Expr *expr){
+ast_decl_var(Token *pos, Ast_Typespec *typespec, Intern_String name, Ast_Expr *expr){
-  AST_NEW(Decl, AK_Decl_Const, pos);
+  AST_NEW(Decl, AK_Decl_Var, pos);
   result->var.expr     = expr;
+  result->var.typespec = typespec;
   result->name         = name;
   return result;
 }
+
+function Ast_Package *
+ast_package(Token *pos, String name){
+  AST_NEW(Package, AK_Package, pos);
+  result->name = intern_string(&ctx->interns, name);
+  return result;
+}

new_lex.cpp

@@ -138,7 +138,7 @@ lexcp(Lex_Stream *s){
 
 function B32
 lex_is_whitespace(U8 c){
-  B32 result = c == '\r' || c == ' ' || c == '\r';
+  B32 result = c == ' ' || c == '\r';
   return result;
 }
 
@@ -263,8 +263,6 @@ break
 function void
 lex__stream(Intern_Table *table, Array<Token> *array, Lex_Stream *s){
   while(lexc(s)){
-    while(lex_is_whitespace(lexc(s)))
-      lex_advance(s);
     
     Token t = {};
     t.str = lexcp(s);
@@ -300,14 +298,18 @@ lex__stream(Intern_Table *table, Array<Token> *array, Lex_Stream *s){
         t.kind = TK_Semicolon; 
       }break;
       
+      case ' ' : s->stream.str -= 1;
       case '\n': {
         t.kind = TK_NewLine;
         if(lexc(s) == '\r') 
           lex_advance(s);
         
         for(;;){
-          if(lexc(s) == ' ') t.indent++;
+          if(lexc(s) == ' ') {
-          else if(lexc(s) == '\t') t.indent += 2;
+            t.indent++;
+            // @Todo(Krzosa): Detect indentation method, file an error while methods are mixed
+          }
+          else if(lexc(s) == '\t') t.indent++;
           else break;
           lex_advance(s);
         }
@@ -482,6 +484,12 @@ lex__stream(Intern_Table *table, Array<Token> *array, Lex_Stream *s){
       lex_set_len(s,&t);
     
     array->add(t);
+    
+    while(lex_is_whitespace(lexc(s)))
+      lex_advance(s);
+    
+    if(s->iter >= s->stream.len) // End of stream
+      break;
   }
 }
 

new_parse.cpp

@@ -5,8 +5,11 @@ token_get(S64 i = 0){ Get_Ctx(Parse_Ctx);
   if(i >= ctx->tokens.len){
     return &ctx->empty_token;
   }
-  
   Token *result = &ctx->tokens[i];
+  if(result->kind == TK_NewLine){
+    ctx->indent = result->indent;
+  }
+  
   return result;
 }
 
@@ -207,25 +210,38 @@ test_parse_expr(){
 // Parsing declarations
 //-----------------------------------------------------------------------------
 function Ast_Decl *
-parse_decl(){ 
+parse_decl(){Get_Ctx(Parse_Ctx);
   Ast_Decl *result = 0;
   Token *token = token_match(TK_Identifier);
   if(token){
-    if(token_match(TK_DoubleColon)){
+    if(token_match(TK_ColonAssign)){
-      if(token_match_keyword(keyword_const)){
+      if(ctx->indent != 0)
+        parsing_error(token, "Top level declarations shouldn't be indented");
+      
       Ast_Expr *expr = parse_expr();
-        result = ast_decl_const(token, token->intern_val, expr);
+      result = ast_decl_var(token, 0, token->intern_val, expr);
-      }
     }
     else parsing_error(token, "Encountered unexpected token while parsing a top level declarations");
   }
   return result;
 }
 
+function Ast_Package *
+parse_file(){
+  Ast_Package *result = ast_package(token_get(), token_get()->file);
+  while(!token_is(TK_End)){
+    while(token_match(TK_NewLine));
+    Ast_Decl *decl = parse_decl();
+    if(!decl) break;
+    result->decls.add(decl);
+  }
+  return result;
+}
+
 function void
 test_parse_decl(){
   TEST_PARSER();
-  lex_restream(&ctx, "thing :: const 24252\n"_s, "test_parse_decl"_s);
+  lex_restream(&ctx, "thing := 24252\nanother_thing := \"string\"\n\nref := thing"_s, "test_parse_decl"_s);
-  Ast_Decl *result = parse_decl();
+  Ast_Package *result = parse_file();
 }