Organization cleanup, stage arena to Scratch_Arena

base.cpp

@@ -158,6 +158,25 @@ struct Allocator {
   Deallocate *deallocate;
 };
 
+CORE_Static void memory_zero(void *p, size_t size);
+CORE_Static void deallocate_stub(Allocator *, void *) {}
+
+#define allocate_array(a, T, size,...) (T *)allocate_size(a, sizeof(T)*(size),##__VA_ARGS__)
+#define allocate_struct(a, T, ...) allocate_array(a, T, 1,##__VA_ARGS__)
+CORE_Static void *allocate_size(Allocator *allocator, size_t size, bool zero_memory = true) {
+  void *result = allocator->allocate(allocator, size);
+  if (zero_memory) {
+    memory_zero(result, size);
+  }
+  return result;
+}
+
+CORE_Static void deallocate(Allocator *allocator, void *p) {
+  assert(p);
+  allocator->deallocate(allocator, p);
+}
+
+
 //-----------------------------------------------------------------------------
 // Utilities
 //-----------------------------------------------------------------------------
@@ -406,119 +425,3 @@ operator!=(Intern_String a, Intern_String b){
 
 #define Iter_Named(list, it) for(auto it = iterate(list); should_we_continue(&it); advance(&it))
 #define Iter(list) Iter_Named(list, it)
-
-#define allocate_array(a, T, size,...) (T *)allocate_size(a, sizeof(T)*(size),##__VA_ARGS__)
-#define allocate_struct(a, T, ...) allocate_array(a, T, 1,##__VA_ARGS__)
-CORE_Static void *allocate_size(Allocator *allocator, size_t size, bool zero_memory = true) {
-void *result = allocator->allocate(allocator, size);
-  if (zero_memory) {
-    memory_zero(result, size);
-  }
-  return result;
-}
-
-CORE_Static void deallocate(Allocator *allocator, void *p) {
-  assert(p);
-  allocator->deallocate(allocator, p);
-}
-
-//-----------------------------------------------------------------------------
-// Memory OS
-//-----------------------------------------------------------------------------
-struct OS_Memory{
-  size_t commit, reserve;
-  U8 *data;
-};
-CORE_Static OS_Memory os_reserve(size_t size);
-CORE_Static B32 os_commit(OS_Memory *m, size_t size);
-CORE_Static void os_release(OS_Memory *m);
-CORE_Static B32 os_decommit_pos(OS_Memory *m, size_t pos);
-
-//-----------------------------------------------------------------------------
-// Memory arenas
-//-----------------------------------------------------------------------------
-global const size_t default_reserve_size   = gib(4);
-global const size_t default_alignment      = 8;
-global const size_t additional_commit_size = mib(1);
-struct Arena : Allocator {
-  OS_Memory memory;
-  size_t alignment;
-  size_t len;
-  String debug_string;
-};
-
-CORE_Static void
-arena_pop_pos(Arena *arena, size_t pos){
-  pos = clamp_top(pos, arena->len);
-  arena->len = pos;
-}
-
-CORE_Static void *
-arena_pop(Arena *arena, size_t size){
-  size = clamp_top(size, arena->len);
-  arena->len -= size;
-  return arena->memory.data + arena->len;
-}
-
-CORE_Static void
-arena_release(Arena *arena){
-  os_release(&arena->memory);
-}
-
-CORE_Static void
-arena_clear(Arena *arena){
-  arena_pop_pos(arena, 0);
-}
-
-CORE_Static void
-deallocate_stub(Allocator *, void *) {
-}
-
-CORE_Static void *
-arena_push_size(Arena *a, size_t size){
-  size_t generous_size = size + a->alignment;
-  if(a->len+generous_size>a->memory.commit){
-    assert(a->memory.reserve > 0);
-    B32 result = os_commit(&a->memory, generous_size+additional_commit_size);
-    assert(result);
-  }
-
-  a->len = align_up(a->len, a->alignment);
-  assert(a->memory.reserve > a->len + size);
-  void *result = (U8*)a->memory.data + a->len;
-  a->len += size;
-
-  return result;
-}
-
-CORE_Static Arena
-push_arena(Allocator *allocator, size_t size, String debug_name) {
-  Arena result = {};
-  result.memory.data = (U8 *)allocate_size(allocator, size);
-  result.memory.reserve = size;
-  result.alignment = default_alignment;
-  result.debug_string = debug_name;
-  result.allocate     = (Allocator::Allocate *)arena_push_size;
-  result.deallocate   = (Allocator::Deallocate *)deallocate_stub;
-  return result;
-}
-
-CORE_Static void
-arena_init(Arena *a, String debug_name){
-  a->memory       = os_reserve(default_reserve_size);
-  a->alignment    = default_alignment;
-  a->debug_string = debug_name;
-  a->allocate     = (Allocator::Allocate *)arena_push_size;
-  a->deallocate   = (Allocator::Deallocate *)deallocate_stub;
-}
-
-CORE_Static Arena
-arena_sub(Arena *base, size_t size, String debug_name) {
-  Arena result = {};
-  result.memory.data = (U8 *)arena_push_size(base, size);
-  result.memory.commit = size;
-  result.memory.reserve = size;
-  result.alignment = default_alignment;
-  result.len = 0;
-  return result;
-}

base_arena.cpp

@@ -114,3 +114,100 @@ static CRT_Heap make_crt_heap() {
   result.deallocate = crt_deallocate;
   return result;
 }
+
+//-----------------------------------------------------------------------------
+// Memory OS
+//-----------------------------------------------------------------------------
+struct OS_Memory{
+  size_t commit, reserve;
+  U8 *data;
+};
+CORE_Static OS_Memory os_reserve(size_t size);
+CORE_Static B32 os_commit(OS_Memory *m, size_t size);
+CORE_Static void os_release(OS_Memory *m);
+CORE_Static B32 os_decommit_pos(OS_Memory *m, size_t pos);
+
+//-----------------------------------------------------------------------------
+// Memory arenas
+//-----------------------------------------------------------------------------
+global const size_t default_reserve_size   = gib(4);
+global const size_t default_alignment      = 8;
+global const size_t additional_commit_size = mib(1);
+struct Arena : Allocator {
+  OS_Memory memory;
+  size_t alignment;
+  size_t len;
+  String debug_string;
+};
+
+CORE_Static void
+  arena_pop_pos(Arena *arena, size_t pos){
+  pos = clamp_top(pos, arena->len);
+  arena->len = pos;
+}
+
+CORE_Static void *
+  arena_pop(Arena *arena, size_t size){
+  size = clamp_top(size, arena->len);
+  arena->len -= size;
+  return arena->memory.data + arena->len;
+}
+
+CORE_Static void
+  arena_release(Arena *arena){
+  os_release(&arena->memory);
+}
+
+CORE_Static void
+  arena_clear(Arena *arena){
+  arena_pop_pos(arena, 0);
+}
+
+CORE_Static void *
+  arena_push_size(Arena *a, size_t size){
+  size_t generous_size = size + a->alignment;
+  if(a->len+generous_size>a->memory.commit){
+    assert(a->memory.reserve > 0);
+    B32 result = os_commit(&a->memory, generous_size+additional_commit_size);
+    assert(result);
+  }
+
+  a->len = align_up(a->len, a->alignment);
+  assert(a->memory.reserve > a->len + size);
+  void *result = (U8*)a->memory.data + a->len;
+  a->len += size;
+
+  return result;
+}
+
+CORE_Static Arena
+  push_arena(Allocator *allocator, size_t size, String debug_name) {
+  Arena result = {};
+  result.memory.data = (U8 *)allocate_size(allocator, size);
+  result.memory.reserve = size;
+  result.alignment = default_alignment;
+  result.debug_string = debug_name;
+  result.allocate     = (Allocator::Allocate *)arena_push_size;
+  result.deallocate   = (Allocator::Deallocate *)deallocate_stub;
+  return result;
+}
+
+CORE_Static void
+  arena_init(Arena *a, String debug_name){
+  a->memory       = os_reserve(default_reserve_size);
+  a->alignment    = default_alignment;
+  a->debug_string = debug_name;
+  a->allocate     = (Allocator::Allocate *)arena_push_size;
+  a->deallocate   = (Allocator::Deallocate *)deallocate_stub;
+}
+
+CORE_Static Arena
+  arena_sub(Arena *base, size_t size, String debug_name) {
+  Arena result = {};
+  result.memory.data = (U8 *)arena_push_size(base, size);
+  result.memory.commit = size;
+  result.memory.reserve = size;
+  result.alignment = default_alignment;
+  result.len = 0;
+  return result;
+}

c3_big_int.h

@@ -13,7 +13,7 @@ enum CmpRes
 #define FATAL_ERROR(x) compiler_error(0, x)
 
 CORE_Static void compiler_error(Token *token, const char *str, ...);
-const char *bigint_to_error_string(Arena *allocator, const BigInt *bigint, uint64_t base);
+const char *bigint_to_error_string(Allocator *allocator, const BigInt *bigint, uint64_t base);
 void bigint_init_unsigned(BigInt *big_int, uint64_t value);
 void bigint_init_signed(BigInt *big_int, int64_t value);
 void bigint_init_bigint(BigInt *dest, const BigInt *src);

core_compiler.cpp

@@ -6,13 +6,13 @@ core_init_compiler(Core_Ctx *ctx, Allocator *allocator) {
   ctx->perm_push_only = make_push_arena(allocator);
   ctx->perm = &ctx->perm_push_only;
   ctx->scratch = allocate_scratch_arena(ctx->perm, mib(1));
+  ctx->stage_arena = allocate_scratch_arena(ctx->perm, mib(1));
   ctx->heap = allocator;
   ctx->type_map       = map_make(ctx->heap, 2048);
   ctx->gen            = {ctx->perm};
   ctx->helper_builder = {ctx->perm};
   ctx->scope_ids      = 1;
   bigint_allocator    = ctx->perm;
-  arena_init(&ctx->stage_arena, "Compiler stage arena"_s);
 
   ctx->tokens  = array_make<Token>(ctx->heap, 4096 * 4);
   ctx->interns = intern_table_make(ctx->perm, ctx->heap, 2048);
@@ -98,7 +98,6 @@ core_init_compiler(Core_Ctx *ctx, Allocator *allocator) {
 CORE_Static void
 core_bootstrap_compiler(Allocator *allocator) {
   Core_Ctx *ctx = allocate_struct(allocator, Core_Ctx);
-  assert((uintptr_t)allocator->allocate == (uintptr_t)arena_push_size);
   core_init_compiler(ctx, allocator);
 }
 
@@ -215,9 +214,9 @@ resolve_everything_in_module(Ast_Module *module) {
 }
 
 CORE_Static String
-compile_file_to_string(Arena *arena, String filename) {
+compile_file_to_string(Allocator *allocator, String filename) {
   F64 total_time = os_time();
-  core_bootstrap_compiler(arena);
+  core_bootstrap_compiler(allocator);
   pctx->total_time = total_time;
   {
     Ast_Module *module = add_module(0, pctx->intern("Language.core"_s));
@@ -280,7 +279,8 @@ compile_file_to_string(Arena *arena, String filename) {
   assert(module);
   resolve_everything_in_module(module);
 
-  arena_clear(&pctx->stage_arena);
+  
+  pctx->stage_arena->len = 0;
   String result = compile_to_c_code();
   return result;
 }
@@ -292,8 +292,8 @@ const U32 COMPILE_AND_RUN                              = 0x4;
 const U32 COMPILE_TESTING                              = 0x8;
 
 CORE_Static void
-compile_file(Arena *arena, String filename, U32 compile_flags = COMPILE_NULL) {
+compile_file(Allocator *allocator, String filename, U32 compile_flags = COMPILE_NULL) {
-  String result = compile_file_to_string(arena, filename);
+  String result = compile_file_to_string(allocator, filename);
   if (is_flag_set(compile_flags, COMPILE_AND_RUN)) {
     log_info_no_nl("%Q - ", filename);
   }

core_compiler.h

@@ -1,12 +1,21 @@
 
 /*
-  * Get rid of Scratch, move that concept onto the main context, create scratch memory from user allocator
+  @! Change type of Stage allocator 
-  * Simplify allocators, only should be derived from the user allocator
+  @! Look into stage allocator and perhaps 
-  * Cleanup big int allocator
+    use it more often to reduce scenarios
-  * Replace iterator interface
+    with 2 allocators, and simplify stuff
+  @! Cleanup big int allocator
+  @! Replace iterator interface
 
-  * Reset compiler
+  @! Clean way to free all memory and reset the compiler
+  @! Bring the Table<>
+  @! Look into List, check if that's neccessary
 
+  @! Compute time statistics inside context and expose them properly
+ 
+  Probably want to implement a Red Black Tree then
+  probably I wouldn't need any sort of heap based
+  data structure.
   */
 
 struct Lex_Stream{
@@ -17,17 +26,15 @@ struct Lex_Stream{
   Intern_String  file;
   S32            line;
   S32            inside_brace_paren;
-  Array<Token *> indent_stack;
+  Array<Token *> indent_stack; // @scratch_allocated
 };
 
 struct Core_Ctx{
+  Allocator *heap;
   Push_Arena perm_push_only;
   Push_Arena *perm; // Stores: AST, tokens, interns
-  Allocator *heap;
-
   Scratch_Arena *scratch;
-
+  Scratch_Arena *stage_arena;
-  Arena  stage_arena;
 
   // Lexer stuff
   Lex_Stream   stream;

core_main.cpp

@@ -258,10 +258,8 @@ For modules it's a bit different cause they should be distributed as valid.
 */
 
 #include "base.cpp"
-
 #define STB_SPRINTF_IMPLEMENTATION
 #include "stb_sprintf.h"
-
 #include "base_unicode.cpp"
 #include "base_arena.cpp"
 #include "base_data_structures.cpp"

core_types.cpp

@@ -335,5 +335,5 @@ get_typename(Allocator *a, Ast_Type *type){
 
 CORE_Static String
 typestring(Ast_Type *type){
-  return get_typename(&pctx->stage_arena, type);
+  return get_typename(pctx->stage_arena, type);
 }

os_linux.cpp

@@ -6,6 +6,7 @@
 #include <errno.h>
 #include <string.h>
 #include <linux/limits.h>
+#include <stdio.h>
 
 #define POSIX_PAGE_SIZE 4096