Moving to C++ with new idea

main.cpp

@@ -29,7 +29,14 @@ typedef double   F64;
 #define mib(x) (kib(x)*1024llu)
 #define gib(x) (mib(x)*1024llu)
 struct String{U8 *str;S64 len;}; 
+union Intern_String{ 
+  String s; 
+  struct{ U8 *str; S64 len; };
+};
 
+//-----------------------------------------------------------------------------
+// Utilities
+//-----------------------------------------------------------------------------
 function SizeU
 get_align_offset(SizeU size, SizeU align){
   SizeU mask = align - 1;
@@ -69,6 +76,18 @@ memory_zero(void *p, SizeU size){
     pp[i] = 0;
 }
 
+template<class T>
+T max(T a, T b){
+  if(a > b) return a;
+  return b;
+}
+
+template<class T>
+T min(T a, T b){
+  if(a > b) return b;
+  return a;
+}
+
 template<class T>
 T clamp_top(T val, T max){
   if(val > max) val = max;
@@ -88,6 +107,53 @@ T clamp(T min, T val, T max){
   return val;
 }
 
+function U64 
+hash_string(String string) {
+  U64 hash = (U64)14695981039346656037ULL;
+  for (U64 i = 0; i < string.len; i++) {
+    hash = hash ^ (U64)(string.str[i]);
+    hash = hash * (U64)1099511628211ULL;
+  }
+  return hash;
+}
+
+function U64 
+hash_u64(U64 x) {
+  x *= 0xff51afd7ed558ccd;
+  x ^= x >> 32;
+  return x;
+}
+
+function U64 
+hash_ptr(const void *ptr) {
+  return hash_u64((uintptr_t)ptr);
+}
+
+function U64 
+hash_mix(U64 x, U64 y) {
+  x ^= y;
+  x *= 0xff51afd7ed558ccd;
+  x ^= x >> 32;
+  return x;
+}
+
+function U64
+is_pow2(U64 x) {
+  assert(x != 0);
+  B32 result = (x & (x - 1llu)) == 0;
+  return result;
+}
+
+function U64 
+wrap_around_pow2(U64 x, U64 power_of_2) {
+  assert(is_pow2(power_of_2));
+  U64 r = (((x)&((power_of_2)-1llu)));
+  return r;
+}
+
+//-----------------------------------------------------------------------------
+// OS Memory
+//-----------------------------------------------------------------------------
 constexpr SizeU os_page_size = 4096;
 struct OS_Memory{
   SizeU commit, reserve;
@@ -136,8 +202,8 @@ os_decommit_pos(OS_Memory *m, SizeU pos){
   SizeU adjusted_pos = clamp_top(aligned, m->commit);
   SizeU size_to_decommit = m->commit - adjusted_pos;
   if(size_to_decommit){
-    U8 *imp_address = m->data + adjusted_pos;
-    BOOL result = VirtualFree(imp_address, size_to_decommit, MEM_DECOMMIT);
+    U8 *base_address = m->data + adjusted_pos;
+    BOOL result = VirtualFree(base_address, size_to_decommit, MEM_DECOMMIT);
     if(result){
       m->commit -= size_to_decommit;
       return true;
@@ -173,11 +239,17 @@ test_os_memory(){
   assert(memory.data == 0);
 }
 
-enum Allocation_Kind{Allocation_Alloc,Allocation_Resize,Allocation_FreeAll,Allocation_Free};
+//-----------------------------------------------------------------------------
+// Base Allocator stuff
+//-----------------------------------------------------------------------------
+enum Allocation_Kind{Allocation_Alloc,Allocation_Resize,Allocation_FreeAll,Allocation_Free,Allocation_Destroy};
 struct Allocator;
 typedef void *Allocator_Proc(Allocator*, Allocation_Kind, void *, SizeU);
 struct Allocator{Allocator_Proc *proc;};
 
+//-----------------------------------------------------------------------------
+// Memory arenas
+//-----------------------------------------------------------------------------
 global const SizeU default_reserve_size   = gib(4);
 global const SizeU default_alignment      = 8;
 global const SizeU additional_commit_size = mib(1);
@@ -195,6 +267,11 @@ arena_pop_pos(Arena *arena, SizeU pos){
   arena->len = pos;
 }
 
+function void
+arena_release(Arena *arena){
+  os_release(&arena->memory);
+}
+
 function void
 arena_clear(Arena *arena){
   arena_pop_pos(arena, 0);
@@ -202,8 +279,8 @@ arena_clear(Arena *arena){
 
 function void *
 arena_push_size(Arena *a, SizeU size){
-  SizeU generous_size = size;
-  if(a->memory.commit+generous_size>a->memory.commit){
+  SizeU generous_size = size + a->alignment;
+  if(a->len+generous_size>a->memory.commit){
     if(a->memory.reserve == 0){
       arena_init(a);
     }
@@ -223,14 +300,21 @@ arena_allocator_proc(Allocator *a, Allocation_Kind kind, void *old_pointer, Size
   Arena *arena = (Arena *)a;
   switch(kind){
     case Allocation_Alloc: return arena_push_size(arena, size);
-    case Allocation_Resize: return arena_push_size(arena, size);
+    case Allocation_Resize:{
+      void *result = arena_push_size(arena, size);
+      memory_copy(result, old_pointer, size);
+      return result;
+    } 
     case Allocation_Free : invalid_codepath; return 0;
     case Allocation_FreeAll: arena_clear(arena); return 0;
+    case Allocation_Destroy: arena_release(arena); return 0;
   }
+  invalid_codepath;
+  return 0;
 }
 
 force_inline void *
-big_personal_arena_allocator_proc(Allocator *a, Allocation_Kind kind, void *old_pointer, SizeU size){
+personal_arena_allocator_proc(Allocator *a, Allocation_Kind kind, void *old_pointer, SizeU size){
   Arena *arena = (Arena *)a;
   arena->alignment = 1;
   return arena_allocator_proc(a, kind, old_pointer, size);
@@ -243,6 +327,17 @@ arena_init(Arena *a){
   if(!a->proc) a->proc = arena_allocator_proc;
 }
 
+function Arena
+arena_make_personal(){
+  Arena arena = {};
+  arena.proc = personal_arena_allocator_proc;
+  arena_init(&arena);
+  return arena;
+}
+
+//-----------------------------------------------------------------------------
+// OS Heap allocator
+//-----------------------------------------------------------------------------
 struct OS_Heap:Allocator{
   HANDLE handle;
 };
@@ -252,8 +347,14 @@ os_heap_allocator_proc(Allocator *a, Allocation_Kind kind, void *old_pointer, Si
   OS_Heap *heap = (OS_Heap *)a;
   switch(kind){
     case Allocation_FreeAll:{
+      invalid_codepath;
+      return 0;
+    }
+    case Allocation_Destroy:{
       BOOL result = HeapDestroy(heap->handle);
       assert(result != 0);
+      heap->handle = 0;
+      heap->proc = 0;
       return 0;
     }
     case Allocation_Free:{
@@ -271,7 +372,9 @@ os_heap_allocator_proc(Allocator *a, Allocation_Kind kind, void *old_pointer, Si
       assert(result);
       return result;
     }
+    default: invalid_codepath;
   }
+  return 0;
 }
 
 function OS_Heap // max_size == 0 == growing heap
@@ -283,9 +386,16 @@ win32_os_heap_create(B32 multithreaded, SizeU initial_size, SizeU max_size){
   return result;
 }
 
+enum Log_Kind{Log_Kind_Normal, Log_Kind_Error};
+typedef void Log_Proc(Log_Kind kind, String string, char *file, int line);
+//-----------------------------------------------------------------------------
+// Thread Context
+//-----------------------------------------------------------------------------
 struct Thread_Ctx{
-  Arena scratch[2];
+  Arena      scratch[2];
   Allocator *implicit_allocator;
+  void      *ctx;
+  Log_Proc  *log_proc;
 };
 thread_local Thread_Ctx thread_ctx;
 global       Arena pernament_arena;
@@ -323,12 +433,26 @@ struct Scoped_Allocator{
   }
 };
 
-#define exp_alloc_array(a, T, size) (T *)exp_alloc(a, sizeof(T)*(size))
-#define exp_alloc_type(a, T) exp_alloc_array(a, T, 1)
-#define exp_resize_array(a, p, T, size) expr_resize(a, p, sizeof(T)*(size))
+#define Get_Ctx(T)   T *ctx = (T *)thread_ctx.ctx
+#define Set_Ctx(ctx) Scoped_Ctx scoped_ctx_##__LINE__((void *)ctx)
+struct Scoped_Ctx{
+  void *prev_ctx;
+  Scoped_Ctx(void *in_ctx){
+    prev_ctx = thread_ctx.ctx;
+    thread_ctx.ctx = in_ctx;
+  }
+  ~Scoped_Ctx(){thread_ctx.ctx = prev_ctx;}
+};
+
+enum Alloc_Flag{AF_None,AF_ZeroMemory};
+#define exp_alloc_array(a, T, size,...) (T *)exp_alloc(a, sizeof(T)*(size), ## __VA_ARGS__)
+#define exp_alloc_type(a, T, ...) exp_alloc_array(a, T, 1, ## __VA_ARGS__)
+#define exp_resize_array(a, p, T, size, ...) (T *)exp_resize(a, p, sizeof(T)*(size),## __VA_ARGS__)
 force_inline void *
-exp_alloc(Allocator *a, SizeU size){
-  return a->proc(a, Allocation_Alloc, 0, size);
+exp_alloc(Allocator *a, SizeU size, Alloc_Flag flag = AF_None){
+  void *result = a->proc(a, Allocation_Alloc, 0, size);
+  if(flag & AF_ZeroMemory) memory_zero(result, size);
+  return result;
 }
 force_inline void *
 exp_resize(Allocator *a, void *pointer, SizeU size){
@@ -342,13 +466,17 @@ force_inline void
 exp_free_all(Allocator *a){
   a->proc(a, Allocation_FreeAll, 0, 0);
 }
+force_inline void
+exp_destroy(Allocator *a){
+  a->proc(a, Allocation_Destroy, 0, 0);
+}
 
-#define imp_alloc_array(T,size) (T *)imp_alloc(sizeof(T) * (size))
-#define imp_alloc_type (T) imp_alloc_array(T,1)
-#define imp_resize_array(p, T,size) (T *)imp_resize(p, sizeof(T) * (size))
+#define imp_alloc_array(T,size,...) (T *)imp_alloc(sizeof(T) * (size),##__VA_ARGS__)
+#define imp_alloc_type (T,...) imp_alloc_array(T,1,## __VA_ARGS__)
+#define imp_resize_array(p, T,size, ...) (T *)imp_resize(p, sizeof(T) * (size),##__VA_ARGS__)
 force_inline void *
-imp_alloc(SizeU size){
-  return exp_alloc(thread_ctx.implicit_allocator, size);
+imp_alloc(SizeU size, Alloc_Flag flag=AF_None){
+  return exp_alloc(thread_ctx.implicit_allocator, size, flag);
 }
 force_inline void *
 imp_resize(void *pointer, SizeU size){
@@ -362,6 +490,16 @@ force_inline void
 imp_free_all(){
   exp_free_all(thread_ctx.implicit_allocator);
 }
+force_inline void
+imp_destroy(){
+  exp_destroy(thread_ctx.implicit_allocator);
+}
+
+force_inline Allocator *
+imp_get(){
+  assert(thread_ctx.implicit_allocator);
+  return thread_ctx.implicit_allocator;
+}
 
 function void 
 thread_ctx_init(){
@@ -370,6 +508,54 @@ thread_ctx_init(){
   arena_init(&pernament_arena);
   os_process_heap.proc = os_heap_allocator_proc;
   os_process_heap.handle = GetProcessHeap();
+  thread_ctx.implicit_allocator = &os_process_heap;
+}
+
+function String
+string_copy(Allocator *a, String string){
+  U8 *copy = exp_alloc_array(a, U8, string.len+1);
+  memory_copy(copy, string.str, string.len);
+  copy[string.len] = 0;
+  return (String){copy, string.len};
+}
+
+#include <stdio.h>
+function String
+string_fmtv(Allocator *a, const char *str, va_list args1) {
+  va_list args2;
+  va_copy(args2, args1);
+  S64 len = vsnprintf(0, 0, str, args2);
+  va_end(args2);
+  
+  char *result = exp_alloc_array(a, char, len + 1);
+  vsnprintf(result, len + 1, str, args1);
+  
+  String res = {(U8 *)result, len};
+  return res;
+}
+
+#define STRING_FMT(alloc, str, result) \
+va_list args1; \
+va_start(args1, str); \
+String result = string_fmtv(alloc, str, args1); \
+va_end(args1)
+
+function String 
+string_fmt(Allocator *a, const char *str, ...) {
+  STRING_FMT(a, str, result);
+  return result;
+}
+
+#define log(...) handle_log_message(Log_Kind_Normal, __LINE__, __FILE__, ## __VA_ARGS__)
+#define log_error(...) handle_log_message(Log_Kind_Error, __LINE__, __FILE__, ## __VA_ARGS__)
+function void 
+handle_log_message(Log_Kind kind, int line, const char *file, const char *str, ...){
+  Set_Backup_Scratch();
+  STRING_FMT(imp_get(), str, message);
+  if(thread_ctx.log_proc) thread_ctx.log_proc(kind, message, (char *)file, line);
+  else{
+    printf("%s", message.str);
+  }
 }
 
 function void
@@ -382,7 +568,7 @@ test_heap_allocator(){
   result[1023] = 1;
   result = exp_alloc_type(&heap, U8);
   *result = 0;
-  imp_free_all();
+  imp_destroy();
   
   assert(thread_ctx.implicit_allocator == &heap);
   {
@@ -393,8 +579,498 @@ test_heap_allocator(){
   assert(thread_ctx.implicit_allocator == &heap);
 }
 
-int main(){
-  test_heap_allocator();
-  thread_ctx_init();
-  
+struct Test_Context{
+  int value;
+};
+
+function void
+test_custom_context_2(){
+  Get_Ctx(Test_Context);
+  ctx->value += 10;
+}
+
+function void
+test_custom_context_1(){
+  Test_Context context = {};
+  Set_Ctx(&context);
+  Get_Ctx(Test_Context);
+  ctx->value = 10;
+  test_custom_context_2();
+  test_custom_context_2();
+  test_custom_context_2();
+  assert(ctx->value == 40);
+  assert(thread_ctx.ctx == &context);
+}
+
+function void
+test_custom_context(){
+  assert(thread_ctx.ctx == 0);
+  test_custom_context_1();
+  assert(thread_ctx.ctx == 0);
+}
+
+
+//-----------------------------------------------------------------------------
+// Defer
+// http://www.gingerbill.org/article/2015/08/19/defer-in-cpp/
+//-----------------------------------------------------------------------------
+template <typename F>
+struct Defer_Scope {
+  F f;
+  Defer_Scope(F f) : f(f) {}
+  ~Defer_Scope() { f(); }
+};
+
+template <typename F>
+Defer_Scope<F> defer_func(F f) {
+  return Defer_Scope<F>(f);
+}
+#define DEFER_1(x, y) x##y
+#define DEFER_2(x, y) DEFER_1(x, y)
+#define DEFER_3(x)    DEFER_2(x, __COUNTER__)
+#define defer(code)   auto DEFER_3(_defer_) = defer_func([&](){code;})
+
+//-----------------------------------------------------------------------------
+// Array
+//-----------------------------------------------------------------------------
+template<class T>
+struct Array{
+  T  *data;
+  S64 cap;
+  S64 len;
+  Allocator *allocator;
+  
+  T *begin(){ return data; }
+  T *end  (){ return data + len; }
+  T &operator[](S64 i){ return data[i]; }
+};
+#define For(array,it,i) for(SizeU i = 0; i < array.len; i++) for(auto *it = &array[i]; it; it = 0)
+#define IterList(list,it) for(auto *it = list->first; it; it=it->next)
+
+template<class T>
+void array_init(Array<T> *a, S64 size){
+  if(!a->allocator) a->allocator = thread_ctx.implicit_allocator;
+  a->data = exp_alloc_array(a->allocator, T, size);
+  a->cap = size;
+}
+
+template<class T>
+void array_grow(Array<T> *a, S64 required_size){
+  if(a->cap == 0){
+    S64 cap = max(required_size*2, (S64)16);
+    array_init(a, cap);
+  }
+  else if(a->len + required_size > a->cap){
+    S64 cap = (a->len + required_size)*2;
+    a->data = exp_resize_array(a->allocator, a->data, T, cap);
+    a->cap = cap;
+  }
+}
+
+template<class T>
+Array<T> array_make(S64 size){
+  Array<T> result = {};
+  array_init(&result, size);
+  return result;
+}
+
+template<class T>
+T *array_alloc(Array<T> *a, S64 count){
+  array_grow(a, count);
+  T *result = a->data + a->len;
+  a->len += count;
+  return result;
+}
+
+template<class T>
+void array_push(Array<T> *a, T &item){
+  array_grow(a, 1);
+  a->data[a->len++] = item;
+}
+
+template<class T>
+T array_pop_get(Array<T> *a){
+  assert(a->len > 0);
+  return a->data[--a->len];
+}
+
+template<class T>
+void array_pop(Array<T> *a){
+  assert(a->len > 0);
+  --a->len;
+}
+
+template<class T>
+void array_clear(Array<T> *array){
+  array->len = 0;
+}
+
+function void
+test_array(){
+  Set_Scratch();
+  Array<int> array = {};
+  int size = 1000;
+  for(int i = 0; i < size; i++){
+    array_push(&array, i);
+  }
+  For(array, it, i){
+    assert(*it == i);
+  }
+  
+  Arena arena = arena_make_personal();
+  Array<int> array2 = {};
+  array2.allocator = &arena;
+  for(int i = 0; i < size; i++){
+    array_push(&array2, i);
+  }
+  For(array2, iterator, count){
+    assert(*iterator == count);
+  }
+  for(int i = 999; i > 950; i--){
+    assert(array_pop_get(&array) == i);
+  }
+  for(int i = 0; i < 10; i++){
+    array_pop(&array2);
+  }
+  exp_destroy(&arena);
+  assert(arena.memory.data == 0);
+  assert(thread_ctx.scratch->memory.data != 0);
+  assert(thread_ctx.scratch == thread_ctx.implicit_allocator);
+}
+
+//-----------------------------------------------------------------------------
+// Map
+//-----------------------------------------------------------------------------
+struct Map_Key_Val{
+  U64   key;
+  void *value;
+};
+
+struct Map{
+  Map_Key_Val *data;
+  S64 len;
+  S64 cap;
+  Allocator *allocator;
+};
+function void map_insert_u64(Map *map, U64 key, void *val);
+
+function void
+map_grow(Map *map, S64 new_size){
+  new_size = max((S64)16, new_size);
+  assert(new_size > map->cap);
+  assert(is_pow2(new_size));
+  if(!map->allocator) map->allocator = imp_get();
+  
+  Map new_map       = {};
+  new_map.data      = exp_alloc_array(map->allocator, Map_Key_Val, new_size, AF_ZeroMemory),
+  new_map.cap       = new_size,
+  new_map.allocator = map->allocator;
+  
+  for(S64 i = 0; i < map->cap; i++){
+    if(map->data[i].key){
+      map_insert_u64(&new_map, map->data[i].key, map->data[i].value);
+    }
+  }
+  if(map->data) free(map->data);
+  *map = new_map;
+}
+
+function void
+map_insert_u64(Map *map, U64 key, void *val){
+  assert(val);
+  if(key == 0) key++;
+  if((2*map->len) + 1 > map->cap){
+    map_grow(map, 2*map->cap);
+  }
+  U64 hash  = hash_u64(key);
+  U64 index = wrap_around_pow2(hash, map->cap);
+  U64 i     = index;
+  for(;;){
+    if(map->data[i].key == 0){
+      map->len++;
+      map->data[i].key = key;
+      map->data[i].value = val;
+      return;
+    }
+    else if(map->data[i].key == key){
+      map->data[i].value = val;
+      return;
+    }
+    
+    i = wrap_around_pow2(i+1, map->cap);
+    if(i == map->cap){
+      return;
+    }
+  }
+}
+
+function void *
+map_get_u64(Map *map, U64 key){
+  if(map->len == 0) return 0;
+  if(key == 0) key++;
+  U64 hash  = hash_u64(key);
+  U64 index = wrap_around_pow2(hash, map->cap);
+  U64 i     = index;
+  for(;;){
+    if(map->data[i].key == key){
+      return map->data[i].value;
+    }
+    else if(map->data[i].key == 0){
+      return 0;
+    }
+    
+    i = wrap_around_pow2(i+1, map->cap);
+    if(i == map->cap){
+      return 0;
+    }
+  }
+}
+
+function void *
+map_get(Map *map, void *pointer){
+  return map_get_u64(map, (U64)pointer);
+}
+
+function void
+map_insert(Map *map, void *key, void *value){
+  map_insert_u64(map, (U64)key, value);
+}
+
+function void
+map_test(){
+  Map map = {0};
+  const SizeU size = 1025;
+  for(SizeU i = 1; i < size; i++){
+    map_insert_u64(&map, i, (void *)i);
+  }
+  for(SizeU i = 1; i < size; i++){
+    SizeU val = (SizeU)map_get_u64(&map, i);
+    assert(val == i);
+  }
+}
+
+//-----------------------------------------------------------------------------
+// Bucket Array
+//-----------------------------------------------------------------------------
+template<class T>
+struct Bucket_Array{
+  struct Bucket{
+    Bucket *next;
+    S64 len, cap;
+    T data[0];
+  };
+  
+  Allocator *allocator;
+  Bucket *first;
+  Bucket *last;
+  
+  Bucket *iter;
+  S64 iter_len;
+};
+
+function void
+test_bucket_array(){
+  Bucket_Array<int> arr = {};
+}
+
+//-----------------------------------------------------------------------------
+// Linked lists
+//-----------------------------------------------------------------------------
+#define SLLQueuePushMod(f,l,n,next) do{\
+if((f)==0){\
+(f)=(l)=(n);\
+}\
+else{\
+(l)=(l)->next=(n);\
+} \
+}while(0)
+#define SLLQueuePush(f,l,n) SLLQueuePushMod(f,l,n,next)
+
+#define SLLStackPush(l,n) do{\
+(n)->next = (l);\
+(l) = (n);\
+}while(0)
+
+#define SLLStackPop(l,n) do{\
+if(l){\
+(n) = (l);\
+(l) = (l)->next;\
+(n)->next = 0;\
+}\
+}while(0)
+
+//-----------------------------------------------------------------------------
+// String builder
+//-----------------------------------------------------------------------------
+#include <stdio.h>
+struct String_Builder_Block{
+  String_Builder_Block *next;
+  S64 cap;
+  S64 len;
+  U8 data[0];
+};
+
+struct String_Builder{
+  String_Builder_Block *first;
+  String_Builder_Block *last;
+  Allocator *allocator;
+};
+
+function void
+string_builder_push_block(String_Builder *b, SizeU size){
+  String_Builder_Block *block = (String_Builder_Block *)imp_alloc(sizeof(String_Builder_Block) + size);
+  memory_zero(block, sizeof(String_Builder_Block)+1); // Also clear first byte of character data
+  block->cap = size;
+  SLLQueuePush(b->first, b->last, block);
+}
+
+function void
+string_builder_init(String_Builder *b, SizeU size = 4096){
+  if(!b->allocator) b->allocator = imp_get();
+  string_builder_push_block(b, size);
+}
+
+function void
+appendf(String_Builder *b, const char *str, ...){
+  if(b->first == 0){
+    string_builder_init(b);
+  }
+  va_list args, args2;
+  va_start(args, str); defer(va_end(args));
+  retry:{
+    String_Builder_Block *block = b->last;
+    int block_size = block->cap - block->len;
+    char *write_address = (char *)block->data + block->len;
+    
+    va_copy(args2, args); defer(va_end(args2));
+    int written = vsnprintf(write_address, block_size, str, args2);
+    
+    if(written > block_size){
+      int new_block_size = max(4096, (written+1)*2);
+      string_builder_push_block(b, new_block_size);
+      goto retry;
+    }
+    block->len += written;
+  }
+}
+
+function String
+string_flatten(String_Builder *b){
+  // @Note(Krzosa): Only single block, no need to flatten, vsnprintf null terminates too
+  if(b->first == b->last){
+    String result = {b->first->data, b->first->len};
+    return result;
+  }
+  
+  // @Note(Krzosa): Compute size to allocate
+  S64 size = 1;
+  IterList(b, it){
+    size += it->len;
+  }
+  String result = {};
+  result.str = (U8 *)exp_alloc(b->allocator, size);
+  
+  // @Note(Krzosa): Copy the content of each block into the string
+  IterList(b, it){
+    memory_copy(result.str + result.len, it->data, it->len);
+    result.len += it->len;
+  }
+  result.str[result.len] = 0;
+  return result;
+}
+
+function B32
+string_compare(String a, String b){
+  if(a.len != b.len)
+    return false;
+  for(S64 i = 0; i < a.len; i++){
+    if(a.str[i] != b.str[i])
+      return false;
+  }
+  return true;
+}
+
+function U8
+char_to_lower(U8 c){
+  if(c >= 'A' && c <= 'Z')
+    c += 32;
+  return c;
+}
+
+function U8
+char_to_upper(U8 c){
+  if(c >= 'a' && c <= 'z')
+    c -= 32;
+  return c;
+}
+
+force_inline String
+operator""_s(const char *str, size_t size){
+  return String{(U8 *)str, (S64)size};
+}
+
+function void
+test_string_builder(){
+  Set_Scratch();
+  String_Builder sb = {};
+  string_builder_init(&sb, 4);
+  appendf(&sb, "Thing, %d", 242252);
+  String f = string_flatten(&sb);
+  assert(string_compare(f, "Thing, 242252"_s));
+  appendf(&sb, "-%f %f %f", 23.0, 42.29, 2925.2);
+  f = string_flatten(&sb);
+}
+
+//-----------------------------------------------------------------------------
+// String intern
+//-----------------------------------------------------------------------------
+struct Intern_Table{
+  Allocator *string_allocator;
+  Map map;
+};
+
+function Intern_String
+intern_string(Intern_Table *t, String string){
+  if(!t->string_allocator) t->string_allocator = imp_get();
+  U64 hash = hash_string(string);
+  U8 *slot = (U8 *)map_get_u64(&t->map, hash);
+  if(slot){
+    Intern_String result = {{slot, *(slot-sizeof(S64))}};
+    return result;
+  }
+  
+  S64 *len_address = (S64 *)exp_alloc(t->string_allocator, string.len+1+sizeof(S64));
+  *len_address = string.len;
+  
+  U8 *string_address = (U8 *)(len_address + 1);
+  memory_copy(string_address, string.str, string.len);
+  string_address[string.len] = 0;
+  
+  map_insert_u64(&t->map, hash, string_address);
+  Intern_String result = {{string_address, *len_address}};
+  
+  return result;
+}
+
+function void
+test_intern_table(){ Set_Scratch();
+  Intern_Table table = {};
+  Intern_String intern1 = intern_string(&table, "Thing"_s);
+  Intern_String intern2 = intern_string(&table, "Thing"_s);
+  Intern_String intern3 = intern_string(&table, "Not Thing"_s);
+  assert(intern1.str == intern2.str);
+  assert(intern3.str != intern2.str);
+}
+
+#include "new_lex.cpp"
+int main(){
+  test_custom_context();
+  test_heap_allocator();
+  test_os_memory();
+  thread_ctx_init();
+  test_array();
+  map_test();
+  test_string_builder();
+  test_intern_table();
+  lex_test();
 }