New approach, new lexer

common.c

@@ -1,4 +1,3 @@
-global String string_empty;
 
 function S64
 clamp_top_s64(S64 val, S64 max){
@@ -52,6 +51,11 @@ wrap_around_pow2(U64 x, U64 power_of_2) {
   return r;
 }
 
+//-----------------------------------------------------------------------------
+// Strings
+//-----------------------------------------------------------------------------
+global String string_empty;
+
 function B32
 string_compare(String a, String b){
   if(a.len != b.len)
@@ -77,6 +81,12 @@ char_to_upper(U8 c){
   return c;
 }
 
+function B32 
+string_is_empty(String a){
+  B32 result = a.len == 0;
+  return result;
+}
+
 function String
 string_to_lower(Arena *arena, String string){
   String result = arena_push_string_copy(arena, string);
@@ -85,3 +95,103 @@ string_to_lower(Arena *arena, String string){
   }
   return result;
 }
+
+//-----------------------------------------------------------------------------
+// String interning
+//-----------------------------------------------------------------------------
+typedef struct Table_Index{
+  U64 hash;
+  U64 index;
+  U64 iter;
+  U64 max_size;
+}Table_Index;
+
+typedef struct Intern_String{
+  String s;
+}Intern_String;
+
+typedef struct Intern_Table{
+  S64 interns_in_bytes;
+  S64 interns_inserted;
+  S64 interns_max;
+  Intern_String *interns;
+  Arena *arena;
+}Intern_Table;
+
+function Intern_Table
+intern_table(Arena *arena, SizeU size){
+  Intern_Table result = {0};
+  result.arena = arena;
+  result.interns = arena_push_array(arena, Intern_String, size);
+  result.interns_max = size;
+  return result;
+}
+
+function Table_Index
+table_index_from_hash(U64 hash, U64 max_size){
+  Table_Index result = {0};
+  result.hash = hash;
+  result.index = result.hash % max_size;
+  result.iter = result.index;
+  result.max_size = max_size;
+  return result;
+}
+
+function Table_Index
+table_index_from_string(String string, U64 max_size){
+  U64 hash = hash_fnv(string);
+  Table_Index result = table_index_from_hash(hash, max_size);
+  return result;
+}
+
+function B32
+table_index_advance(Table_Index *index){
+  index->iter = wrap_around_pow2(index->iter + 1, index->max_size);
+  B32 result = index->iter == index->index;
+  return result;
+}
+
+function Intern_String
+intern_string(Intern_Table *p, String string){
+  Intern_String result = {0};
+  Table_Index index = table_index_from_string(string, p->interns_max);
+  for(;;){
+    Intern_String *intern = p->interns + index.iter;
+    if(intern->s.str == 0){
+      result.s = arena_push_string_copy(p->arena, string);
+      p->interns_in_bytes += string.len;
+      p->interns_inserted += 1;
+      *intern = result;
+      break;
+    }
+    else if(string_compare(intern->s, string)){
+      result = *intern;
+      break;
+    }
+    
+    if (table_index_advance(&index)) 
+      break;
+    
+  }
+  
+  return result;
+}
+
+function B32
+intern_compare(Intern_String a, Intern_String b){
+  B32 result = a.s.str == b.s.str;
+  return result;
+}
+
+function void
+intern_test(){
+  Arena *scratch = arena_begin_scratch();
+  Intern_Table table = intern_table(scratch, 512);
+  assert(intern_compare(intern_string(&table, lit("Thing")), 
+                        intern_string(&table, lit("Thing"))));
+  assert(!intern_compare(intern_string(&table, lit("Thing")), 
+                         intern_string(&table, lit("No_Thing"))));
+  assert(intern_compare(intern_string(&table, lit("No_Thing")), 
+                        intern_string(&table, lit("No_Thing"))));
+  
+}