New approach, new lexer

generated.c

@@ -0,0 +1,69 @@
+typedef struct Token_Bucket Token_Bucket;
+typedef struct Token_Array Token_Array;
+struct Token_Bucket{
+Token_Bucket*next;
+Token data[4096];
+};
+struct Token_Array{
+Intern_Table interns;
+Token_Bucket first;
+Token_Bucket*last;
+S64 len;
+S64 block;
+Arena*arena;
+Token_Bucket*iter_bucket;
+S64 iter_len;
+S64 iter_block;
+};
+function Token_Array token_array_make(Arena*arena){
+Token_Array result={
+.last=&result.first,.arena=arena,.interns=intern_table(arena,4096*4)};
+return result;
+}
+function void token_array_push(Token_Array*array,Token*p){
+if(array->len>=buff_cap(array->first.data)){
+Token_Bucket*bucket=arena_push_struct(array->arena,Token_Bucket);
+array->last=array->last->next=bucket;
+array->len=0;
+array->block+=1;
+}
+array->last->data[array->len++]=*p;
+}
+function B32 token_array_iter_is_end(Token_Array*array){
+B32 result=array->iter_len==array->len&&array->iter_block==array->block;
+return result;
+}
+function Token*token_array_iter_next(Token_Array*array){
+if(token_array_iter_is_end(array)){
+return0;
+}
+if(array->iter_len>=buff_cap(array->first.data)){
+array->iter_len=0;
+array->iter_block+=1;
+array->iter_bucket=array->iter_bucket->next;
+}
+Token*result=array->iter_bucket->data+array->iter_len++;
+return result;
+}
+function Token*token_array_iter_peek(Token_Array*array,S64 i){
+S64 save_len=array->iter_len;
+Token_Bucket*save_bucket=array->iter_bucket;
+assert(i<buff_cap(array->first.data));
+S64 over=i;
+if(array->iter_len+i>=buff_cap(array->first.data)){
+over=buff_cap(array->first.data)-(array->iter_len+i);
+array->iter_len=0;
+array->iter_bucket=array->iter_bucket->next;
+}
+Token*result=array->iter_bucket->data+array->iter_len+over;
+array->iter_len=save_len;
+array->iter_bucket=save_bucket;
+return result;
+}
+function Token*token_array_iter_begin(Token_Array*array){
+array->iter_len=0;
+array->iter_block=0;
+array->iter_bucket=&array->first;
+Token*result=token_array_iter_next(array);
+return result;
+}