New approach, new lexer

generated_lex.cpp

@@ -0,0 +1,53 @@
+
+// Generated Array block
+
+function
+ Token_Array token_array_make(Arena*arena){
+Token_Array result={
+0};
+result.arena=arena;
+result.last=&result.first;
+return result;
+}
+
+function
+ void token_array_push(Token_Array*array, Token*item){
+if(array->len+1>4096){
+assert(array->arena);
+Token_Array_Block*block=arena_push_struct(array->arena, Token_Array_Block);
+array->last=array->last->next=block;
+array->len=0;
+array->block+=1;
+}
+array->last->data[array->len++]=*item;
+}
+
+function
+ Token_Array_Iter token_array_iter(Token_Array*array){
+return((Token_Array_Iter){
+.total_len=array->len, .total_block=array->block, .block=&array->first, }
+);
+}
+
+function
+ Token*token_array_iter_next(Token_Array_Iter*it){
+if(it->iter_len+1>4096){
+it->iter_len=0;
+it->iter_block+=1;
+it->block=it->block->next;
+}
+return it->block->data+it->iter_len++;
+}
+
+function
+ B32 token_array_iter_is_end(Token_Array_Iter*it){
+B32 result=it->iter_len==it->total_len&&it->iter_block==it->total_block;
+return result;
+}
+
+function
+ Token*token_array_iter_begin(Token_Array*array){
+array->it=token_array_iter(array);
+Token*result=token_array_iter_next(&array->it);
+return result;
+}