New approach, new lexer

generate.c

@@ -0,0 +1,36 @@
+#define _CRT_SECURE_NO_WARNINGS
+#include <stdio.h>
+#include <windows.h>
+
+#include "lang.h"
+#include "os.h"
+#include "memory.h"
+
+#include "common.c"
+#include "memory.c"
+#include "scratch.c"
+#include "os_win32.c"
+
+#include "new_lex.c"
+
+int main(){
+  
+  FILE *f = fopen("generated.c", "w");
+  assert(f);
+  
+  Arena *scratch = arena_begin_scratch();
+  String file = os_read_file(scratch, lit("token_array.c"));
+  Token_Array array = lex_stream(scratch, file, lit("token_array.c"));
+  for(Token *t = token_array_iter_begin(&array); t; t = token_array_iter_next(&array)){
+    
+    fprintf(f, "%.*s", (int)t->len, t->str);
+    if(t->kind == TK_OpenBrace) fprintf(f, "\n");
+    if(t->kind == TK_CloseBrace && token_array_iter_peek(&array, 0)->kind != TK_Semicolon) fprintf(f, "\n");
+    if(t->kind == TK_Semicolon) fprintf(f, "\n");
+    if(t[0].kind == TK_Identifier && token_array_iter_peek(&array, 0)->kind == TK_Identifier) fprintf(f, " ");
+    token_array_iter_peek(&array, 1);
+  }
+  arena_end_scratch();
+  
+  fclose(f);
+}