New approach, new lexer

lex.c

@@ -1,4 +1,4 @@
-global Token token_end_of_stream = {};
+global Token token_end_of_stream = {0};
 
 function Token *
 token_alloc(Tokens *t){
@@ -363,7 +363,7 @@ break
 function Tokens
 lex_stream(String in_stream, String filename){
   Lex_Stream stream = {in_stream.str, in_stream.str, filename, 0};
-  Tokens tokens = {};
+  Tokens tokens = {0};
   lex_base(&stream, &tokens);
   return tokens;
 }
@@ -498,3 +498,32 @@ token_peek_is_keyword(Parser *p, S64 count, Intern_String keyword){
   }
   return 0;
 }
+
+function void
+lex_test(){
+  Tokens t;
+  t = lex_stream(lit("3252342510 42524 \"U8Literal\""), lit("test"));
+  //tokens_print(t);
+  assert(t.len == 3);
+  assert(t.tokens[0].int_val == 3252342510);
+  assert(t.tokens[1].int_val == 42524);
+  assert(t.tokens[2].kind == TK_U8Lit);
+  assert(token_compare(t.tokens + 2, lit("U8Literal")));
+  
+  t = lex_stream(lit("_identifier Thing Thing2 lit(\"String_Test\")"), lit("test"));
+  //tokens_print(t);
+  assert(t.tokens[0].kind == TK_Identifier);
+  assert(t.tokens[1].kind == TK_Identifier);
+  assert(t.tokens[2].kind == TK_Identifier);
+  assert(t.tokens[3].kind == TK_StringLit);
+  assert(token_compare(t.tokens, lit("_identifier")));
+  assert(token_compare(t.tokens+1, lit("Thing")));
+  assert(token_compare(t.tokens+2, lit("Thing2")));
+  assert(token_compare(t.tokens+3, lit("String_Test")));
+  
+  t = lex_stream(lit("lit(\"String_Test\"{})(324*=+=-/ *% // Comment \n"
+                     "Thing /*Thing*/ += -= =- +/%^&*&&|| |>> << <<= >>=/*Error"), 
+                 lit("test"));
+  assert(t.tokens[0].kind == TK_Error);
+  //tokens_print(t);
+}