New approach, new lexer

generated_lex.h

@@ -0,0 +1,136 @@
+typedef struct Intern_String Intern_String;
+
+
+typedef struct Token Token;
+
+typedef struct Lex_Stream Lex_Stream;
+
+typedef struct Tokens Tokens;
+
+struct Intern_String{
+String s;
+};
+
+typedef enum Token_Kind{
+TK_End,
+TK_Mul,
+TK_Div,
+TK_Add,
+TK_Sub,
+TK_Mod,
+TK_BitAnd,
+TK_BitOr,
+TK_BitXor,
+TK_Neg,
+TK_Not,
+TK_OpenParen,
+TK_CloseParen,
+TK_OpenBrace,
+TK_CloseBrace,
+TK_OpenBracket,
+TK_CloseBracket,
+TK_Comma,
+TK_Pound,
+TK_Question,
+TK_ThreeDots,
+TK_Semicolon,
+TK_Dot,
+TK_LesserThen,
+TK_GreaterThen,
+TK_Colon,
+TK_Assign,
+TK_DivAssign,
+TK_MulAssign,
+TK_ModAssign,
+TK_SubAssign,
+TK_AddAssign,
+TK_AndAssign,
+TK_OrAssign,
+TK_XorAssign,
+TK_LeftShiftAssign,
+TK_RightShiftAssign,
+TK_DoubleColon,
+TK_At,
+TK_Decrement,
+TK_Increment,
+TK_PostDecrement,
+TK_PostIncrement,
+TK_LesserThenOrEqual,
+TK_GreaterThenOrEqual,
+TK_Equals,
+TK_And,
+TK_Or,
+TK_NotEquals,
+TK_LeftShift,
+TK_RightShift,
+TK_Arrow,
+TK_ExprSizeof,
+TK_DocComment,
+TK_Comment,
+TK_Identifier,
+TK_StringLit,
+TK_U8Lit,
+TK_Character,
+TK_Error,
+TK_Float,
+TK_Int,
+TK_Keyword,
+}Token_Kind;
+
+struct Token{
+Token_Kind kind;
+union{
+struct{
+U8 (*str);
+S64 len;
+};
+String string;
+};
+union{
+S64 int_val;
+String error_val;
+Intern_String intern_val;
+};
+String file;
+S64 line;
+U8 (*line_begin);
+};
+
+// Generated Array block
+typedef struct Token_Array Token_Array;
+typedef struct Token_Array_Block Token_Array_Block;
+typedef struct Token_Array_Iter Token_Array_Iter;
+struct Token_Array_Block{
+Token_Array_Block*next;
+Token data[4096];
+};
+struct Token_Array_Iter{
+Token_Array_Block*block;
+U32 iter_len;
+U32 iter_block;
+U32 total_len;
+U32 total_block;
+};
+struct Token_Array{
+Token_Array_Block first;
+Token_Array_Block*last;
+Token_Array_Iter it;
+Arena*arena;
+S64 block;
+S64 len;
+};
+
+struct Lex_Stream{
+U8 (*stream);
+U8 (*line_begin);
+String filename;
+S64 line;
+};
+
+struct Tokens{
+Token (*tokens);
+S64 len;
+S64 cap;
+S64 iter;
+};
+