typedef struct Intern_String Intern_String;


typedef struct Token Token;

typedef struct Lex_Stream Lex_Stream;

typedef struct Tokens Tokens;

struct Intern_String{
String s;
};

typedef enum Token_Kind{
TK_End,
TK_Mul,
TK_Div,
TK_Add,
TK_Sub,
TK_Mod,
TK_BitAnd,
TK_BitOr,
TK_BitXor,
TK_Neg,
TK_Not,
TK_OpenParen,
TK_CloseParen,
TK_OpenBrace,
TK_CloseBrace,
TK_OpenBracket,
TK_CloseBracket,
TK_Comma,
TK_Pound,
TK_Question,
TK_ThreeDots,
TK_Semicolon,
TK_Dot,
TK_LesserThen,
TK_GreaterThen,
TK_Colon,
TK_Assign,
TK_DivAssign,
TK_MulAssign,
TK_ModAssign,
TK_SubAssign,
TK_AddAssign,
TK_AndAssign,
TK_OrAssign,
TK_XorAssign,
TK_LeftShiftAssign,
TK_RightShiftAssign,
TK_DoubleColon,
TK_At,
TK_Decrement,
TK_Increment,
TK_PostDecrement,
TK_PostIncrement,
TK_LesserThenOrEqual,
TK_GreaterThenOrEqual,
TK_Equals,
TK_And,
TK_Or,
TK_NotEquals,
TK_LeftShift,
TK_RightShift,
TK_Arrow,
TK_ExprSizeof,
TK_DocComment,
TK_Comment,
TK_Identifier,
TK_StringLit,
TK_U8Lit,
TK_Character,
TK_Error,
TK_Float,
TK_Int,
TK_Keyword,
}Token_Kind;

struct Token{
Token_Kind kind;
union{
struct{
U8 (*str);
S64 len;
};
String string;
};
union{
S64 int_val;
String error_val;
Intern_String intern_val;
};
String file;
S64 line;
U8 (*line_begin);
};

// Generated Array block
typedef struct Token_Array Token_Array;
typedef struct Token_Array_Block Token_Array_Block;
typedef struct Token_Array_Iter Token_Array_Iter;
struct Token_Array_Block{
Token_Array_Block*next;
Token data[4096];
};
struct Token_Array_Iter{
Token_Array_Block*block;
U32 iter_len;
U32 iter_block;
U32 total_len;
U32 total_block;
};
struct Token_Array{
Token_Array_Block first;
Token_Array_Block*last;
Token_Array_Iter it;
Arena*arena;
S64 block;
S64 len;
};

struct Lex_Stream{
U8 (*stream);
U8 (*line_begin);
String filename;
S64 line;
};

struct Tokens{
Token (*tokens);
S64 len;
S64 cap;
S64 iter;
};