137 lines
1.8 KiB
C
137 lines
1.8 KiB
C
typedef struct Intern_String Intern_String;
|
|
|
|
|
|
typedef struct Token Token;
|
|
|
|
typedef struct Lex_Stream Lex_Stream;
|
|
|
|
typedef struct Tokens Tokens;
|
|
|
|
struct Intern_String{
|
|
String s;
|
|
};
|
|
|
|
typedef enum Token_Kind{
|
|
TK_End,
|
|
TK_Mul,
|
|
TK_Div,
|
|
TK_Add,
|
|
TK_Sub,
|
|
TK_Mod,
|
|
TK_BitAnd,
|
|
TK_BitOr,
|
|
TK_BitXor,
|
|
TK_Neg,
|
|
TK_Not,
|
|
TK_OpenParen,
|
|
TK_CloseParen,
|
|
TK_OpenBrace,
|
|
TK_CloseBrace,
|
|
TK_OpenBracket,
|
|
TK_CloseBracket,
|
|
TK_Comma,
|
|
TK_Pound,
|
|
TK_Question,
|
|
TK_ThreeDots,
|
|
TK_Semicolon,
|
|
TK_Dot,
|
|
TK_LesserThen,
|
|
TK_GreaterThen,
|
|
TK_Colon,
|
|
TK_Assign,
|
|
TK_DivAssign,
|
|
TK_MulAssign,
|
|
TK_ModAssign,
|
|
TK_SubAssign,
|
|
TK_AddAssign,
|
|
TK_AndAssign,
|
|
TK_OrAssign,
|
|
TK_XorAssign,
|
|
TK_LeftShiftAssign,
|
|
TK_RightShiftAssign,
|
|
TK_DoubleColon,
|
|
TK_At,
|
|
TK_Decrement,
|
|
TK_Increment,
|
|
TK_PostDecrement,
|
|
TK_PostIncrement,
|
|
TK_LesserThenOrEqual,
|
|
TK_GreaterThenOrEqual,
|
|
TK_Equals,
|
|
TK_And,
|
|
TK_Or,
|
|
TK_NotEquals,
|
|
TK_LeftShift,
|
|
TK_RightShift,
|
|
TK_Arrow,
|
|
TK_ExprSizeof,
|
|
TK_DocComment,
|
|
TK_Comment,
|
|
TK_Identifier,
|
|
TK_StringLit,
|
|
TK_U8Lit,
|
|
TK_Character,
|
|
TK_Error,
|
|
TK_Float,
|
|
TK_Int,
|
|
TK_Keyword,
|
|
}Token_Kind;
|
|
|
|
struct Token{
|
|
Token_Kind kind;
|
|
union{
|
|
struct{
|
|
U8 (*str);
|
|
S64 len;
|
|
};
|
|
String string;
|
|
};
|
|
union{
|
|
S64 int_val;
|
|
String error_val;
|
|
Intern_String intern_val;
|
|
};
|
|
String file;
|
|
S64 line;
|
|
U8 (*line_begin);
|
|
};
|
|
|
|
// Generated Array block
|
|
typedef struct Token_Array Token_Array;
|
|
typedef struct Token_Array_Block Token_Array_Block;
|
|
typedef struct Token_Array_Iter Token_Array_Iter;
|
|
struct Token_Array_Block{
|
|
Token_Array_Block*next;
|
|
Token data[4096];
|
|
};
|
|
struct Token_Array_Iter{
|
|
Token_Array_Block*block;
|
|
U32 iter_len;
|
|
U32 iter_block;
|
|
U32 total_len;
|
|
U32 total_block;
|
|
};
|
|
struct Token_Array{
|
|
Token_Array_Block first;
|
|
Token_Array_Block*last;
|
|
Token_Array_Iter it;
|
|
Arena*arena;
|
|
S64 block;
|
|
S64 len;
|
|
};
|
|
|
|
struct Lex_Stream{
|
|
U8 (*stream);
|
|
U8 (*line_begin);
|
|
String filename;
|
|
S64 line;
|
|
};
|
|
|
|
struct Tokens{
|
|
Token (*tokens);
|
|
S64 len;
|
|
S64 cap;
|
|
S64 iter;
|
|
};
|
|
|