corelang/lex.h

#pragma once
typedef struct Intern_String{
  String s;
}Intern_String;

typedef enum Token_Kind{
  meta("End of stream")TK_End,
  meta("*")TK_Mul,
  meta("/")TK_Div,
  meta("+")TK_Add,
  meta("-")TK_Sub,
  meta("%")TK_Mod,
  meta("&")TK_BitAnd,
  meta("|")TK_BitOr,
  meta("^")TK_BitXor,
  meta("~")TK_Neg,
  meta("!")TK_Not,
  meta("(")TK_OpenParen,
  meta(")")TK_CloseParen,
  meta("{")TK_OpenBrace,
  meta("}")TK_CloseBrace,
  meta("[")TK_OpenBracket,
  meta("]")TK_CloseBracket,
  meta(",")TK_Comma,
  meta("#")TK_Pound,
  meta("?")TK_Question,
  meta("...")TK_ThreeDots,
  meta(";")TK_Semicolon,
  meta(".")TK_Dot,
  meta("<")TK_LesserThen,
  meta(">")TK_GreaterThen,
  meta(":")TK_Colon,
  meta("=")TK_Assign,
  meta("/=")TK_DivAssign,
  meta("*=")TK_MulAssign,
  meta("%=")TK_ModAssign,
  meta("-=")TK_SubAssign,
  meta("+=")TK_AddAssign,
  meta("&=")TK_AndAssign,
  meta("|=")TK_OrAssign,
  meta("^=")TK_XorAssign,
  meta("<<=")TK_LeftShiftAssign,
  meta(">>=")TK_RightShiftAssign,
  meta("::")TK_DoubleColon,
  meta("@")TK_At,
  meta("--")TK_Decrement,
  meta("++")TK_Increment,
  meta("--")TK_PostDecrement,
  meta("++")TK_PostIncrement,
  meta("<=")TK_LesserThenOrEqual,
  meta(">=")TK_GreaterThenOrEqual,
  meta("==")TK_Equals,
  meta("&&")TK_And,
  meta("||")TK_Or,
  meta("!=")TK_NotEquals,
  meta("<<")TK_LeftShift,
  meta(">>")TK_RightShift,
  meta("->")TK_Arrow,
  meta("sizeof")TK_ExprSizeof,
  TK_DocComment,
  TK_Comment,
  TK_Identifier,
  TK_StringLit,
  TK_U8Lit,
  TK_Character,
  TK_Error,
  TK_Float,
  TK_Int,
  TK_Keyword,
}Token_Kind;

typedef struct Token{
  Token_Kind kind;
  union{
    struct{
      U8 *str;
      S64 len;
    };
    String string;
  };
  union {
    S64 int_val;
    String error_val;
    Intern_String intern_val;
  };
  
  String file;
  S64 line;
  U8 *line_begin;
} Token;

typedef struct Tokens{
  Token *tokens;
  S64 len;
  S64 cap;
  S64 iter;
}Tokens;

typedef struct Lex_Stream{
  U8 *stream;
  U8 *line_begin;
  String filename;
  S64 line;
}Lex_Stream;

//-----------------------------------------------------------------------------
// 
//-----------------------------------------------------------------------------

global String token_kind_string[] = {
  [TK_End] = lit("End of stream"),
  [TK_Error] = lit("Error"),
  [TK_Comment] = lit("Comment"),
  [TK_Identifier] = lit("Identifier"),
  [TK_Keyword] = lit("Keyword"),
  [TK_Character] = lit("Character"),
  [TK_StringLit] = lit("StringLiteral"),
  [TK_U8Lit] = lit("U8Literal"),
  [TK_Float] = lit("Float"),
  [TK_Int] = lit("Integer"),
  [TK_Mul] = lit("*"),
  [TK_Div] = lit("/"),
  [TK_Add] = lit("+"),
  [TK_Sub] = lit("-"),
  [TK_Mod] = lit("%"),
  [TK_BitAnd] = lit("&"),
  [TK_BitOr] = lit("|"),
  [TK_BitXor] = lit("^"),
  [TK_Neg] = lit("~"),
  [TK_Not] = lit("!"),
  [TK_OpenParen] = lit("("),
  [TK_CloseParen] = lit(")"),
  [TK_OpenBrace] = lit("{"),
  [TK_CloseBrace] = lit("}"),
  [TK_OpenBracket] = lit("["),
  [TK_CloseBracket] = lit("]"),
  [TK_Comma] = lit(","),
  [TK_Pound] = lit("#"),
  [TK_Question] = lit("?"),
  [TK_ThreeDots] = lit("..."),
  [TK_Semicolon] = lit(";"),
  [TK_Dot] = lit("."),
  [TK_LesserThen] = lit("<"),
  [TK_GreaterThen] = lit(">"),
  [TK_Colon] = lit(":"),
  [TK_Assign] = lit("="),
  [TK_DivAssign] = lit("/="),
  [TK_MulAssign] = lit("*="),
  [TK_ModAssign] = lit("%="),
  [TK_SubAssign] = lit("-="),
  [TK_AddAssign] = lit("+="),
  [TK_AndAssign] = lit("&="),
  [TK_OrAssign] = lit("|="),
  [TK_XorAssign] = lit("^="),
  [TK_LeftShiftAssign] = lit("<<="),
  [TK_RightShiftAssign] = lit(">>="),
  [TK_DoubleColon] = lit("::"),
  [TK_At] = lit("@"),
  [TK_Decrement] = lit("--"),
  [TK_Increment] = lit("++"),
  [TK_PostDecrement] = lit("--"),
  [TK_PostIncrement] = lit("++"),
  [TK_LesserThenOrEqual] = lit("<="),
  [TK_GreaterThenOrEqual] = lit(">="),
  [TK_Equals] = lit("=="),
  [TK_And] = lit("&&"),
  [TK_Or] = lit("||"),
  [TK_NotEquals] = lit("!="),
  [TK_LeftShift] = lit("<<"),
  [TK_RightShift] = lit(">>"),
  [TK_Arrow] = lit("->"),
  [TK_ExprSizeof] = lit("sizeof"),
};

global String Token_Kind_metadata[] = {
  [TK_End] = lit("End of stream"),
  [TK_Mul] = lit("*"),
  [TK_Div] = lit("/"),
  [TK_Add] = lit("+"),
  [TK_Sub] = lit("-"),
  [TK_Mod] = lit("%"),
  [TK_BitAnd] = lit("&"),
  [TK_BitOr] = lit("|"),
  [TK_BitXor] = lit("^"),
  [TK_Neg] = lit("~"),
  [TK_Not] = lit("!"),
  [TK_OpenParen] = lit("("),
  [TK_CloseParen] = lit(" "),
  [TK_OpenBrace] = lit("{"),
  [TK_CloseBrace] = lit("}"),
  [TK_OpenBracket] = lit("["),
  [TK_CloseBracket] = lit("]"),
  [TK_Comma] = lit(","),
  [TK_Pound] = lit("#"),
  [TK_Question] = lit("?"),
  [TK_ThreeDots] = lit("..."),
  [TK_Semicolon] = lit(";"),
  [TK_Dot] = lit("."),
  [TK_LesserThen] = lit("<"),
  [TK_GreaterThen] = lit(">"),
  [TK_Colon] = lit(":"),
  [TK_Assign] = lit("="),
  [TK_DivAssign] = lit("/="),
  [TK_MulAssign] = lit("*="),
  [TK_ModAssign] = lit("%="),
  [TK_SubAssign] = lit("-="),
  [TK_AddAssign] = lit("+="),
  [TK_AndAssign] = lit("&="),
  [TK_OrAssign] = lit("|="),
  [TK_XorAssign] = lit("^="),
  [TK_LeftShiftAssign] = lit("<<="),
  [TK_RightShiftAssign] = lit(">>="),
  [TK_DoubleColon] = lit("::"),
  [TK_At] = lit("@"),
  [TK_Decrement] = lit("--"),
  [TK_Increment] = lit("++"),
  [TK_PostDecrement] = lit("--"),
  [TK_PostIncrement] = lit("++"),
  [TK_LesserThenOrEqual] = lit("<="),
  [TK_GreaterThenOrEqual] = lit(">="),
  [TK_Equals] = lit("=="),
  [TK_And] = lit("&&"),
  [TK_Or] = lit("||"),
  [TK_NotEquals] = lit("!="),
  [TK_LeftShift] = lit("<<"),
  [TK_RightShift] = lit(">>"),
  [TK_Arrow] = lit("->"),
  [TK_ExprSizeof] = lit("sizeof"),
  [TK_DocComment] = lit("DocComment"),
  [TK_Comment] = lit("Comment"),
  [TK_Identifier] = lit("Identifier"),
  [TK_StringLit] = lit("StringLit"),
  [TK_U8Lit] = lit("U8Lit"),
  [TK_Character] = lit("Character"),
  [TK_Error] = lit("Error"),
  [TK_Float] = lit("Float"),
  [TK_Int] = lit("Int"),
  [TK_Keyword] = lit("Keyword"),
};