corelang/lex.h

#pragma once
#pragma once
#if 0
Intern_String :: struct {
  s: String;
}

@prefix = "TK_"
Token_Kind :: enum {
  @str = "End of stream" End,
  @str = "*" Mul,
  @str = "/" Div,
  @str = "+" Add,
  @str = "-" Sub,
  @str = "%" Mod,
  @str = "&" BitAnd,
  @str = "|" BitOr,
  @str = "^" BitXor,
  @str = "~" Neg,
  @str = "!" Not,
  @str = "(" OpenParen,
  @str = " " CloseParen,
  @str = "{" OpenBrace,
  @str = "}" CloseBrace,
  @str = "[" OpenBracket,
  @str = "]" CloseBracket,
  @str = "," Comma,
  @str = "#" Pound,
  @str = "?" Question,
  @str = "..." ThreeDots,
  @str = ";" Semicolon,
  @str = "." Dot,
  @str = "<" LesserThen,
  @str = ">" GreaterThen,
  @str = ":" Colon,
  @str = "=" Assign,
  @str = "/=" DivAssign,
  @str = "*=" MulAssign,
  @str = "%=" ModAssign,
  @str = "-=" SubAssign,
  @str = "+=" AddAssign,
  @str = "&=" AndAssign,
  @str = "|=" OrAssign,
  @str = "^=" XorAssign,
  @str = "<<=" LeftShiftAssign,
  @str = ">>=" RightShiftAssign,
  @str = "::" DoubleColon,
  @str = "@" At,
  @str = "--" Decrement,
  @str = "++" Increment,
  @str = "--" PostDecrement,
  @str = "++" PostIncrement,
  @str = "<=" LesserThenOrEqual,
  @str = ">=" GreaterThenOrEqual,
  @str = "==" Equals,
  @str = "&&" And,
  @str = "||" Or,
  @str = "!=" NotEquals,
  @str = "<<" LeftShift,
  @str = ">>" RightShift,
  @str = "->" Arrow,
  @str = "sizeof" ExprSizeof,
  DocComment,
  Comment,
  Identifier,
  StringLit,
  U8Lit,
  Character,
  Error,
  Float,
  Int,
  Keyword,
}Token_Kind;

@array(type=block)
Token :: struct{
  kind: Token_Kind;
  union {
    struct{
      str: U8*;
      len: S64;
    }
    string: String;
  }
  union {
    int_val: S64;
    error_val: String;
    intern_val: Intern_String;
  }
  
  file: String;
  line: S64;
  line_begin: U8*;
}

Lex_Stream :: struct{
  stream: U8 *;
  line_begin: U8 *;
  filename: String;
  line: S64;
}

Tokens :: struct{
  tokens: Token*;
  len:  S64;
  cap:  S64;
  iter: S64;
}

#endif
#include "generated_lex.h"
#include "generated_lex.cpp"

//-----------------------------------------------------------------------------
// 
//-----------------------------------------------------------------------------

global String token_kind_string[] = {
  [TK_End] = lit("End of stream"),
  [TK_Error] = lit("Error"),
  [TK_Comment] = lit("Comment"),
  [TK_Identifier] = lit("Identifier"),
  [TK_Keyword] = lit("Keyword"),
  [TK_Character] = lit("Character"),
  [TK_StringLit] = lit("StringLiteral"),
  [TK_U8Lit] = lit("U8Literal"),
  [TK_Float] = lit("Float"),
  [TK_Int] = lit("Integer"),
  [TK_Mul] = lit("*"),
  [TK_Div] = lit("/"),
  [TK_Add] = lit("+"),
  [TK_Sub] = lit("-"),
  [TK_Mod] = lit("%"),
  [TK_BitAnd] = lit("&"),
  [TK_BitOr] = lit("|"),
  [TK_BitXor] = lit("^"),
  [TK_Neg] = lit("~"),
  [TK_Not] = lit("!"),
  [TK_OpenParen] = lit("("),
  [TK_CloseParen] = lit(")"),
  [TK_OpenBrace] = lit("{"),
  [TK_CloseBrace] = lit("}"),
  [TK_OpenBracket] = lit("["),
  [TK_CloseBracket] = lit("]"),
  [TK_Comma] = lit(","),
  [TK_Pound] = lit("#"),
  [TK_Question] = lit("?"),
  [TK_ThreeDots] = lit("..."),
  [TK_Semicolon] = lit(";"),
  [TK_Dot] = lit("."),
  [TK_LesserThen] = lit("<"),
  [TK_GreaterThen] = lit(">"),
  [TK_Colon] = lit(":"),
  [TK_Assign] = lit("="),
  [TK_DivAssign] = lit("/="),
  [TK_MulAssign] = lit("*="),
  [TK_ModAssign] = lit("%="),
  [TK_SubAssign] = lit("-="),
  [TK_AddAssign] = lit("+="),
  [TK_AndAssign] = lit("&="),
  [TK_OrAssign] = lit("|="),
  [TK_XorAssign] = lit("^="),
  [TK_LeftShiftAssign] = lit("<<="),
  [TK_RightShiftAssign] = lit(">>="),
  [TK_DoubleColon] = lit("::"),
  [TK_At] = lit("@"),
  [TK_Decrement] = lit("--"),
  [TK_Increment] = lit("++"),
  [TK_PostDecrement] = lit("--"),
  [TK_PostIncrement] = lit("++"),
  [TK_LesserThenOrEqual] = lit("<="),
  [TK_GreaterThenOrEqual] = lit(">="),
  [TK_Equals] = lit("=="),
  [TK_And] = lit("&&"),
  [TK_Or] = lit("||"),
  [TK_NotEquals] = lit("!="),
  [TK_LeftShift] = lit("<<"),
  [TK_RightShift] = lit(">>"),
  [TK_Arrow] = lit("->"),
  [TK_ExprSizeof] = lit("sizeof"),
};

global String Token_Kind_metadata[] = {
  [TK_End] = lit("End of stream"),
  [TK_Mul] = lit("*"),
  [TK_Div] = lit("/"),
  [TK_Add] = lit("+"),
  [TK_Sub] = lit("-"),
  [TK_Mod] = lit("%"),
  [TK_BitAnd] = lit("&"),
  [TK_BitOr] = lit("|"),
  [TK_BitXor] = lit("^"),
  [TK_Neg] = lit("~"),
  [TK_Not] = lit("!"),
  [TK_OpenParen] = lit("("),
  [TK_CloseParen] = lit(" "),
  [TK_OpenBrace] = lit("{"),
  [TK_CloseBrace] = lit("}"),
  [TK_OpenBracket] = lit("["),
  [TK_CloseBracket] = lit("]"),
  [TK_Comma] = lit(","),
  [TK_Pound] = lit("#"),
  [TK_Question] = lit("?"),
  [TK_ThreeDots] = lit("..."),
  [TK_Semicolon] = lit(";"),
  [TK_Dot] = lit("."),
  [TK_LesserThen] = lit("<"),
  [TK_GreaterThen] = lit(">"),
  [TK_Colon] = lit(":"),
  [TK_Assign] = lit("="),
  [TK_DivAssign] = lit("/="),
  [TK_MulAssign] = lit("*="),
  [TK_ModAssign] = lit("%="),
  [TK_SubAssign] = lit("-="),
  [TK_AddAssign] = lit("+="),
  [TK_AndAssign] = lit("&="),
  [TK_OrAssign] = lit("|="),
  [TK_XorAssign] = lit("^="),
  [TK_LeftShiftAssign] = lit("<<="),
  [TK_RightShiftAssign] = lit(">>="),
  [TK_DoubleColon] = lit("::"),
  [TK_At] = lit("@"),
  [TK_Decrement] = lit("--"),
  [TK_Increment] = lit("++"),
  [TK_PostDecrement] = lit("--"),
  [TK_PostIncrement] = lit("++"),
  [TK_LesserThenOrEqual] = lit("<="),
  [TK_GreaterThenOrEqual] = lit(">="),
  [TK_Equals] = lit("=="),
  [TK_And] = lit("&&"),
  [TK_Or] = lit("||"),
  [TK_NotEquals] = lit("!="),
  [TK_LeftShift] = lit("<<"),
  [TK_RightShift] = lit(">>"),
  [TK_Arrow] = lit("->"),
  [TK_ExprSizeof] = lit("sizeof"),
  [TK_DocComment] = lit("DocComment"),
  [TK_Comment] = lit("Comment"),
  [TK_Identifier] = lit("Identifier"),
  [TK_StringLit] = lit("StringLit"),
  [TK_U8Lit] = lit("U8Lit"),
  [TK_Character] = lit("Character"),
  [TK_Error] = lit("Error"),
  [TK_Float] = lit("Float"),
  [TK_Int] = lit("Int"),
  [TK_Keyword] = lit("Keyword"),
};