corelang/core_compiler.h

struct Ast_Scope;
struct Ast_Decl;
struct Ast_File_Namespace;
struct Ast_File;
struct Ast_Module;
struct Ast_Type;
function Ast_Module *ast_module(Token *pos, Intern_String filename);
function void insert_builtin_types_into_scope(Ast_Scope *p);

enum Token_Kind{
  TK_End,

  TK_Mul,
  TK_Div,
  TK_Mod,
  TK_LeftShift,
  TK_RightShift,
  TK_FirstMul = TK_Mul,
  TK_LastMul = TK_RightShift,

  TK_Add,
  TK_Sub,
  TK_FirstAdd = TK_Add,
  TK_LastAdd = TK_Sub,

  TK_Equals,
  TK_LesserThenOrEqual,
  TK_GreaterThenOrEqual,
  TK_LesserThen,
  TK_GreaterThen,
  TK_NotEquals,
  TK_FirstCompare = TK_Equals,
  TK_LastCompare = TK_NotEquals,

  TK_BitAnd,
  TK_BitOr,
  TK_BitXor,
  TK_And,
  TK_Or,
  TK_FirstLogical = TK_BitAnd,
  TK_LastLogical = TK_Or,

  TK_Neg,
  TK_Not,
  TK_OpenParen,
  TK_CloseParen,
  TK_OpenBrace,
  TK_CloseBrace,
  TK_OpenBracket,
  TK_CloseBracket,
  TK_Comma,
  TK_Pound,
  TK_Question,
  TK_ThreeDots,
  TK_Semicolon,
  TK_Dot,
  TK_TwoDots,

  TK_NewLine,
  TK_Colon,

  TK_Assign,
  TK_ColonAssign,
  TK_DivAssign,
  TK_MulAssign,
  TK_ModAssign,
  TK_SubAssign,
  TK_AddAssign,
  TK_AndAssign,
  TK_OrAssign,
  TK_XorAssign,
  TK_LeftShiftAssign,
  TK_RightShiftAssign,
  TK_FirstAssign = TK_Assign,
  TK_LastAssign = TK_RightShiftAssign,

  TK_DoubleColon,
  TK_At,
  TK_Decrement,
  TK_Increment,
  TK_PostDecrement,
  TK_PostIncrement,

  TK_Arrow,
  TK_ExprSizeof,
  TK_DocComment,
  TK_Comment,
  TK_Identifier,
  TK_UnicodeLit,
  TK_StringLit,
  TK_Error,
  TK_Float,
  TK_Integer,
  TK_Keyword,

  TK_Pointer     = TK_Mul,
  TK_Dereference = TK_BitAnd,


  OPEN_SCOPE = 128,
  CLOSE_SCOPE,
  SAME_SCOPE,
};

struct Token{
  Token_Kind kind;
  U32 di; // debug_id
  union{
    String string;
    struct{U8 *str; S64 len;};
  };

  union {
    U32 unicode;
    BigInt int_val;
    F64 f64_val;
    String error_val;
    Intern_String intern_val;
    S64 indent;
  };

  Intern_String file;
  S32           line;
  U8           *line_begin;
};
global Token null_token;

struct Lex_Stream{
  String stream;
  S64    iter;

  U8            *line_begin;
  Intern_String  file;
  S32            line;
  S32            inside_brace_paren;
  Array<Token *> indent_stack;
};

struct Lexer{
  Allocator *arena;
  Lex_Stream   stream;
  Array<Token> tokens;
  Intern_Table interns;
  S64 token_iter;
  U32 token_debug_ids;

  Intern_String intern(String string){
    return intern_string(&interns, string);
  }
};

// Lexer::interns::map::allocator - array allocator, resizing
// Lexer::tokens                  - array allocator, resizing
//
// Parser::ast_arena                - arena for asts
// Lexer::interns::string_allocator - arena for interns
//
Intern_String keyword_struct;
Intern_String keyword_union;
Intern_String keyword_return;
Intern_String keyword_if;
Intern_String keyword_else;
Intern_String keyword_true;
Intern_String keyword_false;
Intern_String keyword_for;
Intern_String keyword_pass;
Intern_String keyword_default;
Intern_String keyword_switch;
Intern_String keyword_break;
Intern_String keyword_elif;
Intern_String keyword_assert;
Intern_String keyword_enum;

Intern_String intern_sizeof;
Intern_String intern_alignof;
Intern_String intern_lengthof;
Intern_String intern_void;
Intern_String intern_foreign;
Intern_String intern_it;
Intern_String intern_strict;
Intern_String intern_flag;

struct Parse_Ctx:Lexer{
  Allocator *perm; // Stores: AST, tokens, interns
  Allocator *heap;
  Arena      stage_arena;

  Array<Ast_Type *> all_types;
  S32 type_ids;
  U64 unique_ids; // @Debug
  Map type_map;

  Ast_Module *language_base_module;

  Array<Ast_File    *> files;
  Array<Ast_Module  *> modules;
  Array<Ast_Decl    *> ordered_decls;
  S32 base_language_ordered_decl_len;

  Ast_Scope         *currently_parsed_scope;
  Ast_File          *currently_parsed_file;
  U32 scope_ids;

  Array<String> module_folders;
  String module_folder;
  String exe_folder;
  String working_folder;

  S64 indent;
  String_Builder gen;
  String_Builder helper_builder;
};

global B32 emit_line_directives;
function void init_type();

//-----------------------------------------------------------------------------
// Constructors
//-----------------------------------------------------------------------------
thread_local Parse_Ctx *pctx;
function void
lex_init(Allocator *token_string_arena, Allocator *map_allocator, Lexer *l){
  l->arena = token_string_arena;
  l->tokens = array_make<Token>(token_string_arena, 1024*2);
  l->interns= intern_table_make(token_string_arena, map_allocator, 1024);

  keyword_struct= l->intern("struct"_s);
  keyword_union = l->intern("union"_s);
  keyword_true = l->intern("true"_s);
  keyword_default = l->intern("default"_s);
  keyword_break = l->intern("break"_s);
  keyword_false = l->intern("false"_s);
  keyword_return = l->intern("return"_s);
  keyword_switch = l->intern("switch"_s);
  keyword_assert = l->intern("Assert"_s);
  keyword_if = l->intern("if"_s);
  keyword_elif = l->intern("elif"_s);
  keyword_pass = l->intern("pass"_s);
  keyword_else = l->intern("else"_s);
  keyword_for = l->intern("for"_s);
  keyword_enum  = intern_string(&l->interns, "enum"_s);
  l->interns.first_keyword = keyword_struct.str;
  l->interns.last_keyword  = keyword_enum.str;

  intern_sizeof = l->intern("SizeOf"_s);
  intern_lengthof = l->intern("Length"_s);
  intern_alignof = l->intern("AlignOf"_s);
  intern_foreign = intern_string(&l->interns, "foreign"_s);
  intern_strict = intern_string(&l->interns, "strict"_s);
  intern_void = intern_string(&l->interns, "void"_s);
  intern_flag = intern_string(&l->interns, "flag"_s);
  intern_it = intern_string(&l->interns, "it"_s);
}

function void
parse_init(Parse_Ctx *ctx, Allocator *perm_allocator, Allocator *heap_allocator){
  pctx               = ctx;
  ctx->perm          = perm_allocator;
  ctx->heap          = heap_allocator;
  ctx->gen           = {ctx->heap};
  ctx->ordered_decls = {ctx->heap};
  ctx->type_map      = {ctx->heap};
  ctx->modules       = {ctx->heap};
  ctx->all_types     = {ctx->heap};
  ctx->helper_builder= {ctx->heap};
  ctx->files         = {ctx->heap};
  ctx->scope_ids = 1;
  bigint_allocator   = ctx->perm;
  arena_init(&ctx->stage_arena, "Compiler stage arena"_s);

  lex_init(ctx->perm, ctx->heap, ctx);
  init_type();

  // Init paths
  ctx->exe_folder = os_get_exe_dir(ctx->perm);
  ctx->working_folder = os_get_working_dir(ctx->perm);

  ctx->module_folders = {ctx->heap};
  String main_module = string_fmt(ctx->perm, "%Q/modules", ctx->exe_folder);
  ctx->module_folders.add(main_module);
}