krzosa/corelang
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Using arena as token array, remove arenas idea

Browse Source
This commit is contained in:
Krzosa Karol
2023-02-09 12:36:36 +01:00
parent 5138ba0097
commit 7370e8b716
16 changed files with 295 additions and 407 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
base.cpp
View File

@@ -260,7 +260,7 @@ T clamp(T min, T val, T max){
CORE_Static U64
hash_string(String string) {
U64 hash = (U64)14695981039346656037ULL;
for (U64 i = 0; i < string.len; i++) {
for (S64 i = 0; i < string.len; i++) {
hash = hash ^ (U64)(string.str[i]);
hash = hash * (U64)1099511628211ULL;
}

126
base_arena.cpp
View File

@@ -1,110 +1,6 @@
constexpr size_t ARENA_BLOCK_SIZE = mib(4);
constexpr size_t ARENA_ALIGNMENT = 8;
struct Scratch_Arena : Allocator {
int cap, len;
uint8_t memory[];
};
struct Scratch_Scope {
Scratch_Arena *arena;
int pos;
Scratch_Scope(Scratch_Arena *arena) { this->arena = arena; this->pos = arena->len; }
~Scratch_Scope() { this->arena->len = this->pos; }
};
static void *scratch_arena_push_size(Scratch_Arena *arena, size_t size) {
size_t generous_size = size + ARENA_ALIGNMENT;
if (arena->len + generous_size > arena->cap) {
assert(size < arena->cap);
assert_message(0, "Scratch arena is too small, failed to handle a request, capacity: %d, occupied: %d, allocation: %d", generous_size);
return 0;
}
size_t aligned_fill = align_up((size_t)arena->memory + arena->len, ARENA_ALIGNMENT) - (size_t)arena->memory;
assert(((int64_t)aligned_fill - (int64_t)arena->len) >= 0);
assert(((int64_t)aligned_fill - (int64_t)arena->len) <= ARENA_ALIGNMENT);
arena->len = aligned_fill;
uint8_t *result = arena->memory + arena->len;
arena->len += size;
return (void *)result;
}
static Scratch_Arena *make_scratch_arena(void *buff, size_t size) {
Scratch_Arena *scratch = (Scratch_Arena *)buff;
scratch->len = 0;
scratch->cap = size - sizeof(Scratch_Arena);
scratch->allocate = (Allocator::Allocate *)scratch_arena_push_size;
scratch->deallocate = deallocate_stub;
return scratch;
}
static Scratch_Arena *allocate_scratch_arena(Allocator *allocator, size_t size_without_members) {
Scratch_Arena *scratch = (Scratch_Arena *)allocate_size(allocator, size_without_members + sizeof(Scratch_Arena), false);
scratch->cap = size_without_members;
scratch->len = 0;
scratch->allocate = (Allocator::Allocate *)scratch_arena_push_size;
scratch->deallocate = deallocate_stub;
return scratch;
}
struct Push_Arena_Block {
Push_Arena_Block *next;
uint8_t memory[];
};
struct Push_Arena : Allocator {
Allocator *block_allocator;
Push_Arena_Block *blocks;
size_t allocated_block_count;
size_t block_fill;
size_t wasted_memory;
};
static void push_arena_add_block(Push_Arena *arena) {
Push_Arena_Block *block = (Push_Arena_Block *)allocate_size(arena->block_allocator, sizeof(Push_Arena_Block) + ARENA_BLOCK_SIZE, false);
memory_zero(block, sizeof(Push_Arena_Block));
SLL_STACK_ADD(arena->blocks, block);
if (arena->allocated_block_count) arena->wasted_memory += ARENA_BLOCK_SIZE - arena->block_fill;
arena->allocated_block_count += 1;
arena->block_fill = 0;
}
static void push_arena_deallocate(Push_Arena *arena) {
for(Push_Arena_Block *it = arena->blocks; it; it = it->next) {
deallocate(arena->block_allocator, it);
}
memory_zero(arena, sizeof(*arena));
}
static void *push_arena_push_size(Push_Arena *arena, size_t size) {
assert(arena->block_allocator);
assert(size < ARENA_BLOCK_SIZE / 2);
size_t generous_size = size + ARENA_ALIGNMENT;
if (arena->blocks == 0 || (arena->block_fill + generous_size > ARENA_BLOCK_SIZE)) {
push_arena_add_block(arena);
}
Push_Arena_Block *L = arena->blocks;
size_t aligned_fill = align_up((size_t)L->memory + arena->block_fill, ARENA_ALIGNMENT) - (size_t)L->memory;
assert(((int64_t)aligned_fill - (int64_t)arena->block_fill) >= 0);
assert(((int64_t)aligned_fill - (int64_t)arena->block_fill) <= ARENA_ALIGNMENT);
arena->block_fill = aligned_fill;
uint8_t *result = L->memory + arena->block_fill;
arena->block_fill += size;
return (void *)result;
}
static Push_Arena make_push_arena(Allocator *allocator) {
Push_Arena result = {};
result.block_allocator = allocator;
result.allocate = (Allocator::Allocate *)push_arena_push_size;
result.deallocate = (Allocator::Deallocate *)deallocate_stub;
return result;
}
struct CRT_Heap : Allocator {};
static void *crt_allocate(Allocator *allocator, size_t size) { return malloc(size); }
static void crt_deallocate(Allocator *allocator, void *p) { return free(p); }
@@ -208,6 +104,26 @@ CORE_Static Arena
result.memory.commit = size;
result.memory.reserve = size;
result.alignment = default_alignment;
result.len = 0;
result.allocate = (Allocator::Allocate *)arena_push_size;
result.deallocate= (Allocator::Deallocate *)deallocate_stub;
return result;
}
CORE_Static Arena
arena_from_buffer(void *buffer, size_t size) {
Arena result = {};
result.memory.data = (U8 *)buffer;
result.memory.commit = size;
result.memory.reserve = size;
result.alignment = default_alignment;
result.allocate = (Allocator::Allocate *)arena_push_size;
result.deallocate= (Allocator::Deallocate *)deallocate_stub;
return result;
}
struct Scratch_Scope {
Arena *arena;
int pos;
Scratch_Scope(Arena *arena) { this->arena = arena; this->pos = arena->len; }
~Scratch_Scope() { this->arena->len = this->pos; }
};

12
base_string.cpp
View File

@@ -91,7 +91,7 @@ string_fmtv(Allocator *a, const char *str, va_list args1) {
va_end(args2);
char *result = allocate_array(a, char, len + 1);
stbsp_vsnprintf(result, len + 1, str, args1);
stbsp_vsnprintf(result, (int)(len + 1), str, args1);
String res = {(U8 *)result, len};
return res;
@@ -187,7 +187,7 @@ struct String_Builder{
char *write_address = (char *)block->data + block->len;
va_copy(args2, args);
int written = stbsp_vsnprintf(write_address, block_size, str, args2);
int written = stbsp_vsnprintf(write_address, (int)block_size, str, args2);
va_end(args2);
if(written > (block_size-1)){
@@ -348,7 +348,7 @@ string_trim_end(String string) {
CORE_Static String
string_to_lower_case(Allocator *arena, String s) {
String copy = string_copy(arena, s);
for (U64 i = 0; i < copy.len; i++) {
for (S64 i = 0; i < copy.len; i++) {
copy.str[i] = to_lower_case(copy.str[i]);
}
return copy;
@@ -357,7 +357,7 @@ string_to_lower_case(Allocator *arena, String s) {
CORE_Static String
string_to_upper_case(Allocator *arena, String s) {
String copy = string_copy(arena, s);
for (U64 i = 0; i < copy.len; i++) {
for (S64 i = 0; i < copy.len; i++) {
copy.str[i] = to_upper_case(copy.str[i]);
}
return copy;
@@ -455,10 +455,10 @@ struct String_Replace {
};
CORE_Static String
string_replace(Scratch_Arena *scratch, Allocator *allocator, String string, Array<String_Replace> pairs){
string_replace(Arena *scratch, Allocator *allocator, String string, Array<String_Replace> pairs){
Scratch_Scope _scope(scratch);
String_Builder builder = {scratch};
for(int i = 0; i < string.len; i++){
for(S64 i = 0; i < string.len; i++){
For(pairs){
String current = string_skip(string, i);
current = string_get_prefix(current, it.find.len);

4
base_unicode.cpp
View File

@@ -162,7 +162,7 @@ CORE_Static String32
string16_to_string32(Allocator *allocator, String16 string){
String32 result = {allocate_array(allocator, U32, string.len+1)};
for(S64 i = 0; i < string.len;){
UTF32_Result decode = utf16_to_utf32(string.str + i, string.len - i);
UTF32_Result decode = utf16_to_utf32(string.str + i, (S32)(string.len - i));
if(!decode.error){
i += decode.advance;
result.str[result.len++] = decode.out_str;
@@ -215,7 +215,7 @@ CORE_Static String
string16_to_string8(Allocator *allocator, String16 in){
String result = {allocate_array(allocator, U8, in.len*4+1)};
for(S64 i = 0; i < in.len;){
UTF32_Result decode = utf16_to_utf32(in.str + i, in.len - i);
UTF32_Result decode = utf16_to_utf32(in.str + i, (S32)(in.len - i));
if(!decode.error){
i += decode.advance;
UTF8_Result encode = utf32_to_utf8(decode.out_str);

5
build.bat
View File

@@ -1,10 +1,11 @@
@echo off
if exist "..\misc\compile_setup.bat" call "..\misc\compile_setup.bat"
set clang-flags=-O0 -Wall -Wno-unused-function -fno-exceptions -fdiagnostics-absolute-paths -g -o main.exe -Wl,user32.lib
pushd %~dp0
rem cl main.cpp -I.. user32.lib
clang core_main.cpp %clang-flags%
cl core_main.cpp -Zi -nologo -W3 -wd4200 -wd4267 -wd4244 -Fe:main.exe user32.lib
rem clang core_main.cpp %clang-flags%
rem ubuntu run clang core_main.cpp -O0 -Wall -Wno-unused-function -fno-exceptions -fdiagnostics-absolute-paths -g -o core.out
rem clang core_arena.cpp %clang-flags%

6
c3_big_int.cpp
View File

@@ -632,6 +632,10 @@ bool mul_u64_overflow(uint64_t op1, uint64_t op2, uint64_t *result)
#endif
// @! Big int: Maybe look into those bool operations?
#pragma warning( push )
#pragma warning( disable : 4804 )
void bigint_add(BigInt *dest, const BigInt *op1, const BigInt *op2)
{
assert(dest != op1);
@@ -772,6 +776,8 @@ void bigint_add(BigInt *dest, const BigInt *op1, const BigInt *op2)
dest->digit_count = i;
normalize(dest);
}
#pragma warning( pop )
void bigint_add_wrap(BigInt *dest, const BigInt *op1, const BigInt *op2, size_t bit_count, bool is_signed)
{

12
core_codegen_c_language.cpp
View File

@@ -44,7 +44,7 @@ string_scope_name(Allocator *a, Ast_Scope *scope){
CORE_Static void
gen_scope_name(Ast_Scope *scope){
Scratch_Arena *scratch = pctx->scratch;
Arena *scratch = pctx->scratch;
Scratch_Scope _scope(scratch);
String string = string_scope_name(scratch, scope);
gen("%.*s", (int)string.len, string.str);
@@ -169,7 +169,7 @@ string_simple_decl(Allocator *a, Ast_Type *ast, Intern_String name = {}){
CORE_Static void
gen_simple_decl(Ast_Type *ast, Intern_String name = {}){
Scratch_Arena *scratch = pctx->scratch;
Arena *scratch = pctx->scratch;
Scratch_Scope _scope(scratch);
String string = string_simple_decl(scratch, ast, name);
@@ -205,7 +205,7 @@ gen_value(Token *pos, Value a){
switch(type->kind){
CASE_INT: {
Scratch_Arena *scratch = pctx->scratch;
Arena *scratch = pctx->scratch;
Scratch_Scope _scope(scratch);
String postfix = get_type_postfix(type);
@@ -531,7 +531,7 @@ gen_ast(Ast *ast){
CASE(RETURN, Return){
if(is_tuple(node->resolved_type)) {
Scratch_Arena *scratch = pctx->scratch;
Arena *scratch = pctx->scratch;
Scratch_Scope _scope(scratch);
Intern_String var_name = pctx->intern(unique_name_scratch(scratch, node));
@@ -742,7 +742,7 @@ gen_ast(Ast *ast){
For(node->vars)
gen_ast(it);
Scratch_Arena *scratch = pctx->scratch;
Arena *scratch = pctx->scratch;
Scratch_Scope _scope(scratch);
Intern_String var_name = pctx->intern(unique_name_scratch(scratch, node));
@@ -844,7 +844,7 @@ typedef struct String{
// Generate slice and tuple types
For_Named(pctx->all_types, type){
Scratch_Arena *scratch = pctx->scratch;
Arena *scratch = pctx->scratch;
Scratch_Scope _scope(scratch);
if(type->kind == TYPE_SLICE){

87
core_compiler.cpp
View File

@@ -10,10 +10,12 @@ static void core_init_compiler(Core_Ctx *ctx, Allocator *allocator) {
ctx->color_codes_enabled = true;
ctx->same_scope_token = { SAME_SCOPE };
ctx->perm_push_only = make_push_arena(allocator);
arena_init(&ctx->perm_push_only, "Perm Push Only"_s);
arena_init(&ctx->scratch_, "Scratch"_s);
arena_init(&ctx->stage_arena_, "Stage Arena"_s);
ctx->scratch = &ctx->scratch_;
ctx->stage_arena = &ctx->stage_arena_;
ctx->perm = &ctx->perm_push_only;
ctx->scratch = allocate_scratch_arena(ctx->perm, mib(1));
ctx->stage_arena = allocate_scratch_arena(ctx->perm, mib(1));
ctx->heap = allocator;
ctx->type_map = map_make(ctx->heap, 2048);
ctx->gen = {ctx->perm};
@@ -197,12 +199,17 @@ parse_all_modules() {
}
CORE_Static Ast_Module *
add_module(Token *pos, Intern_String filename, B32 command_line_module) {
Scratch_Arena *scratch = pctx->scratch;
add_module(Token *pos, Intern_String filename, B32 command_line_module, bool string_only_module) {
Arena *scratch = pctx->scratch;
Scratch_Scope _scope(scratch);
String absolute_file_path = {};
String absolute_base_folder = {};
if (string_only_module) {
absolute_file_path = filename.s;
absolute_base_folder = filename.s;
}
else {
//
// Find in working directory
//
@@ -232,6 +239,7 @@ add_module(Token *pos, Intern_String filename, B32 command_line_module) {
if (absolute_file_path.len == 0) {
compiler_error(pos, "Couldn't find the module with name %Q", filename);
}
}
For(pctx->modules) {
if (string_compare(it->absolute_file_path, absolute_file_path)) {
@@ -280,7 +288,69 @@ compile_file_to_string(Allocator *allocator, String filename) {
pctx->time.total = total_time;
pctx->time.start = total_time;
{
Ast_Module *module = add_module(0, pctx->intern("Language.core"_s));
Ast_Module *module = add_module(0, pctx->intern("Language.core"_s), false, true);
get(&module->all_loaded_files, 0)->filecontent =
R"(
Any :: struct
data: *void
type: Type
Type_Info_Kind :: enum
S64 // FIRST_NUMERIC
S32
S16
S8
INT
CHAR
U64
U32
U16
U8
F32
F64
POINTER
BOOL // LAST_NUMERIC
STRING
VOID
ARRAY
LAMBDA
STRUCT
UNION
ENUM
TYPE
SLICE
TUPLE
Type_Info_Struct_Member :: struct
name: String
type: Type
offset: S64
Type_Info :: struct
kind: Type_Info_Kind
size: S64
align: S64
is_unsigned: Bool
type: Type
base_type: Type
array_size: S64
struct_member_count: S64
struct_members: *Type_Info_Struct_Member
lambda_argument_count: S64
lambda_arguments: *Type_Info
lambda_return: Type
type_infos_len: S64 #foreign
type_infos : *Type_Info #foreign
GetTypeInfo :: (type: Type): *Type_Info
id := type->S64
if id >= type_infos_len
return 0
return type_infos + id
)"_s;
{
insert_builtin_type_into_scope(module, "S64"_s, pctx->type_s64);
insert_builtin_type_into_scope(module, "S32"_s, pctx->type_s32);
@@ -300,6 +370,11 @@ compile_file_to_string(Allocator *allocator, String filename) {
insert_builtin_type_into_scope(module, "Type"_s, pctx->type_type);
}
/*
Ast_Decl *namespace = core_create_namespace("name");
Ast_Decl *core_create_constant("name",
*/
{
Ast_Scope *scope = ast_decl_scope(&pctx->null_token, pctx->perm, get(&module->all_loaded_files, 0));
Ast_Decl * decl = ast_namespace(&pctx->null_token, scope, pctx->intern("Const"_s));

30
core_compiler.h
View File

@@ -1,24 +1,4 @@
/*
@! Dont fully rely on files!!!
@! Language.core shouldnt be a file!
@! Separate out the codegen stage cause that can change
@! Look into String_Builders in Core_Ctx
@! Look into stage allocator and perhaps
use it more often to reduce scenarios
with 2 allocators, and simplify stuff
@! Cleanup big int allocator
@! Errors and logs should be placed on the compiler context
@! Clean way to free all memory and reset the compiler
@! Bring the Table<>
@! Look into List, check if that's neccessary
Probably want to implement a Red Black Tree then
probably I wouldn't need any sort of heap based
data structure.
*/
struct Lex_Stream{
String stream;
S64 iter;
@@ -33,11 +13,13 @@ struct Lex_Stream{
struct Core_Ctx{
Allocator *heap;
Push_Arena perm_push_only;
Push_Arena *perm; // Stores: AST, tokens, interns
Arena perm_push_only;
Arena *perm; // Stores: AST, tokens, interns
Scratch_Arena *scratch;
Scratch_Arena *stage_arena;
Arena *scratch;
Arena scratch_;
Arena stage_arena_;
Arena *stage_arena;
String_Builder helper_builder;
int errors_occured;

46
core_compiler_interface.hpp
View File

@@ -648,5 +648,51 @@ struct Core_Message {
Core_Context *core_create_context(Allocator *allocator);
Array<Token> core_lex_string(Core_Context *ctx, char *string, size_t size);
AST Modification API
AST Query API
Tree walk interpreter / Bytecode interpreter
C code generation backend
Control of compilation with many hooks
Ast tags
* Accessing and modifying the AST
* Using as a scripting language
* Tree walk interp or Bytecode
* Using as a compiler
* C code backend
* C++ code backend ?
* Hooking into raw strings to preprocess
* Hooking into tokens to preprocess
* Hooking into AST to manipulate
* Hooking into Parsing to add syntax sugar
* Hooking into Parsing and typechecking to add language features
* Hooking into typechecking to add for example builtin functions
* Creating a custom code generation backend
* Performing compilation passes multiple times to get some sort of effect
* Accessing the typechecked AST and modifying it then typechecking modifications ??
* Tagging syntax for creating language features
Possible use cases:
I want to use it as a scripting language for my game
It should be able to not leak memory while rerunning scripts contnously
It should allow me to constrain certain features (pointer arithmetic)
It should allow me to add possible functions
I want to use this as a code generation tool
I should be able to access and modify the AST
I should be able to access the tokens for pre processing
I should be able to add my own syntax
I should be albe to add my own features
I should be able to create or modify a code generating back end
Code generation
Making DSLs
*/

35
core_lexing.cpp
View File

@@ -205,24 +205,32 @@ lex_is_scope(Token *t){
}
CORE_Static void
lex_unwind_indent_stack(Token *t, Lex_Stream *s, Array<Token> *array){
lex_add_token(Core_Ctx *ctx, Token *token) {
Token *top = (Token *)arena_push_size(ctx->stage_arena, sizeof(Token));
*top = *token;
ctx->tokens.len += 1;
ctx->tokens.data = (Token *)ctx->stage_arena->memory.data;
}
CORE_Static void
lex_unwind_indent_stack(Core_Ctx *ctx, Token *t, Lex_Stream *s){
for(S64 i = s->indent_stack.len-1; i >= 0; i-=1){
auto it = s->indent_stack.data[i];
assert(lex_is_scope(it));
if(it->indent == t->indent){
t->kind = SAME_SCOPE;
array->add(*t);
lex_add_token(ctx, t);
break;
}
else if(it->indent < t->indent){
token_error(t, "Bad indentation"_s);
array->add(*t);
lex_add_token(ctx, t);
break;
}
else{
s->indent_stack.pop();
t->kind = CLOSE_SCOPE;
array->add(*t);
lex_add_token(ctx, t);
}
}
}
@@ -230,7 +238,6 @@ lex_unwind_indent_stack(Token *t, Lex_Stream *s, Array<Token> *array){
CORE_Static void
lex__stream(Core_Ctx *lexer){
Intern_Table *table = &lexer->interns;
Array<Token> *array = &lexer->tokens;
Lex_Stream *s = &lexer->stream;
B32 beginning = true;
@@ -238,7 +245,7 @@ lex__stream(Core_Ctx *lexer){
if(lexc(s) == 0 || s->iter >= s->stream.len){
end_of_stream:
Token t = token_make(lexer);
lex_unwind_indent_stack(&t, s, array);
lex_unwind_indent_stack(lexer, &t, s);
break;
}
@@ -307,11 +314,11 @@ lex__stream(Core_Ctx *lexer){
lex_advance(s);
semi.kind = OPEN_SCOPE;
semi.indent = last->indent + 2; // @todo: proper detection of indentation
array->add(semi);
s->indent_stack.add(array->last());
lex_add_token(lexer, &semi);
s->indent_stack.add(lexer->tokens.last());
} else{
semi.kind = SAME_SCOPE;
array->add(semi);
lex_add_token(lexer, &semi);
}
} break;
@@ -327,16 +334,16 @@ lex__stream(Core_Ctx *lexer){
Token *last = lex_last_indent_token(s);
if(t.indent > last->indent){
t.kind = OPEN_SCOPE;
array->add(t);
s->indent_stack.add(array->last());
lex_add_token(lexer, &t);
s->indent_stack.add(lexer->tokens.last());
}
else if(t.indent < last->indent){
lex_unwind_indent_stack(&t, s, array);
lex_unwind_indent_stack(lexer, &t, s);
}
else {
t.kind = SAME_SCOPE;
array->add(t);
lex_add_token(lexer, &t);
}
}
@@ -579,7 +586,7 @@ lex__stream(Core_Ctx *lexer){
if(t.len==0)
lex_set_len(s,&t);
array->add(t);
lex_add_token(lexer, &t);
}
#undef CASE2
#undef CASE3

203
core_main.cpp
View File

@@ -1,4 +1,25 @@
/*
@! Dont fully rely on files!!!
@! Language.core shouldnt be a file!
@! Nice way of creating valid declarations programatically
@! Add a token global that implies Ast is in the executable
@! Ast_Module should be Ast_File
@! Stringify Core_Message list
@! Separate out the codegen stage cause that can change
@! Look into String_Builders in Core_Ctx
@! Look into stage allocator and perhaps
use it more often to reduce scenarios
with 2 allocators, and simplify stuff
@! Cleanup big int allocator
@! Clean way to free all memory and reset the compiler
@! Bring the Table<>
core_create_namespace("memes", {
core_basic_type(
}, 4);
Fix backlog
- [ ] Fix invalid error message: AlmostLinearToSRGB :: (a: Color);; return {sqrtf(a.r), sqrtf(a.g), sqrtf(a.b), a.a}
@@ -86,175 +107,6 @@ Ideas
- [ ] Cast from array to pointer?
- [ ] Ternary operator?
- [ ] Optionally pass size and alignment calculations to C ?
-------------------------------------------------------------------------------
2022.09.29 - Function overloads, operator overloads, namespaces idea
I thought about adding function overloading but I'm not sure if that's
actually good. It seems pretty hard to implement and there are many problems
with it. It's probably better to implement descent namespacing mechanism.
Currently I'm thinking of something like this: operator overloads are this special
additional thing. They can be overloaded cause they are handled differently.
You are not looking up names, you are looking into an array of operators
during resolution of binary expressions etc. It shouldn't complicate things, hopefully.
-------------------------------------------------------------------------------
2022.05.28 - On lambda expressions
I think it's not wise to add the multiline lambda expressions
As is the case with python it would require new alternative syntax.
The idea was to imply that the whitespace significant syntax is just
inserting '{' '}' ';' automatically and if you decide to write a brace
it stops being whitespace significant and you can type everything with semicolons and braces
Problem is first of all it's kind of weird to have a completly different syntax
in a language to solve something that is a minor inconvenience,
second of all it turned out to be kind of bad, maybe if it would be more
complicated then it would be ok but it implies that you have to semicolon end
a lot of thing unintuitively.
Probably single line lambdas should be ok. Currently braces turn off indentation
awareness. There might be something you can do by trying to turn that on
problem is that complicates parsing a lot cause you have to account for multiple
indent styles, which means error messages become bad.
-------------------------------------------------------------------------------
2022.05.30 - On constructors and compound expressions
I unified the compound expression syntax (Type){} with function call syntax Type().
It's differentiating on whether you used type. I think there is a benefit to the language
not having an idea of overloading the type constructor. You will always know what will happen.
Unlike C++ for example. It seems like a minefield that can fuck your mind up. So many corner cases
and variants. having the idea of compounds doing one thing is reassuring I think.
You can always do a constructor by writing a function with lower case type if you want that.
For now I don't thing it should be overloadable.
-------------------------------------------------------------------------------
Imports
We compile lot's of files, we keep track of them in Parse_Ctx, making sure we don't
parse same thing twice. Files belong to a module, files can be loaded #load "file".
All the files see all the decls from all the files in that module. We can import
other modules using a different directive #import. #import perhaps should be lazily
evaluated, making sure we don't resolve stuff we don't require. Currently probably
want to export all the symbols, we can namespace them optionally.
2022.06.26 - import revision
Current design is a bit weird, there can be situations where you have colissions and stuff.
That's not cool. I was thinking of bringing back this idea where you have modules, implicit or
explicit(module directive at top of the file). Then each file that belongs to a module sees other
decls in that module and stuff like that. Other modules would be imported using a single keyword.
BUT on second thought I don't really like that design. It's not really clear how files that
belong to a module are found and concatenated. Then you would need to specify a directory to
compile instead of a single file and stuff like that which seems annoying! Current idea is
to just disallow having same file loaded twice! It's the same result as with the module stuff,
we cant reuse files, but we avoid file colissions where we load a file and a module we import loads
a file. I like the load files approach to a module cause we specify which files to compile and load.
Imports also could be this other thing where we load lazily and we decrease the amount of code
that we output. Code in our project shouldn't be evaluated lazily cause that's counter intuitive.
For modules it's a bit different cause they should be distributed as valid.
-------------------------------------------------------------------------------
## Done
- [x] Conditional compilation, maybe based on some pattern matching
- [x] #import "$OS.core"
- [x] You can't alias Lambdas because they are not evaluated as constant.
I used a little simplification where lambdas and structs were marked as such
in parsing I think. BUT Structs work so it's maybe just a little fix of constant
propagation using Operands, where I will need to modify Operand of lambda to
be constant AND rewrite_const to not rewrite a lambda OR SOMETHING, maybe it's cool
dont know. BUT not sure if we wont need to rewrite the idea that Lambdas can be Decls.
- [x] Operator Overloading
- [x] '.' Operator doesn't handle expressions inside the dot chain, no good, so casts don't work
- [x] Introduce List to reduce heap allocations and make it more arena friendly, can we get rid of heap completly?
- [x] Function renaming to prevent colissions, we can't really touch other stuff cause I want it to be easily debuggable
- [x] Fix Length etc. they should be function calls not operators
- [x] Idea to fix overshoot when debugging and it goes to the close bracket and there is not enough line directives. Store the last outputed line and propagate it on the close brace etc.
- [x] Disable .len for Strings, are there other things that use this convention?
- [x] Calculate size and alignment of struct data types
- [x] Consider changing syntax of scopes to use braces { } NO
- [x] Disable ability to parse inner structs, functions, constants etc. ?
- [x] Fix language_basics.kl string index error
- [x] Type as a parameter to a function, alloc :: (size: U64, type: Type)
- [x] Add token information to instructions
- [-] Mixing loads and imports leads to code duplication, is that what we want???
- [x] print those token lines nicely
- [x] Improve the python metaprogram
- [x] Implementing type Any
- [x] Runtime TypeInfo
- [x] Proper type Type support
- [x] Switch
- [x] Type aliases :: should probably be strictly typed, but assigning constant values should work
- [x] Array of inferred size
- [x] Casting pointers to and from void should be implicit
- [x] Multiple return values
- [x] Add c string
- [-] Should compound resolution use an algorithm to reorder compounds to initialize all fields in order
- [x] slices should be properly displayed in debugger
- [x] Imports inside of import shouldn't spill outside
- [x] Scope
- [x] #Assert that handles constants at compile time and vars at runtime
- [x] Hex 0x42
- [x] Rewrite where # happen,
- [x] elif
- [x] cast ->
- [x] Remodel compound from call to {}
- [x] Fix codegen renames
- [x] Field access rewrite
- [-] Constants embeded in structs should be able to refer to other constants in that namespace without prefix
- [-] Order independent constants in structs
- [-] Fix recursive lambdas in structs
- [x] Error message when file not found
- [x] Better error messages when type difference
- [-] Fixing access to functions/structs, in C we cant have functons inside of structs / functions so we need to rewrite the tree
- [x] Emitting #line
- [x] Making sure debugger works
- [x] We need ++ -- operators
- [x] Order independent declarations in structs ? NO MORE
- [x] Arrays with size passed
- [x] Values inited to 0 by default
- [x] Some way to call foreign functions
- [x] We are parsing wrong here: (t.str=(&string_to_lex.str)[i]);
- [x] Test new operators, add constant eval for them
- [x] lvalue, rvalue concept so we cant assign value to some arbitrary weird expression
- [x] Passing down program to compile through command line
- [x] More basic types
- [x] Implementing required operations int128
- [x] Fix casting
- [x] More for loop variations
- [x] Add basic support for floats
- [x] Converting from U64 token to S64 Atom introduces unnanounced error (negates) - probably need big int
- [x] Add basic setup for new type system
- [x] Access through struct names to constants Arena.CONSTANT
- [x] Enums
- [x] Make sure pointer arithmetic works
- [x] Initial for loop
- [x] Enum . access to values
- [x] Character literal
- [x] Compiling and running a program
- [x] Infinite for loop
- [x] in new typesystem: Fix calls, fix all example programs
- [x] Fix arithmetic operations in new type system
- [x] Init statements, different kinds [+=] [-=] etc.
- [x] Struct calls
- [x] Operators: Bit negation, Not
- [x] Default values in calls
- [x] Resolving calls with default values
- [x] Pass statement
- [x] Lexer: Need to insert scope endings when hitting End of file
- [x] Resolving calls with named args, with indexed args
- [x] Structs
- [x] Struct field access
- [x] Struct field access with dots while compiling to arrows in c
- [x] Typespecs should probably be expressions so stuff like would be possible :: \*[32]int
- [x] Initial order independence algorithm
- [x] Think about compound expressions, unify with calls - maybe Thing(a=1) instead of Thing{a=1}
*/
#include "core_compiler_includes.cpp"
@@ -276,7 +128,7 @@ static void compile_file(Allocator *allocator, String filename, U32 compile_flag
F64 total_compiler_time = os_time() - pctx->time.start;
printf("%f - ", total_compiler_time);
Scratch_Arena *scratch = pctx->scratch;
Arena *scratch = pctx->scratch;
Scratch_Scope _scope(scratch);
F64 begin = os_time();
@@ -324,10 +176,11 @@ static void compile_file(Allocator *allocator, String filename, U32 compile_flag
int main(int argument_count, char **arguments){
Arena arena = {};
Arena scratch = {};
arena_init(&arena, "Pernament arena"_s);
Scratch_Arena *scratch_arena = allocate_scratch_arena(&arena, mib(1));
arena_init(&scratch, "Pernament scratch arena"_s);
Array<String> args = {scratch_arena};
Array<String> args = {&scratch};
for(int i = 1; i < argument_count; i+=1){
String arg = string_from_cstring(arguments[i]);
args.add(arg);
@@ -371,9 +224,9 @@ int main(int argument_count, char **arguments){
For(args){
if(it == "-testing"_s){
Scratch_Scope _scope(scratch_arena);
Array<OS_File_Info> examples = os_list_dir(scratch_arena, scratch_arena, "examples"_s);
Array<OS_File_Info> tests = os_list_dir(scratch_arena, scratch_arena, "tests"_s);
Scratch_Scope _scope(&scratch);
Array<OS_File_Info> examples = os_list_dir(&scratch, &scratch, "examples"_s);
Array<OS_File_Info> tests = os_list_dir(&scratch, &scratch, "tests"_s);
For(examples){
if(it.is_directory) continue;
compile_file(&arena, it.absolute_path, COMPILE_AND_RUN | COMPILE_TESTING);

42
core_parsing.cpp
View File

@@ -43,33 +43,33 @@ String core_stringify_message(Core_Ctx *pctx, Allocator *allocator, Core_Message
b.addf("%Q", msg->string);
for (int i = 0; i < buff_cap(msg->tokens); i += 1) {
for (S64 i = 0; i < buff_cap(msg->tokens); i += 1) {
Token *token = msg->tokens[i];
if (token) {
b.addf("\n");
// Print from line begin to token
int i1 = token->str - token->line_begin;
S64 i1 = token->str - token->line_begin;
b.addf("%.*s", i1, token->line_begin);
// Print token part
if(color_codes_enabled){
b.addf( PRINTF_RED "%.*s" PRINTF_RESET, (int)token->len, token->str);
b.addf( PRINTF_RED "%.*s" PRINTF_RESET, (S64)token->len, token->str);
} else {
b.addf("%.*s", (int)token->len, token->str);
b.addf("%.*s", (S64)token->len, token->str);
}
// Print to end of line from token
int iend = 0;
S64 iend = 0;
U8 *pointer = token->str + token->len;
while(pointer[iend]!='\n' && pointer[iend]!=0) iend++;
b.addf("%.*s", iend, pointer);
}
}
for (int i = 0; i < buff_cap(msg->tokens); i += 1) {
for (S64 i = 0; i < buff_cap(msg->tokens); i += 1) {
Token *it = msg->tokens[i];
if (it) {
b.addf("\n%s:%d", it->file.str, (int)it->line + 1);
b.addf("\n%s:%d", it->file.str, (S64)it->line + 1);
}
}
@@ -222,7 +222,7 @@ parse_init_stmt(Ast_Expr *expr){
CORE_Static Ast_Call *
parse_expr_call(Ast_Expr *left, Token_Kind close_kind){
Scratch_Arena *scratch = pctx->scratch;
Arena *scratch = pctx->scratch;
Scratch_Scope __scope(scratch);
Token *pos = token_get();
Array<Ast_Call_Item *> exprs = {scratch};
@@ -275,7 +275,7 @@ parse_stmt_scope(Ast_Scope *scope_defined_outside = 0){
if(token_expect(OPEN_SCOPE)){ // @todo: Fix error message here, it doesn't show proper token context
Token *token_block = token_get();
Scratch_Arena *scratch = pctx->scratch;
Arena *scratch = pctx->scratch;
Scratch_Scope __scope(scratch);
if(!scope_defined_outside) scope = begin_stmt_scope(scratch, token_block);
do{
@@ -460,7 +460,7 @@ parse_stmt_scope(Ast_Scope *scope_defined_outside = 0){
CORE_Static Ast_Lambda *
parse_lambda(Token *token){
Scratch_Arena *scratch = pctx->scratch;
Arena *scratch = pctx->scratch;
Scratch_Scope __scope(scratch);
Array<Ast_Decl *> params = {scratch};
@@ -688,7 +688,7 @@ parse_assign_expr(){
CORE_Static Ast_Decl *
parse_struct(Token *pos){
Scratch_Arena *scratch = pctx->scratch;
Arena *scratch = pctx->scratch;
Scratch_Scope __scope(scratch);
token_match(OPEN_SCOPE);
@@ -713,7 +713,7 @@ parse_struct(Token *pos){
CORE_Static Ast_Decl *
parse_enum(Token *pos){
Scratch_Arena *scratch = pctx->scratch;
Arena *scratch = pctx->scratch;
Scratch_Scope __scope(scratch);
Ast_Expr *typespec = parse_optional_type();
Token *flag = token_match_pound(pctx->intern_flag);
@@ -756,15 +756,15 @@ CORE_Static Ast_File *
register_ast_file(Token *pos, String absolute_file_path, Ast_Module *module, B32 global_implicit_load){
Ast_File *file = 0;
For_Named(pctx->files, it_file){
if(string_compare(it_file->absolute_file_path, absolute_file_path)){
if(module == it_file->module){
For(pctx->files){
if(string_compare(it->absolute_file_path, absolute_file_path)){
if(module == it->module){
log_trace("%Q :: Returning registered file: %Q\n", module->absolute_file_path, absolute_file_path);
file = it_file;
file = it;
break;
}
compiler_error(it_file->pos, pos, "This file is already loaded by module: %Q, try importing that module to get access to it", module->absolute_file_path);
compiler_error(it->pos, pos, "This file is already loaded by module: %Q, try importing that module to get access to it", module->absolute_file_path);
}
}
@@ -812,7 +812,7 @@ parse_load(B32 global_implicit_load){
return result;
}
CORE_Static Ast_Module *add_module(Token *pos, Intern_String filename, B32 command_line_module = false);
CORE_Static Ast_Module *add_module(Token *pos, Intern_String filename, B32 command_line_module = false, bool string_only_module = false);
CORE_Static Ast_Module *
parse_import(B32 global_implicit_import){
Token *file = token_expect(TK_StringLit);
@@ -938,13 +938,15 @@ parse_decl(B32 is_global){
CORE_Static void
parse_file(Ast_File *file){
assert(file);
Scratch_Arena *scratch = pctx->scratch;
Arena *scratch = pctx->scratch;
Scratch_Scope __scope(scratch);
if (!file->filecontent.len) {
file->filecontent = os_read_file(pctx->perm, file->absolute_file_path);
if(file->filecontent.len == 0){
compiler_error(file->pos, "Failed to open file \"%Q\"", file->absolute_file_path);
}
}
pctx->currently_parsed_file = file;
pctx->currently_parsed_scope = file;

24
core_typechecking.cpp
View File

@@ -114,7 +114,7 @@ CORE_Static void
check_value_bounds(Token *pos, Value *a){
if(!is_int(a->type)) return;
Scratch_Arena *scratch = pctx->scratch;
Arena *scratch = pctx->scratch;
Scratch_Scope __scope(scratch);
if(!bigint_fits_in_bits(&a->big_int_val, a->type->size*8, is_signed_int(a->type))){
const char *string = bigint_to_error_string(scratch, &a->big_int_val, 10);
@@ -502,7 +502,7 @@ make_scope_search(Allocator *arena, Ast_Scope *scope, Intern_String name){
CORE_Static Ast_Decl *
search_for_single_decl(Ast_Scope *scope, Intern_String name){
Scratch_Arena *scratch = pctx->scratch;
Arena *scratch = pctx->scratch;
Scratch_Scope _scope(scratch);
Scope_Search search = make_scope_search(scratch, scope, name);
@@ -520,7 +520,7 @@ search_for_single_decl(Ast_Scope *scope, Intern_String name){
CORE_Static Ast_Decl *
resolve_name(Ast_Scope *scope, Token *pos, Intern_String name, Search_Flag search_flags){
Scratch_Arena *scratch = pctx->scratch;
Arena *scratch = pctx->scratch;
Scratch_Scope _scope(scratch);
Scope_Search search = make_scope_search(scratch, scope, name);
search.search_only_current_scope = search_flags & SEARCH_ONLY_CURRENT_SCOPE;
@@ -554,7 +554,7 @@ resolve_operator_overload(Ast_Scope *scope, Ast_Type *left, Ast_Type *right, Tok
// Search for all possible candidates in three scopes
// The current module, left type definition module, right type definition module
Scratch_Arena *scratch = pctx->scratch;
Arena *scratch = pctx->scratch;
Scratch_Scope _scope(scratch);
Scope_Search search = make_scope_search(scratch, scope, op_info->op);
if( left->ast && left->ast->parent_scope) search.scopes.add(left->ast->parent_scope);
@@ -589,7 +589,7 @@ insert_into_scope(Ast_Scope *scope, Ast_Decl *decl){
//
// It's also called when scanning top level declarations of a module
// as such we probably don't want to call any resolve stuff here
Scratch_Arena *scratch = pctx->scratch;
Arena *scratch = pctx->scratch;
Scratch_Scope _scope(scratch);
Scope_Search search = make_scope_search(scratch, scope, decl->name);
search.search_only_current_scope = true;
@@ -675,7 +675,7 @@ resolve_typespec(Ast_Expr *ast, Resolve_Flag flags){
if(!ast && is_flag_set(flags, AST_CAN_BE_NULL))
return 0;
Scratch_Arena *scratch = pctx->scratch;
Arena *scratch = pctx->scratch;
Scratch_Scope _scope(scratch);
Operand resolved = resolve_expr(ast, flags | RESOLVE_TYPESPEC, 0, 0);
if(is_flag_set(flags, RESOLVE_TYPESPEC_COMPLETE))
@@ -693,7 +693,7 @@ resolve_and_require_bool(const char *error, Ast_Expr *expr, Resolve_Flag flags){
else compiler_error(0, "Compiler error: Null expression");
}
Scratch_Arena *scratch = pctx->scratch;
Arena *scratch = pctx->scratch;
Scratch_Scope _scope(scratch);
Operand op = resolve_expr(expr, flags, 0, 0);
if(!is_bool(op.type)){
@@ -730,7 +730,7 @@ resolve_stmt(Ast *ast, Ast_Type *ret){
switch(ast->kind){
CASE(RETURN, Return){ // @todo: need to check if all paths return a value
Scratch_Arena *scratch = pctx->scratch;
Arena *scratch = pctx->scratch;
Scratch_Scope _scope(scratch);
Array<Ast_Type *> types = {scratch};
@@ -897,7 +897,7 @@ resolve_stmt(Ast *ast, Ast_Type *ret){
CORE_Static Ast_Type *
resolve_lambda_type(Ast_Lambda *lambda){
Scratch_Arena *scratch = pctx->scratch;
Arena *scratch = pctx->scratch;
Scratch_Scope _scope(scratch);
Array<Ast_Type *> args = {scratch};
Array<Ast_Type *> ret = {scratch};
@@ -1016,7 +1016,7 @@ resolve_compound_array(Ast_Call *node, Ast_Type *type){
if(i >= size)
compiler_error(it->pos, "Index %lld is out of range of array of size %lld", i, size);
it->resolved_index = i;
it->resolved_index = (S32)i;
}
else{
if(default_counter == -1)
@@ -1521,7 +1521,7 @@ resolve_expr(Ast_Expr *ast, Resolve_Flag flags, Ast_Type *compound_and_const_str
}
node->resolved_decl = name.resolved_decl;
Scratch_Arena *scratch = pctx->scratch;
Arena *scratch = pctx->scratch;
Scratch_Scope _scope(scratch);
Array<Ast_Call_Item *> items = {scratch};
S64 default_iter = 0;
@@ -1645,7 +1645,7 @@ resolve_decl(Ast_Decl *ast){
try_resolving_lambda_scope(&result, lambda, node->type);
node->value = result.value;
Scratch_Arena *scratch = pctx->scratch;
Arena *scratch = pctx->scratch;
Scratch_Scope _scope(scratch);
node->unique_name = node->name;

2
core_types.cpp
View File

@@ -226,7 +226,7 @@ type_struct_complete(Ast_Type *type, Ast_Decl *node){
assert(node->kind == AST_STRUCT);
// @todo: compute size, alignement, offset !!!
// @note: resolve all the struct members first
Scratch_Arena *scratch = pctx->scratch;
Arena *scratch = pctx->scratch;
Scratch_Scope __scope(scratch);
Array<Ast_Resolved_Member> members = {scratch};

12
os_windows.cpp
View File

@@ -161,10 +161,10 @@ os_get_exe_dir(Allocator *a){
CORE_Static String
os_get_absolute_path(Allocator *a, String path){
char buff[2048];
Scratch_Arena *scratch = make_scratch_arena(buff, 2048);
String16 path16 = string8_to_string16(scratch, path);
Arena scratch = arena_from_buffer(buff, 2048);
String16 path16 = string8_to_string16(&scratch, path);
wchar_t *buffer = allocate_array(scratch, wchar_t, 512);
wchar_t *buffer = allocate_array(&scratch, wchar_t, 512);
DWORD written = GetFullPathNameW((wchar_t *)path16.str, 512, buffer, 0);
if(written == 0) return {};
@@ -177,15 +177,15 @@ os_get_absolute_path(Allocator *a, String path){
CORE_Static B32
os_does_file_exist(String path){
char buff[2048];
Scratch_Arena *scratch = make_scratch_arena(buff, buff_cap(buff));
String16 path16 = string8_to_string16(scratch, path);
Arena scratch = arena_from_buffer(buff, buff_cap(buff));
String16 path16 = string8_to_string16(&scratch, path);
DWORD attribs = GetFileAttributesW((wchar_t *)path16.str);
B32 result = attribs == INVALID_FILE_ATTRIBUTES ? false : true;
return result;
}
CORE_Static Array<OS_File_Info>
os_list_dir(Scratch_Arena *scratch, Allocator *a, String dir, U32 flags = LIST_NO_FLAGS){
os_list_dir(Arena *scratch, Allocator *a, String dir, U32 flags = LIST_NO_FLAGS){
Scratch_Scope _scope(scratch);
Array<String> dirs_to_read = {scratch};
dirs_to_read.add(dir);