diff --git a/.gitignore b/.gitignore index b75f097..b21d3fd 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ *.ilk *.pdb *.txt -*.4c \ No newline at end of file +*.4c +*.bin \ No newline at end of file diff --git a/build.bat b/build.bat index 9b51886..8ae8a3b 100644 --- a/build.bat +++ b/build.bat @@ -1,3 +1,3 @@ @echo off -clang main.cpp -Wall -Wno-unused-function -fno-exceptions -fdiagnostics-absolute-paths -g -o main.exe -Wl,user32.lib +clang main.cpp -I.. -Wall -Wno-unused-function -fno-exceptions -fdiagnostics-absolute-paths -g -o main.exe -Wl,user32.lib diff --git a/main.c b/main.c deleted file mode 100644 index 0ff1cac..0000000 --- a/main.c +++ /dev/null @@ -1,29 +0,0 @@ -#define _CRT_SECURE_NO_WARNINGS -#include -#include - -#include "lang.h" -#include "os.h" -#include "memory.h" - -#include "common.c" -#include "memory.c" -#include "os_win32.c" - -#include "lex.c" -#include "ast.c" -#include "print.c" -#include "parse.c" -#include "type.c" -#include "resolve.c" - -int main(){ - lex_test(); - parse_test(); - ast_test(); - intern_test(); - map_test(); - array_test(); - test_types(); - resolve_test(); -} \ No newline at end of file diff --git a/main.cpp b/main.cpp index 961ba86..76045b6 100644 --- a/main.cpp +++ b/main.cpp @@ -1,1133 +1,18 @@ -#define NOMINMAX -#include -#include -typedef int8_t S8; -typedef int16_t S16; -typedef int32_t S32; -typedef int64_t S64; -typedef uint8_t U8; -typedef uint16_t U16; -typedef uint32_t U32; -typedef uint64_t U64; -typedef S8 B8; -typedef S16 B16; -typedef S32 B32; -typedef S64 B64; -typedef U64 SizeU; -typedef S64 SizeI; -typedef float F32; -typedef double F64; -#define function static -#define global static -#define force_inline __forceinline -#define assert(x) do{if(!(x))__debugbreak();}while(0) -#define assert_msg(x,...) assert(x) -#define invalid_codepath assert_msg(0, "Invalid codepath") -#define not_implemented assert_msg(0, "Not implemented") -#define buff_cap(x) (sizeof(x)/sizeof((x)[0])) -#define kib(x) ((x)*1024llu) -#define mib(x) (kib(x)*1024llu) -#define gib(x) (mib(x)*1024llu) -struct String{U8 *str;S64 len;}; -union Intern_String{ String s; struct{ U8 *str; S64 len; }; }; // Basically just String - -#define JOIN1(X,Y) X##Y // helper macro -#define JOIN(X,Y) JOIN1(X,Y) -//----------------------------------------------------------------------------- -// Utilities -//----------------------------------------------------------------------------- -function SizeU -get_align_offset(SizeU size, SizeU align){ - SizeU mask = align - 1; - SizeU val = size & mask; - if(val){ - val = align - val; - } - return val; -} - -function SizeU -align_up(SizeU size, SizeU align){ - SizeU result = size + get_align_offset(size, align); - return result; -} - -function SizeU -align_down(SizeU size, SizeU align){ - size += 1; // Make sure 8 when align is 8 doesn't get rounded down to 0 - SizeU result = size - (align - get_align_offset(size, align)); - return result; -} - -function void -memory_copy(void *dst, void *src, SizeU size){ - U8 *d = (U8*)dst; - U8 *s = (U8*)src; - for(SizeU i = 0; i < size; i++){ - d[i] = s[i]; - } -} - -function void -memory_zero(void *p, SizeU size){ - U8 *pp = (U8 *)p; - for(SizeU i = 0; i < size; i++) - pp[i] = 0; -} - -template -T max(T a, T b){ - if(a > b) return a; - return b; -} - -template -T min(T a, T b){ - if(a > b) return b; - return a; -} - -template -T clamp_top(T val, T max){ - if(val > max) val = max; - return val; -} - -template -T clamp_bot(T bot, T val){ - if(val < bot) val = bot; - return val; -} - -template -T clamp(T min, T val, T max){ - if(val > max) val = max; - if(val < min) val = min; - return val; -} - -function U64 -hash_string(String string) { - U64 hash = (U64)14695981039346656037ULL; - for (U64 i = 0; i < string.len; i++) { - hash = hash ^ (U64)(string.str[i]); - hash = hash * (U64)1099511628211ULL; - } - return hash; -} - -function U64 -hash_u64(U64 x) { - x *= 0xff51afd7ed558ccd; - x ^= x >> 32; - return x; -} - -function U64 -hash_ptr(const void *ptr) { - return hash_u64((uintptr_t)ptr); -} - -function U64 -hash_mix(U64 x, U64 y) { - x ^= y; - x *= 0xff51afd7ed558ccd; - x ^= x >> 32; - return x; -} - -function U64 -is_pow2(U64 x) { - assert(x != 0); - B32 result = (x & (x - 1llu)) == 0; - return result; -} - -function U64 -wrap_around_pow2(U64 x, U64 power_of_2) { - assert(is_pow2(power_of_2)); - U64 r = (((x)&((power_of_2)-1llu))); - return r; -} - -function B32 -string_compare(String a, String b){ - if(a.len != b.len) - return false; - for(S64 i = 0; i < a.len; i++){ - if(a.str[i] != b.str[i]) - return false; - } - return true; -} - -function U8 -char_to_lower(U8 c){ - if(c >= 'A' && c <= 'Z') - c += 32; - return c; -} - -function U8 -char_to_upper(U8 c){ - if(c >= 'a' && c <= 'z') - c -= 32; - return c; -} - -force_inline String -operator""_s(const char *str, size_t size){ - return String{(U8 *)str, (S64)size}; -} - -//----------------------------------------------------------------------------- -// OS Memory -//----------------------------------------------------------------------------- -const SizeU os_page_size = 4096; -struct OS_Memory{ - SizeU commit, reserve; - U8 *data; -}; - -function OS_Memory -os_reserve(SizeU size){ - OS_Memory result = {}; - SizeU adjusted_size = align_up(size, os_page_size); - result.data = (U8*)VirtualAlloc(0, adjusted_size, MEM_RESERVE, PAGE_READWRITE); - assert_msg(result.data, "Failed to reserve virtual memory"); - result.reserve = adjusted_size; - return result; -} - -function B32 -os_commit(OS_Memory *m, SizeU size){ - SizeU commit = align_up(size, os_page_size); - SizeU total_commit = m->commit + commit; - total_commit = clamp_top(total_commit, m->reserve); - SizeU adjusted_commit = total_commit - m->commit; - if(adjusted_commit != 0){ - void *result = VirtualAlloc((U8*)m->data + m->commit, adjusted_commit, MEM_COMMIT, PAGE_READWRITE); - assert_msg(result, "Failed to commit more memory"); - m->commit += adjusted_commit; - return true; - } - return false; -} - -function void -os_release(OS_Memory *m){ - BOOL result = VirtualFree(m->data, 0, MEM_RELEASE); - assert_msg(result != 0, "Failed to release OS_Memory"); - if(result){ - m->data = 0; - m->commit = 0; - m->reserve = 0; - } -} - -function B32 -os_decommit_pos(OS_Memory *m, SizeU pos){ - SizeU aligned = align_down(pos, os_page_size); - SizeU adjusted_pos = clamp_top(aligned, m->commit); - SizeU size_to_decommit = m->commit - adjusted_pos; - if(size_to_decommit){ - U8 *base_address = m->data + adjusted_pos; - BOOL result = VirtualFree(base_address, size_to_decommit, MEM_DECOMMIT); - if(result){ - m->commit -= size_to_decommit; - return true; - } - } - return false; -} - -function void -test_os_memory(){ - assert(align_down(4096, 4096) == 4096); - assert(align_down(4095, 4096) == 0); - - OS_Memory memory = os_reserve(9000); - assert(memory.reserve == 4096*3 && memory.data && memory.commit == 0); - os_commit(&memory, 100); - assert(memory.commit == 4096); - os_commit(&memory, 100); - assert(memory.commit == 4096*2); - os_commit(&memory, 9000); - assert(memory.commit == 4096*3); - os_commit(&memory, 9000); - assert(memory.commit == 4096*3); - - os_decommit_pos(&memory, 4096); - assert(memory.commit == 4096); - os_decommit_pos(&memory, 4096); - assert(memory.commit == 4096); - os_decommit_pos(&memory, 0); - assert(memory.commit == 0); - - os_release(&memory); - assert(memory.data == 0); -} - -//----------------------------------------------------------------------------- -// Base Allocator stuff -//----------------------------------------------------------------------------- -enum Allocation_Kind{Allocation_Alloc,Allocation_Resize,Allocation_FreeAll,Allocation_Free,Allocation_Destroy}; -struct Allocator; -typedef void *Allocator_Proc(Allocator*, Allocation_Kind, void *, SizeU); -struct Allocator{Allocator_Proc *proc; String debug_name;}; - -//----------------------------------------------------------------------------- -// Memory arenas -//----------------------------------------------------------------------------- -global const SizeU default_reserve_size = gib(4); -global const SizeU default_alignment = 8; -global const SizeU additional_commit_size = mib(1); -struct Arena:Allocator{ - OS_Memory memory; - SizeU alignment; - SizeU len; - - // Personal arena memes so we can compute correct size when resizing - // Also a pointer so that we can make sure it didn't change - SizeU old_size; - void *debug_prev_pointer; -}; - -function void arena_init(Arena *arena, String debug_name); - -function void -arena_pop_pos(Arena *arena, SizeU pos){ - pos = clamp_top(pos, arena->len); - arena->len = pos; -} - -function void -arena_release(Arena *arena){ - os_release(&arena->memory); -} - -function void -arena_clear(Arena *arena){ - arena_pop_pos(arena, 0); -} - -function void * -arena_push_size(Arena *a, SizeU size){ - SizeU generous_size = size + a->alignment; - if(a->len+generous_size>a->memory.commit){ - if(a->memory.reserve == 0){ - arena_init(a, "Zero initialized arena"_s); - } - B32 result = os_commit(&a->memory, generous_size+additional_commit_size); - assert(result); - } - - a->len = align_up(a->len, a->alignment); - assert(a->memory.reserve > a->len + a->memory.commit); - void *result = (U8*)a->memory.data + a->len; - a->len += size; - return result; -} - -force_inline void * -arena_allocator_proc(Allocator *a, Allocation_Kind kind, void *old_pointer, SizeU size){ - Arena *arena = (Arena *)a; - switch(kind){ - case Allocation_Alloc: return arena_push_size(arena, size); - case Allocation_Resize:{ - void *result = arena_push_size(arena, size); - memory_copy(result, old_pointer, size); - return result; - } - case Allocation_Free : return 0; - case Allocation_FreeAll: arena_clear(arena); return 0; - case Allocation_Destroy: arena_release(arena); return 0; - } - invalid_codepath; - return 0; -} - -force_inline void * -personal_arena_allocator_proc(Allocator *a, Allocation_Kind kind, void *old_pointer, SizeU size){ - Arena *arena = (Arena *)a; - arena->alignment = 1; - - void *result = 0; - switch(kind){ - case Allocation_Resize: { - assert(arena->old_size); - assert(arena->old_size < size); - assert(arena->debug_prev_pointer == old_pointer); - result = arena_push_size(arena, size - arena->old_size); - result = old_pointer; - } break; - default: { - result = arena_allocator_proc(a, kind, old_pointer, size); - arena->debug_prev_pointer = result; - } - } - - arena->old_size = size; - return result; -} - -function void -arena_init(Arena *a, String debug_name){ - a->memory = os_reserve(default_reserve_size); - a->alignment = default_alignment; - a->debug_name = debug_name; - if(!a->proc) a->proc = arena_allocator_proc; -} - -function Arena -arena_make_personal(String debug_name){ - Arena arena = {}; - arena.proc = personal_arena_allocator_proc; - arena_init(&arena, debug_name); - return arena; -} - -//----------------------------------------------------------------------------- -// OS Heap allocator -//----------------------------------------------------------------------------- -struct OS_Heap:Allocator{ - HANDLE handle; -}; - -function void * -os_heap_allocator_proc(Allocator *a, Allocation_Kind kind, void *old_pointer, SizeU size){ - OS_Heap *heap = (OS_Heap *)a; - switch(kind){ - case Allocation_FreeAll:{ - invalid_codepath; - return 0; - } - case Allocation_Destroy:{ - BOOL result = HeapDestroy(heap->handle); - assert(result != 0); - heap->handle = 0; - heap->proc = 0; - return 0; - } - case Allocation_Free:{ - BOOL result = HeapFree(heap->handle, 0, old_pointer); - assert(result != 0); - return 0; - } - case Allocation_Alloc:{ - void *result = HeapAlloc(heap->handle, 0, size); - assert(result); - return result; - } - case Allocation_Resize:{ - void *result = HeapReAlloc(heap->handle, 0, old_pointer, size); - assert(result); - return result; - } - default: invalid_codepath; - } - return 0; -} - -function OS_Heap // max_size == 0 == growing heap -win32_os_heap_create(B32 multithreaded, SizeU initial_size, SizeU max_size){ - OS_Heap result = {}; - result.proc = os_heap_allocator_proc; - result.handle = HeapCreate(multithreaded ? 0 : HEAP_NO_SERIALIZE, initial_size, max_size); - assert(result.handle); - return result; -} - -enum Log_Kind{Log_Kind_Normal, Log_Kind_Error}; -typedef void Log_Proc(Log_Kind kind, String string, char *file, int line); -//----------------------------------------------------------------------------- -// Thread Context -//----------------------------------------------------------------------------- -struct Thread_Ctx{ - Arena scratch[2]; - Allocator *implicit_allocator; - void *ctx; - U64 ctx_id; - Log_Proc *log_proc; - - int line; - char *file; -}; -thread_local Thread_Ctx thread_ctx; -global Arena pernament_arena; -global OS_Heap os_process_heap; - -#define report_file_and_line() report__file_and_line(__FILE__, __LINE__) -#define DEBUG_TRACE(x) (report_file_and_line(), (x)) -function void -report__file_and_line(const char *file, int line){ - thread_ctx.file = (char *)file; - thread_ctx.line = line; -} - -//----------------------------------------------------------------------------- -// Implicit scratch stack -//----------------------------------------------------------------------------- -#define Set_Scratch() Scoped_Scratch JOIN(scratch, __LINE__) -#define Set_Backup_Scratch() Scoped_Scratch JOIN(scratch, __LINE__)(true) -struct Scoped_Scratch{ - SizeU saved_pos; - Allocator *saved_allocator; - Arena *arena; - - Scoped_Scratch(B32 backup_scratch=false){ - if(!backup_scratch) arena = thread_ctx.scratch; - else arena = thread_ctx.scratch + 1; - saved_allocator = thread_ctx.implicit_allocator; - saved_pos = arena->len; - thread_ctx.implicit_allocator = arena; - } - ~Scoped_Scratch(){ - arena_pop_pos(arena, saved_pos); - thread_ctx.implicit_allocator = saved_allocator; - } -}; - -//----------------------------------------------------------------------------- -// Implicit allocator stack -//----------------------------------------------------------------------------- -#define Set_Allocator(a) Scoped_Allocator JOIN(scoped,__LINE__)(a) -struct Scoped_Allocator{ - Allocator *allocator; - Scoped_Allocator(Allocator *a){ - allocator = thread_ctx.implicit_allocator; - thread_ctx.implicit_allocator = a; - } - ~Scoped_Allocator(){ - thread_ctx.implicit_allocator = allocator; - } -}; - -function void * -thread_ctx_get_user_ctx(U64 id){ - assert(id == thread_ctx.ctx_id); - assert(id != 0); - assert(thread_ctx.ctx_id != 0); - assert(thread_ctx.ctx != 0); - return thread_ctx.ctx; -} - -#define Get_Ctx(T) T *ctx = (T *)thread_ctx_get_user_ctx(T##_ID) -#define Set_Ctx(ctx, id) Scoped_Ctx JOIN(scoped_ctx, __LINE__)((void *)ctx, id) -struct Scoped_Ctx{ - void *prev_ctx; - U64 prev_id; - Scoped_Ctx(void *in_ctx, U64 id){ - prev_ctx = thread_ctx.ctx; - prev_id = thread_ctx.ctx_id; - thread_ctx.ctx = in_ctx; - thread_ctx.ctx_id = id; - } - ~Scoped_Ctx(){thread_ctx.ctx = prev_ctx; thread_ctx.ctx_id = prev_id;} -}; - - -enum Alloc_Flag{AF_None,AF_ZeroMemory}; -//----------------------------------------------------------------------------- -// Explicit allocator -//----------------------------------------------------------------------------- -#define exp_alloc_array(a, T, size,...) (T *)exp_alloc(a, sizeof(T)*(size), ## __VA_ARGS__) -#define exp_alloc_type(a, T, ...) exp_alloc_array(a, T, 1, ## __VA_ARGS__) -#define exp_alloc(a, size, ...) DEBUG_TRACE(exp__alloc(a, size, ## __VA_ARGS__)) -#define exp_resize(a,p,size) DEBUG_TRACE(exp__resize(a, p, size)) -#define exp_resize_array(a, p, T, size) DEBUG_TRACE((T *)exp_resize(a, p, sizeof(T)*(size))) -#define exp_free(a, p) DEBUG_TRACE(exp__free(a, p)) -#define exp_free_all(a) DEBUG_TRACE(exp__free_all(a)) -#define exp_destroy(a) DEBUG_TRACE(exp__destroy(a)) - -#include -force_inline void * -exp__alloc(Allocator *a, SizeU size, Alloc_Flag flag = AF_None){ - printf("Alloc(%s) %s:%d %u\n", a->debug_name.str, thread_ctx.file, thread_ctx.line, (U32)size); - void *result = a->proc(a, Allocation_Alloc, 0, size); - if(flag & AF_ZeroMemory) memory_zero(result, size); - return result; -} -force_inline void * -exp__resize(Allocator *a, void *pointer, SizeU size){ - printf("Resize(%s) %s:%d %u\n", a->debug_name.str, thread_ctx.file, thread_ctx.line, (U32)size); - return a->proc(a, Allocation_Resize, pointer, size); -} -force_inline void -exp__free(Allocator *a, void *pointer){ - printf("Free(%s) %s:%d\n", a->debug_name.str, thread_ctx.file, thread_ctx.line); - a->proc(a, Allocation_Free, pointer, 0); -} -force_inline void -exp__free_all(Allocator *a){ - printf("FreeAll(%s) %s:%d\n", a->debug_name.str, thread_ctx.file, thread_ctx.line); - a->proc(a, Allocation_FreeAll, 0, 0); -} -force_inline void -exp__destroy(Allocator *a){ - printf("Destroy(%s) %s:%d\n", a->debug_name.str, thread_ctx.file, thread_ctx.line); - a->proc(a, Allocation_Destroy, 0, 0); -} - -//----------------------------------------------------------------------------- -// Implicit allocator -//----------------------------------------------------------------------------- -#define imp_alloc_array(T,size,...) (T *)imp_alloc(sizeof(T) * (size), ##__VA_ARGS__) -#define imp_alloc_type (T,...) imp_alloc_array(T,1, ## __VA_ARGS__) -#define imp_alloc(size,...) DEBUG_TRACE(imp__alloc(size, ##__VA_ARGS__)) -#define imp_resize_array(p, T,size, ...) (T *)imp_resize(p, sizeof(T) * (size), ##__VA_ARGS__) -#define imp_resize(p,size) DEBUG_TRACE(imp_resize(p, size)) -#define imp_free(p) DEBUG_TRACE(imp__free(p)) -#define imp_free_all() DEBUG_TRACE(imp__free_all()) -#define imp_destroy() DEBUG_TRACE(imp__destroy()) - -force_inline void * -imp__alloc(SizeU size, Alloc_Flag flag=AF_None){ - return exp__alloc(thread_ctx.implicit_allocator, size, flag); -} -force_inline void * -imp__resize(void *pointer, SizeU size){ - return exp__resize(thread_ctx.implicit_allocator, pointer, size); -} -force_inline void -imp__free(void *pointer){ - exp__free(thread_ctx.implicit_allocator, pointer); -} -force_inline void -imp__free_all(){ - exp__free_all(thread_ctx.implicit_allocator); -} -force_inline void -imp__destroy(){ - exp__destroy(thread_ctx.implicit_allocator); -} - -force_inline Allocator * -imp_get(){ - assert(thread_ctx.implicit_allocator); - return thread_ctx.implicit_allocator; -} - -function void -thread_ctx_init(){ - arena_init(thread_ctx.scratch, "Scratch1"_s); - arena_init(thread_ctx.scratch+1, "Scratch2"_s); - arena_init(&pernament_arena, "Pernament Arena"_s); - os_process_heap.proc = os_heap_allocator_proc; - os_process_heap.handle = GetProcessHeap(); - os_process_heap.debug_name = "Win32 Process Heap"_s; - - thread_ctx.implicit_allocator = &os_process_heap; -} - -function String -string_copy(Allocator *a, String string){ - U8 *copy = exp_alloc_array(a, U8, string.len+1); - memory_copy(copy, string.str, string.len); - copy[string.len] = 0; - return (String){copy, string.len}; -} - -#include -function String -string_fmtv(Allocator *a, const char *str, va_list args1) { - va_list args2; - va_copy(args2, args1); - S64 len = vsnprintf(0, 0, str, args2); - va_end(args2); - - char *result = exp_alloc_array(a, char, len + 1); - vsnprintf(result, len + 1, str, args1); - - String res = {(U8 *)result, len}; - return res; -} - -#define STRING_FMT(alloc, str, result) \ -va_list args1; \ -va_start(args1, str); \ -String result = string_fmtv(alloc, str, args1); \ -va_end(args1) - -function String -string_fmt(Allocator *a, const char *str, ...) { - STRING_FMT(a, str, result); - return result; -} - -#define log(...) handle_log_message(Log_Kind_Normal, __LINE__, __FILE__, ## __VA_ARGS__) -#define log_error(...) handle_log_message(Log_Kind_Error, __LINE__, __FILE__, ## __VA_ARGS__) -function void -handle_log_message(Log_Kind kind, int line, const char *file, const char *str, ...){ - Set_Backup_Scratch(); - STRING_FMT(imp_get(), str, message); - if(thread_ctx.log_proc) thread_ctx.log_proc(kind, message, (char *)file, line); - else{ - printf("%s", message.str); - } -} - -function void -test_heap_allocator(){ - OS_Heap heap = win32_os_heap_create(false, mib(1), 0); - heap.debug_name = "Test heap"_s; - Set_Allocator(&heap); - assert(thread_ctx.implicit_allocator == &heap); - - U8 *result = imp_alloc_array(U8,1024); - result[1023] = 1; - result = exp_alloc_type(&heap, U8); - *result = 0; - imp_destroy(); - - assert(thread_ctx.implicit_allocator == &heap); - { - Set_Scratch(); - assert(thread_ctx.implicit_allocator != &heap); - assert(thread_ctx.implicit_allocator == thread_ctx.scratch); - } - assert(thread_ctx.implicit_allocator == &heap); -} - -const U64 Test_Context_ID = 14242; -struct Test_Context{ - int value; -}; - -function void -test_custom_context_2(){ - Get_Ctx(Test_Context); - ctx->value += 10; -} - -function void -test_custom_context_1(){ - Test_Context context = {}; - Set_Ctx(&context, Test_Context_ID); - Get_Ctx(Test_Context); - ctx->value = 10; - test_custom_context_2(); - test_custom_context_2(); - test_custom_context_2(); - assert(ctx->value == 40); - assert(thread_ctx.ctx == &context); -} - -function void -test_custom_context(){ - assert(thread_ctx.ctx == 0); - test_custom_context_1(); - assert(thread_ctx.ctx == 0); -} - -//----------------------------------------------------------------------------- -// Defer -// http://www.gingerbill.org/article/2015/08/19/defer-in-cpp/ -//----------------------------------------------------------------------------- -template -struct Defer_Scope { - F f; - Defer_Scope(F f) : f(f) {} - ~Defer_Scope() { f(); } -}; - -template -Defer_Scope defer_func(F f) { - return Defer_Scope(f); -} -#define DEFER_1(x, y) x##y -#define DEFER_2(x, y) DEFER_1(x, y) -#define DEFER_3(x) DEFER_2(x, __COUNTER__) -#define defer(code) auto DEFER_3(_defer_) = defer_func([&](){code;}) - -//----------------------------------------------------------------------------- -// Array -//----------------------------------------------------------------------------- -template -struct Array{ - T *data; - S64 cap; - S64 len; - Allocator *allocator; - - void init(S64 size){ - if(!allocator) allocator = imp_get(); - data = exp_alloc_array(allocator, T, size); - cap = size; - } - - void grow(S64 required_size){ - if(cap == 0){ - S64 cap = max(required_size*2, (S64)16); - init(cap); - } - else if(len + required_size > cap){ - U64 new_cap = max(cap * 2, len+required_size+1); - data = exp_resize_array(allocator, data, T, new_cap); - cap = new_cap; - } - } - - void add(T item){ - grow(1); - data[len++] = item; - } - - void clear(){ - len = 0; - } - - Array copy(Allocator *a){ - Array result = {}; - result.len = len; - result.cap = len; - result.allocator = 0; - result.data = exp_alloc_array(a, T, len); - memory_copy(result.data, data, sizeof(T)*len); - return result; - } - - T *begin(){ return data; } - T *end (){ return data + len; } - T &operator[](S64 i){ return data[i]; } -}; -#define For(array,it,i) for(SizeU i = 0; i < array.len; i++) for(auto *it = &array[i]; it; it = 0) -#define IFor(array) for(auto *it = array.begin(); it != array.end(); it++) -#define IterList(list,it) for(auto *it = list->first; it; it=it->next) - -template -Array array_make(S64 size){ - Array result = {}; - result.init(size); - return result; -} - -function void -test_array(){ - Set_Scratch(); - Array array = {}; - int size = 1000; - for(int i = 0; i < size; i++){ - array.add(i); - } - For(array, it, i){ - assert(*it == i); - } - - Arena arena = arena_make_personal("Test personal arena"_s); - Array array2 = {}; - array2.allocator = &arena; - for(int i = 0; i < size; i++){ - array2.add(i); - } - For(array2, iterator, count){ - assert(*iterator == count); - } - exp_destroy(&arena); - assert(arena.memory.data == 0); - assert(thread_ctx.scratch->memory.data != 0); - assert(thread_ctx.scratch == thread_ctx.implicit_allocator); -} - -//----------------------------------------------------------------------------- -// Map -//----------------------------------------------------------------------------- -struct Map_Key_Val{ - U64 key; - void *value; -}; - -struct Map{ - Map_Key_Val *data; - S64 len; - S64 cap; - Allocator *allocator; -}; -function void map_insert_u64(Map *map, U64 key, void *val); - -function void -map_grow(Map *map, S64 new_size){ - new_size = max((S64)16, new_size); - assert(new_size > map->cap); - assert(is_pow2(new_size)); - if(!map->allocator) map->allocator = imp_get(); - - Map new_map = {}; - new_map.data = exp_alloc_array(map->allocator, Map_Key_Val, new_size, AF_ZeroMemory), - new_map.cap = new_size, - new_map.allocator = map->allocator; - - for(S64 i = 0; i < map->cap; i++){ - if(map->data[i].key){ - map_insert_u64(&new_map, map->data[i].key, map->data[i].value); - } - } - if(map->data) exp_free(map->allocator, map->data); - *map = new_map; -} - -function Map -map_make(S64 size){ - Map result = {}; - map_grow(&result, size); - return result; -} - -function void -map_insert_u64(Map *map, U64 key, void *val){ - assert(val); - if(key == 0) key++; - if((2*map->len) + 1 > map->cap){ - map_grow(map, 2*map->cap); - } - U64 hash = hash_u64(key); - U64 index = wrap_around_pow2(hash, map->cap); - U64 i = index; - for(;;){ - if(map->data[i].key == 0){ - map->len++; - map->data[i].key = key; - map->data[i].value = val; - return; - } - else if(map->data[i].key == key){ - map->data[i].value = val; - return; - } - - i = wrap_around_pow2(i+1, map->cap); - if(i == map->cap){ - return; - } - } -} - -function void * -map_get_u64(Map *map, U64 key){ - if(map->len == 0) return 0; - if(key == 0) key++; - U64 hash = hash_u64(key); - U64 index = wrap_around_pow2(hash, map->cap); - U64 i = index; - for(;;){ - if(map->data[i].key == key){ - return map->data[i].value; - } - else if(map->data[i].key == 0){ - return 0; - } - - i = wrap_around_pow2(i+1, map->cap); - if(i == map->cap){ - return 0; - } - } -} - -function void * -map_get(Map *map, void *pointer){ - return map_get_u64(map, (U64)pointer); -} - -function void -map_insert(Map *map, void *key, void *value){ - map_insert_u64(map, (U64)key, value); -} - -function void -map_test(){ - Set_Scratch(); - Map map = {0}; - const SizeU size = 1025; - for(SizeU i = 1; i < size; i++){ - map_insert_u64(&map, i, (void *)i); - } - for(SizeU i = 1; i < size; i++){ - SizeU val = (SizeU)map_get_u64(&map, i); - assert(val == i); - } -} - -//----------------------------------------------------------------------------- -// Linked lists -//----------------------------------------------------------------------------- -#define SLLQueuePushMod(f,l,n,next) do{\ -if((f)==0){\ -(f)=(l)=(n);\ -}\ -else{\ -(l)=(l)->next=(n);\ -} \ -}while(0) -#define SLLQueuePush(f,l,n) SLLQueuePushMod(f,l,n,next) - -#define SLLStackPush(l,n) do{\ -(n)->next = (l);\ -(l) = (n);\ -}while(0) - -#define SLLStackPop(l,n) do{\ -if(l){\ -(n) = (l);\ -(l) = (l)->next;\ -(n)->next = 0;\ -}\ -}while(0) - -//----------------------------------------------------------------------------- -// String builder -//----------------------------------------------------------------------------- -struct String_Builder_Block{ - String_Builder_Block *next; - S64 cap; - S64 len; - U8 data[0]; -}; - -struct String_Builder{ - String_Builder_Block *first; - String_Builder_Block *last; - Allocator *allocator; -}; - -function void -string_builder_push_block(String_Builder *b, SizeU size){ - String_Builder_Block *block = (String_Builder_Block *)imp_alloc(sizeof(String_Builder_Block) + size); - memory_zero(block, sizeof(String_Builder_Block)+1); // Also clear first byte of character data - block->cap = size; - SLLQueuePush(b->first, b->last, block); -} - -function void -string_builder_init(String_Builder *b, SizeU size = 4096){ - if(!b->allocator) b->allocator = imp_get(); - string_builder_push_block(b, size); -} - -function void -appendf(String_Builder *b, const char *str, ...){ - if(b->first == 0){ - string_builder_init(b); - } - va_list args, args2; - va_start(args, str); defer(va_end(args)); - retry:{ - String_Builder_Block *block = b->last; - int block_size = block->cap - block->len; - char *write_address = (char *)block->data + block->len; - - va_copy(args2, args); defer(va_end(args2)); - int written = vsnprintf(write_address, block_size, str, args2); - - if(written > block_size){ - int new_block_size = max(4096, (written+1)*2); - string_builder_push_block(b, new_block_size); - goto retry; - } - block->len += written; - } -} - -function String -string_flatten(String_Builder *b){ - // @Note(Krzosa): Only single block, no need to flatten, vsnprintf null terminates too - if(b->first == b->last){ - String result = {b->first->data, b->first->len}; - return result; - } - - // @Note(Krzosa): Compute size to allocate - S64 size = 1; - IterList(b, it){ - size += it->len; - } - String result = {}; - result.str = (U8 *)exp_alloc(b->allocator, size); - - // @Note(Krzosa): Copy the content of each block into the string - IterList(b, it){ - memory_copy(result.str + result.len, it->data, it->len); - result.len += it->len; - } - result.str[result.len] = 0; - return result; -} - -function void -test_string_builder(){ - Set_Scratch(); - String_Builder sb = {}; - string_builder_init(&sb, 4); - appendf(&sb, "Thing, %d", 242252); - String f = string_flatten(&sb); - assert(string_compare(f, "Thing, 242252"_s)); - appendf(&sb, "-%f %f %f", 23.0, 42.29, 2925.2); - f = string_flatten(&sb); -} - -//----------------------------------------------------------------------------- -// String intern -//----------------------------------------------------------------------------- -struct Intern_Table{ - Allocator *string_allocator; - Map map; - U8 *first_keyword; - U8 *last_keyword; -}; - -function Intern_Table -intern_table_make(S64 initial_size){ - Intern_Table result = {}; - result.map = map_make(initial_size); - result.string_allocator = imp_get(); - return result; -} - -function Intern_String -intern_string(Intern_Table *t, String string){ - if(!t->string_allocator) t->string_allocator = imp_get(); - U64 hash = hash_string(string); - U8 *slot = (U8 *)map_get_u64(&t->map, hash); - if(slot){ - Intern_String result = {{slot, *(slot-sizeof(S64))}}; - return result; - } - - S64 *len_address = (S64 *)exp_alloc(t->string_allocator, string.len+1+sizeof(S64)); - *len_address = string.len; - - U8 *string_address = (U8 *)(len_address + 1); - memory_copy(string_address, string.str, string.len); - string_address[string.len] = 0; - - map_insert_u64(&t->map, hash, string_address); - Intern_String result = {{string_address, *len_address}}; - - return result; -} - -function void -test_intern_table(){ Set_Scratch(); - Intern_Table table = {}; - Intern_String intern1 = intern_string(&table, "Thing"_s); - Intern_String intern2 = intern_string(&table, "Thing"_s); - Intern_String intern3 = intern_string(&table, "Not Thing"_s); - assert(intern1.str == intern2.str); - assert(intern3.str != intern2.str); -} - +#include "base.cpp" +#include "base_unicode.cpp" #include "new_lex.cpp" #include "new_ast.cpp" #include "new_parse.cpp" #include "new_type.cpp" +#include "new_resolve.cpp" + int main(){ - test_custom_context(); - test_heap_allocator(); test_os_memory(); - thread_ctx_init(); + + test_unicode(); + test_resolve(); test_types(); - test_parse_decl(); test_parse_expr(); map_test(); @@ -1136,3 +21,5 @@ int main(){ test_intern_table(); lex_test(); } + + diff --git a/new_ast.cpp b/new_ast.cpp index d0c6733..90e3a05 100644 --- a/new_ast.cpp +++ b/new_ast.cpp @@ -5,20 +5,29 @@ // Parser::ast_arena - arena for asts // Lexer::interns::string_allocator - arena for interns // -Intern_String keyword_const; Intern_String keyword_struct; Intern_String keyword_union; +Intern_String keyword_cast; Intern_String keyword_enum; +Intern_String intern_void; +Intern_String intern_int; +Intern_String intern_str; +Intern_String intern_unsigned; + const U64 Parse_Ctx_ID = 115151; struct Parse_Ctx:Lexer{ - Arena ast_arena; + Allocator *perm; // Stores: AST, tokens, interns + Allocator *heap; + + Map global_syms; + Map type_map; + Token empty_token; S64 indent; S64 pt[256]; // precedence table - Map type_map; - void init(){ + void init(Allocator *perm_allocator, Allocator *heap_allocator){ const S64 addp = 1; const S64 mulp = 2; pt[TK_Add] = addp; @@ -26,17 +35,28 @@ struct Parse_Ctx:Lexer{ pt[TK_Div] = mulp; pt[TK_Mul] = mulp; - arena_init(&ast_arena, "AST Arena"_s); - lex_init(this); - keyword_const = intern_string(&interns, "const"_s); + perm = perm_allocator; + heap = heap_allocator; + + global_syms = {heap}; + type_map = {heap}; + + lex_init(perm, heap, this); keyword_struct= intern_string(&interns, "struct"_s); keyword_union = intern_string(&interns, "union"_s); + keyword_cast = intern_string(&interns, "cast"_s); keyword_enum = intern_string(&interns, "enum"_s); - interns.first_keyword = keyword_const.str; + interns.first_keyword = keyword_struct.str; interns.last_keyword = keyword_enum.str; + + intern_void = intern_string(&interns, "void"_s); + intern_int = intern_string(&interns, "int"_s); + intern_str = intern_string(&interns, "String"_s); + intern_unsigned = intern_string(&interns, "unsigned"_s); } }; +thread_local Parse_Ctx *pctx; //----------------------------------------------------------------------------- // AST //----------------------------------------------------------------------------- @@ -47,8 +67,11 @@ enum Ast_Kind{ AK_Expr_Str, AK_Expr_Int, + AK_Expr_Cast, AK_Expr_Ident, AK_Expr_Binary, + AK_Expr_CompoundItem, + AK_Expr_Compound, AK_Decl_Func, AK_Decl_Func_Arg, @@ -66,6 +89,7 @@ struct Ast{ Token *pos; }; +struct Ast_Typespec; struct Ast_Expr:Ast{ union{ Intern_String intern_val; @@ -75,6 +99,19 @@ struct Ast_Expr:Ast{ Ast_Expr *left; Ast_Expr *right; } binary; + struct{ + Ast_Typespec *typespec; + Array exprs; + }compound; + struct{ + Ast_Expr *name; // index | name + Ast_Expr *index; + Ast_Expr *item; + }compound_item; + struct{ + Ast_Expr *expr; + Ast_Typespec *typespec; + }cast; }; }; @@ -98,7 +135,6 @@ struct Ast_Decl:Ast{ union{ struct{ Ast_Typespec *typespec; - Intern_String name; Ast_Expr *expr; }var; struct{ @@ -117,8 +153,7 @@ struct Ast_Package:Ast{ // AST Constructors beginning with expressions //----------------------------------------------------------------------------- #define AST_NEW(T,ikind,ipos) \ -Get_Ctx(Parse_Ctx); \ -Ast_##T *result = exp_alloc_type(&ctx->ast_arena, Ast_##T); \ +Ast_##T *result = exp_alloc_type(pctx->perm, Ast_##T); \ result->kind = ikind; \ result->pos = ipos @@ -152,6 +187,31 @@ ast_expr_binary(Ast_Expr *left, Ast_Expr *right, Token *op){ return result; } +function Ast_Expr * +ast_expr_compound(Token *pos, Ast_Typespec *typespec, Array exprs){ + AST_NEW(Expr, AK_Expr_Compound, pos); + result->compound.typespec = typespec; + result->compound.exprs = exprs.tight_copy(pctx->perm); + return result; +} + +function Ast_Expr * +ast_expr_compound_item(Token *pos, Ast_Expr *index, Ast_Expr *name, Ast_Expr *item){ + AST_NEW(Expr, AK_Expr_CompoundItem, pos); + result->compound_item.name = name; + result->compound_item.index = index; + result->compound_item.item = item; + return result; +} + +function Ast_Expr * +ast_expr_cast(Token *pos, Ast_Expr *expr, Ast_Typespec *typespec){ + AST_NEW(Expr, AK_Expr_Cast, pos); + result->cast.expr = expr; + result->cast.typespec = typespec; + return result; +} + //----------------------------------------------------------------------------- // Typespecs //----------------------------------------------------------------------------- @@ -177,6 +237,14 @@ ast_typespec_array(Token *pos, Ast_Typespec *base, Ast_Expr *expr){ return result; } +function Ast_Typespec * +ast_typespec_func(Token *pos, Ast_Typespec *ret, Array args){ + AST_NEW(Typespec, AK_Typespec_Func, pos); + result->func.ret = ret; + result->func.args = args.tight_copy(pctx->perm); + return result; +} + //----------------------------------------------------------------------------- // Declarations //----------------------------------------------------------------------------- @@ -196,9 +264,17 @@ ast_decl_var(Token *pos, Ast_Typespec *typespec, Intern_String name, Ast_Expr *e return result; } +function Ast_Decl * +ast_decl_const(Token *pos, Intern_String name, Ast_Expr *expr){ + Ast_Decl *result = ast_decl_var(pos, 0, name, expr); + result->kind = AK_Decl_Const; + return result; +} + function Ast_Package * -ast_package(Token *pos, String name){ +ast_package(Token *pos, String name, Array decls){ AST_NEW(Package, AK_Package, pos); - result->name = intern_string(&ctx->interns, name); + result->decls = decls.tight_copy(pctx->perm); + result->name = intern_string(&pctx->interns, name); return result; } \ No newline at end of file diff --git a/new_lex.cpp b/new_lex.cpp index 21cef24..8a90cef 100644 --- a/new_lex.cpp +++ b/new_lex.cpp @@ -51,7 +51,6 @@ enum Token_Kind{ TK_Colon, TK_Assign, - TK_ColonAssign, TK_DivAssign, TK_MulAssign, TK_ModAssign, @@ -112,6 +111,7 @@ struct Lex_Stream{ U8 *line_begin; String file; S32 line; + S32 inside_brace_paren; }; struct Lexer{ @@ -169,7 +169,7 @@ lex_set_len(Lex_Stream *s, Token *token){ function void lex_set_keywords(Lexer *lexer, Array keywords){ Intern_String keyword = {}; - IFor(keywords){ + Iter(keywords){ keyword = intern_string(&lexer->interns, *it); if(it == keywords.begin()) lexer->interns.first_keyword = keyword.str; @@ -272,14 +272,14 @@ lex__stream(Intern_Table *table, Array *array, Lex_Stream *s){ lex_advance(s); switch(*t.str){ - case 0: break; + case 0 : break; case '@': t.kind = TK_At; break; - case '(': t.kind = TK_OpenParen; break; - case ')': t.kind = TK_CloseParen; break; - case '{': t.kind = TK_OpenBrace; break; - case '}': t.kind = TK_CloseBrace; break; - case '[': t.kind = TK_OpenBracket; break; - case ']': t.kind = TK_CloseBracket; break; + case '(': s->inside_brace_paren++; t.kind = TK_OpenParen; break; + case ')': s->inside_brace_paren--; t.kind = TK_CloseParen; break; + case '{': s->inside_brace_paren++; t.kind = TK_OpenBrace; break; + case '}': s->inside_brace_paren--; t.kind = TK_CloseBrace; break; + case '[': s->inside_brace_paren++; t.kind = TK_OpenBracket; break; + case ']': s->inside_brace_paren--; t.kind = TK_CloseBracket; break; case ',': t.kind = TK_Comma; break; case '~': t.kind = TK_Neg; break; case '?': t.kind = TK_Question; break; @@ -371,10 +371,6 @@ lex__stream(Intern_Table *table, Array *array, Lex_Stream *s){ lex_advance(s); t.kind = TK_DoubleColon; } - else if(lexc(s) == '='){ - lex_advance(s); - t.kind = TK_ColonAssign; - } else { t.kind = TK_Colon; } @@ -483,7 +479,10 @@ lex__stream(Intern_Table *table, Array *array, Lex_Stream *s){ if(t.len==0) lex_set_len(s,&t); - array->add(t); + B32 skip = t.kind == TK_NewLine && s->inside_brace_paren > 0; + if(!skip){ + array->add(t); + } while(lex_is_whitespace(lexc(s))) lex_advance(s); @@ -494,15 +493,15 @@ lex__stream(Intern_Table *table, Array *array, Lex_Stream *s){ } function void -lex_init(Lexer *l){ - l->tokens = array_make(1024*2); - l->interns= intern_table_make(1024); +lex_init(Allocator *token_string_arena, Allocator *map_allocator, Lexer *l){ + l->tokens = array_make(token_string_arena, 1024*2); + l->interns= intern_table_make(token_string_arena, map_allocator, 1024); } function Lexer -lex_make(){ +lex_make(Allocator *token_string_arena, Allocator *map_allocator){ Lexer result = {}; - lex_init(&result); + lex_init(token_string_arena, map_allocator, &result); return result; } @@ -515,29 +514,30 @@ lex_restream(Lexer *lexer, String istream, String file){ } function Lexer -lex_stream(String istream, String file){ - Lexer result = lex_make(); +lex_stream(Allocator *token_string_arena, Allocator *map_allocator, String istream, String file){ + Lexer result = lex_make(token_string_arena, map_allocator); lex_restream(&result, istream, file); return result; } function void -lex_test(){ Set_Scratch(); +lex_test(){ + Scratch scratch; String test = "Keyword //R\n 18446744073709551616{})(@?&+-;....->,:::/**/\"Thing\" Thingy" "\"Test_Meme\"+=-===42524 4294967295 18446744073709551615" "for if while switch :="_s; - Array keywords = {}; + Array keywords = {scratch}; keywords.add("Keyword"_s); keywords.add("for"_s); keywords.add("if"_s); keywords.add("while"_s); keywords.add("switch"_s); - Lexer lexer = lex_make(); + Lexer lexer = lex_make(scratch, scratch); lex_set_keywords(&lexer, keywords); lex_restream(&lexer, test, "Test1"_s); - Array array = lexer.tokens; + Array arr = lexer.tokens; Token_Kind kind[] = { TK_Keyword, TK_NewLine, TK_Error,TK_OpenBrace,TK_CloseBrace,TK_CloseParen,TK_OpenParen, @@ -546,7 +546,7 @@ lex_test(){ Set_Scratch(); TK_StringLit, TK_Identifier, TK_StringLit, TK_AddAssign, TK_SubAssign, TK_Equals, TK_Integer, TK_Integer, TK_Integer, TK_Keyword, TK_Keyword, TK_Keyword, TK_Keyword, - TK_ColonAssign, TK_End + TK_Colon, TK_Assign, TK_End }; String strs[] = { "Keyword"_s, "\n "_s, "18446744073709551616"_s,"{"_s,"}"_s,")"_s,"("_s, @@ -554,18 +554,18 @@ lex_test(){ Set_Scratch(); "..."_s,"."_s,"->"_s,","_s,"::"_s,":"_s, "Thing"_s,"Thingy"_s,"Test_Meme"_s, "+="_s,"-="_s, "=="_s,"42524"_s,"4294967295"_s,"18446744073709551615"_s, - "for"_s, "if"_s, "while"_s, "switch"_s, ":="_s, ""_s, + "for"_s, "if"_s, "while"_s, "switch"_s, ":"_s, "="_s, ""_s, }; U64 vals[] = { 42524, 4294967295, 18446744073709551615llu }; int ui = 0; - For(array, t, i){ - assert(t->kind == kind[i]); - assert(string_compare(t->string, strs[i])); - if(t->kind == TK_Integer){ - assert(t->int_val == vals[ui++]); + For(arr){ + assert(it->kind == kind[i]); + assert(string_compare(it->string, strs[i])); + if(it->kind == TK_Integer){ + assert(it->int_val == vals[ui++]); } } } @@ -603,7 +603,6 @@ token_kind_string(Token_Kind kind){ case TK_GreaterThen: return ">"_s; case TK_Colon: return ":"_s; case TK_Assign: return "="_s; - case TK_ColonAssign: return ":="_s; case TK_DivAssign: return "/="_s; case TK_MulAssign: return "*="_s; case TK_ModAssign: return "%="_s; @@ -629,6 +628,7 @@ token_kind_string(Token_Kind kind){ case TK_LeftShift: return "<<"_s; case TK_RightShift: return ">>"_s; case TK_Arrow: return "->"_s; + case TK_NewLine: return "NewLine"_s; case TK_ExprSizeof: return "sizeof"_s; case TK_DocComment: return "DocComment"_s; case TK_Comment: return "Comment"_s; diff --git a/new_parse.cpp b/new_parse.cpp index 0bc164a..63813a1 100644 --- a/new_parse.cpp +++ b/new_parse.cpp @@ -1,22 +1,22 @@ function Token * -token_get(S64 i = 0){ Get_Ctx(Parse_Ctx); - i += ctx->token_iter; - if(i >= ctx->tokens.len){ - return &ctx->empty_token; +token_get(S64 i = 0){ + i += pctx->token_iter; + if(i >= pctx->tokens.len){ + return &pctx->empty_token; } - Token *result = &ctx->tokens[i]; + Token *result = &pctx->tokens[i]; if(result->kind == TK_NewLine){ - ctx->indent = result->indent; + pctx->indent = result->indent; } return result; } function Token * -token_next(){ Get_Ctx(Parse_Ctx); +token_next(){ Token *token = token_get(); - ctx->token_iter++; + pctx->token_iter++; return token; } @@ -52,8 +52,8 @@ token_match_keyword(Intern_String string){ function void parsing_error(Token *token, const char *str, ...){ - Set_Scratch(); - STRING_FMT(imp_get(), str, string); + Scratch scratch; + STRING_FMT(scratch, str, string); // @Note(Krzosa): Print nice error message printf("\nError: %s", string.str); @@ -122,6 +122,41 @@ Compound literals - { } */ function Ast_Expr *parse_expr(S64 rbp = 0); +function Ast_Typespec *parse_typespec(); + +function Ast_Expr * +parse_expr_compound(){ + Scratch scratch; + Token *pos = token_get(); + Array exprs = {scratch}; + while(!token_is(TK_CloseBrace)){ + Token *token = token_get(); + Ast_Expr *index = 0; + Ast_Expr *name = 0; + if(token_match(TK_OpenBracket)){ + index = parse_expr(); + token_expect(TK_CloseBracket); + token_expect(TK_Assign); + } + else if(token_is(TK_Identifier)){ + token = token_next(); + name = ast_expr_identifier(token, token->intern_val); + token_expect(TK_Assign); + } + + Ast_Expr *item = parse_expr(); + Ast_Expr *item_comp = ast_expr_compound_item(token, index, name, item); + exprs.add(item_comp); + + if(!token_match(TK_Comma)){ + break; + } + } + token_expect(TK_CloseBrace); + + Ast_Expr *result = ast_expr_compound(pos, 0, exprs); + return result; +} function Ast_Expr * parse_expr_nud(Token *token){ @@ -129,14 +164,29 @@ parse_expr_nud(Token *token){ case TK_StringLit: return ast_expr_string(token, token->intern_val); case TK_Identifier: return ast_expr_identifier(token, token->intern_val); case TK_Integer: return ast_expr_integer(token, token->int_val); + case TK_Keyword: { + if(token->intern_val == keyword_cast){ + token_expect(TK_OpenParen); + Ast_Expr *expr = parse_expr(); + token_expect(TK_Colon); + Ast_Typespec *typespec = parse_typespec(); + token_expect(TK_CloseParen); + return ast_expr_cast(token, expr, typespec); + } + else { + parsing_error(token, "Unexpected keyword: [%s], expected keyword [cast]", token->intern_val.str); + return 0; + } + }break; + case TK_OpenBrace: return parse_expr_compound(); case TK_OpenParen: {Ast_Expr *result = parse_expr(); token_expect(TK_CloseParen); return result;} default: parsing_error(token, "Unexpected token of kind: [%s] in expression", token_kind_string(token->kind).str); return 0; } } function S64 -op_precedence(Token_Kind kind){ Get_Ctx(Parse_Ctx); - S64 result = ctx->pt[kind]; +op_precedence(Token_Kind kind){ + S64 result = pctx->pt[kind]; return result; } @@ -182,23 +232,23 @@ expr_eval(Ast_Expr *expr){ return 0; } -#define TEST_PARSER() \ -Set_Scratch(); \ -Parse_Ctx ctx = {}; \ -ctx.init(); \ -Set_Ctx(&ctx, Parse_Ctx_ID) +#define TEST_PARSER() \ +Scratch scratch(thread_ctx.scratch);\ +Parse_Ctx ctx = {}; \ +ctx.init(scratch, scratch); \ +pctx = &ctx function void test_parse_expr(){ TEST_PARSER(); struct Test{String str;S64 val;}; - Array exprs = {}; + Array exprs = {scratch}; exprs.add({"(4+5)*2"_s, (4+5)*2}); exprs.add({"4+5*2"_s, 4+5*2}); exprs.add({"4*5+5"_s, 4*5+5}); exprs.add({"4+5+5+3"_s, 4+5+5+3}); - For(exprs,it,i){ + Iter(exprs){ lex_restream(&ctx, it->str, "test_expr"_s); Ast_Expr *result = parse_expr(); S64 val = expr_eval(result); @@ -209,38 +259,115 @@ test_parse_expr(){ //----------------------------------------------------------------------------- // Parsing declarations //----------------------------------------------------------------------------- -function Ast_Decl * -parse_decl(){Get_Ctx(Parse_Ctx); - Ast_Decl *result = 0; - Token *token = token_match(TK_Identifier); - if(token){ - if(token_match(TK_ColonAssign)){ - if(ctx->indent != 0) - parsing_error(token, "Top level declarations shouldn't be indented"); - - Ast_Expr *expr = parse_expr(); - result = ast_decl_var(token, 0, token->intern_val, expr); +function Ast_Typespec * +parse_optional_type(){ + Ast_Typespec *result = 0; + if(token_match(TK_Colon)) result = parse_typespec(); + return result; +} + +function Ast_Typespec * +parse_typespec_function(Token *token){ + Scratch scratch; + Array args = {scratch}; + if(!token_is(TK_CloseParen)) { + for(;;) { + args.add(parse_typespec()); + if(!token_match(TK_Comma)){ + break; + } } - else parsing_error(token, "Encountered unexpected token while parsing a top level declarations"); + } + token_expect(TK_CloseParen); + Ast_Typespec *ret = parse_optional_type(); + Ast_Typespec *result = ast_typespec_func(token, ret, args); + return result; +} + +// [10]*int - Array of 10 pointers to ints +function Ast_Typespec * +parse_typespec_recurse(){ + Token *token = token_get(); + if(token_match(TK_Mul)){ + Ast_Typespec *result = parse_typespec_recurse(); + result = ast_typespec_pointer(token, result); + return result; + } + else if(token_match(TK_OpenBracket)){ + Ast_Expr *expr = parse_expr(); + token_expect(TK_CloseBracket); + Ast_Typespec *result = parse_typespec_recurse(); + result = ast_typespec_array(token, result, expr); + return result; + } + else if(token_match(TK_OpenParen)){ + Ast_Typespec *result = parse_typespec_function(token); + return result; + } + else if(token_match(TK_Identifier)){ + Ast_Typespec *result = ast_typespec_name(token, token->intern_val); + return result; + } + else{ + parsing_error(token, "Failed to parse type, unexpected token of kind", token_kind_string(token->kind).str); + return 0; + } +} + +function Ast_Typespec * +parse_typespec(){ + Ast_Typespec *result = parse_typespec_recurse(); + return result; +} + +function Ast_Expr * +parse_assign_expr(){ + Ast_Expr *result = 0; + if(token_match(TK_Assign)) result = parse_expr(); + return result; +} + +function Ast_Decl * +parse_decl(){ + Ast_Decl *result = 0; + if(token_is(TK_Identifier)){ + if(pctx->indent != 0) parsing_error(token_get(), "Top level declarations shouldn't be indented"); + Token *name = token_next(); + if(token_match(TK_DoubleColon)){ // Constant + Ast_Expr *expr = parse_expr(); + result = ast_decl_const(name, name->intern_val, expr); + } + else if(token_match(TK_Colon)){ + Ast_Typespec *typespec = 0; + Ast_Expr *expr = 0; + if(!token_is(TK_Assign)) typespec = parse_typespec(); + if(token_match(TK_Assign)) expr = parse_expr(); + if(!expr && !typespec) parsing_error(name, "invalid declaration, no type or value"); + + result = ast_decl_var(name, typespec, name->intern_val, expr); + } + else{ + Token *token = token_get(); + parsing_error(token, "Unexpected token: [%s] when parsing a declaration", token_kind_string(token->kind).str); + } + } return result; } function Ast_Package * parse_file(){ - Ast_Package *result = ast_package(token_get(), token_get()->file); + Scratch scratch; + Token *token = token_get(); + Arraydecls = {scratch}; while(!token_is(TK_End)){ while(token_match(TK_NewLine)); Ast_Decl *decl = parse_decl(); if(!decl) break; - result->decls.add(decl); + decls.add(decl); } + Ast_Package *result = ast_package(token, token->file, decls); return result; } -function void -test_parse_decl(){ - TEST_PARSER(); - lex_restream(&ctx, "thing := 24252\nanother_thing := \"string\"\n\nref := thing"_s, "test_parse_decl"_s); - Ast_Package *result = parse_file(); -} + diff --git a/new_resolve.cpp b/new_resolve.cpp new file mode 100644 index 0000000..64a85e4 --- /dev/null +++ b/new_resolve.cpp @@ -0,0 +1,309 @@ +#define Ast_Begin(kind,type) case kind: { type *node = (type *)ast; +#define Ast_End() } break + +enum Sym_Kind{ + SYM_None, + SYM_Type, + SYM_Const, + SYM_Var, +}; + +struct Sym{ + Intern_String name; + Sym_Kind kind; + Ast_Decl *decl; + Type *type; + union{ + S64 int_val; + Intern_String intern_val; + }; +}; + +struct Operand{ + Type *type; + bool is_const; + union { + S64 int_val; + Intern_String intern_val; + }; +}; + +function void +sym_insert(Sym *sym){ + U64 hash = hash_string(sym->name.s); + Sym *is_sym = (Sym *)map_get_u64(&pctx->global_syms, hash); + if(is_sym){ + parsing_error(sym->decl->pos, "Symbol with name: [%s] defined multiple times", sym->name.s.str); + } + + map_insert_u64(&pctx->global_syms, hash, sym); +} + +function Sym * +sym_get(Intern_String name){ + Sym *result = (Sym *)map_get_u64(&pctx->global_syms, hash_string(name.s)); + return result; +} + +function Sym * +sym_new(Sym_Kind kind, Intern_String name, Type *type, Ast_Decl *decl){ + Sym *result = exp_alloc_type(pctx->perm, Sym); + result->name = name; + result->kind = kind; + result->type = type; + result->decl = decl; + return result; +} + +global Ast_Decl empty_decl = {}; +function void +sym_insert_builtin_type(String name, Type *type){ + Intern_String string = intern_string(&pctx->interns, name); + Sym *sym = sym_new(SYM_Type, string, type, &empty_decl); + sym_insert(sym); +} + +function void +sym_insert_builtins(){ + sym_insert_builtin_type("void"_s, type_void); + sym_insert_builtin_type("bool"_s, type_bool); + sym_insert_builtin_type("int"_s, type_int); + sym_insert_builtin_type("String"_s, type_string); + + { + Intern_String string = intern_string(&pctx->interns, "true"_s); + Sym *sym = sym_new(SYM_Const, string, type_bool, &empty_decl); + sym_insert(sym); + } + + { + Intern_String string = intern_string(&pctx->interns, "false"_s); + Sym *sym = sym_new(SYM_Const, string, type_bool, &empty_decl); + sym_insert(sym); + } + + { + Intern_String string = intern_string(&pctx->interns, "null"_s); + Sym *sym = sym_new(SYM_Const, string, type_null, &empty_decl); + sym_insert(sym); + } + + +} + +function Operand eval_expr(Ast_Expr *ast, Type *compound_required_type = 0); +function Type * +eval_typespec(Ast_Typespec *ast){ + if(!ast) return 0; + + switch(ast->kind){ + Ast_Begin(AK_Typespec_Ident, Ast_Typespec){ + Sym *type_sym = sym_get(node->name); + if(!type_sym){ + parsing_error(node->pos, "This type is not defined"); + } + if(type_sym->kind != SYM_Type){ + parsing_error(node->pos, "This identifier is not a type"); + } + return type_sym->type; + Ast_End(); + } + + Ast_Begin(AK_Typespec_Pointer, Ast_Typespec){ + Type *type = eval_typespec(node->base); + Type *result = type_pointer(type); + return result; + Ast_End(); + } + + Ast_Begin(AK_Typespec_Array, Ast_Typespec){ + Type *type = eval_typespec(node->arr.base); + Operand expr = eval_expr(node->arr.expr); + if(!expr.is_const) parsing_error(node->pos, "Array size is not a constant"); + if(expr.type != type_int) parsing_error(node->pos, "Array size is expected to be of type int"); + Type *result = type_array(type, expr.int_val); + return result; + Ast_End(); + } + invalid_default_case; + } + return 0; +} + +function Type * +resolve_type_pair(Token *pos, Type *a, Type *b){ + Type *result = 0; + if(!a && b) result = b; + else if(a && !b) result = a; + else if(!a && !b) parsing_error(pos, "Trying to resolve a type pair where both types are null"); + else{ // a && b + if(b->kind == TYPE_Null) result = a; + else if(a->kind == TYPE_Null) result = b; + else if(a != b) parsing_error(pos, "Expression and type specification are differing"); + else result = a; // Types are the same + } + + return result; +} + +function Operand +eval_expr(Ast_Expr *ast, Type *exp_compound_type){ + switch(ast->kind){ + Ast_Begin(AK_Expr_Int, Ast_Expr){ + Operand result = {type_int, true, {.int_val=(S64)node->int_val}}; + return result; + Ast_End(); + } + + Ast_Begin(AK_Expr_Str, Ast_Expr){ + Operand result = {type_string, true, {.intern_val = node->intern_val}}; + return result; + Ast_End(); + } + + Ast_Begin(AK_Expr_Compound, Ast_Expr){ + Type *type = eval_typespec(node->compound.typespec); + Type *variable_type = exp_compound_type; + if(!type && variable_type) type = variable_type; + else if(!variable_type && type); + else if(variable_type != type) parsing_error(node->pos, "Variable type different from explicit compound type"); + + if(type->kind == TYPE_Array){ + if(node->compound.exprs.len > type->arr.size) parsing_error(node->pos, "compound statement has too many items for this type"); + Type *item_type = type->arr.base; + + For(node->compound.exprs){ + assert(it[0]->kind == AK_Expr_CompoundItem); + if(it[0]->compound_item.name) parsing_error(it[0]->pos, "Invalid array indexing in compound expression"); + if(it[0]->compound_item.index){ + Operand index_op = eval_expr(it[0]->compound_item.index); + if(!index_op.is_const) parsing_error(it[0]->pos, "Index in a compound expression is not a constant"); + if(index_op.type != type_int) parsing_error(it[0]->pos, "Index should be of type int"); + if(index_op.int_val > (type->arr.size - 1)) parsing_error(it[0]->pos, "Invalid index in compound expression, larger then type can store"); + } + Operand expr = eval_expr(it[0]->compound_item.item); + if(expr.type != item_type) parsing_error(it[0]->pos, "Invalid type of item in compound expression"); + } + } + else parsing_error(node->pos, "Invalid compound expression type"); + + Operand result = {type, false}; + return result; + Ast_End(); + } + + Ast_Begin(AK_Expr_Ident, Ast_Expr){ + Sym *sym = sym_get(node->intern_val); + if(!sym){ + parsing_error(node->pos, "Identifier is undefined"); + } + + Operand result = {sym->type, sym->kind == SYM_Const ? true : false, {.int_val = sym->int_val}}; + return result; + Ast_End(); + } + + Ast_Begin(AK_Expr_Cast, Ast_Expr){ + Operand expr = eval_expr(node->cast.expr); + Type *type = eval_typespec(node->cast.typespec); + + if(type == expr.type) return expr; + + else if(expr.type == type_int && type == type_bool){ + expr.type = type_bool; + return expr; + } + + else if(expr.type == type_bool && type == type_int){ + expr.type = type_int; + return expr; + } + + else if(expr.type == type_null){ + expr.type = type; + return expr; + } + + else parsing_error(node->pos, "Failed to cast, incompatible types"); + + Ast_End(); + } + + Ast_Begin(AK_Expr_Binary, Ast_Expr){ + Operand left = eval_expr(ast->binary.left); + Operand right = eval_expr(ast->binary.right); + Operand result = {}; + result.type = resolve_type_pair(node->pos, left.type, right.type); + if(left.is_const && right.is_const){ + result.is_const = true; + if(result.type == type_int){ + switch(node->binary.op){ + case TK_Add: result.int_val = left.int_val + right.int_val; break; + case TK_Sub: result.int_val = left.int_val - right.int_val; break; + case TK_Mul: result.int_val = left.int_val * right.int_val; break; + case TK_Div: result.int_val = left.int_val / right.int_val; break; + invalid_default_case; + } + } + else parsing_error(node->pos, "Arithmetic on type [TODO] is not supported"); + } + + return result; + Ast_End(); + } + + invalid_default_case; + } + + return {}; +} + +function void +eval_decl(Ast *ast){ + switch(ast->kind){ + + Ast_Begin(AK_Package, Ast_Package){ + For(node->decls) eval_decl(*it); + Ast_End(); + } + + Ast_Begin(AK_Decl_Var, Ast_Decl){ + Type *type = eval_typespec(node->var.typespec); + Operand expr = eval_expr(node->var.expr, type); + Type *resolved_type = resolve_type_pair(node->pos, type, expr.type); + + Sym *sym = sym_new(SYM_Var, node->name, resolved_type, node); + sym_insert(sym); + Ast_End(); + } + + Ast_Begin(AK_Decl_Const, Ast_Decl){ + Type *type = eval_typespec(node->var.typespec); + if(type && type->kind == TYPE_Pointer) parsing_error(node->pos, "Const cant be a pointer"); + Operand expr = eval_expr(node->var.expr); + if(!expr.type) parsing_error(node->pos, "Constant value without expression"); + if(!expr.is_const) parsing_error(node->pos, "Value of constant variable is not a constant expression"); + Type *resolved_type = resolve_type_pair(node->pos, type, expr.type); + + Sym *sym = sym_new(SYM_Const, node->name, resolved_type, node); + if(resolved_type == type_int) sym->int_val = expr.int_val; + else if(resolved_type == type_string) sym->intern_val = expr.intern_val; + sym_insert(sym); + Ast_End(); + } + + invalid_default_case; + } +} + +function void +test_resolve(){ + TEST_PARSER(); + String filename = "test3.kl"_s; + String file_content = os_read_file(scratch, filename); + lex_restream(&ctx, file_content, filename); + Ast_Package *result = parse_file(); + sym_insert_builtins(); + eval_decl(result); +} + diff --git a/new_type.cpp b/new_type.cpp index 1e8cd39..d3f9716 100644 --- a/new_type.cpp +++ b/new_type.cpp @@ -1,7 +1,9 @@ - enum Type_Kind{ TYPE_None, + TYPE_Null, TYPE_Int, + TYPE_Bool, + TYPE_Unsigned, TYPE_String, TYPE_Void, TYPE_Pointer, @@ -12,6 +14,22 @@ enum Type_Kind{ TYPE_Enum, }; +const char *type_names[] = { + "[Invalid Type]", + "[TYPE_Null]", + "[TYPE_Int]", + "[TYPE_Bool]", + "[TYPE_Unsigned]", + "[TYPE_String]", + "[TYPE_Void]", + "[TYPE_Pointer]", + "[TYPE_Array]", + "[TYPE_Func]", + "[TYPE_Struct]", + "[TYPE_Union]", + "[TYPE_Enum]", +}; + struct Type{ Type_Kind kind; SizeU size; @@ -32,13 +50,19 @@ struct Type{ const SizeU pointer_size = sizeof(SizeU); const SizeU pointer_align = __alignof(SizeU); +global Type type__null = {TYPE_Null}; global Type type__void = {TYPE_Void}; global Type type__int = {TYPE_Int, sizeof(int), __alignof(int)}; +global Type type__unsigned = {TYPE_Int, sizeof(unsigned), __alignof(unsigned)}; global Type type__string = {TYPE_String, sizeof(String), __alignof(String)}; +global Type type__bool = {TYPE_Bool, sizeof(bool), __alignof(bool)}; global Type *type_void = &type__void; global Type *type_int = &type__int; +global Type *type_unsigned = &type__unsigned; global Type *type_string = &type__string; +global Type *type_bool = &type__bool; +global Type *type_null = &type__null; function Type * type_new(Allocator *allocator, Type_Kind kind, SizeU size, SizeU align){ @@ -58,12 +82,11 @@ type_copy(Allocator *a, Type *type){ function Type * type_pointer(Type *base){ - Get_Ctx(Parse_Ctx); - Type *result = (Type *)map_get(&ctx->type_map, (void *)base); + Type *result = (Type *)map_get(&pctx->type_map, (void *)base); if(!result){ - result = type_new(&ctx->ast_arena, TYPE_Pointer, pointer_size, pointer_align); + result = type_new(pctx->perm, TYPE_Pointer, pointer_size, pointer_align); result->base = base; - map_insert(&ctx->type_map, base, result); + map_insert(&pctx->type_map, base, result); } assert(result->kind == TYPE_Pointer); return result; @@ -71,9 +94,8 @@ type_pointer(Type *base){ function Type * type_array(Type *base, SizeU size){ - Get_Ctx(Parse_Ctx); U64 hash = hash_mix(hash_ptr(base), hash_u64(size)); - Type *result = (Type *)map_get_u64(&ctx->type_map, hash); + Type *result = (Type *)map_get_u64(&pctx->type_map, hash); if(result){ assert(result->kind == TYPE_Array); assert(result->arr.size == size); @@ -81,44 +103,40 @@ type_array(Type *base, SizeU size){ return result; } - result = type_new(&ctx->ast_arena, TYPE_Array, pointer_size, pointer_align); + result = type_new(pctx->perm, TYPE_Array, pointer_size, pointer_align); result->arr.base = base; result->arr.size = size; - map_insert_u64(&ctx->type_map, hash, result); + map_insert_u64(&pctx->type_map, hash, result); return result; } function Type * type_function(Type *ret, Array args){ - Get_Ctx(Parse_Ctx); - U64 hash = hash_ptr(ret); - IFor(args){ - hash = hash_mix(hash, hash_ptr(*it)); - } + For(args) hash = hash_mix(hash, hash_ptr(*it)); + Type *result = (Type *)map_get_u64(&pctx->type_map, hash); - Type *result = (Type *)map_get_u64(&ctx->type_map, hash); if(result){ assert(result->kind == TYPE_Func); assert(result->func.ret == ret); assert(result->func.args.len == args.len); return result; } - result = type_new(&ctx->ast_arena, TYPE_Func, pointer_size, pointer_align); + + result = type_new(pctx->perm, TYPE_Func, pointer_size, pointer_align); result->func.ret = ret; - result->func.args = args.copy(&ctx->ast_arena); - map_insert_u64(&ctx->type_map, hash, result); + result->func.args = args.tight_copy(pctx->perm); + map_insert_u64(&pctx->type_map, hash, result); return result; } function void test_types(){ - Set_Backup_Scratch(); - Set_Scratch(); + Scratch scratch; Parse_Ctx ctx = {}; - Set_Ctx(&ctx, Parse_Ctx_ID); - ctx.ast_arena = thread_ctx.scratch[1]; + ctx.init(scratch, scratch); + pctx = &ctx; Type *array_type1 = type_array(type_int, 32); Type *array_type2 = type_array(type_int, 32); @@ -133,13 +151,13 @@ test_types(){ assert(pointer_type3 != pointer_type1); assert(pointer_type3 == pointer_type4); - Array types = {}; + Array types = {scratch}; types.add(type_array(type_int, 32)); Type *func_type1 = type_function(types[0], types); Type *func_type2 = type_function(types[0], types); assert(func_type1 == func_type2); - Array types2 = {}; + Array types2 = {scratch}; { types2.add(type_array(type_int, 32)); types2.add(type_int); diff --git a/os_win32.c b/os_win32.c index 4b9e8c4..116d7a8 100644 --- a/os_win32.c +++ b/os_win32.c @@ -1,23 +1,6 @@ function S32 os_main(); const SizeU page_size = 4096; -function String -os_read_file(Arena *arena, String name){ - String result = {0}; - FILE *f = fopen((char *)name.str, "rb"); - assert(f); - fseek(f, 0, SEEK_END); - result.len = ftell(f); - fseek(f, 0, SEEK_SET); - - result.str = (U8 *)arena_push_size(arena, result.len + 1); - fread(result.str, result.len, 1, f); - fclose(f); - - result.str[result.len] = 0; - return result; -} - function SizeU get_align_offset(SizeU size, SizeU align){ SizeU mask = align - 1; diff --git a/resolve.c b/resolve.c deleted file mode 100644 index c95b926..0000000 --- a/resolve.c +++ /dev/null @@ -1,28 +0,0 @@ -typedef struct Sym Sym; - -typedef enum Sym_Kind{ - Sym_Kind_None, - Sym_Kind_Type, - Sym_Kind_Const, - Sym_Kind_Func, -}Sym_Kind; - -typedef enum Sym_State{ - Sym_State_NotVisited, - Sym_State_Visited, -}; - -struct Sym{ - Sym_Kind kind; - Sym_State state; - AST *ast; - Type *type; -}; - -global Sym *global_syms; - - - -function void -resolve_test(){ -} diff --git a/test3.kl b/test3.kl new file mode 100644 index 0000000..8ddbde6 --- /dev/null +++ b/test3.kl @@ -0,0 +1,58 @@ + +boolean: bool = true +value_of_bool: int = cast(boolean: int) +base := null + +array1: [4]int = {1,2,3,4} +array2: [32]int = {1,2,3,4} +array3: [32]int = { + [0] = 0, + [31] = 31, +} + +/* + +pointer: *int +variable: int = *pointer +pointer_from_var := ^variable + +pointer := &base +base_from_pointer := *pointer + +structure: Data = {} + +array2: [4]int = { + [0] = 0, + [1] = 1 +} +array_to_pointer: *[4]int = &array1 +array_to_pointer_imp : = &array1 + + +*/ +//----------------------------------------------------------------------------- +// Implicit type +//----------------------------------------------------------------------------- +implicit_int :: 10 +implicit_str :: "Hello world" + +//----------------------------------------------------------------------------- +// Pointers +//----------------------------------------------------------------------------- +pointer: *int = cast(null: *int) +pointer2: *int = null +pointer3: **int = null + +//----------------------------------------------------------------------------- +// String types +//----------------------------------------------------------------------------- +string1 :: "Test" +string2 :: string1 + +//----------------------------------------------------------------------------- +// Constant int variables +//----------------------------------------------------------------------------- +thing0 :: 10 +thing1 :: thing0 + 11 +thing2 :: thing1 + 20 +combin :: thing0 + thing1 + thing2