krzosa/corelang
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Parsing exprs, enum_decls, Introduce intern table, symbol table

Browse Source
This commit is contained in:
Krzosa Karol
2022-04-29 11:22:10 +02:00
parent d462892e14
commit 9cbbb4d616
20 changed files with 1831 additions and 335 deletions
Download Patch File Download Diff File Expand all files Collapse all files

438
main.c
View File

@@ -1,341 +1,41 @@
#include "os.cpp"
#define _CRT_SECURE_NO_WARNINGS
#include <stdio.h>
#include <windows.h>
#include "lang.h"
#include "os.h"
#include "memory.h"
#include "lex.h"
global FILE *global_output_file;
#define lex_print(...) fprintf(global_output_file, __VA_ARGS__)
#include "parser.h"
#include "type.h"
#include "expr.h"
#include "decl.h"
function void
memory_zero(void *p, SizeU size){
U8 *pp = p;
for(SizeU i = 0; i < size; i++)
pp[i] = 0;
}
function B32
string_compare(String a, String b){
if(a.len != b.len)
return false;
for(S64 i = 0; i < a.len; i++){
if(a.str[i] != b.str[i])
return false;
}
return true;
}
function Token *
token_alloc(Tokens *t){
if(t->cap == 0){
t->cap = 1024;
t->tokens = malloc(sizeof(Token)*t->cap);
}
else if(t->len+1 > t->cap){
t->cap *= 2;
t->tokens = realloc(t->tokens, sizeof(Token)*t->cap);
}
Token *result = t->tokens + t->len++;
memory_zero(result, sizeof(*result));
return result;
}
function void
lex_advance(Lex_Stream *s){
if(*s->stream == '\n'){
s->stream++;
s->line++;
s->line_begin = s->stream;
}
else if(*s->stream == 0){
// Don't advance, end of stream
}
else{
s->stream++;
}
}
function U64
parse_u64(U8 *str, S64 len){
U64 result = 0;
U64 m = 1;
for(S64 i = len - 1; i >= 0; --i){
U64 val = str[i] - '0';
result += val * m;
m *= 10;
}
return result;
}
function B32
lex_is_whitespace(U8 c){
B32 result = c == '\n' || c == '\r' || c == ' ' || c == '\r';
return result;
}
function B32
lex_is_alphabetic(U8 c){
B32 result = (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z');
return result;
}
function B32
lex_is_numeric(U8 c){
B32 result = c >= '0' && c <= '9';
return result;
}
function B32
lex_is_alphanumeric(U8 c){
B32 result = lex_is_numeric(c) || lex_is_alphabetic(c);
return result;
}
function void
lex_set_len(Lex_Stream *s, Token *token){
assert(s->stream > token->str);
token->len = s->stream - token->str;
}
function U8
lexc(Lex_Stream *s){
return *s->stream;
}
function void
token_error(Token *t, String error_val){
t->kind = TK_Error;
t->error_val = error_val;
}
function void
lex_parse_string(Lex_Stream *s, Token *t, U8 c){
for(;;){
if(lexc(s) == '\\') lex_advance(s);
else if(lexc(s) == c) break;
else if(lexc(s) == 0){
token_error(t, lit("Unterminated string, reached end of file"));
break;
}
lex_advance(s);
}
if(t->kind != TK_Error){
lex_advance(s);
lex_set_len(s,t);
}
}
function void
lex_token_seed(Lex_Stream *s, Token *t){
t->str = s->stream;
t->file = s->filename;
t->line = s->line;
t->line_begin = s->line_begin;
}
function void
token_push_error(Lex_Stream *stream, Tokens *tokens, String error_val){
Token *token = token_alloc(tokens);
token->kind = TK_Error;
token->error_val = error_val;
lex_token_seed(stream, token);
}
function void
lex_base(Lex_Stream *s, Tokens *tokens){
while(*s->stream){
while(lex_is_whitespace(*s->stream))
lex_advance(s);
#define CASE2(op, OpName, Assign) \
case op: \
if (lexc(s) == '=') { \
lex_advance(s); \
t->kind = Assign; \
} else { \
t->kind = OpName; \
} \
break
#define CASE3(op, OpName, Assign, Incr) \
case op: \
if (lexc(s) == '=') { \
lex_advance(s); \
t->kind = Assign; \
} else if (lexc(s) == op) { \
lex_advance(s); \
t->kind = Incr; \
} else { \
t->kind = OpName; \
} \
break
Token *t = token_alloc(tokens);
lex_token_seed(s, t);
lex_advance(s);
switch(*t->str) {
CASE2('!', TK_Not, TK_NotEquals);
CASE2('^', TK_BitXor, TK_XorAssign);
CASE2('=', TK_Assign, TK_Equals);
CASE2('*', TK_Mul, TK_MulAssign);
CASE2('%', TK_Mod, TK_ModAssign);
CASE3('+', TK_Add, TK_AddAssign, TK_Increment);
CASE3('&', TK_BitAnd, TK_AndAssign, TK_And);
CASE3('|', TK_BitOr, TK_OrAssign, TK_Or);
#undef CASE2
#undef CASE3
case '@': t->kind = TK_At; break;
case '(': t->kind = TK_OpenParen; break;
case ')': t->kind = TK_CloseParen; break;
case '{': t->kind = TK_OpenBrace; break;
case '}': t->kind = TK_CloseBrace; break;
case '[': t->kind = TK_OpenBracket; break;
case ']': t->kind = TK_CloseBracket; break;
case ',': t->kind = TK_Comma; break;
case '~': t->kind = TK_Neg; break;
case '?': t->kind = TK_Question; break;
case ';': t->kind = TK_Semicolon; break;
case '-':{
if (lexc(s) == '=') {
lex_advance(s);
t->kind = TK_SubAssign;
}
else if (lexc(s) == '-') {
lex_advance(s);
t->kind = TK_Decrement;
}
else if (lexc(s) == '>') {
lex_advance(s);
t->kind = TK_Arrow;
}
else {
t->kind = TK_Sub;
}
} break;
case '\'':{not_implemented;} break;
case '"': {
t->kind = TK_U8Lit;
lex_parse_string(s,t,'"');
if(t->kind != TK_Error){
t->str += 1;
t->len -= 2;
}
} break;
case '/': {
if(lexc(s) == '/'){
lex_advance(s);
if(lexc(s) == '/'){
lex_advance(s);
t->kind = TK_DocComment;
}
else {
t->kind = TK_Comment;
}
for(;;){
if(lexc(s) == '\n' || lexc(s) == 0) break;
lex_advance(s);
}
lex_set_len(s,t);
}
else if(lexc(s) == '*'){
lex_advance(s);
t->kind = TK_Comment;
for(;;){
if(s->stream[0] == '*' && s->stream[1] == '/'){
lex_advance(s);
lex_advance(s);
break;
}
else if(lexc(s) == 0){
token_error(t, lit("Unterminated block comment"));
break;
}
lex_advance(s);
}
lex_set_len(s,t);
}
else t->kind = TK_Div;
} break;
case '0':
case '1':case '2':case '3':
case '4':case '5':case '6':
case '7':case '8':case '9': {
t->kind = TK_Int;
while(lex_is_numeric(lexc(s)))
lex_advance(s);
lex_set_len(s, t);
t->int_val = parse_u64(t->str, t->len);
} break;
case 'l':{
if(s->stream[0] == 'i' && s->stream[1] == 't' && s->stream[2] == '(' && s->stream[3] == '"'){
t->kind = TK_StringLit;
lex_advance(s);lex_advance(s);lex_advance(s);lex_advance(s);
lex_parse_string(s,t,'"');
if(s->stream[0] == ')') {
t->str += 5;
t->len -= 6;
lex_advance(s);
}
else token_error(t, lit("Unterminated string literal, missing closing parenthesis"));
break;
}
};
case 'A':case 'a':case 'M':case 'm':case 'B':
case 'b':case 'N':case 'n':case 'C':case 'c':case 'O':
case 'o':case 'D':case 'd':case 'P':case 'p':case 'E':
case 'e':case 'Q':case 'q':case 'F':case 'f':case 'R':
case 'r':case 'G':case 'g':case 'S':case 's':case 'H':
case 'h':case 'T':case 't':case 'I':case 'i':case 'U':
case 'u':case 'J':case 'j':case 'V':case 'v':case 'K':
case 'k':case 'W':case 'w':case 'L':case 'X':
case 'x':case 'Z':case 'z':case 'Y':case 'y':case '_': {
t->kind = TK_Identifier;
while(lex_is_alphanumeric(lexc(s)) || lexc(s) == '_')
lex_advance(s);
lex_set_len(s,t);
} break;
default: {
token_error(t, lit("Unknown token"));
} break;
}
}
}
function Tokens
lex_stream(String in_stream, String filename){
Lex_Stream stream = {in_stream.str, in_stream.str, filename, 0};
Tokens tokens = {};
lex_base(&stream, &tokens);
return tokens;
}
function void
token_print(Tokens tokens){
lex_print("\n== Token count = %d\n", (S32)tokens.len);
for(Token *t = tokens.tokens; t != tokens.tokens + tokens.len; t++){
lex_print("%s %.*s\n", token_kind_string[t->kind].str, (S32)t->len, t->str);
}
}
function B32
token_compare(Token *t, String str){
B32 result = string_compare(t->string, str);
return result;
}
#include "common.c"
#include "memory.c"
#include "parser.c"
#include "os_win32.c"
#include "lex.c"
#include "expr.c"
#include "decl.c"
#include "type.c"
#include "parse_expr.c"
#include "parse_decl.c"
#include "print.c"
function void
lex_test(){
Tokens t;
t = lex_stream(lit("32523 42524 \"U8Literal\""), lit("test"));
token_print(t);
t = lex_stream(lit("3252342510 42524 \"U8Literal\""), lit("test"));
//tokens_print(t);
assert(t.len == 3);
assert(t.tokens[0].int_val == 32523);
assert(t.tokens[0].int_val == 3252342510);
assert(t.tokens[1].int_val == 42524);
assert(t.tokens[2].kind == TK_U8Lit);
assert(token_compare(t.tokens + 2, lit("U8Literal")));
t = lex_stream(lit("_identifier Thing Thing2 lit(\"String_Test\")"), lit("test"));
token_print(t);
//tokens_print(t);
assert(t.tokens[0].kind == TK_Identifier);
assert(t.tokens[1].kind == TK_Identifier);
assert(t.tokens[2].kind == TK_Identifier);
@@ -345,10 +45,68 @@ lex_test(){
assert(token_compare(t.tokens+2, lit("Thing2")));
assert(token_compare(t.tokens+3, lit("String_Test")));
t = lex_stream(lit("lit(\"String_Test\"{})(324*=+=-/ *% // Comment \n Thing /*Thing*/ /*Error"), lit("test"));
t = lex_stream(lit("lit(\"String_Test\"{})(324*=+=-/ *% // Comment \n"
"Thing /*Thing*/ += -= =- +/%^&*&&|| |>> << <<= >>=/*Error"),
lit("test"));
assert(t.tokens[0].kind == TK_Error);
token_print(t);
//tokens_print(t);
}
function void
parser_test(){
Parser p = {};
{
parser_init(&p);
Intern_String a = intern_string(&p, lit("Thing"));
Intern_String b = intern_string(&p, lit("Thing"));
assert(a.s.str == b.s.str);
}
parser_lex_stream(&p, lit("S64 thing; S64 second_var = 10;"), lit("File"));
assert(token_match(&p, TK_Keyword));
assert(token_match(&p, TK_Identifier));
assert(token_match(&p, TK_Semicolon));
assert(token_match(&p, TK_Keyword));
assert(token_match(&p, TK_Identifier));
assert(token_match(&p, TK_Assign));
assert(token_match(&p, TK_Int));
assert(token_match(&p, TK_Semicolon));
assert(token_match(&p, TK_End));
assert(token_match(&p, TK_End));
assert(token_match(&p, TK_End));
String exprs[] = {
lit("(4+2*53)"),
lit("((4+2)*53"),
lit("++5"),
lit("5--"), // @Todo(Krzosa):
lit("-5"),
lit("(+5)"),
lit("(S64)5"),
lit("(S64)5+3"),
lit("534>43?435:42"),
lit("(534>43?435:42,234,(S64)42,Thing[10][2],Thing(1,2))"),
};
for(S64 i = 0; i < buff_cap(exprs); i++){
parser_lex_stream(&p, exprs[i], lit("File"));
Expr *expr = parse_expr(&p);
assert(expr);
expr_print(expr);
lex_print("\n");
}
type_test(&p);
lex_new_line();
String decls[] = {
lit("enum Thing{ Thing_1, Thing_2 = 2}"),
};
for(S64 i = 0; i < buff_cap(decls); i++){
parser_lex_stream(&p, decls[i], lit("File"));
Decl *decl = parse_decl_global(&p);
assert(decl);
decl_print(decl);
lex_new_line();
}
}
function S32
@@ -356,8 +114,28 @@ os_main(){
global_output_file = fopen("output.txt", "w");
assert_msg(global_output_file, "Failed to open output.txt");
lex_test();
parser_test();
/*
String keywords[]={
lit("S64"),
lit("U64"),
lit("void"),
lit("SizeU"),
lit("struct"),
lit("union"),
lit("function"),
lit("global"),
};
for(S64 i = 0; i < buff_cap(keywords); i++){
lex_print("global Intern_String keyword_%s;\n", keywords[i].str);
}
for(S64 i = 0; i < buff_cap(keywords); i++){
lex_print("keyword_%s = intern_string(p, lit(\"%s\"));\n", keywords[i].str, keywords[i].str);
}
*/
fclose(global_output_file);
return 0;
}