krzosa/smallprojects
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Expand C tokens and parser expression support

Browse Source
This commit is contained in:
Krzosa Karol
2026-05-10 14:28:57 +02:00
parent 9792517c41
commit f0e5b6a273
5 changed files with 161 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
main.c
View File

@@ -4,6 +4,7 @@
#include <stdlib.h>
#include <ctype.h>
#include <stdint.h>
#include <stdarg.h>
#include "base.c"
#include "meta_gen.c"
@@ -115,13 +116,31 @@ Token lex_token(Lexer *lex) {
switch (c) {
case 0: t.kind = TOK_EOF; break;
case '(': t.kind = TOK_LPAREN; break;
case ')': t.kind = TOK_RPAREN; break;
case '[': t.kind = TOK_LBRACKET; break;
case ']': t.kind = TOK_RBRACKET; break;
case '{': t.kind = TOK_LBRACE; break;
case '}': t.kind = TOK_RBRACE; break;
case ',': t.kind = TOK_COMMA; break;
case '.': t.kind = TOK_DOT; break;
case ':': t.kind = TOK_COLON; break;
case ';': t.kind = TOK_SEMICOLON; break;
case '?': t.kind = TOK_QUESTION; break;
case '#': t.kind = TOK_HASH; break;
case '+': t.kind = TOK_PLUS; break;
case '-': t.kind = TOK_MINUS; break;
case '*': t.kind = TOK_STAR; break;
case '/': t.kind = TOK_SLASH; break;
case '%': t.kind = TOK_PERCENT; break;
case '{': t.kind = TOK_LBRACE; break;
case '}': t.kind = TOK_RBRACE; break;
case '=': t.kind = TOK_ASSIGN; break;
case '<': t.kind = TOK_LT; break;
case '>': t.kind = TOK_GT; break;
case '!': t.kind = TOK_NOT; break;
case '~': t.kind = TOK_BITNOT; break;
case '&': t.kind = TOK_BITAND; break;
case '|': t.kind = TOK_BITOR; break;
case '^': t.kind = TOK_BITXOR; break;
default: {
// @todo: lexer perhaps should have a static buffer of size 1024, error message
// should be put there and piped to the upper program. The token should be filled
@@ -198,6 +217,13 @@ Token *match_token(Parser *p, Token_Kind kind) {
return NULL;
}
Token *expect_token(Parser *p, Token_Kind kind) {
if (p->at->kind == kind) {
return next_token(p);
}
panicf("expected token kind: %s, got instead: %s", token_to_name[p->at->kind], token_to_name[kind]);
}
Ast *create_ast(Token *token, Ast_Kind kind) {
Ast *result = calloc(1, sizeof(Ast));
result->pos = token;
@@ -224,28 +250,37 @@ Ast *parse_atom(Parser *p) {
} else if (match_token(p, TOK_LPAREN)) {
// @todo: do a comma list here
n = parse_expr(p, 0);
expect_token(p, TOK_RPAREN);
} else {
fprintf(stderr, "encountered invalid token while parsing atom: %.*s\n", token->len, token->str);
exit(1);
panicf("unknown token in %s. %.*s (%s/%d), ", __FUNCTION__, token->len, token->str, token_to_name[token->kind], token->kind);
}
return n;
}
int get_binding_power(Token *tok) {
switch (tok->kind) {
case TOK_LSHIFT: case TOK_RSHIFT: return 100;
case TOK_PLUS: case TOK_MINUS: return 110;
case TOK_STAR: case TOK_SLASH: case TOK_PERCENT: return 120;
case TOK_PLUS: case TOK_MINUS: return 110;
case TOK_LSHIFT: case TOK_RSHIFT: return 100;
case TOK_LT: case TOK_LEQ: case TOK_GT: case TOK_GEQ: return 90;
case TOK_EQ: case TOK_NEQ: return 80;
case TOK_BITAND: return 70;
case TOK_BITXOR: return 60;
case TOK_BITOR: return 50;
case TOK_AND: return 40;
case TOK_OR: return 30;
default: return 0;
}
}
Ast *parse_valid_left_binding(Parser *p, Token *tok, Ast *left) {
switch (tok->kind) {
case TOK_LSHIFT: case TOK_RSHIFT: case TOK_PLUS: case TOK_MINUS: case TOK_SLASH: case TOK_STAR: case TOK_PERCENT: {
case TOK_PLUS: case TOK_MINUS: case TOK_STAR: case TOK_SLASH: case TOK_PERCENT:
case TOK_EQ: case TOK_NEQ: case TOK_LT: case TOK_LEQ: case TOK_GT: case TOK_GEQ: case TOK_BITAND:
case TOK_BITOR: case TOK_BITXOR: case TOK_AND: case TOK_OR: case TOK_LSHIFT: case TOK_RSHIFT: {
return create_binary_expr(tok, tok->kind, left, parse_expr(p, get_binding_power(tok)));
} break;
default: fprintf(stderr, "ERROR"); exit(1);
default: panicf("unknown token in %s. %.*s (%s/%d), ", __FUNCTION__, tok->len, tok->str, token_to_name[tok->kind], tok->kind);
}
return NULL;
}
@@ -271,16 +306,23 @@ int64_t eval_expr(Ast *n) {
case TOK_STAR: return left * right;
case TOK_SLASH: return left / right;
case TOK_PERCENT: return left % right;
default: {
fprintf(stderr, "invalid token kind in eval_expr, binary");
exit(1);
}
case TOK_EQ: return left == right;
case TOK_NEQ: return left != right;
case TOK_LT: return left < right;
case TOK_LEQ: return left <= right;
case TOK_GT: return left > right;
case TOK_GEQ: return left >= right;
case TOK_BITAND: return left & right;
case TOK_BITOR: return left | right;
case TOK_BITXOR: return left ^ right;
case TOK_AND: return left && right;
case TOK_OR: return left || right;
case TOK_LSHIFT: return left << right;
case TOK_RSHIFT: return left >> right;
default: panicf("invalid token kind in eval_expr, binary");
}
} break;
default: {
fprintf(stderr, "invalid ast kind in eval_expr");
exit(1);
} break;
default: panicf("invalid ast kind in eval_expr");
}
}
@@ -292,20 +334,37 @@ void print_expr(Ast *n) {
printf(" %s ", token_to_op[n->op]);
print_expr(n->r);
} break;
default: fprintf(stderr, "memes"); exit(1);
default: panicf("encountered invalid ast kind in %s of kind: %d\n", __FUNCTION__, n->kind);
}
}
void parser_test(void) {
#define TEST_EVAL(expr) do { \
Token_Array tokens = lex_file("eval_test", #expr, strlen(#expr));\
Token_Array tokens = lex_file("eval_test", (#expr), strlen((#expr)));\
Parser p = {tokens.data, tokens.data + tokens.len};\
Ast *result = parse_expr(&p, 0);\
assert(eval_expr(result) == expr);\
int64_t left = eval_expr(result);\
int64_t right = (expr);\
if (left != right) {\
printf("%s:%d expected: %ld, got: %ld\n expression: ", __FILE__, __LINE__, left, right);\
print_expr(result);\
printf("\n");\
}\
} while (0)
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wparentheses"
TEST_EVAL(32+5-4);
TEST_EVAL(16/2/2);
TEST_EVAL(5125-42|(4&3)^2|2%1242);
TEST_EVAL((45%2)^(23&3));
TEST_EVAL(1<16*2);
TEST_EVAL(1+1+1+1+2-2-3-4-5);
TEST_EVAL(5%2^5&6|3);
#pragma clang diagnostic pop
printf("parser tests passed\n");
}