krzosa/corelang
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

More parsing of expressions

Browse Source
This commit is contained in:
Krzosa Karol
2022-05-06 21:58:09 +02:00
parent 76a8cc779c
commit 501e42be19
6 changed files with 661 additions and 267 deletions
Download Patch File Download Diff File Expand all files Collapse all files

438
new_parse.c
View File

@@ -29,7 +29,23 @@ parser_push_error(Parser *p, Token *token, char *str, ...){
va_end(args1);
}
printf("Error: %s %s:%d\n", string.str, token->file.str, (S32)token->line);
// @Note(Krzosa): Print nice error message
{
printf("Error: %s %s:%d\n", string.str, token->file.str, (S32)token->line);
// @Note(Krzosa): Print error line
{
int i = 0;
while(token->line_begin[i]!='\n' && token->line_begin[i]!=0) i++;
printf("%.*s\n", i, token->line_begin);
// @Note(Krzosa): Print error marker
int token_i = token->str - token->line_begin;
for(int i = 0; i < token_i-2; i++) printf(" ");
printf("^^^^^^\n");
}
}
Parser_Error *error = arena_push_struct(p->arena, Parser_Error);
error->message = string;
error->next = 0;
@@ -48,6 +64,20 @@ token_get(Parser *p){
return result;
}
function Token *
token_peek(Parser *p, S64 i){
Token *result = token_array_iter_peek(&p->tokens, i);
return result;
}
function Token *
token_peek_is(Parser *p, Token_Kind kind, S64 i){
Token *result = token_peek(p, i);
if(result->kind == kind)
return result;
return 0;
}
function Token *
token_is(Parser *p, Token_Kind kind){
Token *result = token_get(p);
@@ -95,6 +125,8 @@ token_expect(Parser *p, Token_Kind kind){
return 0;
}
function Typespec *parse_typespec(Parser *p);
function Expr *parse_expr(Parser *p);
//-----------------------------------------------------------------------------
// Expression parsing
//-----------------------------------------------------------------------------
@@ -112,6 +144,9 @@ atom_expr = Int
| 'cast' '(' typespec ',' expr ')'
| 'size_type' '(' typespec ')'
| 'size_expr' '(' expr ')'
| '{' compound_expr '}'
| '(' expr ')'
| '(' ':' typespec ')' '{' compound_expr '}'
postfix_expr = atom_expr ('[' expr ']' | '.' Identifier | ++ | -- | '(' expr_list ')')*
unary_expr = unary ? unary_expr : atom_expr
mul_expr = atom_expr (mul atom_expr)*
@@ -120,57 +155,16 @@ logical_expr = add_expr (logical add_expr)*
compare_expr = logical_expr (compare logical_expr)*
ternary_expr = compare_expr ('?' ternary_expr ':' ternary_expr)?
expr = logical_expr
Compound literals
- (:[23]*Type){}
- Type{}
- { }
*/
function Expr *parse_expr(Parser *p);
function Expr *
parse_expr_atom(Parser *p){
Token *token = 0;
if((token = token_match(p, TK_StringLit))){
Expr *result = expr_str(p->arena, token);
return result;
}
else if((token = token_match(p, TK_Identifier))){
Expr *result = expr_identifier(p->arena, token);
return result;
}
else if((token = token_match(p, TK_Int))){
Expr *result = expr_int(p->arena, token);
return result;
}
else if((token = token_match(p, TK_OpenParen))){
Expr *expr = parse_expr(p);
token_expect(p, TK_CloseParen);
Expr *result = expr_paren(p->arena, token, expr);
return result;
}
else if((token = token_match_keyword(p, keyword_cast))){
token_expect(p, TK_OpenParen);
token_expect(p, TK_Identifier);
token_expect(p, TK_Comma);
token_expect(p, TK_Identifier);
token_expect(p, TK_CloseParen);
return 0;
}
else{
parser_push_error(p, token_get(p), "Failed to parse expression");
return 0;
}
}
function B32
token_is_postfix(Parser *p){
Token *token = token_get(p);
B32 result = token->kind == TK_OpenBracket
|| token->kind == TK_OpenParen
|| token->kind == TK_Dot
|| token->kind == TK_Increment
|| token->kind == TK_Decrement;
return result;
}
function Expr_Compound_Field *
parse_expr_compound(Parser *p){
parse_expr_compound_field(Parser *p){
Token *token = token_get(p);
Expr_Compound_Field *result = 0;
if(token_match(p, TK_OpenBracket)){
@@ -196,6 +190,81 @@ parse_expr_compound(Parser *p){
return result;
}
function Expr *
parse_expr_compound(Parser *p, Typespec *typespec){
Token *token = token_expect(p, TK_OpenBrace);
Expr *expr = expr_compound(p->arena, token, typespec);
while(!token_is(p, TK_CloseBrace)){
Expr_Compound_Field *field = parse_expr_compound_field(p);
expr_compound_push(expr, field);
if(!token_match(p, TK_Comma)){
break;
}
}
token_expect(p, TK_CloseBrace);
return expr;
}
function Expr *
parse_expr_atom(Parser *p){
Expr *result = 0;
Token *token = token_get(p);
if(token_match(p, TK_StringLit)){
result = expr_str(p->arena, token);
}
else if(token_match(p, TK_Identifier)){
if(token_is(p, TK_OpenBrace)){
Typespec *typespec = typespec_name(p->arena, token, token->intern_val);
result = parse_expr_compound(p, typespec);
}
else{
result = expr_identifier(p->arena, token);
}
}
else if(token_match(p, TK_Int)){
result = expr_int(p->arena, token);
}
else if(token_is(p, TK_OpenBrace)){
result = parse_expr_compound(p, 0);
}
else if(token_match(p, TK_OpenParen)){
if(token_match(p, TK_Colon)){
Typespec *typespec = parse_typespec(p);
token_expect(p, TK_CloseParen);
result = parse_expr_compound(p, typespec);
}
else{
Expr *expr = parse_expr(p);
token_expect(p, TK_CloseParen);
result = expr_paren(p->arena, token, expr);
}
}
else if(token_match_keyword(p, keyword_cast)){
token_expect(p, TK_OpenParen);
Typespec *typespec = parse_typespec(p);
token_expect(p, TK_Comma);
Expr *expr = parse_expr(p);
token_expect(p, TK_CloseParen);
result = expr_cast(p->arena, token, typespec, expr);
}
else{
parser_push_error(p, token, "Failed to parse expression");
}
return result;
}
function B32
token_is_postfix(Parser *p){
Token *token = token_get(p);
B32 result = token->kind == TK_OpenBracket
|| token->kind == TK_OpenParen
|| token->kind == TK_Dot
|| token->kind == TK_Increment
|| token->kind == TK_Decrement;
return result;
}
function Expr_Compound_Field *
parse_expr_function_argument(Parser *p){
Token *token = token_get(p);
@@ -365,10 +434,171 @@ parse_expr(Parser *p){
return parse_expr_ternary(p);
}
//-----------------------------------------------------------------------------
// Type specifier parsing
//-----------------------------------------------------------------------------
/*
base_type = NAME
| '(' type_list? ')' type?
type = ('*' | '[' expr ']')* base_type
Examples:
[32]*U32 - Array of 32 pointers to U32
**CustomDataType - Pointer to pointer of CustomDataType
(*U32, S64) **S64 - Function pointer
(CoolType: optional, S32) - Implicit void return value
*/
function Typespec *
parse_typespec_function(Parser *p, Token *token){
Typespec *result = typespec_function(p->arena, token, 0);
if(!token_is(p, TK_CloseParen))
for(;;) {
// Optional name
Token *name = 0;
if((token = token_is(p, TK_Identifier))){
if(token_peek_is(p, TK_Colon, 1)){
token_next(p);
token_next(p);
name = token;
}
}
// Parse type
Typespec *arg = parse_typespec(p);
if(name)
arg = typespec_named_argument(p->arena, name, arg, name->intern_val);
typespec_function_push(result, arg);
if(!token_match(p, TK_Comma)){
break;
}
}
token_expect(p, TK_CloseParen);
if(token_is(p, TK_Identifier)
|| token_is(p, TK_OpenParen)
|| token_is(p, TK_Mul)
|| token_is(p, TK_OpenBracket))
result->func.ret = parse_typespec(p);
else
result->func.ret = typespec_name(p->arena, token_get(p), intern_void);
return result;
}
// [10]*int - Array of 10 pointers to ints
function Typespec *
parse_typespec_recurse(Parser *p){
Token *token = token_get(p);
if(token_match(p, TK_Mul)){
Typespec *result = parse_typespec_recurse(p);
result = typespec_pointer(p->arena, token, result);
return result;
}
else if(token_match(p, TK_OpenBracket)){
Expr *expr = parse_expr(p);
token_expect(p, TK_CloseBracket);
Typespec *result = parse_typespec_recurse(p);
result = typespec_array(p->arena, token, result, expr);
return result;
}
else if(token_match(p, TK_OpenParen)){
Typespec *result = parse_typespec_function(p, token);
return result;
}
else if(token_match(p, TK_Identifier)){
Typespec *result = typespec_name(p->arena, token, token->intern_val);
return result;
}
else{
parser_push_error(p, token, "Failed to parse type, unexpected token");
return 0;
}
}
function Typespec *
parse_typespec(Parser *p){
Typespec *result = parse_typespec_recurse(p);
return result;
}
//-----------------------------------------------------------------------------
// Parsing decls
//-----------------------------------------------------------------------------
/*
name::(param:U32)*U32{}
name::struct{}
name::union{}
name::enum{}
name::typedef = name2;
name::const = 4254;
*/
function Decl *
parse_enum(Parser *p, Token *name){
Typespec *typespec = 0;
if(token_match(p, TK_Colon)){
typespec = parse_typespec(p);
}
else{
typespec = typespec_name(p->arena, token_get(p), intern_int);
}
Decl *result = decl_enum(p->arena, name, name->intern_val, typespec);
token_expect(p, TK_OpenBrace);
do{
Token *val = token_expect(p, TK_Identifier);
Expr *expr = 0;
if(token_match(p, TK_Assign)){
expr = parse_expr(p);
}
decl_enum_push(p->arena, result, val, val->intern_val, expr, 0);
} while(token_is(p, TK_Comma));
token_expect(p, TK_CloseBrace);
return result;
}
function Decl *
parse_decl(Parser *p){
Decl *result = 0;
Token *name = 0;
if((name = token_match(p, TK_Identifier))){
if(token_match(p, TK_DoubleColon)){
Token *token = 0;
if((token = token_match_keyword(p, keyword_enum))){
result = parse_enum(p, name);
}
else if((token = token_match_keyword(p, keyword_union))){
// Union
}
else if((token = token_match_keyword(p, keyword_struct))){
// Struct
}
else if((token = token_match_keyword(p, keyword_const))){
// Const value
}
else if((token = token_match(p, TK_OpenParen))){
// Function
}
else{
parser_push_error(p, token_get(p), "Expected token of kind todo:decl_tokens");
}
}
else{
parser_push_error(p, token_get(p), "Expected token of kind '::'");
}
}
return result;
}
//-----------------------------------------------------------------------------
// Test code
//-----------------------------------------------------------------------------
function void expr_print(Expr *expr);
function B32 typespec_print(Typespec *spec);
function void
token_print(Token *token){
printf("%.*s", (S32)token->len, token->str);
@@ -408,7 +638,11 @@ expr_print(Expr *expr){
}break;
case EK_Compound:{
// typespec_print(p);
if(expr->compound.typespec){
printf("(");
typespec_print(expr->compound.typespec);
printf(")");
}
printf("{");
for(Expr_Compound_Field *n = expr->compound.first; n; n=n->next){
expr_compound_print(n);
@@ -468,7 +702,7 @@ expr_print(Expr *expr){
case EK_Cast:{
printf("(");
printf("(");
//print_typespec(expr->cast.type);
typespec_print(expr->cast.typespec);
printf(")");
expr_print(expr->cast.expr);
printf(")");
@@ -494,6 +728,46 @@ expr_print(Expr *expr){
}
}
function B32
typespec_print(Typespec *spec){
switch(spec->kind) {
case TS_Name: {
printf("%s", spec->name.s.str);
} break;
case TS_NamedArgument: {
printf("%s: ", spec->named.name.s.str);
typespec_print(spec->named.base);
}break;
case TS_Pointer: {
typespec_print(spec->base);
printf("*");
} break;
case TS_Array: {
typespec_print(spec->arr.base);
printf("[");
expr_print(spec->arr.size);
printf("]");
} break;
case TS_Function: {
printf("(");
for(Typespec *n = spec->func.first; n; n=n->next){
typespec_print(n);
if(n!=spec->func.last)
printf(", ");
}
printf(")");
typespec_print(spec->func.ret);
} break;
default: {invalid_codepath;} break;
}
return true;
}
function S64
eval_expr(Expr *expr){
@@ -545,8 +819,34 @@ eval_expr(Expr *expr){
return 0;
}
function Parser
parser_make(Arena *arena){
Parser result = {
.tokens = lex_make_token_array(arena),
.arena = arena,
};
return result;
}
function void
parse_test(){
parser_restream(Parser *p, String stream, String file){
lex_restream(&p->tokens, stream, file);
}
function Parser
parser_make_stream(Arena *arena, String stream, String file){
Parser parser = parser_make(arena);
lex_restream(&parser.tokens, stream, file);
return parser;
}
function void
parser_add_stream(Parser *p, String string, String file){
lex_add_stream(&p->tokens, string, file);
}
function void
parse_test_expr(){
Arena *scratch = arena_begin_scratch();
String test_case = lit("32+52-242*2/424%5-23"
" 1<<5>>6<<2 "
@@ -558,12 +858,9 @@ parse_test(){
" 1>5 ? 1 : 2 "
" !!!!!1 "
" ~~1 + -!2 "
" 1 + ++Thing[12]++ + ++Thing[12].expr + --Not_Thing[156](Thing) + test_func(asd=func1, af=func2, gg=func3)"
" 1 + ++Thing[12]++ + ++Thing[12].expr +"
);
Parser parser = {
.tokens = lex_stream(scratch, test_case, lit("expr_test")),
.arena = scratch,
};
Parser parser = parser_make_stream(scratch, test_case, lit("Big_Expr"));
Parser *p = &parser;
S64 t = 5;
@@ -585,10 +882,33 @@ parse_test(){
assert(val == test_val[i]);
}
Expr *expr = parse_expr(p);
expr_print(expr);
String exprs[] = {
lit("cast([12](thing: U32, qwe: *U32) [32]Result, (123+234))"),
lit("cast((thing: U32, qwe: *U32), (123+234))"),
lit("(:(U32,U32)){Thing=10}"),
lit("--Not_Thing[156](Thing) + test_func(asd=func1, af=func2, gg=func3)"),
lit("(:[23]*Type){Thing=10}"),
lit("cast(**Data,{Thing=10})"),
lit("(:[64]S64){1,2,3,4,5}"),
lit("Data_Type{1,2,3,4,5}"),
};
for(SizeU i = 0; i < buff_cap(exprs); i++){
parser_restream(p, exprs[i], lit("Test_Exprs"));
Expr *expr = parse_expr(p);
expr_print(expr);
printf("\n");
}
arena_end_scratch();
}
function void
parse_test_decls(){
}
function void
parse_test(){
parse_test_expr();
parse_test_decls();
}