diff --git a/lex.c b/lex.c new file mode 100644 index 0000000..ef43d56 --- /dev/null +++ b/lex.c @@ -0,0 +1,194 @@ +typedef struct Token { + Token_Kind kind; + char *str; + int len; + + char *file; + int line, column; + + union { + uint64_t u; + }; +} Token; + +typedef Vec(Token) Token_Array; + +typedef struct Lexer { + char *at; + char *end; + char *file; + int line; + int column; +} Lexer; + +void lex_advance(Lexer *lex) { + if (lex->at >= lex->end) { + return; + } + + if (*lex->at == '\n') { + lex->line++; + lex->column = 0; + } else { + lex->column++; + } + lex->at += 1; +} + +void eat_whitespace(Lexer *lex) { + while (lex->at < lex->end) { + switch (*lex->at) { + case ' ': + case '\t': + case '\r': + case '\n': + lex_advance(lex); + break; + default: + return; + } + } +} + +Lexer make_lexer(char *file, char *src, int len) { + Lexer lex = { + .at = src, + .end = src + len, + .file = file, + .line = 0, + .column = 0, + }; + return lex; +} + +bool lex_peek_is(Lexer *lex, char c) { + return lex->at < lex->end && *lex->at == c; +} + +bool lex_match(Lexer *lex, char c) { + if (lex_peek_is(lex, c)) { + lex_advance(lex); + return true; + } + return false; +} + +Token_Kind lex_repeat_or_assign(Lexer *lex, char repeated_char, Token_Kind single, Token_Kind repeated, Token_Kind assigned) { + if (lex_match(lex, repeated_char)) return repeated; + if (lex_match(lex, '=')) return assigned; + return single; +} + +Token_Kind lex_assign_variant(Lexer *lex, Token_Kind single, Token_Kind assigned) { + return lex_match(lex, '=') ? assigned : single; +} + +Token_Kind lex_shift_family(Lexer *lex, char repeated_char, Token_Kind single, Token_Kind single_eq, Token_Kind doubled, Token_Kind doubled_eq) { + if (lex_match(lex, repeated_char)) { + return lex_match(lex, '=') ? doubled_eq : doubled; + } + return lex_match(lex, '=') ? single_eq : single; +} + +Token lex_token(Lexer *lex) { + eat_whitespace(lex); + Token t = { + .str = lex->at, + .line = lex->line, + .column = lex->column, + .file = lex->file, + }; + + if (lex->at >= lex->end) { + t.kind = TOK_EOF; + t.len = 0; + return t; + } + + char c = *lex->at; + + if (isdigit(c)) { + t.kind = TOK_INT; + while (lex->at < lex->end && isdigit(*lex->at)) { + lex_advance(lex); + } + + // @todo: proper lexing of floats (as well as postfixes) + if (lex->at < lex->end && *lex->at == '.') { + t.kind = TOK_FLOAT; + lex_advance(lex); + + while (lex->at < lex->end && isdigit(*lex->at)) { + lex_advance(lex); + } + } + + if (t.kind == TOK_INT) { + t.u = strtoull(t.str, NULL, 10); + } + + t.len = (int)(lex->at - t.str); + return t; + } + + lex_advance(lex); + + switch (c) { + case 0: t.kind = TOK_EOF; break; + case '(': t.kind = TOK_LPAREN; break; + case ')': t.kind = TOK_RPAREN; break; + case '[': t.kind = TOK_LBRACKET; break; + case ']': t.kind = TOK_RBRACKET; break; + case '{': t.kind = TOK_LBRACE; break; + case '}': t.kind = TOK_RBRACE; break; + case ',': t.kind = TOK_COMMA; break; + case '.': t.kind = TOK_DOT; break; + case ':': t.kind = TOK_COLON; break; + case ';': t.kind = TOK_SEMICOLON; break; + case '?': t.kind = TOK_QUESTION; break; + case '#': t.kind = TOK_HASH; break; + case '+': t.kind = lex_repeat_or_assign(lex, '+', TOK_PLUS, TOK_INC, TOK_PLUS_ASSIGN); break; + case '-': { + if (lex_match(lex, '-')) t.kind = TOK_DEC; + else if (lex_match(lex, '=')) t.kind = TOK_MINUS_ASSIGN; + else if (lex_match(lex, '>')) t.kind = TOK_ARROW; + else t.kind = TOK_MINUS; + } break; + case '*': t.kind = lex_assign_variant(lex, TOK_STAR, TOK_MUL_ASSIGN); break; + case '/': t.kind = lex_assign_variant(lex, TOK_SLASH, TOK_DIV_ASSIGN); break; + case '%': t.kind = lex_assign_variant(lex, TOK_PERCENT, TOK_MOD_ASSIGN); break; + case '=': t.kind = lex_assign_variant(lex, TOK_ASSIGN, TOK_EQ); break; + case '<': t.kind = lex_shift_family(lex, '<', TOK_LT, TOK_LEQ, TOK_LSHIFT, TOK_LSHIFT_ASSIGN); break; + case '>': t.kind = lex_shift_family(lex, '>', TOK_GT, TOK_GEQ, TOK_RSHIFT, TOK_RSHIFT_ASSIGN); break; + case '!': t.kind = lex_assign_variant(lex, TOK_NOT, TOK_NEQ); break; + case '~': t.kind = TOK_BITNOT; break; + case '&': t.kind = lex_repeat_or_assign(lex, '&', TOK_BITAND, TOK_AND, TOK_AND_ASSIGN); break; + case '|': t.kind = lex_repeat_or_assign(lex, '|', TOK_BITOR, TOK_OR, TOK_OR_ASSIGN); break; + case '^': t.kind = lex_assign_variant(lex, TOK_BITXOR, TOK_XOR_ASSIGN); break; + default: panicf("unrecognized character: '%c', can't match with any of the token kinds", c); + } + + t.len = (int)(lex->at - t.str); + return t; +} + +void assert_token(Token t, Token_Kind kind, char *text, int line, int column) { + assert(t.kind == kind); + assert(t.line == line); + assert(t.column == column); + assert(t.len == (int)strlen(text)); + assert(strncmp(t.str, text, t.len) == 0); +} + +Token_Array lex_file(char *file, char *src, int len) { + Lexer lex = make_lexer(file, src, len); + Token_Array result = {0}; + for (;;) { + Token token = lex_token(&lex); + vec_push(&result, token); + if (token.kind == TOK_EOF) { + break; + } + } + return result; +} \ No newline at end of file diff --git a/main.c b/main.c index 890a428..65d2a59 100644 --- a/main.c +++ b/main.c @@ -8,206 +8,7 @@ #include #include "base.c" #include "meta_gen.c" - -typedef struct Token { - Token_Kind kind; - char *str; - int len; - - char *file; - int line, column; - - union { - uint64_t u; - }; -} Token; - -typedef Vec(Token) Token_Array; - -typedef struct Lexer { - char *at; - char *end; - char *file; - int line; - int column; -} Lexer; - -void lex_advance(Lexer *lex) { - if (lex->at >= lex->end) { - return; - } - - if (*lex->at == '\n') { - lex->line++; - lex->column = 0; - } else { - lex->column++; - } - lex->at += 1; -} - -void eat_whitespace(Lexer *lex) { - while (lex->at < lex->end) { - switch (*lex->at) { - case ' ': - case '\t': - case '\r': - case '\n': - lex_advance(lex); - break; - default: - return; - } - } -} - -Lexer make_lexer(char *file, char *src, int len) { - Lexer lex = { - .at = src, - .end = src + len, - .file = file, - .line = 0, - .column = 0, - }; - return lex; -} - -bool lex_peek_is(Lexer *lex, char c) { - return lex->at < lex->end && *lex->at == c; -} - -bool lex_match(Lexer *lex, char c) { - if (lex_peek_is(lex, c)) { - lex_advance(lex); - return true; - } - return false; -} - -Token_Kind lex_repeat_or_assign(Lexer *lex, char repeated_char, Token_Kind single, Token_Kind repeated, Token_Kind assigned) { - if (lex_match(lex, repeated_char)) return repeated; - if (lex_match(lex, '=')) return assigned; - return single; -} - -Token_Kind lex_assign_variant(Lexer *lex, Token_Kind single, Token_Kind assigned) { - return lex_match(lex, '=') ? assigned : single; -} - -Token_Kind lex_shift_family(Lexer *lex, char repeated_char, Token_Kind single, Token_Kind single_eq, Token_Kind doubled, Token_Kind doubled_eq) { - if (lex_match(lex, repeated_char)) { - return lex_match(lex, '=') ? doubled_eq : doubled; - } - return lex_match(lex, '=') ? single_eq : single; -} - -Token lex_token(Lexer *lex) { - eat_whitespace(lex); - Token t = { - .str = lex->at, - .line = lex->line, - .column = lex->column, - .file = lex->file, - }; - - if (lex->at >= lex->end) { - t.kind = TOK_EOF; - t.len = 0; - return t; - } - - char c = *lex->at; - - if (isdigit(c)) { - t.kind = TOK_INT; - while (lex->at < lex->end && isdigit(*lex->at)) { - lex_advance(lex); - } - - // @todo: proper lexing of floats (as well as postfixes) - if (lex->at < lex->end && *lex->at == '.') { - t.kind = TOK_FLOAT; - lex_advance(lex); - - while (lex->at < lex->end && isdigit(*lex->at)) { - lex_advance(lex); - } - } - - if (t.kind == TOK_INT) { - t.u = strtoull(t.str, NULL, 10); - } - - t.len = (int)(lex->at - t.str); - return t; - } - - lex_advance(lex); - - switch (c) { - case 0: t.kind = TOK_EOF; break; - case '(': t.kind = TOK_LPAREN; break; - case ')': t.kind = TOK_RPAREN; break; - case '[': t.kind = TOK_LBRACKET; break; - case ']': t.kind = TOK_RBRACKET; break; - case '{': t.kind = TOK_LBRACE; break; - case '}': t.kind = TOK_RBRACE; break; - case ',': t.kind = TOK_COMMA; break; - case '.': t.kind = TOK_DOT; break; - case ':': t.kind = TOK_COLON; break; - case ';': t.kind = TOK_SEMICOLON; break; - case '?': t.kind = TOK_QUESTION; break; - case '#': t.kind = TOK_HASH; break; - case '+': t.kind = lex_repeat_or_assign(lex, '+', TOK_PLUS, TOK_INC, TOK_PLUS_ASSIGN); break; - case '-': { - if (lex_match(lex, '-')) t.kind = TOK_DEC; - else if (lex_match(lex, '=')) t.kind = TOK_MINUS_ASSIGN; - else if (lex_match(lex, '>')) t.kind = TOK_ARROW; - else t.kind = TOK_MINUS; - } break; - case '*': t.kind = lex_assign_variant(lex, TOK_STAR, TOK_MUL_ASSIGN); break; - case '/': t.kind = lex_assign_variant(lex, TOK_SLASH, TOK_DIV_ASSIGN); break; - case '%': t.kind = lex_assign_variant(lex, TOK_PERCENT, TOK_MOD_ASSIGN); break; - case '=': t.kind = lex_assign_variant(lex, TOK_ASSIGN, TOK_EQ); break; - case '<': t.kind = lex_shift_family(lex, '<', TOK_LT, TOK_LEQ, TOK_LSHIFT, TOK_LSHIFT_ASSIGN); break; - case '>': t.kind = lex_shift_family(lex, '>', TOK_GT, TOK_GEQ, TOK_RSHIFT, TOK_RSHIFT_ASSIGN); break; - case '!': t.kind = lex_assign_variant(lex, TOK_NOT, TOK_NEQ); break; - case '~': t.kind = TOK_BITNOT; break; - case '&': t.kind = lex_repeat_or_assign(lex, '&', TOK_BITAND, TOK_AND, TOK_AND_ASSIGN); break; - case '|': t.kind = lex_repeat_or_assign(lex, '|', TOK_BITOR, TOK_OR, TOK_OR_ASSIGN); break; - case '^': t.kind = lex_assign_variant(lex, TOK_BITXOR, TOK_XOR_ASSIGN); break; - default: { - // @todo: lexer perhaps should have a static buffer of size 1024, error message - // should be put there and piped to the upper program. The token should be filled - // with that message - t.kind = TOK_ERROR; - } - } - - t.len = (int)(lex->at - t.str); - return t; -} - -void assert_token(Token t, Token_Kind kind, char *text, int line, int column) { - assert(t.kind == kind); - assert(t.line == line); - assert(t.column == column); - assert(t.len == (int)strlen(text)); - assert(strncmp(t.str, text, t.len) == 0); -} - -Token_Array lex_file(char *file, char *src, int len) { - Lexer lex = make_lexer(file, src, len); - Token_Array result = {0}; - for (;;) { - Token token = lex_token(&lex); - vec_push(&result, token); - if (token.kind == TOK_EOF) { - break; - } - } - return result; -} +#include "lex.c" typedef struct Parser { Token *at; @@ -256,7 +57,7 @@ Token *expect_token(Parser *p, Token_Kind kind) { if (p->at->kind == kind) { return next_token(p); } - panicf("expected token kind: %s, got instead: %s", token_to_name[p->at->kind], token_to_name[kind]); + panicf("expected token kind: %s, got instead: %s", token_to_name(p->at->kind), token_to_name(kind)); } Ast *create_ast(Token *token, Ast_Kind kind) { @@ -287,7 +88,7 @@ Ast *parse_atom(Parser *p) { n = parse_expr(p, 0); expect_token(p, TOK_RPAREN); } else { - panicf("unknown token in %s. %.*s (%s/%d), ", __FUNCTION__, token->len, token->str, token_to_name[token->kind], token->kind); + panicf("unknown token in %s. %.*s (%s/%d), ", __FUNCTION__, token->len, token->str, token_to_name(token->kind), token->kind); } return n; } @@ -315,7 +116,7 @@ Ast *parse_valid_left_binding(Parser *p, Token *tok, Ast *left) { case TOK_BITOR: case TOK_BITXOR: case TOK_AND: case TOK_OR: case TOK_LSHIFT: case TOK_RSHIFT: { return create_binary_expr(tok, tok->kind, left, parse_expr(p, get_binding_power(tok))); } break; - default: panicf("unknown token in %s. %.*s (%s/%d), ", __FUNCTION__, tok->len, tok->str, token_to_name[tok->kind], tok->kind); + default: panicf("unknown token in %s. %.*s (%s/%d), ", __FUNCTION__, tok->len, tok->str, token_to_name(tok->kind), tok->kind); } return NULL; } @@ -366,7 +167,7 @@ void print_expr(Ast *n) { case AST_INT: printf("%lu", n->u); break; case AST_BINARY: { print_expr(n->l); - printf(" %s ", token_to_op[n->op]); + printf(" %s ", token_to_op(n->op)); print_expr(n->r); } break; default: panicf("encountered invalid ast kind in %s of kind: %d\n", __FUNCTION__, n->kind); @@ -415,18 +216,17 @@ void parser_test(void) { TEST_EVAL((2+3)*(4+5)); TEST_EVAL(9>3&1); TEST_EVAL(8|1<4); - - // TEST_EVAL(7<=3+4); - // TEST_EVAL(8>=2*4); - // TEST_EVAL(4==2+2); - // TEST_EVAL(5!=2+2); - // TEST_EVAL(1&&2); - // TEST_EVAL(0||3); - // TEST_EVAL(1||0&&0); - // TEST_EVAL(8<<2); - // TEST_EVAL(32>>3); - // TEST_EVAL(1+2<<3); - // TEST_EVAL(16>>1+1); + TEST_EVAL(7<=3+4); + TEST_EVAL(8>=2*4); + TEST_EVAL(4==2+2); + TEST_EVAL(5!=2+2); + TEST_EVAL(1&&2); + TEST_EVAL(0||3); + TEST_EVAL(1||0&&0); + TEST_EVAL(8<<2); + TEST_EVAL(32>>3); + TEST_EVAL(1+2<<3); + TEST_EVAL(16>>1+1); #pragma clang diagnostic pop diff --git a/meta.c b/meta.c index 3cbc900..e32e686 100644 --- a/meta.c +++ b/meta.c @@ -81,19 +81,25 @@ int main() { } printf("} Token_Kind;\n"); - printf("char *token_to_op[] = {\n"); + printf("char *token_to_op(Token_Kind kind) {\n"); + printf(" switch (kind) {\n"); for (int i = 0; i < ilen(kinds); i += 1) { if (kinds[i].serialized_operator) { - printf(" [TOK_%s] = \"%s\",\n", kinds[i].name, kinds[i].serialized_operator); + printf(" case TOK_%s: return \"%s\";\n", kinds[i].name, kinds[i].serialized_operator); } } - printf("};\n"); + printf(" default: return 0;\n"); + printf(" }\n"); + printf("}\n"); - printf("char *token_to_name[] = {\n"); + printf("char *token_to_name(Token_Kind kind) {\n"); + printf(" switch (kind) {\n"); for (int i = 0; i < ilen(kinds); i += 1) { if (kinds[i].name) { - printf(" [TOK_%s] = \"%s\",\n", kinds[i].name, kinds[i].name); + printf(" case TOK_%s: return \"%s\";\n", kinds[i].name, kinds[i].name); } } - printf("};\n"); + printf(" default: return \"\";\n"); + printf(" }\n"); + printf("}\n"); } diff --git a/meta_gen.c b/meta_gen.c index 5ca6a54..4a23997 100644 --- a/meta_gen.c +++ b/meta_gen.c @@ -57,111 +57,117 @@ typedef enum { TOK_LSHIFT, TOK_RSHIFT, } Token_Kind; -char *token_to_op[] = { - [TOK_LPAREN] = "(", - [TOK_RPAREN] = ")", - [TOK_LBRACKET] = "[", - [TOK_RBRACKET] = "]", - [TOK_LBRACE] = "{", - [TOK_RBRACE] = "}", - [TOK_COMMA] = ",", - [TOK_DOT] = ".", - [TOK_ARROW] = "->", - [TOK_ELLIPSIS] = "...", - [TOK_COLON] = ":", - [TOK_SEMICOLON] = ";", - [TOK_QUESTION] = "?", - [TOK_HASH] = "#", - [TOK_HASHHASH] = "##", - [TOK_PLUS] = "+", - [TOK_MINUS] = "-", - [TOK_STAR] = "*", - [TOK_SLASH] = "/", - [TOK_PERCENT] = "%", - [TOK_INC] = "++", - [TOK_DEC] = "--", - [TOK_ASSIGN] = "=", - [TOK_PLUS_ASSIGN] = "+=", - [TOK_MINUS_ASSIGN] = "-=", - [TOK_MUL_ASSIGN] = "*=", - [TOK_DIV_ASSIGN] = "/=", - [TOK_MOD_ASSIGN] = "%=", - [TOK_LSHIFT_ASSIGN] = "<<=", - [TOK_RSHIFT_ASSIGN] = ">>=", - [TOK_AND_ASSIGN] = "&=", - [TOK_XOR_ASSIGN] = "^=", - [TOK_OR_ASSIGN] = "|=", - [TOK_EQ] = "==", - [TOK_NEQ] = "!=", - [TOK_LT] = "<", - [TOK_LEQ] = "<=", - [TOK_GT] = ">", - [TOK_GEQ] = ">=", - [TOK_NOT] = "!", - [TOK_BITNOT] = "~", - [TOK_BITAND] = "&", - [TOK_BITOR] = "|", - [TOK_BITXOR] = "^", - [TOK_AND] = "&&", - [TOK_OR] = "||", - [TOK_LSHIFT] = "<<", - [TOK_RSHIFT] = ">>", -}; -char *token_to_name[] = { - [TOK_EOF] = "EOF", - [TOK_ERROR] = "ERROR", - [TOK_IDENT] = "IDENT", - [TOK_KEYWORD] = "KEYWORD", - [TOK_INT] = "INT", - [TOK_FLOAT] = "FLOAT", - [TOK_CHAR] = "CHAR", - [TOK_STRING] = "STRING", - [TOK_LPAREN] = "LPAREN", - [TOK_RPAREN] = "RPAREN", - [TOK_LBRACKET] = "LBRACKET", - [TOK_RBRACKET] = "RBRACKET", - [TOK_LBRACE] = "LBRACE", - [TOK_RBRACE] = "RBRACE", - [TOK_COMMA] = "COMMA", - [TOK_DOT] = "DOT", - [TOK_ARROW] = "ARROW", - [TOK_ELLIPSIS] = "ELLIPSIS", - [TOK_COLON] = "COLON", - [TOK_SEMICOLON] = "SEMICOLON", - [TOK_QUESTION] = "QUESTION", - [TOK_HASH] = "HASH", - [TOK_HASHHASH] = "HASHHASH", - [TOK_PLUS] = "PLUS", - [TOK_MINUS] = "MINUS", - [TOK_STAR] = "STAR", - [TOK_SLASH] = "SLASH", - [TOK_PERCENT] = "PERCENT", - [TOK_INC] = "INC", - [TOK_DEC] = "DEC", - [TOK_ASSIGN] = "ASSIGN", - [TOK_PLUS_ASSIGN] = "PLUS_ASSIGN", - [TOK_MINUS_ASSIGN] = "MINUS_ASSIGN", - [TOK_MUL_ASSIGN] = "MUL_ASSIGN", - [TOK_DIV_ASSIGN] = "DIV_ASSIGN", - [TOK_MOD_ASSIGN] = "MOD_ASSIGN", - [TOK_LSHIFT_ASSIGN] = "LSHIFT_ASSIGN", - [TOK_RSHIFT_ASSIGN] = "RSHIFT_ASSIGN", - [TOK_AND_ASSIGN] = "AND_ASSIGN", - [TOK_XOR_ASSIGN] = "XOR_ASSIGN", - [TOK_OR_ASSIGN] = "OR_ASSIGN", - [TOK_EQ] = "EQ", - [TOK_NEQ] = "NEQ", - [TOK_LT] = "LT", - [TOK_LEQ] = "LEQ", - [TOK_GT] = "GT", - [TOK_GEQ] = "GEQ", - [TOK_NOT] = "NOT", - [TOK_BITNOT] = "BITNOT", - [TOK_BITAND] = "BITAND", - [TOK_BITOR] = "BITOR", - [TOK_BITXOR] = "BITXOR", - [TOK_AND] = "AND", - [TOK_OR] = "OR", - [TOK_LSHIFT] = "LSHIFT", - [TOK_RSHIFT] = "RSHIFT", -}; +char *token_to_op(Token_Kind kind) { + switch (kind) { + case TOK_LPAREN: return "("; + case TOK_RPAREN: return ")"; + case TOK_LBRACKET: return "["; + case TOK_RBRACKET: return "]"; + case TOK_LBRACE: return "{"; + case TOK_RBRACE: return "}"; + case TOK_COMMA: return ","; + case TOK_DOT: return "."; + case TOK_ARROW: return "->"; + case TOK_ELLIPSIS: return "..."; + case TOK_COLON: return ":"; + case TOK_SEMICOLON: return ";"; + case TOK_QUESTION: return "?"; + case TOK_HASH: return "#"; + case TOK_HASHHASH: return "##"; + case TOK_PLUS: return "+"; + case TOK_MINUS: return "-"; + case TOK_STAR: return "*"; + case TOK_SLASH: return "/"; + case TOK_PERCENT: return "%"; + case TOK_INC: return "++"; + case TOK_DEC: return "--"; + case TOK_ASSIGN: return "="; + case TOK_PLUS_ASSIGN: return "+="; + case TOK_MINUS_ASSIGN: return "-="; + case TOK_MUL_ASSIGN: return "*="; + case TOK_DIV_ASSIGN: return "/="; + case TOK_MOD_ASSIGN: return "%="; + case TOK_LSHIFT_ASSIGN: return "<<="; + case TOK_RSHIFT_ASSIGN: return ">>="; + case TOK_AND_ASSIGN: return "&="; + case TOK_XOR_ASSIGN: return "^="; + case TOK_OR_ASSIGN: return "|="; + case TOK_EQ: return "=="; + case TOK_NEQ: return "!="; + case TOK_LT: return "<"; + case TOK_LEQ: return "<="; + case TOK_GT: return ">"; + case TOK_GEQ: return ">="; + case TOK_NOT: return "!"; + case TOK_BITNOT: return "~"; + case TOK_BITAND: return "&"; + case TOK_BITOR: return "|"; + case TOK_BITXOR: return "^"; + case TOK_AND: return "&&"; + case TOK_OR: return "||"; + case TOK_LSHIFT: return "<<"; + case TOK_RSHIFT: return ">>"; + default: return 0; + } +} +char *token_to_name(Token_Kind kind) { + switch (kind) { + case TOK_EOF: return "EOF"; + case TOK_ERROR: return "ERROR"; + case TOK_IDENT: return "IDENT"; + case TOK_KEYWORD: return "KEYWORD"; + case TOK_INT: return "INT"; + case TOK_FLOAT: return "FLOAT"; + case TOK_CHAR: return "CHAR"; + case TOK_STRING: return "STRING"; + case TOK_LPAREN: return "LPAREN"; + case TOK_RPAREN: return "RPAREN"; + case TOK_LBRACKET: return "LBRACKET"; + case TOK_RBRACKET: return "RBRACKET"; + case TOK_LBRACE: return "LBRACE"; + case TOK_RBRACE: return "RBRACE"; + case TOK_COMMA: return "COMMA"; + case TOK_DOT: return "DOT"; + case TOK_ARROW: return "ARROW"; + case TOK_ELLIPSIS: return "ELLIPSIS"; + case TOK_COLON: return "COLON"; + case TOK_SEMICOLON: return "SEMICOLON"; + case TOK_QUESTION: return "QUESTION"; + case TOK_HASH: return "HASH"; + case TOK_HASHHASH: return "HASHHASH"; + case TOK_PLUS: return "PLUS"; + case TOK_MINUS: return "MINUS"; + case TOK_STAR: return "STAR"; + case TOK_SLASH: return "SLASH"; + case TOK_PERCENT: return "PERCENT"; + case TOK_INC: return "INC"; + case TOK_DEC: return "DEC"; + case TOK_ASSIGN: return "ASSIGN"; + case TOK_PLUS_ASSIGN: return "PLUS_ASSIGN"; + case TOK_MINUS_ASSIGN: return "MINUS_ASSIGN"; + case TOK_MUL_ASSIGN: return "MUL_ASSIGN"; + case TOK_DIV_ASSIGN: return "DIV_ASSIGN"; + case TOK_MOD_ASSIGN: return "MOD_ASSIGN"; + case TOK_LSHIFT_ASSIGN: return "LSHIFT_ASSIGN"; + case TOK_RSHIFT_ASSIGN: return "RSHIFT_ASSIGN"; + case TOK_AND_ASSIGN: return "AND_ASSIGN"; + case TOK_XOR_ASSIGN: return "XOR_ASSIGN"; + case TOK_OR_ASSIGN: return "OR_ASSIGN"; + case TOK_EQ: return "EQ"; + case TOK_NEQ: return "NEQ"; + case TOK_LT: return "LT"; + case TOK_LEQ: return "LEQ"; + case TOK_GT: return "GT"; + case TOK_GEQ: return "GEQ"; + case TOK_NOT: return "NOT"; + case TOK_BITNOT: return "BITNOT"; + case TOK_BITAND: return "BITAND"; + case TOK_BITOR: return "BITOR"; + case TOK_BITXOR: return "BITXOR"; + case TOK_AND: return "AND"; + case TOK_OR: return "OR"; + case TOK_LSHIFT: return "LSHIFT"; + case TOK_RSHIFT: return "RSHIFT"; + default: return ""; + } +}