krzosa/smallprojects
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Extract lexer and switch token metadata to helpers

Browse Source
This commit is contained in:
Krzosa Karol
2026-05-12 08:42:11 +02:00
parent df7f1f27a1
commit 4cffa1f69a
4 changed files with 336 additions and 330 deletions
Download Patch File Download Diff File Expand all files Collapse all files

194
lex.c Normal file
View File

@@ -0,0 +1,194 @@
typedef struct Token {
Token_Kind kind;
char *str;
int len;
char *file;
int line, column;
union {
uint64_t u;
};
} Token;
typedef Vec(Token) Token_Array;
typedef struct Lexer {
char *at;
char *end;
char *file;
int line;
int column;
} Lexer;
void lex_advance(Lexer *lex) {
if (lex->at >= lex->end) {
return;
}
if (*lex->at == '\n') {
lex->line++;
lex->column = 0;
} else {
lex->column++;
}
lex->at += 1;
}
void eat_whitespace(Lexer *lex) {
while (lex->at < lex->end) {
switch (*lex->at) {
case ' ':
case '\t':
case '\r':
case '\n':
lex_advance(lex);
break;
default:
return;
}
}
}
Lexer make_lexer(char *file, char *src, int len) {
Lexer lex = {
.at = src,
.end = src + len,
.file = file,
.line = 0,
.column = 0,
};
return lex;
}
bool lex_peek_is(Lexer *lex, char c) {
return lex->at < lex->end && *lex->at == c;
}
bool lex_match(Lexer *lex, char c) {
if (lex_peek_is(lex, c)) {
lex_advance(lex);
return true;
}
return false;
}
Token_Kind lex_repeat_or_assign(Lexer *lex, char repeated_char, Token_Kind single, Token_Kind repeated, Token_Kind assigned) {
if (lex_match(lex, repeated_char)) return repeated;
if (lex_match(lex, '=')) return assigned;
return single;
}
Token_Kind lex_assign_variant(Lexer *lex, Token_Kind single, Token_Kind assigned) {
return lex_match(lex, '=') ? assigned : single;
}
Token_Kind lex_shift_family(Lexer *lex, char repeated_char, Token_Kind single, Token_Kind single_eq, Token_Kind doubled, Token_Kind doubled_eq) {
if (lex_match(lex, repeated_char)) {
return lex_match(lex, '=') ? doubled_eq : doubled;
}
return lex_match(lex, '=') ? single_eq : single;
}
Token lex_token(Lexer *lex) {
eat_whitespace(lex);
Token t = {
.str = lex->at,
.line = lex->line,
.column = lex->column,
.file = lex->file,
};
if (lex->at >= lex->end) {
t.kind = TOK_EOF;
t.len = 0;
return t;
}
char c = *lex->at;
if (isdigit(c)) {
t.kind = TOK_INT;
while (lex->at < lex->end && isdigit(*lex->at)) {
lex_advance(lex);
}
// @todo: proper lexing of floats (as well as postfixes)
if (lex->at < lex->end && *lex->at == '.') {
t.kind = TOK_FLOAT;
lex_advance(lex);
while (lex->at < lex->end && isdigit(*lex->at)) {
lex_advance(lex);
}
}
if (t.kind == TOK_INT) {
t.u = strtoull(t.str, NULL, 10);
}
t.len = (int)(lex->at - t.str);
return t;
}
lex_advance(lex);
switch (c) {
case 0: t.kind = TOK_EOF; break;
case '(': t.kind = TOK_LPAREN; break;
case ')': t.kind = TOK_RPAREN; break;
case '[': t.kind = TOK_LBRACKET; break;
case ']': t.kind = TOK_RBRACKET; break;
case '{': t.kind = TOK_LBRACE; break;
case '}': t.kind = TOK_RBRACE; break;
case ',': t.kind = TOK_COMMA; break;
case '.': t.kind = TOK_DOT; break;
case ':': t.kind = TOK_COLON; break;
case ';': t.kind = TOK_SEMICOLON; break;
case '?': t.kind = TOK_QUESTION; break;
case '#': t.kind = TOK_HASH; break;
case '+': t.kind = lex_repeat_or_assign(lex, '+', TOK_PLUS, TOK_INC, TOK_PLUS_ASSIGN); break;
case '-': {
if (lex_match(lex, '-')) t.kind = TOK_DEC;
else if (lex_match(lex, '=')) t.kind = TOK_MINUS_ASSIGN;
else if (lex_match(lex, '>')) t.kind = TOK_ARROW;
else t.kind = TOK_MINUS;
} break;
case '*': t.kind = lex_assign_variant(lex, TOK_STAR, TOK_MUL_ASSIGN); break;
case '/': t.kind = lex_assign_variant(lex, TOK_SLASH, TOK_DIV_ASSIGN); break;
case '%': t.kind = lex_assign_variant(lex, TOK_PERCENT, TOK_MOD_ASSIGN); break;
case '=': t.kind = lex_assign_variant(lex, TOK_ASSIGN, TOK_EQ); break;
case '<': t.kind = lex_shift_family(lex, '<', TOK_LT, TOK_LEQ, TOK_LSHIFT, TOK_LSHIFT_ASSIGN); break;
case '>': t.kind = lex_shift_family(lex, '>', TOK_GT, TOK_GEQ, TOK_RSHIFT, TOK_RSHIFT_ASSIGN); break;
case '!': t.kind = lex_assign_variant(lex, TOK_NOT, TOK_NEQ); break;
case '~': t.kind = TOK_BITNOT; break;
case '&': t.kind = lex_repeat_or_assign(lex, '&', TOK_BITAND, TOK_AND, TOK_AND_ASSIGN); break;
case '|': t.kind = lex_repeat_or_assign(lex, '|', TOK_BITOR, TOK_OR, TOK_OR_ASSIGN); break;
case '^': t.kind = lex_assign_variant(lex, TOK_BITXOR, TOK_XOR_ASSIGN); break;
default: panicf("unrecognized character: '%c', can't match with any of the token kinds", c);
}
t.len = (int)(lex->at - t.str);
return t;
}
void assert_token(Token t, Token_Kind kind, char *text, int line, int column) {
assert(t.kind == kind);
assert(t.line == line);
assert(t.column == column);
assert(t.len == (int)strlen(text));
assert(strncmp(t.str, text, t.len) == 0);
}
Token_Array lex_file(char *file, char *src, int len) {
Lexer lex = make_lexer(file, src, len);
Token_Array result = {0};
for (;;) {
Token token = lex_token(&lex);
vec_push(&result, token);
if (token.kind == TOK_EOF) {
break;
}
}
return result;
}

232
main.c
View File

@@ -8,206 +8,7 @@
#include <stdarg.h> #include <stdarg.h>
#include "base.c" #include "base.c"
#include "meta_gen.c" #include "meta_gen.c"
#include "lex.c"
typedef struct Token {
Token_Kind kind;
char *str;
int len;
char *file;
int line, column;
union {
uint64_t u;
};
} Token;
typedef Vec(Token) Token_Array;
typedef struct Lexer {
char *at;
char *end;
char *file;
int line;
int column;
} Lexer;
void lex_advance(Lexer *lex) {
if (lex->at >= lex->end) {
return;
}
if (*lex->at == '\n') {
lex->line++;
lex->column = 0;
} else {
lex->column++;
}
lex->at += 1;
}
void eat_whitespace(Lexer *lex) {
while (lex->at < lex->end) {
switch (*lex->at) {
case ' ':
case '\t':
case '\r':
case '\n':
lex_advance(lex);
break;
default:
return;
}
}
}
Lexer make_lexer(char *file, char *src, int len) {
Lexer lex = {
.at = src,
.end = src + len,
.file = file,
.line = 0,
.column = 0,
};
return lex;
}
bool lex_peek_is(Lexer *lex, char c) {
return lex->at < lex->end && *lex->at == c;
}
bool lex_match(Lexer *lex, char c) {
if (lex_peek_is(lex, c)) {
lex_advance(lex);
return true;
}
return false;
}
Token_Kind lex_repeat_or_assign(Lexer *lex, char repeated_char, Token_Kind single, Token_Kind repeated, Token_Kind assigned) {
if (lex_match(lex, repeated_char)) return repeated;
if (lex_match(lex, '=')) return assigned;
return single;
}
Token_Kind lex_assign_variant(Lexer *lex, Token_Kind single, Token_Kind assigned) {
return lex_match(lex, '=') ? assigned : single;
}
Token_Kind lex_shift_family(Lexer *lex, char repeated_char, Token_Kind single, Token_Kind single_eq, Token_Kind doubled, Token_Kind doubled_eq) {
if (lex_match(lex, repeated_char)) {
return lex_match(lex, '=') ? doubled_eq : doubled;
}
return lex_match(lex, '=') ? single_eq : single;
}
Token lex_token(Lexer *lex) {
eat_whitespace(lex);
Token t = {
.str = lex->at,
.line = lex->line,
.column = lex->column,
.file = lex->file,
};
if (lex->at >= lex->end) {
t.kind = TOK_EOF;
t.len = 0;
return t;
}
char c = *lex->at;
if (isdigit(c)) {
t.kind = TOK_INT;
while (lex->at < lex->end && isdigit(*lex->at)) {
lex_advance(lex);
}
// @todo: proper lexing of floats (as well as postfixes)
if (lex->at < lex->end && *lex->at == '.') {
t.kind = TOK_FLOAT;
lex_advance(lex);
while (lex->at < lex->end && isdigit(*lex->at)) {
lex_advance(lex);
}
}
if (t.kind == TOK_INT) {
t.u = strtoull(t.str, NULL, 10);
}
t.len = (int)(lex->at - t.str);
return t;
}
lex_advance(lex);
switch (c) {
case 0: t.kind = TOK_EOF; break;
case '(': t.kind = TOK_LPAREN; break;
case ')': t.kind = TOK_RPAREN; break;
case '[': t.kind = TOK_LBRACKET; break;
case ']': t.kind = TOK_RBRACKET; break;
case '{': t.kind = TOK_LBRACE; break;
case '}': t.kind = TOK_RBRACE; break;
case ',': t.kind = TOK_COMMA; break;
case '.': t.kind = TOK_DOT; break;
case ':': t.kind = TOK_COLON; break;
case ';': t.kind = TOK_SEMICOLON; break;
case '?': t.kind = TOK_QUESTION; break;
case '#': t.kind = TOK_HASH; break;
case '+': t.kind = lex_repeat_or_assign(lex, '+', TOK_PLUS, TOK_INC, TOK_PLUS_ASSIGN); break;
case '-': {
if (lex_match(lex, '-')) t.kind = TOK_DEC;
else if (lex_match(lex, '=')) t.kind = TOK_MINUS_ASSIGN;
else if (lex_match(lex, '>')) t.kind = TOK_ARROW;
else t.kind = TOK_MINUS;
} break;
case '*': t.kind = lex_assign_variant(lex, TOK_STAR, TOK_MUL_ASSIGN); break;
case '/': t.kind = lex_assign_variant(lex, TOK_SLASH, TOK_DIV_ASSIGN); break;
case '%': t.kind = lex_assign_variant(lex, TOK_PERCENT, TOK_MOD_ASSIGN); break;
case '=': t.kind = lex_assign_variant(lex, TOK_ASSIGN, TOK_EQ); break;
case '<': t.kind = lex_shift_family(lex, '<', TOK_LT, TOK_LEQ, TOK_LSHIFT, TOK_LSHIFT_ASSIGN); break;
case '>': t.kind = lex_shift_family(lex, '>', TOK_GT, TOK_GEQ, TOK_RSHIFT, TOK_RSHIFT_ASSIGN); break;
case '!': t.kind = lex_assign_variant(lex, TOK_NOT, TOK_NEQ); break;
case '~': t.kind = TOK_BITNOT; break;
case '&': t.kind = lex_repeat_or_assign(lex, '&', TOK_BITAND, TOK_AND, TOK_AND_ASSIGN); break;
case '|': t.kind = lex_repeat_or_assign(lex, '|', TOK_BITOR, TOK_OR, TOK_OR_ASSIGN); break;
case '^': t.kind = lex_assign_variant(lex, TOK_BITXOR, TOK_XOR_ASSIGN); break;
default: {
// @todo: lexer perhaps should have a static buffer of size 1024, error message
// should be put there and piped to the upper program. The token should be filled
// with that message
t.kind = TOK_ERROR;
}
}
t.len = (int)(lex->at - t.str);
return t;
}
void assert_token(Token t, Token_Kind kind, char *text, int line, int column) {
assert(t.kind == kind);
assert(t.line == line);
assert(t.column == column);
assert(t.len == (int)strlen(text));
assert(strncmp(t.str, text, t.len) == 0);
}
Token_Array lex_file(char *file, char *src, int len) {
Lexer lex = make_lexer(file, src, len);
Token_Array result = {0};
for (;;) {
Token token = lex_token(&lex);
vec_push(&result, token);
if (token.kind == TOK_EOF) {
break;
}
}
return result;
}
typedef struct Parser { typedef struct Parser {
Token *at; Token *at;
@@ -256,7 +57,7 @@ Token *expect_token(Parser *p, Token_Kind kind) {
if (p->at->kind == kind) { if (p->at->kind == kind) {
return next_token(p); return next_token(p);
} }
panicf("expected token kind: %s, got instead: %s", token_to_name[p->at->kind], token_to_name[kind]); panicf("expected token kind: %s, got instead: %s", token_to_name(p->at->kind), token_to_name(kind));
} }
Ast *create_ast(Token *token, Ast_Kind kind) { Ast *create_ast(Token *token, Ast_Kind kind) {
@@ -287,7 +88,7 @@ Ast *parse_atom(Parser *p) {
n = parse_expr(p, 0); n = parse_expr(p, 0);
expect_token(p, TOK_RPAREN); expect_token(p, TOK_RPAREN);
} else { } else {
panicf("unknown token in %s. %.*s (%s/%d), ", __FUNCTION__, token->len, token->str, token_to_name[token->kind], token->kind); panicf("unknown token in %s. %.*s (%s/%d), ", __FUNCTION__, token->len, token->str, token_to_name(token->kind), token->kind);
} }
return n; return n;
} }
@@ -315,7 +116,7 @@ Ast *parse_valid_left_binding(Parser *p, Token *tok, Ast *left) {
case TOK_BITOR: case TOK_BITXOR: case TOK_AND: case TOK_OR: case TOK_LSHIFT: case TOK_RSHIFT: { case TOK_BITOR: case TOK_BITXOR: case TOK_AND: case TOK_OR: case TOK_LSHIFT: case TOK_RSHIFT: {
return create_binary_expr(tok, tok->kind, left, parse_expr(p, get_binding_power(tok))); return create_binary_expr(tok, tok->kind, left, parse_expr(p, get_binding_power(tok)));
} break; } break;
default: panicf("unknown token in %s. %.*s (%s/%d), ", __FUNCTION__, tok->len, tok->str, token_to_name[tok->kind], tok->kind); default: panicf("unknown token in %s. %.*s (%s/%d), ", __FUNCTION__, tok->len, tok->str, token_to_name(tok->kind), tok->kind);
} }
return NULL; return NULL;
} }
@@ -366,7 +167,7 @@ void print_expr(Ast *n) {
case AST_INT: printf("%lu", n->u); break; case AST_INT: printf("%lu", n->u); break;
case AST_BINARY: { case AST_BINARY: {
print_expr(n->l); print_expr(n->l);
printf(" %s ", token_to_op[n->op]); printf(" %s ", token_to_op(n->op));
print_expr(n->r); print_expr(n->r);
} break; } break;
default: panicf("encountered invalid ast kind in %s of kind: %d\n", __FUNCTION__, n->kind); default: panicf("encountered invalid ast kind in %s of kind: %d\n", __FUNCTION__, n->kind);
@@ -415,18 +216,17 @@ void parser_test(void) {
TEST_EVAL((2+3)*(4+5)); TEST_EVAL((2+3)*(4+5));
TEST_EVAL(9>3&1); TEST_EVAL(9>3&1);
TEST_EVAL(8|1<4); TEST_EVAL(8|1<4);
TEST_EVAL(7<=3+4);
// TEST_EVAL(7<=3+4); TEST_EVAL(8>=2*4);
// TEST_EVAL(8>=2*4); TEST_EVAL(4==2+2);
// TEST_EVAL(4==2+2); TEST_EVAL(5!=2+2);
// TEST_EVAL(5!=2+2); TEST_EVAL(1&&2);
// TEST_EVAL(1&&2); TEST_EVAL(0||3);
// TEST_EVAL(0||3); TEST_EVAL(1||0&&0);
// TEST_EVAL(1||0&&0); TEST_EVAL(8<<2);
// TEST_EVAL(8<<2); TEST_EVAL(32>>3);
// TEST_EVAL(32>>3); TEST_EVAL(1+2<<3);
// TEST_EVAL(1+2<<3); TEST_EVAL(16>>1+1);
// TEST_EVAL(16>>1+1);
#pragma clang diagnostic pop #pragma clang diagnostic pop

18
meta.c
View File

@@ -81,19 +81,25 @@ int main() {
} }
printf("} Token_Kind;\n"); printf("} Token_Kind;\n");
printf("char *token_to_op[] = {\n"); printf("char *token_to_op(Token_Kind kind) {\n");
printf(" switch (kind) {\n");
for (int i = 0; i < ilen(kinds); i += 1) { for (int i = 0; i < ilen(kinds); i += 1) {
if (kinds[i].serialized_operator) { if (kinds[i].serialized_operator) {
printf(" [TOK_%s] = \"%s\",\n", kinds[i].name, kinds[i].serialized_operator); printf(" case TOK_%s: return \"%s\";\n", kinds[i].name, kinds[i].serialized_operator);
} }
} }
printf("};\n"); printf(" default: return 0;\n");
printf(" }\n");
printf("}\n");
printf("char *token_to_name[] = {\n"); printf("char *token_to_name(Token_Kind kind) {\n");
printf(" switch (kind) {\n");
for (int i = 0; i < ilen(kinds); i += 1) { for (int i = 0; i < ilen(kinds); i += 1) {
if (kinds[i].name) { if (kinds[i].name) {
printf(" [TOK_%s] = \"%s\",\n", kinds[i].name, kinds[i].name); printf(" case TOK_%s: return \"%s\";\n", kinds[i].name, kinds[i].name);
} }
} }
printf("};\n"); printf(" default: return \"<invalid-token-kind>\";\n");
printf(" }\n");
printf("}\n");
} }

222
meta_gen.c
View File

@@ -57,111 +57,117 @@ typedef enum {
TOK_LSHIFT, TOK_LSHIFT,
TOK_RSHIFT, TOK_RSHIFT,
} Token_Kind; } Token_Kind;
char *token_to_op[] = { char *token_to_op(Token_Kind kind) {
[TOK_LPAREN] = "(", switch (kind) {
[TOK_RPAREN] = ")", case TOK_LPAREN: return "(";
[TOK_LBRACKET] = "[", case TOK_RPAREN: return ")";
[TOK_RBRACKET] = "]", case TOK_LBRACKET: return "[";
[TOK_LBRACE] = "{", case TOK_RBRACKET: return "]";
[TOK_RBRACE] = "}", case TOK_LBRACE: return "{";
[TOK_COMMA] = ",", case TOK_RBRACE: return "}";
[TOK_DOT] = ".", case TOK_COMMA: return ",";
[TOK_ARROW] = "->", case TOK_DOT: return ".";
[TOK_ELLIPSIS] = "...", case TOK_ARROW: return "->";
[TOK_COLON] = ":", case TOK_ELLIPSIS: return "...";
[TOK_SEMICOLON] = ";", case TOK_COLON: return ":";
[TOK_QUESTION] = "?", case TOK_SEMICOLON: return ";";
[TOK_HASH] = "#", case TOK_QUESTION: return "?";
[TOK_HASHHASH] = "##", case TOK_HASH: return "#";
[TOK_PLUS] = "+", case TOK_HASHHASH: return "##";
[TOK_MINUS] = "-", case TOK_PLUS: return "+";
[TOK_STAR] = "*", case TOK_MINUS: return "-";
[TOK_SLASH] = "/", case TOK_STAR: return "*";
[TOK_PERCENT] = "%", case TOK_SLASH: return "/";
[TOK_INC] = "++", case TOK_PERCENT: return "%";
[TOK_DEC] = "--", case TOK_INC: return "++";
[TOK_ASSIGN] = "=", case TOK_DEC: return "--";
[TOK_PLUS_ASSIGN] = "+=", case TOK_ASSIGN: return "=";
[TOK_MINUS_ASSIGN] = "-=", case TOK_PLUS_ASSIGN: return "+=";
[TOK_MUL_ASSIGN] = "*=", case TOK_MINUS_ASSIGN: return "-=";
[TOK_DIV_ASSIGN] = "/=", case TOK_MUL_ASSIGN: return "*=";
[TOK_MOD_ASSIGN] = "%=", case TOK_DIV_ASSIGN: return "/=";
[TOK_LSHIFT_ASSIGN] = "<<=", case TOK_MOD_ASSIGN: return "%=";
[TOK_RSHIFT_ASSIGN] = ">>=", case TOK_LSHIFT_ASSIGN: return "<<=";
[TOK_AND_ASSIGN] = "&=", case TOK_RSHIFT_ASSIGN: return ">>=";
[TOK_XOR_ASSIGN] = "^=", case TOK_AND_ASSIGN: return "&=";
[TOK_OR_ASSIGN] = "|=", case TOK_XOR_ASSIGN: return "^=";
[TOK_EQ] = "==", case TOK_OR_ASSIGN: return "|=";
[TOK_NEQ] = "!=", case TOK_EQ: return "==";
[TOK_LT] = "<", case TOK_NEQ: return "!=";
[TOK_LEQ] = "<=", case TOK_LT: return "<";
[TOK_GT] = ">", case TOK_LEQ: return "<=";
[TOK_GEQ] = ">=", case TOK_GT: return ">";
[TOK_NOT] = "!", case TOK_GEQ: return ">=";
[TOK_BITNOT] = "~", case TOK_NOT: return "!";
[TOK_BITAND] = "&", case TOK_BITNOT: return "~";
[TOK_BITOR] = "|", case TOK_BITAND: return "&";
[TOK_BITXOR] = "^", case TOK_BITOR: return "|";
[TOK_AND] = "&&", case TOK_BITXOR: return "^";
[TOK_OR] = "||", case TOK_AND: return "&&";
[TOK_LSHIFT] = "<<", case TOK_OR: return "||";
[TOK_RSHIFT] = ">>", case TOK_LSHIFT: return "<<";
}; case TOK_RSHIFT: return ">>";
char *token_to_name[] = { default: return 0;
[TOK_EOF] = "EOF", }
[TOK_ERROR] = "ERROR", }
[TOK_IDENT] = "IDENT", char *token_to_name(Token_Kind kind) {
[TOK_KEYWORD] = "KEYWORD", switch (kind) {
[TOK_INT] = "INT", case TOK_EOF: return "EOF";
[TOK_FLOAT] = "FLOAT", case TOK_ERROR: return "ERROR";
[TOK_CHAR] = "CHAR", case TOK_IDENT: return "IDENT";
[TOK_STRING] = "STRING", case TOK_KEYWORD: return "KEYWORD";
[TOK_LPAREN] = "LPAREN", case TOK_INT: return "INT";
[TOK_RPAREN] = "RPAREN", case TOK_FLOAT: return "FLOAT";
[TOK_LBRACKET] = "LBRACKET", case TOK_CHAR: return "CHAR";
[TOK_RBRACKET] = "RBRACKET", case TOK_STRING: return "STRING";
[TOK_LBRACE] = "LBRACE", case TOK_LPAREN: return "LPAREN";
[TOK_RBRACE] = "RBRACE", case TOK_RPAREN: return "RPAREN";
[TOK_COMMA] = "COMMA", case TOK_LBRACKET: return "LBRACKET";
[TOK_DOT] = "DOT", case TOK_RBRACKET: return "RBRACKET";
[TOK_ARROW] = "ARROW", case TOK_LBRACE: return "LBRACE";
[TOK_ELLIPSIS] = "ELLIPSIS", case TOK_RBRACE: return "RBRACE";
[TOK_COLON] = "COLON", case TOK_COMMA: return "COMMA";
[TOK_SEMICOLON] = "SEMICOLON", case TOK_DOT: return "DOT";
[TOK_QUESTION] = "QUESTION", case TOK_ARROW: return "ARROW";
[TOK_HASH] = "HASH", case TOK_ELLIPSIS: return "ELLIPSIS";
[TOK_HASHHASH] = "HASHHASH", case TOK_COLON: return "COLON";
[TOK_PLUS] = "PLUS", case TOK_SEMICOLON: return "SEMICOLON";
[TOK_MINUS] = "MINUS", case TOK_QUESTION: return "QUESTION";
[TOK_STAR] = "STAR", case TOK_HASH: return "HASH";
[TOK_SLASH] = "SLASH", case TOK_HASHHASH: return "HASHHASH";
[TOK_PERCENT] = "PERCENT", case TOK_PLUS: return "PLUS";
[TOK_INC] = "INC", case TOK_MINUS: return "MINUS";
[TOK_DEC] = "DEC", case TOK_STAR: return "STAR";
[TOK_ASSIGN] = "ASSIGN", case TOK_SLASH: return "SLASH";
[TOK_PLUS_ASSIGN] = "PLUS_ASSIGN", case TOK_PERCENT: return "PERCENT";
[TOK_MINUS_ASSIGN] = "MINUS_ASSIGN", case TOK_INC: return "INC";
[TOK_MUL_ASSIGN] = "MUL_ASSIGN", case TOK_DEC: return "DEC";
[TOK_DIV_ASSIGN] = "DIV_ASSIGN", case TOK_ASSIGN: return "ASSIGN";
[TOK_MOD_ASSIGN] = "MOD_ASSIGN", case TOK_PLUS_ASSIGN: return "PLUS_ASSIGN";
[TOK_LSHIFT_ASSIGN] = "LSHIFT_ASSIGN", case TOK_MINUS_ASSIGN: return "MINUS_ASSIGN";
[TOK_RSHIFT_ASSIGN] = "RSHIFT_ASSIGN", case TOK_MUL_ASSIGN: return "MUL_ASSIGN";
[TOK_AND_ASSIGN] = "AND_ASSIGN", case TOK_DIV_ASSIGN: return "DIV_ASSIGN";
[TOK_XOR_ASSIGN] = "XOR_ASSIGN", case TOK_MOD_ASSIGN: return "MOD_ASSIGN";
[TOK_OR_ASSIGN] = "OR_ASSIGN", case TOK_LSHIFT_ASSIGN: return "LSHIFT_ASSIGN";
[TOK_EQ] = "EQ", case TOK_RSHIFT_ASSIGN: return "RSHIFT_ASSIGN";
[TOK_NEQ] = "NEQ", case TOK_AND_ASSIGN: return "AND_ASSIGN";
[TOK_LT] = "LT", case TOK_XOR_ASSIGN: return "XOR_ASSIGN";
[TOK_LEQ] = "LEQ", case TOK_OR_ASSIGN: return "OR_ASSIGN";
[TOK_GT] = "GT", case TOK_EQ: return "EQ";
[TOK_GEQ] = "GEQ", case TOK_NEQ: return "NEQ";
[TOK_NOT] = "NOT", case TOK_LT: return "LT";
[TOK_BITNOT] = "BITNOT", case TOK_LEQ: return "LEQ";
[TOK_BITAND] = "BITAND", case TOK_GT: return "GT";
[TOK_BITOR] = "BITOR", case TOK_GEQ: return "GEQ";
[TOK_BITXOR] = "BITXOR", case TOK_NOT: return "NOT";
[TOK_AND] = "AND", case TOK_BITNOT: return "BITNOT";
[TOK_OR] = "OR", case TOK_BITAND: return "BITAND";
[TOK_LSHIFT] = "LSHIFT", case TOK_BITOR: return "BITOR";
[TOK_RSHIFT] = "RSHIFT", case TOK_BITXOR: return "BITXOR";
}; case TOK_AND: return "AND";
case TOK_OR: return "OR";
case TOK_LSHIFT: return "LSHIFT";
case TOK_RSHIFT: return "RSHIFT";
default: return "<invalid-token-kind>";
}
}