krzosa/corelang
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Starting from scratch on smaller scale, typechecking global and constant variables, compound expressions for arrays

Browse Source
This commit is contained in:
Krzosa Karol
2022-05-20 10:54:20 +02:00
parent d993623a50
commit 30736d37b6
12 changed files with 709 additions and 1307 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
new_lex.cpp
View File

@@ -51,7 +51,6 @@ enum Token_Kind{
TK_Colon,
TK_Assign,
TK_ColonAssign,
TK_DivAssign,
TK_MulAssign,
TK_ModAssign,
@@ -112,6 +111,7 @@ struct Lex_Stream{
U8 *line_begin;
String file;
S32 line;
S32 inside_brace_paren;
};
struct Lexer{
@@ -169,7 +169,7 @@ lex_set_len(Lex_Stream *s, Token *token){
function void
lex_set_keywords(Lexer *lexer, Array<String> keywords){
Intern_String keyword = {};
IFor(keywords){
Iter(keywords){
keyword = intern_string(&lexer->interns, *it);
if(it == keywords.begin())
lexer->interns.first_keyword = keyword.str;
@@ -272,14 +272,14 @@ lex__stream(Intern_Table *table, Array<Token> *array, Lex_Stream *s){
lex_advance(s);
switch(*t.str){
case 0: break;
case 0 : break;
case '@': t.kind = TK_At; break;
case '(': t.kind = TK_OpenParen; break;
case ')': t.kind = TK_CloseParen; break;
case '{': t.kind = TK_OpenBrace; break;
case '}': t.kind = TK_CloseBrace; break;
case '[': t.kind = TK_OpenBracket; break;
case ']': t.kind = TK_CloseBracket; break;
case '(': s->inside_brace_paren++; t.kind = TK_OpenParen; break;
case ')': s->inside_brace_paren--; t.kind = TK_CloseParen; break;
case '{': s->inside_brace_paren++; t.kind = TK_OpenBrace; break;
case '}': s->inside_brace_paren--; t.kind = TK_CloseBrace; break;
case '[': s->inside_brace_paren++; t.kind = TK_OpenBracket; break;
case ']': s->inside_brace_paren--; t.kind = TK_CloseBracket; break;
case ',': t.kind = TK_Comma; break;
case '~': t.kind = TK_Neg; break;
case '?': t.kind = TK_Question; break;
@@ -371,10 +371,6 @@ lex__stream(Intern_Table *table, Array<Token> *array, Lex_Stream *s){
lex_advance(s);
t.kind = TK_DoubleColon;
}
else if(lexc(s) == '='){
lex_advance(s);
t.kind = TK_ColonAssign;
}
else {
t.kind = TK_Colon;
}
@@ -483,7 +479,10 @@ lex__stream(Intern_Table *table, Array<Token> *array, Lex_Stream *s){
if(t.len==0)
lex_set_len(s,&t);
array->add(t);
B32 skip = t.kind == TK_NewLine && s->inside_brace_paren > 0;
if(!skip){
array->add(t);
}
while(lex_is_whitespace(lexc(s)))
lex_advance(s);
@@ -494,15 +493,15 @@ lex__stream(Intern_Table *table, Array<Token> *array, Lex_Stream *s){
}
function void
lex_init(Lexer *l){
l->tokens = array_make<Token>(1024*2);
l->interns= intern_table_make(1024);
lex_init(Allocator *token_string_arena, Allocator *map_allocator, Lexer *l){
l->tokens = array_make<Token>(token_string_arena, 1024*2);
l->interns= intern_table_make(token_string_arena, map_allocator, 1024);
}
function Lexer
lex_make(){
lex_make(Allocator *token_string_arena, Allocator *map_allocator){
Lexer result = {};
lex_init(&result);
lex_init(token_string_arena, map_allocator, &result);
return result;
}
@@ -515,29 +514,30 @@ lex_restream(Lexer *lexer, String istream, String file){
}
function Lexer
lex_stream(String istream, String file){
Lexer result = lex_make();
lex_stream(Allocator *token_string_arena, Allocator *map_allocator, String istream, String file){
Lexer result = lex_make(token_string_arena, map_allocator);
lex_restream(&result, istream, file);
return result;
}
function void
lex_test(){ Set_Scratch();
lex_test(){
Scratch scratch;
String test = "Keyword //R\n 18446744073709551616{})(@?&+-;....->,:::/**/\"Thing\" Thingy"
"\"Test_Meme\"+=-===42524 4294967295 18446744073709551615"
"for if while switch :="_s;
Array<String> keywords = {};
Array<String> keywords = {scratch};
keywords.add("Keyword"_s);
keywords.add("for"_s);
keywords.add("if"_s);
keywords.add("while"_s);
keywords.add("switch"_s);
Lexer lexer = lex_make();
Lexer lexer = lex_make(scratch, scratch);
lex_set_keywords(&lexer, keywords);
lex_restream(&lexer, test, "Test1"_s);
Array<Token> array = lexer.tokens;
Array<Token> arr = lexer.tokens;
Token_Kind kind[] = {
TK_Keyword, TK_NewLine, TK_Error,TK_OpenBrace,TK_CloseBrace,TK_CloseParen,TK_OpenParen,
@@ -546,7 +546,7 @@ lex_test(){ Set_Scratch();
TK_StringLit, TK_Identifier, TK_StringLit, TK_AddAssign, TK_SubAssign,
TK_Equals, TK_Integer, TK_Integer, TK_Integer,
TK_Keyword, TK_Keyword, TK_Keyword, TK_Keyword,
TK_ColonAssign, TK_End
TK_Colon, TK_Assign, TK_End
};
String strs[] = {
"Keyword"_s, "\n "_s, "18446744073709551616"_s,"{"_s,"}"_s,")"_s,"("_s,
@@ -554,18 +554,18 @@ lex_test(){ Set_Scratch();
"..."_s,"."_s,"->"_s,","_s,"::"_s,":"_s,
"Thing"_s,"Thingy"_s,"Test_Meme"_s, "+="_s,"-="_s,
"=="_s,"42524"_s,"4294967295"_s,"18446744073709551615"_s,
"for"_s, "if"_s, "while"_s, "switch"_s, ":="_s, ""_s,
"for"_s, "if"_s, "while"_s, "switch"_s, ":"_s, "="_s, ""_s,
};
U64 vals[] = {
42524, 4294967295, 18446744073709551615llu
};
int ui = 0;
For(array, t, i){
assert(t->kind == kind[i]);
assert(string_compare(t->string, strs[i]));
if(t->kind == TK_Integer){
assert(t->int_val == vals[ui++]);
For(arr){
assert(it->kind == kind[i]);
assert(string_compare(it->string, strs[i]));
if(it->kind == TK_Integer){
assert(it->int_val == vals[ui++]);
}
}
}
@@ -603,7 +603,6 @@ token_kind_string(Token_Kind kind){
case TK_GreaterThen: return ">"_s;
case TK_Colon: return ":"_s;
case TK_Assign: return "="_s;
case TK_ColonAssign: return ":="_s;
case TK_DivAssign: return "/="_s;
case TK_MulAssign: return "*="_s;
case TK_ModAssign: return "%="_s;
@@ -629,6 +628,7 @@ token_kind_string(Token_Kind kind){
case TK_LeftShift: return "<<"_s;
case TK_RightShift: return ">>"_s;
case TK_Arrow: return "->"_s;
case TK_NewLine: return "NewLine"_s;
case TK_ExprSizeof: return "sizeof"_s;
case TK_DocComment: return "DocComment"_s;
case TK_Comment: return "Comment"_s;