krzosa/corelang
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Cleanup

Browse Source
This commit is contained in:
Krzosa Karol
2022-05-31 22:52:56 +02:00
parent 7ea0dfc7a6
commit 98d2389c9f
5 changed files with 119 additions and 152 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
new_lex.cpp
View File

@@ -51,6 +51,7 @@ enum Token_Kind{
TK_Colon,
TK_Assign,
TK_ColonAssign,
TK_DivAssign,
TK_MulAssign,
TK_ModAssign,
@@ -131,6 +132,8 @@ struct Lexer{
S64 token_iter;
};
force_inline B32 token_is_assign(Token *token){return token->kind >= TK_FirstAssign && token->kind <= TK_LastAssign;}
function U8
lexc(Lex_Stream *s){
return s->stream.str[s->iter];
@@ -507,6 +510,10 @@ lex__stream(Intern_Table *table, Array<Token> *array, Lex_Stream *s){
lex_advance(s);
t.kind = TK_DoubleColon;
}
else if(lexc(s) == '='){
lex_advance(s);
t.kind = TK_ColonAssign;
}
else {
t.kind = TK_Colon;
}
@@ -630,59 +637,6 @@ lex_stream(Allocator *token_string_arena, Allocator *map_allocator, String istre
return result;
}
function void
lex_test(){
Scratch scratch;
String test = "Keyword //R\n 18446744073709551616\n {}\n)(@?&+-....->,:::/**/\"Thing\" Thingy"
"\"Test_Meme\"+=-===42524 4294967295 18446744073709551615"
"for if while switch :="_s;
Array<String> keywords = {scratch};
keywords.add("Keyword"_s);
keywords.add("for"_s);
keywords.add("if"_s);
keywords.add("while"_s);
keywords.add("switch"_s);
Lexer lexer = lex_make(scratch, scratch);
lex_set_keywords(&lexer, keywords);
lex_restream(&lexer, test, "Test1"_s);
Array<Token> arr = lexer.tokens;
Token_Kind kind[] = {
SAME_SCOPE,
TK_Keyword, OPEN_SCOPE, TK_Error, OPEN_SCOPE, TK_OpenBrace,TK_CloseBrace,CLOSE_SCOPE, CLOSE_SCOPE, SAME_SCOPE, TK_CloseParen,TK_OpenParen,
TK_At,TK_Question,TK_BitAnd,TK_Add,TK_Sub,
TK_ThreeDots, TK_Dot, TK_Arrow, TK_Comma, TK_DoubleColon, TK_Colon,
TK_StringLit, TK_Identifier, TK_StringLit, TK_AddAssign, TK_SubAssign,
TK_Equals, TK_Integer, TK_Integer, TK_Integer,
TK_Keyword, TK_Keyword, TK_Keyword, TK_Keyword,
TK_Colon, TK_Assign, TK_End
};
String strs[] = {
""_s, "Keyword"_s, ""_s, "18446744073709551616"_s, ""_s, "{"_s,"}"_s, ""_s, ""_s, ""_s, ")"_s, "("_s,
"@"_s,"?"_s,"&"_s,"+"_s,"-"_s,
"..."_s,"."_s,"->"_s,","_s,"::"_s,":"_s,
"Thing"_s,"Thingy"_s,"Test_Meme"_s, "+="_s,"-="_s,
"=="_s,"42524"_s,"4294967295"_s,"18446744073709551615"_s,
"for"_s, "if"_s, "while"_s, "switch"_s, ":"_s, "="_s, ""_s,
};
U64 vals[] = {
42524, 4294967295, 18446744073709551615llu
};
int ui = 0;
S32 i = 0;
For(arr){
assert(it.kind == kind[i]);
assert(string_compare(it.string, strs[i]));
if(it.kind == TK_Integer){
assert(it.int_val == vals[ui++]);
}
i++;
}
}
//-----------------------------------------------------------------------------
// Token metadata
//-----------------------------------------------------------------------------
@@ -706,6 +660,7 @@ token_kind_string(Token_Kind kind){
case TK_CloseBrace: return "}"_s;
case TK_OpenBracket: return "["_s;
case TK_CloseBracket: return "]"_s;
case TK_ColonAssign: return ":="_s;
case TK_Comma: return ","_s;
case TK_Pound: return "#"_s;
case TK_Question: return "?"_s;