diff --git a/build.bat b/build.bat index 604904b..c33e84e 100644 --- a/build.bat +++ b/build.bat @@ -1,6 +1,4 @@ @echo off -call ../misc/compile_setup.bat - mkdir build cd build cl -Fe:bld.exe ../build_tool/main.cpp -FC -WX -W3 -wd4200 -diagnostics:column -nologo -Zi -D_CRT_SECURE_NO_WARNINGS diff --git a/build_file.cpp b/build_file.cpp index 54f0ff6..d52d431 100644 --- a/build_file.cpp +++ b/build_file.cpp @@ -21,7 +21,7 @@ void Compile(S8_String cc, S8_String file) { Array flags = Split("/MP /Zi -D_CRT_SECURE_NO_WARNINGS /GF /Gm- /Oi"); flags += Split("/FC /WX /W3 /wd4200 /diagnostics:column /nologo"); flags += Split("/GR- /EHa-"); - flags += Split("-fsanitize=address -RTC1"); + flags += Split("-RTC1"); Array link = Split("/link /incremental:no"); S8_String name = Fmt("/Fe:%.*s", S8_Expand(exe)); @@ -55,6 +55,6 @@ void Compile(S8_String cc, S8_String file) { result = Run(cc + file + name + flags); } - if (result == 0) result = OS_SystemF(IF_WINDOWS_ELSE("", "./") "%.*s.exe", S8_Expand(exe)); + if (result == 0) result = OS_SystemF(IF_WINDOWS_ELSE("", "./") "%.*s", S8_Expand(exe)); else ReturnValue = result; } diff --git a/standalone_libraries/clexer.c b/standalone_libraries/clexer.c index a60c0dc..c4a16dc 100644 --- a/standalone_libraries/clexer.c +++ b/standalone_libraries/clexer.c @@ -615,9 +615,17 @@ CL_PRIVATE_FUNCTION void CL_IsIdentifierKeyword(CL_Token *token) { } } +CL_PRIVATE_FUNCTION void CL_EatMacroWhitespace(CL_Lexer *T) { + while (T->stream[0] == ' ' || T->stream[0] == '\t') CL_Advance(T); +} + +CL_PRIVATE_FUNCTION void CL_EatUntil(CL_Lexer *T, char c) { + while (T->stream[0] != c && T->stream[0] != 0) CL_Advance(T); +} + CL_PRIVATE_FUNCTION void CL_LexMacroInclude(CL_Lexer *T, CL_Token *token) { token->kind = CL_PREPROC_INCLUDE; - while (*T->stream == ' ') CL_Advance(T); + CL_EatMacroWhitespace(T); char end = 0; if (*T->stream == '"') { end = '"'; @@ -650,7 +658,7 @@ CL_PRIVATE_FUNCTION void CL_LexMacroInclude(CL_Lexer *T, CL_Token *token) { } CL_PRIVATE_FUNCTION bool CL_LexMacro(CL_Lexer *T, CL_Token *token) { - while (*T->stream == ' ' || T->stream[0] == '\t') CL_Advance(T); + CL_EatMacroWhitespace(T); token->str = T->stream; while (CL_IsAlphabetic(*T->stream)) CL_Advance(T); CL_SetTokenLength(T, token); @@ -684,6 +692,10 @@ CL_PRIVATE_FUNCTION bool CL_LexMacro(CL_Lexer *T, CL_Token *token) { } else if (CL_StringsAreEqual(token->str, token->len, "error", 5)) { token->kind = CL_PREPROC_ERROR; + CL_EatMacroWhitespace(T); + token->str = T->stream; + CL_EatUntil(T, '\n'); + CL_SetTokenLength(T, token); } else if (CL_StringsAreEqual(token->str, token->len, "else", 4)) { token->kind = CL_PREPROC_ELSE; @@ -750,10 +762,7 @@ CL_PRIVATE_FUNCTION void CL_DefaultTokenize(CL_Lexer *T, CL_Token *token) { if (*T->stream == '/') { token->kind = CL_COMMENT; CL_Advance(T); - - while (*T->stream != '\n' && *T->stream != 0) { - CL_Advance(T); - } + CL_EatUntil(T, '\n'); CL_SetTokenLength(T, token); } else if (*T->stream == '*') { @@ -1480,13 +1489,13 @@ const char *CL_KindString[] = { }; CL_API_FUNCTION void CL_StringifyMessage(char *buff, int buff_size, CL_Message *msg) { - CL_SNPRINTF(buff, buff_size, "%s:%d %15s", msg->token.file, msg->token.line, msg->string); + CL_SNPRINTF(buff, buff_size, "%s(%d,%d): %15s", msg->token.file, msg->token.line + 1, msg->token.column + 1, msg->string); } CL_API_FUNCTION void CL_Stringify(char *buff, int buff_size, CL_Token *token) { const char *token_kind = "UNKNOWN"; if (token->kind < CL_COUNT) token_kind = CL_KindString[token->kind]; - CL_SNPRINTF(buff, buff_size, "%s:%d %15s %15.*s", token->file, token->line, token_kind, token->len, token->str); + CL_SNPRINTF(buff, buff_size, "%s(%d,%d): %15s %15.*s", token->file, token->line + 1, token->column + 1, token_kind, token->len, token->str); } #define CL_SLL_QUEUE_ADD_MOD(f, l, n, next) \ diff --git a/tests/test_lexer.cpp b/tests/test_lexer.cpp new file mode 100644 index 0000000..d603cda --- /dev/null +++ b/tests/test_lexer.cpp @@ -0,0 +1,39 @@ + +#define CL_Allocator MA_Arena * +#define CL_Allocate(a, s) MA_PushSizeNonZeroed(a, s) +#define CL_ASSERT IO_Assert +#define CL_VSNPRINTF stbsp_vsnprintf +#define CL_SNPRINTF stbsp_snprintf +#include "../standalone_libraries/clexer.c" + +void TestClexer() { + MA_Scratch scratch; + for (OS_FileIter iter = OS_IterateFiles(scratch, "../standalone_libraries"); OS_IsValid(iter); OS_Advance(&iter)) { + S8_String file = OS_ReadFile(scratch, iter.absolute_path); + CL_Lexer lexer = CL_Begin(scratch, file.str, iter.absolute_path.str); + + char buff[1024]; + char msg_buff[1024]; + int token_count = 0; + for (;;) { + if (iter.filename == "preproc_env.h") { + int a = 10; + } + CL_Token token = CL_Next(&lexer); + if (token.kind == CL_EOF) break; + + if (lexer.errors) { + for (CL_Message *it = lexer.first_message; it; it = it->next) { + CL_StringifyMessage(msg_buff, sizeof(msg_buff), it); + IO_Printf("%s\n", msg_buff); + } + } + + CL_Stringify(buff, sizeof(buff), &token); + token_count += 1; + } + // IO_Printf("%.*s token count = %d\n", S8_Expand(iter.absolute_path), token_count); + } + + S8_String filename = "../standalone_libraries/clexer.c"; +} \ No newline at end of file diff --git a/tests/test_main.cpp b/tests/test_main.cpp index 83420f0..37368e9 100644 --- a/tests/test_main.cpp +++ b/tests/test_main.cpp @@ -2,8 +2,7 @@ #include "test_table.cpp" #include "test_array.cpp" #include "test_arena.cpp" - -void TestClexer(); +#include "test_lexer.cpp" int main() { TestSimpleInsertAndIntegrity(); @@ -54,28 +53,3 @@ int main() { IO_Assert(i == 5); } } - -#define CL_Allocator MA_Arena * -#define CL_Allocate(a, s) MA_PushSizeNonZeroed(a, s) -#define CL_ASSERT IO_Assert -#define CL_VSNPRINTF stbsp_vsnprintf -#define CL_SNPRINTF stbsp_snprintf -#include "../standalone_libraries/clexer.c" - -void TestClexer() { - MA_Scratch scratch; - - S8_String filename = "../standalone_libraries/clexer.c"; - S8_String file = OS_ReadFile(scratch, filename); - CL_Lexer lexer = CL_Begin(scratch, file.str, filename.str); - - char buff[1024]; - for (;;) { - CL_Token token = CL_Next(&lexer); - if (token.kind == CL_EOF) break; - if (token.kind != CL_PREPROC_INCLUDE) continue; - - CL_Stringify(buff, sizeof(buff), &token); - // IO_Printf("%s\n", buff); - } -} \ No newline at end of file