Add #error lexing

build.bat

@@ -1,6 +1,4 @@
 @echo off
-call ../misc/compile_setup.bat
-
 mkdir build
 cd build
 cl -Fe:bld.exe ../build_tool/main.cpp -FC -WX -W3 -wd4200 -diagnostics:column -nologo -Zi -D_CRT_SECURE_NO_WARNINGS

build_file.cpp

@@ -21,7 +21,7 @@ void Compile(S8_String cc, S8_String file) {
         Array<S8_String> flags = Split("/MP /Zi -D_CRT_SECURE_NO_WARNINGS /GF /Gm- /Oi");
         flags += Split("/FC /WX /W3 /wd4200 /diagnostics:column /nologo");
         flags += Split("/GR- /EHa-");
-        flags += Split("-fsanitize=address -RTC1");
+        flags += Split("-RTC1");
 
         Array<S8_String> link = Split("/link /incremental:no");
         S8_String name = Fmt("/Fe:%.*s", S8_Expand(exe));
@@ -55,6 +55,6 @@ void Compile(S8_String cc, S8_String file) {
         result = Run(cc + file + name + flags);
     }
 
-    if (result == 0) result = OS_SystemF(IF_WINDOWS_ELSE("", "./") "%.*s.exe", S8_Expand(exe));
+    if (result == 0) result = OS_SystemF(IF_WINDOWS_ELSE("", "./") "%.*s", S8_Expand(exe));
     else ReturnValue = result;
 }

standalone_libraries/clexer.c

@@ -615,9 +615,17 @@ CL_PRIVATE_FUNCTION void CL_IsIdentifierKeyword(CL_Token *token) {
     }
 }
 
+CL_PRIVATE_FUNCTION void CL_EatMacroWhitespace(CL_Lexer *T) {
+    while (T->stream[0] == ' ' || T->stream[0] == '\t') CL_Advance(T);
+}
+
+CL_PRIVATE_FUNCTION void CL_EatUntil(CL_Lexer *T, char c) {
+    while (T->stream[0] != c && T->stream[0] != 0) CL_Advance(T);
+}
+
 CL_PRIVATE_FUNCTION void CL_LexMacroInclude(CL_Lexer *T, CL_Token *token) {
     token->kind = CL_PREPROC_INCLUDE;
-    while (*T->stream == ' ') CL_Advance(T);
+    CL_EatMacroWhitespace(T);
     char end = 0;
     if (*T->stream == '"') {
         end = '"';
@@ -650,7 +658,7 @@ CL_PRIVATE_FUNCTION void CL_LexMacroInclude(CL_Lexer *T, CL_Token *token) {
 }
 
 CL_PRIVATE_FUNCTION bool CL_LexMacro(CL_Lexer *T, CL_Token *token) {
-    while (*T->stream == ' ' || T->stream[0] == '\t') CL_Advance(T);
+    CL_EatMacroWhitespace(T);
     token->str = T->stream;
     while (CL_IsAlphabetic(*T->stream)) CL_Advance(T);
     CL_SetTokenLength(T, token);
@@ -684,6 +692,10 @@ CL_PRIVATE_FUNCTION bool CL_LexMacro(CL_Lexer *T, CL_Token *token) {
             }
             else if (CL_StringsAreEqual(token->str, token->len, "error", 5)) {
                 token->kind = CL_PREPROC_ERROR;
+                CL_EatMacroWhitespace(T);
+                token->str = T->stream;
+                CL_EatUntil(T, '\n');
+                CL_SetTokenLength(T, token);
             }
             else if (CL_StringsAreEqual(token->str, token->len, "else", 4)) {
                 token->kind = CL_PREPROC_ELSE;
@@ -750,10 +762,7 @@ CL_PRIVATE_FUNCTION void CL_DefaultTokenize(CL_Lexer *T, CL_Token *token) {
             if (*T->stream == '/') {
                 token->kind = CL_COMMENT;
                 CL_Advance(T);
-
+                CL_EatUntil(T, '\n');
-                while (*T->stream != '\n' && *T->stream != 0) {
-                    CL_Advance(T);
-                }
                 CL_SetTokenLength(T, token);
             }
             else if (*T->stream == '*') {
@@ -1480,13 +1489,13 @@ const char *CL_KindString[] = {
 };
 
 CL_API_FUNCTION void CL_StringifyMessage(char *buff, int buff_size, CL_Message *msg) {
-    CL_SNPRINTF(buff, buff_size, "%s:%d %15s", msg->token.file, msg->token.line, msg->string);
+    CL_SNPRINTF(buff, buff_size, "%s(%d,%d): %15s", msg->token.file, msg->token.line + 1, msg->token.column + 1, msg->string);
 }
 
 CL_API_FUNCTION void CL_Stringify(char *buff, int buff_size, CL_Token *token) {
     const char *token_kind = "UNKNOWN";
     if (token->kind < CL_COUNT) token_kind = CL_KindString[token->kind];
-    CL_SNPRINTF(buff, buff_size, "%s:%d %15s %15.*s", token->file, token->line, token_kind, token->len, token->str);
+    CL_SNPRINTF(buff, buff_size, "%s(%d,%d): %15s %15.*s", token->file, token->line + 1, token->column + 1, token_kind, token->len, token->str);
 }
 
 #define CL_SLL_QUEUE_ADD_MOD(f, l, n, next) \

tests/test_lexer.cpp

@@ -0,0 +1,39 @@
+
+#define CL_Allocator MA_Arena *
+#define CL_Allocate(a, s) MA_PushSizeNonZeroed(a, s)
+#define CL_ASSERT IO_Assert
+#define CL_VSNPRINTF stbsp_vsnprintf
+#define CL_SNPRINTF stbsp_snprintf
+#include "../standalone_libraries/clexer.c"
+
+void TestClexer() {
+    MA_Scratch scratch;
+    for (OS_FileIter iter = OS_IterateFiles(scratch, "../standalone_libraries"); OS_IsValid(iter); OS_Advance(&iter)) {
+        S8_String file = OS_ReadFile(scratch, iter.absolute_path);
+        CL_Lexer lexer = CL_Begin(scratch, file.str, iter.absolute_path.str);
+
+        char buff[1024];
+        char msg_buff[1024];
+        int token_count = 0;
+        for (;;) {
+            if (iter.filename == "preproc_env.h") {
+                int a = 10;
+            }
+            CL_Token token = CL_Next(&lexer);
+            if (token.kind == CL_EOF) break;
+
+            if (lexer.errors) {
+                for (CL_Message *it = lexer.first_message; it; it = it->next) {
+                    CL_StringifyMessage(msg_buff, sizeof(msg_buff), it);
+                    IO_Printf("%s\n", msg_buff);
+                }
+            }
+
+            CL_Stringify(buff, sizeof(buff), &token);
+            token_count += 1;
+        }
+        // IO_Printf("%.*s token count = %d\n", S8_Expand(iter.absolute_path), token_count);
+    }
+
+    S8_String filename = "../standalone_libraries/clexer.c";
+}

tests/test_main.cpp

@@ -2,8 +2,7 @@
 #include "test_table.cpp"
 #include "test_array.cpp"
 #include "test_arena.cpp"
-
+#include "test_lexer.cpp"
-void TestClexer();
 
 int main() {
     TestSimpleInsertAndIntegrity();
@@ -54,28 +53,3 @@ int main() {
         IO_Assert(i == 5);
     }
 }
-
-#define CL_Allocator MA_Arena *
-#define CL_Allocate(a, s) MA_PushSizeNonZeroed(a, s)
-#define CL_ASSERT IO_Assert
-#define CL_VSNPRINTF stbsp_vsnprintf
-#define CL_SNPRINTF stbsp_snprintf
-#include "../standalone_libraries/clexer.c"
-
-void TestClexer() {
-    MA_Scratch scratch;
-
-    S8_String filename = "../standalone_libraries/clexer.c";
-    S8_String file = OS_ReadFile(scratch, filename);
-    CL_Lexer lexer = CL_Begin(scratch, file.str, filename.str);
-
-    char buff[1024];
-    for (;;) {
-        CL_Token token = CL_Next(&lexer);
-        if (token.kind == CL_EOF) break;
-        if (token.kind != CL_PREPROC_INCLUDE) continue;
-
-        CL_Stringify(buff, sizeof(buff), &token);
-        // IO_Printf("%s\n", buff);
-    }
-}