Improve build tool api, cleanup pass, add lexer tests

build.bat

@@ -3,4 +3,5 @@ mkdir build
 cd build
 cl -Fe:bld.exe ../build_tool/main.cpp -FC -WX -W3 -wd4200 -diagnostics:column -nologo -Zi -D_CRT_SECURE_NO_WARNINGS
 cd ..
+rem build\bld.exe clear_cache
 build\bld.exe

build_file.cpp

@@ -3,8 +3,10 @@
 void Compile(S8_String cc, S8_String files);
 int ReturnValue = 0;
 
-int Main() {
+int main(int argc, char **argv) {
-    S8_String cc = CMDLine.get("cc", ON_WINDOWS("cl") ON_MAC("clang") ON_LINUX("gcc"));
+    Array<S8_String> cmd = CMD_Make(argv, argc);
+    S8_String cc = CMD_Get(cmd, "cc", IF_WINDOWS("cl") IF_MAC("clang") IF_LINUX("gcc"));
+
     Compile(cc, "../tests/test_main.cpp");
     Compile(cc, "../tests/test_filesystem.c");
 

build_tool/cache.cpp

@@ -1,5 +1,5 @@
-
 #define SRC_CACHE_ENTRY_COUNT 1024
+
 struct SRC_CacheEntry {
     uint64_t filepath_hash;
     uint64_t file_hash;
@@ -19,6 +19,11 @@ SRC_Cache *SRC_FromFileCache;
 S8_String SRC_CacheFilename;
 CL_SearchPaths SRC_SearchPaths = {}; // @todo;
 
+#define SRC_CacheScope(cache_filename) \
+    SRC_InitCache(Perm, cache_filename); \
+    defer { SRC_SaveCache(); };
+
+
 void SRC_InitCache(MA_Arena *arena, S8_String cachefilename) {
     SRC_CacheFilename = cachefilename;
 

build_tool/easy_strings.cpp

@@ -75,17 +75,6 @@ S8_String Fmt(const char *str, ...) {
     return str_fmt;
 }
 
-bool CodeWasModified(S8_String str, S8_String artifact = {}) {
-    return SRC_WasModified(str, artifact);
-}
-
-S8_String IfCodeWasModified(S8_String cfile, S8_String objfile) {
-    Array<S8_String> result = {};
-    if (SRC_WasModified(cfile, objfile))
-        return cfile;
-    return objfile;
-}
-
 int Run(Array<S8_String> s) {
     S8_String cmd = Merge(s);
     return OS_SystemF("%.*s", S8_Expand(cmd));
@@ -98,3 +87,36 @@ Array<S8_String> ListDir(char *dir) {
     }
     return result;
 }
+
+Array<S8_String> CMD_Make(char **argv, int argc) {
+    Array<S8_String> result = {Perm};
+    IO_Printf("Command line arguments:\n");
+    for (int i = 1; i < argc; i += 1) {
+        S8_String it = S8_MakeFromChar(argv[i]);
+        result.add(it);
+
+        IO_Printf("[%d] %.*s\n", i, S8_Expand(it));
+    }
+    return result;
+}
+
+S8_String CMD_Get(Array<S8_String> &cmd, S8_String name, S8_String default_value = "") {
+    For(cmd) {
+        int64_t idx = 0;
+        if (S8_Seek(it, "="_s, 0, &idx)) {
+            S8_String key = S8_GetPrefix(it, idx);
+            S8_String value = S8_Skip(it, idx + 1);
+            if (key == name) {
+                return value;
+            }
+        }
+    }
+    return default_value;
+}
+
+bool CMD_Match(Array<S8_String> &cmd, S8_String name) {
+    For(cmd) {
+        if (it == name) return true;
+    }
+    return false;
+}

build_tool/library.cpp

@@ -54,38 +54,3 @@ S8_String GCC_Flags = "-Wno-write-strings";
 S8_String GCC_NoStd = "-fno-exceptions";
 S8_String GCC_Debug = "-fsanitize=address -g";
 
-#ifndef BUILD_MAIN
-int Main();
-
-void BUILD_ReportError(S8_String it) {
-    IO_FatalErrorf("Invalid command line argument syntax! Expected a key value pair!\n"
-                   "Here is the wrong argument: %.*s\n"
-                   "Here is a good example: bld.exe profile=release platform=windows\n",
-                   S8_Expand(it));
-}
-
-int main(int argc, char **argv) {
-
-    if (argc > 1) IO_Printf("Command line arguments:\n");
-    for (int i = 1; i < argc; i += 1) {
-        S8_String it = S8_MakeFromChar(argv[i]);
-
-        int64_t idx = 0;
-        if (S8_Seek(it, "="_s, 0, &idx)) {
-            S8_String key = S8_GetPrefix(it, idx);
-            S8_String value = S8_Skip(it, idx + 1);
-            if (key.len == 0) BUILD_ReportError(it);
-            if (value.len == 0) BUILD_ReportError(it);
-            IO_Printf("[%d] %.*s = %.*s\n", i, S8_Expand(key), S8_Expand(value));
-
-            CMDLine.put(key, value);
-        }
-        else BUILD_ReportError(it);
-    }
-
-    SRC_InitCache(Perm, S8_Lit("build_file.cache"));
-    int result = Main();
-    if (result == 0) SRC_SaveCache();
-    return result;
-}
-#endif

build_tool/main.cpp

@@ -1,4 +1,3 @@
-#define BUILD_MAIN
 #include "library.cpp"
 
 int main(int argument_count, char **arguments) {
@@ -7,21 +6,17 @@ int main(int argument_count, char **arguments) {
     S8_String working_dir = OS_GetWorkingDir(Perm);
     IO_Printf("WORKING DIR: %.*s\n", S8_Expand(working_dir));
 
-    S8_String cc = ON_WINDOWS("cl"_s) ON_MAC("clang"_s) ON_LINUX("gcc"_s);
+    Array<S8_String> cmd = CMD_Make(arguments, argument_count);
-    S8_String cache_filename = "build_tool.cache"_s;
+    if (CMD_Match(cmd, "clear_cache")) {
-
+        for (OS_FileIter it = OS_IterateFiles(Perm, "./"); OS_IsValid(it); OS_Advance(&it)) {
-    S8_String cmdline_args = S8_MakeEmpty();
+            if (!it.is_directory && S8_EndsWith(it.filename, ".cache", true)) {
-    for (int i = 1; i < argument_count; i += 1) {
+                OS_DeleteFile(it.absolute_path);
-        S8_String arg = S8_MakeFromChar(arguments[i]);
+            }
-        if (arg == "clear_cache"_s) {
-            OS_DeleteFile(cache_filename);
-            break;
         }
-
+        return 0;
-        cmdline_args = S8_Format(Perm, "%.*s%.*s ", S8_Expand(cmdline_args), S8_Expand(arg));
     }
 
-    SRC_InitCache(Perm, cache_filename);
+    S8_String cc = CMD_Get(cmd, "cc", IF_WINDOWS("cl") IF_MAC("clang") IF_LINUX("gcc"));
 
     // Search for build file in the project directory
     S8_String build_file = {};
@@ -38,6 +33,7 @@ int main(int argument_count, char **arguments) {
         }
     }
 
+    SRC_CacheScope("build_tool.cache");
     S8_String name_no_ext = S8_GetNameNoExt(build_file);
     S8_String exe_name = S8_Format(Perm, "%.*s.exe", S8_Expand(name_no_ext));
 
@@ -45,14 +41,14 @@ int main(int argument_count, char **arguments) {
     if (SRC_WasModified(build_file, exe_name)) {
         double time = OS_GetTime();
         int result = 0;
-        if (cc == "cl"_s) {
+        if (cc == "cl") {
             result = OS_SystemF("cl %.*s -Fe:%.*s -WX -W3 -wd4200 -diagnostics:column -nologo -Zi", S8_Expand(build_file), S8_Expand(exe_name));
         }
-        else if (cc == "clang"_s) {
+        else if (cc == "clang") {
             result = OS_SystemF("clang++ -std=c++11 -fdiagnostics-absolute-paths -Wno-writable-strings %.*s -o %.*s -g", S8_Expand(build_file), S8_Expand(exe_name));
         }
         else {
-            IO_Assert(cc == "gcc"_s);
+            IO_Assert(cc == "gcc");
             result = OS_SystemF("gcc -Wno-write-strings %.*s -o %.*s -g", S8_Expand(build_file), S8_Expand(exe_name));
         }
 
@@ -64,6 +60,8 @@ int main(int argument_count, char **arguments) {
         IO_Printf("TIME Build file compilation: %f\n", time);
     }
 
+    S8_String cmdline_args = Merge(cmd);
+
     // Run the build file
     double time = OS_GetTime();
     if (build_file.str) {
@@ -75,7 +73,4 @@ int main(int argument_count, char **arguments) {
     }
     time = OS_GetTime() - time;
     IO_Printf("TIME total build file execution: %f\n", time);
-
-    SRC_SaveCache();
-    // OS_SetWorkingDir(S8_Lit(".."));
 }

core_library/filesystem.c

@@ -47,7 +47,7 @@ OS_API S8_String OS_GetExePath(MA_Arena *arena) {
     IO_Assert(wsize != 0);
 
     S8_String path = UTF_CreateStringFromWidechar(arena, wbuffer, wsize);
-    S8_NormalizePath(path);
+    S8_NormalizePathUnsafe(path);
     return path;
 }
 
@@ -67,7 +67,7 @@ OS_API S8_String OS_GetWorkingDir(MA_Arena *arena) {
     wbuffer[wsize] = 0;
 
     S8_String path = UTF_CreateStringFromWidechar(arena, wbuffer, wsize);
-    S8_NormalizePath(path);
+    S8_NormalizePathUnsafe(path);
     return path;
 }
 
@@ -85,7 +85,7 @@ OS_API S8_String OS_GetAbsolutePath(MA_Arena *arena, S8_String relative) {
     if (written == 0)
         return S8_MakeEmpty();
     S8_String path = UTF_CreateStringFromWidechar(arena, wpath_abs, written);
-    S8_NormalizePath(path);
+    S8_NormalizePathUnsafe(path);
     return path;
 }
 
@@ -153,8 +153,8 @@ OS_API void OS_Advance(OS_FileIter *it) {
 
         it->is_directory = data->dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY;
         it->filename = UTF_CreateStringFromWidechar(it->arena, data->cFileName, S8_WideLength(data->cFileName));
-        char *is_dir = it->is_directory ? "/" : "";
+        const char *is_dir = it->is_directory ? "/" : "";
-        char *separator = it->path.str[it->path.len - 1] == '/' ? "" : "/";
+        const char *separator = it->path.str[it->path.len - 1] == '/' ? "" : "/";
         it->relative_path = S8_Format(it->arena, "%.*s%s%.*s%s", S8_Expand(it->path), separator, S8_Expand(it->filename), is_dir);
         it->absolute_path = OS_GetAbsolutePath(it->arena, it->relative_path);
         it->is_valid = true;

standalone_libraries/clexer.c

@@ -722,7 +722,7 @@ CL_PRIVATE_FUNCTION bool CL_LexMacro(CL_Lexer *T, CL_Token *token) {
 }
 
 // Skipped space here is for case #define Memes (a), this is not a function like macro because of space
-static uint32_t CL_TokenID; // @todo: make it stable, thread local?
+static uint32_t CL_TokenID; // @todo: make it read only
 CL_PRIVATE_FUNCTION void CL_PrepareToken(CL_Lexer *T, CL_Token *token, bool skipped_space) {
     CL_MemoryZero(token, sizeof(*token));
     token->str = T->stream;

standalone_libraries/preproc_env.h

@@ -81,24 +81,24 @@
 #endif
 
 #if OS_MAC
-    #define ON_MAC(x) x
+    #define IF_MAC(x) x
 #else
-    #define ON_MAC(x)
+    #define IF_MAC(x)
 #endif
 
 #if OS_WINDOWS
-    #define ON_WINDOWS(x) x
+    #define IF_WINDOWS(x) x
     #define IF_WINDOWS_ELSE(x, y) x
 #else
-    #define ON_WINDOWS(x)
+    #define IF_WINDOWS(x)
     #define IF_WINDOWS_ELSE(x, y) y
 #endif
 
 #if OS_LINUX
-    #define ON_LINUX(x) x
+    #define IF_LINUX(x) x
     #define IF_LINUX_ELSE(x, y) x
 #else
-    #define ON_LINUX(x)
+    #define IF_LINUX(x)
     #define IF_LINUX_ELSE(x, y) y
 #endif
 

standalone_libraries/string.c

@@ -120,7 +120,16 @@ S8_API S8_String S8_CopyChar(S8_Allocator allocator, char *s) {
     return result;
 }
 
-S8_API void S8_NormalizePath(S8_String s) {
+S8_API S8_String S8_NormalizePath(S8_Allocator allocator, S8_String s) {
+    S8_String copy = S8_Copy(allocator, s);
+    for (int64_t i = 0; i < copy.len; i++) {
+        if (copy.str[i] == '\\')
+            copy.str[i] = '/';
+    }
+    return copy;
+}
+
+S8_API void S8_NormalizePathUnsafe(S8_String s) {
     for (int64_t i = 0; i < s.len; i++) {
         if (s.str[i] == '\\')
             s.str[i] = '/';

standalone_libraries/string.h

@@ -131,7 +131,8 @@ S8_API bool S8_EndsWith(S8_String a, S8_String end, unsigned ignore_case S8_IF_C
 S8_API bool S8_StartsWith(S8_String a, S8_String start, unsigned ignore_case S8_IF_CPP(= false));
 S8_API S8_String S8_Make(char *str, int64_t len);
 S8_API S8_String S8_Copy(S8_Allocator allocator, S8_String string);
-S8_API void S8_NormalizePath(S8_String s);
+S8_API S8_String S8_NormalizePath(S8_Allocator allocator, S8_String s);
+S8_API void S8_NormalizePathUnsafe(S8_String s); // make sure there is no way string is const etc.
 S8_API S8_String S8_Chop(S8_String string, int64_t len);
 S8_API S8_String S8_Skip(S8_String string, int64_t len);
 S8_API S8_String S8_GetPostfix(S8_String string, int64_t len);

tests/test_lexer.cpp

@@ -6,19 +6,16 @@
 #define CL_SNPRINTF stbsp_snprintf
 #include "../standalone_libraries/clexer.c"
 
-void TestClexer() {
+void TestLexDir(S8_String dir) {
     MA_Scratch scratch;
-    for (OS_FileIter iter = OS_IterateFiles(scratch, "../standalone_libraries"); OS_IsValid(iter); OS_Advance(&iter)) {
+    char buff[1024];
+    char msg_buff[1024];
+    for (OS_FileIter iter = OS_IterateFiles(scratch, dir); OS_IsValid(iter); OS_Advance(&iter)) {
         S8_String file = OS_ReadFile(scratch, iter.absolute_path);
+        if (!file.str) continue;
         CL_Lexer lexer = CL_Begin(scratch, file.str, iter.absolute_path.str);
 
-        char buff[1024];
+        for (int i = 0;; i += 1) {
-        char msg_buff[1024];
-        int token_count = 0;
-        for (;;) {
-            if (iter.filename == "preproc_env.h") {
-                int a = 10;
-            }
             CL_Token token = CL_Next(&lexer);
             if (token.kind == CL_EOF) break;
 
@@ -30,10 +27,13 @@ void TestClexer() {
             }
 
             CL_Stringify(buff, sizeof(buff), &token);
-            token_count += 1;
         }
-        // IO_Printf("%.*s token count = %d\n", S8_Expand(iter.absolute_path), token_count);
     }
+}
 
-    S8_String filename = "../standalone_libraries/clexer.c";
+void TestClexer() {
+    TestLexDir("../standalone_libraries");
+    TestLexDir("../core_library");
+    TestLexDir("../build_tool");
+    TestLexDir("../tests");
 }