imgui_transcript_browser/src/transcript_browser/loading_thread.cpp

struct ParseThreadIO {
    Array<String> input_files;
};

enum SourceKind {
    SourceKind_Invalid,
    SourceKind_SRT,
    SourceKind_PDF,
    SourceKind_TXT,
};

struct XToSource {
    SourceKind kind;
    String     string; // String inside x arena
    String     filepath;
    union {
        struct {
            uint16_t hour;
            uint16_t minute;
            uint16_t second;
        } srt;
        struct {
            int page;
        } pdf;
        struct {
            int64_t row;
        } txt;
    };
};

struct FileLoadResult {
    String filename;
    String error;
};

Arena                 XArena;
std::mutex            XArenaAddMutex;
Array<XToSource>      XToSourceArray;
Array<FileLoadResult> XFileLoadResults;
int64_t               XLoadThreadComplete;

void XInitLoading() {
    InitArena(&XArena);
    XArena.align = 0;

    XToSourceArray.reserve(1000000);
    XFileLoadResults.reserve(10000);
}

WORK_FUNCTION(ParseFilesWork) {
    ParseThreadIO *io = (ParseThreadIO *)data;
    ForItem(it_time_file, io->input_files) {
        Scratch scratch;
        if (EndsWith(it_time_file, ".srt", true)) {
            Array<TimeString> time_strings = ParseSrtFile(scratch, it_time_file);

            XArenaAddMutex.lock();
            For(time_strings) {
                String s      = Copy(XArena, it.string);
                s.data[s.len] = ' ';

                XToSource t = {SourceKind_SRT, s, it_time_file};
                t.srt       = {it.hour, it.minute, it.second};
                XToSourceArray.add(t);
            }
            XArenaAddMutex.unlock();
            XFileLoadResults.bounded_add({it_time_file});
        } else if (EndsWith(it_time_file, ".txt", true) || EndsWith(it_time_file, ".html", true)) {
            String string = ReadFile(scratch, it_time_file);
            if (string.data) {
                XArenaAddMutex.lock();
                Array<String> lines = Split(scratch, string, "\n");
                For(lines) {
                    String s      = Copy(XArena, it);
                    s.data[s.len] = ' ';

                    XToSource t = {SourceKind_TXT, s, it_time_file};
                    t.txt       = {lines.get_index(it)};
                    XToSourceArray.add(t);
                }
                XArenaAddMutex.unlock();
                XFileLoadResults.bounded_add({it_time_file});
            } else {
                XFileLoadResults.bounded_add({it_time_file, "failed to read the file"});
            }
        } else {
            XFileLoadResults.bounded_add({it_time_file, "internal error: extension is not supported but got propagated to parse stage"});
        }
    }
    AtomicIncrement(&XLoadThreadComplete);
}

String XGetArenaString() {
    XArenaAddMutex.lock();
    String buffer = {(char *)XArena.data, (int64_t)XArena.len};
    XArenaAddMutex.unlock();
    return buffer;
}

Array<FileLoadResult> XLockFileLoadResults() {
    return XFileLoadResults;
}

void XUnlockFileResults() {
}

void XAddFolder(String folder, Array<String> *filenames) {
    Scratch       scratch;
    Array<String> files_to_parse = {scratch};
    for (FileIter iter = IterateFiles(scratch, folder); IsValid(iter); Advance(&iter)) {
        String file = Copy(Perm, iter.absolute_path);
        filenames->add(file);
        if (EndsWith(iter.filename, ".srt", true)) {
            files_to_parse.add(file);
        } else if (EndsWith(iter.filename, ".txt", true) || EndsWith(iter.filename, ".html", true)) {
            files_to_parse.add(file);
        }
    }

    if (files_to_parse.len == 0) {
        XFileLoadResults.add({Copy(Perm, folder), "no files found"});
        return;
    }

    int64_t thread_count     = MainWorkQueue.thread_count;
    int64_t files_per_thread = files_to_parse.len / thread_count;
    int64_t remainder        = files_to_parse.len % thread_count;
    int64_t fi               = 0;

    Array<ParseThreadIO> io = {Perm};
    io.reserve(thread_count);
    for (int ti = 0; ti < thread_count; ti += 1) {
        Array<String> files = {Perm};
        for (int i = 0; fi < files_to_parse.len && i < files_per_thread + remainder; fi += 1, i += 1) {
            files.add(files_to_parse[fi]);
        }
        if (remainder) remainder = 0;

        ParseThreadIO *i = io.alloc();
        i->input_files   = files;
        PushWork(&MainWorkQueue, (void *)i, ParseFilesWork);
    }
}

XToSource *XFindSource(String string) {
    XToSource *result = NULL;

    XArenaAddMutex.lock();
    For(XToSourceArray) {
        uintptr_t begin  = (uintptr_t)(it.string.data);
        uintptr_t end    = (uintptr_t)(it.string.data + it.string.len);
        uintptr_t needle = (uintptr_t)string.data;
        if (needle >= begin && needle < end) {
            result = &it;
            break;
        }
    }
    XArenaAddMutex.unlock();

    return result;
}

bool XLoadingComplete() {
    bool result = MainWorkQueue.thread_count == XLoadThreadComplete;
    return result;
}