Fix lexer end of file not unwinding scopes, Trying to run a program

lexer.kl

@@ -0,0 +1,4 @@
+
+
+main :: ()
+  pass

main.cpp

@@ -32,7 +32,8 @@ For now I don't thing it should be overloadable.
 -------------------------------------------------------------------------------
 
 @todo
-[ ] - lvalue, rvalue concept so we cant assign value to some arbitrary weird expression
+[ ] - Compiling and running a program
+[ ] - Passing down program to compile through command line
 [ ] - More operators
 [ ] - More for loop variations
 [ ] - Fixing access to constants, in C we cant have constants inside of structs / functions so we need to rewrite the tree
@@ -50,6 +51,7 @@ For now I don't thing it should be overloadable.
 [ ] - Constant arrays that evaluate fully at compile time
 
 @donzo
+[x] - lvalue, rvalue concept so we cant assign value to some arbitrary weird expression
 [x] - Access through struct names to constants Arena.CONSTANT
 [x] - Enums
 [x] - Initial for loop
@@ -76,6 +78,10 @@ For now I don't thing it should be overloadable.
 #include "typecheck.h"
 #include "typecheck.cpp"
 #include "ccodegen.cpp"
+
+
+
+
 int main(){
 
   test_os_memory();
@@ -89,6 +95,7 @@ int main(){
   test_intern_table();
 
   String result = {};
+#if 0
   result = compile_file("globals.kl"_s);
   printf("%s", result.str);
   result = compile_file("enums.kl"_s);
@@ -99,6 +106,14 @@ int main(){
   printf("%s", result.str);
   result = compile_file("lambdas.kl"_s);
   printf("%s", result.str);
+#endif
+
+  result = compile_file("lexer.kl"_s);
+
+  FILE *f = fopen("program.c", "w");
+  assert(f);
+  fprintf(f, "%.*s", (int)result.len, result.str);
+  fclose(f);
 
   __debugbreak();
 }

new_lex.cpp

@@ -300,12 +300,38 @@ lex_is_scope(Token *t){
   return result;
 }
 
+function void
+lex_unwind_indent_stack(Token *t, Lex_Stream *s, Array<Token> *array){
+  for(S64 i = s->indent_stack.len-1; i >= 0; i-=1){
+    auto it = s->indent_stack.data[i];
+    assert(lex_is_scope(it));
+    if(it->indent == t->indent){
+      t->kind = SAME_SCOPE;
+      array->add(*t);
+      break;
+    }
+    else if(it->indent < t->indent){
+      token_error(t, "Bad indentation"_s);
+      array->add(*t);
+      break;
+    }
+    else{
+      s->indent_stack.pop();
+      t->kind = CLOSE_SCOPE;
+      array->add(*t);
+    }
+  }
+}
+
 function void
 lex__stream(Intern_Table *table, Array<Token> *array, Lex_Stream *s){
   B32 beginning = true;
-  while(lexc(s)){
-    if(s->iter >= s->stream.len) // End of stream
+  for(;;){
+    if(lexc(s) == 0 || s->iter >= s->stream.len){
+      Token t = token_make(lexcp(s), s->file, s->line, s->line_begin);
+      lex_unwind_indent_stack(&t, s, array);
       break;
+    }
 
     // @note: the lexer is going to be a 2 stage process
     // first we tokenize the indentation and then proceed to tokenize
@@ -388,7 +414,6 @@ lex__stream(Intern_Table *table, Array<Token> *array, Lex_Stream *s){
         //   s->inside_brace_paren--;
         //   t.kind = CLOSE_SCOPE;
         // } break;
-
         default:{
           if(s->inside_brace_paren) should_emit = false;
           if(should_emit){
@@ -400,25 +425,7 @@ lex__stream(Intern_Table *table, Array<Token> *array, Lex_Stream *s){
             }
 
             else if(t.indent < last->indent){
-              for(S64 i = s->indent_stack.len-1; i >= 0; i-=1){
-                auto it = s->indent_stack.data[i];
-                assert(lex_is_scope(it));
-                if(it->indent == t.indent){
-                  t.kind = SAME_SCOPE;
-                  array->add(t);
-                  break;
-                }
-                else if(it->indent < t.indent){
-                  token_error(&t, "Bad indentation"_s);
-                  array->add(t);
-                  break;
-                }
-                else{
-                  s->indent_stack.pop();
-                  t.kind = CLOSE_SCOPE;
-                  array->add(t);
-                }
-              }
+              lex_unwind_indent_stack(&t, s, array);
             }
             else {
               t.kind = SAME_SCOPE;

program.c

@@ -0,0 +1,9 @@
+
+//-------------------------------
+#define NULL_POINTER 0
+#define NULL_LAMBDA  0
+//-------------------------------
+  
+static void main(){
+  //pass
+}