Squashing lexer bugs related to EOF

compiler.h

@@ -97,6 +97,7 @@ enum Token_Kind{
 
 struct Token{
   Token_Kind kind;
+  U32 di; // debug_id
   union{
     String string;
     struct{U8 *str; S64 len;};
@@ -133,6 +134,7 @@ struct Lexer{
   Array<Token> tokens;
   Intern_Table interns;
   S64 token_iter;
+  U32 token_debug_ids;
 
   Intern_String intern(String string){
     return intern_string(&interns, string);
@@ -182,11 +184,12 @@ struct Parse_Ctx:Lexer{
   String_Builder gen;
 };
 
+global B32 emit_line_directives;
+
 //-----------------------------------------------------------------------------
 // Constructors
 //-----------------------------------------------------------------------------
 thread_local Parse_Ctx *pctx;
-
 function void
 lex_init(Allocator *token_string_arena, Allocator *map_allocator, Lexer *l){
   l->arena = token_string_arena;