Swinging in a different direction, different syntax

lex.c

@@ -57,7 +57,7 @@ lex_is_alphanumeric(U8 c){
 
 function void
 lex_set_len(Lex_Stream *s, Token *token){
-  assert(s->stream > token->str);
+  assert(s->stream >= token->str);
   token->len = s->stream - token->str;
 }
 
@@ -150,6 +150,13 @@ break
     
     
     Token *t = token_alloc(tokens);
+    if(0){
+      top:
+      while(lex_is_whitespace(*s->stream))
+        lex_advance(s);
+      if(lexc(s) == 0)
+        break;
+    }
     lex_token_seed(s, t);
     lex_advance(s);
     switch(*t->str) {
@@ -164,6 +171,7 @@ break
 #undef CASE2
 #undef CASE3
       
+      case 0: break;
       case '@': t->kind = TK_At; break;
       case '(': t->kind = TK_OpenParen; break;
       case ')': t->kind = TK_CloseParen; break;
@@ -279,20 +287,21 @@ break
           lex_advance(s);
           if(lexc(s) == '/'){
             lex_advance(s);
-            t->kind = TK_DocComment;
+            //t->kind = TK_DocComment;
           }
           else {
-            t->kind = TK_Comment;
+            //t->kind = TK_Comment;
           }
           for(;;){
             if(lexc(s) == '\n' || lexc(s) == 0) break;
             lex_advance(s);
           }
-          lex_set_len(s,t);
+          goto top;
+          //lex_set_len(s,t);
         }
         else if(lexc(s) == '*'){
           lex_advance(s);
-          t->kind = TK_Comment;
+          //t->kind = TK_Comment;
           for(;;){
             if(s->stream[0] == '*' && s->stream[1] == '/'){
               lex_advance(s);
@@ -305,7 +314,8 @@ break
             }
             lex_advance(s);
           }
-          lex_set_len(s,t);
+          goto top;
+          //lex_set_len(s,t);
         }
         else t->kind = TK_Div;
       } break;
@@ -352,7 +362,7 @@ break
           lex_advance(s);
         lex_set_len(s,t);
       } break;
-      default: {
+      default:{
         token_error(t, lit("Unknown token"));
       } break;
     }
@@ -424,9 +434,7 @@ token_is_keyword(Parser *p, Intern_String keyword){
 
 function void
 token_advance(Parser *p){
-  do{
-    p->tokens.iter = clamp_top_s64(p->tokens.iter + 1, p->tokens.len);
-  }while(token_is_comment(token_get(p)));
+  p->tokens.iter = clamp_top_s64(p->tokens.iter + 1, p->tokens.len);
 }
 
 function Token *
@@ -495,4 +503,15 @@ token_peek_is(Parser *p, S64 count, Token_Kind kind){
   if(token->kind == kind)
     return token;
   return 0;
-}
+}
+
+function Token *
+token_peek_is_keyword(Parser *p, S64 count, Intern_String keyword){
+  Token *token = token_peek(p, count);
+  if(token->kind == TK_Keyword){
+    if(intern_compare(keyword, token->intern_val)){
+      return token;
+    }
+  }
+  return 0;
+}