Add unicode literal

new_lex.cpp

@@ -77,8 +77,8 @@ enum Token_Kind{
   TK_DocComment,
   TK_Comment,
   TK_Identifier,
+  TK_UnicodeLit,
   TK_StringLit,
-  TK_Character,
   TK_Error,
   TK_Float,
   TK_Integer,
@@ -102,6 +102,7 @@ struct Token{
   };
 
   union {
+    U32 unicode;
     U64 int_val;
     F64 f64_val;
     String error_val;
@@ -416,17 +417,6 @@ lex__stream(Intern_Table *table, Array<Token> *array, Lex_Stream *s){
           t = token_make(lexcp(s), s->file, s->line, s->line_begin);
         } break;
 
-        // @todo: add open and close brace handling as OPEN_SCOPE CLOSE_SCOPE
-        // when it comes to compound statements it's going to check for scopes
-        // and then it's going to specialize and look for brace string
-
-        // case '{': {
-        //   s->inside_brace_paren++; t.kind = TK_OpenBrace;
-        // } break;
-        // case '}': {
-        //   s->inside_brace_paren--;
-        //   t.kind = CLOSE_SCOPE;
-        // } break;
         default:{
           if(s->inside_brace_paren) should_emit = false;
           if(should_emit){
@@ -486,6 +476,18 @@ lex__stream(Intern_Table *table, Array<Token> *array, Lex_Stream *s){
         }
       } break;
 
+      case '\'':{
+        assert(s->stream.len >= s->iter);
+        UTF32_Result decode = utf8_to_utf32(lexcp(s), s->stream.len - s->iter);
+        if(!decode.error){
+          for(S32 i = 0; i < decode.advance; i++) lex_advance(s);
+          t.unicode = decode.out_str;
+          t.kind = TK_UnicodeLit;
+        }
+        else{
+          token_error(&t, "Invalid UTF8 sequence in unicode literal"_s);
+        }
+      } break;
 
       case '<': {
         if (lexc(s) == '<') {
@@ -559,8 +561,6 @@ lex__stream(Intern_Table *table, Array<Token> *array, Lex_Stream *s){
         }
       } break;
 
-
-      case '\'':{not_implemented;} break;
       case '"': {
         t.kind = TK_StringLit;
         lex_parse_string(s,&t,'"');
@@ -738,7 +738,7 @@ name(Token_Kind kind){
     case TK_Comment: return "Comment";
     case TK_Identifier: return "Identifier";
     case TK_StringLit: return "String_Lit";
-    case TK_Character: return "Character";
+    case TK_UnicodeLit: return "Unicode_Lit";
     case TK_Error: return "Error";
     case TK_Float: return "Float";
     case TK_Integer: return "int";