Work on unicode

programs/base.kl

@@ -33,7 +33,7 @@ arena_init :: (a: *Arena)
   a.memory    = Os.reserve(a.DEFAULT_RESERVE_SIZE)
   a.alignment = a.DEFAULT_ALIGNMENT
   a.di        = arena_di++
-  a.allocator.proc = arena_allocator_proc
+  // a.allocator.proc = arena_allocator_proc
 
 arena_push_size :: (a: *Arena, size: SizeU): *void
   generous_size := size + a.alignment
@@ -73,6 +73,80 @@ arena_allocator_proc :: (a: *Allocator, action: ALLOCATOR_ACTION, size: SizeU, o
 
 allocate :: (a: *Allocator, size: SizeU): *void
   return a.proc(a, ALLOCATOR_ACTION.ALLOCATE, size, 0)
+*/
 
+//
+// Unicode
+//
+question_mark16 :: 0x003f
+String32 :: struct;; str: *U32; len: S64
+String16 :: struct;; str: *U16; len: S64
 
-*/
+utf8_to_utf32 :: (c: *U8, max_advance: S64): U32, S64
+  out_str: U32
+  advance: S64
+  if (c[0] & 0b10000000) == 0
+    if max_advance >= 1
+      c0 := c[0]->U32
+      out_str = c0
+      advance = 1
+
+  elif (c[0] & 0b11100000) == 0b11000000
+    if (c[1] & 0b11000000) == 0b10000000 // Continuation byte required
+      if max_advance >= 2
+        c0 := c[0]->U32; c1 := c[1]->U32
+        out_str = (c0 & 0b00011111) << 6 | (c1 & 0b00111111)
+        advance = 2
+
+  elif (c[0] & 0b11110000) == 0b11100000
+    if (c[1] & 0b11000000) == 0b10000000 && (c[2] & 0b11000000) == 0b10000000 // Two continuation bytes required
+      if max_advance >= 3
+        c0 := c[0]->U32; c1 := c[1]->U32; c2 := c[2]->U32
+        out_str = (c0 & 0b00001111) << 12 | (c1 & 0b00111111) << 6 | (c2 & 0b00111111)
+        advance = 3
+
+  elif (c[0] & 0b11111000) == 0b11110000
+    if (c[1] & 0b11000000) == 0b10000000 && (c[2] & 0b11000000) == 0b10000000 && (c[3] & 0b11000000) == 0b10000000 // Three continuation bytes required
+      if max_advance >= 4
+        c0 := c[0]->U32; c1 := c[1]->U32; c2 := c[2]->U32; c3 := c[3]->U32
+        out_str = (c0 & 0b00001111) << 18 | (c1 & 0b00111111) << 12 | (c2 & 0b00111111) << 6 | (c3 & 0b00111111)
+        advance = 4
+
+  return out_str, advance
+
+utf32_to_utf16 :: (codepoint: U32): [2]U16, S64
+  str: [2]U16
+  len := 0
+  if codepoint < 0x10000
+    str[0] = codepoint->U16
+    len = 1
+  elif codepoint <= 0x10FFFF
+    code: U32 = (codepoint - 0x10000)
+    str[0] = (0xD800 | (code >> 10))->U16
+    str[1] = (0xDC00 | (code & 0x3FF))->U16
+    len = 2
+
+  return str, len
+
+string_to_string16 :: (arena: *Arena, in: String): String16
+  in_str := &in[0]
+  // @Note(Krzosa): Should be more then enough space
+  alloc_size := (length_of(in)*2)+1
+  result := String16{str = arena_push_size(arena, alloc_size->U64)}
+  for i := 0, i < length_of(in)
+    s32, s32_len := utf8_to_utf32(in_str + i, length_of(in) - i)
+    if s32_len != 0
+      i += s32_len
+      s16, s16_len := utf32_to_utf16(s32)
+      if s16_len != 0
+        for j := 0, j < s16_len, j++
+          result.str[result.len++] = s16[j]
+      else
+        result.str[result.len++] = question_mark16
+        break
+    else
+      result.str[result.len++] = question_mark16
+      break
+
+  result.str[result.len] = 0
+  return result