typedef struct Token_Bucket Token_Bucket;
typedef struct Token_Array Token_Array;
global Token token_end_of_stream;

struct Token_Bucket{
  Token_Bucket *next;
  Token data[4096];
};

// @Note(Krzosa): 
// Interns has an arena - this arena points to arena in Token_Array
// Token_Array arena is used to allocate the token buckets, Intern table and Intern_Strings
// 
// If we want to resize the table in the future we need a second arena or use the heap
// For now lets leave it this way
// or
// We could potentially also use the bucket array and allocate additional buckets when resizing
// the table. In that case we would need to traverse the bucket list when inserting
struct Token_Array{
  Intern_Table interns;
  
  Token_Bucket first;
  Token_Bucket *last;
  S64 len;
  S64 block;
  Arena *arena; 
  
  // @Note(Krzosa): Iterator
  Token_Bucket *iter_bucket;
  S64 iter_len;
  S64 iter_block;
};

function Token_Array
token_array_make(Arena *arena){
  Token_Array result = {
    .last  = &result.first,
    .arena = arena,
    .interns = intern_table(arena, 4096*4),
    .iter_bucket = &result.first
  };
  
  return result;
}

function void
token_array_push(Token_Array *array, Token *p){
  if(array->len >= buff_cap(array->first.data)){
    Token_Bucket *bucket = arena_push_struct(array->arena, Token_Bucket);
    array->last = array->last->next = bucket;
    array->len = 0;
    array->block += 1;
  }
  array->last->data[array->len++] = *p;
}

function Token *
token_array_iter_next(Token_Array *array){
  if((array->iter_len >= array->len) && (array->iter_block >= array->block)){
    return &token_end_of_stream;
  }
  if(array->iter_len >= buff_cap(array->first.data)){
    array->iter_len = 0;
    array->iter_block += 1;
    array->iter_bucket = array->iter_bucket->next;
  }
  Token *result = array->iter_bucket->data + array->iter_len++;
  return result;
}

function Token *
token_array_iter_peek(Token_Array *array, S64 i){
  // @Todo(Krzosa): Simplify, too many calculations here
  // Why even change state?
  S64 save_len = array->iter_len;
  S64 save_block = array->iter_block;
  Token_Bucket *save_bucket = array->iter_bucket;
  assert(i < buff_cap(array->first.data));
  
  S64 over = i;
  if(array->iter_len + i >= buff_cap(array->first.data)){
    over = buff_cap(array->first.data) - (array->iter_len + i);
    array->iter_len    = 0;
    array->iter_block += 1;
    array->iter_bucket = array->iter_bucket->next;
  }
  
  over = array->iter_len + over;
  if(over == array->len && array->iter_block == array->block){
    return &token_end_of_stream;
  }
  
  Token *result = array->iter_bucket->data + over;
  array->iter_len = save_len;
  array->iter_bucket = save_bucket;
  array->iter_block = save_block;
  return result;
}

function void
token_array_reset(Token_Array *array){
  // @Todo(Krzosa): Add free list
  assert(&array->first == array->last);
  array->len = 0;
  array->block = 0;
  array->last = &array->first;
  array->last->next = 0;
  array->iter_bucket = array->last;
  array->iter_len = 0;
  array->iter_block = 0;
}

function Token *
token_array_iter_begin(Token_Array *array){
  array->iter_len = 0;
  array->iter_block = 0;
  array->iter_bucket = &array->first;
  Token *result = token_array_iter_next(array);
  return result;
}