1 files changed, 114 insertions, 0 deletions
diff --git a/src/util/str_tokenizer.h b/src/util/str_tokenizer.h
new file mode 100644
index 0000000..f316104
--- /dev/null
+++ b/src/util/str_tokenizer.h
@@ -0,0 +1,114 @@
+#pragma once
+
+#include "string.h"
+
+using TOKENIZER_COMPARE_FN = FN<U8( char )>;
+
+struct STR_TOKENIZER {
+  STR str;
+  STR ignored;
+
+  I32 cur;
+  I32 last;
+};
+
+inline STR_TOKENIZER tok_init( STR str, STR whitespace_chars = " \t\n\r" ) {
+  return {
+    .str = str,
+    .ignored = { whitespace_chars },
+    .cur = 0,
+    .last = 0,
+  };
+}
+
+inline STR tok_next( STR_TOKENIZER* t ) {
+  if( t->cur >= t->str.size )
+    return "";
+
+  U8 start = 0;
+  for( I32 i = t->cur; i < t->str.size; i++ ) {
+    U8 c = t->str.data[i];
+    if( i == t->str.size - 1 ) {
+      if( !start )
+        return "";
+
+      STR ret = { (U32)i - t->cur + 1, t->str.data + t->cur };
+      t->last = t->cur;
+      t->cur = i + 1;
+      return ret;
+    }
+
+    for( auto& it : t->ignored ) {
+      if( c == it ) {
+        if( !start ) {
+          t->cur = i + 1;
+          continue;
+        } else {
+          STR ret = { (U32)i - t->cur, t->str.data + t->cur };
+          t->last = t->cur;
+          t->cur = i + 1;
+          return ret;
+        }
+      }
+    }
+
+    start = 1;
+  }
+
+  return "";
+}
+
+inline STR tok_peek( STR_TOKENIZER *t ) {
+  if( t->cur >= t->str.size )
+    return "";
+
+  U8 start = 0;
+  I32 cur = t->cur;
+  for( I32 i = cur; i < t->str.size; i++ ) {
+    U8 c = t->str.data[i];
+    if( i == t->str.size - 1 ) {
+      if( !start )
+        return "";
+
+      STR ret = { (U32)i - cur + 1, t->str.data + cur };
+      return ret;
+    }
+
+    for( auto& it : t->ignored ) {
+      if( c == it ) {
+        if( !start ) {
+          cur = i + 1;
+          continue;
+        } else {
+          STR ret = { (U32)i - cur, t->str.data + cur };
+          return ret;
+        }
+      }
+    }
+
+    start = 1;
+  }
+
+  return "";
+}
+
+
+inline STR tok_next( STR_TOKENIZER *t, STR what ) {
+  if( t->cur >= t->str.size )
+    return "";
+
+  if( !what.size )
+    return "";
+
+  for( I32 i = t->cur; i <= t->str.size - what.size; i++ ) {
+    STR slice = { what.size, t->str.data + i };
+    if( slice == what ) {
+      STR ret = { (U32)i - t->cur, t->str.data + i + what.size };
+      t->last = t->cur;
+      t->cur = i + what.size;
+      return ret;
+    }
+  }
+
+  return "";
+}