1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
|
#pragma once
#include "string.h"
using TOKENIZER_COMPARE_FN = FN<U8( char )>;
struct STR_TOKENIZER {
STR str;
STR ignored;
I32 cur;
I32 last;
};
inline STR_TOKENIZER tok_init( STR str, STR whitespace_chars = " \t\n\r" ) {
return {
.str = str,
.ignored = { whitespace_chars },
.cur = 0,
.last = 0,
};
}
inline STR tok_next( STR_TOKENIZER* t ) {
if( t->cur >= t->str.size )
return "";
U8 start = 0;
for( I32 i = t->cur; i < t->str.size; i++ ) {
U8 c = t->str.data[i];
if( i == t->str.size - 1 ) {
if( !start )
return "";
STR ret = { (U32)i - t->cur + 1, t->str.data + t->cur };
t->last = t->cur;
t->cur = i + 1;
return ret;
}
for( auto& it : t->ignored ) {
if( c == it ) {
if( !start ) {
t->cur = i + 1;
continue;
} else {
STR ret = { (U32)i - t->cur, t->str.data + t->cur };
t->last = t->cur;
t->cur = i + 1;
return ret;
}
}
}
start = 1;
}
return "";
}
inline STR tok_peek( STR_TOKENIZER *t ) {
if( t->cur >= t->str.size )
return "";
U8 start = 0;
I32 cur = t->cur;
for( I32 i = cur; i < t->str.size; i++ ) {
U8 c = t->str.data[i];
if( i == t->str.size - 1 ) {
if( !start )
return "";
STR ret = { (U32)i - cur + 1, t->str.data + cur };
return ret;
}
for( auto& it : t->ignored ) {
if( c == it ) {
if( !start ) {
cur = i + 1;
continue;
} else {
STR ret = { (U32)i - cur, t->str.data + cur };
return ret;
}
}
}
start = 1;
}
return "";
}
inline STR tok_next( STR_TOKENIZER *t, STR what ) {
if( t->cur >= t->str.size )
return "";
if( !what.size )
return "";
for( I32 i = t->cur; i <= t->str.size - what.size; i++ ) {
STR slice = { what.size, t->str.data + i };
if( slice == what ) {
STR ret = { (U32)i - t->cur, t->str.data + i + what.size };
t->last = t->cur;
t->cur = i + what.size;
return ret;
}
}
return "";
}
|
