code wiki / _hdl_build / nx_token_counter.nx
nx_token_counter.nx source
↩ module page · 67 lines · 3383 B
1// nx_token_counter.nx -- SOVEREIGN LLM token estimation + budget-aware context packing: the "push our books into
2// an LLM natively / build for RAG" foundation the library lacked (recon: BM25 + rerank exist, but no token counter
3// and no context-packer). tok_estimate uses the well-known ~4-chars/token rule, FLOORED by a word count, as a
4// deliberate slight OVER-estimate so budgeting never overflows the real context window. pack_text greedily keeps
5// input lines IN ORDER until the token budget is hit, and emits an honest truncation marker if the document does
6// not fully fit. HONEST: this is an ESTIMATE (no BPE table) for budgeting with a safety margin, not exact billing.
7// license_tier: ORIGINAL
8import "nx_syscalls.nx"
9
10func tc_isspace(c: i64) -> i64 { if c == 32 { return 1 } if c == 10 { return 1 } if c == 9 { return 1 } if c == 13 { return 1 } return 0 }
11
12// estimate LLM tokens for b[0..n): max(ceil(chars/4), word_count). Over-estimates slightly -> safe for budgeting.
13func tok_estimate(b: *u8, n: i64) -> i64 {
14 if n <= 0 { return 0 }
15 var words: i64 = 0; var inw: i64 = 0; var i: i64 = 0
16 while i < n {
17 if tc_isspace(b[i] as i64) == 0 { if inw == 0 { words = words + 1; inw = 1 } } else { inw = 0 }
18 i = i + 1
19 }
20 let bychar: i64 = (n + 3) / 4
21 if bychar > words { return bychar }
22 return words
23}
24
25func tc_cat(out: *u8, o: i64, s: *u8, outcap: i64) -> i64 {
26 var i: i64 = 0
27 while s[i] != (0 as u8) { if o < outcap - 1 { out[o] = s[i]; o = o + 1 } i = i + 1 }
28 return o
29}
30func tc_catn(out: *u8, o: i64, v: i64, outcap: i64) -> i64 {
31 if v == 0 { if o < outcap - 1 { out[o] = 48 as u8; o = o + 1 } return o }
32 let t: *u8 = sys_mmap(32); var k: i64 = 0; var m: i64 = v
33 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
34 while k > 0 { k = k - 1; if o < outcap - 1 { out[o] = t[k]; o = o + 1 } }
35 return o
36}
37
38// pack b[0..n) line-by-line into out (NUL-terminated), keeping lines IN ORDER until `cap` tokens are used; append
39// a truncation marker if any line was omitted. returns tokens used. incl[0]=lines kept, omit[0]=lines dropped.
40func pack_text(b: *u8, n: i64, cap: i64, out: *u8, outcap: i64, incl: *i64, omit: *i64) -> i64 {
41 var used: i64 = 0; var o: i64 = 0; var included: i64 = 0; var omitted: i64 = 0
42 var i: i64 = 0
43 while i < n {
44 var le: i64 = i
45 var go: i64 = 1
46 while go == 1 { if le >= n { go = 0 } else { if b[le] == (10 as u8) { go = 0 } else { le = le + 1 } } }
47 let lt: i64 = tok_estimate((b as i64 + i) as *u8, le - i)
48 if used + lt <= cap {
49 var k: i64 = i
50 while k <= le { if k < n { if o < outcap - 1 { out[o] = b[k]; o = o + 1 } } k = k + 1 }
51 used = used + lt; included = included + 1
52 } else { omitted = omitted + 1 }
53 i = le + 1
54 }
55 if omitted > 0 {
56 o = tc_cat(out, o, "\n[TRUNCATED: " as *u8, outcap)
57 o = tc_catn(out, o, omitted, outcap)
58 o = tc_cat(out, o, " of " as *u8, outcap)
59 o = tc_catn(out, o, included + omitted, outcap)
60 o = tc_cat(out, o, " lines omitted to fit the token budget; " as *u8, outcap)
61 o = tc_catn(out, o, used, outcap)
62 o = tc_cat(out, o, " tokens packed]\n" as *u8, outcap)
63 }
64 if o < outcap { out[o] = 0 as u8 }
65 incl[0] = included; omit[0] = omitted
66 return used
67}