code wiki / _hdl_build / nx_token_counter_gate.nx
nx_token_counter_gate.nx source
↩ module page · 40 lines · 2706 B
1// nx_token_counter_gate.nx -- proves the LLM token estimate is sane and, critically, that pack_text NEVER exceeds
2// the token budget (the guarantee that makes "fit a book into an LLM" safe) + marks truncation honestly when the
3// document doesn't fully fit (and does NOT mark it when it does). license_tier: ORIGINAL
4import "nx_token_counter.nx"
5
6func tg_slen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
7func tg_contains(b: *u8, n: i64, pat: *u8) -> i64 {
8 let pl: i64 = tg_slen(pat); if pl == 0 { return 1 }
9 var i: i64 = 0
10 while i + pl <= n { var k: i64 = 0; var ok: i64 = 1; while k < pl { if b[i+k] != pat[k] { ok = 0; k = pl } else { k = k + 1 } } if ok == 1 { return 1 } i = i + 1 }
11 return 0
12}
13func tg_w(s: *u8) -> i64 { sys_write(1, s, tg_slen(s)); return 0 }
14
15func main() -> i64 {
16 var fails: i64 = 0
17 // tok_estimate: chars/4 floored by words, slight over-estimate
18 if tok_estimate("hello world" as *u8, 11) != 3 { fails = fails + 1 } // (11+3)/4=3 > 2 words
19 if tok_estimate("" as *u8, 0) != 0 { fails = fails + 1 } // empty
20 if tok_estimate("the quick brown fox jumps over the lazy dog" as *u8, 43) != 11 { fails = fails + 1 } // (43+3)/4=11 > 9 words
21 // pack_text: 3 lines, ~13 tokens total
22 let txt: *u8 = "alpha beta gamma\ndelta epsilon zeta\neta theta iota\n" as *u8
23 let tn: i64 = 51
24 let out: *u8 = sys_mmap(8192)
25 let incl: *i64 = sys_mmap(16) as *i64; let omit: *i64 = sys_mmap(16) as *i64
26 // budget 8 cannot fit all 3 -> something dropped, NEVER exceeds 8
27 let used: i64 = pack_text(txt, tn, 8, out, 8192, incl, omit)
28 if used > 8 { fails = fails + 1 } // THE guarantee: never over budget
29 if incl[0] + omit[0] != 3 { fails = fails + 1 } // every line accounted for
30 if omit[0] < 1 { fails = fails + 1 } // budget 8 forces a drop
31 if tg_contains(out, tg_slen(out), "TRUNCATED" as *u8) == 0 { fails = fails + 1 } // truncation is MARKED
32 // huge budget -> everything fits, NO truncation marker
33 let used2: i64 = pack_text(txt, tn, 100000, out, 8192, incl, omit)
34 if omit[0] != 0 { fails = fails + 1 }
35 if incl[0] != 3 { fails = fails + 1 }
36 if tg_contains(out, tg_slen(out), "TRUNCATED" as *u8) == 1 { fails = fails + 1 } // not marked when complete
37 if fails == 0 { tg_w("GATE nx_token_counter verdict=GREEN pass=10/10 (estimate sane; pack NEVER exceeds budget; truncation marked iff dropped)\n" as *u8); sys_exit(0); return 0 }
38 tg_w("GATE nx_token_counter verdict=RED fails\n" as *u8); sys_exit(1)
39 return 1
40}