nx_slopmeter.nx source
↩ module page · 74 lines · 3392 B
1// nx_slopmeter.nx -- CLI: measure a GENERATED artifact's self-repetition.
2//
3// The first instrument in this estate that reads what the generator actually produced. nx_writebench
4// scores the registry; this scores the bytes. Both are needed and they are not substitutes.
5//
6// nx_slopmeter file <path> repetition + distinct-vocabulary permil for a stored generation
7//
8// Reports numbers, never a verdict: the cutoff between "textured" and "slop" is a policy that has to
9// come from measurements across real generations, not from a constant typed at the keyboard.
10// license_tier: ORIGINAL
11// module: nishi-core.write.slopmeter.cli
12// capability: WRITE_OUTPUT_REPETITION_MEASURE
13import "nx_estate_path.nx"
14import "nx_itoa_lib.nx"
15import "nx_slopmeter_lib.nx"
16
17// Read budget for one generation. PROVENANCE: the write lane caps a request at 1200 tokens and the
18// largest .done observed this session is 1681 bytes; 1 MiB is ~600x that, so a stored generation
19// cannot silently truncate. Oversize is REPORTED, never quietly shrunk.
20const SC_BUF_BYTES: i64 = 1048576
21// Token ceiling. 1200 tokens is the lane's own max_tokens, so 16384 cannot be reached by a
22// well-formed generation; a text that hits it is malformed and the count is declared either way.
23const SC_TOKCAP: i64 = 16384
24
25func cw(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
26func ce(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(2, s, n); return 0 }
27
28func sc_streq(a: *u8, b: *u8) -> i64 {
29 var i: i64 = 0
30 while a[i] != (0 as u8) { if a[i] != b[i] { return 0 } i = i + 1 }
31 if b[i] != (0 as u8) { return 0 }
32 return 1
33}
34
35func main(argc: i64, argv: *i64) -> i64 {
36 if argc < 3 {
37 cw("usage: nx_slopmeter file <path>\n" as *u8)
38 cw(" repetition_permil = share of 3-gram windows that echo an earlier one\n" as *u8)
39 cw(" distinct_permil = distinct tokens per 1000 tokens (an INDEPENDENT axis)\n" as *u8)
40 return 2
41 }
42 if sc_streq(argv[1] as *u8, "file" as *u8) == 0 { ce("nx_slopmeter: unknown mode\n" as *u8); return 2 }
43 let path: *u8 = argv[2] as *u8
44
45 let fd: i64 = ep_open_rd(path)
46 if fd < 0 { ce("nx_slopmeter: cannot open " as *u8); ce(path); ce("\n" as *u8); return 3 }
47 let buf: *u8 = sys_mmap(SC_BUF_BYTES)
48 var n: i64 = 0
49 var go: i64 = 1
50 while go == 1 {
51 if n >= SC_BUF_BYTES { go = 0 } else {
52 let r: i64 = sys_read(fd, ((buf as i64) + n) as *u8, SC_BUF_BYTES - n)
53 if r <= 0 { go = 0 } else { n = n + r }
54 }
55 }
56 sys_close(fd)
57
58 let st: *i64 = sys_mmap(SC_TOKCAP * 8) as *i64
59 let ln: *i64 = sys_mmap(SC_TOKCAP * 8) as *i64
60 let nt: i64 = sm_tokenize(buf, n, st, ln, SC_TOKCAP)
61 let rep: i64 = sm_repeat_permil(buf, st, ln, nt)
62 let dis: i64 = sm_distinct_permil(buf, st, ln, nt)
63
64 cw("NX-SLOPMETER bytes=" as *u8); nxi_out(n)
65 cw(" tokens=" as *u8); nxi_out(nt)
66 cw(" ngram=" as *u8); nxi_out(SM_NGRAM)
67 cw(" repetition_permil=" as *u8)
68 if rep == SM_UNMEASURABLE { cw("UNMEASURABLE" as *u8) } else { nxi_out(rep) }
69 cw(" distinct_permil=" as *u8)
70 if dis == SM_UNMEASURABLE { cw("UNMEASURABLE" as *u8) } else { nxi_out(dis) }
71 // no verdict: see the header. A number with a stated width and unit is the whole product here.
72 cw("\n" as *u8)
73 return 0
74}