code wiki / _hdl_build / nx_dr_semcorpus.nx
nx_dr_semcorpus.nx source
↩ module page · 144 lines · 6660 B
1// nx_dr_semcorpus.nx -- BATCH semantic judge (the at-scale enabler; eats the model-reload debt).
2// nx_dr_semjudge_cli reloads the 41MB PPMI model per PROCESS: a 16-doc x 81-insight run
3// reloaded it 16x = ~21 min, ALL of it model-load. This loads the model ONCE and scores MANY
4// doc FILES against a SHARED insight set, so a full-corpus judge is no longer reload-bound.
5// Behaviour-preserving BY COMPOSITION: it calls the SAME sj_load/sj_tokenize_ids/sj_maxsim
6// (=> nx_ppmi_lib) as nx_dr_semjudge, so per (doc,insight) semantic_permil is bit-identical to
7// the per-call judge -- the gate proves that against the live incumbent, not by assertion.
8// nx_dr_semcorpus <insights_file (one insight per line)> <doc_file_1> [doc_file_2 ...]
9// -> {"tool":"nx_dr_semcorpus","vocab":V,"n_insights":K,"docs":[
10// {"doc":0,"readable":1,"tokens":N,"scores":[s0,..,sK-1]}, ...]}
11// An unreadable doc reports readable:0 (fail-loud), never a silent all-zero row.
12// Every func <=6 params (NAS nx_cc >6-arg skew, seq239). No hardware writes (Rule 26).
13// license_tier: ORIGINAL expect_exit: 0
14// module: nishi-core.research.dr_semcorpus depends: nx_dr_semjudge.nx
15// v3 (OO tranche 1): local sc_puts/sc_q/sc_num DELETED -> the canonical nx_json_emit lib.
16// Migration is BYTE-EXACT gated (same inputs, identical output bytes) -- zero lost function.
17import "nx_dr_semjudge.nx"
18import "nx_json_emit.nx"
19import "nx_syscalls.nx"
20const K_MAGIC_262144: i64 = 262144
21
22const K_DBUF: i64 = 4194304
23const K_DTOK: i64 = 16384
24const K_INS: i64 = 1024
25const K_IPOOL: i64 = 65536
26const K_IBUF: i64 = 4194304
27
28// Read a whole file into buf (cap bytes). Returns bytes read, or -1 if the path won't open.
29func sc_readfile(path: *u8, buf: *u8, cap: i64) -> i64 {
30 let fd: i64 = sys_openat_rd(path)
31 if fd < 0 { return 0 - 1 }
32 var total: i64 = 0
33 var r: i64 = 1
34 while r > 0 {
35 let left: i64 = cap - total
36 if left <= 0 { r = 0 } else {
37 var want: i64 = K_MAGIC_262144
38 if want > left { want = left }
39 r = sys_read(fd, (buf as i64 + total) as *u8, want)
40 if r > 0 { total = total + r }
41 }
42 }
43 sys_close(fd)
44 return total
45}
46
47func sc_kv_err(msg: *u8) -> i64 {
48 je_puts("{" as *u8); je_q(); je_puts("error" as *u8); je_q(); je_puts(":" as *u8)
49 je_q(); je_puts(msg); je_q(); je_puts("}\n" as *u8); return 0
50}
51
52func main(argc: i64, argv: *i64) -> i64 {
53 if argc < 3 { sc_kv_err("usage: nx_dr_semcorpus <insights_file> <doc_file>..." as *u8); sys_exit(2); return 2 }
54 let g: *i64 = sys_mmap(128 * 8) as *i64
55 if sj_load(g) == 0 { sc_kv_err("semppmi_v1.bin load failed (model at knowledge/index/)" as *u8); sys_exit(1); return 1 }
56
57 // Read + tokenize insights (one per line) into a flat id pool.
58 let ibuf: *u8 = sys_mmap(K_IBUF)
59 let ilen: i64 = sc_readfile(argv[1] as *u8, ibuf, K_IBUF)
60 if ilen < 0 { sc_kv_err("insights_file unreadable" as *u8); sys_exit(1); return 1 }
61 let ipool: *i64 = sys_mmap(K_IPOOL * 8) as *i64
62 let ioff: *i64 = sys_mmap(K_INS * 8) as *i64
63 let icnt: *i64 = sys_mmap(K_INS * 8) as *i64
64 var nins: i64 = 0
65 var pool: i64 = 0
66 var ls: i64 = 0
67 var p: i64 = 0
68 while p <= ilen {
69 var brk: i64 = 0
70 if p == ilen { brk = 1 } else { if ibuf[p] == (10 as u8) { brk = 1 } }
71 if brk == 1 {
72 if p > ls {
73 if nins < K_INS {
74 let room: i64 = K_IPOOL - pool
75 if room > 0 {
76 let scratch: *i64 = (ipool as i64 + pool*8) as *i64
77 let nc: i64 = sj_tokenize_ids(g, (ibuf as i64 + ls) as *u8, p - ls, scratch, room)
78 ioff[nins] = pool
79 icnt[nins] = nc
80 pool = pool + nc
81 nins = nins + 1
82 }
83 }
84 }
85 ls = p + 1
86 }
87 p = p + 1
88 }
89
90 let dbuf: *u8 = sys_mmap(K_DBUF)
91 let dtok: *i64 = sys_mmap(K_DTOK * 8) as *i64
92 // v2: dedup scratch. maxsim takes MAX over doc terms, so duplicate valid ids recompute the
93 // same cosine for nothing and OOV ids (-1) always contribute 0. Judging over the UNIQUE
94 // VALID doc terms is bit-identical and cuts the inner loop from ~8000 to the vocab set.
95 let dtok2: *i64 = sys_mmap(K_DTOK * 8) as *i64
96 let seen: *u8 = sys_mmap(g[71] + 16)
97
98 je_puts("{" as *u8)
99 je_q(); je_puts("tool" as *u8); je_q(); je_puts(":" as *u8); je_q(); je_puts("nx_dr_semcorpus" as *u8); je_q(); je_puts("," as *u8)
100 je_q(); je_puts("vocab" as *u8); je_q(); je_puts(":" as *u8); je_num(g[71]); je_puts("," as *u8)
101 je_q(); je_puts("n_insights" as *u8); je_q(); je_puts(":" as *u8); je_num(nins); je_puts("," as *u8)
102 je_q(); je_puts("docs" as *u8); je_q(); je_puts(":[" as *u8)
103
104 var d: i64 = 2
105 while d < argc {
106 if d > 2 { je_puts("," as *u8) }
107 let dl: i64 = sc_readfile(argv[d] as *u8, dbuf, K_DBUF)
108 je_puts("{" as *u8)
109 je_q(); je_puts("doc" as *u8); je_q(); je_puts(":" as *u8); je_num(d - 2); je_puts("," as *u8)
110 if dl < 0 {
111 je_q(); je_puts("readable" as *u8); je_q(); je_puts(":0," as *u8)
112 je_q(); je_puts("tokens" as *u8); je_q(); je_puts(":0," as *u8)
113 je_q(); je_puts("scores" as *u8); je_q(); je_puts(":[]" as *u8)
114 } else {
115 let nd: i64 = sj_tokenize_ids(g, dbuf, dl, dtok, K_DTOK)
116 var nd2: i64 = 0
117 var ti: i64 = 0
118 while ti < nd {
119 let id: i64 = dtok[ti]
120 if id >= 0 { if seen[id] == (0 as u8) { seen[id] = 1 as u8; dtok2[nd2] = id; nd2 = nd2 + 1 } }
121 ti = ti + 1
122 }
123 je_q(); je_puts("readable" as *u8); je_q(); je_puts(":1," as *u8)
124 je_q(); je_puts("tokens" as *u8); je_q(); je_puts(":" as *u8); je_num(nd); je_puts("," as *u8)
125 je_q(); je_puts("unique" as *u8); je_q(); je_puts(":" as *u8); je_num(nd2); je_puts("," as *u8)
126 je_q(); je_puts("scores" as *u8); je_q(); je_puts(":[" as *u8)
127 var k: i64 = 0
128 while k < nins {
129 if k > 0 { je_puts("," as *u8) }
130 let ins: *i64 = (ipool as i64 + ioff[k]*8) as *i64
131 let sv: i64 = sj_maxsim(g, ins, icnt[k], dtok2, nd2)
132 je_num(sv)
133 k = k + 1
134 }
135 je_puts("]" as *u8)
136 var ci: i64 = 0
137 while ci < nd2 { seen[dtok2[ci]] = 0 as u8; ci = ci + 1 }
138 }
139 je_puts("}" as *u8)
140 d = d + 1
141 }
142 je_puts("]}\n" as *u8)
143 sys_exit(0); return 0
144}