nx_dr_corpus_cli.nx source
↩ module page · 173 lines · 8324 B
1// nx_dr_corpus_cli.nx -- the sovereign deep-research engine over a REAL ON-DISK CORPUS (DR-9).
2// Same one-call loop as nx_dr_chain (judge -> conduct -> score + budget-cost) but each source is
3// a FILE PATH, so it runs over the documents the live nx_fetch spine already writes
4// (knowledge/library/*.txt) instead of argv-sized strings. This is the scale path to a
5// full-corpus benchmark run.
6// nx_dr_corpus <threshold_permil> <budget> <n_insights> <insight_1..n> <cost_1> <path_1> ...
7// e.g. nx_dr_corpus 600 3 2 "tournament of ideas" "liver fibrosis" \
8// 1 knowledge/library/coscientist.txt 5 knowledge/library/alphafold.txt
9// FAIL-LOUD: an unreadable source is reported (readable:0, counted in "unreadable"), never
10// silently scored as zero coverage.
11// license_tier: ORIGINAL No hw writes (Rule 26). expect_exit: 0
12import "nx_dr_corpus.nx"
13import "nx_itoa_lib.nx" // shared MSB-first emitter (zero-alloc)
14import "nx_dr_run.nx"
15import "nx_dr_conductor.nx"
16import "nx_dr_drbench.nx"
17import "nx_syscalls.nx"
18const K_MAGIC_8192: i64 = 8192
19const K_MAGIC_1024: i64 = 1024
20const K_MAGIC_262144: i64 = 262144
21const K_MAGIC_65536: i64 = 65536
22
23func cp_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
24func cp_q() -> i64 { let b: *u8 = sys_mmap(1); b[0] = 34 as u8; sys_write(1, b, 1); return 0 }
25func cp_key(name: *u8) -> i64 { cp_q(); cp_puts(name); cp_q(); cp_puts(":" as *u8); return 0 }
26// MIGRATED to the shared emitter (debt 1785563586). The old body mmapped a scratch buffer
27// per call and never freed it. At PAGE granularity that is 4096B leaked PER CALL -- the
28// defect that took 28.5GB of a 36GB host in nx_ts_lumadiff (2MB input, ~3.66M calls).
29// nxi_* is MSB-first, allocates NOTHING, and emits identical bytes including the sign.
30func cp_num(v: i64) -> i64 { nxi_out(v); return 0 }
31func cp_atoi(a: *u8) -> i64 {
32 var v: i64 = 0; var i: i64 = 0
33 while a[i] != (0 as u8) {
34 let c: i64 = a[i] as i64
35 if c >= 48 { if c <= 57 { v = v * 10 + (c - 48) } }
36 i = i + 1
37 }
38 return v
39}
40
41func main(argc: i64, argv: *i64) -> i64 {
42 if argc < 7 { cp_puts("{" as *u8); cp_key("error" as *u8); cp_q(); cp_puts("usage: nx_dr_corpus <threshold> <budget> <n_insights> <insight..> <cost path>.." as *u8); cp_q(); cp_puts("}\n" as *u8); sys_exit(2); return 2 }
43 let threshold: i64 = cp_atoi(argv[1] as *u8)
44 let budget: i64 = cp_atoi(argv[2] as *u8)
45 let ni: i64 = cp_atoi(argv[3] as *u8)
46 if ni < 1 { cp_puts("{" as *u8); cp_key("error" as *u8); cp_q(); cp_puts("n_insights must be >= 1" as *u8); cp_q(); cp_puts("}\n" as *u8); sys_exit(1); return 1 }
47 let base: i64 = 4 + ni
48 let ndocs: i64 = (argc - base) / 2
49 if ndocs < 1 { cp_puts("{" as *u8); cp_key("error" as *u8); cp_q(); cp_puts("need >=1 <cost path> pair" as *u8); cp_q(); cp_puts("}\n" as *u8); sys_exit(1); return 1 }
50
51 // ---- pack the shared groundtruth insight set (insights stay argv strings)
52 let ins_pack: *i64 = sys_mmap((1 + 2 * ni + K_MAGIC_8192) * 8) as *i64
53 ins_pack[0] = ni
54 let ins_tok: *i64 = ((ins_pack as i64) + (1 + 2 * ni) * 8) as *i64
55 var off: i64 = 0
56 var i: i64 = 0
57 while i < ni {
58 let s: *u8 = argv[4 + i] as *u8
59 let dst: *i64 = ((ins_tok as i64) + off * 8) as *i64
60 let ln: i64 = drr_tokenize(s, drr_strlen(s), dst, K_MAGIC_1024)
61 ins_pack[1 + i] = off
62 ins_pack[1 + ni + i] = ln
63 off = off + ln
64 i = i + 1
65 }
66
67 // ---- read + pack every source document FROM DISK (fail-loud on unreadable)
68 let doc_pack: *i64 = sys_mmap((1 + ndocs + K_MAGIC_262144) * 8) as *i64
69 doc_pack[0] = ndocs
70 let doc_tok: *i64 = ((doc_pack as i64) + (1 + ndocs) * 8) as *i64
71 let cost: *i64 = sys_mmap(ndocs * 8) as *i64
72 let readable: *i64 = sys_mmap(ndocs * 8) as *i64
73 var unreadable: i64 = 0
74 var doff: i64 = 0
75 var d: i64 = 0
76 while d < ndocs {
77 cost[d] = cp_atoi(argv[base + d * 2] as *u8)
78 let path: *u8 = argv[base + d * 2 + 1] as *u8
79 let dst: *i64 = ((doc_tok as i64) + doff * 8) as *i64
80 let dl: i64 = corp_tokenize_file(path, dst, K_MAGIC_65536)
81 if dl < 0 {
82 readable[d] = 0
83 doc_pack[1 + d] = 0
84 unreadable = unreadable + 1
85 } else {
86 readable[d] = 1
87 doc_pack[1 + d] = dl
88 doff = doff + dl
89 }
90 d = d + 1
91 }
92
93 // ---- STAGE 1 judge
94 let sup: *i64 = sys_mmap(ndocs * 8) as *i64
95 let cov: *i64 = sys_mmap(ndocs * ni * 8) as *i64
96 chain_judge_all(ins_pack, doc_pack, threshold, sup, cov)
97
98 let allsel: *i64 = sys_mmap(ndocs * 8) as *i64
99 var a: i64 = 0
100 while a < ndocs { allsel[a] = 1; a = a + 1 }
101 let allcov: *i64 = sys_mmap(ni * 8) as *i64
102 chain_sel_cov(cov, allsel, ndocs, ni, allcov)
103 let recall_all: i64 = dr_frac(allcov, ni)
104
105 // ---- STAGE 2 conduct
106 let total: *i64 = sys_mmap(ndocs * 8) as *i64
107 var t2: i64 = 0
108 while t2 < ndocs { total[t2] = ni; t2 = t2 + 1 }
109 let out: *i64 = sys_mmap((5 * ndocs + 1) * 8) as *i64
110 let cnt: i64 = cond_run(sup, total, cost, ndocs, budget, out)
111 let ql: *i64 = out
112 let rt: *i64 = ((out as i64) + ndocs * 8) as *i64
113 let pri: *i64 = ((out as i64) + ndocs * 16) as *i64
114 let ord: *i64 = ((out as i64) + ndocs * 24) as *i64
115 let sel: *i64 = ((out as i64) + ndocs * 32) as *i64
116 let spent: i64 = out[5 * ndocs]
117
118 // ---- STAGE 3 score
119 let selcov: *i64 = sys_mmap(ni * 8) as *i64
120 chain_sel_cov(cov, sel, ndocs, ni, selcov)
121 let recall_sel: i64 = dr_frac(selcov, ni)
122 var retained: i64 = 1000
123 if recall_all > 0 { retained = (recall_sel * 1000) / recall_all }
124
125 cp_puts("{" as *u8)
126 cp_key("tool" as *u8); cp_q(); cp_puts("nx_dr_corpus" as *u8); cp_q(); cp_puts("," as *u8)
127 cp_key("n_insights" as *u8); cp_num(ni); cp_puts("," as *u8)
128 cp_key("n_sources" as *u8); cp_num(ndocs); cp_puts("," as *u8)
129 cp_key("unreadable" as *u8); cp_num(unreadable); cp_puts("," as *u8)
130 cp_key("threshold_permil" as *u8); cp_num(threshold); cp_puts("," as *u8)
131 cp_key("budget" as *u8); cp_num(budget); cp_puts("," as *u8)
132 cp_key("selected" as *u8); cp_num(cnt); cp_puts("," as *u8)
133 cp_key("spent" as *u8); cp_num(spent); cp_puts("," as *u8)
134 cp_key("recall_all_permil" as *u8); cp_num(recall_all); cp_puts("," as *u8)
135 cp_key("recall_selected_permil" as *u8); cp_num(recall_sel); cp_puts("," as *u8)
136 cp_key("budget_cost_permil" as *u8); cp_num(recall_all - recall_sel); cp_puts("," as *u8)
137 cp_key("recall_retained_permil" as *u8); cp_num(retained); cp_puts("," as *u8)
138 cp_key("sources" as *u8); cp_puts("[" as *u8)
139 var r: i64 = 0
140 while r < ndocs {
141 if r > 0 { cp_puts("," as *u8) }
142 let id: i64 = ord[r]
143 cp_puts("{" as *u8)
144 cp_key("rank" as *u8); cp_num(r + 1); cp_puts("," as *u8)
145 cp_key("id" as *u8); cp_num(id); cp_puts("," as *u8)
146 cp_key("readable" as *u8); cp_num(readable[id]); cp_puts("," as *u8)
147 cp_key("doc_tokens" as *u8); cp_num(doc_pack[1 + id]); cp_puts("," as *u8)
148 cp_key("supported" as *u8); cp_num(sup[id]); cp_puts("," as *u8)
149 cp_key("quality_permil" as *u8); cp_num(ql[id]); cp_puts("," as *u8)
150 cp_key("elo" as *u8); cp_num(rt[id]); cp_puts("," as *u8)
151 cp_key("priority" as *u8); cp_num(pri[id]); cp_puts("," as *u8)
152 cp_key("cost" as *u8); cp_num(cost[id]); cp_puts("," as *u8)
153 cp_key("selected" as *u8); cp_num(sel[id])
154 cp_puts("}" as *u8)
155 r = r + 1
156 }
157 cp_puts("]," as *u8)
158 cp_key("insights" as *u8); cp_puts("[" as *u8)
159 var k2: i64 = 0
160 while k2 < ni {
161 if k2 > 0 { cp_puts("," as *u8) }
162 cp_puts("{" as *u8)
163 cp_key("idx" as *u8); cp_num(k2); cp_puts("," as *u8)
164 cp_key("achievable" as *u8); cp_num(allcov[k2]); cp_puts("," as *u8)
165 cp_key("obtained" as *u8); cp_num(selcov[k2])
166 cp_puts("}" as *u8)
167 k2 = k2 + 1
168 }
169 cp_puts("]," as *u8)
170 cp_key("note" as *u8); cp_q(); cp_puts("on-disk corpus: judge(lexical containment)->conduct(Elo+opp-cost budget)->score; unreadable sources reported not silently zeroed; sovereign integer" as *u8); cp_q()
171 cp_puts("}\n" as *u8)
172 sys_exit(0); return 0
173}