nx_code_index.nx source
↩ module page · 480 lines · 16672 B
1// nx_code_index.nx -- E-ARC e3 (2026-07-15): SEMANTIC ORGAN RETRIEVAL for the forge (pure core;
2// CLI = nx_code_index_cli.nx, gate = nx_code_index_gate.nx -- the CRM idiom).
3// INDEX: per source file, embed "FILE: <basename>" + the leading comment header via the LIVE
4// no-float coder serve POST /embed (127.0.0.1:<port>, jina recipe arXiv:2508.21290, kind=passage),
5// append a record to a sovereign binary store. QUERY: embed kind=query, integer-cosine scan, top-1.
6// STORE NXCI1 (sovereign binary, no TSV): magic 8B "NXCI1\n\0\0" then records
7// [nlen i64][name nlen bytes][ntok i64][vec CI_NE i64].
8// IDEMPOTENT: a name already present is SKIPPED on build (crash-resumable re-runs, rule 10).
9// OVERFLOW LAW: cosine pre-scales both vecs by the SAME pow2 so max|e| <= 2^20 (raw q24 elements
10// reach ~2^29 and their squares would overflow i64 across 896 dims).
11// Production embeds run on OUR BASE coder weights (jina-tuned = benchmark oracle only, cc-by-nc).
12// license_tier: ORIGINAL No hw writes (Rule 26).
13import "nx_forge_model.nx"
14import "nx_connect.nx" // bounded connect: a raw sys_connect hangs ~127s on a black-holed host
15import "nx_lib_std.nx"
16const CI_MAGIC_1048575: i64 = 1048575
17const CI_MAGIC_262144: i64 = 262144
18const CI_MAGIC_8192: i64 = 8192
19const CI_MAGIC_65536: i64 = 65536
20const CI_MAGIC_8190: i64 = 8190
21const CI_MAGIC_2000: i64 = 2000
22
23const CI_NE: i64 = 896
24const CI_HDRCAP: i64 = 480
25const CI_NAMECAP: i64 = 256
26
27// ---- tiny out helpers ----
28func ci_ws(s: *u8) -> i64 {
29 var n: i64 = 0
30 while s[n] != (0 as u8) { n = n + 1 }
31 sys_write(1, s, n)
32 return 0
33}
34
35func ci_kv(label: *u8, v: i64) -> i64 {
36 ci_ws(label)
37 ci_ws("=" as *u8)
38 std_pdec(v)
39 ci_ws(" " as *u8)
40 return 0
41}
42
43// ---- integer cosine (per-mille), overflow-safe pre-scale ----
44func ci_isqrt(v: i64) -> i64 {
45 if v < 2 { return v }
46 var x: i64 = v
47 var y: i64 = (x + 1) / 2
48 while y < x { x = y; y = (x + v / x) / 2 }
49 return x
50}
51
52func ci_cos_pm(a: *i64, b: *i64) -> i64 {
53 var mx: i64 = 1
54 var i: i64 = 0
55 while i < CI_NE {
56 var va: i64 = a[i]
57 if va < 0 { va = 0 - va }
58 var vb: i64 = b[i]
59 if vb < 0 { vb = 0 - vb }
60 if va > mx { mx = va }
61 if vb > mx { mx = vb }
62 i = i + 1
63 }
64 var dv: i64 = 1
65 while mx > CI_MAGIC_1048575 { mx = mx / 2; dv = dv * 2 }
66 var dab: i64 = 0
67 var daa: i64 = 0
68 var dbb: i64 = 0
69 i = 0
70 while i < CI_NE {
71 let ea: i64 = a[i] / dv
72 let eb: i64 = b[i] / dv
73 dab = dab + ea * eb
74 daa = daa + ea * ea
75 dbb = dbb + eb * eb
76 i = i + 1
77 }
78 let sa: i64 = ci_isqrt(daa)
79 let sb: i64 = ci_isqrt(dbb)
80 if sa == 0 { return 0 }
81 if sb == 0 { return 0 }
82 return (dab * 1000) / (sa * sb)
83}
84
85// ---- HTTP POST /embed to 127.0.0.1:<port> (fm_post_gen pattern, /embed path) ----
86func ci_post_embed(port: i64, body: *u8, blen: i64, resp: *u8, rescap: i64) -> i64 {
87 let fd: i64 = sys_socket(2, 1, 0)
88 if fd < 0 { return 0 - 1 }
89 let addr: *u8 = sys_mmap(16) as *u8
90 nx_http_client_sockaddr_ipv4(addr, 127, 0, 0, 1, port)
91 let cr: i64 = nx_connect_bounded(fd, addr, 16, NX_CONN_DEFAULT_MS)
92 if cr < 0 { sys_close(fd); return 0 - 1 }
93 let req: *u8 = sys_mmap(CI_MAGIC_262144) as *u8
94 let host: *u8 = "127.0.0.1" as *u8
95 let ct: *u8 = "application/json" as *u8
96 let rl: i64 = nx_http_client_build_request_post("/embed" as *u8, 6, host, 9, ct, 16, body, blen, req)
97 let wr: i64 = sys_write(fd, req, rl)
98 if wr != rl { sys_close(fd); return 0 - 1 }
99 let got: i64 = _drain(fd, resp, rescap)
100 sys_close(fd)
101 return got
102}
103
104// build {"text":"<esc>","task":"nl2code","kind":"query|passage","mode":"i8"}; only the text is
105// escaped (fc_json_esc_bytes); the structural chunks are raw copies. ret len
106func ci_build_body(text: *u8, tlen: i64, kindq: i64, out: *u8) -> i64 {
107 var o: i64 = 0
108 let head: *u8 = "{\"text\":\"" as *u8
109 var j: i64 = 0
110 while head[j] != (0 as u8) { out[o] = head[j]; o = o + 1; j = j + 1 }
111 o = fc_json_esc_bytes(out, o, text, tlen)
112 let mid: *u8 = "\",\"task\":\"nl2code\",\"kind\":\"" as *u8
113 j = 0
114 while mid[j] != (0 as u8) { out[o] = mid[j]; o = o + 1; j = j + 1 }
115 var kw: *u8 = "passage" as *u8
116 if kindq == 1 { kw = "query" as *u8 }
117 j = 0
118 while kw[j] != (0 as u8) { out[o] = kw[j]; o = o + 1; j = j + 1 }
119 let tail: *u8 = "\",\"mode\":\"i8\"}" as *u8
120 j = 0
121 while tail[j] != (0 as u8) { out[o] = tail[j]; o = o + 1; j = j + 1 }
122 out[o] = 0 as u8
123 return o
124}
125
126// find needle in hay[0,hn); -1 if absent
127func ci_find(hay: *u8, hn: i64, needle: *u8) -> i64 {
128 var nl: i64 = 0
129 while needle[nl] != (0 as u8) { nl = nl + 1 }
130 if nl == 0 { return 0 }
131 var i: i64 = 0
132 while i + nl <= hn {
133 var k: i64 = 0
134 var ok: i64 = 1
135 while k < nl { if hay[i+k] != needle[k] { ok = 0; k = nl } else { k = k + 1 } }
136 if ok == 1 { return i }
137 i = i + 1
138 }
139 return 0 - 1
140}
141
142// parse response: "ntok" value into ntok_out[0]; then "vec":[ signed ints into vec (CI_NE required).
143// returns count parsed (CI_NE = success), negative on structural failure.
144func ci_parse_vec(resp: *u8, rlen: i64, vec: *i64, ntok_out: *i64) -> i64 {
145 ntok_out[0] = 0
146 let kn: i64 = ci_find(resp, rlen, "\"ntok\":" as *u8)
147 if kn >= 0 {
148 var p: i64 = kn + 7
149 var nv: i64 = 0
150 var numing: i64 = 1
151 while numing == 1 {
152 if p >= rlen { numing = 0 } else {
153 let c: i64 = resp[p] as i64
154 if c >= 48 { if c <= 57 { nv = nv * 10 + (c - 48); p = p + 1 } else { numing = 0 } } else { numing = 0 }
155 }
156 }
157 ntok_out[0] = nv
158 }
159 let kv: i64 = ci_find(resp, rlen, "\"vec\":[" as *u8)
160 if kv < 0 { return 0 - 2 }
161 var p2: i64 = kv + 7
162 var cnt: i64 = 0
163 var scanning: i64 = 1
164 while scanning == 1 {
165 if p2 >= rlen { scanning = 0 } else {
166 let c2: i64 = resp[p2] as i64
167 if c2 == 93 { scanning = 0 } else {
168 var neg: i64 = 0
169 if c2 == 45 { neg = 1; p2 = p2 + 1 }
170 var v: i64 = 0
171 var digits: i64 = 0
172 var numing2: i64 = 1
173 while numing2 == 1 {
174 if p2 >= rlen { numing2 = 0 } else {
175 let d: i64 = resp[p2] as i64
176 if d >= 48 { if d <= 57 { v = v * 10 + (d - 48); digits = digits + 1; p2 = p2 + 1 } else { numing2 = 0 } } else { numing2 = 0 }
177 }
178 }
179 if digits == 0 { p2 = p2 + 1 } else {
180 if neg == 1 { v = 0 - v }
181 if cnt < CI_NE { vec[cnt] = v }
182 cnt = cnt + 1
183 }
184 }
185 }
186 }
187 return cnt
188}
189
190// embed text through the live serve; vec must hold CI_NE i64. ret ntok (>0) or negative.
191func ci_embed(port: i64, text: *u8, tlen: i64, kindq: i64, vec: *i64) -> i64 {
192 let body: *u8 = sys_mmap(CI_MAGIC_8192) as *u8
193 let bl: i64 = ci_build_body(text, tlen, kindq, body)
194 let resp: *u8 = sys_mmap(CI_MAGIC_65536) as *u8
195 let got: i64 = ci_post_embed(port, body, bl, resp, CI_MAGIC_65536)
196 if got <= 0 { return 0 - 1 }
197 let nt: *i64 = sys_mmap(8) as *i64
198 let cnt: i64 = ci_parse_vec(resp, got, vec, nt)
199 if cnt != CI_NE { return 0 - 3 }
200 var r: i64 = nt[0]
201 if r < 1 { r = 1 }
202 return r
203}
204
205// ---- header extraction: "FILE: <basename>\n" + leading //-or-backslash comment lines ----
206func ci_header(path: *u8, out: *u8, cap: i64) -> i64 {
207 var o: i64 = 0
208 let pre: *u8 = "FILE: " as *u8
209 var j: i64 = 0
210 while pre[j] != (0 as u8) { out[o] = pre[j]; o = o + 1; j = j + 1 }
211 var last: i64 = 0
212 var i: i64 = 0
213 while path[i] != (0 as u8) {
214 if path[i] == (47 as u8) { last = i + 1 }
215 i = i + 1
216 }
217 var b: i64 = last
218 while path[b] != (0 as u8) { out[o] = path[b]; o = o + 1; b = b + 1 }
219 out[o] = 10 as u8
220 o = o + 1
221 let fd: i64 = sys_openat_rd(path)
222 if fd < 0 { return 0 - 1 }
223 let buf: *u8 = sys_mmap(CI_MAGIC_8192) as *u8
224 let n: i64 = sys_read(fd, buf, CI_MAGIC_8190)
225 sys_close(fd)
226 if n <= 0 { return o }
227 var p: i64 = 0
228 var lining: i64 = 1
229 while lining == 1 {
230 if p >= n { lining = 0 } else {
231 var iscmt: i64 = 0
232 if buf[p] == (92 as u8) { iscmt = 1 }
233 if p + 1 < n { if buf[p] == (47 as u8) { if buf[p+1] == (47 as u8) { iscmt = 1 } } }
234 if iscmt == 0 { lining = 0 } else {
235 var copying: i64 = 1
236 while copying == 1 {
237 if p >= n { copying = 0 } else {
238 let c: i64 = buf[p] as i64
239 if o < cap - 2 { out[o] = c as u8; o = o + 1 }
240 p = p + 1
241 if c == 10 { copying = 0 }
242 }
243 }
244 if o >= cap - 2 { lining = 0 }
245 }
246 }
247 }
248 out[o] = 0 as u8
249 return o
250}
251
252// ---- store ----
253func ci_store_ensure(store: *u8) -> i64 {
254 let fd: i64 = sys_openat_rd(store)
255 if fd >= 0 {
256 let hb: *u8 = sys_mmap(16) as *u8
257 let r: i64 = sys_read(fd, hb, 8)
258 sys_close(fd)
259 if r == 8 { if hb[0] == (78 as u8) { return 0 } }
260 }
261 let wf: i64 = sys_openat_wr(store, 420)
262 if wf < 0 { return 0 - 1 }
263 let mg: *u8 = sys_mmap(8) as *u8
264 mg[0] = 78 as u8
265 mg[1] = 88 as u8
266 mg[2] = 67 as u8
267 mg[3] = 73 as u8
268 mg[4] = 49 as u8
269 mg[5] = 10 as u8
270 mg[6] = 0 as u8
271 mg[7] = 0 as u8
272 sys_write(wf, mg, 8)
273 sys_close(wf)
274 return 0
275}
276
277// load whole store; ret len or negative; buf_out[0]=ptr
278func ci_store_load(store: *u8, buf_out: *i64) -> i64 {
279 let lo: *i64 = sys_mmap(8) as *i64
280 lo[0] = 0
281 let b: *u8 = sys_read_file(store, lo)
282 if (b as i64) == 0 { return 0 - 1 }
283 if lo[0] < 8 { return 0 - 1 }
284 if b[0] != (78 as u8) { return 0 - 2 }
285 if b[4] != (49 as u8) { return 0 - 2 }
286 buf_out[0] = b as i64
287 return lo[0]
288}
289
290// does store contain name? 1/0
291func ci_store_has(store: *u8, name: *u8) -> i64 {
292 let bo: *i64 = sys_mmap(8) as *i64
293 let n: i64 = ci_store_load(store, bo)
294 if n < 8 { return 0 }
295 let b: *u8 = bo[0] as *u8
296 let nl_want: i64 = std_slen(name)
297 var off: i64 = 8
298 while off + 16 <= n {
299 let pl: *i64 = (b + off) as *i64
300 let nlen: i64 = pl[0]
301 if nlen < 1 { return 0 }
302 if nlen > CI_NAMECAP { return 0 }
303 let rec_end: i64 = off + 8 + nlen + 8 + CI_NE * 8
304 if rec_end > n { return 0 }
305 if nlen == nl_want {
306 var eq: i64 = 1
307 var k: i64 = 0
308 while k < nlen { if b[off + 8 + k] != name[k] { eq = 0; k = nlen } else { k = k + 1 } }
309 if eq == 1 { return 1 }
310 }
311 off = rec_end
312 }
313 return 0
314}
315
316func ci_store_append(store: *u8, name: *u8, ntok: i64, vec: *i64) -> i64 {
317 let fd: i64 = sys_openat_append(store, 420)
318 if fd < 0 { return 0 - 1 }
319 let nlen: i64 = std_slen(name)
320 let hdr: *i64 = sys_mmap(16) as *i64
321 hdr[0] = nlen
322 sys_write(fd, hdr as *u8, 8)
323 sys_write(fd, name, nlen)
324 hdr[1] = ntok
325 let hp: i64 = (hdr as i64) + 8
326 sys_write(fd, hp as *u8, 8)
327 sys_write(fd, vec as *u8, CI_NE * 8)
328 sys_close(fd)
329 return 0
330}
331
332// ---- index one file (skip if present). ret 1 indexed, 0 skipped, negative error ----
333func ci_index_file(port: i64, store: *u8, path: *u8) -> i64 {
334 let nameb: *u8 = sys_mmap(CI_NAMECAP) as *u8
335 var last: i64 = 0
336 var i: i64 = 0
337 while path[i] != (0 as u8) {
338 if path[i] == (47 as u8) { last = i + 1 }
339 i = i + 1
340 }
341 var o: i64 = 0
342 var b: i64 = last
343 while path[b] != (0 as u8) { nameb[o] = path[b]; o = o + 1; b = b + 1 }
344 nameb[o] = 0 as u8
345 if ci_store_has(store, nameb) == 1 {
346 ci_ws("[ci] skip (present) " as *u8)
347 ci_ws(nameb)
348 ci_ws("\n" as *u8)
349 return 0
350 }
351 let hdr: *u8 = sys_mmap(CI_HDRCAP + 64) as *u8
352 let hl: i64 = ci_header(path, hdr, CI_HDRCAP)
353 if hl <= 0 { return 0 - 1 }
354 let vec: *i64 = sys_mmap(CI_NE * 8) as *i64
355 let t0: i64 = sys_now_ms()
356 let nt: i64 = ci_embed(port, hdr, hl, 0, vec)
357 let t1: i64 = sys_now_ms()
358 if nt < 1 { return 0 - 2 }
359 let ar: i64 = ci_store_append(store, nameb, nt, vec)
360 if ar != 0 { return 0 - 3 }
361 ci_ws("[ci] indexed " as *u8)
362 ci_ws(nameb)
363 ci_kv(" ntok" as *u8, nt)
364 ci_kv("ms" as *u8, t1 - t0)
365 ci_ws("\n" as *u8)
366 return 1
367}
368
369// list file = newline-separated paths (CR tolerated). ret count newly indexed, negative on error.
370func ci_index_list(port: i64, store: *u8, listfile: *u8) -> i64 {
371 if ci_store_ensure(store) != 0 { return 0 - 1 }
372 let lo: *i64 = sys_mmap(8) as *i64
373 lo[0] = 0
374 let lb: *u8 = sys_read_file(listfile, lo)
375 if (lb as i64) == 0 { return 0 - 1 }
376 let n: i64 = lo[0]
377 var added: i64 = 0
378 var p: i64 = 0
379 let pathb: *u8 = sys_mmap(512) as *u8
380 while p < n {
381 var o: i64 = 0
382 var reading: i64 = 1
383 while reading == 1 {
384 if p >= n { reading = 0 } else {
385 let c: i64 = lb[p] as i64
386 p = p + 1
387 if c == 10 { reading = 0 } else {
388 if c != 13 { if o < 500 { pathb[o] = c as u8; o = o + 1 } }
389 }
390 }
391 }
392 pathb[o] = 0 as u8
393 if o > 2 {
394 let r: i64 = ci_index_file(port, store, pathb)
395 if r == 1 { added = added + 1 }
396 if r < 0 {
397 ci_ws("[ci] ERROR " as *u8)
398 ci_kv("rc" as *u8, r)
399 ci_ws(pathb)
400 ci_ws("\n" as *u8)
401 }
402 }
403 }
404 return added
405}
406
407// count records in store; negative on unreadable/corrupt store
408func ci_store_count(store: *u8) -> i64 {
409 let bo: *i64 = sys_mmap(8) as *i64
410 let n: i64 = ci_store_load(store, bo)
411 if n < 8 { return 0 - 1 }
412 let b: *u8 = bo[0] as *u8
413 var cnt: i64 = 0
414 var off: i64 = 8
415 var scanning: i64 = 1
416 while scanning == 1 {
417 if off + 16 > n { scanning = 0 } else {
418 let pl: *i64 = (b + off) as *i64
419 let nlen: i64 = pl[0]
420 if nlen < 1 { scanning = 0 } else {
421 if nlen > CI_NAMECAP { scanning = 0 } else {
422 let rec_end: i64 = off + 8 + nlen + 8 + CI_NE * 8
423 if rec_end > n { scanning = 0 } else {
424 cnt = cnt + 1
425 off = rec_end
426 }
427 }
428 }
429 }
430 }
431 return cnt
432}
433
434// query: embed text (kind=query), scan store, top-1 name into nameb (cap CI_NAMECAP), pm into
435// pm_out[0]; prints every record's score. ret record count, negative on error (incl corrupt store).
436func ci_query_top1(port: i64, store: *u8, text: *u8, nameb: *u8, pm_out: *i64) -> i64 {
437 let bo: *i64 = sys_mmap(8) as *i64
438 let n: i64 = ci_store_load(store, bo)
439 if n < 8 { return 0 - 2 }
440 let qv: *i64 = sys_mmap(CI_NE * 8) as *i64
441 let nt: i64 = ci_embed(port, text, std_slen(text), 1, qv)
442 if nt < 1 { return 0 - 1 }
443 let b: *u8 = bo[0] as *u8
444 var bestpm: i64 = 0 - CI_MAGIC_2000
445 var cnt: i64 = 0
446 var off: i64 = 8
447 nameb[0] = 0 as u8
448 var scanning: i64 = 1
449 while scanning == 1 {
450 if off + 16 > n { scanning = 0 } else {
451 let pl: *i64 = (b + off) as *i64
452 let nlen: i64 = pl[0]
453 if nlen < 1 { scanning = 0 } else {
454 if nlen > CI_NAMECAP { scanning = 0 } else {
455 let rec_end: i64 = off + 8 + nlen + 8 + CI_NE * 8
456 if rec_end > n { scanning = 0 } else {
457 let vp_i: i64 = (b as i64) + off + 8 + nlen + 8
458 let vp: *i64 = vp_i as *i64
459 let pm: i64 = ci_cos_pm(qv, vp)
460 ci_ws("[ci-score] " as *u8)
461 std_pdec(pm)
462 ci_ws(" " as *u8)
463 sys_write(1, b + off + 8, nlen)
464 ci_ws("\n" as *u8)
465 if pm > bestpm {
466 bestpm = pm
467 var k: i64 = 0
468 while k < nlen { nameb[k] = b[off + 8 + k]; k = k + 1 }
469 nameb[nlen] = 0 as u8
470 }
471 cnt = cnt + 1
472 off = rec_end
473 }
474 }
475 }
476 }
477 }
478 pm_out[0] = bestpm
479 return cnt
480}