nx_uxf_corpus_persist.nx source
↩ module page · 190 lines · 9316 B
1// nx_uxf_corpus_persist.nx -- UXF arc #1 (production wire), rung 1: persist the WHOLE corpus into
2// ONE DURABLE store under knowledge/ (survives reboot, unlike /tmp), then retrieve every record by
3// its content-address CID and verify byte-faithful. Composes D-1 (manifest corpus migrate) + R2b
4// (seg_store persist + binary-search retrieval). This is "replace the TSVs" as durable infra: the
5// 5 registry TSVs become one immutable content-addressed segment. No hardware writes (Rule 26).
6// expect_exit: 0 license_tier: ORIGINAL
7import "nx_syscalls.nx"
8import "nx_canon_cid.nx"
9import "nx_uxf_cid.nx"
10import "nx_seg_store.nx"
11const CAP_MAGIC_8192: i64 = 8192
12const CAP_MAGIC_1048576: i64 = 1048576
13
14const CAP_REC: i64 = 512
15
16func q_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
17func q_putn(v: i64) -> i64 { let bb: *u8 = sys_mmap(28); var m: i64 = v; if m < 0 { m = 0 - m; sys_write(1, "-" as *u8, 1) } let t: *u8 = sys_mmap(28); var k: i64 = 0; if m == 0 { t[0] = 48 as u8; k = 1 } while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 } var i: i64 = 0; while i < k { bb[i] = t[k - 1 - i]; i = i + 1 } sys_write(1, bb, k); return 0 }
18func q_write(p: *u8, a: i64, b: i64) -> i64 { sys_write(1, ((p as i64) + a) as *u8, b - a); return 0 }
19
20func q_keyname(idx: i64, dst: *u8) -> i64 {
21 dst[0] = 99 as u8
22 if idx == 0 { dst[1] = 48 as u8; dst[2] = 0 as u8; return 2 }
23 let tmp: *u8 = sys_mmap(24)
24 var m: i64 = idx
25 var k: i64 = 0
26 while m > 0 { tmp[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
27 var i: i64 = 0
28 while i < k { dst[1 + i] = tmp[k - 1 - i]; i = i + 1 }
29 dst[1 + k] = 0 as u8
30 return k + 1
31}
32
33func q_bytes_eq(a: *u8, b: *u8, n: i64) -> i64 { var i: i64 = 0; while i < n { if a[i] != b[i] { return 0 } i = i + 1 } return 1 }
34
35func q_get(idx: *u8, key: *u8, kl: i64, out: *i64) -> i64 {
36 let m: i64 = ss_r32(idx, 4)
37 let base: i64 = 8 + 4 * m
38 var lo: i64 = 0
39 var hi: i64 = m - 1
40 while lo <= hi {
41 let mid: i64 = (lo + hi) / 2
42 let rel: i64 = ss_r32(idx, 8 + 4 * mid)
43 let eoff: i64 = base + rel
44 let eklen: i64 = ss_r32(idx, eoff + 1)
45 let ekoff: i64 = eoff + 5
46 let cmp: i64 = ss_kcmp(((idx as i64) + ekoff) as *u8, eklen, key, kl)
47 if cmp == 0 { out[0] = ss_r32(idx, ekoff + eklen); out[1] = ss_r32(idx, ekoff + eklen + 4); return mid }
48 if cmp < 0 { lo = mid + 1 } else { hi = mid - 1 }
49 }
50 return 0 - 1
51}
52
53// migrate one TSV's rows into the writer + tracking arrays; nrecp[0] is the running record count.
54func q_migrate_into(tsv: *u8, n: i64, w: *i64, cids: *u8, cptr: *i64, clenr: *i64, nrecp: *i64) -> i64 {
55 let keys: *i64 = sys_mmap(8 * 64) as *i64
56 let vals: *i64 = sys_mmap(8 * 64) as *i64
57 var added: i64 = 0
58 var i: i64 = 0
59 while i < n {
60 let ls: i64 = i
61 var le: i64 = i
62 var f: i64 = 1
63 while f == 1 { if le >= n { f = 0 } else { if tsv[le] == (0x0A as u8) { f = 0 } else { le = le + 1 } } }
64 var lineend: i64 = le
65 if lineend > ls { if tsv[lineend - 1] == (0x0D as u8) { lineend = lineend - 1 } }
66 if lineend > ls {
67 if tsv[ls] != (0x23 as u8) {
68 if nrecp[0] < CAP_REC {
69 var ncol: i64 = 0
70 var fstart: i64 = ls
71 var p: i64 = ls
72 while p <= lineend {
73 var isend: i64 = 0
74 if p == lineend { isend = 1 } else { if tsv[p] == (0x09 as u8) { isend = 1 } }
75 if isend == 1 {
76 if ncol < 64 {
77 let flen: i64 = p - fstart
78 let vbuf: *u8 = sys_mmap(flen + 2)
79 var t: i64 = 0
80 while t < flen { vbuf[t] = tsv[fstart + t]; t = t + 1 }
81 vbuf[flen] = 0 as u8
82 let kbuf: *u8 = sys_mmap(24)
83 q_keyname(ncol, kbuf)
84 keys[ncol] = kbuf as i64
85 vals[ncol] = vbuf as i64
86 ncol = ncol + 1
87 }
88 fstart = p + 1
89 }
90 p = p + 1
91 }
92 if ncol > 0 {
93 let canonbuf: *u8 = sys_mmap(CAP_MAGIC_8192)
94 let clen: i64 = canon_encode(keys, vals, ncol, canonbuf)
95 let r: i64 = nrecp[0]
96 let cidp: *u8 = ((cids as i64) + r * 128) as *u8
97 uxf_cid_profiled(UXF_DATA, canonbuf, clen, cidp)
98 ss_add(w, 1, cidp, canonbuf, clen)
99 cptr[r] = canonbuf as i64
100 clenr[r] = clen
101 nrecp[0] = r + 1
102 added = added + 1
103 }
104 }
105 }
106 }
107 i = le + 1
108 }
109 return added
110}
111
112func main() -> i64 {
113 let lenp: *i64 = sys_mmap(8) as *i64
114 let man: *u8 = sys_read_file("knowledge/staging/uxf/tsv_corpus.tsv\x00" as *u8, lenp)
115 if (man as i64) == 0 { q_puts("corpus manifest not found\n" as *u8); sys_exit(1); return 1 }
116 let mn: i64 = lenp[0]
117 q_puts("#1 CORPUS PERSIST -> durable knowledge/ store -> retrieve-by-CID byte-faithful\n" as *u8)
118
119 let w: *i64 = ss_begin()
120 let cids: *u8 = sys_mmap(CAP_REC * 128)
121 let cptr: *i64 = sys_mmap(CAP_REC * 8) as *i64
122 let clenr: *i64 = sys_mmap(CAP_REC * 8) as *i64
123 let nrecp: *i64 = sys_mmap(8) as *i64
124 nrecp[0] = 0
125 let pathbuf: *u8 = sys_mmap(512)
126
127 var i: i64 = 0
128 while i < mn {
129 let ls: i64 = i
130 var le: i64 = i
131 var f: i64 = 1
132 while f == 1 { if le >= mn { f = 0 } else { if man[le] == (0x0A as u8) { f = 0 } else { le = le + 1 } } }
133 var lineend: i64 = le
134 if lineend > ls { if man[lineend - 1] == (0x0D as u8) { lineend = lineend - 1 } }
135 if lineend > ls {
136 if man[ls] != (0x23 as u8) {
137 var t: i64 = 0
138 while t < (lineend - ls) { pathbuf[t] = man[ls + t]; t = t + 1 }
139 pathbuf[lineend - ls] = 0 as u8
140 let tszp: *i64 = sys_mmap(8) as *i64
141 let tsv: *u8 = sys_read_file(pathbuf, tszp)
142 q_puts(" " as *u8); q_write(man, ls, lineend); q_puts(" " as *u8)
143 if (tsv as i64) == 0 { q_puts("MISSING\n" as *u8) } else {
144 let added: i64 = q_migrate_into(tsv, tszp[0], w, cids, cptr, clenr, nrecp)
145 q_puts("+" as *u8); q_putn(added); q_puts(" records\n" as *u8)
146 }
147 }
148 }
149 i = le + 1
150 }
151
152 let nrec: i64 = nrecp[0]
153 let kout: *u8 = sys_mmap(CAP_MAGIC_1048576)
154 let klen: i64 = ss_build_keys(w, kout)
155 ss_writefile("knowledge/staging/uxf/corpus.docs\x00" as *u8, w[0] as *u8, w[1])
156 ss_writefile("knowledge/staging/uxf/corpus.keys\x00" as *u8, kout, klen)
157 q_puts(" PERSISTED records=" as *u8); q_putn(nrec); q_puts(" docs_bytes=" as *u8); q_putn(w[1]); q_puts(" -> knowledge/staging/uxf/corpus.{docs,keys} (DURABLE)\n" as *u8)
158
159 // reopen fresh from the durable store + retrieve every record by CID
160 let szd: *i64 = sys_mmap(8) as *i64
161 let szi: *i64 = sys_mmap(8) as *i64
162 let docs: *u8 = ss_readall("knowledge/staging/uxf/corpus.docs\x00" as *u8, szd)
163 let idx: *u8 = ss_readall("knowledge/staging/uxf/corpus.keys\x00" as *u8, szi)
164 let out: *i64 = sys_mmap(16) as *i64
165 var hits: i64 = 0
166 var faithful: i64 = 0
167 var r: i64 = 0
168 while r < nrec {
169 let cidp: *u8 = ((cids as i64) + r * 128) as *u8
170 if q_get(idx, cidp, 72, out) >= 0 {
171 hits = hits + 1
172 if out[1] == clenr[r] { if q_bytes_eq(((docs as i64) + out[0]) as *u8, cptr[r] as *u8, out[1]) == 1 { faithful = faithful + 1 } }
173 }
174 r = r + 1
175 }
176 let bogus: *u8 = "nxc1-01-ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff\x00" as *u8
177 let miss: i64 = q_get(idx, bogus, 72, out)
178 q_puts(" RETRIEVED hits=" as *u8); q_putn(hits); q_puts(" byte-faithful=" as *u8); q_putn(faithful); q_puts("\n" as *u8)
179
180 var pass: i64 = 0
181 var ttl: i64 = 0
182 ttl = ttl + 1; q_puts(" T1 records persisted (>0): " as *u8); if nrec > 0 { pass = pass + 1; q_puts("PASS\n" as *u8) } else { q_puts("FAIL\n" as *u8) }
183 ttl = ttl + 1; q_puts(" T2 all retrievable from durable store: " as *u8); if hits == nrec { pass = pass + 1; q_puts("PASS\n" as *u8) } else { q_puts("FAIL\n" as *u8) }
184 ttl = ttl + 1; q_puts(" T3 all byte-IDENTICAL to source: " as *u8); if faithful == nrec { pass = pass + 1; q_puts("PASS\n" as *u8) } else { q_puts("FAIL\n" as *u8) }
185 ttl = ttl + 1; q_puts(" T4 neg-control absent CID misses: " as *u8); if miss < 0 { pass = pass + 1; q_puts("PASS\n" as *u8) } else { q_puts("FAIL\n" as *u8) }
186
187 q_puts("UXF-CORPUS-PERSIST-GATE passed " as *u8); q_putn(pass); q_puts("/" as *u8); q_putn(ttl)
188 if pass == ttl { q_puts(" verdict=GREEN\n" as *u8); sys_exit(0); return 0 }
189 q_puts(" verdict=RED\n" as *u8); sys_exit(1); return 1
190}