nx_uxf_tsv_persist.nx source
↩ module page · 159 lines · 7716 B
1// nx_uxf_tsv_persist.nx -- UXF arc R2b: the FULL data pipeline, end to end on real data.
2// A real TSV -> per-row canonical content-addressed records -> PERSISTED into the immutable
3// seg_store (ss_add -> ss_build_keys -> ss_writefile+fsync) -> REOPENED -> every row RETRIEVED by
4// its content-address CID (binary search over the NXK1 index) and verified BYTE-IDENTICAL to the
5// original. This is "replace the TSVs" made real: not just migrate-in-memory, but store+retrieve.
6// Composes canon_cid + uxf_cid + seg_store. No hardware writes (Rule 26).
7// expect_exit: 0 license_tier: ORIGINAL
8import "nx_syscalls.nx"
9import "nx_canon_cid.nx"
10import "nx_uxf_cid.nx"
11import "nx_seg_store.nx"
12const K_MAGIC_4096: i64 = 4096
13const K_MAGIC_1048576: i64 = 1048576
14
15func p_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
16func p_putn(v: i64) -> i64 { let bb: *u8 = sys_mmap(28); var m: i64 = v; if m < 0 { m = 0 - m; sys_write(1, "-" as *u8, 1) } let t: *u8 = sys_mmap(28); var k: i64 = 0; if m == 0 { t[0] = 48 as u8; k = 1 } while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 } var i: i64 = 0; while i < k { bb[i] = t[k - 1 - i]; i = i + 1 } sys_write(1, bb, k); return 0 }
17
18func p_keyname(idx: i64, dst: *u8) -> i64 {
19 dst[0] = 99 as u8
20 if idx == 0 { dst[1] = 48 as u8; dst[2] = 0 as u8; return 2 }
21 let tmp: *u8 = sys_mmap(24)
22 var m: i64 = idx
23 var k: i64 = 0
24 while m > 0 { tmp[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
25 var i: i64 = 0
26 while i < k { dst[1 + i] = tmp[k - 1 - i]; i = i + 1 }
27 dst[1 + k] = 0 as u8
28 return k + 1
29}
30
31func p_bytes_eq(a: *u8, b: *u8, n: i64) -> i64 { var i: i64 = 0; while i < n { if a[i] != b[i] { return 0 } i = i + 1 } return 1 }
32
33// binary search the NXK1 index for key(kl); out[0]=voff,out[1]=vlen; returns entry idx or -1.
34func p_get(idx: *u8, key: *u8, kl: i64, out: *i64) -> i64 {
35 let m: i64 = ss_r32(idx, 4)
36 let base: i64 = 8 + 4 * m
37 var lo: i64 = 0
38 var hi: i64 = m - 1
39 while lo <= hi {
40 let mid: i64 = (lo + hi) / 2
41 let rel: i64 = ss_r32(idx, 8 + 4 * mid)
42 let eoff: i64 = base + rel
43 let eklen: i64 = ss_r32(idx, eoff + 1)
44 let ekoff: i64 = eoff + 5
45 let cmp: i64 = ss_kcmp(((idx as i64) + ekoff) as *u8, eklen, key, kl)
46 if cmp == 0 { out[0] = ss_r32(idx, ekoff + eklen); out[1] = ss_r32(idx, ekoff + eklen + 4); return mid }
47 if cmp < 0 { lo = mid + 1 } else { hi = mid - 1 }
48 }
49 return 0 - 1
50}
51
52func main() -> i64 {
53 let lenp: *i64 = sys_mmap(8) as *i64
54 let buf: *u8 = sys_read_file("knowledge/registry/sci_units.tsv\x00" as *u8, lenp)
55 if (buf as i64) == 0 { p_puts("sci_units.tsv not found\n" as *u8); sys_exit(1); return 1 }
56 let n: i64 = lenp[0]
57 p_puts("R2b PERSIST: sci_units.tsv -> seg_store -> retrieve-by-CID (byte-faithful)\n" as *u8)
58
59 let cids: *u8 = sys_mmap(256 * 128) // per-row CID (72+1 bytes), row r at cids + r*128
60 let cptr: *i64 = sys_mmap(256 * 8) as *i64 // per-row original canon ptr
61 let clenr: *i64 = sys_mmap(256 * 8) as *i64 // per-row canon length
62 var nrows: i64 = 0
63
64 let w: *i64 = ss_begin()
65 let keys: *i64 = sys_mmap(8 * 64) as *i64
66 let vals: *i64 = sys_mmap(8 * 64) as *i64
67
68 var i: i64 = 0
69 while i < n {
70 let ls: i64 = i
71 var le: i64 = i
72 var f: i64 = 1
73 while f == 1 { if le >= n { f = 0 } else { if buf[le] == (0x0A as u8) { f = 0 } else { le = le + 1 } } }
74 if le > ls {
75 if buf[ls] != (0x23 as u8) {
76 var ncol: i64 = 0
77 var fstart: i64 = ls
78 var p: i64 = ls
79 while p <= le {
80 var isend: i64 = 0
81 if p == le { isend = 1 } else { if buf[p] == (0x09 as u8) { isend = 1 } }
82 if isend == 1 {
83 let flen: i64 = p - fstart
84 let vbuf: *u8 = sys_mmap(flen + 2)
85 var t: i64 = 0
86 while t < flen { vbuf[t] = buf[fstart + t]; t = t + 1 }
87 vbuf[flen] = 0 as u8
88 let kbuf: *u8 = sys_mmap(24)
89 p_keyname(ncol, kbuf)
90 keys[ncol] = kbuf as i64
91 vals[ncol] = vbuf as i64
92 ncol = ncol + 1
93 fstart = p + 1
94 }
95 p = p + 1
96 }
97 let canonbuf: *u8 = sys_mmap(K_MAGIC_4096)
98 let clen: i64 = canon_encode(keys, vals, ncol, canonbuf)
99 let cidp: *u8 = ((cids as i64) + nrows * 128) as *u8
100 uxf_cid_profiled(UXF_DATA, canonbuf, clen, cidp)
101 // PERSIST: key = content-address CID, value = canonical record bytes
102 ss_add(w, 1, cidp, canonbuf, clen)
103 cptr[nrows] = canonbuf as i64
104 clenr[nrows] = clen
105 nrows = nrows + 1
106 }
107 }
108 i = le + 1
109 }
110
111 // build index, commit segment + index to disk (fsync inside ss_writefile)
112 let kout: *u8 = sys_mmap(K_MAGIC_1048576)
113 let klen: i64 = ss_build_keys(w, kout)
114 ss_writefile("/tmp/uxf_sci.docs\x00" as *u8, w[0] as *u8, w[1])
115 ss_writefile("/tmp/uxf_sci.keys\x00" as *u8, kout, klen)
116 p_puts(" persisted rows=" as *u8); p_putn(nrows); p_puts(" docs_bytes=" as *u8); p_putn(w[1]); p_puts(" index_bytes=" as *u8); p_putn(klen); p_puts("\n" as *u8)
117
118 // REOPEN fresh from disk
119 let szd: *i64 = sys_mmap(8) as *i64
120 let szi: *i64 = sys_mmap(8) as *i64
121 let docs: *u8 = ss_readall("/tmp/uxf_sci.docs\x00" as *u8, szd)
122 let idx: *u8 = ss_readall("/tmp/uxf_sci.keys\x00" as *u8, szi)
123
124 // retrieve every row by its CID, verify byte-faithful
125 let out: *i64 = sys_mmap(16) as *i64
126 var hits: i64 = 0
127 var faithful: i64 = 0
128 var r: i64 = 0
129 while r < nrows {
130 let cidp: *u8 = ((cids as i64) + r * 128) as *u8
131 let e: i64 = p_get(idx, cidp, 72, out)
132 if e >= 0 {
133 hits = hits + 1
134 let voff: i64 = out[0]
135 let vlen: i64 = out[1]
136 if vlen == clenr[r] {
137 if p_bytes_eq(((docs as i64) + voff) as *u8, cptr[r] as *u8, vlen) == 1 { faithful = faithful + 1 }
138 }
139 }
140 r = r + 1
141 }
142
143 // neg-control: a well-formed but non-existent CID must MISS
144 let bogus: *u8 = "nxc1-01-ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff\x00" as *u8
145 let miss: i64 = p_get(idx, bogus, 72, out)
146
147 p_puts(" retrieved hits=" as *u8); p_putn(hits); p_puts(" byte-faithful=" as *u8); p_putn(faithful); p_puts("\n" as *u8)
148
149 var pass: i64 = 0
150 var total: i64 = 0
151 total = total + 1; p_puts(" T1 rows persisted (>0): " as *u8); if nrows > 0 { pass = pass + 1; p_puts("PASS\n" as *u8) } else { p_puts("FAIL\n" as *u8) }
152 total = total + 1; p_puts(" T2 every CID retrievable: " as *u8); if hits == nrows { pass = pass + 1; p_puts("PASS\n" as *u8) } else { p_puts("FAIL\n" as *u8) }
153 total = total + 1; p_puts(" T3 retrieved bytes byte-IDENTICAL to source: " as *u8); if faithful == nrows { pass = pass + 1; p_puts("PASS\n" as *u8) } else { p_puts("FAIL\n" as *u8) }
154 total = total + 1; p_puts(" T4 neg-control absent CID misses: " as *u8); if miss < 0 { pass = pass + 1; p_puts("PASS\n" as *u8) } else { p_puts("FAIL\n" as *u8) }
155
156 p_puts("UXF-TSV-PERSIST-GATE passed " as *u8); p_putn(pass); p_puts("/" as *u8); p_putn(total)
157 if pass == total { p_puts(" verdict=GREEN\n" as *u8); sys_exit(0); return 0 }
158 p_puts(" verdict=RED\n" as *u8); sys_exit(1); return 1
159}