nx_uxf_corpus_migrate.nx source
↩ module page · 136 lines · 6556 B
1// nx_uxf_corpus_migrate.nx -- UXF arc D-1: SCALE the migration -- "replace the TSVs" (plural).
2// Reads a manifest (knowledge/staging/uxf/tsv_corpus.tsv = one TSV path per line) and runs the
3// proven general migrator over EACH real registry TSV: per data row -> positional canon record ->
4// content-address profiled CID (UXF_DATA). Reports per-file row counts + corpus total. Proves
5// the migrator generalizes across HETEROGENEOUS real TSVs at volume (Rule 7 real data, Rule 11
6// data-driven manifest). Composes canon_cid + uxf_cid. No hardware writes (Rule 26).
7// expect_exit: 0 license_tier: ORIGINAL
8import "nx_syscalls.nx"
9import "nx_canon_cid.nx"
10import "nx_uxf_cid.nx"
11const K_MAGIC_8192: i64 = 8192
12
13func c_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
14func c_putn(v: i64) -> i64 { let bb: *u8 = sys_mmap(28); var m: i64 = v; if m < 0 { m = 0 - m; sys_write(1, "-" as *u8, 1) } let t: *u8 = sys_mmap(28); var k: i64 = 0; if m == 0 { t[0] = 48 as u8; k = 1 } while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 } var i: i64 = 0; while i < k { bb[i] = t[k - 1 - i]; i = i + 1 } sys_write(1, bb, k); return 0 }
15func c_write(p: *u8, a: i64, b: i64) -> i64 { sys_write(1, ((p as i64) + a) as *u8, b - a); return 0 }
16
17func c_keyname(idx: i64, dst: *u8) -> i64 {
18 dst[0] = 99 as u8
19 if idx == 0 { dst[1] = 48 as u8; dst[2] = 0 as u8; return 2 }
20 let tmp: *u8 = sys_mmap(24)
21 var m: i64 = idx
22 var k: i64 = 0
23 while m > 0 { tmp[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
24 var i: i64 = 0
25 while i < k { dst[1 + i] = tmp[k - 1 - i]; i = i + 1 }
26 dst[1 + k] = 0 as u8
27 return k + 1
28}
29
30// migrate one TSV buffer: per data row -> canon -> CID; returns row count.
31func c_migrate_count(tsv: *u8, n: i64) -> i64 {
32 let keys: *i64 = sys_mmap(8 * 64) as *i64
33 let vals: *i64 = sys_mmap(8 * 64) as *i64
34 var nrows: i64 = 0
35 var i: i64 = 0
36 while i < n {
37 let ls: i64 = i
38 var le: i64 = i
39 var f: i64 = 1
40 while f == 1 { if le >= n { f = 0 } else { if tsv[le] == (0x0A as u8) { f = 0 } else { le = le + 1 } } }
41 var lineend: i64 = le
42 if lineend > ls { if tsv[lineend - 1] == (0x0D as u8) { lineend = lineend - 1 } }
43 if lineend > ls {
44 if tsv[ls] != (0x23 as u8) {
45 var ncol: i64 = 0
46 var fstart: i64 = ls
47 var p: i64 = ls
48 while p <= lineend {
49 var isend: i64 = 0
50 if p == lineend { isend = 1 } else { if tsv[p] == (0x09 as u8) { isend = 1 } }
51 if isend == 1 {
52 if ncol < 64 {
53 let flen: i64 = p - fstart
54 let vbuf: *u8 = sys_mmap(flen + 2)
55 var t: i64 = 0
56 while t < flen { vbuf[t] = tsv[fstart + t]; t = t + 1 }
57 vbuf[flen] = 0 as u8
58 let kbuf: *u8 = sys_mmap(24)
59 c_keyname(ncol, kbuf)
60 keys[ncol] = kbuf as i64
61 vals[ncol] = vbuf as i64
62 ncol = ncol + 1
63 }
64 fstart = p + 1
65 }
66 p = p + 1
67 }
68 if ncol > 0 {
69 let canonbuf: *u8 = sys_mmap(K_MAGIC_8192)
70 let clen: i64 = canon_encode(keys, vals, ncol, canonbuf)
71 let cidbuf: *u8 = sys_mmap(128)
72 uxf_cid_profiled(UXF_DATA, canonbuf, clen, cidbuf)
73 nrows = nrows + 1
74 }
75 }
76 }
77 i = le + 1
78 }
79 return nrows
80}
81
82func main() -> i64 {
83 let lenp: *i64 = sys_mmap(8) as *i64
84 let man: *u8 = sys_read_file("knowledge/staging/uxf/tsv_corpus.tsv\x00" as *u8, lenp)
85 if (man as i64) == 0 { c_puts("corpus manifest not found\n" as *u8); sys_exit(1); return 1 }
86 let mn: i64 = lenp[0]
87 c_puts("D-1 CORPUS MIGRATE (manifest-driven, real registry TSVs -> content-addressed records)\n" as *u8)
88
89 let pathbuf: *u8 = sys_mmap(512)
90 var total: i64 = 0
91 var npaths: i64 = 0
92 var loaded: i64 = 0
93
94 var i: i64 = 0
95 while i < mn {
96 let ls: i64 = i
97 var le: i64 = i
98 var f: i64 = 1
99 while f == 1 { if le >= mn { f = 0 } else { if man[le] == (0x0A as u8) { f = 0 } else { le = le + 1 } } }
100 var lineend: i64 = le
101 if lineend > ls { if man[lineend - 1] == (0x0D as u8) { lineend = lineend - 1 } }
102 if lineend > ls {
103 if man[ls] != (0x23 as u8) {
104 npaths = npaths + 1
105 var t: i64 = 0
106 while t < (lineend - ls) { pathbuf[t] = man[ls + t]; t = t + 1 }
107 pathbuf[lineend - ls] = 0 as u8
108 let tszp: *i64 = sys_mmap(8) as *i64
109 let tsv: *u8 = sys_read_file(pathbuf, tszp)
110 c_puts(" " as *u8); c_write(man, ls, lineend); c_puts(" " as *u8)
111 if (tsv as i64) == 0 { c_puts("MISSING\n" as *u8) } else {
112 let tsz: i64 = tszp[0]
113 if tsz <= 0 { c_puts("EMPTY\n" as *u8) } else {
114 loaded = loaded + 1
115 let rc: i64 = c_migrate_count(tsv, tsz)
116 total = total + rc
117 c_puts("rows=" as *u8); c_putn(rc); c_puts("\n" as *u8)
118 }
119 }
120 }
121 }
122 i = le + 1
123 }
124
125 c_puts("CORPUS: manifest_paths=" as *u8); c_putn(npaths); c_puts(" loaded=" as *u8); c_putn(loaded); c_puts(" total_records=" as *u8); c_putn(total); c_puts("\n" as *u8)
126
127 var pass: i64 = 0
128 var ttl: i64 = 0
129 ttl = ttl + 1; c_puts("T1 manifest has paths: " as *u8); if npaths > 0 { pass = pass + 1; c_puts("PASS\n" as *u8) } else { c_puts("FAIL\n" as *u8) }
130 ttl = ttl + 1; c_puts("T2 every listed TSV loaded: " as *u8); if loaded == npaths { pass = pass + 1; c_puts("PASS\n" as *u8) } else { c_puts("FAIL\n" as *u8) }
131 ttl = ttl + 1; c_puts("T3 corpus produced records (>0): " as *u8); if total > 0 { pass = pass + 1; c_puts("PASS\n" as *u8) } else { c_puts("FAIL\n" as *u8) }
132
133 c_puts("UXF-CORPUS-MIGRATE-GATE passed " as *u8); c_putn(pass); c_puts("/" as *u8); c_putn(ttl)
134 if pass == ttl { c_puts(" verdict=GREEN\n" as *u8); sys_exit(0); return 0 }
135 c_puts(" verdict=RED\n" as *u8); sys_exit(1); return 1
136}