code wiki / _hdl_build / nx_corpus_ingest_gate.nx
nx_corpus_ingest_gate.nx source
↩ module page · 161 lines · 8553 B
1// nx_corpus_ingest_gate.nx -- GATE for the corpus-ingest front door (dir -> searchable sovereign shard).
2// Builds a real test dir (/tmp) with: a raw .txt, an .html (tags/script/entities exercise nx_html_to_text),
3// a .bak (must be skipped), and TWO ~600KB files (forces the 1MB writer to batch across >=2 segments).
4// Assertions are RUN-INDEPENDENT (content-addressed skip-if-present makes re-runs clean): search truths +
5// the persisted manifest segment count, never per-run counters alone. license_tier: ORIGINAL
6import "nx_corpus_ingest.nx"
7import "nx_g_puts_lib.nx"
8
9func g_len(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
10func g_num(v: i64) -> i64 {
11 let bb: *u8 = sys_mmap(28); var m: i64 = v
12 if m < 0 { sys_write(1, "-" as *u8, 1); m = 0 - m }
13 let t: *u8 = sys_mmap(28); var k: i64 = 0
14 if m == 0 { t[0] = 48 as u8; k = 1 }
15 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
16 var i: i64 = 0; while i < k { bb[i] = t[k - 1 - i]; i = i + 1 }
17 sys_write(1, bb, k); return 0
18}
19func g_contains(hay: *u8, hn: i64, needle: *u8) -> i64 {
20 let nl: i64 = g_len(needle)
21 if nl == 0 { return 1 }
22 var i: i64 = 0
23 while i + nl <= hn {
24 var m: i64 = 1
25 var j: i64 = 0
26 while j < nl { if hay[i + j] != needle[j] { m = 0; j = nl } else { j = j + 1 } }
27 if m == 1 { return 1 }
28 i = i + 1
29 }
30 return 0
31}
32func g_write(path: *u8, data: *u8, n: i64) -> i64 {
33 let fd: i64 = sys_openat_wr(path, 0x1a4)
34 if fd < 0 { return 0 - 1 }
35 sys_write(fd, data, n); sys_close(fd); return 0
36}
37func g_check(name: *u8, cond: i64, pass: *i64, total: *i64) -> i64 {
38 total[0] = total[0] + 1
39 g_puts(name)
40 if cond == 1 { pass[0] = pass[0] + 1; g_puts(" PASS\n" as *u8) } else { g_puts(" FAIL\n" as *u8) }
41 return 0
42}
43
44func main() -> i64 {
45 g_puts("=== nx_corpus_ingest gate (dir -> sovereign searchable shard; html->text; url rows; batching) ===\n" as *u8)
46 let dir: *u8 = "/tmp/ci_gate_dir" as *u8
47 sys_mkdir(dir, 0x1ed) // exists-already is fine; files are rewritten each run
48 let fa: *u8 = "Alpha zenithterm ingestion check for the sovereign corpus front door. Tokenizer needs words." as *u8
49 g_write("/tmp/ci_gate_dir/a.txt" as *u8, fa, g_len(fa))
50 let fb: *u8 = "<html><head><style>x{color:red}</style></head><body><h1>Beta estateterm page</h1><p>Planning & probate content here.</p><script>evilmarker()</script></body></html>" as *u8
51 g_write("/tmp/ci_gate_dir/b.html" as *u8, fb, g_len(fb))
52 let fc: *u8 = "bakmarkerterm should never be indexed from a .bak file" as *u8
53 g_write("/tmp/ci_gate_dir/c.bak" as *u8, fc, g_len(fc))
54 // two ~600KB files -> the 1MB writer MUST split them across segments
55 let big: *u8 = sys_mmap(700000)
56 var bo: i64 = 0
57 var r: i64 = 0
58 while r < 30000 { bo = bo + 0; let w: *u8 = "filler corpus words " as *u8; var j: i64 = 0; while w[j] != (0 as u8) { big[bo] = w[j]; bo = bo + 1; j = j + 1 } r = r + 1 }
59 let m1: *u8 = "bigsegterm1 " as *u8
60 var j1: i64 = 0; while m1[j1] != (0 as u8) { big[bo] = m1[j1]; bo = bo + 1; j1 = j1 + 1 }
61 g_write("/tmp/ci_gate_dir/big1.txt" as *u8, big, bo)
62 // swap the unique marker for the second file (same filler, different tail = different cid)
63 var bo2: i64 = bo - 12
64 let m2: *u8 = "bigsegterm2 " as *u8
65 var j2: i64 = 0; while m2[j2] != (0 as u8) { big[bo2] = m2[j2]; bo2 = bo2 + 1; j2 = j2 + 1 }
66 g_write("/tmp/ci_gate_dir/big2.txt" as *u8, big, bo2)
67
68 let dom: *u8 = "cigatetest" as *u8
69 let counts: *i64 = sys_mmap(64) as *i64
70 let rc: i64 = ci_run(dir, dom, "/" as *u8, counts)
71 g_puts(" ci_run rc=" as *u8); g_num(rc)
72 g_puts(" ingested=" as *u8); g_num(counts[0]); g_puts(" present=" as *u8); g_num(counts[1])
73 g_puts(" other=" as *u8); g_num(counts[2]); g_puts(" segs=" as *u8); g_num(counts[3]); g_puts("\n" as *u8)
74
75 let pass: *i64 = sys_mmap(16) as *i64; pass[0] = 0
76 let total: *i64 = sys_mmap(16) as *i64; total[0] = 0
77 let cids: *i64 = sys_mmap(64 * 8) as *i64
78 let scores: *i64 = sys_mmap(64 * 8) as *i64
79
80 // T1: the raw .txt doc is searchable
81 let n1: i64 = dss_search(dom, "zenithterm" as *u8, 10, cids, scores, 10)
82 g_check("T1 raw .txt searchable" as *u8, (n1 == 1) as i64, pass, total)
83
84 // T2: the .html doc was TEXT-EXTRACTED (searchable term; no tags; script suppressed; entity decoded)
85 let n2: i64 = dss_search(dom, "estateterm" as *u8, 10, cids, scores, 10)
86 var t2: i64 = 0
87 if n2 == 1 {
88 let prefix: *u8 = sys_mmap(512); dss_prefix(dom, prefix)
89 let h: *i64 = ss_open(prefix)
90 let key: *u8 = sys_mmap(64); dss_mkkey(cids[0], key)
91 let dp: *i64 = sys_mmap(16) as *i64; let dl: *i64 = sys_mmap(16) as *i64
92 if (h as i64) != 0 { if ss_hget(h, key, dp, dl) == 1 {
93 let txt: *u8 = dp[0] as *u8
94 if g_contains(txt, dl[0], "Beta estateterm page" as *u8) == 1 {
95 if g_contains(txt, dl[0], "<h1>" as *u8) == 0 {
96 if g_contains(txt, dl[0], "evilmarker" as *u8) == 0 {
97 if g_contains(txt, dl[0], "Planning & probate" as *u8) == 1 { t2 = 1 }
98 }
99 }
100 }
101 } }
102 }
103 g_check("T2 html->text (tags gone, script suppressed, entity decoded)" as *u8, t2, pass, total)
104
105 // T3: the html doc carries its url:<cid> row = "/b.html" (SERP links to the real page)
106 var t3: i64 = 0
107 if n2 == 1 {
108 let prefix3: *u8 = sys_mmap(512); dss_prefix(dom, prefix3)
109 let h3: *i64 = ss_open(prefix3)
110 let uk: *u8 = sys_mmap(64); ci_mkurlkey(cids[0], uk)
111 let up: *i64 = sys_mmap(16) as *i64; let ul: *i64 = sys_mmap(16) as *i64
112 if (h3 as i64) != 0 { if ss_hget(h3, uk, up, ul) == 1 {
113 if ul[0] == 7 { if g_contains(up[0] as *u8, ul[0], "/b.html" as *u8) == 1 { t3 = 1 } }
114 } }
115 }
116 g_check("T3 url:<cid> row = /b.html" as *u8, t3, pass, total)
117
118 // T4 NEG: the .bak file was never indexed
119 let n4: i64 = dss_search(dom, "bakmarkerterm" as *u8, 13, cids, scores, 10)
120 g_check("T4 NEG .bak skipped" as *u8, (n4 == 0) as i64, pass, total)
121
122 // T5: BOTH big docs searchable (the writer batched across segments without losing either)
123 let n5a: i64 = dss_search(dom, "bigsegterm1" as *u8, 11, cids, scores, 10)
124 let n5b: i64 = dss_search(dom, "bigsegterm2" as *u8, 11, cids, scores, 10)
125 g_check("T5 both >0.5MB docs searchable across segments" as *u8, ((n5a == 1) as i64) * ((n5b == 1) as i64), pass, total)
126
127 // T6: the shard REALLY spans >=2 segments (persisted manifest truth, run-independent)
128 let prefix6: *u8 = sys_mmap(512); dss_prefix(dom, prefix6)
129 let segs: *i64 = sys_mmap(256 * 8) as *i64
130 let nseg: i64 = ss_manifest_file(prefix6, "manifest.txt" as *u8, segs)
131 g_check("T6 manifest >=2 segments" as *u8, (nseg >= 2) as i64, pass, total)
132
133 // T7: idempotent re-run -- nothing double-ingested (zenithterm still exactly 1 hit)
134 let counts2: *i64 = sys_mmap(64) as *i64
135 ci_run(dir, dom, "/" as *u8, counts2)
136 let n7: i64 = dss_search(dom, "zenithterm" as *u8, 10, cids, scores, 10)
137 var t7: i64 = 0
138 if n7 == 1 { if counts2[0] == 0 { if counts2[1] >= 4 { t7 = 1 } } }
139 g_check("T7 idempotent re-run (0 new, all present, still 1 hit)" as *u8, t7, pass, total)
140
141 // T8 NEG (the 2026-07-03 library-url regression): NO urlprefix (null) -> NO url row may exist.
142 // The `"" as *u8` empty literal miscompiled to an unterminated pool pointer and silently stamped
143 // every doc with a junk href -- this row pins the no-urlprefix path clean forever.
144 let dom2: *u8 = "cigatetest2" as *u8
145 let counts3: *i64 = sys_mmap(64) as *i64
146 ci_run(dir, dom2, 0 as *u8, counts3)
147 let n8b: i64 = dss_search(dom2, "estateterm" as *u8, 10, cids, scores, 10)
148 var t8: i64 = 0
149 if n8b == 1 {
150 let prefix8: *u8 = sys_mmap(512); dss_prefix(dom2, prefix8)
151 let h8: *i64 = ss_open(prefix8)
152 let uk8: *u8 = sys_mmap(64); ci_mkurlkey(cids[0], uk8)
153 let up8: *i64 = sys_mmap(16) as *i64; let ul8: *i64 = sys_mmap(16) as *i64
154 if (h8 as i64) != 0 { if ss_hget(h8, uk8, up8, ul8) != 1 { t8 = 1 } }
155 }
156 g_check("T8 NEG null urlprefix -> NO url row stamped" as *u8, t8, pass, total)
157
158 g_puts("=== corpus-ingest gate: " as *u8); g_num(pass[0]); g_puts("/" as *u8); g_num(total[0]); g_puts(" " as *u8)
159 if pass[0] == total[0] { g_puts("GREEN\n" as *u8); return 0 }
160 g_puts("RED\n" as *u8); return 1
161}