nx_warc_index_gate.nx source
↩ module page · 79 lines · 4506 B
1// nx_warc_index_gate.nx -- proves the C3 ingest pipeline: WARC 'response' records -> extract content -> index ->
2// query retrieves the right docs. In-memory WARC fixture (no network); the live CC fetch + gz-inflate run on the NAS.
3// license_tier: ORIGINAL
4import "nx_warc_index.nx"
5import "nx_gate.nx"
6
7func wig_slen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
8func wig_cat(buf: *u8, off: i64, s: *u8) -> i64 { var i: i64=0; while s[i]!=(0 as u8){ buf[off+i]=s[i]; i=i+1 } return off+i }
9func wig_int(buf: *u8, off: i64, v: i64) -> i64 { if v==0 { buf[off]=48 as u8; return off+1 } var m: i64=v; let t: *u8=sys_mmap(24); var k: i64=0; while m>0 { t[k]=(48+(m%10)) as u8; m=m/10; k=k+1 } var o: i64=off; var j: i64=k-1; while j>=0 { buf[o]=t[j]; o=o+1; j=j-1 } return o }
10func wig_append(buf: *u8, off: i64, uri: *u8, content: *u8) -> i64 {
11 let cl: i64 = wig_slen(content)
12 var o: i64 = off
13 o = wig_cat(buf, o, "WARC/1.0\r\nWARC-Type: response\r\nWARC-Target-URI: " as *u8)
14 o = wig_cat(buf, o, uri)
15 o = wig_cat(buf, o, "\r\nContent-Length: " as *u8)
16 o = wig_int(buf, o, cl)
17 o = wig_cat(buf, o, "\r\n\r\n" as *u8)
18 var i: i64=0; while i<cl { buf[o]=content[i]; o=o+1; i=i+1 }
19 o = wig_cat(buf, o, "\r\n\r\n" as *u8)
20 return o
21}
22func wig_query(idx: *NxInvIndex, term: *u8, out: *i64) -> i64 {
23 let h: i64 = nx_inv_hash_bytes_lower(term, wig_slen(term))
24 let slot: *u8 = nx_inv_lookup_slot(idx, h)
25 if (slot as i64)==0 { return 0 }
26 let cnt: i64 = nx_inv_slot_postings_count(slot)
27 let off: i64 = nx_inv_slot_postings_offset(slot)
28 var k: i64=0
29 while k<cnt { let p: *u8=((idx.postings_ptr as i64)+off+k*4) as *u8; out[k]=(p[0] as i64)|((p[1] as i64)<<8)|((p[2] as i64)<<16)|((p[3] as i64)<<24); k=k+1 }
30 return cnt
31}
32
33func main() -> i64 {
34 gw("=== nx_warc_index_gate: WARC ingest -> inverted index -> query (the C3 pipeline) ===\n" as *u8)
35 var pass: i64=0; var tot: i64=0
36
37 let wb: *u8 = sys_mmap(65536)
38 var off: i64 = 0
39 off = wig_append(wb, off, "http://example.com/a" as *u8, "sovereign index alpha beta" as *u8)
40 off = wig_append(wb, off, "http://example.com/b" as *u8, "mojeek gamma sovereign delta" as *u8)
41 let n: i64 = off
42
43 let idx: *NxInvIndex = nx_inv_new(4096)
44 let coff: *i64 = sys_mmap(8*64) as *i64; let clen: *i64 = sys_mmap(8*64) as *i64
45 let ndoc: i64 = wi_build_index(wb, n, idx, coff, clen, 64)
46
47 tot=tot+1; if ndoc==2 { pass=pass+1; gw(" [PASS] " as *u8) } else { gw(" [FAIL] " as *u8) }
48 gw("T1 WARC walk indexed " as *u8); gn(ndoc); gw(" 'response' records\n" as *u8)
49
50 let a: *i64 = sys_mmap(8*8) as *i64
51 let cs: i64 = wig_query(idx, "sovereign" as *u8, a)
52 var t2: i64=0; if cs==2 { if a[0]==0 { if a[1]==1 { t2=1 } } }
53 tot=tot+1; if t2==1 { pass=pass+1; gw(" [PASS] " as *u8) } else { gw(" [FAIL] " as *u8) }
54 gw("T2 query 'sovereign' -> docs {0,1} (cnt=" as *u8); gn(cs); gw(")\n" as *u8)
55
56 let cm: i64 = wig_query(idx, "mojeek" as *u8, a)
57 var t3: i64=0; if cm==1 { if a[0]==1 { t3=1 } }
58 tot=tot+1; if t3==1 { pass=pass+1; gw(" [PASS] " as *u8) } else { gw(" [FAIL] " as *u8) }
59 gw("T3 query 'mojeek' -> doc {1} only (cnt=" as *u8); gn(cm); gw(")\n" as *u8)
60
61 let ca: i64 = wig_query(idx, "alpha" as *u8, a)
62 var t4: i64=0; if ca==1 { if a[0]==0 { t4=1 } }
63 tot=tot+1; if t4==1 { pass=pass+1; gw(" [PASS] " as *u8) } else { gw(" [FAIL] " as *u8) }
64 gw("T4 query 'alpha' -> doc {0} only (cnt=" as *u8); gn(ca); gw(")\n" as *u8)
65
66 let cz: i64 = wig_query(idx, "zzzznotpresent" as *u8, a)
67 tot=tot+1; if cz==0 { pass=pass+1; gw(" [PASS] " as *u8) } else { gw(" [FAIL] " as *u8) }
68 gw("T5 negative control: absent term -> 0 (cnt=" as *u8); gn(cz); gw(")\n" as *u8)
69
70 gw("\n=== nx_warc_index_gate " as *u8); gn(pass); gw("/" as *u8); gn(tot); gw(" ===\n" as *u8)
71 let lfd: i64=sys_openat_append("knowledge/status/warc_index_gate.log" as *u8, 420)
72 if pass==tot {
73 if lfd>=0 { sys_write(lfd, "WARC-INDEX-GATE verdict=GREEN ingest->index->query\n" as *u8, 50); sys_close(lfd) }
74 gw("WARC-INDEX GREEN -- the C3 ingest pipeline works: WARC records -> our inverted index -> queryable. Runs at NAS scale over CC WET/WARC segments (composing gz-inflate+compression+shard+decouple).\n" as *u8)
75 sys_exit(0); return 0
76 }
77 if lfd>=0 { sys_write(lfd, "WARC-INDEX-GATE verdict=RED\n" as *u8, 27); sys_close(lfd) }
78 gw("WARC-INDEX RED\n" as *u8); sys_exit(1); return 1
79}