code wiki / _hdl_build / nx_bulk_index.nx
nx_bulk_index.nx source
↩ module page · 142 lines · 8970 B
1// nx_bulk_index.nx -- TARGETED bulk indexer (operator 2026-07-06: "we should have thousands of results ...
2// index the individual works, not 3 search pages"). Reads a URL list (one per line), fetches each over the
3// best-effort stack, PER-HOST paces (nx_crawl_pace -- don't get banned), extracts (nx_html_to_text +
4// nx_struct_extract title/desc), and indexes each into dp-web-pub- with ONE shard handle + batched commits.
5// This is nx_url_index's core looped -- the efficient path to index a performer's whole catalog.
6// usage: nx_bulk_index <url-list-file>
7// license_tier: ORIGINAL
8import "nx_corpus_ingest.nx" // ci_hash / ci_mkurlkey / dss_prefix / dss_mkkey / seg_store / nx_html_to_text
9import "nx_struct_extract.nx"
10import "nx_js_eval.nx" // R-JS-HEADLESS: js_render_page + js_has_inline_script (hydrate JS pages)
11import "nx_crawl_pace.nx"
12import "nx_x509_trust_store.nx"
13import "nx_trust_store_load_from_certdata.nx"
14import "nx_https_fetch_follow.nx"
15const K_MAGIC_4194304: i64 = 4194304
16const K_MAGIC_4096: i64 = 4096
17const K_MAGIC_1048576: i64 = 1048576
18const K_MAGIC_8192: i64 = 8192
19const K_MAGIC_1024: i64 = 1024
20const K_MAGIC_2048: i64 = 2048
21const K_MAGIC_4095: i64 = 4095
22const K_MAGIC_8190: i64 = 8190
23
24func bw(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
25func bn(v: i64) -> i64 { if v==0 { sys_write(1,"0" as *u8,1); return 0 } var m: i64=v; if m<0 { sys_write(1,"-" as *u8,1); m=0-m } let t: *u8=sys_mmap(24); var k: i64=0; while m>0 { t[k]=(48+(m%10)) as u8; m=m/10; k=k+1 } let o: *u8=sys_mmap(24); var q: i64=k-1; var x: i64=0; while q>=0 { o[x]=t[q]; x=x+1; q=q-1 } sys_write(1,o,x); return 0 }
26func bslen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
27func bhas(hay: *u8, hn: i64, ndl: *u8) -> i64 { var nl: i64=0; while ndl[nl]!=(0 as u8){nl=nl+1} var i: i64=0; while i+nl<=hn { var m: i64=1; var j: i64=0; while j<nl { if hay[i+j]!=ndl[j] { m=0; j=nl } else { j=j+1 } } if m==1 { return 1 } i=i+1 } return 0 }
28func bhost(url: *u8, out: *u8) -> i64 {
29 var sp: i64=0-1; var i: i64=0; var d: i64=0
30 while d==0 { if url[i]==(0 as u8) { d=1 } else { if url[i]==(58 as u8) { if url[i+1]==(47 as u8) { if url[i+2]==(47 as u8) { sp=i; d=1 } } } if d==0 { i=i+1 } } }
31 var s: i64=0; if sp>=0 { s=sp+3 }
32 var o: i64=0; var d2: i64=0
33 while d2==0 { let c: i64=url[s] as i64; if c==0 { d2=1 } else { if c==47 { d2=1 } else { if c==58 { d2=1 } else { out[o]=url[s]; o=o+1; s=s+1 } } } }
34 out[o]=0 as u8; return o
35}
36
37func main(argc: i64, argv: *i64) -> i64 {
38 if argc < 2 { bw("usage: nx_bulk_index <url-list-file>\n" as *u8); return 1 }
39 let path: *u8 = argv[1] as *u8
40 let r: i64 = nx_trust_store_load_from_certdata("data/mozilla_certdata.txt" as *u8, 512, K_MAGIC_4194304)
41 if r <= 0 { bw("trust store load failed\n" as *u8); return 2 }
42 let store: *TrustStore = r as *TrustStore
43 let lp: *i64 = sys_mmap(16) as *i64
44 let urls: *u8 = sys_read_file(path, lp)
45 if (urls as i64) == 0 { bw("cannot read url file\n" as *u8); return 3 }
46 let ulen: i64 = lp[0]
47 bw("=== nx_bulk_index: "); bw(path); bw(" ("); bn(ulen); bw(" bytes) -> dp-web-pub- ===\n" as *u8)
48
49 let prefix: *u8 = sys_mmap(512); dss_prefix("web" as *u8, prefix)
50 let segs: *i64 = sys_mmap(K_MAGIC_4096*8) as *i64
51 let nseg: i64 = ss_manifest_cap(prefix, segs, K_MAGIC_4096)
52 var segid: i64 = 1; var si: i64 = 0
53 // ROOT FIX seq1730 (id 1785450987): segs[] holds POINTERS to seg-<id> name strings
54 // (nx_seg_store.nx:1234 stores segs[cnt] = name as i64), NOT ids -- so segs[si] + 1 produced
55 // MMAP_ADDRESS+1 (~1.4e14), the pointer-shaped poison that PINS a plane forever: every later
56 // epoch id sorts BELOW it in supersede order and its rows are silently shadowed while rc=0.
57 // Parse the DIGITS, as nx_web_shard_compact.nx:55-58 already does on this SAME array.
58 while si < nseg {
59 let sg_nm: *u8 = segs[si] as *u8
60 var sg_v: i64 = 0
61 var sg_ci: i64 = 0
62 while sg_nm[sg_ci] != (0 as u8) { let sg_c: i64 = sg_nm[sg_ci] as i64; if sg_c >= 48 { if sg_c <= 57 { sg_v = sg_v * 10 + (sg_c - 48) } } sg_ci = sg_ci + 1 }
63 if sg_v >= segid { segid = sg_v + 1 }
64 si = si + 1
65 }
66 let h: *i64 = ss_open(prefix)
67 let w: *i64 = ss_begin()
68
69 let cap: i64 = K_MAGIC_4194304
70 let out: *u8 = sys_mmap(cap)
71 let hydrbuf: *u8 = sys_mmap(cap) // R-JS-HEADLESS: serialized hydrated DOM for inline-script pages
72 let st: *i64 = sys_mmap(16) as *i64
73 let body: *u8 = sys_mmap(K_MAGIC_1048576)
74 let text: *u8 = sys_mmap(K_MAGIC_1048576 + K_MAGIC_8192)
75 let stitle: *u8 = sys_mmap(K_MAGIC_1024); let sdesc: *u8 = sys_mmap(K_MAGIC_2048); let sty: *u8 = sys_mmap(256); let ssi: *u8 = sys_mmap(256)
76 let key: *u8 = sys_mmap(64); let ukey: *u8 = sys_mmap(64)
77 let pbox: *i64 = sys_mmap(16) as *i64; let lbox: *i64 = sys_mmap(16) as *i64
78 let urlbuf: *u8 = sys_mmap(K_MAGIC_4096)
79 let phost: *u8 = sys_mmap(512)
80
81 var indexed: i64 = 0; var skipped: i64 = 0; var already: i64 = 0
82 var i: i64 = 0
83 while i < ulen {
84 // extract one line -> urlbuf (consume up to and including the newline)
85 var o: i64 = 0
86 var eol: i64 = 0
87 while eol == 0 {
88 if i >= ulen { eol = 1 } else {
89 let c: i64 = urls[i] as i64
90 if c == 10 { i = i + 1; eol = 1 }
91 else { if c == 13 { i = i + 1 } else { if o < K_MAGIC_4095 { urlbuf[o] = urls[i]; o = o + 1 } i = i + 1 } }
92 }
93 }
94 urlbuf[o] = 0 as u8
95 // skip empty lines
96 if o >= 8 {
97 let hl: i64 = bhost(urlbuf, phost)
98 pace_before(phost, hl)
99 let n: i64 = nx_https_fetch_follow_best(urlbuf, store, out, cap, 6, st)
100 pace_after(phost, hl, st[0], 0)
101 if st[0] != 200 { skipped = skipped + 1 } else {
102 var intr: i64 = 0
103 if n > 0 { if bhas(out, n, "Just a moment" as *u8) == 1 { intr = 1 } if bhas(out, n, "Please wait for verification" as *u8) == 1 { intr = 1 } }
104 if intr == 1 { skipped = skipped + 1 } else {
105 // R-JS-HEADLESS: if the page has inline scripts, HYDRATE it (run its JS async-drained,
106 // serialize the mutated DOM) so JS-built content becomes indexable; else use the raw HTML.
107 var phtml: *u8 = out; var phn: i64 = n
108 if js_has_inline_script(out, n) == 1 { let hyr: i64 = js_render_page(out, n, hydrbuf, cap); if hyr > 0 { phtml = hydrbuf; phn = hyr } }
109 let blen: i64 = nx_html_to_text(phtml, phn, body, K_MAGIC_1048576)
110 let sgot: i64 = nx_struct_extract(phtml, phn, stitle, K_MAGIC_1024, sdesc, K_MAGIC_2048, sty, 256, ssi, 256)
111 var okdoc: i64 = 0
112 if blen >= 200 { okdoc = 1 } else { if sgot >= 2 { okdoc = 1 } }
113 if okdoc == 1 {
114 var el: i64 = 0; var c2: i64 = 0
115 while stitle[c2] != (0 as u8) { if el < K_MAGIC_1048576+K_MAGIC_8190 { text[el]=stitle[c2]; el=el+1 } c2=c2+1 }
116 if el < K_MAGIC_1048576+K_MAGIC_8190 { text[el]=10 as u8; el=el+1 }
117 c2 = 0
118 while sdesc[c2] != (0 as u8) { if el < K_MAGIC_1048576+K_MAGIC_8190 { text[el]=sdesc[c2]; el=el+1 } c2=c2+1 }
119 if el < K_MAGIC_1048576+K_MAGIC_8190 { text[el]=10 as u8; el=el+1 }
120 var bi: i64 = 0
121 while bi < blen { if el < K_MAGIC_1048576+K_MAGIC_8190 { text[el]=body[bi]; el=el+1 } bi=bi+1 }
122 var tn: i64 = el; if tn > CI_DOCCAP { tn = CI_DOCCAP }
123 let cid: i64 = ci_hash(text, tn)
124 dss_mkkey(cid, key)
125 var isdup: i64 = 0
126 if (h as i64) != 0 { if ss_hget(h, key, pbox, lbox) == 1 { isdup = 1 } }
127 if isdup == 1 { already = already + 1 } else {
128 if ss_add(w, 1, key, text, tn) < 0 { if ss_commit(prefix, w, segid) == 0 { } segid = segid + 1; w[1] = 0; ss_add(w, 1, key, text, tn) }
129 ci_mkurlkey(cid, ukey)
130 if ss_add(w, 1, ukey, urlbuf, bslen(urlbuf)) < 0 { if ss_commit(prefix, w, segid) == 0 { } segid = segid + 1; w[1] = 0; ss_add(w, 1, ukey, urlbuf, bslen(urlbuf)) }
131 indexed = indexed + 1
132 if (indexed % 10) == 0 { bw(" indexed "); bn(indexed); bw(" ...\n" as *u8) }
133 }
134 } else { skipped = skipped + 1 }
135 }
136 }
137 }
138 }
139 if w[1] > 0 { ss_commit(prefix, w, segid) }
140 bw("=== BULK-INDEX done: indexed="); bn(indexed); bw(" already="); bn(already); bw(" skipped="); bn(skipped); bw(" ===\n" as *u8)
141 return 0
142}