code wiki / _hdl_build / nx_page_ingest.nx
nx_page_ingest.nx source
↩ module page · 239 lines · 12633 B
1// nx_page_ingest.nx -- ENTITY-DEPTH live ingester: ONE operator-chosen seed page + up to N of its
2// SAME-HOST outlinks -> the dp-web-pub- web shard. The targeted complement to the bulk CC pipeline
3// (breadth) and the frontier crawler (which stalls on stale frontiers -- debt seq628): every URL here
4// is FRESH (seed + its own live links), fetches are sequential + politely paced, the job is bounded.
5// usage: nx_page_ingest <seed-url> [maxfollow]
6// Seed is ingested UNCONDITIONALLY (operator chose it; only a minimal length floor applies);
7// followed links pass the normal thin filter. Writes doc:/url: + out: edges (same node identity as
8// serve/pagerank). No loc: row (live pages have no CC locator; absent loc is a supported state).
9// exit 0 + verdict=PASS when the seed ingested (or was already present). license_tier: ORIGINAL
10import "nx_corpus_ingest.nx" // ci_hash / ci_mkurlkey / dss_prefix / dss_mkkey / seg_store / nx_html_to_text
11import "nx_x509_trust_store.nx"
12import "nx_trust_store_load_from_certdata.nx"
13import "nx_https_fetch_follow.nx"
14import "nx_outlink_harvest.nx" // olh_scan / olh_outkey
15import "nx_gzip_wrap.nx" // nx_gzip_inflate -- see the INFLATE note in pi_one
16const PI_MAGIC_1048576: i64 = 1048576
17const PI_MAGIC_4194304: i64 = 4194304
18const PI_MAGIC_8192: i64 = 8192
19const PI_MAGIC_2048: i64 = 2048
20const PI_MAGIC_1500: i64 = 1500
21
22const PI_MAXFOLLOW: i64 = 60 // hard cap on followed links per run (bounded job by design)
23const PI_FETCHCAP: i64 = 4194304 // page byte cap
24const PI_MINSEED: i64 = 100 // seed min extracted chars (operator chose it -- floor only)
25const PI_MINFOLLOW: i64 = 300 // followed-page min extracted chars (thin filter)
26const PI_PACE_MS: i64 = 500 // politeness delay between same-host fetches
27const PI_SEGCAP: i64 = 16777216 // writer segment size
28// Poison ceiling for manifest-derived segids (2026-08-01): a pre-seq1730 run WROTE an
29// address-NAMED segment row into dp-web-pub-'s manifest (seg-140712850411540), so the
30// digit-parse below faithfully adopts the poison and every later commit is refused by
31// the ss_commit address guard. Real segids are small sequence numbers; any parsed value
32// at or above this bound is a poison row and must be SKIPPED, not adopted.
33const PI_SEGID_SANE: i64 = 100000000
34
35func pi_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
36func pi_num(v: i64) -> i64 {
37 let bb: *u8 = sys_mmap(28); var m: i64 = v
38 if m < 0 { sys_write(1, "-" as *u8, 1); m = 0 - m }
39 let t: *u8 = sys_mmap(28); var k: i64 = 0
40 if m == 0 { t[0] = 48 as u8; k = 1 }
41 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
42 var i: i64 = 0; while i < k { bb[i] = t[k - 1 - i]; i = i + 1 }
43 sys_write(1, bb, k); return 0
44}
45func pi_len(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
46func pi_atoi(s: *u8) -> i64 {
47 var v: i64 = 0; var i: i64 = 0
48 while s[i] != (0 as u8) { let c: i64 = s[i] as i64; if c >= 48 { if c <= 57 { v = v*10 + (c-48) } } i = i + 1 }
49 return v
50}
51// host span of a url: writes offset of host start into ob[0], length into ob[1]; 0 = no host
52func pi_host(u: *u8, ul: i64, ob: *i64) -> i64 {
53 var i: i64 = 0
54 var hs: i64 = 0 - 1
55 while i + 2 < ul {
56 if u[i] == (58 as u8) { if u[i+1] == (47 as u8) { if u[i+2] == (47 as u8) { hs = i + 3; i = ul } } }
57 if hs < 0 { i = i + 1 }
58 }
59 if hs < 0 { return 0 }
60 var he: i64 = hs
61 while he < ul { if u[he] == (47 as u8) { ob[0] = hs; ob[1] = he - hs; return 1 } he = he + 1 }
62 ob[0] = hs
63 ob[1] = ul - hs
64 return 1
65}
66// same host? (byte-equal host spans)
67func pi_samehost(a: *u8, al: i64, b: *u8, bl: i64) -> i64 {
68 let oa: *i64 = sys_mmap(16) as *i64
69 let obx: *i64 = sys_mmap(16) as *i64
70 if pi_host(a, al, oa) == 0 { return 0 }
71 if pi_host(b, bl, obx) == 0 { return 0 }
72 if oa[1] != obx[1] { return 0 }
73 var i: i64 = 0
74 while i < oa[1] { if a[oa[0]+i] != b[obx[0]+i] { return 0 } i = i + 1 }
75 return 1
76}
77// fetch + extract + write one page into the shard. lenbox[0] = fetched html length (for the caller's
78// outlink walk). Returns 1 ingested, 2 already-present, 0 skipped/failed.
79func pi_one(url: *u8, store: *TrustStore, h: *i64, w: *i64, prefix: *u8, segbox: *i64,
80 minchars: i64, page: *u8, text: *u8, ceb: *i64, edged: *i64, lenbox: *i64) -> i64 {
81 let st: *i64 = sys_mmap(8) as *i64
82 var n: i64 = nx_https_fetch_follow(url, store, page, PI_FETCHCAP, 8, st)
83 lenbox[0] = 0
84 if st[0] != 200 { pi_puts(" skip status=" as *u8); pi_num(st[0]); pi_puts(" " as *u8); pi_puts(url); pi_puts("\n" as *u8); return 0 }
85 if n < 64 { return 0 }
86 // INFLATE (2026-08-03): the SAME defect class nx_research_engine.nx:323 fixed on 08-01 -- nx_codec_caps
87 // advertises gzip so servers legitimately compress, and tokenizing the WIRE bytes indexed GARBAGE
88 // (rfc-editor/openjdk/curl all serve gzip; the stored "text" began with the 1f 8b magic and no query
89 // could ever match). The sweep that fixed the research lane missed this crawler-side member.
90 if n >= 2 { if page[0] == (31 as u8) { if page[1] == (139 as u8) {
91 let gr: *NxGzipResult = nx_gzip_inflate(page, n, PI_FETCHCAP)
92 if gr.error_code != 0 { pi_puts(" skip gzip-fail " as *u8); pi_puts(url); pi_puts("\n" as *u8); return 0 }
93 if gr.output_size <= 0 { return 0 }
94 var gn: i64 = gr.output_size
95 if gn > PI_FETCHCAP { gn = PI_FETCHCAP }
96 let gd: *u8 = gr.output_data
97 var gc: i64 = 0
98 while gc < gn { page[gc] = gd[gc]; gc = gc + 1 }
99 n = gn
100 } } }
101 lenbox[0] = n
102 let tlen: i64 = nx_html_to_text(page, n, text, PI_MAGIC_1048576)
103 if tlen < minchars { return 0 }
104 var tn: i64 = tlen; if tn > CI_DOCCAP { tn = CI_DOCCAP }
105 let cid: i64 = ci_hash(text, tn)
106 let key: *u8 = sys_mmap(64)
107 dss_mkkey(cid, key)
108 let pb: *i64 = sys_mmap(16) as *i64
109 let lb: *i64 = sys_mmap(16) as *i64
110 if (h as i64) != 0 { if ss_hget(h, key, pb, lb) == 1 { return 2 } }
111 let ul: i64 = pi_len(url)
112 if ss_add(w, 1, key, text, tn) < 0 { if ss_commit(prefix, w, segbox[0])==0 {} segbox[0]=segbox[0]+1; w[1]=0; ss_add(w, 1, key, text, tn) }
113 let ukey: *u8 = sys_mmap(64)
114 ci_mkurlkey(cid, ukey)
115 if ss_add(w, 1, ukey, url, ul) < 0 { if ss_commit(prefix, w, segbox[0])==0 {} segbox[0]=segbox[0]+1; w[1]=0; ss_add(w, 1, ukey, url, ul) }
116 let cne: i64 = olh_scan(page, n, url, ul, ceb, OLH_MAXEDGE)
117 if cne > 0 {
118 let okey: *u8 = sys_mmap(64)
119 olh_outkey(ci_hash(url, ul), okey)
120 if ss_add(w, 1, okey, ceb as *u8, cne * 8) < 0 { if ss_commit(prefix, w, segbox[0])==0 {} segbox[0]=segbox[0]+1; w[1]=0; ss_add(w, 1, okey, ceb as *u8, cne * 8) }
121 edged[0] = edged[0] + 1
122 }
123 pi_puts(" + " as *u8); pi_puts(url); pi_puts(" (" as *u8); pi_num(tn); pi_puts(" chars)\n" as *u8)
124 return 1
125}
126
127func main(argc: i64, argv: *i64) -> i64 {
128 if argc < 2 { pi_puts("usage: nx_page_ingest <seed-url> [maxfollow]\n" as *u8); return 1 }
129 let seed: *u8 = argv[1] as *u8
130 var maxf: i64 = 30
131 if argc >= 3 { maxf = pi_atoi(argv[2] as *u8) }
132 if maxf > PI_MAXFOLLOW { maxf = PI_MAXFOLLOW }
133 let r: i64 = nx_trust_store_load_from_certdata("data/mozilla_certdata.txt" as *u8, 512, PI_MAGIC_4194304)
134 if r <= 0 { pi_puts("trust store load failed\nverdict=FAIL\n" as *u8); return 2 }
135 let store: *TrustStore = r as *TrustStore
136 pi_puts("=== nx_page_ingest seed=" as *u8); pi_puts(seed); pi_puts(" maxfollow=" as *u8); pi_num(maxf); pi_puts(" ===\n" as *u8)
137
138 let prefix: *u8 = sys_mmap(512); dss_prefix("web" as *u8, prefix)
139 let segs: *i64 = sys_mmap(PI_MAGIC_8192*8) as *i64
140 let nseg: i64 = ss_manifest_cap(prefix, segs, PI_MAGIC_8192)
141 let segbox: *i64 = sys_mmap(16) as *i64
142 segbox[0] = 1
143 var si: i64 = 0
144 // ROOT FIX seq1730 (id 1785450987): segs[] holds POINTERS to seg-<id> name strings
145 // (nx_seg_store.nx:1234 stores segs[cnt] = name as i64), NOT ids -- so segs[si] + 1 produced
146 // MMAP_ADDRESS+1 (~1.4e14), the pointer-shaped poison that PINS a plane forever: every later
147 // epoch id sorts BELOW it in supersede order and its rows are silently shadowed while rc=0.
148 // Parse the DIGITS, as nx_web_shard_compact.nx:55-58 already does on this SAME array.
149 while si < nseg {
150 let sg_nm: *u8 = segs[si] as *u8
151 var sg_v: i64 = 0
152 var sg_ci: i64 = 0
153 while sg_nm[sg_ci] != (0 as u8) { let sg_c: i64 = sg_nm[sg_ci] as i64; if sg_c >= 48 { if sg_c <= 57 { sg_v = sg_v * 10 + (sg_c - 48) } } sg_ci = sg_ci + 1 }
154 if sg_v < PI_SEGID_SANE { if sg_v >= segbox[0] { segbox[0] = sg_v + 1 } }
155 si = si + 1
156 }
157 let h: *i64 = ss_open(prefix)
158 let w: *i64 = ss_begin_cap(PI_SEGCAP)
159 let page: *u8 = sys_mmap(PI_FETCHCAP)
160 let text: *u8 = sys_mmap(PI_MAGIC_1048576)
161 let ceb: *i64 = sys_mmap(OLH_MAXEDGE * 8) as *i64
162 let edged: *i64 = sys_mmap(16) as *i64
163 edged[0] = 0
164
165 // 1) the seed (unconditional beyond the floor). pi_one already wrote the seed's out: edges.
166 let seedlen: *i64 = sys_mmap(16) as *i64
167 let rs: i64 = pi_one(seed, store, h, w, prefix, segbox, PI_MINSEED, page, text, ceb, edged, seedlen)
168 if rs == 0 { pi_puts("PAGE-INGEST seed fetch/extract FAILED\nverdict=FAIL\n" as *u8); return 3 }
169 let sl: i64 = pi_len(seed)
170 let plen: i64 = seedlen[0]
171 // collect the follow list: href="..." absolute http(s) urls on the SAME host, deduped, capped
172 let urls: *u8 = sys_mmap(PI_MAXFOLLOW * PI_MAGIC_2048)
173 var nurl: i64 = 0
174 var i: i64 = 0
175 while i + 8 < plen {
176 var advi: i64 = 1
177 if page[i]==(104 as u8) { if page[i+1]==(114 as u8) { if page[i+2]==(101 as u8) { if page[i+3]==(102 as u8) { if page[i+4]==(61 as u8) { if page[i+5]==(34 as u8) {
178 let j: i64 = i + 6
179 var e: i64 = j
180 var scanq: i64 = 1
181 while scanq == 1 {
182 if e >= plen { scanq = 0 } else { if page[e]==(34 as u8) { scanq = 0 } else { e = e + 1 } }
183 }
184 let ulen: i64 = e - j
185 if nurl < maxf { if ulen > 8 { if ulen < PI_MAGIC_1500 {
186 if page[j]==(104 as u8) { if page[j+1]==(116 as u8) { if page[j+2]==(116 as u8) { if page[j+3]==(112 as u8) {
187 let cand: *u8 = (page as i64 + j) as *u8
188 if pi_samehost(seed, sl, cand, ulen) == 1 {
189 var dup: i64 = 0
190 var d: i64 = 0
191 while d < nurl {
192 let sp: *u8 = (urls as i64 + d * PI_MAGIC_2048) as *u8
193 if pi_len(sp) == ulen {
194 var x: i64 = 0
195 var eq: i64 = 1
196 while x < ulen { if sp[x] != cand[x] { eq = 0; x = ulen } else { x = x + 1 } }
197 if eq == 1 { dup = 1; d = nurl }
198 }
199 if dup == 0 { d = d + 1 }
200 }
201 if dup == 0 {
202 let dst: *u8 = (urls as i64 + nurl * PI_MAGIC_2048) as *u8
203 var y: i64 = 0
204 while y < ulen { dst[y] = cand[y]; y = y + 1 }
205 dst[ulen] = 0 as u8
206 nurl = nurl + 1
207 }
208 }
209 } } } }
210 } } }
211 advi = (e + 1) - i
212 } } } } } }
213 i = i + advi
214 }
215 pi_puts("seed outlinks same-host collected=" as *u8); pi_num(nurl); pi_puts("\n" as *u8)
216
217 // 2) follow them, politely paced
218 var ing: i64 = 1
219 var present: i64 = 0
220 var skipped: i64 = 0
221 var f: i64 = 0
222 let flen: *i64 = sys_mmap(16) as *i64
223 while f < nurl {
224 sys_sleep_ms(PI_PACE_MS)
225 let uf: *u8 = (urls as i64 + f * PI_MAGIC_2048) as *u8
226 let rf: i64 = pi_one(uf, store, h, w, prefix, segbox, PI_MINFOLLOW, page, text, ceb, edged, flen)
227 if rf == 1 { ing = ing + 1 }
228 if rf == 2 { present = present + 1 }
229 if rf == 0 { skipped = skipped + 1 }
230 f = f + 1
231 }
232 if w[1] > 0 { if ss_commit(prefix, w, segbox[0])==0 {} }
233 pi_puts("PAGE-INGEST done: ingested=" as *u8); pi_num(ing)
234 pi_puts(" present=" as *u8); pi_num(present)
235 pi_puts(" skipped=" as *u8); pi_num(skipped)
236 pi_puts(" edged=" as *u8); pi_num(edged[0])
237 pi_puts("\nverdict=PASS\n" as *u8)
238 return 0
239}