nx_entity_seeds.nx source
↩ module page · 374 lines · 15414 B
1// nx_entity_seeds.nx -- SEED DISCOVERY: turn one entity page into the list of OTHER pages about that entity.
2//
3// Why this exists: our own index holds exactly ONE page for some entities, so search cannot supply seeds --
4// a measured dead end, not a guess. Seeds must come from STRUCTURE instead: an entity's page links out to
5// the other pages about it (filmography, official site, profiles, references). This fetches a page over
6// sovereign TLS (dechunk -> gunzip, same wire order as the gatherer), enumerates every <a href>, resolves
7// them absolute, and keeps the ones whose URL carries a token of the entity's name.
8//
9// The output is a plain seed list, one URL per line, so the gatherer can be run once per seed. Cross-seed
10// dedupe is FREE and needs no state: assets are sha256 content-addressed, so the same image reached from two
11// different seeds writes the same filename.
12//
13// usage: nx_entity_seeds <page-url> <name-token> <out-file> [max]
14
15import "nx_str.nx"
16import "nx_syscalls.nx"
17import "nx_csprng.nx"
18import "nx_x509_trust_store.nx"
19import "nx_pem_loader.nx"
20import "nx_https_get.nx"
21import "nx_html_extract_links.nx"
22import "nx_inflate.nx"
23import "nx_http_dechunk.nx"
24
25const ES_PAGE_CAP: i64 = 4194304
26const ES_URLBUF: i64 = 2097152
27const ES_MAX_LINKS: i64 = 4096
28const ES_PATHBUF: i64 = 4096
29
30func es_puts(s: *u8) -> i64 { sys_write(1, s, nx_str_len(s)); return 0 }
31func es_w(fd: i64, s: *u8) -> i64 { sys_write(fd, s, nx_str_len(s)); return 0 }
32
33func es_pi(v: i64) -> i64 {
34 let t: *u8 = sys_mmap(32)
35 let o: *u8 = sys_mmap(32)
36 var m: i64 = v
37 var k: i64 = 0
38 if m < 0 { m = 0 - m; sys_write(1, "-" as *u8, 1) }
39 if m == 0 { t[0] = 48; k = 1 }
40 while m > 0 { t[k] = 48 + (m % 10); m = m / 10; k = k + 1 }
41 var i: i64 = 0
42 while i < k { o[i] = t[k - 1 - i]; i = i + 1 }
43 sys_write(1, o, k)
44 return 0
45}
46
47func es_atoi(s: *u8) -> i64 {
48 var i: i64 = 0
49 var v: i64 = 0
50 while s[i] != (0 as u8) {
51 let c: i64 = s[i] as i64
52 if c >= 48 { if c <= 57 { v = v * 10 + (c - 48) } }
53 i = i + 1
54 }
55 return v
56}
57
58func es_lc(c: i64) -> i64 { if c >= 65 { if c <= 90 { return c + 32 } } return c }
59
60func es_body_off(resp: *u8, n: i64) -> i64 {
61 var i: i64 = 0
62 while i + 3 < n {
63 if resp[i] == (13 as u8) { if resp[i+1] == (10 as u8) { if resp[i+2] == (13 as u8) { if resp[i+3] == (10 as u8) { return i + 4 } } } }
64 i = i + 1
65 }
66 return 0
67}
68
69func es_inflate_if_gzip(src: *u8, n: i64, dst: *u8, cap: i64) -> i64 {
70 if n < 3 { return 0 }
71 if src[0] != (31 as u8) { return 0 }
72 if src[1] != (139 as u8) { return 0 }
73 let dn: i64 = inf_gunzip(src, n, dst, cap)
74 if dn <= 0 { return 0 }
75 return dn
76}
77
78func es_is_http(u: *u8, n: i64) -> i64 {
79 if n < 8 { return 0 }
80 if u[0] != (104 as u8) { return 0 }
81 if u[1] != (116 as u8) { return 0 }
82 if u[2] != (116 as u8) { return 0 }
83 if u[3] != (112 as u8) { return 0 }
84 return 1
85}
86
87// case-insensitive: does url[0..n) contain the NUL-terminated lowercase token?
88func es_has_tok(u: *u8, n: i64, tok: *u8) -> i64 {
89 let tl: i64 = nx_str_len(tok)
90 if tl == 0 { return 0 }
91 var i: i64 = 0
92 while i + tl <= n {
93 var j: i64 = 0
94 var hit: i64 = 1
95 while j < tl {
96 if es_lc(u[i+j] as i64) != (tok[j] as i64) { hit = 0; j = tl } else { j = j + 1 }
97 }
98 if hit == 1 { return 1 }
99 i = i + 1
100 }
101 return 0
102}
103
104func es_hasch(u: *u8, n: i64, ch: i64) -> i64 {
105 var i: i64 = 0
106 while i < n { if (u[i] as i64) == ch { return 1 } i = i + 1 }
107 return 0
108}
109
110// already emitted this exact URL? (linear over a modest kept-set)
111func es_seen(keep: *u8, koff: *i64, klen: *i64, nk: i64, u: *u8, n: i64) -> i64 {
112 var i: i64 = 0
113 while i < nk {
114 if klen[i] == n {
115 var same: i64 = 1
116 var j: i64 = 0
117 while j < n { if keep[koff[i]+j] != u[j] { same = 0; j = n } else { j = j + 1 } }
118 if same == 1 { return 1 }
119 }
120 i = i + 1
121 }
122 return 0
123}
124
125// ---- PLATFORM HANDLE EDGES (2026-08-15) ------------------------------------------------------
126// A HANDLE IS NOT A NAME. "little_hamster" shares no token with "diana rider", so the name-token gate
127// below can NEVER admit it -- measured on the proof run: 241 of 374 links rejected as no-token. A link
128// to a KNOWN PLATFORM HOST, found on a page already established to be about this entity, is a DIFFERENT
129// EDGE KIND: it earns admission on the HOST, not on the name. It bypasses ONLY the token gate; http,
130// fragment, meta, dupe and cap all still apply, so the published partition still reconciles.
131// ABSENT CONF = ZERO platforms = today's behaviour byte-for-byte. The policy lives in the data file,
132// never in a baked list, so adding a platform is an edit and never a recompile.
133const ES_PLAT_CONF: *u8 = "knowledge/entity_platforms.conf"
134const ES_PLAT_MAX: i64 = 32
135const ES_PLAT_SLOT: i64 = 64
136static es_plat_g: *u8
137static es_nplat_g: i64
138static es_ploaded_g: i64
139func es_plat_pat(i: i64) -> *u8 { return ((es_plat_g as i64) + i * ES_PLAT_SLOT) as *u8 }
140func es_load_platforms() -> i64 {
141 if es_ploaded_g == 1 { return es_nplat_g }
142 es_ploaded_g = 1
143 es_plat_g = sys_mmap(ES_PLAT_SLOT * ES_PLAT_MAX)
144 es_nplat_g = 0
145 let lb: *i64 = sys_mmap(16) as *i64
146 lb[0] = 0
147 let b: *u8 = sys_read_file(ES_PLAT_CONF, lb)
148 let n: i64 = lb[0]
149 if (b as i64) == 0 { return 0 }
150 if n <= 0 { return 0 }
151 var i: i64 = 0
152 while i < n {
153 let ls: i64 = i
154 var le: i64 = ls
155 var go: i64 = 1
156 while go == 1 { if le >= n { go = 0 } else { if b[le] == (10 as u8) { go = 0 } else { le = le + 1 } } }
157 i = le + 1
158 if le > ls { if b[ls] != (35 as u8) {
159 var t: i64 = 0 - 1
160 var j: i64 = ls
161 while j < le { if b[j] == (9 as u8) { if t < 0 { t = j } } j = j + 1 }
162 if t > ls { if es_nplat_g < ES_PLAT_MAX {
163 let pl: i64 = le - t - 1
164 if pl > 0 { if pl < ES_PLAT_SLOT - 1 {
165 let dst: *u8 = es_plat_pat(es_nplat_g)
166 var c: i64 = 0
167 while c < pl { dst[c] = b[t + 1 + c]; c = c + 1 }
168 dst[pl] = 0 as u8
169 es_nplat_g = es_nplat_g + 1
170 } }
171 } }
172 } }
173 }
174 return es_nplat_g
175}
176// position of tok in u[0..n), else -1. IDENTICAL matching semantics to es_has_tok (haystack lowercased,
177// token taken as-is) so the two can never disagree about what matched.
178func es_find_tok(u: *u8, n: i64, tok: *u8) -> i64 {
179 let tl: i64 = nx_str_len(tok)
180 if tl == 0 { return 0 - 1 }
181 var i: i64 = 0
182 while i + tl <= n {
183 var j: i64 = 0
184 var hit: i64 = 1
185 while j < tl {
186 if es_lc(u[i+j] as i64) != (tok[j] as i64) { hit = 0; j = tl } else { j = j + 1 }
187 }
188 if hit == 1 { return i }
189 i = i + 1
190 }
191 return 0 - 1
192}
193// 1 iff u[0..n) is a PROFILE url for this platform: exactly ONE path segment after the host prefix.
194// DERIVED FROM URL STRUCTURE, never a per-platform blocklist that would need maintaining:
195// instagram.com/ghst_pilot/ -> one segment -> an IDENTITY
196// instagram.com/p/DLGkphjOiH9/ -> two segments -> a POST, which names nobody
197// MEASURED 2026-08-15: without this, 2 of 3 admitted platform edges were post permalinks, so a
198// "platform_handles" count overstated identities by 2x. Query and fragment are cut before counting.
199func es_is_profile_url(u: *u8, n: i64, pat: *u8) -> i64 {
200 let p: i64 = es_find_tok(u, n, pat)
201 if p < 0 { return 0 }
202 let a: i64 = p + nx_str_len(pat)
203 var e: i64 = n
204 var i: i64 = a
205 var go: i64 = 1
206 while go == 1 {
207 if i >= e { go = 0 } else {
208 let c: i64 = u[i] as i64
209 if c == 63 { e = i; go = 0 } else { if c == 35 { e = i; go = 0 } else { i = i + 1 } }
210 }
211 }
212 if a >= e { return 0 }
213 var slashes: i64 = 0
214 var k: i64 = a
215 while k < e { if (u[k] as i64) == 47 { slashes = slashes + 1 } k = k + 1 }
216 if slashes == 0 { return 1 }
217 if slashes == 1 { if (u[e - 1] as i64) == 47 { return 1 } }
218 return 0
219}
220// index of the platform whose host-prefix appears in u[0..n), else -1. Composes es_has_tok so there is
221// exactly ONE substring matcher in this organ.
222func es_platform_of(u: *u8, n: i64) -> i64 {
223 var i: i64 = 0
224 while i < es_nplat_g {
225 if es_has_tok(u, n, es_plat_pat(i)) == 1 { return i }
226 i = i + 1
227 }
228 return 0 - 1
229}
230func main(argc: i64, argv: *i64) -> i64 {
231 if argc < 4 {
232 es_puts("usage: nx_entity_seeds <page-url> <name-token> <out-file> [max]\n" as *u8)
233 sys_exit(2); return 2
234 }
235 let url: *u8 = argv[1] as *u8
236 let tok: *u8 = argv[2] as *u8
237 let outp: *u8 = argv[3] as *u8
238 var maxn: i64 = 64
239 if argc >= 5 { maxn = es_atoi(argv[4] as *u8) }
240 if maxn <= 0 { maxn = 64 }
241 if maxn > ES_MAX_LINKS { maxn = ES_MAX_LINKS }
242
243 es_puts("=== ENTITY SEED DISCOVERY ===\n page=" as *u8); es_puts(url)
244 es_puts(" token=" as *u8); es_puts(tok)
245 es_puts(" max=" as *u8); es_pi(maxn); es_puts("\n" as *u8)
246
247 let store: *TrustStore = trust_store_alloc(400)
248 let nroots: i64 = nx_pem_trust_load_file("/etc/ssl/certs/ca-certificates.crt" as *u8, store)
249 if nroots <= 0 { es_puts("ENTITY-SEEDS-FAIL no-roots\n" as *u8); sys_exit(1); return 1 }
250
251 let cr: *u8 = sys_mmap(32)
252 let pk: *u8 = sys_mmap(32)
253 nx_csprng_fill(cr, 32)
254 nx_csprng_fill(pk, 32)
255 let page: *u8 = sys_mmap(ES_PAGE_CAP)
256 let pr: i64 = nx_https_get(url, cr, pk, store, sys_now_realtime_sec(), page, ES_PAGE_CAP)
257 if pr <= 0 { es_puts("ENTITY-SEEDS-FAIL page-fetch\n" as *u8); sys_exit(1); return 1 }
258
259 let bo: i64 = es_body_off(page, pr)
260 let body: *u8 = ((page as i64) + bo) as *u8
261 let blen: i64 = pr - bo
262
263 var raw: *u8 = body
264 var rawlen: i64 = blen
265 let dech: *u8 = sys_mmap(ES_PAGE_CAP)
266 let cn: i64 = nx_http_dechunk(body, blen, dech, ES_PAGE_CAP)
267 if cn > 0 { raw = dech; rawlen = cn }
268
269 var html: *u8 = raw
270 var hlen: i64 = rawlen
271 let infb: *u8 = sys_mmap(ES_PAGE_CAP)
272 let pdn: i64 = es_inflate_if_gzip(raw, rawlen, infb, ES_PAGE_CAP)
273 if pdn > 0 { html = infb; hlen = pdn }
274
275 es_puts(" html_bytes=" as *u8); es_pi(hlen); es_puts("\n" as *u8)
276
277 let url_buf: *u8 = sys_mmap(ES_URLBUF)
278 let offs: *i64 = sys_mmap(8 * ES_MAX_LINKS) as *i64
279 let lens: *i64 = sys_mmap(8 * ES_MAX_LINKS) as *i64
280 let nlinks: i64 = nx_html_extract_links(html, hlen, url, nx_str_len(url), url_buf, ES_URLBUF, offs, lens, ES_MAX_LINKS)
281 es_puts(" links extracted=" as *u8); es_pi(nlinks); es_puts("\n" as *u8)
282 if nlinks <= 0 { es_puts("ENTITY-SEEDS-FAIL no-links\n" as *u8); sys_exit(1); return 1 }
283
284 let keep: *u8 = sys_mmap(ES_URLBUF)
285 let koff: *i64 = sys_mmap(8 * ES_MAX_LINKS) as *i64
286 let klen: *i64 = sys_mmap(8 * ES_MAX_LINKS) as *i64
287 var nk: i64 = 0
288 var kpos: i64 = 0
289 var n_nonhttp: i64 = 0
290 var n_notok: i64 = 0
291 var n_dupe: i64 = 0
292 var n_meta: i64 = 0
293 var n_plat: i64 = 0 // PLATFORM edges that are PROFILE urls -- identity edges
294 var n_platpost: i64 = 0 // PLATFORM edges that are post permalinks -- content, not identity
295 var capped: i64 = 0
296
297 // load the platform table BEFORE the filter loop; 0 platforms = no platform edges = old behaviour
298 let nplat: i64 = es_load_platforms()
299 var i: i64 = 0
300 while i < nlinks {
301 let up: *u8 = ((url_buf as i64) + offs[i]) as *u8
302 let ul: i64 = lens[i]
303 var st: i64 = 0
304 if nk >= maxn { st = 9; capped = 1 }
305 if st == 0 { if es_is_http(up, ul) == 0 { st = 1; n_nonhttp = n_nonhttp + 1 } }
306 // The token must appear in the PATH, not the query string. Social-share and widget URLs carry the
307 // whole page URL (and therefore the entity name) inside a query parameter, so matching the full URL
308 // admits every share button on the page as a "seed". Truncating at '?' rejects those by construction
309 // while keeping real paginated paths like /idols/<name>/?ipage=2.
310 var pathlen: i64 = ul
311 var qi: i64 = 0
312 var qdone: i64 = 0
313 while qi < ul {
314 if qdone == 0 { if (up[qi] as i64) == 63 { pathlen = qi; qdone = 1 } }
315 qi = qi + 1
316 }
317 // PLATFORM EDGE: admitted on the HOST, so it skips the token gate and nothing else.
318 var isplat: i64 = 0 - 1
319 if st == 0 { isplat = es_platform_of(up, ul) }
320 if st == 0 { if isplat < 0 { if es_has_tok(up, pathlen, tok) == 0 { st = 1; n_notok = n_notok + 1 } } }
321 // A fragment is the SAME page and a wiki meta/action URL is not about the entity -- both would
322 // spend a polite fetch to re-read the page we already have. Reject before the network, not after.
323 if st == 0 { if es_hasch(up, ul, 35) == 1 { st = 1; n_meta = n_meta + 1 } }
324 if st == 0 { if es_has_tok(up, ul, "special:" as *u8) == 1 { st = 1; n_meta = n_meta + 1 } }
325 if st == 0 { if es_has_tok(up, ul, "talk:" as *u8) == 1 { st = 1; n_meta = n_meta + 1 } }
326 if st == 0 { if es_has_tok(up, ul, "action=" as *u8) == 1 { st = 1; n_meta = n_meta + 1 } }
327 if st == 0 { if es_has_tok(up, ul, "index.php" as *u8) == 1 { st = 1; n_meta = n_meta + 1 } }
328 if st == 0 { if es_seen(keep, koff, klen, nk, up, ul) == 1 { st = 1; n_dupe = n_dupe + 1 } }
329 if st == 0 { if kpos + ul >= ES_URLBUF { st = 1 } }
330 if st == 0 {
331 var c: i64 = 0
332 while c < ul { keep[kpos + c] = up[c]; c = c + 1 }
333 koff[nk] = kpos
334 klen[nk] = ul
335 kpos = kpos + ul
336 // A post permalink IS a legitimate seed (content about the entity) -- it is just not an
337 // IDENTITY. Counted apart rather than rejected, so the handle count means what it says.
338 if isplat >= 0 {
339 if es_is_profile_url(up, ul, es_plat_pat(isplat)) == 1 { n_plat = n_plat + 1 } else { n_platpost = n_platpost + 1 }
340 }
341 nk = nk + 1
342 }
343 i = i + 1
344 }
345
346 let fd: i64 = sys_openat_wr(outp, 0x1a4)
347 if fd < 0 { es_puts("ENTITY-SEEDS-FAIL open-out\n" as *u8); sys_exit(1); return 1 }
348 var k: i64 = 0
349 while k < nk {
350 sys_write(fd, ((keep as i64) + koff[k]) as *u8, klen[k])
351 es_w(fd, "\n" as *u8)
352 sys_write(1, ((keep as i64) + koff[k]) as *u8, klen[k])
353 es_puts("\n" as *u8)
354 k = k + 1
355 }
356 sys_close(fd)
357
358 es_puts("{\"tool\":\"nx_entity_seeds\",\"page\":\"" as *u8); es_puts(url)
359 es_puts("\",\"token\":\"" as *u8); es_puts(tok)
360 es_puts("\",\"links_extracted\":" as *u8); es_pi(nlinks)
361 es_puts(",\"seeds\":" as *u8); es_pi(nk)
362 es_puts(",\"rejected_non_http\":" as *u8); es_pi(n_nonhttp)
363 es_puts(",\"rejected_no_token\":" as *u8); es_pi(n_notok)
364 es_puts(",\"rejected_duplicate\":" as *u8); es_pi(n_dupe)
365 es_puts(",\"rejected_fragment_or_meta\":" as *u8); es_pi(n_meta)
366 es_puts(",\"platform_handles\":" as *u8); es_pi(n_plat)
367 es_puts(",\"platform_posts\":" as *u8); es_pi(n_platpost)
368 es_puts(",\"platforms_loaded\":" as *u8); es_pi(nplat)
369 es_puts(",\"capped_at_max\":" as *u8); es_pi(capped)
370 es_puts(",\"out\":\"" as *u8); es_puts(outp)
371 es_puts("\"}\nENTITY-SEEDS-OK\n" as *u8)
372 sys_exit(0)
373 return 0
374}