nx_entity_seeds.nx source
↩ module page · 254 lines · 9825 B
1// nx_entity_seeds.nx -- SEED DISCOVERY: turn one entity page into the list of OTHER pages about that entity.
2//
3// Why this exists: our own index holds exactly ONE page for some entities, so search cannot supply seeds --
4// a measured dead end, not a guess. Seeds must come from STRUCTURE instead: an entity's page links out to
5// the other pages about it (filmography, official site, profiles, references). This fetches a page over
6// sovereign TLS (dechunk -> gunzip, same wire order as the gatherer), enumerates every <a href>, resolves
7// them absolute, and keeps the ones whose URL carries a token of the entity's name.
8//
9// The output is a plain seed list, one URL per line, so the gatherer can be run once per seed. Cross-seed
10// dedupe is FREE and needs no state: assets are sha256 content-addressed, so the same image reached from two
11// different seeds writes the same filename.
12//
13// usage: nx_entity_seeds <page-url> <name-token> <out-file> [max]
14
15import "nx_str.nx"
16import "nx_syscalls.nx"
17import "nx_csprng.nx"
18import "nx_x509_trust_store.nx"
19import "nx_pem_loader.nx"
20import "nx_https_get.nx"
21import "nx_html_extract_links.nx"
22import "nx_inflate.nx"
23import "nx_http_dechunk.nx"
24
25const ES_PAGE_CAP: i64 = 4194304
26const ES_URLBUF: i64 = 2097152
27const ES_MAX_LINKS: i64 = 4096
28const ES_PATHBUF: i64 = 4096
29
30func es_puts(s: *u8) -> i64 { sys_write(1, s, nx_str_len(s)); return 0 }
31func es_w(fd: i64, s: *u8) -> i64 { sys_write(fd, s, nx_str_len(s)); return 0 }
32
33func es_pi(v: i64) -> i64 {
34 let t: *u8 = sys_mmap(32)
35 let o: *u8 = sys_mmap(32)
36 var m: i64 = v
37 var k: i64 = 0
38 if m < 0 { m = 0 - m; sys_write(1, "-" as *u8, 1) }
39 if m == 0 { t[0] = 48; k = 1 }
40 while m > 0 { t[k] = 48 + (m % 10); m = m / 10; k = k + 1 }
41 var i: i64 = 0
42 while i < k { o[i] = t[k - 1 - i]; i = i + 1 }
43 sys_write(1, o, k)
44 return 0
45}
46
47func es_atoi(s: *u8) -> i64 {
48 var i: i64 = 0
49 var v: i64 = 0
50 while s[i] != (0 as u8) {
51 let c: i64 = s[i] as i64
52 if c >= 48 { if c <= 57 { v = v * 10 + (c - 48) } }
53 i = i + 1
54 }
55 return v
56}
57
58func es_lc(c: i64) -> i64 { if c >= 65 { if c <= 90 { return c + 32 } } return c }
59
60func es_body_off(resp: *u8, n: i64) -> i64 {
61 var i: i64 = 0
62 while i + 3 < n {
63 if resp[i] == (13 as u8) { if resp[i+1] == (10 as u8) { if resp[i+2] == (13 as u8) { if resp[i+3] == (10 as u8) { return i + 4 } } } }
64 i = i + 1
65 }
66 return 0
67}
68
69func es_inflate_if_gzip(src: *u8, n: i64, dst: *u8, cap: i64) -> i64 {
70 if n < 3 { return 0 }
71 if src[0] != (31 as u8) { return 0 }
72 if src[1] != (139 as u8) { return 0 }
73 let dn: i64 = inf_gunzip(src, n, dst, cap)
74 if dn <= 0 { return 0 }
75 return dn
76}
77
78func es_is_http(u: *u8, n: i64) -> i64 {
79 if n < 8 { return 0 }
80 if u[0] != (104 as u8) { return 0 }
81 if u[1] != (116 as u8) { return 0 }
82 if u[2] != (116 as u8) { return 0 }
83 if u[3] != (112 as u8) { return 0 }
84 return 1
85}
86
87// case-insensitive: does url[0..n) contain the NUL-terminated lowercase token?
88func es_has_tok(u: *u8, n: i64, tok: *u8) -> i64 {
89 let tl: i64 = nx_str_len(tok)
90 if tl == 0 { return 0 }
91 var i: i64 = 0
92 while i + tl <= n {
93 var j: i64 = 0
94 var hit: i64 = 1
95 while j < tl {
96 if es_lc(u[i+j] as i64) != (tok[j] as i64) { hit = 0; j = tl } else { j = j + 1 }
97 }
98 if hit == 1 { return 1 }
99 i = i + 1
100 }
101 return 0
102}
103
104func es_hasch(u: *u8, n: i64, ch: i64) -> i64 {
105 var i: i64 = 0
106 while i < n { if (u[i] as i64) == ch { return 1 } i = i + 1 }
107 return 0
108}
109
110// already emitted this exact URL? (linear over a modest kept-set)
111func es_seen(keep: *u8, koff: *i64, klen: *i64, nk: i64, u: *u8, n: i64) -> i64 {
112 var i: i64 = 0
113 while i < nk {
114 if klen[i] == n {
115 var same: i64 = 1
116 var j: i64 = 0
117 while j < n { if keep[koff[i]+j] != u[j] { same = 0; j = n } else { j = j + 1 } }
118 if same == 1 { return 1 }
119 }
120 i = i + 1
121 }
122 return 0
123}
124
125func main(argc: i64, argv: *i64) -> i64 {
126 if argc < 4 {
127 es_puts("usage: nx_entity_seeds <page-url> <name-token> <out-file> [max]\n" as *u8)
128 sys_exit(2); return 2
129 }
130 let url: *u8 = argv[1] as *u8
131 let tok: *u8 = argv[2] as *u8
132 let outp: *u8 = argv[3] as *u8
133 var maxn: i64 = 64
134 if argc >= 5 { maxn = es_atoi(argv[4] as *u8) }
135 if maxn <= 0 { maxn = 64 }
136 if maxn > ES_MAX_LINKS { maxn = ES_MAX_LINKS }
137
138 es_puts("=== ENTITY SEED DISCOVERY ===\n page=" as *u8); es_puts(url)
139 es_puts(" token=" as *u8); es_puts(tok)
140 es_puts(" max=" as *u8); es_pi(maxn); es_puts("\n" as *u8)
141
142 let store: *TrustStore = trust_store_alloc(400)
143 let nroots: i64 = nx_pem_trust_load_file("/etc/ssl/certs/ca-certificates.crt" as *u8, store)
144 if nroots <= 0 { es_puts("ENTITY-SEEDS-FAIL no-roots\n" as *u8); sys_exit(1); return 1 }
145
146 let cr: *u8 = sys_mmap(32)
147 let pk: *u8 = sys_mmap(32)
148 nx_csprng_fill(cr, 32)
149 nx_csprng_fill(pk, 32)
150 let page: *u8 = sys_mmap(ES_PAGE_CAP)
151 let pr: i64 = nx_https_get(url, cr, pk, store, sys_now_realtime_sec(), page, ES_PAGE_CAP)
152 if pr <= 0 { es_puts("ENTITY-SEEDS-FAIL page-fetch\n" as *u8); sys_exit(1); return 1 }
153
154 let bo: i64 = es_body_off(page, pr)
155 let body: *u8 = ((page as i64) + bo) as *u8
156 let blen: i64 = pr - bo
157
158 var raw: *u8 = body
159 var rawlen: i64 = blen
160 let dech: *u8 = sys_mmap(ES_PAGE_CAP)
161 let cn: i64 = nx_http_dechunk(body, blen, dech, ES_PAGE_CAP)
162 if cn > 0 { raw = dech; rawlen = cn }
163
164 var html: *u8 = raw
165 var hlen: i64 = rawlen
166 let infb: *u8 = sys_mmap(ES_PAGE_CAP)
167 let pdn: i64 = es_inflate_if_gzip(raw, rawlen, infb, ES_PAGE_CAP)
168 if pdn > 0 { html = infb; hlen = pdn }
169
170 es_puts(" html_bytes=" as *u8); es_pi(hlen); es_puts("\n" as *u8)
171
172 let url_buf: *u8 = sys_mmap(ES_URLBUF)
173 let offs: *i64 = sys_mmap(8 * ES_MAX_LINKS) as *i64
174 let lens: *i64 = sys_mmap(8 * ES_MAX_LINKS) as *i64
175 let nlinks: i64 = nx_html_extract_links(html, hlen, url, nx_str_len(url), url_buf, ES_URLBUF, offs, lens, ES_MAX_LINKS)
176 es_puts(" links extracted=" as *u8); es_pi(nlinks); es_puts("\n" as *u8)
177 if nlinks <= 0 { es_puts("ENTITY-SEEDS-FAIL no-links\n" as *u8); sys_exit(1); return 1 }
178
179 let keep: *u8 = sys_mmap(ES_URLBUF)
180 let koff: *i64 = sys_mmap(8 * ES_MAX_LINKS) as *i64
181 let klen: *i64 = sys_mmap(8 * ES_MAX_LINKS) as *i64
182 var nk: i64 = 0
183 var kpos: i64 = 0
184 var n_nonhttp: i64 = 0
185 var n_notok: i64 = 0
186 var n_dupe: i64 = 0
187 var n_meta: i64 = 0
188 var capped: i64 = 0
189
190 var i: i64 = 0
191 while i < nlinks {
192 let up: *u8 = ((url_buf as i64) + offs[i]) as *u8
193 let ul: i64 = lens[i]
194 var st: i64 = 0
195 if nk >= maxn { st = 9; capped = 1 }
196 if st == 0 { if es_is_http(up, ul) == 0 { st = 1; n_nonhttp = n_nonhttp + 1 } }
197 // The token must appear in the PATH, not the query string. Social-share and widget URLs carry the
198 // whole page URL (and therefore the entity name) inside a query parameter, so matching the full URL
199 // admits every share button on the page as a "seed". Truncating at '?' rejects those by construction
200 // while keeping real paginated paths like /idols/<name>/?ipage=2.
201 var pathlen: i64 = ul
202 var qi: i64 = 0
203 var qdone: i64 = 0
204 while qi < ul {
205 if qdone == 0 { if (up[qi] as i64) == 63 { pathlen = qi; qdone = 1 } }
206 qi = qi + 1
207 }
208 if st == 0 { if es_has_tok(up, pathlen, tok) == 0 { st = 1; n_notok = n_notok + 1 } }
209 // A fragment is the SAME page and a wiki meta/action URL is not about the entity -- both would
210 // spend a polite fetch to re-read the page we already have. Reject before the network, not after.
211 if st == 0 { if es_hasch(up, ul, 35) == 1 { st = 1; n_meta = n_meta + 1 } }
212 if st == 0 { if es_has_tok(up, ul, "special:" as *u8) == 1 { st = 1; n_meta = n_meta + 1 } }
213 if st == 0 { if es_has_tok(up, ul, "talk:" as *u8) == 1 { st = 1; n_meta = n_meta + 1 } }
214 if st == 0 { if es_has_tok(up, ul, "action=" as *u8) == 1 { st = 1; n_meta = n_meta + 1 } }
215 if st == 0 { if es_has_tok(up, ul, "index.php" as *u8) == 1 { st = 1; n_meta = n_meta + 1 } }
216 if st == 0 { if es_seen(keep, koff, klen, nk, up, ul) == 1 { st = 1; n_dupe = n_dupe + 1 } }
217 if st == 0 { if kpos + ul >= ES_URLBUF { st = 1 } }
218 if st == 0 {
219 var c: i64 = 0
220 while c < ul { keep[kpos + c] = up[c]; c = c + 1 }
221 koff[nk] = kpos
222 klen[nk] = ul
223 kpos = kpos + ul
224 nk = nk + 1
225 }
226 i = i + 1
227 }
228
229 let fd: i64 = sys_openat_wr(outp, 0x1a4)
230 if fd < 0 { es_puts("ENTITY-SEEDS-FAIL open-out\n" as *u8); sys_exit(1); return 1 }
231 var k: i64 = 0
232 while k < nk {
233 sys_write(fd, ((keep as i64) + koff[k]) as *u8, klen[k])
234 es_w(fd, "\n" as *u8)
235 sys_write(1, ((keep as i64) + koff[k]) as *u8, klen[k])
236 es_puts("\n" as *u8)
237 k = k + 1
238 }
239 sys_close(fd)
240
241 es_puts("{\"tool\":\"nx_entity_seeds\",\"page\":\"" as *u8); es_puts(url)
242 es_puts("\",\"token\":\"" as *u8); es_puts(tok)
243 es_puts("\",\"links_extracted\":" as *u8); es_pi(nlinks)
244 es_puts(",\"seeds\":" as *u8); es_pi(nk)
245 es_puts(",\"rejected_non_http\":" as *u8); es_pi(n_nonhttp)
246 es_puts(",\"rejected_no_token\":" as *u8); es_pi(n_notok)
247 es_puts(",\"rejected_duplicate\":" as *u8); es_pi(n_dupe)
248 es_puts(",\"rejected_fragment_or_meta\":" as *u8); es_pi(n_meta)
249 es_puts(",\"capped_at_max\":" as *u8); es_pi(capped)
250 es_puts(",\"out\":\"" as *u8); es_puts(outp)
251 es_puts("\"}\nENTITY-SEEDS-OK\n" as *u8)
252 sys_exit(0)
253 return 0
254}