nx_media_gather.nx source
↩ module page · 238 lines · 11078 B
1// nx_media_gather.nx -- the ENTITY MEDIA GATHER rung (canonical case: Diora Baird, the dossier entity from
2// docs/06): fetch an entity's page over the team's OWN validated HTTPS (sovereign DNS+TCP+TLS1.3+X.509, the
3// proven nx_live_https path), extract its image URLs (nx_html_extract_imgs), politely fetch the FIRST hosted
4// image (1.5s pacing, 2 requests total -- far under any polite floor), sniff the magic (nx_media_sniff),
5// content-hash it (sha256_digest), store it content-addressed under knowledge/media/<entity>/, append a
6// PROVENANCE ledger row (knowledge/media/media_ledger.tsv: entity, source url, image url, bytes, sha256,
7// kind), and write a SEARCHABLE dossier doc (knowledge/library/media-<entity>.txt) so the next harvest makes
8// the gathered media findable by nx_search_cli. SELF-VERIFYING: re-reads the stored file and re-hashes --
9// store verdict is byte-proven, not assumed. v2 rungs: pHash dedup (needs nx_jpeg_decode), multi-image walk,
10// robots+governor wiring for >2-request sessions. Usage: nx_media_gather [url] [entity]
11// license_tier: ORIGINAL
12import "nx_str.nx"
13import "nx_syscalls.nx"
14import "nx_csprng.nx"
15import "nx_x509_trust_store.nx"
16import "nx_pem_loader.nx"
17import "nx_https_get.nx"
18import "nx_html_extract_imgs.nx"
19import "nx_media_sniff.nx"
20import "nx_sha256.nx"
21const MG_MAGIC_131072: i64 = 131072
22const MG_MAGIC_4096: i64 = 4096
23const MG_MAGIC_1500: i64 = 1500
24
25const MG_PAGE_CAP: i64 = 4194304 // 4 MB page buffer
26const MG_IMG_CAP: i64 = 8388608 // 8 MB image buffer
27const MG_MAX_IMGS: i64 = 256
28
29func mg_puts(s: *u8) -> i64 { sys_write(1, s, nx_str_len(s)); return 0 }
30func mg_pi(v: i64) -> i64 { let bb: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m;sys_write(1,"-" as *u8,1)}; let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=48;k=1}; while m>0{t[k]=48+(m%10);m=m/10;k=k+1}; var i: i64=0; while i<k{bb[i]=t[k-1-i];i=i+1}; sys_write(1,bb,k); return 0 }
31func mg_cat(dst: *u8, off: i64, s: *u8) -> i64 { var i: i64=0; while s[i]!=(0 as u8){dst[off+i]=s[i];i=i+1} return off+i }
32func mg_w(fd: i64, s: *u8) -> i64 { sys_write(fd, s, nx_str_len(s)); return 0 }
33func mg_wn(fd: i64, v: i64) -> i64 { let bb: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m;sys_write(fd,"-" as *u8,1)}; let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=48;k=1}; while m>0{t[k]=48+(m%10);m=m/10;k=k+1}; var i: i64=0; while i<k{bb[i]=t[k-1-i];i=i+1}; sys_write(fd,bb,k); return 0 }
34
35// mkdir via x86_64 mkdirat=258 DIRECTLY (x86-only number, passes through both x86 lanes' translators).
36// sys_mkdir's rv64 34 HANGS on the May-29 native lane: that compiler predates the 34->258 table row
37// (2026-06-06), so 34 falls through to x86 pause() = block-forever. Caught live by the mkdir probe.
38// NOT rv64-portable -- this module is x86-lane only until the lane regains the table row.
39func mg_mkdir(path: *u8) -> i64 { return __syscall(258, -100, path, 0x1ed, 0, 0, 0) }
40
41// hex of a 32-byte digest into out (64 chars + NUL)
42func mg_hex(d: *u8, out: *u8) -> i64 {
43 let hx: *u8 = "0123456789abcdef" as *u8
44 var i: i64 = 0
45 while i < 32 {
46 let b: i64 = d[i] as i64
47 out[i*2] = hx[(b >> 4) & 15]
48 out[i*2+1] = hx[b & 15]
49 i = i + 1
50 }
51 out[64] = 0 as u8
52 return 64
53}
54
55// end of HTTP headers (same helper the crawl modules carry)
56func mg_body_off(resp: *u8, n: i64) -> i64 {
57 var i: i64 = 0
58 while i + 3 < n {
59 if resp[i] == (13 as u8) { if resp[i+1] == (10 as u8) { if resp[i+2] == (13 as u8) { if resp[i+3] == (10 as u8) { return i + 4 } } } }
60 i = i + 1
61 }
62 return 0
63}
64
65// substring containment (NUL-terminated needle)
66func mg_has(hay: *u8, n: i64, needle: *u8) -> i64 {
67 let nl: i64 = nx_str_len(needle)
68 if nl == 0 { return 0 }
69 var i: i64 = 0
70 while i + nl <= n {
71 var j: i64 = 0
72 var hit: i64 = 1
73 while j < nl { if hay[i+j] != needle[j] { hit = 0; j = nl } else { j = j + 1 } }
74 if hit == 1 { return 1 }
75 i = i + 1
76 }
77 return 0
78}
79
80// write whole buffer to path (create+truncate); return bytes or -1
81func mg_write_file(path: *u8, buf: *u8, n: i64) -> i64 {
82 let fd: i64 = sys_openat_wr(path, 0x1a4)
83 if fd < 0 { return 0 - 1 }
84 var off: i64 = 0
85 while off < n {
86 let w: i64 = sys_write(fd, ((buf as i64) + off) as *u8, n - off)
87 if w <= 0 { sys_close(fd); return 0 - 1 }
88 off = off + w
89 }
90 sys_close(fd)
91 return n
92}
93
94func main(argc: i64, argv: *i64) -> i64 {
95 var url: *u8 = "https://en.wikipedia.org/wiki/Diora_Baird" as *u8
96 var entity: *u8 = "diora_baird" as *u8
97 if argc >= 2 { url = argv[1] as *u8 }
98 if argc >= 3 { entity = argv[2] as *u8 }
99 mg_puts("=== MEDIA GATHER (validated-HTTPS fetch -> extract -> sniff -> hash -> store -> provenance) ===\n" as *u8)
100 mg_puts(" entity=" as *u8); mg_puts(entity); mg_puts(" page=" as *u8); mg_puts(url); mg_puts("\n" as *u8)
101
102 let store: *TrustStore = trust_store_alloc(400)
103 let nroots: i64 = nx_pem_trust_load_file("/etc/ssl/certs/ca-certificates.crt" as *u8, store)
104 mg_puts(" trust roots=" as *u8); mg_pi(nroots); mg_puts("\n" as *u8)
105 if nroots <= 0 { mg_puts("MEDIA-GATHER-FAIL no-roots\n" as *u8); sys_exit(1); return 1 }
106
107 // ---- request 1: the entity page ----
108 let cr: *u8 = sys_mmap(32); nx_csprng_fill(cr, 32)
109 let pk: *u8 = sys_mmap(32); nx_csprng_fill(pk, 32)
110 let page: *u8 = sys_mmap(MG_PAGE_CAP)
111 let now: i64 = sys_now_realtime_sec()
112 let pr: i64 = nx_https_get(url, cr, pk, store, now, page, MG_PAGE_CAP)
113 mg_puts(" page fetch bytes=" as *u8); mg_pi(pr); mg_puts("\n" as *u8)
114 if pr <= 0 { mg_puts("MEDIA-GATHER-FAIL page-fetch\n" as *u8); sys_exit(1); return 1 }
115 let bo: i64 = mg_body_off(page, pr)
116 let body: *u8 = ((page as i64) + bo) as *u8
117 let blen: i64 = pr - bo
118
119 // ---- extract image URLs ----
120 let url_buf: *u8 = sys_mmap(MG_MAGIC_131072)
121 let offs: *i64 = sys_mmap(8 * MG_MAX_IMGS) as *i64
122 let lens: *i64 = sys_mmap(8 * MG_MAX_IMGS) as *i64
123 let nimgs: i64 = nx_html_extract_imgs(body, blen, url, nx_str_len(url), url_buf, MG_MAGIC_131072, offs, lens, MG_MAX_IMGS)
124 mg_puts(" images extracted=" as *u8); mg_pi(nimgs); mg_puts("\n" as *u8)
125 if nimgs <= 0 {
126 mg_write_file("/tmp/mg_page_debug.bin" as *u8, page, pr) // evidence on fail: the raw response
127 mg_puts("MEDIA-GATHER-FAIL no-images (raw response -> /tmp/mg_page_debug.bin)\n" as *u8)
128 sys_exit(1); return 1
129 }
130
131 // pick the first HOSTED photo (upload.wikimedia.org); NUL-terminate a copy
132 let img_url: *u8 = sys_mmap(MG_MAGIC_4096)
133 var picked: i64 = 0 - 1
134 var i: i64 = 0
135 while i < nimgs {
136 if picked < 0 {
137 let up: *u8 = ((url_buf as i64) + offs[i]) as *u8
138 if mg_has(up, lens[i], "upload.wikimedia.org" as *u8) == 1 {
139 var o: i64 = 0
140 var j: i64 = 0
141 while j < lens[i] { img_url[o] = up[j]; o = o + 1; j = j + 1 }
142 img_url[o] = 0 as u8
143 picked = i
144 }
145 }
146 i = i + 1
147 }
148 if picked < 0 { mg_puts("MEDIA-GATHER-FAIL no-hosted-image\n" as *u8); sys_exit(1); return 1 }
149 mg_puts(" picked image=" as *u8); mg_puts(img_url); mg_puts("\n" as *u8)
150
151 // ---- request 2: the image (polite pause first) ----
152 sys_sleep_ms(MG_MAGIC_1500)
153 let img: *u8 = sys_mmap(MG_IMG_CAP)
154 nx_csprng_fill(cr, 32); nx_csprng_fill(pk, 32)
155 let ir: i64 = nx_https_get(img_url, cr, pk, store, sys_now_realtime_sec(), img, MG_IMG_CAP)
156 mg_puts(" image fetch bytes=" as *u8); mg_pi(ir); mg_puts("\n" as *u8)
157 if ir <= 0 { mg_puts("MEDIA-GATHER-FAIL image-fetch\n" as *u8); sys_exit(1); return 1 }
158 let ibo: i64 = mg_body_off(img, ir)
159 let ibody: *u8 = ((img as i64) + ibo) as *u8
160 let iblen: i64 = ir - ibo
161
162 // ---- sniff + hash + content-addressed store ----
163 var kind: *u8 = "unknown" as *u8
164 var ext: *u8 = ".bin" as *u8
165 if nx_sniff_is_jpeg(ibody, iblen) == 1 { kind = "jpeg" as *u8; ext = ".jpg" as *u8 }
166 if nx_sniff_is_png(ibody, iblen) == 1 { kind = "png" as *u8; ext = ".png" as *u8 }
167 let dg: *u8 = sys_mmap(32)
168 sha256_digest(ibody, iblen, dg)
169 let hex: *u8 = sys_mmap(72)
170 mg_hex(dg, hex)
171 mg_mkdir("knowledge/media" as *u8)
172 let dir: *u8 = sys_mmap(MG_MAGIC_4096)
173 var dofF: i64 = mg_cat(dir, 0, "knowledge/media/" as *u8)
174 dofF = mg_cat(dir, dofF, entity)
175 dir[dofF] = 0 as u8
176 mg_mkdir(dir)
177 let fpath: *u8 = sys_mmap(MG_MAGIC_4096)
178 var fo: i64 = mg_cat(fpath, 0, dir)
179 fo = mg_cat(fpath, fo, "/" as *u8)
180 fo = mg_cat(fpath, fo, hex)
181 fo = mg_cat(fpath, fo, ext)
182 fpath[fo] = 0 as u8
183 let wr: i64 = mg_write_file(fpath, ibody, iblen)
184 mg_puts(" sniff=" as *u8); mg_puts(kind)
185 mg_puts(" sha256=" as *u8); mg_puts(hex)
186 mg_puts("\n stored=" as *u8); mg_puts(fpath); mg_puts(" bytes=" as *u8); mg_pi(wr); mg_puts("\n" as *u8)
187 if wr != iblen { mg_puts("MEDIA-GATHER-FAIL store\n" as *u8); sys_exit(1); return 1 }
188
189 // ---- SELF-VERIFY: re-read + re-hash must match (store proven, not assumed) ----
190 let lenbox: *i64 = sys_mmap(16) as *i64
191 let back: *u8 = sys_read_file(fpath, lenbox)
192 var verified: i64 = 0
193 if back != 0 as *u8 {
194 if lenbox[0] == iblen {
195 let dg2: *u8 = sys_mmap(32)
196 sha256_digest(back, lenbox[0], dg2)
197 var same: i64 = 1
198 var b: i64 = 0
199 while b < 32 { if dg[b] != dg2[b] { same = 0 } b = b + 1 }
200 verified = same
201 }
202 }
203 mg_puts(" store re-read hash-verified=" as *u8); mg_pi(verified); mg_puts("\n" as *u8)
204 if verified != 1 { mg_puts("MEDIA-GATHER-FAIL verify\n" as *u8); sys_exit(1); return 1 }
205
206 // ---- provenance ledger (append-only) ----
207 let lg: i64 = sys_openat_append("knowledge/media/media_ledger.tsv" as *u8, 0x1a4)
208 if lg >= 0 {
209 mg_w(lg, entity); mg_w(lg, "\t" as *u8)
210 mg_w(lg, url); mg_w(lg, "\t" as *u8)
211 mg_w(lg, img_url); mg_w(lg, "\t" as *u8)
212 mg_wn(lg, iblen); mg_w(lg, "\t" as *u8)
213 mg_w(lg, hex); mg_w(lg, "\t" as *u8)
214 mg_w(lg, kind); mg_w(lg, "\tVERIFIED\n" as *u8)
215 sys_close(lg)
216 }
217
218 // ---- searchable dossier doc (the harvest indexes knowledge/library/*.txt) ----
219 let dpath: *u8 = sys_mmap(MG_MAGIC_4096)
220 var dpo: i64 = mg_cat(dpath, 0, "knowledge/library/media-" as *u8)
221 dpo = mg_cat(dpath, dpo, entity)
222 dpo = mg_cat(dpath, dpo, ".txt" as *u8)
223 dpath[dpo] = 0 as u8
224 let df: i64 = sys_openat_append(dpath, 0x1a4)
225 if df >= 0 {
226 mg_w(df, "media dossier entity=" as *u8); mg_w(df, entity)
227 mg_w(df, " source=" as *u8); mg_w(df, url)
228 mg_w(df, " image=" as *u8); mg_w(df, img_url)
229 mg_w(df, " sha256=" as *u8); mg_w(df, hex)
230 mg_w(df, " bytes=" as *u8); mg_wn(df, iblen)
231 mg_w(df, " kind=" as *u8); mg_w(df, kind)
232 mg_w(df, " transport=sovereign-validated-tls13 status=verified\n" as *u8)
233 sys_close(df)
234 }
235 mg_puts("MEDIA-GATHER-OK\n" as *u8)
236 sys_exit(0)
237 return 0
238}