nx_revimg_crawl_run.nx source
↩ module page · 174 lines · 9071 B
1// nx_revimg_crawl_run.nx -- LIVE reverse-image discovery CLI: the operator-driven assembly that runs the
2// whole loop on the real web. Given a SEED page (the entity's media, e.g. the Diora Baird page) and one or
3// more CANDIDATE site pages, it: fetches each over the team's OWN validated HTTPS (sovereign DNS+TCP+
4// TLS1.3+X.509, the nx_media_gather path), extracts image URLs (nx_html_extract_imgs), politely fetches
5// each image, decodes it (nx_img_to_gray -- PNG today, JPEG since the FF-D8-FF wiring, so the real web
6// works), perceptual-hashes it (nx_phash dHash), then runs nx_revimg_discover to report WHICH candidate
7// sites host the seed media (recompression/format-robust, which a sha256 byte-hash can't) and HARVESTS
8// those hosts' OTHER media -- the "grow the search results" answer.
9//
10// SAFETY: with no args it prints usage and exits (the build-time self-run never touches the network).
11// BOUNDED: <= MAX_PER_PAGE images per page, polite 1.2s pacing, total fetches kept under the per-process
12// certloop ceiling (~19) -- the cap is LOGGED, never silent. Composes nx_media_gather's HTTPS pattern +
13// nx_revimg_crawl/discover -- ZERO new transport or ranking logic.
14// Usage: nx_revimg_crawl_run <seed_page_url> [candidate_url ...]
15// license_tier: ORIGINAL
16import "nx_str.nx"
17import "nx_itoa_lib.nx" // shared MSB-first emitter (zero-alloc)
18import "nx_syscalls.nx"
19import "nx_csprng.nx"
20import "nx_x509_trust_store.nx"
21import "nx_pem_loader.nx"
22import "nx_https_get.nx"
23import "nx_html_extract_imgs.nx"
24import "nx_image_gray.nx"
25import "nx_phash.nx"
26import "nx_revimg_discover.nx"
27const CR_MAGIC_131072: i64 = 131072
28const CR_MAGIC_4096: i64 = 4096
29const CR_MAGIC_1200: i64 = 1200
30
31const CR_PAGE_CAP: i64 = 4194304
32const CR_IMG_CAP: i64 = 8388608
33const CR_MAX_EXTRACT: i64 = 256
34const CR_MAX_PER_PAGE: i64 = 5
35const CR_CORPUS_CAP: i64 = 256
36const CR_THRESH: i64 = 10
37
38func cr_puts(s: *u8) -> i64 { sys_write(1, s, nx_str_len(s)); return 0 }
39// MIGRATED to the shared emitter (debt 1785563586). The old body mmapped a scratch buffer
40// per call and never freed it. At PAGE granularity that is 4096B leaked PER CALL -- the
41// defect that took 28.5GB of a 36GB host in nx_ts_lumadiff (2MB input, ~3.66M calls).
42// nxi_* is MSB-first, allocates NOTHING, and emits identical bytes including the sign.
43func cr_num(v: i64) -> i64 { nxi_out(v); return 0 }
44func cr_putn(p: *u8, n: i64) -> i64 { if n>0 { sys_write(1, p, n) } return 0 }
45
46// end of HTTP headers (CRLF CRLF)
47func cr_body_off(resp: *u8, n: i64) -> i64 {
48 var i: i64 = 0
49 while i + 3 < n {
50 if resp[i]==(13 as u8) { if resp[i+1]==(10 as u8) { if resp[i+2]==(13 as u8) { if resp[i+3]==(10 as u8) { return i + 4 } } } }
51 i = i + 1
52 }
53 return 0
54}
55func cr_write_file(path: *u8, buf: *u8, n: i64) -> i64 {
56 let fd: i64 = sys_openat_wr(path, 0x1a4)
57 if fd < 0 { return 0-1 }
58 var off: i64 = 0
59 while off < n { let w: i64 = sys_write(fd, ((buf as i64)+off) as *u8, n-off); if w<=0 { sys_close(fd); return 0-1 } off=off+w }
60 sys_close(fd)
61 return n
62}
63
64// fetch a page, extract <img> URLs, fetch+decode+dHash up to CR_MAX_PER_PAGE of them, appending
65// (fp, site_id) to the corpus arrays. returns the new corpus count. `dropped` is incremented if the page
66// had more images than the per-page cap (so truncation is reported, never silent).
67func cr_ingest_page(page_url: *u8, store: *TrustStore, site_id: i64,
68 out_fp: *i64, out_site: *i64, n_in: i64, cap: i64, dropped: *i64) -> i64 {
69 let cr: *u8 = sys_mmap(32); nx_csprng_fill(cr, 32)
70 let pk: *u8 = sys_mmap(32); nx_csprng_fill(pk, 32)
71 let page: *u8 = sys_mmap(CR_PAGE_CAP)
72 let pr: i64 = nx_https_get(page_url, cr, pk, store, sys_now_realtime_sec(), page, CR_PAGE_CAP)
73 if pr <= 0 { cr_puts(" (page fetch failed)\n" as *u8); return n_in }
74 let bo: i64 = cr_body_off(page, pr)
75 let body: *u8 = ((page as i64)+bo) as *u8
76 let blen: i64 = pr - bo
77 let url_buf: *u8 = sys_mmap(CR_MAGIC_131072)
78 let offs: *i64 = sys_mmap(8*CR_MAX_EXTRACT) as *i64
79 let lens: *i64 = sys_mmap(8*CR_MAX_EXTRACT) as *i64
80 let nimgs: i64 = nx_html_extract_imgs(body, blen, page_url, nx_str_len(page_url), url_buf, CR_MAGIC_131072, offs, lens, CR_MAX_EXTRACT)
81 cr_puts(" images on page=" as *u8); cr_num(nimgs); cr_puts("\n" as *u8)
82 if nimgs > CR_MAX_PER_PAGE { dropped[0] = dropped[0] + (nimgs - CR_MAX_PER_PAGE) }
83 var n: i64 = n_in
84 var got: i64 = 0
85 var i: i64 = 0
86 while i < nimgs {
87 if got < CR_MAX_PER_PAGE { if n < cap {
88 let iu: *u8 = sys_mmap(CR_MAGIC_4096)
89 var k: i64 = 0
90 while k < lens[i] { iu[k] = url_buf[offs[i]+k]; k=k+1 }
91 iu[lens[i]] = 0 as u8
92 sys_sleep_ms(CR_MAGIC_1200)
93 nx_csprng_fill(cr,32); nx_csprng_fill(pk,32)
94 let img: *u8 = sys_mmap(CR_IMG_CAP)
95 let ir: i64 = nx_https_get(iu, cr, pk, store, sys_now_realtime_sec(), img, CR_IMG_CAP)
96 if ir > 0 {
97 let ibo: i64 = cr_body_off(img, ir)
98 let ibody: *u8 = ((img as i64)+ibo) as *u8
99 let iblen: i64 = ir - ibo
100 cr_write_file("/tmp/cr_dl.img" as *u8, ibody, iblen)
101 let wh: *i64 = sys_mmap(16) as *i64
102 let gray: *u8 = nx_img_to_gray("/tmp/cr_dl.img" as *u8, wh)
103 if gray != (0 as *u8) { if wh[0]>0 { if wh[1]>0 {
104 out_fp[n] = nx_phash_dhash(gray, wh[0], wh[1])
105 out_site[n] = site_id
106 n = n + 1
107 got = got + 1
108 cr_puts(" + fingerprinted " as *u8); cr_putn(iu, lens[i]); cr_puts("\n" as *u8)
109 } } }
110 }
111 } }
112 i = i + 1
113 }
114 return n
115}
116
117func main(argc: i64, argv: *i64) -> i64 {
118 if argc < 2 {
119 cr_puts("usage: nx_revimg_crawl_run <seed_page_url> [candidate_url ...]\n" as *u8)
120 cr_puts(" seed page = the entity's media; candidates = other sites to check for reposts.\n" as *u8)
121 sys_exit(2); return 2
122 }
123 let seed_url: *u8 = argv[1] as *u8
124 cr_puts("=== REVIMG CRAWL RUN (live reverse-image discovery over sovereign HTTPS) ===\n" as *u8)
125
126 let store: *TrustStore = trust_store_alloc(400)
127 let nroots: i64 = nx_pem_trust_load_file("/etc/ssl/certs/ca-certificates.crt" as *u8, store)
128 if nroots <= 0 { cr_puts("CRAWL-RUN-FAIL no-trust-roots\n" as *u8); sys_exit(1); return 1 }
129 cr_puts(" trust roots=" as *u8); cr_num(nroots); cr_puts("\n" as *u8)
130
131 // SEEDS = the entity's media on the seed page
132 let seed_fp: *i64 = sys_mmap(8*CR_CORPUS_CAP) as *i64
133 let seed_site: *i64 = sys_mmap(8*CR_CORPUS_CAP) as *i64
134 let drop: *i64 = sys_mmap(8) as *i64; drop[0]=0
135 cr_puts(" seed page: " as *u8); cr_puts(seed_url); cr_puts("\n" as *u8)
136 let nseed: i64 = cr_ingest_page(seed_url, store, 0-1, seed_fp, seed_site, 0, CR_CORPUS_CAP, drop)
137 cr_puts(" seed fingerprints=" as *u8); cr_num(nseed); cr_puts("\n" as *u8)
138 if nseed <= 0 { cr_puts("CRAWL-RUN-FAIL no-seed-media\n" as *u8); sys_exit(1); return 1 }
139
140 // CORPUS = images on each candidate site (site_id = candidate index, 0-based)
141 let cfp: *i64 = sys_mmap(8*CR_CORPUS_CAP) as *i64
142 let csite: *i64 = sys_mmap(8*CR_CORPUS_CAP) as *i64
143 var ncorp: i64 = 0
144 var ai: i64 = 2
145 while ai < argc {
146 let curl: *u8 = argv[ai] as *u8
147 cr_puts(" candidate[" as *u8); cr_num(ai-2); cr_puts("]: " as *u8); cr_puts(curl); cr_puts("\n" as *u8)
148 ncorp = cr_ingest_page(curl, store, ai-2, cfp, csite, ncorp, CR_CORPUS_CAP, drop)
149 ai = ai + 1
150 }
151 cr_puts(" corpus fingerprints=" as *u8); cr_num(ncorp); cr_puts("\n" as *u8)
152 if drop[0] > 0 { cr_puts(" NOTE: " as *u8); cr_num(drop[0]); cr_puts(" images skipped by the per-page cap (raise CR_MAX_PER_PAGE / certloop ceiling)\n" as *u8) }
153
154 // DISCOVER: which candidate sites host the seed media, ranked by coverage
155 let hsite: *i64 = sys_mmap(8*64) as *i64
156 let hhits: *i64 = sys_mmap(8*64) as *i64
157 let nh: i64 = nx_revimg_host_sites(seed_fp, nseed, cfp, csite, ncorp, CR_THRESH, hsite, hhits, 64)
158 cr_puts("---- HOST SITES (candidates hosting the entity's media) ----\n" as *u8)
159 if nh == 0 { cr_puts(" (none of the candidates host the seed media)\n" as *u8) }
160 var hi: i64 = 0
161 while hi < nh {
162 let sid: i64 = hsite[hi]
163 cr_puts(" candidate[" as *u8); cr_num(sid); cr_puts("] " as *u8); cr_puts(argv[sid+2] as *u8)
164 cr_puts(" hosts " as *u8); cr_num(hhits[hi]); cr_puts(" of the entity's media\n" as *u8)
165 let hvfp: *i64 = sys_mmap(8*CR_CORPUS_CAP) as *i64
166 let hvci: *i64 = sys_mmap(8*CR_CORPUS_CAP) as *i64
167 let nha: i64 = nx_revimg_harvest(sid, seed_fp, nseed, cfp, csite, ncorp, CR_THRESH, hvfp, hvci, CR_CORPUS_CAP)
168 cr_puts(" + " as *u8); cr_num(nha); cr_puts(" OTHER media on this host = new results to grow the corpus\n" as *u8)
169 hi = hi + 1
170 }
171 cr_puts("CRAWL-RUN-OK\n" as *u8)
172 sys_exit(0)
173 return 0
174}