nx_revimg_crawl_gate.nx source
↩ module page · 277 lines · 14507 B
1// nx_revimg_crawl_gate.nx -- END-TO-END referee for the discovery DRIVER (nx_revimg_crawl) on a REAL
2// multi-site, MULTI-FORMAT local image corpus. Proves the whole reverse-image discovery loop runs on
3// actual decoded pixels (PNG + JPEG), not synthetic fingerprints:
4// fixtures: seed0 = red vertical-stripes (PNG), seed1 = flat-128 (PNG) = the entity's media.
5// site0 (official): copy of seed0 (PNG), copy of seed1 (PNG), red-checker (other media).
6// site1 (fan): flat-128 JPEG = a CROSS-FORMAT copy of seed1 ; green-checker (other media).
7// site2 (unrelated): red-checker-p5 -- none of the entity's media.
8// row1 REAL MULTI-FORMAT INGEST -- the driver decodes all 6 files (incl the JPEG) into the corpus, and
9// the JPEG entry fingerprints to the flat image (proves JPEG flows through the real pipeline).
10// row2 CROSS-FORMAT CROSS-SITE MATCH -- seed1 is found on site0 (PNG) AND site1 (JPEG): the same media
11// served in different formats on different sites MATCHES (a sha256 byte-hash could never).
12// row3 HOST RANKING -- official site0 (2 seeds) outranks fan site1 (1); unrelated site2 is not a host.
13// row4 HARVEST -- each host site's other media surfaces as new results.
14// row5 PRECISION -- the unrelated site is never a host.
15// GREEN iff 5/5. Durable verdict -> knowledge/status/revimg_crawl_gate.log. license_tier: ORIGINAL
16import "nx_revimg_crawl.nx"
17import "nx_crc32.nx"
18import "nx_adler32.nx"
19
20func g_puts(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
21func g_num(v: i64) -> i64 { let bb: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m;sys_write(1,"-" as *u8,1)}; let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=(48 as u8);k=1}; while m>0{t[k]=((48+(m%10)) as u8);m=m/10;k=k+1}; var i: i64=0; while i<k{bb[i]=t[k-1-i];i=i+1}; sys_write(1,bb,k); return 0 }
22func g_w(fd: i64, s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(fd,s,n); return 0 }
23func g_wn(fd: i64, v: i64) -> i64 { let bb: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m}; let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=(48 as u8);k=1}; while m>0{t[k]=((48+(m%10)) as u8);m=m/10;k=k+1}; var i: i64=0; while i<k{bb[i]=t[k-1-i];i=i+1}; sys_write(fd,bb,k); return 0 }
24func g_clamp255(v: i64) -> i64 { if v<0 { return 0 } if v>255 { return 255 } return v }
25func g_write_bytes(path: *u8, buf: *u8, n: i64) -> i64 { let fd: i64=sys_openat_wr(path, 0x1a4); if fd<0 { return 0-1 } sys_write(fd, buf, n); sys_close(fd); return 0 }
26func g_has_site(arr: *i64, n: i64, sid: i64) -> i64 { var i: i64=0; while i<n { if arr[i]==sid { return 1 } i=i+1 } return 0 }
27func g_hits_of(sites: *i64, hits: *i64, n: i64, sid: i64) -> i64 { var i: i64=0; while i<n { if sites[i]==sid { return hits[i] } i=i+1 } return 0-1 }
28
29// structured color fill: structIdx 0=vertical stripes 1=checker; colorIdx 0=red 1=green.
30func g_synth_rgb_sc(rgb: *u8, w: i64, h: i64, structIdx: i64, colorIdx: i64, p: i64, phase: i64) -> i64 {
31 var y: i64 = 0
32 while y < h {
33 var x: i64 = 0
34 while x < w {
35 var s: i64 = 0
36 if structIdx==0 { s = (((x+phase)/p) & 1) } else { s = ((((x+phase)/p) + (y/p)) & 1) }
37 var r: i64 = 0
38 var g: i64 = 0
39 var b: i64 = 0
40 if colorIdx==0 {
41 if s==1 { r=200; g=0; b=0 } else { r=40; g=0; b=0 }
42 } else {
43 if s==1 { r=0; g=160; b=0 } else { r=0; g=40; b=0 }
44 }
45 let nz: i64 = ((x*7 + y*13 + phase*5) & 3)
46 let idx: i64 = (y*w+x)*3
47 rgb[idx] = g_clamp255(r + nz) as u8
48 rgb[idx+1] = g_clamp255(g + nz) as u8
49 rgb[idx+2] = g_clamp255(b + nz) as u8
50 x = x + 1
51 }
52 y = y + 1
53 }
54 return 0
55}
56func g_fill_rgb(rgb: *u8, w: i64, h: i64, v: i64) -> i64 {
57 var i: i64 = 0
58 let n: i64 = w*h
59 while i < n { rgb[i*3]=v as u8; rgb[i*3+1]=v as u8; rgb[i*3+2]=v as u8; i=i+1 }
60 return 0
61}
62
63// monotonic light->dark horizontal gradient: left>right at every pixel -> dHash all-ones, provably far
64// from both the flat seed (dHash 0) and the vertical-stripe seed -- a clean "other/unrelated" fingerprint.
65func g_synth_grad(rgb: *u8, w: i64, h: i64) -> i64 {
66 var y: i64 = 0
67 while y < h {
68 var x: i64 = 0
69 while x < w {
70 var v: i64 = 240 - (x * 200 / w)
71 v = g_clamp255(v)
72 let idx: i64 = (y*w+x)*3
73 rgb[idx]=v as u8; rgb[idx+1]=v as u8; rgb[idx+2]=v as u8
74 x = x + 1
75 }
76 y = y + 1
77 }
78 return 0
79}
80func gp_wbe(buf: *u8, off: i64, val: i64) -> i64 { buf[off]=((val>>24)&255) as u8; buf[off+1]=((val>>16)&255) as u8; buf[off+2]=((val>>8)&255) as u8; buf[off+3]=(val&255) as u8; return 0 }
81func gp_build_buf(out: *u8, rgb: *u8, w: i64, h: i64) -> i64 {
82 out[0]=137 as u8; out[1]=80 as u8; out[2]=78 as u8; out[3]=71 as u8
83 out[4]=13 as u8; out[5]=10 as u8; out[6]=26 as u8; out[7]=10 as u8
84 var pos: i64=8
85 gp_wbe(out,pos,13); pos=pos+4
86 let ihdr_t: i64=pos
87 out[pos]=73 as u8; out[pos+1]=72 as u8; out[pos+2]=68 as u8; out[pos+3]=82 as u8; pos=pos+4
88 gp_wbe(out,pos,w); pos=pos+4
89 gp_wbe(out,pos,h); pos=pos+4
90 out[pos]=8 as u8; out[pos+1]=2 as u8; out[pos+2]=0 as u8; out[pos+3]=0 as u8; out[pos+4]=0 as u8; pos=pos+5
91 gp_wbe(out,pos, nx_crc32(((out as i64)+ihdr_t) as *u8, 17)); pos=pos+4
92 let payload_len: i64=h*(1+w*3)
93 let zlib_len: i64=2+5+payload_len+4
94 gp_wbe(out,pos,zlib_len); pos=pos+4
95 let idat_t: i64=pos
96 out[pos]=73 as u8; out[pos+1]=68 as u8; out[pos+2]=65 as u8; out[pos+3]=84 as u8; pos=pos+4
97 out[pos]=0x78 as u8; out[pos+1]=0x01 as u8; pos=pos+2
98 out[pos]=0x01 as u8; pos=pos+1
99 out[pos]=(payload_len&255) as u8; out[pos+1]=((payload_len>>8)&255) as u8; pos=pos+2
100 let nlen: i64=(payload_len^0xFFFF)&0xFFFF
101 out[pos]=(nlen&255) as u8; out[pos+1]=((nlen>>8)&255) as u8; pos=pos+2
102 let payload_off: i64=pos
103 var y: i64=0
104 while y<h {
105 out[pos]=0 as u8; pos=pos+1
106 var x: i64=0
107 while x<w {
108 let i: i64=(y*w+x)*3
109 out[pos]=rgb[i] as u8; out[pos+1]=rgb[i+1] as u8; out[pos+2]=rgb[i+2] as u8; pos=pos+3
110 x=x+1
111 }
112 y=y+1
113 }
114 gp_wbe(out,pos, adler32(((out as i64)+payload_off) as *u8, payload_len)); pos=pos+4
115 gp_wbe(out,pos, nx_crc32(((out as i64)+idat_t) as *u8, 4+zlib_len)); pos=pos+4
116 gp_wbe(out,pos,0); pos=pos+4
117 let iend_t: i64=pos
118 out[pos]=73 as u8; out[pos+1]=69 as u8; out[pos+2]=78 as u8; out[pos+3]=68 as u8; pos=pos+4
119 gp_wbe(out,pos, nx_crc32(((out as i64)+iend_t) as *u8, 4)); pos=pos+4
120 return pos
121}
122func g_make_png(path: *u8, rgb: *u8, png: *u8, w: i64, h: i64) -> i64 {
123 let sz: i64 = gp_build_buf(png, rgb, w, h)
124 g_write_bytes(path, png, sz)
125 return 0
126}
127
128// the canonical 137-byte all-128 grayscale baseline JPEG.
129func _build_gray_jpeg(jpeg: *u8) -> i64 {
130 var p: i64 = 0
131 jpeg[p]=0xFF; p=p+1; jpeg[p]=0xD8; p=p+1
132 jpeg[p]=0xFF; p=p+1; jpeg[p]=0xDB; p=p+1
133 jpeg[p]=0x00; p=p+1; jpeg[p]=0x43; p=p+1
134 jpeg[p]=0x00; p=p+1
135 var i: i64 = 0
136 while i < 64 { jpeg[p+i] = 1; i = i + 1 }
137 p = p + 64
138 jpeg[p]=0xFF; p=p+1; jpeg[p]=0xC4; p=p+1
139 jpeg[p]=0x00; p=p+1; jpeg[p]=0x26; p=p+1
140 jpeg[p]=0x00; p=p+1
141 jpeg[p]=0x01; p=p+1
142 i = 0
143 while i < 15 { jpeg[p+i] = 0; i = i + 1 }
144 p = p + 15
145 jpeg[p]=0x00; p=p+1
146 jpeg[p]=0x10; p=p+1
147 jpeg[p]=0x01; p=p+1
148 i = 0
149 while i < 15 { jpeg[p+i] = 0; i = i + 1 }
150 p = p + 15
151 jpeg[p]=0x00; p=p+1
152 jpeg[p]=0xFF; p=p+1; jpeg[p]=0xC0; p=p+1
153 jpeg[p]=0x00; p=p+1; jpeg[p]=0x0B; p=p+1
154 jpeg[p]=0x08; p=p+1
155 jpeg[p]=0x00; p=p+1; jpeg[p]=0x08; p=p+1
156 jpeg[p]=0x00; p=p+1; jpeg[p]=0x08; p=p+1
157 jpeg[p]=0x01; p=p+1
158 jpeg[p]=0x01; p=p+1
159 jpeg[p]=0x11; p=p+1
160 jpeg[p]=0x00; p=p+1
161 jpeg[p]=0xFF; p=p+1; jpeg[p]=0xDA; p=p+1
162 jpeg[p]=0x00; p=p+1; jpeg[p]=0x08; p=p+1
163 jpeg[p]=0x01; p=p+1
164 jpeg[p]=0x01; p=p+1
165 jpeg[p]=0x00; p=p+1
166 jpeg[p]=0x00; p=p+1
167 jpeg[p]=0x3F; p=p+1
168 jpeg[p]=0x00; p=p+1
169 jpeg[p]=0x00; p=p+1
170 jpeg[p]=0xFF; p=p+1; jpeg[p]=0xD9; p=p+1
171 return p
172}
173
174func main() -> i64 {
175 g_puts("=== REVIMG CRAWL GATE (discovery driver e2e: real PNG+JPEG decode -> dHash -> cross-site discover) ===\n" as *u8)
176 let W: i64 = 16
177 let H: i64 = 16
178 let THRESH: i64 = 10
179 let rgb: *u8 = sys_mmap(W*H*3)
180 let png: *u8 = sys_mmap(8192)
181
182 // seeds (the entity's media, fingerprinted from REAL files)
183 g_synth_rgb_sc(rgb, W, H, 0, 0, 3, 0); g_make_png("/tmp/rc_seed0.png" as *u8, rgb, png, W, H)
184 g_fill_rgb(rgb, W, H, 128); g_make_png("/tmp/rc_seed1.png" as *u8, rgb, png, W, H)
185 let seed0: i64 = nx_revimg_seed_file("/tmp/rc_seed0.png" as *u8)
186 let seed1: i64 = nx_revimg_seed_file("/tmp/rc_seed1.png" as *u8)
187 let seeds: *i64 = sys_mmap(8*2) as *i64
188 seeds[0]=seed0; seeds[1]=seed1
189
190 // corpus fixture files
191 g_synth_rgb_sc(rgb, W, H, 0, 0, 3, 0); g_make_png("/tmp/rc_s0a.png" as *u8, rgb, png, W, H) // = seed0
192 g_fill_rgb(rgb, W, H, 128); g_make_png("/tmp/rc_s0b.png" as *u8, rgb, png, W, H) // = seed1
193 g_synth_grad(rgb, W, H); g_make_png("/tmp/rc_s0x.png" as *u8, rgb, png, W, H) // gradient (other; dHash far from both seeds)
194 let jb: *u8 = sys_mmap(256)
195 let jl: i64 = _build_gray_jpeg(jb); g_write_bytes("/tmp/rc_s1j.jpg" as *u8, jb, jl) // flat JPEG = seed1 cross-format
196 g_synth_grad(rgb, W, H); g_make_png("/tmp/rc_s1y.png" as *u8, rgb, png, W, H) // gradient (other)
197 g_synth_grad(rgb, W, H); g_make_png("/tmp/rc_s2u.png" as *u8, rgb, png, W, H) // gradient (unrelated)
198
199 // ingest via the driver (real decode of PNG + JPEG)
200 let cap: i64 = 16
201 let cfp: *i64 = sys_mmap(8*cap) as *i64
202 let csite: *i64 = sys_mmap(8*cap) as *i64
203 let cpid: *i64 = sys_mmap(8*cap) as *i64
204 var n: i64 = 0
205 n = nx_revimg_ingest_file("/tmp/rc_s0a.png" as *u8, 0, 0, cfp, csite, cpid, n, cap)
206 n = nx_revimg_ingest_file("/tmp/rc_s0b.png" as *u8, 0, 1, cfp, csite, cpid, n, cap)
207 n = nx_revimg_ingest_file("/tmp/rc_s0x.png" as *u8, 0, 2, cfp, csite, cpid, n, cap)
208 n = nx_revimg_ingest_file("/tmp/rc_s1j.jpg" as *u8, 1, 3, cfp, csite, cpid, n, cap)
209 n = nx_revimg_ingest_file("/tmp/rc_s1y.png" as *u8, 1, 4, cfp, csite, cpid, n, cap)
210 n = nx_revimg_ingest_file("/tmp/rc_s2u.png" as *u8, 2, 5, cfp, csite, cpid, n, cap)
211
212 var pass: i64 = 0
213 let rows: i64 = 5
214
215 // ---- row1: real multi-format ingest -- 6 files in, JPEG entry fingerprints to the flat seed ----
216 var jpeg_idx: i64 = 0-1
217 var ci: i64 = 0
218 while ci < n { if cpid[ci]==3 { jpeg_idx=ci } ci=ci+1 }
219 var jpeg_ok: i64 = 0
220 if jpeg_idx>=0 { if nx_simhash_hamming(cfp[jpeg_idx], seed1)<=2 { jpeg_ok=1 } }
221 g_puts(" row1 ingest: corpus=" as *u8); g_num(n); g_puts(" files, JPEG decoded->matches flat seed? " as *u8); g_num(jpeg_ok); g_puts(" -> " as *u8)
222 if n==6 { if jpeg_ok==1 { pass=pass+1; g_puts("PASS (real PNG+JPEG ingested)\n" as *u8) } else { g_puts("FAIL\n" as *u8) } } else { g_puts("FAIL\n" as *u8) }
223
224 // ---- row2: cross-format cross-site match -- seed1 on site0 (PNG) AND site1 (JPEG) ----
225 let oci: *i64 = sys_mmap(8*64) as *i64
226 let osi: *i64 = sys_mmap(8*64) as *i64
227 let oham: *i64 = sys_mmap(8*64) as *i64
228 let nm: i64 = nx_revimg_match(seeds, 2, cfp, n, THRESH, oci, osi, oham, 64)
229 var s1_on0: i64 = 0
230 var s1_on1_jpeg: i64 = 0
231 var mi: i64 = 0
232 while mi < nm {
233 if osi[mi]==1 {
234 let cix: i64 = oci[mi]
235 if csite[cix]==0 { s1_on0=1 }
236 if csite[cix]==1 { if cpid[cix]==3 { s1_on1_jpeg=1 } }
237 }
238 mi = mi + 1
239 }
240 g_puts(" row2 cross-format: seed1 on site0(PNG)? " as *u8); g_num(s1_on0); g_puts(" on site1(JPEG)? " as *u8); g_num(s1_on1_jpeg); g_puts(" -> " as *u8)
241 if s1_on0==1 { if s1_on1_jpeg==1 { pass=pass+1; g_puts("PASS (same media, JPEG site + PNG site, MATCHED)\n" as *u8) } else { g_puts("FAIL\n" as *u8) } } else { g_puts("FAIL\n" as *u8) }
242
243 // ---- row3: host ranking ----
244 let hsite: *i64 = sys_mmap(8*16) as *i64
245 let hhits: *i64 = sys_mmap(8*16) as *i64
246 let nh: i64 = nx_revimg_host_sites(seeds, 2, cfp, csite, n, THRESH, hsite, hhits, 16)
247 let hA: i64 = g_hits_of(hsite, hhits, nh, 0)
248 let hB: i64 = g_hits_of(hsite, hhits, nh, 1)
249 let hasC: i64 = g_has_site(hsite, nh, 2)
250 g_puts(" row3 host ranking: site0=" as *u8); g_num(hA); g_puts(" site1=" as *u8); g_num(hB); g_puts(" site2 present? " as *u8); g_num(hasC); g_puts(" -> " as *u8)
251 if hA==2 { if hB==1 { if hasC==0 { pass=pass+1; g_puts("PASS\n" as *u8) } else { g_puts("FAIL\n" as *u8) } } else { g_puts("FAIL\n" as *u8) } } else { g_puts("FAIL\n" as *u8) }
252
253 // ---- row4: harvest each host's other media ----
254 let hvfp: *i64 = sys_mmap(8*16) as *i64
255 let hvci: *i64 = sys_mmap(8*16) as *i64
256 let naA: i64 = nx_revimg_harvest(0, seeds, 2, cfp, csite, n, THRESH, hvfp, hvci, 16)
257 let hvfpB: *i64 = sys_mmap(8*16) as *i64
258 let hvciB: *i64 = sys_mmap(8*16) as *i64
259 let naB: i64 = nx_revimg_harvest(1, seeds, 2, cfp, csite, n, THRESH, hvfpB, hvciB, 16)
260 g_puts(" row4 harvest: site0-other=" as *u8); g_num(naA); g_puts(" site1-other=" as *u8); g_num(naB); g_puts(" -> " as *u8)
261 if naA>=1 { if naB>=1 { pass=pass+1; g_puts("PASS (their other media surfaced)\n" as *u8) } else { g_puts("FAIL\n" as *u8) } } else { g_puts("FAIL\n" as *u8) }
262
263 // ---- row5: precision -- unrelated site2 never a host ----
264 g_puts(" row5 precision: unrelated site2 a host? " as *u8); g_num(hasC); g_puts(" -> " as *u8)
265 if hasC==0 { pass=pass+1; g_puts("PASS (no false host)\n" as *u8) } else { g_puts("FAIL\n" as *u8) }
266
267 g_puts("----\nREVIMG-CRAWL-GATE rows=" as *u8); g_num(rows); g_puts(" pass=" as *u8); g_num(pass); g_puts("\n" as *u8)
268 let lg: i64 = sys_openat_append("knowledge/status/revimg_crawl_gate.log" as *u8, 0x1a4)
269 if lg>=0 {
270 g_w(lg,"REVIMG-CRAWL-GATE rows=" as *u8); g_wn(lg,rows); g_w(lg," pass=" as *u8); g_wn(lg,pass)
271 g_w(lg," corpus=" as *u8); g_wn(lg,n); g_w(lg," hosts=" as *u8); g_wn(lg,nh)
272 if pass==rows { g_w(lg," verdict=GREEN\n" as *u8) } else { g_w(lg," verdict=RED\n" as *u8) }
273 sys_close(lg)
274 }
275 if pass==rows { g_puts("REVIMG-CRAWL-GATE GREEN\n" as *u8); sys_exit(0); return 0 }
276 g_puts("REVIMG-CRAWL-GATE RED\n" as *u8); sys_exit(1); return 1
277}