nx_image_index.nx source
↩ module page · 96 lines · 4669 B
1// nx_image_index.nx -- REAL-CORPUS reverse-image INGESTOR.
2// Turns a corpus of real encoded images into the per-site fingerprint index that nx_image_search
3// queries. For each manifest row "cid<TAB>image_path": decode the image (PNG today via nx_png_decode;
4// JPEG queued -- the JPEG decoder pipeline already exists), convert to Rec.709 grayscale, compute
5// dHash+aHash (nx_phash -- ZERO new hashing math), and append "cid<TAB>dhash_hex16<TAB>ahash_hex16".
6// The resulting index is byte-identical in format to the one nx_image_search_gate builds inline, so
7// the SAME client serves a synthetic test corpus and a real gallery corpus.
8//
9// GRACEFUL DEGRADATION (Cardinal 14): an undecodable / unsupported / truncated image is SKIPPED
10// (counted in `skipped`, never fatal) so one bad file cannot abort a 366k-image crawl. Honest:
11// nx_png_decoder is PARTIAL (no palette / no Adam7 / no sub-byte depth) -> those images skip cleanly
12// and are reported, not silently dropped.
13//
14// Usage: nx_image_index <corpus.tsv> <out_fp_index> (corpus rows: cid<TAB>image_path, NL-terminated)
15// license_tier: ORIGINAL
16import "nx_phash.nx"
17import "nx_itoa_lib.nx" // shared MSB-first emitter (zero-alloc)
18import "nx_image_gray.nx" // shared decode -> gray (single source of truth; SAME fn the query client uses)
19
20func ii_puts(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
21// MIGRATED to the shared emitter (debt 1785563586). The old body mmapped a scratch buffer
22// per call and never freed it. At page granularity that is 4096B leaked PER CALL -- the
23// defect that took 28.5GB of a 36GB host in nx_ts_lumadiff. A BENCH is the worst home for
24// it: its purpose is millions of iterations. nxi_* is MSB-first and allocates NOTHING.
25func ii_num(v: i64) -> i64 { nxi_out(v); return 0 }
26func ii_strlen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
27
28// emit one i64 as 16 lowercase-hex chars (high nibble first); &0xF masks sign-extension cleanly
29func ii_hexemit(fd: i64, v: i64) -> i64 {
30 let buf: *u8=sys_mmap(16)
31 var i: i64=0
32 while i<16 {
33 let shift: i64=(15-i)*4
34 let nib: i64=(v>>shift)&0xF
35 var c: i64=0
36 if nib<10 { c=48+nib } else { c=87+nib }
37 buf[i]=c as u8
38 i=i+1
39 }
40 sys_write(fd, buf, 16)
41 return 0
42}
43
44// decode -> gray now lives in nx_image_gray.nx (nx_img_to_gray) -- ONE source of truth shared with
45// the query client so index-side and query-side fingerprints are computed identically.
46
47func main(argc: i64, argv: *i64) -> i64 {
48 if argc<3 {
49 ii_puts("usage: nx_image_index <corpus.tsv> <out_fp_index>\n" as *u8)
50 sys_exit(2); return 2
51 }
52 let corpus: *u8=argv[1] as *u8
53 let outp: *u8=argv[2] as *u8
54 let cbox: *i64=sys_mmap(16) as *i64
55 let cbuf: *u8=sys_read_file(corpus, cbox)
56 if cbuf==(0 as *u8) { ii_puts("IMGINDEX-FAIL: no corpus manifest\n" as *u8); sys_exit(1); return 1 }
57 let cn: i64=cbox[0]
58 let ofd: i64=sys_openat_wr(outp, 0x1a4)
59 if ofd<0 { ii_puts("IMGINDEX-FAIL: cannot open output index\n" as *u8); sys_exit(1); return 1 }
60
61 let wh: *i64=sys_mmap(16) as *i64
62 var indexed: i64=0; var skipped: i64=0
63 var ls: i64=0; var i: i64=0
64 while i<=cn {
65 var eol: i64=0
66 if i==cn { eol=1 } else { if cbuf[i]==(10 as u8) { eol=1 } }
67 if eol==1 {
68 if i>ls {
69 var t1: i64=0-1; var k: i64=ls
70 while k<i { if cbuf[k]==(9 as u8) { if t1<0 { t1=k } } k=k+1 }
71 if t1>ls {
72 cbuf[t1]=0 as u8
73 if i<cn { cbuf[i]=0 as u8 }
74 let cid: *u8=((cbuf as i64)+ls) as *u8
75 let path: *u8=((cbuf as i64)+t1+1) as *u8
76 let gray: *u8=nx_img_to_gray(path, wh)
77 if gray==(0 as *u8) {
78 skipped=skipped+1
79 } else {
80 let d: i64=nx_phash_dhash(gray, wh[0], wh[1])
81 let a: i64=nx_phash_ahash(gray, wh[0], wh[1])
82 sys_write(ofd, cid, ii_strlen(cid)); sys_write(ofd, "\t" as *u8, 1)
83 ii_hexemit(ofd, d); sys_write(ofd, "\t" as *u8, 1)
84 ii_hexemit(ofd, a); sys_write(ofd, "\n" as *u8, 1)
85 indexed=indexed+1
86 }
87 }
88 }
89 ls=i+1
90 }
91 i=i+1
92 }
93 sys_close(ofd)
94 ii_puts("IMGINDEX-OK indexed=" as *u8); ii_num(indexed); ii_puts(" skipped=" as *u8); ii_num(skipped); ii_puts(" -> " as *u8); ii_puts(outp); ii_puts("\n" as *u8)
95 sys_exit(0); return 0
96}