code wiki / (root) / nx_image_search.nx
`nx_image_search.nx` source

↩ module page · 220 lines · 10328 B
1// nx_image_search.nx -- REUSABLE per-site SEARCH-BY-IMAGE (reverse-image) CLIENT.
2// ONE perceptual-hash engine pointed at ANY site via a data-driven sites registry (Cardinal 11/17).
3// Each site = its OWN durable fingerprint index, so a query is SCOPED to exactly that site BY
4// CONSTRUCTION (site A's index file physically cannot return site B's images). This is the image-side
5// analog of nx_onsite_search (which does text->doc); here we do image->image: "find this image across
6// THIS site's corpus", Yandex/Google/Bing reverse-image style, done sovereignly.
7//
8//   Usage: nx_image_search <sites.tsv> <site> <query_gray_path> <w> <h> [threshold]
9//   Registry row (TAB-separated, trailing newline required): site<TAB>fp_index_path<TAB>base_url
10//   fp_index line (built by nx_image_index / the gate; docid = line number):
11//       cid<TAB>dhash_hex16<TAB>ahash_hex16
12//
13// REUSE not reinvent: dHash/aHash + nearest-fingerprint + 64-bit Hamming all come from nx_phash
14// (Krawetz 2011 / Zauner 2010) -> ZERO new ranking math. The query image is a raw grayscale buffer
15// (w*h u8) so the client stays format-agnostic; the real-corpus path (PNG/JPEG decode -> gray) is a
16// separate ingestor rung. dHash ranks (robust to brightness/recompress/resize); aHash breaks ties.
17// HONEST: this is the FAST near-duplicate tier; semantic "looks-like" needs a trained embedding (later
18// rung). license_tier: ORIGINAL
19import "nx_phash.nx"
20import "nx_itoa_lib.nx"   // shared MSB-first emitter (zero-alloc)
21import "nx_image_gray.nx"   // shared decode -> gray (same fn the indexer uses, so query+index fingerprint identically)
22
23const IS_TOP_K:         i64 = 5
24const IS_DEFAULT_THRESH: i64 = 10
25
26func is_puts(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
27func is_putb(p: *u8, n: i64) -> i64 { if n>0 { sys_write(1,p,n) } return 0 }
28// MIGRATED to the shared emitter (debt 1785563586). The old body mmapped a scratch buffer
29// per call and never freed it. At page granularity that is 4096B leaked PER CALL -- the
30// defect that took 28.5GB of a 36GB host in nx_ts_lumadiff. A BENCH is the worst home for
31// it: its purpose is millions of iterations. nxi_* is MSB-first and allocates NOTHING.
32func is_num(v: i64) -> i64 { nxi_out(v); return 0 }
33func is_strlen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
34func is_streq(a: *u8, b: *u8) -> i64 { var i: i64=0; while a[i]!=(0 as u8){ if a[i]!=b[i]{return 0} i=i+1 } if b[i]!=(0 as u8){return 0} return 1 }
35func is_atoi(s: *u8) -> i64 { var v: i64=0; var i: i64=0; while s[i]!=(0 as u8){ if s[i]>=(48 as u8){ if s[i]<=(57 as u8){ v=v*10+((s[i]-(48 as u8)) as i64) } } i=i+1 } return v }
36
37// parse 16 lowercase-hex chars at p -> i64 bit pattern (unsigned; 0xff..ff round-trips to -1, Hamming-safe)
38func is_hexparse(p: *u8) -> i64 {
  var v: i64=0
  var i: i64=0
  while i<16 {
      let c: i64=p[i] as i64
      var nib: i64=0
      if c>=48 { if c<=57 { nib=c-48 } }
      if c>=97 { if c<=102 { nib=c-87 } }
      v=(v<<4)|nib
      i=i+1
  }
  return v
50}
51
52// registry lookup -- 3 fields site<TAB>fp_index<TAB>base_url. NUL-terminate the row's fields IN PLACE;
53// return field pointers. 1=found, 0=not. Registry must end with a newline.
54func is_lookup_site(reg: *u8, n: i64, site: *u8, out_idx: *i64, out_base: *i64) -> i64 {
  var ls: i64=0; var i: i64=0
  while i<=n {
      var eol: i64=0
      if i==n { eol=1 } else { if reg[i]==(10 as u8) { eol=1 } }
      if eol==1 {
          if i>ls {
              var t1: i64=0-1; var t2: i64=0-1
              var k: i64=ls
              while k<i {
                  if reg[k]==(9 as u8) { if t1<0 { t1=k } else { if t2<0 { t2=k } } }
                  k=k+1
              }
              if t1>ls { if t2>t1 {
                  reg[t1]=0 as u8; reg[t2]=0 as u8
                  if i<n { reg[i]=0 as u8 }
                  let sp: *u8=((reg as i64)+ls) as *u8
                  if is_streq(sp, site)==1 {
                      out_idx[0]=(reg as i64)+t1+1
                      out_base[0]=(reg as i64)+t2+1
                      return 1
                  }
              } }
          }
          ls=i+1
      }
      i=i+1
  }
  return 0
83}
84
85func main(argc: i64, argv: *i64) -> i64 {
  if argc<4 {
      is_puts("usage: nx_image_search <sites.tsv> <site> <query_image_path> [w h] [threshold]\n" as *u8)
      is_puts("       PNG query: w/h auto-detected from the file.  raw-gray query: w h required.\n" as *u8)
      sys_exit(2); return 2
  }
  let sites_tsv: *u8=argv[1] as *u8
  let site: *u8=argv[2] as *u8
  let qpath: *u8=argv[3] as *u8
  var w: i64=0
  var h: i64=0
  if argc>=6 { w=is_atoi(argv[4] as *u8); h=is_atoi(argv[5] as *u8) }
  var thresh: i64=IS_DEFAULT_THRESH
  if argc>=7 { thresh=is_atoi(argv[6] as *u8) }
99
  // registry + site lookup (per-site isolation: we only ever open THIS site's index)
  let regbox: *i64=sys_mmap(16) as *i64
  let reg: *u8=sys_read_file(sites_tsv, regbox)
  if reg==0 as *u8 { is_puts("IMGSEARCH-FAIL: no sites registry\n" as *u8); sys_exit(1); return 1 }
  let oidx: *i64=sys_mmap(8) as *i64
  let obase: *i64=sys_mmap(8) as *i64
  if is_lookup_site(reg, regbox[0], site, oidx, obase)==0 {
      is_puts("IMGSEARCH-FAIL: unknown site '" as *u8); is_puts(site); is_puts("' (not in registry)\n" as *u8)
      sys_exit(1); return 1
  }
  let idx_path: *u8=oidx[0] as *u8
  let base_url: *u8=obase[0] as *u8
112
  // read the query image and perceptual-hash it. AUTO-DETECT: a PNG (magic 89 50 4E 47) is decoded
  // via the shared nx_img_to_gray (same path the indexer used); anything else is treated as a raw
  // grayscale w*h buffer (needs w,h). This is the real "search by image": hand it an actual image.
  let qbox: *i64=sys_mmap(16) as *i64
  let qbuf: *u8=sys_read_file(qpath, qbox)
  if qbuf==0 as *u8 { is_puts("IMGSEARCH-FAIL: cannot read query image\n" as *u8); sys_exit(1); return 1 }
  var gray: *u8=qbuf
  var gw: i64=w
  var gh: i64=h
  var ispng: i64=0
  if qbox[0]>=8 { if qbuf[0]==(137 as u8) { if qbuf[1]==(80 as u8) { if qbuf[2]==(78 as u8) { if qbuf[3]==(71 as u8) { ispng=1 } } } } }
  if ispng==1 {
      let wh: *i64=sys_mmap(16) as *i64
      let g: *u8=nx_img_to_gray(qpath, wh)
      if g==0 as *u8 { is_puts("IMGSEARCH-FAIL: cannot decode PNG query (unsupported/corrupt)\n" as *u8); sys_exit(1); return 1 }
      gray=g; gw=wh[0]; gh=wh[1]
  } else {
      if gw<=0 { is_puts("IMGSEARCH-FAIL: raw-gray query needs <w> <h> args\n" as *u8); sys_exit(1); return 1 }
      if gh<=0 { is_puts("IMGSEARCH-FAIL: raw-gray query needs <w> <h> args\n" as *u8); sys_exit(1); return 1 }
      if qbox[0]<gw*gh { is_puts("IMGSEARCH-FAIL: raw-gray query smaller than w*h\n" as *u8); sys_exit(1); return 1 }
  }
  let qd: i64=nx_phash_dhash(gray, gw, gh)
  let qa: i64=nx_phash_ahash(gray, gw, gh)
136
  // load this site's fingerprint index (one read): lines cid<TAB>dhash_hex<TAB>ahash_hex
  let mbox: *i64=sys_mmap(16) as *i64
  let mbuf: *u8=sys_read_file(idx_path, mbox)
  if mbuf==0 as *u8 { is_puts("IMGSEARCH-FAIL: no index for site (run nx_image_index first)\n" as *u8); sys_exit(1); return 1 }
  let mn: i64=mbox[0]
  var nlines: i64=0; var ii: i64=0
  while ii<mn { if mbuf[ii]==(10 as u8) { nlines=nlines+1 } ii=ii+1 }
  if nlines<=0 { is_puts("IMGSEARCH-FAIL: empty index\n" as *u8); sys_exit(1); return 1 }
  let cidp: *i64=sys_mmap(8*(nlines+8)) as *i64
  let cidl: *i64=sys_mmap(8*(nlines+8)) as *i64
  let dh:   *i64=sys_mmap(8*(nlines+8)) as *i64
  let ah:   *i64=sys_mmap(8*(nlines+8)) as *i64
  var ndocs: i64=0; var ls: i64=0
  ii=0
  while ii<mn {
      if mbuf[ii]==(10 as u8) {
          let le: i64=ii
          var t1: i64=0-1; var t2: i64=0-1
          var k: i64=ls
          while k<le { if mbuf[k]==(9 as u8) { if t1<0 { t1=k } else { if t2<0 { t2=k } } } k=k+1 }
          if t1>ls { if t2>t1 {
              cidp[ndocs]=(mbuf as i64)+ls; cidl[ndocs]=t1-ls
              dh[ndocs]=is_hexparse(((mbuf as i64)+t1+1) as *u8)
              ah[ndocs]=is_hexparse(((mbuf as i64)+t2+1) as *u8)
              ndocs=ndocs+1
          } }
          ls=ii+1
      }
      ii=ii+1
  }
  if ndocs<=0 { is_puts("IMGSEARCH-FAIL: index parse yielded 0 rows\n" as *u8); sys_exit(1); return 1 }
168
  // per-doc Hamming (dHash primary, aHash tiebreak); selection-sort an index permutation ascending
  let order: *i64=sys_mmap(8*(ndocs+8)) as *i64
  let hamd:  *i64=sys_mmap(8*(ndocs+8)) as *i64
  let hama:  *i64=sys_mmap(8*(ndocs+8)) as *i64
  var d: i64=0
  while d<ndocs {
      order[d]=d
      hamd[d]=nx_simhash_hamming(qd, dh[d])
      hama[d]=nx_simhash_hamming(qa, ah[d])
      d=d+1
  }
  var a: i64=0
  while a<ndocs {
      var best: i64=a; var bz: i64=a+1
      while bz<ndocs {
          let cb: i64=order[bz]; let cbest: i64=order[best]
          var better: i64=0
          if hamd[cb]<hamd[cbest] { better=1 } else { if hamd[cb]==hamd[cbest] { if hama[cb]<hama[cbest] { better=1 } } }
          if better==1 { best=bz }
          bz=bz+1
      }
      if best!=a { let tmp: i64=order[a]; order[a]=order[best]; order[best]=tmp }
      a=a+1
  }
193
  // matches = images within the dHash threshold ("near this image")
  var nm: i64=0
  d=0
  while d<ndocs { if hamd[d]<=thresh { nm=nm+1 } d=d+1 }
198
  is_puts("IMGSEARCH-RESULTS site=" as *u8); is_puts(site)
  is_puts(" indexed=" as *u8); is_num(ndocs)
  is_puts(" matches=" as *u8); is_num(nm); is_puts("\n" as *u8)
  if nm==0 {
      is_puts("IMGSEARCH-OK matches=0 (no indexed image within threshold)\n" as *u8)
      sys_exit(0); return 0
  }
  var topk: i64=nm
  if topk>IS_TOP_K { topk=IS_TOP_K }
  var r: i64=0
  while r<topk {
      let di: i64=order[r]
      is_puts(" #" as *u8); is_num(r+1)
      is_puts(" ham=" as *u8); is_num(hamd[di])
      is_puts(" simq16=" as *u8); is_num(nx_phash_similarity_q16(qd, dh[di]))
      is_puts(" " as *u8); is_puts(base_url); is_putb(cidp[di] as *u8, cidl[di])
      is_puts("\n" as *u8)
      r=r+1
  }
  is_puts("IMGSEARCH-OK\n" as *u8)
  sys_exit(0); return 0
220}
nx_image_search.nx source

`nx_image_search.nx` source
