code wiki / (root) / nx_asset_meta_gate.nx
`nx_asset_meta_gate.nx` source

↩ module page · 360 lines · 18386 B
1// nx_asset_meta_gate.nx -- KAT + TEETH for nx_asset_meta + nx_asset_autotag (R3 of the universal
2// organization-tooling arc). ONE gate, both organs.
3//
4// PROVES the catalog can know WHAT each asset IS and WHICH media are NEAR-duplicates:
5//
6//  (a) METADATA (KAT, REAL extraction):
7//      - a REAL PNG byte buffer (signature + IHDR, width=320 height=200) -> am_enrich_image populates
8//        width="320" / height="200" / encoding_format="image/png" extracted FROM THE BYTES;
9//      - those fields ROUND-TRIP through R0's ar_encode -> ar_decode -> ar_get byte-faithfully;
10//      - a DOC record -> am_enrich_doc populates dc:title + dc:subject, round-tripping the same way;
11//      - TEETH: non-PNG bytes -> am_enrich_image refuses (AM_NOT_PNG), inventing NO dimensions.
12//
13//  (b) NEAR-DUP via perceptual hash:
14//      - two NEAR-identical grayscale images (a small perturbation) -> at_neardup GROUPS them
15//        (Hamming < threshold);
16//      - two CLEARLY-DIFFERENT images -> NOT grouped (TEETH: no false-merge);
17//      - identical bytes -> Hamming 0 AND same CID (ar_cid).
18//
19//  (c) THE WHOLE POINT -- perceptual hash catches a near-dup that exact-CID does NOT:
20//      a 1-pixel-changed image has a DIFFERENT CID (ar_cid differs) but a near-zero Hamming distance
21//      -> grouped. Both asserted (CID differs AND phash groups).
22//
23// Verdict logged to knowledge/status/asset_meta_gate.log (append-only; ADDITIVE law #13).
24// expect_exit: 0   license_tier: ORIGINAL
25import "nx_syscalls.nx"
26import "nx_asset_record.nx"
27import "nx_asset_meta.nx"
28import "nx_asset_autotag.nx"
29import "nx_gate_verdict.nx"
30
31// ---- gate I/O (logfd THREADED as a param: compiler supports const globals but not reassigning a
32// module-level var inside a function -- the R0/R2 gate idiom) ----
33func g_puts(logfd: i64, s: *u8) -> i64 {
  var n: i64 = 0
  while s[n] != (0 as u8) { n = n + 1 }
  sys_write(1, s, n)
  if logfd > 0 { sys_write(logfd, s, n) }
  return 0
39}
40func g_putn(logfd: i64, v: i64) -> i64 {
  let bb: *u8 = sys_mmap(28)
  var m: i64 = v
  if m < 0 { g_puts(logfd, "-\x00" as *u8); m = 0 - m }
  let t: *u8 = sys_mmap(28)
  var k: i64 = 0
  if m == 0 { t[0] = 48 as u8; k = 1 }
  while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
  var i: i64 = 0
  while i < k { bb[i] = t[k - 1 - i]; i = i + 1 }
  sys_write(1, bb, k)
  if logfd > 0 { sys_write(logfd, bb, k) }
  return 0
53}
54func g_streq(a: *u8, b: *u8) -> i64 {
  var i: i64 = 0
  while 1 == 1 {
      if a[i] != b[i] { return 0 }
      if a[i] == (0 as u8) { return 1 }
      i = i + 1
  }
  return 1
62}
63// empty-string sentinel for omitted optional fields (mirrors R0/R2's E()).
64func E() -> *u8 { return "\x00" as *u8 }
65
66// build a synthetic 320x200-shaped PNG: 8-byte signature + IHDR (len=13, "IHDR", width/height BE,
67// bit_depth=8 color_type=2 RGB) + 4 CRC bytes (we don't verify CRC; nx_png_header doesn't either).
68// Returns the byte length. Width/height are written big-endian per the IHDR layout.
69func gate_make_png(raw: *u8, w: i64, h: i64) -> i64 {
  var i: i64 = 0
  while i < 64 { raw[i] = 0 as u8; i = i + 1 }
  raw[0] = 0x89 as u8; raw[1] = 0x50 as u8; raw[2] = 0x4E as u8; raw[3] = 0x47 as u8
  raw[4] = 0x0D as u8; raw[5] = 0x0A as u8; raw[6] = 0x1A as u8; raw[7] = 0x0A as u8
  raw[11] = 13 as u8                                   // IHDR length (BE; high bytes already 0)
  raw[12] = 0x49 as u8; raw[13] = 0x48 as u8; raw[14] = 0x44 as u8; raw[15] = 0x52 as u8  // "IHDR"
  raw[16] = ((w >> 24) & 255) as u8; raw[17] = ((w >> 16) & 255) as u8
  raw[18] = ((w >> 8) & 255) as u8;  raw[19] = (w & 255) as u8        // width BE
  raw[20] = ((h >> 24) & 255) as u8; raw[21] = ((h >> 16) & 255) as u8
  raw[22] = ((h >> 8) & 255) as u8;  raw[23] = (h & 255) as u8        // height BE
  raw[24] = 8 as u8                                    // bit_depth
  raw[25] = 2 as u8                                    // color_type RGB
  raw[26] = 0 as u8; raw[27] = 0 as u8; raw[28] = 0 as u8             // compression/filter/interlace
  return 8 + 8 + 13 + 4                                // = 33 bytes
84}
85
86// fill a w*h grayscale buffer with a deterministic horizontal gradient (rich adjacent-pixel structure
87// so dHash produces a meaningful, differentiable fingerprint). pixel(x,y) = (x*7 + y*3) & 255.
88func gate_fill_gradient(g: *u8, w: i64, h: i64) -> i64 {
  var y: i64 = 0
  while y < h {
      var x: i64 = 0
      while x < w {
          g[y * w + x] = ((x * 7 + y * 3) & 255) as u8
          x = x + 1
      }
      y = y + 1
  }
  return 0
99}
100// copy n bytes src->dst.
101func gate_memcpy(dst: *u8, src: *u8, n: i64) -> i64 {
  var i: i64 = 0
  while i < n { dst[i] = src[i]; i = i + 1 }
  return 0
105}
106
107func main() -> i64 {
  let logfd: i64 = sys_openat_append("knowledge/status/asset_meta_gate.log\x00" as *u8, 0x1a4)
  g_puts(logfd, "=== ASSET-META-GATE (R3: metadata enrichment + perceptual near-dup) ===\n\x00" as *u8)
110
  var pass: i64 = 0
  var total: i64 = 0
113
  // =========================================================================================
  // (a) METADATA -- REAL extraction from PNG bytes + Dublin Core round-trip.
  // =========================================================================================
  g_puts(logfd, "-- (a) METADATA enrichment (real PNG extraction + Dublin Core round-trip) --\n\x00" as *u8)
118
  // build a real PNG (width=320, height=200) and REAL-extract its facts.
  let png: *u8 = sys_mmap(64)
  let pnglen: i64 = gate_make_png(png, 320, 200)
  // field set for an image record; start from the core/type via ar_fields, then am_enrich_image
  // EXTENDS it. (keys/vals sized generously; kv[0] = running field count.)
  let keys: *i64 = sys_mmap(8 * 32) as *i64
  let vals: *i64 = sys_mmap(8 * 32) as *i64
  let kv: *i64 = sys_mmap(16) as *i64
  kv[0] = 0
  // seed minimal core: type=image, title (so the record is realistic + we prove enrich is additive).
  ar_addf(keys, vals, kv, "type\x00" as *u8, "image\x00" as *u8)
  ar_addf(keys, vals, kv, "title\x00" as *u8, "Family Photo\x00" as *u8)
  // REAL extraction -> width/height/encoding_format buffers (must OUTLIVE the field set).
  let wbuf: *u8 = sys_mmap(24)
  let hbuf: *u8 = sys_mmap(24)
  let fbuf: *u8 = sys_mmap(24)
  let nf_after: i64 = am_enrich_image(png, pnglen, keys, vals, kv, wbuf, hbuf, fbuf)
  g_puts(logfd, "  am_enrich_image extracted: width=\x00" as *u8); g_puts(logfd, wbuf)
  g_puts(logfd, " height=\x00" as *u8); g_puts(logfd, hbuf)
  g_puts(logfd, " encoding_format=\x00" as *u8); g_puts(logfd, fbuf); g_puts(logfd, "\n\x00" as *u8)
139
  // (a1) KAT: extracted values equal the expected PNG dimensions + format.
  total = total + 1
  g_puts(logfd, "  (a1) KAT image facts == 320 / 200 / image/png (real-extracted): \x00" as *u8)
  var a1: i64 = 0
  if nf_after > 0 {
      a1 = 1
      if g_streq(wbuf, "320\x00" as *u8) == 0 { a1 = 0 }
      if g_streq(hbuf, "200\x00" as *u8) == 0 { a1 = 0 }
      if g_streq(fbuf, "image/png\x00" as *u8) == 0 { a1 = 0 }
  }
  if a1 == 1 { pass = pass + 1; g_puts(logfd, "PASS\n\x00" as *u8) } else { g_puts(logfd, "FAIL\n\x00" as *u8) }
151
  // (a2) ROUND-TRIP: encode the enriched field set -> decode -> ar_get returns the same values.
  let enc: *u8 = sys_mmap(8192)
  let elen: i64 = ar_encode(keys, vals, kv[0], enc)
  let dk: *i64 = sys_mmap(8 * 32) as *i64
  let dv: *i64 = sys_mmap(8 * 32) as *i64
  let nfd: i64 = ar_decode(enc, elen, dk, dv, 32)
  total = total + 1
  g_puts(logfd, "  (a2) fields round-trip ar_encode->ar_decode->ar_get [w=\x00" as *u8)
  g_puts(logfd, ar_get(dk, dv, nfd, "width\x00" as *u8)); g_puts(logfd, " h=\x00" as *u8)
  g_puts(logfd, ar_get(dk, dv, nfd, "height\x00" as *u8)); g_puts(logfd, " fmt=\x00" as *u8)
  g_puts(logfd, ar_get(dk, dv, nfd, "encoding_format\x00" as *u8)); g_puts(logfd, " type=\x00" as *u8)
  g_puts(logfd, ar_get(dk, dv, nfd, "type\x00" as *u8)); g_puts(logfd, "]: \x00" as *u8)
  var a2: i64 = 0
  if nfd > 0 {
      a2 = 1
      if g_streq(ar_get(dk, dv, nfd, "width\x00" as *u8), "320\x00" as *u8) == 0 { a2 = 0 }
      if g_streq(ar_get(dk, dv, nfd, "height\x00" as *u8), "200\x00" as *u8) == 0 { a2 = 0 }
      if g_streq(ar_get(dk, dv, nfd, "encoding_format\x00" as *u8), "image/png\x00" as *u8) == 0 { a2 = 0 }
      if g_streq(ar_get(dk, dv, nfd, "type\x00" as *u8), "image\x00" as *u8) == 0 { a2 = 0 }
  }
  if a2 == 1 { pass = pass + 1; g_puts(logfd, "PASS\n\x00" as *u8) } else { g_puts(logfd, "FAIL\n\x00" as *u8) }
173
  // (a3) DOC record: am_enrich_doc populates dc:title + dc:subject, round-tripping the same way.
  let dkeys: *i64 = sys_mmap(8 * 32) as *i64
  let dvals: *i64 = sys_mmap(8 * 32) as *i64
  let dkv: *i64 = sys_mmap(16) as *i64
  dkv[0] = 0
  ar_addf(dkeys, dvals, dkv, "type\x00" as *u8, "doc\x00" as *u8)
  am_enrich_doc("2025 Tax Return\x00" as *u8, "taxes,finance,2025\x00" as *u8, dkeys, dvals, dkv)
  let denc: *u8 = sys_mmap(8192)
  let delen: i64 = ar_encode(dkeys, dvals, dkv[0], denc)
  let ddk: *i64 = sys_mmap(8 * 32) as *i64
  let ddv: *i64 = sys_mmap(8 * 32) as *i64
  let dnfd: i64 = ar_decode(denc, delen, ddk, ddv, 32)
  total = total + 1
  g_puts(logfd, "  (a3) DOC dc:title+dc:subject round-trip [title=\x00" as *u8)
  g_puts(logfd, ar_get(ddk, ddv, dnfd, "title\x00" as *u8)); g_puts(logfd, " subject=\x00" as *u8)
  g_puts(logfd, ar_get(ddk, ddv, dnfd, "subject\x00" as *u8)); g_puts(logfd, "]: \x00" as *u8)
  var a3: i64 = 0
  if dnfd > 0 {
      a3 = 1
      if g_streq(ar_get(ddk, ddv, dnfd, "title\x00" as *u8), "2025 Tax Return\x00" as *u8) == 0 { a3 = 0 }
      if g_streq(ar_get(ddk, ddv, dnfd, "subject\x00" as *u8), "taxes,finance,2025\x00" as *u8) == 0 { a3 = 0 }
  }
  if a3 == 1 { pass = pass + 1; g_puts(logfd, "PASS\n\x00" as *u8) } else { g_puts(logfd, "FAIL\n\x00" as *u8) }
197
  // (a4) TEETH: non-PNG bytes -> am_enrich_image REFUSES (AM_NOT_PNG), invents no dimensions.
  let notpng: *u8 = sys_mmap(64)
  var z: i64 = 0
  while z < 64 { notpng[z] = (65 + (z % 5)) as u8; z = z + 1 }   // arbitrary ASCII, no PNG signature
  let tkeys: *i64 = sys_mmap(8 * 8) as *i64
  let tvals: *i64 = sys_mmap(8 * 8) as *i64
  let tkv: *i64 = sys_mmap(16) as *i64
  tkv[0] = 0
  let twb: *u8 = sys_mmap(24)
  let thb: *u8 = sys_mmap(24)
  let tfb: *u8 = sys_mmap(24)
  let tret: i64 = am_enrich_image(notpng, 64, tkeys, tvals, tkv, twb, thb, tfb)
  total = total + 1
  g_puts(logfd, "  (a4) TEETH non-PNG -> AM_NOT_PNG, 0 fields invented (ret=\x00" as *u8)
  g_putn(logfd, tret); g_puts(logfd, " fields=\x00" as *u8); g_putn(logfd, tkv[0]); g_puts(logfd, "): \x00" as *u8)
  if tret == AM_NOT_PNG { if tkv[0] == 0 { pass = pass + 1; g_puts(logfd, "PASS\n\x00" as *u8) } else { g_puts(logfd, "FAIL\n\x00" as *u8) } } else { g_puts(logfd, "FAIL\n\x00" as *u8) }
214
  // =========================================================================================
  // (b) NEAR-DUP via perceptual hash.
  // =========================================================================================
  g_puts(logfd, "-- (b) perceptual near-dup (group near-identical, NOT clearly-different) --\n\x00" as *u8)
219
  let W: i64 = 32
  let H: i64 = 32
  let npx: i64 = W * H
223
  // imgA: gradient. imgB: copy of A with a SMALL perturbation (near-identical). imgC: a clearly
  // DIFFERENT pattern (vertical bands inverted from A's gradient).
  let imgA: *u8 = sys_mmap(npx)
  let imgB: *u8 = sys_mmap(npx)
  let imgC: *u8 = sys_mmap(npx)
  gate_fill_gradient(imgA, W, H)
  gate_memcpy(imgB, imgA, npx)
  // LOCALIZED perturbation: strongly brighten one small block (rows 0-7, cols 0-3) so it flips a
  // FEW dHash bits (the relative-order comparisons land differently in those downscale cells) yet
  // leaves A~B well under threshold -- this exercises the near-but-NOT-identical fingerprint case
  // (a non-zero sub-threshold Hamming), not a degenerate Hamming-0. clamps at 255.
  var py: i64 = 0
  while py < 8 {
      var px: i64 = 0
      while px < 4 {
          let idx: i64 = py * W + px
          var nv: i64 = (imgB[idx] as i64) + 120
          if nv > 255 { nv = 255 }
          imgB[idx] = nv as u8
          px = px + 1
      }
      py = py + 1
  }
  // clearly different image: a high-contrast checkerboard (uncorrelated with the gradient's order).
  var cy: i64 = 0
  while cy < H {
      var cx: i64 = 0
      while cx < W {
          if ((cx + cy) % 2) == 0 { imgC[cy * W + cx] = 0 as u8 } else { imgC[cy * W + cx] = 255 as u8 }
          cx = cx + 1
      }
      cy = cy + 1
  }
257
  let hA: i64 = at_phash(imgA, W, H)
  let hB: i64 = at_phash(imgB, W, H)
  let hC: i64 = at_phash(imgC, W, H)
  let hAB: i64 = at_hamming(hA, hB)
  let hAC: i64 = at_hamming(hA, hC)
  g_puts(logfd, "  dHash Hamming: A~B(near-dup)=\x00" as *u8); g_putn(logfd, hAB)
  g_puts(logfd, "  A~C(different)=\x00" as *u8); g_putn(logfd, hAC); g_puts(logfd, "\n\x00" as *u8)
265
  // threshold = 10 (generous near-dup band for 64-bit dHash; A~B must fall under, A~C must not).
  let THRESH: i64 = 10
  let hashes: *i64 = sys_mmap(8 * 3) as *i64
  hashes[0] = hA; hashes[1] = hB; hashes[2] = hC
  let groups: *i64 = sys_mmap(8 * 3) as *i64
  let ngroups: i64 = at_neardup(hashes, 3, THRESH, groups)
  g_puts(logfd, "  at_neardup groups: A=\x00" as *u8); g_putn(logfd, groups[0])
  g_puts(logfd, " B=\x00" as *u8); g_putn(logfd, groups[1])
  g_puts(logfd, " C=\x00" as *u8); g_putn(logfd, groups[2])
  g_puts(logfd, " distinct=\x00" as *u8); g_putn(logfd, ngroups); g_puts(logfd, "\n\x00" as *u8)
276
  // (b1) near-identical A,B grouped (Hamming < threshold AND same group id).
  total = total + 1
  g_puts(logfd, "  (b1) near-identical A,B GROUPED (hamming<thresh + same group): \x00" as *u8)
  var b1: i64 = 0
  if hAB < THRESH { if at_in_same_group(groups, 0, 1) == 1 { b1 = 1 } }
  if b1 == 1 { pass = pass + 1; g_puts(logfd, "PASS\n\x00" as *u8) } else { g_puts(logfd, "FAIL\n\x00" as *u8) }
283
  // (b2) TEETH: clearly-different C NOT merged with A (no false-merge), and 2 distinct groups total.
  total = total + 1
  g_puts(logfd, "  (b2) TEETH different C NOT grouped with A + exactly 2 distinct groups: \x00" as *u8)
  var b2: i64 = 0
  if hAC >= THRESH { if at_in_same_group(groups, 0, 2) == 0 { if ngroups == 2 { b2 = 1 } } }
  if b2 == 1 { pass = pass + 1; g_puts(logfd, "PASS\n\x00" as *u8) } else { g_puts(logfd, "FAIL\n\x00" as *u8) }
290
  // (b3) identical bytes -> Hamming 0 AND same CID (ar_cid over the raw buffers).
  let hAA: i64 = at_hamming(hA, at_phash(imgA, W, H))
  let cidA: *u8 = sys_mmap(128)
  let cidA2: *u8 = sys_mmap(128)
  ar_cid(imgA, npx, cidA)
  ar_cid(imgA, npx, cidA2)
  total = total + 1
  g_puts(logfd, "  (b3) identical bytes -> Hamming 0 (=\x00" as *u8); g_putn(logfd, hAA)
  g_puts(logfd, ") AND same CID: \x00" as *u8)
  var b3: i64 = 0
  if hAA == 0 { if g_streq(cidA, cidA2) == 1 { b3 = 1 } }
  if b3 == 1 { pass = pass + 1; g_puts(logfd, "PASS\n\x00" as *u8) } else { g_puts(logfd, "FAIL\n\x00" as *u8) }
303
  // =========================================================================================
  // (c) THE WHOLE POINT: a 1-pixel-changed image has a DIFFERENT CID but a near-zero Hamming
  //     distance -> the perceptual hash groups what exact-CID dedup MISSES.
  // =========================================================================================
  g_puts(logfd, "-- (c) phash catches what CID misses (1-px change: diff CID, near-zero Hamming) --\n\x00" as *u8)
  let imgA1: *u8 = sys_mmap(npx)
  gate_memcpy(imgA1, imgA, npx)
  imgA1[123] = (imgA[123] + 1) as u8           // ONE pixel changed by 1
312
  let cidOrig: *u8 = sys_mmap(128)
  let cid1px: *u8 = sys_mmap(128)
  ar_cid(imgA, npx, cidOrig)
  ar_cid(imgA1, npx, cid1px)
  let hA1: i64 = at_phash(imgA1, W, H)
  let hamA_A1: i64 = at_hamming(hA, hA1)
319
  // group the original + the 1-px-changed image.
  let h2: *i64 = sys_mmap(8 * 2) as *i64
  h2[0] = hA; h2[1] = hA1
  let g2: *i64 = sys_mmap(8 * 2) as *i64
  let ng2: i64 = at_neardup(h2, 2, THRESH, g2)
325
  g_puts(logfd, "  CID(orig)  = \x00" as *u8); g_puts(logfd, cidOrig); g_puts(logfd, "\n\x00" as *u8)
  g_puts(logfd, "  CID(1px)   = \x00" as *u8); g_puts(logfd, cid1px); g_puts(logfd, "\n\x00" as *u8)
  g_puts(logfd, "  CID differs=\x00" as *u8)
  var ciddiff: i64 = 1
  if g_streq(cidOrig, cid1px) == 1 { ciddiff = 0 }
  g_putn(logfd, ciddiff)
  g_puts(logfd, "  phash Hamming(orig,1px)=\x00" as *u8); g_putn(logfd, hamA_A1)
  g_puts(logfd, "  grouped=\x00" as *u8); g_putn(logfd, at_in_same_group(g2, 0, 1)); g_puts(logfd, "\n\x00" as *u8)
334
  // (c) BOTH must hold: exact-CID says "different" (ciddiff==1) yet perceptual hash says "near-dup"
  // (Hamming < threshold AND grouped + collapses to 1 group). This is the capability exact-CID lacks.
  total = total + 1
  g_puts(logfd, "  (c) CID DIFFERS yet phash GROUPS (the near-dup CID misses): \x00" as *u8)
  var c1: i64 = 0
  if ciddiff == 1 {
      if hamA_A1 < THRESH {
          if at_in_same_group(g2, 0, 1) == 1 {
              if ng2 == 1 { c1 = 1 }
          }
      }
  }
  if c1 == 1 { pass = pass + 1; g_puts(logfd, "PASS\n\x00" as *u8) } else { g_puts(logfd, "FAIL\n\x00" as *u8) }
348
  // =========================================================================================
  g_puts(logfd, "ASSET-META-GATE passed \x00" as *u8); g_putn(logfd, pass); g_puts(logfd, "/\x00" as *u8); g_putn(logfd, total)
  // MIGRATED onto nx_gate_verdict by nx_gate_dry_apply (D001, minimal form): every check
  // row above is untouched, so the PASS/FAIL vector cannot change; only the hand-rolled
  // verdict emission is replaced by the ONE shared base class. Proven by nx_gate_migrate verify.
  let ctr__dry: *i64 = gv_ctr()
  ctr__dry[0] = pass
  ctr__dry[1] = total
  let rc__dry: i64 = gv_verdict("ASSET-META-GATE" as *u8, ctr__dry, "teeth unchanged; verdict emission migrated onto the shared base class" as *u8)
  sys_exit(rc__dry)
  return rc__dry
360}
nx_asset_meta_gate.nx source

`nx_asset_meta_gate.nx` source
