`nx_semruler.nx` source

↩ module page · 299 lines · 14991 B
1// nx_semruler.nx -- THE SEMANTIC RULER, STEP 2: measure the EXISTING descriptors on a real
2// same-concept corpus. license_tier: ORIGINAL
3//
4// nx_imgsearch publishes its own gap honestly and names the order of work:
5//   {"class":"semantic-lookalike","permille":-1,"status":"UNMEASURED",
6//    "note":"... (1) build a semantic ruler from labelled same-concept pairs, (2) measure the EXISTING
7//            descriptors as the honest baseline, (3) only then a quantised learned image embedding."}
8// nx_semruler_lib is step (1)'s decision core. THIS is step (2), and it deliberately stops short of
9// step (3): it builds NO new descriptor and trains nothing.
10//
11// WHERE THE LABELS COME FROM, AND WHY THEY ARE HONEST. knowledge/media/ is one directory per subject
12// (diora_baird, julia_kyoka, barack_obama...). Two images in the same directory are the SAME CONCEPT;
13// two in different directories are not. That is a weak label derived from the estate's own filing, not
14// an opinion, and it is the standard way a retrieval corpus is bootstrapped. NO LABEL IS INVENTED HERE.
15//
16// THE ONE NUMBER THIS EXISTS TO PRODUCE, and why it has to come first:
17//   A "SAME-CONCEPT" PAIR THAT IS ACTUALLY A NEAR-DUPLICATE MEASURES THE COPY TIER, NOT SEMANTICS.
18//   Before anyone can report a semantic recall figure, they have to know how much of the same-subject
19//   corpus is genuinely different photographs and how much is re-uploads, re-crops and re-encodes. Run
20//   a recall benchmark over an unfiltered folder of one person and it will score near 1000 permil and
21//   "prove" the engine is already semantic -- the number real, the subject wrong.
22//   So this organ reports the ADMISSIBILITY CENSUS: of all within-subject pairs, how many survive
23//   sr_pair_admissible (distinct photographs) and how many are duplicates under some dihedral
24//   transform. THE SURVIVING SET IS THE ONLY VALID DENOMINATOR for step 3.
25//
26// AND THE CONTROL THAT KEEPS IT HONEST. If the within-subject inadmissible count is ZERO, this corpus
27// contains no duplicates, the predicate was never exercised on real data, and the census says so rather
28// than reporting a clean sweep. Cross-subject pairs are sampled as the opposite control: they should be
29// overwhelmingly ADMISSIBLE, because two different people are never near-duplicates -- if they are not,
30// the distance is not measuring what it claims and no semantic figure built on it would mean anything.
31import "nx_syscalls.nx"
32import "nx_dir.nx"
33import "nx_image_gray.nx"
34import "nx_imgsearch_tier.nx"
35import "nx_semruler_lib.nx"
36
37const SR_MAXCLASS: i64 = 32
38const SR_MAXIMG: i64 = 48          // per-class image cap -- ANNOUNCED, never silent (see coverage line)
39const SR_PATHCAP: i64 = 1024
40const SR_ROWCAP: i64 = 512
41const SR_ARENA: i64 = 262144
42const SR_XPAIRS: i64 = 64          // cross-subject control pairs
43const SR_DESCCAP: i64 = 16         // i64 slots per descriptor (copy=1, orient index=1, orient query=8)
44
45func sw(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
46func sn(v: i64) -> i64 {
  let t: *u8 = sys_mmap(32)
  var m: i64 = v
  if m < 0 { sw("-" as *u8); m = 0 - m }
  var k: i64 = 0
  if m == 0 { t[0] = 48 as u8; k = 1 }
  while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
  let b: *u8 = sys_mmap(32)
  var j: i64 = 0
  while j < k { b[j] = t[k - 1 - j]; j = j + 1 }
  sys_write(1, b, k)
  return 0
58}
59func swn(p: *u8, n: i64) -> i64 { sys_write(1, p, n); return 0 }
60
61func sr_is_image(name: *u8, nl: i64) -> i64 {
  if nl < 5 { return 0 }
  var i: i64 = nl - 1
  while i > 0 {
      if name[i] == (46 as u8) {
          // .jpg .png .gif  (webp/avif are a declared decoder gap in nx_imgsearch status)
          if name[i+1] == (106 as u8) { return 1 }
          if name[i+1] == (112 as u8) { return 1 }
          if name[i+1] == (103 as u8) { return 1 }
          return 0
      }
      i = i - 1
  }
  return 0
75}
76
77func sr_join(a: *u8, b: *u8, out: *u8) -> i64 {
  var o: i64 = 0
  var i: i64 = 0
  while a[i] != (0 as u8) { out[o] = a[i]; o = o + 1; i = i + 1 }
  out[o] = 47 as u8; o = o + 1
  i = 0
  while b[i] != (0 as u8) { out[o] = b[i]; o = o + 1; i = i + 1 }
  out[o] = 0 as u8
  return o
86}
87
88func main(argc: i64, argv: *i64) -> i64 {
  var root: *u8 = "knowledge/media" as *u8
  if argc > 1 { root = argv[1] as *u8 }
91
  sw("=== nx_semruler -- ADMISSIBILITY CENSUS over a real same-concept corpus ===\n" as *u8)
  sw("root=" as *u8); sw(root); sw("\n" as *u8)
  sw("labels: one directory per subject; same dir = same concept. No label is invented.\n" as *u8)
95
  // the two tiers whose distances define "is this the same photograph"
  let tcopy: *nx_imgtier = nx_imgtier_new(NX_IT_KIND_COPY, NX_IT_COPY_THRESH, 1000)
  let torient: *nx_imgtier = nx_imgtier_new(NX_IT_KIND_ORIENT, NX_IT_COPY_THRESH, 1000)
99
  let rows: *NxDirRow = sys_mmap(NX_DIR_ROW_BYTES * SR_ROWCAP) as *NxDirRow
  let arena: *u8 = sys_mmap(SR_ARENA)
  let res: *NxDirResult = sys_mmap(NX_DIR_RESULT_BYTES) as *NxDirResult
  let dv: i64 = nx_dir_list(root, rows, SR_ROWCAP, arena, SR_ARENA, 0, res)
  if dv == NX_DIR_OPEN_FAILED {
      sw("ROOT UNREADABLE -- no conclusion available in either direction\n" as *u8)
      sys_exit(3); return 3
  }
108
  // per-class descriptor stores. cd[] = copy index desc, od[] = orient index desc, oq[] = orient query desc
  let cd: *i64 = sys_mmap(8 * SR_MAXIMG * SR_MAXCLASS) as *i64
  let od: *i64 = sys_mmap(8 * SR_MAXIMG * SR_MAXCLASS) as *i64
  let oq: *i64 = sys_mmap(8 * SR_MAXIMG * SR_MAXCLASS * 8) as *i64
  let ccount: *i64 = sys_mmap(8 * SR_MAXCLASS) as *i64
  let cnameoff: *i64 = sys_mmap(8 * SR_MAXCLASS) as *i64
  let cnamelen: *i64 = sys_mmap(8 * SR_MAXCLASS) as *i64
116
  let subrows: *NxDirRow = sys_mmap(NX_DIR_ROW_BYTES * SR_ROWCAP) as *NxDirRow
  let subarena: *u8 = sys_mmap(SR_ARENA)
  let subres: *NxDirResult = sys_mmap(NX_DIR_RESULT_BYTES) as *NxDirResult
  let dirpath: *u8 = sys_mmap(SR_PATHCAP)
  let filepath: *u8 = sys_mmap(SR_PATHCAP)
  let wh: *i64 = sys_mmap(16) as *i64
  let dbuf: *i64 = sys_mmap(8 * SR_DESCCAP) as *i64
  // HOISTED 2026-09-01: these three were allocated PER PAIR inside the inner loop, which on a 48-image
  // class is over three thousand mmaps and blew the allocator arena (ARENA-OVERRUN on first live run).
  // NEVER ALLOCATE IN A HOT LOOP -- pass the buffer in. The loop body only ever writes slot 0 before
  // reading it, so one buffer per role is sufficient and carries no state between iterations.
  let qc: *i64 = sys_mmap(8) as *i64
  let dc: *i64 = sys_mmap(8) as *i64
  let doo: *i64 = sys_mmap(8) as *i64
131
  var nclass: i64 = 0
  var total_seen: i64 = 0
  var total_used: i64 = 0
  var total_undecodable: i64 = 0
  var any_capped: i64 = 0
137
  var r: i64 = 0
  while r < res.n_filled {
      if nclass >= SR_MAXCLASS { r = res.n_filled } else {
          let row: *NxDirRow = nx_dir_row_at(rows, r)
          if nx_dir_row_is_regular_file(row) == 0 {
              // a directory: one subject class. skip dot-entries and the _sheets helper dir.
              if row.name_ptr[0] != (46 as u8) {
                  if row.name_ptr[0] != (95 as u8) {
                      sr_join(root, row.name_ptr, dirpath)
                      let sv: i64 = nx_dir_list(dirpath, subrows, SR_ROWCAP, subarena, SR_ARENA, 0, subres)
                      if sv != NX_DIR_OPEN_FAILED {
                          var used: i64 = 0
                          var seen: i64 = 0
                          var sr2: i64 = 0
                          while sr2 < subres.n_filled {
                              let frow: *NxDirRow = nx_dir_row_at(subrows, sr2)
                              if nx_dir_row_is_regular_file(frow) == 1 {
                                  if sr_is_image(frow.name_ptr, frow.name_len) == 1 {
                                      seen = seen + 1
                                      if used < SR_MAXIMG {
                                          sr_join(dirpath, frow.name_ptr, filepath)
                                          let g: *u8 = nx_img_to_gray(filepath, wh)
                                          if (g as i64) == 0 { total_undecodable = total_undecodable + 1 } else {
                                              let base: i64 = nclass * SR_MAXIMG + used
                                              nx_imgtier_describe_index(tcopy, g, 0 as *u8, wh[0], wh[1], dbuf)
                                              cd[base] = dbuf[0]
                                              nx_imgtier_describe_index(torient, g, 0 as *u8, wh[0], wh[1], dbuf)
                                              od[base] = dbuf[0]
                                              nx_imgtier_describe_query(torient, g, 0 as *u8, wh[0], wh[1], dbuf)
                                              var k: i64 = 0
                                              while k < 8 { oq[base * 8 + k] = dbuf[k]; k = k + 1 }
                                              used = used + 1
                                          }
                                      }
                                  }
                              }
                              sr2 = sr2 + 1
                          }
                          if seen > used { any_capped = 1 }
                          if used > 1 {
                              ccount[nclass] = used
                              cnameoff[nclass] = row.name_ptr as i64
                              cnamelen[nclass] = row.name_len
                              total_seen = total_seen + seen
                              total_used = total_used + used
                              nclass = nclass + 1
                          }
                      }
                  }
              }
          }
          r = r + 1
      }
  }
192
  if nclass < 1 {
      sw("NO SUBJECT CLASS WITH >1 DECODABLE IMAGE -- the census has no population and REFUSES to\n" as *u8)
      sw("publish a rate. This is not a pass: a ruler over an empty set proves nothing.\n" as *u8)
      sys_exit(3); return 3
  }
198
  sw("\ncoverage: classes=" as *u8); sn(nclass)
  sw(" images_seen=" as *u8); sn(total_seen)
  sw(" images_used=" as *u8); sn(total_used)
  sw(" undecodable=" as *u8); sn(total_undecodable)
  sw(" per_class_cap=" as *u8); sn(SR_MAXIMG)
  sw(" capped=" as *u8); sn(any_capped)
  if any_capped == 1 { sw("  <== THE CAP BOUND: counts below are a FLOOR over the used set, not the whole corpus" as *u8) }
  sw("\n  (webp/avif are excluded here because they are a DECLARED decoder gap in nx_imgsearch status,\n" as *u8)
  sw("   not because they were sampled away -- undecodable files are counted above, never dropped.)\n\n" as *u8)
208
  // ---- WITHIN-SUBJECT: the census that decides whether a semantic benchmark is even possible ----
  var tot_pairs: i64 = 0
  var tot_adm: i64 = 0
  var tot_dup: i64 = 0
  var ci: i64 = 0
  while ci < nclass {
      let n: i64 = ccount[ci]
      var pairs: i64 = 0
      var adm: i64 = 0
      var a: i64 = 0
      while a < n {
          var b: i64 = a + 1
          while b < n {
              let ba: i64 = ci * SR_MAXIMG + a
              let bb: i64 = ci * SR_MAXIMG + b
              qc[0] = cd[ba]
              dc[0] = cd[bb]
              let cdist: i64 = nx_imgtier_distance(tcopy, qc, dc)
              doo[0] = od[bb]
              let odist: i64 = nx_imgtier_distance(torient, (oq as i64 + ba * 64) as *i64, doo)
              pairs = pairs + 1
              if sr_pair_admissible(cdist, odist) == 1 { adm = adm + 1 }
              b = b + 1
          }
          a = a + 1
      }
      sw("  class " as *u8); swn(cnameoff[ci] as *u8, cnamelen[ci])
      sw(" images=" as *u8); sn(n)
      sw(" pairs=" as *u8); sn(pairs)
      sw(" admissible=" as *u8); sn(adm)
      sw(" duplicate=" as *u8); sn(pairs - adm)
      sw(" admissible_permil=" as *u8); sn(sr_rate_permil(adm, pairs))
      sw("\n" as *u8)
      tot_pairs = tot_pairs + pairs
      tot_adm = tot_adm + adm
      tot_dup = tot_dup + (pairs - adm)
      ci = ci + 1
  }
247
  // ---- CROSS-SUBJECT CONTROL: two different people must not read as near-duplicates ----
  var xp: i64 = 0
  var xadm: i64 = 0
  if nclass > 1 {
      var i2: i64 = 0
      while i2 < SR_XPAIRS {
          let c1: i64 = i2 % nclass
          var c2: i64 = (c1 + 1) % nclass
          let a: i64 = i2 % ccount[c1]
          let b: i64 = i2 % ccount[c2]
          let ba: i64 = c1 * SR_MAXIMG + a
          let bb: i64 = c2 * SR_MAXIMG + b
          qc[0] = cd[ba]
          dc[0] = cd[bb]
          let cdist: i64 = nx_imgtier_distance(tcopy, qc, dc)
          doo[0] = od[bb]
          let odist: i64 = nx_imgtier_distance(torient, (oq as i64 + ba * 64) as *i64, doo)
          xp = xp + 1
          if sr_pair_admissible(cdist, odist) == 1 { xadm = xadm + 1 }
          i2 = i2 + 1
      }
  }
270
  sw("\nWITHIN-SUBJECT TOTAL pairs=" as *u8); sn(tot_pairs)
  sw(" admissible=" as *u8); sn(tot_adm)
  sw(" duplicate=" as *u8); sn(tot_dup)
  sw(" admissible_permil=" as *u8); sn(sr_semantic_permil(tot_adm, tot_pairs))
  sw("\nCROSS-SUBJECT CONTROL pairs=" as *u8); sn(xp)
  sw(" admissible=" as *u8); sn(xadm)
  sw(" admissible_permil=" as *u8); sn(sr_rate_permil(xadm, xp))
  sw("\n\n" as *u8)
279
  // ---- what the numbers MEAN, stated so a reader cannot take the wrong one ----
  sw("READ THIS BEFORE QUOTING ANY NUMBER ABOVE:\n" as *u8)
  sw("  admissible pairs ARE the valid denominator for a semantic recall benchmark (step 3).\n" as *u8)
  sw("  duplicate pairs would be scored by the COPY tiers and must never enter that denominator.\n" as *u8)
  if tot_dup == 0 {
      sw("  ** THE PREDICATE WAS NOT EXERCISED: zero within-subject duplicates were found, so this\n" as *u8)
      sw("     corpus does not demonstrate the guard firing on real data. Do not read 1000 permil as\n" as *u8)
      sw("     evidence the guard works -- read it as evidence this corpus has no duplicates.\n" as *u8)
  }
  if xp > 0 {
      if xadm < xp {
          sw("  ** CONTROL ANOMALY: some CROSS-SUBJECT pairs read as near-duplicates. Two different\n" as *u8)
          sw("     subjects should never be the same photograph, so the distance is not measuring what\n" as *u8)
          sw("     it claims and NO semantic figure built on it is trustworthy until this is explained.\n" as *u8)
      }
  }
  sw("  the semantic-lookalike figure in nx_imgsearch status stays -1 UNMEASURED after this run:\n" as *u8)
  sw("  this organ measures the CORPUS, not the ENGINE. Step 3 is the ranking benchmark.\n" as *u8)
  return 0
299}
nx_semruler.nx source

`nx_semruler.nx` source
