code wiki / _hdl_build / nx_gen_ingest.nx
`nx_gen_ingest.nx` source

↩ module page · 242 lines · 11987 B
1// nx_gen_ingest.nx -- ingest an ON-DISK PNG into the gallery: GENREC tEXt -> CID -> blob -> sidecar.
2//
3// WHY THIS EXISTS. The operator asked, this session: *"are these gens pointable so right now i can
4// see them in gallery"*. The measured answer was NO, and it stayed no while three sovereign images
5// were produced: `grep -c sovgen /volume1/ai/gen/cids.tsv` returned 0. The sovereign renderer wrote
6// PNGs to /volume1/ai/images/sovgen/ -- a directory the gallery does not read. Nothing was broken;
7// the images were simply never handed to the index.
8// ★★★★★ AN ARTEFACT THAT IS NOT IN THE INDEX DOES NOT EXIST TO THE PRODUCT, HOWEVER REAL THE FILE IS.
9//
10// nx_gen_pipeline already does exactly this ingest, but its entry point takes BASE64 because it was
11// written for a GPU worker that answers in JSON. The sovereign path has raw PNG bytes on disk, so
12// this organ reuses that file's blob-path and sidecar-append helpers rather than restating them --
13// the gallery's contract must have ONE definition or the two writers will disagree about where an
14// image lives, which is a corruption nobody would see until a link 404s.
15//
16// Usage:
17//   nx_gen_ingest <blobdir> <sidecar|-> <idx|-> <blob|-> <view|-> <model> <seed> <prompt> <png> ...
18// Production values (measured from the live store, not assumed):
19//   blobdir  /volume1/ai/gen/blob-
20//   idx      /volume1/ai/galx/knowledge/index/galx_cid.idx
21//   blob     /volume1/ai/galx/knowledge/index/galx_cid.blob
22// Pass `-` for the sidecar to skip the legacy TSV entirely -- which is the point: the gen->gallery
23// path now writes the SOVEREIGN store directly (nx_galx_cidput), so no 70-byte text append and no
24// 34MB recompile of 235,445 rows stands between a render and the gallery.
25// The gate points everything at /tmp -- ★ A TEST THAT WRITES TO THE PRODUCTION STORE IS NOT A TEST.
26// license_tier: ORIGINAL
27
28import "nx_syscalls.nx"
29import "nx_strconv.nx"
30import "nx_canon_cid.nx"
31import "nx_png_textw.nx"
32import "nx_store_ingest.nx"
33import "nx_gen_pipeline.nx"
34import "nx_galx_cidput.nx"
35import "nx_galx_viewadd_lib.nx"
36const GI_MAGIC_4096: i64 = 4096
37const GI_MAGIC_1024: i64 = 1024
38
39func gi_puts(s: *u8) -> i64 {
  var n: i64 = 0
  while s[n] != (0 as u8) { n = n + 1 }
  return sys_write(1, s, n)
43}
44
45// Read a whole file into a fresh mapping. Returns byte count, or negative.
46// ⚠ Returns -2 on a SHORT read rather than a partial buffer: a truncated PNG would otherwise be
47// ingested under a CID that no future read of the same file could reproduce.
48func gi_slurp(path: *u8, out: *u8, cap: i64) -> i64 {
  let fd: i64 = sys_openat_rd(path)
  if fd < 0 { return 0 - 1 }
  var n: i64 = 0
  var go: i64 = 1
  while go == 1 {
      let r: i64 = sys_read(fd, ((out as i64) + n) as *u8, cap - n)
      if r <= 0 { go = 0 } else { n = n + r }
      if n >= cap { go = 0 }
  }
  sys_close(fd)
  if n >= cap { return 0 - 2 }
  return n
61}
62
63func gi_be32(b: *u8, off: i64) -> i64 {
  return ((b[off] as i64) << 24) | ((b[off + 1] as i64) << 16)
       | ((b[off + 2] as i64) << 8) | (b[off + 3] as i64)
66}
67
68// "<w>x<h>" from the IHDR. The size string is DERIVED from the file, never passed in -- a caller
69// that mislabels the dimensions would write a GENREC that disagrees with its own pixels.
70func gi_size(png: *u8, n: i64, out: *u8) -> i64 {
  if n < 33 { return 0 - 1 }
  if png[12] != (0x49 as u8) { return 0 - 1 }               // 'I' of IHDR
  var o: i64 = nx_strconv_format_i64(gi_be32(png, 16), out)
  out[o] = 0x78                                             // 'x'
  o = o + 1
  o = o + nx_strconv_format_i64(gi_be32(png, 20), ((out as i64) + o) as *u8)
  out[o] = 0
  return o
79}
80
81// basename, so the report reads as images rather than as directories
82func gi_base(p: *u8) -> *u8 {
  var n: i64 = 0
  var last: i64 = 0
  while p[n] != (0 as u8) {
      if p[n] == (0x2F as u8) { last = n + 1 }
      n = n + 1
  }
  return ((p as i64) + last) as *u8
90}
91
92const GI_CAP: i64 = 33554432
93
94func main(argc: i64, argv: *i64) -> i64 {
  if argc < 10 {
      gi_puts("usage: nx_gen_ingest <blobdir> <sidecar|-> <idx|-> <blob|-> <model> <seed> <prompt> <png> [<png> ...]\n" as *u8)
      return 2
  }
  let blobdir: *u8 = argv[1] as *u8
  let sidecar: *u8 = argv[2] as *u8
  let cidx:    *u8 = argv[3] as *u8
  let cblob:   *u8 = argv[4] as *u8
  let cview:   *u8 = argv[5] as *u8
  let model:   *u8 = argv[6] as *u8
  let seed:    *u8 = argv[7] as *u8
  let prompt:  *u8 = argv[8] as *u8
  // ⚠THE ENUMERATOR IS A SEPARATE INDEX AND IT IS THE ONE THE USER SEES. Resolver-only ingest
  // yields an image fetchable by cid and invisible in the grid -- exactly what shipped once.
  var do_view: i64 = 1
  if cview[0] == (0x2D as u8) { if cview[1] == (0 as u8) { do_view = 0 } }
  // "-" disables a sink. The legacy TSV is opt-IN now, not opt-out.
  var do_side: i64 = 1
  if sidecar[0] == (0x2D as u8) { if sidecar[1] == (0 as u8) { do_side = 0 } }
  var do_native: i64 = 1
  if cidx[0] == (0x2D as u8) { if cidx[1] == (0 as u8) { do_native = 0 } }
116
  let raw:  *u8 = sys_mmap(GI_CAP + GI_MAGIC_4096)
  let wt:   *u8 = sys_mmap(GI_CAP + GI_MAGIC_4096)
  let cid:  *u8 = sys_mmap(128)
  let path: *u8 = sys_mmap(GI_MAGIC_1024)
  let size: *u8 = sys_mmap(64)
  let chash: *u8 = sys_mmap(128)
  let args: *u8 = sys_mmap(512)
124
  var n_new: i64 = 0
  var n_dup: i64 = 0
  var n_err: i64 = 0
  var n_idx: i64 = 0
  var n_view: i64 = 0
130
  var a: i64 = 9
  while a < argc {
      let png: *u8 = argv[a] as *u8
      gi_puts(gi_base(png))
      gi_puts("  " as *u8)
      let rn: i64 = gi_slurp(png, raw, GI_CAP)
      if rn < 0 {
          gi_puts("UNREADABLE (or larger than the 32MB cap)\n" as *u8)
          n_err = n_err + 1
      } else {
          if gi_size(raw, rn, size) < 0 {
              // Refuse rather than ingest something that is not a PNG: the gallery serves these
              // bytes to a browser, and the CID would make a non-image permanent.
              gi_puts("NOT A PNG (no IHDR)\n" as *u8)
              n_err = n_err + 1
          } else {
              // ⚠⚠ THE STORE'S CID IS A HASH OF THE HARVESTED GENREC FACTORS, NOT OF THE PIXELS.
              // That is sound for the GPU-worker path, which increments the seed per image, so
              // its factors differ per image. It is NOT sound for ingesting arbitrary files:
              // ingesting two DIFFERENT sovereign renders (69,600 B and 68,863 B) under one seed
              // produced ONE CID, and the second image was silently dropped as a duplicate.
              // ★★★★★★ CONTENT ADDRESSING THAT HASHES THE LABEL INSTEAD OF THE CONTENT SILENTLY
              // DISCARDS EVERY COLLISION AS A "DUPLICATE" -- and the dedup counter reports it as
              // a success. The dedup path is exactly where the loss becomes invisible.
              // ⇒ fold the TRUE content hash into the factors, so the record identifies the bytes.
              // ⚠PRECISELY WHAT THIS HASH COVERS: the source file AS INGESTED, before the GENREC
              // chunk is inserted. The stored blob is those bytes PLUS the chunk, so re-hashing
              // the served image does NOT reproduce it -- you must hash the original, or strip
              // the injected tEXt first. Stating the wrong subject here would hand the next
              // reader a "verification" that fails for a file that is perfectly intact.
              cid_of(raw, rn, chash)
              let pfx: *u8 = "sovereign-gen-path content=" as *u8
              var ao: i64 = 0
              while pfx[ao] != (0 as u8) { args[ao] = pfx[ao]; ao = ao + 1 }
              var ci: i64 = 0
              while chash[ci] != (0 as u8) { args[ao] = chash[ci]; ao = ao + 1; ci = ci + 1 }
              args[ao] = 0
168
              let wl: i64 = png_insert_genrec(raw, rn, seed, model,
                                              "nishi-sovereign" as *u8, "8" as *u8,
                                              "nishios" as *u8, prompt, size,
                                              args, wt)
              if wl < 0 {
                  gi_puts("GENREC INJECT FAILED\n" as *u8)
                  n_err = n_err + 1
              } else {
                  let rc: i64 = nx_store_ingest_ingest_cid(wt, wl, "nxc1" as *u8, cid)
                  if rc < 0 {
                      gi_puts("STORE INGEST FAILED\n" as *u8)
                      n_err = n_err + 1
                  } else {
                      gp_blob_path(blobdir, cid, path)
                      let fd: i64 = sys_openat_wr(path, 0x1a4)
                      if fd >= 0 { sys_write(fd, wt, wl); sys_close(fd) }
                      // rc==1 is new, rc==0 is already in the store. Appending on a dup would
                      // grow the sidecar without bound and make /img/<cid> ambiguous.
                      if rc == 1 {
                          if do_side == 1 { gp_sidecar_append(sidecar, cid, path) }
                          n_new = n_new + 1
                      } else { n_dup = n_dup + 1 }
                      gi_puts(size)
                      gi_puts("  " as *u8)
                      gi_puts(cid)
                      if rc == 1 { gi_puts("  NEW" as *u8) } else { gi_puts("  dup" as *u8) }
195
                      // ---- the sovereign store, directly. This is what the gallery READS. ----
                      // Done even for rc==0: a blob can exist in the factor store while the
                      // cid->path index has never heard of it, which is precisely the state that
                      // left 109 images 404ing. The put is idempotent, so re-asserting is free.
                      if do_native == 1 {
                          let pr: i64 = nx_galx_cidput(cidx, cblob, cid, path)
                          if pr == CP_INSERTED { gi_puts("  idx:INSERTED" as *u8); n_idx = n_idx + 1 }
                          else {
                              if pr == CP_PRESENT { gi_puts("  idx:present" as *u8) }
                              else {
                                  gi_puts("  idx:FAILED" as *u8)
                                  n_err = n_err + 1
                              }
                          }
                      }
                      // ---- THE ENUMERATOR. Separate index, and the only one the grid reads. ----
                      if do_view == 1 {
                          let vr: i64 = nx_galx_view_prepend(cview, cid)
                          if vr == NXV_ADDED { gi_puts("  view:ADDED" as *u8); n_view = n_view + 1 }
                          else {
                              if vr == NXV_PRESENT { gi_puts("  view:present" as *u8) }
                              else { gi_puts("  view:FAILED" as *u8); n_err = n_err + 1 }
                          }
                      }
                      gi_puts("\n" as *u8)
                  }
              }
          }
      }
      a = a + 1
  }
227
  gi_puts("ingested_new=" as *u8)
  let b: *u8 = sys_mmap(32)
  sys_write(1, b, nx_strconv_format_i64(n_new, b))
  gi_puts(" duplicates=" as *u8)
  sys_write(1, b, nx_strconv_format_i64(n_dup, b))
  gi_puts(" idx_inserted=" as *u8)
  sys_write(1, b, nx_strconv_format_i64(n_idx, b))
  gi_puts(" view_added=" as *u8)
  sys_write(1, b, nx_strconv_format_i64(n_view, b))
  gi_puts(" errors=" as *u8)
  sys_write(1, b, nx_strconv_format_i64(n_err, b))
  gi_puts("\n" as *u8)
  if n_err > 0 { return 1 }
  return 0
242}
nx_gen_ingest.nx source

`nx_gen_ingest.nx` source
