code wiki / (root) / nx_plane_migrate.nx
`nx_plane_migrate.nx` source

↩ module page · 207 lines · 9201 B
1// nx_plane_migrate.nx -- migrate a plane's VALUE ENCODING from flat tab-delimited lines to NXR1 records
2// (seq1326 rung 2, the actual retirement). ADDITIVE BY CONSTRUCTION: it writes to a NEW destination
3// prefix and NEVER touches the source, so rule 13 holds and a bad migration costs a wasted prefix, not data.
4//
5// ORDER OF OPERATIONS IS THE SAFETY PROPERTY:
6//   1. load the source with the HONEST loader
7//   2. REFUSE BEFORE WRITING ANYTHING if the read was truncated or the plane was empty -- migrating a
8//      partial read would silently drop rows into a plane that then looks authoritative. This is the
9//      lesson nx_plane_check taught me by catching my own false GREEN: a checker that has not seen every
10//      row must never act as if it has.
11//   3. write every row as NXR1 under the same q:<seq> key
12//   4. VERIFY by reading the destination back and rendering each record to TSV, comparing byte-for-byte
13//      against the source line. Verification reads the STORE, not the in-memory buffer, so it exercises
14//      the real commit path rather than trusting the writer.
15//
16// WHY THE KEY STAYS q:<seq> FOR NOW: this rung changes the ENCODING only. Re-keying to business keys is a
17// separate rung with its own blast radius, and doing both at once would make a failure impossible to
18// attribute. One change, one gate.
19//
20// EXIT: 0 migrated+verified · 2 usage · 3 verify mismatch · 4 source empty · 6 source read TRUNCATED
21//       7 too many rows for the row table · 8 commit failed
22// license_tier: ORIGINAL   No hw writes (Rule 26).
23import "nx_record_tsv.nx"
24import "nx_store_seed_lib.nx"
25
26const PM_CAP: i64 = 16777216
27const PM_RECCAP: i64 = 262144
28const PM_OUTCAP: i64 = 262144
29const PM_MSGCAP: i64 = 4096
30const PM_TYPECAP: i64 = 512
31const PM_FLAGS: i64 = 64
32const PM_MAXROWS: i64 = 65536
33const PM_KEYCAP: i64 = 64
34const PM_NUMCAP: i64 = 32
35const PM_SLACK: i64 = 1048576
36const PM_GROWTH: i64 = 3          // NXR1 adds 7B/field + 6B header; 3x the flat bytes is a safe arena
37const PM_NL: i64 = 10
38const PM_CH_I: i64 = 105
39const PM_STDOUT: i64 = 1
40const PM_KIND_LIVE: i64 = 1
41const PM_EXIT_USAGE: i64 = 2
42const PM_EXIT_MISMATCH: i64 = 3
43const PM_EXIT_EMPTY: i64 = 4
44const PM_EXIT_TRUNCATED: i64 = 6
45const PM_EXIT_TOOMANY: i64 = 7
46const PM_EXIT_COMMIT: i64 = 8
47const PM_EXIT_LOSSY: i64 = 9
48
49func pm_len(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
50func pm_cat(d: *u8, off: i64, s: *u8) -> i64 {
  var i: i64 = 0
  while s[i] != (0 as u8) { d[off + i] = s[i]; i = i + 1 }
  return off + i
54}
55
56func main(argc: i64, argv: *i64) -> i64 {
  let msg: *u8 = sys_mmap(PM_MSGCAP)
  if argc < 3 {
      var u: i64 = pm_cat(msg, 0, "usage: nx_plane_migrate <src-prefix-> <dst-prefix-> [typespec]\n  writes NXR1 records to dst, never touches src; verifies by rendering back to TSV\n" as *u8)
      sys_write(PM_STDOUT, msg, u)
      return PM_EXIT_USAGE
  }
  let src: *u8 = argv[1] as *u8
  let dst: *u8 = argv[2] as *u8
65
  let types: *u8 = sys_mmap(PM_TYPECAP)
  var ntypes: i64 = 0
  if argc >= 4 {
      let ts: *u8 = argv[3] as *u8
      let tn: i64 = pm_len(ts)
      var i: i64 = 0
      while i < tn {
          if i < PM_TYPECAP {
              if ts[i] == (PM_CH_I as u8) { types[i] = NXR_T_I64 as u8 } else { types[i] = NXR_T_STR as u8 }
          }
          i = i + 1
      }
      ntypes = tn
      if ntypes > PM_TYPECAP { ntypes = PM_TYPECAP }
  }
81
  let buf: *u8 = sys_mmap(PM_CAP)
  let flags: *i64 = sys_mmap(PM_FLAGS) as *i64
  let n: i64 = sts_load_honest(src, buf, PM_CAP, flags)
85
  // index the source lines ONCE so the write pass and the verify pass see exactly the same rows.
  let offs: *i64 = sys_mmap(8 * PM_MAXROWS) as *i64
  let lens: *i64 = sys_mmap(8 * PM_MAXROWS) as *i64
  var rows: i64 = 0
  var over: i64 = 0
  var start: i64 = 0
  var i2: i64 = 0
  while i2 <= n {
      var atend: i64 = 0
      if i2 == n { atend = 1 } else { if buf[i2] == (PM_NL as u8) { atend = 1 } }
      if atend == 1 {
          let llen: i64 = i2 - start
          if llen > 0 {
              if rows < PM_MAXROWS { offs[rows] = start; lens[rows] = llen; rows = rows + 1 } else { over = 1 }
          }
          start = i2 + 1
      }
      i2 = i2 + 1
  }
105
  // ---- REFUSE BEFORE WRITING. Every guard here runs while the destination is still untouched.
  var o: i64 = pm_cat(msg, 0, "NX-PLANE-MIGRATE src=" as *u8)
  o = pm_cat(msg, o, src)
  o = pm_cat(msg, o, " dst=" as *u8)
  o = pm_cat(msg, o, dst)
  o = pm_cat(msg, o, " rows=" as *u8);        o = rtv_itoa(msg, o, rows)
  o = pm_cat(msg, o, " loaded_rows=" as *u8); o = rtv_itoa(msg, o, flags[1])
  o = pm_cat(msg, o, " bytes=" as *u8);       o = rtv_itoa(msg, o, n)
  o = pm_cat(msg, o, " cap=" as *u8);         o = rtv_itoa(msg, o, PM_CAP)
115
  if over == 1 {
      o = pm_cat(msg, o, " verdict=RED-TOO-MANY-ROWS-nothing-written\n" as *u8)
      sys_write(PM_STDOUT, msg, o); return PM_EXIT_TOOMANY
  }
  if rows == 0 {
      o = pm_cat(msg, o, " verdict=RED-EMPTY-nothing-written\n" as *u8)
      sys_write(PM_STDOUT, msg, o); return PM_EXIT_EMPTY
  }
  // ★★★LOSSY-SOURCE GUARD (added 2026-07-30 AFTER I violated it on the debt- plane). flags[0] is the
  // plane's DECLARED q:n and flags[1] is what the loader could actually reach. When declared > loaded,
  // rows are ALREADY unreachable -- and rewriting the plane with a NEW q:n equal to the loaded count
  // BAKES THAT LOSS IN and DESTROYS THE SIGNAL that something is wrong. nx_debt already had exactly this
  // guard ("DEBT-REFUSED lossy load ... committing would BAKE the loss into the next generation") and I
  // shipped a migrator without it, then silently converted a flagged-lossy debt- plane into a
  // self-consistent one. The rows were unreachable either way; what I removed was the ALARM.
  // A MIGRATOR MUST NEVER MAKE A DAMAGED PLANE LOOK HEALTHY.
  var lossy: i64 = 0
  if flags[0] > flags[1] { lossy = 1 }
  var trunc: i64 = 0
  if n >= PM_CAP { trunc = 1 }
  if rows != flags[1] { trunc = 1 }
  if trunc == 1 {
      o = pm_cat(msg, o, " verdict=RED-SOURCE-READ-TRUNCATED-nothing-written\n" as *u8)
      sys_write(PM_STDOUT, msg, o); return PM_EXIT_TRUNCATED
  }
  if lossy == 1 {
      o = pm_cat(msg, o, " declared_qn=" as *u8); o = rtv_itoa(msg, o, flags[0])
      o = pm_cat(msg, o, " verdict=RED-SOURCE-PLANE-IS-LOSSY-declared-exceeds-loadable-REPAIR-FIRST-nothing-written\n" as *u8)
      sys_write(PM_STDOUT, msg, o); return PM_EXIT_LOSSY
  }
146
  // ---- WRITE PASS: one segment carrying every row as NXR1 plus the q:n count.
  let w: *i64 = ss_begin_cap(n * PM_GROWTH + PM_SLACK)
  let rec: *u8 = sys_mmap(PM_RECCAP)
  let key: *u8 = sys_mmap(PM_KEYCAP)
  var r: i64 = 0
  while r < rows {
      let lp: *u8 = (buf as i64 + offs[r]) as *u8
      let rn: i64 = rtv_to_nxr(lp, lens[r], types, ntypes, rec)
      sts_rowkey(r, key)
      ss_add(w, PM_KIND_LIVE, key, rec, rn)
      r = r + 1
  }
  let cntb: *u8 = sys_mmap(PM_NUMCAP)
  let cn: i64 = rtv_itoa(cntb, 0, rows)
  ss_add(w, PM_KIND_LIVE, "q:n" as *u8, cntb, cn)
  if ss_commit(dst, w, ss_next_segid(dst)) != 0 {
      o = pm_cat(msg, o, " verdict=RED-COMMIT-FAILED\n" as *u8)
      sys_write(PM_STDOUT, msg, o); return PM_EXIT_COMMIT
  }
166
  // ---- VERIFY PASS: read the DESTINATION STORE back (not the in-memory buffer) and render each record
  //      to TSV, comparing byte-for-byte with the source line. This exercises the real commit path.
  let h: *i64 = ss_open_cached(dst)
  let pv: *i64 = sys_mmap(32) as *i64
  let lv: *i64 = sys_mmap(32) as *i64
  let out: *u8 = sys_mmap(PM_OUTCAP)
  var same: i64 = 0
  var bad: i64 = 0
  if (h as i64) != 0 {
      var v: i64 = 0
      while v < rows {
          sts_rowkey(v, key)
          var okrow: i64 = 0
          if ss_hget(h, key, pv, lv) == 1 {
              let rp: *u8 = pv[0] as *u8
              let cols: i64 = nxr_count(rp)
              let on: i64 = rtv_to_tsv(rp, lv[0], cols, out, PM_OUTCAP)
              if on == lens[v] {
                  let lp2: *u8 = (buf as i64 + offs[v]) as *u8
                  okrow = 1
                  var k: i64 = 0
                  while k < on { if out[k] != lp2[k] { okrow = 0; k = on } else { k = k + 1 } }
              }
          }
          if okrow == 1 { same = same + 1 } else { bad = bad + 1 }
          v = v + 1
      }
      ss_close(h)
  }
196
  o = pm_cat(msg, o, " written=" as *u8);  o = rtv_itoa(msg, o, rows)
  o = pm_cat(msg, o, " verified=" as *u8); o = rtv_itoa(msg, o, same)
  o = pm_cat(msg, o, " bad=" as *u8);      o = rtv_itoa(msg, o, bad)
  var rc: i64 = 0
  if bad > 0 { rc = PM_EXIT_MISMATCH }
  if same != rows { rc = PM_EXIT_MISMATCH }
  if rc == 0 { o = pm_cat(msg, o, " verdict=GREEN-migrated-and-every-row-renders-back-byte-identical\n" as *u8) }
  if rc != 0 { o = pm_cat(msg, o, " verdict=RED-VERIFY-FAILED-dst-is-suspect-src-UNTOUCHED\n" as *u8) }
  sys_write(PM_STDOUT, msg, o)
  return rc
207}
nx_plane_migrate.nx source

`nx_plane_migrate.nx` source
