code wiki / _hdl_build / nx_web_shard_compact.nx
`nx_web_shard_compact.nx` source

↩ module page · 464 lines · 27291 B
1// nx_web_shard_compact.nx -- SCALE compaction for big multi-segment shards (the web shard's rung).
2// ss_compact/ss_compact_cap have EXACT semantics (last entry per key wins, tombstones kept, retired
3// segments archived then an atomic manifest swap) but their key dedup is a LINEAR table scan =
4// O(entries^2): fine at 4k keys, unusable at the web shard's ~1.4M entries (bulk docs + pr: priors).
5// This organ is the same merge with an open-addressing HASH dedup = O(entries), plus a post-swap
6// verification pass. Semantics kept byte-equal to ss_compact: chronological walk, last wins, ALL kinds
7// carried (incl. kind-2 tombstones), archive-before-swap, every failure path returns BEFORE the swap.
8//
9// BYTE-BUDGETED SUFFIX FOLD (2026-07-29): an unbounded whole-shard merge materializes the merged
10// segment in anonymous RAM (the writer buffer, plus aux blobs and key tables that all scale with
11// merged bytes) -- the OOM killer reaped it twice at bytes-in=1883885124 on 2026-07-25, mid-merge,
12// before the swap (never-brick held; shard untouched). The merge therefore folds a contiguous run
13// whose .docs bytes fit a budget derived from the box itself (MemAvailable/WSC_BUDGET_SHARE, clamped
14// to WSC_BUDGET_CEIL; argv[2] in MB overrides, same clamp). Repeated runs converge the shard
15// LSM-style with bounded memory; a streaming (file-backed) writer that lifts the budget entirely is
16// the named next rung.
17//
18// RANGE FOLD (2026-08-10, supersedes suffix-only selection). The suffix rule stalls forever at the
19// first FAT segment: once a merged product nears the budget it heads every suffix walk (runbytes
20// starts at the tail), so each later fold must carry it and the hundreds of small segments BEHIND it
21// can never be reached -- measured on this shard: ~836 pre-08-08 segments stranded behind a ~240MB
22// merged tail while the live count climbed 878 -> 1205 and query latency followed. A fold of ANY
23// contiguous run, replaced IN PLACE in the manifest, preserves last-wins semantics by the same
24// argument the suffix fold's header makes: shadowing depends only on RELATIVE manifest order, and the
25// merged segment sits exactly where the run sat -- newer kept segments still shadow it, and it still
26// shadows older ones, precisely as the unmerged run did. Kind-2 tombstones are carried as before.
27// FAT SKIP: a segment >= budget/WSC_FAT_DIV is an already-merged product; re-folding it buys almost
28// no count reduction per byte, so runs are chosen from non-fat segments only, newest first.
29//   usage: nx_web_shard_compact <domain> [budget-MB]
30// license_tier: ORIGINAL
31import "nx_docportal_search_seg.nx"    // dss_prefix + nx_seg_store (ss_* primitives)
32import "nx_heavyio_lib.nx"             // estate-wide heavy-I/O bound (2026-09-03): a scale fold is pure seg-store maintenance -- it yields to the storm BEFORE taking the plane lock
33const WSC_MAGIC_1125899906842597: i64 = 1125899906842597
34const WSC_MAGIC_1048576: i64 = 1048576
35const WSC_MAGIC_65536: i64 = 65536
36
37const WSC_MAXSEGS: i64 = 8192        // manifest lines we can fold in one run
38const WSC_MAXENT:  i64 = 134217728  // hard sanity ceiling on entries (2^27; ~an order past any near plan)
39const WSC_HMUL:    i64 = 7046029254386353131  // odd 63-bit multiplicative-mix constant (same family as pagerank_build)
40const WSC_BUDGET_SHARE: i64 = 32        // /32 (was /16) 2026-07-30: the /16 reasoning below is RIGHT about the 9-10x amplification but it reserves a share of MemAvailable AT START on a box with OTHER GROWING CONSUMERS (torrent stack, jellyfin, syno services) -- the share is stale the moment it is computed, and it assumes this job is the only claimant. MEASURED TODAY: with ~25GB available at start the /16 budget projected a peak of ~15GB and this organ was caught at 21GB RSS with 536MB free, minutes from OOM, taking mgmt + the tools daemon down with it. /32 projects ~7-8GB, which also keeps it clear of the 16GiB RLIMIT_AS ceiling now imposed by hc_spawn_searchpipe_job -- TWO CONTROLS THAT CONTRADICT EACH OTHER ARE ONE CONTROL AND ONE OUTAGE, so the soft budget must project a peak BELOW the hard cap. Original note, still accurate on the amplification: fold budget = MemAvailable/16. MEASURED 2026-07-29 (death #3, OOM in ss_build_terms): a 1.88GB run took MemAvailable from 19.4GB to 1.2GB before the kill => PEAK TOUCH ~9-10x run bytes (occ arrays + pairs + writer + key tables + input page cache), not the ~6x the seq1048 memo estimated. /16 caps projected peak at ~60% of MemAvailable so the box keeps serving
41const WSC_BUDGET_CEIL: i64 = 209715200  // 200MB HARD CEILING on the fold budget, argv and auto alike. MEASURED: explicit budgets 128/200/230MB all PASSed (2026-08-08) while 256MB died ENOMEM in ss_build_terms under the 16GiB RLIMIT_AS; on 2026-08-10 the UNCLAMPED auto budget (MemAvailable/32 = ~744MB with ~23GB free) died in merge-write on EVERY spawner run -- the searchcompact spawner passes no budget, so the unattended path MUST be self-capping or compaction is structurally dead exactly when nobody is watching. /32 bounds the SHARE of the box; this bounds the ABSOLUTE peak against the address-space cap. Raise only with a re-measured amplification factor.
42const WSC_FAT_DIV: i64 = 2              // fat threshold divisor: a segment >= budget/WSC_FAT_DIV is an already-merged product -- skipped by run selection, never re-folded at this budget
43const WSC_BUDGET_FALLBACK: i64 = 268435456  // 256MB: used ONLY when /proc/meminfo is unreadable -- conservative enough to be safe on any box that boots this stack, keeps the pinned no-arg hostctl sub from wedging into a permanent refuse (the CEIL clamp above still applies)
44const WSC_BUDGET_FLOOR: i64 = 67108864  // 64MB: below this the box is too tight to merge anything safely -> refuse loudly rather than thrash
45const WSC_LOCK_EX: i64 = 2              // flock LOCK_EX on <prefix>plock -- the plane-lock discipline segguard + nx_seg_compact_cli already honor; a lockless manifest swap under a concurrent fold LOSES rows (the proven seq349 race)
46const WSC_MEMINFO_BUF: i64 = 8192       // scratch size for the bounded /proc/meminfo read (file is ~1.5KB)
47const WSC_MEMINFO_CAP: i64 = 7900       // read cap under the scratch -- headroom below WSC_MEMINFO_BUF
48const WSC_KB: i64 = 1024                // /proc/meminfo reports MemAvailable in kB
49
50func wc9_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
51func wc9_num(v: i64) -> i64 {
  let bb: *u8 = sys_mmap(28); var m: i64 = v
  if m < 0 { sys_write(1, "-" as *u8, 1); m = 0 - m }
  let t: *u8 = sys_mmap(28); var k: i64 = 0
  if m == 0 { t[0] = 48 as u8; k = 1 }
  while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
  var i: i64 = 0; while i < k { bb[i] = t[k - 1 - i]; i = i + 1 }
  sys_write(1, bb, k); return 0
59}
60// polynomial hash of key bytes -> mixed slot (same ci_hash polynomial so behavior is well-understood).
61// hshift/hmask parameterized: the table is sized DATA-DRIVEN from a first-pass entry count.
62func wsc_slot(k: *u8, kl: i64, hshift: i64, hmask: i64) -> i64 {
  var h: i64 = WSC_MAGIC_1125899906842597
  var i: i64 = 0
  while i < kl { h = (h * 131) + (k[i] as i64); i = i + 1 }
  var s: i64 = h * WSC_HMUL
  s = s & 0x7fffffffffffffff
  return (s >> hshift) & hmask
69}
70// MemAvailable in BYTES from /proc/meminfo -- the kernel's own "allocatable without swapping" figure.
71// procfs files report size 0 to lseek, so ss_readall (lseek-sized buffer) CANNOT read them (its 64KB
72// reads would overflow a 64-byte buffer); this is a bounded read loop into a fixed 8KB scratch instead.
73// Returns -1 if unreadable or unparseable -- callers must treat that as "cannot judge", never as 0.
74func wsc_memavail() -> i64 {
  let fd: i64 = sys_openat_rd("/proc/meminfo" as *u8)
  if fd < 0 { return 0 - 1 }
  let buf: *u8 = sys_mmap(WSC_MEMINFO_BUF)
  var got: i64 = 0
  var n: i64 = 1
  while n > 0 {
      if got >= WSC_MEMINFO_CAP { n = 0 } else {
          n = sys_read(fd, (buf as i64 + got) as *u8, WSC_MEMINFO_CAP - got)
          if n > 0 { got = got + n }
      }
  }
  sys_close(fd)
  if got <= 0 { return 0 - 1 }
  let pat: *u8 = "MemAvailable:" as *u8
  var i: i64 = 0
  while i + 13 < got {
      var hit: i64 = 1
      var j: i64 = 0
      while j < 13 { if buf[i + j] != pat[j] { hit = 0; j = 13 } else { j = j + 1 } }
      if hit == 1 {
          var v: i64 = 0
          var seen: i64 = 0
          var k2: i64 = i + 13
          while k2 < got {
              let c: i64 = buf[k2] as i64
              if c == 10 { k2 = got } else {
                  if c >= 48 { if c <= 57 { v = v * 10 + (c - 48); seen = 1 } }
                  k2 = k2 + 1
              }
          }
          if seen == 1 { return v * WSC_KB }
          return 0 - 1
      }
      i = i + 1
  }
  return 0 - 1
111}
112
113func main(argc: i64, argv: *i64) -> i64 {
  var domain: *u8 = "web" as *u8
  if argc >= 2 { domain = argv[1] as *u8 }
  let prefix: *u8 = sys_mmap(512)
  dss_prefix(domain, prefix)
  // HEAVY-I/O ADMISSION BEFORE THE PLANE LOCK (2026-09-03): a scale fold is pure seg-store maintenance, so under
  // a storm it YIELDS (exit 4, verdict=DEFER -- a third state, never FAIL) and must never sit holding <prefix>plock
  // while it waits -- the shape nx_seg_compact_cli adopted. HIO_UNOBSERVABLE (conf unreadable) PROCEEDS: maintenance
  // is never blocked forever by a missing conf, and the announce line names the state that was measured.
  let hio: *i64 = (sys_mmap(HIO_C_BYTES)) as *i64
  let hv: i64 = hio_admit(hio)
  hio_announce(1, hv, hio)
  if hv == HIO_DEFER { wc9_puts("nx_web_shard_compact DEFERRED: heavy-I/O bound reached; requeue and re-fire\nverdict=DEFER\n" as *u8); return 4 }
  // PLANE LOCK FIRST: serialize vs segguard + nx_seg_compact_cli (both flock <prefix>plock).
  // Blocking LOCK_EX -- a short wait behind a sibling fold is correct; running beside one is not.
  let lkp: *u8 = sys_mmap(512)
  var lo: i64 = 0
  lo = ss_cat(lkp, lo, prefix)
  lo = ss_cat(lkp, lo, "plock" as *u8)
  lkp[lo] = 0 as u8
  let lfd: i64 = sys_openat_append(lkp, 0x1a4)
  if lfd < 0 { wc9_puts("cannot open plane lock\nverdict=FAIL\n" as *u8); return 13 }
  if sys_flock(lfd, WSC_LOCK_EX) != 0 { wc9_puts("cannot take plane lock\nverdict=FAIL\n" as *u8); return 13 }
  let segs: *i64 = sys_mmap(8 * WSC_MAXSEGS) as *i64
  let ns: i64 = ss_manifest_cap(prefix, segs, WSC_MAXSEGS)
  wc9_puts("=== nx_web_shard_compact " as *u8); wc9_puts(prefix); wc9_puts(" segments=" as *u8); wc9_num(ns); wc9_puts(" ===\n" as *u8)
  if ns <= 0 { wc9_puts("no live segments\nverdict=FAIL\n" as *u8); return 1 }
  if ns == 1 { wc9_puts("already a single segment -- nothing to fold\nverdict=PASS\n" as *u8); return 0 }
141
  // fold-run byte budget: argv[2] MB if given, else MemAvailable/WSC_BUDGET_SHARE, else fallback.
  var budget: i64 = 0
  if argc >= 3 {
      let a2c: *u8 = argv[2] as *u8
      var bi: i64 = 0
      while a2c[bi] != (0 as u8) { let c: i64 = a2c[bi] as i64; if c >= 48 { if c <= 57 { budget = budget * 10 + (c - 48) } } bi = bi + 1 }
      budget = budget * WSC_MAGIC_1048576
  }
  if budget <= 0 {
      let ma: i64 = wsc_memavail()
      if ma > 0 { budget = ma / WSC_BUDGET_SHARE }
      if ma <= 0 { budget = WSC_BUDGET_FALLBACK; wc9_puts("MemAvailable unreadable -> conservative fallback budget\n" as *u8) }
  }
  // HARD CEILING, argv and auto alike (see WSC_BUDGET_CEIL). Printed so a clamped run is never
  // mistaken for a run at the requested budget -- a silent cap becomes a measurement nobody knows
  // is partial.
  if budget > WSC_BUDGET_CEIL {
      wc9_puts("budget clamped " as *u8); wc9_num(budget); wc9_puts(" -> " as *u8); wc9_num(WSC_BUDGET_CEIL)
      wc9_puts(" (ss_build_terms peak ~9-10x run bytes vs 16GiB RLIMIT_AS; 256MB measured ENOMEM 2026-08-08, unclamped auto ~744MB died every run 2026-08-10)\n" as *u8)
      budget = WSC_BUDGET_CEIL
  }
  wc9_puts("fold-budget bytes=" as *u8); wc9_num(budget); wc9_puts("\n" as *u8)
  if budget < WSC_BUDGET_FLOOR { wc9_puts("budget below floor -- box too tight to merge safely, refusing\nverdict=FAIL\n" as *u8); return 11 }
165
  // fresh merged segid -- DELEGATED to ss_next_segid (the seq1730 REPAIR half), not rolled locally.
  // The local max+1 scan this replaces took the max over EVERY id in the manifest, INCLUDING poisoned
  // pointer-band ones. This shard carries seg-140712850411539 (= 0x7FFF...), written by some earlier
  // caller that stored a pointer where a parsed id belonged, so max+1 inherited the poison and
  // ss_write_seg's guard correctly REFUSED the write -- MEASURED 2026-08-08: fold reached
  // "merge-write: emitting seg" then failed with "segid=140712850411540 is an ADDRESS", verdict=FAIL,
  // NOTHING WRITTEN. seq1730 shipped in two halves and this organ had adopted only the refusing one,
  // so the guard could fire forever and compaction could never succeed.
  // ss_max_segid (which ss_next_segid delegates to) SKIPS the pointer band by design -- "a corrupted
  // manifest heals itself on the next write instead of staying pinned near 1.4e14 forever" -- so this
  // both unblocks the fold and repairs the id sequence. It returns max-SANE + 1, which is greater than
  // every sane id present and distinct from every poisoned one, so it cannot collide.
  // ONE max-segid rule in the tree (rule 15): two copies of it is exactly how the two drift apart.
  let segid: i64 = ss_next_segid(prefix)
  if ss_segid_ok(segid) == 0 { wc9_puts("ss_next_segid returned an unusable id -- refusing before any write\nverdict=FAIL\n" as *u8); return 13 }
181
  // read every live segment (chronological = manifest order)
  let bptrs: *i64 = sys_mmap(8 * WSC_MAXSEGS) as *i64
  let bszs: *i64 = sys_mmap(8 * WSC_MAXSEGS) as *i64
  // HOISTED OUT OF THE LOOP (2026-08-06). These were allocated PER SEGMENT and never freed, so a
  // manifest with N live segments burned N pages for `path` plus N for `szp` -- sys_mmap here is
  // page-granular, so a 512-byte path costs 4096 bytes and its own kernel VMA.
  // THIS IS NOT THEORETICAL: dmesg records this organ dying THREE TIMES, roughly hourly, at
  // `segfault at fffffffffffffff4`. That address is -12 = ENOMEM. mmap failed, NOTHING CHECKED THE
  // RETURN, and the next write went straight through the error code -- the exact endgame written
  // down in sys_munmap's own header, happening on schedule.
  // Reuse is behaviour-identical: path is rebuilt from offset 0 and NUL-terminated every iteration,
  // and szp is overwritten by ss_loadfile every iteration. Same hoist that fixed the tokenizer.
  let path: *u8 = sys_mmap(512)
  let szp: *i64 = sys_mmap(16) as *i64
  // FAIL LOUD RATHER THAN WRITE THROUGH. A negative return is -errno; refusing here turns a silent
  // memory-corrupting crash into a verdict a caller can read.
  if (path as i64) <= 0 { wc9_puts("mmap failed for the path buffer\nverdict=FAIL\n" as *u8); return 12 }
  if (szp as i64) <= 0 { wc9_puts("mmap failed for the size slot\nverdict=FAIL\n" as *u8); return 12 }
  var total: i64 = 0
  var s: i64 = 0
  while s < ns {
      var o: i64 = 0
      o = ss_cat(path, o, prefix)
      o = ss_cat(path, o, segs[s] as *u8)
      o = ss_cat(path, o, ".docs" as *u8)
      path[o] = 0 as u8
      bptrs[s] = ss_loadfile(path, szp, 1) as i64   // mmap-open the input segments: sequential read, low RSS
      bszs[s] = szp[0]
      if bszs[s] < 0 { bszs[s] = 0 }
      total = total + bszs[s]
      s = s + 1
  }
214
  // RANGE-RUN SELECTION (2026-08-10; see the RANGE FOLD header note). Newest-first: find the newest
  // contiguous run of >=2 NON-FAT segments whose total .docs bytes fit the budget. Fat segments are
  // skipped over (they neither join a run nor end the search), so runs BEHIND a fat merged tail are
  // reachable -- the exact population the suffix rule stranded. flo..fhi inclusive is the run.
  // MAX-COUNT WINDOW, not newest-first (2026-08-10 v2). Newest-first starved convergence: the
  // crawler always supplies a fresh 2-3 segment run at the tail, so every round folded THOSE and the
  // ~800-segment old base below the fat products was never reached (measured: one round removed 2
  // lines). Choosing the run with the MOST SEGMENTS under budget digs the base down first; once the
  // whole shard is fat products, the newest small run wins ties and the organ degrades gracefully
  // into tail maintenance. Two-pointer window, restarted at every fat boundary -- O(ns).
  var fhi: i64 = 0 - 1
  var flo: i64 = 0
  var runbytes: i64 = 0
  var bcnt: i64 = 0
  var wlo: i64 = 0
  var wrb: i64 = 0
  var idx: i64 = 0
  while idx < ns {
      if bszs[idx] * WSC_FAT_DIV >= budget {
          wlo = idx + 1
          wrb = 0
      } else {
          wrb = wrb + bszs[idx]
          while wrb > budget { wrb = wrb - bszs[wlo]; wlo = wlo + 1 }
          let wcnt: i64 = idx - wlo + 1
          if wcnt >= 2 { if wcnt >= bcnt { bcnt = wcnt; fhi = idx; flo = wlo; runbytes = wrb } }
      }
      idx = idx + 1
  }
  if fhi < 0 {
      wc9_puts("CONVERGED: no contiguous run of >=2 non-fat segments fits the budget -- nothing left to fold at this budget (streaming writer lifts this)\nverdict=PASS\n" as *u8)
      return 0
  }
  wc9_puts("fold-run: lo=" as *u8); wc9_num(flo); wc9_puts(" hi=" as *u8); wc9_num(fhi); wc9_puts(" fold=" as *u8); wc9_num(fhi - flo + 1)
  wc9_puts(" run-bytes=" as *u8); wc9_num(runbytes); wc9_puts(" total-bytes=" as *u8); wc9_num(total); wc9_puts("\n" as *u8)
250
  // PASS 1 over the RUN ONLY: count real entries (record framing walk, no fixed guess)
  var nent: i64 = 0
  var s0: i64 = flo
  while s0 <= fhi {
      let b0: *u8 = bptrs[s0] as *u8
      let sz0: i64 = bszs[s0]
      var i0: i64 = 0
      while i0 + 9 <= sz0 {
          let kl0: i64 = ss_r32(b0, i0 + 1)
          let vl0: i64 = ss_r32(b0, i0 + 5 + kl0)
          i0 = i0 + 5 + kl0 + 4 + vl0
          nent = nent + 1
      }
      s0 = s0 + 1
  }
  if nent > WSC_MAXENT { wc9_puts("entry count exceeds WSC_MAXENT sanity ceiling\nverdict=FAIL\n" as *u8); return 2 }
  let maxk: i64 = nent + 16
  // hash table = smallest power of two >= 4x entries (load factor <= 0.25)
  var hsize: i64 = WSC_MAGIC_1048576
  var hbits: i64 = 20
  while hsize < nent * 4 { hsize = hsize * 2; hbits = hbits + 1 }
  let hshift: i64 = 63 - hbits
  let hmask: i64 = hsize - 1
  wc9_puts("pass1 entries=" as *u8); wc9_num(nent); wc9_puts(" hash-slots=" as *u8); wc9_num(hsize); wc9_puts("\n" as *u8)
275
  // entry tables (dense) + hash slot -> entry-index (+1 so 0 = empty)
  let tkp: *i64 = sys_mmap(8 * maxk) as *i64
  let tkl: *i64 = sys_mmap(8 * maxk) as *i64
  let tkind: *i64 = sys_mmap(8 * maxk) as *i64
  let tvp: *i64 = sys_mmap(8 * maxk) as *i64
  let tvl: *i64 = sys_mmap(8 * maxk) as *i64
  let hslot: *i64 = sys_mmap(8 * hsize) as *i64
  var nk: i64 = 0
  var entries: i64 = 0
  s = flo
  while s <= fhi {
      let b: *u8 = bptrs[s] as *u8
      let sz: i64 = bszs[s]
      var i: i64 = 0
      while i + 9 <= sz {
          let kind: i64 = b[i]
          let kl: i64 = ss_r32(b, i + 1)
          let koff: i64 = i + 5
          let vl: i64 = ss_r32(b, koff + kl)
          let voff: i64 = koff + kl + 4
          let kp: *u8 = (b as i64 + koff) as *u8
          // hash-probe for this key (byte-verify on hit; O(1) expected)
          var slot: i64 = wsc_slot(kp, kl, hshift, hmask)
          var hit: i64 = 0 - 1
          var probing: i64 = 1
          while probing == 1 {
              let e: i64 = hslot[slot]
              if e == 0 { probing = 0 } else {
                  if ss_kcmp(tkp[e - 1] as *u8, tkl[e - 1], kp, kl) == 0 { hit = e - 1; probing = 0 } else {
                      slot = (slot + 1) & hmask
                  }
              }
          }
          if hit < 0 {
              if nk >= maxk { wc9_puts("key table overflow (impossible: data-driven bound)\nverdict=FAIL\n" as *u8); return 3 }
              hit = nk
              hslot[slot] = nk + 1
              nk = nk + 1
          }
          tkp[hit] = kp as i64
          tkl[hit] = kl
          tkind[hit] = kind
          tvp[hit] = (b as i64) + voff
          tvl[hit] = vl
          entries = entries + 1
          i = voff + vl
      }
      s = s + 1
  }
  wc9_puts("entries=" as *u8); wc9_num(entries); wc9_puts(" live-keys=" as *u8); wc9_num(nk)
  wc9_puts(" run-bytes-in=" as *u8); wc9_num(runbytes); wc9_puts("\n" as *u8)
327
  // merged segment (latest per key, insertion order = first-seen key order, matching ss_compact)
  let w: *i64 = ss_begin_cap(runbytes + WSC_MAGIC_65536)
  var t2: i64 = 0
  while t2 < nk {
      if ss_add2(w, tkind[t2], tkp[t2] as *u8, tkl[t2], tvp[t2] as *u8, tvl[t2]) != 0 { wc9_puts("writer overflow\nverdict=FAIL\n" as *u8); return 4 }
      t2 = t2 + 1
  }
  wc9_puts("merge-write: emitting seg (docs+idx+pos+imp)...\n" as *u8)
  if ss_write_seg(prefix, w, segid) != 0 { wc9_puts("segment write failed\nverdict=FAIL\n" as *u8); return 5 }
  wc9_puts("merged segment written\n" as *u8)
338
  // archive retired names BEFORE the swap (crash-safe ordering, identical to ss_compact)
  let ap: *u8 = sys_mmap(512)
  var ao: i64 = 0
  ao = ss_cat(ap, ao, prefix)
  ao = ss_cat(ap, ao, "manifest-archive.txt" as *u8)
  ap[ao] = 0 as u8
  let afd: i64 = sys_openat_append(ap, 0x1a4)
  if afd < 0 { wc9_puts("archive open failed\nverdict=FAIL\n" as *u8); return 6 }
  s = flo
  while s <= fhi {
      let nm2: *u8 = segs[s] as *u8
      sys_write(afd, nm2, ss_len(nm2))
      sys_write(afd, "\n" as *u8, 1)
      s = s + 1
  }
  sys_fsync(afd)
  sys_close(afd)
  // RE-READ THE MANIFEST UNDER THE PLANE LOCK, immediately before the swap. Crawler commits take
  // <prefix>slock, NOT plock, so rows CAN land while a fold runs -- the old rewrite silently dropped
  // them, orphaning freshly crawled segments (a lost-row class the seq349 note warned about).
  // Appended rows are strictly at the tail (nothing else rewrites while we hold plock: sibling
  // compactors block on plock, commits only append), so carrying segs2[ns..ns2-1] closes the loss
  // window to the instant between this read and the rename.
  let segs2: *i64 = sys_mmap(8 * WSC_MAXSEGS) as *i64
  var ns2: i64 = ss_manifest_cap(prefix, segs2, WSC_MAXSEGS)
  if ns2 < ns { ns2 = ns }
  // atomic manifest swap -> kept lines [0..flo-1] + merged + kept lines [fhi+1..ns-1] + any rows
  // appended past our snapshot (order preserved everywhere; the merged segment sits WHERE THE RUN SAT)
  let mf: *u8 = sys_mmap(512)
  let mt: *u8 = sys_mmap(512)
  var o2: i64 = 0
  o2 = ss_cat(mf, o2, prefix)
  o2 = ss_cat(mf, o2, "manifest.txt" as *u8)
  mf[o2] = 0 as u8
  o2 = 0
  o2 = ss_cat(mt, o2, prefix)
  o2 = ss_cat(mt, o2, "manifest.tmp" as *u8)
  mt[o2] = 0 as u8
  let nb: *u8 = sys_mmap(40 * (ns2 + 4) + 256)
  var no: i64 = 0
  var km: i64 = 0
  while km < flo {
      no = ss_cat(nb, no, segs[km] as *u8)
      nb[no] = 10 as u8
      no = no + 1
      km = km + 1
  }
  no = ss_cat(nb, no, "seg-" as *u8)
  no = ss_catn(nb, no, segid)
  nb[no] = 10 as u8
  no = no + 1
  km = fhi + 1
  while km < ns {
      no = ss_cat(nb, no, segs[km] as *u8)
      nb[no] = 10 as u8
      no = no + 1
      km = km + 1
  }
  var appended: i64 = 0
  km = ns
  while km < ns2 {
      no = ss_cat(nb, no, segs2[km] as *u8)
      nb[no] = 10 as u8
      no = no + 1
      km = km + 1
      appended = appended + 1
  }
  if appended > 0 { wc9_puts("carried " as *u8); wc9_num(appended); wc9_puts(" manifest row(s) appended by concurrent commits during the fold\n" as *u8) }
  if ss_writefile(mt, nb, no) != 0 { wc9_puts("manifest tmp write failed\nverdict=FAIL\n" as *u8); return 7 }
  if sys_renameat(mt, mf) != 0 { wc9_puts("manifest swap failed\nverdict=FAIL\n" as *u8); return 8 }
  ss_syncdir(prefix)
410
  // POST-SWAP VERIFY: reopen; sampled live keys from the folded run must RESOLVE via ss_hget
  // (stride 97 ~= 1% coverage floor + first/last always). STRICT byte-equality is only a valid
  // oracle when the run was the manifest SUFFIX and nothing landed during the fold -- for a MIDDLE
  // run, a kept NEWER segment legitimately shadows a folded key with different bytes, and treating
  // that as corruption would fail every healthy middle fold. So: a MISS is always bad (a key present
  // in the run must resolve to SOMETHING after the swap); a byte MISMATCH is bad only in strict mode,
  // otherwise counted as shadowed and reported. ss_open2(...,1) = mmap open: the read-all ss_open
  // pulls the WHOLE shard into anon RAM -- the same OOM class this rewrite exists to kill.
  var strict: i64 = 1
  if fhi != ns - 1 { strict = 0 }
  if appended > 0 { strict = 0 }
  let h2: *i64 = ss_open2(prefix, 1)
  if (h2 as i64) == 0 { wc9_puts("VERIFY reopen failed\nverdict=FAIL\n" as *u8); return 9 }
  let vp: *i64 = sys_mmap(16) as *i64
  let vl2: *i64 = sys_mmap(16) as *i64
  let kbuf: *u8 = sys_mmap(600)
  var bad: i64 = 0
  var shadowed: i64 = 0
  var checked: i64 = 0
  var t3: i64 = 0
  while t3 < nk {
      var pick: i64 = 0
      if t3 == 0 { pick = 1 }
      if t3 == nk - 1 { pick = 1 }
      if (t3 % 97) == 0 { pick = 1 }
      if pick == 1 { if tkind[t3] == 1 { if tkl[t3] < 590 {
          let sp2: *u8 = tkp[t3] as *u8
          var x: i64 = 0
          while x < tkl[t3] { kbuf[x] = sp2[x]; x = x + 1 }
          kbuf[tkl[t3]] = 0 as u8
          checked = checked + 1
          if ss_hget(h2, kbuf, vp, vl2) != 1 { bad = bad + 1 } else {
              var mism: i64 = 0
              if vl2[0] != tvl[t3] { mism = 1 } else {
                  let a2: *u8 = vp[0] as *u8
                  let b2: *u8 = tvp[t3] as *u8
                  var y: i64 = 0
                  while y < vl2[0] { if a2[y] != b2[y] { mism = 1; y = vl2[0] } else { y = y + 1 } }
              }
              if mism == 1 { if strict == 1 { bad = bad + 1 } else { shadowed = shadowed + 1 } }
          }
      } } }
      t3 = t3 + 1
  }
  wc9_puts("COMPACT done: segments " as *u8); wc9_num(ns2); wc9_puts(" -> " as *u8); wc9_num(ns2 - (fhi - flo + 1) + 1); wc9_puts(" (folded run -> seg-" as *u8); wc9_num(segid)
  wc9_puts(") live-keys=" as *u8); wc9_num(nk); wc9_puts(" strict=" as *u8); wc9_num(strict); wc9_puts(" shadowed=" as *u8); wc9_num(shadowed)
  wc9_puts(" verify-checked=" as *u8); wc9_num(checked); wc9_puts(" verify-bad=" as *u8); wc9_num(bad); wc9_puts("\n" as *u8)
  if bad > 0 {
      wc9_puts("VERIFY FAILED -- restore <prefix>manifest.txt from the retired list in manifest-archive.txt\nverdict=FAIL\n" as *u8)
      return 10
  }
  wc9_puts("verdict=PASS\n" as *u8)
  return 0
464}
nx_web_shard_compact.nx source

`nx_web_shard_compact.nx` source
