`nx_treepack.nx` source

↩ module page · 349 lines · 21559 B
1// nx_treepack.nx -- SOVEREIGN single-blob tree archive. Packs a directory tree into ONE .pack file (and unpacks
2// it) so a whole source tree (runtime/ = ~14,765 files / 72.6 MB) moves in ONE /api/upload instead of ~14,765
3// per-file SSH pushes (hours). THE keystone primitive for build-over-API + source-search-API (both need the
4// source tree on the NAS). Format (streaming, length-prefixed): "NXPK1\n" then repeat { u32 LE pathlen | path
5// (forward-slash, relative to pack root) | u64 LE size | size bytes }. Dirs are implicit (mkdir -p on unpack).
6// Additive-safe: pack reads only; unpack writes only under dest. Reuses nx_dr_tree's getdents walk pattern.
7// Buffers hoisted (shared cbuf/hbuf passed down; per-dir dbuf) -- no mmap in the per-file inner loops.
8//   nx_treepack pack   <src-dir>  <out.pack>
9//   nx_treepack unpack <in.pack>  <dest-dir>
10// license_tier: ORIGINAL
11import "nx_syscalls.nx"
12import "nx_symdrop.nx"   // sd_symbols / sd_has -- ONE definition of what a declaration is (rule 15)
13const K_MAGIC_3900: i64 = 3900
14const K_MAGIC_131072: i64 = 131072
15const K_MAGIC_1048576: i64 = 1048576
16const K_MAGIC_4000: i64 = 4000
17const K_MAGIC_4096: i64 = 4096
18
19func tp_w(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
20func tp_wn(v: i64) -> i64 { var t:i64=v; if t<0{sys_write(1,"-" as *u8,1);t=0-t} let tm:*u8=sys_mmap(24); var k:i64=0; if t==0{tm[0]=48 as u8;k=1} while t>0{tm[k]=(48+(t%10)) as u8;t=t/10;k=k+1} let b:*u8=sys_mmap(24); var j:i64=0; while j<k{b[j]=tm[k-1-j];j=j+1} sys_write(1,b,k); return 0 }
21
22// read EXACTLY n bytes (loop over short reads); returns bytes actually read (< n only at EOF).
23func tp_readn(fd: i64, buf: *u8, n: i64) -> i64 {
  var got: i64 = 0
  while got < n { let r: i64 = sys_read(fd, (buf as i64 + got) as *u8, n - got); if r <= 0 { return got } got = got + r }
  return got
27}
28func tp_put32(b: *u8, v: i64) -> i64 { b[0]=(v&0xff) as u8; b[1]=((v>>8)&0xff) as u8; b[2]=((v>>16)&0xff) as u8; b[3]=((v>>24)&0xff) as u8; return 0 }
29func tp_put64(b: *u8, v: i64) -> i64 { var i:i64=0; while i<8 { b[i]=((v>>(i*8))&0xff) as u8; i=i+1 } return 0 }
30func tp_get32(b: *u8) -> i64 { return (b[0] as i64) | ((b[1] as i64)<<8) | ((b[2] as i64)<<16) | ((b[3] as i64)<<24) }
31func tp_get64(b: *u8) -> i64 { var v:i64=0; var i:i64=0; while i<8 { v = v | ((b[i] as i64)<<(i*8)); i=i+1 } return v }
32
33func tp_isdotdot(nm: *u8) -> i64 { if nm[0]==(46 as u8){ if nm[1]==(0 as u8){return 1} if nm[1]==(46 as u8){ if nm[2]==(0 as u8){return 1} } } return 0 }
34// (canonical wrapper lives in nx_syscalls -- sys_unlinkat; kept as a thin alias for call-site clarity)
35func tp_unlink(path: *u8) -> i64 { return sys_unlinkat(path) }
36func tp_join(buf: *u8, base_n: i64, name: *u8) -> i64 { buf[base_n]=47 as u8; var o:i64=base_n+1; var i:i64=0; while name[i]!=(0 as u8){buf[o]=name[i];o=o+1;i=i+1} return o }
37func tp_size(path: *u8) -> i64 { let fd:i64=sys_openat_rd(path); if fd<0 {return 0-1} let s:i64=sys_lseek(fd,0,2); sys_close(fd); return s }
38
39// PACK walk: src[0..src_n) is the live path; rootlen = length of the original root dir (relpaths begin after
40// rootlen+1). hbuf(>=16)+cbuf(1MB) are shared scratch (used only at leaves, safe across recursion). dbuf is
41// per-call (each dir needs its own getdents buffer, since recursion happens inside the getdents loop).
42func tp_pack_walk(src: *u8, src_n: i64, rootlen: i64, packfd: i64, hbuf: *u8, cbuf: *u8, st: *i64) -> i64 {
  if src_n > K_MAGIC_3900 { return 0 }
  src[src_n] = 0 as u8
  let fd: i64 = sys_openat_rd(src); if fd < 0 { return 0 }
  let dbuf: *u8 = sys_mmap(K_MAGIC_131072)
  var go: i64 = 1
  while go == 1 {
      let nr: i64 = sys_getdents64(fd, dbuf, K_MAGIC_131072)
      if nr <= 0 { go = 0 } else {
          var off: i64 = 0
          while off < nr {
              let rec: *u8 = (dbuf as i64 + off) as *u8
              let ty: i64 = dirent_type(rec)
              let nm: *u8 = dirent_name(rec)
              if tp_isdotdot(nm) == 0 {
                  let cs: i64 = tp_join(src, src_n, nm)
                  if ty == 4 { st[1]=st[1]+1; tp_pack_walk(src, cs, rootlen, packfd, hbuf, cbuf, st) }
                  else {
                      src[cs] = 0 as u8
                      let sz: i64 = tp_size(src)
                      if sz < 0 { st[3]=st[3]+1 } else {
                          let rp: *u8 = (src as i64 + rootlen + 1) as *u8
                          let rlen: i64 = cs - rootlen - 1
                          tp_put32(hbuf, rlen); sys_write(packfd, hbuf, 4)
                          sys_write(packfd, rp, rlen)
                          tp_put64(hbuf, sz); sys_write(packfd, hbuf, 8)
                          let sfd: i64 = sys_openat_rd(src)
                          if sfd < 0 { st[3]=st[3]+1 } else {
                              var remain: i64 = sz
                              while remain > 0 {
                                  var want: i64 = K_MAGIC_1048576; if remain < want { want = remain }
                                  let n2: i64 = sys_read(sfd, cbuf, want)
                                  if n2 <= 0 { remain = 0 } else { sys_write(packfd, cbuf, n2); remain = remain - n2 }
                              }
                              sys_close(sfd)
                              st[0]=st[0]+1; st[2]=st[2]+sz
                          }
                      }
                  }
              }
              off = off + dirent_reclen(rec)
          }
      }
  }
  sys_close(fd)
  return 0
88}
89
90// mkdir every '/'-delimited prefix of fpath BELOW dest_n (so nested relpaths get their parent dirs).
91func tp_mkparents(fpath: *u8, dest_n: i64) -> i64 {
  var i: i64 = dest_n + 1
  while fpath[i] != (0 as u8) {
      if fpath[i] == (47 as u8) { fpath[i]=0 as u8; sys_mkdir(fpath, 0x1ed); fpath[i]=47 as u8 }
      i = i + 1
  }
  return 0
98}
99
100// seq1379 half-2 -- REFUSE TO BACKDATE. An unpack MERGES over the destination, so pushing a stale tree
101// silently reverted canonical sources on 2026-07-30 and re-opened FIVE eaten debts across THREE lanes
102// with no error anywhere (seq1370/1375). The rule that fixes it needs no policy and no config:
103//   A DESTINATION FILE NEWER THAN THE PACK ITSELF CANNOT HAVE COME FROM THAT PACK.
104// So such a file is SKIPPED, COUNTED and NAMED rather than overwritten. `force` overrides deliberately.
105// This was unwritable until 2026-07-30 because the tree had NO way to read an mtime at all (sys_stat and
106// statx: 0 matches in 19,813 files) -- the primitive below is declared LOCALLY via __syscall (the
107// nx_movie_probe precedent) so nx_syscalls.nx, at ~14k importers, stays untouched. Offsets are VERIFIED
108// byte-exact against `stat -c '%Y %s'`, not assumed: st_mtim.tv_sec@88, st_size@48 in x86-64 struct stat.
109// ---- seq1467: content-regression refusal support -------------------------------------------------
110// Buffers + the .nx test used by the REFUSED-WOULD-DROP-SYMBOLS rule in tp_unpack. The symbol scanner
111// itself is NOT re-implemented here -- sd_symbols/sd_has are imported from nx_symdrop (rule 15): the
112// detector that finds these reverts after the fact and the guard that prevents them must agree by
113// CONSTRUCTION, because two copies of "what counts as a declaration" would disagree exactly when it
114// matters and one of them would be silently wrong.
115const TP_SYMBUF: i64 = 4194304    // largest .nx source compared whole (nx_hostctl.nx ~281KB; 14x headroom)
116const TP_SYMCAP: i64 = 1048576    // symbol csv per side
117const TP_DOT:    i64 = 46
118const TP_NAMEMAX: i64 = 256
119
120static tp_sbuf:  *u8    // incoming entry, buffered so it can be compared then written
121static tp_dbuf2: *u8    // destination file
122static tp_isym:  *u8    // incoming symbol csv
123static tp_dsym:  *u8    // destination symbol csv
124static tp_mnm:   *u8    // one symbol name
125
126func tp_sym_init() -> i64 {
  if (tp_sbuf as i64) == 0 {
      tp_sbuf  = sys_mmap(TP_SYMBUF)
      tp_dbuf2 = sys_mmap(TP_SYMBUF)
      tp_isym  = sys_mmap(TP_SYMCAP)
      tp_dsym  = sys_mmap(TP_SYMCAP)
      tp_mnm   = sys_mmap(TP_NAMEMAX)
  }
  return 0
135}
136// does this pack-relative path end in ".nx"? Only sources get the content check: a lost declaration in
137// a source is lost capability, whereas data/blob files legitimately shrink.
138func tp_is_nx(p: *u8) -> i64 {
  var n: i64 = 0
  while p[n] != (0 as u8) { n = n + 1 }
  if n < 3 { return 0 }
  if p[n-3] != (TP_DOT as u8) { return 0 }
  if p[n-2] != (110 as u8) { return 0 }
  if p[n-1] != (120 as u8) { return 0 }
  return 1
146}
147// bounded whole-file read; -1 when absent
148func tp_read_file(path: *u8, b: *u8, cap: i64) -> i64 {
  let fd: i64 = sys_openat_rd(path)
  if fd < 0 { return 0 - 1 }
  var n: i64 = 0
  var go: i64 = 1
  while go == 1 {
      let r: i64 = sys_read(fd, (b as i64 + n) as *u8, cap - n)
      if r > 0 { n = n + r } else { go = 0 }
      if n >= cap { go = 0 }
  }
  sys_close(fd)
  return n
160}
161
162const TP_SYS_FSTAT: i64 = 5
163const TP_SYS_NEWFSTATAT: i64 = 262
164const TP_AT_FDCWD: i64 = 0 - 100
165const TP_STATBUF: i64 = 256
166const TP_OFF_MTIME: i64 = 88
167
168// mtime of an OPEN fd, or 0 when unavailable. 0 is the FAIL-SAFE value: it makes every dest look newer
169// than the pack, which would skip everything loudly rather than overwrite anything silently.
170func tp_fd_mtime(fd: i64) -> i64 {
  let sb: *u8 = sys_mmap(TP_STATBUF)
  if __syscall(TP_SYS_FSTAT, fd, sb as i64, 0, 0, 0, 0) != 0 { return 0 }
  let mp: *i64 = ((sb as i64) + TP_OFF_MTIME) as *i64
  return mp[0]
175}
176// mtime of a PATH, or -1 when the file is absent (absent => free to create, never a backdate)
177func tp_path_mtime(path: *u8) -> i64 {
  let sb: *u8 = sys_mmap(TP_STATBUF)
  if __syscall(TP_SYS_NEWFSTATAT, TP_AT_FDCWD, path as i64, sb as i64, 0, 0, 0) != 0 { return 0 - 1 }
  let mp: *i64 = ((sb as i64) + TP_OFF_MTIME) as *i64
  return mp[0]
182}
183
184// UNPACK: fpath is prebuilt as dest + '/' (relpath is written after dest_n+1). Streams each file out.
185func tp_unpack(packfd: i64, fpath: *u8, dest_n: i64, hbuf: *u8, cbuf: *u8, st: *i64, packmt: i64, force: i64) -> i64 {
  tp_sym_init()
  let mg: i64 = tp_readn(packfd, hbuf, 6)
  if mg < 6 { tp_w("BAD: truncated (no magic)\n" as *u8); return 0-1 }
  if hbuf[0]!=(78 as u8) { tp_w("BAD: not an NXPK archive\n" as *u8); return 0-1 }
  var go: i64 = 1
  while go == 1 {
      let g1: i64 = tp_readn(packfd, hbuf, 4)
      if g1 < 4 { go = 0 } else {
          let plen: i64 = tp_get32(hbuf)
          if plen <= 0 { go = 0 } else { if plen > K_MAGIC_4000 { tp_w("BAD: pathlen overflow\n" as *u8); go = 0 } else {
              let rp: *u8 = (fpath as i64 + dest_n + 1) as *u8
              tp_readn(packfd, rp, plen)
              rp[plen] = 0 as u8
              let g2: i64 = tp_readn(packfd, hbuf, 8)
              if g2 < 8 { go = 0 } else {
                  let sz: i64 = tp_get64(hbuf)
                  tp_mkparents(fpath, dest_n)
                  // overwrite-robust: an existing file can deny O_TRUNC (mixed-ownership buildroot, EACCES/ETXTBSY
                  // -- the seq142/seq124 stale-source root). Unlink-then-recreate wins wherever the DIR is writable.
                  // seq1379: a dest NEWER than the pack cannot have come from it -> skip, count, NAME it.
                  // The bytes are still consumed so the stream stays aligned (same discipline as the fail path).
                  // seq1379 half-2: a dest NEWER than the pack cannot have come from it -> skip, count, NAME.
                  // seq1467 (2026-07-30): that rule alone is NOT enough, and the hole is exactly how five
                  // debts got re-opened. It compares the dest against the PACK'S mtime, so it only catches
                  // REPLAYING AN OLD PACK. Packing a STALE TREE *now* produces a pack whose mtime is the
                  // newest thing in the system -- every destination then looks older, nothing is refused, and
                  // 07-29 content silently overwrote 07-30 sources (measured: NAS nx_syscalls.nx replaced by
                  // the laptop copy, erasing sys_ignore_sigpipe while three files still CALLED it, leaving the
                  // tree unbuildable; my parser guarantee went the same way in the same push).
                  // TIMESTAMPS CANNOT DECIDE THIS. Content can: an incoming source that DROPS top-level
                  // declarations the destination already has is a REGRESSION regardless of any mtime, and that
                  // test is immune to clock skew, touch, copy-preserved times and freshly-made packs alike.
                  // Applied to .nx sources only (where a lost symbol is lost capability), bounded by buffer
                  // size, and `force` still overrides deliberately.
                  var backdate: i64 = 0
                  var why: i64 = 0
                  if force == 0 { if tp_path_mtime(fpath) > packmt { backdate = 1; why = 1 } }
                  // CONTENT-REGRESSION check. Buffer the entry so the same bytes can be compared and then
                  // written -- the stream is consumed exactly once either way, so alignment is preserved on
                  // every path (the same discipline the skip path already keeps).
                  var buffered: i64 = 0
                  if backdate == 0 { if force == 0 { if tp_is_nx(rp) == 1 { if sz > 0 { if sz < TP_SYMBUF {
                      if tp_path_mtime(fpath) >= 0 {
                          if tp_readn(packfd, tp_sbuf, sz) == sz {
                              buffered = 1
                              let dn2: i64 = tp_read_file(fpath, tp_dbuf2, TP_SYMBUF)
                              if dn2 > 0 {
                                  let li: i64 = sd_symbols(tp_sbuf, sz, tp_isym, TP_SYMCAP)
                                  let ld: i64 = sd_symbols(tp_dbuf2, dn2, tp_dsym, TP_SYMCAP)
                                  var miss: i64 = 0
                                  var ms: i64 = 0
                                  var mk: i64 = 0
                                  while mk <= ld {
                                      var msep: i64 = 0
                                      if mk == ld { msep = 1 } else { if tp_dsym[mk] == (44 as u8) { msep = 1 } }
                                      if msep == 1 {
                                          let mnl: i64 = mk - ms
                                          if mnl > 0 { if mnl < 200 {
                                              var mz: i64 = 0
                                              while mz < mnl { tp_mnm[mz] = tp_dsym[ms+mz]; mz = mz + 1 }
                                              tp_mnm[mnl] = 0 as u8
                                              if sd_has(tp_isym, li, tp_mnm, mnl) == 0 { miss = miss + 1 }
                                          } }
                                          ms = mk + 1
                                      }
                                      mk = mk + 1
                                  }
                                  if miss > 0 { backdate = 1; why = 2 }
                              }
                          }
                      }
                  } } } } }
                  if backdate == 1 {
                      st[5]=st[5]+1
                      if st[6]<8 {
                          st[6]=st[6]+1
                          if why == 2 { tp_w("REFUSED-WOULD-DROP-SYMBOLS: " as *u8) } else { tp_w("SKIP-NEWER-THAN-PACK: " as *u8) }
                          tp_w(fpath); tp_w("\n" as *u8)
                      }
                      if buffered == 0 {
                          var br: i64 = sz
                          while br > 0 { var bw: i64=K_MAGIC_1048576; if br<bw {bw=br} let n4: i64=tp_readn(packfd,cbuf,bw); if n4<=0 {br=0} else {br=br-n4} }
                      }
                  } else {
                      var ofd: i64 = sys_openat_wr(fpath, 0x1a4)
                      if ofd < 0 { tp_unlink(fpath); ofd = sys_openat_wr(fpath, 0x1a4) }
                      if ofd < 0 {
                          st[3]=st[3]+1
                          if st[4]<5 { st[4]=st[4]+1; tp_w("UNPACK-FAIL: " as *u8); tp_w(fpath); tp_w("\n" as *u8) }
                          if buffered == 0 {
                              var rr: i64 = sz
                              while rr > 0 { var w: i64=K_MAGIC_1048576; if rr<w {w=rr} let n3: i64=tp_readn(packfd,cbuf,w); if n3<=0 {rr=0} else {rr=rr-n3} }
                          }
                      } else {
                          if buffered == 1 {
                              sys_write(ofd, tp_sbuf, sz)
                              sys_close(ofd)
                              st[0]=st[0]+1; st[2]=st[2]+sz
                          } else {
                              var remain: i64 = sz
                              while remain > 0 { var want: i64=K_MAGIC_1048576; if remain<want {want=remain} let n2: i64=tp_readn(packfd,cbuf,want); if n2<=0 {remain=0} else { sys_write(ofd,cbuf,n2); remain=remain-n2 } }
                              sys_close(ofd)
                              st[0]=st[0]+1; st[2]=st[2]+sz
                          }
                      }
                  }
              }
          } }
      }
  }
  return 0
297}
298
299func main(argc: i64, argv: *i64) -> i64 {
  if argc < 4 { tp_w("usage: nx_treepack pack <src-dir> <out.pack>  |  nx_treepack unpack <in.pack> <dest-dir>\n" as *u8); sys_exit(2); return 2 }
  let mode: *u8 = argv[1] as *u8
  let a1: *u8 = argv[2] as *u8
  let a2: *u8 = argv[3] as *u8
  let hbuf: *u8 = sys_mmap(64)
  let cbuf: *u8 = sys_mmap(K_MAGIC_1048576)
  let st: *i64 = sys_mmap(64) as *i64; st[0]=0; st[1]=0; st[2]=0; st[3]=0; st[4]=0
  if mode[0]==(112 as u8) {
      let src: *u8 = sys_mmap(K_MAGIC_4096); var sn: i64=0; var i: i64=0; while a1[i]!=(0 as u8){src[sn]=a1[i];sn=sn+1;i=i+1}
      if sn>0 { if src[sn-1]==(47 as u8) { sn=sn-1 } }
      let packfd: i64 = sys_openat_wr(a2, 0x1a4)
      if packfd < 0 { tp_w("FAIL: cannot open pack output\n" as *u8); sys_exit(1); return 1 }
      hbuf[0]=78 as u8; hbuf[1]=88 as u8; hbuf[2]=80 as u8; hbuf[3]=75 as u8; hbuf[4]=49 as u8; hbuf[5]=10 as u8
      sys_write(packfd, hbuf, 6)
      tp_pack_walk(src, sn, sn, packfd, hbuf, cbuf, st)
      sys_close(packfd)
      tp_w("=== nx_treepack PACK: files=" as *u8); tp_wn(st[0]); tp_w(" dirs=" as *u8); tp_wn(st[1]); tp_w(" bytes=" as *u8); tp_wn(st[2]); tp_w(" failed=" as *u8); tp_wn(st[3]); tp_w(" -> " as *u8); tp_w(a2); tp_w("\n" as *u8)
      if st[3]==0 { sys_exit(0); return 0 }
      sys_exit(1); return 1
  }
  if mode[0]==(117 as u8) {
      let packfd: i64 = sys_openat_rd(a1)
      if packfd < 0 { tp_w("FAIL: cannot open pack\n" as *u8); sys_exit(1); return 1 }
      let fpath: *u8 = sys_mmap(K_MAGIC_4096); var dn: i64=0; var i: i64=0; while a2[i]!=(0 as u8){fpath[dn]=a2[i];dn=dn+1;i=i+1}
      if dn>0 { if fpath[dn-1]==(47 as u8) { dn=dn-1 } }
      sys_mkdir(a2, 0x1ed)
      fpath[dn]=47 as u8
      // seq1379 half-2: the pack's OWN mtime is the reference clock. `force` (argv[4]) overrides.
      let packmt: i64 = tp_fd_mtime(packfd)
      var force: i64 = 0
      if argc >= 5 { let f: *u8 = argv[4] as *u8; if f[0]==(102 as u8) { force = 1 } }
      tp_unpack(packfd, fpath, dn, hbuf, cbuf, st, packmt, force)
      sys_close(packfd)
      tp_w("=== nx_treepack UNPACK: files=" as *u8); tp_wn(st[0]); tp_w(" bytes=" as *u8); tp_wn(st[2]); tp_w(" failed=" as *u8); tp_wn(st[3]); tp_w(" skipped-newer=" as *u8); tp_wn(st[5]); tp_w(" -> " as *u8); tp_w(a2); tp_w("\n" as *u8)
      if st[5] > 0 { tp_w("NOTE: PRESERVED, not overwritten. Each line above says which rule fired: SKIP-NEWER-THAN-PACK = the destination is newer than the pack, so it cannot have come from it (seq1379); REFUSED-WOULD-DROP-SYMBOLS = the incoming source declares FEWER top-level symbols than the file already on disk, i.e. a stale tree would delete landed work (seq1467) -- that one is decided on CONTENT, so a freshly-made pack of an old tree cannot slip past it. Re-pack from current sources, or pass a 4th arg `force` to overwrite deliberately.
335" as *u8) }
      // seq1519 -- A REFUSAL NOBODY SEES IS NOT A GUARD. This exited 0 whenever nothing FAILED, so a run
      // that PRESERVED files (and therefore did NOT deliver them) reported success; nx_ship swallowed it
      // and a sibling shipped a repair that never landed, leaving a truncated source and three gates
      // unable to compile. A skip is an INCOMPLETE UNPACK, not a success: it now exits 5 -- distinct from
      // 1 (write failure) so a wrapper can tell "some files were withheld on purpose" from "I/O broke".
      if st[3] == 0 { if st[5] > 0 {
          tp_w("UNPACK-INCOMPLETE skipped-newer=" as *u8); tp_wn(st[5])
          tp_w(" -- files were WITHHELD, so this unpack did NOT deliver the pack. exit=5; wrappers MUST surface this, never treat it as success. If the on-disk file is CORRUPT rather than newer-and-good (mtime says nothing about content), re-run with a 4th arg `force`.\n" as *u8)
          sys_exit(5); return 5
      } sys_exit(0); return 0 }
      sys_exit(1); return 1
  }
  tp_w("unknown mode (use pack or unpack)\n" as *u8); sys_exit(2); return 2
349}
nx_treepack.nx source

`nx_treepack.nx` source
