code wiki / (root) / nx_safe_archive_ingest.nx
`nx_safe_archive_ingest.nx` source

↩ module page · 350 lines · 18619 B
1// nx_safe_archive_ingest.nx -- SAFE sovereign ingestion of an UNTRUSTED .zip archive into the
2// permanent library.  Composes the shipped zip + deflate primitives (rule 15 DRY) under a
3// NEVER-POISON-THE-LIBRARY safety envelope -- the #26 NEVER-BRICK law applied to ingestion:
4// an untrusted archive can NEVER, BY CONSTRUCTION:
5//   (1) write outside the dest dir         -- Zip-Slip-proof: the stored path is NEVER used as a
6//                                              filesystem path; output is dest + "/" + a sanitized
7//                                              BASENAME (chars mapped to [A-Za-z0-9._-], ".." rejected).
8//   (2) exhaust memory/disk                 -- zip-bomb caps: per-entry uncompressed cap, total cap,
9//                                              compression-ratio cap, entry-count cap, archive cap.
10//   (3) be executed                         -- data-only: files written 0644 (no exec bit), content is
11//                                              sanitized to printable text; nothing is ever run/chmod+x.
12//   (4) smuggle an unknown codec            -- method whitelist: 0 (stored) / 8 (deflate) only.
13//   (5) smuggle a binary/script as library  -- content-type whitelist by extension; non-text SKIPPED
14//                                              (never banked); even "text" is stripped of control bytes
15//                                              (defuses terminal-escape injection).
16//   (6) link out via symlink                -- entries with unix S_IFLNK mode are SKIPPED.
17//   (7) bomb via nested archive             -- we do NOT recurse; a .zip entry is non-text -> SKIPPED.
18// Zip64-sentinel sizes are SKIPPED (not yet handled = refuse, never guess).
19//
20// usage: nx_safe_archive_ingest <archive.zip> [dest-dir]   (default dest = knowledge/library)
21// exit 0 always on a well-formed run (per-entry verdicts on stdout); nonzero only on unreadable
22// archive / not-a-zip.  Rule 11: every cap is a named const (data-driven). license_tier: ORIGINAL
23import "nx_syscalls.nx"
24import "nx_zip_header.nx"
25import "nx_deflate.nx"
26import "nx_pdf_text.nx"
27const SAI_MAGIC_2048: i64 = 2048
28const SAI_MAGIC_16777216: i64 = 16777216
29
30// ---- safety caps (data-driven; tune in one place) ----
31const SAI_MAX_ARCHIVE:     i64 = 67108864     // 64 MiB total archive bytes
32const SAI_MAX_ENTRIES:     i64 = 4096
33const SAI_MAX_ENTRY_USIZE: i64 = 33554432     // 32 MiB uncompressed per entry
34const SAI_MAX_TOTAL_USIZE: i64 = 268435456    // 256 MiB uncompressed total
35const SAI_MAX_RATIO:       i64 = 200          // uncompressed/compressed ceiling (zip-bomb guard)
36const SAI_CDFH_SIG:        i64 = 0x02014b50
37
38const SAI_BANKED:   i64 = 1
39const SAI_SKIPPED:  i64 = 2
40const SAI_REJECTED: i64 = 3
41
42func sai_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
43func sai_putn(v: i64) -> i64 {
  if v == 0 { sys_write(1, "0" as *u8, 1); return 0 }
  var m: i64 = v
  if m < 0 { sys_write(1, "-" as *u8, 1); m = 0 - m }
  let d: *u8 = sys_mmap(24); var k: i64 = 0
  while m > 0 { d[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
  var j: i64 = k - 1
  while j >= 0 { sys_write(1, ((d as i64)+j) as *u8, 1); j = j - 1 }
  return 0
52}
53// write the raw stored name (for the audit log only -- NEVER used as a path)
54func sai_put_name(buf: *u8, off: i64, len: i64) -> i64 {
  var i: i64 = 0
  while i < len { let o: *u8 = sys_mmap(1); o[0] = buf[off + i]; sys_write(1, o, 1); i = i + 1 }
  return 0
58}
59
60func sai_lc(c: i64) -> i64 { if c >= 0x41 { if c <= 0x5a { return c + 0x20 } } return c }
61
62// is byte a member of the safe basename charset [A-Za-z0-9._-] ?
63func sai_safe_char(c: i64) -> i64 {
  if c >= 0x41 { if c <= 0x5a { return 1 } }   // A-Z
  if c >= 0x61 { if c <= 0x7a { return 1 } }   // a-z
  if c >= 0x30 { if c <= 0x39 { return 1 } }   // 0-9
  if c == 0x2e { return 1 }                    // .
  if c == 0x5f { return 1 }                    // _
  if c == 0x2d { return 1 }                    // -
  return 0
71}
72
73// ZIP-SLIP-PROOF basename: take ONLY the segment after the last '/' or '\', map every char NOT in
74// [A-Za-z0-9._-] to '_', reject empty / "." / "..".  Returns out length, or 0 to reject.
75func sai_safe_basename(name: *u8, off: i64, len: i64, out: *u8, cap: i64) -> i64 {
  if len <= 0 { return 0 }
  var start: i64 = 0
  var i: i64 = 0
  while i < len {
      let c: i64 = name[off + i] as i64
      if c == 0x2f { start = i + 1 }           // '/'
      if c == 0x5c { start = i + 1 }           // '\'
      i = i + 1
  }
  var blen: i64 = len - start
  if blen <= 0 { return 0 }
  if blen > cap - 1 { blen = cap - 1 }
  var o: i64 = 0
  var j: i64 = 0
  while j < blen {
      var c: i64 = name[off + start + j] as i64
      if sai_safe_char(c) == 0 { c = 0x5f }    // map anything unsafe -> '_'
      out[o] = c as u8
      o = o + 1
      j = j + 1
  }
  out[o] = 0 as u8
  if o == 0 { return 0 }
  if o == 1 { if out[0] == (0x2e as u8) { return 0 } }                                  // "."
  if o == 2 { if out[0] == (0x2e as u8) { if out[1] == (0x2e as u8) { return 0 } } }    // ".."
  return o
102}
103
104// case-insensitive: does base[start..end) equal NUL-terminated lit (lit lowercase) exactly?
105func sai_ext_eq(b: *u8, start: i64, end: i64, lit: *u8) -> i64 {
  var i: i64 = 0
  while lit[i] != (0 as u8) {
      if start + i >= end { return 0 }
      if sai_lc(b[start + i] as i64) != (lit[i] as i64) { return 0 }
      i = i + 1
  }
  if start + i != end { return 0 }
  return 1
114}
115
116// SKIP editor/OS cruft (AppleDouble "._*") -- never real standard content.
117func sai_name_is_cruft(b: *u8, blen: i64) -> i64 {
  if blen >= 2 { if b[0] == (0x2e as u8) { if b[1] == (0x5f as u8) { return 1 } } }
  return 0
120}
121// SKIP-LIST of execution-associated AND known-binary/media extensions.  Execution types are skipped
122// for defense-in-depth (we never execute anything regardless); binary/media types are skipped pre-
123// inflate for efficiency.  Anything NOT on this list is inflated and content-sniffed, so genuine
124// source/text (e.g. the .vhdl standard library) is captured while blobs are refused.
125func sai_is_skip_ext(b: *u8, blen: i64) -> i64 {
  var dot: i64 = 0 - 1
  var i: i64 = 0
  while i < blen { if b[i] == (0x2e as u8) { dot = i } i = i + 1 }
  if dot < 0 { return 0 }
  let s: i64 = dot + 1
  // executables / scripts
  if sai_ext_eq(b, s, blen, "sh" as *u8) == 1 { return 1 }
  if sai_ext_eq(b, s, blen, "bash" as *u8) == 1 { return 1 }
  if sai_ext_eq(b, s, blen, "zsh" as *u8) == 1 { return 1 }
  if sai_ext_eq(b, s, blen, "bat" as *u8) == 1 { return 1 }
  if sai_ext_eq(b, s, blen, "cmd" as *u8) == 1 { return 1 }
  if sai_ext_eq(b, s, blen, "com" as *u8) == 1 { return 1 }
  if sai_ext_eq(b, s, blen, "ps1" as *u8) == 1 { return 1 }
  if sai_ext_eq(b, s, blen, "vbs" as *u8) == 1 { return 1 }
  if sai_ext_eq(b, s, blen, "exe" as *u8) == 1 { return 1 }
  if sai_ext_eq(b, s, blen, "dll" as *u8) == 1 { return 1 }
  if sai_ext_eq(b, s, blen, "scr" as *u8) == 1 { return 1 }
  if sai_ext_eq(b, s, blen, "msi" as *u8) == 1 { return 1 }
  if sai_ext_eq(b, s, blen, "jar" as *u8) == 1 { return 1 }
  if sai_ext_eq(b, s, blen, "so" as *u8) == 1 { return 1 }
  if sai_ext_eq(b, s, blen, "dylib" as *u8) == 1 { return 1 }
  // binary / media / archives (skip pre-inflate). NOTE: pdf is NOT here -- PDFs are text-extracted
  // by nx_pdf_text (sovereign) and banked as text, not skipped.
  if sai_ext_eq(b, s, blen, "png" as *u8) == 1 { return 1 }
  if sai_ext_eq(b, s, blen, "jpg" as *u8) == 1 { return 1 }
  if sai_ext_eq(b, s, blen, "jpeg" as *u8) == 1 { return 1 }
  if sai_ext_eq(b, s, blen, "gif" as *u8) == 1 { return 1 }
  if sai_ext_eq(b, s, blen, "bmp" as *u8) == 1 { return 1 }
  if sai_ext_eq(b, s, blen, "ico" as *u8) == 1 { return 1 }
  if sai_ext_eq(b, s, blen, "zip" as *u8) == 1 { return 1 }
  if sai_ext_eq(b, s, blen, "gz" as *u8) == 1 { return 1 }
  if sai_ext_eq(b, s, blen, "tar" as *u8) == 1 { return 1 }
  if sai_ext_eq(b, s, blen, "7z" as *u8) == 1 { return 1 }
  if sai_ext_eq(b, s, blen, "rar" as *u8) == 1 { return 1 }
  if sai_ext_eq(b, s, blen, "bin" as *u8) == 1 { return 1 }
  if sai_ext_eq(b, s, blen, "doc" as *u8) == 1 { return 1 }
  if sai_ext_eq(b, s, blen, "docx" as *u8) == 1 { return 1 }
  if sai_ext_eq(b, s, blen, "xls" as *u8) == 1 { return 1 }
  if sai_ext_eq(b, s, blen, "xlsx" as *u8) == 1 { return 1 }
  return 0
166}
167// Does the safe basename end in ".pdf"? (PDFs are routed to nx_pdf_text for sovereign text extraction.)
168func sai_ext_is_pdf(b: *u8, blen: i64) -> i64 {
  var dot: i64 = 0 - 1
  var i: i64 = 0
  while i < blen { if b[i] == (0x2e as u8) { dot = i } i = i + 1 }
  if dot < 0 { return 0 }
  if sai_ext_eq(b, dot + 1, blen, "pdf" as *u8) == 1 { return 1 }
  return 0
175}
176// CONTENT SNIFF: bank only if the decompressed bytes are predominantly text.  "Hard-binary" bytes
177// (NUL + C0 controls except tab/newline/CR) almost never occur in source/text but saturate PDFs/
178// images/executables.  >1% hard-binary -> treat as a blob -> SKIP (never bank a binary).
179func sai_is_text_content(c: *u8, len: i64) -> i64 {
  if len <= 0 { return 0 }
  var bad: i64 = 0
  var i: i64 = 0
  while i < len {
      let x: i64 = c[i] as i64
      if x < 0x20 {
          if x != 0x09 { if x != 0x0a { if x != 0x0d { bad = bad + 1 } } }
      }
      i = i + 1
  }
  if (bad * 100) > len { return 0 }
  return 1
192}
193
194func sai_scat(dst: *u8, o: i64, s: *u8, slen: i64) -> i64 { var i: i64 = 0; while i < slen { dst[o + i] = s[i]; i = i + 1 } return o + i }
195func sai_strlen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
196
197// sanitize content to PRINTABLE text (keep 0x20-0x7e, \t \n \r; drop everything else) and write it
198// 0644 to dest/<base>.  The output path is OUR controlled (dest) + sanitized basename: an entry named
199// "../../etc/passwd" can only ever land at dest/passwd.  No exec bit, ever.
200func sai_bank_text(dest: *u8, destlen: i64, base: *u8, blen: i64, content: *u8, clen: i64) -> i64 {
  let path: *u8 = sys_mmap(SAI_MAGIC_2048)
  var o: i64 = sai_scat(path, 0, dest, destlen)
  path[o] = 0x2f as u8; o = o + 1
  o = sai_scat(path, o, base, blen)
  path[o] = 0 as u8
  let txt: *u8 = sys_mmap(clen + 16)
  var ti: i64 = 0
  var i: i64 = 0
  while i < clen {
      let c: i64 = content[i] as i64
      var keep: i64 = 0
      if c >= 0x20 { if c <= 0x7e { keep = 1 } }
      if c == 0x09 { keep = 1 }
      if c == 0x0a { keep = 1 }
      if c == 0x0d { keep = 1 }
      if keep == 1 { txt[ti] = c as u8; ti = ti + 1 }
      i = i + 1
  }
  let fd: i64 = sys_openat_wr(path, 0x1a4)    // 0644 -- NO execute bit
  if fd < 0 { return 0 }
  var off: i64 = 0
  while off < ti { let w: i64 = sys_write(fd, ((txt as i64) + off) as *u8, ti - off); if w <= 0 { sys_close(fd); return 0 } off = off + w }
  sys_close(fd)
  return 1
225}
226
227// process ONE central-directory entry at CDFH offset `off`.  Returns a SAI_* verdict; updates
228// running total via tot_p.  Every dangerous shape returns SKIPPED/REJECTED BEFORE any write/inflate.
229func sai_process_entry(buf: *u8, n: i64, off: i64, dest: *u8, destlen: i64, tot_p: *i64) -> i64 {
  let method:   i64 = zip_read_u16(buf, off + 10)
  let csize:    i64 = zip_read_u32(buf, off + 20)
  let usize:    i64 = zip_read_u32(buf, off + 24)
  let nlen:     i64 = zip_read_u16(buf, off + 28)
  let elen:     i64 = zip_read_u16(buf, off + 30)
  let clen:     i64 = zip_read_u16(buf, off + 32)
  let ext_attr: i64 = zip_read_u32(buf, off + 38)
  let lfh_off:  i64 = zip_read_u32(buf, off + 42)
  let name_off: i64 = off + 46
239
  // (6) symlink (unix S_IFLNK = 0xA000 in the high word of external attrs)
  let umode: i64 = (ext_attr >> 16) & 0xffff
  if (umode & 0xf000) == 0xa000 { sai_puts("  SKIP symlink           name="); sai_put_name(buf, name_off, nlen); sai_puts("\n"); return SAI_SKIPPED }
  // refuse Zip64-sentinel sizes (never guess a size)
  if usize == ZIP64_MARKER { sai_puts("  SKIP zip64-usize\n"); return SAI_SKIPPED }
  if csize == ZIP64_MARKER { sai_puts("  SKIP zip64-csize\n"); return SAI_SKIPPED }
  // (4) method whitelist: stored(0) / deflate(8)
  if method != 0 { if method != 8 { sai_puts("  SKIP method="); sai_putn(method); sai_puts("\n"); return SAI_SKIPPED } }
  // (2) zip-bomb caps
  if usize > SAI_MAX_ENTRY_USIZE { sai_puts("  REJECT entry-too-big usize="); sai_putn(usize); sai_puts("\n"); return SAI_REJECTED }
  if (tot_p[0] + usize) > SAI_MAX_TOTAL_USIZE { sai_puts("  REJECT total-too-big\n"); return SAI_REJECTED }
  if csize > 0 { if (usize / csize) > SAI_MAX_RATIO { sai_puts("  REJECT ratio="); sai_putn(usize / csize); sai_puts(":1 (zip-bomb)\n"); return SAI_REJECTED }
  }
  // (1) Zip-Slip-proof basename
  let base: *u8 = sys_mmap(512)
  let blen: i64 = sai_safe_basename(buf, name_off, nlen, base, 512)
  if blen == 0 { sai_puts("  SKIP unsafe/empty-name  raw="); sai_put_name(buf, name_off, nlen); sai_puts("\n"); return SAI_SKIPPED }
  // (5a) skip OS/editor cruft + execution/binary extensions BEFORE inflating
  if sai_name_is_cruft(base, blen) == 1 { sai_puts("  SKIP cruft              name="); sai_puts(base); sai_puts("\n"); return SAI_SKIPPED }
  if sai_is_skip_ext(base, blen) == 1 { sai_puts("  SKIP binary/exec-ext    name="); sai_puts(base); sai_puts("\n"); return SAI_SKIPPED }
  // locate the payload via the LFH (authoritative header length)
  let lh_raw: *u8 = sys_mmap(128)
  let lh: *ZipLocalHeader = lh_raw as *ZipLocalHeader
  if zip_parse_lfh(buf, n, lfh_off, lh) != 0 { sai_puts("  SKIP bad-LFH            name="); sai_puts(base); sai_puts("\n"); return SAI_SKIPPED }
  let payload_off: i64 = lfh_off + lh.header_len
  if (payload_off + csize) > n { sai_puts("  SKIP payload-OOB        name="); sai_puts(base); sai_puts("\n"); return SAI_SKIPPED }
  // decompress (bounded by usize, already <= cap)
  var content: *u8 = 0 as *u8
  var got: i64 = 0
  if method == 0 {
      content = ((buf as i64) + payload_off) as *u8
      got = csize
  } else {
      let res: *NxDeflateResult = nx_deflate_inflate(((buf as i64) + payload_off) as *u8, csize, usize + 16)
      if (res as i64) == 0 { sai_puts("  SKIP inflate-null       name="); sai_puts(base); sai_puts("\n"); return SAI_SKIPPED }
      if res.error_code != NX_DEF_OK { sai_puts("  SKIP inflate-err        name="); sai_puts(base); sai_puts("\n"); return SAI_SKIPPED }
      content = res.output_data
      got = res.output_size
  }
  if got > usize { got = usize }
  tot_p[0] = tot_p[0] + got
  // PDF -> sovereign TEXT extraction (nx_pdf_text). Bank the extracted text, NEVER the binary blob.
  // The text still passes the content sniff + sanitize + 0644 write, so no safety invariant changes.
  if sai_ext_is_pdf(base, blen) == 1 {
      let ptxt: *u8 = sys_mmap(SAI_MAGIC_16777216)
      let plen: i64 = nx_pdf_extract_text(content, got, ptxt, SAI_MAGIC_16777216)
      if plen <= 0 { sai_puts("  SKIP pdf-no-text        name="); sai_puts(base); sai_puts("\n"); return SAI_SKIPPED }
      content = ptxt
      got = plen
  }
  // (5b) content sniff -- only predominantly-text payloads are banked; binary blobs refused
  if sai_is_text_content(content, got) == 0 { sai_puts("  SKIP binary-content     name="); sai_puts(base); sai_puts("\n"); return SAI_SKIPPED }
  if sai_bank_text(dest, destlen, base, blen, content, got) == 0 { sai_puts("  SKIP write-fail         name="); sai_puts(base); sai_puts("\n"); return SAI_SKIPPED }
  sai_puts("  BANK "); sai_puts(base); sai_puts("  ("); sai_putn(got); sai_puts(" bytes text)\n")
  return SAI_BANKED
295}
296
297func main(argc: i64, argv: *i64) -> i64 {
  if argc < 2 { sai_puts("usage: nx_safe_archive_ingest <archive.zip> [dest-dir]\n"); return 1 }
  let zippath: *u8 = argv[1] as *u8
  var dest: *u8 = "knowledge/library" as *u8
  if argc >= 3 { dest = argv[2] as *u8 }
  let destlen: i64 = sai_strlen(dest)
303
  let lb: *i64 = sys_mmap(16) as *i64
  let buf: *u8 = sys_read_file(zippath, lb)
  if (buf as i64) == 0 { sai_puts("REJECT: cannot read archive\n"); return 1 }
  let n: i64 = lb[0]
  sai_puts("nx_safe_archive_ingest: "); sai_puts(zippath); sai_puts("  bytes="); sai_putn(n); sai_puts("  dest="); sai_puts(dest); sai_puts("\n")
  if n < 22 { sai_puts("REJECT: too short to be a zip\n"); return 1 }
  if n > SAI_MAX_ARCHIVE { sai_puts("REJECT: archive exceeds cap\n"); return 1 }
311
  let eocd_off: i64 = zip_find_eocd(buf, n)
  if eocd_off < 0 { sai_puts("REJECT: no EOCD (not a zip)\n"); return 1 }
  let e_raw: *u8 = sys_mmap(128)
  let e: *ZipEndOfCentralDir = e_raw as *ZipEndOfCentralDir
  if zip_parse_eocd(buf, n, eocd_off, e) != 0 { sai_puts("REJECT: bad EOCD\n"); return 1 }
  let entries: i64 = e.entries_total
  if entries > SAI_MAX_ENTRIES { sai_puts("REJECT: too many entries\n"); return 1 }
  if entries < 0 { sai_puts("REJECT: bad entry count\n"); return 1 }
  sai_puts("entries="); sai_putn(entries); sai_puts("  cd_offset="); sai_putn(e.cd_offset); sai_puts("\n")
  sys_mkdir(dest, 0x1ed)
322
  var off: i64 = e.cd_offset
  var idx: i64 = 0
  var banked: i64 = 0
  var skipped: i64 = 0
  var rejected: i64 = 0
  let tot_p: *i64 = sys_mmap(16) as *i64
  tot_p[0] = 0
  while idx < entries {
      if (off + 46) > n { sai_puts("  STOP: CDFH out of bounds\n"); rejected = rejected + 1; idx = entries }
      else {
          if zip_read_u32(buf, off) != SAI_CDFH_SIG { sai_puts("  STOP: bad CDFH sig\n"); rejected = rejected + 1; idx = entries }
          else {
              let nlen: i64 = zip_read_u16(buf, off + 28)
              let elen: i64 = zip_read_u16(buf, off + 30)
              let clen: i64 = zip_read_u16(buf, off + 32)
              let v: i64 = sai_process_entry(buf, n, off, dest, destlen, tot_p)
              if v == SAI_BANKED { banked = banked + 1 }
              if v == SAI_SKIPPED { skipped = skipped + 1 }
              if v == SAI_REJECTED { rejected = rejected + 1 }
              off = off + 46 + nlen + elen + clen
              idx = idx + 1
          }
      }
  }
  sai_puts("DONE: banked="); sai_putn(banked); sai_puts(" skipped="); sai_putn(skipped); sai_puts(" rejected="); sai_putn(rejected)
  sai_puts("  total_text_bytes="); sai_putn(tot_p[0]); sai_puts("\n")
  return 0
350}
nx_safe_archive_ingest.nx source

`nx_safe_archive_ingest.nx` source
