code wiki / (root) / nx_vault_capture.nx
`nx_vault_capture.nx` source

↩ module page · 1074 lines · 42799 B
1// nx_vault_capture.nx -- SAVE, as a first-class sovereign verb.  (v2: ref-root/phys-root split)
2//
3// OPERATOR 2026-08-06: "saving an image doesn't just go to a download folder --
4// you have the ability to save the media or the whole site for future
5// consumption if you choose, from the browser to the OS."
6//
7// WHAT THIS DOES: one verb, three scopes, one destination.
8//   media <url>  the asset alone
9//   page  <url>  the document AS AN ITEM + every image it references
10//   site  <url>  a bounded same-host crawl, every page captured as above
11// Each captured byte becomes a content-addressed vault item (CID = identity, so
12// re-saving is free and duplicates collapse) carrying its provenance at ingest.
13//
14// ONE CHOKEPOINT (vc_ingest_file): the album downloader (mvf_fetch_album_pfx)
15// filed downloads into collections but wrote NO item record, and does not even
16// import the ingest module -- so it structurally COULD NOT. A capture path that
17// does its own recording can forget to record. So there is exactly one way in,
18// and the gate asserts the invariant it holds: EVERY COLLECTION MEMBER RESOLVES
19// TO A RECORD (T6), with a negative control (T7).
20//
21// REF-ROOT vs PHYS-ROOT (v2, the fix for the v1 15/15 live failure). The
22// record's `ref` uses the LOGICAL DSM root /volume1/vault -- a shared folder
23// that needs root/DSM to create and DOES NOT EXIST as a plain dir. All existing
24// records carry that logical root, and nx_vault_gateway SERVES by stripping its
25// 14-char prefix and prepending a writable PHYSICAL root (~/vaultfs). v1 used
26// ONE root for both the ref and the bytes, so every write under the nonexistent
27// /volume1/vault failed (a page + 14 assets = 15/15). v2 keeps the LOGICAL ref
28// in the record (consistent with existing records + the gateway remap) and
29// writes bytes to the PHYSICAL root. Both roots are data-driven
30// (knowledge/status/vault_roots.conf), rule 11/17.
31//
32// ORDERING (nx_mvault_coll): collections are declared BEFORE the bytes land.
33// Composes only proven organs. license_tier: ORIGINAL
34
35import "nx_syscalls.nx"
36import "nx_canon_cid.nx"
37import "nx_registry.nx"
38import "nx_media_pool.nx"
39import "nx_media_sniff.nx"
40import "nx_mvault_ingest.nx"
41import "nx_mvault_context.nx"
42import "nx_mvault_record.nx"
43import "nx_mvault_layout.nx"
44import "nx_mvault_reclaim.nx"
45import "nx_mvault_reindex.nx"
46import "nx_mvault_tag.nx"
47import "nx_mvault_coll.nx"
48import "nx_mvault_fetch.nx"
49import "nx_https_fetch_lib.nx"
50import "nx_paced_fetch.nx"   // R0 (/compare/mediaingest): the ONE pacing hook around every capture fetch -- see vc_pace_hook
51import "nx_ingest_admit.nx"
52import "nx_url_safety_lib.nx"  // R11: the ONE SSRF url ruler, shared with nx_clean_serve_daemon and nx_cleanview  // R2 (/compare/mediaingest): disk-floor admission at the chokepoint -- refuse a torn write on a full volume
53const VC_MAGIC_4096: i64 = 4096
54const VC_DISK_REFUSE: i64 = 0 - 2   // vc_ingest_file's distinct return for a disk-floor refusal (callers read <0 as fail, correctly)
55const VC_EXIT_UNSAFE_URL: i64 = 9   // R11: the edge verb refused an SSRF-unsafe url; distinct from a fetch failure (6) so a caller can tell a REFUSAL from a FAILURE
56const VC_MAGIC_1024: i64 = 1024
57const VC_MAGIC_65536: i64 = 65536
58const VC_MAGIC_8192: i64 = 8192
59
60const VC_URL_CAP:    i64 = 2048
61const VC_PATH_CAP:   i64 = 2048
62const VC_PAGE_CAP:   i64 = 4194304
63const VC_LINKS_CAP:  i64 = 1048576
64const VC_ITEMS_CAP:  i64 = 262144
65const VC_VISIT_CAP:  i64 = 262144
66const VC_HEAD_CAP:   i64 = 8192
67const VC_REC_CAP:    i64 = 4096
68const VC_REPORT_CAP: i64 = 65536
69const VC_EXT_CAP:    i64 = 64
70const VC_HOST_CAP:   i64 = 512
71const VC_CID_CAP:    i64 = 128
72const VC_ROOT_CAP:   i64 = 512
73
74const VC_DEF_ASSETS: i64 = 64
75const VC_DEF_PAGES:  i64 = 16
76const VC_MAX_ASSETS: i64 = 512
77const VC_MAX_PAGES:  i64 = 256
78
79// Compiled defaults; overridable from knowledge/status/vault_roots.conf.
80const VC_REF_ROOT:  *u8 = "/volume1/vault" as *u8
81const VC_PHYS_ROOT: *u8 = "/volume1/homes/elderwesto/vaultfs" as *u8
82const VC_ROOTS_CONF: *u8 = "knowledge/status/vault_roots.conf" as *u8
83
84const VC_ST_FOUND: i64 = 0
85const VC_ST_NEW:   i64 = 1
86const VC_ST_DUP:   i64 = 2
87const VC_ST_FAIL:  i64 = 3
88const VC_ST_PAGE:  i64 = 4
89const VC_ST_PAGES: i64 = 5
90const VC_ST_N:     i64 = 6
91
92func vc_strlen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
93
94func vc_cat(dst: *u8, off: i64, s: *u8) -> i64 {
  var i: i64 = 0
  while s[i] != (0 as u8) { dst[off + i] = s[i]; i = i + 1 }
  return off + i
98}
99
100func vc_eq(a: *u8, b: *u8) -> i64 {
  var i: i64 = 0
  while 1 == 1 {
      if a[i] != b[i] { return 0 }
      if a[i] == (0 as u8) { return 1 }
      i = i + 1
  }
  return 0
108}
109
110func vc_w(s: *u8) -> i64 { sys_write(1, s, vc_strlen(s)); return 0 }
111func vc_werr(s: *u8) -> i64 { sys_write(2, s, vc_strlen(s)); return 0 }
112
113func vc_n(v: i64) -> i64 {
  let d: *u8 = sys_mmap(32)
  let dl: i64 = mv_u_dec(d, 0, v)
  sys_write(1, d, dl)
  sys_munmap(d, 32)
  return 0
119}
120// stderr twin of vc_n -- a refused capture announces its numbers on the error channel (R2)
121func vc_werr_n(v: i64) -> i64 {
  let d: *u8 = sys_mmap(32)
  let dl: i64 = mv_u_dec(d, 0, v)
  sys_write(2, d, dl)
  sys_munmap(d, 32)
  return 0
127}
128
129func vc_lower(c: i64) -> i64 { if c >= 65 { if c <= 90 { return c + 32 } } return c }
130
131func vc_has_pfx(buf: *u8, pfx: *u8) -> i64 {
  var i: i64 = 0
  while pfx[i] != (0 as u8) {
      if buf[i] != pfx[i] { return 0 }
      i = i + 1
  }
  return 1
138}
139
140func vc_phys_of(ref: *u8, refroot: *u8, physroot: *u8, out: *u8, cap: i64) -> i64 {
  if vc_has_pfx(ref, refroot) == 0 {
      let d: i64 = vc_cat(out, 0, ref)
      out[d] = 0 as u8
      return d
  }
  let rl: i64 = vc_strlen(refroot)
  var o: i64 = vc_cat(out, 0, physroot)
  var i: i64 = rl
  while ref[i] != (0 as u8) { out[o] = ref[i]; o = o + 1; i = i + 1 }
  out[o] = 0 as u8
  return o
152}
153
154func vc_conf_field(buf: *u8, n: i64, key: *u8, out: *u8, cap: i64) -> i64 {
  var ls: i64 = 0
  var i: i64 = 0
  while i <= n {
      var eol: i64 = 0
      if i == n { eol = 1 } else { if buf[i] == (10 as u8) { eol = 1 } }
      if eol == 1 {
          var m: i64 = 1
          var k: i64 = 0
          while key[k] != (0 as u8) {
              if ls + k >= i { m = 0 } else { if buf[ls + k] != key[k] { m = 0 } }
              k = k + 1
          }
          if m == 1 {
              var o: i64 = 0
              var j: i64 = ls + k
              while j < i {
                  if buf[j] != (13 as u8) { if o < cap - 1 { out[o] = buf[j]; o = o + 1 } }
                  j = j + 1
              }
              out[o] = 0 as u8
              if o > 0 { return o }
          }
          ls = i + 1
      }
      i = i + 1
  }
  return 0
182}
183
184func vc_load_roots(ref_out: *u8, phys_out: *u8) -> i64 {
  vc_cat(ref_out, 0, VC_REF_ROOT);   ref_out[vc_strlen(VC_REF_ROOT)] = 0 as u8
  vc_cat(phys_out, 0, VC_PHYS_ROOT); phys_out[vc_strlen(VC_PHYS_ROOT)] = 0 as u8
  let szbox: *i64 = sys_mmap(16) as *i64
  let buf: *u8 = sys_read_file(VC_ROOTS_CONF, szbox)
  if (buf as i64) != 0 {
      if szbox[0] > 0 {
          vc_conf_field(buf, szbox[0], "ref_root=" as *u8, ref_out, VC_ROOT_CAP)
          vc_conf_field(buf, szbox[0], "phys_root=" as *u8, phys_out, VC_ROOT_CAP)
      }
  }
  sys_munmap(szbox as *u8, 16)
  return 1
197}
198
199func vc_mkdirp(path: *u8) -> i64 {
  let tmp: *u8 = sys_mmap(VC_PATH_CAP)
  var n: i64 = 0
  while path[n] != (0 as u8) {
      if n < VC_PATH_CAP - 2 { tmp[n] = path[n] }
      n = n + 1
  }
  if n >= VC_PATH_CAP - 2 { sys_munmap(tmp, VC_PATH_CAP); return 0 - 1 }
  tmp[n] = 0 as u8
208
  var last: i64 = 0 - 1
  var i: i64 = 0
  while i < n { if tmp[i] == (47 as u8) { last = i } i = i + 1 }
  if last <= 0 { sys_munmap(tmp, VC_PATH_CAP); return 0 }
213
  var j: i64 = 1
  while j < last {
      if tmp[j] == (47 as u8) {
          tmp[j] = 0 as u8
          sys_mkdir(tmp, 493)
          tmp[j] = 47 as u8
      }
      j = j + 1
  }
  tmp[last] = 0 as u8
  sys_mkdir(tmp, 493)
  sys_munmap(tmp, VC_PATH_CAP)
  return 1
227}
228
229func vc_ext_of(url: *u8, deflt: *u8, out: *u8, cap: i64) -> i64 {
  let base: *u8 = sys_mmap(VC_URL_CAP)
  let have: i64 = mvf_basename(url, base, VC_URL_CAP)
  if have == 0 {
      sys_munmap(base, VC_URL_CAP)
      let d: i64 = vc_cat(out, 0, deflt)
      out[d] = 0 as u8
      return d
  }
  var n: i64 = 0
  var cut: i64 = 0 - 1
  while base[n] != (0 as u8) {
      if cut < 0 {
          if base[n] == (63 as u8) { cut = n }
          if base[n] == (35 as u8) { cut = n }
      }
      n = n + 1
  }
  if cut >= 0 { base[cut] = 0 as u8; n = cut }
248
  var dot: i64 = 0 - 1
  var i: i64 = 0
  while i < n { if base[i] == (46 as u8) { dot = i } i = i + 1 }
252
  var ok: i64 = 1
  if dot < 0 { ok = 0 }
  if dot + 1 >= n { ok = 0 }
  if n - dot - 1 >= cap { ok = 0 }
  if n - dot - 1 > 12 { ok = 0 }
258
  if ok == 1 {
      var k: i64 = dot + 1
      while k < n {
          let c: i64 = base[k] as i64
          var good: i64 = 0
          if c >= 48 { if c <= 57 { good = 1 } }
          if c >= 65 { if c <= 90 { good = 1 } }
          if c >= 97 { if c <= 122 { good = 1 } }
          if good == 0 { ok = 0 }
          k = k + 1
      }
  }
271
  if ok == 0 {
      sys_munmap(base, VC_URL_CAP)
      let d2: i64 = vc_cat(out, 0, deflt)
      out[d2] = 0 as u8
      return d2
  }
  var o: i64 = 0
  var j: i64 = dot + 1
  while j < n { out[o] = vc_lower(base[j] as i64) as u8; o = o + 1; j = j + 1 }
  out[o] = 0 as u8
  sys_munmap(base, VC_URL_CAP)
  return o
284}
285
286func vc_ingest_file(store: *u8, ref_root: *u8, phys_root: *u8, tmppath: *u8,
                  source: *u8, ext: *u8, class_code: i64,
                  cidout: *u8, refout: *u8) -> i64 {
  // R2 (/compare/mediaingest): disk-floor admission BEFORE this item is hashed and committed. The vault volume
  // (phys_root) must hold at least the configured floor free; below it, or if the disk is UNMEASURABLE, REFUSE
  // rather than grow the corpus onto a full volume. The refusal ANNOUNCES the two numbers -- a refusal that does
  // not say how much was free and how much was needed is not actionable. The already-fetched tmp is unlinked so a
  // refused capture leaves nothing behind. Callers read a negative return as a failure, which is correct.
  let da: *i64 = sys_mmap(16) as *i64
  let df: *i64 = sys_mmap(16) as *i64
  let dv: i64 = ig_disk_floor(phys_root, da, df)
  if dv != IG_ADMIT {
      vc_werr("nx_vault_capture: ingest REFUSED -- " as *u8); vc_werr(ig_verdict_str(dv))
      vc_werr(" free=" as *u8); vc_werr_n(da[0]); vc_werr(" floor=" as *u8); vc_werr_n(df[0]); vc_werr("\n" as *u8)
      sys_unlinkat(tmppath)
      sys_munmap(da as *u8, 16)
      sys_munmap(df as *u8, 16)
      return VC_DISK_REFUSE
  }
  sys_munmap(da as *u8, 16)
  sys_munmap(df as *u8, 16)
  let szbox: *i64 = sys_mmap(16) as *i64
  szbox[0] = 0
  if cid_of_file_chunk(tmppath, CID_FILE_CHUNK, szbox, cidout) < 0 {
      sys_munmap(szbox as *u8, 16)
      return 0 - 1
  }
313
  let head: *u8 = sys_mmap(VC_HEAD_CAP)
  let fd: i64 = sys_openat_rd(tmppath)
  if fd < 0 {
      sys_munmap(head, VC_HEAD_CAP)
      sys_munmap(szbox as *u8, 16)
      return 0 - 1
  }
  var hn: i64 = 0
  var go: i64 = 1
  while go == 1 {
      let dst: *u8 = ((head as i64) + hn) as *u8
      let r: i64 = sys_read(fd, dst, VC_HEAD_CAP - hn)
      if r <= 0 { go = 0 } else {
          hn = hn + r
          if hn >= VC_HEAD_CAP { go = 0 }
      }
  }
  sys_close(fd)
332
  let vtype: i64 = mv_classify_plan(head, hn, class_code, MV_CTX_NONE, ref_root,
                                    source, ext, refout, cidout)
  sys_munmap(head, VC_HEAD_CAP)
336
  let gp: *i64 = sys_mmap(16) as *i64
  let gl: *i64 = sys_mmap(16) as *i64
  let found: i64 = mv_store_get(store, cidout, gp, gl)
  sys_munmap(gp as *u8, 16)
  sys_munmap(gl as *u8, 16)
  if found == 1 {
      sys_unlinkat(tmppath)
      sys_munmap(szbox as *u8, 16)
      return 0
  }
347
  let phys: *u8 = sys_mmap(VC_PATH_CAP)
  vc_phys_of(refout, ref_root, phys_root, phys, VC_PATH_CAP)
  vc_mkdirp(phys)
  if sys_renameat(tmppath, phys) < 0 {
      sys_munmap(phys, VC_PATH_CAP)
      sys_munmap(szbox as *u8, 16)
      return 0 - 1
  }
  sys_munmap(phys, VC_PATH_CAP)
357
  let rec: *u8 = sys_mmap(VC_REC_CAP)
  let rl: i64 = mv_record_build(rec, class_code, vtype, MV_LVL_TS, source, refout)
  let rc: i64 = mv_store_put(store, cidout, rec, rl)
  sys_munmap(rec, VC_REC_CAP)
  sys_munmap(szbox as *u8, 16)
  if rc < 0 { return 0 - 1 }
  return 1
365}
366
367// R0 (/compare/mediaingest, 2026-08-30): THE pacing hook. Every fetch this organ makes goes through it, before
368// (phase 0: the polite wait for the url's host, served by sleeping) and after (phase 1: the REAL status fed back so
369// a 429 or 503 becomes a longer next interval, persisted per host). It DELEGATES to nx_paced_fetch, which delegates
370// to nx_crawl_pace -- one ruler, no second copy. MEASURED before this hook: this organ imported no pacing at all,
371// so a site capture hit its host as fast as the asset loop ran. retry_after_s is 0 here on purpose: the fetch lib
372// hands back bytes and a status, not the headers, and a guessed Retry-After would be a fabricated measurement.
373func vc_pace_hook(url: *u8, phase: i64, status: i64) -> i64 {
  if phase == 0 { return pf_before(url) }
  return pf_after(url, status, 0)
376}
377
378func vc_fetch_tmp(store_i: i64, phys_root: *u8, url: *u8, seqn: i64,
                out: *u8, stbox: *i64) -> i64 {
  var o: i64 = vc_cat(out, 0, phys_root)
  o = vc_cat(out, o, "/_incoming/cap" as *u8)
  o = mv_u_dec(out, o, seqn)
  out[o] = 0 as u8
  vc_mkdirp(out)
  let fd: i64 = sys_openat_wr(out, 420)
  if fd < 0 { return 0 - 1 }
  stbox[0] = 0
  vc_pace_hook(url, 0, 0)
  let n: i64 = hf_fetch_to_file(store_i, url, 0, 0, fd, 0, stbox)
  vc_pace_hook(url, 1, stbox[0])
  sys_close(fd)
  if n <= 0 { sys_unlinkat(out); return 0 - 1 }
  if stbox[0] != 200 { sys_unlinkat(out); return 0 - 2 }
  return n
395}
396
397func vc_capture_media(store_i: i64, store: *u8, ref_root: *u8, phys_root: *u8,
                    url: *u8, source: *u8, class_code: i64, seqn: i64,
                    cidout: *u8) -> i64 {
  let tmp: *u8 = sys_mmap(VC_PATH_CAP)
  let stbox: *i64 = sys_mmap(16) as *i64
  let n: i64 = vc_fetch_tmp(store_i, phys_root, url, seqn, tmp, stbox)
  sys_munmap(stbox as *u8, 16)
  if n < 0 { sys_munmap(tmp, VC_PATH_CAP); return n }
405
  let ext: *u8 = sys_mmap(VC_EXT_CAP)
  vc_ext_of(url, "bin" as *u8, ext, VC_EXT_CAP)
  let ref: *u8 = sys_mmap(VC_PATH_CAP)
  let r: i64 = vc_ingest_file(store, ref_root, phys_root, tmp, source, ext,
                              class_code, cidout, ref)
  sys_munmap(ext, VC_EXT_CAP)
  sys_munmap(ref, VC_PATH_CAP)
  sys_munmap(tmp, VC_PATH_CAP)
  return r
415}
416
417func vc_items_add(items: *u8, io: i64, cid: *u8) -> i64 {
  let cl: i64 = vc_strlen(cid)
  if io + cl + 2 >= VC_ITEMS_CAP { return 0 - 1 }
  var y: i64 = 0
  while y < cl { items[io + y] = cid[y]; y = y + 1 }
  items[io + cl] = 10 as u8
  return io + cl + 1
424}
425
426func vc_seen(set: *u8, n: i64, hay: *u8) -> i64 {
  let hl: i64 = vc_strlen(hay)
  var ls: i64 = 0
  var i: i64 = 0
  while i <= n {
      var eol: i64 = 0
      if i == n { eol = 1 } else { if set[i] == (10 as u8) { eol = 1 } }
      if eol == 1 {
          if i - ls == hl {
              var same: i64 = 1
              var k: i64 = 0
              while k < hl { if set[ls + k] != hay[k] { same = 0 } k = k + 1 }
              if same == 1 { return 1 }
          }
          ls = i + 1
      }
      i = i + 1
  }
  return 0
445}
446
447func vc_page_core(store_i: i64, store: *u8, cpfx: *u8, ref_root: *u8, phys_root: *u8,
                url: *u8, source: *u8, body: *u8, blen: i64, max_assets: i64,
                site_col: *u8, page_col: *u8, stats: *i64,
                links_out: *u8, links_cap: i64, links_len: *i64) -> i64 {
  let items: *u8 = sys_mmap(VC_ITEMS_CAP)
  var io: i64 = 0
453
  let ptmp: *u8 = sys_mmap(VC_PATH_CAP)
  var po: i64 = vc_cat(ptmp, 0, phys_root)
  po = vc_cat(ptmp, po, "/_incoming/page" as *u8)
  po = mv_u_dec(ptmp, po, stats[VC_ST_PAGES])
  ptmp[po] = 0 as u8
  vc_mkdirp(ptmp)
  let pcid: *u8 = sys_mmap(VC_CID_CAP)
  let pfd: i64 = sys_openat_wr(ptmp, 420)
  if pfd >= 0 {
      sys_write(pfd, body, blen)
      sys_close(pfd)
      let pref: *u8 = sys_mmap(VC_PATH_CAP)
      let pr: i64 = vc_ingest_file(store, ref_root, phys_root, ptmp, source,
                                   "html" as *u8, MV_CLASS_REAL, pcid, pref)
      sys_munmap(pref, VC_PATH_CAP)
      if pr >= 0 {
          stats[VC_ST_PAGE] = stats[VC_ST_PAGE] + 1
          let r2: i64 = vc_items_add(items, io, pcid)
          if r2 >= 0 { io = r2 }
      } else { stats[VC_ST_FAIL] = stats[VC_ST_FAIL] + 1 }
  } else { stats[VC_ST_FAIL] = stats[VC_ST_FAIL] + 1 }
  sys_munmap(ptmp, VC_PATH_CAP)
476
  let imgs: *u8 = sys_mmap(VC_LINKS_CAP)
  let inn: i64 = mvf_extract(body, blen, "img" as *u8, "src" as *u8,
                             "" as *u8, imgs, VC_LINKS_CAP)
  let one: *u8 = sys_mmap(VC_URL_CAP)
  let href: *u8 = sys_mmap(VC_URL_CAP)
  let acid: *u8 = sys_mmap(VC_CID_CAP)
  var got: i64 = 0
484
  if inn > 0 {
      var ls: i64 = 0
      var i: i64 = 0
      while i <= inn {
          var eol: i64 = 0
          if i == inn { eol = 1 } else { if imgs[i] == (10 as u8) { eol = 1 } }
          if eol == 1 {
              let hl: i64 = i - ls
              if hl > 0 {
                  if hl < VC_URL_CAP - 1 {
                      if got < max_assets {
                          stats[VC_ST_FOUND] = stats[VC_ST_FOUND] + 1
                          var q: i64 = 0
                          while q < hl { href[q] = imgs[ls + q]; q = q + 1 }
                          href[hl] = 0 as u8
                          if mvf_resolve(url, href, one, VC_URL_CAP) > 0 {
                              let seqn: i64 = stats[VC_ST_PAGES] * VC_MAGIC_4096 + got + 1
                              let r: i64 = vc_capture_media(store_i, store, ref_root, phys_root,
                                                            one, source, MV_CLASS_REAL, seqn, acid)
                              if r == 1 {
                                  stats[VC_ST_NEW] = stats[VC_ST_NEW] + 1
                                  let ra: i64 = vc_items_add(items, io, acid)
                                  if ra >= 0 { io = ra }
                                  got = got + 1
                              }
                              if r == 0 {
                                  stats[VC_ST_DUP] = stats[VC_ST_DUP] + 1
                                  let rd: i64 = vc_items_add(items, io, acid)
                                  if rd >= 0 { io = rd }
                                  got = got + 1
                              }
                              if r < 0 { stats[VC_ST_FAIL] = stats[VC_ST_FAIL] + 1 }
                          } else { stats[VC_ST_FAIL] = stats[VC_ST_FAIL] + 1 }
                      }
                  }
              }
              ls = i + 1
          }
          i = i + 1
      }
  }
  sys_munmap(imgs, VC_LINKS_CAP)
527
  if io > 0 {
      mvc_batch_add_pfx(cpfx, page_col, items, io)
      mvc_batch_add_pfx(cpfx, site_col, items, io)
  }
  sys_munmap(items, VC_ITEMS_CAP)
533
  if links_cap > 0 {
      links_len[0] = 0
      let ln: i64 = mvf_extract(body, blen, "a" as *u8, "href" as *u8,
                                "" as *u8, links_out, links_cap)
      if ln > 0 { if ln < links_cap { links_len[0] = ln } }
  }
540
  sys_munmap(one, VC_URL_CAP)
  sys_munmap(href, VC_URL_CAP)
  sys_munmap(acid, VC_CID_CAP)
  sys_munmap(pcid, VC_CID_CAP)
  return got
546}
547
548func vc_declare_page_cols(cpfx: *u8, url: *u8, host: *u8,
                        site_col: *u8, page_col: *u8) -> i64 {
  mvc_declare_pfx(cpfx, "site" as *u8, host, host, host, url, "-" as *u8, site_col)
  let pkey: *u8 = sys_mmap(VC_URL_CAP)
  var i: i64 = 0
  while url[i] != (0 as u8) {
      if i < VC_URL_CAP - 1 { pkey[i] = url[i] }
      i = i + 1
  }
  if i > VC_URL_CAP - 1 { i = VC_URL_CAP - 1 }
  pkey[i] = 0 as u8
  mvc_declare_pfx(cpfx, "page" as *u8, host, pkey, pkey, url, site_col, page_col)
  sys_munmap(pkey, VC_URL_CAP)
  return 1
562}
563
564func vc_report_stats(report: *u8, off: i64, stats: *i64) -> i64 {
  var o: i64 = off
  o = vc_cat(report, o, ",\"page_items\":" as *u8)
  o = mv_u_dec(report, o, stats[VC_ST_PAGE])
  o = vc_cat(report, o, ",\"assets_found\":" as *u8)
  o = mv_u_dec(report, o, stats[VC_ST_FOUND])
  o = vc_cat(report, o, ",\"stored\":" as *u8)
  o = mv_u_dec(report, o, stats[VC_ST_NEW])
  o = vc_cat(report, o, ",\"dup\":" as *u8)
  o = mv_u_dec(report, o, stats[VC_ST_DUP])
  o = vc_cat(report, o, ",\"failed\":" as *u8)
  o = mv_u_dec(report, o, stats[VC_ST_FAIL])
  return o
577}
578
579func vc_capture_page(store_i: i64, store: *u8, cpfx: *u8, ref_root: *u8, phys_root: *u8,
                   url: *u8, source: *u8, max_assets: i64, report: *u8, rcap: i64) -> i64 {
  let host: *u8 = sys_mmap(VC_HOST_CAP)
  mvf_host_of(url, host)
583
  let site_col: *u8 = sys_mmap(VC_CID_CAP)
  let page_col: *u8 = sys_mmap(VC_CID_CAP)
  vc_declare_page_cols(cpfx, url, host, site_col, page_col)
587
  let page: *u8 = sys_mmap(VC_PAGE_CAP)
  vc_pace_hook(url, 0, 0)
  let pn: i64 = hf_fetch(store_i, url, 0, 0, page, VC_PAGE_CAP)
  var pst: i64 = 0
  if pn > 0 { pst = hf_status(page, pn) }
  vc_pace_hook(url, 1, pst)
  if pn <= 0 { sys_munmap(page, VC_PAGE_CAP); return 0 - 2 }
  var boff: i64 = hf_body_off(page, pn)
  if boff < 0 { boff = 0 }
  let body: *u8 = ((page as i64) + boff) as *u8
598
  let stats: *i64 = sys_mmap(8 * VC_ST_N) as *i64
  var z: i64 = 0
  while z < VC_ST_N { stats[z] = 0; z = z + 1 }
  stats[VC_ST_PAGES] = 1
603
  let lbox: *i64 = sys_mmap(16) as *i64
  let got: i64 = vc_page_core(store_i, store, cpfx, ref_root, phys_root, url, source,
                              body, pn - boff, max_assets, site_col, page_col, stats,
                              "" as *u8, 0, lbox)
  sys_munmap(page, VC_PAGE_CAP)
  sys_munmap(lbox as *u8, 16)
610
  var o: i64 = vc_cat(report, 0, "{\"scope\":\"page\",\"url\":\"" as *u8)
  o = vc_cat(report, o, url)
  o = vc_cat(report, o, "\",\"site_col\":\"" as *u8)
  o = vc_cat(report, o, site_col)
  o = vc_cat(report, o, "\",\"page_col\":\"" as *u8)
  o = vc_cat(report, o, page_col)
  o = vc_cat(report, o, "\"" as *u8)
  o = vc_report_stats(report, o, stats)
  o = vc_cat(report, o, "}" as *u8)
  report[o] = 10 as u8
  report[o + 1] = 0 as u8
622
  sys_munmap(stats as *u8, 8 * VC_ST_N)
  sys_munmap(host, VC_HOST_CAP)
  sys_munmap(site_col, VC_CID_CAP)
  sys_munmap(page_col, VC_CID_CAP)
  return got
628}
629
630func vc_capture_site(store_i: i64, store: *u8, cpfx: *u8, ref_root: *u8, phys_root: *u8,
                   url: *u8, source: *u8, max_pages: i64, max_assets: i64,
                   report: *u8, rcap: i64) -> i64 {
  let host: *u8 = sys_mmap(VC_HOST_CAP)
  mvf_host_of(url, host)
635
  let site_col: *u8 = sys_mmap(VC_CID_CAP)
  mvc_declare_pfx(cpfx, "site" as *u8, host, host, host, url, "-" as *u8, site_col)
638
  let frontier: *u8 = sys_mmap(VC_VISIT_CAP)
  let visited:  *u8 = sys_mmap(VC_VISIT_CAP)
  var fo: i64 = vc_cat(frontier, 0, url)
  frontier[fo] = 10 as u8
  fo = fo + 1
  var vo: i64 = 0
645
  let stats: *i64 = sys_mmap(8 * VC_ST_N) as *i64
  var z: i64 = 0
  while z < VC_ST_N { stats[z] = 0; z = z + 1 }
649
  let cur:   *u8 = sys_mmap(VC_URL_CAP)
  let one:   *u8 = sys_mmap(VC_URL_CAP)
  let href:  *u8 = sys_mmap(VC_URL_CAP)
  let lhost: *u8 = sys_mmap(VC_HOST_CAP)
  let page_col: *u8 = sys_mmap(VC_CID_CAP)
  let page:  *u8 = sys_mmap(VC_PAGE_CAP)
  let links: *u8 = sys_mmap(VC_LINKS_CAP)
  let lbox:  *i64 = sys_mmap(16) as *i64
658
  var fp: i64 = 0
  var pages: i64 = 0
  var refused: i64 = 0
662
  while fp < fo {
      if pages >= max_pages { fp = fo } else {
          var stop: i64 = 0 - 1
          var q: i64 = fp
          while q < fo {
              if stop < 0 { if frontier[q] == (10 as u8) { stop = q } }
              q = q + 1
          }
          if stop < 0 { stop = fo }
672
          let cl: i64 = stop - fp
          var ok: i64 = 1
          if cl <= 0 { ok = 0 }
          if cl >= VC_URL_CAP - 1 { ok = 0 }
677
          if ok == 1 {
              var k: i64 = 0
              while k < cl { cur[k] = frontier[fp + k]; k = k + 1 }
              cur[cl] = 0 as u8
              if vc_seen(visited, vo, cur) == 1 { ok = 0 }
          }
          if ok == 1 {
              if vo + cl + 2 < VC_VISIT_CAP {
                  var m: i64 = 0
                  while m < cl { visited[vo + m] = cur[m]; m = m + 1 }
                  visited[vo + cl] = 10 as u8
                  vo = vo + cl + 1
              } else { ok = 0; refused = refused + 1 }
          }
692
          if ok == 1 {
              vc_pace_hook(cur, 0, 0)
              let pn: i64 = hf_fetch(store_i, cur, 0, 0, page, VC_PAGE_CAP)
              var cst: i64 = 0
              if pn > 0 { cst = hf_status(page, pn) }
              vc_pace_hook(cur, 1, cst)
              if pn <= 0 { stats[VC_ST_FAIL] = stats[VC_ST_FAIL] + 1 } else {
                  var boff: i64 = hf_body_off(page, pn)
                  if boff < 0 { boff = 0 }
                  let body: *u8 = ((page as i64) + boff) as *u8
703
                  pages = pages + 1
                  stats[VC_ST_PAGES] = pages
                  vc_declare_page_cols(cpfx, cur, host, site_col, page_col)
                  lbox[0] = 0
                  vc_page_core(store_i, store, cpfx, ref_root, phys_root, cur, source,
                               body, pn - boff, max_assets, site_col, page_col, stats,
                               links, VC_LINKS_CAP, lbox)
711
                  let ln: i64 = lbox[0]
                  var ls: i64 = 0
                  var i: i64 = 0
                  while i <= ln {
                      var eol: i64 = 0
                      if i == ln { eol = 1 } else { if links[i] == (10 as u8) { eol = 1 } }
                      if eol == 1 {
                          let hl: i64 = i - ls
                          if hl > 0 {
                              if hl < VC_URL_CAP - 1 {
                                  var y: i64 = 0
                                  while y < hl { href[y] = links[ls + y]; y = y + 1 }
                                  href[hl] = 0 as u8
                                  if mvf_resolve(cur, href, one, VC_URL_CAP) > 0 {
                                      mvf_host_of(one, lhost)
                                      if vc_eq(lhost, host) == 1 {
                                          if vc_seen(visited, vo, one) == 0 {
                                              if vc_seen(frontier, fo, one) == 0 {
                                                  let ul: i64 = vc_strlen(one)
                                                  if fo + ul + 2 < VC_VISIT_CAP {
                                                      var w: i64 = 0
                                                      while w < ul { frontier[fo + w] = one[w]; w = w + 1 }
                                                      frontier[fo + ul] = 10 as u8
                                                      fo = fo + ul + 1
                                                  } else { refused = refused + 1 }
                                              }
                                          }
                                      }
                                  }
                              }
                          }
                          ls = i + 1
                      }
                      i = i + 1
                  }
              }
          }
          fp = stop + 1
      }
  }
752
  var o: i64 = vc_cat(report, 0, "{\"scope\":\"site\",\"host\":\"" as *u8)
  o = vc_cat(report, o, host)
  o = vc_cat(report, o, "\",\"site_col\":\"" as *u8)
  o = vc_cat(report, o, site_col)
  o = vc_cat(report, o, "\",\"pages\":" as *u8)
  o = mv_u_dec(report, o, pages)
  o = vc_report_stats(report, o, stats)
  o = vc_cat(report, o, ",\"budget_refused\":" as *u8)
  o = mv_u_dec(report, o, refused)
  o = vc_cat(report, o, "}" as *u8)
  report[o] = 10 as u8
  report[o + 1] = 0 as u8
765
  sys_munmap(page, VC_PAGE_CAP)
  sys_munmap(links, VC_LINKS_CAP)
  sys_munmap(frontier, VC_VISIT_CAP)
  sys_munmap(visited, VC_VISIT_CAP)
  sys_munmap(stats as *u8, 8 * VC_ST_N)
  sys_munmap(lbox as *u8, 16)
  return pages
773}
774
775func vc_write_file(path: *u8, content: *u8, n: i64) -> i64 {
  vc_mkdirp(path)
  let fd: i64 = sys_openat_wr(path, 420)
  if fd < 0 { return 0 - 1 }
  sys_write(fd, content, n)
  sys_close(fd)
  return n
782}
783
784func do_selftest() -> i64 {
  var fails: i64 = 0
  let store: *u8 = "knowledge/store/vctest-" as *u8
  let cpfx:  *u8 = "knowledge/store/vctestc-" as *u8
  let refroot:  *u8 = "/volume1/vault" as *u8
  let physroot: *u8 = "/tmp/vcphys" as *u8
790
  let e: *u8 = sys_mmap(VC_EXT_CAP)
  vc_ext_of("https://h/a/b/photo.JPG?x=1" as *u8, "bin" as *u8, e, VC_EXT_CAP)
  if vc_eq(e, "jpg" as *u8) != 1 { fails = fails + 1 }
  vc_w("T1 ext of photo.JPG?x=1 -> " as *u8); vc_w(e); vc_w("\n" as *u8)
795
  vc_ext_of("https://cdn/stream/9f8a2b" as *u8, "bin" as *u8, e, VC_EXT_CAP)
  if vc_eq(e, "bin" as *u8) != 1 { fails = fails + 1 }
  vc_w("T2 extensionless CDN url -> " as *u8); vc_w(e); vc_w("\n" as *u8)
799
  let tmp: *u8 = "/tmp/vcphys/_incoming/t3" as *u8
  let png: *u8 = sys_mmap(64)
  png[0] = 137 as u8; png[1] = 80 as u8; png[2] = 78 as u8; png[3] = 71 as u8
  png[4] = 13 as u8;  png[5] = 10 as u8; png[6] = 26 as u8; png[7] = 10 as u8
  // ★ seed with the current epoch so each gate RUN produces a FRESH CID.
  // A selftest that writes a fixed fixture into a persistent store is not
  // repeatable: the SECOND run sees the FIRST run's record and every
  // new-ingest assertion flips to dup. Uniqueness per run restores isolation.
  let tnonce: i64 = sys_now_realtime_sec()
  var f: i64 = 8
  while f < 16 { png[f] = ((tnonce >> ((f - 8) * 8)) & 255) as u8; f = f + 1 }
  while f < 64 { png[f] = f as u8; f = f + 1 }
  vc_write_file(tmp, png, 64)
  let cid: *u8 = sys_mmap(VC_CID_CAP)
  let ref: *u8 = sys_mmap(VC_PATH_CAP)
  let r1: i64 = vc_ingest_file(store, refroot, physroot, tmp, "test" as *u8,
                               "png" as *u8, MV_CLASS_REAL, cid, ref)
  if r1 != 1 { fails = fails + 1 }
  vc_w("T3 first ingest (want 1=new) -> " as *u8); vc_n(r1)
  vc_w("  ref " as *u8); vc_w(ref); vc_w("\n" as *u8)
820
  if vc_has_pfx(ref, refroot) != 1 { fails = fails + 1 }
  if vc_has_pfx(ref, physroot) == 1 { fails = fails + 1 }
  vc_w("T13 record ref is LOGICAL (/volume1/vault), not phys -> ok\n" as *u8)
824
  let phys: *u8 = sys_mmap(VC_PATH_CAP)
  vc_phys_of(ref, refroot, physroot, phys, VC_PATH_CAP)
  let pchk: i64 = sys_openat_rd(phys)
  if pchk < 0 { fails = fails + 1 } else { sys_close(pchk) }
  let lchk: i64 = sys_openat_rd(ref)
  if lchk >= 0 { fails = fails + 1; sys_close(lchk) }
  vc_w("T14 blob at PHYSICAL " as *u8); vc_w(phys)
  vc_w(" (fd " as *u8); vc_n(pchk); vc_w("), logical absent (fd " as *u8); vc_n(lchk); vc_w(")\n" as *u8)
833
  if vc_has_pfx(phys, physroot) != 1 { fails = fails + 1 }
  vc_w("T15 vc_phys_of round-trips ref->phys under phys_root -> ok\n" as *u8)
836
  let passt: *u8 = sys_mmap(VC_PATH_CAP)
  vc_phys_of("/some/other/path.png" as *u8, refroot, physroot, passt, VC_PATH_CAP)
  if vc_eq(passt, "/some/other/path.png" as *u8) != 1 { fails = fails + 1 }
  vc_w("T16 non-ref-root path passes through unchanged -> " as *u8); vc_w(passt); vc_w("\n" as *u8)
841
  vc_write_file(tmp, png, 64)
  let cid2: *u8 = sys_mmap(VC_CID_CAP)
  let ref2: *u8 = sys_mmap(VC_PATH_CAP)
  let r2: i64 = vc_ingest_file(store, refroot, physroot, tmp, "test" as *u8,
                               "png" as *u8, MV_CLASS_REAL, cid2, ref2)
  if r2 != 0 { fails = fails + 1 }
  if vc_eq(cid, cid2) != 1 { fails = fails + 1 }
  vc_w("T5 re-ingest identical bytes (want 0=dup, same cid) -> " as *u8); vc_n(r2); vc_w("\n" as *u8)
850
  let scol: *u8 = sys_mmap(VC_CID_CAP)
  mvc_declare_pfx(cpfx, "site" as *u8, "t.example" as *u8, "t.example" as *u8,
                  "t.example" as *u8, "https://t.example" as *u8, "-" as *u8, scol)
  let ib: *u8 = sys_mmap(VC_MAGIC_1024)
  let io: i64 = vc_items_add(ib, 0, cid)
  mvc_batch_add_pfx(cpfx, scol, ib, io)
857
  let mem: *u8 = sys_mmap(VC_MAGIC_65536)
  let mn: i64 = mvc_items_pfx(cpfx, scol, mem, VC_MAGIC_65536)
  var members: i64 = 0
  var resolved: i64 = 0
  let gp: *i64 = sys_mmap(16) as *i64
  let gl: *i64 = sys_mmap(16) as *i64
  let mcid: *u8 = sys_mmap(VC_CID_CAP)
  var ls: i64 = 0
  var i: i64 = 0
  while i <= mn {
      var eol: i64 = 0
      if i == mn { eol = 1 } else { if mem[i] == (10 as u8) { eol = 1 } }
      if eol == 1 {
          let hl: i64 = i - ls
          if hl > 0 {
              if hl < VC_CID_CAP - 1 {
                  members = members + 1
                  var k: i64 = 0
                  while k < hl { mcid[k] = mem[ls + k]; k = k + 1 }
                  mcid[hl] = 0 as u8
                  if mv_store_get(store, mcid, gp, gl) == 1 { resolved = resolved + 1 }
              }
          }
          ls = i + 1
      }
      i = i + 1
  }
  if members == 0 { fails = fails + 1 }
  if resolved != members { fails = fails + 1 }
  vc_w("T6 INVARIANT every collection member has a record -> " as *u8)
  vc_n(resolved); vc_w("/" as *u8); vc_n(members); vc_w("\n" as *u8)
889
  let ghost: *u8 = "nxc1-0000000000000000000000000000000000000000000000000000000000000000" as *u8
  if mv_store_get(store, ghost, gp, gl) == 1 { fails = fails + 1 }
  vc_w("T7 NEG never-ingested cid does NOT resolve\n" as *u8)
893
  let html: *u8 = "<html><body><img src=\"/a/one.png\"><a href=\"/next\">n</a><img src=\"https://cdn.x/two.jpg\"></body></html>" as *u8
  let outb: *u8 = sys_mmap(VC_MAGIC_8192)
  let an: i64 = mvf_extract(html, vc_strlen(html), "img" as *u8, "src" as *u8,
                            "" as *u8, outb, VC_MAGIC_8192)
  let acount: i64 = mvc_count_lines(outb, an)
  if acount != 2 { fails = fails + 1 }
  vc_w("T8 img srcs extracted (want 2) -> " as *u8); vc_n(acount); vc_w("\n" as *u8)
901
  let res: *u8 = sys_mmap(VC_URL_CAP)
  mvf_resolve("https://t.example/dir/page.html" as *u8, "/a/one.png" as *u8, res, VC_URL_CAP)
  if vc_eq(res, "https://t.example/a/one.png" as *u8) != 1 { fails = fails + 1 }
  vc_w("T9 relative -> absolute: " as *u8); vc_w(res); vc_w("\n" as *u8)
906
  // T17 -- protocol-relative //host/path inherits the base scheme (the fix for
  // the 8/14 Pima_cotton asset failures; the dominant CDN url shape)
  let pr: *u8 = sys_mmap(VC_URL_CAP)
  mvf_resolve("https://en.wikipedia.org/wiki/X" as *u8, "//upload.wikimedia.org/a/b.jpg" as *u8, pr, VC_URL_CAP)
  if vc_eq(pr, "https://upload.wikimedia.org/a/b.jpg" as *u8) != 1 { fails = fails + 1 }
  vc_w("T17 protocol-relative -> " as *u8); vc_w(pr); vc_w("\n" as *u8)
913
  let h1: *u8 = sys_mmap(VC_HOST_CAP)
  let h2: *u8 = sys_mmap(VC_HOST_CAP)
  mvf_host_of("https://t.example/a" as *u8, h1)
  mvf_host_of("https://evil.other/a" as *u8, h2)
  if vc_eq(h1, h2) == 1 { fails = fails + 1 }
  if vc_eq(h1, "t.example" as *u8) != 1 { fails = fails + 1 }
  vc_w("T10 same-host filter separates " as *u8); vc_w(h1)
  vc_w(" from " as *u8); vc_w(h2); vc_w("\n" as *u8)
922
  let small: *u8 = sys_mmap(64)
  let over: i64 = vc_items_add(small, VC_ITEMS_CAP - 4, cid)
  if over >= 0 { fails = fails + 1 }
  vc_w("T11 over-envelope membership add REFUSES -> " as *u8); vc_n(over); vc_w("\n" as *u8)
927
  let set: *u8 = "https://t.example/abc\n" as *u8
  if vc_seen(set, 21, "https://t.example/ab" as *u8) == 1 { fails = fails + 1 }
  if vc_seen(set, 21, "https://t.example/abc" as *u8) != 1 { fails = fails + 1 }
  vc_w("T12 visited-set matches whole lines only\n" as *u8)
932
  vc_w("verdict=" as *u8)
  if fails == 0 { vc_w("GREEN 17/17\n" as *u8) } else {
      vc_w("RED fails=" as *u8); vc_n(fails); vc_w("\n" as *u8)
  }
  if fails == 0 { return 0 }
  return 1
939}
940
941func vc_usage() -> i64 {
  vc_werr("usage:\n  nx_vault_capture media <url> [source]\n  nx_vault_capture edge  <url> [source]   (media, with the SSRF url guard in front -- for urls chosen by an untrusted page)\n  nx_vault_capture page  <url> [source] [max-assets]\n  nx_vault_capture site  <url> [source] [max-pages] [max-assets]\n  nx_vault_capture selftest\n" as *u8)
  return 2
944}
945
946func argp(argv: *i64, i: i64) -> *u8 { return (argv[i]) as *u8 }
947
948func vc_atoi(s: *u8) -> i64 {
  var v: i64 = 0
  var i: i64 = 0
  while s[i] != (0 as u8) {
      let c: i64 = s[i] as i64
      if c >= 48 { if c <= 57 { v = v * 10 + (c - 48) } }
      i = i + 1
  }
  return v
957}
958
959func main(argc: i64, argv: *i64) -> i64 {
  if argc < 2 { return vc_usage() }
  let verb: *u8 = argp(argv, 1)
962
  if vc_eq(verb, "selftest" as *u8) == 1 { return do_selftest() }
964
  if argc < 3 { return vc_usage() }
  let url: *u8 = argp(argv, 2)
967
  let store: *u8 = "knowledge/store/mvault-" as *u8
  let cpfx:  *u8 = "knowledge/store/mvcoll-" as *u8
970
  let ref_root:  *u8 = sys_mmap(VC_ROOT_CAP)
  let phys_root: *u8 = sys_mmap(VC_ROOT_CAP)
  vc_load_roots(ref_root, phys_root)
974
  var source: *u8 = "web" as *u8
  if argc > 3 { source = argp(argv, 3) }
977
  let store_i: i64 = hf_store_load()
  if store_i <= 0 {
      vc_werr("ERR: trust store load failed (data/mozilla_certdata.txt on CWD?)\n" as *u8)
      return 3
  }
983
  let report: *u8 = sys_mmap(VC_REPORT_CAP)
985
  // ---- THE UNTRUSTED DOOR (/compare/mediaingest R11 ge_edge_push) -------------------------------
  // `media` is the TRUSTED verb: an operator or an in-estate organ hands it a url it already vouches
  // for, and its behaviour here is byte-unchanged. `edge` is the SAME capture with an SSRF admission
  // check in front, and it exists because the browser-extension push hands this organ a url chosen by
  // whatever page the user happened to be looking at. Fetching that unguarded is a server-side request
  // forgery primitive pointed at our own LAN: this estate has already measured the shape once, when an
  // unauthenticated SSRF-capable fetcher bound to INADDR_ANY answered from another host.
  // The rule set is NOT re-implemented here -- it is nx_url_safety_lib, the one ruler shared with
  // nx_clean_serve_daemon and nx_cleanview, so a host one surface learns to refuse is refused here too.
  // The refusal NAMES WHICH RULE FIRED rather than saying "unsafe", because a guard whose refusals are
  // indistinguishable cannot be told apart from a guard that refuses everything.
  var do_media: i64 = 0
  if vc_eq(verb, "media" as *u8) == 1 { do_media = 1 }
  if vc_eq(verb, "edge" as *u8) == 1 {
      let ur: i64 = us_reason(url, vc_strlen(url))
      if ur != US_OK {
          vc_werr("EDGE REFUSED url-unsafe rule=" as *u8)
          vc_werr(us_reason_name(ur))
          vc_werr(" url=" as *u8)
          vc_werr(url)
          vc_werr(" -- refused BEFORE any fetch, so nothing on the private side was ever contacted\n" as *u8)
          return VC_EXIT_UNSAFE_URL
      }
      do_media = 1
  }
1011
  if do_media == 1 {
      let cid: *u8 = sys_mmap(VC_CID_CAP)
      let host: *u8 = sys_mmap(VC_HOST_CAP)
      mvf_host_of(url, host)
      let scol: *u8 = sys_mmap(VC_CID_CAP)
      mvc_declare_pfx(cpfx, "site" as *u8, host, host, host, url, "-" as *u8, scol)
1018
      let r: i64 = vc_capture_media(store_i, store, ref_root, phys_root, url,
                                    source, MV_CLASS_REAL, 1, cid)
      if r < 0 {
          vc_werr("ERR: capture failed (fetch or non-200)\n" as *u8)
          return 6
      }
      let ib: *u8 = sys_mmap(VC_MAGIC_1024)
      let io: i64 = vc_items_add(ib, 0, cid)
      if io > 0 { mvc_batch_add_pfx(cpfx, scol, ib, io) }
1028
      var o: i64 = vc_cat(report, 0, "{\"scope\":\"media\",\"cid\":\"" as *u8)
      o = vc_cat(report, o, cid)
      o = vc_cat(report, o, "\",\"site_col\":\"" as *u8)
      o = vc_cat(report, o, scol)
      o = vc_cat(report, o, "\",\"stored\":" as *u8)
      o = mv_u_dec(report, o, r)
      o = vc_cat(report, o, ",\"dup\":" as *u8)
      if r == 0 { o = mv_u_dec(report, o, 1) } else { o = mv_u_dec(report, o, 0) }
      o = vc_cat(report, o, "}" as *u8)
      report[o] = 10 as u8
      sys_write(1, report, o + 1)
      return 0
  }
1042
  if vc_eq(verb, "page" as *u8) == 1 {
      var ma: i64 = VC_DEF_ASSETS
      if argc > 4 { ma = vc_atoi(argp(argv, 4)) }
      if ma <= 0 { ma = VC_DEF_ASSETS }
      if ma > VC_MAX_ASSETS { ma = VC_MAX_ASSETS }
      let got: i64 = vc_capture_page(store_i, store, cpfx, ref_root, phys_root, url,
                                     source, ma, report, VC_REPORT_CAP)
      if got == (0 - 2) {
          vc_werr("ERR: page fetch failed\n" as *u8)
          return 6
      }
      sys_write(1, report, vc_strlen(report))
      return 0
  }
1057
  if vc_eq(verb, "site" as *u8) == 1 {
      var mp: i64 = VC_DEF_PAGES
      if argc > 4 { mp = vc_atoi(argp(argv, 4)) }
      if mp <= 0 { mp = VC_DEF_PAGES }
      if mp > VC_MAX_PAGES { mp = VC_MAX_PAGES }
      var ma: i64 = VC_DEF_ASSETS
      if argc > 5 { ma = vc_atoi(argp(argv, 5)) }
      if ma <= 0 { ma = VC_DEF_ASSETS }
      if ma > VC_MAX_ASSETS { ma = VC_MAX_ASSETS }
      vc_capture_site(store_i, store, cpfx, ref_root, phys_root, url, source,
                      mp, ma, report, VC_REPORT_CAP)
      sys_write(1, report, vc_strlen(report))
      return 0
  }
1072
  return vc_usage()
1074}
nx_vault_capture.nx source

`nx_vault_capture.nx` source
