code wiki / _hdl_build / nx_crawl_callee_probe.nx
`nx_crawl_callee_probe.nx` source

↩ module page · 203 lines · 9566 B
1// nx_crawl_callee_probe.nx -- PER-CALLEE VmSize PROBE for the crawler's per-page pipeline (debt 1787082132).
2// The +282 MB/min ingest growth survived the seg-store release-list fix; the named per-page mappings
3// (wc_harvest 8 KiB + olh_scan ~9.5 KiB) explain only ~20 KB/page, so the dominant allocator is in the
4// per-page CALLEES. This organ MEASURES each callee instead of reading it (the debt row's own law:
5// "do NOT hoist by reading"): run the SAME page bytes through each stage N times and diff
6// /proc/self/status VmSize around the loop. Per stage it prints BOTH numbers that matter:
7//   warm_kb   = VmSize cost of the FIRST call (one-time lazy tables/policy loads -- not a leak)
8//   steady per-call = (delta over N calls after warm) / N -- the number that scales with pages/min
9// Stages: H2T (nx_html_to_text), FEED (nx_feed_discover + nx_feed_extract), OLH (olh_scan),
10// FETCH (nx_https_fetch_follow -- only when /tmp/mozilla_certdata.txt exists AND the arg is a url;
11// otherwise that stage prints UNOBSERVABLE and abstains rather than acquitting).
12// argv: nx_crawl_callee_probe <page-file-or-https-url> [iters]   (file mode = no network, deterministic)
13// Output: one CALLEE line per stage; the LAST line names the worst steady grower (positional anchor).
14// license_tier: ORIGINAL  expect_exit: 0
15import "nx_x509_trust_store.nx"
16import "nx_trust_store_load_from_certdata.nx"
17import "nx_https_fetch_follow.nx"
18import "nx_outlink_harvest.nx"
19import "nx_feed_extract.nx"
20
21const CP_STATUSB:  i64 = 8192      // /proc/self/status read buffer
22const CP_PAGECAP:  i64 = 8388608   // page fixture cap == the crawler's CI_RAWCAP (largest raw file it reads)
23const CP_TXTCAP:   i64 = 1048576   // html_to_text output cap (crawler docs are <= ~900 KB)
24const CP_FEEDCAP:  i64 = 65536     // feed extractor output cap (== its own K_MAGIC_65536)
25const CP_URLCAP:   i64 = 2048      // discovered feed url
26const CP_EDGES:    i64 = 1024      // olh_scan edge slots (== OLH_MAXEDGE)
27const CP_ITERS:    i64 = 24        // default steady-state iterations per stage
28const CP_CERTDATA: *u8 = "/tmp/mozilla_certdata.txt"
29const CP_FETCH_HOPS: i64 = 5
30
31func cp_w(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
32static cp_nb_g: *u8
33func cp_wn(v: i64) -> i64 {
  if (cp_nb_g as i64) == 0 { cp_nb_g = sys_mmap(64) }
  var m: i64 = v
  if m < 0 { cp_w("-" as *u8); m = 0 - m }
  if m == 0 { cp_w("0" as *u8); return 0 }
  var k: i64 = 0
  while m > 0 { cp_nb_g[32 + k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
  var i: i64 = 0
  while i < k { cp_nb_g[i] = cp_nb_g[32 + k - 1 - i]; i = i + 1 }
  sys_write(1, cp_nb_g, k)
  return 0
44}
45
46// VmSize in kB from /proc/self/status. LINE-ANCHORED: the value is taken only from a line that STARTS
47// with the key (offset 0 or preceded by newline) -- the rm_field lesson, so prose can never be the answer.
48static cp_stb_g: *u8
49func cp_vmsize() -> i64 {
  if (cp_stb_g as i64) == 0 { cp_stb_g = sys_mmap(CP_STATUSB) }
  let fd: i64 = sys_openat_rd("/proc/self/status" as *u8)
  if fd < 0 { return 0 - 1 }
  let n: i64 = sys_read(fd, cp_stb_g, CP_STATUSB - 1)
  sys_close(fd)
  if n <= 0 { return 0 - 1 }
  let key: *u8 = "VmSize:" as *u8
  var i: i64 = 0
  while i < n {
      var atline: i64 = 0
      if i == 0 { atline = 1 } else { if cp_stb_g[i - 1] == (10 as u8) { atline = 1 } }
      if atline == 1 {
          var j: i64 = 0
          while key[j] != (0 as u8) { if i + j >= n { return 0 - 1 } if cp_stb_g[i + j] != key[j] { j = 0 - 1; break } j = j + 1 }
          if j > 0 {
              var p: i64 = i + j
              var v: i64 = 0
              var seen: i64 = 0
              while p < n {
                  let c: i64 = cp_stb_g[p] as i64
                  if c >= 48 { if c <= 57 { v = v * 10 + (c - 48); seen = 1; p = p + 1; continue } }
                  if seen == 1 { break }
                  if c == 10 { break }
                  p = p + 1
              }
              if seen == 1 { return v }
              return 0 - 1
          }
      }
      i = i + 1
  }
  return 0 - 1
82}
83
84func cp_slen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
85
86// one probe stage report line. delta values in kB (VmSize granularity).
87func cp_report(name: *u8, warm_kb: i64, steady_kb: i64, iters: i64) -> i64 {
  cp_w("CALLEE " as *u8); cp_w(name)
  cp_w(" warm_kb=" as *u8); cp_wn(warm_kb)
  cp_w(" steady_delta_kb=" as *u8); cp_wn(steady_kb)
  cp_w(" n=" as *u8); cp_wn(iters)
  cp_w(" per_call_kb=" as *u8); cp_wn(steady_kb / iters)
  cp_w("\n" as *u8)
  return 0
95}
96
97func main(argc: i64, argv: *i64) -> i64 {
  if argc < 2 {
      cp_w("usage: nx_crawl_callee_probe <page-file-or-https-url> [iters]\n" as *u8)
      return 2
  }
  let src: *u8 = argv[1] as *u8
  var iters: i64 = CP_ITERS
  if argc > 2 {
      let a2: *u8 = argv[2] as *u8
      var vv: i64 = 0
      var ii: i64 = 0
      while a2[ii] != (0 as u8) { let c: i64 = a2[ii] as i64; if c >= 48 { if c <= 57 { vv = vv * 10 + (c - 48) } } ii = ii + 1 }
      if vv > 0 { iters = vv }
  }
  var is_url: i64 = 0
  if src[0] == (104 as u8) { if src[1] == (116 as u8) { is_url = 1 } }   // "ht..." = url
113
  // ---- acquire the page fixture (before ANY measurement) ----
  let page: *u8 = sys_mmap(CP_PAGECAP)
  var plen: i64 = 0
  var store: *TrustStore = 0 as *TrustStore
  if is_url == 1 {
      let kfd: i64 = sys_openat_rd(CP_CERTDATA)
      if kfd < 0 {
          cp_w("FETCH UNOBSERVABLE: no /tmp/mozilla_certdata.txt -- pass a page FILE instead\n" as *u8)
          return 2
      }
      sys_close(kfd)
      let r: i64 = nx_trust_store_load_from_certdata(CP_CERTDATA, 300, 4194304)
      if r <= 0 { cp_w("FETCH UNOBSERVABLE: trust store load failed\n" as *u8); return 2 }
      store = r as *TrustStore
      let stp: *i64 = sys_mmap(16) as *i64
      plen = nx_https_fetch_follow(src, store, page, CP_PAGECAP, CP_FETCH_HOPS, stp)
      cp_w("fixture: fetched status=" as *u8); cp_wn(stp[0]); cp_w(" bytes=" as *u8); cp_wn(plen); cp_w("\n" as *u8)
  } else {
      let lp: *i64 = sys_mmap(16) as *i64
      let b: *u8 = sys_read_file(src, lp)
      if (b as i64) == 0 { cp_w("PAGE-FILE UNREADABLE\n" as *u8); return 2 }
      plen = lp[0]
      if plen > CP_PAGECAP { plen = CP_PAGECAP }
      var ci: i64 = 0
      while ci < plen { page[ci] = b[ci]; ci = ci + 1 }
      cp_w("fixture: file bytes=" as *u8); cp_wn(plen); cp_w("\n" as *u8)
  }
  if plen <= 0 { cp_w("EMPTY FIXTURE -- nothing to measure\n" as *u8); return 2 }
142
  // ---- output buffers, all allocated BEFORE the first measurement ----
  let txt: *u8 = sys_mmap(CP_TXTCAP)
  let feed: *u8 = sys_mmap(CP_FEEDCAP)
  let feedurl: *u8 = sys_mmap(CP_URLCAP)
  let edges: *i64 = sys_mmap(8 * CP_EDGES + 64) as *i64
  let base: *u8 = "https://nishifamily.com/" as *u8
  let blen: i64 = cp_slen(base)
150
  var worst_kb: i64 = 0 - 1
  var worst_name: *u8 = "none" as *u8
153
  // ---- H2T: nx_html_to_text ----
  var v0: i64 = cp_vmsize()
  nx_html_to_text(page, plen, txt, CP_TXTCAP)
  var v1: i64 = cp_vmsize()
  var it: i64 = 0
  while it < iters { nx_html_to_text(page, plen, txt, CP_TXTCAP); it = it + 1 }
  var v2: i64 = cp_vmsize()
  cp_report("nx_html_to_text" as *u8, v1 - v0, v2 - v1, iters)
  if (v2 - v1) / iters > worst_kb { worst_kb = (v2 - v1) / iters; worst_name = "nx_html_to_text" as *u8 }
163
  // ---- FEED: nx_feed_discover + nx_feed_extract ----
  v0 = cp_vmsize()
  nx_feed_discover(page, plen, feedurl, CP_URLCAP)
  nx_feed_extract(page, plen, feed, CP_FEEDCAP)
  v1 = cp_vmsize()
  it = 0
  while it < iters { nx_feed_discover(page, plen, feedurl, CP_URLCAP); nx_feed_extract(page, plen, feed, CP_FEEDCAP); it = it + 1 }
  v2 = cp_vmsize()
  cp_report("nx_feed_extract" as *u8, v1 - v0, v2 - v1, iters)
  if (v2 - v1) / iters > worst_kb { worst_kb = (v2 - v1) / iters; worst_name = "nx_feed_extract" as *u8 }
174
  // ---- OLH: olh_scan (the known ~9.5 KiB/call pair rides inside) ----
  v0 = cp_vmsize()
  olh_scan(page, plen, base, blen, edges, CP_EDGES)
  v1 = cp_vmsize()
  it = 0
  while it < iters { olh_scan(page, plen, base, blen, edges, CP_EDGES); it = it + 1 }
  v2 = cp_vmsize()
  cp_report("olh_scan" as *u8, v1 - v0, v2 - v1, iters)
  if (v2 - v1) / iters > worst_kb { worst_kb = (v2 - v1) / iters; worst_name = "olh_scan" as *u8 }
184
  // ---- FETCH: nx_https_fetch_follow (url mode only; store already loaded above) ----
  if is_url == 1 {
      let stf: *i64 = sys_mmap(16) as *i64
      v0 = cp_vmsize()
      nx_https_fetch_follow(src, store, txt, CP_TXTCAP, CP_FETCH_HOPS, stf)
      v1 = cp_vmsize()
      it = 0
      while it < iters { nx_https_fetch_follow(src, store, txt, CP_TXTCAP, CP_FETCH_HOPS, stf); it = it + 1 }
      v2 = cp_vmsize()
      cp_report("nx_https_fetch_follow" as *u8, v1 - v0, v2 - v1, iters)
      if (v2 - v1) / iters > worst_kb { worst_kb = (v2 - v1) / iters; worst_name = "nx_https_fetch_follow" as *u8 }
  } else {
      cp_w("CALLEE nx_https_fetch_follow UNOBSERVABLE (file-mode fixture; pass a url + certdata to measure)\n" as *u8)
  }
199
  // LAST LINE = the verdict the reader anchors on (positional, gv_last_line discipline).
  cp_w("WORST " as *u8); cp_w(worst_name); cp_w(" per_call_kb=" as *u8); cp_wn(worst_kb); cp_w("\n" as *u8)
  return 0
203}
nx_crawl_callee_probe.nx source

`nx_crawl_callee_probe.nx` source
