code wiki / _hdl_build / nx_health_eval.nx
`nx_health_eval.nx` source

↩ module page · 565 lines · 36698 B
1// nx_health_eval.nx -- SOVEREIGN continuous M&E evaluator (operator 2026-07-04: "we have no monitoring
2// and evaluation ... fire and pray for all our tools ... same with web hosting" + "use the apis not
3// awsudo/shell"). Probes the WHOLE live hosting+video surface using the sovereign nx_https_get primitive
4// (TLS-1.3, own trust store -- NO curl, NO awsudo, NO shell), checks each flow's real content (200 +
5// marker; a JS asset MUST NOT be text/html = the "app.js 404 -> Unexpected token <" class), rolls up a
6// single health verdict, writes a served health.json, AND records the pass-count to the metric ledger so
7// every run reports a BETTER/WORSE trend. Designed to run continuously (hostctl-supervised, the sovereign
8// supervisor -- not a shell cron) + register in the tools API. This converts "fire and pray" into
9// measured-every-run. expect_exit: 0 when all critical flows pass.  license_tier: ORIGINAL
10import "nx_https_get.nx"
11import "nx_connect.nx"   // bounded connect: a raw sys_connect hangs ~127s on a black-holed host
12import "nx_trust_store_load_from_certdata.nx"
13import "nx_metric_ledger.nx"
14import "nx_ui_checks.nx"
15import "nx_websocket_client_upgrade.nx"   // seq1135 relay probe: real WS clients through the live relay
16import "nx_websocket_stream.nx"
17import "nx_websocket_frame.nx"   // TOP-DOWN UI eye folded into the continuous loop: verify the LIVE page's buttons
18const HE_MAGIC_8445: i64 = 8445
19const HE_MAGIC_2048: i64 = 2048
20const HE_MAGIC_4096: i64 = 4096
21const HE_MAGIC_8443: i64 = 8443
22const HE_MAGIC_999999: i64 = 999999
23const HE_MAGIC_65536: i64 = 65536
24const HE_MAGIC_1000000000: i64 = 1000000000
25const HE_MAGIC_4194304: i64 = 4194304
                         // are wired + init crash-safe + not truncated every pass (not just serving)
27
28const HE_CAP: i64 = 262144
29// write DIRECTLY to the served location (relative to the server CWD = nishihost): so the supervised loop
30// refreshes the live surface in place every pass. Same relative path on dev (nxc2/sites/...) = the source.
31const HE_HEALTH_OUT: *u8 = "sites/nishifamily/health.json"
32
33func hw(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
34func hn(v: i64) -> i64 {
  let b: *u8=sys_mmap(28); var m: i64=v; if m<0{sys_write(1,"-" as *u8,1);m=0-m}
  let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=48 as u8;k=1} while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1}
  var i: i64=0; while i<k{b[i]=t[k-1-i];i=i+1} sys_write(1,b,k); return 0 }
38func he_has(hay: *u8, n: i64, needle: *u8) -> i64 {
  var nl: i64=0; while needle[nl]!=(0 as u8){nl=nl+1}
  var i: i64=0
  while i+nl<=n { var j: i64=0; var ok: i64=1; while j<nl { if hay[i+j]!=needle[j]{ok=0;j=nl} else {j=j+1} } if ok==1{return 1} i=i+1 }
  return 0 }
43
44// LOOPBACK GET: connect to 127.0.0.1:8443 (sites.elf directly -- reliable from the NAS, NO hairpin/NAT
45// dependency) while SNI + Host + cert-verify use the REAL hostname nishifamily.com (cert is valid there).
46// This is the fix for the hairpin false-RED: the NAS cannot reach its own PUBLIC hostname, but it can
47// always reach loopback -- and loopback:8443 is the actual content server, exactly where the bugs live.
48func he_get_lo(host: *u8, host_len: i64, path: *u8, path_len: i64, store: *TrustStore, now: i64,
             cr: *u8, pk: *u8, out: *u8, cap: i64) -> i64 {
  let fd: i64 = sys_socket(2, 1, 0)          // AF_INET, SOCK_STREAM
  if fd < 0 { return 0 - 1 }
  let sa: *u8 = sys_mmap(16)
  sa[0]=2 as u8; sa[1]=0 as u8
  sa[2]=((HE_MAGIC_8443>>8)&0xff) as u8; sa[3]=(HE_MAGIC_8443&0xff) as u8
  sa[4]=127 as u8; sa[5]=0 as u8; sa[6]=0 as u8; sa[7]=1 as u8
  var zi: i64=8; while zi<16 { sa[zi]=0 as u8; zi=zi+1 }
  if nx_connect_bounded(fd, sa, 16, NX_CONN_DEFAULT_MS) < 0 { sys_close(fd); return 0 - 1 }
  let vc_raw: *u8 = sys_mmap(64)
  let vc: *TlsValidationContext = vc_raw as *TlsValidationContext
  vc.store = store; vc.sni_host = host; vc.sni_host_len = host_len; vc.now_epoch = now
  let sr: i64 = nx_tls13_client_session_run(fd, host, host_len, cr, pk, vc)
  if sr <= 0 { sys_close(fd); return 0 - 1 }
  let session: *Tls13ClientSession = sr as *Tls13ClientSession
  let n: i64 = nx_https_get_complete(session, fd, path, path_len, host, host_len, out, cap)
  sys_close(fd)
  return n
67}
68func he_slen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
69
70// one flow: loopback-GET path, verify "200" status + `marker` present. is_js=1 -> ALSO require NOT text/html.
71// returns 1 pass / 0 fail. writes a JSON row into jout at *joff (advances it).
72func he_check(name: *u8, path: *u8, marker: *u8, is_js: i64, store: *TrustStore, now: i64,
            out: *u8, jout: *u8, joff: *i64) -> i64 {
  let cr: *u8 = sys_mmap(32); var i: i64=0; while i<32 { cr[i]=(0x40 + i) as u8; i=i+1 }
  let pk: *u8 = sys_mmap(32); i=0; while i<32 { pk[i]=(0x80 + i) as u8; i=i+1 }
  // loopback-first (reliable on the NAS, no hairpin), public-fallback (works from dev / any host that
  // can reach the WAN) -> the SAME organ gates on dev AND runs correct on the NAS.
  var rn: i64 = he_get_lo("nishifamily.com\x00" as *u8, 15, path, he_slen(path), store, now, cr, pk, out, HE_CAP)
  if rn <= 0 {
      let fu: *u8 = sys_mmap(512); var fo: i64 = 0
      let pfx: *u8 = "https://nishifamily.com" as *u8; var z: i64=0; while pfx[z]!=(0 as u8){fu[fo]=pfx[z];fo=fo+1;z=z+1}
      z=0; while path[z]!=(0 as u8){fu[fo]=path[z];fo=fo+1;z=z+1}
      fu[fo]=0 as u8
      rn = nx_https_get(fu, cr, pk, store, now, out, HE_CAP)
  }
  var pass: i64 = 0
  var reason: *u8 = "conn-fail" as *u8
  if rn > 0 {
      let ok200: i64 = he_has(out, rn, "200" as *u8)
      let okmark: i64 = he_has(out, rn, marker)
      var okjs: i64 = 1
      if is_js == 1 { if he_has(out, rn, "text/html" as *u8) == 1 { okjs = 0 } }
      if ok200 == 1 { if okmark == 1 { if okjs == 1 { pass = 1 } else { reason = "js-served-as-html" as *u8 } } else { reason = "marker-absent" as *u8 } } else { reason = "no-200" as *u8 }
  }
  hw("  " as *u8); hw(name)
  if pass == 1 { hw("  PASS (" as *u8); hn(rn); hw("B)\n" as *u8) } else { hw("  FAIL " as *u8); hw(reason); hw("\n" as *u8) }
  // JSON row: {"name":"..","ok":0/1,"bytes":N,"why":".."},
  var o: i64 = joff[0]
  let jr0: *u8 = "    {\"name\":\"" as *u8; var z: i64=0; while jr0[z]!=(0 as u8){jout[o]=jr0[z];o=o+1;z=z+1}
  z=0; while name[z]!=(0 as u8){jout[o]=name[z];o=o+1;z=z+1}
  let jr1: *u8 = "\",\"ok\":" as *u8; z=0; while jr1[z]!=(0 as u8){jout[o]=jr1[z];o=o+1;z=z+1}
  jout[o]=(0x30 + pass) as u8; o=o+1
  let jr2: *u8 = ",\"why\":\"" as *u8; z=0; while jr2[z]!=(0 as u8){jout[o]=jr2[z];o=o+1;z=z+1}
  if pass==1 { jout[o]=0x6F as u8; o=o+1; jout[o]=0x6B as u8; o=o+1 } else { z=0; while reason[z]!=(0 as u8){jout[o]=reason[z];o=o+1;z=z+1} }
  let jr3: *u8 = "\"},\n" as *u8; z=0; while jr3[z]!=(0 as u8){jout[o]=jr3[z];o=o+1;z=z+1}
  joff[0] = o
  return pass
108}
109
110// fetch a path loopback-first (NAS) with public fallback (dev) -- mirrors he_check's fetch, for the UI eye.
111func he_fetch_one(path: *u8, store: *TrustStore, now: i64, out: *u8, cap: i64) -> i64 {
  let cr: *u8 = sys_mmap(32); var i: i64=0; while i<32 { cr[i]=(0x40+i) as u8; i=i+1 }
  let pk: *u8 = sys_mmap(32); i=0; while i<32 { pk[i]=(0x80+i) as u8; i=i+1 }
  var rn: i64 = he_get_lo("nishifamily.com\x00" as *u8, 15, path, he_slen(path), store, now, cr, pk, out, cap)
  if rn <= 0 {
      let fu: *u8 = sys_mmap(512); var fo: i64=0
      let pfx: *u8="https://nishifamily.com" as *u8; var z: i64=0; while pfx[z]!=(0 as u8){fu[fo]=pfx[z];fo=fo+1;z=z+1}
      z=0; while path[z]!=(0 as u8){fu[fo]=path[z];fo=fo+1;z=z+1}
      fu[fo]=0 as u8
      rn = nx_https_get(fu, cr, pk, store, now, out, cap)
  }
  return rn }
123
124// TOP-DOWN UI eye (redundant with the bottom-up nx_ui_wiring_census): fetch the LIVE /video + app.v2.js
125// and assert every <button id> is wired + init crash-safe + not truncated. Catches dead buttons / deploy
126// drift on what the USER receives -- the class serving-checks (200+marker) miss. Writes a health.json row.
127func he_ui_check(store: *TrustStore, now: i64, jout: *u8, joff: *i64) -> i64 {
  let hbuf: *u8 = sys_mmap(HE_CAP); let jbuf: *u8 = sys_mmap(HE_CAP)
  let hnv: i64 = he_fetch_one("/video\x00" as *u8, store, now, hbuf, HE_CAP)
  let jnv: i64 = he_fetch_one("/video/app.v2.js\x00" as *u8, store, now, jbuf, HE_CAP)
  var pass: i64 = 0
  var reason: *u8 = "fetch-fail" as *u8
  if hnv > 0 { if jnv > 0 {
      let tally: *i64 = sys_mmap(32) as *i64; tally[0]=0; tally[1]=0
      uc_check_buttons(hbuf, hnv, jbuf, jnv, tally)
      let init_bad: i64 = uc_check_init(jbuf, jnv)
      let intact: i64 = uc_has(jbuf, jnv, "})();" as *u8)
      if tally[1]==0 { if init_bad==0 { if intact==1 { if tally[0]>0 { pass=1 } else { reason="no-buttons" as *u8 } } else { reason="truncated" as *u8 } } else { reason="init-unsafe" as *u8 } } else { reason="dead-button" as *u8 }
  } }
  hw("  ui_wiring " as *u8); if pass==1 { hw("PASS (live buttons wired+safe+intact)\n" as *u8) } else { hw("FAIL " as *u8); hw(reason); hw("\n" as *u8) }
  // health.json row
  let o0: i64 = joff[0]; var o: i64 = o0
  let r0: *u8="  {\"name\":\"ui_wiring\",\"ok\":" as *u8; var z2: i64=0; while r0[z2]!=(0 as u8){jout[o]=r0[z2];o=o+1;z2=z2+1}
  if pass==1 { jout[o]=49 as u8 } else { jout[o]=48 as u8 } o=o+1
  let r1: *u8=",\"why\":\"" as *u8; z2=0; while r1[z2]!=(0 as u8){jout[o]=r1[z2];o=o+1;z2=z2+1}
  if pass==1 { let ok: *u8="ok" as *u8; z2=0; while ok[z2]!=(0 as u8){jout[o]=ok[z2];o=o+1;z2=z2+1} } else { z2=0; while reason[z2]!=(0 as u8){jout[o]=reason[z2];o=o+1;z2=z2+1} }
  let r2: *u8="\"},\n" as *u8; z2=0; while r2[z2]!=(0 as u8){jout[o]=r2[z2];o=o+1;z2=z2+1}
  joff[0]=o
  return pass }
150
151// ---- CALL-QOE AGGREGATION (operator 2026-07-05: a real 3-person call hit 4fps + an invisible peer and
152// NOTHING showed it). Clients beacon {"type":"qoe",...} every ~10s; the signaling relay tees each one into
153// room_telemetry.log (QOE t=<sec> room=<r> {json}). This flow reads the log TAIL, takes the latest beacon,
154// and turns it into a dashboard row + ledger trend: a 4fps participant or a roster/tiles mismatch (mm=1,
155// the invisible-peer class) is now a RED row on /video/health within minutes -- measured, not reported. ----
156const HE_QOE_LOG: *u8 = "/volume1/homes/elderwesto/nishihost/room_telemetry.log"
157const HE_QOE_MIN_FPS: i64 = 8          // below ~8fps conversational video reads as a slideshow (field-cited 2026-07-05)
158const HE_QOE_FRESH_S: i64 = 3600       // only judge calls from the last hour; older = informational
159
160// parse the signed integer right after `key` inside buf[from,to). returns -999999 if absent.
161func he_qi(buf: *u8, from: i64, to: i64, key: *u8) -> i64 {
  let kl: i64 = he_slen(key)
  var i: i64 = from
  var at: i64 = 0 - 1
  while i + kl <= to { var j: i64=0; var ok: i64=1; while j<kl { if buf[i+j]!=key[j]{ok=0;j=kl} else {j=j+1} } if ok==1 { at=i+kl; i=to } i=i+1 }
  if at < 0 { return 0 - HE_MAGIC_999999 }
  var neg: i64 = 0
  if buf[at] == (45 as u8) { neg = 1; at = at + 1 }
  var v: i64 = 0
  var any: i64 = 0
  while at < to { let c: i64 = buf[at] & 0xff; if c >= 48 { if c <= 57 { v = v*10 + (c-48); any=1; at=at+1 } else { at=to } } else { at=to } }
  if any == 0 { return 0 - HE_MAGIC_999999 }
  if neg == 1 { return 0 - v }
  return v }
175func he_wn(dst: *u8, off: *i64, v: i64) -> i64 {
  var m: i64 = v
  if m < 0 { dst[off[0]] = 45 as u8; off[0]=off[0]+1; m = 0 - m }
  let t: *u8 = sys_mmap(28); var k: i64=0; if m==0 { t[0]=48 as u8; k=1 } while m>0 { t[k]=(48+(m%10)) as u8; m=m/10; k=k+1 }
  var i: i64=0; while i<k { dst[off[0]]=t[k-1-i]; off[0]=off[0]+1; i=i+1 }
  return 0 }
181func he_ws2(dst: *u8, off: *i64, s: *u8) -> i64 { var i: i64=0; while s[i]!=(0 as u8) { dst[off[0]]=s[i]; off[0]=off[0]+1; i=i+1 } return 0 }
182
183// one verdict flow: console line + health.json row {"name":<n>,"ok":0|1,"why":<w>}. Shared by
184// call_qoe + video_canary (he_check emits its own richer row with status/bytes fields).
185func he_flow_row(name: *u8, ok: i64, why: *u8, jout: *u8, joff: *i64) -> i64 {
  hw("  " as *u8); hw(name); hw(" " as *u8)
  if ok==1 { hw("PASS " as *u8) } else { hw("FAIL " as *u8) }
  hw(why); hw("
189" as *u8)
  var o: i64 = joff[0]
  let a: *u8 = "  {\"name\":\"" as *u8; var z: i64=0; while a[z]!=(0 as u8){jout[o]=a[z];o=o+1;z=z+1}
  z=0; while name[z]!=(0 as u8){jout[o]=name[z];o=o+1;z=z+1}
  let b: *u8 = "\",\"ok\":" as *u8; z=0; while b[z]!=(0 as u8){jout[o]=b[z];o=o+1;z=z+1}
  if ok==1 { jout[o]=49 as u8 } else { jout[o]=48 as u8 } o=o+1
  let c: *u8 = ",\"why\":\"" as *u8; z=0; while c[z]!=(0 as u8){jout[o]=c[z];o=o+1;z=z+1}
  z=0; while why[z]!=(0 as u8){jout[o]=why[z];o=o+1;z=z+1}
  let d: *u8 = "\"},
198" as *u8; z=0; while d[z]!=(0 as u8){jout[o]=d[z];o=o+1;z=z+1}
  joff[0]=o
  return 0
201}
202
203func he_qoe_check(now: i64, jout: *u8, joff: *i64) -> i64 {
  let box: *i64 = sys_mmap(16) as *i64
  let raw: *u8 = sys_read_file(HE_QOE_LOG, box)
  var pass: i64 = 1
  let why: *u8 = sys_mmap(512)
  let wo: *i64 = sys_mmap(16) as *i64; wo[0]=0
  if (raw as i64) == 0 { he_ws2(why, wo, "no-telemetry-log (no calls yet)" as *u8) }
  else {
      let n: i64 = box[0]
      var from: i64 = 0
      if n > HE_MAGIC_65536 { from = n - HE_MAGIC_65536 }          // tail window: bounded work on a growing log
      // collect the LAST up-to-16 "QOE t=" line starts in the window (ring) -> judge the WHOLE call
      // window worst-of, not just the final beacon (a call is many participants x many beacons).
      let ring: *i64 = sys_mmap(256) as *i64
      var rc2: i64 = 0
      var i: i64 = from
      while i + 6 <= n {
          if raw[i]==(81 as u8) { if raw[i+1]==(79 as u8) { if raw[i+2]==(69 as u8) { if raw[i+3]==(32 as u8) { if raw[i+4]==(116 as u8) { if raw[i+5]==(61 as u8) { ring[rc2 % 16] = i; rc2 = rc2 + 1 } } } } } }
          i = i + 1
      }
      if rc2 == 0 { he_ws2(why, wo, "no-qoe-beacons-yet" as *u8) }
      else {
          // newest line's timestamp anchors freshness. EPOCH GUARD (first field beacons 2026-07-05
          // exposed the tee stamping MONOTONIC time): t < 1e9 = not an epoch -> age UNKNOWN -> still
          // JUDGE (a misjudged stale window self-heals next pass; a skipped bad call hides forever).
          var cnt: i64 = rc2; if cnt > 12 { cnt = 12 }
          var worstTx: i64 = 0 - 1
          var worstRx: i64 = 0 - 1
          var rttMax: i64 = 0 - 1
          var bpMax: i64 = 0 - 1
          var mmAny: i64 = 0
          var vcMin: i64 = 0 - 1
          var encMax: i64 = 0 - 1
          var newestT: i64 = 0 - 1
          var peersMax: i64 = 0 - 1
          var bgAny: i64 = 0
          var wkMin: i64 = 0 - 1
          var k: i64 = 0
          while k < cnt {
              let ls2: i64 = ring[((rc2 - 1 - k) % 16 + 16) % 16]
              var le2: i64 = ls2
              var g: i64 = 1
              while g==1 { if le2<n { if raw[le2]!=(10 as u8) { le2=le2+1 } else { g=0 } } else { g=0 } }
              let t: i64 = he_qi(raw, ls2, le2, "QOE t=" as *u8)
              if k == 0 { newestT = t }
              let fmin: i64 = he_qi(raw, ls2, le2, "\"fpsRxMin\":" as *u8)
              let ftx: i64 = he_qi(raw, ls2, le2, "\"fpsTx\":" as *u8)
              let rtt: i64 = he_qi(raw, ls2, le2, "\"rtt\":" as *u8)
              let bp: i64 = he_qi(raw, ls2, le2, "\"bp\":" as *u8)
              let mm: i64 = he_qi(raw, ls2, le2, "\"mm\":" as *u8)
              let vc: i64 = he_qi(raw, ls2, le2, "\"vc57\":" as *u8)
              let em: i64 = he_qi(raw, ls2, le2, "\"encMs\":" as *u8)
              let vo: i64 = he_qi(raw, ls2, le2, "\"vo\":" as *u8)
              let prs: i64 = he_qi(raw, ls2, le2, "\"peers\":" as *u8)
              let bg: i64 = he_qi(raw, ls2, le2, "\"bg\":" as *u8)
              let wkf: i64 = he_qi(raw, ls2, le2, "\"wk\":" as *u8)
              // only lines within the fresh window of the newest count toward the verdict
              var inwin: i64 = 1
              if newestT > HE_MAGIC_1000000000 { if t > HE_MAGIC_1000000000 { if newestT - t > HE_QOE_FRESH_S { inwin = 0 } } }
              if inwin == 1 {
                  // a BACKGROUNDED tab (bg=1) legitimately collapses to ~1fps (browser timer clamp) --
                  // its fps lines are EXEMPT from the verdict (never a false-RED from an idle tab), but
                  // its mm/vc/bp still count. vo=0 (audio-only plan) exempts fpsTx the same way.
                  if bg != 1 {
                      if ftx >= 0 { if vo != 0 { if worstTx < 0 { worstTx = ftx } if ftx < worstTx { worstTx = ftx } } }
                      if fmin >= 0 { if worstRx < 0 { worstRx = fmin } if fmin < worstRx { worstRx = fmin } }
                  }
                  if bg == 1 { bgAny = 1 }
                  if rtt > rttMax { rttMax = rtt }
                  if bp > bpMax { bpMax = bp }
                  if mm == 1 { mmAny = 1 }
                  if vc >= 0 { if vcMin < 0 { vcMin = vc } if vc < vcMin { vcMin = vc } }
                  if em > encMax { encMax = em }
                  if prs > peersMax { peersMax = prs }
                  if wkf >= 0 { if wkMin < 0 { wkMin = wkf } if wkf < wkMin { wkMin = wkf } }
              }
              k = k + 1
          }
          var age: i64 = 0 - 1
          if newestT > HE_MAGIC_1000000000 { age = now - newestT }
          var judge: i64 = 0
          if age >= 0 { if age <= HE_QOE_FRESH_S { judge = 1 } }
          if age < 0 { judge = 1 }                       // age unknown (monotonic-era lines) -> judge anyway
          if peersMax == 0 { judge = 0 }                 // someone ALONE in a room = not a call; nothing to judge
          if judge == 1 {
              if mmAny == 1 { pass = 0 }
              if worstRx >= 0 { if worstRx < HE_QOE_MIN_FPS { pass = 0 } }
              if worstTx >= 0 { if worstTx < HE_QOE_MIN_FPS { pass = 0 } }
              if worstRx >= 0 { ml_report("call_fps_rx_min" as *u8, worstRx, 1, "fps" as *u8, "nx_health_eval" as *u8) }
              if worstTx >= 0 { ml_report("call_fps_tx" as *u8, worstTx, 1, "fps" as *u8, "nx_health_eval" as *u8) }
              if rttMax >= 0 { ml_report("call_rtt_ms" as *u8, rttMax, 0, "ms" as *u8, "nx_health_eval" as *u8) }
          }
          he_ws2(why, wo, "n=" as *u8); he_wn(why, wo, cnt)
          he_ws2(why, wo, " worstTx=" as *u8); he_wn(why, wo, worstTx)
          he_ws2(why, wo, " worstRx=" as *u8); he_wn(why, wo, worstRx)
          he_ws2(why, wo, " rttMax=" as *u8); he_wn(why, wo, rttMax)
          he_ws2(why, wo, " bpMax=" as *u8); he_wn(why, wo, bpMax)
          he_ws2(why, wo, " mm=" as *u8); he_wn(why, wo, mmAny)
          he_ws2(why, wo, " vc57=" as *u8); he_wn(why, wo, vcMin)
          he_ws2(why, wo, " encMsMax=" as *u8); he_wn(why, wo, encMax)
          he_ws2(why, wo, " peers=" as *u8); he_wn(why, wo, peersMax)
          he_ws2(why, wo, " bg=" as *u8); he_wn(why, wo, bgAny)
          he_ws2(why, wo, " wk=" as *u8); he_wn(why, wo, wkMin)
          he_ws2(why, wo, " age_s=" as *u8); he_wn(why, wo, age)
      }
  }
  why[wo[0]] = 0 as u8
  he_flow_row("call_qoe" as *u8, pass, why, jout, joff)
  return pass }
312
313// seq1100/seq1073 VIDEO CANARY (2026-07-27): the 5-day dead-product incident -- the global CSP silently
314// revoked wasm+workers and every serving check stayed GREEN (a 200 + marker does not EXECUTE policy).
315// call_qoe cannot catch it either: with stale beacons it sets judge=0 and passes, so "unused" and
316// "broken" read identically. This canary asserts the PRODUCT CAN WORK, deterministically, every pass,
317// independent of whether anyone called today: (1) the /video response (raw, headers included) carries
318// the wasm-enabling CSP tokens; (2) the served codec wasm is a real >=2KB \0asm module. A policy or
319// asset regression fails the SAME 300s pass it ships in, instead of surfacing as a quiet week of AMBER.
320// seq1135 RELAY DATA-PLANE PROBE -- proves the media relay RELAYS, not merely listens (the 2026-07-28
321// wedge: 2.9GB daemon, port open, zero frames moving, every "UP" check green). Two real WS clients join
322// a probe room on loopback :8445; A registers + sends one 0x57 video frame; B must RECEIVE the relayed
323// copy inside its 3s socket timeout. A complete data-plane roundtrip, or FAIL.
324func he_relay_join(path: *u8, plen: i64, rq: *u8, rs: *u8) -> i64 {
  let fd: i64 = sys_socket(2, 1, 0)
  if fd < 0 { return 0 - 1 }
  let ad: *u8 = sys_mmap(16)
  ad[0]=2 as u8; ad[1]=0 as u8; ad[2]=((HE_MAGIC_8445>>8)&0xff) as u8; ad[3]=(HE_MAGIC_8445&0xff) as u8
  ad[4]=127 as u8; ad[5]=0 as u8; ad[6]=0 as u8; ad[7]=1 as u8
  var zi: i64=8
  while zi<16 { ad[zi]=0 as u8; zi=zi+1 }
  if nx_connect_bounded(fd, ad, 16, NX_CONN_DEFAULT_MS) < 0 { sys_close(fd); return 0 - 1 }
  if nx_ws_client_upgrade(fd, "127.0.0.1" as *u8, 9, path, plen, HE_MAGIC_8445, 1, rq, HE_MAGIC_2048, rs, HE_MAGIC_4096) != NX_WSCU_OK { sys_close(fd); return 0 - 1 }
  let tv: *i64 = sys_mmap(16) as *i64
  tv[0]=3; tv[1]=0
  sys_setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, tv as *u8, 16)
  return fd }
338func he_relay_probe(jout: *u8, joff: *i64) -> i64 {
  let rq: *u8 = sys_mmap(HE_MAGIC_2048)
  let rs: *u8 = sys_mmap(HE_MAGIC_4096)
  let rbuf: *u8 = sys_mmap(HE_MAGIC_65536)
  var pass: i64 = 0
  let why: *u8 = sys_mmap(128)
  let wo: *i64 = sys_mmap(16) as *i64
  wo[0]=0
  let pth: *u8 = "/signal/_hprobe" as *u8
  let fa: i64 = he_relay_join(pth, 15, rq, rs)
  if fa < 0 { he_ws2(why, wo, "peerA-join-fail" as *u8) } else {
      let fb: i64 = he_relay_join(pth, 15, rq, rs)
      if fb < 0 { he_ws2(why, wo, "peerB-join-fail" as *u8) } else {
          let fr: *u8 = sys_mmap(64)
          var i: i64 = 0
          fr[0]=0x43 as u8
          i=0
          while i<8 { fr[1+i]=(66+i) as u8; i=i+1 }
          i=9
          while i<16 { fr[i]=0 as u8; i=i+1 }
          nx_ws_send_frame_to_fd(fb, 1, WS_OP_BINARY, fr, 16)
          i=0
          while i<8 { fr[1+i]=(97+i) as u8; i=i+1 }
          nx_ws_send_frame_to_fd(fa, 1, WS_OP_BINARY, fr, 16)
          sys_sleep_ms(150)
          fr[0]=0x57 as u8
          i=0
          while i<8 { fr[1+i]=(97+i) as u8; i=i+1 }
          fr[9]=1 as u8; fr[10]=0 as u8; fr[11]=0 as u8; fr[12]=0 as u8
          fr[13]=1 as u8; fr[14]=20 as u8; fr[15]=64 as u8; fr[16]=1 as u8; fr[17]=240 as u8; fr[18]=0 as u8
          i=19
          while i<32 { fr[i]=((i*3)&255) as u8; i=i+1 }
          nx_ws_send_frame_to_fd(fa, 1, WS_OP_BINARY, fr, 32)
          var tries: i64 = 0
          while tries < 5 {
              let f_raw: *u8 = sys_mmap(128)
              let f: *WsFrame = f_raw as *WsFrame
              if nx_ws_read_frame_from_fd(fb, rbuf, HE_MAGIC_65536, f) != NX_WSS_OK { tries = 5 } else {
                  if f.payload_len >= 32 { if rbuf[f.payload_off] == (0x57 as u8) { pass = 1; tries = 5 } }
                  tries = tries + 1 }
          }
          if pass == 1 { he_ws2(why, wo, "frame-relayed-roundtrip-ok" as *u8) } else { he_ws2(why, wo, "NO relayed frame -- relay up but not relaying" as *u8) }
          sys_close(fb) }
      sys_close(fa) }
  why[wo[0]] = 0 as u8
  he_flow_row("relay_dataplane" as *u8, pass, why, jout, joff)
  return pass }
385
386// 2026-07-29 (eaten seq932's monitoring gap): the hub gateway was ABSENT from this 14-service
387// inventory while it grew to 9.78 GiB and exhausted host swap -- a leak nobody was watching. Data-plane
388// tooth: loopback GET /hub on the gateway port must answer HTTP 200 with a real body (>=800B; the live
389// page is ~2182B). Connect-refused / short / non-200 all name themselves in the why.
390const HE_HUB_PORT: i64 = 18792
391func he_hub_probe(jout: *u8, joff: *i64) -> i64 {
  var pass: i64 = 0
  let why: *u8 = sys_mmap(128)
  let wo: *i64 = sys_mmap(16) as *i64
  wo[0]=0
  let fd: i64 = sys_socket(2, 1, 0)
  if fd < 0 { he_ws2(why, wo, "socket-fail " as *u8) } else {
      let ad: *u8 = sys_mmap(16)
      ad[0]=2 as u8; ad[1]=0 as u8; ad[2]=((HE_HUB_PORT>>8)&0xff) as u8; ad[3]=(HE_HUB_PORT&0xff) as u8
      ad[4]=127 as u8; ad[5]=0 as u8; ad[6]=0 as u8; ad[7]=1 as u8
      var zi: i64=8
      while zi<16 { ad[zi]=0 as u8; zi=zi+1 }
      if nx_connect_bounded(fd, ad, 16, NX_CONN_DEFAULT_MS) < 0 { he_ws2(why, wo, "hub-connect-refused " as *u8) } else {
          let tv: *i64 = sys_mmap(16) as *i64
          tv[0]=3; tv[1]=0
          sys_setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, tv as *u8, 16)
          let rq: *u8 = "GET /hub HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: close\r\n\r\n" as *u8
          var rl: i64 = 0
          while rq[rl] != (0 as u8) { rl = rl + 1 }
          sys_write(fd, rq, rl)
          let rb: *u8 = sys_mmap(HE_MAGIC_65536)
          var got: i64 = 0
          var rr: i64 = 1
          while rr > 0 { rr = sys_read(fd, ((rb as i64)+got) as *u8, HE_MAGIC_65536 - got - 1); if rr > 0 { got = got + rr } if got >= HE_MAGIC_65536 - 1 { rr = 0 } }
          if got < 64 { he_ws2(why, wo, "hub-short-read " as *u8) } else {
              if he_has(rb, got, "HTTP/1.1 200" as *u8) == 0 { he_ws2(why, wo, "hub-not-200 " as *u8) }
              else { if got < 800 { he_ws2(why, wo, "hub-body-too-small " as *u8) } else { pass = 1; he_ws2(why, wo, "hub-200-body-ok" as *u8) } }
          }
      }
      sys_close(fd)
  }
  why[wo[0]] = 0 as u8
  he_flow_row("hub_gateway" as *u8, pass, why, jout, joff)
  return pass }
425
426func he_csp_canary(store: *TrustStore, now: i64, jout: *u8, joff: *i64) -> i64 {
  let out: *u8 = sys_mmap(HE_CAP)
  var pass: i64 = 1
  let why: *u8 = sys_mmap(256)
  let wo: *i64 = sys_mmap(16) as *i64; wo[0]=0
  let rn: i64 = he_fetch_one("/video\x00" as *u8, store, now, out, HE_CAP)
  if rn <= 0 { pass = 0; he_ws2(why, wo, "video-unfetchable " as *u8) }
  else {
      if he_has(out, rn, "wasm-unsafe-eval" as *u8) == 0 { pass = 0; he_ws2(why, wo, "csp-missing-wasm-unsafe-eval " as *u8) }
      if he_has(out, rn, "worker-src" as *u8) == 0 { pass = 0; he_ws2(why, wo, "csp-missing-worker-src " as *u8) }
      // F1120 cross-origin isolation: crossOriginIsolated/SAB dies silently if these headers drop (seq1073 class)
      if he_has(out, rn, "Cross-Origin-Opener-Policy: same-origin" as *u8) == 0 { pass = 0; he_ws2(why, wo, "coop-missing " as *u8) }
      if he_has(out, rn, "Cross-Origin-Embedder-Policy: require-corp" as *u8) == 0 { pass = 0; he_ws2(why, wo, "coep-missing " as *u8) }
  }
  let rw: i64 = he_fetch_one("/video/nx_video_client.wasm\x00" as *u8, store, now, out, HE_CAP)
  if rw < HE_MAGIC_2048 { pass = 0; he_ws2(why, wo, "wasm-response-too-small " as *u8) }
  else {
      var found: i64 = 0
      var i: i64 = 0
      while i + 4 <= rw {
          if out[i]==(0 as u8) { if out[i+1]==(97 as u8) { if out[i+2]==(115 as u8) { if out[i+3]==(109 as u8) { found = 1; i = rw } } } }
          i = i + 1
      }
      if found == 0 { pass = 0; he_ws2(why, wo, "wasm-magic-missing " as *u8) }
  }
  if pass == 1 { he_ws2(why, wo, "csp-tokens+wasm-module-ok" as *u8) }
  why[wo[0]] = 0 as u8
  he_flow_row("video_canary" as *u8, pass, why, jout, joff)
  return pass }
455
456// one full evaluation pass against the loaded trust store; returns pass count (total via total_out).
457func he_pass(store: *TrustStore, total_out: *i64) -> i64 {
  let now: i64 = sys_now_realtime_sec()
  let out: *u8 = sys_mmap(HE_CAP)
  let jout: *u8 = sys_mmap(HE_MAGIC_65536)
  let joff: *i64 = sys_mmap(16) as *i64; joff[0]=0
  var pass: i64 = 0
  var total: i64 = 0
  pass = pass + he_check("home" as *u8, "/\x00" as *u8, "200" as *u8, 0, store, now, out, jout, joff); total=total+1
  pass = pass + he_check("video_room" as *u8, "/video\x00" as *u8, "Nishi Family Video" as *u8, 0, store, now, out, jout, joff); total=total+1
  pass = pass + he_check("video_appjs" as *u8, "/video/app.v2.js\x00" as *u8, "callLayout" as *u8, 1, store, now, out, jout, joff); total=total+1
  pass = pass + he_check("video_wasm" as *u8, "/video/nx_video_client.wasm\x00" as *u8, "200" as *u8, 0, store, now, out, jout, joff); total=total+1
  pass = pass + he_check("embed_js" as *u8, "/video/nishi-video.js\x00" as *u8, "customElements.define" as *u8, 1, store, now, out, jout, joff); total=total+1
  pass = pass + he_check("gallery" as *u8, "/gallery/login\x00" as *u8, "Nishi Gallery" as *u8, 0, store, now, out, jout, joff); total=total+1
  pass = pass + he_check("wiki" as *u8, "/wiki\x00" as *u8, "200" as *u8, 0, store, now, out, jout, joff); total=total+1
  pass = pass + he_check("health_json" as *u8, "/health.json\x00" as *u8, "nishi_health" as *u8, 0, store, now, out, jout, joff); total=total+1
  pass = pass + he_check("health_dashboard" as *u8, "/video/health\x00" as *u8, "Nishi Health" as *u8, 0, store, now, out, jout, joff); total=total+1
  pass = pass + he_ui_check(store, now, jout, joff); total=total+1   // TOP-DOWN UI eye: live buttons wired+safe+intact
  pass = pass + he_qoe_check(now, jout, joff); total=total+1   // CALL-QOE: latest beacon's fps/rtt/mismatch judged + trended
  pass = pass + he_csp_canary(store, now, jout, joff); total=total+1
  pass = pass + he_relay_probe(jout, joff); total=total+1   // RELAY DATA-PLANE: frames MOVE (seq1135)   // VIDEO CANARY: policy+asset CAN-WORK check (seq1073/seq1100 class)
  pass = pass + he_hub_probe(jout, joff); total=total+1   // HUB GATEWAY: loopback data-plane (eaten seq932's monitoring gap)
  hw("  HEALTH: " as *u8); hn(pass); hw("/" as *u8); hn(total); hw(" flows pass\n" as *u8)
  // write the served health.json (surfaced -- the operator can SEE it)
  let hj: *u8 = sys_mmap(HE_MAGIC_65536)
  var ho: i64 = 0
  let h0: *u8 = "{\"nishi_health\":1,\"pass\":" as *u8; var z: i64=0; while h0[z]!=(0 as u8){hj[ho]=h0[z];ho=ho+1;z=z+1}
  let pd: *u8=sys_mmap(16); var pv: i64=pass; var pk2: i64=0; if pv==0{pd[0]=48;pk2=1} while pv>0{pd[pk2]=(48+pv%10) as u8;pv=pv/10;pk2=pk2+1} var pi: i64=pk2-1; while pi>=0{hj[ho]=pd[pi];ho=ho+1;pi=pi-1}
  let h1: *u8 = ",\"total\":" as *u8; z=0; while h1[z]!=(0 as u8){hj[ho]=h1[z];ho=ho+1;z=z+1}
  let td: *u8=sys_mmap(16); var tv: i64=total; var tk: i64=0; if tv==0{td[0]=48;tk=1} while tv>0{td[tk]=(48+tv%10) as u8;tv=tv/10;tk=tk+1} var ti: i64=tk-1; while ti>=0{hj[ho]=td[ti];ho=ho+1;ti=ti-1}
  let h2: *u8 = ",\"t_epoch\":" as *u8; z=0; while h2[z]!=(0 as u8){hj[ho]=h2[z];ho=ho+1;z=z+1}
  let nd: *u8=sys_mmap(24); var nv: i64=now; var nk: i64=0; if nv==0{nd[0]=48;nk=1} while nv>0{nd[nk]=(48+nv%10) as u8;nv=nv/10;nk=nk+1} var ni: i64=nk-1; while ni>=0{hj[ho]=nd[ni];ho=ho+1;ni=ni-1}
  let h3: *u8 = ",\"flows\":[\n" as *u8; z=0; while h3[z]!=(0 as u8){hj[ho]=h3[z];ho=ho+1;z=z+1}
  z=0; while z<joff[0]{hj[ho]=jout[z];ho=ho+1;z=z+1}
  let h4: *u8 = "  ]}\n" as *u8; z=0; while h4[z]!=(0 as u8){hj[ho]=h4[z];ho=ho+1;z=z+1}
  let fd: i64 = sys_openat_wr(HE_HEALTH_OUT, 0x1a4)
  if fd >= 0 { var w: i64=0; while w<ho { let ww: i64=sys_write(fd, ((hj as i64)+w) as *u8, ho-w); if ww<=0{w=ho}else{w=w+ww} } sys_close(fd) }
  ml_report("health_flows_pass" as *u8, pass, 1, "flows" as *u8, "nx_health_eval" as *u8)
  ml_report("health_flows_total" as *u8, total, 1, "flows" as *u8, "nx_health_eval" as *u8)
  total_out[0] = total
  return pass
497}
498
499func main(argc: i64, argv: *i64) -> i64 {
  hw("=== nx_health_eval: sovereign M&E over the live hosting surface (nx_https_get, no shell) ===\n" as *u8)
  let r: i64 = nx_trust_store_load_from_certdata("data/mozilla_certdata.txt\x00" as *u8, 512, HE_MAGIC_4194304)
  if r <= 0 { hw("trust store load failed\n" as *u8); return 1 }
  let store: *TrustStore = r as *TrustStore
  // loop mode (supervised continuous M&E): `nx_health_eval loop [interval_s] [sweep_budget]`. One
  // trust-store load, reused every pass. Default 300s. A pass failing does NOT crash the loop -- it
  // records RED + keeps measuring. Exits CLEANLY after sweep_budget sweeps (default 288 = one day at
  // 300s) -> the hostctl guard respawns fresh = supervisor-as-GC (the gateway budget pattern), and a
  // freshly staged elf goes live within one budget window with no manual restart.
  if argc >= 2 {
      let a1: *u8 = argv[1] as *u8
      if a1[0]==(0x6C as u8) { if a1[1]==(0x6F as u8) { if a1[2]==(0x6F as u8) {
          var interval: i64 = 300
          if argc >= 3 { interval = 0; let a2: *u8 = argv[2] as *u8; var q: i64=0; while a2[q]!=(0 as u8) { if a2[q]>=(48 as u8) { if a2[q]<=(57 as u8) { interval = interval*10 + ((a2[q] as i64)-48) } } q=q+1 } }
          if interval < 30 { interval = 30 }
          var budget: i64 = 288
          if argc >= 4 { budget = 0; let a3: *u8 = argv[3] as *u8; var q3: i64=0; while a3[q3]!=(0 as u8) { if a3[q3]>=(48 as u8) { if a3[q3]<=(57 as u8) { budget = budget*10 + ((a3[q3] as i64)-48) } } q3=q3+1 } if budget < 1 { budget = 288 } }
          hw("  LOOP mode: continuous M&E every " as *u8); hn(interval); hw("s (supervised; fork-per-pass; recycle after " as *u8); hn(budget); hw(" sweeps)\n" as *u8)
          // FORK-PER-PASS (2026-07-05 OOM root fix): one sweep's sovereign-TLS fetches sys_mmap
          // hundreds of MB (session internals) the GC-free substrate never frees; in-process that
          // compounded ~650MB/sweep to 26 GiB RSS and a kernel OOM-kill loop every ~3.4h. The child
          // pays the allocation cost and exits; the KERNEL reclaims its whole address space; this
          // parent loop allocates NOTHING per iteration, so its RSS stays flat for the daemon's life.
          let st: *i64 = sys_mmap(16) as *i64
          var sweeps: i64 = 0
          var go: i64 = 1
          while go == 1 {
              let t0: i64 = sys_now_realtime_sec()
              let pid: i64 = sys_fork()
              if pid == 0 {
                  let ctot: *i64 = sys_mmap(16) as *i64
                  he_pass(store, ctot)
                  sys_exit(0)
              }
              if pid > 0 {
                  // bounded reap: WNOHANG poll so a hung TLS connect cannot wedge the cadence forever;
                  // 240s (below the 300s default interval) then SIGKILL + blocking reap (no zombie).
                  var reaped: i64 = 0; var waited: i64 = 0
                  while reaped == 0 {
                      st[0]=0
                      let wr: i64 = sys_wait4(pid, st, 1)
                      if wr == pid { reaped = 1 } else {
                          if waited >= 240 { __syscall(129, pid, 9, 0, 0, 0, 0); sys_wait4(pid, st, 0); reaped = 1; hw("  [hang-guard] sweep child SIGKILLed at 240s\n" as *u8) } else { sys_sleep_ms(1000); waited = waited + 1 }
                      }
                  }
              } else {
                  // fork refused (box-wide memory pressure): recycle cleanly -- a fresh respawn is the
                  // honest retry; running the sweep in-process here would re-open the leak path.
                  hw("  [fork-fail] recycling for a fresh respawn\n" as *u8); go = 0
              }
              sweeps = sweeps + 1
              if go == 1 { if sweeps >= budget { hw("  [recycle] sweep budget reached -> clean exit (guard respawns fresh)\n" as *u8); go = 0 } }
              // WALL-CLOCK interval gate (banked Synology quirk: long nanosleep returns EARLY on the
              // NAS kernel, so a raw 300s sleep can tick far faster): sleep in 60s chunks until elapsed.
              if go == 1 { var waitg: i64 = 1; while waitg == 1 { let nowt: i64 = sys_now_realtime_sec(); if nowt - t0 >= interval { waitg = 0 } else { var chunk: i64 = interval - (nowt - t0); if chunk > 60 { chunk = 60 } sys_sleep_ms(chunk * 1000) } } }
          }
          return 0
      } } }
  }
  let tot: *i64 = sys_mmap(16) as *i64
  let pass: i64 = he_pass(store, tot)
  hw("  wrote " as *u8); hw(HE_HEALTH_OUT); hw(" -- serve at /health.json (surfaced)\n" as *u8)
  if pass == tot[0] { hw("HEALTH-EVAL verdict=GREEN -- every critical flow live + correct (measured, not prayed)\n" as *u8); return 0 }
  hw("HEALTH-EVAL verdict=RED -- " as *u8); hn(tot[0] - pass); hw(" flow(s) broken -- the catch fire-and-pray missed\n" as *u8)
  return 1
565}
nx_health_eval.nx source

`nx_health_eval.nx` source
