nx_apm_sweep.nx source

1// nx_apm_sweep.nx -- SOVEREIGN concurrency-aware APM probe sweep (the "New Relic" core). 2// 3// WHY: the existing monitor (nx_site_monitor/nx_health_probe) is BINARY up/down and SINGLE-request, 4// and the HTTPS fetch used to read-until-close (measuring the ~19s keep-alive idle timeout, not real 5// latency). So a daemon that serves fine alone but STALLS under concurrent load ("switching tabs and 6// tabs don't load") showed all-green. This organ fixes that: it fires N SIMULTANEOUS cold probes per 7// endpoint (= N browser tabs) using the Content-Length-honest nx_fetch_staged, and reports the real 8// latency distribution + stalls + timeouts + error rate, with a durable per-endpoint time-series row. 9// 10// usage: nx_apm_sweep [concurrency] (default 8) 11// out: per-endpoint report on stdout + appended rows in knowledge/status/apm_metrics.tsv 12// license_tier: ORIGINAL 13import "nx_fetch_unit.nx" // nx_fetch_staged, FU_CAP 14import "nx_syscalls.nx" 15const APS_MAGIC_1024: i64 = 1024 16const APS_MAGIC_65536: i64 = 65536 17 18const APS_STALL_MS: i64 = 2000 19const APS_TIMEOUT_MS: i64 = 18000 20const APS_METRICS: *u8 = "knowledge/status/apm_metrics.tsv" 21 22func aps_slen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n } 23func aps_w(s: *u8) -> i64 { return sys_write(1, s, aps_slen(s)) } 24// append decimal v into buf at off, return new off 25func aps_n(buf: *u8, off: i64, v: i64) -> i64 { 26 var o: i64 = off; var m: i64 = v 27 if m < 0 { buf[o] = 45 as u8; o = o + 1; m = 0 - m } 28 let t: *u8 = sys_mmap(24); var k: i64 = 0 29 if m == 0 { t[0] = 48 as u8; k = 1 } 30 while m > 0 { t[k] = (48 + (m - (m/10)*10)) as u8; m = m / 10; k = k + 1 } 31 var i: i64 = 0; while i < k { buf[o + i] = t[k - 1 - i]; i = i + 1 } 32 return o + k 33} 34// append C-string s into buf at off, return new off 35func aps_s(buf: *u8, off: i64, s: *u8) -> i64 { var o: i64 = off; var i: i64 = 0; while s[i] != (0 as u8) { buf[o] = s[i]; o = o + 1; i = i + 1 } return o } 36func aps_wn(v: i64) -> i64 { let b: *u8 = sys_mmap(24); let e: i64 = aps_n(b, 0, v); return sys_write(1, b, e) } 37// pad a number's printed width to w with leading spaces (right-align) 38func aps_wpad(v: i64, w: i64) -> i64 { 39 let b: *u8 = sys_mmap(24); let e: i64 = aps_n(b, 0, v) 40 var pad: i64 = w - e; while pad > 0 { sys_write(1, " " as *u8, 1); pad = pad - 1 } 41 return sys_write(1, b, e) 42} 43 44// parse HTTP status from "HTTP/1.x NNN ..."; -1 if not an HTTP response. 45func aps_status(buf: *u8, n: i64) -> i64 { 46 if n < 13 { return 0 - 1 } 47 if buf[0] != (72 as u8) { return 0 - 1 } // 'H' 48 var i: i64 = 0; var sp: i64 = 0 - 1 49 while i < n { if buf[i] == (32 as u8) { sp = i; i = n } else { i = i + 1 } } 50 if sp < 0 { return 0 - 1 } 51 if sp + 4 > n { return 0 - 1 } 52 let c0: i64 = buf[sp+1] as i64; let c1: i64 = buf[sp+2] as i64; let c2: i64 = buf[sp+3] as i64 53 if c0 < 48 { return 0 - 1 } if c0 > 57 { return 0 - 1 } 54 if c1 < 48 { return 0 - 1 } if c1 > 57 { return 0 - 1 } 55 if c2 < 48 { return 0 - 1 } if c2 > 57 { return 0 - 1 } 56 return (c0-48)*100 + (c1-48)*10 + (c2-48) 57} 58 59// one cold probe; out_status[0]=HTTP code (or -1), returns elapsed ms. 60func aps_probe(url: *u8, out_status: *i64) -> i64 { 61 let buf: *u8 = sys_mmap(FU_CAP) 62 let t0: i64 = sys_now_ms() 63 let n: i64 = nx_fetch_staged(url, buf, FU_CAP) 64 let t1: i64 = sys_now_ms() 65 // nx_fetch_staged returns the BODY byte-count (headers already stripped), >0 for any HTTP 66 // response (incl. 401), <=0 for a connect/TLS failure. So bytes>0 == "server responded". 67 out_status[0] = n 68 return t1 - t0 69} 70 71// build "/tmp/nx_apm_<base>_<k>\0" 72func aps_mkpath(path: *u8, base: i64, k: i64) -> i64 { 73 var o: i64 = aps_s(path, 0, "/tmp/nx_apm_" as *u8) 74 o = aps_n(path, o, base); path[o] = 95 as u8; o = o + 1; o = aps_n(path, o, k) 75 path[o] = 0 as u8; return o 76} 77 78// parse two leading ints (ms, status) from a result file's bytes. 79func aps_2ints(data: *u8, n: i64, out: *i64) -> i64 { 80 out[0] = 0 - 1; out[1] = 0 - 1 81 var i: i64 = 0; var idx: i64 = 0 82 while idx < 2 { 83 var go: i64 = 1 84 while go == 1 { if i >= n { go = 0; idx = 2 } else { let c: i64 = data[i] as i64; if c == 45 { go = 0 } else { if c >= 48 { if c <= 57 { go = 0 } else { i = i + 1 } } else { i = i + 1 } } } } 85 if idx < 2 { 86 var neg: i64 = 0 87 if i < n { if data[i] as i64 == 45 { neg = 1; i = i + 1 } } 88 var v: i64 = 0; var any: i64 = 0; var g2: i64 = 1 89 while g2 == 1 { if i >= n { g2 = 0 } else { let d: i64 = data[i] as i64; if d >= 48 { if d <= 57 { v = v*10 + (d-48); any = 1; i = i + 1 } else { g2 = 0 } } else { g2 = 0 } } } 90 if neg == 1 { v = 0 - v } 91 if any == 1 { out[idx] = v; idx = idx + 1 } else { idx = 2 } 92 } 93 } 94 return 0 95} 96 97// insertion sort ascending 98func aps_sort(a: *i64, n: i64) -> i64 { 99 var i: i64 = 1 100 while i < n { 101 let kv: i64 = a[i]; var j: i64 = i - 1; var go: i64 = 1 102 while go == 1 { if j >= 0 { if a[j] > kv { a[j+1] = a[j]; j = j - 1 } else { go = 0 } } else { go = 0 } } 103 a[j+1] = kv; i = i + 1 104 } 105 return 0 106} 107 108// fire `conc` concurrent cold probes of url; aggregate; print a report row; append a time-series row. 109func aps_burst(url: *u8, label: *u8, conc: i64, base: i64, epoch: i64) -> i64 { 110 var k: i64 = 0 111 while k < conc { 112 let pid: i64 = sys_fork() 113 if pid == 0 { 114 let st: *i64 = sys_mmap(16) as *i64 115 let ms: i64 = aps_probe(url, st) 116 let path: *u8 = sys_mmap(64); aps_mkpath(path, base, k) 117 let ob: *u8 = sys_mmap(48); var o: i64 = aps_n(ob, 0, ms); ob[o] = 32 as u8; o = o + 1; o = aps_n(ob, o, st[0]); ob[o] = 10 as u8; o = o + 1 118 let fd: i64 = sys_openat_wr(path, 0x1a4) 119 if fd >= 0 { sys_write(fd, ob, o); sys_close(fd) } 120 sys_exit(0) 121 } 122 k = k + 1 123 } 124 let stp: *i64 = sys_mmap(16) as *i64; var got: i64 = 0 125 while got < conc { let r: i64 = sys_wait4(0 - 1, stp, 0); if r <= 0 { got = conc } else { got = got + 1 } } 126 127 let ms: *i64 = sys_mmap(8 * (conc + 2)) as *i64 128 var nres: i64 = 0; var oks: i64 = 0; var stalls: i64 = 0; var timeouts: i64 = 0; var errs: i64 = 0 129 var kk: i64 = 0 130 while kk < conc { 131 let path: *u8 = sys_mmap(64); aps_mkpath(path, base, kk) 132 let lp: *i64 = sys_mmap(16) as *i64; lp[0] = 0 133 let data: *u8 = sys_read_file(path, lp) 134 if (data as i64) != 0 { if lp[0] > 0 { 135 let v: *i64 = sys_mmap(24) as *i64; aps_2ints(data, lp[0], v) 136 let mv: i64 = v[0]; let sv: i64 = v[1] // mv = ms, sv = body bytes (>0 = responded) 137 if mv >= 0 { ms[nres] = mv; nres = nres + 1 } 138 if sv > 0 { oks = oks + 1 } else { errs = errs + 1 } 139 if sv <= 0 { timeouts = timeouts + 1 } else { if mv >= APS_TIMEOUT_MS { timeouts = timeouts + 1 } } 140 if mv >= APS_STALL_MS { stalls = stalls + 1 } 141 } } 142 kk = kk + 1 143 } 144 aps_sort(ms, nres) 145 var mn: i64 = 0; var p50: i64 = 0; var p95: i64 = 0; var mx: i64 = 0 146 if nres > 0 { 147 mn = ms[0]; mx = ms[nres-1]; p50 = ms[nres/2] 148 var pi: i64 = (nres*95)/100; if pi >= nres { pi = nres - 1 } p95 = ms[pi] 149 } 150 151 // verdict 152 var verd: *u8 = "GREEN" as *u8 153 if stalls > 0 { verd = "AMBER" as *u8 } 154 if timeouts > 0 { verd = "RED " as *u8 } 155 if errs > 0 { verd = "RED " as *u8 } 156 if oks < conc { if timeouts == 0 { verd = "AMBER" as *u8 } } 157 158 // report row 159 aps_w("[" as *u8); aps_w(verd); aps_w("] " as *u8) 160 let lw: i64 = aps_slen(label); sys_write(1, label, lw); var pad: i64 = 22 - lw; while pad > 0 { sys_write(1, " " as *u8, 1); pad = pad - 1 } 161 aps_w("ok=" as *u8); aps_wn(oks); aps_w("/" as *u8); aps_wn(conc) 162 aps_w(" p50=" as *u8); aps_wpad(p50, 5); aps_w(" p95=" as *u8); aps_wpad(p95, 6); aps_w(" max=" as *u8); aps_wpad(mx, 6); aps_w("ms" as *u8) 163 aps_w(" stall=" as *u8); aps_wn(stalls); aps_w(" to=" as *u8); aps_wn(timeouts); aps_w(" err=" as *u8); aps_wn(errs) 164 aps_w("\n" as *u8) 165 166 // time-series append: epoch label url conc oks nres min p50 p95 max stalls timeouts errs 167 // SOVEREIGN seg-store record (latest-per-endpoint, key:value; NO tsv) -- per-key file + apmsweep-ids index 168 sys_mkdir("knowledge/store" as *u8, 0x1ff) 169 let sp: *u8 = sys_mmap(512); var spo: i64 = aps_s(sp, 0, "knowledge/store/apmsweep-" as *u8); spo = aps_s(sp, spo, label); sp[spo]=0 as u8 170 let rb: *u8 = sys_mmap(APS_MAGIC_1024); var ro: i64 = 0 171 ro=aps_s(rb,ro,"label=" as *u8); ro=aps_s(rb,ro,label); rb[ro]=10 as u8; ro=ro+1 172 ro=aps_s(rb,ro,"epoch=" as *u8); ro=aps_n(rb,ro,epoch); rb[ro]=10 as u8; ro=ro+1 173 ro=aps_s(rb,ro,"url=" as *u8); ro=aps_s(rb,ro,url); rb[ro]=10 as u8; ro=ro+1 174 ro=aps_s(rb,ro,"conc=" as *u8); ro=aps_n(rb,ro,conc); rb[ro]=10 as u8; ro=ro+1 175 ro=aps_s(rb,ro,"oks=" as *u8); ro=aps_n(rb,ro,oks); rb[ro]=10 as u8; ro=ro+1 176 ro=aps_s(rb,ro,"ttfb_p50=" as *u8); ro=aps_n(rb,ro,p50); rb[ro]=10 as u8; ro=ro+1 177 ro=aps_s(rb,ro,"ttfb_p95=" as *u8); ro=aps_n(rb,ro,p95); rb[ro]=10 as u8; ro=ro+1 178 ro=aps_s(rb,ro,"ttfb_max=" as *u8); ro=aps_n(rb,ro,mx); rb[ro]=10 as u8; ro=ro+1 179 ro=aps_s(rb,ro,"stalls=" as *u8); ro=aps_n(rb,ro,stalls); rb[ro]=10 as u8; ro=ro+1 180 ro=aps_s(rb,ro,"timeouts=" as *u8); ro=aps_n(rb,ro,timeouts); rb[ro]=10 as u8; ro=ro+1 181 ro=aps_s(rb,ro,"errs=" as *u8); ro=aps_n(rb,ro,errs); rb[ro]=10 as u8; ro=ro+1 182 let mfd: i64 = sys_openat_wr(sp, 0x1a4); if mfd >= 0 { sys_write(mfd, rb, ro); sys_close(mfd) } 183 aps_seg_index(label) 184 return 0 185} 186 187// ---- SOVEREIGN seg-store (NO tsv): latest-per-endpoint record + apmsweep-ids index ---- 188func aps_line_eq(buf: *u8, start: i64, end: i64, label: *u8) -> i64 { var i: i64=start; var j: i64=0; while i<end { if buf[i]!=label[j] {return 0} i=i+1; j=j+1 } if label[j]!=(0 as u8) {return 0} return 1 } 189func aps_seg_index(label: *u8) -> i64 { 190 let buf: *u8=sys_mmap(APS_MAGIC_65536); var n: i64=0; let fd: i64=sys_openat_rd("knowledge/store/apmsweep-ids" as *u8); if fd>=0 { n=sys_read(fd,buf,APS_MAGIC_65536); sys_close(fd) } 191 var present: i64=0; var ls: i64=0; var i: i64=0 192 while i<=n { var nl: i64=0; if i>=n{nl=1} else { if buf[i]==(10 as u8){nl=1} } if nl==1 { if i>ls { if aps_line_eq(buf,ls,i,label)==1 {present=1} } ls=i+1 } i=i+1 } 193 if present==0 { let afd: i64=sys_openat_append("knowledge/store/apmsweep-ids" as *u8, 0x1a4); if afd>=0 { let lb: *u8=sys_mmap(128); var lo: i64=aps_s(lb,0,label); lb[lo]=10 as u8; lo=lo+1; sys_write(afd,lb,lo); sys_close(afd) } } 194 return 0 195} 196 197func main(argc: i64, argv: *i64) -> i64 { 198 var conc: i64 = 8 199 if argc >= 2 { 200 let a: *u8 = argv[1] as *u8; var v: i64 = 0; var i: i64 = 0 201 while a[i] != (0 as u8) { let c: i64 = a[i] as i64; if c >= 48 { if c <= 57 { v = v*10 + (c-48) } } i = i + 1 } 202 if v > 0 { conc = v } 203 } 204 let base: i64 = sys_now_ms() 205 let epoch: i64 = sys_now_realtime_sec() 206 207 aps_w("=== NISHI APM SWEEP (concurrency=" as *u8); aps_wn(conc); aps_w(", browser-faithful, cold-connection) ===\n" as *u8) 208 aps_w(" stall=>2000ms timeout=>18000ms verdict GREEN/AMBER(stall)/RED(timeout|err|5xx)\n\n" as *u8) 209 210 aps_burst("https://nishifamily.com/" as *u8, "landing /" as *u8, conc, base, epoch) 211 aps_burst("https://nishifamily.com/status.html" as *u8, "status /status.html" as *u8, conc, base, epoch) 212 aps_burst("https://nishifamily.com/wiki" as *u8, "wiki /wiki" as *u8, conc, base, epoch) 213 aps_burst("https://nishifamily.com/gallery/login" as *u8, "gallery /gallery/login" as *u8, conc, base, epoch) 214 aps_burst("https://nishifamily.com/gallery/api/list" as *u8, "gallery /gallery/api" as *u8, conc, base, epoch) 215 aps_burst("https://nishifamily.com/library" as *u8, "library /library" as *u8, conc, base, epoch) 216 aps_burst("https://nishifamily.com/media" as *u8, "media /media" as *u8, conc, base, epoch) 217 218 aps_w("\nappended " as *u8); aps_w(APS_METRICS); aps_w(" (epoch=" as *u8); aps_wn(epoch); aps_w(")\n" as *u8) 219 return 0 220}