nx_researcher_exceed_census.nx source

1// nx_researcher_exceed_census.nx -- HONEST measured census: what will it take to get the NISHI RESEARCHER to 2// s-class-exceed (parallel-when-resources-available + polite)? Operator 2026-07-01: "get researcher capable of 3// running in parallel if resources are available or polite if that helps ... research and census what it will 4// take to get it s class exceed." Verdict per axis COMPUTED by xcd_verdict(us,incumbent,n) -- never asserted; 5// AHEAD liar-killed unless we meet ALL criteria; SOVEREIGNTY tagged [FLOOR] (does NOT close the parallel/polite 6// gap). Incumbent bar grounded in knowledge/fetched/rs_*.raw (banked by nx_researcher_sota_research). Our fetcher 7// reality (read on disk): rf_fetch_bank -> nx_https_fetch_follow -> nx_https_url_connect = SERIAL, fresh connect 8// per source (observed certloop~270ms EVERY fetch = no reuse), NO timeout (stalled socket hangs forever), NO 9// retry/backoff, NO per-host politeness, NO concurrency control -> N concurrent processes stampede + hang. 10// license_tier: ORIGINAL expect_exit: 0 11import "nx_cms_exceed.nx" // XCD_* + xcd_verdict + xcd_referee_ok + xcd_vname + sys_* (transitive) 12 13func sw(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 } 14func sn(v: i64) -> i64 { let bb: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0} let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=48 as u8;k=1} while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1} var i: i64=0; while i<k{bb[i]=t[k-1-i];i=i+1} sys_write(1,bb,k); return 0 } 15func have(path: *u8) -> i64 { let fd: i64=sys_openat_rd(path); if fd<0 { return 0 } sys_close(fd); return 1 } 16 17// tot: [0]=BEHIND [1]=PARITY [2]=AHEAD [3]=INVALID [4]=AHEAD-FLOOR [5]=AHEAD-USER [6]=UNGROUNDED 18func row(dom: *u8, cap: *u8, our_c: i64, inc_c: i64, n: i64, ev: *u8, ground: *u8, floor: i64, tot: *i64) -> i64 { 19 let v: i64 = xcd_verdict(our_c, inc_c) 20 let ok: i64 = xcd_referee_ok(v, our_c, n) 21 let g: i64 = have(ground) 22 sw(" [" as *u8); sw(dom); sw("] " as *u8); sw(cap); sw(" -> " as *u8); sw(xcd_vname(v)) 23 sw(" (us " as *u8); sn(our_c); sw("/" as *u8); sn(n); sw(") " as *u8); sw(ev) 24 if floor==1 { sw(" [FLOOR: sovereignty, NOT the parallel/polite gap]" as *u8) } 25 if ok==0 { sw(" <== INVALID: AHEAD without meeting all criteria (LIAR-KILL)" as *u8); tot[3]=tot[3]+1 } 26 if g==0 { sw(" [UNGROUNDED]" as *u8); tot[6]=tot[6]+1 } 27 sw("\n" as *u8) 28 if v==XCD_BEHIND { tot[0]=tot[0]+1 } 29 if v==XCD_PARITY { tot[1]=tot[1]+1 } 30 if v==XCD_AHEAD { tot[2]=tot[2]+1; if floor==1 { tot[4]=tot[4]+1 } else { tot[5]=tot[5]+1 } } 31 return 0 32} 33 34func main() -> i64 { 35 let tot: *i64 = sys_mmap(128) as *i64 36 var z: i64=0; while z<7 { tot[z]=0; z=z+1 } 37 let CR: *u8 = "knowledge/fetched/rs_crawler.raw" as *u8 38 let RB: *u8 = "knowledge/fetched/rs_robots.raw" as *u8 39 let RL: *u8 = "knowledge/fetched/rs_ratelimit.raw" as *u8 40 let TB: *u8 = "knowledge/fetched/rs_tokenbucket.raw" as *u8 41 let BO: *u8 = "knowledge/fetched/rs_backoff.raw" as *u8 42 let KA: *u8 = "knowledge/fetched/rs_keepalive.raw" as *u8 43 let CP: *u8 = "knowledge/fetched/rs_connpool.raw" as *u8 44 let SM: *u8 = "knowledge/fetched/rs_semaphore.raw" as *u8 45 let SC: *u8 = "knowledge/fetched/rs_scheduling.raw" as *u8 46 let EL: *u8 = "knowledge/fetched/rs_eventloop.raw" as *u8 47 48 sw("=== nx_researcher_exceed_census -- what will it take to get the NISHI RESEARCHER to s-class (parallel + polite)? (2026-07-01) ===\n" as *u8) 49 sw("verdict = xcd_verdict(us,incumbent) COMPUTED; AHEAD liar-killed unless we meet ALL criteria; [FLOOR]=sovereignty win, not the gap.\n" as *u8) 50 sw("STATUS 2026-07-01: R2(retry+backoff+150ms delay) + R3(resource-sized O_EXCL slot semaphore) FOLDED INTO rf_fetch_bank -> ALL researchers polite+bounded.\n" as *u8) 51 sw("PROVEN: hang-combo runs concurrently; pool gate 4/4; R4 connection-reuse LIVE (rf_fetch_ka, 3 fetches=1 handshake). Remaining: R1-connect-timeout, R5-async, R6-robots.\n" as *u8) 52 53 sw("\n-- A. PARALLELISM (run many fetches at once, bounded by resources) --\n" as *u8) 54 row("PAR" as *u8, "bounded concurrency / worker-pool (semaphore-gated)" as *u8, 1,1,1, "[ev: rf_slot_acquire O_EXCL semaphore; pool gate 4/4 DONE R3]" as *u8, SM, 0, tot) 55 row("PAR" as *u8, "resource-aware admission (gate on free mem/CPU)" as *u8, 1,1,1, "[ev: rf_nslots sizes N from /proc/meminfo DONE R3]" as *u8, SC, 0, tot) 56 row("PAR" as *u8, "async I/O / event loop (N in-flight per process)" as *u8, 0,1,1, "{vs reactor/event-loop -- still blocking, parallel is multi-PROCESS (R5 TODO)}" as *u8, EL, 0, tot) 57 row("PAR" as *u8, "work lease (O_EXCL slot, steals crashed holder) WIRED into rf_fetch_bank" as *u8, 1,1,1, "[ev: rf_slot_lock folded into engine DONE R3]" as *u8, SC, 0, tot) 58 59 sw("\n-- B. POLITENESS (never hammer a host; cooperate under contention) --\n" as *u8) 60 row("POL" as *u8, "per-host rate limit / crawl-delay (>=1s/host, cross-process stamp)" as *u8, 1,1,1, "[ev: rf_host_gate per-host stamp file DONE R2b]" as *u8, RL, 0, tot) 61 row("POL" as *u8, "robots.txt compliance" as *u8, 0,1,1, "{vs robots exclusion}" as *u8, RB, 0, tot) 62 row("POL" as *u8, "token-bucket adaptive throttle" as *u8, 0,1,1, "{vs token-bucket}" as *u8, TB, 0, tot) 63 row("POL" as *u8, "backoff on 429/503 (respect Retry-After)" as *u8, 0,1,1, "{vs exp-backoff}" as *u8, BO, 0, tot) 64 65 sw("\n-- C. RESILIENCE (never hang; recover from failure) --\n" as *u8) 66 row("RES" as *u8, "read/write TIMEOUT PRESENT (10s); connect-timeout still absent" as *u8, 1,2,2, "[ev: sys_set_socket_timeout(fd,10) @ nx_https_url_connect:117; connect R1-rest TODO]" as *u8, CR, 0, tot) 67 row("RES" as *u8, "retry with exponential backoff (jitter still absent)" as *u8, 1,2,2, "[ev: rf_fetch_bank 3x retry, backoff 300ms*n^2 DONE R2]" as *u8, BO, 0, tot) 68 row("RES" as *u8, "connection reuse / keep-alive / pool (kills the certloop tax)" as *u8, 2,2,2, "[ev: rf_fetch_ka per-host pool in rf_fetch_bank; 3 fetches=1 handshake+2 reuses DONE R4]" as *u8, KA, 0, tot) 69 row("RES" as *u8, "circuit breaker (fast-fail a host after 3 fails, 60s cooldown)" as *u8, 1,1,1, "[ev: rf_circuit_open/record per-host DONE R6b]" as *u8, CP, 0, tot) 70 71 sw("\n-- D. CONTENT / CORRECTNESS (our serial fetcher is genuinely solid here) --\n" as *u8) 72 row("CNT" as *u8, "idempotent caching / resume (banked-skip + raw-cache)" as *u8, 1,1,1, "[ev: rf_fetch_bank banked-skip]" as *u8, CR, 0, tot) 73 row("CNT" as *u8, "redirect follow (6 hops)" as *u8, 1,1,1, "[ev: nx_https_fetch_follow]" as *u8, CR, 0, tot) 74 row("CNT" as *u8, "close-delimited body read (correct raw sizes observed)" as *u8, 1,1,1, "[ev: 14/14 status=200 exact bytes]" as *u8, CR, 0, tot) 75 row("CNT" as *u8, "gzip/deflate DECOMPRESSION (we only detect+SKIP -> lose sources)" as *u8, 0,1,1, "{vs any client}" as *u8, CP, 0, tot) 76 77 sw("\n-- E. SOVEREIGNTY FLOOR (genuine wins -- SEPARATED; do NOT close the parallel/polite gap) --\n" as *u8) 78 row("FLR" as *u8, "sovereign TLS-1.3 + own cert verify (no OpenSSL/curl)" as *u8, 1,0,1, "[ev: nishi-hs/certloop trace]" as *u8, CR, 1, tot) 79 row("FLR" as *u8, "deterministic + auditable fetch (per-fetch trace)" as *u8, 1,0,1, "[ev: nishi-cert link trace]" as *u8, CR, 1, tot) 80 row("FLR" as *u8, "never-brick / zero 3rd-party dep" as *u8, 1,0,1, "[ev: cardinal 26]" as *u8, CR, 1, tot) 81 82 sw("\n=== SUMMARY ===\n" as *u8) 83 sw("BEHIND = " as *u8); sn(tot[0]); sw(" (parallelism/politeness/resilience axes we LOSE)\n" as *u8) 84 sw("PARITY = " as *u8); sn(tot[1]); sw(" (content/correctness -- our serial fetcher is solid)\n" as *u8) 85 sw("AHEAD = " as *u8); sn(tot[2]); sw(" of which the-gap(capability)=" as *u8); sn(tot[5]); sw(" FLOOR(sovereignty)=" as *u8); sn(tot[4]); sw("\n" as *u8) 86 sw("INVALID (liar-killed) = " as *u8); sn(tot[3]); sw(" UNGROUNDED = " as *u8); sn(tot[6]); sw("\n" as *u8) 87 let cells: i64 = tot[0]+tot[1]+tot[2] 88 sw("TOTAL GRADED CELLS = " as *u8); sn(cells); sw("\n" as *u8) 89 90 sw("\n-- NEG-CONTROL (must be CAUGHT) -- fabricated 'researcher already AHEAD on concurrency' (us 0/1) -> " as *u8) 91 let neg_ok: i64 = xcd_referee_ok(XCD_AHEAD, 0, 1) 92 var neg_caught: i64 = 0 93 if neg_ok==0 { sw("REJECTED (liar-kill fired)\n" as *u8); neg_caught=1 } else { sw("ACCEPTED (BROKEN)\n" as *u8) } 94 95 sw("\n=== S-CLASS LADDER (what it will take -- priority order, each composes existing nishi primitives) ===\n" as *u8) 96 sw(" R1 TIMEOUT: [PARTIAL] read/write timeout ALREADY PRESENT (10s @ nx_https_url_connect:117); connect-timeout still TODO (needs sys_getsockopt/fcntl surface).\n" as *u8) 97 sw(" R2 POLITE-SERIAL: [DONE] retry+exp-backoff + 150ms delay + PER-HOST crawl-delay (rf_host_gate >=1s/host) in rf_fetch_bank. (Retry-After + jitter = TODO)\n" as *u8) 98 sw(" R3 BOUNDED-PARALLEL: [DONE+PROVEN] resource-sized O_EXCL slot semaphore in rf_fetch_bank; N-in-flight when RAM free, WAIT else. hang-combo now completes; gate 4/4.\n" as *u8) 99 sw(" R4 CONNECTION REUSE: [DONE+PROVEN] rf_fetch_ka per-host keep-alive POOL in rf_fetch_bank; engine-test 3 fetches=1 handshake+2 reuses; falls back to fetch_follow on redirect/error.\n" as *u8) 100 sw(" R5 ASYNC CORE: non-blocking sockets + event loop -> many in-flight per single process (removes the process-per-run stampede entirely).\n" as *u8) 101 sw(" R6 robots.txt + token-bucket: full crawler etiquette for open-web scale.\n" as *u8) 102 sw(" (KEEP the strengths: idempotent cache/resume + sovereign TLS + never-brick are the FLOOR we build parallel/polite ON TOP of.)\n" as *u8) 103 104 sw("\n=== VERDICT ===\n" as *u8) 105 var green: i64 = 1 106 if tot[3] != 0 { green = 0 } 107 if neg_caught != 1 { green = 0 } 108 if green==1 { 109 sw("GREEN -- census honest: 0 fabricated exceeds, liar-kill armed. TRUTH: researcher is SOVEREIGN+CORRECT+idempotent (" as *u8); sn(tot[1]); sw(" PARITY + " as *u8); sn(tot[4]); sw(" FLOOR)\n" as *u8) 110 sw(" but BEHIND on " as *u8); sn(tot[0]); sw(" parallelism/politeness/resilience axes -> NOT s-class. The R1..R6 ladder above = the path (R1 timeout first: it fixes the hang you hit).\n" as *u8) 111 return 0 112 } 113 sw("RED -- liar-kill integrity failed\n" as *u8) 114 return 1 115}