code wiki / _hdl_build / nx_crawl_coverage_census.nx
`nx_crawl_coverage_census.nx` source

1// nx_crawl_coverage_census.nx -- HONEST access-coverage census for the sovereign crawler (operator 2026-07-04:
2// "are there other forms of sites like this... make sure we have FULL COVERAGE before we pretend our crawler is
3// state of the art"). Maps the full ACCESS TAXONOMY a SOTA crawler must handle (TLS versions/ciphers/curves,
4// HTTP versions, content encodings, JS-render, anti-bot, discovery) and grades each: WIRED (in the live crawl
5// fetch path) · HAVE (capability organ exists but not wired) · GAP (missing). Grounding = nx_research_fetch
6// cc_* sources (webcrawler/http2/http3/tls/brotli/robots/cipher/botmgmt). Score = permille WIRED-or-HAVE, but
7// SOTA is claimed ONLY when the WIRED set covers the taxonomy -- gaps are named, never hidden. license_tier: ORIGINAL
8import "nx_syscalls.nx"
9
10func w(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
11func wn(v: i64) -> i64 { if v==0 { sys_write(1,"0" as *u8,1); return 0 } var m: i64=v; if m<0 { sys_write(1,"-" as *u8,1); m=0-m } let t: *u8=sys_mmap(24); var k: i64=0; while m>0 { t[k]=(48+(m%10)) as u8; m=m/10; k=k+1 } let o: *u8=sys_mmap(24); var q: i64=k-1; var x: i64=0; while q>=0 { o[x]=t[q]; x=x+1; q=q-1 } sys_write(1,o,x); return 0 }
12func fex(path: *u8) -> i64 { let fd: i64=sys_openat_rd(path); if fd<0 { return 0 } sys_close(fd); return 1 }
13// tier: 2=WIRED (live crawl path), 1=HAVE (organ exists, not wired), 0=GAP
14func row(name: *u8, tier: i64, wired: *i64, have: *i64, total: *i64) -> i64 {
  total[0]=total[0]+1
  if tier==2 { wired[0]=wired[0]+1; have[0]=have[0]+1; w("  [WIRED]" as *u8) }
  else { if tier==1 { have[0]=have[0]+1; w("  [have ]" as *u8) } else { w("  [GAP  ]" as *u8) } }
  w(" " as *u8); w(name); w("\n" as *u8)
  return 0
20}
21
22func main() -> i64 {
  w("=== NISHI CRAWLER ACCESS-COVERAGE CENSUS (full taxonomy; WIRED vs have vs GAP; honest SOTA gate) ===\n" as *u8)
  let wired: *i64=sys_mmap(16) as *i64; wired[0]=0
  let have: *i64=sys_mmap(16) as *i64; have[0]=0
  let total: *i64=sys_mmap(16) as *i64; total[0]=0
27
  w("  -- TLS reach (the layer that just cracked ukdevilz) --\n" as *u8)
  row("TLS 1.3 (minimal fleet hello) -- most of the modern web" as *u8, 2, wired, have, total)
  row("TLS 1.3 Chrome-JA3 hello -- beats Cloudflare JA3 fingerprinting (nhentai-class)" as *u8, 2, wired, have, total)
  row("TLS 1.2 ECDHE-RSA + ECDHE-ECDSA (P-256, secp256r1-forced) -- 1.2-only ECDSA hosts (ukdevilz LIVE 200)" as *u8, 2, wired, have, total)
  row("TLS 1.2 ChaCha20-Poly1305 suites (cipher organ exists; NOT completed in the 1.2 session)" as *u8, fex("runtime/nx_chacha20_poly1305.nx" as *u8), wired, have, total)
  row("TLS 1.2 AES-256-GCM-SHA384 (cipher exists; needs the P_SHA384 PRF/key-schedule in the session)" as *u8, fex("runtime/nx_aes256_gcm.nx" as *u8), wired, have, total)
  row("ECDHE X25519 / P-384 key-agreement in TLS 1.2 (P-256 only today -- rare 1.2 servers that force these)" as *u8, 0, wired, have, total)
  row("cert chains: RSA-2048/3072/4096 + ECDSA-P256/P384 + Ed25519 (full Web-PKI validator)" as *u8, 2, wired, have, total)
36
  w("  -- HTTP version --\n" as *u8)
  row("HTTP/1.1 (chunked, keep-alive, redirects) -- the crawl fetch path" as *u8, 2, wired, have, total)
  row("HTTP/2 (h2) client-over-TLS EXISTS (nx_h2_client_over_tls) but NOT wired into the crawler fetch" as *u8, fex("runtime/nx_h2_client_over_tls.nx" as *u8), wired, have, total)
  row("HTTP/3 / QUIC (wire+pkt pieces exist; no full client) -- a growing slice of top sites are h3-preferred" as *u8, 0, wired, have, total)
41
  w("  -- content decode --\n" as *u8)
  row("gzip / deflate inflate (wired in CC ingest + fetch)" as *u8, fex("runtime/nx_gzip_wrap.nx" as *u8), wired, have, total)
  row("Brotli decode (Content-Encoding: br) -- MANY sites send br to browsers; we'd get unreadable bytes" as *u8, 0, wired, have, total)
  row("zstd decode (Content-Encoding: zstd) -- newer, growing" as *u8, 0, wired, have, total)
  row("charset/encoding normalize (UTF-8 fold WIRED; legacy charsets not transcoded)" as *u8, 1, wired, have, total)
47
  w("  -- render / dynamic --\n" as *u8)
  row("static HTML -> text (nx_html_to_text) -- wired" as *u8, 2, wired, have, total)
  row("SPA content via NON-JS path: RSS/Atom feed extract+discover (nx_feed_extract/_discover) WIRED -- old.reddit 25 items, nyt 14" as *u8, fex("runtime/_hdl_build/nx_feed_extract.nx" as *u8) + 1, wired, have, total)
  row("structured-data mine (og/twitter/JSON-LD title+desc, nx_struct_extract) WIRED -- clean titles/snippets + cards" as *u8, fex("runtime/_hdl_build/nx_struct_extract.nx" as *u8) + 1, wired, have, total)
  row("JS-EXECUTION for true SPAs w/ no feed+no SSR-alt (reddit-verify): engine 15/15 but no async/BOM, not wired" as *u8, 1, wired, have, total)
53
  w("  -- anti-bot walls --\n" as *u8)
  row("JA3/TLS fingerprint walls -- beaten by Chrome-JA3 (nhentai LIVE 200)" as *u8, 2, wired, have, total)
  row("Cloudflare MANAGED JS-challenge ('Just a moment') -- needs browser JS+BOM execution (javdb/jable class)" as *u8, 0, wired, have, total)
  row("CAPTCHA / Turnstile -- interactive; not automatable sovereignly (honest ceiling)" as *u8, 0, wired, have, total)
58
  w("  -- reach / discipline --\n" as *u8)
  row("IPv4 DNS + connect (nx_dns_resolve + sockets)" as *u8, 2, wired, have, total)
  row("IPv6 (AAAA + AF_INET6) -- IPv4-only today; IPv6-only hosts unreachable" as *u8, 0, wired, have, total)
  row("polite PACING: per-host interval + Retry-After + exponential backoff (nx_crawl_pace, WIRED) -- avoids IP-ban" as *u8, fex("runtime/_hdl_build/nx_crawl_pace.nx" as *u8) + 1, wired, have, total)
  row("robots.txt PARSE/RESPECT (crawl-delay HOOK exists in pacer; robots.txt not yet parsed to feed it)" as *u8, 1, wired, have, total)
  row("redirect following (3xx Location, bounded) -- wired incl. the new TLS 1.2 path" as *u8, 2, wired, have, total)
  row("cookies / sessions / login walls (no cookie jar) -- auth-gated content unreachable" as *u8, 0, wired, have, total)
66
  w("  -- discovery / diagnosis --\n" as *u8)
  row("Common Crawl CDX-locate + byte-range ingest (index-only) -- wired" as *u8, 2, wired, have, total)
  row("frontier-persisted resumable crawl -- wired" as *u8, 2, wired, have, total)
  row("feed-driven frontier expansion: crawl a page -> enqueue its feed's article URLs (coverage compounds) WIRED" as *u8, 2, wired, have, total)
  row("sitemap.xml discovery: robots Sitemap: + <loc> parser (nx_sitemap_extract, gate 11/11) -- built, not yet crawler-wired" as *u8, 1, wired, have, total)
  row("nx_net_diag layered troubleshooter (DNS/TCP/TLS-response) -- the self-heal eyes (NEW this session)" as *u8, fex("runtime/nx_net_diag.nx" as *u8) + 1, wired, have, total)
73
  let sc: i64 = ((wired[0] + have[0]) * 1000) / (total[0] * 2)
  w("=== CRAWLER-COVERAGE wired=" as *u8); wn(wired[0]); w(" have=" as *u8); wn(have[0]); w(" /" as *u8); wn(total[0])
  w("  coverage=" as *u8); wn(sc); w(" permille ===\n" as *u8)
  // ★EVIDENCE-CORRECTED PRIORITY (16-site live sweep 2026-07-05, not analysis): the sweep MEASURED the real
  // gaps and REFUTED my Brotli-first guess -- every 200 returned readable HTML (Brotli never blocked). The
  // dominant real barrier is JS EXECUTION: SPAs return only a shell (reddit 8KB, x.com, imdb 202+empty) and
  // Cloudflare managed-challenge sites (javdb 403, cloudflare.com) serve the challenge page -- both need our
  // browser to run async JS + a BOM. Secondary: soft bot-blocks (reuters 401) fixable with h2 + full browser
  // headers. TLS-1.2 fallback now broadly helps (github/stackoverflow/x/nasa/nhentai/ukdevilz all 200 via 1.2).
  // ★MEASURED-NEGATIVE PASS (2026-07-05, live probes -- refuted three plausible "wins" before wiring them):
  //   (a) HTTP/2: fetched reuters/imdb/bbc/x/reddit over the sovereign h2 stack. reuters STILL 401, imdb STILL
  //       202, bbc/x parity 200, reddit REGRESSED 200-shell->403. So h2 = parity-not-reach; NOT wired (would
  //       add cost + a reddit regression for zero unlock). (b) javdb/jable challenge = MEASURED Turnstile/
  //       managed ("Just a moment"/"Attention Required", no old solvable IUAM) -> real browser + fingerprint,
  //       likely NOT sovereignly solvable. (c) cloudflare.com was a detector FALSE-POSITIVE (bare-word match)
  //       -> fixed to interstitial-title/__cf_chl markers; it is REACHABLE. (d) Brotli never blocked a 200.
  w("HONEST SOTA GATE: NOT state-of-the-art, but the reach envelope GREW this session. Reaches static/traditional\n" as *u8)
  w("  + TLS-1.2-ECDSA (ukdevilz) + SEO-structured + NOW SPA content via the NON-JS feed path (RSS/Atom extract+\n" as *u8)
  w("  discover WIRED: old.reddit 25 items, nyt 14). MEASURED-NEGATIVES: HTTP/2 does NOT grow reach (reuters/imdb\n" as *u8)
  w("  still blocked, reddit regressed); Brotli never blocked. RESIDUAL browser-JS-async+BOM arc now NARROWER --\n" as *u8)
  w("  only sites with NO feed AND NO SSR alt-host AND Turnstile (javdb/jable). Priority = feed-conventions(/.rss,\n" as *u8)
  w("  /feed)+sitemap.xml > separate-doc-per-feed-item(frontier) > browser-JS-async(Turnstile) > robots-respect > IPv6.\n" as *u8)
  return 0
97}
nx_crawl_coverage_census.nx source

`nx_crawl_coverage_census.nx` source
