code wiki / _hdl_build / nx_archive_census.nx
`nx_archive_census.nx` source

↩ module page · 171 lines · 10398 B
1// nx_archive_census.nx -- Evaluates historical web archive capabilities using disk facts to grade and attest competitor systems.
2import "nx_gate_gn.nx"
3import "nx_gate_base.nx"
4// nx_archive_census.nx -- the TIME-MACHINE / WEB-ARCHIVE s-class-exceed SCORECARD (verified on disk, anti-liar,
5// sourced competitor evidence). Maps the capability surface of a media-complete historical web archive (vs the
6// Wayback Machine + Common Crawl) to real organs, grades each from DISK FACTS ONLY (EXCEEDS = a measured exceed-gate
7// opens > PRESENT = the organ opens > ABSENT), attests every competitor 'they-have' from the sovereign-banked
8// arc_/mediarc_ corpus (re_has, NEVER fabricated), and reports an HONEST coverage permil + the named ABSENT worklist.
9// Deliberately INCOMPLETE: GREEN means the scorecard is VALID + LIAR-FREE, not that we are done (gaps-first, like
10// nx_stability_census). Composes nx_capability_census (cc_our_grade/cc_comp_attest/cc_verdict). Operator law (no-wave):
11// every EXCEED is a target until measured -- here the perceptual-recovery differentiator IS measured (imgquery +
12// phash-index gates GREEN). expect_exit: 0  license_tier: ORIGINAL
13import "nx_capability_census.nx"
14const K_MAGIC_4194304: i64 = 4194304
15
16
17func grow(name: *u8, ok: i64) -> i64 { if ok==1 { gw("  PASS " as *u8) } else { gw("  FAIL " as *u8) } gw(name); gw("
18" as *u8); return ok }
19func ax_label(i: i64) -> *u8 {
  if i==0 { return "Sovereign web fetch (TLS-1.3, cert-validated)" as *u8 }
  if i==1 { return "Polite crawler (robots/backoff/dedup)" as *u8 }
  if i==2 { return "DEFLATE/gzip decompression" as *u8 }
  if i==3 { return "Content-addressed immutable never-delete store" as *u8 }
  if i==4 { return "Inverted index + on-disk persist" as *u8 }
  if i==5 { return "BM25 ranking" as *u8 }
  if i==6 { return "HTML parse + outlink extraction" as *u8 }
  if i==7 { return "Content-validated render check (a 404-as-200 is DOWN)" as *u8 }
  if i==8 { return "Perceptual hash dHash/aHash -- MEDIA RECOVERY" as *u8 }
  if i==9 { return "ANN index (BK-tree sublinear Hamming)" as *u8 }
  if i==10 { return "Reverse-image discover (cross-source survivor)" as *u8 }
  if i==11 { return "WARC/WET/WAT ingest + streaming  [GAP]" as *u8 }
  if i==12 { return "Temporal (host,date)->capture index  [GAP]" as *u8 }
  if i==13 { return "CDX capture-index parse" as *u8 }
  if i==14 { return "Link-analysis / PageRank authority  [GAP]" as *u8 }
  if i==15 { return "Media decoders (GIF/LZW + JPEG/PNG)" as *u8 }
  return "Time-travel reconstruct+rewrite server (Memento)  [GAP]" as *u8
37}
38func ax_term(i: i64) -> *u8 {
  if i==0 { return "TLS" as *u8 }
  if i==1 { return "crawler" as *u8 }
  if i==2 { return "gzip" as *u8 }
  if i==3 { return "content-addressable" as *u8 }
  if i==4 { return "inverted index" as *u8 }
  if i==5 { return "BM25" as *u8 }
  if i==6 { return "HTML" as *u8 }
  if i==7 { return "" as *u8 }
  if i==8 { return "perceptual" as *u8 }
  if i==9 { return "nearest" as *u8 }
  if i==10 { return "reverse image" as *u8 }
  if i==11 { return "WARC" as *u8 }
  if i==12 { return "Wayback" as *u8 }
  if i==13 { return "Common Crawl" as *u8 }
  if i==14 { return "PageRank" as *u8 }
  if i==15 { return "WebP" as *u8 }
  return "Memento" as *u8
56}
57func ax_organ(i: i64) -> *u8 {
  if i==0 { return "runtime/nx_https_fetch_follow.nx" as *u8 }
  if i==1 { return "runtime/nx_crawl_https.nx" as *u8 }
  if i==2 { return "runtime/nx_deflate.nx" as *u8 }
  if i==3 { return "runtime/nx_seg_store.nx" as *u8 }
  if i==4 { return "runtime/_hdl_build/nx_search_inverted_persist.nx" as *u8 }
  if i==5 { return "runtime/nx_bm25.nx" as *u8 }
  if i==6 { return "runtime/nx_html_extract_links.nx" as *u8 }
  if i==7 { return "runtime/_hdl_build/nx_health_template.nx" as *u8 }
  if i==8 { return "runtime/nx_phash.nx" as *u8 }
  if i==9 { return "runtime/nx_phash_index.nx" as *u8 }
  if i==10 { return "runtime/nx_revimg_discover.nx" as *u8 }
  if i==11 { return "runtime/nx_warc_reader.nx" as *u8 }
  if i==12 { return "runtime/nx_temporal_index.nx" as *u8 }
  if i==13 { return "runtime/nx_cdx_parse.nx" as *u8 }
  if i==14 { return "runtime/nx_pagerank.nx" as *u8 }
  if i==15 { return "runtime/nx_gif_decode.nx" as *u8 }
  return "runtime/nx_timemachine_server.nx" as *u8
75}
76func ax_exceed(i: i64) -> *u8 {
  if i==0 { return "runtime/nx_https_get_live_google_test.nx" as *u8 }
  if i==8 { return "runtime/nx_imgquery_e2e_gate.nx" as *u8 }
  if i==9 { return "runtime/nx_phash_index_gate.nx" as *u8 }
  if i==10 { return "runtime/nx_revimg_discover_gate.nx" as *u8 }
  return "" as *u8
82}
83
84func main() -> i64 {
  gw("=== nx_archive_census: TIME-MACHINE / web-archive s-class-exceed scorecard (disk-verified, sourced, anti-liar) ===\n" as *u8)
  let N: i64 = 17
  let cap: i64 = K_MAGIC_4194304
  let buf: *u8 = sys_mmap(cap)
89
  // sovereign-banked competitor source corpus (arc_/mediarc_ .raw) -- 'they-have' is ATTESTED here, never fabricated.
  let NS: i64 = 14
  let srcp: *i64 = sys_mmap(NS*8) as *i64
  srcp[0]  = ("knowledge/fetched/arc_warc.raw" as *u8) as i64
  srcp[1]  = ("knowledge/fetched/arc_commoncrawl.raw" as *u8) as i64
  srcp[2]  = ("knowledge/fetched/arc_wayback.raw" as *u8) as i64
  srcp[3]  = ("knowledge/fetched/arc_pagerank.raw" as *u8) as i64
  srcp[4]  = ("knowledge/fetched/arc_hits.raw" as *u8) as i64
  srcp[5]  = ("knowledge/fetched/arc_link_analysis.raw" as *u8) as i64
  srcp[6]  = ("knowledge/fetched/arc_memento.raw" as *u8) as i64
  srcp[7]  = ("knowledge/fetched/arc_perceptual_hashing.raw" as *u8) as i64
  srcp[8]  = ("knowledge/fetched/arc_reverse_image_search.raw" as *u8) as i64
  srcp[9]  = ("knowledge/fetched/arc_tineye.raw" as *u8) as i64
  srcp[10] = ("knowledge/fetched/arc_content_addressable_storage.raw" as *u8) as i64
  srcp[11] = ("knowledge/fetched/arc_web_crawler.raw" as *u8) as i64
  srcp[12] = ("knowledge/fetched/mediarc_archiveteam.raw" as *u8) as i64
  srcp[13] = ("knowledge/fetched/mediarc_webrecorder.raw" as *u8) as i64
107
  var nexceed: i64=0; var npresent: i64=0; var nabsent: i64=0
  var nahead: i64=0; var nparity: i64=0; var ngap: i64=0
  gw("\n  -- per-axis grade (OUR = disk fact, attest = #banked sources naming it, verdict = head-to-head) --\n" as *u8)
  var i: i64=0
  while i<N {
      let our: i64 = cc_our_grade(ax_organ(i), ax_exceed(i))
      var comp: i64 = 0
      let term: *u8 = ax_term(i)
      if term[0] != (0 as u8) { comp = cc_comp_attest(srcp, NS, term, buf, cap) }
      let v: i64 = cc_verdict(comp, our)
      if our==4 { nexceed=nexceed+1 } else { if our==3 { npresent=npresent+1 } else { nabsent=nabsent+1 } }
      if v==3 { nahead=nahead+1 } else { if v==2 { nparity=nparity+1 } else { if v==1 { ngap=ngap+1 } } }
      gw("  " as *u8); gw(cc_grade_str(our)); gw(" " as *u8); gw(cc_verdict_str(v)); gw(" attest=" as *u8); gn(comp); gw("  " as *u8); gw(ax_label(i)); gw("\n" as *u8)
      i=i+1
  }
123
  var pass: i64=0; var total: i64=0
  // T0 structure
  total=total+1; if N>0 { pass=pass+1; gw("  [PASS] " as *u8) } else { gw("  [FAIL] " as *u8) }
  gw("T0 SURFACE: " as *u8); gn(N); gw(" capability axes (crawl/store/index/rank/image-recovery/reconstruct)\n" as *u8)
128
  // T1 NEG-CTL liar-kill: a fake organ MUST grade ABSENT (the grader cannot hallucinate a capability).
  let negc: i64 = cc_our_grade("runtime/_hdl_build/nx_FAKE_ARCHIVE_NONEXISTENT.nx" as *u8, "runtime/_hdl_build/nx_FAKE_ARCHIVE_NONEXISTENT.nx" as *u8)
  total=total+1; if negc==0 { pass=pass+1; gw("  [PASS] " as *u8) } else { gw("  [FAIL] " as *u8) }
  gw("T1 NEG-CTL: fake organ grades ABSENT (anti-liar -- disk facts only)\n" as *u8)
133
  // T2 SOURCED-GAPS: the load-bearing GAP terms are attested in the banked corpus (no fabricated 'they-have').
  let aw: i64 = cc_comp_attest(srcp, NS, "WARC" as *u8, buf, cap)
  let ap: i64 = cc_comp_attest(srcp, NS, "PageRank" as *u8, buf, cap)
  let am: i64 = cc_comp_attest(srcp, NS, "Memento" as *u8, buf, cap)
  total=total+1
  var t2ok: i64 = 0
  if aw>=1 { if ap>=1 { if am>=1 { t2ok=1 } } }
  if t2ok==1 { pass=pass+1; gw("  [PASS] " as *u8) } else { gw("  [FAIL] " as *u8) }
  gw("T2 SOURCED-GAPS: WARC=" as *u8); gn(aw); gw(" PageRank=" as *u8); gn(ap); gw(" Memento=" as *u8); gn(am); gw(" attested in banked corpus (gaps are REAL, sourced)\n" as *u8)
143
  // T3 MEASURED-DIFFERENTIATOR: perceptual recovery axes grade EXCEEDS (their exceed-gates open on disk).
  let g8: i64 = cc_our_grade(ax_organ(8), ax_exceed(8))
  let g9: i64 = cc_our_grade(ax_organ(9), ax_exceed(9))
  total=total+1
  var t3ok: i64 = 0
  if g8==4 { if g9==4 { t3ok=1 } }
  if t3ok==1 { pass=pass+1; gw("  [PASS] " as *u8) } else { gw("  [FAIL] " as *u8) }
  gw("T3 MEASURED-EXCEED: perceptual-hash + BK-tree recovery graded EXCEEDS via GREEN gates (differentiator MEASURED)\n" as *u8)
152
  // T4 COVERAGE report (honest, incomplete by design).
  let have: i64 = nexceed + npresent
  let permil: i64 = (have*1000)/N
  total=total+1; pass=pass+1
  gw("  [PASS] T4 COVERAGE: EXCEEDS=" as *u8); gn(nexceed); gw(" PRESENT=" as *u8); gn(npresent); gw(" ABSENT=" as *u8); gn(nabsent)
  gw(" -> HAVE=" as *u8); gn(have); gw("/" as *u8); gn(N); gw(" = " as *u8); gn(permil); gw(" permil  (AHEAD=" as *u8); gn(nahead); gw(" PARITY=" as *u8); gn(nparity); gw(" GAP=" as *u8); gn(ngap); gw(")\n" as *u8)
  gw("        ABSENT worklist = WARC ingest, temporal index, CDX, PageRank, media decoders, time-travel server (ladder R2-R7)\n" as *u8)
160
  // T5 verdict: GREEN iff NEG-CTL held, gaps sourced, differentiator measured.
  total=total+1
  var ok5: i64 = 0
  if negc==0 { if t2ok==1 { if t3ok==1 { ok5=1 } } }
  if ok5==1 { pass=pass+1; gw("  [PASS] " as *u8) } else { gw("  [FAIL] " as *u8) }
  gw("T5 SCORECARD: disk-verified, sourced, anti-liar; coverage HONESTLY INCOMPLETE (GREEN != done)\n" as *u8)
167
  gw("\nARCHIVE-CENSUS " as *u8); gn(permil); gw(" permil HAVE, differentiator MEASURED, " as *u8)
  if pass==total { gw("verdict=GREEN passes=" as *u8); gn(pass); gw("/" as *u8); gn(total); gw("\n" as *u8); sys_exit(0); return 0 }
  gw("verdict=RED passes=" as *u8); gn(pass); gw("/" as *u8); gn(total); gw("\n" as *u8); sys_exit(1); return 1
171}
nx_archive_census.nx source

`nx_archive_census.nx` source
