code wiki / _hdl_build / nx_domain_map_gate.nx
`nx_domain_map_gate.nx` source

↩ module page · 127 lines · 8858 B
1// nx_domain_map_gate.nx -- GATE for nx_domain_map (the /compare/webscraping R8 contracts dm_scan + dm_soft404).
2// Network-free: every parser and the fingerprint classifier are PURE and are tested here on runtime fixtures; the
3// live composition is proven by running the organ on a real domain (its summary line carries the partition and the
4// fetch counts). Accept rule pre-declared on the board: a 200 that matches the not-found fingerprint is SOFT404, a
5// 200 with a different title or a different length is LIVE (the false-positive control), a host whose fingerprint
6// is a real 404 can never produce SOFT404; robots paths are mined without wildcards; crt.sh hosts are deduplicated,
7// lowercased, wildcard-stripped and kept under the domain only.
8// license_tier: ORIGINAL
9import "nx_syscalls.nx"
10import "nx_domain_map.nx"
11import "nx_gate_verdict.nx"
12
13const DG_CAP: i64 = 8192
14const DG_SLOT: i64 = 256
15const DG_MAX: i64 = 32
16
17func dg_len(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
18func dg_slot(b: *u8, i: i64) -> *u8 { return (b as i64 + i * DG_SLOT) as *u8 }
19
20func main() -> i64 {
  gv_head("=== nx_domain_map gate (soft-404 fingerprint, robots path mining, crt.sh hosts, sitemap recursion; network-free) ===" as *u8)
  let ctr: *i64 = gv_ctr()
  dm_load_conf()
  gv_puts("  conf_src=" as *u8); if dm_conf_src_g == 1 { gv_puts("file" as *u8) } else { gv_puts("defaults" as *u8) }
  gv_puts(" max_fetches=" as *u8); gv_num(dm_max_fetches_g); gv_puts(" tol_permil=" as *u8); gv_num(dm_tol_g); gv_puts("\n" as *u8)
26
  // ---- T1 title hash ----
  let h1: *u8 = "<html><head><TITLE>Not Found</TITLE></head><body>x</body></html>" as *u8
  let h2: *u8 = "<html><head><title>Not Found</title></head><body>yy</body></html>" as *u8
  let h3: *u8 = "<html><head><title>Docs</title></head><body>real</body></html>" as *u8
  let h4: *u8 = "<html><body>no title here</body></html>" as *u8
  var t1: i64 = 0
  if dm_title_hash(h1, dg_len(h1)) == dm_title_hash(h2, dg_len(h2)) { if dm_title_hash(h1, dg_len(h1)) != dm_title_hash(h3, dg_len(h3)) { if dm_title_hash(h4, dg_len(h4)) == 0 { t1 = 1 } } }
  gv_check("T1 title hash: case-insensitive tag, equal titles equal, different titles differ, no title -> 0" as *u8, t1, ctr)
35
  // ---- T2 soft-404 classifier ----
  let fpt: i64 = dm_title_hash(h1, dg_len(h1))
  let doct: i64 = dm_title_hash(h3, dg_len(h3))
  var t2: i64 = 1
  if dm_soft404_classify(200, 1000, fpt, 200, 1050, fpt, 100) != 1 { t2 = 0 }   // same status, title, +5 pct -> SOFT404
  if dm_soft404_classify(200, 1000, fpt, 200, 1200, fpt, 100) != 0 { t2 = 0 }   // +20 pct -> LIVE
  if dm_soft404_classify(200, 1000, fpt, 200, 1000, doct, 100) != 0 { t2 = 0 }  // different title -> LIVE
  if dm_soft404_classify(404, 1000, fpt, 200, 1000, fpt, 100) != 0 { t2 = 0 }   // real-404 host -> never SOFT404
  if dm_soft404_classify(200, 1000, fpt, 404, 1000, fpt, 100) != 0 { t2 = 0 }   // a real 404 probe is not soft
  if dm_soft404_classify(200, 0, 0, 200, 0, 0, 100) != 1 { t2 = 0 }             // empty fingerprint, empty probe -> soft (nothing served either way)
  gv_check("T2 soft-404 classifier: status + title + length tolerance, real-404 hosts exempt (6 KATs)" as *u8, t2, ctr)
  var bad: i64 = dm_soft404_classify(200, 1000, fpt, 200, 1020, fpt, 100)
  var good: i64 = dm_soft404_classify(200, 1000, fpt, 200, 5400, doct, 100)
  gv_bite("T3 BITE: fires on the not-found page wearing a 200, silent on a real docs page" as *u8, bad, good, ctr)
50
  // ---- T3b the dm_soft404 contract: abstains before a fingerprint exists, then classifies from the run's own ----
  var t3b: i64 = 0
  dm_fp_taken_g = 0
  if dm_soft404(200, 1000, fpt) == 0 - 1 {
      dm_fp_taken_g = 1; dm_fp_status_g = 200; dm_fp_len_g = 1000; dm_fp_title_g = fpt
      if dm_soft404(200, 1020, fpt) == 1 { if dm_soft404(200, 1020, doct) == 0 { t3b = 1 } }
  }
  gv_check("T3b dm_soft404 returns -1 (UNOBSERVABLE) before a fingerprint exists, then classifies from the run's own fingerprint" as *u8, t3b, ctr)
59
  // ---- T4 robots path mining ----
  let rob: *u8 = "User-agent: *\nDisallow: /private/\nAllow: /api/public\nDisallow: /*.pdf$\nDisallow: /\nDisallow: /private/\nSitemap: https://a.org/sitemap.xml\nDisallow: /admin\n" as *u8
  let paths: *u8 = sys_mmap(DG_SLOT * DG_MAX)
  let np: i64 = dm_robots_paths(rob, dg_len(rob), paths, DG_SLOT, DG_MAX)
  gv_puts("  robots paths=" as *u8); gv_num(np); gv_puts("\n" as *u8)
  var t4: i64 = 0
  if np == 3 { if dm_streq(dg_slot(paths, 0), "/private/" as *u8) == 1 { if dm_streq(dg_slot(paths, 1), "/api/public" as *u8) == 1 { if dm_streq(dg_slot(paths, 2), "/admin" as *u8) == 1 { t4 = 1 } } } }
  gv_check("T4 robots mining: Allow and Disallow paths kept, wildcard and root rules skipped, duplicates collapsed, Sitemap line ignored" as *u8, t4, ctr)
68
  // ---- T5 crt.sh hosts ----
  let js: *u8 = "[{\"issuer_name\":\"x\",\"name_value\":\"*.a.org\\nAPI.a.org\\nwww.a.org\"},{\"name_value\":\"evil.b.org\\nnota-a.org\"},{\"name_value\":\"www.a.org\"},{\"name_value\":\"a.org\"}]" as *u8
  let hosts: *u8 = sys_mmap(DG_SLOT * DG_MAX)
  let ov: *i64 = sys_mmap(16) as *i64
  ov[0] = 0
  let nh: i64 = dm_crt_hosts(js, dg_len(js), "a.org" as *u8, hosts, DG_SLOT, DG_MAX, ov)
  gv_puts("  crt hosts=" as *u8); gv_num(nh); gv_puts(":" as *u8)
  var hi: i64 = 0
  while hi < nh { gv_puts(" " as *u8); gv_puts(dg_slot(hosts, hi)); hi = hi + 1 }
  gv_puts("\n" as *u8)
  var t5: i64 = 0
  if nh == 3 { if dm_streq(dg_slot(hosts, 0), "a.org" as *u8) == 1 { if dm_streq(dg_slot(hosts, 1), "api.a.org" as *u8) == 1 { if dm_streq(dg_slot(hosts, 2), "www.a.org" as *u8) == 1 { t5 = 1 } } } }
  gv_check("T5 crt.sh: wildcard stripped, lowercased, deduplicated, foreign and lookalike hosts refused (nota-a.org)" as *u8, t5, ctr)
  var t5b: i64 = 0
  ov[0] = 0
  let nh2: i64 = dm_crt_hosts(js, dg_len(js), "a.org" as *u8, hosts, DG_SLOT, 2, ov)
  if nh2 == 2 { if ov[0] == 2 { t5b = 1 } }
  gv_check("T5b the host cap COUNTS each refused occurrence (www.a.org twice past the cap -> oversize=2); repeats of kept hosts are neither" as *u8, t5b, ctr)
87
  // ---- T6 gzip magic + sitemap recursion classification (the estate's own sm_is_subsitemap) ----
  let gz: *u8 = sys_mmap(16)
  gz[0] = 31 as u8; gz[1] = 139 as u8; gz[2] = 8 as u8
  var t6: i64 = 0
  if dm_is_gzip(gz, 3) == 1 { if dm_is_gzip("<?xml" as *u8, 5) == 0 { if dm_is_gzip(gz, 1) == 0 { t6 = 1 } } }
  gv_check("T6 gzip magic detected, XML and a 1-byte body are not gzip" as *u8, t6, ctr)
  var t7: i64 = 0
  if sm_is_subsitemap("https://a.org/sitemap-posts.xml.gz" as *u8, 34) == 1 { if sm_is_subsitemap("https://a.org/post/1" as *u8, 20) == 0 { t7 = 1 } }
  gv_check("T7 a .xml.gz loc is a sub-sitemap to recurse into, a page loc is a url row" as *u8, t7, ctr)
97
  // ---- T8 the nonce path and the url builder ----
  let p: *u8 = sys_mmap(DG_SLOT)
  let pl: i64 = dm_nonce_path(p, "a.org" as *u8)
  let p2: *u8 = sys_mmap(DG_SLOT)
  dm_nonce_path(p2, "b.org" as *u8)
  let u: *u8 = sys_mmap(DG_SLOT)
  let ul: i64 = dm_url(u, "a.org" as *u8, "/docs" as *u8)
  var t8: i64 = 0
  if pl > 20 { if (p[0] as i64) == 47 { if dm_streq(p, p2) == 0 { if dm_streq(u, "https://a.org/docs" as *u8) == 1 { if ul == 18 { t8 = 1 } } } } }
  gv_check("T8 the fingerprint path is per-domain and cannot collide with a real path; url builder is exact" as *u8, t8, ctr)
108
  // ---- T8b a budget refusal is OUR decision, never the host's: status box carries DM_STATUS_BUDGET ----
  let sb: *i64 = sys_mmap(16) as *i64
  dm_max_fetches_g = 0
  let fb: *u8 = sys_mmap(DG_SLOT)
  let nfb: i64 = dm_fetch("https://a.org/x" as *u8, 0 as *TrustStore, fb, DG_SLOT, sb)
  var t8b: i64 = 0
  if nfb == 0 { if sb[0] == DM_STATUS_BUDGET { t8b = 1 } }
  gv_check("T8b past the fetch budget dm_fetch refuses without a network call and marks the status box BUDGET, not UNREACHABLE" as *u8, t8b, ctr)
  dm_load_conf()
118
  // ---- T9 partition arithmetic on the counters (the summary line's own rule) ----
  dm_rows_total_g = 0; dm_rows_robots_g = 0; dm_rows_sitemap_g = 0; dm_rows_feed_g = 0; dm_rows_wayback_g = 0; dm_rows_crt_g = 0; dm_rows_probe_g = 0
  dm_outfd_g = 0
  var t9: i64 = 0
  if dm_row("probe" as *u8, "path" as *u8, DM_V_LIVE, "/x" as *u8, 2) == 0 { if dm_rows_total_g == 0 { t9 = 1 } }
  gv_check("T9 with no output file open, a row is refused (0) and not counted -- rows are what was WRITTEN" as *u8, t9, ctr)
125
  return gv_verdict("DOMAIN-MAP-GATE" as *u8, ctr, "soft-404 fingerprint classifies by status+title+length with real-404 hosts exempt; robots paths, crt.sh hosts, gzip and sitemap recursion parse as declared; caps count what they refuse" as *u8)
127}
nx_domain_map_gate.nx source

`nx_domain_map_gate.nx` source
