nx_domain_map.nx source

1// nx_domain_map.nx -- DOMAIN MAP: every URL under a domain BEFORE crawling it -- the /compare/webscraping R8 contracts 2// dm_scan (the composed census) and dm_soft404 (the fingerprint that keeps a 200-that-is-really-404 out of it). 3// Concept source: crawl4ai DomainMapper (eight discovery sources + soft-404 fingerprint). Sovereign form composes the 4// estate's own parsers, never a second one: 5// robots nx_sitemap_extract.sm_robots_sitemaps (Sitemap: lines) + dm_robots_paths (Allow/Disallow paths a site 6// ACKNOWLEDGES exist) ............................................................ source=robots 7// sitemap sm_extract_locs over every sitemap, RECURSING sitemapindex (sm_is_subsitemap) and inflating .gz 8// bodies (nx_gzip_wrap) ........................................................... source=sitemap 9// feed nx_feed_extract on the homepage (feed href -> items) ............................ source=feed 10// wayback the Internet Archive CDX API, original urls, 200s only ............................ source=wayback 11// crt crt.sh certificate-transparency JSON -> subdomains ............................... source=crt (kind=host) 12// probe knowledge/domain_map_probe.conf paths (robots-checked) classified against the SOFT-404 FINGERPRINT taken 13// first from a path that cannot exist ............................................. source=probe 14// EVERY ROW IS APPENDED AS IT IS DECIDED (a run killed at 95 pct keeps 95 pct; the estate's long-sweep law), to 15// knowledge/status/domain_map/<domain>.tsv: ts TAB source TAB kind(url|host|path) TAB verdict TAB value. The final 16// line prints the partition (rows per source) and it SUMS to rows written; fetches used, refused (robots), oversize 17// (a <loc> past the url slot) and truncated (a body that filled the cap) are counted, never silent. 18// PACED: every fetch goes through nx_crawl_pace (per host, Retry-After, robots Crawl-delay); probes are refused when 19// robots forbids the path, and a refusal is a ROW, not a silence. 20// usage: nx_domain_map <domain> [max_fetches] [out.tsv] (rows and caps: knowledge/domain_map.conf) 21// exit: 0 rows written | 2 usage | 3 trust store | 4 nothing reachable (0 fetches succeeded) 22// license_tier: ORIGINAL No hw writes (Rule 26). 23import "nx_syscalls.nx" 24import "nx_lane_conf.nx" 25import "nx_x509_trust_store.nx" 26import "nx_trust_store_load_from_certdata.nx" 27import "nx_https_fetch_follow.nx" 28import "nx_gzip_wrap.nx" 29import "nx_sitemap_extract.nx" 30import "nx_feed_extract.nx" 31import "nx_robots.nx" 32import "nx_crawl_pace.nx" 33import "nx_itoa_lib.nx" 34 35const DM_CONF: *u8 = "knowledge/domain_map.conf" 36const DM_PROBES: *u8 = "knowledge/domain_map_probe.conf" 37const DM_OUTDIR: *u8 = "knowledge/status/domain_map" 38const DM_CERTDATA: *u8 = "data/mozilla_certdata.txt" 39// bootstrap defaults (rule 17: argv > conf > these). Reason per row: 40const DM_DEF_MAX_FETCHES: i64 = 64 // one map is a reconnaissance, not a crawl: 64 fetches covers robots + a sitemap tree + probes 41const DM_DEF_WAYBACK_LIMIT: i64 = 500 // CDX rows requested; the API accepts a limit and 500 keeps the body under the fetch cap 42const DM_DEF_MAX_SITEMAPS: i64 = 32 // sitemap files followed through a sitemapindex before the map says so and stops 43const DM_DEF_MAX_LOCS: i64 = 50000 // sitemaps.org's own per-file ceiling; a file past it is out of spec 44const DM_DEF_MAX_HOSTS: i64 = 256 // distinct subdomains kept from crt.sh; the rest are counted as oversize 45const DM_DEF_FETCH_CAP: i64 = 8388608 // 8 MiB body cap per fetch (announced when filled); crt.sh answers can be MBs 46const DM_DEF_SOFT404_TOL_PERMIL: i64 = 100 // crawl4ai Soft404Fingerprint compares status, title and content length; +-10 pct length 47const DM_DEF_MAX_HOPS: i64 = 6 // redirect hops, the crawler's own number 48const DM_URL_SLOT: i64 = 2048 // one url slot; sitemaps.org allows a 2048-char <loc>, longer is out of spec 49const DM_HOST_SLOT: i64 = 256 // one host slot (a DNS name is at most 253 bytes) 50const DM_PATH_SLOT: i64 = 512 // one probe or robots path slot 51const DM_TS_BUF: i64 = 32 52const DM_LINE_CAP: i64 = 4096 53const DM_SMALL: i64 = 1024 54const DM_TRUST_ANCHORS: i64 = 512 // the crawler's own trust-store sizing 55const DM_TRUST_PARSE_CAP: i64 = 4194304 56const DM_MODE_0644: i64 = 420 57const DM_MODE_0755: i64 = 493 58const DM_GZ_ID1: i64 = 31 59const DM_GZ_ID2: i64 = 139 60const DM_UALEN: i64 = 8 // len("nishibot"), the crawler's UA token 61const DM_TAB: i64 = 9 62const DM_NL: i64 = 10 63const DM_CR: i64 = 13 64const DM_HASH: i64 = 35 65const DM_QUOTE: i64 = 34 66const DM_BACKSLASH: i64 = 92 67const DM_SLASH: i64 = 47 68const DM_COLON: i64 = 58 69const DM_LT: i64 = 60 70const DM_GT: i64 = 62 71const DM_DIGIT0: i64 = 48 72const DM_DIGIT9: i64 = 57 73const DM_HEX_BASE: i64 = 16 74const DM_HASH_SEED: i64 = 5381 75const DM_HASH_MASK: i64 = 0x7fffffffffffffff 76const DM_PROBE_DEFAULTS: i64 = 15 77// verdicts 78const DM_V_DECLARED: *u8 = "DECLARED" 79const DM_V_LISTED: *u8 = "LISTED" 80const DM_V_ARCHIVED: *u8 = "ARCHIVED" 81const DM_V_CERT: *u8 = "CERT" 82const DM_V_LIVE: *u8 = "LIVE" 83const DM_V_SOFT404: *u8 = "SOFT404" 84const DM_V_MISSING: *u8 = "MISSING" 85const DM_V_BLOCKED: *u8 = "BLOCKED" 86const DM_V_UNREACHABLE: *u8 = "UNREACHABLE" 87const DM_V_ROBOTS_REFUSED: *u8 = "ROBOTS-REFUSED" 88const DM_V_FINGERPRINT: *u8 = "FINGERPRINT" 89const DM_HTTP_OK: i64 = 200 90const DM_HTTP_NOTFOUND: i64 = 404 91const DM_HTTP_ERR_FLOOR: i64 = 400 92const DM_STATUS_BUDGET: i64 = 0 - 1 // status-box value meaning: this fetch was refused by OUR budget, the host was never asked 93const DM_V_BUDGET: *u8 = "BUDGET-REFUSED" 94 95static dm_outfd_g: i64 96static dm_fetches_g: i64 97static dm_fetch_ok_g: i64 98static dm_refused_g: i64 99static dm_oversize_g: i64 100static dm_truncated_g: i64 101static dm_rows_robots_g: i64 102static dm_rows_sitemap_g: i64 103static dm_rows_feed_g: i64 104static dm_rows_wayback_g: i64 105static dm_rows_crt_g: i64 106static dm_rows_probe_g: i64 107static dm_rows_total_g: i64 108static dm_max_fetches_g: i64 109static dm_wayback_limit_g: i64 110static dm_max_sitemaps_g: i64 111static dm_max_locs_g: i64 112static dm_max_hosts_g: i64 113static dm_fetch_cap_g: i64 114static dm_tol_g: i64 115static dm_max_hops_g: i64 116static dm_conf_src_g: i64 117// soft-404 fingerprint 118static dm_fp_status_g: i64 119static dm_fp_len_g: i64 120static dm_fp_title_g: i64 121static dm_fp_taken_g: i64 122 123func dm_len(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n } 124func dm_puts(s: *u8) -> i64 { sys_write(1, s, dm_len(s)); return 0 } 125func dm_num(v: i64) -> i64 { nxi_out(v); return 0 } 126func dm_cat(d: *u8, o: i64, s: *u8) -> i64 { var i: i64 = 0; var p: i64 = o; while s[i] != (0 as u8) { d[p] = s[i]; p = p + 1; i = i + 1 } d[p] = 0 as u8; return p } 127func dm_catn(d: *u8, o: i64, s: *u8, n: i64) -> i64 { var i: i64 = 0; var p: i64 = o; while i < n { d[p] = s[i]; p = p + 1; i = i + 1 } d[p] = 0 as u8; return p } 128func dm_streq(a: *u8, b: *u8) -> i64 { var i: i64 = 0; while a[i] == b[i] { if a[i] == (0 as u8) { return 1 } i = i + 1 } return 0 } 129func dm_lc(c: i64) -> i64 { if c >= 65 { if c <= 90 { return c + 32 } } return c } 130func dm_conf_one(key: *u8, dflt: i64) -> i64 { 131 let v: i64 = lc_geti(DM_CONF, "" as *u8, key, 0 - 1) 132 if v < 0 { return dflt } 133 dm_conf_src_g = 1 134 return v 135} 136func dm_load_conf() -> i64 { 137 dm_conf_src_g = 0 138 dm_max_fetches_g = dm_conf_one("max_fetches" as *u8, DM_DEF_MAX_FETCHES) 139 dm_wayback_limit_g = dm_conf_one("wayback_limit" as *u8, DM_DEF_WAYBACK_LIMIT) 140 dm_max_sitemaps_g = dm_conf_one("max_sitemaps" as *u8, DM_DEF_MAX_SITEMAPS) 141 dm_max_locs_g = dm_conf_one("max_locs_per_sitemap" as *u8, DM_DEF_MAX_LOCS) 142 dm_max_hosts_g = dm_conf_one("max_hosts" as *u8, DM_DEF_MAX_HOSTS) 143 dm_fetch_cap_g = dm_conf_one("fetch_cap_bytes" as *u8, DM_DEF_FETCH_CAP) 144 dm_tol_g = dm_conf_one("soft404_len_tolerance_permil" as *u8, DM_DEF_SOFT404_TOL_PERMIL) 145 dm_max_hops_g = dm_conf_one("max_hops" as *u8, DM_DEF_MAX_HOPS) 146 return dm_conf_src_g 147} 148// djb2 over bytes, masked positive 149func dm_hash(b: *u8, n: i64) -> i64 { 150 var h: i64 = DM_HASH_SEED 151 var i: i64 = 0 152 while i < n { h = (((h << 5) + h) + (b[i] as i64)) & DM_HASH_MASK; i = i + 1 } 153 return h 154} 155// ---- PURE: the soft-404 fingerprint ------------------------------------------------------------------------- 156// title hash of an HTML body: hash of the bytes between <title> and </title> (case-insensitive tag), 0 when absent 157func dm_title_hash(html: *u8, n: i64) -> i64 { 158 var i: i64 = 0 159 while i + 7 <= n { 160 if (html[i] as i64) == DM_LT { if dm_lc(html[i + 1] as i64) == 116 { if dm_lc(html[i + 2] as i64) == 105 { if dm_lc(html[i + 3] as i64) == 116 { if dm_lc(html[i + 4] as i64) == 108 { if dm_lc(html[i + 5] as i64) == 101 { 161 var s: i64 = i + 6 162 var f: i64 = 0 163 while f == 0 { if s >= n { f = 1 } else { if (html[s] as i64) == DM_GT { s = s + 1; f = 1 } else { s = s + 1 } } } 164 var e: i64 = s 165 var g: i64 = 0 166 while g == 0 { if e >= n { g = 1 } else { if (html[e] as i64) == DM_LT { g = 1 } else { e = e + 1 } } } 167 if e <= s { return 0 } 168 return dm_hash((html as i64 + s) as *u8, e - s) 169 } } } } } } 170 i = i + 1 171 } 172 return 0 173} 174// 1 when a probe response is the not-found page wearing a 200: same status, same title, length within tolerance. 175// A host whose fingerprint is a real 404 has no soft-404 problem at all (fp_status != 200 -> never SOFT404). 176func dm_soft404_classify(fp_status: i64, fp_len: i64, fp_title: i64, status: i64, len: i64, title: i64, tol_permil: i64) -> i64 { 177 if fp_status != DM_HTTP_OK { return 0 } 178 if status != fp_status { return 0 } 179 if title != fp_title { return 0 } 180 var d: i64 = len - fp_len 181 if d < 0 { d = 0 - d } 182 if d * 1000 <= tol_permil * fp_len { return 1 } 183 return 0 184} 185// THE CONTRACT dm_soft404: classify a probe response against the fingerprint THIS RUN took (dm_source_probe takes it 186// first). Returns 1 SOFT404, 0 not the not-found page, -1 when no fingerprint exists yet (UNOBSERVABLE, never a guess). 187func dm_soft404(status: i64, len: i64, title: i64) -> i64 { 188 if dm_fp_taken_g == 0 { return 0 - 1 } 189 return dm_soft404_classify(dm_fp_status_g, dm_fp_len_g, dm_fp_title_g, status, len, title, dm_tol_g) 190} 191// ---- PURE: robots.txt path mining (Allow/Disallow values are paths the site acknowledges) --------------------- 192func dm_robots_paths(rob: *u8, n: i64, out: *u8, slot: i64, max: i64) -> i64 { 193 var count: i64 = 0 194 var i: i64 = 0 195 while i < n { 196 var at_line: i64 = 0 197 if i == 0 { at_line = 1 } else { if (rob[i - 1] as i64) == DM_NL { at_line = 1 } } 198 if at_line == 1 { 199 var key: i64 = 0 200 if sm_ci_at(rob, i, n, "disallow:" as *u8, 9) == 1 { key = 9 } 201 if sm_ci_at(rob, i, n, "allow:" as *u8, 6) == 1 { key = 6 } 202 if key > 0 { 203 var e: i64 = i 204 while e < n { if (rob[e] as i64) == DM_NL { e = n + e } else { e = e + 1 } } 205 if e > n { e = e - n } 206 if count < max { 207 let dst: *u8 = (out as i64 + count * slot) as *u8 208 let l: i64 = sm_copy_url(rob, i + key, e, dst, slot) 209 // a path row must start with '/', be more than the root and carry no wildcard 210 var ok: i64 = 0 211 if l > 1 { if (dst[0] as i64) == DM_SLASH { ok = 1 } } 212 var k: i64 = 0 213 while k < l { let c: i64 = dst[k] as i64; if c == 42 { ok = 0 } if c == 36 { ok = 0 } k = k + 1 } 214 if ok == 1 { 215 var dup: i64 = 0 216 var d: i64 = 0 217 while d < count { if dm_streq((out as i64 + d * slot) as *u8, dst) == 1 { dup = 1; d = count } else { d = d + 1 } } 218 if dup == 0 { count = count + 1 } 219 } 220 } 221 i = e 222 } 223 } 224 i = i + 1 225 } 226 return count 227} 228// ---- PURE: crt.sh JSON -> distinct hosts (name_value fields, newline-separated, wildcards stripped) ----------- 229func dm_crt_hosts(js: *u8, n: i64, domain: *u8, out: *u8, slot: i64, max: i64, oversize: *i64) -> i64 { 230 let key: *u8 = "\"name_value\":\"" as *u8 231 let kl: i64 = dm_len(key) 232 let dl: i64 = dm_len(domain) 233 var count: i64 = 0 234 var i: i64 = 0 235 while i + kl < n { 236 var m: i64 = 1 237 var k: i64 = 0 238 while k < kl { if js[i + k] != key[k] { m = 0; k = kl } else { k = k + 1 } } 239 if m == 1 { 240 var p: i64 = i + kl 241 var hs: i64 = p 242 var f: i64 = 0 243 while f == 0 { 244 var c: i64 = 0 245 var endhost: i64 = 0 246 if p >= n { endhost = 1; f = 1 } else { 247 c = js[p] as i64 248 if c == DM_QUOTE { endhost = 1; f = 1 } 249 if c == DM_BACKSLASH { if p + 1 < n { if (js[p + 1] as i64) == 110 { endhost = 1 } } } 250 } 251 if endhost == 1 { 252 var hl: i64 = p - hs 253 var ho: i64 = hs 254 if hl >= 2 { if (js[ho] as i64) == 42 { if (js[ho + 1] as i64) == 46 { ho = ho + 2; hl = hl - 2 } } } 255 // keep only hosts under the domain (exact or a dot-suffix), lowercase compare 256 var under: i64 = 0 257 if hl >= dl { 258 var eq: i64 = 1 259 var q: i64 = 0 260 while q < dl { if dm_lc(js[ho + hl - dl + q] as i64) != dm_lc(domain[q] as i64) { eq = 0; q = dl } else { q = q + 1 } } 261 if eq == 1 { if hl == dl { under = 1 } else { if (js[ho + hl - dl - 1] as i64) == 46 { under = 1 } } } 262 } 263 if under == 1 { if hl < slot { 264 // lowercase the candidate into the slot past the kept ones (scratch; callers allocate max+1 slots), 265 // dedup against the kept, then commit it or count it as refused past the cap -- a repeat of a 266 // kept host is neither a new host nor a refusal 267 let dst: *u8 = (out as i64 + count * slot) as *u8 268 var w: i64 = 0 269 while w < hl { dst[w] = dm_lc(js[ho + w] as i64) as u8; w = w + 1 } 270 dst[hl] = 0 as u8 271 var dup: i64 = 0 272 var d: i64 = 0 273 while d < count { if dm_streq((out as i64 + d * slot) as *u8, dst) == 1 { dup = 1; d = count } else { d = d + 1 } } 274 if dup == 0 { if count < max { count = count + 1 } else { oversize[0] = oversize[0] + 1 } } 275 } } 276 if f == 0 { p = p + 2; hs = p } else { p = p + 1 } 277 } else { p = p + 1 } 278 } 279 i = p 280 } else { i = i + 1 } 281 } 282 return count 283} 284func dm_is_gzip(b: *u8, n: i64) -> i64 { 285 if n < 2 { return 0 } 286 if (b[0] as i64) == DM_GZ_ID1 { if (b[1] as i64) == DM_GZ_ID2 { return 1 } } 287 return 0 288} 289 290// ---- output rows ------------------------------------------------------------------------------------------------ 291func dm_row(source: *u8, kind: *u8, verdict: *u8, value: *u8, vlen: i64) -> i64 { 292 if dm_outfd_g <= 0 { return 0 } 293 let b: *u8 = sys_mmap(DM_LINE_CAP + vlen) 294 var o: i64 = 0 295 o = ccz_cat_num(b, o, sys_now_realtime_sec()) 296 b[o] = DM_TAB as u8; o = o + 1 297 o = dm_cat(b, o, source) 298 b[o] = DM_TAB as u8; o = o + 1 299 o = dm_cat(b, o, kind) 300 b[o] = DM_TAB as u8; o = o + 1 301 o = dm_cat(b, o, verdict) 302 b[o] = DM_TAB as u8; o = o + 1 303 o = dm_catn(b, o, value, vlen) 304 b[o] = DM_NL as u8; o = o + 1 305 sys_write(dm_outfd_g, b, o) 306 sys_munmap(b, DM_LINE_CAP + vlen) 307 dm_rows_total_g = dm_rows_total_g + 1 308 return 1 309} 310// ---- paced fetch under the run budget: returns body bytes (0 on failure), status in stbox[0] ------------------- 311func dm_fetch(url: *u8, store: *TrustStore, out: *u8, cap: i64, stbox: *i64) -> i64 { 312 stbox[0] = 0 313 // MEASURED on the first live map (python.org, 2026-08-24): the last four probes were budget-refused and 314 // printed UNREACHABLE -- a host verdict for a decision that was ours. A refused fetch says so in the status box. 315 if dm_fetches_g >= dm_max_fetches_g { stbox[0] = DM_STATUS_BUDGET; return 0 } 316 dm_fetches_g = dm_fetches_g + 1 317 let hb: *u8 = sys_mmap(DM_HOST_SLOT) 318 // host = between "://" and the next '/' 319 var i: i64 = 0 320 var s: i64 = 0 321 var f: i64 = 0 322 while f == 0 { if url[i] == (0 as u8) { f = 1 } else { if url[i] == (58 as u8) { if url[i + 1] == (47 as u8) { if url[i + 2] == (47 as u8) { s = i + 3; f = 1 } } } if f == 0 { i = i + 1 } } } 323 var hl: i64 = 0 324 var g: i64 = 0 325 while g == 0 { let c: i64 = url[s + hl] as i64; if c == 0 { g = 1 } else { if c == DM_SLASH { g = 1 } else { if hl + 1 >= DM_HOST_SLOT { g = 1 } else { hb[hl] = url[s + hl]; hl = hl + 1 } } } } 326 hb[hl] = 0 as u8 327 pace_before(hb, hl) 328 let n: i64 = nx_https_fetch_follow_best(url, store, out, cap, dm_max_hops_g, stbox) 329 pace_after(hb, hl, stbox[0], 0) 330 sys_munmap(hb, DM_HOST_SLOT) 331 if n > 0 { dm_fetch_ok_g = dm_fetch_ok_g + 1 } 332 if n >= cap { dm_truncated_g = dm_truncated_g + 1 } 333 if n < 0 { return 0 } 334 return n 335} 336func dm_url(dst: *u8, domain: *u8, path: *u8) -> i64 { 337 var o: i64 = dm_cat(dst, 0, "https://" as *u8) 338 o = dm_cat(dst, o, domain) 339 o = dm_cat(dst, o, path) 340 return o 341} 342// the nonce path no site serves: /nx-dm-<hex of hash(domain)>-does-not-exist 343func dm_nonce_path(dst: *u8, domain: *u8) -> i64 { 344 var o: i64 = dm_cat(dst, 0, "/nx-dm-" as *u8) 345 var h: i64 = dm_hash(domain, dm_len(domain)) 346 var k: i64 = 0 347 while k < DM_HEX_BASE { let d: i64 = h % DM_HEX_BASE; if d < 10 { dst[o] = (DM_DIGIT0 + d) as u8 } else { dst[o] = (97 + d - 10) as u8 } o = o + 1; h = h / DM_HEX_BASE; k = k + 1 } 348 o = dm_cat(dst, o, "-does-not-exist" as *u8) 349 return o 350} 351 352// ---- sources ---------------------------------------------------------------------------------------------------- 353func dm_source_sitemaps(domain: *u8, store: *TrustStore, robots: *u8, rn: i64, body: *u8, cap: i64, stbox: *i64) -> i64 { 354 // queue of sitemap urls: from robots Sitemap: lines, else /sitemap.xml and /sitemap_index.xml 355 let q: *u8 = sys_mmap(DM_URL_SLOT * (dm_max_sitemaps_g + 2)) 356 var qn: i64 = 0 357 if rn > 0 { qn = sm_robots_sitemaps(robots, rn, q, DM_URL_SLOT, dm_max_sitemaps_g) } 358 if qn == 0 { 359 dm_url(q, domain, "/sitemap.xml" as *u8) 360 dm_url((q as i64 + DM_URL_SLOT) as *u8, domain, "/sitemap_index.xml" as *u8) 361 qn = 2 362 } 363 let locs: *u8 = sys_mmap(DM_URL_SLOT * (dm_max_locs_g + 1)) 364 var qi: i64 = 0 365 var emitted: i64 = 0 366 while qi < qn { 367 let su: *u8 = (q as i64 + qi * DM_URL_SLOT) as *u8 368 var n: i64 = dm_fetch(su, store, body, cap, stbox) 369 var xml: *u8 = body 370 var xn: i64 = n 371 var gzres: *NxGzipResult = 0 as *NxGzipResult 372 if n > 0 { if dm_is_gzip(body, n) == 1 { 373 gzres = nx_gzip_inflate(body, n, cap) 374 if gzres.error_code == 0 { xml = gzres.output_data; xn = gzres.output_size } else { xn = 0 } 375 } } 376 if xn > 0 { if stbox[0] == DM_HTTP_OK { 377 let nl: i64 = sm_extract_locs(xml, xn, locs, DM_URL_SLOT, dm_max_locs_g) 378 if nl >= dm_max_locs_g { dm_oversize_g = dm_oversize_g + 1 } 379 var li: i64 = 0 380 while li < nl { 381 let lp: *u8 = (locs as i64 + li * DM_URL_SLOT) as *u8 382 let ll: i64 = dm_len(lp) 383 if sm_is_subsitemap(lp, ll) == 1 { 384 var seen: i64 = 0 385 var z: i64 = 0 386 while z < qn { if dm_streq((q as i64 + z * DM_URL_SLOT) as *u8, lp) == 1 { seen = 1; z = qn } else { z = z + 1 } } 387 if seen == 0 { if qn < dm_max_sitemaps_g { dm_cat((q as i64 + qn * DM_URL_SLOT) as *u8, 0, lp); qn = qn + 1 } else { dm_oversize_g = dm_oversize_g + 1 } } 388 } else { 389 dm_row("sitemap" as *u8, "url" as *u8, DM_V_LISTED, lp, ll) 390 dm_rows_sitemap_g = dm_rows_sitemap_g + 1 391 emitted = emitted + 1 392 } 393 li = li + 1 394 } 395 } } 396 qi = qi + 1 397 } 398 sys_munmap(locs, DM_URL_SLOT * (dm_max_locs_g + 1)) 399 sys_munmap(q, DM_URL_SLOT * (dm_max_sitemaps_g + 2)) 400 return emitted 401} 402func dm_source_feed(domain: *u8, store: *TrustStore, body: *u8, cap: i64, stbox: *i64, feedxml: *u8) -> i64 { 403 let home: *u8 = sys_mmap(DM_URL_SLOT) 404 dm_url(home, domain, "/" as *u8) 405 let n: i64 = dm_fetch(home, store, body, cap, stbox) 406 if n <= 0 { return 0 } 407 let fh: *u8 = sys_mmap(DM_URL_SLOT) 408 if nx_feed_discover(body, n, fh, DM_URL_SLOT) != 1 { return 0 } 409 // absolute or root-relative feed href 410 let fu: *u8 = sys_mmap(DM_URL_SLOT) 411 if (fh[0] as i64) == DM_SLASH { dm_url(fu, domain, fh) } else { dm_cat(fu, 0, fh) } 412 let fn: i64 = dm_fetch(fu, store, feedxml, cap, stbox) 413 if fn <= 0 { return 0 } 414 let t: *u8 = sys_mmap(DM_URL_SLOT) 415 let l: *u8 = sys_mmap(DM_URL_SLOT) 416 let sm: *u8 = sys_mmap(DM_URL_SLOT) 417 var idx: i64 = 0 418 var emitted: i64 = 0 419 var go: i64 = 1 420 while go == 1 { 421 if nx_feed_item_at(feedxml, fn, idx, t, DM_URL_SLOT, l, DM_URL_SLOT, sm, DM_URL_SLOT) == 1 { 422 let ll: i64 = dm_len(l) 423 if ll > 0 { dm_row("feed" as *u8, "url" as *u8, DM_V_LISTED, l, ll); dm_rows_feed_g = dm_rows_feed_g + 1; emitted = emitted + 1 } 424 idx = idx + 1 425 } else { go = 0 } 426 } 427 return emitted 428} 429func dm_source_wayback(domain: *u8, store: *TrustStore, body: *u8, cap: i64, stbox: *i64) -> i64 { 430 let u: *u8 = sys_mmap(DM_URL_SLOT) 431 var o: i64 = dm_cat(u, 0, "https://web.archive.org/cdx/search/cdx?url=" as *u8) 432 o = dm_cat(u, o, domain) 433 o = dm_cat(u, o, "/*&fl=original&collapse=urlkey&filter=statuscode:200&limit=" as *u8) 434 o = ccz_cat_num(u, o, dm_wayback_limit_g) 435 let n: i64 = dm_fetch(u, store, body, cap, stbox) 436 if n <= 0 { return 0 } 437 if stbox[0] != DM_HTTP_OK { return 0 } 438 var emitted: i64 = 0 439 var s: i64 = 0 440 while s < n { 441 var e: i64 = s 442 while e < n { if (body[e] as i64) == DM_NL { e = n + e } else { e = e + 1 } } 443 if e > n { e = e - n } 444 var e2: i64 = e 445 if e2 > s { if (body[e2 - 1] as i64) == DM_CR { e2 = e2 - 1 } } 446 if e2 > s { dm_row("wayback" as *u8, "url" as *u8, DM_V_ARCHIVED, (body as i64 + s) as *u8, e2 - s); dm_rows_wayback_g = dm_rows_wayback_g + 1; emitted = emitted + 1 } 447 s = e + 1 448 } 449 return emitted 450} 451func dm_source_crt(domain: *u8, store: *TrustStore, body: *u8, cap: i64, stbox: *i64) -> i64 { 452 let u: *u8 = sys_mmap(DM_URL_SLOT) 453 var o: i64 = dm_cat(u, 0, "https://crt.sh/?q=%25." as *u8) 454 o = dm_cat(u, o, domain) 455 o = dm_cat(u, o, "&output=json" as *u8) 456 let n: i64 = dm_fetch(u, store, body, cap, stbox) 457 if n <= 0 { return 0 } 458 if stbox[0] != DM_HTTP_OK { return 0 } 459 let hosts: *u8 = sys_mmap(DM_HOST_SLOT * (dm_max_hosts_g + 1)) 460 let ov: *i64 = sys_mmap(16) as *i64 461 ov[0] = 0 462 let nh: i64 = dm_crt_hosts(body, n, domain, hosts, DM_HOST_SLOT, dm_max_hosts_g, ov) 463 dm_oversize_g = dm_oversize_g + ov[0] 464 var i: i64 = 0 465 while i < nh { 466 let hp: *u8 = (hosts as i64 + i * DM_HOST_SLOT) as *u8 467 dm_row("crt" as *u8, "host" as *u8, DM_V_CERT, hp, dm_len(hp)) 468 dm_rows_crt_g = dm_rows_crt_g + 1 469 i = i + 1 470 } 471 sys_munmap(hosts, DM_HOST_SLOT * (dm_max_hosts_g + 1)) 472 return nh 473} 474// probes: the fingerprint first, then every conf path robots allows 475func dm_probe_paths_load(out: *u8, slot: i64, max: i64) -> i64 { 476 let lb: *i64 = sys_mmap(16) as *i64 477 lb[0] = 0 478 let src: *u8 = sys_read_file(DM_PROBES, lb) 479 var count: i64 = 0 480 if (src as i64) != 0 { 481 let n: i64 = lb[0] 482 var s: i64 = 0 483 while s < n { 484 var e: i64 = s 485 while e < n { if (src[e] as i64) == DM_NL { e = n + e } else { e = e + 1 } } 486 if e > n { e = e - n } 487 var e2: i64 = e 488 if e2 > s { if (src[e2 - 1] as i64) == DM_CR { e2 = e2 - 1 } } 489 if e2 > s { if (src[s] as i64) == DM_SLASH { if count < max { if e2 - s < slot { 490 dm_catn((out as i64 + count * slot) as *u8, 0, (src as i64 + s) as *u8, e2 - s) 491 count = count + 1 492 } } } } 493 s = e + 1 494 } 495 sys_free_file(src, n) 496 return count 497 } 498 // bootstrap defaults: the well-known paths crawl4ai's DomainMapper probes, trimmed to the ones a public site serves 499 let d0: *u8 = "/docs" as *u8 500 let d1: *u8 = "/api" as *u8 501 let d2: *u8 = "/login" as *u8 502 let d3: *u8 = "/dashboard" as *u8 503 let d4: *u8 = "/openapi.json" as *u8 504 let d5: *u8 = "/sitemap.xml" as *u8 505 let d6: *u8 = "/feed" as *u8 506 let d7: *u8 = "/rss" as *u8 507 let d8: *u8 = "/atom.xml" as *u8 508 let d9: *u8 = "/blog" as *u8 509 let d10: *u8 = "/about" as *u8 510 let d11: *u8 = "/status" as *u8 511 let d12: *u8 = "/admin" as *u8 512 let d13: *u8 = "/graphql" as *u8 513 let d14: *u8 = "/.well-known/security.txt" as *u8 514 var c: i64 = 0 515 if c < max { dm_cat((out as i64 + c * slot) as *u8, 0, d0); c = c + 1 } 516 if c < max { dm_cat((out as i64 + c * slot) as *u8, 0, d1); c = c + 1 } 517 if c < max { dm_cat((out as i64 + c * slot) as *u8, 0, d2); c = c + 1 } 518 if c < max { dm_cat((out as i64 + c * slot) as *u8, 0, d3); c = c + 1 } 519 if c < max { dm_cat((out as i64 + c * slot) as *u8, 0, d4); c = c + 1 } 520 if c < max { dm_cat((out as i64 + c * slot) as *u8, 0, d5); c = c + 1 } 521 if c < max { dm_cat((out as i64 + c * slot) as *u8, 0, d6); c = c + 1 } 522 if c < max { dm_cat((out as i64 + c * slot) as *u8, 0, d7); c = c + 1 } 523 if c < max { dm_cat((out as i64 + c * slot) as *u8, 0, d8); c = c + 1 } 524 if c < max { dm_cat((out as i64 + c * slot) as *u8, 0, d9); c = c + 1 } 525 if c < max { dm_cat((out as i64 + c * slot) as *u8, 0, d10); c = c + 1 } 526 if c < max { dm_cat((out as i64 + c * slot) as *u8, 0, d11); c = c + 1 } 527 if c < max { dm_cat((out as i64 + c * slot) as *u8, 0, d12); c = c + 1 } 528 if c < max { dm_cat((out as i64 + c * slot) as *u8, 0, d13); c = c + 1 } 529 if c < max { dm_cat((out as i64 + c * slot) as *u8, 0, d14); c = c + 1 } 530 return c 531} 532func dm_source_probe(domain: *u8, store: *TrustStore, robots: *u8, rn: i64, body: *u8, cap: i64, stbox: *i64) -> i64 { 533 let u: *u8 = sys_mmap(DM_URL_SLOT) 534 let p: *u8 = sys_mmap(DM_PATH_SLOT) 535 // 1. the fingerprint: a path that cannot exist 536 let pl: i64 = dm_nonce_path(p, domain) 537 dm_url(u, domain, p) 538 let fn: i64 = dm_fetch(u, store, body, cap, stbox) 539 dm_fp_status_g = stbox[0] 540 dm_fp_len_g = fn 541 dm_fp_title_g = 0 542 if fn > 0 { dm_fp_title_g = dm_title_hash(body, fn) } 543 if stbox[0] > 0 { dm_fp_taken_g = 1 } 544 let fpline: *u8 = sys_mmap(DM_SMALL) 545 var fo: i64 = dm_cat(fpline, 0, "status=" as *u8) 546 fo = ccz_cat_num(fpline, fo, dm_fp_status_g) 547 fo = dm_cat(fpline, fo, " len=" as *u8) 548 fo = ccz_cat_num(fpline, fo, dm_fp_len_g) 549 fo = dm_cat(fpline, fo, " title_hash=" as *u8) 550 fo = ccz_cat_num(fpline, fo, dm_fp_title_g) 551 fo = dm_cat(fpline, fo, " path=" as *u8) 552 fo = dm_cat(fpline, fo, p) 553 dm_row("probe" as *u8, "path" as *u8, DM_V_FINGERPRINT, fpline, fo) 554 dm_rows_probe_g = dm_rows_probe_g + 1 555 // 2. the paths 556 let paths: *u8 = sys_mmap(DM_PATH_SLOT * (DM_PROBE_DEFAULTS * 4)) 557 let np: i64 = dm_probe_paths_load(paths, DM_PATH_SLOT, DM_PROBE_DEFAULTS * 4) 558 var i: i64 = 0 559 var emitted: i64 = 1 560 while i < np { 561 let pp: *u8 = (paths as i64 + i * DM_PATH_SLOT) as *u8 562 let ppl: i64 = dm_len(pp) 563 var allowed: i64 = 1 564 if rn > 0 { allowed = nx_robots_allowed(robots, rn, "nishibot" as *u8, DM_UALEN, pp, ppl) } 565 if allowed == 0 { 566 dm_row("probe" as *u8, "path" as *u8, DM_V_ROBOTS_REFUSED, pp, ppl) 567 dm_refused_g = dm_refused_g + 1 568 } else { 569 dm_url(u, domain, pp) 570 let n: i64 = dm_fetch(u, store, body, cap, stbox) 571 var v: *u8 = DM_V_UNREACHABLE 572 if stbox[0] == DM_STATUS_BUDGET { v = DM_V_BUDGET } 573 if stbox[0] > 0 { 574 if stbox[0] == DM_HTTP_OK { 575 var th: i64 = 0 576 if n > 0 { th = dm_title_hash(body, n) } 577 if dm_soft404_classify(dm_fp_status_g, dm_fp_len_g, dm_fp_title_g, stbox[0], n, th, dm_tol_g) == 1 { v = DM_V_SOFT404 } else { v = DM_V_LIVE } 578 } else { 579 if stbox[0] == DM_HTTP_NOTFOUND { v = DM_V_MISSING } else { if stbox[0] >= DM_HTTP_ERR_FLOOR { v = DM_V_BLOCKED } else { v = DM_V_LIVE } } 580 } 581 } 582 dm_row("probe" as *u8, "path" as *u8, v, pp, ppl) 583 } 584 dm_rows_probe_g = dm_rows_probe_g + 1 585 emitted = emitted + 1 586 i = i + 1 587 } 588 return emitted 589} 590 591// THE CONTRACT dm_scan: the composed census -- robots, sitemaps, feed, wayback, crt, probes -- in that order, every row 592// appended as it is decided. Returns the rows written; the caller prints the partition (which must sum). 593func dm_scan(domain: *u8, store: *TrustStore, body: *u8, feedxml: *u8, robots: *u8, cap: i64, stbox: *i64) -> i64 { 594 let ru: *u8 = sys_mmap(DM_URL_SLOT) 595 dm_url(ru, domain, "/robots.txt" as *u8) 596 var rn: i64 = dm_fetch(ru, store, robots, cap, stbox) 597 if stbox[0] != DM_HTTP_OK { rn = 0 } 598 if rn > 0 { 599 let paths: *u8 = sys_mmap(DM_PATH_SLOT * (dm_max_hosts_g + 1)) 600 let np: i64 = dm_robots_paths(robots, rn, paths, DM_PATH_SLOT, dm_max_hosts_g) 601 var i: i64 = 0 602 while i < np { let pp: *u8 = (paths as i64 + i * DM_PATH_SLOT) as *u8; dm_row("robots" as *u8, "path" as *u8, DM_V_DECLARED, pp, dm_len(pp)); dm_rows_robots_g = dm_rows_robots_g + 1; i = i + 1 } 603 dm_puts(" robots.txt bytes=" as *u8); dm_num(rn); dm_puts(" declared_paths=" as *u8); dm_num(np); dm_puts("\n" as *u8) 604 } else { dm_puts(" robots.txt absent or unreadable (status=" as *u8); dm_num(stbox[0]); dm_puts(") -- sitemaps guessed, probes unrestricted per RFC 9309\n" as *u8) } 605 let ns: i64 = dm_source_sitemaps(domain, store, robots, rn, body, cap, stbox) 606 dm_puts(" sitemap urls=" as *u8); dm_num(ns); dm_puts("\n" as *u8) 607 let nf: i64 = dm_source_feed(domain, store, body, cap, stbox, feedxml) 608 dm_puts(" feed urls=" as *u8); dm_num(nf); dm_puts("\n" as *u8) 609 let nw: i64 = dm_source_wayback(domain, store, body, cap, stbox) 610 dm_puts(" wayback urls=" as *u8); dm_num(nw); dm_puts(" last_status=" as *u8); dm_num(stbox[0]); dm_puts("\n" as *u8) 611 let nc: i64 = dm_source_crt(domain, store, body, cap, stbox) 612 dm_puts(" crt hosts=" as *u8); dm_num(nc); dm_puts(" last_status=" as *u8); dm_num(stbox[0]); dm_puts("\n" as *u8) 613 let npb: i64 = dm_source_probe(domain, store, robots, rn, body, cap, stbox) 614 dm_puts(" probe rows=" as *u8); dm_num(npb); dm_puts(" fingerprint status=" as *u8); dm_num(dm_fp_status_g); dm_puts(" len=" as *u8); dm_num(dm_fp_len_g); dm_puts("\n" as *u8) 615 return dm_rows_total_g 616} 617 618func main(argc: i64, argv: *i64) -> i64 { 619 if argc < 2 { dm_puts("usage: nx_domain_map <domain> [max_fetches] [out.tsv]\n" as *u8); return 2 } 620 let domain: *u8 = argv[1] as *u8 621 dm_load_conf() 622 if argc >= 3 { let mf: i64 = wc_atoi_dm(argv[2] as *u8); if mf > 0 { dm_max_fetches_g = mf } } 623 let r: i64 = nx_trust_store_load_from_certdata(DM_CERTDATA, DM_TRUST_ANCHORS, DM_TRUST_PARSE_CAP) 624 if r <= 0 { dm_puts("DOMAIN-MAP UNMEASURED: trust store failed to load\n" as *u8); return 3 } 625 let store: *TrustStore = r as *TrustStore 626 // output: knowledge/status/domain_map/<domain>.tsv, appended 627 sys_mkdir(DM_OUTDIR, DM_MODE_0755) 628 let outp: *u8 = sys_mmap(DM_URL_SLOT) 629 if argc >= 4 { dm_cat(outp, 0, argv[3] as *u8) } else { 630 var o: i64 = dm_cat(outp, 0, DM_OUTDIR) 631 o = dm_cat(outp, o, "/" as *u8) 632 o = dm_cat(outp, o, domain) 633 o = dm_cat(outp, o, ".tsv" as *u8) 634 } 635 dm_outfd_g = sys_openat_append(outp, DM_MODE_0644) 636 if dm_outfd_g <= 0 { dm_puts("DOMAIN-MAP UNMEASURED: cannot open " as *u8); dm_puts(outp); dm_puts("\n" as *u8); return 3 } 637 dm_puts("=== nx_domain_map " as *u8); dm_puts(domain); dm_puts(" -> " as *u8); dm_puts(outp) 638 dm_puts(" (max_fetches=" as *u8); dm_num(dm_max_fetches_g); dm_puts(" conf_src=" as *u8) 639 if dm_conf_src_g == 1 { dm_puts("file" as *u8) } else { dm_puts("defaults" as *u8) } 640 dm_puts(")\n" as *u8) 641 let cap: i64 = dm_fetch_cap_g 642 let body: *u8 = sys_mmap(cap) 643 let feedxml: *u8 = sys_mmap(cap) 644 let robots: *u8 = sys_mmap(cap) 645 let stbox: *i64 = sys_mmap(16) as *i64 646 // THE CONTRACT: one composed census, rows appended as decided 647 let scanned: i64 = dm_scan(domain, store, body, feedxml, robots, cap, stbox) 648 dm_puts(" scanned rows=" as *u8); dm_num(scanned); dm_puts("\n" as *u8) 649 // THE PARTITION, and it sums 650 let sum: i64 = dm_rows_robots_g + dm_rows_sitemap_g + dm_rows_feed_g + dm_rows_wayback_g + dm_rows_crt_g + dm_rows_probe_g 651 dm_puts("DOMAIN-MAP " as *u8); dm_puts(domain) 652 dm_puts(" rows=" as *u8); dm_num(dm_rows_total_g) 653 dm_puts(" robots=" as *u8); dm_num(dm_rows_robots_g) 654 dm_puts(" sitemap=" as *u8); dm_num(dm_rows_sitemap_g) 655 dm_puts(" feed=" as *u8); dm_num(dm_rows_feed_g) 656 dm_puts(" wayback=" as *u8); dm_num(dm_rows_wayback_g) 657 dm_puts(" crt=" as *u8); dm_num(dm_rows_crt_g) 658 dm_puts(" probe=" as *u8); dm_num(dm_rows_probe_g) 659 if sum == dm_rows_total_g { dm_puts(" partition=SUMS" as *u8) } else { dm_puts(" partition=LEAK" as *u8) } 660 dm_puts(" fetches=" as *u8); dm_num(dm_fetches_g); dm_puts("/" as *u8); dm_num(dm_max_fetches_g) 661 dm_puts(" fetch_ok=" as *u8); dm_num(dm_fetch_ok_g) 662 dm_puts(" robots_refused=" as *u8); dm_num(dm_refused_g) 663 dm_puts(" oversize=" as *u8); dm_num(dm_oversize_g) 664 dm_puts(" truncated=" as *u8); dm_num(dm_truncated_g) 665 dm_puts(" soft404_fingerprint=" as *u8); if dm_fp_taken_g == 1 { dm_puts("TAKEN" as *u8) } else { dm_puts("UNTAKEN" as *u8) } 666 dm_puts("\n" as *u8) 667 sys_close(dm_outfd_g) 668 if dm_fetch_ok_g == 0 { dm_puts("DOMAIN-MAP UNREACHABLE (no fetch succeeded)\n" as *u8); return 4 } 669 return 0 670} 671func wc_atoi_dm(s: *u8) -> i64 { var v: i64 = 0; var i: i64 = 0; while s[i] != (0 as u8) { let c: i64 = s[i] as i64; if c >= DM_DIGIT0 { if c <= DM_DIGIT9 { v = v * 10 + (c - DM_DIGIT0) } } i = i + 1 } return v }