code wiki / _hdl_build / nx_crawl_web.nx
`nx_crawl_web.nx` source

↩ module page · 198 lines · 11681 B
1// nx_crawl_web.nx -- REAL-WEB BFS crawler: "crawl whatever is out there", sovereign + polite. Combines what the
2// ecosystem already had in pieces: nx_crawl_https (validated TLS fetch, but single-seed) + nx_crawl_frontier
3// (link extraction, absolute-only) + adds RELATIVE-URL resolution (the v2 the frontier flagged) so the spider
4// actually WALKS a site's links, not just the seed. Pipeline per page: dequeue -> polite sleep -> sovereign
5// nx_https_fetch_follow (TLS-1.3 + Mozilla CA, the proven fetcher) -> nx_html_to_text (now OOM-safe) -> simhash
6// dedup -> record -> extract+resolve outbound links -> enqueue. Bounded (max_pages + the per-process certloop
7// leak ~19 fetches = the NEXT OOM-class fix for crawl-at-scale). Seeded on a cost-intelligence page so the crawl
8// grows the corpus. license_tier: ORIGINAL
9import "nx_syscalls.nx"
10import "nx_x509_trust_store.nx"
11import "nx_trust_store_load_from_certdata.nx"
12import "nx_https_fetch_follow.nx"
13import "nx_html_to_text.nx"
14import "nx_simhash.nx"
15const K_MAGIC_8192: i64 = 8192
16const K_MAGIC_4194304: i64 = 4194304
17const K_MAGIC_8388608: i64 = 8388608
18const K_MAGIC_1048576: i64 = 1048576
19const K_MAGIC_2048: i64 = 2048
20
21func gp(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
22func gn(v0: i64) -> i64 { var v: i64=v0; if v<0{sys_write(1,"-" as *u8,1);v=0-v} let b: *u8=sys_mmap(24); var k: i64=0; if v==0{b[0]=48 as u8;k=1} while v>0{b[k]=(48+(v%10)) as u8;v=v/10;k=k+1} let o: *u8=sys_mmap(24); var j: i64=0; while j<k{o[j]=b[k-1-j];j=j+1} sys_write(1,o,k); return 0 }
23func slen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
24func streq(a: *u8, b: *u8) -> i64 { var i: i64=0; while a[i]!=(0 as u8){ if a[i]!=b[i]{return 0} i=i+1 } if b[i]!=(0 as u8){return 0} return 1 }
25func cw_lc(c: i64) -> i64 { if c>=0x41 { if c<=0x5a { return c+0x20 } } return c }
26// html[off..] begins with literal lit?
27func cw_pre(h: *u8, hlen: i64, off: i64, lit: *u8, litlen: i64) -> i64 {
  if off+litlen > hlen { return 0 }
  var k: i64=0; while k<litlen { if h[off+k]!=lit[k] { return 0 } k=k+1 } return 1
30}
31// "href=" (case-insensitive) at off?
32func cw_href_at(h: *u8, hlen: i64, off: i64) -> i64 {
  if off+5 > hlen { return 0 }
  if cw_lc(h[off] as i64)!=0x68 { return 0 }
  if cw_lc(h[off+1] as i64)!=0x72 { return 0 }
  if cw_lc(h[off+2] as i64)!=0x65 { return 0 }
  if cw_lc(h[off+3] as i64)!=0x66 { return 0 }
  if (h[off+4] as i64)!=0x3d { return 0 }
  return 1
40}
41// enqueue a null-term URL (copied) if not already SEEN + room. seen is the dedup set (all ever enqueued).
42func eq_nl(a: *u8, alen: i64, b: *u8) -> i64 {
  var i: i64=0; while i<alen { if a[i]!=b[i] { return 0 } i=i+1 }
  if b[alen]!=(0 as u8) { return 0 }
  return 1
46}
47func cw_enqueue(url: *u8, ulen: i64, seen: **u8, nseen: *i64, queue: **u8, qt: *i64, cap: i64) -> i64 {
  // S-class: dedup + capacity check BEFORE allocating -> the common dup/full path leaks NOTHING (was the page-11 OOM).
  var s: i64=0; while s<nseen[0] { if eq_nl(url, ulen, seen[s])==1 { return 0 } s=s+1 }
  if nseen[0] >= cap { return 0 }
  let nu: *u8 = sys_mmap(ulen+1); var i: i64=0; while i<ulen { nu[i]=url[i]; i=i+1 } nu[ulen]=0 as u8
  seen[nseen[0]] = nu; queue[qt[0]] = nu; nseen[0]=nseen[0]+1; qt[0]=qt[0]+1
  return 1
54}
55// extract outbound links from HTML + enqueue (absolute http(s) kept; relative "/path" resolved against base host).
56func cw_links(h: *u8, hlen: i64, base: *u8, blen: i64, seen: **u8, nseen: *i64, queue: **u8, qt: *i64, cap: i64) -> i64 {
  var found: i64=0
  let scratch: *u8 = sys_mmap(K_MAGIC_8192)   // reused per relative link -> no per-link buffer leak
  var i: i64=0
  while i < hlen {
      var step: i64=1
      if cw_href_at(h, hlen, i)==1 {
          let q: i64 = h[i+5] as i64
          if q==0x22 { step=0 } else { if q==0x27 { step=0 } else { step=1 } }
          if step==0 {
              let cs: i64 = i+6
              var e: i64 = cs
              var run: i64=1
              while run==1 { run=0; if e<hlen { if (h[e] as i64)!=q { e=e+1; run=1 } } }
              let clen: i64 = e-cs
              if clen>0 {
                  if cw_pre(h,hlen,cs,"http://" as *u8,7)==1 {
                      if cw_enqueue(((h as i64)+cs) as *u8, clen, seen, nseen, queue, qt, cap)==1 { found=found+1 }
                  } else { if cw_pre(h,hlen,cs,"https://" as *u8,8)==1 {
                      if cw_enqueue(((h as i64)+cs) as *u8, clen, seen, nseen, queue, qt, cap)==1 { found=found+1 }
                  } else {
                      // relative "/path" (but not "//host"): resolve against base host
                      if (h[cs] as i64)==0x2f {
                          var protorel: i64=0
                          if clen>=2 { if (h[cs+1] as i64)==0x2f { protorel=1 } }
                          if protorel==0 { if blen+clen+1 < K_MAGIC_8192 {
                              var b: i64=0; while b<blen { scratch[b]=base[b]; b=b+1 }
                              var c: i64=0; while c<clen { scratch[blen+c]=h[cs+c]; c=c+1 }
                              scratch[blen+clen]=0 as u8
                              if cw_enqueue(scratch, blen+clen, seen, nseen, queue, qt, cap)==1 { found=found+1 }
                          } }
                      }
                  } }
              }
              step = (e-i)+1
          } else { step=1 }
      }
      i = i + step
  }
  return found
96}
97
98// S-CLASS arg-driven helpers (2026-06-25, operator: "not seed fixed ... s-class capable"): derive the
99// scheme://host base from ANY seed URL (so relative links resolve on any site), and parse an int arg.
100func cw_base(url: *u8, out: *u8) -> i64 {
  let n: i64 = slen(url)
  var schemeend: i64 = 0
  var i: i64 = 0
  var hit: i64 = 0
  while hit == 0 {
      if i + 2 >= n { hit = 1 } else {
          if url[i]==(58 as u8) { if url[i+1]==(47 as u8) { if url[i+2]==(47 as u8) { schemeend = i+3; hit = 1 } } }
          if hit == 0 { i = i + 1 }
      }
  }
  var e: i64 = schemeend
  var go: i64 = 1
  while go == 1 { if e >= n { go = 0 } else { if url[e]==(47 as u8) { go = 0 } else { e = e + 1 } } }
  var k: i64 = 0
  while k < e { out[k] = url[k]; k = k + 1 }
  out[e] = 0 as u8
  return e
118}
119func cw_atoi(s: *u8) -> i64 { var v: i64=0; var i: i64=0; while s[i]!=(0 as u8){ let c: i64=s[i] as i64; if c>=48 { if c<=57 { v=v*10+(c-48) } } i=i+1 } return v }
120
121func main(argc: i64, argv: *i64) -> i64 {
  var pass: i64=0; var fail: i64=0
  let r: i64 = nx_trust_store_load_from_certdata("data/mozilla_certdata.txt" as *u8, 512, K_MAGIC_4194304)
  if r <= 0 { gp("CRAWL-WEB FAIL: certdata load\n" as *u8); sys_exit(1); return 1 }
  let store: *TrustStore = r as *TrustStore
  gp("=== nx_crawl_web: REAL-WEB BFS crawl (sovereign TLS, polite, link-following) ===\n" as *u8)
  gp("  CA roots=" as *u8); gn(trust_store_count(store)); gp("\n" as *u8)
128
  let cap: i64 = K_MAGIC_8388608
  let out: *u8 = sys_mmap(cap)
  let tcap: i64 = K_MAGIC_1048576
  let text: *u8 = sys_mmap(tcap)
  let status: *i64 = sys_mmap(8) as *i64
  let QCAP: i64 = 256
  let queue: **u8 = sys_mmap(QCAP*8) as **u8
  let seen:  **u8 = sys_mmap(QCAP*8) as **u8
  let qt: *i64 = sys_mmap(8) as *i64; qt[0]=0
  let nseen: *i64 = sys_mmap(8) as *i64; nseen[0]=0
  var qh: i64 = 0
  let keptfp: *i64 = sys_mmap(64*8) as *i64
  // S-CLASS 2026-06-25: seed URL + max_pages are ARG-DRIVEN; base (scheme://host, for relative-link
  // resolution) is DERIVED from the seed -> crawl ANY site, not just wikipedia. No-arg keeps the default
  // Lithium seed + the standing self-gate (so the crawler's CI gate still proves out).
  var seed: *u8 = "https://en.wikipedia.org/wiki/Lithium" as *u8
  var argmode: i64 = 0
  var max_pages: i64 = 12
  if argc >= 2 { seed = argv[1] as *u8; argmode = 1 }
  if argc >= 3 { max_pages = cw_atoi(argv[2] as *u8) }
  let base: *u8 = sys_mmap(K_MAGIC_2048)
  let blen: i64 = cw_base(seed, base)
  gp("  seed=" as *u8); gp(seed); gp("  base=" as *u8); gp(base); gp("  max_pages=" as *u8); gn(max_pages); gp("\n" as *u8)
  cw_enqueue(seed, slen(seed), seen, nseen, queue, qt, QCAP)
  var crawled: i64=0; var fails: i64=0; var dups: i64=0; var discovered: i64=0
  var running: i64=1
  while running==1 {
      if qh >= qt[0] { running=0 }
      if crawled >= max_pages { running=0 }
      if running==1 {
          let url: *u8 = queue[qh]; qh=qh+1
          sys_sleep_ms(350)   // polite, ISP-safe
          let n: i64 = nx_https_fetch_follow(url, store, out, cap, 6, status)
          if n <= 0 { fails=fails+1; gp("  fetch-fail verdict=" as *u8); gn(n); gp(" " as *u8); gp(url); gp("\n" as *u8) }
          else { if status[0] != 200 { fails=fails+1; gp("  status=" as *u8); gn(status[0]); gp(" " as *u8); gp(url); gp("\n" as *u8) }
          else {
              let tlen: i64 = nx_html_to_text(out, n, text, tcap)
              let fp: i64 = nx_simhash_fingerprint(text, tlen)
              var dup: i64=0; var k: i64=0
              while k<crawled { if nx_simhash_hamming(fp, keptfp[k]) <= 4 { dup=1 } k=k+1 }
              if dup==1 { dups=dups+1; gp("  dup " as *u8); gp(url); gp("\n" as *u8) }
              else {
                  keptfp[crawled]=fp; crawled=crawled+1
                  gp("  crawled #" as *u8); gn(crawled); gp(" " as *u8); gp(url); gp("  (" as *u8); gn(n); gp(" bytes -> " as *u8); gn(tlen); gp(" chars)\n" as *u8)
                  let nl: i64 = cw_links(out, n, base, blen, seen, nseen, queue, qt, QCAP)
                  discovered = discovered + nl
              }
          } }
      }
  }
  gp("CRAWL-WEB done: crawled=" as *u8); gn(crawled); gp(" pages, discovered=" as *u8); gn(discovered); gp(" new links, dups=" as *u8); gn(dups); gp(" fails=" as *u8); gn(fails); gp(" (frontier qsize=" as *u8); gn(qt[0]); gp(")\n" as *u8)
180
  // ===== verdict ===== ARG mode (operator seed): success iff the seed actually fetched -- a single arbitrary
  // URL (or a PDF with no outbound HTML links) must NOT false-RED the crawler. NO-ARG mode: the standing
  // self-gate below proves a real multi-page link-following crawl (the CI guarantee, unchanged).
  if argmode == 1 {
      if crawled >= 1 { gp("CRAWL-WEB verdict=GREEN (arg-seed crawl: " as *u8); gn(crawled); gp(" page(s) fetched + " as *u8); gn(discovered); gp(" links discovered from the operator seed)\n" as *u8); sys_exit(0); return 0 }
      gp("CRAWL-WEB verdict=RED (arg-seed fetched nothing -- TLS / redirect / host / paywall?)\n" as *u8); sys_exit(1); return 1
  }
  // ===== self-gate (default seed): PROVE a real multi-page link-following crawl happened =====
  if crawled >= 5 { pass=pass+1 } else { fail=fail+1; gp("  FAIL crawled<5 (not a multi-page crawl)\n" as *u8) }
  if discovered >= 10 { pass=pass+1 } else { fail=fail+1; gp("  FAIL discovered<10 (link-following broken)\n" as *u8) }
  if qt[0] > crawled { pass=pass+1 } else { fail=fail+1; gp("  FAIL frontier didn't grow (no real link discovery)\n" as *u8) }
  // the seed itself must have fetched (crawled>=1 means TLS worked on the real web)
  if crawled >= 1 { pass=pass+1 } else { fail=fail+1; gp("  FAIL nothing fetched (TLS/network)\n" as *u8) }
194
  gp("CRAWL-WEB pass=" as *u8); gn(pass); gp(" fail=" as *u8); gn(fail)
  if fail==0 { gp(" verdict=GREEN (our own spider crawled the REAL web over sovereign TLS, followed real links, deduped)\n" as *u8); sys_exit(0); return 0 }
  gp(" verdict=RED\n" as *u8); sys_exit(1); return 1
198}
nx_crawl_web.nx source

`nx_crawl_web.nx` source
