code wiki / _hdl_build / nx_web_ingest.nx
`nx_web_ingest.nx` source

↩ module page · 235 lines · 11769 B
1// nx_web_ingest.nx -- OPEN-WEB front of nishi search: BFS-crawl the REAL web (sovereign TLS-1.3 + Mozilla CA,
2// polite, simhash near-dup killed) and ingest each kept page STRAIGHT into the shared "web" seg_store shard
3// (dp-web-pub-): doc:<cid> = the page's extracted text, url:<cid> = its ABSOLUTE url -> the /search?scope=web
4// SERP links results to the real pages. Composes the PROVEN pieces only: nx_crawl_web's fetch/link spine +
5// nx_corpus_ingest's writer/key/hash mechanics (ci_hash / ci_mkurlkey / fresh-segid batching, gate 7/7).
6//   usage: nx_web_ingest <seed-url> <max_pages>
7// Bounded by max_pages AND the known per-process certloop budget (~19 fetches) -- crawl-at-scale = many small
8// runs, each committing its own segments (append-only, idempotent by content cid). license_tier: ORIGINAL
9import "nx_corpus_ingest.nx"   // ci_hash / ci_mkurlkey / dss_prefix / dss_mkkey / seg_store (transitively)
10import "nx_x509_trust_store.nx"
11import "nx_trust_store_load_from_certdata.nx"
12import "nx_https_fetch_follow.nx"
13import "nx_simhash.nx"
14const K_MAGIC_8192: i64 = 8192
15const K_MAGIC_4194304: i64 = 4194304
16const K_MAGIC_8388608: i64 = 8388608
17const K_MAGIC_1048576: i64 = 1048576
18const K_MAGIC_2048: i64 = 2048
19
20func wi_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
21func wi_num(v: i64) -> i64 {
  let bb: *u8 = sys_mmap(28); var m: i64 = v
  if m < 0 { sys_write(1, "-" as *u8, 1); m = 0 - m }
  let t: *u8 = sys_mmap(28); var k: i64 = 0
  if m == 0 { t[0] = 48 as u8; k = 1 }
  while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
  var i: i64 = 0; while i < k { bb[i] = t[k - 1 - i]; i = i + 1 }
  sys_write(1, bb, k); return 0
29}
30func wi_len(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
31func wi_atoi(s: *u8) -> i64 { var v: i64 = 0; var i: i64 = 0; while s[i] != (0 as u8) { let c: i64 = s[i] as i64; if c >= 48 { if c <= 57 { v = v * 10 + (c - 48) } } i = i + 1 } return v }
32
33// scheme://host base from the seed (relative-link resolution) -- mirrors nx_crawl_web.cw_base
34func wi_base(url: *u8, out: *u8) -> i64 {
  let n: i64 = wi_len(url)
  var schemeend: i64 = 0
  var i: i64 = 0
  var hit: i64 = 0
  while hit == 0 {
      if i + 2 >= n { hit = 1 } else {
          if url[i] == (58 as u8) { if url[i+1] == (47 as u8) { if url[i+2] == (47 as u8) { schemeend = i + 3; hit = 1 } } }
          if hit == 0 { i = i + 1 }
      }
  }
  var e: i64 = schemeend
  var go: i64 = 1
  while go == 1 { if e >= n { go = 0 } else { if url[e] == (47 as u8) { go = 0 } else { e = e + 1 } } }
  var k: i64 = 0
  while k < e { out[k] = url[k]; k = k + 1 }
  out[e] = 0 as u8
  return e
52}
53func wi_eqn(a: *u8, alen: i64, b: *u8) -> i64 {
  var i: i64 = 0; while i < alen { if a[i] != b[i] { return 0 } i = i + 1 }
  if b[alen] != (0 as u8) { return 0 }
  return 1
57}
58func wi_enqueue(url: *u8, ulen: i64, seen: **u8, nseen: *i64, queue: **u8, qt: *i64, cap: i64) -> i64 {
  var s: i64 = 0; while s < nseen[0] { if wi_eqn(url, ulen, seen[s]) == 1 { return 0 } s = s + 1 }
  if nseen[0] >= cap { return 0 }
  let nu: *u8 = sys_mmap(ulen + 1); var i: i64 = 0; while i < ulen { nu[i] = url[i]; i = i + 1 } nu[ulen] = 0 as u8
  seen[nseen[0]] = nu; queue[qt[0]] = nu; nseen[0] = nseen[0] + 1; qt[0] = qt[0] + 1
  return 1
64}
65func wi_pre(h: *u8, hlen: i64, off: i64, lit: *u8, litlen: i64) -> i64 {
  if off + litlen > hlen { return 0 }
  var k: i64 = 0; while k < litlen { if h[off+k] != lit[k] { return 0 } k = k + 1 } return 1
68}
69func wi_lc(c: i64) -> i64 { if c >= 0x41 { if c <= 0x5a { return c + 0x20 } } return c }
70func wi_href_at(h: *u8, hlen: i64, off: i64) -> i64 {
  if off + 5 > hlen { return 0 }
  if wi_lc(h[off] as i64) != 0x68 { return 0 }
  if wi_lc(h[off+1] as i64) != 0x72 { return 0 }
  if wi_lc(h[off+2] as i64) != 0x65 { return 0 }
  if wi_lc(h[off+3] as i64) != 0x66 { return 0 }
  if (h[off+4] as i64) != 0x3d { return 0 }
  return 1
78}
79// extract + resolve + enqueue outbound links (absolute http(s) + root-relative)
80func wi_links(h: *u8, hlen: i64, base: *u8, blen: i64, seen: **u8, nseen: *i64, queue: **u8, qt: *i64, cap: i64) -> i64 {
  var found: i64 = 0
  let scratch: *u8 = sys_mmap(K_MAGIC_8192)
  var i: i64 = 0
  while i < hlen {
      var step: i64 = 1
      if wi_href_at(h, hlen, i) == 1 {
          let q: i64 = h[i+5] as i64
          var quoted: i64 = 0
          if q == 0x22 { quoted = 1 }
          if q == 0x27 { quoted = 1 }
          if quoted == 1 {
              let cs: i64 = i + 6
              var e: i64 = cs
              var run: i64 = 1
              while run == 1 { run = 0; if e < hlen { if (h[e] as i64) != q { e = e + 1; run = 1 } } }
              let clen: i64 = e - cs
              if clen > 0 {
                  if wi_pre(h, hlen, cs, "https://" as *u8, 8) == 1 {
                      if wi_enqueue(((h as i64)+cs) as *u8, clen, seen, nseen, queue, qt, cap) == 1 { found = found + 1 }
                  } else { if wi_pre(h, hlen, cs, "http://" as *u8, 7) == 1 {
                      if wi_enqueue(((h as i64)+cs) as *u8, clen, seen, nseen, queue, qt, cap) == 1 { found = found + 1 }
                  } else {
                      if (h[cs] as i64) == 0x2f {
                          var protorel: i64 = 0
                          if clen >= 2 { if (h[cs+1] as i64) == 0x2f { protorel = 1 } }
                          if protorel == 0 { if blen + clen + 1 < K_MAGIC_8192 {
                              var b: i64 = 0; while b < blen { scratch[b] = base[b]; b = b + 1 }
                              var c: i64 = 0; while c < clen { scratch[blen+c] = h[cs+c]; c = c + 1 }
                              scratch[blen+clen] = 0 as u8
                              if wi_enqueue(scratch, blen + clen, seen, nseen, queue, qt, cap) == 1 { found = found + 1 }
                          } }
                      }
                  } }
              }
              step = (e - i) + 1
          }
      }
      i = i + step
  }
  return found
121}
122
123func main(argc: i64, argv: *i64) -> i64 {
  if argc < 3 { wi_puts("usage: nx_web_ingest <seed-url> <max_pages>\n" as *u8); return 1 }
  let seed: *u8 = argv[1] as *u8
  let max_pages: i64 = wi_atoi(argv[2] as *u8)
  let r: i64 = nx_trust_store_load_from_certdata("data/mozilla_certdata.txt" as *u8, 512, K_MAGIC_4194304)
  if r <= 0 { wi_puts("certdata load failed\n" as *u8); return 2 }
  let store: *TrustStore = r as *TrustStore
  wi_puts("=== nx_web_ingest: crawl -> dp-web-pub- (sovereign open-web search corpus) ===\n" as *u8)
131
  let prefix: *u8 = sys_mmap(512)
  dss_prefix("web" as *u8, prefix)
  // fresh segid past the manifest max (append-only, never clobber)
  let segs: *i64 = sys_mmap(256 * 8) as *i64
  let nseg: i64 = ss_manifest_file(prefix, "manifest.txt" as *u8, segs)
  var segid: i64 = 1
  var si: i64 = 0
  // ROOT FIX seq1730 (id 1785450987): segs[] holds POINTERS to seg-<id> name strings
  // (nx_seg_store.nx:1234 stores segs[cnt] = name as i64), NOT ids -- so segs[si] + 1 produced
  // MMAP_ADDRESS+1 (~1.4e14), the pointer-shaped poison that PINS a plane forever: every later
  // epoch id sorts BELOW it in supersede order and its rows are silently shadowed while rc=0.
  // Parse the DIGITS, as nx_web_shard_compact.nx:55-58 already does on this SAME array.
  while si < nseg {
      let sg_nm: *u8 = segs[si] as *u8
      var sg_v: i64 = 0
      var sg_ci: i64 = 0
      while sg_nm[sg_ci] != (0 as u8) { let sg_c: i64 = sg_nm[sg_ci] as i64; if sg_c >= 48 { if sg_c <= 57 { sg_v = sg_v * 10 + (sg_c - 48) } } sg_ci = sg_ci + 1 }
      if sg_v >= segid { segid = sg_v + 1 }
      si = si + 1
  }
  let h: *i64 = ss_open(prefix)
  let pbox: *i64 = sys_mmap(16) as *i64
  let lbox: *i64 = sys_mmap(16) as *i64
155
  let cap: i64 = K_MAGIC_8388608
  let out: *u8 = sys_mmap(cap)
  let tcap: i64 = K_MAGIC_1048576
  let text: *u8 = sys_mmap(tcap)
  let status: *i64 = sys_mmap(8) as *i64
  let QCAP: i64 = 512
  let queue: **u8 = sys_mmap(QCAP * 8) as **u8
  let seen: **u8 = sys_mmap(QCAP * 8) as **u8
  let qt: *i64 = sys_mmap(8) as *i64; qt[0] = 0
  let nseen: *i64 = sys_mmap(8) as *i64; nseen[0] = 0
  var qh: i64 = 0
  let keptfp: *i64 = sys_mmap(128 * 8) as *i64
  let base: *u8 = sys_mmap(K_MAGIC_2048)
  let blen: i64 = wi_base(seed, base)
  wi_enqueue(seed, wi_len(seed), seen, nseen, queue, qt, QCAP)
171
  let key: *u8 = sys_mmap(64)
  let ukey: *u8 = sys_mmap(64)
  var w: *i64 = ss_begin()
  var crawled: i64 = 0
  var ingested: i64 = 0
  var present: i64 = 0
  var segments: i64 = 0
  var running: i64 = 1
  while running == 1 {
      if qh >= qt[0] { running = 0 }
      if crawled >= max_pages { running = 0 }
      if running == 1 {
          let url: *u8 = queue[qh]; qh = qh + 1
          sys_sleep_ms(350)
          let n: i64 = nx_https_fetch_follow(url, store, out, cap, 6, status)
          if n > 0 { if status[0] == 200 {
              let tlen: i64 = nx_html_to_text(out, n, text, tcap)
              if tlen >= CI_MINDOC {
                  let fp: i64 = nx_simhash_fingerprint(text, tlen)
                  var dup: i64 = 0
                  var k: i64 = 0
                  while k < crawled { if nx_simhash_hamming(fp, keptfp[k]) <= 4 { dup = 1 } k = k + 1 }
                  if dup == 0 {
                      if crawled < 128 { keptfp[crawled] = fp }
                      crawled = crawled + 1
                      var tn: i64 = tlen
                      if tn > CI_DOCCAP { tn = CI_DOCCAP }
                      let cid: i64 = ci_hash(text, tn)
                      dss_mkkey(cid, key)
                      var already: i64 = 0
                      if (h as i64) != 0 { if ss_hget(h, key, pbox, lbox) == 1 { already = 1 } }
                      if already == 1 { present = present + 1 } else {
                          if ss_add(w, 1, key, text, tn) < 0 {
                              if ss_commit(prefix, w, segid) == 0 { segments = segments + 1 }
                              segid = segid + 1
                              w = ss_begin()
                              ss_add(w, 1, key, text, tn)
                          }
                          ci_mkurlkey(cid, ukey)
                          if ss_add(w, 1, ukey, url, wi_len(url)) < 0 {
                              if ss_commit(prefix, w, segid) == 0 { segments = segments + 1 }
                              segid = segid + 1
                              w = ss_begin()
                              ss_add(w, 1, ukey, url, wi_len(url))
                          }
                          ingested = ingested + 1
                          wi_puts("  ingested #" as *u8); wi_num(ingested); wi_puts(" " as *u8); wi_puts(url); wi_puts(" (" as *u8); wi_num(tn); wi_puts(" chars)\n" as *u8)
                      }
                      wi_links(out, n, base, blen, seen, nseen, queue, qt, QCAP)
                  }
              }
          } }
      }
  }
  if w[1] > 0 { if ss_commit(prefix, w, segid) == 0 { segments = segments + 1 } }
  wi_puts("WEB-INGEST done: crawled=" as *u8); wi_num(crawled)
  wi_puts(" ingested=" as *u8); wi_num(ingested)
  wi_puts(" already_present=" as *u8); wi_num(present)
  wi_puts(" segments=" as *u8); wi_num(segments)
  wi_puts(" -> dp-web-pub-\n" as *u8)
  if ingested + present >= 1 { wi_puts("WEB-INGEST GREEN\n" as *u8); return 0 }
  wi_puts("WEB-INGEST RED (nothing ingested)\n" as *u8)
  return 3
235}
nx_web_ingest.nx source

`nx_web_ingest.nx` source
