code wiki / _hdl_build / nx_url_index.nx
`nx_url_index.nx` source

↩ module page · 231 lines · 14190 B
1// nx_url_index.nx -- Fetches and indexes a single live URL into the web scope using a neutral, sovereign stack.
2import "nx_gate_gn.nx"
3import "nx_gate_base.nx"
4// nx_url_index.nx -- DIRECT neutral-access indexer: fetch ONE live url over our sovereign stack (best-effort:
5// minimal hello then Chrome-JA3) and index it INTO the web scope (dp-web-pub-, INDEX-ONLY + serve-LIVE).
6// This is the path for content Common Crawl MISSES because the site blocks CC's crawler (adult-media sources
7// major engines de-rank: supjav/nhentai/etc.). We are NEUTRAL: only US-illegal content (CSAM floor) is out of
8// scope -- lawful adult media is indexed like any other page. Query '?' urls allowed (unlike the BFS crawler).
9//   usage: nx_url_index <url> [display-url]
10// license_tier: ORIGINAL
11import "nx_corpus_ingest.nx"          // ci_hash / ci_mkurlkey / dss_prefix / dss_mkkey / seg_store / nx_html_to_text
12import "nx_struct_extract.nx"         // nx_struct_extract -- mine og/twitter/JSON-LD title+desc for enrichment
13import "nx_feed_extract.nx"           // nx_feed_discover + nx_feed_extract -- non-JS content path for SPA shells
14import "nx_crawl_pace.nx"             // per-host polite pacing (pace_before/after) -- don't get IP-banned
15import "nx_x509_trust_store.nx"
16import "nx_trust_store_load_from_certdata.nx"
17import "nx_https_fetch_follow.nx"
18const K_MAGIC_4194304: i64 = 4194304
19const K_MAGIC_1048576: i64 = 1048576
20const K_MAGIC_1024: i64 = 1024
21const K_MAGIC_2048: i64 = 2048
22const K_MAGIC_4096: i64 = 4096
23const K_MAGIC_8192: i64 = 8192
24const K_MAGIC_8191: i64 = 8191
25const K_MAGIC_65536: i64 = 65536
26const K_MAGIC_70000: i64 = 70000
27const K_MAGIC_69998: i64 = 69998
28
29func grow(name: *u8, ok: i64) -> i64 { if ok==1 { gw("  PASS " as *u8) } else { gw("  FAIL " as *u8) } gw(name); gw("
30" as *u8); return ok }
31func slen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
32func has(hay: *u8, hn: i64, ndl: *u8) -> i64 { var nl: i64=0; while ndl[nl]!=(0 as u8){nl=nl+1} var i: i64=0; while i+nl<=hn { var m: i64=1; var j: i64=0; while j<nl { if hay[i+j]!=ndl[j] { m=0; j=nl } else { j=j+1 } } if m==1 { return 1 } i=i+1 } return 0 }
33
34// extract the host (between "://" and the next '/' or ':') from a url -> out. returns host length.
35// (flag-based loops -- do NOT jump the index past the string, that re-reads OOB in the while condition.)
36func ui_host(url: *u8, out: *u8) -> i64 {
  var sp: i64 = 0 - 1; var i: i64 = 0; var d: i64 = 0
  while d == 0 {
      if url[i]==(0 as u8) { d = 1 } else {
          if url[i]==(58 as u8) { if url[i+1]==(47 as u8) { if url[i+2]==(47 as u8) { sp = i; d = 1 } } }
          if d == 0 { i = i + 1 }
      }
  }
  var s: i64 = 0; if sp >= 0 { s = sp + 3 }
  var o: i64 = 0; var d2: i64 = 0
  while d2 == 0 {
      let c: i64 = url[s] as i64
      if c==0 { d2 = 1 } else { if c==47 { d2 = 1 } else { if c==58 { d2 = 1 } else { out[o]=url[s]; o=o+1; s=s+1 } } }
  }
  out[o] = 0 as u8
  return o
52}
53
54// resolve a feed href against the page url: absolute (http...) kept; "/path" -> scheme://host/path.
55func ui_resolve(base: *u8, href: *u8, out: *u8) -> i64 {
  if href[0]==(104 as u8) { if href[1]==(116 as u8) { if href[2]==(116 as u8) { if href[3]==(112 as u8) {
      var i: i64=0; while href[i]!=(0 as u8) { out[i]=href[i]; i=i+1 } out[i]=0 as u8; return 0
  } } } }
  // find "://" in base (flag-based; never jump the index past the string)
  var sp: i64 = 0 - 1; var i: i64 = 0; var df: i64 = 0
  while df == 0 {
      if base[i]==(0 as u8) { df = 1 } else {
          if base[i]==(58 as u8) { if base[i+1]==(47 as u8) { if base[i+2]==(47 as u8) { sp = i; df = 1 } } }
          if df == 0 { i = i + 1 }
      }
  }
  if sp < 0 { var k: i64=0; while href[k]!=(0 as u8) { out[k]=href[k]; k=k+1 } out[k]=0 as u8; return 0 }
  // he = first '/' at/after sp+3, else end of string
  var he: i64 = sp + 3
  var done: i64 = 0
  while done==0 { if base[he]==(0 as u8) { done=1 } else { if base[he]==(47 as u8) { done=1 } else { he=he+1 } } }
  var o: i64 = 0
  while o < he { out[o]=base[o]; o=o+1 }
  if href[0]==(47 as u8) { var j: i64=0; while href[j]!=(0 as u8) { out[o]=href[j]; o=o+1; j=j+1 } }
  else { out[o]=47 as u8; o=o+1; var j: i64=0; while href[j]!=(0 as u8) { out[o]=href[j]; o=o+1; j=j+1 } }
  out[o]=0 as u8
  return 0
78}
79
80func main(argc: i64, argv: *i64) -> i64 {
  if argc < 2 { gw("usage: nx_url_index <url> [display-url]\n" as *u8); return 1 }
  let url: *u8 = argv[1] as *u8
  var disp: *u8 = url
  if argc >= 3 { disp = argv[2] as *u8 }
  let r: i64 = nx_trust_store_load_from_certdata("data/mozilla_certdata.txt" as *u8, 512, K_MAGIC_4194304)
  if r <= 0 { gw("trust store load failed\n" as *u8); return 2 }
  let store: *TrustStore = r as *TrustStore
  gw("=== nx_url_index: " as *u8); gw(url); gw(" -> dp-web-pub- (INDEX-ONLY, serve-LIVE) ===\n" as *u8)
89
  let cap: i64 = K_MAGIC_4194304
  let out: *u8 = sys_mmap(cap)
  let st: *i64 = sys_mmap(8) as *i64
  let hbuf: *u8 = sys_mmap(512); let hl: i64 = ui_host(url, hbuf)
  let waited: i64 = pace_before(hbuf, hl)       // polite per-host interval (0 on first-ever hit)
  if waited > 0 { gw("  [pace] waited " as *u8); gn(waited); gw("ms for " as *u8); gw(hbuf); gw("\n" as *u8) }
  let n: i64 = nx_https_fetch_follow_best(url, store, out, cap, 8, st)
  pace_after(hbuf, hl, st[0], 0)                // status-based backoff (429/503 -> exponential; 200 -> reset)
  gw("  fetch status=" as *u8); gn(st[0]); gw(" bytes=" as *u8); gn(n); gw("\n" as *u8)
  if st[0] != 200 {
      var cf: i64 = 0
      if n > 0 { if has(out, n, "Just a moment" as *u8) == 1 { cf = 1 } if has(out, n, "Cloudflare" as *u8) == 1 { cf = 1 } }
      if cf == 1 { gw("  BLOCKED by Cloudflare bot-challenge (needs the browser JS-solver) -- NOT a neutrality issue\n" as *u8) }
      return 3
  }
  // QUALITY GUARD: a 200 can still be a bot-interstitial (reddit "Please wait for verification", Cloudflare
  // "Just a moment" / "Attention Required"). Don't index the wall as if it were content.
  if has(out, n, "Please wait for verification" as *u8) == 1 { gw("  SKIP: bot-verification interstitial (not real content)\n" as *u8); return 6 }
  if has(out, n, "Just a moment" as *u8) == 1 { gw("  SKIP: Cloudflare JS-challenge interstitial\n" as *u8); return 6 }
  if has(out, n, "Attention Required! | Cloudflare" as *u8) == 1 { gw("  SKIP: Cloudflare block interstitial\n" as *u8); return 6 }
  let body: *u8 = sys_mmap(K_MAGIC_1048576)
  let blen: i64 = nx_html_to_text(out, n, body, K_MAGIC_1048576)
  // STRUCTURED ENRICHMENT (grow-capabilities 2026-07-05): mine og/twitter/JSON-LD title+desc and PREPEND
  // them -- they become searchable + the snippet, and a structured-rich SPA shell with a thin body is now
  // indexable (title+desc present) instead of dropped as "too small". nx_struct_extract fabricates nothing.
  let stitle: *u8 = sys_mmap(K_MAGIC_1024); let sdesc: *u8 = sys_mmap(K_MAGIC_2048)
  let sty: *u8 = sys_mmap(256); let ssi: *u8 = sys_mmap(256)
  let sgot: i64 = nx_struct_extract(out, n, stitle, K_MAGIC_1024, sdesc, K_MAGIC_2048, sty, 256, ssi, 256)
  // FEED FALLBACK (MEASURED: reddit's SPA shell is empty but its /.rss is a real Atom feed): discover the
  // page's RSS/Atom <link>, fetch it, extract clean item text -> turns a JS-only shell into real indexable
  // content WITHOUT a JS engine. Also enriches rich pages with fresh feed items.
  let feedtext: *u8 = sys_mmap(K_MAGIC_1048576)
  let feedxml: *u8 = sys_mmap(cap)      // kept at fn scope so per-item docs can be emitted at commit time
  var feedxml_len: i64 = 0
  var feed_items: i64 = 0
  var feedlen: i64 = 0
  let feedurl: *u8 = sys_mmap(K_MAGIC_4096)
  if nx_feed_discover(out, n, feedurl, K_MAGIC_4096) == 1 {
      let feedabs: *u8 = sys_mmap(K_MAGIC_4096)
      ui_resolve(url, feedurl, feedabs)
      gw("  feed discovered: " as *u8); gw(feedabs); gw("\n" as *u8)
      let fst: *i64 = sys_mmap(8) as *i64
      let fhbuf: *u8 = sys_mmap(512); let fhl: i64 = ui_host(feedabs, fhbuf)
      pace_before(fhbuf, fhl)                   // pace the feed host too (may differ, e.g. rss.nytimes.com)
      let fn: i64 = nx_https_fetch_follow_best(feedabs, store, feedxml, cap, 8, fst)
      pace_after(fhbuf, fhl, fst[0], 0)
      if fst[0] != 200 { gw("  feed fetch status=" as *u8); gn(fst[0]); gw(" (throttled? paced for next time)\n" as *u8) }
      if fst[0] == 200 { if fn > 0 { feedxml_len = fn; feed_items = nx_feed_extract(feedxml, fn, feedtext, K_MAGIC_1048576); feedlen = slen(feedtext); gw("  feed items: " as *u8); gn(feed_items); gw("\n" as *u8) } }
  }
  if blen < CI_MINDOC { if sgot < 2 { if feed_items == 0 { gw("  extract too small (thin body + no structured data + no feed)\n" as *u8); return 4 } } }
  if stitle[0] != (0 as u8) { gw("  struct-title: " as *u8); gw(stitle); gw("\n" as *u8) }
  let text: *u8 = sys_mmap(2*K_MAGIC_1048576 + K_MAGIC_8192)
  var el: i64 = 0
  var ci2: i64 = 0
  while stitle[ci2] != (0 as u8) { text[el] = stitle[ci2]; el = el + 1; ci2 = ci2 + 1 }
  text[el] = 10 as u8; el = el + 1
  ci2 = 0
  while sdesc[ci2] != (0 as u8) { text[el] = sdesc[ci2]; el = el + 1; ci2 = ci2 + 1 }
  text[el] = 10 as u8; el = el + 1
  var bi: i64 = 0
  while bi < blen { if el < 2*K_MAGIC_1048576 + K_MAGIC_8191 { text[el] = body[bi]; el = el + 1 } bi = bi + 1 }
  if feedlen > 0 {
      if el < 2*K_MAGIC_1048576 + K_MAGIC_8191 { text[el] = 10 as u8; el = el + 1 }
      var fbi: i64 = 0
      while fbi < feedlen { if el < 2*K_MAGIC_1048576 + K_MAGIC_8191 { text[el] = feedtext[fbi]; el = el + 1 } fbi = fbi + 1 }
  }
  let tlen: i64 = el
  var tn: i64 = tlen; if tn > CI_DOCCAP { tn = CI_DOCCAP }
  let cid: i64 = ci_hash(text, tn)
159
  let prefix: *u8 = sys_mmap(512); dss_prefix("web" as *u8, prefix)
  let segs: *i64 = sys_mmap(K_MAGIC_4096*8) as *i64
  let nseg: i64 = ss_manifest_cap(prefix, segs, K_MAGIC_4096)
  var segid: i64 = 1; var si: i64 = 0
  // ROOT FIX seq1730 (id 1785450987): segs[] holds POINTERS to seg-<id> name strings
  // (nx_seg_store.nx:1234 stores segs[cnt] = name as i64), NOT ids -- so segs[si] + 1 produced
  // MMAP_ADDRESS+1 (~1.4e14), the pointer-shaped poison that PINS a plane forever: every later
  // epoch id sorts BELOW it in supersede order and its rows are silently shadowed while rc=0.
  // Parse the DIGITS, as nx_web_shard_compact.nx:55-58 already does on this SAME array.
  while si < nseg {
      let sg_nm: *u8 = segs[si] as *u8
      var sg_v: i64 = 0
      var sg_ci: i64 = 0
      while sg_nm[sg_ci] != (0 as u8) { let sg_c: i64 = sg_nm[sg_ci] as i64; if sg_c >= 48 { if sg_c <= 57 { sg_v = sg_v * 10 + (sg_c - 48) } } sg_ci = sg_ci + 1 }
      if sg_v >= segid { segid = sg_v + 1 }
      si = si + 1
  }
  let h: *i64 = ss_open(prefix)
  let key: *u8 = sys_mmap(64); let ukey: *u8 = sys_mmap(64)
  let pbox: *i64 = sys_mmap(16) as *i64; let lbox: *i64 = sys_mmap(16) as *i64
  dss_mkkey(cid, key)
  var already: i64 = 0
  if (h as i64) != 0 { if ss_hget(h, key, pbox, lbox) == 1 { already = 1 } }
  let w: *i64 = ss_begin()
  if already == 0 {
      ss_add(w, 1, key, text, tn)
      ci_mkurlkey(cid, ukey)
      ss_add(w, 1, ukey, disp, slen(disp))
  }
  // PER-ITEM DOCS: index each feed item as its OWN document with its OWN article URL. This makes a specific
  // article retrievable (not just the homepage) AND seeds the frontier with fresh URLs. Idempotent: skip an
  // item whose content-cid already exists.
  var item_docs: i64 = 0
  if feed_items > 0 { if feedxml_len > 0 {
      let ititle: *u8 = sys_mmap(K_MAGIC_4096); let ilink: *u8 = sys_mmap(K_MAGIC_4096); let isum: *u8 = sys_mmap(K_MAGIC_65536)
      let itext: *u8 = sys_mmap(K_MAGIC_70000)
      let ikey: *u8 = sys_mmap(64); let iukey: *u8 = sys_mmap(64)
      let ipb: *i64 = sys_mmap(16) as *i64; let ilb: *i64 = sys_mmap(16) as *i64
      var idx: i64 = 0
      while idx < feed_items { if idx < 1000 {
          if nx_feed_item_at(feedxml, feedxml_len, idx, ititle, K_MAGIC_4096, ilink, K_MAGIC_4096, isum, K_MAGIC_65536) == 1 {
              if slen(ilink) >= 8 {                       // require a real per-article URL
                  var il: i64 = 0; var c: i64 = 0
                  while ititle[c] != (0 as u8) { if il < K_MAGIC_69998 { itext[il] = ititle[c]; il = il + 1 } c = c + 1 }
                  if il < K_MAGIC_69998 { itext[il] = 46 as u8; il = il + 1 } if il < K_MAGIC_69998 { itext[il] = 32 as u8; il = il + 1 }
                  c = 0
                  while isum[c] != (0 as u8) { if il < K_MAGIC_69998 { itext[il] = isum[c]; il = il + 1 } c = c + 1 }
                  if il >= CI_MINDOC {
                      let icid: i64 = ci_hash(itext, il)
                      dss_mkkey(icid, ikey)
                      var iex: i64 = 0
                      if (h as i64) != 0 { if ss_hget(h, ikey, ipb, ilb) == 1 { iex = 1 } }
                      if iex == 0 {
                          ss_add(w, 1, ikey, itext, il)
                          ci_mkurlkey(icid, iukey)
                          ss_add(w, 1, iukey, ilink, slen(ilink))
                          item_docs = item_docs + 1
                      }
                  }
              }
          }
          idx = idx + 1
      } }
  } }
  if already == 1 { if item_docs == 0 { gw("  already present, no new feed items (idempotent)\n" as *u8); gw("URL-INDEX GREEN\n" as *u8); return 0 } }
  if ss_commit(prefix, w, segid) != 0 { gw("  commit failed\n" as *u8); return 5 }
  if already == 0 { gw("  INDEXED cid=" as *u8); gn(cid); gw(" (" as *u8); gn(tn); gw(" chars) url=" as *u8); gw(disp); gw("\n" as *u8) }
  else { gw("  page already present; added only new feed items\n" as *u8) }
  if item_docs > 0 { gw("  + " as *u8); gn(item_docs); gw(" feed-item docs (each its own article URL -> specific-article retrieval + frontier)\n" as *u8) }
  gw("URL-INDEX GREEN\n" as *u8)
  return 0
231}
nx_url_index.nx source

`nx_url_index.nx` source
