code wiki / _hdl_build / nx_outlink_harvest.nx
`nx_outlink_harvest.nx` source

↩ module page · 228 lines · 11769 B
1// nx_outlink_harvest.nx — shared outlink -> link-graph-edge harvester (P1 PageRank feed). LIB (no main).
2// Extracted (by copy) from nx_web_crawl_step.wc_harvest so BOTH the live crawler AND nx_cc_ingest emit the same
3// row shape: out:<ci_hash(page_url)> = packed i64 ci_hash(target_url). Node identity = ci_hash(url) ==
4// dss_urlcid(url) at serve time, so an outlink resolves to a graph node with zero extra index; serve-time
5// authority = content_cid -> url:<cid> -> ci_hash(url) -> pr:<cid>. ABSOLUTE cross-site links (the authority-
6// bearing ones) produce byte-identical cids to the crawler.
7//
8// Three intentional improvements over the crawler's inline copy (all to backport when the crawler adopts this
9// lib): (1) root-relative "/path" resolves HOST-based (scheme://host + path) not base+path — the crawler's
10// concatenation makes ".../Trust_law" + "/about" a phantom node; (2) edge capture is DECOUPLED from the frontier
11// cap (the crawler stops recording edges once discbox hits WC_MAXDISC — a graph must not depend on frontier
12// state); (3) one REUSED url buffer instead of an mmap per outlink (bounds address-space growth for CC-scale
13// ingests of thousands of pages). license_tier: ORIGINAL
14import "nx_corpus_ingest.nx"          // ci_hash
15const OLH_MAGIC_8192: i64 = 8192
16
17const OLH_MAXURL: i64 = 1500      // == the crawler's WC_MAXURL (measured junk/tracking-monster threshold) so
                                // edge nodes and frontier urls follow ONE canonical length policy
19const OLH_MAXEDGE: i64 = 1024
20
21func olh_len(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
22
23// literal substring present in u[0..ul)?  (verbatim wc_hassub)
24func olh_hassub(u: *u8, ul: i64, lit: *u8) -> i64 {
  let ll: i64 = olh_len(lit)
  if ll == 0 { return 0 }
  var i: i64 = 0
  while i + ll <= ul {
      var m: i64 = 1
      var k: i64 = 0
      while k < ll { if u[i + k] != lit[k] { m = 0; k = ll } else { k = k + 1 } }
      if m == 1 { return 1 }
      i = i + 1
  }
  return 0
36}
37// u ends with lit?  (verbatim wc_endswith)
38func olh_endswith(u: *u8, ul: i64, lit: *u8) -> i64 {
  let ll: i64 = olh_len(lit)
  if ul < ll { return 0 }
  var k: i64 = 0
  while k < ll { if u[ul - ll + k] != lit[k] { return 0 } k = k + 1 }
  return 1
44}
45// CONTENT-URL FILTER: 1 = a real page worth an authority edge. (verbatim wc_urlok; WC_MAXURL -> OLH_MAXURL)
46// R8 (2026-08-05): the body moved into olh_urlok_q with an allowq flag; olh_urlok keeps the exact
47// old contract (allowq=0) so CC ingest + the link graph are byte-identical. WHY: the blanket '?'
48// reject is correct for UNCURATED hosts (tracking junk, pagination traps) and WRONG for a curated
49// seed host whose CONTENT rides query strings -- measured: graphis.ne.jp galleries are
50// model.php?ID=..., so the whole entity vertical was structurally unreachable. Same law as the
51// R4b depth cap: re-derive every limit when the trust of its subject changes. ONE body, ONE flag
52// -- never a twin (twin-drift is what the 08-04 shared-policy fix killed).
53func olh_urlok_q(u: *u8, ul: i64, allowq: i64) -> i64 {
  if ul < 12 { return 0 }
  if ul > OLH_MAXURL { return 0 }
  var scheme: i64 = 0
  if olh_hassub(u, 9, "https://" as *u8) == 1 { scheme = 1 }
  if olh_hassub(u, 8, "http://" as *u8) == 1 { scheme = 1 }
  if scheme == 0 { return 0 }
  var i: i64 = 0
  while i < ul {
      if u[i] == (35 as u8) { return 0 }    // '#' fragment
      if allowq == 0 { if u[i] == (63 as u8) { return 0 } }    // '?' query -- curated hosts may override (R8)
      i = i + 1
  }
  if olh_hassub(u, ul, "mailto:" as *u8) == 1 { return 0 }
  if olh_hassub(u, ul, "javascript:" as *u8) == 1 { return 0 }
  if olh_hassub(u, ul, ".wikipedia.org/" as *u8) == 1 {
      if olh_hassub(u, ul, "://en.wikipedia.org/" as *u8) == 0 { return 0 }
  }
  if olh_hassub(u, ul, ".wikimedia.org/" as *u8) == 1 { return 0 }
  if olh_hassub(u, ul, ".wiktionary.org/" as *u8) == 1 { return 0 }
  if olh_hassub(u, ul, ".wikidata.org/" as *u8) == 1 { return 0 }
  if olh_hassub(u, ul, "/Special:" as *u8) == 1 { return 0 }
  if olh_hassub(u, ul, "/File:" as *u8) == 1 { return 0 }
  if olh_hassub(u, ul, "/Talk:" as *u8) == 1 { return 0 }
  if olh_hassub(u, ul, "/Help:" as *u8) == 1 { return 0 }
  if olh_hassub(u, ul, "/Template:" as *u8) == 1 { return 0 }
  if olh_hassub(u, ul, "/Category:" as *u8) == 1 { return 0 }
  if olh_hassub(u, ul, "/Portal:" as *u8) == 1 { return 0 }
  if olh_hassub(u, ul, "/Wikipedia:" as *u8) == 1 { return 0 }
  if olh_endswith(u, ul, ".png" as *u8) == 1 { return 0 }
  if olh_endswith(u, ul, ".jpg" as *u8) == 1 { return 0 }
  if olh_endswith(u, ul, ".jpeg" as *u8) == 1 { return 0 }
  if olh_endswith(u, ul, ".gif" as *u8) == 1 { return 0 }
  if olh_endswith(u, ul, ".svg" as *u8) == 1 { return 0 }
  if olh_endswith(u, ul, ".css" as *u8) == 1 { return 0 }
  if olh_endswith(u, ul, ".js" as *u8) == 1 { return 0 }
  if olh_endswith(u, ul, ".ico" as *u8) == 1 { return 0 }
  if olh_endswith(u, ul, ".pdf" as *u8) == 1 { return 0 }
  if olh_endswith(u, ul, ".zip" as *u8) == 1 { return 0 }
  if olh_endswith(u, ul, ".xml" as *u8) == 1 { return 0 }
  if olh_hassub(u, ul, "/wp-json/" as *u8) == 1 { return 0 }
  if olh_endswith(u, ul, ".json" as *u8) == 1 { return 0 }
  if olh_endswith(u, ul, ".rss" as *u8) == 1 { return 0 }
  if olh_endswith(u, ul, "/feed" as *u8) == 1 { return 0 }
  if olh_endswith(u, ul, "/feed/" as *u8) == 1 { return 0 }
  // ANTI-SLOP URL CLASSES (2026-08-04, debt 1785895889): tag/category LISTING pages are
  // pagination-trap link farms, not content -- tag-farm pages (healthquerys.com/tag/*)
  // reached page 1 of natural-language SERPs while the ruler read 367 vs its 721 ratchet.
  // This is the ONE policy file, so crawler frontier, CC ingest and the link graph all
  // refuse them together. Article pages link each other directly; losing tag-page link
  // harvest costs little and stops the trap class at admission.
  if olh_hassub(u, ul, "/tag/" as *u8) == 1 { return 0 }
  if olh_hassub(u, ul, "/tags/" as *u8) == 1 { return 0 }
  if olh_hassub(u, ul, "/category/" as *u8) == 1 { return 0 }
  if olh_endswith(u, ul, "/tag" as *u8) == 1 { return 0 }
  if olh_endswith(u, ul, "/category" as *u8) == 1 { return 0 }
  return 1
110}
111func olh_urlok(u: *u8, ul: i64) -> i64 { return olh_urlok_q(u, ul, 0) }
112// out:<decimal cid> key. (verbatim wc_outkey)
113func olh_outkey(cid: i64, out: *u8) -> i64 {
  out[0] = 111 as u8   // o
  out[1] = 117 as u8   // u
  out[2] = 116 as u8   // t
  out[3] = 58 as u8    // :
  var m: i64 = cid
  let t: *u8 = sys_mmap(28)
  var k: i64 = 0
  if m == 0 { t[0] = 48 as u8; k = 1 }
  while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
  var i: i64 = 0
  while i < k { out[4 + i] = t[k - 1 - i]; i = i + 1 }
  out[4 + k] = 0 as u8
  return 4 + k
127}
128// resolve a root-relative href ("/path", NOT protocol-relative "//") against a NUL-terminated base url:
129// scheme://host + path (NOT base+path concat — that makes ".../Trust_law"+"/about" a phantom node). hp[0..clen)
130// = the raw href bytes. Writes the resolved NUL-terminated url into scratch (>= host + clen + 1 bytes; callers
131// pass an 8192 arena). Returns resolved length, or 0 if base has no "://" or the result would overflow.
132func olh_resolve_root(base: *u8, hp: *u8, clen: i64, scratch: *u8) -> i64 {
  var sp: i64 = 0 - 1
  var bi: i64 = 0
  var bf: i64 = 0
  while bf == 0 {
      if base[bi] == (0 as u8) { bf = 1 }
      else {
          if base[bi] == (58 as u8) { if base[bi+1] == (47 as u8) { if base[bi+2] == (47 as u8) { sp = bi; bf = 1 } } }
          if bf == 0 { bi = bi + 1 }
      }
  }
  if sp < 0 { return 0 }
  var he: i64 = sp + 3
  var hf: i64 = 0
  while hf == 0 { if base[he] == (0 as u8) { hf = 1 } else { if base[he] == (47 as u8) { hf = 1 } else { he = he + 1 } } }
  if he + clen + 1 >= OLH_MAGIC_8192 { return 0 }
  var b: i64 = 0
  while b < he { scratch[b] = base[b]; b = b + 1 }
  var cc: i64 = 0
  while cc < clen { scratch[he + cc] = hp[cc]; cc = cc + 1 }
  scratch[he + clen] = 0 as u8
  return he + clen
154}
155// scan hh[0..hlen) for <a href="..."> outlinks -> edgebuf[0..nedge) = ci_hash(resolved_url); returns nedge.
156// base = NUL-terminated page url (for root-relative resolution). Absolute http(s) kept as-is; "/path" ->
157// scheme://host + path; protocol-relative "//..." and other relative forms skipped (no authoritative base).
158func olh_scan(hh: *u8, hlen: i64, base: *u8, blen: i64, edgebuf: *i64, maxedge: i64) -> i64 {
  var nedge: i64 = 0
  let scratch: *u8 = sys_mmap(OLH_MAGIC_8192)
  let ubuf: *u8 = sys_mmap(OLH_MAXURL + 2)      // reused per outlink (no per-link mmap)
  var i: i64 = 0
  while i < hlen {
      var step: i64 = 1
      var ish: i64 = 0
      if i + 5 < hlen {
          var c0: i64 = hh[i] as i64
          if c0 >= 0x41 { if c0 <= 0x5a { c0 = c0 + 0x20 } }
          if c0 == 0x68 {
              var c1: i64 = hh[i+1] as i64
              if c1 >= 0x41 { if c1 <= 0x5a { c1 = c1 + 0x20 } }
              var c2: i64 = hh[i+2] as i64
              if c2 >= 0x41 { if c2 <= 0x5a { c2 = c2 + 0x20 } }
              var c3: i64 = hh[i+3] as i64
              if c3 >= 0x41 { if c3 <= 0x5a { c3 = c3 + 0x20 } }
              if c1 == 0x72 { if c2 == 0x65 { if c3 == 0x66 { if (hh[i+4] as i64) == 0x3d { ish = 1 } } } }
          }
      }
      if ish == 1 {
          let q: i64 = hh[i+5] as i64
          var quoted: i64 = 0
          if q == 0x22 { quoted = 1 }
          if q == 0x27 { quoted = 1 }
          if quoted == 1 {
              let cs: i64 = i + 6
              var e: i64 = cs
              var run: i64 = 1
              while run == 1 { run = 0; if e < hlen { if (hh[e] as i64) != q { e = e + 1; run = 1 } } }
              let clen: i64 = e - cs
              if clen > 0 { if clen <= OLH_MAXURL {
                  let hp: *u8 = ((hh as i64) + cs) as *u8
                  var cand: *u8 = 0 as *u8
                  var candlen: i64 = 0
                  var abs: i64 = 0
                  if clen >= 8 { if olh_hassub(hp, 8, "https://" as *u8) == 1 { abs = 1 } }
                  if abs == 0 { if clen >= 7 { if olh_hassub(hp, 7, "http://" as *u8) == 1 { abs = 1 } } }
                  if abs == 1 {
                      cand = hp
                      candlen = clen
                  } else {
                      if (hp[0] as i64) == 0x2f {
                          var protorel: i64 = 0
                          if clen >= 2 { if (hp[1] as i64) == 0x2f { protorel = 1 } }
                          if protorel == 0 {
                              let rl: i64 = olh_resolve_root(base, hp, clen, scratch)
                              if rl > 0 {
                                  cand = scratch
                                  candlen = rl
                              }
                          }
                      }
                  }
                  if (cand as i64) != 0 { if candlen <= OLH_MAXURL {
                      var z: i64 = 0
                      while z < candlen { ubuf[z] = cand[z]; z = z + 1 }
                      ubuf[candlen] = 0 as u8
                      if olh_urlok(ubuf, candlen) == 1 {
                          if nedge < maxedge { edgebuf[nedge] = ci_hash(ubuf, candlen); nedge = nedge + 1 }
                      }
                  } }
              } }
              step = (e - i) + 1
          }
      }
      i = i + step
  }
  return nedge
228}
nx_outlink_harvest.nx source

`nx_outlink_harvest.nx` source
