code wiki / _hdl_build / nx_outlink_harvest.nx
`nx_outlink_harvest.nx` source

↩ module page · 374 lines · 19339 B
1// nx_outlink_harvest.nx — shared outlink -> link-graph-edge harvester (P1 PageRank feed). LIB (no main).
2// Extracted (by copy) from nx_web_crawl_step.wc_harvest so BOTH the live crawler AND nx_cc_ingest emit the same
3// row shape: out:<ci_hash(page_url)> = packed i64 ci_hash(target_url). Node identity = ci_hash(url) ==
4// dss_urlcid(url) at serve time, so an outlink resolves to a graph node with zero extra index; serve-time
5// authority = content_cid -> url:<cid> -> ci_hash(url) -> pr:<cid>. ABSOLUTE cross-site links (the authority-
6// bearing ones) produce byte-identical cids to the crawler.
7//
8// Three intentional improvements over the crawler's inline copy (all to backport when the crawler adopts this
9// lib): (1) root-relative "/path" resolves HOST-based (scheme://host + path) not base+path — the crawler's
10// concatenation makes ".../Trust_law" + "/about" a phantom node; (2) edge capture is DECOUPLED from the frontier
11// cap (the crawler stops recording edges once discbox hits WC_MAXDISC — a graph must not depend on frontier
12// state); (3) one REUSED url buffer instead of an mmap per outlink (bounds address-space growth for CC-scale
13// ingests of thousands of pages). license_tier: ORIGINAL
14import "nx_corpus_ingest.nx"
15import "nx_store_seed_lib.nx"   // sts_load -> knowledge/store/urlpolicy- (R10: the denylist is DATA, not code)
16import "nx_url_canon.nx"        // nx_url_canon -- ONE definition, shared with the HTML extractors          // ci_hash
17const OLH_MAGIC_8192: i64 = 8192
18
19const OLH_MAXURL: i64 = 1500      // == the crawler's WC_MAXURL (measured junk/tracking-monster threshold) so
                                // edge nodes and frontier urls follow ONE canonical length policy
21const OLH_MAXEDGE: i64 = 1024
22
23func olh_len(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
24
25// literal substring present in u[0..ul)?  (verbatim wc_hassub)
26func olh_hassub(u: *u8, ul: i64, lit: *u8) -> i64 {
  let ll: i64 = olh_len(lit)
  if ll == 0 { return 0 }
  var i: i64 = 0
  while i + ll <= ul {
      var m: i64 = 1
      var k: i64 = 0
      while k < ll { if u[i + k] != lit[k] { m = 0; k = ll } else { k = k + 1 } }
      if m == 1 { return 1 }
      i = i + 1
  }
  return 0
38}
39// u ends with lit?  (verbatim wc_endswith)
40func olh_endswith(u: *u8, ul: i64, lit: *u8) -> i64 {
  let ll: i64 = olh_len(lit)
  if ul < ll { return 0 }
  var k: i64 = 0
  while k < ll { if u[ul - ll + k] != lit[k] { return 0 } k = k + 1 }
  return 1
46}
47// CONTENT-URL FILTER: 1 = a real page worth an authority edge. (verbatim wc_urlok; WC_MAXURL -> OLH_MAXURL)
48// R8 (2026-08-05): the body moved into olh_urlok_q with an allowq flag; olh_urlok keeps the exact
49// old contract (allowq=0) so CC ingest + the link graph are byte-identical. WHY: the blanket '?'
50// reject is correct for UNCURATED hosts (tracking junk, pagination traps) and WRONG for a curated
51// seed host whose CONTENT rides query strings -- measured: graphis.ne.jp galleries are
52// model.php?ID=..., so the whole entity vertical was structurally unreachable. Same law as the
53// R4b depth cap: re-derive every limit when the trust of its subject changes. ONE body, ONE flag
54// -- never a twin (twin-drift is what the 08-04 shared-policy fix killed).
55// ---- R11 PATH-CLASS POLICY AS DATA (2026-08-14) -------------------------------------------------
56// THE PATH AXIS OF R8, and the same defect on the same class of site. R8 moved the blanket '?' reject
57// onto a data-driven judge after measuring that graphis.ne.jp's whole gravure vertical was structurally
58// unreachable because its CONTENT rides query strings. The PATH axis kept five hardcoded literals and
59// is unreachable in exactly the same way: a tag-ORGANISED host's navigation IS its /tag/ and /category/
60// pages, so refusing the class refuses the host. MEASURED 2026-08-14: beeg.com and asianude4u.net are
61// absent from the index while their SEO doorway clones (bestbeeg.top, raj.red) are present -- doorways
62// publish flat article-shaped urls, real tag-organised sites do not. The rule selected AGAINST the
63// sites that serve content and FOR the ones that do not.
64// R10 already declared "the denylist is DATA, not code" and shipped knowledge/store/urlpolicy- to hold
65// it; the path rules never adopted it. This is an ADOPTION GAP, not a missing primitive.
66// * A DENYLIST IN CODE IS A POLICY NOBODY CAN CHANGE WITHOUT A COMPILER.
67// SELF-DEFAULTING, NOT LOADER-DEPENDENT (the law wc_query_ok learned the hard way): an absent or
68// path-free plane reproduces the EXACT 2026-08-04 list, so a missing data file can neither silently
69// open the gate nor silently close it. To make the plane authoritative -- INCLUDING "deny nothing" --
70// set __pathpolicy, which is the only way to express an intentionally empty policy.
71// Plane rows (ONE plane, TWO namespaces, no shared bucket -- the query axis owns the bare names):
72//   path:<substr>      deny   -- substring anywhere in the url
73//   pathend:<suffix>   deny   -- url ends with this
74//   __pathpolicy       plane  -- the plane is authoritative even if it lists no path rules
75const OLH_PATHPOL_PLANE: *u8 = "knowledge/store/urlpolicy-"
76const OLH_PSLOT:     i64 = 64      // bytes per rule: [kind byte][NUL-terminated pattern]
77const OLH_PMAXRULE:  i64 = 128
78const OLH_PPLANECAP: i64 = 65536   // ONE-TIME read buffer (per process, not per call); bounded and named
79const OLH_PKIND_SUB: i64 = 1
80const OLH_PKIND_END: i64 = 2
81static olh_prule_g:   *u8
82static olh_nprule_g:  i64
83static olh_ploaded_g: i64
84static olh_ppath_g:   *u8   // NULL = production plane; set = isolated fixture prefix (TEST ISOLATION)
85func olh_pathpol_set(p: *u8) -> i64 { olh_ppath_g = p; olh_ploaded_g = 0; return 0 }
86
87func olh_prule_add(kind: i64, pat: *u8, pl: i64) -> i64 {
  if olh_nprule_g >= OLH_PMAXRULE { return 1 }
  if pl >= OLH_PSLOT - 2 { return 1 }
  let dst: *u8 = (olh_prule_g as i64 + olh_nprule_g * OLH_PSLOT) as *u8
  dst[0] = kind as u8
  var k: i64 = 0
  while k < pl { dst[1 + k] = pat[k]; k = k + 1 }
  dst[1 + pl] = 0 as u8
  olh_nprule_g = olh_nprule_g + 1
  return 0
97}
98func olh_prule_addc(kind: i64, pat: *u8) -> i64 { return olh_prule_add(kind, pat, olh_len(pat)) }
99// THE 2026-08-04 LIST, VERBATIM. This is the behavioural contract when the plane says nothing.
100func olh_prule_defaults() -> i64 {
  olh_prule_addc(OLH_PKIND_SUB, "/tag/" as *u8)
  olh_prule_addc(OLH_PKIND_SUB, "/tags/" as *u8)
  olh_prule_addc(OLH_PKIND_SUB, "/category/" as *u8)
  olh_prule_addc(OLH_PKIND_END, "/tag" as *u8)
  olh_prule_addc(OLH_PKIND_END, "/category" as *u8)
  return olh_nprule_g
107}
108func olh_pname_is(b: *u8, s: i64, nl: i64, lit: *u8) -> i64 {
  let ll: i64 = olh_len(lit)
  if ll != nl { return 0 }
  var k: i64 = 0
  while k < ll { if b[s + k] != lit[k] { return 0 } k = k + 1 }
  return 1
114}
115// returns the prefix LENGTH when b[s..s+nl) starts with lit and is strictly longer, else 0
116func olh_pname_pref(b: *u8, s: i64, nl: i64, lit: *u8) -> i64 {
  let ll: i64 = olh_len(lit)
  if nl <= ll { return 0 }
  var k: i64 = 0
  while k < ll { if b[s + k] != lit[k] { return 0 } k = k + 1 }
  return ll
122}
123func olh_load_pathpolicy() -> i64 {
  if olh_ploaded_g == 1 { return olh_nprule_g }
  olh_ploaded_g = 1
  olh_prule_g = sys_mmap(OLH_PSLOT * OLH_PMAXRULE)
  olh_nprule_g = 0
  let b: *u8 = sys_mmap(OLH_PPLANECAP)
  var pp: *u8 = olh_ppath_g
  if (pp as i64) == 0 { pp = OLH_PATHPOL_PLANE }
  let n: i64 = sts_load(pp, b, OLH_PPLANECAP - 16)
  var authoritative: i64 = 0
  if n > 0 {
      var i: i64 = 0
      while i < n {
          let ls: i64 = i
          var le: i64 = ls
          var s: i64 = 1
          while s == 1 { if le >= n { s = 0 } else { if b[le] == (10 as u8) { s = 0 } else { le = le + 1 } } }
          i = le + 1
          if le - ls > 2 { if b[ls] != (35 as u8) {
              var t: i64 = 0 - 1
              var j: i64 = ls
              while j < le { if b[j] == (9 as u8) { t = j; j = le } else { j = j + 1 } }
              if t > ls {
                  let nl: i64 = t - ls
                  if olh_pname_is(b, ls, nl, "__pathpolicy" as *u8) == 1 { authoritative = 1 }
                  else {
                      let po: i64 = olh_pname_pref(b, ls, nl, "path:" as *u8)
                      if po > 0 {
                          authoritative = 1
                          olh_prule_add(OLH_PKIND_SUB, (b as i64 + ls + po) as *u8, nl - po)
                      }
                      else {
                          let qo: i64 = olh_pname_pref(b, ls, nl, "pathend:" as *u8)
                          if qo > 0 {
                              authoritative = 1
                              olh_prule_add(OLH_PKIND_END, (b as i64 + ls + qo) as *u8, nl - qo)
                          }
                      }
                  }
              }
          } }
      }
  }
  if authoritative == 0 { olh_prule_defaults() }
  return olh_nprule_g
168}
169// 1 = this url is on the path denylist. The plane is loaded ONCE per process, so this is hot-loop safe:
170// the mmap and the read happen on first call only, never per outlink.
171func olh_path_denied(u: *u8, ul: i64) -> i64 {
  olh_load_pathpolicy()
  var i: i64 = 0
  while i < olh_nprule_g {
      let sp: *u8 = (olh_prule_g as i64 + i * OLH_PSLOT) as *u8
      let kind: i64 = sp[0] as i64
      let pat: *u8 = (sp as i64 + 1) as *u8
      if kind == OLH_PKIND_SUB { if olh_hassub(u, ul, pat) == 1 { return 1 } }
      if kind == OLH_PKIND_END { if olh_endswith(u, ul, pat) == 1 { return 1 } }
      i = i + 1
  }
  return 0
183}
184func olh_urlok_q(u: *u8, ul: i64, allowq: i64) -> i64 {
  if ul < 12 { return 0 }
  if ul > OLH_MAXURL { return 0 }
  var scheme: i64 = 0
  if olh_hassub(u, 9, "https://" as *u8) == 1 { scheme = 1 }
  if olh_hassub(u, 8, "http://" as *u8) == 1 { scheme = 1 }
  if scheme == 0 { return 0 }
  var i: i64 = 0
  while i < ul {
      if u[i] == (35 as u8) { return 0 }    // '#' fragment
      if allowq == 0 { if u[i] == (63 as u8) { return 0 } }    // '?' query -- curated hosts may override (R8)
      i = i + 1
  }
  if olh_hassub(u, ul, "mailto:" as *u8) == 1 { return 0 }
  if olh_hassub(u, ul, "javascript:" as *u8) == 1 { return 0 }
  if olh_hassub(u, ul, ".wikipedia.org/" as *u8) == 1 {
      if olh_hassub(u, ul, "://en.wikipedia.org/" as *u8) == 0 { return 0 }
  }
  if olh_hassub(u, ul, ".wikimedia.org/" as *u8) == 1 { return 0 }
  if olh_hassub(u, ul, ".wiktionary.org/" as *u8) == 1 { return 0 }
  if olh_hassub(u, ul, ".wikidata.org/" as *u8) == 1 { return 0 }
  if olh_hassub(u, ul, "/Special:" as *u8) == 1 { return 0 }
  if olh_hassub(u, ul, "/File:" as *u8) == 1 { return 0 }
  if olh_hassub(u, ul, "/Talk:" as *u8) == 1 { return 0 }
  if olh_hassub(u, ul, "/Help:" as *u8) == 1 { return 0 }
  if olh_hassub(u, ul, "/Template:" as *u8) == 1 { return 0 }
  if olh_hassub(u, ul, "/Category:" as *u8) == 1 { return 0 }
  if olh_hassub(u, ul, "/Portal:" as *u8) == 1 { return 0 }
  if olh_hassub(u, ul, "/Wikipedia:" as *u8) == 1 { return 0 }
  if olh_endswith(u, ul, ".png" as *u8) == 1 { return 0 }
  if olh_endswith(u, ul, ".jpg" as *u8) == 1 { return 0 }
  if olh_endswith(u, ul, ".jpeg" as *u8) == 1 { return 0 }
  if olh_endswith(u, ul, ".gif" as *u8) == 1 { return 0 }
  if olh_endswith(u, ul, ".svg" as *u8) == 1 { return 0 }
  if olh_endswith(u, ul, ".css" as *u8) == 1 { return 0 }
  if olh_endswith(u, ul, ".js" as *u8) == 1 { return 0 }
  if olh_endswith(u, ul, ".ico" as *u8) == 1 { return 0 }
  if olh_endswith(u, ul, ".pdf" as *u8) == 1 { return 0 }
  if olh_endswith(u, ul, ".zip" as *u8) == 1 { return 0 }
  if olh_endswith(u, ul, ".xml" as *u8) == 1 { return 0 }
  if olh_hassub(u, ul, "/wp-json/" as *u8) == 1 { return 0 }
  if olh_endswith(u, ul, ".json" as *u8) == 1 { return 0 }
  if olh_endswith(u, ul, ".rss" as *u8) == 1 { return 0 }
  if olh_endswith(u, ul, "/feed" as *u8) == 1 { return 0 }
  if olh_endswith(u, ul, "/feed/" as *u8) == 1 { return 0 }
  // ANTI-SLOP URL CLASSES (2026-08-04, debt 1785895889): tag/category LISTING pages are
  // pagination-trap link farms, not content -- tag-farm pages (healthquerys.com/tag/*)
  // reached page 1 of natural-language SERPs while the ruler read 367 vs its 721 ratchet.
  // This is the ONE policy file, so crawler frontier, CC ingest and the link graph all
  // refuse them together. Article pages link each other directly; losing tag-page link
  // harvest costs little and stops the trap class at admission.
  // R11 (2026-08-14): the five literals that were here are now DATA (olh_path_denied above).
  // SAME rules by default and the SAME one-policy-file property -- crawler frontier, CC ingest,
  // WARC ingest, page ingest and the link graph all reach this one predicate -- but narrowing or
  // lifting the class is now a PLANE EDIT and a re-run, not a recompile and a promote across five
  // consumers. That difference is the whole point: the rule was measured wrong on 2026-08-14 and
  // nobody could change it without a build.
  if olh_path_denied(u, ul) == 1 { return 0 }
  return 1
243}
244func olh_urlok(u: *u8, ul: i64) -> i64 { return olh_urlok_q(u, ul, 0) }
245// out:<decimal cid> key. (verbatim wc_outkey)
246func olh_outkey(cid: i64, out: *u8) -> i64 {
  out[0] = 111 as u8   // o
  out[1] = 117 as u8   // u
  out[2] = 116 as u8   // t
  out[3] = 58 as u8    // :
  var m: i64 = cid
  let t: *u8 = sys_mmap(28)
  var k: i64 = 0
  if m == 0 { t[0] = 48 as u8; k = 1 }
  while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
  var i: i64 = 0
  while i < k { out[4 + i] = t[k - 1 - i]; i = i + 1 }
  out[4 + k] = 0 as u8
  return 4 + k
260}
261// resolve a root-relative href ("/path", NOT protocol-relative "//") against a NUL-terminated base url:
262// scheme://host + path (NOT base+path concat — that makes ".../Trust_law"+"/about" a phantom node). hp[0..clen)
263// = the raw href bytes. Writes the resolved NUL-terminated url into scratch (>= host + clen + 1 bytes; callers
264// pass an 8192 arena). Returns resolved length, or 0 if base has no "://" or the result would overflow.
265func olh_resolve_root(base: *u8, hp: *u8, clen: i64, scratch: *u8) -> i64 {
  var sp: i64 = 0 - 1
  var bi: i64 = 0
  var bf: i64 = 0
  while bf == 0 {
      if base[bi] == (0 as u8) { bf = 1 }
      else {
          if base[bi] == (58 as u8) { if base[bi+1] == (47 as u8) { if base[bi+2] == (47 as u8) { sp = bi; bf = 1 } } }
          if bf == 0 { bi = bi + 1 }
      }
  }
  if sp < 0 { return 0 }
  var he: i64 = sp + 3
  var hf: i64 = 0
  while hf == 0 { if base[he] == (0 as u8) { hf = 1 } else { if base[he] == (47 as u8) { hf = 1 } else { he = he + 1 } } }
  if he + clen + 1 >= OLH_MAGIC_8192 { return 0 }
  var b: i64 = 0
  while b < he { scratch[b] = base[b]; b = b + 1 }
  var cc: i64 = 0
  while cc < clen { scratch[he + cc] = hp[cc]; cc = cc + 1 }
  scratch[he + clen] = 0 as u8
  return he + clen
287}
288// scan hh[0..hlen) for <a href="..."> outlinks -> edgebuf[0..nedge) = ci_hash(resolved_url); returns nedge.
289// base = NUL-terminated page url (for root-relative resolution). Absolute http(s) kept as-is; "/path" ->
290// scheme://host + path; protocol-relative "//..." and other relative forms skipped (no authoritative base).
291// PER-CALL SCRATCH -> LAZY STATICS (2026-08-19 lane F, measured by nx_crawl_callee_probe: 12 kB/call
292// here after the nx_feed_extract fix -- the last named per-page allocator, ~2.5 pages/call page-rounded).
293// Rewritten per call; crawler fetch children fork their own copies. One-time cost ~10 KiB per process.
294static olh_scr_g: *u8
295static olh_ubuf_g: *u8
296func olh_scratch() -> i64 {
  if (olh_scr_g as i64) == 0 {
      olh_scr_g = sys_mmap(OLH_MAGIC_8192)
      olh_ubuf_g = sys_mmap(OLH_MAXURL + 2)
  }
  return 0
302}
303func olh_scan(hh: *u8, hlen: i64, base: *u8, blen: i64, edgebuf: *i64, maxedge: i64) -> i64 {
  var nedge: i64 = 0
  olh_scratch()
  let scratch: *u8 = olh_scr_g
  let ubuf: *u8 = olh_ubuf_g      // reused per outlink (no per-link mmap)
  var i: i64 = 0
  while i < hlen {
      var step: i64 = 1
      var ish: i64 = 0
      if i + 5 < hlen {
          var c0: i64 = hh[i] as i64
          if c0 >= 0x41 { if c0 <= 0x5a { c0 = c0 + 0x20 } }
          if c0 == 0x68 {
              var c1: i64 = hh[i+1] as i64
              if c1 >= 0x41 { if c1 <= 0x5a { c1 = c1 + 0x20 } }
              var c2: i64 = hh[i+2] as i64
              if c2 >= 0x41 { if c2 <= 0x5a { c2 = c2 + 0x20 } }
              var c3: i64 = hh[i+3] as i64
              if c3 >= 0x41 { if c3 <= 0x5a { c3 = c3 + 0x20 } }
              if c1 == 0x72 { if c2 == 0x65 { if c3 == 0x66 { if (hh[i+4] as i64) == 0x3d { ish = 1 } } } }
          }
      }
      if ish == 1 {
          let q: i64 = hh[i+5] as i64
          var quoted: i64 = 0
          if q == 0x22 { quoted = 1 }
          if q == 0x27 { quoted = 1 }
          if quoted == 1 {
              let cs: i64 = i + 6
              var e: i64 = cs
              var run: i64 = 1
              while run == 1 { run = 0; if e < hlen { if (hh[e] as i64) != q { e = e + 1; run = 1 } } }
              let clen: i64 = e - cs
              if clen > 0 { if clen <= OLH_MAXURL {
                  let hp: *u8 = ((hh as i64) + cs) as *u8
                  var cand: *u8 = 0 as *u8
                  var candlen: i64 = 0
                  var abs: i64 = 0
                  if clen >= 8 { if olh_hassub(hp, 8, "https://" as *u8) == 1 { abs = 1 } }
                  if abs == 0 { if clen >= 7 { if olh_hassub(hp, 7, "http://" as *u8) == 1 { abs = 1 } } }
                  if abs == 1 {
                      cand = hp
                      candlen = clen
                  } else {
                      if (hp[0] as i64) == 0x2f {
                          var protorel: i64 = 0
                          if clen >= 2 { if (hp[1] as i64) == 0x2f { protorel = 1 } }
                          if protorel == 0 {
                              let rl: i64 = olh_resolve_root(base, hp, clen, scratch)
                              if rl > 0 {
                                  cand = scratch
                                  candlen = rl
                              }
                          }
                      }
                  }
                  if (cand as i64) != 0 { if candlen <= OLH_MAXURL {
                      // CANONICALISE BEFORE JUDGING OR HASHING: node identity must be the identity of
                      // the url we would actually FETCH, or the link graph and the frontier disagree.
                      let cn: i64 = nx_url_canon(cand, candlen, ubuf, OLH_MAXURL)
                      if cn > 0 { if olh_urlok(ubuf, cn) == 1 {
                          if nedge < maxedge { edgebuf[nedge] = ci_hash(ubuf, cn); nedge = nedge + 1 }
                      } }
                  } }
              } }
              step = (e - i) + 1
          }
      }
      i = i + step
  }
  return nedge
374}
nx_outlink_harvest.nx source

`nx_outlink_harvest.nx` source
