code wiki / (root) / nx_entity_discover.nx
`nx_entity_discover.nx` source

↩ module page · 250 lines · 11619 B
1// nx_entity_discover.nx -- SREACH R3b: CRAWLER-DRIVEN ENTITY DISCOVERY.
2//
3// module: nishi-core.search.entity_discover
4// depends: nx_guarded_run.nx, nx_html_links.nx, nx_bm25.nx,
5//          /tmp/_ed_fetch_html.sov.elf (seed HTML), /tmp/_dlr_fetch_one.sov.elf (text)
6// capability: GATE + EVIDENCE
7//
8// OPERATOR LINE CLOSED HERE: seeds are no longer hand-curated. The crawler
9// starts from entity hub pages OUR index already reaches (wikidata + wikipedia),
10// extracts their outbound links with the nx_html_links kernel, keeps the
11// CROSS-HOST ones whose URL names the entity, then guarded-fetches each
12// DISCOVERED page over the sovereign stack, indexes it, and retrieves the
13// entity query from THAT index. Every result row is durable evidence; walls
14// stay named-not-faked. GATE (Rule #11 bars):
15//   discovered >= 3  -- the link harvest found genuinely new cross-host pages
16//   retrieved  >= 1  -- at least one DISCOVERED page fetched+indexed+retrievable
17//   (conservative: anonymous fetches of social hosts often hit login walls --
18//    a wall is a finding, not a discovery failure)
19
20import "nx_str.nx"
21import "nx_syscalls.nx"
22import "nx_search_inverted.nx"   // nx_bm25_tf + nx_inv_hash_bytes_lower. The header above lists nx_bm25.nx
                               // as the dependency for these -- it never carried them.
24import "nx_guarded_run.nx"
25import "nx_html_links.nx"
26import "nx_bm25.nx"
27import "nx_html_to_text.nx"
28import "nx_content_measure.nx"
29const NX_MAGIC_131072: i64 = 131072
30const NX_MAGIC_524288: i64 = 524288
31const NX_MAGIC_2048: i64 = 2048
32
33const NX_ED_SEEDS:    i64 = 2
34const NX_ED_MAXDISC:  i64 = 8
35const NX_ED_DEADLINE: i64 = 18000
36const NX_ED_MIN_DISCOVERED: i64 = 3
37const NX_ED_MIN_RETRIEVED:  i64 = 1
38
39func ed_p(s: *u8) -> i64 { sys_write(1, s, nx_str_len(s)); return 0 }
40func ed_n(v: i64) -> i64 {
  if v == 0 { sys_write(1, "0" as *u8, 1); return 0 }
  var m: i64 = v; if m < 0 { sys_write(1, "-" as *u8, 1); m = 0 - m }
  let t: *u8 = sys_mmap(28); var k: i64 = 0
  while m > 0 { t[k] = 0x30 + (m - (m/10)*10); m = m/10; k = k + 1 }
  while k > 0 { k = k - 1; sys_write(1, (((t as i64)+k) as *u8), 1) }
  return 0
47}
48func ed_f(fd: i64, s: *u8) -> i64 { sys_write(fd, s, nx_str_len(s)); return 0 }
49func ed_fn(fd: i64, v: i64) -> i64 {
  let b: *u8 = sys_mmap(28); let t: *u8 = sys_mmap(28)
  var m: i64 = v; if m < 0 { m = 0 - m }
  var k: i64 = 0
  if m == 0 { t[0] = 48; k = 1 }
  while m > 0 { t[k] = 48 + (m % 10); m = m / 10; k = k + 1 }
  var i: i64 = 0
  while i < k { b[i] = t[k-1-i]; i = i + 1 }
  sys_write(fd, b, k)
  return 0
59}
60func ed_lc(c: i64) -> i64 { if c >= 0x41 { if c <= 0x5A { return c + 0x20 } } return c }
61func ed_ci_contains(hay: *u8, needle: *u8) -> i64 {
  let hn: i64 = nx_str_len(hay)
  let nn: i64 = nx_str_len(needle)
  if nn > hn { return 0 }
  var i: i64 = 0
  while i <= hn - nn {
      var j: i64 = 0
      var ok: i64 = 1
      while j < nn {
          if ed_lc(hay[i+j] as i64) != ed_lc(needle[j] as i64) { ok = 0; j = nn } else { j = j + 1 }
      }
      if ok == 1 { return 1 }
      i = i + 1
  }
  return 0
76}
77func ed_streq(a: *u8, b: *u8) -> i64 {
  var i: i64 = 0
  while a[i] != (0 as u8) { if a[i] != b[i] { return 0 } i = i + 1 }
  if b[i] != (0 as u8) { return 0 }
  return 1
82}
83
84func ed_seed_url(e: i64) -> *u8 {
  if e == 0 { return "https://www.wikidata.org/wiki/Q283504" }
  if e == 1 { return "https://en.wikipedia.org/wiki/Diora_Baird" }
  return "?"
88}
89func ed_seed_out(e: i64) -> *u8 {
  if e == 0 { return "/tmp/ed_seed_0.html\x00" }
  return "/tmp/ed_seed_1.html\x00"
92}
93func ed_disc_out(d: i64) -> *u8 {
  if d == 0 { return "/tmp/ed_disc_0.html\x00" }
  if d == 1 { return "/tmp/ed_disc_1.html\x00" }
  if d == 2 { return "/tmp/ed_disc_2.html\x00" }
  if d == 3 { return "/tmp/ed_disc_3.html\x00" }
  if d == 4 { return "/tmp/ed_disc_4.html\x00" }
  if d == 5 { return "/tmp/ed_disc_5.html\x00" }
  if d == 6 { return "/tmp/ed_disc_6.html\x00" }
  return "/tmp/ed_disc_7.html\x00"
102}
103
104func ed_read_all(path: *u8, buf: *u8, cap: i64) -> i64 {
  let fd: i64 = sys_openat_rd(path)
  if fd < 0 { return 0 }
  var total: i64 = 0
  var go: i64 = 1
  while go == 1 {
      let n: i64 = sys_read(fd, (((buf as i64) + total) as *u8), cap - total)
      if n <= 0 { go = 0 }
      if n > 0 { total = total + n; if total >= cap { go = 0 } }
  }
  sys_close(fd)
  return total
116}
117
118// own-corpus host (the seeds' hosts) -> NOT a discovery
119func ed_is_seed_host(url: *u8) -> i64 {
  if ed_ci_contains(url, "wikipedia.org") == 1 { return 1 }
  if ed_ci_contains(url, "wikidata.org")  == 1 { return 1 }
  if ed_ci_contains(url, "wikimedia.org") == 1 { return 1 }
  return 0
124}
125
126func main() -> i64 {
  ed_p("=== ENTITY DISCOVERY (R3b): crawler finds the pages, no hand-curated seeds ===\n")
  let html_child: *u8 = "/tmp/_ed_fetch_html.sov.elf\x00"
  let pr1: i64 = sys_openat_rd(html_child)
  if pr1 < 0 { ed_p(" prereq missing: _ed_fetch_html\n"); sys_exit(3); return 3 }
  sys_close(pr1)
  let envp: *i64 = sys_mmap(16) as *i64
  envp[0] = "PATH=/usr/bin:/bin" as *u8 as i64; envp[1] = 0
  let devnull: i64 = sys_openat_wr("/dev/null" as *u8, 0x1a4)
  let lfd: i64 = sys_openat_append("knowledge/status/search_discovery.log" as *u8, 0x1a4)
  if lfd < 0 { ed_p(" log open failed\n"); sys_exit(1); return 1 }
  ed_f(lfd, "ED-RUN epoch=" as *u8); ed_fn(lfd, sys_now_realtime_sec()); ed_f(lfd, "\n" as *u8)
138
  // ---- harvest links from the seed hub pages ----
  let store: *u8 = sys_mmap(NX_MAGIC_131072)
  let offs: *i64 = sys_mmap(512 * 8) as *i64
  let durl: **u8 = sys_mmap(NX_ED_MAXDISC * 8) as **u8
  var ndisc: i64 = 0
  var nlinks_total: i64 = 0
  var e: i64 = 0
  while e < NX_ED_SEEDS {
      let av: *i64 = sys_mmap(8 * 4) as *i64
      av[0] = html_child as i64; av[1] = ed_seed_url(e) as i64; av[2] = ed_seed_out(e) as i64; av[3] = 0
      let rc: i64 = nx_guarded_run(html_child, av, envp, NX_ED_DEADLINE, devnull, devnull)
      ed_p("  seed "); ed_p(ed_seed_url(e)); ed_p(" -> ")
      if rc != 0 { ed_p("fetch rc="); ed_n(rc); ed_p(" (named)\n") } else {
          let html: *u8 = sys_mmap(NX_MAGIC_524288)
          let hn: i64 = ed_read_all(ed_seed_out(e), html, NX_MAGIC_524288)
          let nl: i64 = nx_html_links(html, hn, store, NX_MAGIC_131072, offs, 512)
          nlinks_total = nlinks_total + nl
          ed_p("OK "); ed_n(hn); ed_p(" html bytes, "); ed_n(nl); ed_p(" https links\n")
          // filter: entity-named URL, cross-host, deduped, capped
          var li: i64 = 0
          while li < nl {
              let u: *u8 = (((store as i64) + offs[li]) as *u8)
              var keep: i64 = 0
              if ed_ci_contains(u, "diora") == 1 { keep = 1 }
              if keep == 1 { if ed_is_seed_host(u) == 1 { keep = 0 } }
              if keep == 1 {
                  var dd: i64 = 0
                  while dd < ndisc { if ed_streq(u, durl[dd]) == 1 { keep = 0; dd = ndisc } else { dd = dd + 1 } }
              }
              if keep == 1 {
                  if ndisc < NX_ED_MAXDISC {
                      // persist a copy (store is reused across seeds)
                      let cp: *u8 = sys_mmap(NX_MAGIC_2048)
                      var c: i64 = 0
                      while u[c] != (0 as u8) { cp[c] = u[c]; c = c + 1 }
                      cp[c] = 0 as u8
                      durl[ndisc] = cp
                      ndisc = ndisc + 1
                  }
              }
              li = li + 1
          }
      }
      e = e + 1
  }
  ed_p(" discovered (cross-host, entity-named, deduped): "); ed_n(ndisc); ed_p(" of "); ed_n(nlinks_total); ed_p(" harvested links\n")
  ed_f(lfd, "ED-HARVEST links=" as *u8); ed_fn(lfd, nlinks_total)
  ed_f(lfd, " discovered=" as *u8); ed_fn(lfd, ndisc); ed_f(lfd, "\n" as *u8)
187
  // ---- fetch + index + retrieve each DISCOVERED page ----
  let h_diora: i64 = nx_inv_hash_bytes_lower("diora", 5)
  let h_baird: i64 = nx_inv_hash_bytes_lower("baird", 5)
  // R3c-a: discovered docs are fetched as HTML so content value is MEASURED
  // from the served bytes (nx_content_measure: text-length + media tags),
  // never inferred; text for retrieval is extracted in-process. Retrieved
  // docs feed the durable corpus (knowledge/index/discovered_corpus.txt) the
  // live SERP grows from -- corpus growth by crawl, not by seed-table edit.
  let cfd: i64 = sys_openat_append("knowledge/index/discovered_corpus.txt" as *u8, 0x1a4)
  var nret: i64 = 0
  var d: i64 = 0
  while d < ndisc {
      let av2: *i64 = sys_mmap(8 * 4) as *i64
      av2[0] = html_child as i64; av2[1] = durl[d] as i64; av2[2] = ed_disc_out(d) as i64; av2[3] = 0
      let rc2: i64 = nx_guarded_run(html_child, av2, envp, NX_ED_DEADLINE, devnull, devnull)
      ed_p("  #"); ed_n(d + 1); ed_p(" "); ed_p(durl[d]); ed_p(" -> ")
      ed_f(lfd, "ED-DOC url=" as *u8); ed_f(lfd, durl[d]); ed_f(lfd, " rc=" as *u8); ed_fn(lfd, rc2)
      if rc2 != 0 {
          if rc2 == 124 { ed_p("TIMEOUT-REAPED\n"); ed_f(lfd, " status=TIMEOUT\n" as *u8) }
          if rc2 != 124 { ed_p("walled/fail rc="); ed_n(rc2); ed_p(" (named)\n"); ed_f(lfd, " status=WALLED\n" as *u8) }
      } else {
          let html: *u8 = sys_mmap(NX_MAGIC_524288)
          let hn: i64 = ed_read_all(ed_disc_out(d), html, NX_MAGIC_524288)
          let text: *u8 = sys_mmap(NX_MAGIC_524288)
          let tl: i64 = nx_html_to_text(html, hn, text, NX_MAGIC_524288)
          let tf: i64 = nx_bm25_tf(text, tl, h_diora) + nx_bm25_tf(text, tl, h_baird)
          let cm: i64 = nx_content_measure(html, hn, tl)
          let cm_permil: i64 = (cm * 1000) >> 16
          ed_p("OK "); ed_n(tl); ed_p(" chars, entity-tf "); ed_n(tf)
          ed_p(", content-permil "); ed_n(cm_permil)
          ed_f(lfd, " status=FETCHED chars=" as *u8); ed_fn(lfd, tl)
          ed_f(lfd, " entity_tf=" as *u8); ed_fn(lfd, tf)
          ed_f(lfd, " content_permil=" as *u8); ed_fn(lfd, cm_permil)
          if tf > 0 {
              nret = nret + 1
              ed_p("  <- RETRIEVED\n"); ed_f(lfd, " retrieved=1\n" as *u8)
              if cfd >= 0 {
                  ed_f(cfd, "DC-URL url=" as *u8); ed_f(cfd, durl[d])
                  ed_f(cfd, " tf=" as *u8); ed_fn(cfd, tf)
                  ed_f(cfd, " content_permil=" as *u8); ed_fn(cfd, cm_permil)
                  ed_f(cfd, " epoch=" as *u8); ed_fn(cfd, sys_now_realtime_sec())
                  ed_f(cfd, "\n" as *u8)
              }
          } else { ed_p("\n"); ed_f(lfd, " retrieved=0\n" as *u8) }
      }
      d = d + 1
  }
  if cfd >= 0 { sys_close(cfd) }
236
  var green: i64 = 0
  if ndisc >= NX_ED_MIN_DISCOVERED { if nret >= NX_ED_MIN_RETRIEVED { green = 1 } }
  ed_p(" discovery: "); ed_n(ndisc); ed_p(" found / "); ed_n(nret); ed_p(" retrieved from OUR index\n")
  ed_f(lfd, "ED-GATE discovered=" as *u8); ed_fn(lfd, ndisc)
  ed_f(lfd, " retrieved=" as *u8); ed_fn(lfd, nret)
  ed_f(lfd, " bars=" as *u8); ed_fn(lfd, NX_ED_MIN_DISCOVERED)
  ed_f(lfd, "/" as *u8); ed_fn(lfd, NX_ED_MIN_RETRIEVED)
  if green == 1 { ed_f(lfd, " verdict=GREEN\n" as *u8) } else { ed_f(lfd, " verdict=RED\n" as *u8) }
  sys_close(lfd)
  if green == 1 { ed_p(" ENTITY-DISCOVERY GATE: GREEN (the crawler found them, we did not)\n"); sys_exit(0); return 0 }
  ed_p(" ENTITY-DISCOVERY GATE: RED\n")
  sys_exit(1)
  return 1
250}
nx_entity_discover.nx source

`nx_entity_discover.nx` source
