`nx_crawl_main.nx` source

↩ module page · 256 lines · 12740 B
1// nx_crawl_main.nx -- the deployable Nishi crawl+serve daemon.
2//
3// module: nishi-core.search.crawl_main
4// capability: APP_RUNNABLE (DAEMON)
5//
6// One binary that does it all, ISP-safely:
7//   STARTUP CRAWL: governed (nx_crawl_governor) polite-BFS (nx_robots +
8//   crawl-delay + descriptive UA), URL-canonicalized (nx_urlcanon) so we never
9//   re-fetch a variant, content-deduped (nx_simhash), indexed
10//   (nx_search_inverted) and PERSISTED (nx_docstore) so it resumes.
11//   SERVE: a live search site ranking the crawled corpus (nx_rank_fused:
12//   tier + BM25 + RRF). Surfaces everything the hosts allow, by relevance.
13//
14// ISP profile = slow ordinary browser: standard ports (80/443 -- here loopback
15// 8088 for the demo), GET-only (never seed/upload like BitTorrent), sequential
16// concurrency=1, GLOBAL min-interval + request/byte budgets (governor). In the
17// networked env, change the seed host/port to real targets over DNS/TLS.
18
19import "fx.nx"
20import "nx_str.nx"
21import "nx_syscalls.nx"
22import "nx_http_client.nx"
23import "nx_http_server.nx"
24import "nx_html_to_text.nx"
25import "nx_simhash.nx"
26import "nx_crawl_frontier.nx"
27import "nx_robots.nx"
28import "nx_urlcanon.nx"
29import "nx_docstore.nx"
30import "nx_net_governor.nx"
31import "nx_http_get_ua.nx"
32import "nx_rank_fused.nx"
33const NX_MAGIC_1048576: i64 = 1048576
34const NX_MAGIC_65536: i64 = 65536
35const NX_MAGIC_8192: i64 = 8192
36
37const NX_CM_CRAWL_PORT: i64 = 8088
38const NX_CM_SERVE_PORT: i64 = 8791
39
40func nx_putc(c: i64) -> i64 { let b: *u8 = sys_mmap(1); b[0] = c; sys_write(1, b, 1); return 0 }
41func nx_puts(s: *u8) -> i64 { sys_write(1, s, nx_str_len(s)); return 0 }
42func nx_pi(n: i64) -> i64 {
  if n == 0 { nx_putc(0x30); return 0 }
  var v: i64 = n; let t: *u8 = sys_mmap(32); var k: i64 = 0
  while v > 0 { t[k] = 0x30 + (v - (v/10)*10); v = v/10; k = k+1 }
  while k > 0 { k = k-1; sys_write(1, (((t as i64)+k) as *u8), 1) }
  return 0
48}
49func bput(buf: *u8, p: *i64, cap: i64, s: *u8) -> i64 { let n: i64 = nx_str_len(s); var i: i64 = 0; while i < n { if p[0] < cap { buf[p[0]] = s[i]; p[0] = p[0]+1 } i = i+1 } return 0 }
50func nx_body_off(resp: *u8, n: i64) -> i64 {
  var i: i64 = 0
  while i < n - 3 { if (resp[i] as i64)==0x0D { if (resp[i+1] as i64)==0x0A { if (resp[i+2] as i64)==0x0D { if (resp[i+3] as i64)==0x0A { return i+4 } } } } i = i + 1 }
  return 0
54}
55
56// GET with UA, capped read (governor resp cap), status parsed.
57// thin wrapper over the shared primitive (DRY)
58func nx_cm_fetch(addr: *u8, path: *u8, plen: i64, host: *u8, hlen: i64, ua: *u8, ualen: i64, out: *u8, cap: i64, out_status: *i64) -> i64 {
  return nx_http_get_ua(addr, path, plen, host, hlen, ua, ualen, out, cap, out_status)
60}
61
62// extract the path of a full URL (after scheme://host) into out (null-term).
63func nx_cm_path(url: *u8, out: *u8, outcap: i64) -> i64 {
  let n: i64 = nx_str_len(url)
  var i: i64 = 0
  var afters: i64 = 0
  var fnd: i64 = 0
  while i < n - 2 { if fnd == 0 { if (url[i] as i64)==0x3A { if (url[i+1] as i64)==0x2F { if (url[i+2] as i64)==0x2F { afters = i+3; fnd = 1 } } } } i = i + 1 }
  var ps: i64 = 0 - 1
  var p: i64 = afters
  while p < n { if ps < 0 { if (url[p] as i64)==0x2F { ps = p } } p = p + 1 }
  if ps < 0 { out[0] = 0x2F; out[1] = 0; return 1 }
  var o: i64 = 0
  var q: i64 = ps
  while q < n { if o < outcap-1 { out[o] = url[q]; o = o + 1 } q = q + 1 }
  out[o] = 0
  return o
78}
79
80func main() -> i64 {
  let addr: *u8 = sys_mmap(16)
  nx_http_client_sockaddr_ipv4(addr, 127, 0, 0, 1, NX_CM_CRAWL_PORT)
  let host: *u8 = "127.0.0.1"
  let hlen: i64 = nx_str_len(host)
  let ua: *u8 = "NishiBot/1.0 (+http://nishi.example/bot)"
  let ualen: i64 = nx_str_len(ua)
  let uatok: *u8 = "NishiBot"
  let uatoklen: i64 = nx_str_len(uatok)
  let storepath: *u8 = "_offc/nx_crawl_main.store\x00"
  let st: *i64 = sys_mmap(8) as *i64
91
  // ---- ISP-safe governor: slow, low-volume, browser-like ----
  let gov: *NxNetGov = sys_mmap(NX_NETGOV_BYTES) as *NxNetGov
  nx_gov_init(gov, 200, 20, NX_MAGIC_1048576, NX_MAGIC_65536)   // 200ms min-interval, 20 reqs, 1MB, 64KB/resp
95
  // ---- robots.txt ----
  let rbb: *u8 = sys_mmap(NX_MAGIC_8192)
  let rbn: i64 = nx_cm_fetch(addr, "/robots.txt", 11, host, hlen, ua, ualen, rbb, NX_MAGIC_8192, st)
  var robots: *u8 = rbb
  var robolen: i64 = 0
  if rbn > 0 { let rbo: i64 = nx_body_off(rbb, rbn); robots = ((rbb as i64) + rbo) as *u8; robolen = rbn - rbo }
  nx_gov_record(gov, sys_now_ms(), rbn)
  let cd: i64 = nx_robots_crawl_delay(robots, robolen, uatok, uatoklen)
  var hostdelay: i64 = cd * 1000
  if hostdelay <= 0 { hostdelay = 200 }
106
  nx_docstore_reset(storepath)
108
  // ---- governed polite BFS ----
  let QCAP: i64 = 128
  let queue: **u8 = sys_mmap(QCAP * 8) as **u8
  var qh: i64 = 0
  var qt: i64 = 0
  let visited: **u8 = sys_mmap(QCAP * 8) as **u8
  var nv: i64 = 0
  let seed: *u8 = sys_mmap(8); seed[0] = 0x2F; seed[1] = 0
  queue[qt] = seed; qt = qt + 1
118
  let curl:  **u8 = sys_mmap(64 * 8) as **u8
  let ctext: **u8 = sys_mmap(64 * 8) as **u8
  let ctlen: *i64 = sys_mmap(64 * 8) as *i64
  let keptfp: *i64 = sys_mmap(64 * 8) as *i64
  var ndoc: i64 = 0
  let idx: *NxInvIndex = nx_inv_new(64)
125
  nx_puts("=== Nishi crawl daemon: ISP-safe governed polite BFS ===\n")
  var running: i64 = 1
  while running == 1 {
      if qh >= qt { running = 0 }
      if nx_gov_can_fetch(gov) == 0 { running = 0; nx_puts("  [governor: budget reached -> stop]\n") }
      if running == 1 {
          let raw: *u8 = queue[qh]; qh = qh + 1
          // canonicalize the path for the visited-set (dedup variants)
          let cpath: *u8 = sys_mmap(512)
          nx_url_canon(raw, nx_str_len(raw), cpath, 512)
          var seen: i64 = 0
          var vi: i64 = 0
          while vi < nv { if nx_str_eq(cpath, visited[vi]) == 1 { seen = 1 } vi = vi + 1 }
          if seen == 0 {
              visited[nv] = cpath; nv = nv + 1
              if nx_robots_allowed(robots, robolen, uatok, uatoklen, cpath, nx_str_len(cpath)) == 0 {
                  nx_puts("  robots-skip: "); nx_puts(cpath); nx_putc(0x0A)
              } else {
                  let w: i64 = nx_gov_wait_ms(gov, sys_now_ms())
                  var hd: i64 = hostdelay
                  if w > hd { hd = w }
                  sys_sleep_ms(hd)
                  let resp: *u8 = sys_mmap(NX_MAGIC_65536)
                  let n: i64 = nx_cm_fetch(addr, cpath, nx_str_len(cpath), host, hlen, ua, ualen, resp, nx_gov_resp_cap(gov), st)
                  nx_gov_record(gov, sys_now_ms(), n)
                  if st[0] == 429 { nx_puts("  backoff 429\n"); sys_sleep_ms(hd * 4) }
                  else { if n > 0 {
                      let bo: i64 = nx_body_off(resp, n)
                      let bptr: *u8 = ((resp as i64) + bo) as *u8
                      let blen: i64 = n - bo
                      let text: *u8 = sys_mmap(NX_MAGIC_8192)
                      let tlen: i64 = nx_html_to_text(bptr, blen, text, NX_MAGIC_8192)
                      let fp: i64 = nx_simhash_fingerprint(text, tlen)
                      var dup: i64 = 0
                      var k: i64 = 0
                      while k < ndoc { if nx_simhash_hamming(fp, keptfp[k]) <= 6 { dup = 1 } k = k + 1 }
                      if dup == 0 {
                          curl[ndoc] = cpath; ctext[ndoc] = text; ctlen[ndoc] = tlen; keptfp[ndoc] = fp
                          nx_inv_index_row(idx, text, tlen, ndoc)
                          nx_docstore_append(storepath, cpath, nx_str_len(cpath), text, tlen)
                          ndoc = ndoc + 1
                          nx_puts("  crawled+persisted: "); nx_puts(cpath); nx_putc(0x0A)
                      } else { nx_puts("  content-dup: "); nx_puts(cpath); nx_putc(0x0A) }
                      // discover links
                      let lub: *u8 = sys_mmap(NX_MAGIC_8192)
                      let offs: *i64 = sys_mmap(64*8) as *i64
                      let lens: *i64 = sys_mmap(64*8) as *i64
                      let nl: i64 = nx_crawl_extract_links(bptr, blen, lub, NX_MAGIC_8192, offs, lens, 64)
                      var li: i64 = 0
                      while li < nl {
                          let lk: *u8 = sys_mmap(512)
                          var c: i64 = 0
                          while c < lens[li] { if c < 511 { lk[c] = lub[offs[li]+c] } c = c + 1 }
                          lk[lens[li]] = 0
                          let lp: *u8 = sys_mmap(256)
                          nx_cm_path(lk, lp, 256)
                          if qt < QCAP { queue[qt] = lp; qt = qt + 1 }
                          li = li + 1
                      }
                  } }
              }
          }
      }
  }
  nx_inv_finalize_offsets(idx)
  var d: i64 = 0
  while d < ndoc { nx_inv_emit_row(idx, ctext[d], ctlen[d], d); d = d + 1 }
  nx_puts("crawl done: "); nx_pi(ndoc); nx_puts(" docs indexed+persisted; bytes="); nx_pi(gov.bytes_used); nx_puts(" reqs="); nx_pi(gov.n_requests); nx_putc(0x0A)
194
  // ---- SERVE over the crawled corpus ----
  let saddr: *u8 = sys_mmap(16)
  nx_http_server_addr_any(saddr, NX_CM_SERVE_PORT)
  let vbox: *i64 = sys_mmap(8) as *i64
  let lfd: i64 = nx_http_server_listen(saddr, 16, vbox)
  if lfd < 0 { return 2 }
  nx_puts("serving search on :"); nx_pi(NX_CM_SERVE_PORT); nx_puts(" over the crawl\n")
202
  var serving: i64 = 1
  while serving == 1 {
      let cfd: i64 = nx_http_server_accept_one(lfd, vbox)
      if cfd >= 0 {
          let req: *u8 = sys_mmap(NX_MAGIC_8192)
          let m: *i64 = sys_mmap(8) as *i64
          let po: *i64 = sys_mmap(8) as *i64
          let pl: *i64 = sys_mmap(8) as *i64
          let cl: *i64 = sys_mmap(8) as *i64
          let bo2: *i64 = sys_mmap(8) as *i64
          let rn2: *i64 = sys_mmap(8) as *i64
          let rc: i64 = nx_http_server_read_request(cfd, req, NX_MAGIC_8192, m, po, pl, cl, bo2, rn2)
          let resp: *u8 = sys_mmap(NX_MAGIC_65536)
          let pos: *i64 = sys_mmap(8) as *i64
          pos[0] = 0
          bput(resp, pos, NX_MAGIC_65536, "HTTP/1.1 200 OK\r\nContent-Type: text/html; charset=utf-8\r\nConnection: close\r\n\r\n<!doctype html><body style='font-family:system-ui;background:#0b0e14;color:#dde'><h3>Nishi Search (over our crawl)</h3><form action='/search'><input name='q'><button>go</button></form>")
          // extract q
          let qbuf: *u8 = sys_mmap(512)
          var qlen: i64 = 0
          if rc == NXS_OK {
              var i: i64 = po[0]
              let endp: i64 = po[0] + pl[0]
              var qs: i64 = 0 - 1
              while i < endp - 1 { if (req[i] as i64)==0x71 { if (req[i+1] as i64)==0x3D { qs = i+2; i = endp } } if qs < 0 { i = i + 1 } }
              if qs >= 0 { var p2: i64 = qs; while p2 < endp { let c: i64 = req[p2] as i64; if c == 0x26 { p2 = endp } else { var e: i64 = c; if c == 0x2B { e = 0x20 } if qlen < 512 { qbuf[qlen] = e; qlen = qlen + 1 } p2 = p2 + 1 } } }
          }
          if qlen > 0 {
              let tp: **u8 = sys_mmap(8*8) as **u8
              let tl: *i64 = sys_mmap(8*8) as *i64
              var nt: i64 = 0
              var z: i64 = 0
              while z < qlen { if (qbuf[z] as i64)==0x20 { z = z+1 } else { let s0: i64 = z; var rn3: i64 = 1; while rn3==1 { rn3=0; if z<qlen { if (qbuf[z] as i64)!=0x20 { z=z+1; rn3=1 } } } if nt<8 { tp[nt]=((qbuf as i64)+s0) as *u8; tl[nt]=z-s0; nt=nt+1 } } }
              let order: *i64 = sys_mmap(64*8) as *i64
              nx_rank_fused(curl, curl, ctext, ctlen, ndoc, tp, tl, nt, order)
              bput(resp, pos, NX_MAGIC_65536, "<ul>")
              var shown: i64 = 0
              var oi: i64 = 0
              while oi < ndoc {
                  let dd: i64 = order[oi]
                  var mt: i64 = 0
                  var ti: i64 = 0
                  while ti < nt { if nx_bm25_tf(ctext[dd], ctlen[dd], nx_inv_hash_bytes_lower(tp[ti], tl[ti])) > 0 { mt = 1 } ti = ti + 1 }
                  if mt == 1 { bput(resp, pos, NX_MAGIC_65536, "<li>"); bput(resp, pos, NX_MAGIC_65536, curl[dd]); bput(resp, pos, NX_MAGIC_65536, "</li>"); shown = shown + 1 }
                  oi = oi + 1
              }
              if shown == 0 { bput(resp, pos, NX_MAGIC_65536, "<li>no match</li>") }
              bput(resp, pos, NX_MAGIC_65536, "</ul>")
          }
          bput(resp, pos, NX_MAGIC_65536, "</body>")
          nx_http_server_send_response(cfd, resp, pos[0])
      }
  }
  return 0
256}
nx_crawl_main.nx source

`nx_crawl_main.nx` source
