code wiki / (root) / nx_search_crawl_server.nx
`nx_search_crawl_server.nx` source

↩ module page · 224 lines · 10015 B
1// nx_search_crawl_server.nx -- LIVE search site over CRAWLED content.
2//
3// module: nishi-core.search.crawl_server
4// depends: nx_http_server, nx_http_client, nx_html_to_text, nx_simhash,
5//          nx_rank_fused, fx, nx_str
6// capability: APP_RUNNABLE (DAEMON)
7//
8// The integrated end-state: at startup the daemon CRAWLS a seed list over our
9// own HTTP client (fetch -> extract text -> content-dedup -> index), then
10// serves a live search site that ranks the SCRAPED corpus per query
11// (nx_rank_fused: tier + BM25 content via RRF). Everything is surfaced by
12// relevance -- no category censorship, no provenance weighting (operator
13// decision 2026-05-29: maximal coverage). Demonstrated on loopback against
14// nx_mock_web; in the networked env the seeds are real URLs over DNS/TLS/HTTP.
15
16import "fx.nx"
17import "nx_str.nx"
18import "nx_http_server.nx"
19import "nx_http_client.nx"
20import "nx_html_to_text.nx"
21import "nx_simhash.nx"
22import "nx_rank_fused.nx"
23
24const NX_SCS_PORT: i64 = 8790
25const NX_SCS_CRAWL_PORT: i64 = 8088   // the mock-web (or any seed host)
26
27func bputc(buf: *u8, pos: *i64, cap: i64, c: i64) -> i64 { if pos[0] < cap { buf[pos[0]] = c; pos[0] = pos[0] + 1 } return 0 }
28func bput(buf: *u8, pos: *i64, cap: i64, s: *u8) -> i64 { let n: i64 = nx_str_len(s); var i: i64 = 0; while i < n { bputc(buf, pos, cap, s[i] as i64); i = i + 1 } return 0 }
29
30func nx_body_off(resp: *u8, n: i64) -> i64 {
  var i: i64 = 0
  while i < n - 3 {
      if (resp[i] as i64) == 0x0D { if (resp[i+1] as i64) == 0x0A { if (resp[i+2] as i64) == 0x0D { if (resp[i+3] as i64) == 0x0A { return i + 4 } } } }
      i = i + 1
  }
  return 0
37}
38
39// extract <title>..</title> text into out; returns len (0 if none).
40func nx_extract_title(html: *u8, n: i64, out: *u8, outcap: i64) -> i64 {
  let tag: *u8 = "<title>"
  let tl: i64 = 7
  var i: i64 = 0
  var ts: i64 = 0 - 1
  while i <= n - tl {
      if ts < 0 {
          var j: i64 = 0
          var ok: i64 = 1
          while j < tl { if (html[i + j] as i64) != (tag[j] as i64) { ok = 0; j = tl } else { j = j + 1 } }
          if ok == 1 { ts = i + tl }
      }
      i = i + 1
  }
  if ts < 0 { return 0 }
  var olen: i64 = 0
  var p: i64 = ts
  var run: i64 = 1
  while run == 1 {
      run = 0
      if p < n { if (html[p] as i64) != 0x3C { if olen < outcap { out[olen] = html[p]; olen = olen + 1 } p = p + 1; run = 1 } }
  }
  return olen
63}
64
65func nx_extract_query(req: *u8, path_off: i64, path_len: i64, out: *u8, out_cap: i64) -> i64 {
  var i: i64 = path_off
  let end: i64 = path_off + path_len
  var qstart: i64 = 0 - 1
  while i < end - 1 {
      if (req[i] as i64) == 0x71 { if (req[i + 1] as i64) == 0x3D { qstart = i + 2; i = end } }
      if qstart < 0 { i = i + 1 }
  }
  if qstart < 0 { return 0 }
  var p: i64 = qstart
  var olen: i64 = 0
  var run: i64 = 1
  while run == 1 {
      run = 0
      if p < end {
          let c: i64 = req[p] as i64
          if c == 0x26 { run = 0 } else {
              var emit: i64 = c
              if c == 0x2B { emit = 0x20 }
              if olen < out_cap { out[olen] = emit; olen = olen + 1 }
              p = p + 1; run = 1
          }
      }
  }
  return olen
90}
91
92func nx_split_terms(qbuf: *u8, qlen: i64, tp: **u8, tl: *i64, max: i64) -> i64 {
  var n: i64 = 0
  var i: i64 = 0
  while i < qlen {
      if (qbuf[i] as i64) == 0x20 { i = i + 1 } else {
          let start: i64 = i
          var run: i64 = 1
          while run == 1 { run = 0; if i < qlen { if (qbuf[i] as i64) != 0x20 { i = i + 1; run = 1 } } }
          if n < max { tp[n] = ((qbuf as i64) + start) as *u8; tl[n] = i - start; n = n + 1 }
      }
      i = i + 1
  }
  return n
105}
106
107func main() -> i64 {
  // ---- STARTUP CRAWL: fetch seeds over our HTTP client, extract, dedup, index ----
  let caddr: *u8 = sys_mmap(16)
  nx_http_client_sockaddr_ipv4(caddr, 127, 0, 0, 1, NX_SCS_CRAWL_PORT)
  let host: *u8 = "127.0.0.1"
  let hlen: i64 = nx_str_len(host)
113
  let NSEED: i64 = 3
  let seeds: **u8 = sys_mmap(NSEED * 8) as **u8
  seeds[0] = "/wiki"; seeds[1] = "/thothub"; seeds[2] = "/fan"
117
  let curl:  **u8 = sys_mmap(64 * 8) as **u8
  let ctit:  **u8 = sys_mmap(64 * 8) as **u8
  let ctext: **u8 = sys_mmap(64 * 8) as **u8
  let ctlen: *i64 = sys_mmap(64 * 8) as *i64
  let keptfp: *i64 = sys_mmap(64 * 8) as *i64
  var ndoc: i64 = 0
124
  var s: i64 = 0
  while s < NSEED {
      let resp: *u8 = sys_mmap(16384)
      let vb: *i64 = sys_mmap(8) as *i64
      let n: i64 = nx_http_client_get(caddr, seeds[s], nx_str_len(seeds[s]), host, hlen, resp, 16384, vb)
      if n > 0 {
          let bo: i64 = nx_body_off(resp, n)
          let text: *u8 = sys_mmap(8192)
          let tlen: i64 = nx_html_to_text(((resp as i64) + bo) as *u8, n - bo, text, 8192)
          let fp: i64 = nx_simhash_fingerprint(text, tlen)
          var dup: i64 = 0
          var k: i64 = 0
          while k < ndoc { if nx_simhash_hamming(fp, keptfp[k]) <= 6 { dup = 1 } k = k + 1 }
          if dup == 0 {
              let title: *u8 = sys_mmap(256)
              let tnl: i64 = nx_extract_title(((resp as i64) + bo) as *u8, n - bo, title, 255)
              title[tnl] = 0
              // full url for display: host + path
              let fu: *u8 = sys_mmap(256)
              let fp2: *i64 = sys_mmap(8) as *i64
              fp2[0] = 0
              bput(fu, fp2, 256, "http://127.0.0.1:8088")
              bput(fu, fp2, 256, seeds[s])
              fu[fp2[0]] = 0
              curl[ndoc] = fu; ctit[ndoc] = title; ctext[ndoc] = text; ctlen[ndoc] = tlen; keptfp[ndoc] = fp
              ndoc = ndoc + 1
          }
      }
      s = s + 1
  }
155
  // ---- SERVE ----
  let addr: *u8 = sys_mmap(16)
  nx_http_server_addr_any(addr, NX_SCS_PORT)
  let vbox: *i64 = sys_mmap(8) as *i64
  let lfd: i64 = nx_http_server_listen(addr, 16, vbox)
  if lfd < 0 { return 2 }
162
  var serving: i64 = 1
  while serving == 1 {
      let cfd: i64 = nx_http_server_accept_one(lfd, vbox)
      if cfd >= 0 {
          let req: *u8 = sys_mmap(8192)
          let m: *i64 = sys_mmap(8) as *i64
          let po: *i64 = sys_mmap(8) as *i64
          let pl: *i64 = sys_mmap(8) as *i64
          let cl: *i64 = sys_mmap(8) as *i64
          let bo2: *i64 = sys_mmap(8) as *i64
          let rn: *i64 = sys_mmap(8) as *i64
          let rc: i64 = nx_http_server_read_request(cfd, req, 8192, m, po, pl, cl, bo2, rn)
175
          let resp: *u8 = sys_mmap(131072)
          let pos: *i64 = sys_mmap(8) as *i64
          pos[0] = 0
          bput(resp, pos, 131072, "HTTP/1.1 200 OK\r\nContent-Type: text/html; charset=utf-8\r\nConnection: close\r\n\r\n")
          bput(resp, pos, 131072, "<!doctype html><html><head><meta charset='utf-8'><title>Nishi Search (crawled)</title><style>body{font-family:system-ui;background:#0b0e14;color:#dde3ee;max-width:820px;margin:0 auto;padding:28px}h1{font-size:19px}form{display:flex;gap:8px;margin:14px 0}input{flex:1;padding:10px 13px;border-radius:9px;border:1px solid #25304a;background:#0f1422;color:#fff}button{padding:10px 16px;border:0;border-radius:9px;background:#2a6df4;color:#fff;font-weight:600}ol{list-style:none;padding:0;counter-reset:r}li{counter-increment:r;padding:9px 0;border-bottom:1px solid #161d29}li::before{content:counter(r) '. ';color:#5b6b86}.t{color:#eef2f8}.u{color:#5b6b86;font-size:12px}.sub{color:#8ea0c0;font-size:13px}</style></head><body>")
          bput(resp, pos, 131072, "<h1>NISHI SEARCH &mdash; over our own crawl</h1><p class='sub'>corpus: ")
          bput(resp, pos, 131072, "")
          // crawled-count
          let cnt: *u8 = sys_mmap(16); let cp: *i64 = sys_mmap(8) as *i64; cp[0] = 0; bputc(cnt, cp, 16, 0x30 + ndoc); cnt[cp[0]] = 0
          bput(resp, pos, 131072, cnt)
          bput(resp, pos, 131072, " pages scraped from the loopback web (live crawl in the networked env)</p>")
          bput(resp, pos, 131072, "<form action='/search'><input name='q' placeholder='Search the crawl&hellip;' value='")
188
          let qbuf: *u8 = sys_mmap(1024)
          var qlen: i64 = 0
          if rc == NXS_OK { qlen = nx_extract_query(req, po[0], pl[0], qbuf, 1024) }
          var qe: i64 = 0
          while qe < qlen { bputc(resp, pos, 131072, qbuf[qe] as i64); qe = qe + 1 }
          bput(resp, pos, 131072, "' autofocus><button>Search</button></form>")
195
          if qlen > 0 {
              let tp: **u8 = sys_mmap(8 * 8) as **u8
              let tl: *i64 = sys_mmap(8 * 8) as *i64
              let nterms: i64 = nx_split_terms(qbuf, qlen, tp, tl, 8)
              let order: *i64 = sys_mmap(64 * 8) as *i64
              nx_rank_fused(curl, ctit, ctext, ctlen, ndoc, tp, tl, nterms, order)
              bput(resp, pos, 131072, "<ol>")
              var shown: i64 = 0
              var i: i64 = 0
              while i < ndoc {
                  let d: i64 = order[i]
                  var matched: i64 = 0
                  var t: i64 = 0
                  while t < nterms { if nx_bm25_tf(ctext[d], ctlen[d], nx_inv_hash_bytes_lower(tp[t], tl[t])) > 0 { matched = 1 } t = t + 1 }
                  if matched == 1 {
                      bput(resp, pos, 131072, "<li><span class='t'>"); bput(resp, pos, 131072, ctit[d]); bput(resp, pos, 131072, "</span><div class='u'>"); bput(resp, pos, 131072, curl[d]); bput(resp, pos, 131072, "</div></li>")
                      shown = shown + 1
                  }
                  i = i + 1
              }
              if shown == 0 { bput(resp, pos, 131072, "<li class='sub'>no matches in the crawl</li>") }
              bput(resp, pos, 131072, "</ol>")
          }
          bput(resp, pos, 131072, "</body></html>")
          nx_http_server_send_response(cfd, resp, pos[0])
      }
  }
  return 0
224}
nx_search_crawl_server.nx source

`nx_search_crawl_server.nx` source
