code wiki / (root) / nx_search_server.nx
`nx_search_server.nx` source

↩ module page · 263 lines · 12455 B
1// nx_search_server.nx -- the LIVE bits-up search site (sovereign HTTP server).
2//
3// module: nishi-core.search.server
4// depends: nx_http_server.nx, nx_rank_fused.nx, nx_diora_data.nx, fx.nx, nx_str.nx
5// capability: APP_RUNNABLE (DAEMON)
6//
7// Serves a real search site over our own HTTP server (no nginx/libc):
8//   GET /            -> the search page (query box)
9//   GET /search?q=.. -> runs nx_rank_fused over the indexed corpus and returns
10//                       ranked results (facet + source tier + primary star)
11// The corpus is loaded at startup from nx_diora_data (the verified Diora docs --
12// the stand-in for crawled content); in the networked env the crawler
13// (nx_crawl_doc/frontier) fills the same arrays from live scraping. Ranking is
14// LIVE per query: tier prior + BM25 content, fused via RRF -- so a unique
15// low-tier page outranks a thin high-tier one. Deterministic Q16.16.
16
17import "fx.nx"
18import "nx_str.nx"
19import "nx_http_server.nx"
20import "nx_rank_fused.nx"
21import "nx_search_inverted.nx"   // nx_bm25_tf + nx_inv_hash_bytes_lower, used on the /search path
22import "nx_diora_data.nx"
23
24const NX_SEARCH_PORT: i64 = 8787
25
26// ---- response-buffer append helpers -------------------------------
27func bputc(buf: *u8, pos: *i64, cap: i64, c: i64) -> i64 {
  if pos[0] < cap { buf[pos[0]] = c; pos[0] = pos[0] + 1 }
  return 0
30}
31func bput(buf: *u8, pos: *i64, cap: i64, s: *u8) -> i64 {
  let n: i64 = nx_str_len(s)
  var i: i64 = 0
  while i < n { bputc(buf, pos, cap, s[i] as i64); i = i + 1 }
  return 0
36}
37func bput_i64(buf: *u8, pos: *i64, cap: i64, n: i64) -> i64 {
  if n == 0 { bputc(buf, pos, cap, 0x30); return 0 }
  var v: i64 = n
  if v < 0 { bputc(buf, pos, cap, 0x2D); v = 0 - v }
  let tmp: *u8 = sys_mmap(32)
  var k: i64 = 0
  while v > 0 { tmp[k] = 0x30 + (v - (v / 10) * 10); v = v / 10; k = k + 1 }
  while k > 0 { k = k - 1; bputc(buf, pos, cap, tmp[k] as i64) }
  return 0
46}
47func bput_q16(buf: *u8, pos: *i64, cap: i64, x: i64) -> i64 {
  bput_i64(buf, pos, cap, x >> 16); bputc(buf, pos, cap, 0x2E)
  let low: i64 = x - ((x >> 16) << 16)
  let frac: i64 = (low * 10000) >> 16
  let d3: i64 = frac / 1000
  let r3: i64 = frac - d3 * 1000
  let d2: i64 = r3 / 100
  let r2: i64 = r3 - d2 * 100
  let d1: i64 = r2 / 10
  bputc(buf, pos, cap, 0x30 + d3); bputc(buf, pos, cap, 0x30 + d2)
  bputc(buf, pos, cap, 0x30 + d1); bputc(buf, pos, cap, 0x30 + (r2 - d1 * 10))
  return 0
59}
60
61func nx_facet_name(f: i64) -> *u8 {
  if f == 0 { return "identity" }
  if f == 1 { return "works" }
  if f == 2 { return "official-media" }
  if f == 3 { return "interviews" }
  if f == 4 { return "press" }
  if f == 5 { return "self-social" }
  if f == 6 { return "long-tail" }
  return "&mdash;"
70}
71func nx_tier_name(t: i64) -> *u8 {
  if t == NX_TIER_PRIMARY   { return "primary" }
  if t == NX_TIER_EDITORIAL { return "editorial" }
  if t == NX_TIER_PLATFORM  { return "platform" }
  if t == NX_TIER_ARCHIVE   { return "archive" }
  if t == NX_TIER_COMMUNITY { return "community" }
  return "unknown"
78}
79
80// hex digit value or -1
81func nx_hexval(c: i64) -> i64 {
  if c >= 0x30 { if c <= 0x39 { return c - 0x30 } }
  if c >= 0x61 { if c <= 0x66 { return c - 0x61 + 10 } }
  if c >= 0x41 { if c <= 0x46 { return c - 0x41 + 10 } }
  return 0 - 1
86}
87
88// extract + url-decode the q= value from a request path region into out.
89// Returns decoded length. Handles '+' -> space and %XX.
90func nx_extract_query(req: *u8, path_off: i64, path_len: i64, out: *u8, out_cap: i64) -> i64 {
  // find "q=" within [path_off, path_off+path_len)
  var i: i64 = path_off
  let end: i64 = path_off + path_len
  var qstart: i64 = 0 - 1
  while i < end - 1 {
      if (req[i] as i64) == 0x71 {           // 'q'
          if (req[i + 1] as i64) == 0x3D {    // '='
              qstart = i + 2
              i = end
          }
      }
      if qstart < 0 { i = i + 1 }
  }
  if qstart < 0 { return 0 }
  var p: i64 = qstart
  var olen: i64 = 0
  var run: i64 = 1
  while run == 1 {
      run = 0
      if p < end {
          let c: i64 = req[p] as i64
          if c == 0x26 { run = 0 }                 // '&' ends the value
          else {
              var emit: i64 = c
              if c == 0x2B { emit = 0x20 }         // '+' -> space
              if c == 0x25 {                       // '%XX'
                  if p + 2 < end {
                      let hi: i64 = nx_hexval(req[p + 1] as i64)
                      let lo: i64 = nx_hexval(req[p + 2] as i64)
                      if hi >= 0 { if lo >= 0 { emit = hi * 16 + lo; p = p + 2 } }
                  }
              }
              if olen < out_cap { out[olen] = emit; olen = olen + 1 }
              p = p + 1
              run = 1
          }
      }
  }
  return olen
130}
131
132// split a decoded query into up to max terms (token slices into qbuf).
133func nx_split_terms(qbuf: *u8, qlen: i64, term_ptrs: **u8, term_lens: *i64, max: i64) -> i64 {
  var n: i64 = 0
  var i: i64 = 0
  while i < qlen {
      if (qbuf[i] as i64) == 0x20 { i = i + 1 }
      else {
          let start: i64 = i
          var run: i64 = 1
          while run == 1 {
              run = 0
              if i < qlen { if (qbuf[i] as i64) != 0x20 { i = i + 1; run = 1 } }
          }
          if n < max {
              term_ptrs[n] = ((qbuf as i64) + start) as *u8
              term_lens[n] = i - start
              n = n + 1
          }
      }
  }
  return n
153}
154
155func main() -> i64 {
  // ---- load corpus (urls/titles/facets); titles are the indexed text
  //      field until the live crawl supplies full page content ----
  let qn: i64 = nx_diora_qn()
  let curl: **u8 = sys_mmap(qn * 8) as **u8
  let ctit: **u8 = sys_mmap(qn * 8) as **u8
  let cfac: *i64 = sys_mmap(qn * 8) as *i64
  let cgrd: *i64 = sys_mmap(qn * 8) as *i64
  let cpri: *i64 = sys_mmap(qn * 8) as *i64
  nx_diora_fill_qrels(curl, ctit, cfac, cgrd, cpri)
  let ctlen: *i64 = sys_mmap(qn * 8) as *i64
  var ci: i64 = 0
  while ci < qn { ctlen[ci] = nx_str_len(ctit[ci]); ci = ci + 1 }
168
  // ---- listen ----
  let addr: *u8 = sys_mmap(16)
  nx_http_server_addr_any(addr, NX_SEARCH_PORT)
  let vbox: *i64 = sys_mmap(8) as *i64
  let lfd: i64 = nx_http_server_listen(addr, 16, vbox)
  if lfd < 0 { return 2 }
175
  let reqcap: i64 = 8192
  let respcap: i64 = 131072
  let v: *NxTierVerdict = sys_mmap(NX_TIER_VERDICT_BYTES) as *NxTierVerdict
179
  var serving: i64 = 1
  while serving == 1 {
      let cfd: i64 = nx_http_server_accept_one(lfd, vbox)
      if cfd >= 0 {
          let req: *u8 = sys_mmap(reqcap)
          let m: *i64 = sys_mmap(8) as *i64
          let po: *i64 = sys_mmap(8) as *i64
          let pl: *i64 = sys_mmap(8) as *i64
          let cl: *i64 = sys_mmap(8) as *i64
          let bo: *i64 = sys_mmap(8) as *i64
          let rn: *i64 = sys_mmap(8) as *i64
          let rc: i64 = nx_http_server_read_request(cfd, req, reqcap, m, po, pl, cl, bo, rn)
192
          let resp: *u8 = sys_mmap(respcap)
          let pos: *i64 = sys_mmap(8) as *i64
          pos[0] = 0
          bput(resp, pos, respcap, "HTTP/1.1 200 OK\r\nContent-Type: text/html; charset=utf-8\r\nConnection: close\r\n\r\n")
          bput(resp, pos, respcap, "<!doctype html><html lang='en'><head><meta charset='utf-8'><meta name='viewport' content='width=device-width,initial-scale=1'><title>Nishi Search</title><style>")
          bput(resp, pos, respcap, "body{font-family:ui-sans-serif,system-ui,Segoe UI,Roboto,sans-serif;background:#0b0e14;color:#dde3ee;margin:0;padding:28px;max-width:880px}")
          bput(resp, pos, respcap, "h1{font-size:20px;letter-spacing:.5px}form{display:flex;gap:8px;margin:14px 0 6px}")
          bput(resp, pos, respcap, "input{flex:1;padding:11px 14px;border-radius:10px;border:1px solid #25304a;background:#0f1422;color:#fff;font-size:15px}")
          bput(resp, pos, respcap, "button{padding:11px 18px;border-radius:10px;border:0;background:#2a6df4;color:#fff;font-weight:600;cursor:pointer}")
          bput(resp, pos, respcap, ".sub{color:#8ea0c0;font-size:13px}ol{padding-left:0;list-style:none;counter-reset:r}")
          bput(resp, pos, respcap, "li{counter-increment:r;padding:10px 0;border-bottom:1px solid #161d29}li::before{content:counter(r) '. ';color:#5b6b86}")
          bput(resp, pos, respcap, ".tag{font-size:11px;padding:1px 7px;border-radius:99px;background:#172033;color:#9db4dc;margin-right:4px}")
          bput(resp, pos, respcap, ".tag.primary{background:#0f2a1c;color:#6ee7a8}.title{color:#eef2f8}.url{color:#5b6b86;font-size:12px;word-break:break-all}.star{color:#fbbf24}")
          bput(resp, pos, respcap, "</style></head><body><h1>NISHI SEARCH &mdash; bits-up sovereign</h1>")
          bput(resp, pos, respcap, "<form action='/search' method='get'><input name='q' placeholder='Search&hellip;' value='")
208
          // extract query (if any)
          let qbuf: *u8 = sys_mmap(1024)
          var qlen: i64 = 0
          if rc == NXS_OK { qlen = nx_extract_query(req, po[0], pl[0], qbuf, 1024) }
          // echo query into the input value (already plain text)
          var qe: i64 = 0
          while qe < qlen { bputc(resp, pos, respcap, qbuf[qe] as i64); qe = qe + 1 }
          bput(resp, pos, respcap, "' autofocus><button>Search</button></form>")
217
          if qlen > 0 {
              // rank the corpus for this query
              let tp: **u8 = sys_mmap(8 * 8) as **u8
              let tl: *i64 = sys_mmap(8 * 8) as *i64
              let nterms: i64 = nx_split_terms(qbuf, qlen, tp, tl, 8)
              let order: *i64 = sys_mmap(qn * 8) as *i64
              nx_rank_fused(curl, ctit, ctit, ctlen, qn, tp, tl, nterms, order)
              bput(resp, pos, respcap, "<p class='sub'>ranked by tier prior + BM25 content, fused (RRF) &middot; primary-source-first</p><ol>")
              var shown: i64 = 0
              var i: i64 = 0
              while i < qn {
                  if shown < 10 {
                      let d: i64 = order[i]
                      // only show docs that actually contain a query term
                      var matched: i64 = 0
                      var t: i64 = 0
                      while t < nterms {
                          if nx_bm25_tf(ctit[d], ctlen[d], nx_inv_hash_bytes_lower(tp[t], tl[t])) > 0 { matched = 1 }
                          t = t + 1
                      }
                      if matched == 1 {
                          nx_tier_classify(curl[d], ctit[d], v)
                          bput(resp, pos, respcap, "<li><span class='tag'>")
                          bput(resp, pos, respcap, nx_facet_name(cfac[d]))
                          bput(resp, pos, respcap, "</span>")
                          if v.is_primary == 1 { bput(resp, pos, respcap, "<span class='tag primary'>"); bput(resp, pos, respcap, nx_tier_name(v.tier)); bput(resp, pos, respcap, " <span class='star'>&#9733;</span></span>") }
                          else { bput(resp, pos, respcap, "<span class='tag'>"); bput(resp, pos, respcap, nx_tier_name(v.tier)); bput(resp, pos, respcap, "</span>") }
                          bput(resp, pos, respcap, " <span class='title'>"); bput(resp, pos, respcap, ctit[d]); bput(resp, pos, respcap, "</span>")
                          bput(resp, pos, respcap, "<div class='url'>"); bput(resp, pos, respcap, curl[d]); bput(resp, pos, respcap, "</div></li>")
                          shown = shown + 1
                      }
                  }
                  i = i + 1
              }
              if shown == 0 { bput(resp, pos, respcap, "<li class='sub'>no matches in the indexed corpus</li>") }
              bput(resp, pos, respcap, "</ol>")
          } else {
              bput(resp, pos, respcap, "<p class='sub'>try: Diora Baird &middot; wedding &middot; interview &middot; ")
              bput(resp, pos, respcap, "indexed corpus: "); bput_i64(resp, pos, respcap, qn); bput(resp, pos, respcap, " docs (live crawl fills this in the networked env)</p>")
          }
          bput(resp, pos, respcap, "</body></html>")
          nx_http_server_send_response(cfd, resp, pos[0])
      }
  }
  return 0
263}
nx_search_server.nx source

`nx_search_server.nx` source
