code wiki / _hdl_build / nx_docportal_search_serve.nx
nx_docportal_search_serve.nx source
↩ module page · 1035 lines · 57794 B
1// nx_docportal_search_serve.nx -- R2: the PUBLIC /search serve handler over the SOVEREIGN seg_store search.
2// bytes-in -> bytes-out (no socket -- the gate drives it directly, like dad_handle). Parses GET /search?q=<query>,
3// runs dss_search over the domain's PUBLIC seg_store shard (NO tsv), reads each hit's text (ss_hget) and renders a
4// branded results page with Content-Length (sites.elf's buffered reverse proxy needs the length). A stale/old-
5// format shard (-2) degrades to "0 results", never a 500. license_tier: ORIGINAL
6import "nx_docportal_search_seg.nx"
7
8// Consts for the zero-alloc numeric emitter + timespec buffer; MUST precede their first readers.
9const DSV_ASCII_0: i64 = 48
10const DSV_DEC: i64 = 10
11const DSV_TSBUF: i64 = 16
12
13const DSV_MAGIC_1000000: i64 = 1000000
14const DSV_MAGIC_1024: i64 = 1024
15const DSV_MAGIC_80000: i64 = 80000
16
17const DSV_OUTCAP: i64 = 262144
18const DSV_BODYCAP: i64 = 786432 // uniform per-request body mmap size (>= the largest body 600000); munmap'd every request to kill the leak
19const DSV_MAXR: i64 = 30 // results per page (20->30 2026-07-24: fuller SERPs; the shrunk tf-scan cap keeps it fast)
20const DSV_SNIP: i64 = 240 // chars of a hit's text to show
21
22func dsv_slen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
23func dsv_cat(out: *u8, o: i64, s: *u8) -> i64 { var i: i64 = 0; while s[i] != (0 as u8) { out[o] = s[i]; o = o + 1; i = i + 1 } return o }
24func dsv_catn(out: *u8, o: i64, v: i64) -> i64 {
25 if v == 0 { out[o] = 48 as u8; return o + 1 }
26 var m: i64 = v
27 if m < 0 { out[o] = 45 as u8; o = o + 1; m = 0 - m }
28 // MSB-FIRST: zero allocation (2026-07-31, debt 1785516350). Byte-identical output.
29 var pw: i64 = 1
30 while m / pw >= DSV_DEC { pw = pw * DSV_DEC }
31 while pw > 0 { out[o] = (DSV_ASCII_0 + ((m / pw) % DSV_DEC)) as u8; o = o + 1; pw = pw / DSV_DEC }
32 return o
33}
34// HTML-escape src[0..n) into out (defensive at the boundary: query echo + doc text are untrusted)
35func dsv_esc(out: *u8, o: i64, src: *u8, n: i64) -> i64 {
36 var i: i64 = 0
37 while i < n {
38 let c: u8 = src[i]
39 if c == (60 as u8) { o = dsv_cat(out, o, "<" as *u8) }
40 else { if c == (62 as u8) { o = dsv_cat(out, o, ">" as *u8) }
41 else { if c == (38 as u8) { o = dsv_cat(out, o, "&" as *u8) }
42 else { if c == (34 as u8) { o = dsv_cat(out, o, """ as *u8) }
43 else { out[o] = c; o = o + 1 } } } }
44 i = i + 1
45 }
46 return o
47}
48func dsv_hexval(c: u8) -> i64 {
49 if c >= (48 as u8) { if c <= (57 as u8) { return (c - (48 as u8)) as i64 } }
50 if c >= (97 as u8) { if c <= (102 as u8) { return ((c - (97 as u8)) as i64) + 10 } }
51 if c >= (65 as u8) { if c <= (70 as u8) { return ((c - (65 as u8)) as i64) + 10 } }
52 return 0 - 1
53}
54// url-decode src[0..n) -> out (cap outcap). %XX -> byte, '+' -> space. Returns out length.
55func dsv_urldecode(src: *u8, n: i64, out: *u8, outcap: i64) -> i64 {
56 var i: i64 = 0
57 var o: i64 = 0
58 while i < n {
59 if o >= outcap { return o }
60 let c: u8 = src[i]
61 if c == (37 as u8) {
62 var handled: i64 = 0
63 if i + 2 < n {
64 let hh: i64 = dsv_hexval(src[i + 1])
65 let ll: i64 = dsv_hexval(src[i + 2])
66 if hh >= 0 { if ll >= 0 { out[o] = (hh * 16 + ll) as u8; o = o + 1; i = i + 3; handled = 1 } }
67 }
68 if handled == 0 { out[o] = c; o = o + 1; i = i + 1 }
69 } else {
70 if c == (43 as u8) { out[o] = 32 as u8 } else { out[o] = c }
71 o = o + 1; i = i + 1
72 }
73 }
74 return o
75}
76// extract query param `name` (nlen chars) from the request line "M <url>?a=b&name=val HTTP/1.1". urldecoded -> out.
77// Returns value length, or 0 if absent.
78func dsv_qparam(req: *u8, req_n: i64, name: *u8, nlen: i64, out: *u8, outcap: i64) -> i64 {
79 var us: i64 = 0 - 1
80 var ue: i64 = req_n
81 var sp: i64 = 0
82 var i: i64 = 0
83 while i < req_n {
84 if req[i] == (32 as u8) {
85 if sp == 0 { us = i + 1; sp = 1 } else { if sp == 1 { ue = i; sp = 2 } }
86 }
87 i = i + 1
88 }
89 if us < 0 { return 0 }
90 var qs: i64 = 0 - 1
91 i = us
92 while i < ue { if req[i] == (63 as u8) { if qs < 0 { qs = i + 1 } } i = i + 1 }
93 if qs < 0 { return 0 }
94 var p: i64 = qs
95 while p < ue {
96 var m: i64 = 1
97 var x: i64 = 0
98 while x < nlen { if (p + x) >= ue { m = 0 } else { if req[p + x] != name[x] { m = 0 } } x = x + 1 }
99 if m == 1 { if (p + nlen) < ue { if req[p + nlen] == (61 as u8) {
100 let vs: i64 = p + nlen + 1
101 var vend: i64 = 0 - 1
102 var ve: i64 = vs
103 while ve < ue { if req[ve] == (38 as u8) { if vend < 0 { vend = ve } } ve = ve + 1 }
104 if vend < 0 { vend = ue }
105 return dsv_urldecode(((req as i64) + vs) as *u8, vend - vs, out, outcap)
106 } } }
107 var adv: i64 = 0 - 1
108 var y: i64 = p
109 while y < ue { if req[y] == (38 as u8) { if adv < 0 { adv = y + 1 } } y = y + 1 }
110 if adv < 0 { p = ue } else { p = adv }
111 }
112 return 0
113}
114// dsv_host: extract the Host header value (the domain) from an HTTP request -> the search shard is dp-<domain>-pub-.
115// Port stripped; empty if absent (search then finds no shard -> graceful "0 results"). Matches "Host:" and "host:".
116func dsv_host(req: *u8, req_n: i64, out: *u8, cap: i64) -> i64 {
117 var i: i64 = 0
118 var hs: i64 = 0 - 1
119 while i + 5 < req_n {
120 if req[i] == (10 as u8) {
121 var hmatch: i64 = 0
122 if req[i + 1] == (72 as u8) { hmatch = 1 }
123 if req[i + 1] == (104 as u8) { hmatch = 1 }
124 if hmatch == 1 { if req[i + 2] == (111 as u8) { if req[i + 3] == (115 as u8) { if req[i + 4] == (116 as u8) { if req[i + 5] == (58 as u8) { hs = i + 6 } } } } }
125 }
126 i = i + 1
127 }
128 if hs < 0 { out[0] = 0 as u8; return 0 }
129 var p: i64 = hs
130 var g1: i64 = 1
131 while g1 == 1 { if p < req_n { if req[p] == (32 as u8) { p = p + 1 } else { g1 = 0 } } else { g1 = 0 } }
132 var o: i64 = 0
133 var g2: i64 = 1
134 while g2 == 1 {
135 if p >= req_n { g2 = 0 } else {
136 let c: u8 = req[p]
137 if c == (13 as u8) { g2 = 0 } else { if c == (10 as u8) { g2 = 0 } else { if c == (58 as u8) { g2 = 0 } else {
138 if o < cap { out[o] = c; o = o + 1 }
139 p = p + 1
140 } } }
141 }
142 }
143 out[o] = 0 as u8
144 return o
145}
146// tokenize the query with the INDEX'S OWN tokenizer (mirrors dss_search's inline block) so highlight
147// decisions align exactly with what matched. termstore = 16x64 bytes; returns nterms.
148func dsv_tokq(q: *u8, qn: i64, termstore: *u8, termptrs: *i64, tbl: *u8) -> i64 {
149 var nterms: i64 = 0
150 let qpos: *i64 = sys_mmap(16) as *i64
151 qpos[0] = 0
152 let tb: *u8 = sys_mmap(64)
153 var qgo: i64 = 1
154 while qgo == 1 {
155 let l: i64 = ss_tok_next2(q, qn, qpos, tb, tbl)
156 if l < 0 { qgo = 0 } else {
157 if nterms < 16 {
158 let dst: *u8 = (termstore as i64 + nterms * 64) as *u8
159 var i: i64 = 0
160 while tb[i] != (0 as u8) { dst[i] = tb[i]; i = i + 1 }
161 dst[i] = 0 as u8
162 termptrs[nterms] = dst as i64
163 nterms = nterms + 1
164 }
165 }
166 }
167 return nterms
168}
169// escape src[0..n) into out while <b>-wrapping every token that matches a query term. Own scanner over the
170// SAME classifier table as the index tokenizer (lowercase alnum runs, len>=2, 32-char compare cap), with
171// explicit run offsets -- so highlights == matches and the raw bytes still pass through dsv_esc (XSS-safe).
172func dsv_hl(out: *u8, o: i64, src: *u8, n: i64, termptrs: *i64, nterms: i64, tbl: *u8) -> i64 {
173 let tok: *u8 = sys_mmap(40)
174 var i: i64 = 0
175 while i < n {
176 let m: i64 = tbl[src[i]] as i64
177 if m == 0 {
178 o = dsv_esc(out, o, ((src as i64) + i) as *u8, 1)
179 i = i + 1
180 } else {
181 var e: i64 = i
182 var l: i64 = 0
183 var run: i64 = 1
184 while run == 1 {
185 if e >= n { run = 0 } else {
186 let c: i64 = tbl[src[e]] as i64
187 if c == 0 { run = 0 } else {
188 if l < 32 { tok[l] = c as u8; l = l + 1 }
189 e = e + 1
190 }
191 }
192 }
193 tok[l] = 0 as u8
194 var hit: i64 = 0
195 if l >= 2 {
196 var t: i64 = 0
197 while t < nterms { if dss_streq(tok, termptrs[t] as *u8) == 1 { hit = 1 } t = t + 1 }
198 }
199 if hit == 1 { o = dsv_cat(out, o, "<b>" as *u8) }
200 o = dsv_esc(out, o, ((src as i64) + i) as *u8, e - i)
201 if hit == 1 { o = dsv_cat(out, o, "</b>" as *u8) }
202 i = e
203 }
204 }
205 return o
206}
207// url-encode src[0..n) -> out (alnum passthrough, else %XX) so a decoded query can ride a result href.
208func dsv_urlenc(out: *u8, o: i64, src: *u8, n: i64) -> i64 {
209 let hex: *u8 = "0123456789ABCDEF" as *u8
210 var i: i64 = 0
211 while i < n {
212 let c: i64 = src[i] as i64
213 var plain: i64 = 0
214 if c >= 48 { if c <= 57 { plain = 1 } }
215 if c >= 97 { if c <= 122 { plain = 1 } }
216 if c >= 65 { if c <= 90 { plain = 1 } }
217 if plain == 1 { out[o] = c as u8; o = o + 1 }
218 else {
219 out[o] = 37 as u8
220 out[o + 1] = hex[(c >> 4) & 15]
221 out[o + 2] = hex[c & 15]
222 o = o + 3
223 }
224 i = i + 1
225 }
226 return o
227}
228// build the source-url key "url:<cid>" -- an ingested SITE PAGE / crawled web page carries the page's real
229// location in this row, so its result links to the PAGE itself instead of the /doc text view.
230func dsv_mkurlkey(cid: i64, out: *u8) -> i64 {
231 out[0] = 117 as u8; out[1] = 114 as u8; out[2] = 108 as u8; out[3] = 58 as u8 // "url:"
232 var o: i64 = 4
233 if cid == 0 { out[o] = 48 as u8; o = o + 1; out[o] = 0 as u8; return o }
234 let t: *u8 = sys_mmap(24)
235 var k: i64 = 0
236 var m: i64 = cid
237 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
238 var j: i64 = 0
239 while j < k { out[o] = t[k - 1 - j]; o = o + 1; j = j + 1 }
240 out[o] = 0 as u8
241 return o
242}
243// render one result: linked title + snippet, query terms <b>-highlighted. Link target: the doc's url:<cid>
244// row when present (a real page), else the /doc?cid text view (q + web scope ride the /doc link).
245func dsv_result(body: *u8, b: i64, txt: *u8, tn: i64, cid: i64, q: *u8, qn: i64, termptrs: *i64, nterms: i64, tbl: *u8, url: *u8, ul: i64, webscope: i64) -> i64 {
246 // ---- clean TITLE: the first non-empty LINE (a real <title>/H1), not the first-N-bytes of nav chrome.
247 // Skip leading whitespace/newlines, then take up to the next newline or 72 bytes. ----
248 var ts: i64 = 0
249 var g1: i64 = 1
250 while g1 == 1 {
251 if ts >= tn { g1 = 0 } else {
252 let c: i64 = txt[ts] as i64
253 if c == 32 { ts = ts + 1 } else { if c == 10 { ts = ts + 1 } else { if c == 13 { ts = ts + 1 } else { if c == 9 { ts = ts + 1 } else { g1 = 0 } } } }
254 }
255 }
256 var te: i64 = ts
257 var g2: i64 = 1
258 while g2 == 1 {
259 if te >= tn { g2 = 0 } else { if te - ts >= 72 { g2 = 0 } else {
260 let c: i64 = txt[te] as i64
261 if c == 10 { g2 = 0 } else { if c == 13 { g2 = 0 } else { te = te + 1 } }
262 } }
263 }
264 if te <= ts { ts = 0; te = tn; if te > 56 { te = 56 } }
265 // semantic result: <article><h3><a> -- real structure, query terms bolded in the title
266 var o: i64 = dsv_cat(body, b, "<article class=r><h3 class=t><a href=\"" as *u8)
267 if ul > 0 {
268 o = dsv_esc(body, o, url, ul)
269 } else {
270 o = dsv_cat(body, o, "/doc?cid=" as *u8)
271 o = dsv_catn(body, o, cid)
272 if qn > 0 { o = dsv_cat(body, o, "&q=" as *u8); o = dsv_urlenc(body, o, q, qn) }
273 o = dsv_scope_link(body, o, webscope)
274 }
275 o = dsv_cat(body, o, "\">" as *u8)
276 o = dsv_hl(body, o, ((txt as i64) + ts) as *u8, te - ts, termptrs, nterms, tbl)
277 o = dsv_cat(body, o, "</a></h3>" as *u8)
278 if ul > 0 {
279 o = dsv_cat(body, o, "<div class=u>" as *u8)
280 o = dsv_esc(body, o, url, ul)
281 o = dsv_cat(body, o, "</div>" as *u8)
282 }
283 // ---- query-biased SNIPPET (the SOTA fix): show the best query-COVERING sentence (Google/Mojeek style),
284 // NOT the page's leading nav chrome. Fall back to content just past the title only when the query isn't
285 // found in prose. dsv_best_sentence already filters newline-dense nav blobs, so snippets read like content. ----
286 o = dsv_cat(body, o, "<div class=s>" as *u8)
287 var scanlen: i64 = tn
288 if scanlen > 8000 { scanlen = 8000 }
289 let aoff: *i64 = sys_mmap(16) as *i64
290 let alen: *i64 = sys_mmap(16) as *i64
291 let am: i64 = dsv_best_sentence(txt, scanlen, termptrs, nterms, tbl, aoff, alen)
292 if am > 0 {
293 o = dsv_hl(body, o, ((txt as i64) + aoff[0]) as *u8, alen[0], termptrs, nterms, tbl)
294 } else {
295 var ss: i64 = te
296 if ss >= tn { ss = 0 }
297 var sl: i64 = tn - ss
298 if sl > DSV_SNIP { sl = DSV_SNIP }
299 o = dsv_hl(body, o, ((txt as i64) + ss) as *u8, sl, termptrs, nterms, tbl)
300 }
301 o = dsv_cat(body, o, "</div></article>" as *u8)
302 return o
303}
304// shared page style (search + doc views render as one system). MOBILE-FIRST by measured markers:
305// 16px inputs (iOS never auto-zooms), >=44px touch targets (button + tabs), text-size-adjust locked,
306// prefers-color-scheme dark palette, overflow-wrap on urls -- the serve gate asserts each marker.
307func dsv_style(body: *u8, b: i64) -> i64 {
308 var o: i64 = dsv_cat(body, b, "<style>html{-webkit-text-size-adjust:100%;text-size-adjust:100%}body{font-family:-apple-system,Segoe UI,sans-serif;max-width:680px;margin:3vh auto;padding:0 16px;color:#1c1c1e}header{margin:0 0 1rem}.brand{font-size:1.15rem;font-weight:700;color:#1c1c1e;text-decoration:none;letter-spacing:-.02em}.brand span{color:#0a6}form{display:flex;gap:8px}input[name=q]{flex:1;min-width:0;padding:12px;border:1px solid #ccc;border-radius:10px;font-size:16px}button{min-height:44px;padding:10px 20px;border:0;border-radius:10px;background:#0a6;color:#fff;font-size:16px}.meta{color:#666;font-size:.9rem;margin:1rem 0}.meta a{color:#0a6;text-decoration:none}.tabs{margin:.6rem 0 0}.tabs a{display:inline-block;min-height:44px;line-height:24px;color:#666;text-decoration:none;font-size:.95rem;padding:10px 14px;border-radius:8px}.tabs a.on{background:#e6f6ef;color:#064;font-weight:600}.ans{border:1px solid #bfe8d4;background:#f3fbf7;border-radius:12px;padding:14px 16px;margin:0 0 .4rem}.anstxt{font-size:1.02rem;line-height:1.5}.anssrc{color:#666;font-size:.8rem;margin-top:6px}.r{padding:.9rem 0;border-top:1px solid #eee;margin:0}.t{font-weight:600;font-size:1rem;margin:0}.t a{color:inherit;text-decoration:none}.t a:hover{text-decoration:underline;color:#0a6}.u{color:#0a6;font-size:.82rem;overflow-wrap:anywhere}.s{color:#444;font-size:.94rem}b{background:#e6f6ef}.doc{white-space:pre-wrap;line-height:1.55;overflow-wrap:anywhere}.pager{padding:1rem 0;border-top:1px solid #eee}.pager a{display:inline-block;min-height:44px;line-height:24px;padding:10px 14px;color:#0a6;text-decoration:none;font-weight:600}.credo{color:#8e8e93;font-size:.78rem;padding:1.4rem 0 2rem;border-top:1px solid #eee;margin-top:1rem}" as *u8)
309 o = dsv_cat(body, o, "@media (prefers-color-scheme:dark){body{background:#111214;color:#e8e8ea}.brand{color:#e8e8ea}input[name=q]{background:#1c1c1e;color:#e8e8ea;border-color:#3a3a3c}.s{color:#a8a8ad}.meta{color:#8e8e93}.r,.pager,.credo{border-top-color:#2c2c2e}b{background:#0f3d2e;color:#b7f0d4}.tabs a{color:#a8a8ad}.tabs a.on{background:#0f3d2e;color:#7fd4ae}.t a:hover{color:#3fbf8f}.u{color:#3fbf8f}.meta a{color:#3fbf8f}.ans{background:#12241c;border-color:#1d4a36}.anssrc{color:#8e8e93}.pager a{color:#3fbf8f}}</style>" as *u8)
310 return o
311}
312// JSON string escaper (API-first boundary: title/snippet/text are untrusted bytes -> \" \\ and controls)
313func dsv_jesc(out: *u8, o: i64, src: *u8, n: i64) -> i64 {
314 let hex: *u8 = "0123456789abcdef" as *u8
315 var i: i64 = 0
316 while i < n {
317 let c: i64 = src[i] as i64
318 if c == 34 { o = dsv_cat(out, o, "\\\"" as *u8) }
319 else { if c == 92 { o = dsv_cat(out, o, "\\\\" as *u8) }
320 else { if c == 10 { o = dsv_cat(out, o, "\\n" as *u8) }
321 else { if c == 13 { o = dsv_cat(out, o, "\\r" as *u8) }
322 else { if c == 9 { o = dsv_cat(out, o, "\\t" as *u8) }
323 else { if c < 32 {
324 o = dsv_cat(out, o, "\\u00" as *u8)
325 out[o] = hex[(c >> 4) & 15]; o = o + 1
326 out[o] = hex[c & 15]; o = o + 1
327 } else { out[o] = c as u8; o = o + 1 } } } } } }
328 i = i + 1
329 }
330 return o
331}
332// wrap a finished JSON body in a response with CORS (a PUBLIC read API any site/app may call)
333// wrap a JSON body + FREE it. EVERY per-request body is mmap'd at DSV_BODYCAP (uniform) so munmap of that
334// size is always exact. This eliminates the ~256-586KB/request leak that forced the daemon to recycle every
335// 5000 requests -- the ~15s respawn gap was the "takes forever" the operator hit (2026-07-03). With the leak
336// gone the daemon runs a high budget without growing RSS. rule 16/21.
337func dsv_respond_json(out: *u8, status: *u8, body: *u8, b: i64) -> i64 {
338 var o: i64 = 0
339 o = dsv_cat(out, o, "HTTP/1.1 " as *u8)
340 o = dsv_cat(out, o, status)
341 o = dsv_cat(out, o, "\r\nContent-Type: application/json; charset=utf-8\r\nAccess-Control-Allow-Origin: *\r\nCache-Control: no-store\r\nConnection: close\r\nContent-Length: " as *u8)
342 o = dsv_catn(out, o, b)
343 o = dsv_cat(out, o, "\r\n\r\n" as *u8)
344 var z: i64 = 0
345 while z < b { out[o] = body[z]; o = o + 1; z = z + 1 }
346 sys_munmap(body, DSV_BODYCAP)
347 return o
348}
349// wrap an HTML body + FREE it (Content-Length for the buffered proxy).
350func dsv_respond(out: *u8, status: *u8, body: *u8, b: i64) -> i64 {
351 var o: i64 = 0
352 o = dsv_cat(out, o, "HTTP/1.1 " as *u8)
353 o = dsv_cat(out, o, status)
354 o = dsv_cat(out, o, "\r\nContent-Type: text/html; charset=utf-8\r\nConnection: close\r\nContent-Length: " as *u8)
355 o = dsv_catn(out, o, b)
356 o = dsv_cat(out, o, "\r\n\r\n" as *u8)
357 var z: i64 = 0
358 while z < b { out[o] = body[z]; o = o + 1; z = z + 1 }
359 sys_munmap(body, DSV_BODYCAP)
360 return o
361}
362// scope resolution: THREE scopes (the operator's model 2026-07-04) -> 0 SITE (this Host domain's own shard),
363// 1 WEB (the broad crawled-everything corpus "web"), 2 TRUSTED (client-flagged good resources "trusted").
364// effdom = the resolved shard name. The return value drives the tabs + all scope-preserving links.
365func dsv_scope(req: *u8, req_n: i64, domain: *u8, effdom: *u8) -> i64 {
366 let sc: *u8 = sys_mmap(32)
367 let sn: i64 = dsv_qparam(req, req_n, "scope" as *u8, 5, sc, 15)
368 var scope: i64 = 1 // DEFAULT = WEB (2026-07-24): nishifamily.com/search IS a web search engine; the site
369 // corpus + trusted are opt-in tabs. Was 0/site -> users landed on a tiny site-search and
370 // never saw the CC/entity web index or any ranking work. Explicit ?scope=site selects site.
371 if sn == 4 { if sc[0] == (115 as u8) { if sc[1] == (105 as u8) { if sc[2] == (116 as u8) { if sc[3] == (101 as u8) { scope = 0 } } } } }
372 if sn == 3 { if sc[0] == (119 as u8) { if sc[1] == (101 as u8) { if sc[2] == (98 as u8) { scope = 1 } } } }
373 if sn == 7 {
374 if sc[0] == (116 as u8) { if sc[1] == (114 as u8) { if sc[2] == (117 as u8) { if sc[3] == (115 as u8) { if sc[4] == (116 as u8) { if sc[5] == (101 as u8) { if sc[6] == (100 as u8) { scope = 2 } } } } } } }
375 }
376 if scope == 1 {
377 effdom[0] = 119 as u8; effdom[1] = 101 as u8; effdom[2] = 98 as u8; effdom[3] = 0 as u8 // "web"
378 return 1
379 }
380 if scope == 2 {
381 let tn0: *u8 = "trusted" as *u8
382 var t: i64 = 0
383 while tn0[t] != (0 as u8) { effdom[t] = tn0[t]; t = t + 1 }
384 effdom[t] = 0 as u8
385 return 2
386 }
387 var i: i64 = 0
388 while domain[i] != (0 as u8) { effdom[i] = domain[i]; i = i + 1 }
389 effdom[i] = 0 as u8
390 return 0
391}
392// append the scope-preserving query param for a link (nothing for site, &scope=web, &scope=trusted)
393func dsv_scope_link(body: *u8, b: i64, scope: i64) -> i64 {
394 if scope == 1 { return dsv_cat(body, b, "&scope=web" as *u8) }
395 if scope == 2 { return dsv_cat(body, b, "&scope=trusted" as *u8) }
396 return dsv_cat(body, b, "&scope=site" as *u8) // web is the default now -> site links MUST be explicit
397}
398// the JSON scope name (site/web/trusted)
399func dsv_scope_name(body: *u8, b: i64, scope: i64) -> i64 {
400 if scope == 1 { return dsv_cat(body, b, "web" as *u8) }
401 if scope == 2 { return dsv_cat(body, b, "trusted" as *u8) }
402 return dsv_cat(body, b, "site" as *u8)
403}
404// the scope tabs: [This site] [Trusted] [Web] -- each preserves the current query, active one highlighted
405func dsv_tabs(body: *u8, b: i64, q: *u8, qn: i64, scope: i64) -> i64 {
406 var o: i64 = dsv_cat(body, b, "<p class=tabs><a " as *u8)
407 if scope == 1 { o = dsv_cat(body, o, "class=on " as *u8) }
408 o = dsv_cat(body, o, "href=\"/search?scope=web" as *u8)
409 if qn > 0 { o = dsv_cat(body, o, "&q=" as *u8); o = dsv_urlenc(body, o, q, qn) }
410 o = dsv_cat(body, o, "\">Web</a> <a " as *u8)
411 if scope == 0 { o = dsv_cat(body, o, "class=on " as *u8) }
412 o = dsv_cat(body, o, "href=\"/search?scope=site" as *u8)
413 if qn > 0 { o = dsv_cat(body, o, "&q=" as *u8); o = dsv_urlenc(body, o, q, qn) }
414 o = dsv_cat(body, o, "\">This site</a> <a " as *u8)
415 if scope == 2 { o = dsv_cat(body, o, "class=on " as *u8) }
416 o = dsv_cat(body, o, "href=\"/search?scope=trusted" as *u8)
417 if qn > 0 { o = dsv_cat(body, o, "&q=" as *u8); o = dsv_urlenc(body, o, q, qn) }
418 o = dsv_cat(body, o, "\">Trusted</a></p>" as *u8)
419 return o
420}
421// monotonic microseconds (integer; the SERP shows real measured query latency like the big engines --
422// except ours is TRUE per-request wall time, not a cached estimate)
423func dsv_now_us() -> i64 {
424 let ts: *i64 = sys_mmap(DSV_TSBUF) as *i64
425 sys_clock_gettime_mono(ts)
426 // read BOTH fields out before releasing -- computing after the munmap would be a use-after-free.
427 let sec: i64 = ts[0]
428 let nsec: i64 = ts[1]
429 sys_munmap(ts as *u8, DSV_TSBUF)
430 return sec * DSV_MAGIC_1000000 + nsec / 1000
431}
432// FEATURED ANSWER (our own featured-snippet, extracted not asserted): scan the text's sentences
433// (. ! ? boundaries; 40..320 bytes) and return the one matching the MOST distinct query terms via the
434// index's own tokenizer. offout/lenout = the winning sentence; returns distinct terms matched (0 = none).
435func dsv_best_sentence(txt: *u8, tn: i64, termptrs: *i64, nterms: i64, tbl: *u8, offout: *i64, lenout: *i64) -> i64 {
436 let tok: *u8 = sys_mmap(40)
437 var best: i64 = 0
438 offout[0] = 0
439 lenout[0] = 0
440 var s: i64 = 0
441 while s < tn {
442 // sentence end: next . ! ? or hard cap
443 var e: i64 = s
444 var run: i64 = 1
445 while run == 1 {
446 if e >= tn { run = 0 } else {
447 let c: i64 = txt[e] as i64
448 if c == 46 { run = 0 } else { if c == 33 { run = 0 } else { if c == 63 { run = 0 } else {
449 if e - s >= 320 { run = 0 } else { e = e + 1 }
450 } } }
451 }
452 }
453 let slen: i64 = e - s
454 // PROSE filter: extracted-page nav chrome ("Jump to content\nMain menu\n...") arrives as newline-
455 // dense blobs between real periods -- a genuine sentence carries at most a stray wrap. >2 newlines
456 // = not prose, never an answer (the okapi live catch, 2026-07-03).
457 var nlcount: i64 = 0
458 var nz: i64 = s
459 while nz < e { if txt[nz] == (10 as u8) { nlcount = nlcount + 1 } nz = nz + 1 }
460 if slen >= 40 { if nlcount <= 2 {
461 // count distinct query terms present in [s, e)
462 var matched: i64 = 0
463 var t: i64 = 0
464 while t < nterms {
465 var hit: i64 = 0
466 var i: i64 = s
467 while i < e {
468 let m: i64 = tbl[txt[i]] as i64
469 if m == 0 { i = i + 1 } else {
470 var e2: i64 = i
471 var l: i64 = 0
472 var r2: i64 = 1
473 while r2 == 1 {
474 if e2 >= e { r2 = 0 } else {
475 let c2: i64 = tbl[txt[e2]] as i64
476 if c2 == 0 { r2 = 0 } else {
477 if l < 32 { tok[l] = c2 as u8; l = l + 1 }
478 e2 = e2 + 1
479 }
480 }
481 }
482 tok[l] = 0 as u8
483 if l >= 2 { if dss_streq(tok, termptrs[t] as *u8) == 1 { hit = 1; i = e } }
484 if i < e { i = e2 }
485 }
486 }
487 matched = matched + hit
488 t = t + 1
489 }
490 if matched > best {
491 best = matched
492 offout[0] = s
493 lenout[0] = slen
494 }
495 } }
496 s = e + 1
497 }
498 return best
499}
500// THE SERVE: (domain, request bytes) -> branded HTTP results page bytes into out. Returns out length.
501func dss_serve(domain: *u8, req: *u8, req_n: i64, out: *u8) -> i64 {
502 let t0: i64 = dsv_now_us()
503 let q: *u8 = sys_mmap(DSV_MAGIC_1024)
504 let qn: i64 = dsv_qparam(req, req_n, "q" as *u8, 1, q, 1023)
505 let effdom: *u8 = sys_mmap(256)
506 let webscope: i64 = dsv_scope(req, req_n, domain, effdom)
507 // page param (digits; junk -> page 0)
508 let pbuf: *u8 = sys_mmap(32)
509 let pbn: i64 = dsv_qparam(req, req_n, "p" as *u8, 1, pbuf, 15)
510 let pok: *i64 = sys_mmap(16) as *i64
511 var page: i64 = dsv_atoin(pbuf, pbn, pok)
512 if pok[0] == 0 { page = 0 }
513 if page > 25 { page = 25 }
514 let cids: *i64 = sys_mmap(DSV_MAXR * 8) as *i64
515 let scores: *i64 = sys_mmap(DSV_MAXR * 8) as *i64
516 let totalbox: *i64 = sys_mmap(16) as *i64
517 var nres: i64 = 0
518 if qn > 0 { nres = dss_search_off(effdom, q, qn, cids, scores, DSV_MAXR, page * DSV_MAXR, totalbox) }
519 if nres < 0 { nres = 0 }
520 let total: i64 = totalbox[0]
521 let qus: i64 = dsv_now_us() - t0
522 // query terms + classifier table, once per request: dsv_result highlights with the index's own tokens
523 let tbl: *u8 = sys_mmap(272)
524 ss_tok_table(tbl)
525 let termstore: *u8 = sys_mmap(16 * 64)
526 let termptrs: *i64 = sys_mmap(16 * 8) as *i64
527 var nterms: i64 = 0
528 if qn > 0 { nterms = dsv_tokq(q, qn, termstore, termptrs, tbl) }
529 let body: *u8 = sys_mmap(DSV_BODYCAP)
530 var b: i64 = 0
531 b = dsv_cat(body, b, "<!DOCTYPE html><meta charset=utf-8><meta name=viewport content=\"width=device-width,initial-scale=1\"><title>" as *u8)
532 if qn > 0 { b = dsv_esc(body, b, q, qn); b = dsv_cat(body, b, " — " as *u8) }
533 b = dsv_cat(body, b, "Nishi Search</title>" as *u8)
534 b = dsv_style(body, b)
535 b = dsv_cat(body, b, "<header><a class=brand href=\"/search\">Nishi<span>Search</span></a></header><main>" as *u8)
536 b = dsv_cat(body, b, "<form action=/search method=get role=search>" as *u8)
537 if webscope == 1 { b = dsv_cat(body, b, "<input type=hidden name=scope value=web>" as *u8) }
538 if webscope == 2 { b = dsv_cat(body, b, "<input type=hidden name=scope value=trusted>" as *u8) }
539 b = dsv_cat(body, b, "<input name=q value=\"" as *u8)
540 b = dsv_esc(body, b, q, qn)
541 b = dsv_cat(body, b, "\" placeholder=\"Search\" autofocus aria-label=\"Search query\"><button>Search</button></form>" as *u8)
542 b = dsv_tabs(body, b, q, qn, webscope)
543 if qn == 0 {
544 b = dsv_cat(body, b, "<p class=meta>Enter a search term.</p>" as *u8)
545 } else {
546 b = dsv_cat(body, b, "<p class=meta>" as *u8); b = dsv_catn(body, b, nres); b = dsv_cat(body, b, " result(s) for “" as *u8)
547 b = dsv_esc(body, b, q, qn); b = dsv_cat(body, b, "”" as *u8)
548 if total > nres { b = dsv_cat(body, b, " of " as *u8); b = dsv_catn(body, b, total); b = dsv_cat(body, b, " matched" as *u8) }
549 if page > 0 { b = dsv_cat(body, b, " · page " as *u8); b = dsv_catn(body, b, page + 1) }
550 b = dsv_cat(body, b, " · " as *u8)
551 if qus < 1000 { b = dsv_cat(body, b, "<1" as *u8) } else { b = dsv_catn(body, b, qus / 1000) }
552 b = dsv_cat(body, b, " ms" as *u8)
553 if totalbox[1] == 0 { b = dsv_cat(body, b, " · phrase matched loosely (index upgrading)" as *u8) }
554 b = dsv_cat(body, b, "</p>" as *u8)
555 // ★COVERAGE HONESTY (operator 2026-07-04: "we dont want partial results thinking we return full
556 // results"): the web scope is NISHI'S OWN INDEX (our crawl + Common Crawl), NOT the whole live web.
557 // This line appears on EVERY web result set so N results is never mistaken for "all the web has".
558 if webscope == 1 { b = dsv_cat(body, b, "<p class=meta style=\"color:#666;font-size:.85rem\">Scope: Nishi’s own index (our crawl + Common Crawl) — not the whole live web. Some sites block all automated crawlers (anti-bot walls), so their pages may be absent here even though we never filter lawful content.</p>" as *u8) }
559 // DID-YOU-MEAN: on zero hits, offer the dictionary's closest reading of the query (typo rung)
560 if nres == 0 {
561 let fix: *u8 = sys_mmap(DSV_MAGIC_1024)
562 let fl: i64 = dss_correct(effdom, q, qn, fix, 1023)
563 if fl > 0 {
564 b = dsv_cat(body, b, "<p class=meta>Did you mean <a href=\"/search?q=" as *u8)
565 b = dsv_urlenc(body, b, fix, fl)
566 b = dsv_scope_link(body, b, webscope)
567 b = dsv_cat(body, b, "\"><b>" as *u8)
568 b = dsv_esc(body, b, fix, fl)
569 b = dsv_cat(body, b, "</b></a>?</p>" as *u8)
570 }
571 // HONEST scope empty states: our "web" is a sovereign crawl (a bounded index), NOT the live
572 // internet; "trusted" is the client-curated good-resources set (small by design).
573 if webscope == 1 { b = dsv_cat(body, b, "<p class=meta>The Web tab searches Nishi’s own sovereign crawl — a growing index of pages we’ve fetched, not the whole live internet. Try a broad topic, or search <a href=\"/search?q=" as *u8); b = dsv_urlenc(body, b, q, qn); b = dsv_cat(body, b, "\">this site</a>.</p>" as *u8) }
574 if webscope == 2 { b = dsv_cat(body, b, "<p class=meta>The Trusted tab searches resources clients have flagged as good references. Nothing matched yet — try the <a href=\"/search?scope=web&q=" as *u8); b = dsv_urlenc(body, b, q, qn); b = dsv_cat(body, b, "\">Web</a> tab.</p>" as *u8) }
575 }
576 let prefix: *u8 = sys_mmap(512); dss_prefix(effdom, prefix)
577 let h: *i64 = dss_open_maybe_cached(prefix)
578 let key: *u8 = sys_mmap(64)
579 let ukey: *u8 = sys_mmap(64)
580 let dptr: *i64 = sys_mmap(16) as *i64; let dlen: *i64 = sys_mmap(16) as *i64
581 let uptr: *i64 = sys_mmap(16) as *i64; let ulen2: *i64 = sys_mmap(16) as *i64
582 // host FACET accumulator (the faceted rung): distinct hosts among this page's results
583 let fhosts: *u8 = sys_mmap(256 * 12)
584 let fcnt: *i64 = sys_mmap(8 * 12) as *i64
585 var nfh: i64 = 0
586 let hbuf: *u8 = sys_mmap(256)
587 var i: i64 = 0
588 while i < nres {
589 dss_mkkey(cids[i], key)
590 if (h as i64) != 0 { if ss_hget(h, key, dptr, dlen) == 1 {
591 dsv_mkurlkey(cids[i], ukey)
592 var up: i64 = 0
593 var ul: i64 = 0
594 if ss_hget(h, ukey, uptr, ulen2) == 1 { up = uptr[0]; ul = ulen2[0] }
595 if ul > 0 {
596 let hl9: i64 = dss_url_host(up as *u8, ul, hbuf)
597 if hl9 > 0 {
598 var ff: i64 = 0 - 1
599 var fz: i64 = 0
600 while fz < nfh {
601 let fp9: *u8 = (fhosts as i64 + fz * 256) as *u8
602 var feq: i64 = 1
603 var fx: i64 = 0
604 var fgo: i64 = 1
605 while fgo == 1 { if fp9[fx] != hbuf[fx] { feq = 0; fgo = 0 } else { if hbuf[fx] == (0 as u8) { fgo = 0 } else { fx = fx + 1 } } }
606 if feq == 1 { ff = fz; fz = nfh } else { fz = fz + 1 }
607 }
608 if ff >= 0 { fcnt[ff] = fcnt[ff] + 1 } else { if nfh < 12 {
609 let fd9: *u8 = (fhosts as i64 + nfh * 256) as *u8
610 var fc9: i64 = 0
611 var fgo2: i64 = 1
612 while fgo2 == 1 { fd9[fc9] = hbuf[fc9]; if hbuf[fc9] == (0 as u8) { fgo2 = 0 } else { fc9 = fc9 + 1 } }
613 fcnt[nfh] = 1
614 nfh = nfh + 1
615 } }
616 }
617 }
618 // FEATURED ANSWER on the top hit of page 0: the best sentence, only when it actually
619 // covers the query (all terms, or 2+ of a multi-term query) -- extracted, never asserted
620 if i == 0 { if page == 0 {
621 let aoff: *i64 = sys_mmap(16) as *i64
622 let alen: *i64 = sys_mmap(16) as *i64
623 let am: i64 = dsv_best_sentence(dptr[0] as *u8, dlen[0], termptrs, nterms, tbl, aoff, alen)
624 var show: i64 = 0
625 if am >= nterms { if nterms > 0 { show = 1 } }
626 if nterms >= 3 { if am >= 2 { show = 1 } }
627 if show == 1 {
628 b = dsv_cat(body, b, "<div class=ans><div class=anstxt>" as *u8)
629 b = dsv_hl(body, b, ((dptr[0] as i64) + aoff[0]) as *u8, alen[0], termptrs, nterms, tbl)
630 b = dsv_cat(body, b, "</div><div class=anssrc>from the top result below</div></div>" as *u8)
631 }
632 } }
633 b = dsv_result(body, b, dptr[0] as *u8, dlen[0], cids[i], q, qn, termptrs, nterms, tbl, up as *u8, ul, webscope)
634 } }
635 i = i + 1
636 }
637 // host FACETS (zero-JS): when this page's results span 2+ sites, offer one-click site: narrowing.
638 // Skipped when the query already carries a site: clause (narrowing twice is noise).
639 var hassite: i64 = 0
640 var hz: i64 = 0
641 while hz + 5 <= qn {
642 if q[hz] == (115 as u8) { if q[hz+1] == (105 as u8) { if q[hz+2] == (116 as u8) { if q[hz+3] == (101 as u8) { if q[hz+4] == (58 as u8) { hassite = 1 } } } } }
643 hz = hz + 1
644 }
645 if nfh >= 2 { if hassite == 0 {
646 b = dsv_cat(body, b, "<p class=meta>Sites: " as *u8)
647 // top 4 hosts by count (selection)
648 let fused: *i64 = sys_mmap(8 * 12) as *i64
649 var fu: i64 = 0
650 while fu < nfh { fused[fu] = 0; fu = fu + 1 }
651 var fshown: i64 = 0
652 var frank: i64 = 0
653 while frank < 4 {
654 var fbest: i64 = 0 - 1
655 var fk: i64 = 0
656 while fk < nfh {
657 if fused[fk] == 0 { if fbest < 0 { fbest = fk } else { if fcnt[fk] > fcnt[fbest] { fbest = fk } } }
658 fk = fk + 1
659 }
660 if fbest < 0 { frank = 4 } else {
661 fused[fbest] = 1
662 let fh9: *u8 = (fhosts as i64 + fbest * 256) as *u8
663 if fshown > 0 { b = dsv_cat(body, b, " · " as *u8) }
664 b = dsv_cat(body, b, "<a href=\"/search?q=" as *u8)
665 b = dsv_urlenc(body, b, q, qn)
666 b = dsv_cat(body, b, "+site%3A" as *u8)
667 b = dsv_cat(body, b, fh9)
668 b = dsv_scope_link(body, b, webscope)
669 b = dsv_cat(body, b, "\">" as *u8)
670 b = dsv_cat(body, b, fh9)
671 b = dsv_cat(body, b, "</a> (" as *u8)
672 b = dsv_catn(body, b, fcnt[fbest])
673 b = dsv_cat(body, b, ")" as *u8)
674 fshown = fshown + 1
675 frank = frank + 1
676 }
677 }
678 b = dsv_cat(body, b, "</p>" as *u8)
679 } }
680 // pager: Prev when past page 0; Next while ranked matches remain (total is the truth)
681 if qn > 0 { if total > DSV_MAXR {
682 b = dsv_cat(body, b, "<nav class=pager>" as *u8)
683 if page > 0 {
684 b = dsv_cat(body, b, "<a href=\"/search?q=" as *u8)
685 b = dsv_urlenc(body, b, q, qn)
686 b = dsv_scope_link(body, b, webscope)
687 b = dsv_cat(body, b, "&p=" as *u8); b = dsv_catn(body, b, page - 1)
688 b = dsv_cat(body, b, "\">← Prev</a> " as *u8)
689 }
690 if (page + 1) * DSV_MAXR < total {
691 b = dsv_cat(body, b, "<a href=\"/search?q=" as *u8)
692 b = dsv_urlenc(body, b, q, qn)
693 b = dsv_scope_link(body, b, webscope)
694 b = dsv_cat(body, b, "&p=" as *u8); b = dsv_catn(body, b, page + 1)
695 b = dsv_cat(body, b, "\">Next →</a>" as *u8)
696 }
697 b = dsv_cat(body, b, "</nav>" as *u8)
698 } }
699 }
700 // the credo footer: MEASURED page facts the big engines cannot print (their SERPs ship megabytes of
701 // script and ad/tracker payloads; this page is what it says it is)
702 b = dsv_cat(body, b, "</main><footer class=credo>~" as *u8)
703 b = dsv_catn(body, b, (b + 400 + 1023) / DSV_MAGIC_1024)
704 b = dsv_cat(body, b, " KB · 0 JavaScript · 0 ads · 0 trackers · sovereign nishi search · <a href=\"/api/openapi.json\">API</a></footer></body></html>" as *u8)
705 return dsv_respond(out, "200 OK" as *u8, body, b)
706}
707
708// digits-only parse of s[0..n); okbox[0]=1 iff every byte was a digit and n>0 (defensive: a cid is ONLY digits)
709func dsv_atoin(s: *u8, n: i64, okbox: *i64) -> i64 {
710 okbox[0] = 0
711 if n <= 0 { return 0 }
712 if n > 19 { return 0 }
713 var v: i64 = 0
714 var i: i64 = 0
715 while i < n {
716 let c: i64 = s[i] as i64
717 if c < 48 { return 0 }
718 if c > 57 { return 0 }
719 v = v * 10 + (c - 48)
720 i = i + 1
721 }
722 okbox[0] = 1
723 return v
724}
725// /doc?cid=<cid>[&q=<query>] -- the PUBLIC document view that completes the search loop (result -> readable
726// doc). Serves ONLY the -pub- shard (dss_prefix); the -prv- shard is physically never opened. Visibility is
727// the owner's public/private choice at upload; the SEARCH flag governs indexing consent, not public
728// readability, so /doc serves any public doc by cid. q (if present) re-highlights + links back to results.
729func dsv_doc_serve(domain: *u8, req: *u8, req_n: i64, out: *u8) -> i64 {
730 let tbl: *u8 = sys_mmap(272)
731 ss_tok_table(tbl)
732 let q: *u8 = sys_mmap(DSV_MAGIC_1024)
733 let qn: i64 = dsv_qparam(req, req_n, "q" as *u8, 1, q, 1023)
734 let effdom: *u8 = sys_mmap(256)
735 let webscope: i64 = dsv_scope(req, req_n, domain, effdom)
736 let termstore: *u8 = sys_mmap(16 * 64)
737 let termptrs: *i64 = sys_mmap(16 * 8) as *i64
738 var nterms: i64 = 0
739 if qn > 0 { nterms = dsv_tokq(q, qn, termstore, termptrs, tbl) }
740 let cidbuf: *u8 = sys_mmap(64)
741 let cn: i64 = dsv_qparam(req, req_n, "cid" as *u8, 3, cidbuf, 31)
742 let okbox: *i64 = sys_mmap(16) as *i64
743 let cid: i64 = dsv_atoin(cidbuf, cn, okbox)
744 var found: i64 = 0
745 let dptr: *i64 = sys_mmap(16) as *i64
746 let dlen: *i64 = sys_mmap(16) as *i64
747 if okbox[0] == 1 {
748 let prefix: *u8 = sys_mmap(512); dss_prefix(effdom, prefix)
749 let h: *i64 = dss_open_maybe_cached(prefix)
750 if (h as i64) != 0 {
751 let key: *u8 = sys_mmap(64); dss_mkkey(cid, key)
752 if ss_hget(h, key, dptr, dlen) == 1 { found = 1 }
753 }
754 }
755 let body: *u8 = sys_mmap(DSV_BODYCAP)
756 var b: i64 = 0
757 b = dsv_cat(body, b, "<!DOCTYPE html><meta charset=utf-8><meta name=viewport content=\"width=device-width,initial-scale=1\"><title>Document</title>" as *u8)
758 b = dsv_style(body, b)
759 b = dsv_cat(body, b, "<p class=meta><a href=\"/search" as *u8)
760 if qn > 0 { b = dsv_cat(body, b, "?q=" as *u8); b = dsv_urlenc(body, b, q, qn) }
761 if webscope == 1 {
762 if qn > 0 { b = dsv_cat(body, b, "&scope=web" as *u8) } else { b = dsv_cat(body, b, "?scope=web" as *u8) }
763 }
764 b = dsv_cat(body, b, "\">← " as *u8)
765 if qn > 0 { b = dsv_cat(body, b, "Back to results" as *u8) } else { b = dsv_cat(body, b, "Search" as *u8) }
766 b = dsv_cat(body, b, "</a></p>" as *u8)
767 if found == 0 {
768 b = dsv_cat(body, b, "<div class=t>Document not found</div><p class=s>No public document has that id.</p></body></html>" as *u8)
769 return dsv_respond(out, "404 Not Found" as *u8, body, b)
770 }
771 let txt: *u8 = dptr[0] as *u8
772 var tn: i64 = dlen[0]
773 // bounded BY CONSTRUCTION: html-escape expands up to 6x ("""), and the daemon's out buffer is
774 // 512KB -- 80000 * 6 + page chrome stays safely inside it. Longer docs render truncated, marked.
775 var trunc: i64 = 0
776 if tn > DSV_MAGIC_80000 { tn = DSV_MAGIC_80000; trunc = 1 }
777 // title = the same leading-sentence rule the results page uses
778 var dot: i64 = 0 - 1
779 var i: i64 = 0
780 while i < tn { if i < 80 { if txt[i] == (46 as u8) { if dot < 0 { dot = i } } } i = i + 1 }
781 var tt: i64 = dot
782 if tt < 0 { tt = 56 }
783 if tt > tn { tt = tn }
784 b = dsv_cat(body, b, "<div class=t>" as *u8)
785 b = dsv_hl(body, b, txt, tt, termptrs, nterms, tbl)
786 b = dsv_cat(body, b, "</div><p class=\"s doc\">" as *u8)
787 if tt < tn { b = dsv_hl(body, b, ((txt as i64) + tt) as *u8, tn - tt, termptrs, nterms, tbl) }
788 if trunc == 1 { b = dsv_cat(body, b, " … [truncated]" as *u8) }
789 b = dsv_cat(body, b, "</p></body></html>" as *u8)
790 return dsv_respond(out, "200 OK" as *u8, body, b)
791}
792
793// ================= API-FIRST SURFACE (versioned JSON; the HTML SERP is just one client) =================
794// GET /api/openapi.json -> the machine-readable OpenAPI 3.1 contract for the search API (enterprise-grade:
795// client codegen, Swagger UI, contract testing all consume this). CORS-open like the data endpoints. The
796// spec is generated in code (SSOT = this file) so it can never drift from the handlers. `srv` = the Host
797// so the served spec names the caller's own domain in the servers[] block.
798func dss_api_openapi(srv: *u8, out: *u8) -> i64 {
799 let body: *u8 = sys_mmap(DSV_BODYCAP)
800 var b: i64 = 0
801 b = dsv_cat(body, b, "{\"openapi\":\"3.1.0\",\"info\":{\"title\":\"Nishi Search API\",\"version\":\"1.0.0\",\"description\":\"Sovereign full-text search: a domain's docs + site pages, the nishi library, and the crawled open web. Integer BM25 (b=0.75), stemmed recall, phrase/boolean queries, did-you-mean. Zero third-party dependencies.\",\"x-notes\":\"cid is a STRING: 63-bit content ids exceed JSON 2^53 safe-integer range.\"}," as *u8)
802 b = dsv_cat(body, b, "\"servers\":[{\"url\":\"https://" as *u8)
803 b = dsv_jesc(body, b, srv, dsv_slen(srv))
804 b = dsv_cat(body, b, "\"}]," as *u8)
805 b = dsv_cat(body, b, "\"paths\":{" as *u8)
806 // /api/search
807 b = dsv_cat(body, b, "\"/api/search\":{\"get\":{\"operationId\":\"search\",\"summary\":\"Ranked full-text search\",\"parameters\":[" as *u8)
808 b = dsv_cat(body, b, "{\"name\":\"q\",\"in\":\"query\",\"required\":true,\"schema\":{\"type\":\"string\"},\"description\":\"Query. Supports OR (default), +required, -excluded, \\\"quoted phrase\\\", and site:host field scoping (dot-suffix match); terms are stemmed for recall.\"}," as *u8)
809 b = dsv_cat(body, b, "{\"name\":\"scope\",\"in\":\"query\",\"required\":false,\"schema\":{\"type\":\"string\",\"enum\":[\"web\",\"trusted\"]},\"description\":\"web = the broad crawled open-web corpus; trusted = resources clients flagged as good references; omit = this site's own corpus.\"}," as *u8)
810 b = dsv_cat(body, b, "{\"name\":\"page\",\"in\":\"query\",\"required\":false,\"schema\":{\"type\":\"integer\",\"minimum\":0},\"description\":\"0-based results page, 20 per page.\"}]," as *u8)
811 b = dsv_cat(body, b, "\"responses\":{\"200\":{\"description\":\"Ranked results\",\"content\":{\"application/json\":{\"schema\":{\"$ref\":\"#/components/schemas/SearchResponse\"}}}}}}}," as *u8)
812 // /api/doc
813 b = dsv_cat(body, b, "\"/api/doc\":{\"get\":{\"operationId\":\"getDoc\",\"summary\":\"Fetch a document by content id\",\"parameters\":[" as *u8)
814 b = dsv_cat(body, b, "{\"name\":\"cid\",\"in\":\"query\",\"required\":true,\"schema\":{\"type\":\"string\"},\"description\":\"Decimal content id (string).\"}," as *u8)
815 b = dsv_cat(body, b, "{\"name\":\"scope\",\"in\":\"query\",\"required\":false,\"schema\":{\"type\":\"string\",\"enum\":[\"web\"]}}]," as *u8)
816 b = dsv_cat(body, b, "\"responses\":{\"200\":{\"description\":\"The document\",\"content\":{\"application/json\":{\"schema\":{\"$ref\":\"#/components/schemas/Doc\"}}}},\"400\":{\"description\":\"bad_cid\"},\"404\":{\"description\":\"not_found\"}}}}," as *u8)
817 // /api/suggest
818 b = dsv_cat(body, b, "\"/api/suggest\":{\"get\":{\"operationId\":\"suggest\",\"summary\":\"Autocomplete term completions\",\"parameters\":[" as *u8)
819 b = dsv_cat(body, b, "{\"name\":\"q\",\"in\":\"query\",\"required\":true,\"schema\":{\"type\":\"string\"},\"description\":\"Term prefix.\"}," as *u8)
820 b = dsv_cat(body, b, "{\"name\":\"scope\",\"in\":\"query\",\"required\":false,\"schema\":{\"type\":\"string\",\"enum\":[\"web\"]}}]," as *u8)
821 b = dsv_cat(body, b, "\"responses\":{\"200\":{\"description\":\"Completions\",\"content\":{\"application/json\":{\"schema\":{\"$ref\":\"#/components/schemas/Suggest\"}}}}}}}" as *u8)
822 b = dsv_cat(body, b, "}," as *u8)
823 // components
824 b = dsv_cat(body, b, "\"components\":{\"schemas\":{" as *u8)
825 b = dsv_cat(body, b, "\"Result\":{\"type\":\"object\",\"properties\":{\"cid\":{\"type\":\"string\"},\"score\":{\"type\":\"integer\"},\"title\":{\"type\":\"string\"},\"snippet\":{\"type\":\"string\"},\"doc\":{\"type\":\"string\"},\"url\":{\"type\":\"string\"},\"host\":{\"type\":\"string\"}},\"required\":[\"cid\",\"score\",\"title\"]}," as *u8)
826 b = dsv_cat(body, b, "\"SearchResponse\":{\"type\":\"object\",\"properties\":{\"v\":{\"type\":\"integer\"},\"query\":{\"type\":\"string\"},\"scope\":{\"type\":\"string\"},\"page\":{\"type\":\"integer\"},\"total\":{\"type\":\"integer\"},\"nresults\":{\"type\":\"integer\"},\"did_you_mean\":{\"type\":\"string\"},\"phrase_exact\":{\"type\":\"boolean\"},\"results\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/components/schemas/Result\"}}},\"required\":[\"v\",\"nresults\",\"results\"]}," as *u8)
827 b = dsv_cat(body, b, "\"Doc\":{\"type\":\"object\",\"properties\":{\"v\":{\"type\":\"integer\"},\"found\":{\"type\":\"boolean\"},\"cid\":{\"type\":\"string\"},\"url\":{\"type\":\"string\"},\"truncated\":{\"type\":\"boolean\"},\"text\":{\"type\":\"string\"}}}," as *u8)
828 b = dsv_cat(body, b, "\"Suggest\":{\"type\":\"object\",\"properties\":{\"v\":{\"type\":\"integer\"},\"prefix\":{\"type\":\"string\"},\"suggestions\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}}}}" as *u8)
829 b = dsv_cat(body, b, "}}}" as *u8)
830 return dsv_respond_json(out, "200 OK" as *u8, body, b)
831}
832// GET /api/search?q=<terms>[&scope=web] -> {"v":1,"query","scope","nresults","results":[{cid,score,title,
833// snippet,doc[,url]}]}. cid is a STRING on purpose (63-bit content ids exceed JSON's 2^53 safe-integer
834// range -- an enterprise client must never receive a silently-rounded id). No q -> a 200 self-describing
835// usage document (the endpoint documents itself); structured errors carry machine-readable codes (rule 6).
836func dss_api_search(domain: *u8, req: *u8, req_n: i64, out: *u8) -> i64 {
837 let q: *u8 = sys_mmap(DSV_MAGIC_1024)
838 let qn: i64 = dsv_qparam(req, req_n, "q" as *u8, 1, q, 1023)
839 let effdom: *u8 = sys_mmap(256)
840 let webscope: i64 = dsv_scope(req, req_n, domain, effdom)
841 let body: *u8 = sys_mmap(DSV_BODYCAP)
842 var b: i64 = 0
843 if qn == 0 {
844 b = dsv_cat(body, b, "{\"v\":1,\"endpoint\":\"/api/search\",\"params\":{\"q\":\"required: search terms\",\"scope\":\"optional: web = the sovereign-crawled open-web corpus; default = this site's corpus\",\"page\":\"optional: 0-based results page (20/page)\"},\"results_cap\":20,\"doc_endpoint\":\"/api/doc?cid=<cid>[&scope=web]\"}" as *u8)
845 return dsv_respond_json(out, "200 OK" as *u8, body, b)
846 }
847 let pgbuf: *u8 = sys_mmap(32)
848 let pgn: i64 = dsv_qparam(req, req_n, "page" as *u8, 4, pgbuf, 15)
849 let pgok: *i64 = sys_mmap(16) as *i64
850 var page: i64 = dsv_atoin(pgbuf, pgn, pgok)
851 if pgok[0] == 0 { page = 0 }
852 if page > 25 { page = 25 }
853 let cids: *i64 = sys_mmap(DSV_MAXR * 8) as *i64
854 let scores: *i64 = sys_mmap(DSV_MAXR * 8) as *i64
855 let totalbox: *i64 = sys_mmap(16) as *i64
856 let rc: i64 = dss_search_off(effdom, q, qn, cids, scores, DSV_MAXR, page * DSV_MAXR, totalbox)
857 var nres: i64 = rc
858 if nres < 0 { nres = 0 }
859 b = dsv_cat(body, b, "{\"v\":1,\"query\":\"" as *u8)
860 b = dsv_jesc(body, b, q, qn)
861 b = dsv_cat(body, b, "\",\"scope\":\"" as *u8)
862 b = dsv_scope_name(body, b, webscope)
863 b = dsv_cat(body, b, "\",\"page\":" as *u8)
864 b = dsv_catn(body, b, page)
865 b = dsv_cat(body, b, ",\"total\":" as *u8)
866 b = dsv_catn(body, b, totalbox[0])
867 b = dsv_cat(body, b, ",\"nresults\":" as *u8)
868 b = dsv_catn(body, b, nres)
869 if rc == (0 - 2) { b = dsv_cat(body, b, ",\"degraded\":true" as *u8) }
870 if totalbox[1] != 2 {
871 b = dsv_cat(body, b, ",\"phrase_exact\":" as *u8)
872 if totalbox[1] == 1 { b = dsv_cat(body, b, "true" as *u8) } else { b = dsv_cat(body, b, "false" as *u8) }
873 }
874 if nres == 0 {
875 let fixj: *u8 = sys_mmap(DSV_MAGIC_1024)
876 let flj: i64 = dss_correct(effdom, q, qn, fixj, 1023)
877 if flj > 0 {
878 b = dsv_cat(body, b, ",\"did_you_mean\":\"" as *u8)
879 b = dsv_jesc(body, b, fixj, flj)
880 b = dsv_cat(body, b, "\"" as *u8)
881 }
882 }
883 // COVERAGE HONESTY (operator 2026-07-04): the web scope is our own index, NOT the whole web -- the
884 // machine-readable contract flag so a client never treats partial coverage as complete.
885 if webscope == 1 { b = dsv_cat(body, b, ",\"coverage\":{\"index\":\"nishi\",\"complete\":false,\"note\":\"Nishi's own index (crawl + Common Crawl); not the whole live web. Some sites block all crawlers; lawful content is never filtered.\"}" as *u8) }
886 b = dsv_cat(body, b, ",\"results\":[" as *u8)
887 let prefix: *u8 = sys_mmap(512); dss_prefix(effdom, prefix)
888 let h: *i64 = dss_open_maybe_cached(prefix)
889 let key: *u8 = sys_mmap(64)
890 let ukey: *u8 = sys_mmap(64)
891 let dptr: *i64 = sys_mmap(16) as *i64; let dlen: *i64 = sys_mmap(16) as *i64
892 let uptr: *i64 = sys_mmap(16) as *i64; let ulen2: *i64 = sys_mmap(16) as *i64
893 var emitted: i64 = 0
894 var i: i64 = 0
895 while i < nres {
896 dss_mkkey(cids[i], key)
897 if (h as i64) != 0 { if ss_hget(h, key, dptr, dlen) == 1 {
898 if emitted > 0 { b = dsv_cat(body, b, "," as *u8) }
899 emitted = emitted + 1
900 let txt: *u8 = dptr[0] as *u8
901 var n2: i64 = dlen[0]
902 if n2 > DSV_SNIP { n2 = DSV_SNIP }
903 var dot: i64 = 0 - 1
904 var x: i64 = 0
905 while x < n2 { if x < 80 { if txt[x] == (46 as u8) { if dot < 0 { dot = x } } } x = x + 1 }
906 var tt: i64 = dot
907 if tt < 0 { tt = 56 }
908 if tt > n2 { tt = n2 }
909 b = dsv_cat(body, b, "{\"cid\":\"" as *u8)
910 b = dsv_catn(body, b, cids[i])
911 b = dsv_cat(body, b, "\",\"score\":" as *u8)
912 b = dsv_catn(body, b, scores[i])
913 b = dsv_cat(body, b, ",\"title\":\"" as *u8)
914 b = dsv_jesc(body, b, txt, tt)
915 b = dsv_cat(body, b, "\",\"snippet\":\"" as *u8)
916 if tt < n2 { b = dsv_jesc(body, b, ((txt as i64) + tt) as *u8, n2 - tt) }
917 b = dsv_cat(body, b, "\",\"doc\":\"/doc?cid=" as *u8)
918 b = dsv_catn(body, b, cids[i])
919 if webscope == 1 { b = dsv_cat(body, b, "&scope=web" as *u8) }
920 b = dsv_cat(body, b, "\"" as *u8)
921 dsv_mkurlkey(cids[i], ukey)
922 if ss_hget(h, ukey, uptr, ulen2) == 1 {
923 if ulen2[0] > 0 {
924 b = dsv_cat(body, b, ",\"url\":\"" as *u8)
925 b = dsv_jesc(body, b, uptr[0] as *u8, ulen2[0])
926 b = dsv_cat(body, b, "\"" as *u8)
927 // host: the site: filter/facet key, parsed from the url (additive contract field)
928 let hostb: *u8 = sys_mmap(256)
929 let hostl: i64 = dss_url_host(uptr[0] as *u8, ulen2[0], hostb)
930 if hostl > 0 {
931 b = dsv_cat(body, b, ",\"host\":\"" as *u8)
932 b = dsv_jesc(body, b, hostb, hostl)
933 b = dsv_cat(body, b, "\"" as *u8)
934 }
935 }
936 }
937 b = dsv_cat(body, b, "}" as *u8)
938 } }
939 i = i + 1
940 }
941 b = dsv_cat(body, b, "]}" as *u8)
942 return dsv_respond_json(out, "200 OK" as *u8, body, b)
943}
944// GET /api/suggest?q=<prefix>[&scope=web] -> ranked completions from the index's OWN term dictionary
945// (API-first autocomplete: apps/clients wire the keystroke loop; the HTML SERP stays zero-JS by design)
946func dss_api_suggest(domain: *u8, req: *u8, req_n: i64, out: *u8) -> i64 {
947 let q: *u8 = sys_mmap(256)
948 let qn: i64 = dsv_qparam(req, req_n, "q" as *u8, 1, q, 63)
949 let effdom: *u8 = sys_mmap(256)
950 let webscope: i64 = dsv_scope(req, req_n, domain, effdom)
951 let body: *u8 = sys_mmap(DSV_BODYCAP)
952 var b: i64 = 0
953 if qn == 0 {
954 b = dsv_cat(body, b, "{\"v\":1,\"endpoint\":\"/api/suggest\",\"params\":{\"q\":\"required: term prefix\",\"scope\":\"optional: web\"},\"max\":8}" as *u8)
955 return dsv_respond_json(out, "200 OK" as *u8, body, b)
956 }
957 // normalize the prefix with the index's own classifier (lowercase alnum)
958 let tbl: *u8 = sys_mmap(272)
959 ss_tok_table(tbl)
960 let pfx: *u8 = sys_mmap(64)
961 var pn: i64 = 0
962 var i: i64 = 0
963 while i < qn { if pn < 32 { let m: i64 = tbl[q[i]] as i64; if m != 0 { pfx[pn] = m as u8; pn = pn + 1 } } i = i + 1 }
964 let sg: *u8 = sys_mmap(8 * 64)
965 var n: i64 = 0
966 if pn >= 1 { n = dss_suggest(effdom, pfx, pn, sg, 8) }
967 b = dsv_cat(body, b, "{\"v\":1,\"prefix\":\"" as *u8)
968 b = dsv_jesc(body, b, pfx, pn)
969 b = dsv_cat(body, b, "\",\"scope\":\"" as *u8)
970 b = dsv_scope_name(body, b, webscope)
971 b = dsv_cat(body, b, "\",\"suggestions\":[" as *u8)
972 var k: i64 = 0
973 while k < n {
974 if k > 0 { b = dsv_cat(body, b, "," as *u8) }
975 b = dsv_cat(body, b, "\"" as *u8)
976 let sp: *u8 = (sg as i64 + k * 64) as *u8
977 var sl: i64 = 0
978 while sp[sl] != (0 as u8) { sl = sl + 1 }
979 b = dsv_jesc(body, b, sp, sl)
980 b = dsv_cat(body, b, "\"" as *u8)
981 k = k + 1
982 }
983 b = dsv_cat(body, b, "]}" as *u8)
984 return dsv_respond_json(out, "200 OK" as *u8, body, b)
985}
986// GET /api/doc?cid=<cid>[&scope=web] -> the full public document as JSON (text bounded like the HTML view)
987func dss_api_doc(domain: *u8, req: *u8, req_n: i64, out: *u8) -> i64 {
988 let effdom: *u8 = sys_mmap(256)
989 let webscope: i64 = dsv_scope(req, req_n, domain, effdom)
990 let cidbuf: *u8 = sys_mmap(64)
991 let cn: i64 = dsv_qparam(req, req_n, "cid" as *u8, 3, cidbuf, 31)
992 let okbox: *i64 = sys_mmap(16) as *i64
993 let cid: i64 = dsv_atoin(cidbuf, cn, okbox)
994 let body: *u8 = sys_mmap(DSV_BODYCAP)
995 var b: i64 = 0
996 if okbox[0] == 0 {
997 b = dsv_cat(body, b, "{\"v\":1,\"error\":{\"code\":\"bad_cid\",\"message\":\"cid must be a decimal content id\"}}" as *u8)
998 return dsv_respond_json(out, "400 Bad Request" as *u8, body, b)
999 }
1000 let dptr: *i64 = sys_mmap(16) as *i64
1001 let dlen: *i64 = sys_mmap(16) as *i64
1002 var found: i64 = 0
1003 let prefix: *u8 = sys_mmap(512); dss_prefix(effdom, prefix)
1004 let h: *i64 = dss_open_maybe_cached(prefix)
1005 let key: *u8 = sys_mmap(64)
1006 if (h as i64) != 0 {
1007 dss_mkkey(cid, key)
1008 if ss_hget(h, key, dptr, dlen) == 1 { found = 1 }
1009 }
1010 if found == 0 {
1011 b = dsv_cat(body, b, "{\"v\":1,\"error\":{\"code\":\"not_found\",\"message\":\"no public document with that cid\"}}" as *u8)
1012 return dsv_respond_json(out, "404 Not Found" as *u8, body, b)
1013 }
1014 var tn: i64 = dlen[0]
1015 var trunc: i64 = 0
1016 if tn > DSV_MAGIC_80000 { tn = DSV_MAGIC_80000; trunc = 1 }
1017 b = dsv_cat(body, b, "{\"v\":1,\"found\":true,\"cid\":\"" as *u8)
1018 b = dsv_catn(body, b, cid)
1019 b = dsv_cat(body, b, "\",\"scope\":\"" as *u8)
1020 b = dsv_scope_name(body, b, webscope)
1021 b = dsv_cat(body, b, "\",\"truncated\":" as *u8)
1022 if trunc == 1 { b = dsv_cat(body, b, "true" as *u8) } else { b = dsv_cat(body, b, "false" as *u8) }
1023 let ukey: *u8 = sys_mmap(64)
1024 let uptr: *i64 = sys_mmap(16) as *i64; let ulen2: *i64 = sys_mmap(16) as *i64
1025 dsv_mkurlkey(cid, ukey)
1026 if ss_hget(h, ukey, uptr, ulen2) == 1 {
1027 b = dsv_cat(body, b, ",\"url\":\"" as *u8)
1028 b = dsv_jesc(body, b, uptr[0] as *u8, ulen2[0])
1029 b = dsv_cat(body, b, "\"" as *u8)
1030 }
1031 b = dsv_cat(body, b, ",\"text\":\"" as *u8)
1032 b = dsv_jesc(body, b, dptr[0] as *u8, tn)
1033 b = dsv_cat(body, b, "\"}" as *u8)
1034 return dsv_respond_json(out, "200 OK" as *u8, body, b)
1035}