nx_f32_llm_serve.nx source
↩ module page · 351 lines · 17315 B
1// nx_f32_llm_serve.nx -- the SOVEREIGN LLM generation SEAT. Loads real Qwen2.5-0.5B GGUF ONCE at startup, binds
2// 127.0.0.1:11434 (the seam nx_writehub_seat already POSTs to), and on POST /chat runs nx_f32_llm_run_v3 ->
3// returns OpenAI-chat-shaped JSON. Drop-in replacement for the Docker gen-llm:11434 seat -- 100% sovereign (our
4// nx_f32 stack + our own HTTP accept-loop, the nx_writer_serve pattern), NO Docker, NO 3rd-party lib, no shell.
5// NOT yet supervised on this box (no nishi_supervisor.sh here -- launch via knowledge/shell/seat_probe.sh or
6// setsid; supervision = follow-on when the seat moves to its target host). 127.0.0.1-only = seam by construction.
7// PERF CAVEAT (honest): scalar-CPU matmul + re-dequant per token -> slow; MAX_NEW capped low. The engine's
8// COHERENCE is gated separately by nx_f32_llm_live_gen_test; this organ is the serving layer around it.
9// expect: LIVE on :11434 (blocks). license_tier: ORIGINAL module: nishi-core.seat.llm
10import "nx_syscalls.nx"
11import "nx_connect.nx" // bounded connect: a raw sys_connect hangs ~127s on a black-holed host
12import "nx_tier.nx"
13import "nx_bpe.nx"
14import "nx_gguf.nx"
15import "nx_gguf_load.nx"
16import "nx_gguf_meta.nx"
17import "nx_f32.nx"
18import "nx_f32_kv_cache.nx"
19import "nx_f32_lazy_weight.nx"
20import "nx_f32_llama_block.nx"
21import "nx_f32_llama_block_v4.nx"
22import "nx_f32_llama_stack_v4.nx"
23import "nx_f32_llama_layer_lazy_load.nx"
24import "nx_f32_llm.nx"
25import "nx_f32_llm_v4.nx"
26import "nx_f32_llm_read_dims.nx"
27import "nx_f32_bpe_load.nx"
28import "nx_f32_llm_special_tokens.nx"
29import "nx_f32_sampler.nx"
30import "nx_f32_llm_run_v2.nx"
31import "nx_f32_llm_run_v3.nx"
32import "nx_http_client.nx"
33import "nx_f32_div.nx"
34import "nx_f32_cvt.nx"
35import "nx_prng.nx"
36import "nx_reasoning.nx"
37const NX_MAGIC_67108864: i64 = 67108864
38const NX_MAGIC_262144: i64 = 262144
39const NX_MAGIC_524288: i64 = 524288
40const NX_MAGIC_151644: i64 = 151644
41const NX_MAGIC_151645: i64 = 151645
42const NX_MAGIC_1048576: i64 = 1048576
43const NX_MAGIC_65536: i64 = 65536
44const NX_MAGIC_131072: i64 = 131072
45const NX_MAGIC_200000: i64 = 200000
46const NX_MAGIC_65534: i64 = 65534
47
48// The seat's ONE port constant; sockaddr bytes are COMPUTED from it (the
49// shared helper), and the startup SELF-PROBE dials it independently.
50const NX_LSEAT_PORT: i64 = 11434
51
52func ls_puts(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
53func ls_slen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
54func ls_find(buf: *u8, n: i64, ndl: *u8, nl: i64) -> i64 {
55 if nl<=0 { return 0-1 }
56 var i: i64=0
57 while i+nl<=n { var j: i64=0; var hit: i64=1; while j<nl { if buf[i+j]!=ndl[j] { hit=0; j=nl } else { j=j+1 } } if hit==1 { return i } i=i+1 }
58 return 0-1
59}
60func ls_cat(dst: *u8, off: i64, s: *u8) -> i64 { var i: i64=0; while s[i]!=(0 as u8){ dst[off]=s[i]; off=off+1; i=i+1 } return off }
61func ls_write_all(fd: i64, buf: *u8, n: i64) -> i64 { var off: i64=0; while off<n { let w: i64=sys_write(fd, ((buf as i64)+off) as *u8, n-off); if w<=0 { off=n } else { off=off+w } } return 0 }
62func ls_atoi(s: *u8) -> i64 {
63 var v: i64 = 0
64 var i: i64 = 0
65 while s[i] != (0 as u8) {
66 let d: i64 = s[i] as i64
67 if d < 48 { return v }
68 if d > 57 { return v }
69 v = v * 10 + (d - 48)
70 i = i + 1
71 }
72 return v
73}
74// Find "<key>": in body and parse the integer after it (skipping spaces).
75// Returns -1 if absent/malformed.
76func ls_body_int(body: *u8, blen: i64, key: *u8) -> i64 {
77 let kl: i64 = ls_slen(key)
78 let pos: i64 = ls_find(body, blen, key, kl)
79 if pos < 0 { return 0 - 1 }
80 var i: i64 = pos + kl
81 while i < blen { if (body[i] as i64) == 32 { i = i + 1 } else { break } }
82 var v: i64 = 0
83 var nd: i64 = 0
84 while i < blen {
85 let d: i64 = body[i] as i64
86 if d < 48 { break }
87 if d > 57 { break }
88 v = v * 10 + (d - 48)
89 nd = nd + 1
90 i = i + 1
91 }
92 if nd == 0 { return 0 - 1 }
93 return v
94}
95// Parse "temperature": as TENTHS (one decimal digit honored: "0.8"->8,
96// "1.0"->10, "2"->20; extra decimals truncated). -1 if absent.
97func ls_body_temp_tenths(body: *u8, blen: i64) -> i64 {
98 let key: *u8 = "\"temperature\":" as *u8
99 let kl: i64 = ls_slen(key)
100 let pos: i64 = ls_find(body, blen, key, kl)
101 if pos < 0 { return 0 - 1 }
102 var i: i64 = pos + kl
103 while i < blen { if (body[i] as i64) == 32 { i = i + 1 } else { break } }
104 var whole: i64 = 0
105 var nd: i64 = 0
106 while i < blen {
107 let d: i64 = body[i] as i64
108 if d < 48 { break }
109 if d > 57 { break }
110 whole = whole * 10 + (d - 48)
111 nd = nd + 1
112 i = i + 1
113 }
114 if nd == 0 { return 0 - 1 }
115 var tenths: i64 = whole * 10
116 if i < blen {
117 if (body[i] as i64) == 46 {
118 i = i + 1
119 if i < blen {
120 let d2: i64 = body[i] as i64
121 if d2 >= 48 { if d2 <= 57 { tenths = tenths + (d2 - 48) } }
122 }
123 }
124 }
125 return tenths
126}
127
128// Read an HTTP request until headers AND the Content-Length body are
129// complete. FIXES the latent single-read bug (2026-07-09, measured live on
130// the batch seat: the request line can arrive ALONE -- first read = 21
131// bytes "POST /chat H..." -- so one sys_read truncated the body and the
132// prompt extraction found nothing). Caller must have set a recv timeout
133// on fd so a stalled client can't wedge the single-threaded seat.
134func ls_read_request(fd: i64, req: *u8, cap: i64) -> i64 {
135 var rn: i64 = 0
136 var tries: i64 = 0
137 while tries < 8 {
138 let r1: i64 = sys_read(fd, ((req as i64)+rn) as *u8, cap-rn)
139 if r1 <= 0 { tries = 8 } else {
140 rn = rn + r1
141 let hb: i64 = ls_find(req, rn, "\r\n\r\n" as *u8, 4)
142 if hb >= 0 {
143 var cl: i64 = 0
144 let cp: i64 = ls_find(req, rn, "Content-Length: " as *u8, 16)
145 if cp >= 0 { cl = ls_atoi(((req as i64)+cp+16) as *u8) }
146 if rn >= hb + 4 + cl { tries = 8 } else { tries = tries + 1 }
147 } else { tries = tries + 1 }
148 }
149 }
150 return rn
151}
152// extract the first "content":"..." value from a JSON body into out (up to cap); returns length
153func ls_extract_prompt(body: *u8, blen: i64, out: *u8, cap: i64) -> i64 {
154 let key: *u8 = "\"content\":\"" as *u8
155 let kl: i64 = ls_slen(key)
156 let pos: i64 = ls_find(body, blen, key, kl)
157 if pos<0 { return 0 }
158 var i: i64 = pos+kl
159 var o: i64 = 0
160 var go: i64 = 1
161 while go==1 {
162 if i>=blen { go=0 } else {
163 let c: i64 = body[i] as i64
164 if c==92 {
165 if i+1<blen { let d: i64=body[i+1] as i64
166 if d==110 { if o<cap { out[o]=10 as u8; o=o+1 } } else { if o<cap { out[o]=body[i+1]; o=o+1 } }
167 i=i+2
168 } else { i=i+1 }
169 } else { if c==34 { go=0 } else { if o<cap { out[o]=body[i]; o=o+1 } i=i+1 } }
170 }
171 }
172 return o
173}
174// JSON-escape src -> dst (escape " \ \n \t; drop other control); returns dstlen
175func ls_json_esc(src: *u8, sn: i64, dst: *u8, cap: i64) -> i64 {
176 var o: i64=0; var i: i64=0
177 while i<sn { if o+2>=cap { i=sn } else {
178 let c: i64=src[i] as i64
179 if c==34 { dst[o]=92 as u8; dst[o+1]=34 as u8; o=o+2 }
180 else { if c==92 { dst[o]=92 as u8; dst[o+1]=92 as u8; o=o+2 }
181 else { if c==10 { dst[o]=92 as u8; dst[o+1]=110 as u8; o=o+2 }
182 else { if c==13 { }
183 else { if c==9 { dst[o]=92 as u8; dst[o+1]=116 as u8; o=o+2 }
184 else { if c<32 { } else { dst[o]=src[i] as u8; o=o+1 } } } } } }
185 i=i+1
186 } }
187 return o
188}
189
190func main(argc: i64, argv: *i64) -> i64 {
191 // Model path: argv[1] overrides; default = the PERSISTENT sovereign NAS path (NOT /tmp -- /tmp is
192 // volatile tmpfs, wastes RAM for a 491MB model, and evaporates on distro/host teardown -- the exact
193 // fragility that broke the laptop-WSL seat). Host on the NAS reading the persistent volume.
194 var path: *u8 = "/volume1/homes/elderwesto/nx_bench/model/nx_real_model.gguf" as *u8
195 if argc > 1 { path = argv[1] as *u8 }
196 ls_puts("nx_f32_llm_serve: loading model (persistent NAS path or argv[1]) ...\n" as *u8)
197 let len_out: *i64 = sys_mmap(8) as *i64
198 let buf: *u8 = sys_read_file(path, len_out)
199 if buf==(0 as *u8) { ls_puts("seat: no model at the given path\n" as *u8); sys_exit(10); return 10 }
200 let hdr: *NxGgufHeader = sys_mmap(NX_GGUF_HDR_BYTES) as *NxGgufHeader
201 if nx_gguf_parse(buf, len_out[0], hdr) != NX_GGUF_OK { ls_puts("seat: gguf parse fail\n" as *u8); sys_exit(20); return 20 }
202 let model: *NxF32LlamaModel = nx_f32_llama_model_alloc()
203 let out_err: *i64 = sys_mmap(8) as *i64
204 if nx_f32_llm_read_dims_from_gguf(buf, len_out[0], hdr, model, out_err) != NX_FLD_OK { ls_puts("seat: read_dims fail\n" as *u8); sys_exit(30); return 30 }
205 if nx_f32_llm_load_weights_v4_from_gguf(buf, hdr, model, out_err) != NX_FLV4_OK { ls_puts("seat: load_weights fail\n" as *u8); sys_exit(40); return 40 }
206 let vocab: *NxBpeVocab = nx_bpe_vocab_new(NX_MAGIC_67108864, NX_MAGIC_262144, NX_MAGIC_524288)
207 let nt: *i64 = sys_mmap(8) as *i64
208 let nm: *i64 = sys_mmap(8) as *i64
209 let bpe_v: nx_int = nx_f32_bpe_load_from_gguf(buf, len_out[0], hdr, vocab, nt, nm, out_err)
210 if bpe_v != NX_FBL_OK { ls_puts("seat: bpe load fail\n" as *u8); sys_exit(50); return 50 }
211 let eos: nx_int = nx_f32_llm_read_eos(buf, len_out[0], hdr)
212 let cache: *NxF32KVCache = nx_f32_kv_cache_alloc(model.n_layers, model.n_kv_heads, 512, model.head_dim)
213 if cache==(0 as *NxF32KVCache) { ls_puts("seat: kv cache alloc fail\n" as *u8); sys_exit(53); return 53 }
214 let cfg: *NxF32SamplerCfg = nx_f32_sampler_cfg_alloc()
215 let eps: i64 = 0x358637BD
216 let attn_scale: i64 = 0x3E000000
217 let rope_base: i64 = 0x415D0EAB
218 // reasoning-organ config: the /chat (ChatML instruct) path. Reuses the
219 // PROVEN nx_reason_chat_gen (template as token ids, chunked prefill,
220 // im_end/eos stop) -- raw continuation stays on /complete.
221 let rc: *NxReasonCfg = nx_reason_cfg_alloc()
222 rc.model = model
223 rc.vocab = vocab
224 rc.cache = cache
225 rc.max_new = 6
226 rc.inv_temp_f32 = 0x3F800000
227 rc.top_k = 0
228 rc.eps = eps
229 rc.attn_scale = attn_scale
230 rc.rope_log_base = rope_base
231 rc.eos = eos
232 rc.im_start = NX_MAGIC_151644
233 rc.im_end = NX_MAGIC_151645
234 ls_puts("nx_f32_llm_serve: model loaded (Qwen2.5-0.5B, sovereign). binding 127.0.0.1:11434 ...\n" as *u8)
235
236 let lfd: i64 = sys_socket(2, 1 | 0x80000, 0)
237 if lfd<0 { ls_puts("seat: socket fail\n" as *u8); sys_exit(60); return 60 }
238 let optval: *i64 = sys_mmap(8) as *i64; optval[0]=1
239 sys_setsockopt(lfd, 1, 2, optval as *u8, 4)
240 // sockaddr from the INTEGER port via the shared helper -- the previous
241 // HAND-ENCODED bytes were 0x2C,0xAE = port 11438, NOT 11434: the seat
242 // NEVER listened where writehub (which uses this helper, port int
243 // 11434) connects. Found 2026-07-10 -- the "loopback wedge" was this.
244 let addr: *u8 = sys_mmap(16)
245 nx_http_client_sockaddr_ipv4(addr, 127, 0, 0, 1, NX_LSEAT_PORT)
246 if sys_bind(lfd, addr, 16) < 0 { ls_puts("seat: bind fail (11434 in use?)\n" as *u8); sys_exit(70); return 70 }
247 sys_listen(lfd, 16)
248 // SELF-PROBE before claiming LIVE (self-verify at the EXECUTION level:
249 // serving != working). Connect to the INTENDED port from a fresh
250 // socket; only a real listener there lets the claim through. Catches
251 // the wrong-port class AND genuine socket-layer wedges at startup.
252 let pfd: i64 = sys_socket(2, 1, 0)
253 let paddr: *u8 = sys_mmap(16)
254 nx_http_client_sockaddr_ipv4(paddr, 127, 0, 0, 1, NX_LSEAT_PORT)
255 if nx_connect_bounded(pfd, paddr, 16, NX_CONN_DEFAULT_MS) < 0 {
256 ls_puts("seat: SELF-PROBE FAILED -- bound socket is NOT reachable on 127.0.0.1:11434; refusing to claim LIVE\n" as *u8)
257 sys_exit(71); return 71
258 }
259 sys_close(pfd)
260 let dfd: i64 = sys_accept(lfd) // drain the probe connection
261 if dfd >= 0 { sys_close(dfd) }
262 ls_puts("nx_f32_llm_serve: SELF-PROBE OK -- LIVE sovereign LLM seat on 127.0.0.1:11434 (POST /chat = ChatML instruct, POST /complete = raw continuation)\n" as *u8)
263
264 let req: *u8 = sys_mmap(NX_MAGIC_1048576)
265 let prompt: *u8 = sys_mmap(NX_MAGIC_65536)
266 let out_bytes: *u8 = sys_mmap(NX_MAGIC_65536)
267 let esc: *u8 = sys_mmap(NX_MAGIC_131072)
268 let resp: *u8 = sys_mmap(NX_MAGIC_262144)
269 let jbuf: *u8 = sys_mmap(NX_MAGIC_200000)
270 let prng: *i64 = sys_mmap(8) as *i64; prng[0]=1
271 let max_new: nx_int = 6
272
273 var alive: i64 = 1
274 while alive==1 {
275 let fd: i64 = sys_accept(lfd)
276 if fd>=0 {
277 sys_set_socket_timeout(fd, 2)
278 let rn: i64 = ls_read_request(fd, req, NX_MAGIC_1048576)
279 if rn>0 {
280 // routes: POST /complete = RAW continuation (the writehub prose
281 // seam); POST /chat = CHATML INSTRUCT via nx_reason_chat_gen
282 // (OpenAI-shaped messages deserve the template the model was
283 // trained on -- raw arithmetic measured garbage, ChatML "85").
284 let r_chat: i64 = ls_find(req, rn, "POST /chat" as *u8, 10)
285 let r_comp: i64 = ls_find(req, rn, "POST /complete" as *u8, 14)
286 var routed: i64 = 0
287 if r_chat >= 0 { routed = 1 }
288 if r_comp >= 0 { routed = 1 }
289 if routed == 1 {
290 let hb: i64 = ls_find(req, rn, "\r\n\r\n" as *u8, 4)
291 var bstart: i64 = 0; if hb>=0 { bstart=hb+4 }
292 let pl: i64 = ls_extract_prompt(((req as i64)+bstart) as *u8, rn-bstart, prompt, NX_MAGIC_65534)
293 prompt[pl]=0 as u8
294 // HONOR the request body: max_tokens -> clamp [1,64]
295 // (single-stream guard); temperature (tenths) -> exact
296 // inv_temp = f32(10)/f32(tenths) + top_k=40; absent ->
297 // greedy argmax.
298 let bp: *u8 = ((req as i64)+bstart) as *u8
299 let bl: i64 = rn - bstart
300 var req_max: nx_int = max_new
301 let mt: i64 = ls_body_int(bp, bl, "\"max_tokens\":" as *u8)
302 if mt >= 1 { req_max = mt as nx_int; if req_max > 64 { req_max = 64 } }
303 var inv_t: i64 = 0x3F800000
304 var tk: nx_int = 0
305 let tt: i64 = ls_body_temp_tenths(bp, bl)
306 if tt >= 1 {
307 inv_t = nx_f32_div(nx_i32_to_f32(10), nx_i32_to_f32(tt))
308 tk = 40
309 }
310 var n_emit: nx_int = 0
311 if r_comp >= 0 {
312 // RAW continuation (run_v3; BPE from prompt bytes).
313 cfg.inv_temp_f32 = inv_t
314 cfg.top_k = tk
315 // fresh positions per request (2026-07-09): without this
316 // reset request 2+ attended request 1's K/V (cross-talk)
317 // and seq_len marched to the 512 cap.
318 nx_f32_kv_cache_reset(cache)
319 n_emit = nx_f32_llm_run_v3(model, vocab, cache, prompt, pl as nx_int, req_max, cfg, eps, attn_scale, rope_base, 1, prng, eos, out_bytes, NX_MAGIC_65536)
320 } else {
321 // CHATML instruct (organ resets the cache itself).
322 rc.max_new = req_max
323 rc.inv_temp_f32 = inv_t
324 rc.top_k = tk
325 var pr: *i64 = 0 as *i64
326 if tt >= 1 { pr = prng }
327 n_emit = nx_reason_chat_gen(rc, prompt, pl as nx_int, pr, out_bytes, NX_MAGIC_65536)
328 }
329 var el: i64 = 0
330 if n_emit>0 { el = ls_json_esc(out_bytes, n_emit as i64, esc, NX_MAGIC_131072) }
331 var jo: i64 = 0
332 jo=ls_cat(jbuf,jo,"{\"model\":\"nishi-qwen-0.5b-sovereign\",\"choices\":[{\"message\":{\"role\":\"assistant\",\"content\":\"" as *u8)
333 var k: i64=0; while k<el { jbuf[jo]=esc[k]; jo=jo+1; k=k+1 }
334 jo=ls_cat(jbuf,jo,"\"}}]}" as *u8)
335 var ro: i64=0
336 ro=ls_cat(resp,ro,"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nConnection: close\r\nContent-Length: " as *u8)
337 let db: *u8=sys_mmap(24); var m: i64=jo; var dc: i64=0; if m==0 { db[0]=48 as u8; dc=1 } while m>0 { db[dc]=(48+(m%10)) as u8; m=m/10; dc=dc+1 }
338 var di: i64=dc-1; while di>=0 { resp[ro]=db[di]; ro=ro+1; di=di-1 }
339 ro=ls_cat(resp,ro,"\r\n\r\n" as *u8)
340 var jj: i64=0; while jj<jo { resp[ro]=jbuf[jj]; ro=ro+1; jj=jj+1 }
341 ls_write_all(fd, resp, ro)
342 } else {
343 let hm: *u8 = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nConnection: close\r\nContent-Length: 18\r\n\r\nnishi-llm-seat ok\n" as *u8
344 ls_write_all(fd, hm, ls_slen(hm))
345 }
346 }
347 sys_close(fd)
348 }
349 }
350 sys_exit(0); return 0
351}