code wiki / _hdl_build / nx_gen_pipeline.nx
nx_gen_pipeline.nx source
↩ module page · 257 lines · 10679 B
1// nx_gen_pipeline.nx -- the sovereign GENERATE pipeline core (the gen-img logic, off Docker).
2// Given a GPU-worker response body {"images":["<base64 png>",...]}, for each image it:
3// decode base64 -> inject GENREC tEXt (nx_png_textw) -> ingest factors (nx_store_ingest, R1) ->
4// save the PNG bytes to a content-addressed blob file -> append the CID->path sidecar the gallery
5// serves /img/<cid> from. Seeds increment per image (seed0, seed0+1, ...) like the old batch path.
6// Pure logic core (no sockets/HTTP) so it gates offline by feeding a synthetic response. The HTTP
7// daemon nx_gen_orchestrator wraps this. license_tier: ORIGINAL
8import "nx_png_textw.nx"
9import "nx_store_ingest.nx"
10import "nx_syscalls.nx"
11
12// --- self-contained base64 decode (avoids importing base64.nx, which carries a smoke main) ---
13func gp_b64_dec_char(c: i64) -> i64 {
14 if c >= 65 { if c <= 90 { return c - 65 } }
15 if c >= 97 { if c <= 122 { return c - 97 + 26 } }
16 if c >= 48 { if c <= 57 { return c - 48 + 52 } }
17 if c == 43 { return 62 }
18 if c == 47 { return 63 }
19 if c == 45 { return 62 }
20 if c == 95 { return 63 }
21 return 0 - 1
22}
23// decode n base64 chars -> out (raw bytes). returns byte count, or -1 on an invalid char. stops at '='.
24func gp_b64_decode(inp: *u8, n: i64, out: *u8) -> i64 {
25 var i: i64 = 0
26 var o: i64 = 0
27 var acc: i64 = 0
28 var bits: i64 = 0
29 while i < n {
30 let c: i64 = inp[i] & 0xff
31 if c == 61 { i = n }
32 else {
33 let v: i64 = gp_b64_dec_char(c)
34 if v < 0 { return 0 - 1 }
35 acc = (acc << 6) | v
36 bits = bits + 6
37 if bits >= 8 { bits = bits - 8; out[o] = ((acc >> bits) & 0xff) as u8; o = o + 1 }
38 i = i + 1
39 }
40 }
41 return o
42}
43
44// base64 ENCODE raw bytes -> out (standard alphabet, '=' padded). returns char count. The img2img path needs this to
45// wrap an on-disk gallery PNG as init_images for the worker; decode's inverse (kept adjacent so the codec is one unit).
46func gp_b64_encode(inp: *u8, n: i64, out: *u8) -> i64 {
47 let tbl: *u8 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" as *u8
48 var i: i64 = 0
49 var o: i64 = 0
50 while i + 3 <= n {
51 let b0: i64 = inp[i]&0xff; let b1: i64 = inp[i+1]&0xff; let b2: i64 = inp[i+2]&0xff
52 out[o] = tbl[b0>>2]; o=o+1
53 out[o] = tbl[((b0&3)<<4)|(b1>>4)]; o=o+1
54 out[o] = tbl[((b1&15)<<2)|(b2>>6)]; o=o+1
55 out[o] = tbl[b2&63]; o=o+1
56 i=i+3
57 }
58 let rem: i64 = n - i
59 if rem == 1 {
60 let b0: i64 = inp[i]&0xff
61 out[o] = tbl[b0>>2]; o=o+1
62 out[o] = tbl[(b0&3)<<4]; o=o+1
63 out[o] = 61 as u8; o=o+1
64 out[o] = 61 as u8; o=o+1
65 }
66 if rem == 2 {
67 let b0: i64 = inp[i]&0xff; let b1: i64 = inp[i+1]&0xff
68 out[o] = tbl[b0>>2]; o=o+1
69 out[o] = tbl[((b0&3)<<4)|(b1>>4)]; o=o+1
70 out[o] = tbl[(b1&15)<<2]; o=o+1
71 out[o] = 61 as u8; o=o+1
72 }
73 return o
74}
75
76func gp_itoa(dst: *u8, off: i64, v: i64) -> i64 {
77 let t: *u8 = sys_mmap(32)
78 var m: i64 = v
79 var k: i64 = 0
80 if m == 0 { t[0] = 48 as u8; k = 1 }
81 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
82 var o: i64 = off
83 var q: i64 = k - 1
84 while q >= 0 { dst[o] = t[q]; o = o + 1; q = q - 1 }
85 dst[o] = 0 as u8
86 return o
87}
88
89// find the "images" array: locate the literal "images" then the next '['. returns offset just after '[', or -1.
90func gp_find_images(resp: *u8, resplen: i64) -> i64 {
91 var i: i64 = 0
92 var found: i64 = 0 - 1
93 while i + 8 <= resplen {
94 if (resp[i]&0xff)==34 { if (resp[i+1]&0xff)==105 { if (resp[i+2]&0xff)==109 { if (resp[i+3]&0xff)==97 { if (resp[i+4]&0xff)==103 { if (resp[i+5]&0xff)==101 { if (resp[i+6]&0xff)==115 { if (resp[i+7]&0xff)==34 { found = i + 8; i = resplen } } } } } } } }
95 if found < 0 { i = i + 1 }
96 }
97 if found < 0 { return 0 - 1 }
98 var k: i64 = found
99 while k < resplen { if (resp[k]&0xff)==91 { return k + 1 } k = k + 1 }
100 return 0 - 1
101}
102
103// extract the next "..."-quoted string into out (base64 has no embedded quotes). advances pos[0].
104// returns length, or -1 when ']' (end of array) is reached first.
105func gp_next_str(resp: *u8, resplen: i64, pos: *i64, out: *u8, outcap: i64) -> i64 {
106 var i: i64 = pos[0]
107 var opened: i64 = 0 - 1
108 var stop: i64 = 0
109 while stop == 0 {
110 if i >= resplen { pos[0] = i; return 0 - 1 }
111 let c: i64 = resp[i] & 0xff
112 if c == 93 { pos[0] = i + 1; return 0 - 1 }
113 if c == 34 { opened = i + 1; stop = 1 }
114 i = i + 1
115 }
116 var j: i64 = opened
117 var o: i64 = 0
118 var stop2: i64 = 0
119 while stop2 == 0 {
120 if j >= resplen { stop2 = 1 }
121 else {
122 let c2: i64 = resp[j] & 0xff
123 if c2 == 34 { stop2 = 1; j = j + 1 }
124 else { if o < outcap - 1 { out[o] = resp[j] as u8; o = o + 1 } j = j + 1 }
125 }
126 }
127 out[o] = 0 as u8
128 pos[0] = j
129 return o
130}
131
132// build <blobdir><cid>.png into out (NUL-term). returns length.
133func gp_blob_path(blobdir: *u8, cid: *u8, out: *u8) -> i64 {
134 var o: i64 = 0
135 var i: i64 = 0
136 while blobdir[i] != (0 as u8) { out[o] = blobdir[i]; o = o + 1; i = i + 1 }
137 i = 0
138 while cid[i] != (0 as u8) { out[o] = cid[i]; o = o + 1; i = i + 1 }
139 out[o]=46 as u8; o=o+1
140 out[o]=112 as u8; o=o+1
141 out[o]=110 as u8; o=o+1
142 out[o]=103 as u8; o=o+1
143 out[o]=0 as u8
144 return o
145}
146
147// append "<cid>\t<path>\n" to the gallery sidecar (CID->path map serving /img/<cid>).
148func gp_sidecar_append(sidecar: *u8, cid: *u8, path: *u8) -> i64 {
149 let fd: i64 = sys_openat_append(sidecar, 0x1a4)
150 if fd < 0 { return 0 - 1 }
151 let line: *u8 = sys_mmap(1024)
152 var o: i64 = 0
153 var i: i64 = 0
154 while cid[i] != (0 as u8) { line[o] = cid[i]; o = o + 1; i = i + 1 }
155 line[o] = 9 as u8; o = o + 1
156 i = 0
157 while path[i] != (0 as u8) { line[o] = path[i]; o = o + 1; i = i + 1 }
158 line[o] = 10 as u8; o = o + 1
159 sys_write(fd, line, o)
160 sys_close(fd)
161 return 0
162}
163
164// process ONE base64 image: decode -> inject GENREC -> ingest factors -> save blob -> sidecar.
165// writes the 69B CID to cidout. returns 1=new, 0=dup (already in store), -1=error.
166func gp_one(b64: *u8, b64len: i64, seed: *u8, model: *u8, sampler: *u8, steps: *u8, host: *u8, prompt: *u8, size: *u8, args: *u8, storeprefix: *u8, blobdir: *u8, sidecar: *u8, cidout: *u8) -> i64 {
167 let raw: *u8 = sys_mmap(8388608)
168 let rawlen: i64 = gp_b64_decode(b64, b64len, raw)
169 if rawlen < 8 { return 0 - 1 }
170 let withtext: *u8 = sys_mmap(8388608)
171 let wlen: i64 = png_insert_genrec(raw, rawlen, seed, model, sampler, steps, host, prompt, size, args, withtext)
172 if wlen < 0 { return 0 - 1 }
173 let rc: i64 = nx_store_ingest_ingest_cid(withtext, wlen, storeprefix, cidout)
174 if rc < 0 { return 0 - 1 }
175 let path: *u8 = sys_mmap(512)
176 gp_blob_path(blobdir, cidout, path)
177 let fd: i64 = sys_openat_wr(path, 0x1a4)
178 if fd >= 0 { sys_write(fd, withtext, wlen); sys_close(fd) }
179 if rc == 1 { gp_sidecar_append(sidecar, cidout, path) }
180 return rc
181}
182
183func gp_slen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
184// find NUL-term `needle` in buf[from..n). returns offset or -1.
185func gp_find_from(buf: *u8, n: i64, needle: *u8, from: i64) -> i64 {
186 let nl: i64 = gp_slen(needle)
187 if nl==0 { return from }
188 var i: i64 = from
189 while i + nl <= n { var j: i64=0; var ok: i64=1; while j<nl { if (buf[i+j]&0xff)!=(needle[j]&0xff){ok=0;j=nl} else {j=j+1} } if ok==1 {return i} i=i+1 }
190 return 0-1
191}
192// extract the next "b64_json":"<value>" (OpenAI shape) from pos. updates pos. returns len, or -1 if none.
193func gp_next_b64json(resp: *u8, resplen: i64, pos: *i64, out: *u8, outcap: i64) -> i64 {
194 let m: i64 = gp_find_from(resp, resplen, "\"b64_json\"" as *u8, pos[0])
195 if m < 0 { return 0 - 1 }
196 var i: i64 = m + 10
197 var s1: i64 = 0
198 while s1 == 0 { if i >= resplen { return 0 - 1 } if (resp[i]&0xff)==58 { s1=1 } i=i+1 }
199 var s2: i64 = 0
200 while s2 == 0 { if i >= resplen { return 0 - 1 } let c: i64 = resp[i]&0xff; if c==34 { s2=1; i=i+1 } else { if c==32 { i=i+1 } else { return 0 - 1 } } }
201 var o: i64 = 0
202 var s3: i64 = 0
203 while s3 == 0 { if i>=resplen { s3=1 } else { let c2: i64 = resp[i]&0xff; if c2==34 { s3=1; i=i+1 } else { if o<outcap-1 { out[o]=resp[i] as u8; o=o+1 } i=i+1 } } }
204 out[o]=0 as u8
205 pos[0]=i
206 return o
207}
208// process the worker response. Handles BOTH the OpenAI shape {"data":[{"b64_json":"..."}]} (the
209// /v1/images/generations endpoint, which renders Z-Image correctly) AND the A1111 shape
210// {"images":["..."]}. gp_one with seed0+i; writes each 69B CID at cids_out[i*80]. returns count, -1 if neither.
211func gp_response(resp: *u8, resplen: i64, seed0: i64, model: *u8, sampler: *u8, steps: *u8, host: *u8, prompt: *u8, size: *u8, args: *u8, storeprefix: *u8, blobdir: *u8, sidecar: *u8, cids_out: *u8, maxc: i64) -> i64 {
212 let posbox: *i64 = sys_mmap(16) as *i64
213 var count: i64 = 0
214 var seed: i64 = seed0
215 let b64: *u8 = sys_mmap(12582912)
216 let oai: i64 = gp_find_from(resp, resplen, "\"b64_json\"" as *u8, 0)
217 if oai >= 0 {
218 posbox[0] = 0
219 var go: i64 = 1
220 while go == 1 {
221 if count >= maxc { go = 0 }
222 else {
223 let slen: i64 = gp_next_b64json(resp, resplen, posbox, b64, 12582912)
224 if slen < 0 { go = 0 }
225 else {
226 let sbuf: *u8 = sys_mmap(32)
227 gp_itoa(sbuf, 0, seed)
228 let cidout: *u8 = (cids_out as i64 + count * 80) as *u8
229 gp_one(b64, slen, sbuf, model, sampler, steps, host, prompt, size, args, storeprefix, blobdir, sidecar, cidout)
230 count = count + 1
231 seed = seed + 1
232 }
233 }
234 }
235 return count
236 }
237 let arr: i64 = gp_find_images(resp, resplen)
238 if arr < 0 { return 0 - 1 }
239 posbox[0] = arr
240 var go2: i64 = 1
241 while go2 == 1 {
242 if count >= maxc { go2 = 0 }
243 else {
244 let slen: i64 = gp_next_str(resp, resplen, posbox, b64, 12582912)
245 if slen < 0 { go2 = 0 }
246 else {
247 let sbuf: *u8 = sys_mmap(32)
248 gp_itoa(sbuf, 0, seed)
249 let cidout: *u8 = (cids_out as i64 + count * 80) as *u8
250 gp_one(b64, slen, sbuf, model, sampler, steps, host, prompt, size, args, storeprefix, blobdir, sidecar, cidout)
251 count = count + 1
252 seed = seed + 1
253 }
254 }
255 }
256 return count
257}