nx_galx_search_index.nx source
↩ module page · 162 lines · 8023 B
1// nx_galx_search_index.nx -- SOVEREIGN search-index builder. For each ingested image it reads the PNG's
2// metadata prefix, harvests the tEXt + uncompressed-iTXt VALUES (prompt / negative_prompt / model / sampler
3// / ...), lowercases + sanitizes them into one searchable line "cid<TAB>text", and APPENDS it to
4// knowledge/status/galx_search.tsv. The daemon's /api/list?q= substring-scans that file. FORK-PER-IMAGE so
5// each PNG read's mmap is reclaimed on exit; each child does ONE atomic O_APPEND write (<4096 bytes) so any
6// number of parallel workers can append to the same file with no merge step and no interleaving.
7// Only a 256KB prefix is read (PNG tEXt/iTXt sit before the big IDAT) -> ~16x less NAS I/O than full read.
8// Usage: nx_galx_search_index <start> <count>. license_tier: ORIGINAL
9import "nx_syscalls.nx"
10import "nx_itoa_lib.nx" // shared MSB-first emitter (zero-alloc)
11const SI_MAGIC_100000: i64 = 100000
12const SI_MAGIC_1000000000: i64 = 1000000000
13const SI_MAGIC_2048: i64 = 2048
14
15const SI_READ: i64 = 262144 // bytes of each PNG to scan (metadata is pre-IDAT)
16const SI_TEXTCAP: i64 = 3000 // cap searchable text so cid+tab+text+nl < 4096 (atomic append)
17const SI_OUT: *u8 = "knowledge/status/galx_search.tsv" as *u8
18
19func si_p(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
20// MIGRATED to the shared emitter (debt 1785563586). The old body mmapped a scratch buffer
21// per call and never freed it. At PAGE granularity that is 4096B leaked PER CALL -- the
22// defect that took 28.5GB of a 36GB host in nx_ts_lumadiff (2MB input, ~3.66M calls).
23// nxi_* is MSB-first, allocates NOTHING, and emits identical bytes including the sign.
24func si_n(v: i64) -> i64 { nxi_out(v); return 0 }
25func si_atoi(s: *u8) -> i64 { var v: i64=0; var i: i64=0; while s[i]!=(0 as u8){ if s[i]>=(48 as u8){ if s[i]<=(57 as u8){ v=v*10+((s[i]-(48 as u8)) as i64) } } i=i+1 } return v }
26func si_be32(b: *u8, o: i64) -> i64 { return ((b[o] as i64)<<24)|((b[o+1] as i64)<<16)|((b[o+2] as i64)<<8)|(b[o+3] as i64) }
27func si_low(c: i64) -> i64 { if c >= 65 { if c <= 90 { return c + 32 } } if c == 9 { return 32 } if c == 10 { return 32 } if c == 13 { return 32 } return c }
28
29// append one byte (lowercased/sanitized) to out if room; return new length.
30func si_put(out: *u8, o: i64, ch: i64) -> i64 { if o < SI_TEXTCAP { out[o] = si_low(ch) as u8; return o + 1 } return o }
31
32// harvest tEXt + uncompressed-iTXt values from png[0..n) into out (lowercased, space-joined). returns len.
33func si_extract(png: *u8, n: i64, out: *u8) -> i64 {
34 var o: i64 = 0
35 if n <= 8 { return 0 }
36 var off: i64 = 8
37 var guard: i64 = 0
38 while off + 12 <= n {
39 if guard > SI_MAGIC_100000 { off = n } else {
40 guard = guard + 1
41 let len: i64 = si_be32(png, off)
42 let t0: i64 = png[off+4] as i64
43 let t1: i64 = png[off+5] as i64
44 let t2: i64 = png[off+6] as i64
45 let t3: i64 = png[off+7] as i64
46 let dataoff: i64 = off + 8
47 if len < 0 { off = n } else {
48 var dend: i64 = dataoff + len
49 if dend > n { dend = n } // truncated prefix: scan what we have
50 var istext: i64 = 0
51 if t0==116 { if t1==69 { if t2==88 { if t3==116 { istext = 1 } } } }
52 var isitxt: i64 = 0
53 if t0==105 { if t1==84 { if t2==88 { if t3==116 { isitxt = 1 } } } }
54 if istext == 1 {
55 var kwe: i64 = dend
56 var p: i64 = dataoff
57 while p < dend { if png[p] == (0 as u8) { if kwe == dend { kwe = p } } p = p + 1 }
58 if kwe < dend {
59 var v: i64 = kwe + 1
60 while v < dend { o = si_put(out, o, png[v] as i64); v = v + 1 }
61 o = si_put(out, o, 32)
62 }
63 }
64 if isitxt == 1 {
65 var kwe2: i64 = dend
66 var p2: i64 = dataoff
67 while p2 < dend { if png[p2] == (0 as u8) { if kwe2 == dend { kwe2 = p2 } } p2 = p2 + 1 }
68 if kwe2 + 2 < dend {
69 let cflag: i64 = png[kwe2+1] as i64
70 var lange: i64 = dend
71 var p3: i64 = kwe2 + 3
72 while p3 < dend { if png[p3] == (0 as u8) { if lange == dend { lange = p3 } } p3 = p3 + 1 }
73 var transe: i64 = dend
74 var p4: i64 = lange + 1
75 while p4 < dend { if png[p4] == (0 as u8) { if transe == dend { transe = p4 } } p4 = p4 + 1 }
76 let textoff: i64 = transe + 1
77 if cflag == 0 { if textoff <= dend {
78 var v2: i64 = textoff
79 while v2 < dend { o = si_put(out, o, png[v2] as i64); v2 = v2 + 1 }
80 o = si_put(out, o, 32)
81 } }
82 }
83 }
84 let nextoff: i64 = dataoff + len + 4
85 if nextoff <= off { off = n } else { off = nextoff }
86 }
87 }
88 }
89 return o
90}
91
92// child: read the PNG prefix at `path`, extract text, atomically append "cid\ttext\n".
93func si_one(cid: *u8, path: *u8) -> i64 {
94 let fd: i64 = sys_openat_rd(path)
95 if fd < 0 { return 0 - 1 }
96 let png: *u8 = sys_mmap(SI_READ + 16)
97 let nrd: i64 = sys_read(fd, png, SI_READ)
98 sys_close(fd)
99 if nrd <= 8 { return 0 - 1 }
100 let text: *u8 = sys_mmap(SI_TEXTCAP + 16)
101 let tl: i64 = si_extract(png, nrd, text)
102 let line: *u8 = sys_mmap(SI_TEXTCAP + 128)
103 var lo: i64 = 0
104 var c: i64 = 0
105 while c < 69 { line[lo] = cid[c]; lo = lo + 1; c = c + 1 }
106 line[lo] = 9 as u8; lo = lo + 1
107 var i: i64 = 0
108 while i < tl { line[lo] = text[i]; lo = lo + 1; i = i + 1 }
109 line[lo] = 10 as u8; lo = lo + 1
110 let ofd: i64 = sys_openat_append(SI_OUT, 0x1a4)
111 if ofd < 0 { return 0 - 1 }
112 sys_write(ofd, line, lo)
113 sys_close(ofd)
114 return 0
115}
116
117func main(argc: i64, argv: *i64) -> i64 {
118 var start: i64 = 0
119 var count: i64 = SI_MAGIC_1000000000
120 if argc >= 3 { start = si_atoi(argv[1] as *u8); count = si_atoi(argv[2] as *u8) }
121 let endl: i64 = start + count
122 let szp: *i64 = sys_mmap(16) as *i64
123 let buf: *u8 = sys_read_file("knowledge/status/galx_cid_paths.tsv" as *u8, szp)
124 let sz: i64 = szp[0]
125 if (buf as i64) == 0 { si_p("SEARCHIDX FAIL: no sidecar\n" as *u8); sys_exit(1); return 1 }
126 let st: *i64 = sys_mmap(16) as *i64
127 let cid: *u8 = sys_mmap(96)
128 let path: *u8 = sys_mmap(SI_MAGIC_2048)
129 var ls: i64 = 0
130 var i: i64 = 0
131 var lineno: i64 = 0
132 var done: i64 = 0
133 while i <= sz {
134 var isnl: i64 = 0
135 if i == sz { isnl = 1 } else { if buf[i] == (10 as u8) { isnl = 1 } }
136 if isnl == 1 {
137 let llen: i64 = i - ls
138 if llen > 70 {
139 if lineno >= start { if lineno < endl {
140 if buf[ls + 69] == (9 as u8) {
141 var c: i64 = 0
142 while c < 69 { cid[c] = buf[ls + c]; c = c + 1 } cid[69] = 0 as u8
143 var po: i64 = 0
144 var p: i64 = ls + 70
145 while p < i { path[po] = buf[p]; po = po + 1; p = p + 1 } path[po] = 0 as u8
146 let pid: i64 = sys_fork()
147 if pid == 0 { si_one(cid, path); sys_exit(0) }
148 sys_wait4(pid, st, 0)
149 done = done + 1
150 if (done % 500) == 0 { si_p(" search-idx done=" as *u8); si_n(done); si_p("\n" as *u8) }
151 }
152 }}
153 lineno = lineno + 1
154 }
155 ls = i + 1
156 }
157 i = i + 1
158 }
159 si_p("SEARCHIDX range done=" as *u8); si_n(done); si_p("\n" as *u8)
160 sys_exit(0)
161 return 0
162}