nx_galx_lora_export.nx source
↩ module page · 178 lines · 9330 B
1// nx_galx_lora_export.nx -- SOVEREIGN LoRA dataset export. Reads the favorite flag set (galx_fav.log,
2// latest-vote-per-cid, keep v==1), resolves each favorited image to its source path (sidecar) and its
3// prompt caption (PNG tEXt/iTXt "prompt" key), and writes a training manifest:
4// knowledge/lora/favorites/manifest.tsv -> <image_path>\t<caption> (one line per favorite)
5// This is the ready-to-train dataset a LoRA / i2i pipeline consumes (the next rung). NishiLang only.
6// Usage: nx_galx_lora_export [favlog] [sidecar] -- license_tier: ORIGINAL
7// Default signal = the GALLERY-SIDE SSOT (/volume1/ai/galx/knowledge/status/*) when readable: the fav
8// log is WRITTEN by the gallery daemon across the GALLERY!=VAULT boundary; absolute reads cross it.
9// Falls back to the nishihost-relative pair when the gallery volume is unreadable. The summary line
10// ALWAYS prints favsrc= -- the source actually read announces itself.
11import "nx_syscalls.nx"
12const FAV_MAGIC_200000: i64 = 200000
13const FAV_MAGIC_2048: i64 = 2048
14const FAV_MAGIC_16384: i64 = 16384
15const FAV_MAGIC_262144: i64 = 262144
16
17const FAV_LOG: *u8 = "knowledge/status/galx_fav.log" as *u8
18const SIDECAR: *u8 = "knowledge/status/galx_cid_paths.tsv" as *u8
19const OUT: *u8 = "knowledge/lora/favorites/manifest.tsv" as *u8
20const FAV_LOG_GX: *u8 = "/volume1/ai/galx/knowledge/status/galx_fav.log" as *u8
21const SIDECAR_GX: *u8 = "/volume1/ai/galx/knowledge/status/galx_cid_paths.tsv" as *u8
22
23func le_p(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
24func le_n(v: i64) -> i64 { let b: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m;sys_write(1,"-" as *u8,1)}; let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=48;k=1}; while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1}; var i: i64=0; while i<k{b[i]=t[k-1-i];i=i+1}; sys_write(1,b,k); return 0 }
25func le_mkdir(path: *u8) -> i64 { return __syscall(258, 0-100, path as i64, 0x1ff, 0, 0, 0) }
26func le_be32(b: *u8, o: i64) -> i64 { return ((b[o] as i64)<<24)|((b[o+1] as i64)<<16)|((b[o+2] as i64)<<8)|(b[o+3] as i64) }
27func le_cideq(a: *u8, ao: i64, b: *u8, bo: i64) -> i64 { var k: i64=0; while k<69 { if a[ao+k]!=b[bo+k] { return 0 } k=k+1 } return 1 }
28
29// keyword at [ko,ke) equals "prompt"?
30func le_is_prompt(buf: *u8, ko: i64, ke: i64) -> i64 {
31 if ke - ko != 6 { return 0 }
32 let p: *u8 = "prompt" as *u8
33 var i: i64 = 0
34 while i < 6 { if buf[ko+i] != p[i] { return 0 } i = i + 1 }
35 return 1
36}
37// extract the "prompt" tEXt/iTXt value from a PNG prefix into out (sanitize tab/newline->space). len.
38func le_prompt(png: *u8, n: i64, out: *u8, cap: i64) -> i64 {
39 if n <= 8 { return 0 }
40 var off: i64 = 8
41 var guard: i64 = 0
42 while off + 12 <= n {
43 if guard > FAV_MAGIC_200000 { off = n } else {
44 guard = guard + 1
45 let len: i64 = le_be32(png, off)
46 let t0: i64=png[off+4] as i64; let t1: i64=png[off+5] as i64; let t2: i64=png[off+6] as i64; let t3: i64=png[off+7] as i64
47 let dataoff: i64 = off + 8
48 if len < 0 { off = n } else {
49 var dend: i64 = dataoff + len
50 if dend > n { dend = n }
51 var istext: i64=0; if t0==116 { if t1==69 { if t2==88 { if t3==116 { istext=1 } } } }
52 var isitxt: i64=0; if t0==105 { if t1==84 { if t2==88 { if t3==116 { isitxt=1 } } } }
53 if istext == 1 {
54 var kwe: i64 = dend; var p: i64 = dataoff
55 while p < dend { if png[p]==(0 as u8) { if kwe==dend { kwe=p } } p=p+1 }
56 if kwe < dend { if le_is_prompt(png, dataoff, kwe) == 1 {
57 var o: i64 = 0; var v: i64 = kwe + 1
58 while v < dend { if o < cap-1 { var c: i64=png[v] as i64; if c==9 {c=32} if c==10 {c=32} if c==13 {c=32} out[o]=c as u8; o=o+1 } v=v+1 }
59 out[o]=0 as u8; return o
60 } }
61 }
62 if isitxt == 1 {
63 var kwe2: i64 = dend; var p2: i64 = dataoff
64 while p2 < dend { if png[p2]==(0 as u8) { if kwe2==dend { kwe2=p2 } } p2=p2+1 }
65 if kwe2 + 2 < dend { if le_is_prompt(png, dataoff, kwe2) == 1 {
66 let cflag: i64 = png[kwe2+1] as i64
67 var lange: i64=dend; var p3: i64=kwe2+3
68 while p3<dend { if png[p3]==(0 as u8) { if lange==dend { lange=p3 } } p3=p3+1 }
69 var transe: i64=dend; var p4: i64=lange+1
70 while p4<dend { if png[p4]==(0 as u8) { if transe==dend { transe=p4 } } p4=p4+1 }
71 let textoff: i64=transe+1
72 if cflag==0 { if textoff<=dend {
73 var o: i64=0; var v2: i64=textoff
74 while v2<dend { if o<cap-1 { var c: i64=png[v2] as i64; if c==9{c=32} if c==10{c=32} if c==13{c=32} out[o]=c as u8; o=o+1 } v2=v2+1 }
75 out[o]=0 as u8; return o
76 } }
77 } }
78 }
79 let nextoff: i64 = dataoff + len + 4
80 if nextoff <= off { off = n } else { off = nextoff }
81 }
82 }
83 }
84 return 0
85}
86// sidecar: cid (69) -> path into out. 1 if found.
87func le_sidecar(sc: *u8, cid: *u8, out: *u8) -> i64 {
88 let szp: *i64 = sys_mmap(16) as *i64
89 let b: *u8 = sys_read_file(sc, szp)
90 let sz: i64 = szp[0]
91 if (b as i64) == 0 { return 0 }
92 var i: i64 = 0; var ls: i64 = 0
93 while i < sz {
94 if b[i]==(10 as u8) {
95 var eq: i64=1; var k: i64=0
96 while k<69 { if b[ls+k]!=cid[k]{eq=0} k=k+1 }
97 if b[ls+69]!=(9 as u8){eq=0}
98 if eq==1 { var o: i64=0; var p: i64=ls+70; while p<i { out[o]=b[p]; o=o+1; p=p+1 } out[o]=0 as u8; return 1 }
99 ls=i+1
100 }
101 i=i+1
102 }
103 return 0
104}
105
106func main(argc: i64, argv: *i64) -> i64 {
107 // GALLERY!=VAULT bridge: prefer the gallery-side SSOT pair; argv overrides exist for fixtures.
108 var favlog: *u8 = FAV_LOG
109 var sidecar: *u8 = SIDECAR
110 let gfd: i64 = sys_openat_rd(FAV_LOG_GX)
111 if gfd >= 0 { sys_close(gfd); favlog = FAV_LOG_GX; sidecar = SIDECAR_GX }
112 if argc >= 2 { favlog = argv[1] as *u8 }
113 if argc >= 3 { sidecar = argv[2] as *u8 }
114 le_mkdir("knowledge/lora" as *u8)
115 le_mkdir("knowledge/lora/favorites" as *u8)
116 let szp: *i64 = sys_mmap(16) as *i64
117 let fav: *u8 = sys_read_file(favlog, szp)
118 let fsz: i64 = szp[0]
119 let outfd: i64 = sys_openat_wr(OUT, 0x1a4)
120 if outfd < 0 { le_p("LORA-EXPORT FAIL: cannot write manifest\n" as *u8); sys_exit(1); return 1 }
121 var exported: i64 = 0
122 if (fav as i64) != 0 {
123 // collect line offsets of cid (and v) ; then walk backward dedup keep latest v==1
124 let CAP: i64 = FAV_MAGIC_200000
125 let coff: *i64 = sys_mmap(8 * CAP) as *i64
126 let cv: *i64 = sys_mmap(8 * CAP) as *i64
127 var m: i64 = 0
128 var i: i64 = 0; var ls: i64 = 0
129 while i <= fsz {
130 var nl: i64 = 0
131 if i == fsz { nl = 1 } else { if fav[i]==(10 as u8) { nl = 1 } }
132 if nl == 1 { if i - ls >= 69 { if m < CAP { coff[m]=ls; if fav[ls+70]==(48 as u8){cv[m]=0}else{cv[m]=1} m=m+1 } } ls=i+1 }
133 i = i + 1
134 }
135 let seen: *i64 = sys_mmap(8 * CAP) as *i64
136 var sn: i64 = 0
137 let cid: *u8 = sys_mmap(96)
138 let path: *u8 = sys_mmap(FAV_MAGIC_2048)
139 let cap2: *u8 = sys_mmap(FAV_MAGIC_16384)
140 var x: i64 = m - 1
141 while x >= 0 {
142 let off: i64 = coff[x]
143 var dup: i64 = 0
144 var y: i64 = 0
145 while y < sn { if le_cideq(fav, seen[y], fav, off) == 1 { dup = 1; y = sn } else { y = y + 1 } }
146 if dup == 0 {
147 seen[sn] = off; sn = sn + 1
148 if cv[x] == 1 {
149 var c: i64 = 0
150 while c < 69 { cid[c] = fav[off+c]; c = c + 1 } cid[69] = 0 as u8
151 if le_sidecar(sidecar, cid, path) == 1 {
152 // read a 256KB prefix for the prompt
153 let pfd: i64 = sys_openat_rd(path)
154 if pfd >= 0 {
155 let png: *u8 = sys_mmap(FAV_MAGIC_262144 + 16)
156 let nrd: i64 = sys_read(pfd, png, FAV_MAGIC_262144)
157 sys_close(pfd)
158 le_prompt(png, nrd, cap2, FAV_MAGIC_16384)
159 // write <path>\t<caption>\n
160 var pl: i64 = 0; while path[pl]!=(0 as u8){pl=pl+1}
161 sys_write(outfd, path, pl)
162 sys_write(outfd, "\t" as *u8, 1)
163 var cl: i64 = 0; while cap2[cl]!=(0 as u8){cl=cl+1}
164 sys_write(outfd, cap2, cl)
165 sys_write(outfd, "\n" as *u8, 1)
166 exported = exported + 1
167 }
168 }
169 }
170 }
171 x = x - 1
172 }
173 }
174 sys_close(outfd)
175 le_p("LORA-EXPORT ok favorites_exported=" as *u8); le_n(exported); le_p(" favsrc=" as *u8); le_p(favlog); le_p(" -> " as *u8); le_p(OUT); le_p("\n" as *u8)
176 sys_exit(0)
177 return 0
178}