nx_galx_lora_export.nx source
↩ module page · 165 lines · 8297 B
1// nx_galx_lora_export.nx -- SOVEREIGN LoRA dataset export. Reads the favorite flag set (galx_fav.log,
2// latest-vote-per-cid, keep v==1), resolves each favorited image to its source path (sidecar) and its
3// prompt caption (PNG tEXt/iTXt "prompt" key), and writes a training manifest:
4// knowledge/lora/favorites/manifest.tsv -> <image_path>\t<caption> (one line per favorite)
5// This is the ready-to-train dataset a LoRA / i2i pipeline consumes (the next rung). NishiLang only.
6// Usage: nx_galx_lora_export (no args). license_tier: ORIGINAL
7import "nx_syscalls.nx"
8const FAV_MAGIC_200000: i64 = 200000
9const FAV_MAGIC_2048: i64 = 2048
10const FAV_MAGIC_16384: i64 = 16384
11const FAV_MAGIC_262144: i64 = 262144
12
13const FAV_LOG: *u8 = "knowledge/status/galx_fav.log" as *u8
14const SIDECAR: *u8 = "knowledge/status/galx_cid_paths.tsv" as *u8
15const OUT: *u8 = "knowledge/lora/favorites/manifest.tsv" as *u8
16
17func le_p(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
18func le_n(v: i64) -> i64 { let b: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m}; let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=48;k=1}; while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1}; var i: i64=0; while i<k{b[i]=t[k-1-i];i=i+1}; sys_write(1,b,k); return 0 }
19func le_mkdir(path: *u8) -> i64 { return __syscall(258, 0-100, path as i64, 0x1ff, 0, 0, 0) }
20func le_be32(b: *u8, o: i64) -> i64 { return ((b[o] as i64)<<24)|((b[o+1] as i64)<<16)|((b[o+2] as i64)<<8)|(b[o+3] as i64) }
21func le_cideq(a: *u8, ao: i64, b: *u8, bo: i64) -> i64 { var k: i64=0; while k<69 { if a[ao+k]!=b[bo+k] { return 0 } k=k+1 } return 1 }
22
23// keyword at [ko,ke) equals "prompt"?
24func le_is_prompt(buf: *u8, ko: i64, ke: i64) -> i64 {
25 if ke - ko != 6 { return 0 }
26 let p: *u8 = "prompt" as *u8
27 var i: i64 = 0
28 while i < 6 { if buf[ko+i] != p[i] { return 0 } i = i + 1 }
29 return 1
30}
31// extract the "prompt" tEXt/iTXt value from a PNG prefix into out (sanitize tab/newline->space). len.
32func le_prompt(png: *u8, n: i64, out: *u8, cap: i64) -> i64 {
33 if n <= 8 { return 0 }
34 var off: i64 = 8
35 var guard: i64 = 0
36 while off + 12 <= n {
37 if guard > FAV_MAGIC_200000 { off = n } else {
38 guard = guard + 1
39 let len: i64 = le_be32(png, off)
40 let t0: i64=png[off+4] as i64; let t1: i64=png[off+5] as i64; let t2: i64=png[off+6] as i64; let t3: i64=png[off+7] as i64
41 let dataoff: i64 = off + 8
42 if len < 0 { off = n } else {
43 var dend: i64 = dataoff + len
44 if dend > n { dend = n }
45 var istext: i64=0; if t0==116 { if t1==69 { if t2==88 { if t3==116 { istext=1 } } } }
46 var isitxt: i64=0; if t0==105 { if t1==84 { if t2==88 { if t3==116 { isitxt=1 } } } }
47 if istext == 1 {
48 var kwe: i64 = dend; var p: i64 = dataoff
49 while p < dend { if png[p]==(0 as u8) { if kwe==dend { kwe=p } } p=p+1 }
50 if kwe < dend { if le_is_prompt(png, dataoff, kwe) == 1 {
51 var o: i64 = 0; var v: i64 = kwe + 1
52 while v < dend { if o < cap-1 { var c: i64=png[v] as i64; if c==9 {c=32} if c==10 {c=32} if c==13 {c=32} out[o]=c as u8; o=o+1 } v=v+1 }
53 out[o]=0 as u8; return o
54 } }
55 }
56 if isitxt == 1 {
57 var kwe2: i64 = dend; var p2: i64 = dataoff
58 while p2 < dend { if png[p2]==(0 as u8) { if kwe2==dend { kwe2=p2 } } p2=p2+1 }
59 if kwe2 + 2 < dend { if le_is_prompt(png, dataoff, kwe2) == 1 {
60 let cflag: i64 = png[kwe2+1] as i64
61 var lange: i64=dend; var p3: i64=kwe2+3
62 while p3<dend { if png[p3]==(0 as u8) { if lange==dend { lange=p3 } } p3=p3+1 }
63 var transe: i64=dend; var p4: i64=lange+1
64 while p4<dend { if png[p4]==(0 as u8) { if transe==dend { transe=p4 } } p4=p4+1 }
65 let textoff: i64=transe+1
66 if cflag==0 { if textoff<=dend {
67 var o: i64=0; var v2: i64=textoff
68 while v2<dend { if o<cap-1 { var c: i64=png[v2] as i64; if c==9{c=32} if c==10{c=32} if c==13{c=32} out[o]=c as u8; o=o+1 } v2=v2+1 }
69 out[o]=0 as u8; return o
70 } }
71 } }
72 }
73 let nextoff: i64 = dataoff + len + 4
74 if nextoff <= off { off = n } else { off = nextoff }
75 }
76 }
77 }
78 return 0
79}
80// sidecar: cid (69) -> path into out. 1 if found.
81func le_sidecar(cid: *u8, out: *u8) -> i64 {
82 let szp: *i64 = sys_mmap(16) as *i64
83 let b: *u8 = sys_read_file(SIDECAR, szp)
84 let sz: i64 = szp[0]
85 if (b as i64) == 0 { return 0 }
86 var i: i64 = 0; var ls: i64 = 0
87 while i < sz {
88 if b[i]==(10 as u8) {
89 var eq: i64=1; var k: i64=0
90 while k<69 { if b[ls+k]!=cid[k]{eq=0} k=k+1 }
91 if b[ls+69]!=(9 as u8){eq=0}
92 if eq==1 { var o: i64=0; var p: i64=ls+70; while p<i { out[o]=b[p]; o=o+1; p=p+1 } out[o]=0 as u8; return 1 }
93 ls=i+1
94 }
95 i=i+1
96 }
97 return 0
98}
99
100func main() -> i64 {
101 le_mkdir("knowledge/lora" as *u8)
102 le_mkdir("knowledge/lora/favorites" as *u8)
103 let szp: *i64 = sys_mmap(16) as *i64
104 let fav: *u8 = sys_read_file(FAV_LOG, szp)
105 let fsz: i64 = szp[0]
106 let outfd: i64 = sys_openat_wr(OUT, 0x1a4)
107 if outfd < 0 { le_p("LORA-EXPORT FAIL: cannot write manifest\n" as *u8); sys_exit(1); return 1 }
108 var exported: i64 = 0
109 if (fav as i64) != 0 {
110 // collect line offsets of cid (and v) ; then walk backward dedup keep latest v==1
111 let CAP: i64 = FAV_MAGIC_200000
112 let coff: *i64 = sys_mmap(8 * CAP) as *i64
113 let cv: *i64 = sys_mmap(8 * CAP) as *i64
114 var m: i64 = 0
115 var i: i64 = 0; var ls: i64 = 0
116 while i <= fsz {
117 var nl: i64 = 0
118 if i == fsz { nl = 1 } else { if fav[i]==(10 as u8) { nl = 1 } }
119 if nl == 1 { if i - ls >= 69 { if m < CAP { coff[m]=ls; if fav[ls+70]==(48 as u8){cv[m]=0}else{cv[m]=1} m=m+1 } } ls=i+1 }
120 i = i + 1
121 }
122 let seen: *i64 = sys_mmap(8 * CAP) as *i64
123 var sn: i64 = 0
124 let cid: *u8 = sys_mmap(96)
125 let path: *u8 = sys_mmap(FAV_MAGIC_2048)
126 let cap2: *u8 = sys_mmap(FAV_MAGIC_16384)
127 var x: i64 = m - 1
128 while x >= 0 {
129 let off: i64 = coff[x]
130 var dup: i64 = 0
131 var y: i64 = 0
132 while y < sn { if le_cideq(fav, seen[y], fav, off) == 1 { dup = 1; y = sn } else { y = y + 1 } }
133 if dup == 0 {
134 seen[sn] = off; sn = sn + 1
135 if cv[x] == 1 {
136 var c: i64 = 0
137 while c < 69 { cid[c] = fav[off+c]; c = c + 1 } cid[69] = 0 as u8
138 if le_sidecar(cid, path) == 1 {
139 // read a 256KB prefix for the prompt
140 let pfd: i64 = sys_openat_rd(path)
141 if pfd >= 0 {
142 let png: *u8 = sys_mmap(FAV_MAGIC_262144 + 16)
143 let nrd: i64 = sys_read(pfd, png, FAV_MAGIC_262144)
144 sys_close(pfd)
145 le_prompt(png, nrd, cap2, FAV_MAGIC_16384)
146 // write <path>\t<caption>\n
147 var pl: i64 = 0; while path[pl]!=(0 as u8){pl=pl+1}
148 sys_write(outfd, path, pl)
149 sys_write(outfd, "\t" as *u8, 1)
150 var cl: i64 = 0; while cap2[cl]!=(0 as u8){cl=cl+1}
151 sys_write(outfd, cap2, cl)
152 sys_write(outfd, "\n" as *u8, 1)
153 exported = exported + 1
154 }
155 }
156 }
157 }
158 x = x - 1
159 }
160 }
161 sys_close(outfd)
162 le_p("LORA-EXPORT ok favorites_exported=" as *u8); le_n(exported); le_p(" -> " as *u8); le_p(OUT); le_p("\n" as *u8)
163 sys_exit(0)
164 return 0
165}