nx_galx_index.nx source
↩ module page · 148 lines · 6230 B
1// nx_galx_index.nx -- SOVEREIGN gallery index builder (replaces the bash build_view_index.sh; all logic
2// is NishiLang per the ecosystem law). Reads the CID->path sidecar, derives each image's capture timestamp
3// from its filename (YYYYMMDD_HHMMSS), sorts NEWEST-FIRST (iterative heapsort -- no recursion depth risk at
4// 230k), de-dups by CID, and writes knowledge/status/galx_view_index.txt (one 69-char CID per line) that
5// the daemon's /api/list + grid read. NO image is skipped: a row whose filename has no parseable timestamp
6// still gets indexed with key 0 (sorts to the end), so every ingested image is browsable.
7// Usage: nx_galx_index (no args). license_tier: ORIGINAL
8import "nx_syscalls.nx"
9import "nx_itoa_lib.nx" // shared MSB-first emitter (zero-alloc)
10const GI_MAGIC_1000000: i64 = 1000000
11
12const GI_CAP: i64 = 300000 // max images indexed (corpus is ~230k)
13const GI_SIDE: *u8 = "knowledge/status/galx_cid_paths.tsv" as *u8
14const GI_OUT: *u8 = "knowledge/status/galx_view_index.txt" as *u8
15
16func gi_p(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
17// MIGRATED to the shared emitter (debt 1785563586). The old body mmapped a scratch buffer
18// per call and never freed it. At PAGE granularity that is 4096B leaked PER CALL -- the
19// defect that took 28.5GB of a 36GB host in nx_ts_lumadiff (2MB input, ~3.66M calls).
20// nxi_* is MSB-first, allocates NOTHING, and emits identical bytes including the sign.
21func gi_n(v: i64) -> i64 { nxi_out(v); return 0 }
22
23func gi_isdig(c: i64) -> i64 { if c < 48 { return 0 } if c > 57 { return 0 } return 1 }
24
25// derive the capture timestamp (YYYYMMDD_HHMMSS) anywhere in the path -> i64 = YYYYMMDD*1000000+HHMMSS.
26// 0 if none (the image is still indexed, just sorted last).
27func gi_ts(buf: *u8, off: i64, end: i64) -> i64 {
28 var i: i64 = off
29 while i + 15 <= end {
30 var ok: i64 = 1
31 var j: i64 = 0
32 while j < 8 { if gi_isdig(buf[i+j] as i64) == 0 { ok = 0; j = 8 } else { j = j + 1 } }
33 if ok == 1 { if buf[i+8] != (95 as u8) { ok = 0 } } // '_'
34 if ok == 1 {
35 var k: i64 = 0
36 while k < 6 { if gi_isdig(buf[i+9+k] as i64) == 0 { ok = 0; k = 6 } else { k = k + 1 } }
37 }
38 if ok == 1 {
39 var d: i64 = 0
40 var q: i64 = 0
41 while q < 8 { d = d*10 + ((buf[i+q] as i64) - 48); q = q + 1 }
42 var t2: i64 = 0
43 q = 0
44 while q < 6 { t2 = t2*10 + ((buf[i+9+q] as i64) - 48); q = q + 1 }
45 return d * GI_MAGIC_1000000 + t2
46 }
47 i = i + 1
48 }
49 return 0
50}
51
52// iterative sift-down for heapsort: restore the max-heap property at `root` within heap size `n`.
53func gi_sift(idx: *i64, key: *i64, root0: i64, n: i64) -> i64 {
54 var root: i64 = root0
55 var go: i64 = 1
56 while go == 1 {
57 let child: i64 = 2 * root + 1
58 if child >= n { go = 0 } else {
59 var swap: i64 = root
60 if key[idx[swap]] < key[idx[child]] { swap = child }
61 if child + 1 < n { if key[idx[swap]] < key[idx[child+1]] { swap = child + 1 } }
62 if swap == root { go = 0 } else {
63 let t: i64 = idx[root]; idx[root] = idx[swap]; idx[swap] = t
64 root = swap
65 }
66 }
67 }
68 return 0
69}
70// heapsort idx[0..n) ASCENDING by key[idx[*]] (we emit in reverse for newest-first).
71func gi_heapsort(idx: *i64, key: *i64, n: i64) -> i64 {
72 if n < 2 { return 0 }
73 var start: i64 = n / 2 - 1
74 while start >= 0 { gi_sift(idx, key, start, n); start = start - 1 }
75 var end: i64 = n - 1
76 while end > 0 {
77 let t: i64 = idx[0]; idx[0] = idx[end]; idx[end] = t
78 gi_sift(idx, key, 0, end)
79 end = end - 1
80 }
81 return 0
82}
83// two CIDs (69 bytes at rowoff in the sidecar buffer) equal?
84func gi_cideq(buf: *u8, a: i64, b: i64) -> i64 {
85 var i: i64 = 0
86 while i < 69 { if buf[a+i] != buf[b+i] { return 0 } i = i + 1 }
87 return 1
88}
89
90func main() -> i64 {
91 let szp: *i64 = sys_mmap(16) as *i64
92 let buf: *u8 = sys_read_file(GI_SIDE, szp)
93 let n: i64 = szp[0]
94 if (buf as i64) == 0 { gi_p("INDEX FAIL: no sidecar\n" as *u8); sys_exit(1); return 1 }
95 let rowoff: *i64 = sys_mmap(8 * GI_CAP) as *i64 // byte offset of each row's CID in buf
96 let key: *i64 = sys_mmap(8 * GI_CAP) as *i64 // timestamp sort key per row
97 let idx: *i64 = sys_mmap(8 * GI_CAP) as *i64 // permutation to sort
98 var cnt: i64 = 0
99 var ls: i64 = 0
100 var i: i64 = 0
101 while i <= n {
102 var isnl: i64 = 0
103 if i == n { isnl = 1 } else { if buf[i] == (10 as u8) { isnl = 1 } }
104 if isnl == 1 {
105 let llen: i64 = i - ls
106 if llen > 70 { // 69 CID + tab + >=1 path char
107 if buf[ls + 69] == (9 as u8) {
108 if cnt < GI_CAP {
109 rowoff[cnt] = ls
110 key[cnt] = gi_ts(buf, ls + 70, i)
111 idx[cnt] = cnt
112 cnt = cnt + 1
113 }
114 }
115 }
116 ls = i + 1
117 }
118 i = i + 1
119 }
120 gi_p("INDEX rows=" as *u8); gi_n(cnt); gi_p(" sorting...\n" as *u8)
121 gi_heapsort(idx, key, cnt)
122 // write newest-first (reverse of ascending sort), de-duping consecutive identical CIDs.
123 let fd: i64 = sys_openat_wr(GI_OUT, 0x1a4)
124 if fd < 0 { gi_p("INDEX FAIL: cannot write index\n" as *u8); sys_exit(1); return 1 }
125 let out: *u8 = sys_mmap(70 * GI_CAP)
126 var o: i64 = 0
127 var prev: i64 = 0 - 1
128 var written: i64 = 0
129 var r: i64 = cnt - 1
130 while r >= 0 {
131 let ro: i64 = rowoff[idx[r]]
132 var dup: i64 = 0
133 if prev >= 0 { if gi_cideq(buf, ro, prev) == 1 { dup = 1 } }
134 if dup == 0 {
135 var c: i64 = 0
136 while c < 69 { out[o] = buf[ro + c]; o = o + 1; c = c + 1 }
137 out[o] = 10 as u8; o = o + 1
138 written = written + 1
139 prev = ro
140 }
141 r = r - 1
142 }
143 sys_write(fd, out, o)
144 sys_close(fd)
145 gi_p("INDEX OK written=" as *u8); gi_n(written); gi_p(" -> " as *u8); gi_p(GI_OUT); gi_p("\n" as *u8)
146 sys_exit(0)
147 return 0
148}