code wiki / _hdl_build / nx_galx_join.nx
`nx_galx_join.nx` source

↩ module page · 181 lines · 7094 B
1// nx_galx_join.nx -- join a gen sidecar into the gallery's READER index, idempotently.
2//
3// MEASURED DEFECT THIS CLOSES (2026-08-07). The gen pipeline appends every image it produces to
4// /volume1/ai/gen/cids.tsv. The gallery server resolves /img/<cid> from a DIFFERENT file --
5// /volume1/ai/galx/knowledge/status/galx_cid_paths.tsv, compiled into a sovereign O(1) idx+blob at
6// startup. Nothing ever joined the two. Measured:
7//     gen sidecar rows ................ 109
8//     gen blobs present on disk ....... 109   (nothing lost)
9//     gen CIDs in the reader index .....  0   of 109
10// So EVERY image the gen pipeline has ever produced 404s in the gallery, and has since at least the
11// reader TSV's mtime (2026-06-18). The images were fine; the index simply never learned about them.
12// ★★★★★★ TWO INDEXES, ONE WRITTEN AND ONE READ, WITH NOTHING JOINING THEM, IS INDISTINGUISHABLE
13// FROM AN EMPTY GALLERY -- and BOTH FILES LOOK HEALTHY IN ISOLATION, which is why it survived.
14// ★ THE WRITER'S SUCCESS AND THE READER'S INPUT ARE DIFFERENT CLAIMS; ONLY THE READER'S DECIDES.
15//
16// Idempotent by construction (rule 10): a CID already in the reader index is never appended again,
17// so this is safe to run on every gen, on a clock, or twice by accident.
18//
19// ⚠THE APPEND ALONE DOES NOT PUBLISH. The server compiles the TSV into knowledge/index/galx_cid.*
20// ONCE AT STARTUP, so a joined row is invisible until the gallery restarts. This organ says so
21// explicitly rather than reporting a success that the reader cannot yet see.
22//
23// Usage:  nx_galx_join <reader_tsv> <sidecar_tsv> [--apply]
24// Default is a DRY RUN -- it reports what it would join and changes nothing.
25// license_tier: ORIGINAL
26
27import "nx_syscalls.nx"
28import "nx_strconv.nx"
29import "nx_galx_cid_index.nx"
30
31const GJ_BUCKETS: i64 = 1048576          // 2^20 >> 2*235k entries; open addressing wants headroom
32
33func gj_puts(s: *u8) -> i64 {
34    var n: i64 = 0
35    while s[n] != (0 as u8) { n = n + 1 }
36    return sys_write(1, s, n)
37}
38func gj_i(v: i64) -> i64 {
39    let b: *u8 = sys_mmap(32)
40    return sys_write(1, b, nx_strconv_format_i64(v, b))
41}
42func gj_eq(s: *u8) -> i64 {
43    var n: i64 = 0
44    while s[n] != (0 as u8) { n = n + 1 }
45    return n
46}
47
48// End of the line starting at `from` (index of the '\n', or n).
49// ⚠THIS IS A HELPER BECAUSE I GOT IT WRONG TWICE. The tempting inline form --
50//     while e < n { if buf[e] == NL { e = n + 1 } else { e = e + 1 } }
51// -- uses the sentinel to exit and THROWS AWAY THE INDEX IT JUST FOUND, so every file parses as a
52// single line. It read 235,445 rows as 1. I had already fixed this exact shape in nx_genpipe.nx and
53// reproduced it here from memory an hour later.
54// ★★★★★ A BUG YOU FIXED BY REWRITING THE LINE, RATHER THAN BY EXTRACTING THE FIX, IS A BUG YOU
55// WILL WRITE AGAIN -- the correction has to live somewhere callable or it only ever fixed one site.
56func gj_eol(buf: *u8, n: i64, from: i64) -> i64 {
57    var e: i64 = from
58    var eol: i64 = n
59    var scanning: i64 = 1
60    while scanning == 1 {
61        if e >= n { eol = n; scanning = 0 }
62        else {
63            if buf[e] == (0x0A as u8) { eol = e; scanning = 0 } else { e = e + 1 }
64        }
65    }
66    return eol
67}
68// Length of the CID field: up to the first tab, else the whole line.
69func gj_cidlen(buf: *u8, from: i64, eol: i64) -> i64 {
70    var q: i64 = from
71    while q < eol {
72        if buf[q] == (0x09 as u8) { return q - from }
73        q = q + 1
74    }
75    return eol - from
76}
77
78// open-addressed membership set over FNV-1a(cid). hash 0 means empty, so remap it -- the same
79// convention the gallery's own index builder uses, deliberately, so the two agree.
80func gj_ins(tab: *i64, h0: i64) -> i64 {
81    var h: i64 = h0
82    if h == 0 { h = 1 }
83    var b: i64 = h & (GJ_BUCKETS - 1)
84    var go: i64 = 1
85    while go == 1 {
86        if tab[b] == 0 { tab[b] = h; return 1 }
87        if tab[b] == h { return 0 }                      // already present
88        b = (b + 1) & (GJ_BUCKETS - 1)
89    }
90    return 0
91}
92func gj_has(tab: *i64, h0: i64) -> i64 {
93    var h: i64 = h0
94    if h == 0 { h = 1 }
95    var b: i64 = h & (GJ_BUCKETS - 1)
96    var go: i64 = 1
97    while go == 1 {
98        if tab[b] == 0 { return 0 }
99        if tab[b] == h { return 1 }
100        b = (b + 1) & (GJ_BUCKETS - 1)
101    }
102    return 0
103}
104
105func main(argc: i64, argv: *i64) -> i64 {
106    if argc < 3 {
107        gj_puts("usage: nx_galx_join <reader_tsv> <sidecar_tsv> [--apply]\n" as *u8)
108        return 2
109    }
110    var apply: i64 = 0
111    if argc > 3 {
112        let f: *u8 = argv[3] as *u8
113        if f[0] == (0x2D as u8) { apply = 1 }
114    }
115
116    let rp: *i64 = sys_mmap(16) as *i64
117    let rbuf: *u8 = sys_read_file(argv[1] as *u8, rp)
118    if (rbuf as i64) == 0 { gj_puts("reader index unreadable\n" as *u8); return 3 }
119    let rn: i64 = rp[0]
120
121    let sp: *i64 = sys_mmap(16) as *i64
122    let sbuf: *u8 = sys_read_file(argv[2] as *u8, sp)
123    if (sbuf as i64) == 0 { gj_puts("sidecar unreadable\n" as *u8); return 4 }
124    let sn: i64 = sp[0]
125
126    // ---- load every reader CID into the set ----
127    let tab: *i64 = sys_mmap(GJ_BUCKETS * 8 + 64) as *i64
128    var have: i64 = 0
129    var p: i64 = 0
130    while p < rn {
131        let eol: i64 = gj_eol(rbuf, rn, p)
132        let cl: i64 = gj_cidlen(rbuf, p, eol)
133        if cl > 0 {
134            gj_ins(tab, cidx_fnv1a(((rbuf as i64) + p) as *u8, cl))
135            have = have + 1
136        }
137        p = eol + 1
138    }
139
140    // ---- stream the sidecar, join what is missing ----
141    var joined: i64 = 0
142    var already: i64 = 0
143    var fd: i64 = 0 - 1
144    if apply == 1 {
145        fd = sys_openat_append(argv[1] as *u8, 0x1a4)
146        if fd < 0 { gj_puts("cannot open reader index for append\n" as *u8); return 5 }
147    }
148    p = 0
149    while p < sn {
150        let eol: i64 = gj_eol(sbuf, sn, p)
151        let cl: i64 = gj_cidlen(sbuf, p, eol)
152        if cl > 0 {
153            let h: i64 = cidx_fnv1a(((sbuf as i64) + p) as *u8, cl)
154            if gj_has(tab, h) == 1 { already = already + 1 }
155            else {
156                gj_ins(tab, h)                            // guards duplicates WITHIN the sidecar too
157                joined = joined + 1
158                if apply == 1 {
159                    sys_write(fd, ((sbuf as i64) + p) as *u8, eol - p)
160                    sys_write(fd, "\n" as *u8, 1)
161                }
162            }
163        }
164        p = eol + 1
165    }
166    if fd >= 0 { sys_close(fd) }
167
168    gj_puts("reader_rows=" as *u8);   gj_i(have)
169    gj_puts(" already_present=" as *u8); gj_i(already)
170    gj_puts(" joined=" as *u8);       gj_i(joined)
171    if apply == 1 { gj_puts(" mode=APPLY\n" as *u8) } else { gj_puts(" mode=DRY-RUN (pass --apply to write)\n" as *u8) }
172
173    if apply == 1 {
174        if joined > 0 {
175            // Say the unfinished part out loud. A row in the TSV is not yet a served image.
176            gj_puts("NOT YET SERVED: the gallery compiles this TSV into knowledge/index/galx_cid.* at STARTUP.\n" as *u8)
177            gj_puts("Restart nx_gallery_serve (cwd /volume1/ai/galx) before these CIDs resolve.\n" as *u8)
178        }
179    }
180    return 0
181}
nx_galx_join.nx source

`nx_galx_join.nx` source
