code wiki / _hdl_build / nx_galx_repair.nx
nx_galx_repair.nx source
↩ module page · 218 lines · 10062 B
1// nx_galx_repair.nx -- recover the index entries the builder silently drops.
2//
3// FOUND BY MEASURING THE WHOLE POPULATION, NOT A SAMPLE. nx_galx_audit emitted two counters for the
4// same set -- `header_entries 235,555` and `occupied 233,312` -- and the 2,243 gap reconciled
5// EXACTLY with the TSV rows whose char-69 is not a tab.
6// ★★★★★★ TWO COUNTERS OF THE SAME POPULATION ARE A FREE AUDIT; ONE COUNTER IS A CLAIM.
7// The builder counts every newline into n_entries but only PLACES rows it can frame, so 2,243
8// images were counted, dropped, and never servable -- with no error, and a header that said they
9// were there.
10//
11// The classes, measured (they sum to the 2,243, no residual):
12// 1,171 no cid at all — 622 empty lines + 549 lines starting with a tab (path, no cid)
13// 619 cid field 138 chars = TWO 69-char cids run together
14// 327/106/10 cid field 74-76 chars
15// 5 207 chars = THREE cids run together
16// 138 = 2x69 and 207 = 3x69 is the signature of a writer that never terminated its line: the row
17// was flushed as a bare `<cid>` with no tab, no path and no newline, and the next row landed on it.
18// ⇒ THE PATH FOR THOSE CIDS IS NOT IN THIS FILE AT ALL. It cannot be parsed back out; it has to be
19// recovered from somewhere else. Two somewhere-elses exist:
20// B) an orphaned cid whose blob is still on disk at the conventional <blobdir><cid>.png
21// C) an orphaned PATH whose cid can be RECOMPUTED from the image's own GENREC factors
22// ★ A ROW THAT CANNOT BE PARSED IS NOT NECESSARILY DATA THAT IS LOST — ASK WHAT ELSE KNOWS IT.
23//
24// Usage: nx_galx_repair <tsv> <idx> <blob> <blobdir> [--apply]
25// Dry run by default. The insert is idempotent, so --apply is safe to repeat.
26// license_tier: ORIGINAL
27
28import "nx_syscalls.nx"
29import "nx_strconv.nx"
30import "nx_store_ingest.nx"
31import "nx_galx_cid_index.nx"
32import "nx_galx_cidput.nx"
33const RP_MAGIC_1024: i64 = 1024
34const RP_MAGIC_4096: i64 = 4096
35
36const RP_CIDLEN: i64 = 69
37const RP_MAXPNG: i64 = 33554432
38
39func rp_puts(s: *u8) -> i64 {
40 var n: i64 = 0
41 while s[n] != (0 as u8) { n = n + 1 }
42 return sys_write(1, s, n)
43}
44func rp_i(v: i64) -> i64 {
45 let b: *u8 = sys_mmap(32)
46 return sys_write(1, b, nx_strconv_format_i64(v, b))
47}
48// The eol helper, extracted -- see nx_galx_join. Writing this inline is how it got written wrong
49// twice: the sentinel-exit form discards the newline index it just found.
50func rp_eol(buf: *u8, n: i64, from: i64) -> i64 {
51 var e: i64 = from
52 var eol: i64 = n
53 var scanning: i64 = 1
54 while scanning == 1 {
55 if e >= n { eol = n; scanning = 0 }
56 else {
57 if buf[e] == (0x0A as u8) { eol = e; scanning = 0 } else { e = e + 1 }
58 }
59 }
60 return eol
61}
62func rp_ishex(c: i64) -> i64 {
63 if c >= 0x30 { if c <= 0x39 { return 1 } }
64 if c >= 0x61 { if c <= 0x66 { return 1 } }
65 if c >= 0x41 { if c <= 0x46 { return 1 } }
66 return 0
67}
68// Is there a well-formed cid (nxc1- + 64 hex) at buf[at]? Framing is checked, never assumed --
69// accepting a near-miss would insert a key nothing can ever look up.
70func rp_iscid(buf: *u8, at: i64, lim: i64) -> i64 {
71 if at + RP_CIDLEN > lim { return 0 }
72 if buf[at] != (0x6E as u8) { return 0 } // n
73 if buf[at+1] != (0x78 as u8) { return 0 } // x
74 if buf[at+2] != (0x63 as u8) { return 0 } // c
75 if buf[at+3] != (0x31 as u8) { return 0 } // 1
76 if buf[at+4] != (0x2D as u8) { return 0 } // -
77 var i: i64 = 5
78 while i < RP_CIDLEN {
79 if rp_ishex(buf[at + i] as i64) == 0 { return 0 }
80 i = i + 1
81 }
82 return 1
83}
84func rp_slurp(path: *u8, out: *u8, cap: i64) -> i64 {
85 let fd: i64 = sys_openat_rd(path)
86 if fd < 0 { return 0 - 1 }
87 var n: i64 = 0
88 var go: i64 = 1
89 while go == 1 {
90 let r: i64 = sys_read(fd, ((out as i64) + n) as *u8, cap - n)
91 if r <= 0 { go = 0 } else { n = n + r }
92 if n >= cap { go = 0 }
93 }
94 sys_close(fd)
95 if n >= cap { return 0 - 2 }
96 return n
97}
98
99func main(argc: i64, argv: *i64) -> i64 {
100 if argc < 5 {
101 rp_puts("usage: nx_galx_repair <tsv> <idx> <blob> <blobdir> [--apply]\n" as *u8)
102 return 2
103 }
104 var apply: i64 = 0
105 var emit: i64 = 0
106 if argc > 5 { let f: *u8 = argv[5] as *u8; if f[0] == (0x2D as u8) { if f[2] == (0x61 as u8) { apply = 1 } else { emit = 1 } } }
107 let idxp: *u8 = argv[2] as *u8
108 let blobp: *u8 = argv[3] as *u8
109 let blobdir: *u8 = argv[4] as *u8
110
111 let sp: *i64 = sys_mmap(16) as *i64
112 let buf: *u8 = sys_read_file(argv[1] as *u8, sp)
113 if (buf as i64) == 0 { rp_puts("tsv unreadable\n" as *u8); return 3 }
114 let n: i64 = sp[0]
115
116 let cid: *u8 = sys_mmap(128)
117 let probe: *u8 = sys_mmap(RP_MAGIC_1024)
118 let png: *u8 = sys_mmap(RP_MAXPNG + RP_MAGIC_4096)
119 let ccid: *u8 = sys_mmap(128)
120 let canon: *u8 = sys_mmap(K_MAGIC_8192)
121 let rk: *u8 = sys_mmap(128)
122 let nlb: *u8 = sys_mmap(16)
123
124 var rows: i64 = 0
125 var wellformed: i64 = 0
126 var orphan_cid: i64 = 0
127 var orphan_cid_recovered: i64 = 0
128 var orphan_path: i64 = 0
129 var orphan_path_recovered: i64 = 0
130 var unrecoverable: i64 = 0
131
132 var p: i64 = 0
133 while p < n {
134 let eol: i64 = rp_eol(buf, n, p)
135 if eol > p {
136 rows = rows + 1
137 // A well-formed row is exactly: cid at offset 0, tab at 69. Everything else is damage.
138 var wf: i64 = 0
139 if rp_iscid(buf, p, eol) == 1 {
140 if p + RP_CIDLEN < eol { if buf[p + RP_CIDLEN] == (0x09 as u8) { wf = 1 } }
141 }
142 if wf == 1 { wellformed = wellformed + 1 }
143 else {
144 // ---- damaged row: harvest every cid token and any absolute path ----
145 var q: i64 = p
146 var ncid: i64 = 0
147 var pathat: i64 = 0 - 1
148 var pathlen: i64 = 0
149 while q < eol {
150 if rp_iscid(buf, q, eol) == 1 {
151 ncid = ncid + 1
152 // B) orphaned cid -> is its blob still on disk under the convention?
153 var o: i64 = 0
154 var k: i64 = 0
155 while blobdir[k] != (0 as u8) { probe[o] = blobdir[k]; o = o + 1; k = k + 1 }
156 k = 0
157 while k < RP_CIDLEN { cid[k] = buf[q + k]; probe[o] = buf[q + k]; o = o + 1; k = k + 1 }
158 cid[RP_CIDLEN] = 0
159 probe[o] = 0x2E; probe[o+1] = 0x70; probe[o+2] = 0x6E; probe[o+3] = 0x67
160 probe[o+4] = 0
161 orphan_cid = orphan_cid + 1
162 let stb: *u8 = sys_mmap(256)
163 if sys_fstatat(probe, stb) >= 0 {
164 if apply == 1 {
165 if nx_galx_cidput(idxp, blobp, cid, probe) >= 0 { orphan_cid_recovered = orphan_cid_recovered + 1 }
166 } else { orphan_cid_recovered = orphan_cid_recovered + 1 }
167 } else { unrecoverable = unrecoverable + 1 }
168 q = q + RP_CIDLEN
169 } else {
170 if buf[q] == (0x2F as u8) {
171 if pathat < 0 { pathat = q; pathlen = eol - q }
172 }
173 q = q + 1
174 }
175 }
176 // C) a path with NO cid: recompute the cid from the image's own GENREC factors.
177 if ncid == 0 {
178 if pathat >= 0 {
179 orphan_path = orphan_path + 1
180 var k: i64 = 0
181 while k < pathlen { probe[k] = buf[pathat + k]; k = k + 1 }
182 probe[pathlen] = 0
183 let rn: i64 = rp_slurp(probe, png, RP_MAXPNG)
184 if rn > 0 {
185 if nx_store_ingest_compute_cid(png, rn, ccid, canon, rk) >= 0 {
186 if apply == 1 {
187 if nx_galx_cidput(idxp, blobp, ccid, probe) >= 0 { orphan_path_recovered = orphan_path_recovered + 1 }
188 } else { orphan_path_recovered = orphan_path_recovered + 1 }
189 // ⚠CONSTRUCT the newline byte; do NOT write a string literal that
190 // IS one. A literal newline inside a "..." is ambiguous to this
191 // lexer -- 780 cids came out correctly and completely UNSEPARATED,
192 // which reads as "only 1 was emitted" and sent me looking for a
193 // logic bug that did not exist.
194 // ★ AN OUTPUT DEFECT CAN PERFECTLY IMITATE A COMPUTATION DEFECT.
195 if emit == 1 { sys_write(1, ccid, RP_CIDLEN); nlb[0] = 10 as u8; sys_write(1, nlb, 1) }
196 } else { unrecoverable = unrecoverable + 1 }
197 } else { unrecoverable = unrecoverable + 1 }
198 }
199 }
200 }
201 }
202 p = eol + 1
203 }
204
205 rp_puts("rows=" as *u8); rp_i(rows)
206 rp_puts(" wellformed=" as *u8); rp_i(wellformed)
207 rp_puts(" damaged=" as *u8); rp_i(rows - wellformed)
208 rp_puts("\n" as *u8)
209 rp_puts("orphan_cids=" as *u8); rp_i(orphan_cid)
210 rp_puts(" recovered=" as *u8); rp_i(orphan_cid_recovered)
211 rp_puts(" | orphan_paths=" as *u8); rp_i(orphan_path)
212 rp_puts(" recovered=" as *u8); rp_i(orphan_path_recovered)
213 rp_puts(" | unrecoverable=" as *u8); rp_i(unrecoverable)
214 rp_puts("\n" as *u8)
215 if apply == 1 { rp_puts("mode=APPLY -- restart the gallery to serve these\n" as *u8) }
216 else { rp_puts("mode=DRY-RUN (pass --apply to write)\n" as *u8) }
217 return 0
218}