nx_gfxpapers_extract.nx source
↩ module page · 290 lines · 14427 B
1// nx_gfxpapers_extract.nx -- TURN THE MIRRORED GRAPHICS LISTINGS INTO A CITABLE PAPER INDEX.
2//
3// The library mirror (nx_gfxpapers_ingest) stores BYTES, which is what makes the corpus non-rottable but
4// does not yet make it usable: /compare needs to cite a specific paper, not a 360 KB page. This organ reads
5// the mirror OFFLINE -- no network, no re-fetch, runnable as many times as it takes to get the parse right,
6// which is precisely why the ingest deliberately stored bytes and deferred extraction rather than doing a
7// brittle scrape on the wire where every correction costs another crawl of someone else's server.
8//
9// THE ANCHOR IS `<dt><B>` ... `</B>`, AND IT WAS CHECKED FOR VARIANCE BEFORE IT WAS TRUSTED. Verified by
10// reading real mirrored bytes at both ends of the corpus -- i3d2005Papers.htm (2005, .htm) and sig2025.html
11// (2025, .html) -- so it is not a pattern fitted to the one page that motivated it. Both cases are counted
12// and reported SEPARATELY rather than merged, because a case split that is silently accepted would halve a
13// corpus without ever showing up as an error.
14//
15// ⚠HTML COMMENTS ARE SKIPPED, AND THAT IS NOT OPTIONAL. Every one of these pages carries a commented-out
16// TEMPLATE entry near the top with the same tag shape as a real one. A scanner that does not skip comments
17// measures the documentation, not the code -- the estate has paid for this law twice, and this session paid
18// for it a third time when a contract symbol written in a COMMENT flipped a published compare row to landed
19// with no capability behind it.
20//
21// ⚠TITLES ARE STRIPPED OF TAB AND NEWLINE. The row format is tab-delimited, so a title containing the
22// delimiter would silently split into phantom columns -- a producer must never emit the byte its consumer
23// uses as a separator, which is the same producer/consumer wire defect that has bitten the status planes.
24//
25// argv: <library-dir> <prefix> <out-index>
26// license_tier: ORIGINAL No hw writes (Rule 26).
27import "nx_syscalls.nx"
28import "nx_itoa_lib.nx"
29
30const GX_DIRBUF: i64 = 65536
31const GX_NAMELEN: i64 = 256
32const GX_PATHLEN: i64 = 512
33const GX_OUTBUF: i64 = 1048576
34const GX_TITLEMAX: i64 = 512
35const GX_DIRENT_RECLEN_OFF: i64 = 16
36const GX_DIRENT_NAME_OFF: i64 = 19
37const GX_TAB: i64 = 9
38const GX_NL: i64 = 10
39const GX_CR: i64 = 13
40const GX_SP: i64 = 32
41const GX_DOT: i64 = 46
42const GX_LT: i64 = 60
43const GX_GT: i64 = 62
44
45func gx_p(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
46func gx_n(v: i64) -> i64 { nxi_out(v); return 0 }
47func gx_len(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
48
49func gx_match_at(buf: *u8, n: i64, at: i64, needle: *u8) -> i64 {
50 let nn: i64 = gx_len(needle)
51 if at + nn > n { return 0 }
52 var k: i64 = 0
53 while k < nn {
54 if buf[at+k] != needle[k] { return 0 }
55 k = k + 1
56 }
57 return 1
58}
59
60func gx_streq(a: *u8, b: *u8) -> i64 {
61 var i: i64 = 0
62 while a[i] != (0 as u8) {
63 if a[i] != b[i] { return 0 }
64 i = i + 1
65 }
66 if b[i] != (0 as u8) { return 0 }
67 return 1
68}
69
70func gx_starts(name: *u8, pre: *u8) -> i64 {
71 var i: i64 = 0
72 while pre[i] != (0 as u8) {
73 if name[i] != pre[i] { return 0 }
74 i = i + 1
75 }
76 return 1
77}
78
79// Scan one mirrored page and append `<label>\t<title>` rows to out. Returns papers found; upper/lower
80// anchor counts land in ctr so a case split is VISIBLE rather than silently merged.
81func gx_scan(buf: *u8, n: i64, label: *u8, out: *u8, outn: i64, outcap: i64, ctr: *i64) -> i64 {
82 var o: i64 = outn
83 var i: i64 = 0
84 while i < n {
85 // COMMENT SKIP FIRST: a commented-out template entry is present in every one of these pages, and
86 // counting it would add one phantom paper per venue-year across the whole corpus.
87 if gx_match_at(buf, n, i, "<!--" as *u8) == 1 {
88 var j: i64 = i + 4
89 var closed: i64 = 0
90 while closed == 0 {
91 if j >= n { closed = 1; j = n }
92 if closed == 0 { if gx_match_at(buf, n, j, "-->" as *u8) == 1 { closed = 1; j = j + 3 } }
93 if closed == 0 { j = j + 1 }
94 }
95 i = j
96 }
97 if i < n {
98 var hit: i64 = 0
99 var tstart: i64 = 0
100 if gx_match_at(buf, n, i, "<dt><B>" as *u8) == 1 { hit = 1; tstart = i + 7; ctr[0] = ctr[0] + 1 }
101 if hit == 0 { if gx_match_at(buf, n, i, "<dt><b>" as *u8) == 1 { hit = 1; tstart = i + 7; ctr[1] = ctr[1] + 1 } }
102 if hit == 1 {
103 var e: i64 = tstart
104 var done: i64 = 0
105 while done == 0 {
106 if e >= n { done = 1 }
107 if done == 0 { if gx_match_at(buf, n, e, "</B>" as *u8) == 1 { done = 1 } }
108 if done == 0 { if gx_match_at(buf, n, e, "</b>" as *u8) == 1 { done = 1 } }
109 if done == 0 { e = e + 1 }
110 }
111 var tl: i64 = e - tstart
112 if tl > GX_TITLEMAX { tl = GX_TITLEMAX }
113 // ★AN EMPTY TITLE IS ITS OWN BUCKET, NOT A SILENT SKIP. First run: anchors=9984 but
114 // rows=9762, and the partition check REFUSED to publish the 222 difference. They are
115 // <dt><B></B> shells -- one per page across 235 pages, i.e. the boilerplate template these
116 // listings all carry, in a comment form the skip above does not catch. Counting them as
117 // papers would have inflated the corpus by one per venue-year; dropping them silently would
118 // have left a census that disagrees with its own scan and quietly rounds in our favour.
119 if tl <= 0 { ctr[4] = ctr[4] + 1 }
120 if tl > 0 {
121 // room check BEFORE writing: a buffer overrun here would corrupt the index rather than
122 // truncate it, and a corrupt index is worse than a short one because it still parses.
123 let need: i64 = gx_len(label) + tl + 2
124 if o + need < outcap {
125 var q: i64 = 0
126 while label[q] != (0 as u8) { out[o] = label[q]; o = o + 1; q = q + 1 }
127 out[o] = GX_TAB as u8
128 o = o + 1
129 // ★A TITLE IS TEXT, NOT MARKUP. Most listings put plain text between <B> and </B>,
130 // but the hub pages nest anchors inside it, so a raw byte copy emitted rows like
131 // `<a href="...">Symposium on Point-Based Graphics 2006</a>`. An index whose rows
132 // are half markup is not citable, and the difference is one state variable.
133 var w: i64 = 0
134 var prevsp: i64 = 0
135 var intag: i64 = 0
136 while w < tl {
137 var c: i64 = buf[tstart+w] as i64
138 var skip: i64 = 0
139 if c == GX_LT { intag = 1 }
140 if intag == 1 { skip = 1 }
141 if c == GX_GT { intag = 0 }
142 if skip == 0 {
143 // collapse the delimiter and every layout byte into single spaces
144 if c == GX_TAB { c = GX_SP }
145 if c == GX_NL { c = GX_SP }
146 if c == GX_CR { c = GX_SP }
147 var emit: i64 = 1
148 if c == GX_SP { if prevsp == 1 { emit = 0 } }
149 if emit == 1 { out[o] = c as u8; o = o + 1 }
150 if c == GX_SP { prevsp = 1 } else { prevsp = 0 }
151 }
152 w = w + 1
153 }
154 // trim trailing spaces so a row never ends in the separator it just collapsed
155 var trim: i64 = 1
156 while trim == 1 {
157 if o <= 0 { trim = 0 }
158 if trim == 1 { if out[o-1] != (GX_SP as u8) { trim = 0 } }
159 if trim == 1 { o = o - 1 }
160 }
161 out[o] = GX_NL as u8
162 o = o + 1
163 ctr[2] = ctr[2] + 1
164 } else { ctr[3] = ctr[3] + 1 }
165 }
166 i = e
167 }
168 if hit == 0 { i = i + 1 }
169 }
170 }
171 return o
172}
173
174func main(argc: i64, argv: *i64) -> i64 {
175 if argc < 4 { gx_p("usage: nx_gfxpapers_extract <library-dir> <prefix> <out-index>\n" as *u8); return 3 }
176 let dir: *u8 = argv[1] as *u8
177 let pre: *u8 = argv[2] as *u8
178 let outp: *u8 = argv[3] as *u8
179 let dfd: i64 = sys_openat_rd(dir)
180 if dfd < 0 { gx_p("EXTRACT-REFUSED cannot open dir=" as *u8); gx_p(dir); gx_p("\n" as *u8); return 2 }
181 let dbuf: *u8 = sys_mmap(GX_DIRBUF)
182 let path: *u8 = sys_mmap(GX_PATHLEN)
183 let label: *u8 = sys_mmap(GX_NAMELEN)
184 let out: *u8 = sys_mmap(GX_OUTBUF)
185 let lenp: *i64 = sys_mmap(16) as *i64
186 let ctr: *i64 = sys_mmap(64) as *i64
187 ctr[0] = 0
188 ctr[1] = 0
189 ctr[2] = 0
190 ctr[3] = 0
191 ctr[4] = 0
192 var seen: i64 = 0
193 var matched: i64 = 0
194 var read_fail: i64 = 0
195 var self_skip: i64 = 0
196 var o: i64 = 0
197 // ★ONE getdents64 CALL IS NOT A DIRECTORY LISTING. Loop until it returns 0, or a big directory is
198 // silently read as a prefix and its total published as fact -- and this directory holds 7,580 files.
199 var more: i64 = 1
200 while more == 1 {
201 let got: i64 = sys_getdents64(dfd, dbuf, GX_DIRBUF)
202 if got <= 0 { more = 0 }
203 if more == 1 {
204 var off: i64 = 0
205 while off < got {
206 let rl: *i64 = ((dbuf as i64) + off + GX_DIRENT_RECLEN_OFF) as *i64
207 var reclen: i64 = rl[0] & 65535
208 if reclen <= 0 { reclen = got }
209 let nm: *u8 = ((dbuf as i64) + off + GX_DIRENT_NAME_OFF) as *u8
210 seen = seen + 1
211 var take: i64 = 0
212 if gx_starts(nm, pre) == 1 { take = 1 }
213 if take == 1 {
214 var po: i64 = 0
215 var dz: i64 = 0
216 while dir[dz] != (0 as u8) { path[po] = dir[dz]; po = po + 1; dz = dz + 1 }
217 path[po] = 47 as u8
218 po = po + 1
219 var mz: i64 = 0
220 while nm[mz] != (0 as u8) { if po < GX_PATHLEN-1 { path[po] = nm[mz]; po = po + 1 } mz = mz + 1 }
221 path[po] = 0 as u8
222 // ★AN EXTRACTOR MUST NEVER CONSUME ITS OWN OUTPUT. The index now lives in the Library
223 // beside the pages it catalogues, so it MATCHES the very prefix this scan walks. It
224 // carries no <dt><B> anchors, so the damage would not be phantom papers -- it would be
225 // a silently inflated matched= count, i.e. a census that reports one more source than
226 // exists and re-inflates by one on every future run. Skipping by exact path costs one
227 // comparison and makes the scan idempotent by construction.
228 if gx_streq(path, outp) == 1 { take = 0; self_skip = self_skip + 1 }
229 if take == 1 {
230 matched = matched + 1
231 // label = filename minus the prefix and minus the extension: sig2025, hpg2025Papers
232 var lo: i64 = 0
233 var sk: i64 = gx_len(pre)
234 var lz: i64 = sk
235 while nm[lz] != (0 as u8) {
236 if nm[lz] == (GX_DOT as u8) { lz = lz } else { if lo < GX_NAMELEN-1 { label[lo] = nm[lz]; lo = lo + 1 } }
237 if nm[lz] == (GX_DOT as u8) { lz = gx_len(nm) } else { lz = lz + 1 }
238 }
239 label[lo] = 0 as u8
240 lenp[0] = 0
241 let fb: *u8 = sys_read_file(path, lenp)
242 if (fb as i64) == 0 { read_fail = read_fail + 1 }
243 if (fb as i64) != 0 {
244 o = gx_scan(fb, lenp[0], label, out, o, GX_OUTBUF, ctr)
245 sys_munmap(fb, lenp[0])
246 }
247 }
248 }
249 off = off + reclen
250 }
251 }
252 }
253 sys_close(dfd)
254 let ofd: i64 = sys_openat_wr(outp, MODE_0644)
255 var wrote: i64 = 0
256 if ofd >= 0 { wrote = sys_write(ofd, out, o); sys_close(ofd) }
257 gx_p("\n=== GFXPAPERS EXTRACT ===\n" as *u8)
258 gx_p("dir_entries=" as *u8); gx_n(seen)
259 gx_p(" matched=" as *u8); gx_n(matched)
260 gx_p(" self_skipped=" as *u8); gx_n(self_skip)
261 gx_p(" unreadable=" as *u8); gx_n(read_fail)
262 gx_p("\n" as *u8)
263 gx_p("anchor_upper=" as *u8); gx_n(ctr[0])
264 gx_p(" anchor_lower=" as *u8); gx_n(ctr[1])
265 gx_p(" papers=" as *u8); gx_n(ctr[2])
266 gx_p(" empty_template=" as *u8); gx_n(ctr[4])
267 gx_p(" dropped_no_room=" as *u8); gx_n(ctr[3])
268 gx_p("\n" as *u8)
269 gx_p("index_bytes=" as *u8); gx_n(o)
270 gx_p(" wrote=" as *u8); gx_n(wrote)
271 gx_p(" path=" as *u8); gx_p(outp)
272 gx_p("\n" as *u8)
273 // A PARTITION IS A CLAIM: the anchors found must equal the rows emitted plus the rows dropped, or the
274 // index silently disagrees with its own scan and neither number is quotable.
275 var recon: i64 = 0
276 if ctr[0] + ctr[1] == ctr[2] + ctr[3] + ctr[4] { recon = 1 }
277 gx_p("partition anchors=" as *u8); gx_n(ctr[0]+ctr[1])
278 gx_p(" rows+empty+dropped=" as *u8); gx_n(ctr[2]+ctr[3]+ctr[4])
279 if recon == 1 { gx_p(" RECONCILES\n" as *u8) }
280 if recon == 0 { gx_p(" LEAKS -- these numbers are not publishable\n" as *u8) }
281 // NO SILENT CAP: a dropped row means the output buffer filled, so papers is a FLOOR, not a total.
282 if ctr[3] > 0 {
283 gx_p("REFUSED-CAP output buffer filled -- papers is a FLOOR; raise GX_OUTBUF and re-run\n" as *u8)
284 return 1
285 }
286 if wrote != o { gx_p("REFUSED short-write -- the index on disk is not the index measured\n" as *u8); return 1 }
287 if recon == 0 { return 1 }
288 if ctr[2] == 0 { gx_p("EXTRACT UNMEASURED -- zero papers found\n" as *u8); return 1 }
289 return 0
290}