nx_gfxpapers_ingest.nx source
↩ module page · 439 lines · 19427 B
1// nx_gfxpapers_ingest.nx -- MIRROR THE GRAPHICS PAPER INDEX INTO THE ESTATE LIBRARY SO IT CANNOT ROT.
2//
3// Operator 2026-08-15: ingest kesen.realtimerendering.com "in a non rottable way", and then -- correctly --
4// "shouldnt these be in papers and our library?". The first cut wrote to knowledge/gfxmirror/, an ad-hoc
5// directory beside the three homes the estate already has, which is the orphan the no-orphans doctrine
6// forbids and which nx_spendgate had already warned about by naming nx_papers_index as the top incumbent.
7// The homes, and why THIS content belongs in the middle one:
8// knowledge/fetched/ -- raw bodies from nx_research_fetch, transient working material
9// knowledge/library/ -- CURATED external reference documents, topic-prefixed, .fail markers for misses
10// knowledge/papers/ -- .nxpap native papers (nx_paper_native) plus authored .conf/.md
11// Ke-Sen Huang's index is the canonical catalogue of real-time-rendering conference papers (SIGGRAPH,
12// SIGGRAPH Asia, EG, I3D, EGSR, SCA, SGP, HPG, PG, NPAR, SMI, GI; 2000-2026). Its venue pages are reference
13// documents, so they land in the library under a caller-supplied prefix that keeps them attributable in a
14// flat namespace shared with 7,000 other references.
15//
16// WHAT "NON-ROTTABLE" MEANS, precisely: a bookmark is not an ingest. A URL list rots the moment a site
17// moves or a domain lapses, and the failure is SILENT because a dead link and an unvisited link look
18// identical in a registry. This stores the BYTES with their provenance. MEASURED on the first run: one
19// indexed page (conference2012.html) was ALREADY 404 on capture day, and the entire www.realtimerendering.com
20// host returns 403 to this client for all 236 pages while kesen.realtimerendering.com serves them -- so a
21// registry of the hrefs exactly as the index writes them would have been 236 dead entries that look healthy.
22//
23// CLOSURE, NOT ONE HOP. "All of its content" is a fixed point, not a depth guess: every mirrored page is
24// itself scanned for further pages in the SAME namespace and the queue grows until a pass adds nothing.
25// The anchor is the site's own `kesen/` path segment, so the crawl can never wander onto third-party hosts
26// -- mirroring every paper PDF this index points at would be a different, and far ruder, program.
27//
28// WHAT IT DELIBERATELY DOES NOT DO: parse paper titles. A brittle HTML scrape would be an unvalidated
29// measurement dressed as data. Bytes and provenance are exact and checkable today; extraction is a later
30// rung that can run against the mirror OFFLINE, repeatedly, without re-fetching a single page.
31//
32// argv: <index-url> <out-dir> [fetcher-elf] [name-prefix]
33// license_tier: ORIGINAL No hw writes (Rule 26).
34import "nx_syscalls.nx"
35import "nx_itoa_lib.nx"
36import "nx_tool_run.nx"
37
38const GP_CAP: i64 = 4194304
39const GP_NAMES_MAX: i64 = 4096
40const GP_NAMELEN: i64 = 128
41const GP_URLLEN: i64 = 512
42const GP_TIMEOUT_MS: i64 = 120000
43const GP_ARGV_SLOTS: i64 = 4
44const GP_MODE_DIR: i64 = 493
45const GP_QUOTE: i64 = 34
46const GP_SQUOTE: i64 = 39
47const GP_GT: i64 = 62
48const GP_SP: i64 = 32
49const GP_NL: i64 = 10
50const GP_ANCHOR_LEN: i64 = 6
51
52func gp_p(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
53func gp_n(v: i64) -> i64 { nxi_out(v); return 0 }
54func gp_len(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
55
56func gp_find(buf: *u8, n: i64, from: i64, needle: *u8) -> i64 {
57 let nn: i64 = gp_len(needle)
58 if nn == 0 { return 0-1 }
59 var i: i64 = from
60 while i + nn <= n {
61 var k: i64 = 0
62 var hit: i64 = 1
63 while k < nn {
64 if buf[i+k] != needle[k] { hit = 0; k = nn }
65 if hit == 1 { k = k + 1 }
66 }
67 if hit == 1 { return i }
68 i = i + 1
69 }
70 return 0-1
71}
72
73// The fetcher prints TLS timing lines before the response, so archived bytes must start at the status line
74// or the mirror is polluted with this run's latencies -- which would also make every re-fetch differ from
75// the last for reasons having nothing to do with the page.
76func gp_http_start(buf: *u8, n: i64) -> i64 { return gp_find(buf, n, 0, "HTTP/" as *u8) }
77
78// A 404 IS A VALID HTTP RESPONSE, AND STORING ITS BODY IS HOW AN ARCHIVE SILENTLY FILLS WITH ROT.
79// Measured: three pages returned EXACTLY 8073 bytes each -- one error template, three "mirrored pages".
80func gp_status(buf: *u8, n: i64, at: i64) -> i64 {
81 var i: i64 = at
82 var go: i64 = 1
83 while go == 1 {
84 if i >= n { return 0-1 }
85 if buf[i] == (GP_SP as u8) { go = 0 }
86 if go == 1 { i = i + 1 }
87 }
88 i = i + 1
89 if i + 2 >= n { return 0-1 }
90 var v: i64 = 0
91 var d: i64 = 0
92 while d < 3 {
93 let c: i64 = buf[i+d] as i64
94 if c < 48 { return 0-1 }
95 if c > 57 { return 0-1 }
96 v = v*10 + (c-48)
97 d = d + 1
98 }
99 return v
100}
101
102// Collect every in-namespace page name in buf into names[], skipping duplicates. Returns the new count, or
103// -1 if the table filled -- a name table that quietly stopped collecting is a SMALLER number that reads
104// like a cleaner site, and every later count would inherit it as a total.
105func gp_collect(buf: *u8, n: i64, names: *u8, nn0: i64) -> i64 {
106 var nn: i64 = nn0
107 var at: i64 = 0
108 var go: i64 = 1
109 while go == 1 {
110 let h: i64 = gp_find(buf, n, at, "kesen/" as *u8)
111 if h < 0 { go = 0 }
112 if go == 1 {
113 var q: i64 = h + GP_ANCHOR_LEN
114 var ln: i64 = 0
115 let nm: *u8 = ((names as i64) + nn*GP_NAMELEN) as *u8
116 var rd: i64 = 1
117 while rd == 1 {
118 if q >= n { rd = 0 }
119 if rd == 1 {
120 let c: i64 = buf[q] as i64
121 if c == GP_QUOTE { rd = 0 }
122 if c == GP_SQUOTE { rd = 0 }
123 if c == GP_GT { rd = 0 }
124 if c == GP_SP { rd = 0 }
125 if c == GP_NL { rd = 0 }
126 if rd == 1 { if ln < GP_NAMELEN-1 { nm[ln] = buf[q] as u8; ln = ln + 1 } }
127 if rd == 1 { q = q + 1 }
128 }
129 }
130 nm[ln] = 0 as u8
131 var keep: i64 = 0
132 if ln > 4 { keep = 1 }
133 if keep == 1 {
134 var d: i64 = 0
135 while d < nn {
136 let other: *u8 = ((names as i64) + d*GP_NAMELEN) as *u8
137 var same: i64 = 1
138 var z: i64 = 0
139 while z <= ln {
140 if other[z] != nm[z] { same = 0; z = ln + 1 }
141 if same == 1 { z = z + 1 }
142 }
143 if same == 1 { keep = 0; d = nn }
144 if keep == 1 { d = d + 1 }
145 }
146 }
147 if keep == 1 {
148 if nn < GP_NAMES_MAX-1 { nn = nn + 1 } else { return 0-1 }
149 }
150 at = h + GP_ANCHOR_LEN
151 }
152 }
153 return nn
154}
155
156// GR7 (2026-08-17): a SECOND ROOT through the same organ. gp_root_of derives the rebuild host from the
157// index url itself -- the host that ANSWERED -- instead of the kesen literal, so any archive whose pages
158// live under one namespace can be mirrored. gp_has_sub is the bounded substring test the collector uses
159// to refuse junk names (index_files MSO artifacts, dot-dot traversal).
160func gp_root_of(idxurl: *u8, out: *u8) -> i64 {
161 var i: i64 = 0
162 var slashes: i64 = 0
163 var o: i64 = 0
164 var go: i64 = 1
165 while go == 1 {
166 let c: i64 = idxurl[i] as i64
167 if c == 0 { go = 0 }
168 if go == 1 {
169 out[o] = c as u8
170 o = o + 1
171 if c == 47 { slashes = slashes + 1; if slashes == 3 { go = 0 } }
172 i = i + 1
173 if o >= GP_URLLEN - 2 { go = 0 }
174 }
175 }
176 if slashes < 3 { out[o] = 47 as u8; o = o + 1 }
177 out[o] = 0 as u8
178 return o
179}
180func gp_has_sub(nm: *u8, ln: i64, needle: *u8) -> i64 {
181 var i: i64 = 0
182 while i < ln {
183 var j: i64 = 0
184 var ok: i64 = 1
185 var go2: i64 = 1
186 while go2 == 1 {
187 if needle[j] == (0 as u8) { go2 = 0 }
188 if go2 == 1 { if i + j >= ln { ok = 0; go2 = 0 } }
189 if go2 == 1 { if nm[i+j] != needle[j] { ok = 0; go2 = 0 } }
190 if go2 == 1 { j = j + 1 }
191 }
192 if ok == 1 { return 1 }
193 i = i + 1
194 }
195 return 0
196}
197// The Advances idiom (measured 2026-08-17 on the live index): year pages are RELATIVE hrefs like
198// s2021/index.html, plus absolute http://advances.../<path>/index.html forms and index_files MSO junk.
199// So this collector finds the ANCHOR (e.g. /index.htm), walks BACK to the opening quote for the name
200// start, forward to the closing delimiter for its end, strips an absolute link to its path, refuses
201// junk, and dedupes exactly as the legacy collector does. The legacy kesen lane is untouched.
202func gp_collect2(buf: *u8, n: i64, names: *u8, nn0: i64, anchor: *u8) -> i64 {
203 var nn: i64 = nn0
204 var at: i64 = 0
205 var go: i64 = 1
206 while go == 1 {
207 let h: i64 = gp_find(buf, n, at, anchor)
208 if h < 0 { go = 0 }
209 if go == 1 {
210 var s: i64 = h
211 var back: i64 = 1
212 while back == 1 {
213 if s <= 0 { back = 0 }
214 if back == 1 {
215 let c0: i64 = buf[s-1] as i64
216 if c0 == GP_QUOTE { back = 0 }
217 if c0 == GP_SQUOTE { back = 0 }
218 if c0 == GP_GT { back = 0 }
219 if c0 == GP_SP { back = 0 }
220 if c0 == GP_NL { back = 0 }
221 if back == 1 { s = s - 1 }
222 if back == 1 { if h - s > GP_NAMELEN { back = 0; s = h } }
223 }
224 }
225 var q: i64 = h
226 var rd: i64 = 1
227 while rd == 1 {
228 if q >= n { rd = 0 }
229 if rd == 1 {
230 let c: i64 = buf[q] as i64
231 if c == GP_QUOTE { rd = 0 }
232 if c == GP_SQUOTE { rd = 0 }
233 if c == GP_GT { rd = 0 }
234 if c == GP_SP { rd = 0 }
235 if c == GP_NL { rd = 0 }
236 if rd == 1 { q = q + 1 }
237 }
238 }
239 let nm: *u8 = ((names as i64) + nn*GP_NAMELEN) as *u8
240 var ln: i64 = 0
241 var z: i64 = s
242 while z < q { if ln < GP_NAMELEN-1 { nm[ln] = buf[z]; ln = ln + 1 } z = z + 1 }
243 nm[ln] = 0 as u8
244 if gp_has_sub(nm, ln, "://" as *u8) == 1 {
245 var p3: i64 = 0
246 var f3: i64 = 0-1
247 while p3 + 2 < ln { if f3 < 0 { if nm[p3] == (58 as u8) { if nm[p3+1] == (47 as u8) { if nm[p3+2] == (47 as u8) { f3 = p3 } } } } p3 = p3 + 1 }
248 var hs: i64 = f3 + 3
249 var f4: i64 = 0-1
250 while hs < ln { if f4 < 0 { if nm[hs] == (47 as u8) { f4 = hs } } hs = hs + 1 }
251 if f4 < 0 { ln = 0; nm[0] = 0 as u8 } else {
252 var w2: i64 = 0
253 while f4 + 1 + w2 < ln { nm[w2] = nm[f4 + 1 + w2]; w2 = w2 + 1 }
254 ln = w2
255 nm[ln] = 0 as u8
256 }
257 }
258 var keep: i64 = 0
259 if ln > 4 { keep = 1 }
260 if keep == 1 { if gp_has_sub(nm, ln, ".." as *u8) == 1 { keep = 0 } }
261 if keep == 1 { if gp_has_sub(nm, ln, "index_files" as *u8) == 1 { keep = 0 } }
262 if keep == 1 { if nm[0] == (47 as u8) {
263 var w3: i64 = 0
264 while w3 + 1 < ln { nm[w3] = nm[w3+1]; w3 = w3 + 1 }
265 ln = ln - 1
266 nm[ln] = 0 as u8
267 if ln <= 4 { keep = 0 }
268 } }
269 if keep == 1 {
270 var d: i64 = 0
271 while d < nn {
272 let other: *u8 = ((names as i64) + d*GP_NAMELEN) as *u8
273 var same: i64 = 1
274 var z2: i64 = 0
275 while z2 <= ln {
276 if other[z2] != nm[z2] { same = 0; z2 = ln + 1 }
277 if same == 1 { z2 = z2 + 1 }
278 }
279 if same == 1 { keep = 0; d = nn }
280 if keep == 1 { d = d + 1 }
281 }
282 }
283 if keep == 1 {
284 if nn < GP_NAMES_MAX-1 { nn = nn + 1 } else { return 0-1 }
285 }
286 at = h + 1
287 }
288 }
289 return nn
290}
291func main(argc: i64, argv: *i64) -> i64 {
292 if argc < 3 { gp_p("usage: nx_gfxpapers_ingest <index-url> <out-dir> [fetcher-elf] [name-prefix]\n" as *u8); return 3 }
293 let idxurl: *u8 = argv[1] as *u8
294 let outdir: *u8 = argv[2] as *u8
295 var fetch: *u8 = "./nx_https_get_cli.elf" as *u8
296 if argc >= 4 { fetch = argv[3] as *u8 }
297 var prefix: *u8 = "" as *u8
298 if argc >= 5 { prefix = argv[4] as *u8 }
299 // GR7: optional 6th arg = href anchor -> collect2 + host derived from the index url; absent = legacy kesen lane, byte-identical
300 var anchor: *u8 = 0 as *u8
301 if argc >= 6 { anchor = argv[5] as *u8 }
302 let rootbuf: *u8 = sys_mmap(GP_URLLEN)
303 gp_root_of(idxurl, rootbuf)
304 sys_mkdir(outdir, GP_MODE_DIR)
305 let buf: *u8 = sys_mmap(GP_CAP)
306 let olen: *i64 = sys_mmap(16) as *i64
307 let trunc: *i64 = sys_mmap(16) as *i64
308 let av: *i64 = sys_mmap(GP_ARGV_SLOTS*8) as *i64
309 let url: *u8 = sys_mmap(GP_URLLEN)
310 let dst: *u8 = sys_mmap(GP_URLLEN)
311 let names: *u8 = sys_mmap(GP_NAMES_MAX*GP_NAMELEN)
312 av[0] = fetch as i64
313 av[1] = idxurl as i64
314 av[2] = 0
315 olen[0] = 0
316 trunc[0] = 0
317 let rc0: i64 = tr_run_capture_tr(fetch, av, buf, GP_CAP, olen, GP_TIMEOUT_MS, trunc)
318 if olen[0] <= 0 { gp_p("INGEST-REFUSED index-fetch-empty rc=" as *u8); gp_n(rc0); gp_p("\n" as *u8); return 2 }
319 if trunc[0] != 0 {
320 gp_p("INGEST-REFUSED index-truncated -- a partial index yields a partial population\n" as *u8)
321 return 2
322 }
323 var nn: i64 = 0
324 if (anchor as i64) != 0 { nn = gp_collect2(buf, olen[0], names, 0, anchor) } else { nn = gp_collect(buf, olen[0], names, 0) }
325 if nn < 0 { gp_p("INGEST-REFUSED name-table-full on the index alone\n" as *u8); return 2 }
326 gp_p("index url=" as *u8); gp_p(idxurl)
327 gp_p(" captured=" as *u8); gp_n(olen[0])
328 gp_p(" seeded=" as *u8); gp_n(nn)
329 gp_p("\n" as *u8)
330 var okc: i64 = 0
331 var failc: i64 = 0
332 var deadc: i64 = 0
333 var truncc: i64 = 0
334 var total_bytes: i64 = 0
335 var capped: i64 = 0
336 var i2: i64 = 0
337 // nn GROWS inside this loop as pages reveal further pages; the walk ends only when the queue is
338 // exhausted, which IS the fixed point.
339 while i2 < nn {
340 let nm2: *u8 = ((names as i64) + i2*GP_NAMELEN) as *u8
341 // Rebuild on the host that ANSWERED, not the one the hrefs name: www.realtimerendering.com 403s
342 // this client for every page while kesen.realtimerendering.com serves the same files.
343 var uo: i64 = 0
344 var pfx: *u8 = "https://kesen.realtimerendering.com/" as *u8
345 if (anchor as i64) != 0 { pfx = rootbuf }
346 var pz: i64 = 0
347 while pfx[pz] != (0 as u8) { url[uo] = pfx[pz]; uo = uo + 1; pz = pz + 1 }
348 var mz: i64 = 0
349 while nm2[mz] != (0 as u8) { if uo < GP_URLLEN-1 { url[uo] = nm2[mz]; uo = uo + 1 } mz = mz + 1 }
350 url[uo] = 0 as u8
351 var doff: i64 = 0
352 var dz: i64 = 0
353 while outdir[dz] != (0 as u8) { dst[doff] = outdir[dz]; doff = doff + 1; dz = dz + 1 }
354 dst[doff] = 47 as u8
355 doff = doff + 1
356 var qz: i64 = 0
357 while prefix[qz] != (0 as u8) { if doff < GP_URLLEN-1 { dst[doff] = prefix[qz]; doff = doff + 1 } qz = qz + 1 }
358 mz = 0
359 while nm2[mz] != (0 as u8) { if doff < GP_URLLEN-1 { var dc: i64 = nm2[mz] as i64; if dc == 47 { dc = 95 } dst[doff] = dc as u8; doff = doff + 1 } mz = mz + 1 }
360 dst[doff] = 0 as u8
361 av[0] = fetch as i64
362 av[1] = url as i64
363 av[2] = 0
364 olen[0] = 0
365 trunc[0] = 0
366 let rc: i64 = tr_run_capture_tr(fetch, av, buf, GP_CAP, olen, GP_TIMEOUT_MS, trunc)
367 var hb: i64 = 0-1
368 if olen[0] > 0 { hb = gp_http_start(buf, olen[0]) }
369 if trunc[0] != 0 { truncc = truncc + 1 }
370 var st: i64 = 0-1
371 if hb >= 0 { st = gp_status(buf, olen[0], hb) }
372 var alive: i64 = 0
373 if st >= 200 { if st < 300 { alive = 1 } }
374 if hb >= 0 { if alive == 0 {
375 deadc = deadc + 1
376 gp_p("GP-DEAD status=" as *u8); gp_n(st)
377 gp_p(" url=" as *u8); gp_p(url); gp_p("\n" as *u8)
378 } }
379 if alive == 1 {
380 let body: *u8 = ((buf as i64) + hb) as *u8
381 let blen: i64 = olen[0] - hb
382 let fd: i64 = sys_openat_wr(dst, MODE_0644)
383 var wrote: i64 = 0
384 if fd >= 0 { wrote = sys_write(fd, body, blen); sys_close(fd) }
385 if wrote == blen {
386 okc = okc + 1
387 total_bytes = total_bytes + blen
388 gp_p("GP\t" as *u8); gp_n(blen)
389 gp_p("\t" as *u8); gp_p(dst)
390 gp_p("\t" as *u8); gp_p(url)
391 gp_p("\n" as *u8)
392 // CLOSURE: this page may name pages the index never did.
393 var grown: i64 = 0
394 if (anchor as i64) != 0 { grown = gp_collect2(body, blen, names, nn, anchor) } else { grown = gp_collect(body, blen, names, nn) }
395 if grown < 0 { capped = 1 } else { nn = grown }
396 }
397 // A SHORT WRITE IS A CORRUPT MIRROR and must never be counted as a stored page.
398 if wrote != blen {
399 failc = failc + 1
400 gp_p("GP-FAILED short-write wrote=" as *u8); gp_n(wrote)
401 gp_p(" of=" as *u8); gp_n(blen)
402 gp_p(" path=" as *u8); gp_p(dst); gp_p("\n" as *u8)
403 }
404 }
405 if hb < 0 {
406 failc = failc + 1
407 gp_p("GP-FAILED no-http-response rc=" as *u8); gp_n(rc)
408 gp_p(" captured=" as *u8); gp_n(olen[0])
409 gp_p(" url=" as *u8); gp_p(url); gp_p("\n" as *u8)
410 }
411 i2 = i2 + 1
412 }
413 gp_p("\n=== GFXPAPERS LIBRARY MIRROR ===\n" as *u8)
414 gp_p("reachable=" as *u8); gp_n(nn)
415 gp_p(" mirrored=" as *u8); gp_n(okc)
416 gp_p(" dead=" as *u8); gp_n(deadc)
417 gp_p(" failed=" as *u8); gp_n(failc)
418 gp_p(" truncated=" as *u8); gp_n(truncc)
419 gp_p(" bytes=" as *u8); gp_n(total_bytes)
420 gp_p("\n" as *u8)
421 // dead is its OWN bucket, never folded into failed: a 404 has already rotted and needs a different
422 // remedy (find where it moved) than a page we could not reach at all.
423 var recon: i64 = 0
424 if okc + failc + deadc == nn { recon = 1 }
425 gp_p("partition mirrored+dead+failed=" as *u8); gp_n(okc+failc+deadc)
426 gp_p(" reachable=" as *u8); gp_n(nn)
427 if recon == 1 { gp_p(" RECONCILES\n" as *u8) }
428 if recon == 0 { gp_p(" LEAKS -- these numbers are not publishable\n" as *u8) }
429 // NO SILENT CAP: if the queue filled, the crawl is a FLOOR and must say so.
430 if capped == 1 {
431 gp_p("REFUSED-CAP name-table filled at " as *u8); gp_n(GP_NAMES_MAX)
432 gp_p(" -- reachable is a FLOOR, not a total; raise the cap and re-run\n" as *u8)
433 return 1
434 }
435 if recon == 0 { return 1 }
436 if truncc > 0 { return 1 }
437 if okc == 0 { gp_p("MIRROR UNMEASURED -- zero pages stored\n" as *u8); return 1 }
438 return 0
439}