nx_medrec.nx source
↩ module page · 254 lines · 11195 B
1// nx_medrec.nx -- MEDIA-RECOVERY ADMISSION CORE: decide whether an archived capture is REALLY
2// the media, or the host's tombstone wearing a valid image header.
3//
4// module: nishi-core.archive.medrec
5// depends: nx_cdx_parse.nx (rows/fields -- NOT re-parsed here, Rule 15)
6// capability: PURE DECISION CORE (zero I/O)
7//
8// WHY THIS EXISTS -- MEASURED LIVE 2026-08-06 against the Wayback CDX index, never assumed (Rule 4):
9// nx_media_xrecover admits a recovered capture on is_image(): GIF/JPEG/PNG/BMP magic bytes. That is a
10// STRUCTURE test and it CANNOT fail on a host placeholder, because a placeholder IS a structurally
11// valid JPEG. Measured:
12// i.imgur.com "image not found" -> digest VXEDWGPHSNERWHDOUD6YWRWNT4ZOLEX4, status 200, image/jpeg,
13// sizes 335 / 339 / 457 / 789 / 827 B -- the SAME digest across captures 2013-04-20 .. 2026-06-17.
14// Real media recovered from that same host -> 6PYVXUDUV5WT3VSTAK33KZYTSI4BA56Z at 1,090,473 B and
15// INAME2WHG3WPFAVNJ3URHYKXTB6FH46R at 937,110 B -- each digest UNIQUE to its image.
16// So a pipeline gated only on magic bytes reports SUCCESS on an unbounded number of identical ~400-byte
17// "not found" images, and its success count RISES as the host deletes MORE.
18// *A ROW COUNT IS NOT EVIDENCE. Structure is not identity.
19//
20// *TWO SIGNALS, BOTH READ FROM THE CDX ROW ITSELF -- a tombstone is rejected for ZERO bandwidth:
21// (1) DIGEST IDENTITY -- a known placeholder digest is never the media.
22// (2) SIZE FLOOR -- supplied by the CALLER, never baked (Rule 11).
23//
24// *AND THE GENERAL FORM, which needs no list -- but ONLY as a CONJUNCTION, and the reason matters.
25// A placeholder is byte-identical every time it is served, so its captures collapse to ONE digest at a
26// SMALL size. Digest-collapse ALONE proves nothing: measured, the real 1 MB image OT6mUiy.jpg also
27// carries one digest (6PYVXUD..) across all 3 of its captures -- an unchanged file always does. What
28// separates them is SIZE, and note the CDX length column is the COMPRESSED WARC RECORD size, not the
29// payload (OT6mUiy reads 1,090,473 / 1,083,440 / 1,083,439 for bit-identical content), so it is a
30// coarse floor and never an equality test. mr_rowset_is_tombstone therefore requires small AND
31// collapsed, and answers 0 whenever either leg fails. Build intelligence, do not maintain a blocklist
32// (Rule 25) -- but do not let a heuristic claim evidence it does not have.
33//
34// *FAIL-CLOSED: an absent or unparseable field is MR_UNKNOWN, never ADMIT. A CDX length of "-" (revisit
35// records carry it) is UNKNOWN, not zero. "We cannot tell" and "it is the media" are different answers.
36// license_tier: ORIGINAL LIB.
37
38import "nx_syscalls.nx"
39import "nx_cdx_parse.nx"
40const MR_MAGIC_1125899906842597: i64 = 1125899906842597
41
42const MR_ADMIT: i64 = 1
43const MR_UNKNOWN: i64 = 0 - 1
44const MR_REJ_STATUS: i64 = 0 - 2
45const MR_REJ_TOMBSTONE: i64 = 0 - 3
46const MR_REJ_TOOSMALL: i64 = 0 - 4
47
48// *MOST REDDIT MEDIA URLS ARE NOT IMAGE URLS. Measured on 12 real r/pics posts from 2016-01 via the
49// arctic-shift archive: 3 direct, 8 host PAGE urls (imgur.com/<id>), 2 gallery, 1 offsite article.
50// A CDX query on a PAGE url recovers HTML, never pixels -- so the kind decides the next step, and a
51// pipeline that skips this step silently archives markup and calls it media.
52const MR_URL_DIRECT: i64 = 1
53const MR_URL_PAGE: i64 = 2
54const MR_URL_GALLERY: i64 = 3
55const MR_URL_OTHER: i64 = 4
56
57// CDX field ordinals: urlkey timestamp original mimetype statuscode digest length
58const MR_F_TS: i64 = 1
59const MR_F_ORIG: i64 = 2
60const MR_F_MIME: i64 = 3
61const MR_F_STATUS: i64 = 4
62const MR_F_DIGEST: i64 = 5
63const MR_F_LENGTH: i64 = 6
64
65func mr_lc(c: i64) -> i64 {
66 if c >= 0x41 { if c <= 0x5A { return c + 0x20 } }
67 return c
68}
69
70// case-insensitive substring test over NUL-terminated strings.
71func mr_ci_has(hay: *u8, needle: *u8) -> i64 {
72 var hn: i64 = 0
73 while hay[hn] != (0 as u8) { hn = hn + 1 }
74 var nn: i64 = 0
75 while needle[nn] != (0 as u8) { nn = nn + 1 }
76 if nn == 0 { return 1 }
77 if nn > hn { return 0 }
78 var i: i64 = 0
79 while i <= hn - nn {
80 var j: i64 = 0
81 var ok: i64 = 1
82 while j < nn {
83 if mr_lc(hay[i+j] as i64) != mr_lc(needle[j] as i64) { ok = 0; j = nn } else { j = j + 1 }
84 }
85 if ok == 1 { return 1 }
86 i = i + 1
87 }
88 return 0
89}
90
91// compare a buffer slice against a NUL-terminated string; exact length match required.
92func mr_slice_eq(buf: *u8, off: i64, len: i64, s: *u8) -> i64 {
93 if len < 0 { return 0 }
94 var i: i64 = 0
95 while i < len {
96 if s[i] == (0 as u8) { return 0 }
97 if buf[off+i] != s[i] { return 0 }
98 i = i + 1
99 }
100 if s[len] != (0 as u8) { return 0 }
101 return 1
102}
103
104// parse a slice as a non-negative integer. ANY non-digit (notably the "-" a revisit record carries in
105// the length column) yields MR_UNKNOWN -- fail-closed, never a silent 0.
106func mr_slice_int(buf: *u8, off: i64, len: i64) -> i64 {
107 if len <= 0 { return MR_UNKNOWN }
108 var v: i64 = 0
109 var i: i64 = 0
110 while i < len {
111 let c: i64 = buf[off+i] as i64
112 if c < 0x30 { return MR_UNKNOWN }
113 if c > 0x39 { return MR_UNKNOWN }
114 v = v * 10 + (c - 0x30)
115 i = i + 1
116 }
117 return v
118}
119
120// KNOWN placeholder digests. One row per entry, each MEASURED, with the host and observation window
121// recorded above. Data-driven (Rule 11) -- new rows need no code change.
122func mr_tombstone_count() -> i64 { return 2 }
123func mr_tombstone_at(i: i64) -> *u8 {
124 // i.imgur.com "image not found", observed 2013-04-20 .. 2026-06-17 at 335..827 B
125 if i == 0 { return "VXEDWGPHSNERWHDOUD6YWRWNT4ZOLEX4" }
126 // the empty-payload SHA1 every archive writes for a bodiless capture (301/302/redirect rows).
127 // Measured: 13 of 40 baseline reddit user-page rows carried exactly this one digest.
128 if i == 1 { return "3I42H3S6NNFQ2MSVX7XZKYAYSCX5QBYJ" }
129 return ""
130}
131
132func mr_digest_is_tombstone(buf: *u8, off: i64, len: i64) -> i64 {
133 let n: i64 = mr_tombstone_count()
134 var i: i64 = 0
135 while i < n {
136 if mr_slice_eq(buf, off, len, mr_tombstone_at(i)) == 1 { return 1 }
137 i = i + 1
138 }
139 return 0
140}
141
142// the whole admission decision for ONE CDX row. o2 is a caller-owned 2-slot scratch for cdx_field.
143func mr_row_admit(buf: *u8, rs: i64, re: i64, min_bytes: i64, o2: *i64) -> i64 {
144 if min_bytes < 0 { return MR_UNKNOWN }
145 if cdx_field(buf, rs, re, MR_F_STATUS, o2) == 0 { return MR_UNKNOWN }
146 let st: i64 = mr_slice_int(buf, o2[0], o2[1])
147 if st == MR_UNKNOWN { return MR_UNKNOWN }
148 if st != 200 { return MR_REJ_STATUS }
149 if cdx_field(buf, rs, re, MR_F_DIGEST, o2) == 0 { return MR_UNKNOWN }
150 if mr_digest_is_tombstone(buf, o2[0], o2[1]) == 1 { return MR_REJ_TOMBSTONE }
151 if cdx_field(buf, rs, re, MR_F_LENGTH, o2) == 0 { return MR_UNKNOWN }
152 let sz: i64 = mr_slice_int(buf, o2[0], o2[1])
153 if sz == MR_UNKNOWN { return MR_UNKNOWN }
154 if sz < min_bytes { return MR_REJ_TOOSMALL }
155 return MR_ADMIT
156}
157
158// row payload size, or MR_UNKNOWN.
159func mr_row_size(buf: *u8, rs: i64, re: i64, o2: *i64) -> i64 {
160 if cdx_field(buf, rs, re, MR_F_LENGTH, o2) == 0 { return MR_UNKNOWN }
161 return mr_slice_int(buf, o2[0], o2[1])
162}
163
164// *THE GENERAL TOMBSTONE SIGNATURE. Requires >= 2 captures -- a single capture is not a pattern, so it
165// answers 0 rather than risking a false positive on a one-shot save.
166func mr_rowset_is_tombstone(buf: *u8, rowstarts: *i64, rowends: *i64, rn: i64, small_bytes: i64, o2: *i64) -> i64 {
167 if rn < 2 { return 0 }
168 if small_bytes <= 0 { return 0 }
169 if cdx_field(buf, rowstarts[0], rowends[0], MR_F_DIGEST, o2) == 0 { return 0 }
170 let d0off: i64 = o2[0]
171 let d0len: i64 = o2[1]
172 var i: i64 = 0
173 while i < rn {
174 let sz: i64 = mr_row_size(buf, rowstarts[i], rowends[i], o2)
175 if sz == MR_UNKNOWN { return 0 }
176 if sz >= small_bytes { return 0 }
177 if cdx_field(buf, rowstarts[i], rowends[i], MR_F_DIGEST, o2) == 0 { return 0 }
178 if o2[1] != d0len { return 0 }
179 var j: i64 = 0
180 while j < d0len {
181 if buf[o2[0]+j] != buf[d0off+j] { return 0 }
182 j = j + 1
183 }
184 i = i + 1
185 }
186 return 1
187}
188
189// pick the capture most likely to BE the media: admitted rows only, largest payload wins. *THE KNOWN-GOOD
190// nx_archive_site_viewer TAKES THE EARLIEST 200 CAPTURE, and the earliest crawl of a site is routinely its
191// most partial -- selection by RECORD SIZE beats selection by DATE for media completeness. Returns the ROW
192// INDEX into rowstarts/rowends, or MR_UNKNOWN when nothing qualifies.
193func mr_pick_best(buf: *u8, rowstarts: *i64, rowends: *i64, rn: i64, min_bytes: i64, o2: *i64) -> i64 {
194 if rn <= 0 { return MR_UNKNOWN }
195 var best: i64 = MR_UNKNOWN
196 var bestsz: i64 = 0 - 1
197 var i: i64 = 0
198 while i < rn {
199 if mr_row_admit(buf, rowstarts[i], rowends[i], min_bytes, o2) == MR_ADMIT {
200 let sz: i64 = mr_row_size(buf, rowstarts[i], rowends[i], o2)
201 if sz > bestsz { bestsz = sz; best = i }
202 }
203 i = i + 1
204 }
205 return best
206}
207
208// *THE URL->MEDIA JOIN KEY. Hashes the PATH ONLY, deliberately excluding the host, because the CDX
209// 'original' column is absolute (http://www.page3.com:80/x.jpg) while page markup is usually relative
210// (/x.jpg) -- a host+path key hashes those differently and the join then fails SILENTLY, looking like
211// "the archive lacks this image". The host is not lost: the index lives under archive/<host>/.
212// Query dropped; case folded (megastar serves 3pxShadow.png whose CDX urlkey is 3pxshadow.png).
213// In the LIB so a GATE can import and prove it -- an organ with main() cannot be imported.
214func mr_canon_hash(buf: *u8, off: i64, len: i64) -> i64 {
215 let end0: i64 = off + len
216 var p: i64 = off
217 var i: i64 = off
218 var sch: i64 = 0 - 1
219 while i + 2 < end0 {
220 if buf[i]==(58 as u8) { if buf[i+1]==(47 as u8) { if buf[i+2]==(47 as u8) { if sch < 0 { sch = i + 3 } } } }
221 i = i + 1
222 }
223 if sch >= 0 {
224 p = end0
225 i = sch
226 var found: i64 = 0
227 while i < end0 {
228 if found == 0 { if buf[i]==(47 as u8) { p = i; found = 1 } }
229 i = i + 1
230 }
231 }
232 var h: i64 = MR_MAGIC_1125899906842597
233 i = p
234 while i < end0 {
235 let c: i64 = buf[i] as i64
236 if c == 63 { i = end0 } else {
237 h = (h * 131) + mr_lc(c)
238 i = i + 1
239 }
240 }
241 if h < 0 { h = 0 - h }
242 return h & 0x7fffffffffffffff
243}
244
245// classify a media URL so the caller knows whether it can be fetched directly or must be resolved first.
246func mr_url_kind(url: *u8) -> i64 {
247 if mr_ci_has(url, "imgur.com/gallery/") == 1 { return MR_URL_GALLERY }
248 if mr_ci_has(url, "imgur.com/a/") == 1 { return MR_URL_GALLERY }
249 if mr_ci_has(url, "i.imgur.com/") == 1 { return MR_URL_DIRECT }
250 if mr_ci_has(url, "i.redd.it/") == 1 { return MR_URL_DIRECT }
251 if mr_ci_has(url, "preview.redd.it/") == 1 { return MR_URL_DIRECT }
252 if mr_ci_has(url, "imgur.com/") == 1 { return MR_URL_PAGE }
253 return MR_URL_OTHER
254}