nx_mvault_fetch.nx source
↩ module page · 531 lines · 24579 B
1// nx_mvault_fetch.nx -- album/gallery ingest for the vault (a BunkrDownloader-class
2// capability, sovereign and data-driven).
3//
4// OPERATOR DECISION 2026-07-31: adapters are a DATA TABLE, not code. A new site is
5// an INSERT, not a rebuild. That is rule 11 (policy in data, never buried in code)
6// and it is also what keeps this from becoming the patch cascade that per-site
7// scraper code always becomes -- every host that needs a `if host == "..."` branch
8// is a redeploy, and the branch list is never complete.
9//
10// WHAT AN ADAPTER SAYS: album pages differ only in WHERE the links live. So a row
11// describes that shape --
12// link_tag/link_attr/link_pfx where the per-item links are on an ALBUM page
13// item_tag/item_attr where the DIRECT media url is on an ITEM page
14// Two-hop by design: album -> item page -> CDN url is the near-universal shape,
15// and a one-hop site is expressed by leaving the item rule empty.
16//
17// ★★★ ORDERING: this writes into the COLLECTION AXIS (nx_mvault_coll) so every
18// fetched file records WHICH ALBUM and WHICH SITE it came from AT INGEST TIME.
19// Ingesting first and grouping later is what produced the 07-30 backfill of
20// 35,615 mis-typed records; provenance captured at ingest is free, recovered
21// later it is a full-corpus pass.
22//
23// This module is the OFFLINE half -- adapter storage + extraction. It is pure and
24// fully gateable with no network, which is deliberate: the parsing rules are where
25// the bugs live, and they should be provable without a live site.
26// license_tier: ORIGINAL
27
28import "nx_syscalls.nx"
29import "nx_canon_cid.nx"
30import "nx_registry.nx"
31import "nx_mvault_record.nx" // mv_rec_field -- read adapter record fields
32import "nx_mvault_tag.nx" // mv_u_dec -- decimal into the JSON report
33import "nx_mvault_coll.nx" // the collection axis this ingest writes into
34import "nx_https_fetch_lib.nx" // hf_fetch (pages) / hf_fetch_to_file (media)
35import "nx_paced_fetch.nx" // R0 (/compare/mediaingest): pf_before/pf_after around every album, item and media fetch
36
37const MVF_PFX: *u8 = "knowledge/store/mvfetch-\x00"
38const MVF_KP: *u8 = "ad:\x00"
39const MVF_IDS: *u8 = "ad:ids\x00"
40
41const MVF_REC_CAP: i64 = 4096
42const MVF_URL_CAP: i64 = 2048
43
44func mvf_strlen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
45func mvf_cat(dst: *u8, off: i64, s: *u8) -> i64 { var i: i64=0; while s[i]!=(0 as u8){dst[off+i]=s[i]; i=i+1} return off+i }
46
47func mvf_lower(c: i64) -> i64 { if c >= 65 { if c <= 90 { return c + 32 } } return c }
48
49// case-insensitive: does buf[at..] begin with s?
50func mvf_at_ci(buf: *u8, n: i64, at: i64, s: *u8) -> i64 {
51 var i: i64 = 0
52 while s[i] != (0 as u8) {
53 if at + i >= n { return 0 }
54 if mvf_lower(buf[at + i] as i64) != mvf_lower(s[i] as i64) { return 0 }
55 i = i + 1
56 }
57 return 1
58}
59
60// does buf[at..at+len) begin with the NUL-terminated prefix?
61func mvf_has_pfx(buf: *u8, at: i64, len: i64, pfx: *u8) -> i64 {
62 var i: i64 = 0
63 while pfx[i] != (0 as u8) {
64 if i >= len { return 0 }
65 if buf[at + i] != pfx[i] { return 0 }
66 i = i + 1
67 }
68 return 1
69}
70
71// ---- URL HELPERS ----------------------------------------------------------
72// host of "scheme://host/path" -> out (NUL-terminated); returns length.
73func mvf_host_of(url: *u8, out: *u8) -> i64 {
74 let n: i64 = mvf_strlen(url)
75 var i: i64 = 0
76 var start: i64 = 0
77 while i + 2 < n {
78 if url[i] == (58 as u8) { if url[i+1] == (47 as u8) { if url[i+2] == (47 as u8) { start = i + 3; i = n } } }
79 i = i + 1
80 }
81 var o: i64 = 0
82 var j: i64 = start
83 while j < n {
84 let c: u8 = url[j]
85 if c == (47 as u8) { j = n } else {
86 if c == (58 as u8) { j = n } else { out[o] = c; o = o + 1; j = j + 1 }
87 }
88 }
89 out[o] = 0 as u8
90 return o
91}
92
93// scheme://host of a url -> out; returns length.
94func mvf_origin_of(url: *u8, out: *u8) -> i64 {
95 let n: i64 = mvf_strlen(url)
96 var start: i64 = 0
97 var i: i64 = 0
98 while i + 2 < n {
99 if url[i] == (58 as u8) { if url[i+1] == (47 as u8) { if url[i+2] == (47 as u8) { start = i + 3; i = n } } }
100 i = i + 1
101 }
102 var o: i64 = 0
103 var j: i64 = 0
104 while j < start { out[o] = url[j]; o = o + 1; j = j + 1 }
105 j = start
106 while j < n { if url[j] == (47 as u8) { j = n } else { out[o] = url[j]; o = o + 1; j = j + 1 } }
107 out[o] = 0 as u8
108 return o
109}
110
111// Resolve href against a base url. Absolute (has "://") passes through;
112// root-relative ("/f/x") gets scheme://host; anything else is refused (0) rather
113// than guessed -- a mis-resolved url silently downloads the WRONG file, so a
114// shape we do not understand must not be turned into a plausible-looking one.
115func mvf_resolve(base: *u8, href: *u8, out: *u8, cap: i64) -> i64 {
116 let hn: i64 = mvf_strlen(href)
117 if hn == 0 { return 0 }
118 var i: i64 = 0
119 var abs: i64 = 0
120 while i + 2 < hn { if href[i]==(58 as u8) { if href[i+1]==(47 as u8) { if href[i+2]==(47 as u8) { abs=1; i=hn } } } i = i + 1 }
121 if abs == 1 {
122 if hn + 1 > cap { return 0 - 1 }
123 var k: i64 = 0
124 while k < hn { out[k] = href[k]; k = k + 1 }
125 out[hn] = 0 as u8
126 return hn
127 }
128 // protocol-relative "//host/path" inherits the BASE's scheme -- the dominant
129 // image url shape on wikimedia/CDN pages. Without this it falls into the
130 // origin-relative branch below and becomes "https://<base>//host/path" (wrong
131 // host) -- measured 2026-08-06 as why 8/14 Pima_cotton assets failed.
132 if hn >= 2 { if href[0]==(47 as u8) { if href[1]==(47 as u8) {
133 var col: i64 = 0 - 1
134 var b: i64 = 0
135 while base[b] != (0 as u8) { if col < 0 { if base[b]==(58 as u8) { col = b } } b = b + 1 }
136 if col < 0 { return 0 - 1 }
137 let slen: i64 = col + 1
138 if slen + hn + 1 > cap { return 0 - 1 }
139 var w: i64 = 0
140 while w < slen { out[w] = base[w]; w = w + 1 }
141 var z: i64 = 0
142 while z < hn { out[slen + z] = href[z]; z = z + 1 }
143 out[slen + hn] = 0 as u8
144 return slen + hn
145 } } }
146 if href[0] != (47 as u8) { return 0 }
147 let org: *u8 = sys_mmap(MVF_URL_CAP)
148 let ol: i64 = mvf_origin_of(base, org)
149 if ol + hn + 1 > cap { sys_munmap(org, MVF_URL_CAP); return 0 - 1 }
150 var o: i64 = 0
151 while o < ol { out[o] = org[o]; o = o + 1 }
152 var q: i64 = 0
153 while q < hn { out[o] = href[q]; o = o + 1; q = q + 1 }
154 out[o] = 0 as u8
155 sys_munmap(org, MVF_URL_CAP)
156 return o
157}
158
159// ---- EXTRACTION -----------------------------------------------------------
160// Pull every <tag ... attr="value"> whose value starts with pfx (pfx "" = any),
161// newline-separated, into out. Returns bytes written, or -1 if it does not fit.
162//
163// REFUSES rather than truncating, same contract as reg_index after the
164// 2026-07-31 root fix: a half-read album that looks complete is how a downloader
165// silently skips files and reports success.
166//
167// Deliberately a TAG+ATTRIBUTE scanner, not a regex over the whole document:
168// the adapter names the tag and attribute, so the rule stays legible as DATA and
169// cannot accidentally match a url mentioned in prose or a script blob.
170func mvf_extract(html: *u8, n: i64, tag: *u8, attr: *u8, pfx: *u8, out: *u8, cap: i64) -> i64 {
171 var o: i64 = 0
172 var i: i64 = 0
173 while i < n {
174 if html[i] == (60 as u8) {
175 if mvf_at_ci(html, n, i + 1, tag) == 1 {
176 let tl: i64 = mvf_strlen(tag)
177 var p: i64 = i + 1 + tl
178 var okb: i64 = 0
179 if p < n { if html[p]==(32 as u8) { okb=1 } if html[p]==(9 as u8) { okb=1 } if html[p]==(10 as u8) { okb=1 } if html[p]==(13 as u8) { okb=1 } }
180 if okb == 1 {
181 // scan attributes until the tag closes
182 var g: i64 = 1
183 while g == 1 {
184 if p >= n { g = 0 } else {
185 if html[p] == (62 as u8) { g = 0 } else {
186 if mvf_at_ci(html, n, p, attr) == 1 {
187 // NOTE: NishiLang has no `break`, and the first cut of this used
188 // `q = n + 1` as a loop-exit sentinel -- which exits the loop but
189 // DESTROYS the very index the next step needs, so every extraction
190 // returned 0. Caught by the gate before this shipped. Skip whitespace
191 // with an explicit flag and leave the cursor where it landed.
192 var q: i64 = p + mvf_strlen(attr)
193 var gq: i64 = 1
194 while gq == 1 { if q >= n { gq = 0 } else { if html[q]==(32 as u8) { q = q + 1 } else { gq = 0 } } }
195 if q <= n { if q < n { if html[q] == (61 as u8) {
196 var v: i64 = q + 1
197 var gv: i64 = 1
198 while gv == 1 { if v >= n { gv = 0 } else { if html[v]==(32 as u8) { v = v + 1 } else { gv = 0 } } }
199 if v <= n {
200 var quote: i64 = 0
201 if v < n { if html[v]==(34 as u8) { quote=34; v=v+1 } else { if html[v]==(39 as u8) { quote=39; v=v+1 } } }
202 var e: i64 = v
203 var g2: i64 = 1
204 while g2 == 1 {
205 if e >= n { g2 = 0 } else {
206 if quote != 0 { if (html[e] as i64) == quote { g2 = 0 } else { e = e + 1 } }
207 else { if html[e]==(32 as u8) { g2=0 } else { if html[e]==(62 as u8) { g2=0 } else { e=e+1 } } }
208 }
209 }
210 let vlen: i64 = e - v
211 if vlen > 0 { if mvf_has_pfx(html, v, vlen, pfx) == 1 {
212 if o + vlen + 1 > cap { return 0 - 1 }
213 var w: i64 = 0
214 while w < vlen { out[o] = html[v + w]; o = o + 1; w = w + 1 }
215 out[o] = 10 as u8; o = o + 1
216 } }
217 p = e
218 }
219 } } }
220 }
221 p = p + 1
222 }
223 }
224 }
225 i = p
226 }
227 }
228 }
229 i = i + 1
230 }
231 return o
232}
233
234// ---- ADAPTER TABLE (the data) ---------------------------------------------
235func mvf_adapter_rec(out: *u8, host: *u8, link_tag: *u8, link_attr: *u8, link_pfx: *u8,
236 item_tag: *u8, item_attr: *u8, ua: *u8) -> i64 {
237 let keys: *i64 = sys_mmap(8 * 7) as *i64
238 let vals: *i64 = sys_mmap(8 * 7) as *i64
239 let k_host: *u8 = "host" as *u8
240 let k_lt: *u8 = "link_tag" as *u8
241 let k_la: *u8 = "link_attr" as *u8
242 let k_lp: *u8 = "link_pfx" as *u8
243 let k_it: *u8 = "item_tag" as *u8
244 let k_ia: *u8 = "item_attr" as *u8
245 let k_ua: *u8 = "ua" as *u8
246 keys[0]=k_host as i64; vals[0]=host as i64
247 keys[1]=k_lt as i64; vals[1]=link_tag as i64
248 keys[2]=k_la as i64; vals[2]=link_attr as i64
249 keys[3]=k_lp as i64; vals[3]=link_pfx as i64
250 keys[4]=k_it as i64; vals[4]=item_tag as i64
251 keys[5]=k_ia as i64; vals[5]=item_attr as i64
252 keys[6]=k_ua as i64; vals[6]=ua as i64
253 let rl: i64 = canon_encode(keys, vals, 7, out)
254 sys_munmap(keys as *u8, 8 * 7)
255 sys_munmap(vals as *u8, 8 * 7)
256 return rl
257}
258
259func mvf_adapter_put_pfx(prefix: *u8, host: *u8, link_tag: *u8, link_attr: *u8, link_pfx: *u8,
260 item_tag: *u8, item_attr: *u8, ua: *u8) -> i64 {
261 let rec: *u8 = sys_mmap(MVF_REC_CAP)
262 let rl: i64 = mvf_adapter_rec(rec, host, link_tag, link_attr, link_pfx, item_tag, item_attr, ua)
263 let rc: i64 = reg_put(prefix, MVF_KP, MVF_IDS, host, rec, rl)
264 sys_munmap(rec, MVF_REC_CAP)
265 return rc
266}
267
268// Find the adapter for a host, trying the most specific first and then dropping
269// leading labels: img.bunkr.si -> bunkr.si -> si. So one row covers a whole
270// family of mirror domains, which is exactly how these hosts multiply, WITHOUT
271// the row having to enumerate them.
272func mvf_adapter_find_pfx(prefix: *u8, host: *u8, ptrout: *i64, lenout: *i64) -> i64 {
273 let n: i64 = mvf_strlen(host)
274 var start: i64 = 0
275 while start < n {
276 let cand: *u8 = ((host as i64) + start) as *u8
277 if reg_get(prefix, MVF_KP, cand, ptrout, lenout) == 1 { return 1 }
278 // drop to just past the next dot
279 var d: i64 = start
280 var moved: i64 = 0
281 while d < n { if host[d] == (46 as u8) { start = d + 1; moved = 1; d = n } else { d = d + 1 } }
282 if moved == 0 { start = n }
283 }
284 return 0
285}
286
287func mvf_adapter_list_pfx(prefix: *u8, out: *u8, cap: i64) -> i64 {
288 return reg_index(prefix, MVF_IDS, out, cap)
289}
290
291// ---- ONLINE HALF: album -> vault ------------------------------------------
292
293// Last path segment of a url, SANITISED, into out. Refuses (0) a name that is
294// empty or carries a path separator or ".." -- a remote server chooses this
295// string, so it is untrusted input at a boundary (rule 12) and a crafted one
296// would otherwise write outside the destination directory.
297func mvf_basename(url: *u8, out: *u8, cap: i64) -> i64 {
298 let n: i64 = mvf_strlen(url)
299 var start: i64 = 0
300 var i: i64 = 0
301 while i < n { if url[i] == (47 as u8) { start = i + 1 } i = i + 1 }
302 var e: i64 = start
303 while e < n { if url[e] == (63 as u8) { e = n } else { e = e + 1 } }
304 var end: i64 = start
305 var j: i64 = start
306 while j < n { if url[j] == (63 as u8) { j = n } else { end = j + 1; j = j + 1 } }
307 let ln: i64 = end - start
308 if ln <= 0 { return 0 }
309 if ln + 1 > cap { return 0 }
310 var o: i64 = 0
311 while o < ln {
312 let c: u8 = url[start + o]
313 if c == (47 as u8) { return 0 }
314 if c == (92 as u8) { return 0 }
315 out[o] = c
316 o = o + 1
317 }
318 out[ln] = 0 as u8
319 if ln >= 2 { if out[0] == (46 as u8) { if out[1] == (46 as u8) { return 0 } } }
320 return ln
321}
322
323// Read a NUL-terminated field out of an adapter record.
324func mvf_field(rec: *u8, rl: i64, key: *u8, out: *u8, cap: i64) -> i64 {
325 let po: *i64 = sys_mmap(16) as *i64
326 let lo: *i64 = sys_mmap(16) as *i64
327 if mv_rec_field(rec, rl, key, mvf_strlen(key), po, lo) != 1 { out[0] = 0 as u8; return 0 }
328 let n: i64 = lo[0]
329 if n + 1 > cap { out[0] = 0 as u8; return 0 }
330 let src: *u8 = po[0] as *u8
331 var i: i64 = 0
332 while i < n { out[i] = src[i]; i = i + 1 }
333 out[n] = 0 as u8
334 return n
335}
336
337const MVF_PAGE_CAP: i64 = 8388608
338const MVF_LINKS_CAP: i64 = 1048576
339const MVF_ITEMS_CAP: i64 = 1048576
340const MVF_MAX_ITEMS: i64 = 4096
341
342// Ingest one album URL into the vault, recording its SITE and ALBUM provenance
343// at ingest time.
344//
345// ★ PROVENANCE IS CAPTURED HERE, NOT LATER. Recovering "which album was this"
346// after the fact means a full-corpus pass -- that is exactly what the 07-30
347// classifier backfill cost (35,615 records). Captured at ingest it is free.
348//
349// Returns items ingested, or negative: -1 no adapter, -2 album fetch failed,
350// -3 no links found. Writes a JSON summary into report.
351func mvf_fetch_album_pfx(cpfx: *u8, store: i64, album_url: *u8, dest_dir: *u8,
352 report: *u8, rcap: i64) -> i64 {
353 let host: *u8 = sys_mmap(512)
354 mvf_host_of(album_url, host)
355
356 let po: *i64 = sys_mmap(16) as *i64
357 let lo: *i64 = sys_mmap(16) as *i64
358 if mvf_adapter_find(host, po, lo) != 1 { return 0 - 1 }
359 let rec: *u8 = po[0] as *u8
360 let rl: i64 = lo[0]
361
362 let l_tag: *u8 = sys_mmap(128)
363 let l_attr: *u8 = sys_mmap(128)
364 let l_pfx: *u8 = sys_mmap(256)
365 let i_tag: *u8 = sys_mmap(128)
366 let i_attr: *u8 = sys_mmap(128)
367 mvf_field(rec, rl, "link_tag" as *u8, l_tag, 128)
368 mvf_field(rec, rl, "link_attr" as *u8, l_attr, 128)
369 mvf_field(rec, rl, "link_pfx" as *u8, l_pfx, 256)
370 mvf_field(rec, rl, "item_tag" as *u8, i_tag, 128)
371 mvf_field(rec, rl, "item_attr" as *u8, i_attr, 128)
372
373 // ---- album page ----
374 let page: *u8 = sys_mmap(MVF_PAGE_CAP)
375 pf_before(album_url) // R0: polite wait for the album host
376 let pn: i64 = hf_fetch(store, album_url, 0, 0, page, MVF_PAGE_CAP)
377 var pst: i64 = 0
378 if pn > 0 { pst = hf_status(page, pn) }
379 pf_after(album_url, pst, 0) // R0: the REAL status feeds the pacer (0 = no response)
380 if pn <= 0 { return 0 - 2 }
381 var boff: i64 = hf_body_off(page, pn)
382 if boff < 0 { boff = 0 }
383 let body: *u8 = ((page as i64) + boff) as *u8
384
385 let links: *u8 = sys_mmap(MVF_LINKS_CAP)
386 let ln: i64 = mvf_extract(body, pn - boff, l_tag, l_attr, l_pfx, links, MVF_LINKS_CAP)
387 if ln <= 0 { return 0 - 3 }
388
389 // ---- declare the two collections this album's files belong to ----
390 let site_col: *u8 = sys_mmap(256)
391 let alb_col: *u8 = sys_mmap(256)
392 mvc_declare_pfx(cpfx, "site" as *u8, host, host, host, album_url, "-" as *u8, site_col)
393 let akey: *u8 = sys_mmap(1024)
394 if mvf_basename(album_url, akey, 1024) == 0 { mvf_cat(akey, 0, "album" as *u8); akey[5] = 0 as u8 }
395 mvc_declare_pfx(cpfx, "album" as *u8, host, akey, akey, album_url, site_col, alb_col)
396
397 let items: *u8 = sys_mmap(MVF_ITEMS_CAP)
398 var io: i64 = 0
399 var got: i64 = 0
400 var failed: i64 = 0
401 var truncated: i64 = 0
402
403 let one: *u8 = sys_mmap(MVF_URL_CAP)
404 let direct: *u8 = sys_mmap(MVF_URL_CAP)
405 let ipage: *u8 = sys_mmap(MVF_PAGE_CAP)
406 let dbuf: *u8 = sys_mmap(MVF_LINKS_CAP)
407 let path: *u8 = sys_mmap(2048)
408 let base: *u8 = sys_mmap(1024)
409 let cid: *u8 = sys_mmap(256)
410 let szbox: *i64 = sys_mmap(16) as *i64
411 let stbox: *i64 = sys_mmap(16) as *i64
412
413 var ls: i64 = 0
414 var i: i64 = 0
415 while i <= ln {
416 var eol: i64 = 0
417 if i == ln { eol = 1 } else { if links[i] == (10 as u8) { eol = 1 } }
418 if eol == 1 {
419 let hl: i64 = i - ls
420 if hl > 0 {
421 if got >= MVF_MAX_ITEMS { truncated = 1 } else {
422 let href: *u8 = sys_mmap(MVF_URL_CAP)
423 var q: i64 = 0
424 while q < hl { href[q] = links[ls + q]; q = q + 1 }
425 href[hl] = 0 as u8
426
427 var ok: i64 = 1
428 if mvf_resolve(album_url, href, one, MVF_URL_CAP) <= 0 { ok = 0 }
429
430 // second hop: the item PAGE carries the direct CDN url
431 if ok == 1 { if i_tag[0] != (0 as u8) {
432 pf_before(one) // R0: polite wait for the item host
433 let inn: i64 = hf_fetch(store, one, 0, 0, ipage, MVF_PAGE_CAP)
434 var ist: i64 = 0
435 if inn > 0 { ist = hf_status(ipage, inn) }
436 pf_after(one, ist, 0)
437 if inn <= 0 { ok = 0 } else {
438 var ibo: i64 = hf_body_off(ipage, inn)
439 if ibo < 0 { ibo = 0 }
440 let dn: i64 = mvf_extract(((ipage as i64) + ibo) as *u8, inn - ibo, i_tag, i_attr, "" as *u8, dbuf, MVF_LINKS_CAP)
441 if dn <= 0 { ok = 0 } else {
442 // take the FIRST extracted direct url
443 var cut: i64 = 0
444 var z: i64 = 0
445 while z < dn { if dbuf[z] == (10 as u8) { cut = z; z = dn } else { z = z + 1 } }
446 if cut == 0 { cut = dn }
447 let tmp: *u8 = sys_mmap(MVF_URL_CAP)
448 var w: i64 = 0
449 while w < cut { tmp[w] = dbuf[w]; w = w + 1 }
450 tmp[cut] = 0 as u8
451 if mvf_resolve(one, tmp, direct, MVF_URL_CAP) <= 0 { ok = 0 }
452 }
453 }
454 } else {
455 var c2: i64 = 0
456 while one[c2] != (0 as u8) { direct[c2] = one[c2]; c2 = c2 + 1 }
457 direct[c2] = 0 as u8
458 } }
459
460 // ---- stream to disk, then CID the FILE (never the buffer) ----
461 if ok == 1 {
462 if mvf_basename(direct, base, 1024) == 0 { ok = 0 } else {
463 var pO: i64 = mvf_cat(path, 0, dest_dir)
464 path[pO] = 47 as u8; pO = pO + 1
465 pO = mvf_cat(path, pO, base)
466 path[pO] = 0 as u8
467 let wfd: i64 = sys_openat_wr(path, 420)
468 if wfd < 0 { ok = 0 } else {
469 pf_before(direct) // R0: polite wait for the CDN host
470 stbox[0] = 0
471 let wn: i64 = hf_fetch_to_file(store, direct, 0, 0, wfd, 0, stbox)
472 pf_after(direct, stbox[0], 0) // R0: the REAL status feeds the pacer
473 sys_close(wfd)
474 if wn <= 0 { ok = 0; sys_unlinkat(path) }
475 }
476 }
477 }
478 if ok == 1 {
479 if cid_of_file_chunk(path, 4194304, szbox, cid) <= 0 { ok = 0 }
480 }
481 if ok == 1 {
482 let cl: i64 = mvf_strlen(cid)
483 if io + cl + 2 < MVF_ITEMS_CAP {
484 var y: i64 = 0
485 while y < cl { items[io] = cid[y]; io = io + 1; y = y + 1 }
486 items[io] = 10 as u8; io = io + 1
487 got = got + 1
488 }
489 } else { failed = failed + 1 }
490 }
491 }
492 ls = i + 1
493 }
494 i = i + 1
495 }
496
497 // ---- one segment per collection, both directions of provenance ----
498 var linked: i64 = 0
499 if io > 0 {
500 linked = mvc_batch_add_pfx(cpfx, alb_col, items, io)
501 mvc_batch_add_pfx(cpfx, site_col, items, io)
502 }
503
504 var ro: i64 = mvf_cat(report, 0, "{\"album\":\"" as *u8)
505 ro = mvf_cat(report, ro, alb_col)
506 ro = mvf_cat(report, ro, "\",\"site\":\"" as *u8)
507 ro = mvf_cat(report, ro, site_col)
508 ro = mvf_cat(report, ro, "\",\"found\":" as *u8)
509 ro = mv_u_dec(report, ro, mvc_count_lines(links, ln))
510 ro = mvf_cat(report, ro, ",\"ingested\":" as *u8)
511 ro = mv_u_dec(report, ro, got)
512 ro = mvf_cat(report, ro, ",\"linked\":" as *u8)
513 ro = mv_u_dec(report, ro, linked)
514 ro = mvf_cat(report, ro, ",\"failed\":" as *u8)
515 ro = mv_u_dec(report, ro, failed)
516 // NEVER a silent cap: if the envelope truncated the album, SAY SO in the
517 // report, or a partial ingest reads as a complete one.
518 ro = mvf_cat(report, ro, ",\"truncated\":" as *u8)
519 ro = mv_u_dec(report, ro, truncated)
520 ro = mvf_cat(report, ro, "}\n" as *u8)
521 report[ro] = 0 as u8
522 return got
523}
524
525// ---- production-prefix wrappers -------------------------------------------
526func mvf_adapter_put(host: *u8, link_tag: *u8, link_attr: *u8, link_pfx: *u8, item_tag: *u8, item_attr: *u8, ua: *u8) -> i64 {
527 return mvf_adapter_put_pfx(MVF_PFX, host, link_tag, link_attr, link_pfx, item_tag, item_attr, ua)
528}
529func mvf_adapter_find(host: *u8, ptrout: *i64, lenout: *i64) -> i64 { return mvf_adapter_find_pfx(MVF_PFX, host, ptrout, lenout) }
530func mvf_adapter_list(out: *u8, cap: i64) -> i64 { return mvf_adapter_list_pfx(MVF_PFX, out, cap) }
531func mvf_fetch_album(store: i64, album_url: *u8, dest_dir: *u8, report: *u8, rcap: i64) -> i64 { return mvf_fetch_album_pfx(MVC_PFX, store, album_url, dest_dir, report, rcap) }