nx_asset_ingest_lib.nx source
↩ module page · 154 lines · 8039 B
1// nx_asset_ingest_lib.nx -- shared ingest CORES for the production catalog populator (R5).
2//
3// Factored out of nx_asset_ingest_real so BOTH the runnable populator (nx_asset_ingest_real, keeps
4// main() + the slug list) AND its gate (nx_asset_ingest_real_gate) drive the SAME real
5// read -> record -> dedup pipeline (DRY #15; mirrors _pdf_reflow_lib shared by the reader + its gate).
6// The *_at cores take an ARBITRARY catalog prefix + an ARBITRARY full path so a gate can exercise them
7// against fixtures + a fresh /tmp catalog (never the production store). NO main(). license_tier: ORIGINAL
8import "nx_syscalls.nx"
9import "nx_asset_record.nx"
10import "nx_asset_catalog.nx"
11import "nx_asset_meta.nx"
12const K_MAGIC_4194304: i64 = 4194304
13const K_MAGIC_8192: i64 = 8192
14const K_MAGIC_2097152: i64 = 2097152
15
16// the PRODUCTION catalog prefix (one SSOT inventory; distinct from the /tmp prefixes the gates use and
17// from the production workstream board knowledge/store/ws-).
18func ai_prefix() -> *u8 { return "knowledge/store/asset-catalog-\x00" as *u8 }
19// the real gallery directory (machine-generated PNG frames).
20func ai_gallery_dir() -> *u8 { return "knowledge/staging/media/gallery/\x00" as *u8 }
21
22// ---- small helpers ----
23func ai_w(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
24func ai_num(v: i64) -> i64 {
25 let bb: *u8 = sys_mmap(28); var m: i64 = v; if m < 0 { sys_write(1, "-\x00" as *u8, 1); m = 0 - m }
26 let t: *u8 = sys_mmap(28); var k: i64 = 0
27 if m == 0 { t[0] = 48 as u8; k = 1 }
28 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
29 var i: i64 = 0; while i < k { bb[i] = t[k - 1 - i]; i = i + 1 }
30 sys_write(1, bb, k); return 0
31}
32func ai_cat(dst: *u8, off: i64, src: *u8) -> i64 { var i: i64 = 0; while src[i] != (0 as u8) { dst[off + i] = src[i]; i = i + 1 } dst[off + i] = 0 as u8; return off + i }
33func E() -> *u8 { return "\x00" as *u8 }
34func sset(a: *i64, i: i64, s: *u8) -> i64 { a[i] = s as i64; return 0 }
35
36// ---- BOUNDED file reader (the sys_read_file 4 GiB-VA landmine fix) ----
37// Reserve only `cap` bytes, read up to cap into a tight buffer; out_len[0] = bytes read (>=0).
38// Returns the buffer pointer, or 0 on open failure.
39func ai_read_cap(path: *u8, cap: i64, out_len: *i64) -> *u8 {
40 out_len[0] = 0
41 let fd: i64 = sys_openat_rd(path)
42 if fd < 0 { return 0 as *u8 }
43 let buf: *u8 = sys_mmap(cap + 16)
44 var total: i64 = 0
45 var go: i64 = 1
46 while go == 1 {
47 let tail: *u8 = ((buf as i64) + total) as *u8
48 let n: i64 = sys_read(fd, tail, cap - total)
49 if n <= 0 { go = 0 }
50 if n > 0 { total = total + n }
51 if total >= cap { go = 0 }
52 }
53 sys_close(fd)
54 out_len[0] = total
55 return buf
56}
57
58// ---- build the canonical record bytes from grouped field arrays -> out; returns length ----
59func ai_mk_rec(core: *i64, prov: *i64, media: *i64, org: *i64, out: *u8) -> i64 {
60 let keys: *i64 = sys_mmap(8 * 48) as *i64
61 let vals: *i64 = sys_mmap(8 * 48) as *i64
62 let n: i64 = ar_fields(core, prov, media, org, keys, vals)
63 return ar_encode(keys, vals, n, out)
64}
65
66// ---- CORE: ingest ONE doc asset (a real on-disk HTML page) into an ARBITRARY catalog prefix ----
67// type=doc, machine provenance (tool=builder, model=slug), location=full path, classification=
68// professional. Returns 1 NEW / 2 DUP / 0 read-fail. REAL bytes: a missing file -> 0 (graceful).
69func ai_ingest_doc_at(cat_prefix: *u8, path: *u8, slug: *u8, title: *u8, date: *u8, tags: *u8, builder: *u8) -> i64 {
70 let nbox: *i64 = sys_mmap(16) as *i64
71 let body: *u8 = ai_read_cap(path, K_MAGIC_4194304, nbox) // 4 MiB cap (> any wiki page)
72 if (body as i64) == 0 { return 0 }
73 if nbox[0] < 1 { return 0 }
74
75 let core: *i64 = sys_mmap(8 * 8) as *i64
76 sset(core, 0, "doc\x00" as *u8); sset(core, 1, title); sset(core, 2, E()); sset(core, 3, date)
77 sset(core, 4, slug); sset(core, 5, E()); sset(core, 6, E())
78 let prov: *i64 = sys_mmap(8 * 8) as *i64
79 sset(prov, 0, "machine\x00" as *u8); sset(prov, 1, "software\x00" as *u8); sset(prov, 2, builder)
80 sset(prov, 3, slug); sset(prov, 4, E()); sset(prov, 5, E())
81 let media: *i64 = sys_mmap(8 * 8) as *i64
82 sset(media, 0, E()); sset(media, 1, E()); sset(media, 2, E()); sset(media, 3, "text/html\x00" as *u8)
83 let org: *i64 = sys_mmap(8 * 8) as *i64
84 sset(org, 0, tags); sset(org, 1, path); sset(org, 2, "use\x00" as *u8); sset(org, 3, "1\x00" as *u8)
85 sset(org, 4, "professional\x00" as *u8); sset(org, 5, E())
86
87 let rec: *u8 = sys_mmap(K_MAGIC_8192)
88 let rlen: i64 = ai_mk_rec(core, prov, media, org, rec)
89 let cid: *u8 = sys_mmap(128)
90 let st: *i64 = sys_mmap(16) as *i64
91 let rc: i64 = cat_ingest_cid(cat_prefix, rec, rlen, cid, st)
92 if rc < 0 { ai_w(" ERR ingest doc rc="); ai_num(rc); ai_w("\n"); return 0 }
93 ai_w(" DOC "); ai_w(slug)
94 if st[0] == CAT_NEW() { ai_w(" NEW \x00" as *u8) } else { ai_w(" DUP \x00" as *u8) }
95 ai_w(cid); ai_w("\n")
96 return st[0]
97}
98
99// production wrapper: web_assets/<file_rel> into the production catalog prefix.
100func ai_ingest_doc(slug: *u8, file_rel: *u8, title: *u8, date: *u8, tags: *u8, builder: *u8) -> i64 {
101 let path: *u8 = sys_mmap(256)
102 var po: i64 = ai_cat(path, 0, "web_assets/\x00" as *u8)
103 po = ai_cat(path, po, file_rel)
104 return ai_ingest_doc_at(ai_prefix(), path, slug, title, date, tags, builder)
105}
106
107// ---- CORE: ingest ONE image asset (a real PNG) into an ARBITRARY catalog prefix ----
108// REAL width/height from the actual IHDR (am_extract_image; AM_NOT_PNG -> SKIP, never fabricated);
109// type=image, machine provenance, encoding_format from the bytes. Returns 1 NEW / 2 DUP / 0 skip/fail.
110func ai_ingest_image_at(cat_prefix: *u8, path: *u8, fname: *u8, title: *u8, date: *u8, tags: *u8) -> i64 {
111 let nbox: *i64 = sys_mmap(16) as *i64
112 let body: *u8 = ai_read_cap(path, K_MAGIC_2097152, nbox) // 2 MiB cap (> largest gallery PNG ~1.1 MB)
113 if (body as i64) == 0 { return 0 }
114 if nbox[0] < 1 { return 0 }
115
116 let w_buf: *u8 = sys_mmap(24)
117 let h_buf: *u8 = sys_mmap(24)
118 let fmt_buf: *u8 = sys_mmap(24)
119 if am_extract_image(body, nbox[0], w_buf, h_buf, fmt_buf) != 0 {
120 ai_w(" SKIP non-PNG "); ai_w(fname); ai_w("\n"); return 0
121 }
122
123 let core: *i64 = sys_mmap(8 * 8) as *i64
124 sset(core, 0, "image\x00" as *u8); sset(core, 1, title); sset(core, 2, E()); sset(core, 3, date)
125 sset(core, 4, fname); sset(core, 5, E()); sset(core, 6, E())
126 let prov: *i64 = sys_mmap(8 * 8) as *i64
127 sset(prov, 0, "machine\x00" as *u8); sset(prov, 1, "software\x00" as *u8)
128 sset(prov, 2, "nishi-media-pipeline\x00" as *u8); sset(prov, 3, "laptop-gen\x00" as *u8)
129 sset(prov, 4, E()); sset(prov, 5, E())
130 let media: *i64 = sys_mmap(8 * 8) as *i64
131 sset(media, 0, w_buf); sset(media, 1, h_buf); sset(media, 2, E()); sset(media, 3, fmt_buf)
132 let org: *i64 = sys_mmap(8 * 8) as *i64
133 sset(org, 0, tags); sset(org, 1, path); sset(org, 2, "manage\x00" as *u8); sset(org, 3, "1\x00" as *u8)
134 sset(org, 4, "professional\x00" as *u8); sset(org, 5, E())
135
136 let rec: *u8 = sys_mmap(K_MAGIC_8192)
137 let rlen: i64 = ai_mk_rec(core, prov, media, org, rec)
138 let cid: *u8 = sys_mmap(128)
139 let st: *i64 = sys_mmap(16) as *i64
140 let rc: i64 = cat_ingest_cid(cat_prefix, rec, rlen, cid, st)
141 if rc < 0 { ai_w(" ERR ingest image rc="); ai_num(rc); ai_w("\n"); return 0 }
142 ai_w(" IMAGE "); ai_w(fname); ai_w(" "); ai_w(w_buf); ai_w("x"); ai_w(h_buf)
143 if st[0] == CAT_NEW() { ai_w(" NEW \x00" as *u8) } else { ai_w(" DUP \x00" as *u8) }
144 ai_w(cid); ai_w("\n")
145 return st[0]
146}
147
148// production wrapper: gallery_dir/<fname> into the production catalog prefix.
149func ai_ingest_image(fname: *u8, title: *u8, date: *u8, tags: *u8) -> i64 {
150 let path: *u8 = sys_mmap(256)
151 var po: i64 = ai_cat(path, 0, ai_gallery_dir())
152 po = ai_cat(path, po, fname)
153 return ai_ingest_image_at(ai_prefix(), path, fname, title, date, tags)
154}