nx_media_pool.nx source
↩ module page · 211 lines · 8663 B
1// nx_media_pool.nx -- container for ingested media bytes + provenance.
2//
3// Foundation primitive of the multi-media ingestion system. Every
4// per-media-type ingester (text / image / gif / video / audio)
5// writes its raw bytes into a MediaPool record, then StoryBeats
6// reference that media_id when they describe scenes / frames /
7// spans extracted from it.
8//
9// Why pool, not per-beat allocation: a single gif might contribute
10// 8 beats (one per scene-change frame group); they all share the
11// same source bytes. Same for a video keyframe sequence, or a
12// chapter's worth of paragraphs sharing one source text blob.
13// Pool keeps bytes in one place, beats hold cheap i64 media_ids.
14//
15// Substrate-native: no external deps; pure sys_mmap allocation.
16//
17// See: docs/NISHI_MEDIA_INGESTION_ROADMAP.md for the full
18// architecture. Composes with:
19// nx_storybeat.nx -- beats hold media_id pointing here
20// nx_storydb.nx -- single MediaPool per StoryDb
21// nx_media_sniff.nx -- determines media_type field
22// nx_media_fetch.nx -- adds bytes after fetching
23
24// nx_safety_envelope:
25// intended_use: "Hold raw bytes of ingested media items
26// (text/image/gif/video/audio) and their
27// provenance. Indexed by media_id for
28// StoryBeat references."
29// sil_target: SIL2 (every beat references through here;
30// a wrong media_id maps to wrong content)
31// asil_target: QM
32// dal_target: DAL C
33// iec_62304_class: NONE
34// evidence: [no_floating_point,
35// fixed_capacity_arena_no_realloc,
36// sealed_media_type_enum,
37// provenance_url_tracked_for_audit_chain]
38// hazard_register: [bug-tape-pool-cap-overflow-silent,
39// bug-tape-media-id-collision-across-pools]
40// residual_risk: "Pool cap is static; large ingestion sessions
41// may exhaust 1024-entry default. Caller can
42// size pool larger at alloc time."
43// verdict: NOT_YET_EVALUATED
44
45import "nx_syscalls.nx"
46
47// ===== sealed media type codes =====================================
48//
49// Every media item in the pool carries one of these. Per-media
50// ingesters dispatch on this value. Stays stable forever
51// (additive-only: new types append at the end, old codes never
52// change meaning).
53
54const NX_MEDIA_TYPE_UNKNOWN: i64 = 0
55const NX_MEDIA_TYPE_TEXT: i64 = 1 // plain text / markdown / prose
56const NX_MEDIA_TYPE_HTML: i64 = 2 // text/html (literotica wrapping)
57const NX_MEDIA_TYPE_IMAGE: i64 = 3 // PNG / JPEG / WebP / single frame
58const NX_MEDIA_TYPE_GIF: i64 = 4 // animated GIF (frame sequence)
59const NX_MEDIA_TYPE_VIDEO: i64 = 5 // MP4 / WebM / Matroska (A/V container)
60const NX_MEDIA_TYPE_AUDIO: i64 = 6 // WAV / MP3 / Opus / Vorbis
61
62// Appended 2026-07-30 (additive-only per the contract above; codes 0-6 keep
63// their meaning, so every existing consumer is unaffected). These close the
64// vault's type hole: MV_TYPE_MODEL3D and MV_TYPE_BOOK were previously
65// UNREACHABLE from bytes because the sniffer had no code to return.
66const NX_MEDIA_TYPE_MODEL3D: i64 = 7 // STL / OBJ / PLY / glTF / 3MF / FBX / STEP mesh
67const NX_MEDIA_TYPE_BOOK: i64 = 8 // PDF / EPUB / MOBI / DJVU ebook
68const NX_MEDIA_TYPE_AUDIOBOOK: i64 = 9 // M4B / AAX narrated long-form audio
69const NX_MEDIA_TYPE_MANGA: i64 = 10 // CBZ / CBR / CB7 comic archive
70const NX_MEDIA_TYPE_WARC: i64 = 11 // WARC / WACZ archived web capture
71const NX_MEDIA_TYPE_ARCHIVE: i64 = 12 // generic zip/rar/7z/tar (container, not media)
72
73// ===== sealed source kinds =========================================
74//
75// Where did this media come from? Used by content classifier and
76// consent boundary checker to apply per-source policy.
77
78const NX_MEDIA_SRC_UNKNOWN: i64 = 0
79const NX_MEDIA_SRC_USER_PASTE: i64 = 1 // user pasted via stdin / clipboard
80const NX_MEDIA_SRC_USER_FILE: i64 = 2 // user pointed at filesystem path
81const NX_MEDIA_SRC_BROWSER: i64 = 3 // nishi-browser URL fetch
82const NX_MEDIA_SRC_VOICE: i64 = 4 // voice dictation / mic capture
83const NX_MEDIA_SRC_BATCH: i64 = 5 // nx_ingest_runner directory walk
84const NX_MEDIA_SRC_LLM_GEN: i64 = 6 // synthetic content from LLM
85
86// ===== MediaItem record ============================================
87
88struct MediaItem {
89 media_type: i64, // NX_MEDIA_TYPE_*
90 src_kind: i64, // NX_MEDIA_SRC_*
91
92 // Raw bytes (owned by pool; caller does not free).
93 bytes: *u8,
94 n_bytes: i64,
95
96 // MIME string as reported by source (e.g. "text/html; charset=utf-8")
97 // or sniffed. May be 0 if unknown. null-terminated.
98 mime: *u8,
99 mime_len: i64,
100
101 // Origin URL / filesystem path / "stdin" / "voice". Null-term.
102 origin: *u8,
103 origin_len: i64,
104
105 // Wall-clock fetch timestamp (seconds since epoch).
106 fetched_at: i64,
107
108 // Cached SHA-256 of bytes for dedup + audit chain. Computed
109 // lazily on first request (zero until populated). 16-byte
110 // truncated hash stored as two i64s.
111 hash_lo: i64,
112 hash_hi: i64,
113}
114
115const NX_MEDIA_ITEM_BYTES: i64 = 80
116
117// ===== MediaPool container =========================================
118//
119// One MediaPool per StoryDb. Holds up to NX_MEDIA_POOL_CAP items.
120// Items are append-only (no delete); ingestion sessions usually
121// add a handful (1 URL fetched + maybe inline-images extracted from
122// HTML). Batch sessions may load thousands; size at alloc-time.
123
124const NX_MEDIA_POOL_DEFAULT_CAP: i64 = 1024
125
126struct MediaPool {
127 items: *MediaItem,
128 n_items: i64,
129 cap: i64,
130}
131
132func nx_media_pool_alloc(cap: i64) -> *MediaPool {
133 var c: i64 = cap
134 if c <= 0 { c = NX_MEDIA_POOL_DEFAULT_CAP }
135 let raw: *u8 = sys_mmap(24)
136 let p: *MediaPool = raw as *MediaPool
137 p.items = (sys_mmap(c * NX_MEDIA_ITEM_BYTES)) as *MediaItem
138 p.n_items = 0
139 p.cap = c
140 return p
141}
142
143func nx_media_pool_alloc_default() -> *MediaPool {
144 return nx_media_pool_alloc(NX_MEDIA_POOL_DEFAULT_CAP)
145}
146
147func nx_media_item_at(p: *MediaPool, i: i64) -> *MediaItem {
148 return (((p.items as i64) + i * NX_MEDIA_ITEM_BYTES) as *MediaItem)
149}
150
151// ===== add an item =================================================
152//
153// Returns the new media_id (index into items) on success, or -1 on
154// pool-overflow. Bytes are NOT copied -- caller's buffer must
155// outlive the pool (most ingesters call sys_mmap themselves for
156// the source bytes; that mmap persists for the program lifetime
157// per the no-free-path convention).
158
159func nx_media_pool_add(p: *MediaPool, media_type: i64, src_kind: i64,
160 bytes: *u8, n_bytes: i64,
161 mime: *u8, mime_len: i64,
162 origin: *u8, origin_len: i64,
163 fetched_at: i64) -> i64 {
164 if p.n_items >= p.cap { return -1 }
165 let id: i64 = p.n_items
166 let it: *MediaItem = nx_media_item_at(p, id)
167 it.media_type = media_type
168 it.src_kind = src_kind
169 it.bytes = bytes
170 it.n_bytes = n_bytes
171 it.mime = mime
172 it.mime_len = mime_len
173 it.origin = origin
174 it.origin_len = origin_len
175 it.fetched_at = fetched_at
176 it.hash_lo = 0
177 it.hash_hi = 0
178 p.n_items = p.n_items + 1
179 return id
180}
181
182// ===== lookup helpers ==============================================
183
184// Get the bytes pointer + length for a media_id. Returns NULL via
185// out params on invalid id (caller checks bytes != 0).
186func nx_media_pool_get(p: *MediaPool, media_id: i64,
187 out_bytes: **u8, out_len: *i64) -> i64 {
188 if media_id < 0 { *out_bytes = 0 as *u8; *out_len = 0; return -1 }
189 if media_id >= p.n_items { *out_bytes = 0 as *u8; *out_len = 0; return -1 }
190 let it: *MediaItem = nx_media_item_at(p, media_id)
191 *out_bytes = it.bytes
192 *out_len = it.n_bytes
193 return 0
194}
195
196// Get the media_type for a media_id. Returns NX_MEDIA_TYPE_UNKNOWN
197// on invalid id.
198func nx_media_pool_type(p: *MediaPool, media_id: i64) -> i64 {
199 if media_id < 0 { return NX_MEDIA_TYPE_UNKNOWN }
200 if media_id >= p.n_items { return NX_MEDIA_TYPE_UNKNOWN }
201 let it: *MediaItem = nx_media_item_at(p, media_id)
202 return it.media_type
203}
204
205// Get the source kind for a media_id.
206func nx_media_pool_src(p: *MediaPool, media_id: i64) -> i64 {
207 if media_id < 0 { return NX_MEDIA_SRC_UNKNOWN }
208 if media_id >= p.n_items { return NX_MEDIA_SRC_UNKNOWN }
209 let it: *MediaItem = nx_media_item_at(p, media_id)
210 return it.src_kind
211}