nx_text_ingest.nx source
↩ module page · 442 lines · 16893 B
1// nx_text_ingest.nx -- text-medium ingester targeting media-agnostic schema.
2//
3// Tier-1 ingester in the multi-media stack. Consumes a MediaItem of
4// type TEXT or HTML from a StoryDb's MediaPool and emits StoryBeats
5// + StoryScenes + StoryChapters into the same StoryDb.
6//
7// Supersedes nx_narrative_ingest.nx (kept in tree as deprecation
8// pointer until concurrent-agent wiring catches up). The two
9// differ in ONE thing: this one writes beats into the canonical
10// nx_storybeat.nx schema so the rest of the director pipeline
11// (nx_arc / nx_moment_resolve / nx_directors_note) consumes them
12// uniformly with image / gif / video / audio beats.
13//
14// Scope: structural parse only. Marks paragraphs as NARRATION,
15// scene breaks as TRANSITION, headings as HEADING. Semantic
16// classification (DIALOGUE / ACTION / DESCRIPTION / INTERIORITY /
17// GESTURE) is the next layer (nx_text_beat_classify.nx).
18//
19// Sources expected:
20// * URL via nishi-browser fetch -> nx_media_fetch.nx
21// * Manual paste via stdin -> nx_media_fetch.nx
22// * Filesystem path -> nx_media_fetch.nx
23// * Voice dictation -> nx_media_fetch.nx
24// * Batch directory walk -> nx_ingest_runner.nx (Tier 5)
25//
26// genealogy_id: propp_morphology_1928 (narrative structural roots)
27// + commonmark_2014 (heading / blank-line rules)
28//
29// Composes with:
30// nx_storybeat.nx -- writes here
31// nx_storydb.nx -- container
32// nx_media_pool.nx -- reads source bytes via media_id
33// nx_media_sniff.nx -- caller already classified
34//
35// nx_safety_envelope:
36// intended_use: "Parse text-medium MediaItem into structured
37// beats / scenes / chapters in the canonical
38// media-agnostic StoryBeat schema."
39// sil_target: SIL2
40// asil_target: QM
41// dal_target: DAL C
42// iec_62304_class: NONE
43// evidence: [no_floating_point,
44// media_agnostic_schema_output,
45// bounded_inner_loops_per_jpl_rule_2,
46// structural_only_no_semantic_judgement]
47// hazard_register: [bug-tape-chapter-boundary-false-positive,
48// bug-tape-scene-break-marker-missed,
49// bug-tape-paragraph-overflow-budget]
50// residual_risk: "Structural-only -- semantic scene boundaries
51// without visual markers (POV shift in prose)
52// require nx_text_beat_classify.nx for refinement."
53// verdict: NOT_YET_EVALUATED
54
55import "nx_syscalls.nx"
56import "nx_storybeat.nx"
57import "nx_storydb.nx"
58import "nx_media_pool.nx"
59
60// ===== verdict codes (per JPL Rule 2 sealed verdict pattern) ======
61
62const NX_TEXT_INGEST_OK: i64 = 0
63const NX_TEXT_INGEST_BAD_MEDIA_ID: i64 = -1
64const NX_TEXT_INGEST_EMPTY: i64 = -2
65const NX_TEXT_INGEST_NOT_TEXT: i64 = -3
66const NX_TEXT_INGEST_BUDGET_EXHAUSTED: i64 = -4
67const NX_TEXT_INGEST_BEATS_FULL: i64 = -5
68
69// ===== per-call loop budget (JPL Rule 2) ==========================
70//
71// Hard ceiling on the outer "walk source" loop. src_len + 1
72// iterations is the natural bound, plus a small safety margin.
73
74const NX_TEXT_INGEST_BUDGET_PAD: i64 = 16
75
76// ===== byte-level helpers =========================================
77
78func nx_text_load_u8(p: *u8, i: i64) -> i64 {
79 let q: *u8 = ((p as i64) + i) as *u8
80 return *q
81}
82
83func nx_text_is_ws(c: i64) -> i64 {
84 if c == 0x20 { return 1 }
85 if c == 0x09 { return 1 }
86 if c == 0x0A { return 1 }
87 if c == 0x0D { return 1 }
88 return 0
89}
90
91func nx_text_is_horiz_ws(c: i64) -> i64 {
92 if c == 0x20 { return 1 }
93 if c == 0x09 { return 1 }
94 return 0
95}
96
97// ===== structural detectors =======================================
98
99// Scene-break marker: 3+ of *, -, _ or 4+ of = on their own line,
100// possibly with intervening single spaces ("* * *").
101func nx_text_is_scene_break(src: *u8, pos: i64, len: i64) -> i64 {
102 if pos >= len { return 0 }
103 let c: i64 = nx_text_load_u8(src, pos)
104 if c != 0x2A {
105 if c != 0x2D {
106 if c != 0x5F {
107 if c != 0x3D { return 0 }
108 }
109 }
110 }
111 var p: i64 = pos
112 var run: i64 = 0
113 let BUDGET: i64 = len + 1
114 var iter: i64 = 0
115 var scanning: i64 = 1
116 while scanning == 1 {
117 if iter >= BUDGET { return 0 }
118 if p >= len { scanning = 0 }
119 if scanning == 1 {
120 let v: i64 = nx_text_load_u8(src, p)
121 if v == c { run = run + 1; p = p + 1 }
122 if v != c {
123 if v == 0x20 { p = p + 1 }
124 if v != 0x20 { scanning = 0 }
125 }
126 }
127 iter = iter + 1
128 }
129 if run < 3 { return 0 }
130 // Rest of line whitespace.
131 var tail_iter: i64 = 0
132 while p < len {
133 if tail_iter >= BUDGET { return 0 }
134 let v2: i64 = nx_text_load_u8(src, p)
135 if v2 == 0x0A { return 1 }
136 if nx_text_is_horiz_ws(v2) == 0 { return 0 }
137 p = p + 1
138 tail_iter = tail_iter + 1
139 }
140 return 1
141}
142
143// Markdown heading: 1-6 hashes followed by space.
144func nx_text_is_heading(src: *u8, pos: i64, len: i64) -> i64 {
145 if pos >= len { return 0 }
146 if nx_text_load_u8(src, pos) != 0x23 { return 0 }
147 var p: i64 = pos
148 var hashes: i64 = 0
149 let HBUDGET: i64 = 8
150 var hiter: i64 = 0
151 var hscan: i64 = 1
152 while hscan == 1 {
153 if hiter >= HBUDGET { hscan = 0 }
154 if hscan == 1 {
155 if p >= len { hscan = 0 }
156 if hscan == 1 {
157 if nx_text_load_u8(src, p) != 0x23 { hscan = 0 }
158 if hscan == 1 {
159 hashes = hashes + 1
160 p = p + 1
161 }
162 }
163 }
164 hiter = hiter + 1
165 }
166 if hashes < 1 { return 0 }
167 if hashes > 6 { return 0 }
168 if p >= len { return 0 }
169 if nx_text_load_u8(src, p) != 0x20 { return 0 }
170 return 1
171}
172
173// Skip blank lines starting at *cur. Returns count of blanks skipped.
174func nx_text_skip_blank_lines(src: *u8, cur: *i64, len: i64) -> i64 {
175 var n: i64 = 0
176 let BUDGET: i64 = len + 1
177 var iter: i64 = 0
178 var keep: i64 = 1
179 while keep == 1 {
180 if iter >= BUDGET { keep = 0 }
181 if keep == 1 {
182 if *cur >= len { keep = 0 }
183 if keep == 1 {
184 let save: i64 = *cur
185 var pp: i64 = *cur
186 // Skip horizontal whitespace.
187 var hs: i64 = 1
188 let HSB: i64 = len + 1
189 var hs_iter: i64 = 0
190 while hs == 1 {
191 if hs_iter >= HSB { hs = 0 }
192 if hs == 1 {
193 if pp >= len { hs = 0 }
194 if hs == 1 {
195 let v: i64 = nx_text_load_u8(src, pp)
196 if nx_text_is_horiz_ws(v) == 1 { pp = pp + 1 }
197 if nx_text_is_horiz_ws(v) == 0 { hs = 0 }
198 }
199 }
200 hs_iter = hs_iter + 1
201 }
202 if pp >= len { *cur = pp; keep = 0 }
203 if keep == 1 {
204 if nx_text_load_u8(src, pp) == 0x0A {
205 *cur = pp + 1
206 n = n + 1
207 }
208 if nx_text_load_u8(src, pp) != 0x0A {
209 *cur = save
210 keep = 0
211 }
212 }
213 }
214 }
215 iter = iter + 1
216 }
217 return n
218}
219
220// Find end of current line (offset of '\n' or len).
221func nx_text_line_end(src: *u8, pos: i64, len: i64) -> i64 {
222 var p: i64 = pos
223 let BUDGET: i64 = len + 1
224 var iter: i64 = 0
225 while p < len {
226 if iter >= BUDGET { return p }
227 if nx_text_load_u8(src, p) == 0x0A { return p }
228 p = p + 1
229 iter = iter + 1
230 }
231 return p
232}
233
234// ===== heuristic beat-kind classification =========================
235//
236// Cheap signals available at structural layer:
237// * '"' count > 0 -> DIALOGUE
238// * '*' wrappers -> INTERIORITY (italic text)
239// * very short + verb-led -> ACTION (defer to classifier for now)
240// Default: NARRATION.
241
242func nx_text_classify_paragraph(src: *u8, start: i64, end: i64) -> i64 {
243 var quotes: i64 = 0
244 var stars: i64 = 0
245 var p: i64 = start
246 let BUDGET: i64 = end - start + 1
247 var iter: i64 = 0
248 while p < end {
249 if iter >= BUDGET { return NX_BEAT_NARRATION }
250 let c: i64 = nx_text_load_u8(src, p)
251 if c == 0x22 { quotes = quotes + 1 }
252 if c == 0x2A { stars = stars + 1 }
253 p = p + 1
254 iter = iter + 1
255 }
256 if quotes >= 2 { return NX_BEAT_DIALOGUE }
257 if stars >= 2 { return NX_BEAT_INTERIORITY }
258 return NX_BEAT_NARRATION
259}
260
261// ===== main ingester ==============================================
262//
263// Walks `src` (looked up from db.media_pool[media_id]) emitting:
264// - one StoryBeat per paragraph / scene-break / heading
265// - one StoryScene per non-empty run between scene-breaks
266// - one StoryChapter per heading
267//
268// Returns NX_TEXT_INGEST_OK on success, negative verdict on failure.
269
270func nx_text_ingest(db: *StoryDb, media_id: i64) -> i64 {
271 // Validate media reference.
272 var src: *u8 = 0 as *u8
273 var src_len: i64 = 0
274 let rc: i64 = nx_media_pool_get(db.media_pool, media_id, &src, &src_len)
275 if rc != 0 { return NX_TEXT_INGEST_BAD_MEDIA_ID }
276 if src_len <= 0 { return NX_TEXT_INGEST_EMPTY }
277 let mtype: i64 = nx_media_pool_type(db.media_pool, media_id)
278 if mtype != NX_MEDIA_TYPE_TEXT {
279 if mtype != NX_MEDIA_TYPE_HTML {
280 return NX_TEXT_INGEST_NOT_TEXT
281 }
282 }
283
284 let pos_raw: *u8 = sys_mmap(16)
285 let pos: *i64 = pos_raw as *i64
286 *pos = 0
287
288 var cur_chapter: i64 = -1
289 var cur_chapter_scene_first: i64 = 0
290 var cur_scene: i64 = -1
291
292 // Skip leading blank lines.
293 let _bi: i64 = nx_text_skip_blank_lines(src, pos, src_len)
294
295 let OUTER_BUDGET: i64 = src_len + NX_TEXT_INGEST_BUDGET_PAD
296 var outer_iter: i64 = 0
297 var done: i64 = 0
298 while done == 0 {
299 if outer_iter >= OUTER_BUDGET { return NX_TEXT_INGEST_BUDGET_EXHAUSTED }
300 if *pos >= src_len { done = 1 }
301
302 if done == 0 {
303 // === heading ===
304 if nx_text_is_heading(src, *pos, src_len) == 1 {
305 // Close any open scene.
306 if cur_scene >= 0 {
307 nx_storydb_close_scene(db, cur_scene, db.n_beats - 1)
308 cur_scene = -1
309 }
310 // Close any open chapter.
311 if cur_chapter >= 0 {
312 nx_storydb_close_chapter(db, cur_chapter, db.n_scenes - 1)
313 }
314 // Emit heading beat.
315 let h_start: i64 = *pos
316 let h_end: i64 = nx_text_line_end(src, *pos, src_len)
317 let beat_id: i64 = nx_storydb_push_beat(db, NX_BEAT_HEADING,
318 NX_MEDIA_TYPE_TEXT,
319 media_id, h_start, h_end)
320 if beat_id < 0 { return NX_TEXT_INGEST_BEATS_FULL }
321 // Open new chapter starting at the next scene we'll create.
322 cur_chapter = nx_storydb_open_chapter(db, db.n_scenes, 0)
323 cur_chapter_scene_first = db.n_scenes
324 // Advance past line.
325 *pos = h_end
326 if *pos < src_len {
327 if nx_text_load_u8(src, *pos) == 0x0A { *pos = *pos + 1 }
328 }
329 let _bh: i64 = nx_text_skip_blank_lines(src, pos, src_len)
330 outer_iter = outer_iter + 1
331 continue
332 }
333
334 // === scene break ===
335 if nx_text_is_scene_break(src, *pos, src_len) == 1 {
336 if cur_scene >= 0 {
337 nx_storydb_close_scene(db, cur_scene, db.n_beats - 1)
338 cur_scene = -1
339 }
340 let t_start: i64 = *pos
341 let t_end: i64 = nx_text_line_end(src, *pos, src_len)
342 let tbeat: i64 = nx_storydb_push_beat(db, NX_BEAT_TRANSITION,
343 NX_MEDIA_TYPE_TEXT,
344 media_id, t_start, t_end)
345 if tbeat < 0 { return NX_TEXT_INGEST_BEATS_FULL }
346 *pos = t_end
347 if *pos < src_len {
348 if nx_text_load_u8(src, *pos) == 0x0A { *pos = *pos + 1 }
349 }
350 let _bt: i64 = nx_text_skip_blank_lines(src, pos, src_len)
351 outer_iter = outer_iter + 1
352 continue
353 }
354
355 // === regular paragraph ===
356 let para_start: i64 = *pos
357 var para_end: i64 = para_start
358 let PARA_BUDGET: i64 = src_len + 1
359 var pa_iter: i64 = 0
360 var pa_done: i64 = 0
361 while pa_done == 0 {
362 if pa_iter >= PARA_BUDGET { pa_done = 1 }
363 if pa_done == 0 {
364 let le: i64 = nx_text_line_end(src, *pos, src_len)
365 para_end = le
366 if le < src_len { *pos = le + 1 }
367 if le >= src_len { *pos = src_len; pa_done = 1 }
368 if pa_done == 0 {
369 // Peek next line: blank? scene-break? heading?
370 if *pos >= src_len { pa_done = 1 }
371 if pa_done == 0 {
372 var peek: i64 = *pos
373 var only_ws: i64 = 1
374 let PB: i64 = src_len + 1
375 var pk_iter: i64 = 0
376 var pk_scan: i64 = 1
377 while pk_scan == 1 {
378 if pk_iter >= PB { pk_scan = 0 }
379 if pk_scan == 1 {
380 if peek >= src_len { pk_scan = 0 }
381 if pk_scan == 1 {
382 let pv: i64 = nx_text_load_u8(src, peek)
383 if pv == 0x0A { pk_scan = 0 }
384 if pk_scan == 1 {
385 if nx_text_is_horiz_ws(pv) == 0 {
386 only_ws = 0
387 pk_scan = 0
388 }
389 if pk_scan == 1 { peek = peek + 1 }
390 }
391 }
392 }
393 pk_iter = pk_iter + 1
394 }
395 if peek >= src_len { pa_done = 1 }
396 if pa_done == 0 {
397 if only_ws == 1 { pa_done = 1 }
398 if pa_done == 0 {
399 if nx_text_load_u8(src, peek) == 0x0A { pa_done = 1 }
400 if pa_done == 0 {
401 if nx_text_is_scene_break(src, peek, src_len) == 1 {
402 pa_done = 1
403 }
404 if pa_done == 0 {
405 if nx_text_is_heading(src, peek, src_len) == 1 {
406 pa_done = 1
407 }
408 }
409 }
410 }
411 }
412 }
413 }
414 }
415 pa_iter = pa_iter + 1
416 }
417
418 // Open scene if needed.
419 if cur_scene < 0 {
420 cur_scene = nx_storydb_open_scene(db, db.n_beats, 0,
421 NX_ARCSTAGE_UNKNOWN,
422 NX_DOMAIN_UNKNOWN)
423 }
424 let kind: i64 = nx_text_classify_paragraph(src, para_start, para_end)
425 let pbeat: i64 = nx_storydb_push_beat(db, kind, NX_MEDIA_TYPE_TEXT,
426 media_id, para_start, para_end)
427 if pbeat < 0 { return NX_TEXT_INGEST_BEATS_FULL }
428 let _bp: i64 = nx_text_skip_blank_lines(src, pos, src_len)
429 }
430 outer_iter = outer_iter + 1
431 }
432
433 // Close trailing scene + chapter.
434 if cur_scene >= 0 {
435 nx_storydb_close_scene(db, cur_scene, db.n_beats - 1)
436 }
437 if cur_chapter >= 0 {
438 nx_storydb_close_chapter(db, cur_chapter, db.n_scenes - 1)
439 }
440
441 return NX_TEXT_INGEST_OK
442}