nx_narrative_ingest.nx source
↩ module page · 530 lines · 19222 B
1// nx_narrative_ingest.nx -- DEPRECATED in favor of nx_text_ingest.nx.
2//
3// >>> NEW CODE: import "nx_text_ingest.nx" and call nx_text_ingest(db, media_id).
4// >>> Targets the canonical media-agnostic StoryBeat schema
5// >>> (nx_storybeat.nx + nx_storydb.nx + nx_media_pool.nx).
6// >>>
7// >>> This file is kept in tree only to avoid breaking any in-flight
8// >>> dependents from the concurrent-agent edit window 2026-05-16/17.
9// >>> Remove after audit that no consumer remains.
10//
11// ----- ORIGINAL HEADER BELOW (predates the multi-media schema) -----
12//
13// Foundation S-class ingester following the proven nx_coq_ingest /
14// nx_lean_ingest / nx_rfc_ingest pattern. Takes raw story text
15// (literotica chapter, hentai script, novel excerpt, fan-fic, etc.)
16// and parses it into a structured StoryDb of scenes + paragraphs +
17// beats that downstream extractors / arc recognizers / director
18// pipelines can consume.
19//
20// Scope of THIS file: structural ingestion only. No NLP / no
21// semantic extraction / no domain tagging / no arc recognition.
22// That's the next layer (nx_story_element_extract.nx,
23// nx_story_beat_extract.nx, nx_plot_arc_recognize.nx).
24//
25// Why structural-first: the same prose-tree shape works for ANY
26// narrative source. Browser-driven URL fetch, manual paste, voice
27// dictation, file-system batch -- all converge here.
28//
29// Sources expected:
30// * URL via nishi-browser fetch (Phase 7)
31// * Manual paste (user pastes chapter text)
32// * Voice/conversation ("ingest this scene for me")
33// * Batch (directory of .txt files)
34//
35// Substrate-native: no AI, no external network, no libc. Uses
36// canonical nx_lex / nx_ascii / nx_syscalls per cardinals.
37//
38// genealogy_id: wiedijk_qed_1994 (the unified-ingestion vision)
39// + propp_morphology_1928 (narrative structure roots)
40// lineage_id: formal_statement_parsing -> narrative_prose_parsing
41//
42// Composes with (downstream):
43// nx_story_element_extract.nx -- per-paragraph element extractors
44// nx_story_beat_extract.nx -- beat-kind classification
45// nx_plot_arc_recognize.nx -- exposition->climax->resolution
46// nx_scene_to_directive.nx -- StoryBeat -> DirectorsNote
47// nx_arc_pool_generate.nx -- recognized arc -> Arc with pools
48//
49// Composes with (upstream / driver options):
50// nx_html_parser.nx -- strip web wrapping from literotica
51// nishi-browser fetch -- URL -> raw text
52// manual stdin -- user paste
53
54// nx_safety_envelope:
55// intended_use: "Parse raw narrative prose into structured
56// StoryDb (chapters, scenes, paragraphs, beats).
57// Foundation for arc extraction + director
58// intent generation. No content judgment at
59// this layer."
60// sil_target: SIL2 (downstream arcs ride on this shape;
61// wrong boundaries = wrong scenes)
62// asil_target: QM
63// dal_target: DAL C
64// iec_62304_class: NONE
65// evidence: [no_floating_point,
66// fixed_capacity_arena_no_realloc,
67// sealed_beat_kind_enum,
68// canonical_lex_ascii_reuse]
69// hazard_register: [bug-tape-chapter-boundary-false-positive,
70// bug-tape-scene-break-marker-missed,
71// bug-tape-paragraph-cap-overflow]
72// residual_risk: "Structural-only -- doesn't catch semantic
73// scene boundaries that lack visual markers
74// (e.g. a scene-change indicated only by POV
75// shift in prose). Downstream beat-extractor
76// compensates."
77// verdict: NOT_YET_EVALUATED
78
79import "nx_syscalls.nx"
80import "nx_lex.nx"
81import "nx_ascii.nx"
82
83// ===== sealed beat kinds ============================================
84//
85// First-pass classification at the structural layer. The
86// semantic-classification layer (nx_story_beat_extract.nx) refines
87// to specific beat kinds (EROTIC_BUILD, INTIMATE_REVEAL,
88// CHARACTER_INTRO, etc.). Here we only mark what the LEXER can
89// know -- paragraph-shape signals.
90
91const NX_NARR_BEAT_UNKNOWN: i64 = 0 // not yet classified
92const NX_NARR_BEAT_NARRATION: i64 = 1 // pure prose paragraph
93const NX_NARR_BEAT_DIALOGUE: i64 = 2 // contains quoted speech
94const NX_NARR_BEAT_ACTION: i64 = 3 // short, verb-heavy
95const NX_NARR_BEAT_DESCRIPTION: i64 = 4 // long, adjective-heavy
96const NX_NARR_BEAT_TRANSITION: i64 = 5 // scene-break marker (***, ---)
97const NX_NARR_BEAT_HEADING: i64 = 6 // chapter/section title
98
99// ===== StoryBeat record ============================================
100//
101// One paragraph (or one transition marker / heading). Carries the
102// byte-span into the original source buffer so downstream extractors
103// can re-tokenize without copying.
104
105struct StoryBeat {
106 kind: i64, // NX_NARR_BEAT_*
107 char_start: i64, // byte offset in source buffer
108 char_end: i64, // exclusive
109 line_start: i64, // 1-based source line where this beat begins
110 chapter_idx: i64, // index into StoryDb.chapters (-1 if unbound)
111 scene_idx: i64, // index into StoryDb.scenes within chapter (-1)
112 n_chars: i64, // char_end - char_start (cached for fast filter)
113 n_words: i64, // approx word count (whitespace-split)
114 n_quotes: i64, // count of '"' characters (dialogue signal)
115}
116
117const NX_NARR_BEAT_BYTES: i64 = 72
118
119// ===== StoryScene + StoryChapter ===================================
120
121struct StoryScene {
122 chapter_idx: i64,
123 beat_start_idx: i64, // index into StoryDb.beats
124 beat_end_idx: i64, // exclusive
125 char_start: i64,
126 char_end: i64,
127}
128
129const NX_NARR_SCENE_BYTES: i64 = 40
130
131struct StoryChapter {
132 title_start: i64, // byte offset of title in source
133 title_len: i64,
134 scene_start_idx: i64,
135 scene_end_idx: i64,
136 beat_start_idx: i64,
137 beat_end_idx: i64,
138}
139
140const NX_NARR_CHAPTER_BYTES: i64 = 48
141
142// ===== StoryDb container ===========================================
143
144const NX_NARR_MAX_BEATS: i64 = 65536 // ~65k paragraphs / chapter book
145const NX_NARR_MAX_SCENES: i64 = 4096
146const NX_NARR_MAX_CHAPTERS: i64 = 256
147
148struct StoryDb {
149 src: *u8, // original buffer (not owned; caller frees)
150 src_len: i64,
151
152 beats: *StoryBeat,
153 n_beats: i64,
154 cap_beats: i64,
155
156 scenes: *StoryScene,
157 n_scenes: i64,
158 cap_scenes: i64,
159
160 chapters: *StoryChapter,
161 n_chapters: i64,
162 cap_chapters: i64,
163}
164
165func nx_narr_db_alloc(src: *u8, src_len: i64) -> *StoryDb {
166 let raw: *u8 = sys_mmap(80)
167 let db: *StoryDb = raw as *StoryDb
168 db.src = src
169 db.src_len = src_len
170 db.beats = (sys_mmap(NX_NARR_MAX_BEATS * NX_NARR_BEAT_BYTES)) as *StoryBeat
171 db.n_beats = 0
172 db.cap_beats = NX_NARR_MAX_BEATS
173 db.scenes = (sys_mmap(NX_NARR_MAX_SCENES * NX_NARR_SCENE_BYTES)) as *StoryScene
174 db.n_scenes = 0
175 db.cap_scenes = NX_NARR_MAX_SCENES
176 db.chapters = (sys_mmap(NX_NARR_MAX_CHAPTERS * NX_NARR_CHAPTER_BYTES)) as *StoryChapter
177 db.n_chapters = 0
178 db.cap_chapters = NX_NARR_MAX_CHAPTERS
179 return db
180}
181
182func nx_narr_beat_at(db: *StoryDb, i: i64) -> *StoryBeat {
183 return (((db.beats as i64) + i * NX_NARR_BEAT_BYTES) as *StoryBeat)
184}
185
186func nx_narr_scene_at(db: *StoryDb, i: i64) -> *StoryScene {
187 return (((db.scenes as i64) + i * NX_NARR_SCENE_BYTES) as *StoryScene)
188}
189
190func nx_narr_chapter_at(db: *StoryDb, i: i64) -> *StoryChapter {
191 return (((db.chapters as i64) + i * NX_NARR_CHAPTER_BYTES) as *StoryChapter)
192}
193
194// ===== lexer helpers ===============================================
195//
196// Reuses canonical nx_lex / nx_ascii. Prose-specific helpers below.
197
198// Is the byte at position `pos` the start of a scene-break marker?
199// Recognizes: 3+ asterisks on their own line (`***`, `* * *`),
200// 3+ hyphens (`---`), 3+ underscores (`___`),
201// 4+ equals (`====`). All surrounded by blank lines or
202// start/end of input.
203func nx_narr_is_scene_break(src: *u8, pos: i64, len: i64) -> i64 {
204 if pos >= len { return 0 }
205 let c: i64 = src[pos]
206 // Must be one of *, -, _, =
207 if c != 0x2A {
208 if c != 0x2D {
209 if c != 0x5F {
210 if c != 0x3D { return 0 }
211 }
212 }
213 }
214 // Count run of same char (allowing spaces between for "* * *")
215 var p: i64 = pos
216 var run: i64 = 0
217 while p < len {
218 if src[p] == c { run = run + 1; p = p + 1; continue }
219 if src[p] == 0x20 { p = p + 1; continue }
220 break
221 }
222 if run < 3 { return 0 }
223 // Rest of line must be whitespace only
224 while p < len {
225 if src[p] == 0x0A { return 1 }
226 if src[p] == 0x20 { p = p + 1; continue }
227 if src[p] == 0x09 { p = p + 1; continue }
228 return 0
229 }
230 return 1 // ran off end
231}
232
233// Is the byte at position `pos` the start of a markdown-style heading?
234// Recognizes `# Chapter ...`, `## ...`, `### ...`.
235func nx_narr_is_heading(src: *u8, pos: i64, len: i64) -> i64 {
236 if pos >= len { return 0 }
237 if src[pos] != 0x23 { return 0 } // '#'
238 // Count hashes
239 var p: i64 = pos
240 var hashes: i64 = 0
241 while p < len {
242 if src[p] != 0x23 { break }
243 hashes = hashes + 1
244 p = p + 1
245 }
246 if hashes < 1 { return 0 }
247 if hashes > 6 { return 0 }
248 // Must be followed by whitespace + text
249 if p >= len { return 0 }
250 if src[p] != 0x20 { return 0 }
251 return 1
252}
253
254// Advance pos past one line.
255func nx_narr_advance_line(src: *u8, pos: *i64, len: i64) -> i64 {
256 while pos[0] < len {
257 let c: i64 = src[pos[0]]
258 pos[0] = pos[0] + 1
259 if c == 0x0A { return 0 }
260 }
261 return 0
262}
263
264// Skip consecutive blank lines starting at pos. Returns count of
265// blank lines skipped.
266func nx_narr_skip_blank_lines(src: *u8, pos: *i64, len: i64) -> i64 {
267 var n: i64 = 0
268 while pos[0] < len {
269 let save: i64 = pos[0]
270 // Skip horizontal whitespace
271 while pos[0] < len {
272 let c: i64 = src[pos[0]]
273 if c != 0x20 {
274 if c != 0x09 { break }
275 }
276 pos[0] = pos[0] + 1
277 }
278 // Either newline (blank line) or non-blank content
279 if pos[0] >= len { return n }
280 if src[pos[0]] == 0x0A {
281 pos[0] = pos[0] + 1
282 n = n + 1
283 continue
284 }
285 // Not blank -- back up to start of this line
286 pos[0] = save
287 return n
288 }
289 return n
290}
291
292// ===== main ingester ===============================================
293//
294// Parses raw prose bytes into beats + scenes + chapters.
295//
296// Algorithm:
297// 1. Walk lines, tracking current line number.
298// 2. Each non-blank paragraph (sequence of non-blank lines until
299// a blank line / EOF / scene-break / heading) becomes one
300// StoryBeat with kind=UNKNOWN (semantic layer refines).
301// 3. Scene-break markers emit a BEAT_TRANSITION and close the
302// current scene.
303// 4. Headings emit a BEAT_HEADING and start a new chapter.
304// 5. End: close any open scene/chapter.
305
306func nx_narr_emit_beat(db: *StoryDb, kind: i64, start: i64, end: i64,
307 line: i64, chapter_idx: i64, scene_idx: i64) -> i64 {
308 if db.n_beats >= db.cap_beats { return -1 }
309 let b: *StoryBeat = nx_narr_beat_at(db, db.n_beats)
310 b.kind = kind
311 b.char_start = start
312 b.char_end = end
313 b.line_start = line
314 b.chapter_idx = chapter_idx
315 b.scene_idx = scene_idx
316 b.n_chars = end - start
317 // Quick word + quote count
318 var nw: i64 = 0
319 var nq: i64 = 0
320 var in_word: i64 = 0
321 var i: i64 = start
322 while i < end {
323 let c: i64 = db.src[i]
324 if c == 0x22 { nq = nq + 1 }
325 let is_ws: i64 = nx_lex_is_ws(c)
326 if is_ws == 1 {
327 in_word = 0
328 }
329 if is_ws == 0 {
330 if in_word == 0 {
331 nw = nw + 1
332 in_word = 1
333 }
334 }
335 i = i + 1
336 }
337 b.n_words = nw
338 b.n_quotes = nq
339 db.n_beats = db.n_beats + 1
340 return 0
341}
342
343func nx_narr_ingest(src: *u8, src_len: i64) -> *StoryDb {
344 let db: *StoryDb = nx_narr_db_alloc(src, src_len)
345
346 let pos_raw: *u8 = sys_mmap(16)
347 let pos: *i64 = pos_raw as *i64
348 pos[0] = 0
349
350 var line: i64 = 1
351 var cur_chapter: i64 = -1 // -1 = no chapter yet (implicit chapter 0)
352 var cur_scene: i64 = -1
353 var cur_scene_beat_start: i64 = 0
354
355 // Skip leading blank lines
356 let blanks_init: i64 = nx_narr_skip_blank_lines(src, pos, src_len)
357 line = line + blanks_init
358
359 while pos[0] < src_len {
360 // Check for heading
361 if nx_narr_is_heading(src, pos[0], src_len) == 1 {
362 // Close any open scene
363 if cur_scene >= 0 {
364 let s: *StoryScene = nx_narr_scene_at(db, cur_scene)
365 s.beat_end_idx = db.n_beats
366 s.char_end = pos[0]
367 cur_scene = -1
368 }
369 // Close any open chapter
370 if cur_chapter >= 0 {
371 let ch: *StoryChapter = nx_narr_chapter_at(db, cur_chapter)
372 ch.scene_end_idx = db.n_scenes
373 ch.beat_end_idx = db.n_beats
374 }
375 // Find title text: skip hashes + space
376 let title_line_start: i64 = pos[0]
377 var th: i64 = pos[0]
378 while th < src_len {
379 if src[th] != 0x23 { break }
380 th = th + 1
381 }
382 while th < src_len {
383 if src[th] != 0x20 { break }
384 th = th + 1
385 }
386 let title_start: i64 = th
387 // Find end of line for title
388 var line_end: i64 = th
389 while line_end < src_len {
390 if src[line_end] == 0x0A { break }
391 line_end = line_end + 1
392 }
393 // Emit chapter
394 if db.n_chapters < db.cap_chapters {
395 let ch_new: *StoryChapter = nx_narr_chapter_at(db, db.n_chapters)
396 ch_new.title_start = title_start
397 ch_new.title_len = line_end - title_start
398 ch_new.scene_start_idx = db.n_scenes
399 ch_new.scene_end_idx = db.n_scenes
400 ch_new.beat_start_idx = db.n_beats
401 ch_new.beat_end_idx = db.n_beats
402 cur_chapter = db.n_chapters
403 db.n_chapters = db.n_chapters + 1
404 }
405 // Emit heading beat
406 nx_narr_emit_beat(db, NX_NARR_BEAT_HEADING, title_line_start,
407 line_end, line, cur_chapter, -1)
408 pos[0] = line_end
409 if pos[0] < src_len {
410 if src[pos[0]] == 0x0A { pos[0] = pos[0] + 1 }
411 }
412 line = line + 1
413 let bb: i64 = nx_narr_skip_blank_lines(src, pos, src_len)
414 line = line + bb
415 continue
416 }
417
418 // Check for scene break
419 if nx_narr_is_scene_break(src, pos[0], src_len) == 1 {
420 // Close current scene
421 if cur_scene >= 0 {
422 let s2: *StoryScene = nx_narr_scene_at(db, cur_scene)
423 s2.beat_end_idx = db.n_beats
424 s2.char_end = pos[0]
425 cur_scene = -1
426 }
427 // Emit transition beat
428 let trans_start: i64 = pos[0]
429 var trans_end: i64 = pos[0]
430 while trans_end < src_len {
431 if src[trans_end] == 0x0A { break }
432 trans_end = trans_end + 1
433 }
434 nx_narr_emit_beat(db, NX_NARR_BEAT_TRANSITION, trans_start,
435 trans_end, line, cur_chapter, -1)
436 pos[0] = trans_end
437 if pos[0] < src_len {
438 if src[pos[0]] == 0x0A { pos[0] = pos[0] + 1 }
439 }
440 line = line + 1
441 let bb2: i64 = nx_narr_skip_blank_lines(src, pos, src_len)
442 line = line + bb2
443 continue
444 }
445
446 // It's a regular paragraph -- find end (blank line or EOF)
447 let para_start: i64 = pos[0]
448 let para_line: i64 = line
449 var para_end: i64 = para_start
450 while pos[0] < src_len {
451 // Find end of THIS line
452 var line_end2: i64 = pos[0]
453 while line_end2 < src_len {
454 if src[line_end2] == 0x0A { break }
455 line_end2 = line_end2 + 1
456 }
457 // Update para_end to end of this line
458 para_end = line_end2
459 // Advance past newline
460 if line_end2 < src_len {
461 pos[0] = line_end2 + 1
462 line = line + 1
463 }
464 if line_end2 >= src_len {
465 pos[0] = src_len
466 }
467 // Check: is the NEXT line blank (or scene-break or heading)?
468 if pos[0] >= src_len { break }
469 // Peek: if next non-empty char before \n is whitespace-only,
470 // or if it's a scene break / heading, the paragraph ends.
471 var peek: i64 = pos[0]
472 var only_ws: i64 = 1
473 while peek < src_len {
474 let cc: i64 = src[peek]
475 if cc == 0x0A {
476 if only_ws == 1 { break }
477 break
478 }
479 if cc != 0x20 {
480 if cc != 0x09 { only_ws = 0; break }
481 }
482 peek = peek + 1
483 }
484 if peek >= src_len { break }
485 if only_ws == 1 { break }
486 if src[peek] == 0x0A { break }
487 if nx_narr_is_scene_break(src, peek, src_len) == 1 { break }
488 if nx_narr_is_heading(src, peek, src_len) == 1 { break }
489 // Otherwise continue collecting this paragraph
490 }
491
492 // Open a scene if needed
493 if cur_scene < 0 {
494 if db.n_scenes < db.cap_scenes {
495 let s3: *StoryScene = nx_narr_scene_at(db, db.n_scenes)
496 s3.chapter_idx = cur_chapter
497 s3.beat_start_idx = db.n_beats
498 s3.beat_end_idx = db.n_beats
499 s3.char_start = para_start
500 s3.char_end = para_end
501 cur_scene = db.n_scenes
502 db.n_scenes = db.n_scenes + 1
503 if cur_chapter >= 0 {
504 let chp: *StoryChapter = nx_narr_chapter_at(db, cur_chapter)
505 chp.scene_end_idx = db.n_scenes
506 }
507 }
508 }
509
510 nx_narr_emit_beat(db, NX_NARR_BEAT_NARRATION, para_start,
511 para_end, para_line, cur_chapter, cur_scene)
512
513 let bb3: i64 = nx_narr_skip_blank_lines(src, pos, src_len)
514 line = line + bb3
515 }
516
517 // Close any open scene/chapter
518 if cur_scene >= 0 {
519 let sf: *StoryScene = nx_narr_scene_at(db, cur_scene)
520 sf.beat_end_idx = db.n_beats
521 sf.char_end = pos[0]
522 }
523 if cur_chapter >= 0 {
524 let chf: *StoryChapter = nx_narr_chapter_at(db, cur_chapter)
525 chf.scene_end_idx = db.n_scenes
526 chf.beat_end_idx = db.n_beats
527 }
528
529 return db
530}