nx_term_extract.nx source
↩ module page · 164 lines · 6481 B
1// nx_term_extract.nx -- end-to-end learning orchestrator.
2//
3// For each beat in a StoryDb:
4// 1. Look up the beat's source bytes from the MediaPool.
5// 2. Run nx_noun_phrase_scan against the beat's span.
6// 3. For each candidate, observe it in nx_term_registry
7// (registry grows itself; no pre-seeded dictionary).
8// 4. Back-fill beat.outfit_id / location_id / character_id with
9// the registry slot indices for the strongest matches.
10//
11// Closes the loop between the ingestion (Tier 0+1) and the
12// director pipeline: downstream nx_arc / nx_moment_resolve consume
13// beats with populated element ids, with the actual vocabulary
14// surfaced from the ingested corpus itself.
15//
16// Per cardinals:
17// feedback-self-surfacing-intelligence:
18// Registry stays in STAGE1 SUPERVISED today. Promotion to
19// STAGE2 happens automatically after N sightings + 0
20// contradictions (per nx_term_registry policy). STAGE3
21// requires user pin or arc-anchor confirmation.
22// feedback-loras-and-negatives-are-patches:
23// Confidence is per-term auditable basis points, not an opaque
24// embedding. Every beat-field assignment is provenance-traceable
25// back to (beat_id, span_start, span_end, cue_index).
26//
27// nx_safety_envelope:
28// intended_use: "Walk a StoryDb's beats, surface noun-phrase
29// candidates, register them, and back-fill
30// beat.outfit_id / location_id / character_id."
31// sil_target: SIL2
32// asil_target: QM
33// dal_target: DAL C
34// iec_62304_class: NONE
35// evidence: [no_floating_point,
36// bounded_outer_beat_loop,
37// bounded_inner_candidate_loop,
38// registry_overflow_returns_error,
39// no_topical_dictionary_dependency]
40// hazard_register: [bug-tape-candidate-spans-cross-paragraph-boundary,
41// bug-tape-strongest-match-tie-breaker-undefined,
42// bug-tape-non-text-beat-skipped-silently]
43// residual_risk: "Per-beat candidate cap is 32 today; longer
44// paragraphs may saturate, dropping later
45// candidates. Caller sizes via the extract_sized
46// variant when ingesting large blocks."
47// verdict: NOT_YET_EVALUATED
48
49import "nx_syscalls.nx"
50import "nx_storybeat.nx"
51import "nx_storydb.nx"
52import "nx_media_pool.nx"
53import "nx_term_registry.nx"
54import "nx_noun_phrase_scan.nx"
55
56// ===== verdicts ===================================================
57
58const NX_EXTR_OK: i64 = 0
59const NX_EXTR_REGISTRY_FULL: i64 = -1
60const NX_EXTR_BUDGET_EXHAUSTED: i64 = -2
61
62const NX_EXTR_CAND_CAP: i64 = 32
63
64// ===== per-beat extraction ========================================
65//
66// Scan the beat's span, register every candidate, store the
67// strongest-confidence outfit / location / character slot indices
68// in the beat's corresponding fields.
69
70func nx_term_extract_beat(db: *StoryDb, registry: *TermRegistry,
71 beat_id: i64) -> i64 {
72 if beat_id < 0 { return NX_EXTR_OK }
73 if beat_id >= db.n_beats { return NX_EXTR_OK }
74 let b: *StoryBeat = nx_storydb_beat_at(db, beat_id)
75
76 // Only text-bearing media types yield text candidates.
77 if b.media_type != NX_MEDIA_TYPE_TEXT {
78 if b.media_type != NX_MEDIA_TYPE_HTML { return NX_EXTR_OK }
79 }
80
81 var src: *u8 = 0 as *u8
82 var src_len: i64 = 0
83 if nx_media_pool_get(db.media_pool, b.media_id,
84 &src, &src_len) != 0 {
85 return NX_EXTR_OK
86 }
87 if b.span_start < 0 { return NX_EXTR_OK }
88 if b.span_end > src_len { return NX_EXTR_OK }
89 if b.span_end <= b.span_start { return NX_EXTR_OK }
90
91 let cand_buf: *u8 = sys_mmap(NX_EXTR_CAND_CAP * NX_NP_CAND_BYTES)
92 let cands: *PhraseCandidate = cand_buf as *PhraseCandidate
93 let n_cands: i64 = nx_noun_phrase_scan(src, b.span_start, b.span_end,
94 cands, NX_EXTR_CAND_CAP)
95 if n_cands <= 0 { return NX_EXTR_OK }
96
97 var best_outfit_slot: i64 = -1
98 var best_outfit_conf: i64 = 0
99 var best_location_slot: i64 = -1
100 var best_location_conf: i64 = 0
101 var best_character_slot: i64 = -1
102 var best_character_conf: i64 = 0
103
104 let LOOP_BUDGET: i64 = n_cands + 1
105 var iter: i64 = 0
106 var i: i64 = 0
107 while i < n_cands {
108 if iter >= LOOP_BUDGET { return NX_EXTR_BUDGET_EXHAUSTED }
109 let c: *PhraseCandidate =
110 (((cand_buf as i64) + i * NX_NP_CAND_BYTES) as *PhraseCandidate)
111
112 let slot: i64 = nx_term_observe(registry, src,
113 c.span_start, c.span_end,
114 c.role_hint, c.confidence,
115 beat_id)
116 if slot < 0 { return NX_EXTR_REGISTRY_FULL }
117
118 if c.role_hint == NX_TERM_ROLE_OUTFIT {
119 if c.confidence > best_outfit_conf {
120 best_outfit_slot = slot
121 best_outfit_conf = c.confidence
122 }
123 }
124 if c.role_hint == NX_TERM_ROLE_LOCATION {
125 if c.confidence > best_location_conf {
126 best_location_slot = slot
127 best_location_conf = c.confidence
128 }
129 }
130 if c.role_hint == NX_TERM_ROLE_CHARACTER {
131 if c.confidence > best_character_conf {
132 best_character_slot = slot
133 best_character_conf = c.confidence
134 }
135 }
136 i = i + 1
137 iter = iter + 1
138 }
139
140 if best_outfit_slot >= 0 { b.outfit_id = best_outfit_slot }
141 if best_location_slot >= 0 { b.location_id = best_location_slot }
142 if best_character_slot >= 0 { b.character_id = best_character_slot }
143
144 return NX_EXTR_OK
145}
146
147// ===== whole-corpus extraction ====================================
148//
149// Walk every beat in db, run nx_term_extract_beat. Returns OK on
150// full success or first non-OK verdict encountered.
151
152func nx_term_extract_all(db: *StoryDb, registry: *TermRegistry) -> i64 {
153 let LOOP_BUDGET: i64 = db.n_beats + 1
154 var iter: i64 = 0
155 var i: i64 = 0
156 while i < db.n_beats {
157 if iter >= LOOP_BUDGET { return NX_EXTR_BUDGET_EXHAUSTED }
158 let rc: i64 = nx_term_extract_beat(db, registry, i)
159 if rc != NX_EXTR_OK { return rc }
160 i = i + 1
161 iter = iter + 1
162 }
163 return NX_EXTR_OK
164}