nx_term_registry.nx source
↩ module page · 437 lines · 15458 B
1// nx_term_registry.nx -- open string-id pool with sightings + stage.
2//
3// Foundation of the LEARNING ingestion layer. Replaces hand-written
4// sealed-enum dictionaries (nx_outfit_vocabulary, nx_location_dict,
5// etc.) with a self-grown registry that LEARNS its vocabulary from
6// the ingested corpus.
7//
8// Every noun phrase the discovery layer finds gets registered here
9// with provenance: where first seen, how many times sighted across
10// the corpus, what semantic role (outfit / location / character /
11// action / object), and which self-surfacing-intelligence stage it
12// has reached.
13//
14// Per cardinals:
15// feedback-self-surfacing-intelligence-staged-autonomy:
16// STAGE1 SUPERVISED (default for fresh sightings),
17// STAGE2 PROMPTED (>= N sightings + 0 contradictions),
18// STAGE3 UNSUPERVISED (user-pinned or arc-anchor confirmed).
19// feedback-user-owns-every-bit:
20// this primitive holds the LIVE counts in-memory; durable
21// snapshots ship to nishi-library/seeds/*.toml on demand,
22// never via background daemon.
23// feedback-loras-and-negatives-are-patches:
24// the registry is a substrate-native CLOSED-LOOP measurement
25// surface -- not a black-box embedding -- so confidence is
26// auditable per-term.
27//
28// nx_safety_envelope:
29// intended_use: "Open string-id pool for discovered noun
30// phrases, with sighting counts, semantic
31// role tags, and self-surfacing stage."
32// sil_target: SIL2
33// asil_target: QM
34// dal_target: DAL C
35// iec_62304_class: NONE
36// evidence: [no_floating_point,
37// fixed_capacity_hash_table,
38// bounded_probe_length_per_jpl_rule_2,
39// sealed_role_enum,
40// sealed_stage_enum,
41// sighting_count_monotonic_only]
42// hazard_register: [bug-tape-hash-collision-shadowing,
43// bug-tape-registry-overflow-silent,
44// bug-tape-stale-stage-after-corpus-flush]
45// residual_risk: "Hash table is open-addressing with bounded
46// probe length; collisions beyond probe limit
47// return REGISTRY_FULL. Caller resizes via
48// nx_term_registry_alloc_sized when sizing
49// the workload."
50// verdict: NOT_YET_EVALUATED
51
52import "nx_syscalls.nx"
53
54// ===== sealed semantic roles ======================================
55//
56// Coarse classification of what KIND of thing this term is. A
57// single term can carry only one role today; future ambiguity gets
58// resolved via co-occurrence (the same surface form "robe" can
59// mean OUTFIT in one corpus and OBJECT in another -- the registry
60// records the dominant role).
61
62const NX_TERM_ROLE_UNKNOWN: i64 = 0
63const NX_TERM_ROLE_OUTFIT: i64 = 1
64const NX_TERM_ROLE_LOCATION: i64 = 2
65const NX_TERM_ROLE_CHARACTER: i64 = 3
66const NX_TERM_ROLE_ACTION: i64 = 4
67const NX_TERM_ROLE_OBJECT: i64 = 5
68const NX_TERM_ROLE_AFFECT: i64 = 6
69const NX_TERM_ROLE_TIME: i64 = 7
70
71// ===== sealed self-surfacing stages ===============================
72//
73// Mirrors nx_intelligence_stage.nx ordinals.
74
75const NX_TERM_STAGE_UNKNOWN: i64 = 0
76const NX_TERM_STAGE_SUPERVISED: i64 = 1 // fresh sighting, human reviews
77const NX_TERM_STAGE_PROMPTED: i64 = 2 // proposes; human approves
78const NX_TERM_STAGE_UNSUPERVISED: i64 = 3 // auto-applies; logs only
79
80// Default promotion thresholds (caller can override per role).
81const NX_TERM_PROMOTE_N_PROMPTED: i64 = 5 // 5 sightings -> STAGE2
82const NX_TERM_PROMOTE_N_UNSUPERVISED: i64 = 30 // user pin or 30+ sights
83
84// ===== Term record ================================================
85//
86// Fixed 96 bytes (12 * i64). Schema FROZEN; append-only.
87
88struct Term {
89 canon_hash: i64, // FNV-1a of lowercased canonical form
90 canon_off: i64, // offset into TermRegistry.str_arena
91 canon_len: i64, // byte length of canonical form
92
93 role: i64, // NX_TERM_ROLE_*
94 stage: i64, // NX_TERM_STAGE_*
95
96 sighting_count: i64, // total times this term observed
97 first_seen_beat: i64, // beat_id of first sighting
98 last_seen_beat: i64, // beat_id of most recent sighting
99
100 role_confidence: i64, // 0..10000 (basis points)
101 contradiction_ct: i64, // sightings tagged with different role
102 user_pinned: i64, // 1 if user manually promoted
103 reserved: i64, // future expansion
104}
105
106const NX_TERM_BYTES: i64 = 96
107
108// ===== TermRegistry container =====================================
109//
110// Open-addressing hash table indexed by canon_hash mod cap. Linear
111// probing with bounded probe length per JPL Rule 2. Strings stored
112// in a separate arena referenced by canon_off / canon_len.
113
114const NX_TERM_REGISTRY_DEFAULT_CAP: i64 = 16384
115const NX_TERM_REGISTRY_STR_ARENA: i64 = 1048576 // 1 MiB
116const NX_TERM_PROBE_LIMIT: i64 = 64
117
118struct TermRegistry {
119 terms: *Term, // length = cap; canon_hash == 0 means empty
120 cap: i64,
121 n_terms: i64,
122
123 str_arena: *u8,
124 str_len: i64,
125 str_cap: i64,
126}
127
128const NX_TERM_REGISTRY_BYTES: i64 = 48
129
130// ===== byte helpers ===============================================
131
132func nx_term_load_u8(p: *u8, i: i64) -> i64 {
133 let q: *u8 = ((p as i64) + i) as *u8
134 return *q
135}
136
137func nx_term_store_u8(p: *u8, i: i64, v: i64) {
138 let q: *u8 = ((p as i64) + i) as *u8
139 *q = v as u8
140}
141
142func nx_term_to_lower(c: i64) -> i64 {
143 if c >= 0x41 {
144 if c <= 0x5A { return c | 0x20 }
145 }
146 return c
147}
148
149// ===== FNV-1a 64-bit hash =========================================
150//
151// Re-implemented here to avoid pulling nx_string_ops's nx_int /
152// nx_tier / nx_essentials import chain. Lowercases ASCII before
153// hashing so "Apron" and "apron" collide intentionally.
154
155func nx_term_fnv1a_lower(buf: *u8, n: i64) -> i64 {
156 var h: i64 = 0xcbf29ce484222325
157 if n <= 0 { return h }
158 let BUDGET: i64 = n + 1
159 var iter: i64 = 0
160 var i: i64 = 0
161 while i < n {
162 if iter >= BUDGET { return h }
163 let c: i64 = nx_term_to_lower(nx_term_load_u8(buf, i))
164 h = h ^ c
165 h = h * 0x100000001b3
166 i = i + 1
167 iter = iter + 1
168 }
169 return h
170}
171
172// ===== allocation =================================================
173
174func nx_term_registry_alloc_sized(cap: i64, str_cap: i64) -> *TermRegistry {
175 var c: i64 = cap
176 var s: i64 = str_cap
177 if c <= 0 { c = NX_TERM_REGISTRY_DEFAULT_CAP }
178 if s <= 0 { s = NX_TERM_REGISTRY_STR_ARENA }
179
180 let raw: *u8 = sys_mmap(NX_TERM_REGISTRY_BYTES)
181 let r: *TermRegistry = raw as *TermRegistry
182 r.terms = (sys_mmap(c * NX_TERM_BYTES)) as *Term
183 r.cap = c
184 r.n_terms = 0
185 r.str_arena = sys_mmap(s)
186 r.str_len = 0
187 r.str_cap = s
188 return r
189}
190
191func nx_term_registry_alloc() -> *TermRegistry {
192 return nx_term_registry_alloc_sized(0, 0)
193}
194
195func nx_term_at(r: *TermRegistry, i: i64) -> *Term {
196 return (((r.terms as i64) + i * NX_TERM_BYTES) as *Term)
197}
198
199// ===== canonicalize + intern ======================================
200//
201// Take the raw byte span [start, end) from a beat's source buffer,
202// trim ASCII whitespace + simple punctuation from both ends, copy
203// to lowercase into the string arena, return (off, len). Returns
204// -1 off on arena overflow.
205
206func nx_term_is_trim(c: i64) -> i64 {
207 if c == 0x20 { return 1 } // space
208 if c == 0x09 { return 1 } // tab
209 if c == 0x0A { return 1 } // lf
210 if c == 0x0D { return 1 } // cr
211 if c == 0x2E { return 1 } // .
212 if c == 0x2C { return 1 } // ,
213 if c == 0x3B { return 1 } // ;
214 if c == 0x3A { return 1 } // :
215 if c == 0x21 { return 1 } // !
216 if c == 0x3F { return 1 } // ?
217 if c == 0x22 { return 1 } // "
218 if c == 0x27 { return 1 } // '
219 return 0
220}
221
222func nx_term_intern_canonical(r: *TermRegistry, src: *u8,
223 start: i64, end: i64,
224 out_off: *i64, out_len: *i64) -> i64 {
225 var s: i64 = start
226 var e: i64 = end
227 let TRIM_BUDGET: i64 = (end - start) + 2
228 var t1: i64 = 0
229 var advancing_l: i64 = 1
230 while advancing_l == 1 {
231 if t1 >= TRIM_BUDGET { advancing_l = 0 }
232 if advancing_l == 1 {
233 if s >= e { advancing_l = 0 }
234 if advancing_l == 1 {
235 if nx_term_is_trim(nx_term_load_u8(src, s)) == 1 { s = s + 1 }
236 if nx_term_is_trim(nx_term_load_u8(src, s)) == 0 { advancing_l = 0 }
237 }
238 }
239 t1 = t1 + 1
240 }
241 var t2: i64 = 0
242 var advancing_r: i64 = 1
243 while advancing_r == 1 {
244 if t2 >= TRIM_BUDGET { advancing_r = 0 }
245 if advancing_r == 1 {
246 if e <= s { advancing_r = 0 }
247 if advancing_r == 1 {
248 if nx_term_is_trim(nx_term_load_u8(src, e - 1)) == 1 { e = e - 1 }
249 if nx_term_is_trim(nx_term_load_u8(src, e - 1)) == 0 { advancing_r = 0 }
250 }
251 }
252 t2 = t2 + 1
253 }
254 let n: i64 = e - s
255 if n <= 0 { *out_off = -1; *out_len = 0; return -1 }
256 if r.str_len + n > r.str_cap { *out_off = -1; *out_len = 0; return -1 }
257
258 let off: i64 = r.str_len
259 var i: i64 = 0
260 while i < n {
261 let c: i64 = nx_term_to_lower(nx_term_load_u8(src, s + i))
262 nx_term_store_u8(r.str_arena, off + i, c)
263 i = i + 1
264 }
265 r.str_len = r.str_len + n
266 *out_off = off
267 *out_len = n
268 return 0
269}
270
271// ===== compare two canonical strings ==============================
272
273func nx_term_canon_eq(r: *TermRegistry, off1: i64, len1: i64,
274 off2: i64, len2: i64) -> i64 {
275 if len1 != len2 { return 0 }
276 var i: i64 = 0
277 while i < len1 {
278 let a: i64 = nx_term_load_u8(r.str_arena, off1 + i)
279 let b: i64 = nx_term_load_u8(r.str_arena, off2 + i)
280 if a != b { return 0 }
281 i = i + 1
282 }
283 return 1
284}
285
286// ===== lookup / insert ============================================
287//
288// Find the slot for `canon_hash` + canonical bytes via linear
289// probing. Returns the slot index, or -1 if registry is full
290// past NX_TERM_PROBE_LIMIT and the key was not present. When key
291// found, *was_present = 1; when free slot returned, *was_present = 0.
292//
293// Hash 0 reserved as "empty"; if input hash is 0 we shift to 1.
294
295func nx_term_find_slot(r: *TermRegistry, h: i64,
296 canon_off: i64, canon_len: i64,
297 was_present: *i64) -> i64 {
298 var hh: i64 = h
299 if hh == 0 { hh = 1 }
300 let cap: i64 = r.cap
301 var start: i64 = hh & 0x7FFFFFFFFFFFFFFF
302 start = start - ((start / cap) * cap)
303 var probe: i64 = 0
304 while probe < NX_TERM_PROBE_LIMIT {
305 let slot: i64 = (start + probe) - (((start + probe) / cap) * cap)
306 let t: *Term = nx_term_at(r, slot)
307 if t.canon_hash == 0 {
308 *was_present = 0
309 return slot
310 }
311 if t.canon_hash == hh {
312 if nx_term_canon_eq(r, t.canon_off, t.canon_len,
313 canon_off, canon_len) == 1 {
314 *was_present = 1
315 return slot
316 }
317 }
318 probe = probe + 1
319 }
320 *was_present = 0
321 return -1
322}
323
324// ===== observe ====================================================
325//
326// Top-level "I saw this term in this beat" call. Canonicalizes,
327// looks up (or inserts), bumps sighting_count, updates last_seen,
328// merges role/confidence using a simple weighted scheme.
329//
330// Returns the slot index of the term on success, or -1 on overflow.
331// On insert: role=`role`, confidence=`confidence`, stage=STAGE1.
332// On re-observe: sighting_count += 1; if existing role matches,
333// confidence increases (capped at 10000); if differs, contradiction_ct
334// rises and confidence falls.
335
336func nx_term_observe(r: *TermRegistry, src: *u8,
337 start: i64, end: i64,
338 role: i64, confidence: i64,
339 beat_id: i64) -> i64 {
340 var canon_off: i64 = -1
341 var canon_len: i64 = 0
342 if nx_term_intern_canonical(r, src, start, end,
343 &canon_off, &canon_len) != 0 {
344 return -1
345 }
346 let h: i64 = nx_term_fnv1a_lower(((r.str_arena as i64) + canon_off) as *u8,
347 canon_len)
348 var was: i64 = 0
349 let slot: i64 = nx_term_find_slot(r, h, canon_off, canon_len, &was)
350 if slot < 0 {
351 // Probe exhausted; arena bytes were spent on the canon copy
352 // we now can't keep -- caller treats -1 as registry overflow.
353 return -1
354 }
355 let t: *Term = nx_term_at(r, slot)
356 if was == 0 {
357 // Fresh insert.
358 t.canon_hash = h
359 if t.canon_hash == 0 { t.canon_hash = 1 }
360 t.canon_off = canon_off
361 t.canon_len = canon_len
362 t.role = role
363 t.stage = NX_TERM_STAGE_SUPERVISED
364 t.sighting_count = 1
365 t.first_seen_beat = beat_id
366 t.last_seen_beat = beat_id
367 t.role_confidence = confidence
368 t.contradiction_ct = 0
369 t.user_pinned = 0
370 t.reserved = 0
371 r.n_terms = r.n_terms + 1
372 return slot
373 }
374
375 // Re-observe: roll back the freshly-interned bytes we don't need.
376 if canon_off + canon_len == r.str_len {
377 r.str_len = canon_off
378 }
379 t.sighting_count = t.sighting_count + 1
380 t.last_seen_beat = beat_id
381 if t.role == role {
382 var nc: i64 = t.role_confidence + confidence
383 if nc > 10000 { nc = 10000 }
384 t.role_confidence = nc
385 }
386 if t.role != role {
387 if role != NX_TERM_ROLE_UNKNOWN {
388 t.contradiction_ct = t.contradiction_ct + 1
389 var nc2: i64 = t.role_confidence - confidence
390 if nc2 < 0 { nc2 = 0 }
391 t.role_confidence = nc2
392 }
393 }
394 // Auto-promote on sighting count (STAGE2 only; STAGE3 needs pin).
395 if t.user_pinned == 0 {
396 if t.stage == NX_TERM_STAGE_SUPERVISED {
397 if t.sighting_count >= NX_TERM_PROMOTE_N_PROMPTED {
398 if t.contradiction_ct == 0 {
399 t.stage = NX_TERM_STAGE_PROMPTED
400 }
401 }
402 }
403 }
404 return slot
405}
406
407// ===== user-pinning + lookup ======================================
408
409func nx_term_pin(r: *TermRegistry, slot: i64) -> i64 {
410 if slot < 0 { return -1 }
411 if slot >= r.cap { return -1 }
412 let t: *Term = nx_term_at(r, slot)
413 if t.canon_hash == 0 { return -1 }
414 t.user_pinned = 1
415 t.stage = NX_TERM_STAGE_UNSUPERVISED
416 return 0
417}
418
419func nx_term_lookup(r: *TermRegistry, src: *u8,
420 start: i64, end: i64) -> i64 {
421 // Canonicalize WITHOUT permanently interning by saving str_len.
422 let saved_len: i64 = r.str_len
423 var canon_off: i64 = -1
424 var canon_len: i64 = 0
425 if nx_term_intern_canonical(r, src, start, end,
426 &canon_off, &canon_len) != 0 {
427 return -1
428 }
429 let h: i64 = nx_term_fnv1a_lower(((r.str_arena as i64) + canon_off) as *u8,
430 canon_len)
431 var was: i64 = 0
432 let slot: i64 = nx_term_find_slot(r, h, canon_off, canon_len, &was)
433 // Roll back the speculative canon bytes.
434 r.str_len = saved_len
435 if was == 1 { return slot }
436 return -1
437}