nx_embfeat_lib.nx source
↩ module page · 160 lines · 6957 B
1// nx_embfeat_lib.nx -- DENSE EMBEDDING FEATURES for a hashed linear sequence model: the ONE library reader of the estate's
2// trained word embeddings (knowledge/index/embed_v1.bin, written by nx_embed_train: a 24-byte header [NXEMB1, nv, dim] then
3// nv x dim Q10 integers, row index shared with the PPMI vocabulary that nx_ppmi_lib resolves) turned into discrete features a
4// perceptron or CRF can weight. Each coordinate is quantised to a signed bucket at half-RMS steps and clamped at
5// +-EF_BUCKET_MAX; the step is DERIVED from the table itself at load (the RMS over every coordinate), never typed, so the
6// buckets follow the model's own scale. A word the vocabulary does not know reads EF_NONE at every coordinate and its
7// feature spells a single byte, degrading to the base features rather than to garbage. Two gates already read this file
8// inline (the reader gates); this is the copy both can import. license_tier: ORIGINAL No hw writes (Rule 26). LIB.
9import "nx_syscalls.nx"
10import "nx_reviewmine_lib.nx"
11import "nx_ppmi_lib.nx"
12
13const EF_G_BYTES: i64 = 1024 // the PPMI lib's g-block (its contract: a zeroed block of at least 632 bytes)
14const EF_HDR: i64 = 24 // magic 8 + nv 8 + dim 8
15const EF_I64_BYTES: i64 = 8
16const EF_MAGIC_N: i64 = 78
17const EF_MAGIC_X: i64 = 88
18const EF_MAGIC_E: i64 = 69
19const EF_MAGIC_M: i64 = 77
20const EF_MAGIC_B: i64 = 66
21const EF_MAGIC_1: i64 = 49
22const EF_BUCKET_MAX: i64 = 4 // buckets -4..4: nine cells at half-RMS steps resolve a bell-shaped coordinate
23const EF_STEP_DIV: i64 = 2 // step = rms / 2
24const EF_NONE: i64 = 0 - 1
25const EF_CH_ZERO: i64 = 48
26const EF_CH_AT: i64 = 64
27
28static ef_g: *i64
29static ef_tab: *i64 // the whole file, mapped; rows start at EF_HDR bytes
30static ef_nv: i64
31static ef_dim: i64
32static ef_step: i64 // Q10, derived
33static ef_rms: i64 // Q10, derived
34static ef_ready: i64
35static ef_lookups: i64 // words asked for since load
36static ef_hits: i64 // of those, known to the vocabulary
37
38func ef_reset() -> i64 {
39 if (ef_g as i64) == 0 { ef_g = sys_mmap(EF_G_BYTES) as *i64 }
40 ef_ready = 0
41 ef_nv = 0
42 ef_dim = 0
43 ef_step = 0
44 ef_rms = 0
45 ef_lookups = 0
46 ef_hits = 0
47 return 0
48}
49// load the PPMI vocabulary (the ONE loader) and the embedding table; 1 on success, 0 fail-closed with ef_ready 0
50func ef_load(ppmi_path: *u8, emb_path: *u8) -> i64 {
51 ef_reset()
52 if ppl_load(ef_g, ppmi_path) != 1 { return 0 }
53 let lp: *i64 = sys_mmap(RM_I64_PAIR) as *i64
54 lp[0] = 0
55 let b: *u8 = sys_read_file(emb_path, lp)
56 if (b as i64) == 0 { return 0 }
57 let n: i64 = lp[0]
58 if n < EF_HDR { return 0 }
59 if (b[0] as i64) != EF_MAGIC_N { return 0 }
60 if (b[1] as i64) != EF_MAGIC_X { return 0 }
61 if (b[2] as i64) != EF_MAGIC_E { return 0 }
62 if (b[3] as i64) != EF_MAGIC_M { return 0 }
63 if (b[4] as i64) != EF_MAGIC_B { return 0 }
64 if (b[5] as i64) != EF_MAGIC_1 { return 0 }
65 let h: *i64 = (b as i64 + 8) as *i64
66 let nv: i64 = h[0]
67 let dim: i64 = h[1]
68 if nv <= 0 { return 0 }
69 if dim <= 0 { return 0 }
70 if n < EF_HDR + nv * dim * EF_I64_BYTES { return 0 }
71 ef_tab = (b as i64 + EF_HDR) as *i64
72 ef_nv = nv
73 ef_dim = dim
74 // the scale: RMS over every coordinate, in Q10 like the values (squares of Q10 values are Q20; the root brings them back)
75 var ss: i64 = 0
76 var i: i64 = 0
77 let cnt: i64 = nv * dim
78 while i < cnt { let v: i64 = ef_tab[i]; ss = ss + v * v; i = i + 1 }
79 ef_rms = ppl_isqrt(ss / cnt)
80 ef_step = ef_rms / EF_STEP_DIV
81 if ef_step < 1 { ef_step = 1 }
82 ef_ready = 1
83 return 1
84}
85func ef_ok() -> i64 { return ef_ready }
86func ef_dim_of() -> i64 { return ef_dim }
87func ef_nv_of() -> i64 { return ef_nv }
88func ef_step_of() -> i64 { return ef_step }
89func ef_rms_of() -> i64 { return ef_rms }
90func ef_lookup_count() -> i64 { return ef_lookups }
91func ef_hit_count() -> i64 { return ef_hits }
92// the vocabulary id of a word, or EF_NONE (counted)
93func ef_wid(s: *u8, n: i64) -> i64 {
94 if ef_ready != 1 { return EF_NONE }
95 ef_lookups = ef_lookups + 1
96 let w: i64 = ppl_wid_range(ef_g, s, 0, n)
97 if w < 0 { return EF_NONE }
98 if w >= ef_nv { return EF_NONE }
99 ef_hits = ef_hits + 1
100 return w
101}
102// coordinate d of word wid, Q10
103func ef_coord(wid: i64, d: i64) -> i64 { return ef_tab[wid * ef_dim + d] }
104// a Q10 value to a signed bucket at half-RMS steps, clamped
105func ef_bucket(v: i64) -> i64 {
106 var b: i64 = v / ef_step
107 if b > EF_BUCKET_MAX { b = EF_BUCKET_MAX }
108 if b < (0 - EF_BUCKET_MAX) { b = 0 - EF_BUCKET_MAX }
109 return b
110}
111// spell coordinate d of a word into out as two bytes [dim byte, bucket byte]; an unknown word spells [dim byte, '@']
112func ef_spell(wid: i64, d: i64, out: *u8) -> i64 {
113 out[0] = (EF_CH_ZERO + d) as u8
114 if wid < 0 { out[1] = EF_CH_AT as u8; return 2 }
115 out[1] = (EF_CH_ZERO + EF_BUCKET_MAX + ef_bucket(ef_coord(wid, d))) as u8
116 return 2
117}
118// ---- FIXTURE: write a tiny NXEMB1 table for the words of a planted PPMI model. Rows must follow the vocabulary's sorted-hash
119// order (the same order nx_wordclust's wc_plant_model writes), so the words are sorted by db_semhash here exactly as there;
120// vecs holds nw x dim Q10 values in the CALLER's word order and is written in the sorted order.
121func ef_plant(path: *u8, buf: *u8, offs: *i64, lens: *i64, vecs: *i64, nw: i64, dim: i64) -> i64 {
122 let hashes: *i64 = sys_mmap(nw * EF_I64_BYTES) as *i64
123 let order: *i64 = sys_mmap(nw * EF_I64_BYTES) as *i64
124 var i: i64 = 0
125 while i < nw { hashes[i] = db_semhash(buf, offs[i], lens[i]); order[i] = i; i = i + 1 }
126 var a: i64 = 0
127 while a < nw {
128 var c: i64 = a + 1
129 while c < nw {
130 if hashes[order[c]] < hashes[order[a]] { let t: i64 = order[a]; order[a] = order[c]; order[c] = t }
131 c = c + 1
132 }
133 a = a + 1
134 }
135 let total: i64 = EF_HDR + nw * dim * EF_I64_BYTES
136 let blob: *u8 = sys_mmap(total + 64)
137 blob[0] = EF_MAGIC_N as u8; blob[1] = EF_MAGIC_X as u8; blob[2] = EF_MAGIC_E as u8; blob[3] = EF_MAGIC_M as u8
138 blob[4] = EF_MAGIC_B as u8; blob[5] = EF_MAGIC_1 as u8; blob[6] = 0; blob[7] = 0
139 let h: *i64 = (blob as i64 + 8) as *i64
140 h[0] = nw
141 h[1] = dim
142 let rows: *i64 = (blob as i64 + EF_HDR) as *i64
143 var r: i64 = 0
144 while r < nw {
145 let w: i64 = order[r]
146 var d: i64 = 0
147 while d < dim { rows[r * dim + d] = vecs[w * dim + d]; d = d + 1 }
148 r = r + 1
149 }
150 let fd: i64 = sys_openat_wr(path, MODE_0644)
151 if fd < 0 { return 0 }
152 var wrote: i64 = 0
153 while wrote < total {
154 let n: i64 = sys_write(fd, (blob as i64 + wrote) as *u8, total - wrote)
155 if n <= 0 { sys_close(fd); return 0 }
156 wrote = wrote + n
157 }
158 sys_close(fd)
159 return 1
160}