code wiki / _hdl_build / nx_stem.nx
nx_stem.nx source
↩ module page · 368 lines · 14381 B
1// nx_stem.nx -- SOVEREIGN morphological stemming: FULL Porter (1980), the estate's ONE stemmer.
2//
3// HISTORY: Step-1a-only -> PORTER-LITE 2026-07-03 (1a/1b/1c + a step-4 subset + -ly) -> FULL PORTER
4// 2026-08-13 (all 5 steps, first-match-stops inside each step, the paper's suffix tables complete).
5// The gate's own HONEST line named "fuller Porter steps" as the extension; the trigger was
6// nx_beir_eval measuring our BM25 at nDCG@10 305 permil vs the stemmed Anserini baseline 320 --
7// its header named stemming as the gap.
8//
9// CONSUMERS (grep-verified 2026-08-13, coverage_complete=1):
10// - nx_docportal_search_seg.nx: QUERY-TIME dictionary stem-expansion (both sides computed per
11// query, nothing persisted -> a stemmer upgrade shifts conflation keys CONSISTENTLY; the change
12// is DORMANT in the live daemon until its next rebuild -- RE-RUN THE MRR RULER on that rebuild).
13// - nx_stem_gate.nx (R-UX-2): every pre-upgrade vector verified to hold under full Porter.
14// - nx_beir_eval.nx: index+query tokenizer (the BEIR stemming rung).
15// HONEST SCOPE: no irregulars (children->child needs a lexicon). Buffer contract: step 1b can APPEND
16// one byte (-e restore) and nx_stem null-terminates, so out needs n+2 capacity.
17// exports: nx_stem (buffered) + vr_stem (in-place, backward-compat) + pst_stem (in-place core).
18// license_tier: ORIGINAL
19import "nx_syscalls.nx"
20
21// consonant test; y = consonant at 0, else the opposite of its predecessor
22func pst_cons(b: *u8, i: i64) -> i64 {
23 let c: i64 = b[i] as i64
24 if c == 97 { return 0 }
25 if c == 101 { return 0 }
26 if c == 105 { return 0 }
27 if c == 111 { return 0 }
28 if c == 117 { return 0 }
29 if c == 121 {
30 if i == 0 { return 1 }
31 if pst_cons(b, i - 1) == 1 { return 0 }
32 return 1
33 }
34 return 1
35}
36// measure m of b[0..j] inclusive (count of VC sequences); j<0 -> 0
37func pst_m(b: *u8, j: i64) -> i64 {
38 var n: i64 = 0
39 var i: i64 = 0
40 var go: i64 = 1
41 while go == 1 { if i > j { return n } if pst_cons(b, i) == 0 { go = 0 } else { i = i + 1 } }
42 var lp: i64 = 1
43 while lp == 1 {
44 go = 1
45 while go == 1 { if i > j { return n } if pst_cons(b, i) == 1 { go = 0 } else { i = i + 1 } }
46 n = n + 1
47 go = 1
48 while go == 1 { if i > j { return n } if pst_cons(b, i) == 0 { go = 0 } else { i = i + 1 } }
49 }
50 return n
51}
52func pst_vowelin(b: *u8, j: i64) -> i64 {
53 var i: i64 = 0
54 while i <= j { if pst_cons(b, i) == 0 { return 1 } i = i + 1 }
55 return 0
56}
57func pst_doublec(b: *u8, j: i64) -> i64 {
58 if j < 1 { return 0 }
59 if b[j] != b[j - 1] { return 0 }
60 return pst_cons(b, j)
61}
62// cvc ending at j, last c not w/x/y (the -e restore condition)
63func pst_cvc(b: *u8, j: i64) -> i64 {
64 if j < 2 { return 0 }
65 if pst_cons(b, j) == 0 { return 0 }
66 if pst_cons(b, j - 1) == 1 { return 0 }
67 if pst_cons(b, j - 2) == 0 { return 0 }
68 let c: i64 = b[j] as i64
69 if c == 119 { return 0 }
70 if c == 120 { return 0 }
71 if c == 121 { return 0 }
72 return 1
73}
74// stem length k-sl when b[0..k) ends with s[0..sl), else -1 (suffix may equal the whole word)
75func pst_ends(b: *u8, k: i64, s: *u8, sl: i64) -> i64 {
76 if sl > k { return 0 - 1 }
77 var i: i64 = 0
78 while i < sl { if b[k - sl + i] != s[i] { return 0 - 1 } i = i + 1 }
79 return k - sl
80}
81func pst_set(b: *u8, js: i64, s: *u8, sl: i64) -> i64 {
82 var i: i64 = 0
83 while i < sl { b[js + i] = s[i]; i = i + 1 }
84 return js + sl
85}
86
87func pst_step1a(b: *u8, k: i64) -> i64 {
88 let s4: *u8 = "sses" as *u8
89 var js: i64 = pst_ends(b, k, s4, 4)
90 if js >= 0 { return js + 2 }
91 let s3: *u8 = "ies" as *u8
92 js = pst_ends(b, k, s3, 3)
93 if js >= 0 { return js + 1 }
94 let s2: *u8 = "ss" as *u8
95 js = pst_ends(b, k, s2, 2)
96 if js >= 0 { return k }
97 let s1: *u8 = "s" as *u8
98 js = pst_ends(b, k, s1, 1)
99 if js >= 0 { return js }
100 return k
101}
102// shared tail of 1b after -ed/-ing removal
103func pst_step1b2(b: *u8, k: i64) -> i64 {
104 let sat: *u8 = "at" as *u8
105 let sate: *u8 = "ate" as *u8
106 var js: i64 = pst_ends(b, k, sat, 2)
107 if js >= 0 { return pst_set(b, js, sate, 3) }
108 let sbl: *u8 = "bl" as *u8
109 let sble: *u8 = "ble" as *u8
110 js = pst_ends(b, k, sbl, 2)
111 if js >= 0 { return pst_set(b, js, sble, 3) }
112 let siz: *u8 = "iz" as *u8
113 let size2: *u8 = "ize" as *u8
114 js = pst_ends(b, k, siz, 2)
115 if js >= 0 { return pst_set(b, js, size2, 3) }
116 if pst_doublec(b, k - 1) == 1 {
117 let c: i64 = b[k - 1] as i64
118 if c != 108 { if c != 115 { if c != 122 { return k - 1 } } }
119 return k
120 }
121 if pst_m(b, k - 1) == 1 { if pst_cvc(b, k - 1) == 1 { b[k] = 101 as u8; return k + 1 } }
122 return k
123}
124func pst_step1b(b: *u8, k: i64) -> i64 {
125 let seed: *u8 = "eed" as *u8
126 var js: i64 = pst_ends(b, k, seed, 3)
127 if js >= 0 {
128 if pst_m(b, js - 1) > 0 { return k - 1 }
129 return k
130 }
131 let sed: *u8 = "ed" as *u8
132 js = pst_ends(b, k, sed, 2)
133 if js >= 0 {
134 if pst_vowelin(b, js - 1) == 1 { return pst_step1b2(b, js) }
135 return k
136 }
137 let sing: *u8 = "ing" as *u8
138 js = pst_ends(b, k, sing, 3)
139 if js >= 0 {
140 if pst_vowelin(b, js - 1) == 1 { return pst_step1b2(b, js) }
141 return k
142 }
143 return k
144}
145func pst_step1c(b: *u8, k: i64) -> i64 {
146 let sy: *u8 = "y" as *u8
147 let js: i64 = pst_ends(b, k, sy, 1)
148 if js >= 0 { if pst_vowelin(b, js - 1) == 1 { b[k - 1] = 105 as u8 } }
149 return k
150}
151func pst_step2(b: *u8, k: i64) -> i64 {
152 let a1: *u8 = "ational" as *u8
153 let r1: *u8 = "ate" as *u8
154 var js: i64 = pst_ends(b, k, a1, 7)
155 if js >= 0 { if pst_m(b, js - 1) > 0 { return pst_set(b, js, r1, 3) } return k }
156 let a2: *u8 = "tional" as *u8
157 let r2: *u8 = "tion" as *u8
158 js = pst_ends(b, k, a2, 6)
159 if js >= 0 { if pst_m(b, js - 1) > 0 { return pst_set(b, js, r2, 4) } return k }
160 let a3: *u8 = "enci" as *u8
161 let r3: *u8 = "ence" as *u8
162 js = pst_ends(b, k, a3, 4)
163 if js >= 0 { if pst_m(b, js - 1) > 0 { return pst_set(b, js, r3, 4) } return k }
164 let a4: *u8 = "anci" as *u8
165 let r4: *u8 = "ance" as *u8
166 js = pst_ends(b, k, a4, 4)
167 if js >= 0 { if pst_m(b, js - 1) > 0 { return pst_set(b, js, r4, 4) } return k }
168 let a5: *u8 = "izer" as *u8
169 let r5: *u8 = "ize" as *u8
170 js = pst_ends(b, k, a5, 4)
171 if js >= 0 { if pst_m(b, js - 1) > 0 { return pst_set(b, js, r5, 3) } return k }
172 let a6: *u8 = "abli" as *u8
173 let r6: *u8 = "able" as *u8
174 js = pst_ends(b, k, a6, 4)
175 if js >= 0 { if pst_m(b, js - 1) > 0 { return pst_set(b, js, r6, 4) } return k }
176 let a7: *u8 = "alli" as *u8
177 let r7: *u8 = "al" as *u8
178 js = pst_ends(b, k, a7, 4)
179 if js >= 0 { if pst_m(b, js - 1) > 0 { return pst_set(b, js, r7, 2) } return k }
180 let a8: *u8 = "entli" as *u8
181 let r8: *u8 = "ent" as *u8
182 js = pst_ends(b, k, a8, 5)
183 if js >= 0 { if pst_m(b, js - 1) > 0 { return pst_set(b, js, r8, 3) } return k }
184 let a9: *u8 = "eli" as *u8
185 let r9: *u8 = "e" as *u8
186 js = pst_ends(b, k, a9, 3)
187 if js >= 0 { if pst_m(b, js - 1) > 0 { return pst_set(b, js, r9, 1) } return k }
188 let a10: *u8 = "ousli" as *u8
189 let r10: *u8 = "ous" as *u8
190 js = pst_ends(b, k, a10, 5)
191 if js >= 0 { if pst_m(b, js - 1) > 0 { return pst_set(b, js, r10, 3) } return k }
192 let a11: *u8 = "ization" as *u8
193 let r11: *u8 = "ize" as *u8
194 js = pst_ends(b, k, a11, 7)
195 if js >= 0 { if pst_m(b, js - 1) > 0 { return pst_set(b, js, r11, 3) } return k }
196 let a12: *u8 = "ation" as *u8
197 let r12: *u8 = "ate" as *u8
198 js = pst_ends(b, k, a12, 5)
199 if js >= 0 { if pst_m(b, js - 1) > 0 { return pst_set(b, js, r12, 3) } return k }
200 let a13: *u8 = "ator" as *u8
201 let r13: *u8 = "ate" as *u8
202 js = pst_ends(b, k, a13, 4)
203 if js >= 0 { if pst_m(b, js - 1) > 0 { return pst_set(b, js, r13, 3) } return k }
204 let a14: *u8 = "alism" as *u8
205 let r14: *u8 = "al" as *u8
206 js = pst_ends(b, k, a14, 5)
207 if js >= 0 { if pst_m(b, js - 1) > 0 { return pst_set(b, js, r14, 2) } return k }
208 let a15: *u8 = "iveness" as *u8
209 let r15: *u8 = "ive" as *u8
210 js = pst_ends(b, k, a15, 7)
211 if js >= 0 { if pst_m(b, js - 1) > 0 { return pst_set(b, js, r15, 3) } return k }
212 let a16: *u8 = "fulness" as *u8
213 let r16: *u8 = "ful" as *u8
214 js = pst_ends(b, k, a16, 7)
215 if js >= 0 { if pst_m(b, js - 1) > 0 { return pst_set(b, js, r16, 3) } return k }
216 let a17: *u8 = "ousness" as *u8
217 let r17: *u8 = "ous" as *u8
218 js = pst_ends(b, k, a17, 7)
219 if js >= 0 { if pst_m(b, js - 1) > 0 { return pst_set(b, js, r17, 3) } return k }
220 let a18: *u8 = "aliti" as *u8
221 let r18: *u8 = "al" as *u8
222 js = pst_ends(b, k, a18, 5)
223 if js >= 0 { if pst_m(b, js - 1) > 0 { return pst_set(b, js, r18, 2) } return k }
224 let a19: *u8 = "iviti" as *u8
225 let r19: *u8 = "ive" as *u8
226 js = pst_ends(b, k, a19, 5)
227 if js >= 0 { if pst_m(b, js - 1) > 0 { return pst_set(b, js, r19, 3) } return k }
228 let a20: *u8 = "biliti" as *u8
229 let r20: *u8 = "ble" as *u8
230 js = pst_ends(b, k, a20, 6)
231 if js >= 0 { if pst_m(b, js - 1) > 0 { return pst_set(b, js, r20, 3) } return k }
232 return k
233}
234func pst_step3(b: *u8, k: i64) -> i64 {
235 let a1: *u8 = "icate" as *u8
236 let r1: *u8 = "ic" as *u8
237 var js: i64 = pst_ends(b, k, a1, 5)
238 if js >= 0 { if pst_m(b, js - 1) > 0 { return pst_set(b, js, r1, 2) } return k }
239 let a2: *u8 = "ative" as *u8
240 js = pst_ends(b, k, a2, 5)
241 if js >= 0 { if pst_m(b, js - 1) > 0 { return js } return k }
242 let a3: *u8 = "alize" as *u8
243 let r3: *u8 = "al" as *u8
244 js = pst_ends(b, k, a3, 5)
245 if js >= 0 { if pst_m(b, js - 1) > 0 { return pst_set(b, js, r3, 2) } return k }
246 let a4: *u8 = "iciti" as *u8
247 let r4: *u8 = "ic" as *u8
248 js = pst_ends(b, k, a4, 5)
249 if js >= 0 { if pst_m(b, js - 1) > 0 { return pst_set(b, js, r4, 2) } return k }
250 let a5: *u8 = "ical" as *u8
251 let r5: *u8 = "ic" as *u8
252 js = pst_ends(b, k, a5, 4)
253 if js >= 0 { if pst_m(b, js - 1) > 0 { return pst_set(b, js, r5, 2) } return k }
254 let a6: *u8 = "ful" as *u8
255 js = pst_ends(b, k, a6, 3)
256 if js >= 0 { if pst_m(b, js - 1) > 0 { return js } return k }
257 let a7: *u8 = "ness" as *u8
258 js = pst_ends(b, k, a7, 4)
259 if js >= 0 { if pst_m(b, js - 1) > 0 { return js } return k }
260 return k
261}
262func pst_step4(b: *u8, k: i64) -> i64 {
263 let a1: *u8 = "al" as *u8
264 var js: i64 = pst_ends(b, k, a1, 2)
265 if js >= 0 { if pst_m(b, js - 1) > 1 { return js } return k }
266 let a2: *u8 = "ance" as *u8
267 js = pst_ends(b, k, a2, 4)
268 if js >= 0 { if pst_m(b, js - 1) > 1 { return js } return k }
269 let a3: *u8 = "ence" as *u8
270 js = pst_ends(b, k, a3, 4)
271 if js >= 0 { if pst_m(b, js - 1) > 1 { return js } return k }
272 let a4: *u8 = "er" as *u8
273 js = pst_ends(b, k, a4, 2)
274 if js >= 0 { if pst_m(b, js - 1) > 1 { return js } return k }
275 let a5: *u8 = "ic" as *u8
276 js = pst_ends(b, k, a5, 2)
277 if js >= 0 { if pst_m(b, js - 1) > 1 { return js } return k }
278 let a6: *u8 = "able" as *u8
279 js = pst_ends(b, k, a6, 4)
280 if js >= 0 { if pst_m(b, js - 1) > 1 { return js } return k }
281 let a7: *u8 = "ible" as *u8
282 js = pst_ends(b, k, a7, 4)
283 if js >= 0 { if pst_m(b, js - 1) > 1 { return js } return k }
284 let a8: *u8 = "ant" as *u8
285 js = pst_ends(b, k, a8, 3)
286 if js >= 0 { if pst_m(b, js - 1) > 1 { return js } return k }
287 let a9: *u8 = "ement" as *u8
288 js = pst_ends(b, k, a9, 5)
289 if js >= 0 { if pst_m(b, js - 1) > 1 { return js } return k }
290 let a10: *u8 = "ment" as *u8
291 js = pst_ends(b, k, a10, 4)
292 if js >= 0 { if pst_m(b, js - 1) > 1 { return js } return k }
293 let a11: *u8 = "ent" as *u8
294 js = pst_ends(b, k, a11, 3)
295 if js >= 0 { if pst_m(b, js - 1) > 1 { return js } return k }
296 let a12: *u8 = "ion" as *u8
297 js = pst_ends(b, k, a12, 3)
298 if js >= 0 {
299 if js > 0 {
300 let c: i64 = b[js - 1] as i64
301 if c == 115 { if pst_m(b, js - 1) > 1 { return js } return k }
302 if c == 116 { if pst_m(b, js - 1) > 1 { return js } return k }
303 }
304 return k
305 }
306 let a13: *u8 = "ou" as *u8
307 js = pst_ends(b, k, a13, 2)
308 if js >= 0 { if pst_m(b, js - 1) > 1 { return js } return k }
309 let a14: *u8 = "ism" as *u8
310 js = pst_ends(b, k, a14, 3)
311 if js >= 0 { if pst_m(b, js - 1) > 1 { return js } return k }
312 let a15: *u8 = "ate" as *u8
313 js = pst_ends(b, k, a15, 3)
314 if js >= 0 { if pst_m(b, js - 1) > 1 { return js } return k }
315 let a16: *u8 = "iti" as *u8
316 js = pst_ends(b, k, a16, 3)
317 if js >= 0 { if pst_m(b, js - 1) > 1 { return js } return k }
318 let a17: *u8 = "ous" as *u8
319 js = pst_ends(b, k, a17, 3)
320 if js >= 0 { if pst_m(b, js - 1) > 1 { return js } return k }
321 let a18: *u8 = "ive" as *u8
322 js = pst_ends(b, k, a18, 3)
323 if js >= 0 { if pst_m(b, js - 1) > 1 { return js } return k }
324 let a19: *u8 = "ize" as *u8
325 js = pst_ends(b, k, a19, 3)
326 if js >= 0 { if pst_m(b, js - 1) > 1 { return js } return k }
327 return k
328}
329func pst_step5a(b: *u8, k: i64) -> i64 {
330 if b[k - 1] != (101 as u8) { return k }
331 let a: i64 = pst_m(b, k - 2)
332 if a > 1 { return k - 1 }
333 if a == 1 { if pst_cvc(b, k - 2) == 0 { return k - 1 } }
334 return k
335}
336func pst_step5b(b: *u8, k: i64) -> i64 {
337 if b[k - 1] != (108 as u8) { return k }
338 if pst_doublec(b, k - 1) == 0 { return k }
339 if pst_m(b, k - 1) > 1 { return k - 1 }
340 return k
341}
342// full Porter, in place. b = lowercase [a-z0-9] word, length n, capacity >= n+2. Returns new length.
343func pst_stem(b: *u8, n: i64) -> i64 {
344 if n <= 2 { return n }
345 var k: i64 = n
346 k = pst_step1a(b, k)
347 k = pst_step1b(b, k)
348 k = pst_step1c(b, k)
349 k = pst_step2(b, k)
350 k = pst_step3(b, k)
351 k = pst_step4(b, k)
352 k = pst_step5a(b, k)
353 k = pst_step5b(b, k)
354 return k
355}
356
357// nx_stem: buffered (out may alias inp -- the initial copy is a safe self-copy then). Returns out length.
358func nx_stem(inp: *u8, n0: i64, out: *u8) -> i64 {
359 var i: i64 = 0
360 while i < n0 { out[i] = inp[i]; i = i + 1 }
361 let n: i64 = pst_stem(out, n0)
362 out[n] = 0 as u8
363 return n
364}
365// backward-compat in-place wrapper (the R-UX-2 gate + dss_stem_expand): stem w[0..n) in place, return new len
366func vr_stem(w: *u8, n: i64) -> i64 {
367 return nx_stem(w, n, w)
368}