nx_stem.nx source

1// nx_stem.nx -- SOVEREIGN morphological stemming (the stemming/lemmatization SOTA-parity rung; sota_stemming 2// canon). Inflected forms map to a common conflation root so a query recalls all forms: run/running/runs, 3// rank/ranking/ranked, index/indexes/indexing. Upgraded 2026-07-03 from Step-1a-only to a PORTER-LITE stemmer 4// (the high-recall steps: plurals + -ed/-ing + -ly + common derivational suffixes) with Porter's measure(m) 5// guard that stops over-stemming short words. NO index-format change: it drives QUERY-TIME dictionary 6// expansion (dss_stem_expand), so indexing is untouched. HONEST SCOPE: no irregulars (children->child needs a 7// lexicon). exports: nx_stem (buffered) + vr_stem (in-place, backward-compat). license_tier: ORIGINAL 8import "nx_syscalls.nx" 9 10func stm_isvowel(c: u8, prev: u8) -> i64 { 11 if c == (97 as u8) { return 1 } 12 if c == (101 as u8) { return 1 } 13 if c == (105 as u8) { return 1 } 14 if c == (111 as u8) { return 1 } 15 if c == (117 as u8) { return 1 } 16 if c == (121 as u8) { // y is a vowel iff preceded by a consonant 17 if prev == (0 as u8) { return 0 } 18 if stm_isvowel(prev, 0 as u8) == 1 { return 0 } 19 return 1 20 } 21 return 0 22} 23// Porter measure m = count of vowel->consonant transitions 24func stm_measure(s: *u8, n: i64) -> i64 { 25 var m: i64 = 0 26 var prevv: i64 = 0 - 1 27 var i: i64 = 0 28 var pc: u8 = 0 as u8 29 while i < n { 30 let v: i64 = stm_isvowel(s[i], pc) 31 if prevv == 1 { if v == 0 { m = m + 1 } } 32 prevv = v 33 pc = s[i] 34 i = i + 1 35 } 36 return m 37} 38func stm_hasvowel(s: *u8, n: i64) -> i64 { 39 var i: i64 = 0 40 var pc: u8 = 0 as u8 41 while i < n { if stm_isvowel(s[i], pc) == 1 { return 1 } pc = s[i]; i = i + 1 } 42 return 0 43} 44func stm_ends(s: *u8, n: i64, suf: *u8, sl: i64) -> i64 { 45 if n < sl { return 0 } 46 var i: i64 = 0 47 while i < sl { if s[n - sl + i] != suf[i] { return 0 } i = i + 1 } 48 return 1 49} 50func stm_dbl_consonant_end(s: *u8, n: i64) -> i64 { 51 if n < 2 { return 0 } 52 if s[n - 1] != s[n - 2] { return 0 } 53 if stm_isvowel(s[n - 1], s[n - 2]) == 1 { return 0 } 54 return 1 55} 56func stm_cvc(s: *u8, n: i64) -> i64 { 57 if n < 3 { return 0 } 58 let c1: u8 = s[n - 3]; let c2: u8 = s[n - 2]; let c3: u8 = s[n - 1] 59 if stm_isvowel(c1, 0 as u8) == 1 { return 0 } 60 if stm_isvowel(c2, c1) == 0 { return 0 } 61 if stm_isvowel(c3, c2) == 1 { return 0 } 62 if c3 == (119 as u8) { return 0 } 63 if c3 == (120 as u8) { return 0 } 64 if c3 == (121 as u8) { return 0 } 65 return 1 66} 67 68// nx_stem: buffered (out may alias inp -- the initial copy is a safe self-copy then). Returns out length. 69func nx_stem(inp: *u8, n0: i64, out: *u8) -> i64 { 70 var n: i64 = n0 71 var i: i64 = 0 72 while i < n { out[i] = inp[i]; i = i + 1 } 73 if n <= 2 { out[n] = 0 as u8; return n } 74 75 // Step 1a: plurals 76 if stm_ends(out, n, "sses" as *u8, 4) == 1 { n = n - 2 } 77 else { if stm_ends(out, n, "ies" as *u8, 3) == 1 { n = n - 2 } 78 else { if stm_ends(out, n, "ss" as *u8, 2) == 1 { n = n } 79 else { if stm_ends(out, n, "s" as *u8, 1) == 1 { n = n - 1 } } } } 80 81 // Step 1b: -eed / -ed / -ing 82 var did1b: i64 = 0 83 if stm_ends(out, n, "eed" as *u8, 3) == 1 { 84 if stm_measure(out, n - 3) > 0 { n = n - 1 } 85 } else { 86 if stm_ends(out, n, "ed" as *u8, 2) == 1 { 87 if stm_hasvowel(out, n - 2) == 1 { n = n - 2; did1b = 1 } 88 } else { 89 if stm_ends(out, n, "ing" as *u8, 3) == 1 { 90 if stm_hasvowel(out, n - 3) == 1 { n = n - 3; did1b = 1 } 91 } 92 } 93 } 94 if did1b == 1 { 95 if stm_ends(out, n, "at" as *u8, 2) == 1 { out[n] = 101 as u8; n = n + 1 } 96 else { if stm_ends(out, n, "bl" as *u8, 2) == 1 { out[n] = 101 as u8; n = n + 1 } 97 else { if stm_ends(out, n, "iz" as *u8, 2) == 1 { out[n] = 101 as u8; n = n + 1 } 98 else { 99 if stm_dbl_consonant_end(out, n) == 1 { 100 let last: u8 = out[n - 1] 101 if last != (108 as u8) { if last != (115 as u8) { if last != (122 as u8) { n = n - 1 } } } 102 } else { 103 if stm_measure(out, n) == 1 { if stm_cvc(out, n) == 1 { out[n] = 101 as u8; n = n + 1 } } 104 } 105 } } } 106 } 107 108 // Step 1c: y -> i when the stem has a vowel 109 if stm_ends(out, n, "y" as *u8, 1) == 1 { if stm_hasvowel(out, n - 1) == 1 { out[n - 1] = 105 as u8 } } 110 111 // Step 4 (subset): common derivational suffixes when (m>1) 112 if stm_ends(out, n, "ement" as *u8, 5) == 1 { if stm_measure(out, n - 5) > 1 { n = n - 5 } } 113 else { if stm_ends(out, n, "ment" as *u8, 4) == 1 { if stm_measure(out, n - 4) > 1 { n = n - 4 } } 114 else { if stm_ends(out, n, "ness" as *u8, 4) == 1 { if stm_measure(out, n - 4) > 1 { n = n - 4 } } 115 else { if stm_ends(out, n, "able" as *u8, 4) == 1 { if stm_measure(out, n - 4) > 1 { n = n - 4 } } 116 else { if stm_ends(out, n, "ible" as *u8, 4) == 1 { if stm_measure(out, n - 4) > 1 { n = n - 4 } } 117 else { if stm_ends(out, n, "al" as *u8, 2) == 1 { if stm_measure(out, n - 2) > 1 { n = n - 2 } } 118 else { if stm_ends(out, n, "er" as *u8, 2) == 1 { if stm_measure(out, n - 2) > 1 { n = n - 2 } } } } } } } } 119 120 // Step 1d: -ly (adverbs) when (m>1) 121 if stm_ends(out, n, "ly" as *u8, 2) == 1 { if stm_measure(out, n - 2) > 1 { n = n - 2 } } 122 123 out[n] = 0 as u8 124 return n 125} 126// backward-compat in-place wrapper (the R-UX-2 gate + any prior caller): stem w[0..n) in place, return new len 127func vr_stem(w: *u8, n: i64) -> i64 { 128 return nx_stem(w, n, w) 129}