nx_keyness.nx source

1// nx_keyness.nx -- substrate-native keyword extraction. 2// 3// "Keyness" answers the canonical corpus-linguistics question: 4// what words are distinctively COMMON in target corpus vs reference? 5// 6// Standard metric is Dunning's log-likelihood applied to a 2x2 7// contingency table comparing token frequencies between corpora. 8// This is identical in shape to the collocation log-likelihood -- 9// the difference is the meaning of the cells: 10// 11// Collocation: word_a vs word_b in same corpus, n_ab co-occurrences 12// Keyness: same word in target vs reference corpus 13// 14// Composes: 15// nx_collocation.nx -- log-likelihood already there; we reuse it 16// 17// Sealed verdict bands (Sketch Engine convention, see Kilgarriff 2009): 18// POSITIVE_KEYWORD_STRONG -- strongly over-represented in target 19// POSITIVE_KEYWORD_WEAK -- modestly over-represented 20// NEUTRAL -- about the same in both 21// NEGATIVE_KEYWORD -- under-represented in target 22// 23// genealogy_id: dunning_1993_log_likelihood + scott_1997_wordsmith_keywords + 24// kilgarriff_2009_simple_maths 25// lineage_id: keyness_extraction_v1 26 27// nx_safety_envelope: 28// intended_use: AUTO_APPLIED -- primitive-specific tuning queued 29// sil_target: SIL1 30// evidence: [bulk_applied_2026-05-16, see-file-comment-for-detail] 31// verdict: NOT_YET_EVALUATED 32 33import "nx_syscalls.nx" 34import "nx_tier.nx" 35import "nx_collocation.nx" 36const NX_MAGIC_1024: i64 = 1024 37const NX_MAGIC_2048: i64 = 2048 38 39// ===== Sealed-enum: KeynessKind ==================================== 40// 41// Direction + magnitude of the keyness verdict. Caller routes the 42// output: POSITIVE keywords describe the target corpus's distinctive 43// vocabulary; NEGATIVE keywords describe what target LACKS vs ref. 44 45const NX_KEY_NEGATIVE_STRONG: nx_int = 0 // strongly under in target 46const NX_KEY_NEGATIVE_WEAK: nx_int = 1 47const NX_KEY_NEUTRAL: nx_int = 2 // ratio close to 1 48const NX_KEY_POSITIVE_WEAK: nx_int = 3 49const NX_KEY_POSITIVE_STRONG: nx_int = 4 // strongly over in target 50const NX_KEY_N_KINDS: nx_int = 5 51 52func nx_key_kind_is_valid(k: nx_int) -> nx_int { 53 if k < 0 { return 0 } 54 if k >= NX_KEY_N_KINDS { return 0 } 55 return 1 56} 57 58// ===== Direction: is target over-represented? ====================== 59// 60// observed_target / total_target vs observed_ref / total_ref 61// 62// Returns 1 if target rate exceeds reference rate, 0 otherwise. 63// Cross-multiplied to avoid division-by-zero + floating-point. 64 65func nx_key_target_overrepresented(observed_target: nx_int, total_target: nx_int, 66 observed_ref: nx_int, total_ref: nx_int) -> nx_int { 67 // target_rate > ref_rate 68 // observed_target * total_ref > observed_ref * total_target 69 let lhs: nx_int = observed_target * total_ref 70 let rhs: nx_int = observed_ref * total_target 71 if lhs > rhs { return 1 } 72 return 0 73} 74 75// ===== Core: keyness log-likelihood ================================ 76// 77// G^2 (Dunning 1993) -- log-likelihood ratio statistic on the 2x2 78// contingency table: 79// 80// target_corpus reference_corpus 81// word_w O_t O_r 82// not_w N_t - O_t N_r - O_r 83// 84// Returns the SIGNED Q10 G^2 value: 85// positive -- target over-represents the word 86// negative -- target under-represents the word 87// zero -- exact rate match 88// 89// Reuses nx_col_log_likelihood_q10 which already implements G^2; we 90// just feed it the (n_a=O_t+O_r, n_b=N_t, n_ab=O_t, n_total=N_t+N_r) 91// shape so the math comes out right, then apply a sign based on the 92// over/under direction. 93 94func nx_keyness_g2_q10(observed_target: nx_int, total_target: nx_int, 95 observed_ref: nx_int, total_ref: nx_int) -> nx_int { 96 let observed_total: nx_int = observed_target + observed_ref 97 let n_total: nx_int = total_target + total_ref 98 let g2_mag: nx_int = nx_col_log_likelihood_q10(observed_total, total_target, 99 observed_target, n_total) 100 // Sign: G^2 magnitude is always non-negative; we report direction 101 // as the signed version of the magnitude. 102 if nx_key_target_overrepresented(observed_target, total_target, 103 observed_ref, total_ref) == 1 { 104 return g2_mag 105 } 106 return 0 - g2_mag 107} 108 109// ===== Keyness band classifier ===================================== 110// 111// Thresholds from Rayson 2008 ("From keywords to key semantic 112// domains") on G^2 distribution at p < 0.0001 (15.13) and p < 0.05 113// (3.84). We Q10-encode: 114// 115// |G^2| < 3.84 (3932 Q10) -- NEUTRAL 116// |G^2| < 15.13 (15493 Q10) -- WEAK (positive or negative by sign) 117// |G^2| >= 15.13 -- STRONG 118 119const NX_KEY_THRESH_WEAK_Q10: nx_int = 3932 // 3.84 * 1024 120const NX_KEY_THRESH_STRONG_Q10: nx_int = 15493 // 15.13 * 1024 121 122func nx_keyness_band(g2_q10: nx_int) -> nx_int { 123 var mag: nx_int = g2_q10 124 if mag < 0 { mag = 0 - mag } 125 126 if mag < NX_KEY_THRESH_WEAK_Q10 { 127 return NX_KEY_NEUTRAL 128 } 129 if g2_q10 >= 0 { 130 if mag >= NX_KEY_THRESH_STRONG_Q10 { return NX_KEY_POSITIVE_STRONG } 131 return NX_KEY_POSITIVE_WEAK 132 } 133 if mag >= NX_KEY_THRESH_STRONG_Q10 { return NX_KEY_NEGATIVE_STRONG } 134 return NX_KEY_NEGATIVE_WEAK 135} 136 137// ===== Effect-size companion: log-ratio Q10 ======================== 138// 139// G^2 is statistically valid but magnitude depends on corpus size. 140// Hardie 2014 ("Log ratio: an informal introduction") proposes 141// log-ratio as the magnitude-only counterpart: 142// 143// LR = log2((O_t / N_t) / (O_r / N_r)) 144// = log2((O_t * N_r) / (O_r * N_t)) 145// 146// Pair LR with G^2 the way doctors pair p-value with effect size. 147// Caller decides which to rank on; we expose both. 148// 149// Cells receive Yates-style +0.5 smoothing internally to avoid 150// log2(0) on missing words (a common keyword case). Smoothing 151// applied in Q10 via doubling and adding 1 so the ratio shifts to 152// (2*O_t + 1) / (2*O_r + 1) and (2*N_t) / (2*N_r). 153 154// Inlined Mitchell 1962 integer log2 (same shape as nx_collocation's 155// _col_log2_q10). Q10 output. Defined BEFORE the public wrapper so 156// nxc2's single-pass resolver finds it. 157func _key_log2_ratio_q10(num: nx_int, den: nx_int) -> nx_int { 158 if num <= 0 { return 0 } 159 if den <= 0 { return 0 } 160 var lo_n: nx_int = 0 161 var v: nx_int = num 162 while v > 1 { 163 v = v / 2 164 lo_n = lo_n + 1 165 } 166 var lo_d: nx_int = 0 167 var w: nx_int = den 168 while w > 1 { 169 w = w / 2 170 lo_d = lo_d + 1 171 } 172 let int_part: nx_int = lo_n - lo_d 173 174 // Fractional part via fixed-point bit-by-bit refinement of the 175 // mantissa of num/den scaled into [1, 2). Cheap + branch-free. 176 // We compute (num * 2^10) / den then subtract 1024 to land in 177 // [0, 1024) -> directly the Q10 mantissa. 178 let mantissa_q10: nx_int = (num * NX_MAGIC_1024) / den 179 var frac: nx_int = mantissa_q10 180 // Bring mantissa to [1024, 2048) by halving until in range 181 while frac >= NX_MAGIC_2048 { 182 frac = frac / 2 183 } 184 let frac_clean: nx_int = frac - NX_MAGIC_1024 // approx Q10 frac 185 186 return int_part * NX_MAGIC_1024 + frac_clean 187} 188 189// Public wrapper: log-ratio between target and reference rates. 190func nx_keyness_log_ratio_q10(observed_target: nx_int, total_target: nx_int, 191 observed_ref: nx_int, total_ref: nx_int) -> nx_int { 192 if total_target <= 0 { return 0 } 193 if total_ref <= 0 { return 0 } 194 let num: nx_int = (2 * observed_target + 1) * total_ref 195 let den: nx_int = (2 * observed_ref + 1) * total_target 196 if num <= 0 { return 0 } 197 if den <= 0 { return 0 } 198 return _key_log2_ratio_q10(num, den) 199} 200 201// ===== Convenience emit: G^2 + band + log-ratio + direction ======= 202 203const NX_KEY_OUT_G2_Q10: nx_int = 0 204const NX_KEY_OUT_LOG_RATIO_Q10: nx_int = 1 205const NX_KEY_OUT_BAND: nx_int = 2 206const NX_KEY_OUT_OVER: nx_int = 3 // 1 = target over-represented 207const NX_KEY_OUT_FIELDS: nx_int = 4 208 209func nx_keyness_emit_all(observed_target: nx_int, total_target: nx_int, 210 observed_ref: nx_int, total_ref: nx_int, 211 out: *i64) -> nx_int { 212 let g2: nx_int = nx_keyness_g2_q10(observed_target, total_target, 213 observed_ref, total_ref) 214 let lr: nx_int = nx_keyness_log_ratio_q10(observed_target, total_target, 215 observed_ref, total_ref) 216 out[NX_KEY_OUT_G2_Q10] = g2 217 out[NX_KEY_OUT_LOG_RATIO_Q10] = lr 218 out[NX_KEY_OUT_BAND] = nx_keyness_band(g2) 219 out[NX_KEY_OUT_OVER] = nx_key_target_overrepresented(observed_target, total_target, 220 observed_ref, total_ref) 221 return 0 222}