nx_keyness_test.nx source
↩ module page · 94 lines · 4069 B
1// nx_keyness_test.nx -- exercise keyness extraction primitive.
2
3import "nx_syscalls.nx"
4import "nx_tier.nx"
5import "nx_keyness.nx"
6
7func main() -> nx_int {
8 // ===== Direction sense ============================================
9 //
10 // Target: word appears 50/1000 = 5%
11 // Ref: word appears 10/10000 = 0.1%
12 // -> target over-represented (50x rate).
13 if nx_key_target_overrepresented(50, 1000, 10, 10000) != 1 { return 1 }
14
15 // Target rate < reference rate -> under-represented
16 if nx_key_target_overrepresented(10, 10000, 50, 1000) != 0 { return 2 }
17
18 // Equal rates -> not over (LHS == RHS, function returns 0)
19 if nx_key_target_overrepresented(100, 1000, 100, 1000) != 0 { return 3 }
20
21 // ===== G^2 sign convention ========================================
22 //
23 // Strong positive keyness: target 5%, ref 0.1%. G^2 large positive.
24 let g2_pos: nx_int = nx_keyness_g2_q10(50, 1000, 10, 10000)
25 if g2_pos <= 0 { return 10 }
26
27 // Strong negative keyness: target 0.1%, ref 5%. G^2 large negative.
28 let g2_neg: nx_int = nx_keyness_g2_q10(10, 10000, 50, 1000)
29 if g2_neg >= 0 { return 11 }
30
31 // Approximately symmetric magnitudes (same 2x2 just flipped corpora)
32 var mag_pos: nx_int = g2_pos
33 var mag_neg: nx_int = g2_neg
34 if mag_neg < 0 { mag_neg = 0 - mag_neg }
35 // Should agree within ~5% (Q10). Q10 ratio test:
36 let diff: nx_int = mag_pos - mag_neg
37 var abs_diff: nx_int = diff
38 if abs_diff < 0 { abs_diff = 0 - abs_diff }
39 let tol: nx_int = mag_pos / 20 // 5%
40 if abs_diff > tol { return 12 }
41
42 // ===== Band thresholds ============================================
43 //
44 // G^2 < 3.84 (Q10=3932) -> NEUTRAL regardless of sign
45 if nx_keyness_band(2000) != NX_KEY_NEUTRAL { return 20 }
46 if nx_keyness_band(0 - 2000) != NX_KEY_NEUTRAL { return 21 }
47 if nx_keyness_band(0) != NX_KEY_NEUTRAL { return 22 }
48
49 // 3.84 <= G^2 < 15.13 -> WEAK positive / negative
50 if nx_keyness_band(8000) != NX_KEY_POSITIVE_WEAK { return 23 }
51 if nx_keyness_band(0 - 8000) != NX_KEY_NEGATIVE_WEAK { return 24 }
52
53 // G^2 >= 15.13 -> STRONG positive / negative
54 if nx_keyness_band(20000) != NX_KEY_POSITIVE_STRONG { return 25 }
55 if nx_keyness_band(0 - 20000) != NX_KEY_NEGATIVE_STRONG { return 26 }
56
57 // Threshold edges
58 if nx_keyness_band(NX_KEY_THRESH_WEAK_Q10 - 1) != NX_KEY_NEUTRAL { return 27 }
59 if nx_keyness_band(NX_KEY_THRESH_STRONG_Q10) != NX_KEY_POSITIVE_STRONG { return 28 }
60
61 // ===== Log-ratio shape =============================================
62 //
63 // Target rate 5%, ref rate 0.1% -> log2(50) ~ 5.64 -> Q10 ~ 5775
64 // Yates smoothing shifts slightly but order-of-magnitude must hold.
65 let lr_strong: nx_int = nx_keyness_log_ratio_q10(50, 1000, 10, 10000)
66 if lr_strong < 2048 { return 30 } // > log2(4)
67
68 // Equal rates -> log-ratio near 0
69 let lr_equal: nx_int = nx_keyness_log_ratio_q10(100, 1000, 100, 1000)
70 var lr_equal_abs: nx_int = lr_equal
71 if lr_equal_abs < 0 { lr_equal_abs = 0 - lr_equal_abs }
72 if lr_equal_abs > 200 { return 31 } // small magnitude
73
74 // ===== Yates smoothing rescues zero-occurrence rows ===============
75 //
76 // Word absent from reference: ref count = 0 would log2(0). Yates
77 // shifts to log2((2*O+1)/1). Result should be finite + positive.
78 let lr_unique: nx_int = nx_keyness_log_ratio_q10(10, 1000, 0, 10000)
79 if lr_unique <= 0 { return 32 }
80
81 // Emit_all consolidates everything ==================================
82 let out: *i64 = (sys_mmap(NX_KEY_OUT_FIELDS * 8)) as *i64
83 nx_keyness_emit_all(50, 1000, 10, 10000, out)
84 if out[NX_KEY_OUT_G2_Q10] <= 0 { return 40 }
85 if out[NX_KEY_OUT_OVER] != 1 { return 41 }
86 if nx_key_kind_is_valid(out[NX_KEY_OUT_BAND]) == 0 { return 42 }
87 if out[NX_KEY_OUT_BAND] == NX_KEY_NEUTRAL { return 43 } // 50x rate is not neutral
88
89 // Equal-rate emit -> NEUTRAL band
90 nx_keyness_emit_all(100, 1000, 100, 1000, out)
91 if out[NX_KEY_OUT_BAND] != NX_KEY_NEUTRAL { return 44 }
92
93 return 0
94}