nx_ngram_test.nx source
↩ module page · 171 lines · 6867 B
1// nx_ngram_test.nx -- smoke for n-gram extraction + frequency + dispersion.
2
3// Single import of nx_ngram brings the whole chain transitively.
4import "nx_ngram.nx"
5
6func main() -> nx_int {
7 // === Test 1: hash determinism ===================================
8 // Same token sequence -> same hash.
9 let tok_a: *i64 = (sys_mmap(3 * NX_SIZEOF_NX_INT)) as *i64
10 tok_a[0] = 100
11 tok_a[1] = 200
12 tok_a[2] = 300
13 let tok_b: *i64 = (sys_mmap(3 * NX_SIZEOF_NX_INT)) as *i64
14 tok_b[0] = 100
15 tok_b[1] = 200
16 tok_b[2] = 300
17 let h_a: nx_int = nx_ng_hash(tok_a, 3)
18 let h_b: nx_int = nx_ng_hash(tok_b, 3)
19 if h_a != h_b { return 1 }
20
21 // Different order -> different hash
22 let tok_c: *i64 = (sys_mmap(3 * NX_SIZEOF_NX_INT)) as *i64
23 tok_c[0] = 200
24 tok_c[1] = 100
25 tok_c[2] = 300
26 let h_c: nx_int = nx_ng_hash(tok_c, 3)
27 if h_c == h_a { return 2 }
28
29 // Different content -> different hash
30 let tok_d: *i64 = (sys_mmap(3 * NX_SIZEOF_NX_INT)) as *i64
31 tok_d[0] = 100
32 tok_d[1] = 200
33 tok_d[2] = 999
34 let h_d: nx_int = nx_ng_hash(tok_d, 3)
35 if h_d == h_a { return 3 }
36
37 // === Test 2: extraction over a stream ==========================
38 // Stream: [10, 20, 30, 40, 50] -> bigrams = [(10,20), (20,30), (30,40), (40,50)]
39 let stream: *i64 = (sys_mmap(5 * NX_SIZEOF_NX_INT)) as *i64
40 stream[0] = 10
41 stream[1] = 20
42 stream[2] = 30
43 stream[3] = 40
44 stream[4] = 50
45 let hashes: *i64 = (sys_mmap(8 * NX_SIZEOF_NX_INT)) as *i64
46 let n_bigrams: nx_int = nx_ng_extract(stream, 5, 2, hashes)
47 if n_bigrams != 4 { return 10 }
48 // All four bigram hashes should be distinct (sequences differ)
49 if hashes[0] == hashes[1] { return 11 }
50 if hashes[1] == hashes[2] { return 12 }
51 if hashes[2] == hashes[3] { return 13 }
52
53 // Trigrams: 3 of them
54 let n_trigrams: nx_int = nx_ng_extract(stream, 5, 3, hashes)
55 if n_trigrams != 3 { return 14 }
56
57 // Edge: stream shorter than n -> 0
58 if nx_ng_extract(stream, 2, 3, hashes) != 0 { return 15 }
59 if nx_ng_extract(stream, 5, 0, hashes) != 0 { return 16 }
60
61 // === Test 3: frequency counting =================================
62 // Stream with repeats: [1, 2, 1, 2, 1] -> bigrams [(1,2),(2,1),(1,2),(2,1)]
63 // Distinct: 2. Counts: (1,2)=2, (2,1)=2.
64 let rep_stream: *i64 = (sys_mmap(5 * NX_SIZEOF_NX_INT)) as *i64
65 rep_stream[0] = 1
66 rep_stream[1] = 2
67 rep_stream[2] = 1
68 rep_stream[3] = 2
69 rep_stream[4] = 1
70 let rep_hashes: *i64 = (sys_mmap(8 * NX_SIZEOF_NX_INT)) as *i64
71 let n_rep: nx_int = nx_ng_extract(rep_stream, 5, 2, rep_hashes)
72 if n_rep != 4 { return 20 }
73
74 let table_cap: nx_int = 16 // power of 2
75 let table_keys: *i64 = (sys_mmap(16 * NX_SIZEOF_NX_INT)) as *i64
76 let table_counts: *i64 = (sys_mmap(16 * NX_SIZEOF_NX_INT)) as *i64
77 let n_distinct: nx_int = nx_ng_count(rep_hashes, n_rep,
78 table_keys, table_counts, table_cap)
79 if n_distinct != 2 { return 21 }
80
81 // Look up the two bigrams
82 let h_12: nx_int = rep_hashes[0] // (1,2)
83 let h_21: nx_int = rep_hashes[1] // (2,1)
84 if nx_ng_lookup(h_12, table_keys, table_counts, table_cap) != 2 { return 22 }
85 if nx_ng_lookup(h_21, table_keys, table_counts, table_cap) != 2 { return 23 }
86 if nx_ng_lookup(999999, table_keys, table_counts, table_cap) != 0 { return 24 }
87
88 // === Test 4: frequency band classifier ==========================
89 if nx_ng_classify_freq(1) != NX_NG_FREQ_SINGLETON_RARE { return 30 }
90 if nx_ng_classify_freq(7) != NX_NG_FREQ_SINGLETON_RARE { return 31 }
91 if nx_ng_classify_freq(8) != NX_NG_FREQ_RARE { return 32 }
92 if nx_ng_classify_freq(127) != NX_NG_FREQ_RARE { return 33 }
93 if nx_ng_classify_freq(128) != NX_NG_FREQ_COMMON { return 34 }
94 if nx_ng_classify_freq(4095) != NX_NG_FREQ_COMMON { return 35 }
95 if nx_ng_classify_freq(4096) != NX_NG_FREQ_VERY_COMMON { return 36 }
96 if nx_ng_classify_freq(1000000) != NX_NG_FREQ_VERY_COMMON { return 37 }
97
98 if nx_ng_freq_band_is_valid(NX_NG_FREQ_VERY_COMMON) != 1 { return 38 }
99 if nx_ng_freq_band_is_valid(NX_NG_FREQ_N_BANDS) != 0 { return 39 }
100
101 // === Test 5: dispersion (Juilland D) ============================
102 // Uniform distribution: parts = [100, 100, 100, 100, 100]
103 // mean=100, sd=0, cv=0, D=1 -> Q10 = 1024
104 let parts_unif: *i64 = (sys_mmap(5 * NX_SIZEOF_NX_INT)) as *i64
105 parts_unif[0] = 100
106 parts_unif[1] = 100
107 parts_unif[2] = 100
108 parts_unif[3] = 100
109 parts_unif[4] = 100
110 let d_unif: nx_int = nx_ng_dispersion_q10(parts_unif, 5)
111 if d_unif != NX_NG_Q { return 40 }
112 if nx_ng_classify_disp(d_unif) != NX_NG_DISP_EVEN { return 41 }
113
114 // Highly clumped: parts = [500, 0, 0, 0, 0]
115 // All in part 0; D should be low.
116 let parts_clump: *i64 = (sys_mmap(5 * NX_SIZEOF_NX_INT)) as *i64
117 parts_clump[0] = 500
118 parts_clump[1] = 0
119 parts_clump[2] = 0
120 parts_clump[3] = 0
121 parts_clump[4] = 0
122 let d_clump: nx_int = nx_ng_dispersion_q10(parts_clump, 5)
123 // D should be close to 0; classify CLUMPED
124 if nx_ng_classify_disp(d_clump) != NX_NG_DISP_CLUMPED { return 42 }
125
126 // Single-part corpus -> trivially uniform (Q)
127 let single: *i64 = (sys_mmap(8)) as *i64
128 single[0] = 100
129 if nx_ng_dispersion_q10(single, 1) != NX_NG_Q { return 43 }
130
131 // Empty parts -> 0
132 if nx_ng_dispersion_q10(parts_unif, 0) != NX_NG_Q { return 44 }
133
134 if nx_ng_disp_band_is_valid(NX_NG_DISP_EVEN) != 1 { return 45 }
135 if nx_ng_disp_band_is_valid(NX_NG_DISP_N_BANDS) != 0 { return 46 }
136
137 // === Test 6: top-K extraction ===================================
138 // Stream: [1,2,1,2,1,3,3,4,4,4]
139 // Bigrams: (1,2),(2,1),(1,2),(2,1),(1,3),(3,3),(3,4),(4,4),(4,4)
140 // Frequencies: (1,2)=2, (2,1)=2, (1,3)=1, (3,3)=1, (3,4)=1, (4,4)=2
141 // Top-3: (1,2)=2, (2,1)=2, (4,4)=2 (ties; any 3 of the 2-counts)
142 let s2: *i64 = (sys_mmap(10 * NX_SIZEOF_NX_INT)) as *i64
143 s2[0] = 1
144 s2[1] = 2
145 s2[2] = 1
146 s2[3] = 2
147 s2[4] = 1
148 s2[5] = 3
149 s2[6] = 3
150 s2[7] = 4
151 s2[8] = 4
152 s2[9] = 4
153 let h2: *i64 = (sys_mmap(16 * NX_SIZEOF_NX_INT)) as *i64
154 let nh2: nx_int = nx_ng_extract(s2, 10, 2, h2)
155 if nh2 != 9 { return 50 }
156 let tk: *i64 = (sys_mmap(32 * NX_SIZEOF_NX_INT)) as *i64
157 let tc: *i64 = (sys_mmap(32 * NX_SIZEOF_NX_INT)) as *i64
158 let nd: nx_int = nx_ng_count(h2, nh2, tk, tc, 32)
159 if nd != 6 { return 51 }
160
161 let top_h: *i64 = (sys_mmap(3 * NX_SIZEOF_NX_INT)) as *i64
162 let top_c: *i64 = (sys_mmap(3 * NX_SIZEOF_NX_INT)) as *i64
163 let n_top: nx_int = nx_ng_top_k(tk, tc, 32, 3, top_h, top_c)
164 if n_top != 3 { return 52 }
165 // All three should have count 2 (the three tied entries)
166 if top_c[0] != 2 { return 53 }
167 if top_c[1] != 2 { return 54 }
168 if top_c[2] != 2 { return 55 }
169
170 return 0
171}