nx_word_sketch.nx source
↩ module page · 201 lines · 8139 B
1// nx_word_sketch.nx -- substrate-native word sketch (Sketch Engine
2// displacement, see feedback-corpus-linguistics-s-class-substrate-...).
3//
4// A "word sketch" is the canonical Sketch Engine feature: given a
5// target word, list its statistically most significant collocates.
6// Sketch Engine groups by grammatical relation (subj_of/obj_of/...),
7// which requires a parser; this v1 primitive groups by POSITIONAL
8// WINDOW instead (collocates within +/- W of the target), which is
9// what corpus linguists call a "raw" word sketch.
10//
11// Composes:
12// nx_ngram.nx -- token-stream + hash-table count primitives
13// nx_collocation.nx -- PMI / log-likelihood / T-score / log-Dice
14//
15// Caller flow:
16// 1. tokenise corpus into i64 token stream (one ID per word)
17// 2. compute global per-token totals via nx_ng_count (1-grams)
18// 3. call nx_word_sketch_count to fill a collocate table
19// 4. call nx_word_sketch_score_one for each collocate to get log_dice_q10
20// 5. (optional) sort by log_dice and pick top-K
21//
22// No memory allocation inside the primitive — caller supplies all
23// buffers, keeping the substrate scale-agnostic from MCU to HPC.
24//
25// genealogy_id: rychly_2008_logDice + church_hanks_1990_pmi +
26// kilgarriff_2014_sketch_engine
27// lineage_id: positional_word_sketch_v1
28
29// nx_safety_envelope:
30// intended_use: AUTO_APPLIED -- primitive-specific tuning queued
31// sil_target: SIL1
32// evidence: [bulk_applied_2026-05-16, see-file-comment-for-detail]
33// verdict: NOT_YET_EVALUATED
34
35import "nx_syscalls.nx"
36import "nx_tier.nx"
37import "nx_ngram.nx"
38import "nx_collocation.nx"
39
40// ===== Sealed-enum: RankMetric =====================================
41//
42// Which metric to score collocates by. Same as nx_collocation's
43// NX_COL_METRIC_* but re-exported here under a sketch-specific name
44// so caller code reads naturally.
45
46const NX_WS_METRIC_PMI: nx_int = 0
47const NX_WS_METRIC_LOG_LIKELIHOOD: nx_int = 1
48const NX_WS_METRIC_T_SCORE: nx_int = 2
49const NX_WS_METRIC_LOG_DICE: nx_int = 3
50const NX_WS_N_METRICS: nx_int = 4
51
52func nx_ws_metric_is_valid(m: nx_int) -> nx_int {
53 if m < 0 { return 0 }
54 if m >= NX_WS_N_METRICS { return 0 }
55 return 1
56}
57
58// ===== Sealed-enum: SketchKind =====================================
59//
60// Diagnostic verdict for a finished sketch. Helps the caller route
61// the output: a HEALTHY sketch is publishable; a SPARSE sketch needs
62// more corpus before claims are statistically valid; an EMPTY sketch
63// means the target never appeared at all.
64
65const NX_WS_KIND_EMPTY: nx_int = 0 // target absent from stream
66const NX_WS_KIND_SPARSE: nx_int = 1 // <5 occurrences -- low confidence
67const NX_WS_KIND_HEALTHY: nx_int = 2 // >=5 occurrences -- publishable
68const NX_WS_KIND_DENSE: nx_int = 3 // >=100 occurrences -- high confidence
69const NX_WS_N_KINDS: nx_int = 4
70
71func nx_ws_kind_is_valid(k: nx_int) -> nx_int {
72 if k < 0 { return 0 }
73 if k >= NX_WS_N_KINDS { return 0 }
74 return 1
75}
76
77func nx_ws_classify(n_target_occurrences: nx_int) -> nx_int {
78 if n_target_occurrences <= 0 { return NX_WS_KIND_EMPTY }
79 if n_target_occurrences < 5 { return NX_WS_KIND_SPARSE }
80 if n_target_occurrences < 100 { return NX_WS_KIND_HEALTHY }
81 return NX_WS_KIND_DENSE
82}
83
84// ===== Sentinel: empty bucket ======================================
85//
86// We use 0 as the empty-slot key in the collocate table, matching
87// nx_ngram's convention. Caller-supplied target tokens that hash to
88// 0 will collide with empty-slot detection; not a concern for ID-based
89// streams where IDs start at 1.
90
91const NX_WS_EMPTY_KEY: nx_int = 0
92
93// ===== Window collocate counter ====================================
94//
95// Walk the token stream; for every position i where stream[i] == target,
96// bump the count of stream[i+offset] for offset in [-window, +window]
97// (skipping 0 = the target itself).
98//
99// Skip positions where i+offset is out of bounds. Caller supplies the
100// collocate hash table (open-addressed, power-of-2 capacity) -- same
101// shape as nx_ngram's count table.
102//
103// Returns the number of TARGET occurrences found (= the "n_target"
104// total needed for log-Dice scoring).
105
106func nx_word_sketch_count(stream: *i64, stream_len: nx_int,
107 target: nx_int, window: nx_int,
108 table_keys: *i64, table_counts: *i64,
109 cap: nx_int) -> nx_int {
110 // Zero table
111 var z: nx_int = 0
112 while z < cap {
113 table_keys[z] = NX_WS_EMPTY_KEY
114 table_counts[z] = 0
115 z = z + 1
116 }
117 let mask: nx_int = cap - 1 // caller guarantees cap is power-of-2
118
119 var n_target: nx_int = 0
120 var i: nx_int = 0
121 while i < stream_len {
122 if stream[i] == target {
123 n_target = n_target + 1
124 var offset: nx_int = 0 - window
125 while offset <= window {
126 if offset != 0 {
127 let j: nx_int = i + offset
128 if j >= 0 {
129 if j < stream_len {
130 let neighbor: nx_int = stream[j]
131 // Open-addressed insert/bump
132 var slot: nx_int = neighbor & mask
133 if slot < 0 { slot = slot + cap }
134 var placed: nx_int = 0
135 while placed == 0 {
136 if table_keys[slot] == NX_WS_EMPTY_KEY {
137 table_keys[slot] = neighbor
138 table_counts[slot] = 1
139 placed = 1
140 } else {
141 if table_keys[slot] == neighbor {
142 table_counts[slot] = table_counts[slot] + 1
143 placed = 1
144 } else {
145 slot = (slot + 1) & mask
146 if slot < 0 { slot = slot + cap }
147 }
148 }
149 }
150 }
151 }
152 }
153 offset = offset + 1
154 }
155 }
156 i = i + 1
157 }
158 return n_target
159}
160
161// ===== Per-collocate scorer ========================================
162//
163// Given one collocate's counts, emit the 4 metrics + bands via the
164// existing nx_col_emit_all primitive. Output layout matches
165// nx_collocation: NX_COL_OUT_FIELDS i64 slots (PMI / LL / T / DICE +
166// 4 band classifiers).
167//
168// n_target -- total occurrences of the target word
169// n_collocate -- total occurrences of the collocate in the corpus
170// n_cooccur -- count of (target, collocate) co-occurrences in window
171// n_total -- total tokens in the corpus
172
173func nx_word_sketch_score_one(n_target: nx_int, n_collocate: nx_int,
174 n_cooccur: nx_int, n_total: nx_int,
175 out: *i64) -> nx_int {
176 return nx_col_emit_all(n_target, n_collocate, n_cooccur, n_total, out)
177}
178
179// ===== Convenience scorer: just the chosen metric, no bands ========
180//
181// Returns the Q10 score for one metric. Useful for ranking when the
182// caller already trusts a single metric (e.g. log-Dice for sketch
183// engine parity).
184
185func nx_word_sketch_metric_q10(n_target: nx_int, n_collocate: nx_int,
186 n_cooccur: nx_int, n_total: nx_int,
187 metric: nx_int) -> nx_int {
188 if metric == NX_WS_METRIC_PMI {
189 return nx_col_pmi_q10(n_target, n_collocate, n_cooccur, n_total)
190 }
191 if metric == NX_WS_METRIC_LOG_LIKELIHOOD {
192 return nx_col_log_likelihood_q10(n_target, n_collocate, n_cooccur, n_total)
193 }
194 if metric == NX_WS_METRIC_T_SCORE {
195 return nx_col_t_score_q10(n_target, n_collocate, n_cooccur, n_total)
196 }
197 if metric == NX_WS_METRIC_LOG_DICE {
198 return nx_col_log_dice_q10(n_target, n_collocate, n_cooccur)
199 }
200 return 0
201}