nx_calibration.nx source
↩ module page · 335 lines · 12063 B
1// nx_calibration.nx -- statistical calibration of predicted confidences.
2//
3// When the substrate emits "predicted age 22 with 90% confidence"
4// (or any probability claim), DOES 90% confidence empirically
5// produce a hit 90% of the time? Without calibration, model
6// confidences are opaque numbers users learn to ignore. With
7// calibration, they become RELIABILITY-GUARANTEED measurements.
8//
9// References:
10// Brier 1950 "Verification of Forecasts Expressed in Terms of
11// Probability" -- the Brier score
12// BS = (1/N) * sum_i (p_i - o_i)^2
13// where p_i in [0,1] is forecast probability and o_i in {0,1}.
14// Naeini/Cooper/Hauskrecht 2015 "Obtaining Well Calibrated
15// Probabilities Using Bayesian Binning" -- ECE / MCE.
16// DeGroot/Fienberg 1983 -- "well-calibrated" definition.
17//
18// Closes calibration field in
19// nxc2/docs/TOPDOWN_BOTTOMUP_GAP_ANALYSIS.md. No existing image-
20// gen / LLM stack measures calibration of its OWN confidences in
21// a bits-up form; "confidence" floats around with no empirical
22// anchor.
23//
24// genealogy_id: brier_1950_naeini_2015
25// lineage_id: substrate_calibration_v1
26//
27// 5W+H+GLP linkage: emits VERDICT records with what=VERDICT,
28// how=brier|ece|mce, performance=score_q10.
29
30// nx_safety_envelope:
31// intended_use: AUTO_APPLIED -- primitive-specific tuning queued
32// sil_target: SIL1
33// evidence: [bulk_applied_2026-05-16, see-file-comment-for-detail]
34// verdict: NOT_YET_EVALUATED
35
36import "nx_syscalls.nx"
37import "nx_runtime.nx"
38import "nx_tier.nx"
39const NX_MAGIC_1024: i64 = 1024
40
41// ===== Q10 fixed-point base ========================================
42
43const NX_CAL_Q10: nx_int = 1024
44
45// ===== bounds ======================================================
46
47const NX_CAL_MAX_SAMPLES: nx_int = 1048576
48const NX_CAL_MAX_BINS: nx_int = 100
49const NX_CAL_DEFAULT_BINS: nx_int = 10
50
51// ===== sealed enum: calibration verdict ===========================
52//
53// Empirical thresholds. ECE_q10 reported per Naeini 2015 convention.
54
55const NX_CAL_VERDICT_EXCELLENT: nx_int = 0 // ECE < 0.02
56const NX_CAL_VERDICT_GOOD: nx_int = 1 // ECE < 0.05
57const NX_CAL_VERDICT_FAIR: nx_int = 2 // ECE < 0.10
58const NX_CAL_VERDICT_POOR: nx_int = 3 // ECE < 0.20
59const NX_CAL_VERDICT_BROKEN: nx_int = 4 // ECE >= 0.20
60
61const NX_CAL_THR_EXCELLENT_Q10: nx_int = 20 // 0.02 * 1024
62const NX_CAL_THR_GOOD_Q10: nx_int = 51 // 0.05 * 1024
63const NX_CAL_THR_FAIR_Q10: nx_int = 102 // 0.10 * 1024
64const NX_CAL_THR_POOR_Q10: nx_int = 204 // 0.20 * 1024
65
66func nx_cal_verdict_from_ece(ece_q10: nx_int) -> nx_int {
67 if ece_q10 < NX_CAL_THR_EXCELLENT_Q10 { return NX_CAL_VERDICT_EXCELLENT }
68 if ece_q10 < NX_CAL_THR_GOOD_Q10 { return NX_CAL_VERDICT_GOOD }
69 if ece_q10 < NX_CAL_THR_FAIR_Q10 { return NX_CAL_VERDICT_FAIR }
70 if ece_q10 < NX_CAL_THR_POOR_Q10 { return NX_CAL_VERDICT_POOR }
71 return NX_CAL_VERDICT_BROKEN
72}
73
74// ===== result struct ==============================================
75
76struct NxCalibrationResult {
77 n_samples: nx_int,
78 n_bins: nx_int,
79 brier_q10: nx_int, // mean squared error
80 ece_q10: nx_int, // expected calibration error
81 mce_q10: nx_int, // maximum calibration error
82 verdict: nx_int, // sealed enum above
83 bin_counts: *nx_int,
84 bin_pred_q10: *nx_int, // average predicted prob per bin (Q10)
85 bin_obs_q10: *nx_int, // empirical hit rate per bin (Q10)
86}
87
88const NX_CAL_RESULT_BYTES: nx_size = 72
89
90// ===== Brier score =================================================
91//
92// p_q10[] predicted probabilities in Q10 ([0, 1024])
93// o[] binary outcomes {0, 1}
94// n_samples >= 1
95// Returns Brier score in Q20 (scaled by Q10^2) -- caller divides.
96// Actually return Q10 by computing in Q10 throughout:
97// BS_q10 = mean( ((p_q10 - o*Q10)/Q10)^2 ) * Q10
98// = mean( (p_q10 - o*Q10)^2 ) / Q10
99
100func nx_cal_brier_q10(p_q10: *nx_int, o: *nx_int, n_samples: nx_int) -> nx_int {
101 if n_samples <= 0 { return 0 - 1 }
102 if n_samples > NX_CAL_MAX_SAMPLES { return 0 - 1 }
103 var acc: nx_int = 0
104 var i: nx_int = 0
105 while i < n_samples {
106 let p: nx_int = p_q10[i]
107 let outcome_q10: nx_int = o[i] * NX_CAL_Q10
108 let diff: nx_int = p - outcome_q10
109 let sq: nx_int = diff * diff
110 acc = acc + (sq / NX_CAL_Q10)
111 i = i + 1
112 }
113 return acc / n_samples
114}
115
116// ===== bin assignment ==============================================
117//
118// Map p_q10 in [0, Q10] to bin index in [0, n_bins). Standard
119// equal-width binning per Naeini 2015.
120
121func _cal_bin_idx(p_q10: nx_int, n_bins: nx_int) -> nx_int {
122 if p_q10 < 0 { return 0 }
123 if p_q10 >= NX_CAL_Q10 { return n_bins - 1 }
124 let bin_width_q10: nx_int = NX_CAL_Q10 / n_bins
125 if bin_width_q10 <= 0 { return 0 }
126 var idx: nx_int = p_q10 / bin_width_q10
127 if idx >= n_bins { idx = n_bins - 1 }
128 return idx
129}
130
131// ===== reliability-diagram binning =================================
132//
133// Allocates and fills bin_counts[], bin_pred_q10[], bin_obs_q10[].
134// Caller frees via discard.
135
136func nx_cal_bin(
137 p_q10: *nx_int, o: *nx_int, n_samples: nx_int, n_bins: nx_int,
138 bin_counts: *nx_int, bin_pred_q10: *nx_int, bin_obs_q10: *nx_int) -> nx_int {
139
140 if n_samples <= 0 { return 0 - 1 }
141 if n_bins <= 0 { return 0 - 1 }
142 if n_bins > NX_CAL_MAX_BINS { return 0 - 1 }
143
144 // Zero accumulators.
145 var b: nx_int = 0
146 while b < n_bins {
147 bin_counts[b] = 0
148 bin_pred_q10[b] = 0
149 bin_obs_q10[b] = 0
150 b = b + 1
151 }
152
153 // Accumulate per bin.
154 var i: nx_int = 0
155 while i < n_samples {
156 let bin: nx_int = _cal_bin_idx(p_q10[i], n_bins)
157 bin_counts[bin] = bin_counts[bin] + 1
158 bin_pred_q10[bin] = bin_pred_q10[bin] + p_q10[i]
159 bin_obs_q10[bin] = bin_obs_q10[bin] + o[i] * NX_CAL_Q10
160 i = i + 1
161 }
162
163 // Divide to get per-bin means.
164 var bn: nx_int = 0
165 while bn < n_bins {
166 let cnt: nx_int = bin_counts[bn]
167 if cnt > 0 {
168 bin_pred_q10[bn] = bin_pred_q10[bn] / cnt
169 bin_obs_q10[bn] = bin_obs_q10[bn] / cnt
170 }
171 bn = bn + 1
172 }
173 return 0
174}
175
176// ===== ECE: expected calibration error =============================
177//
178// ECE = sum_b (|B_b| / N) * |acc_b - conf_b|
179// where acc_b = empirical hit rate in bin b
180// conf_b = average predicted prob in bin b
181//
182// Returns Q10.
183
184func nx_cal_ece_q10(
185 bin_counts: *nx_int, bin_pred_q10: *nx_int, bin_obs_q10: *nx_int,
186 n_bins: nx_int, n_samples: nx_int) -> nx_int {
187
188 if n_samples <= 0 { return 0 - 1 }
189 var acc: nx_int = 0
190 var b: nx_int = 0
191 while b < n_bins {
192 let cnt: nx_int = bin_counts[b]
193 if cnt > 0 {
194 var gap: nx_int = bin_obs_q10[b] - bin_pred_q10[b]
195 if gap < 0 { gap = 0 - gap }
196 let weighted: nx_int = gap * cnt
197 acc = acc + (weighted / n_samples)
198 }
199 b = b + 1
200 }
201 return acc
202}
203
204// ===== MCE: maximum calibration error =============================
205//
206// MCE = max_b |acc_b - conf_b|. Returns Q10.
207
208func nx_cal_mce_q10(
209 bin_counts: *nx_int, bin_pred_q10: *nx_int, bin_obs_q10: *nx_int,
210 n_bins: nx_int) -> nx_int {
211
212 var worst: nx_int = 0
213 var b: nx_int = 0
214 while b < n_bins {
215 if bin_counts[b] > 0 {
216 var gap: nx_int = bin_obs_q10[b] - bin_pred_q10[b]
217 if gap < 0 { gap = 0 - gap }
218 if gap > worst { worst = gap }
219 }
220 b = b + 1
221 }
222 return worst
223}
224
225// ===== full report ================================================
226
227func nx_cal_report(
228 p_q10: *nx_int, o: *nx_int, n_samples: nx_int, n_bins: nx_int) -> *NxCalibrationResult {
229
230 if n_samples <= 0 { return 0 as *NxCalibrationResult }
231 if n_samples > NX_CAL_MAX_SAMPLES { return 0 as *NxCalibrationResult }
232 var bins: nx_int = n_bins
233 if bins <= 0 { bins = NX_CAL_DEFAULT_BINS }
234 if bins > NX_CAL_MAX_BINS { bins = NX_CAL_MAX_BINS }
235
236 let result_ptr: *u8 = sys_mmap(NX_CAL_RESULT_BYTES)
237 let r: *NxCalibrationResult = result_ptr as *NxCalibrationResult
238
239 let bin_bytes: nx_size = (bins as nx_size) * 8
240 let bc: *nx_int = (sys_mmap(bin_bytes)) as *nx_int
241 let bp: *nx_int = (sys_mmap(bin_bytes)) as *nx_int
242 let bo: *nx_int = (sys_mmap(bin_bytes)) as *nx_int
243
244 let rc_bin: nx_int = nx_cal_bin(p_q10, o, n_samples, bins, bc, bp, bo)
245 if rc_bin != 0 { return 0 as *NxCalibrationResult }
246
247 let brier: nx_int = nx_cal_brier_q10(p_q10, o, n_samples)
248 let ece: nx_int = nx_cal_ece_q10(bc, bp, bo, bins, n_samples)
249 let mce: nx_int = nx_cal_mce_q10(bc, bp, bo, bins)
250
251 r.n_samples = n_samples
252 r.n_bins = bins
253 r.brier_q10 = brier
254 r.ece_q10 = ece
255 r.mce_q10 = mce
256 r.verdict = nx_cal_verdict_from_ece(ece)
257 r.bin_counts = bc
258 r.bin_pred_q10 = bp
259 r.bin_obs_q10 = bo
260 return r
261}
262
263// ===== self-test ===================================================
264//
265// Two scenarios:
266// PERFECT -- p=0.5 (Q10=512), outcomes 50/50. ECE = 0.
267// POOR -- always says 1.0 (Q10=1024), only hits 50% of the time.
268// ECE = 0.5 (Q10=512).
269
270func main() -> nx_int {
271 // ---- PERFECT: 8 samples, p=512 all, outcomes [1,0,1,0,1,0,1,0] ----
272 let p_perf: *nx_int = (sys_mmap(64)) as *nx_int
273 let o_perf: *nx_int = (sys_mmap(64)) as *nx_int
274 var i: nx_int = 0
275 while i < 8 {
276 p_perf[i] = 512
277 if i % 2 == 0 { o_perf[i] = 1 } else { o_perf[i] = 0 }
278 i = i + 1
279 }
280
281 let r_perf: *NxCalibrationResult = nx_cal_report(p_perf, o_perf, 8, 10)
282 if r_perf == (0 as *NxCalibrationResult) { return 1 }
283 if r_perf.n_samples != 8 { return 2 }
284 if r_perf.n_bins != 10 { return 3 }
285 // p=0.5 always -> bin 5, all samples in that bin.
286 // bin_pred = 512, bin_obs = 4/8 * 1024 = 512. ECE = 0.
287 if r_perf.ece_q10 != 0 { return 4 }
288 if r_perf.verdict != NX_CAL_VERDICT_EXCELLENT { return 5 }
289 // Brier: (512-1024)^2/1024 = 262144/1024 = 256 for hit samples,
290 // (512-0)^2/1024 = 262144/1024 = 256 for miss samples.
291 // Mean = 256.
292 if r_perf.brier_q10 != 256 { return 6 }
293 // MCE per bin same as ECE for single-bin distribution.
294 if r_perf.mce_q10 != 0 { return 7 }
295
296 // ---- POOR: 8 samples, p=1024 always, half hit half miss ----
297 let p_poor: *nx_int = (sys_mmap(64)) as *nx_int
298 let o_poor: *nx_int = (sys_mmap(64)) as *nx_int
299 i = 0
300 while i < 8 {
301 p_poor[i] = NX_MAGIC_1024
302 if i < 4 { o_poor[i] = 1 } else { o_poor[i] = 0 }
303 i = i + 1
304 }
305 let r_poor: *NxCalibrationResult = nx_cal_report(p_poor, o_poor, 8, 10)
306 if r_poor == (0 as *NxCalibrationResult) { return 10 }
307 // All in bin 9. bin_pred = 1024, bin_obs = 512. gap = 512.
308 // ECE = 512 * 8 / 8 = 512.
309 if r_poor.ece_q10 != 512 { return 11 }
310 if r_poor.mce_q10 != 512 { return 12 }
311 if r_poor.verdict != NX_CAL_VERDICT_BROKEN { return 13 }
312
313 // ---- verdict thresholds ----
314 if nx_cal_verdict_from_ece(10) != NX_CAL_VERDICT_EXCELLENT { return 20 }
315 if nx_cal_verdict_from_ece(30) != NX_CAL_VERDICT_GOOD { return 21 }
316 if nx_cal_verdict_from_ece(75) != NX_CAL_VERDICT_FAIR { return 22 }
317 if nx_cal_verdict_from_ece(150) != NX_CAL_VERDICT_POOR { return 23 }
318 if nx_cal_verdict_from_ece(300) != NX_CAL_VERDICT_BROKEN { return 24 }
319
320 // ---- bin assignment edges ----
321 // bin_width = 1024/10 = 102 (truncated). Bin 0 covers [0..102),
322 // bin 1 [102..204), ..., bin 9 saturates at 1024.
323 if _cal_bin_idx(0, 10) != 0 { return 30 }
324 if _cal_bin_idx(101, 10) != 0 { return 31 }
325 if _cal_bin_idx(102, 10) != 1 { return 32 }
326 if _cal_bin_idx(NX_MAGIC_1024, 10) != 9 { return 33 }
327
328 // ---- input validation ----
329 let bad1: nx_int = nx_cal_brier_q10(p_perf, o_perf, 0)
330 if bad1 != 0 - 1 { return 40 }
331 let bad2: nx_int = nx_cal_brier_q10(p_perf, o_perf, NX_CAL_MAX_SAMPLES + 1)
332 if bad2 != 0 - 1 { return 41 }
333
334 return 0
335}