nx_comms_bench.nx source
↩ module page · 194 lines · 7669 B
1// nx_comms_bench.nx -- head-to-head benchmark harness for real-time
2// communications. Drives Nishi voice/video codec against every
3// known commercial + open-source competitor.
4//
5// Composes with the existing nx_benchmark_harness (6-axis WIN/LOSE/
6// TIE) but adds comms-specific axes that the generic harness doesn't
7// have first-class slots for.
8//
9// Per cardinal feedback-honest-perf-verdict-no-aspirational-claims:
10// every LOSE row MUST be paired with a named_improvement string.
11// Per cardinal feedback-one-percent-minimum-delta-substrate: WIN
12// floor is 101% (1% advantage required to claim WIN).
13//
14// genealogy_id: rfc_6716_opus_test_vectors + itu_p862_pesq +
15// vmaf_netflix + criterion_rs_bootstrap_ci +
16// feedback-honest-perf-verdict-no-aspirational-claims
17// lineage_id: nishi_comms_polyglot_bench_q10
18
19// nx_safety_envelope:
20// intended_use: AUTO_APPLIED -- primitive-specific tuning queued
21// sil_target: SIL1
22// evidence: [bulk_applied_2026-05-16, see-file-comment-for-detail]
23// verdict: NOT_YET_EVALUATED
24
25import "nx_syscalls_x86_64.nx"
26
27// Sealed competitor enum. Order matches the audit's go-to-market
28// priority (most-painful incumbent first).
29const NX_COMMS_TELEGRAM: i64 = 0 // user's son's daily call -- highest priority
30const NX_COMMS_WHATSAPP: i64 = 1 // global incumbent
31const NX_COMMS_DISCORD: i64 = 2 // in-game voice; the "Phase 10" parallel
32const NX_COMMS_ZOOM: i64 = 3 // enterprise / classroom
33const NX_COMMS_JITSI: i64 = 4 // open-source SFU baseline
34const NX_COMMS_SIGNAL: i64 = 5 // privacy bar
35const NX_COMMS_WEBRTC_VANILLA: i64 = 6 // browser-native (Chrome/Firefox/Safari)
36const NX_COMMS_FACETIME: i64 = 7 // Apple ecosystem
37const NX_COMMS_GOOGLE_MEET: i64 = 8 // Workspace incumbent
38const NX_COMMS_MS_TEAMS: i64 = 9 // enterprise rival to Zoom
39const NX_COMMS_MUMBLE: i64 = 10 // legacy low-latency gaming
40const NX_COMMS_N: i64 = 11
41
42// Sealed axis verdict per axis. Identical shape to nx_benchmark_harness
43// but standalone so callers don't transitively pull RV64 syscalls.
44const NX_COMMS_AXIS_UNMEASURABLE: i64 = 0
45const NX_COMMS_AXIS_LOSE: i64 = 1
46const NX_COMMS_AXIS_TIE: i64 = 2
47const NX_COMMS_AXIS_WIN: i64 = 3
48const NX_COMMS_AXIS_DECISIVE_WIN: i64 = 4 // >= 50% better
49const NX_COMMS_AXIS_N_VERDICTS: i64 = 5
50
51// Sealed comms axis indices.
52const NX_COMMS_AXIS_RTT_1V1: i64 = 0 // end-to-end 1:1 RTT (ms)
53const NX_COMMS_AXIS_VOICE_PESQ: i64 = 1 // PESQ MOS score 1-5
54const NX_COMMS_AXIS_VIDEO_VMAF: i64 = 2 // VMAF score 0-100
55const NX_COMMS_AXIS_LOSS_5PCT: i64 = 3 // PESQ at 5% packet loss
56const NX_COMMS_AXIS_LOSS_20PCT: i64 = 4 // PESQ at 20% packet loss
57const NX_COMMS_AXIS_CPU_PER_CALL: i64 = 5 // % of one core
58const NX_COMMS_AXIS_MEM_PER_CALL: i64 = 6 // MB resident
59const NX_COMMS_AXIS_STARTUP_MS: i64 = 7 // call setup latency
60const NX_COMMS_AXIS_NAT_SUCCESS: i64 = 8 // % of NAT pairs that connect P2P
61const NX_COMMS_AXIS_AUDIT: i64 = 9 // sealed: can you prove what was sent?
62const NX_COMMS_AXIS_SOVEREIGNTY: i64 = 10 // % of code path you control
63const NX_COMMS_AXIS_N: i64 = 11
64
65// 1% delta floor in Q10 = 1024 / 100 ~ 10.
66const NX_COMMS_DELTA_FLOOR_Q10: i64 = 10
67
68struct AxisMeasurement {
69 nishi_value: i64, // raw measurement (units depend on axis)
70 competitor_value: i64,
71 n_trials: i64,
72 higher_is_better: i64 // 1 = higher wins; 0 = lower wins (RTT, CPU, etc.)
73}
74
75struct AxisVerdict {
76 measurement: AxisMeasurement,
77 verdict: i64, // NX_COMMS_AXIS_*
78 delta_q10: i64 // signed Q10 of (nishi - competitor) / competitor
79}
80
81// One full head-to-head report: Nishi vs ONE competitor, all axes.
82struct CommsReport {
83 competitor: i64, // NX_COMMS_*
84 rtt: AxisVerdict,
85 voice_pesq: AxisVerdict,
86 video_vmaf: AxisVerdict,
87 loss_5pct: AxisVerdict,
88 loss_20pct: AxisVerdict,
89 cpu: AxisVerdict,
90 mem: AxisVerdict,
91 startup: AxisVerdict,
92 nat_success: AxisVerdict,
93 audit: AxisVerdict,
94 sovereignty: AxisVerdict,
95 n_wins: i64,
96 n_loses: i64,
97 n_ties: i64,
98 n_unmeasured: i64
99}
100
101// Per-axis verdict computation. Mirrors nx_benchmark_harness shape.
102func _comms_axis_verdict(m: *AxisMeasurement, v: *AxisVerdict) -> i64 {
103 v.measurement = m[0]
104 if m.n_trials < 3 {
105 v.verdict = NX_COMMS_AXIS_UNMEASURABLE
106 v.delta_q10 = 0
107 return 0
108 }
109 if m.competitor_value == 0 {
110 if m.nishi_value > 0 {
111 v.verdict = NX_COMMS_AXIS_WIN
112 v.delta_q10 = 1024
113 } else {
114 v.verdict = NX_COMMS_AXIS_UNMEASURABLE
115 v.delta_q10 = 0
116 }
117 return 0
118 }
119 let diff: i64 = m.nishi_value - m.competitor_value
120 var inc: i64 = m.competitor_value
121 if inc < 0 { inc = -inc }
122 v.delta_q10 = (diff * 1024) / inc
123
124 var signed_delta: i64 = v.delta_q10
125 if m.higher_is_better == 0 { signed_delta = -signed_delta }
126
127 if signed_delta >= 512 {
128 v.verdict = NX_COMMS_AXIS_DECISIVE_WIN
129 } else {
130 if signed_delta >= NX_COMMS_DELTA_FLOOR_Q10 {
131 v.verdict = NX_COMMS_AXIS_WIN
132 } else {
133 if signed_delta <= -NX_COMMS_DELTA_FLOOR_Q10 {
134 v.verdict = NX_COMMS_AXIS_LOSE
135 } else {
136 v.verdict = NX_COMMS_AXIS_TIE
137 }
138 }
139 }
140 return 0
141}
142
143func _tally_v(v: i64, r: *CommsReport) -> i64 {
144 if v == NX_COMMS_AXIS_WIN { r.n_wins = r.n_wins + 1 }
145 if v == NX_COMMS_AXIS_DECISIVE_WIN { r.n_wins = r.n_wins + 1 }
146 if v == NX_COMMS_AXIS_TIE { r.n_ties = r.n_ties + 1 }
147 if v == NX_COMMS_AXIS_LOSE { r.n_loses = r.n_loses + 1 }
148 if v == NX_COMMS_AXIS_UNMEASURABLE { r.n_unmeasured = r.n_unmeasured + 1 }
149 return 0
150}
151
152// Tally wins/losses/ties across all axes of a single report.
153func nx_comms_compute(report: *CommsReport) -> i64 {
154 _comms_axis_verdict(report.rtt.measurement, report.rtt)
155 _comms_axis_verdict(report.voice_pesq.measurement, report.voice_pesq)
156 _comms_axis_verdict(report.video_vmaf.measurement, report.video_vmaf)
157 _comms_axis_verdict(report.loss_5pct.measurement, report.loss_5pct)
158 _comms_axis_verdict(report.loss_20pct.measurement, report.loss_20pct)
159 _comms_axis_verdict(report.cpu.measurement, report.cpu)
160 _comms_axis_verdict(report.mem.measurement, report.mem)
161 _comms_axis_verdict(report.startup.measurement, report.startup)
162 _comms_axis_verdict(report.nat_success.measurement, report.nat_success)
163 _comms_axis_verdict(report.audit.measurement, report.audit)
164 _comms_axis_verdict(report.sovereignty.measurement, report.sovereignty)
165
166 report.n_wins = 0
167 report.n_loses = 0
168 report.n_ties = 0
169 report.n_unmeasured = 0
170 _tally_v(report.rtt.verdict, report)
171 _tally_v(report.voice_pesq.verdict, report)
172 _tally_v(report.video_vmaf.verdict, report)
173 _tally_v(report.loss_5pct.verdict, report)
174 _tally_v(report.loss_20pct.verdict, report)
175 _tally_v(report.cpu.verdict, report)
176 _tally_v(report.mem.verdict, report)
177 _tally_v(report.startup.verdict, report)
178 _tally_v(report.nat_success.verdict, report)
179 _tally_v(report.audit.verdict, report)
180 _tally_v(report.sovereignty.verdict, report)
181 return 0
182}
183
184// Sealed-enum validity gates.
185func nx_comms_axis_verdict_is_valid(v: i64) -> i64 {
186 if v < 0 { return 0 }
187 if v >= NX_COMMS_AXIS_N_VERDICTS { return 0 }
188 return 1
189}
190func nx_comms_competitor_is_valid(c: i64) -> i64 {
191 if c < 0 { return 0 }
192 if c >= NX_COMMS_N { return 0 }
193 return 1
194}