nx_smoke_harness.nx source
↩ module page · 256 lines · 9486 B
1// nx_smoke_harness.nx -- adapter smoke-test runner.
2//
3// module: nishi-core.ingest.smoke_harness
4// depends: nishi-core.ingest.synthetic_upstream,
5// nishi-core.ingest.source_adapter,
6// nishi-core.io.syscalls, nishi-core.io.iso8601
7// disk_kb: 5
8// capability: CORE_IO
9//
10// license_tier: PUBLIC_NISHI_SUBSTRATE
11// genealogy_id: golang_testing_pattern + xunit_pattern_meyer_2003 +
12// aaa_pattern_arrange_act_assert +
13// nishi_build_the_system_cardinal_2026
14//
15// Test-runner for NX-INGEST adapters. Per cardinal [[feedback-build-
16// the-system-not-manual-output]]: this is THE SYSTEM that proves
17// adapter correctness without manual catalog entry.
18//
19// Each smoke case:
20// 1. ARRANGE: load adapter + synthetic upstream + expected output
21// 2. ACT: invoke adapter.fetch_and_canonicalize() with fixture
22// 3. ASSERT: emitted JSONL matches expected canonical record
23//
24// Per Cardinal 14 graceful-degradation: failed assertions don't
25// crash the harness — they emit DIAGNOSTIC events + continue with
26// next case. Aggregate report at end.
27
28// nx_safety_envelope:
29// intended_use: AUTO_APPLIED -- primitive-specific tuning queued
30// sil_target: SIL1
31// evidence: [bulk_applied_2026-05-16, see-file-comment-for-detail]
32// verdict: NOT_YET_EVALUATED
33
34import "nx_syscalls.nx"
35import "nx_iso8601.nx"
36import "nx_synthetic_upstream.nx"
37import "nx_source_adapter.nx"
38
39// ===== Smoke case state ===========================================
40
41const NX_SMOKE_PENDING: i64 = 1
42const NX_SMOKE_RUNNING: i64 = 2
43const NX_SMOKE_PASSED: i64 = 3
44const NX_SMOKE_FAILED: i64 = 4
45const NX_SMOKE_FIXTURE_MISSING: i64 = 5
46const NX_SMOKE_ADAPTER_CRASHED: i64 = 6
47const NX_SMOKE_TIMEOUT: i64 = 7
48const NX_SMOKE_SKIPPED: i64 = 8
49
50func nx_smoke_state_name(s: i64) -> *u8 {
51 if s == NX_SMOKE_PENDING { return "PENDING" }
52 if s == NX_SMOKE_RUNNING { return "RUNNING" }
53 if s == NX_SMOKE_PASSED { return "PASSED" }
54 if s == NX_SMOKE_FAILED { return "FAILED" }
55 if s == NX_SMOKE_FIXTURE_MISSING { return "FIXTURE_MISSING" }
56 if s == NX_SMOKE_ADAPTER_CRASHED { return "ADAPTER_CRASHED" }
57 if s == NX_SMOKE_TIMEOUT { return "TIMEOUT" }
58 if s == NX_SMOKE_SKIPPED { return "SKIPPED" }
59 return "UNKNOWN"
60}
61
62// ===== SmokeCase ==================================================
63//
64// One test case per (adapter, fixture-input → expected-output) pair.
65
66struct SmokeCase {
67 case_hk: i64,
68 case_name_ptr: *u8, // "usda_hardiness_zone_98101_seattle"
69 adapter_source_id: i64, // NX_ORCH_SRC_* from orchestrator
70 fixture_input_path_ptr: *u8, // "fixtures/usda_hardiness/98101.fixture"
71 expected_output_path_ptr: *u8, // "fixtures/expected/usda_hardiness_98101.jsonl"
72 state: i64, // NX_SMOKE_*
73 started_at_unix: i64,
74 completed_at_unix: i64,
75 elapsed_ms: i64,
76 bytes_consumed: i64,
77 records_emitted: i64,
78 records_expected: i64,
79 failure_detail_ptr: *u8,
80 is_current: i64,
81}
82
83const NX_SMOKE_CASE_BYTES: i64 = 104 // 13 fields * 8 bytes
84
85// ===== SmokeRun (aggregate of N cases) ============================
86
87struct SmokeRun {
88 run_hk: i64,
89 run_started_unix: i64,
90 run_completed_unix: i64,
91 n_cases: i64,
92 n_passed: i64,
93 n_failed: i64,
94 n_skipped: i64,
95 n_fixture_missing: i64,
96 n_adapter_crashed: i64,
97 verdict: i64, // NX_SMOKE_RUN_*
98}
99
100const NX_SMOKE_RUN_BYTES: i64 = 80 // 10 fields * 8 bytes
101
102// ===== Run verdict ================================================
103
104const NX_SMOKE_RUN_ALL_PASS: i64 = 1
105const NX_SMOKE_RUN_PARTIAL: i64 = 2
106const NX_SMOKE_RUN_ALL_FAIL: i64 = 3
107const NX_SMOKE_RUN_INFRASTRUCTURE_FAIL: i64 = 4 // can't even load fixtures
108
109func nx_smoke_run_verdict_name(v: i64) -> *u8 {
110 if v == NX_SMOKE_RUN_ALL_PASS { return "ALL_PASS" }
111 if v == NX_SMOKE_RUN_PARTIAL { return "PARTIAL" }
112 if v == NX_SMOKE_RUN_ALL_FAIL { return "ALL_FAIL" }
113 if v == NX_SMOKE_RUN_INFRASTRUCTURE_FAIL { return "INFRASTRUCTURE_FAIL" }
114 return "UNKNOWN"
115}
116
117// ===== Run a single smoke case ====================================
118//
119// Composes:
120// 1. nx_synthetic_upstream_serve (returns FixtureResponse)
121// 2. adapter's canonicalize function (caller-supplied via case)
122// 3. compare emitted JSONL bytes against expected output file
123// 4. emit DIAGNOSTIC if mismatch
124
125func nx_smoke_run_case(
126 c: *SmokeCase,
127 upstream: *SyntheticUpstream,
128 now_unix: i64
129) -> i64 {
130 if c == 0 as *SmokeCase { return NX_SMOKE_FIXTURE_MISSING }
131 c.state = NX_SMOKE_RUNNING
132 c.started_at_unix = now_unix
133
134 // Run the adapter against the synthetic upstream.
135 // This requires per-adapter dispatch (nx_orch_dispatch_source style)
136 // — for v1, caller passes the canonicalize function pointer
137 // or invokes the adapter directly.
138 //
139 // v1 structure: substrate composes fixture-input → adapter →
140 // emitted JSONL → byte-compare against expected output.
141 //
142 // Real wire-through depends on:
143 // - nx_synthetic_upstream file-IO landing
144 // - per-adapter function-pointer table
145 //
146 // For first proof-of-life: hardcode dispatch for one adapter
147 // (usda_hardiness) and run the round trip.
148
149 c.completed_at_unix = now_unix
150 c.elapsed_ms = (c.completed_at_unix - c.started_at_unix) * 1000
151
152 // v1 substrate-stub: marks the case PENDING (no real execution yet)
153 c.state = NX_SMOKE_PENDING
154 return c.state
155}
156
157// ===== Run an array of cases ======================================
158
159const NX_SMOKE_MAX_CASES: i64 = 1024
160
161func nx_smoke_run_cases(
162 cases: **SmokeCase,
163 n_cases: i64,
164 upstream: *SyntheticUpstream,
165 now_unix: i64
166) -> *SmokeRun {
167 let raw: *u8 = sys_mmap(NX_SMOKE_RUN_BYTES)
168 let r: *SmokeRun = raw as *SmokeRun
169 r.run_hk = 0
170 r.run_started_unix = now_unix
171 r.run_completed_unix = 0
172 r.n_cases = n_cases
173 r.n_passed = 0
174 r.n_failed = 0
175 r.n_skipped = 0
176 r.n_fixture_missing = 0
177 r.n_adapter_crashed = 0
178 r.verdict = NX_SMOKE_RUN_PARTIAL
179
180 if n_cases <= 0 { return r }
181 if n_cases > NX_SMOKE_MAX_CASES { return r }
182
183 var i: i64 = 0
184 var iter: i64 = 0
185 var verdict: i64 = 0
186 while verdict == 0 && iter < NX_SMOKE_MAX_CASES {
187 if i >= n_cases { verdict = 1 }
188 if verdict == 0 {
189 let c: *SmokeCase = cases[i]
190 let case_verdict: i64 = nx_smoke_run_case(c, upstream, now_unix)
191 if case_verdict == NX_SMOKE_PASSED { r.n_passed = r.n_passed + 1 }
192 if case_verdict == NX_SMOKE_FAILED { r.n_failed = r.n_failed + 1 }
193 if case_verdict == NX_SMOKE_SKIPPED { r.n_skipped = r.n_skipped + 1 }
194 if case_verdict == NX_SMOKE_FIXTURE_MISSING { r.n_fixture_missing = r.n_fixture_missing + 1 }
195 if case_verdict == NX_SMOKE_ADAPTER_CRASHED { r.n_adapter_crashed = r.n_adapter_crashed + 1 }
196 i = i + 1
197 }
198 iter = iter + 1
199 }
200 r.run_completed_unix = now_unix
201
202 if r.n_passed == r.n_cases { r.verdict = NX_SMOKE_RUN_ALL_PASS }
203 if r.n_passed == 0 {
204 if r.n_cases > 0 { r.verdict = NX_SMOKE_RUN_ALL_FAIL }
205 }
206 if r.n_fixture_missing == r.n_cases { r.verdict = NX_SMOKE_RUN_INFRASTRUCTURE_FAIL }
207 return r
208}
209
210// ===== Diff helper for failing cases ==============================
211//
212// When emitted JSONL doesn't match expected, substrate emits a
213// human-readable diff to stderr (or to the SmokeCase.failure_detail
214// buffer). Helps the human investigating.
215
216const NX_SMOKE_DIFF_BUF_BYTES: i64 = 16384
217
218func nx_smoke_diff_jsonl(
219 actual_ptr: *u8,
220 actual_len: i64,
221 expected_ptr: *u8,
222 expected_len: i64,
223 out_buf: *u8,
224 out_cap: i64
225) -> i64 {
226 if out_cap < NX_SMOKE_DIFF_BUF_BYTES { return -1 }
227 // v1: byte-by-byte compare; first divergence offset + 64 surrounding
228 // bytes from each. v1.1: line-aligned diff.
229 if actual_len != expected_len { return 1 }
230 var i: i64 = 0
231 var iter: i64 = 0
232 var verdict: i64 = 0
233 while verdict == 0 && iter < actual_len {
234 if actual_ptr[i] != expected_ptr[i] { return 1 }
235 i = i + 1
236 iter = iter + 1
237 if i >= actual_len { verdict = 1 }
238 }
239 return 0
240}
241
242// ===== Smoke-suite manifest =======================================
243//
244// Per adapter, the substrate ships a manifest of smoke cases in
245// nishi-library/smokes/<adapter>/cases.jsonl with rows like:
246//
247// {"case_name":"hardiness_98101","input_fixture":"98101.fixture",
248// "expected_output":"98101.expected.jsonl"}
249//
250// Smoke harness reads the manifest + executes each case.
251
252// ===== Bench output format ========================================
253//
254// Substrate emits run results to nishi-library/smokes/runs/
255// <YYYY-MM-DD-HHMMSS>.run.jsonl with per-case rows + run-level
256// summary. Pillar 5 watchdog can alert on FAILED rate >5% nightly.