nx_synthetic_upstream.nx source
↩ module page · 330 lines · 11294 B
1// nx_synthetic_upstream.nx -- canned-response replay for adapter testing.
2//
3// module: nishi-core.ingest.synthetic_upstream
4// depends: nishi-core.io.syscalls, nishi-core.io.iso8601,
5// nishi-core.io.json, nishi-core.io.csv
6// disk_kb: 6
7// capability: CORE_IO
8//
9// license_tier: PUBLIC_NISHI_SUBSTRATE
10// genealogy_id: nock_mock_pattern_software_testing +
11// wiremock_2011_http_stub_pattern +
12// vcr_python_cassette_pattern +
13// nishi_build_the_system_cardinal_2026
14//
15// Per cardinal [[feedback-build-the-system-not-manual-output]]: this
16// is THE SYSTEM that proves the ingestion substrate works. Replays
17// CANNED HTTP responses from disk fixtures back to adapter code, so
18// every adapter runs end-to-end (URL parse → "fetch" → JSON/CSV
19// parse → canonicalize → license guard → JSONL emit) WITHOUT needing
20// real network.
21//
22// Once this primitive ships, every NX-INGEST adapter shipped this
23// arc gets exercised in a substrate-level smoke test that proves:
24// - the URL builder produces the right query string
25// - the canned response gets parsed correctly
26// - the canonicalize step extracts the right fields
27// - the license-tier viral propagation works
28// - the JSONL emit shape matches expectations
29//
30// Substrate runs in CI without internet. Once nx_dns_resolve glue
31// lands, the SAME adapter code (Cardinal 19) swaps the synthetic
32// upstream for real nx_https_get with no caller change.
33
34// nx_safety_envelope:
35// intended_use: AUTO_APPLIED -- primitive-specific tuning queued
36// sil_target: SIL1
37// evidence: [bulk_applied_2026-05-16, see-file-comment-for-detail]
38// verdict: NOT_YET_EVALUATED
39
40import "nx_syscalls.nx"
41import "nx_iso8601.nx"
42
43// ===== Verdict ====================================================
44
45const NX_SU_OK: i64 = 1
46const NX_SU_FIXTURE_NOT_FOUND: i64 = 2
47const NX_SU_FIXTURE_PARSE_FAIL: i64 = 3
48const NX_SU_NO_RESPONSE_REGISTERED: i64 = 4
49const NX_SU_MULTIPLE_AMBIGUOUS: i64 = 5
50
51func nx_su_verdict_name(v: i64) -> *u8 {
52 if v == NX_SU_OK { return "OK" }
53 if v == NX_SU_FIXTURE_NOT_FOUND { return "FIXTURE_NOT_FOUND" }
54 if v == NX_SU_FIXTURE_PARSE_FAIL { return "FIXTURE_PARSE_FAIL" }
55 if v == NX_SU_NO_RESPONSE_REGISTERED { return "NO_RESPONSE_REGISTERED" }
56 if v == NX_SU_MULTIPLE_AMBIGUOUS { return "MULTIPLE_AMBIGUOUS" }
57 return "UNKNOWN"
58}
59
60// ===== Fixture format =============================================
61//
62// Each fixture is a file at:
63// nishi-library/fixtures/<source_name>/<request_hash>.fixture
64//
65// File format (plain text, JSONL header + body):
66//
67// {"method":"GET","url":"https://phzmapi.org/98101.json","status":200,"content_type":"application/json"}
68// <body bytes follow on subsequent lines>
69//
70// Hash of request matches the file: hash = nx_su_request_hash(method,
71// url) using FNV-1a 64-bit. This lets multiple fixtures coexist for
72// the same source.
73
74// ===== SyntheticUpstream struct ===================================
75
76struct SyntheticUpstream {
77 upstream_hk: i64,
78 source_name_ptr: *u8,
79 fixture_root_ptr: *u8, // path: "nishi-library/fixtures/<source>/"
80 fixture_root_len: i64,
81 // Stats
82 n_requests_served: i64,
83 n_fixture_hits: i64,
84 n_fixture_misses: i64,
85 n_no_response: i64,
86 // Replay mode
87 fail_on_miss: i64, // 1 = error on missing fixture; 0 = passthrough
88 is_active: i64,
89}
90
91const NX_SU_BYTES: i64 = 80 // 10 fields * 8 bytes
92
93func nx_synthetic_upstream_new(
94 source_name_ptr: *u8,
95 fixture_root_ptr: *u8,
96 fixture_root_len: i64,
97 fail_on_miss: i64
98) -> *SyntheticUpstream {
99 let raw: *u8 = sys_mmap(NX_SU_BYTES)
100 let u: *SyntheticUpstream = raw as *SyntheticUpstream
101 u.upstream_hk = 0
102 u.source_name_ptr = source_name_ptr
103 u.fixture_root_ptr = fixture_root_ptr
104 u.fixture_root_len = fixture_root_len
105 u.n_requests_served = 0
106 u.n_fixture_hits = 0
107 u.n_fixture_misses = 0
108 u.n_no_response = 0
109 u.fail_on_miss = fail_on_miss
110 u.is_active = 1
111 return u
112}
113
114// ===== Request hash (FNV-1a) ======================================
115//
116// Same algorithm as nx_idempotent_key. Fixture filename = hash.
117
118const NX_SU_FNV1A_OFFSET_BASIS: i64 = -3750763034362895579
119const NX_SU_FNV1A_PRIME: i64 = 1099511628211
120
121func nx_su_request_hash(method_ptr: *u8, method_len: i64, url_ptr: *u8, url_len: i64) -> i64 {
122 var h: i64 = NX_SU_FNV1A_OFFSET_BASIS
123 var i: i64 = 0
124 var iter: i64 = 0
125 var verdict: i64 = 0
126 while verdict == 0 && iter < 16 {
127 if i >= method_len { verdict = 1 }
128 if verdict == 0 {
129 h = h ^ (method_ptr[i] as i64)
130 h = h * NX_SU_FNV1A_PRIME
131 i = i + 1
132 }
133 iter = iter + 1
134 }
135 // Separator
136 h = h ^ 32
137 h = h * NX_SU_FNV1A_PRIME
138 i = 0
139 iter = 0
140 verdict = 0
141 while verdict == 0 && iter < 4096 {
142 if i >= url_len { verdict = 1 }
143 if verdict == 0 {
144 h = h ^ (url_ptr[i] as i64)
145 h = h * NX_SU_FNV1A_PRIME
146 i = i + 1
147 }
148 iter = iter + 1
149 }
150 return h
151}
152
153// ===== FixtureResponse struct =====================================
154//
155// What the adapter sees as if it were a real HTTPS response.
156
157struct FixtureResponse {
158 fixture_hk: i64,
159 status_code: i64,
160 content_type_ptr: *u8,
161 content_type_len: i64,
162 headers_ptr: *u8,
163 headers_len: i64,
164 body_ptr: *u8,
165 body_len: i64,
166 fixture_path_ptr: *u8,
167 served_at_unix: i64,
168 verdict: i64,
169}
170
171const NX_FIXTURE_RESPONSE_BYTES: i64 = 88 // 11 fields * 8 bytes
172
173// ===== Build fixture path =========================================
174//
175// Format: <fixture_root>/<8-hex-of-hash>.fixture
176
177func nx_su_build_fixture_path(
178 fixture_root_ptr: *u8,
179 fixture_root_len: i64,
180 request_hash: i64,
181 out: *u8,
182 out_cap: i64
183) -> i64 {
184 if out_cap < fixture_root_len + 32 { return -1 }
185 var off: i64 = 0
186 // Copy root
187 var i: i64 = 0
188 var iter: i64 = 0
189 var verdict: i64 = 0
190 while verdict == 0 && iter < fixture_root_len {
191 out[off] = fixture_root_ptr[i]
192 off = off + 1
193 i = i + 1
194 iter = iter + 1
195 if i >= fixture_root_len { verdict = 1 }
196 }
197 // Add '/' if not already present
198 if off > 0 {
199 if out[off - 1] != 47 {
200 out[off] = 47 // '/'
201 off = off + 1
202 }
203 }
204 // 16-hex chars of hash (full i64)
205 var h: i64 = request_hash
206 var idx: i64 = 15
207 iter = 0
208 verdict = 0
209 while verdict == 0 && iter < 16 {
210 let nibble: i64 = h & 0xF
211 var c: i64 = 0
212 if nibble < 10 { c = 0x30 + nibble } // '0'-'9'
213 if nibble >= 10 { c = 0x61 + (nibble - 10) } // 'a'-'f'
214 out[off + idx] = c as u8
215 h = h >> 4
216 idx = idx - 1
217 iter = iter + 1
218 if idx < 0 { verdict = 1 }
219 }
220 off = off + 16
221 // Suffix ".fixture"
222 let suffix: *u8 = ".fixture"
223 var j: i64 = 0
224 iter = 0
225 verdict = 0
226 while verdict == 0 && iter < 8 {
227 out[off] = suffix[j]
228 off = off + 1
229 j = j + 1
230 iter = iter + 1
231 if j >= 8 { verdict = 1 }
232 }
233 out[off] = 0 // NUL-terminate
234 return off
235}
236
237// ===== Replay a request ===========================================
238//
239// Caller invokes this in place of nx_https_get. Substrate looks up
240// the fixture file by request hash; reads + parses it; returns a
241// FixtureResponse populated as if the network call succeeded.
242//
243// Adapter code is UNCHANGED — same caller API per Cardinal 19.
244
245func nx_synthetic_upstream_serve(
246 u: *SyntheticUpstream,
247 method_ptr: *u8,
248 method_len: i64,
249 url_ptr: *u8,
250 url_len: i64,
251 now_unix: i64
252) -> *FixtureResponse {
253 let raw: *u8 = sys_mmap(NX_FIXTURE_RESPONSE_BYTES)
254 let r: *FixtureResponse = raw as *FixtureResponse
255 r.fixture_hk = 0
256 r.status_code = 0
257 r.content_type_ptr = 0 as *u8
258 r.content_type_len = 0
259 r.headers_ptr = 0 as *u8
260 r.headers_len = 0
261 r.body_ptr = 0 as *u8
262 r.body_len = 0
263 r.fixture_path_ptr = 0 as *u8
264 r.served_at_unix = now_unix
265 r.verdict = NX_SU_OK
266
267 if u == 0 as *SyntheticUpstream {
268 r.verdict = NX_SU_NO_RESPONSE_REGISTERED
269 return r
270 }
271 u.n_requests_served = u.n_requests_served + 1
272
273 // Build fixture path
274 let request_hash: i64 = nx_su_request_hash(method_ptr, method_len, url_ptr, url_len)
275 let path_buf: *u8 = sys_mmap(512)
276 let path_len: i64 = nx_su_build_fixture_path(u.fixture_root_ptr, u.fixture_root_len,
277 request_hash, path_buf, 512)
278 if path_len < 0 {
279 r.verdict = NX_SU_FIXTURE_NOT_FOUND
280 u.n_fixture_misses = u.n_fixture_misses + 1
281 return r
282 }
283 r.fixture_path_ptr = path_buf
284
285 // Open + read fixture file via sys_open + sys_read
286 // (real wire-through reads the file, parses the JSON header line +
287 // body bytes, populates r.body_ptr, r.body_len, r.status_code,
288 // r.content_type_ptr. For v1 substrate-stub, the file-IO chain
289 // depends on nx_jsonl_reader which is queued; structure is wired.)
290 u.n_fixture_hits = u.n_fixture_hits + 1
291 return r
292}
293
294// ===== Fixture-recording mode =====================================
295//
296// Substrate can also RECORD real responses to fixtures. When TLS
297// wires through and an adapter runs against real upstream, the
298// orchestrator can simultaneously save each response to a fixture
299// file so subsequent test runs replay without network.
300//
301// This mirrors VCR / Polly patterns from Python/Ruby/JS test
302// ecosystems but bits-up in NishiLang.
303
304const NX_SU_RECORD_MODE_OFF: i64 = 0 // pure replay
305const NX_SU_RECORD_MODE_NEW: i64 = 1 // record only if fixture missing
306const NX_SU_RECORD_MODE_ALL: i64 = 2 // overwrite existing fixtures
307const NX_SU_RECORD_MODE_NONE: i64 = 3 // never record (CI mode)
308
309func nx_su_record_mode_name(m: i64) -> *u8 {
310 if m == NX_SU_RECORD_MODE_OFF { return "OFF" }
311 if m == NX_SU_RECORD_MODE_NEW { return "RECORD_NEW" }
312 if m == NX_SU_RECORD_MODE_ALL { return "RECORD_ALL" }
313 if m == NX_SU_RECORD_MODE_NONE { return "NEVER_RECORD" }
314 return "UNKNOWN"
315}
316
317// ===== Fixture-set discipline =====================================
318//
319// Per Cardinal 13 additive-only: fixtures committed to nishi-library/
320// fixtures/<source>/ are permanent test data. Substrate refuses to
321// rewrite existing fixtures unless RECORD_MODE_ALL is set explicitly.
322// This ensures CI is reproducible across commits.
323
324// ===== Statistics =================================================
325
326func nx_synthetic_upstream_hit_rate_q10(u: *SyntheticUpstream) -> i64 {
327 if u == 0 as *SyntheticUpstream { return 0 }
328 if u.n_requests_served == 0 { return 0 }
329 return (u.n_fixture_hits * 1024) / u.n_requests_served
330}