nx_log_chunked.nx source
↩ module page · 276 lines · 11219 B
1// nx_log_chunked.nx -- append-only Merkle-linked chunk log.
2//
3// Sovereign storage substrate for live-fire trace data. Foundation
4// for the upcoming nx_trace_query and substrate-native dashboards.
5//
6// Architecture (per QMDB 2025, Trillian, IPFS Merkle-DAG, Git):
7//
8// * Spans land in an in-memory chunk buffer.
9// * When the chunk fills (or is explicitly flushed), it is
10// SHA-256 hashed via the existing nx_sha256 primitive.
11// * The chunk's hash + the PARENT chunk's hash + chunk metadata
12// form a Merkle-linked log: any tamper of an earlier chunk
13// invalidates every subsequent chunk hash.
14// * A separate manifest table records (chunk_hash, n_spans,
15// first_trace_id, last_trace_id, written_at) per chunk for
16// replay + integrity verification.
17//
18// Why not SQLite / LMDB / RocksDB:
19// * each ships ~250KB+ of C; not sovereign.
20// * the LSM-tree write path can be expressed in NishiLang directly
21// (memtable = current chunk, segment files = flushed chunks).
22// * Bloom filters + segment indexes queued for v2.
23//
24// Why not Parquet/Arrow:
25// * spans are small irregular shapes; columnar wins are tiny here.
26// * column store can land later as a separate projection (CQRS:
27// log is source of truth; columns are derived).
28//
29// Wire format inside chunks:
30// * each span is CBOR-encoded via nx_cbor.nx (machine-readable
31// binary; spec-compliant; multiple-language decoders exist).
32// * the JSONL human-readable wire stays on nx_trace_emit.nx.
33//
34// genealogy_id: oneill_1996_lsm_tree + merkle_1979 + trillian_transparency +
35// qmdb_2025_arxiv_2501_05262
36// lineage_id: sovereign_log_v1
37
38// nx_safety_envelope:
39// intended_use: AUTO_APPLIED -- primitive-specific tuning queued
40// sil_target: SIL1
41// evidence: [bulk_applied_2026-05-16, see-file-comment-for-detail]
42// verdict: NOT_YET_EVALUATED
43
44import "nx_syscalls.nx"
45import "nx_tier.nx"
46import "nx_sha256.nx"
47import "nx_cbor.nx"
48
49const NX_LC_HASH_BYTES: nx_int = 32 // SHA-256
50const NX_LC_DEFAULT_CHUNK: nx_int = 4096 // bytes per chunk before flush
51
52// ===== Sealed-enum: ChunkVerdict ==================================
53//
54// Integrity outcome when verifying a chunk against its manifest.
55
56const NX_LC_VERIFY_VALID: nx_int = 0
57const NX_LC_VERIFY_HASH_MISMATCH: nx_int = 1 // chunk bytes don't hash to recorded hash
58const NX_LC_VERIFY_PARENT_BROKEN: nx_int = 2 // recorded parent hash != prior chunk's hash
59const NX_LC_VERIFY_ORDER_REVERSED: nx_int = 3 // chunk index decreased
60const NX_LC_VERIFY_N_VERDICTS: nx_int = 4
61
62func nx_lc_verify_verdict_is_valid(v: nx_int) -> nx_int {
63 if v < 0 { return 0 }
64 if v >= NX_LC_VERIFY_N_VERDICTS { return 0 }
65 return 1
66}
67
68// ===== Chunk record (in manifest) =================================
69//
70// One manifest entry per flushed chunk. Flat-array layout matches
71// the substrate convention (nx_canon_proportions, nx_arc). Fields:
72//
73// 0 chunk_index monotonic
74// 1 n_spans spans in this chunk
75// 2 written_at_ms epoch ms when chunk flushed
76// 3 hash_off offset into manifest.hash_buf for this chunk's SHA-256
77// 4 parent_hash_off offset for the parent chunk's SHA-256 (-1 for root)
78// 5 first_trace_id smallest trace_id in chunk
79// 6 last_trace_id largest trace_id in chunk
80
81const NX_LC_REC_F_INDEX: nx_int = 0
82const NX_LC_REC_F_N_SPANS: nx_int = 1
83const NX_LC_REC_F_WRITTEN_AT: nx_int = 2
84const NX_LC_REC_F_HASH_OFF: nx_int = 3
85const NX_LC_REC_F_PARENT_HASH_OFF: nx_int = 4
86const NX_LC_REC_F_FIRST_TRACE: nx_int = 5
87const NX_LC_REC_F_LAST_TRACE: nx_int = 6
88const NX_LC_REC_FIELDS: nx_int = 7
89
90// ===== ChunkedLog struct ==========================================
91
92struct ChunkedLog {
93 chunk_cap: nx_int, // bytes per chunk before flush
94 chunk_buf: *u8, // in-memory chunk buffer
95 chunk_pos: nx_int, // write position inside chunk_buf
96 chunk_n_spans: nx_int, // spans accumulated in current chunk
97 chunk_first_tid: nx_int, // first trace_id in current chunk
98 chunk_last_tid: nx_int, // last trace_id in current chunk
99 n_chunks: nx_int, // total flushed chunks
100 manifest_cap: nx_int, // capacity in chunk-record slots
101 manifest: *i64, // n_chunks * NX_LC_REC_FIELDS
102 hash_buf: *u8, // contiguous 32-byte SHA-256 blobs; n_chunks * 32
103 last_hash_off: nx_int // offset of the most recent chunk hash (-1 if none)
104}
105
106const NX_LC_LOG_BYTES: nx_int = 88 // 11 fields * 8
107
108// ===== Builder ====================================================
109
110func nx_lc_alloc(chunk_cap: nx_int, manifest_cap: nx_int) -> *ChunkedLog {
111 let log: *ChunkedLog = (sys_mmap(NX_LC_LOG_BYTES)) as *ChunkedLog
112 log.chunk_cap = chunk_cap
113 log.chunk_buf = (sys_mmap(chunk_cap)) as *u8
114 log.chunk_pos = 0
115 log.chunk_n_spans = 0
116 log.chunk_first_tid = 0
117 log.chunk_last_tid = 0
118 log.n_chunks = 0
119 log.manifest_cap = manifest_cap
120 let manifest_bytes: nx_int = manifest_cap * NX_LC_REC_FIELDS * NX_SIZEOF_NX_INT
121 log.manifest = (sys_mmap(manifest_bytes)) as *i64
122 let hash_bytes: nx_int = manifest_cap * NX_LC_HASH_BYTES
123 log.hash_buf = (sys_mmap(hash_bytes)) as *u8
124 log.last_hash_off = 0 - 1 // -1 = no parent yet
125 return log
126}
127
128// ===== Flush the current chunk ====================================
129//
130// SHA-256 the chunk bytes, record (hash, parent_hash, metadata) in
131// the manifest, reset chunk buffer. No syscalls — caller is
132// responsible for persisting the manifest to disk via a separate
133// nx_lc_persist (v2).
134//
135// Returns 0 on success or -1 if the manifest is full.
136//
137// Hoisted above nx_lc_append_span because nxc2's resolver is
138// single-pass and append calls flush on overflow.
139
140func nx_lc_flush(log: *ChunkedLog, now_ms: nx_int) -> nx_int {
141 if log.chunk_n_spans == 0 { return 0 } // nothing to flush
142 if log.n_chunks >= log.manifest_cap { return 0 - 1 }
143
144 // Hash the chunk bytes
145 let hash_off: nx_int = log.n_chunks * NX_LC_HASH_BYTES
146 let hash_ptr: *u8 = ((log.hash_buf as nx_int) + hash_off) as *u8
147 sha256_digest(log.chunk_buf, log.chunk_pos, hash_ptr)
148
149 // Record manifest entry
150 let rec_base: nx_int = log.n_chunks * NX_LC_REC_FIELDS
151 log.manifest[rec_base + NX_LC_REC_F_INDEX] = log.n_chunks
152 log.manifest[rec_base + NX_LC_REC_F_N_SPANS] = log.chunk_n_spans
153 log.manifest[rec_base + NX_LC_REC_F_WRITTEN_AT] = now_ms
154 log.manifest[rec_base + NX_LC_REC_F_HASH_OFF] = hash_off
155 log.manifest[rec_base + NX_LC_REC_F_PARENT_HASH_OFF] = log.last_hash_off
156 log.manifest[rec_base + NX_LC_REC_F_FIRST_TRACE] = log.chunk_first_tid
157 log.manifest[rec_base + NX_LC_REC_F_LAST_TRACE] = log.chunk_last_tid
158
159 // Advance bookkeeping
160 log.n_chunks = log.n_chunks + 1
161 log.last_hash_off = hash_off
162 log.chunk_pos = 0
163 log.chunk_n_spans = 0
164 log.chunk_first_tid = 0
165 log.chunk_last_tid = 0
166 return 0
167}
168
169// ===== Append a span (CBOR-encoded) ==============================
170//
171// Appends one span to the current chunk. If the chunk would
172// overflow, flushes first then appends to the fresh chunk.
173//
174// Returns 0 on success or the rc from the flush path on overflow.
175
176func nx_lc_append_span(log: *ChunkedLog,
177 trace_id: nx_int, span_id: nx_int,
178 parent: nx_int, kind: nx_int,
179 start_ms: nx_int, end_ms: nx_int,
180 attrs: *i64, n_attrs: nx_int,
181 now_ms: nx_int) -> nx_int {
182 if log.chunk_cap - log.chunk_pos < 128 {
183 let rc: nx_int = nx_lc_flush(log, now_ms)
184 if rc < 0 { return rc }
185 }
186
187 let new_pos: nx_int = nx_cbor_emit_span(log.chunk_buf, log.chunk_pos,
188 trace_id, span_id, parent, kind,
189 start_ms, end_ms,
190 attrs, n_attrs)
191 if log.chunk_n_spans == 0 {
192 log.chunk_first_tid = trace_id
193 }
194 log.chunk_last_tid = trace_id
195 log.chunk_pos = new_pos
196 log.chunk_n_spans = log.chunk_n_spans + 1
197 return 0
198}
199
200// ===== Hash byte comparator (hoisted before verify_chunk_bytes) ===
201//
202// Returns 1 if two 32-byte SHA-256 hashes are byte-equal, 0 otherwise.
203
204func _nx_lc_hash_eq(a: *u8, b: *u8) -> nx_int {
205 var i: nx_int = 0
206 while i < NX_LC_HASH_BYTES {
207 if a[i] != b[i] { return 0 }
208 i = i + 1
209 }
210 return 1
211}
212
213// ===== Manifest reader ============================================
214//
215// Field accessor for chunk records (no struct fields exposed; flat
216// array keeps the substrate's <=8-arg-per-fn convention happy).
217
218func nx_lc_rec_get(log: *ChunkedLog, chunk_index: nx_int, field: nx_int) -> nx_int {
219 if chunk_index < 0 { return 0 - 1 }
220 if chunk_index >= log.n_chunks { return 0 - 1 }
221 return log.manifest[chunk_index * NX_LC_REC_FIELDS + field]
222}
223
224// ===== Integrity verifier ========================================
225//
226// Walks the chunk chain. For each chunk, the recorded parent_hash
227// must match the previous chunk's hash; index must be monotonic.
228// We don't re-hash the chunk bytes here because they may have been
229// flushed to disk (v2 nx_lc_load); this verifier checks the
230// manifest's internal chain consistency. Bytes-vs-hash check is
231// the caller's job once they've reloaded the chunk bytes.
232
233func nx_lc_verify_chain(log: *ChunkedLog) -> nx_int {
234 var i: nx_int = 0
235 var prev_hash_off: nx_int = 0 - 1
236 while i < log.n_chunks {
237 let rec_base: nx_int = i * NX_LC_REC_FIELDS
238
239 // Index monotonicity
240 if log.manifest[rec_base + NX_LC_REC_F_INDEX] != i {
241 return NX_LC_VERIFY_ORDER_REVERSED
242 }
243
244 // Parent hash chain
245 let recorded_parent: nx_int = log.manifest[rec_base + NX_LC_REC_F_PARENT_HASH_OFF]
246 if recorded_parent != prev_hash_off {
247 return NX_LC_VERIFY_PARENT_BROKEN
248 }
249
250 prev_hash_off = log.manifest[rec_base + NX_LC_REC_F_HASH_OFF]
251 i = i + 1
252 }
253 return NX_LC_VERIFY_VALID
254}
255
256// ===== Hash-bytes verifier =======================================
257//
258// For one specific chunk: recompute SHA-256 of its bytes (passed in
259// since the bytes may live on disk) and compare to the recorded
260// hash in the manifest.
261
262func nx_lc_verify_chunk_bytes(log: *ChunkedLog, chunk_index: nx_int,
263 bytes: *u8, n_bytes: nx_int) -> nx_int {
264 if chunk_index < 0 { return NX_LC_VERIFY_HASH_MISMATCH }
265 if chunk_index >= log.n_chunks { return NX_LC_VERIFY_HASH_MISMATCH }
266
267 let rec_base: nx_int = chunk_index * NX_LC_REC_FIELDS
268 let recorded_off: nx_int = log.manifest[rec_base + NX_LC_REC_F_HASH_OFF]
269 let recorded: *u8 = ((log.hash_buf as nx_int) + recorded_off) as *u8
270
271 let computed: *u8 = (sys_mmap(NX_LC_HASH_BYTES)) as *u8
272 sha256_digest(bytes, n_bytes, computed)
273
274 if _nx_lc_hash_eq(recorded, computed) == 1 { return NX_LC_VERIFY_VALID }
275 return NX_LC_VERIFY_HASH_MISMATCH
276}