nx_canon_cid.nx source
↩ module page · 200 lines · 6911 B
1// nx_canon_cid.nx -- PART B of the sovereign storage substrate rung 1
2// (knowledge/specs/2026-06-09-tutoring-storage-substrate-rung1.md).
3//
4// Content-addressed records demand BYTE-DETERMINISTIC canonical encoding
5// BEFORE hashing (the spec's hard requirement: IPLD does not give this;
6// without it dedup-by-content silently breaks). Canonical form here:
7// magic "NXR1" | u32be field-count | per field (key-sorted byte-lex):
8// u32be klen | key | u32be vlen | value
9// One logical record => exactly one byte string => exactly one CID.
10// CID = "nxc1-" + 64 lowercase hex of sha256(canonical bytes); the hash
11// is the substrate-canonical sha256 (nx_sha256.nx, FIPS 180-4 KAT'd).
12//
13// Pure NishiLang, NO SQL -- this is the dedup/provenance foundation of
14// the S-class-exceed information-management ladder.
15// license_tier: ORIGINAL
16
17import "nx_syscalls.nx"
18import "nx_sha256.nx"
19
20func cc_len(s: *u8) -> i64 {
21 var n: i64 = 0
22 while s[n] != (0 as u8) { n = n + 1 }
23 return n
24}
25
26// byte-lexicographic compare of null-terminated keys: <0, 0, >0
27func cc_cmp(a: *u8, b: *u8) -> i64 {
28 var i: i64 = 0
29 while 1 == 1 {
30 let ca: i64 = a[i]
31 let cb: i64 = b[i]
32 if ca != cb { return ca - cb }
33 if ca == 0 { return 0 }
34 i = i + 1
35 }
36 return 0
37}
38
39func cc_w32(p: *u8, off: i64, v: i64) -> i64 {
40 p[off] = ((v >> 24) & 0xff) as u8
41 p[off + 1] = ((v >> 16) & 0xff) as u8
42 p[off + 2] = ((v >> 8) & 0xff) as u8
43 p[off + 3] = (v & 0xff) as u8
44 return off + 4
45}
46
47// Encode n (key,value) string fields into `out` canonically; returns byte length.
48// keys/vals are arrays of pointers; insertion order MUST NOT matter -- that is
49// the whole proof (total key order = canonicalization rule 1).
50func canon_encode(keys: *i64, vals: *i64, n: i64, out: *u8) -> i64 {
51 let idx: *i64 = sys_mmap(8 * (n + 2)) as *i64
52 var i: i64 = 0
53 while i < n { idx[i] = i; i = i + 1 }
54 // insertion sort of field indices by key bytes
55 i = 1
56 while i < n {
57 let cur: i64 = idx[i]
58 var j: i64 = i - 1
59 var go: i64 = 1
60 while go == 1 {
61 if j < 0 { go = 0 }
62 if go == 1 {
63 if cc_cmp(keys[idx[j]] as *u8, keys[cur] as *u8) > 0 {
64 idx[j + 1] = idx[j]
65 j = j - 1
66 } else { go = 0 }
67 }
68 }
69 idx[j + 1] = cur
70 i = i + 1
71 }
72 var o: i64 = 0
73 out[0] = 78 as u8 // N
74 out[1] = 88 as u8 // X
75 out[2] = 82 as u8 // R
76 out[3] = 49 as u8 // 1
77 o = 4
78 o = cc_w32(out, o, n)
79 i = 0
80 while i < n {
81 let k: *u8 = keys[idx[i]] as *u8
82 let v: *u8 = vals[idx[i]] as *u8
83 let kl: i64 = cc_len(k)
84 let vl: i64 = cc_len(v)
85 o = cc_w32(out, o, kl)
86 var t: i64 = 0
87 while t < kl { out[o] = k[t]; o = o + 1; t = t + 1 }
88 o = cc_w32(out, o, vl)
89 t = 0
90 while t < vl { out[o] = v[t]; o = o + 1; t = t + 1 }
91 i = i + 1
92 }
93 // bulk drivers encode once per file -- free the sort scratch or it
94 // compounds to GBs of touched pages across a full-corpus walk
95 sys_munmap(idx as *u8, 8 * (n + 2))
96 return o
97}
98
99// EXACT byte length canon_encode will return for these same (keys, vals, n).
100// The format is length-prefixed binary with NO escaping, so this is arithmetic,
101// not a bound to guess: 4 ("NXR1") + 4 (field count) + per field 4 (key length)
102// + key + 4 (value length) + value. Key order does not affect the total, which
103// is why the sort is skipped here.
104// Callers MUST size their output buffer from this instead of writing a constant
105// beside the call -- the encoder is the only thing that knows its own width, and
106// a capacity guessed next to it is a second ruler that silently truncates once
107// the inputs grow. nx_canon_size_gate asserts this equals canon_encode's own
108// return value, so the two cannot drift apart.
109func canon_encode_size(keys: *i64, vals: *i64, n: i64) -> i64 {
110 var total: i64 = 8
111 var i: i64 = 0
112 while i < n {
113 total = total + 8 + cc_len(keys[i] as *u8) + cc_len(vals[i] as *u8)
114 i = i + 1
115 }
116 return total
117}
118
119// digest(32B) -> "nxc1-" + 64 lowercase hex, null-terminated; returns 69.
120func cid_from_digest(dg: *u8, cid: *u8) -> i64 {
121 cid[0] = 110 as u8 // n
122 cid[1] = 120 as u8 // x
123 cid[2] = 99 as u8 // c
124 cid[3] = 49 as u8 // 1
125 cid[4] = 45 as u8 // -
126 var i: i64 = 0
127 while i < 32 {
128 let b: i64 = dg[i] & 0xff
129 let hi: i64 = (b >> 4) & 15
130 let lo: i64 = b & 15
131 var c1: i64 = 48 + hi
132 if hi > 9 { c1 = 87 + hi }
133 var c2: i64 = 48 + lo
134 if lo > 9 { c2 = 87 + lo }
135 cid[5 + i * 2] = c1 as u8
136 cid[6 + i * 2] = c2 as u8
137 i = i + 1
138 }
139 cid[69] = 0 as u8
140 return 69
141}
142
143// CID of canonical bytes: "nxc1-" + 64 hex chars, null-terminated; returns 69.
144func cid_of(bytes: *u8, n: i64, cid: *u8) -> i64 {
145 let dg: *u8 = sys_mmap(40)
146 sha256_digest(bytes, n, dg)
147 let r: i64 = cid_from_digest(dg, cid)
148 sys_munmap(dg, 40)
149 return r
150}
151
152const CID_FILE_CHUNK: i64 = 4194304 // 4 MiB read chunk for the file-streaming path
153
154// Streaming CID of a FILE's raw bytes -- IDENTICAL to cid_of over the full
155// content (same sha256, fed via sha256_update), so dedup keys agree across
156// both paths. Exists so multi-GB media never full-reads into RAM (the seq119
157// OOM guard used to SKIP such files instead of hashing them). chunk = read
158// size; tests pass a tiny prime to force multi-update block-boundary coverage.
159// Frees ALL its scratch before returning (callers loop over many files).
160// Writes total file bytes to szbox[0]; returns 69, or -1 on open/read error.
161func cid_of_file_chunk(path: *u8, chunk: i64, szbox: *i64, cid: *u8) -> i64 {
162 let fd: i64 = sys_openat_rd(path)
163 if fd < 0 { return 0 - 1 }
164 let ctx: *Sha256 = sys_mmap(256) as *Sha256
165 sha256_init(ctx)
166 let buf: *u8 = sys_mmap(chunk)
167 var total: i64 = 0
168 var bad: i64 = 0
169 var done: i64 = 0
170 while done == 0 {
171 let n: i64 = sys_read(fd, buf, chunk)
172 if n < 0 { bad = 1 }
173 if n <= 0 { done = 1 }
174 if done == 0 {
175 sha256_update(ctx, buf, n)
176 total = total + n
177 }
178 }
179 sys_close(fd)
180 var r: i64 = 0 - 1
181 if bad == 0 {
182 let dg: *u8 = sys_mmap(40)
183 sha256_final(ctx, dg)
184 r = cid_from_digest(dg, cid)
185 szbox[0] = total
186 sys_munmap(dg, 40)
187 }
188 sys_munmap(ctx.bufptr as *u8, 64)
189 sys_munmap(ctx.kptr as *u8, 512)
190 sys_munmap(ctx.wptr as *u8, 512)
191 sys_munmap(ctx.k32ptr as *u8, 256)
192 sys_munmap(ctx.st8ptr as *u8, 32)
193 sys_munmap(ctx as *u8, 256)
194 sys_munmap(buf, chunk)
195 return r
196}
197
198func cid_of_file(path: *u8, szbox: *i64, cid: *u8) -> i64 {
199 return cid_of_file_chunk(path, CID_FILE_CHUNK, szbox, cid)
200}