nx_canon_cid.nx source
↩ module page · 180 lines · 5942 B
1// nx_canon_cid.nx -- PART B of the sovereign storage substrate rung 1
2// (knowledge/specs/2026-06-09-tutoring-storage-substrate-rung1.md).
3//
4// Content-addressed records demand BYTE-DETERMINISTIC canonical encoding
5// BEFORE hashing (the spec's hard requirement: IPLD does not give this;
6// without it dedup-by-content silently breaks). Canonical form here:
7// magic "NXR1" | u32be field-count | per field (key-sorted byte-lex):
8// u32be klen | key | u32be vlen | value
9// One logical record => exactly one byte string => exactly one CID.
10// CID = "nxc1-" + 64 lowercase hex of sha256(canonical bytes); the hash
11// is the substrate-canonical sha256 (nx_sha256.nx, FIPS 180-4 KAT'd).
12//
13// Pure NishiLang, NO SQL -- this is the dedup/provenance foundation of
14// the S-class-exceed information-management ladder.
15// license_tier: ORIGINAL
16
17import "nx_syscalls.nx"
18import "nx_sha256.nx"
19
20func cc_len(s: *u8) -> i64 {
21 var n: i64 = 0
22 while s[n] != (0 as u8) { n = n + 1 }
23 return n
24}
25
26// byte-lexicographic compare of null-terminated keys: <0, 0, >0
27func cc_cmp(a: *u8, b: *u8) -> i64 {
28 var i: i64 = 0
29 while 1 == 1 {
30 let ca: i64 = a[i]
31 let cb: i64 = b[i]
32 if ca != cb { return ca - cb }
33 if ca == 0 { return 0 }
34 i = i + 1
35 }
36 return 0
37}
38
39func cc_w32(p: *u8, off: i64, v: i64) -> i64 {
40 p[off] = ((v >> 24) & 0xff) as u8
41 p[off + 1] = ((v >> 16) & 0xff) as u8
42 p[off + 2] = ((v >> 8) & 0xff) as u8
43 p[off + 3] = (v & 0xff) as u8
44 return off + 4
45}
46
47// Encode n (key,value) string fields into `out` canonically; returns byte length.
48// keys/vals are arrays of pointers; insertion order MUST NOT matter -- that is
49// the whole proof (total key order = canonicalization rule 1).
50func canon_encode(keys: *i64, vals: *i64, n: i64, out: *u8) -> i64 {
51 let idx: *i64 = sys_mmap(8 * (n + 2)) as *i64
52 var i: i64 = 0
53 while i < n { idx[i] = i; i = i + 1 }
54 // insertion sort of field indices by key bytes
55 i = 1
56 while i < n {
57 let cur: i64 = idx[i]
58 var j: i64 = i - 1
59 var go: i64 = 1
60 while go == 1 {
61 if j < 0 { go = 0 }
62 if go == 1 {
63 if cc_cmp(keys[idx[j]] as *u8, keys[cur] as *u8) > 0 {
64 idx[j + 1] = idx[j]
65 j = j - 1
66 } else { go = 0 }
67 }
68 }
69 idx[j + 1] = cur
70 i = i + 1
71 }
72 var o: i64 = 0
73 out[0] = 78 as u8 // N
74 out[1] = 88 as u8 // X
75 out[2] = 82 as u8 // R
76 out[3] = 49 as u8 // 1
77 o = 4
78 o = cc_w32(out, o, n)
79 i = 0
80 while i < n {
81 let k: *u8 = keys[idx[i]] as *u8
82 let v: *u8 = vals[idx[i]] as *u8
83 let kl: i64 = cc_len(k)
84 let vl: i64 = cc_len(v)
85 o = cc_w32(out, o, kl)
86 var t: i64 = 0
87 while t < kl { out[o] = k[t]; o = o + 1; t = t + 1 }
88 o = cc_w32(out, o, vl)
89 t = 0
90 while t < vl { out[o] = v[t]; o = o + 1; t = t + 1 }
91 i = i + 1
92 }
93 // bulk drivers encode once per file -- free the sort scratch or it
94 // compounds to GBs of touched pages across a full-corpus walk
95 sys_munmap(idx as *u8, 8 * (n + 2))
96 return o
97}
98
99// digest(32B) -> "nxc1-" + 64 lowercase hex, null-terminated; returns 69.
100func cid_from_digest(dg: *u8, cid: *u8) -> i64 {
101 cid[0] = 110 as u8 // n
102 cid[1] = 120 as u8 // x
103 cid[2] = 99 as u8 // c
104 cid[3] = 49 as u8 // 1
105 cid[4] = 45 as u8 // -
106 var i: i64 = 0
107 while i < 32 {
108 let b: i64 = dg[i] & 0xff
109 let hi: i64 = (b >> 4) & 15
110 let lo: i64 = b & 15
111 var c1: i64 = 48 + hi
112 if hi > 9 { c1 = 87 + hi }
113 var c2: i64 = 48 + lo
114 if lo > 9 { c2 = 87 + lo }
115 cid[5 + i * 2] = c1 as u8
116 cid[6 + i * 2] = c2 as u8
117 i = i + 1
118 }
119 cid[69] = 0 as u8
120 return 69
121}
122
123// CID of canonical bytes: "nxc1-" + 64 hex chars, null-terminated; returns 69.
124func cid_of(bytes: *u8, n: i64, cid: *u8) -> i64 {
125 let dg: *u8 = sys_mmap(40)
126 sha256_digest(bytes, n, dg)
127 let r: i64 = cid_from_digest(dg, cid)
128 sys_munmap(dg, 40)
129 return r
130}
131
132const CID_FILE_CHUNK: i64 = 4194304 // 4 MiB read chunk for the file-streaming path
133
134// Streaming CID of a FILE's raw bytes -- IDENTICAL to cid_of over the full
135// content (same sha256, fed via sha256_update), so dedup keys agree across
136// both paths. Exists so multi-GB media never full-reads into RAM (the seq119
137// OOM guard used to SKIP such files instead of hashing them). chunk = read
138// size; tests pass a tiny prime to force multi-update block-boundary coverage.
139// Frees ALL its scratch before returning (callers loop over many files).
140// Writes total file bytes to szbox[0]; returns 69, or -1 on open/read error.
141func cid_of_file_chunk(path: *u8, chunk: i64, szbox: *i64, cid: *u8) -> i64 {
142 let fd: i64 = sys_openat_rd(path)
143 if fd < 0 { return 0 - 1 }
144 let ctx: *Sha256 = sys_mmap(256) as *Sha256
145 sha256_init(ctx)
146 let buf: *u8 = sys_mmap(chunk)
147 var total: i64 = 0
148 var bad: i64 = 0
149 var done: i64 = 0
150 while done == 0 {
151 let n: i64 = sys_read(fd, buf, chunk)
152 if n < 0 { bad = 1 }
153 if n <= 0 { done = 1 }
154 if done == 0 {
155 sha256_update(ctx, buf, n)
156 total = total + n
157 }
158 }
159 sys_close(fd)
160 var r: i64 = 0 - 1
161 if bad == 0 {
162 let dg: *u8 = sys_mmap(40)
163 sha256_final(ctx, dg)
164 r = cid_from_digest(dg, cid)
165 szbox[0] = total
166 sys_munmap(dg, 40)
167 }
168 sys_munmap(ctx.bufptr as *u8, 64)
169 sys_munmap(ctx.kptr as *u8, 512)
170 sys_munmap(ctx.wptr as *u8, 512)
171 sys_munmap(ctx.k32ptr as *u8, 256)
172 sys_munmap(ctx.st8ptr as *u8, 32)
173 sys_munmap(ctx as *u8, 256)
174 sys_munmap(buf, chunk)
175 return r
176}
177
178func cid_of_file(path: *u8, szbox: *i64, cid: *u8) -> i64 {
179 return cid_of_file_chunk(path, CID_FILE_CHUNK, szbox, cid)
180}