nx_wiki_archive.nx source
↩ module page · 266 lines · 12370 B
1// nx_wiki_archive.nx -- NO-LINK-ROT for the sovereign wiki.
2//
3// THE LAW (operator, standing): "No link rot. Archive all information into the
4// Nishi Library so nothing that helped us get here is lost; links should always
5// work." This organ makes rot STRUCTURALLY IMPOSSIBLE via content-addressing.
6//
7// MECHANISM (why rot cannot happen here):
8// Each page's raw bytes are stored under a KEY that IS the hash of those
9// bytes -- the CID = "nxc1-" + 64 hex of sha256(body), from nx_canon_cid's
10// cid_of() (substrate-canonical, FIPS 180-4 KAT'd via nx_sha256). A CID can
11// ONLY ever name one byte string: hand back a CID, you get back the EXACT
12// bytes that produced it, forever. So:
13// - identical content collapses to ONE blob (same bytes -> same CID -> dedup)
14// - a CID link NEVER rots: the key is derived from the content, so as long
15// as the immutable append-only seg_store keeps the blob, the link resolves
16// - this is HOW we exceed Wikipedia, which leans on the Internet Archive to
17// paper over rot after the fact; here rot is a non-event by construction.
18//
19// STORE: the content-addressed append-only seg_store (runtime/nx_seg_store.nx),
20// prefix "knowledge/store/wikiarchive-" (the library's own durability
21// substrate). No nx_library_*.nx ingest organ exists yet, so we land in the
22// seg_store directly; full library-corpus / BM25 unification = a flagged
23// follow-on, NOT silently done here.
24//
25// LAYOUT (two key families in one store):
26// wikiblob:<cid> -> the page bytes (content-addressed; idempotent dedup)
27// wikicid:<slug> -> the page's CURRENT cid (slug -> content pointer; mutable
28// by ADDITIVE re-archive, old versions stay in history)
29// Dropping a wikicid:<slug> pointer never loses data -- the blob is still keyed
30// by its CID (the no-loss proof).
31//
32// IMPORTS: nx_seg_store + nx_canon_cid. Both transitively pull nx_syscalls; the
33// NishiLang resolver de-dups by module identity (nx_infomgmt_gate.nx imports the
34// same trio and builds), so no double-import nxasm-rc6 landmine.
35// Pure NishiLang, NO SQL, NO .sh/.py/.js, no new .tsv/.conf. license_tier: ORIGINAL
36import "nx_seg_store.nx"
37import "nx_canon_cid.nx"
38
39const WAR_PREFIX: *u8 = "knowledge/store/wikiarchive-"
40
41// ---- tiny local helpers (war_ namespace; no clash with ss_/cc_) ----
42func war_len(s: *u8) -> i64 {
43 var n: i64 = 0
44 while s[n] != (0 as u8) { n = n + 1 }
45 return n
46}
47
48// concat NUL-terminated s into dst at off; returns new off (no terminator)
49func war_cat(dst: *u8, off: i64, s: *u8) -> i64 {
50 var i: i64 = 0
51 while s[i] != (0 as u8) { dst[off + i] = s[i]; i = i + 1 }
52 return off + i
53}
54
55// build "wikiblob:<cid>" (cid NUL-terminated) into out (NUL-terminated)
56func war_blobkey(cid: *u8, out: *u8) -> i64 {
57 var o: i64 = 0
58 o = war_cat(out, o, "wikiblob:" as *u8)
59 o = war_cat(out, o, cid)
60 out[o] = 0 as u8
61 return o
62}
63
64// build "wikicid:<slug>" (slug NUL-terminated) into out (NUL-terminated)
65func war_cidkey(slug: *u8, out: *u8) -> i64 {
66 var o: i64 = 0
67 o = war_cat(out, o, "wikicid:" as *u8)
68 o = war_cat(out, o, slug)
69 out[o] = 0 as u8
70 return o
71}
72
73// ---- CID of a page's raw bytes ----
74// The archive key generator. cid_of() canonicalizes nothing here on purpose:
75// a page body is ALREADY a single byte string (not a multi-field record), so
76// its CID is just nxc1-+sha256(body) -- exactly the content address we want.
77// Writes a 69-byte NUL-terminated "nxc1-<64hex>" into cidout; returns its len.
78func war_cid_of_body(body: *u8, body_n: i64, cidout: *u8) -> i64 {
79 return cid_of(body, body_n, cidout)
80}
81
82// ---- ARCHIVE one page into an open writer (caller commits) ----
83// Computes cid; appends wikiblob:<cid> -> body and wikicid:<slug> -> cid.
84// Idempotent by content: same body -> same cid -> same blob key (the seg_store
85// LAST-writer-wins index means a re-add of identical bytes adds no NEW logical
86// blob the reader can distinguish; the cid pointer simply re-states the same
87// value). cidout (>=72) receives the page's cid so the caller can prove
88// retrieval. Returns 0 ok, <0 on a writer-full error from ss_add.
89func war_archive_page(w: *i64, slug: *u8, body: *u8, body_n: i64, cidout: *u8) -> i64 {
90 let cidn: i64 = war_cid_of_body(body, body_n, cidout)
91 let bk: *u8 = sys_mmap(128)
92 war_blobkey(cidout, bk)
93 // wikiblob:<cid> -> the immutable page bytes (the no-rot anchor)
94 let r1: i64 = ss_add(w, 1, bk, body, body_n)
95 if r1 < 0 { return r1 }
96 // wikicid:<slug> -> current cid (pointer; cid is NUL-terminated, len cidn)
97 let ck: *u8 = sys_mmap(128)
98 war_cidkey(slug, ck)
99 let r2: i64 = ss_add(w, 1, ck, cidout, cidn)
100 if r2 < 0 { return r2 }
101 return 0
102}
103
104// ---- NO-LOSS RETRIEVAL: read a page back by its CID ----
105// Reads wikiblob:<cid> from the committed store. Works even if the slug
106// pointer is gone (proves the content is never lost -- the CID alone resolves).
107// Returns the byte length (>=0) with the bytes pointer in ptrout[0], or
108// -1 absent, 0 if tombstoned. cap = manifest segment-scan cap (>= live segs).
109// prefix = the store prefix (production callers pass WAR_PREFIX).
110func war_get_by_cid(prefix: *u8, cid: *u8, ptrout: *i64, cap: i64) -> i64 {
111 let bk: *u8 = sys_mmap(128)
112 war_blobkey(cid, bk)
113 let lenout: *i64 = sys_mmap(16) as *i64
114 let r: i64 = ss_get_cap(prefix, bk, ptrout, lenout, cap)
115 if r == 1 { return lenout[0] }
116 if r == 0 { return 0 }
117 return 0 - 1
118}
119
120// ---- look up a slug's current CID from the committed store ----
121// Returns the cid length (>=0) with cid bytes copied (NUL-terminated) into
122// cidout, or -1 if the slug pointer is absent/dropped. prefix = store prefix.
123func war_cid_of_slug(prefix: *u8, slug: *u8, cidout: *u8, cap: i64) -> i64 {
124 let ck: *u8 = sys_mmap(128)
125 war_cidkey(slug, ck)
126 let pp: *i64 = sys_mmap(16) as *i64
127 let ll: *i64 = sys_mmap(16) as *i64
128 let r: i64 = ss_get_cap(prefix, ck, pp, ll, cap)
129 if r != 1 { return 0 - 1 }
130 let src: *u8 = pp[0] as *u8
131 let n: i64 = ll[0]
132 var i: i64 = 0
133 while i < n { cidout[i] = src[i]; i = i + 1 }
134 cidout[n] = 0 as u8
135 return n
136}
137
138// does slug X exist in the committed archive? (its wikicid:<slug> pointer is
139// present). This IS the rot resolver: a [[X]] link resolves iff X was archived.
140// NishiLang has no function-pointer params, so the resolver is this concrete
141// store lookup (the archive is the source of truth for "what slugs exist").
142// Returns 1 if present, 0 if absent. prefix = store prefix; cap = scan cap.
143func war_slug_exists(prefix: *u8, slug: *u8, slug_n: i64, cap: i64) -> i64 {
144 let ck: *u8 = sys_mmap(128)
145 war_cidkey(slug, ck)
146 let pp: *i64 = sys_mmap(16) as *i64
147 let ll: *i64 = sys_mmap(16) as *i64
148 let r: i64 = ss_get_cap(prefix, ck, pp, ll, cap)
149 if r == 1 { return 1 }
150 return 0
151}
152
153// ===== LINK-INTEGRITY (ROT) AUDIT ==============================================
154// For a page body, scan internal [[X]] wikilinks (SAME scan shape as
155// nx_wiki_doc_render.nx's preprocessor) and external http(s):// links.
156// - internal [[X]]: target slug X must exist in the committed archive
157// (war_slug_exists). Unresolved internal link => +1 rot.
158// - external http(s)://: COUNTED as external_pending (snapshotting them needs
159// network egress = a follow-on; we do NOT fetch).
160//
161// out[] (caller-provided, >=4 i64): out[0]=internal_total out[1]=rot
162// out[2]=external_pending out[3]=resolved_internal
163// The first BROKEN target (if any) is copied NUL-terminated into brokbuf
164// (>=64) for the gate's evidence; brokbuf[0]==0 if none. Returns rot count.
165// prefix = store prefix; cap = segment-scan cap handed to war_slug_exists.
166func nx_wiki_rot_audit_page(prefix: *u8, body: *u8, body_n: i64, cap: i64,
167 out: *i64, brokbuf: *u8) -> i64 {
168 out[0] = 0
169 out[1] = 0
170 out[2] = 0
171 out[3] = 0
172 brokbuf[0] = 0 as u8
173 let namebuf: *u8 = sys_mmap(256)
174 var i: i64 = 0
175 // bounded outer walk (one pass over the body)
176 while i < body_n {
177 var advanced: i64 = 0
178 // ---- internal [[X]] ----
179 if i + 1 < body_n {
180 if body[i] == (91 as u8) { // '['
181 if body[i + 1] == (91 as u8) { // second '['
182 let start_name: i64 = i + 2
183 // PHASE 1: pure scan for the closing "]]" (no side effects;
184 // separating scan from processing avoids the break-in-loop
185 // codegen-desync the doc-render preprocessor warns about).
186 var end_name: i64 = start_name
187 var found: i64 = 0
188 var scan: i64 = start_name
189 while scan < body_n - 1 {
190 if found == 0 {
191 if body[scan] == (93 as u8) { // ']'
192 if body[scan + 1] == (93 as u8) { // ']'
193 found = 1
194 end_name = scan
195 }
196 }
197 }
198 scan = scan + 1
199 }
200 // PHASE 2: process once, after the scan
201 if found == 1 {
202 let name_n: i64 = end_name - start_name
203 if name_n > 0 {
204 if name_n < 255 {
205 var c: i64 = 0
206 while c < name_n { namebuf[c] = body[start_name + c]; c = c + 1 }
207 namebuf[name_n] = 0 as u8
208 out[0] = out[0] + 1
209 let ok: i64 = war_slug_exists(prefix, namebuf, name_n, cap)
210 if ok == 1 { out[3] = out[3] + 1 }
211 if ok != 1 {
212 out[1] = out[1] + 1
213 // record FIRST broken target for evidence
214 if brokbuf[0] == (0 as u8) {
215 var b2: i64 = 0
216 while b2 < name_n { brokbuf[b2] = namebuf[b2]; b2 = b2 + 1 }
217 brokbuf[name_n] = 0 as u8
218 }
219 }
220 }
221 }
222 // resume AFTER the closing ]]
223 i = end_name + 2
224 advanced = 1
225 }
226 if found == 0 {
227 // unterminated [[ -- step one byte
228 i = i + 1
229 advanced = 1
230 }
231 }
232 }
233 }
234 // ---- external http(s):// ----
235 if advanced == 0 {
236 if i + 6 < body_n {
237 // match "http" then "s?://"
238 if body[i] == (104 as u8) { // h
239 if body[i + 1] == (116 as u8) { // t
240 if body[i + 2] == (116 as u8) { // t
241 if body[i + 3] == (112 as u8) { // p
242 // optional 's', then "://"
243 var j: i64 = i + 4
244 if body[j] == (115 as u8) { j = j + 1 } // s
245 if j + 2 < body_n {
246 if body[j] == (58 as u8) { // ':'
247 if body[j + 1] == (47 as u8) { // '/'
248 if body[j + 2] == (47 as u8) { // '/'
249 out[2] = out[2] + 1
250 // skip past the scheme so we don't recount
251 i = j + 3
252 advanced = 1
253 }
254 }
255 }
256 }
257 }
258 }
259 }
260 }
261 }
262 }
263 if advanced == 0 { i = i + 1 }
264 }
265 return out[1]
266}