nx_mvault_coll.nx source
↩ module page · 296 lines · 12853 B
1// nx_mvault_coll.nx -- the vault's COLLECTION / PROVENANCE axis.
2//
3// WHY THIS EXISTS (operator 2026-07-31: "though we split by media i want to be
4// able to look by site or album or other tagged or original source groupings"):
5// the core vault record is {class,type,source,mls,ref} -- five fields, none of
6// which can say WHICH ALBUM or WHICH SITE an item came from. `source` is a
7// single flat string ("cam"/"gen"). So grouping by album/site was not merely
8// missing from the UI, it was UNREPRESENTABLE IN THE SCHEMA.
9//
10// ★★★★★ THE ORDERING LAW THIS ENFORCES: build the grouping axis BEFORE the
11// downloader. Ingesting albums into a schema that cannot remember their album
12// would repeat the 07-30 classifier disaster exactly -- 53,567 files typed
13// `unknown` because the axis was broken while the bulk migration ran, then a
14// 35,615-record backfill to repair it. An axis added AFTER the ingest is a
15// backfill; added BEFORE, it is free.
16//
17// DESIGN
18// A collection is a NAMED GROUPING with a kind: site | album | source | set.
19// Membership is MANY-TO-MANY -- one file can sit in several albums, and an
20// album holds many files -- so it is a RELATION, not a field on the item.
21// That is also why this did not belong on the core record: a `album` field
22// would have permitted exactly one, and the first re-post would have broken it.
23//
24// colid = CID of the canonical {kind,site,key} record. DERIVED, never minted:
25// the same album on the same site always yields the same colid, so declaring
26// it twice is a no-op and re-running an ingest cannot fork a duplicate
27// collection (rule 10, idempotent BY CONSTRUCTION rather than by remembering).
28//
29// Item identity is untouched. The vault keys records by CONTENT CID, so this
30// plane only ever ADDS relations -- dedup, history and every existing record
31// keep working unchanged (rule 13, additive-only).
32//
33// Lives in its OWN store prefix. The mvault mv:ids index is already ~10.7 MiB
34// at 160,269 items; folding memberships into it would bloat the very index
35// that just overflowed (see nx_registry.nx 2026-07-31).
36// license_tier: ORIGINAL
37
38import "nx_syscalls.nx"
39import "nx_canon_cid.nx"
40import "nx_registry.nx"
41import "nx_seg_store.nx"
42
43const MVC_PFX: *u8 = "knowledge/store/mvcoll-\x00"
44const MVC_KP: *u8 = "col:\x00"
45const MVC_IDS: *u8 = "col:ids\x00"
46
47// Declared envelopes. Named and fail-closed rather than buried: ss_add2 REFUSES
48// when a write would exceed the writer buffer, so exceeding these produces a
49// refusal, never a truncated commit.
50const MVC_REC_CAP: i64 = 8192
51const MVC_KEY_CAP: i64 = 512
52const MVC_IC_CAP: i64 = 65536
53
54func mvc_strlen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
55func mvc_cat(dst: *u8, off: i64, s: *u8) -> i64 { var i: i64=0; while s[i]!=(0 as u8){dst[off+i]=s[i]; i=i+1} return off+i }
56
57// ---- IDENTITY -------------------------------------------------------------
58// colid = cid(canon{kind,site,key}). canon_encode sorts by key, so field order
59// here cannot change the id.
60func mvc_colid(kind: *u8, site: *u8, key: *u8, cidout: *u8) -> i64 {
61 let keys: *i64 = sys_mmap(8 * 3) as *i64
62 let vals: *i64 = sys_mmap(8 * 3) as *i64
63 let k_kind: *u8 = "kind" as *u8
64 let k_site: *u8 = "site" as *u8
65 let k_key: *u8 = "key" as *u8
66 keys[0] = k_kind as i64; vals[0] = kind as i64
67 keys[1] = k_site as i64; vals[1] = site as i64
68 keys[2] = k_key as i64; vals[2] = key as i64
69 let buf: *u8 = sys_mmap(MVC_REC_CAP)
70 let n: i64 = canon_encode(keys, vals, 3, buf)
71 let r: i64 = cid_of(buf, n, cidout)
72 sys_munmap(keys as *u8, 8 * 3)
73 sys_munmap(vals as *u8, 8 * 3)
74 sys_munmap(buf, MVC_REC_CAP)
75 return r
76}
77
78func mvc_rec_build(out: *u8, kind: *u8, site: *u8, key: *u8, name: *u8, url: *u8, parent: *u8) -> i64 {
79 let keys: *i64 = sys_mmap(8 * 6) as *i64
80 let vals: *i64 = sys_mmap(8 * 6) as *i64
81 let k_kind: *u8 = "kind" as *u8
82 let k_site: *u8 = "site" as *u8
83 let k_key: *u8 = "key" as *u8
84 let k_name: *u8 = "name" as *u8
85 let k_url: *u8 = "url" as *u8
86 let k_parent: *u8 = "parent" as *u8
87 keys[0] = k_kind as i64; vals[0] = kind as i64
88 keys[1] = k_site as i64; vals[1] = site as i64
89 keys[2] = k_key as i64; vals[2] = key as i64
90 keys[3] = k_name as i64; vals[3] = name as i64
91 keys[4] = k_url as i64; vals[4] = url as i64
92 keys[5] = k_parent as i64; vals[5] = parent as i64
93 let rl: i64 = canon_encode(keys, vals, 6, out)
94 sys_munmap(keys as *u8, 8 * 6)
95 sys_munmap(vals as *u8, 8 * 6)
96 return rl
97}
98
99// Declare a collection. Idempotent: same {kind,site,key} -> same colid, and
100// reg_put is content-idempotent, so a repeat declaration mints NO segment.
101func mvc_declare_pfx(prefix: *u8, kind: *u8, site: *u8, key: *u8, name: *u8, url: *u8, parent: *u8, cidout: *u8) -> i64 {
102 mvc_colid(kind, site, key, cidout)
103 let rec: *u8 = sys_mmap(MVC_REC_CAP)
104 let rl: i64 = mvc_rec_build(rec, kind, site, key, name, url, parent)
105 let rc: i64 = reg_put(prefix, MVC_KP, MVC_IDS, cidout, rec, rl)
106 sys_munmap(rec, MVC_REC_CAP)
107 return rc
108}
109
110func mvc_get_pfx(prefix: *u8, colid: *u8, ptrout: *i64, lenout: *i64) -> i64 {
111 return reg_get(prefix, MVC_KP, colid, ptrout, lenout)
112}
113
114// ---- MEMBERSHIP KEYS ------------------------------------------------------
115// ci:<colid> -> the items in a collection (forward)
116// ic:<item> -> the collections holding an item (reverse)
117// Both directions are kept because both questions get asked: "show me this
118// album" and "where did this file come from". Deriving one from the other would
119// mean scanning every collection to answer the second.
120func mvc_ci_key(out: *u8, colid: *u8) -> i64 {
121 var o: i64 = mvc_cat(out, 0, "ci:" as *u8)
122 o = mvc_cat(out, o, colid)
123 out[o] = 0 as u8
124 return o
125}
126func mvc_ic_key(out: *u8, item: *u8) -> i64 {
127 var o: i64 = mvc_cat(out, 0, "ic:" as *u8)
128 o = mvc_cat(out, o, item)
129 out[o] = 0 as u8
130 return o
131}
132
133// count newline-terminated entries in buf[0..n)
134func mvc_count_lines(buf: *u8, n: i64) -> i64 {
135 var c: i64 = 0
136 var i: i64 = 0
137 while i < n { if buf[i] == (10 as u8) { c = c + 1 } i = i + 1 }
138 return c
139}
140
141// ---- BATCH MEMBERSHIP -----------------------------------------------------
142// Add MANY items to ONE collection in ONE segment.
143//
144// ★ WHY BATCHED: reg_put mints a NEW segment per call AND rewrites the whole
145// index each time. Adding a 300-file album one reg_put at a time would mint 300
146// segments and make every later ss_get walk all of them -- the exact quadratic
147// that took tools/list to 10s per page and that the mvault backfill had to be
148// re-cut to avoid. One album = one segment.
149//
150// `items` is newline-separated item CIDs. Returns items newly linked, or -1.
151func mvc_batch_add_pfx(prefix: *u8, colid: *u8, items: *u8, items_n: i64) -> i64 {
152 let cikey: *u8 = sys_mmap(MVC_KEY_CAP)
153 let cikl: i64 = mvc_ci_key(cikey, colid)
154 let colid_l: i64 = mvc_strlen(colid)
155
156 let po: *i64 = sys_mmap(16) as *i64
157 let lo: *i64 = sys_mmap(16) as *i64
158 var have: i64 = 0
159 if ss_get(prefix, cikey, po, lo) >= 0 { have = lo[0] }
160
161 let cand: i64 = mvc_count_lines(items, items_n) + 1
162
163 // merged forward list = existing + the new ids not already present
164 let merged: *u8 = sys_mmap(have + items_n + 16)
165 var mlen: i64 = 0
166 if have > 0 {
167 let src: *u8 = po[0] as *u8
168 var c: i64 = 0
169 while c < have { merged[mlen] = src[c]; mlen = mlen + 1; c = c + 1 }
170 }
171
172 // per-item reverse payloads, staged then written in one commit
173 let icbuf: *u8 = sys_mmap((MVC_IC_CAP + 256) * cand)
174 let icoff: *i64 = sys_mmap(8 * cand) as *i64
175 let iclen: *i64 = sys_mmap(8 * cand) as *i64
176 let ickey: *i64 = sys_mmap(8 * cand) as *i64
177 var icn: i64 = 0
178 var icw: i64 = 0
179 var added: i64 = 0
180
181 var ls: i64 = 0
182 var i: i64 = 0
183 while i <= items_n {
184 var eol: i64 = 0
185 if i == items_n { eol = 1 } else { if items[i] == (10 as u8) { eol = 1 } }
186 if eol == 1 {
187 let idlen: i64 = i - ls
188 if idlen > 0 {
189 let id: *u8 = sys_mmap(MVC_KEY_CAP)
190 var q: i64 = 0
191 while q < idlen { id[q] = items[ls + q]; q = q + 1 }
192 id[idlen] = 0 as u8
193
194 if reg_id_present(merged, mlen, id, idlen) == 0 {
195 var w: i64 = 0
196 while w < idlen { merged[mlen] = id[w]; mlen = mlen + 1; w = w + 1 }
197 merged[mlen] = 10 as u8; mlen = mlen + 1
198 added = added + 1
199
200 // reverse: append this colid to ic:<id> if not already there
201 let ik: *u8 = sys_mmap(MVC_KEY_CAP)
202 mvc_ic_key(ik, id)
203 let ipo: *i64 = sys_mmap(16) as *i64
204 let ilo: *i64 = sys_mmap(16) as *i64
205 var ihave: i64 = 0
206 if ss_get(prefix, ik, ipo, ilo) >= 0 { ihave = ilo[0] }
207 let base: i64 = icw
208 if ihave > 0 {
209 if ihave <= MVC_IC_CAP {
210 let isrc: *u8 = ipo[0] as *u8
211 var z: i64 = 0
212 while z < ihave { icbuf[icw] = isrc[z]; icw = icw + 1; z = z + 1 }
213 }
214 }
215 var dup: i64 = reg_id_present(((icbuf as i64) + base) as *u8, icw - base, colid, colid_l)
216 if dup == 0 {
217 var y: i64 = 0
218 while y < colid_l { icbuf[icw] = colid[y]; icw = icw + 1; y = y + 1 }
219 icbuf[icw] = 10 as u8; icw = icw + 1
220 ickey[icn] = ik as i64
221 icoff[icn] = base
222 iclen[icn] = icw - base
223 icn = icn + 1
224 } else { icw = base }
225 }
226 }
227 ls = i + 1
228 }
229 i = i + 1
230 }
231
232 if added == 0 { return 0 }
233
234 var cap: i64 = mlen + cikl + 64
235 var s: i64 = 0
236 while s < icn { cap = cap + iclen[s] + MVC_KEY_CAP + 32; s = s + 1 }
237
238 let w2: *i64 = ss_begin_cap(cap)
239 var bad: i64 = 0
240 if ss_add(w2, 1, cikey, merged, mlen) < 0 { bad = 1 }
241 var t: i64 = 0
242 while t < icn {
243 let kp: *u8 = ickey[t] as *u8
244 let vp: *u8 = ((icbuf as i64) + icoff[t]) as *u8
245 if ss_add(w2, 1, kp, vp, iclen[t]) < 0 { bad = 1 }
246 t = t + 1
247 }
248 if bad == 1 { return 0 - 1 }
249
250 let segid: i64 = ss_next_segid(prefix)
251 if ss_commit(prefix, w2, segid) < 0 { return 0 - 1 }
252 return added
253}
254
255// ---- READ SIDE ------------------------------------------------------------
256// All bounded and REFUSING (-1) rather than truncating -- same contract as
257// reg_index after the 2026-07-31 root fix, so no caller can mistake a partial
258// album for a complete one.
259func mvc_items_pfx(prefix: *u8, colid: *u8, out: *u8, cap: i64) -> i64 {
260 let k: *u8 = sys_mmap(MVC_KEY_CAP)
261 mvc_ci_key(k, colid)
262 return reg_index(prefix, k, out, cap)
263}
264
265func mvc_of_item_pfx(prefix: *u8, item: *u8, out: *u8, cap: i64) -> i64 {
266 let k: *u8 = sys_mmap(MVC_KEY_CAP)
267 mvc_ic_key(k, item)
268 return reg_index(prefix, k, out, cap)
269}
270
271func mvc_list_pfx(prefix: *u8, out: *u8, cap: i64) -> i64 {
272 return reg_index(prefix, MVC_IDS, out, cap)
273}
274
275func mvc_count_pfx(prefix: *u8, colid: *u8) -> i64 {
276 let k: *u8 = sys_mmap(MVC_KEY_CAP)
277 mvc_ci_key(k, colid)
278 let po: *i64 = sys_mmap(16) as *i64
279 let lo: *i64 = sys_mmap(16) as *i64
280 if reg_index_open(prefix, k, po, lo) == 0 { return 0 }
281 return mvc_count_lines(po[0] as *u8, lo[0])
282}
283
284// ---- production-prefix convenience wrappers -------------------------------
285// Same shape the rest of the ecosystem uses (persona_list_pfx / persona_list,
286// svc_list_pfx / svc_list): the _pfx form is the testable one, these bind the
287// live store so callers do not repeat the prefix.
288func mvc_declare(kind: *u8, site: *u8, key: *u8, name: *u8, url: *u8, parent: *u8, cidout: *u8) -> i64 {
289 return mvc_declare_pfx(MVC_PFX, kind, site, key, name, url, parent, cidout)
290}
291func mvc_get(colid: *u8, ptrout: *i64, lenout: *i64) -> i64 { return mvc_get_pfx(MVC_PFX, colid, ptrout, lenout) }
292func mvc_batch_add(colid: *u8, items: *u8, items_n: i64) -> i64 { return mvc_batch_add_pfx(MVC_PFX, colid, items, items_n) }
293func mvc_items(colid: *u8, out: *u8, cap: i64) -> i64 { return mvc_items_pfx(MVC_PFX, colid, out, cap) }
294func mvc_of_item(item: *u8, out: *u8, cap: i64) -> i64 { return mvc_of_item_pfx(MVC_PFX, item, out, cap) }
295func mvc_list(out: *u8, cap: i64) -> i64 { return mvc_list_pfx(MVC_PFX, out, cap) }
296func mvc_count(colid: *u8) -> i64 { return mvc_count_pfx(MVC_PFX, colid) }