nx_asset_catalog.nx source
↩ module page · 176 lines · 8166 B
1// nx_asset_catalog.nx -- UNIVERSAL ORGANIZATION TOOLING arc, R1.
2//
3// Turns the R0 content-addressed asset RECORDS (nx_asset_record) into an actual CATALOG: the single
4// source-of-truth inventory (org_research.tsv CONFIRMED dam-ssot-catalog) with ingest / list /
5// get-by-CID, IDEMPOTENT re-ingest, and DEDUP-by-CID (content-addressed-dedup: identical content ->
6// exactly ONE record, never a second copy). This is PURE COMPOSITION -- zero new storage/identity:
7// * record identity + bytes -> nx_asset_record (ar_cid -> the record's CID = its catalog key)
8// * durable append-only KV -> nx_seg_store (ss_open snapshot + ss_hget / ss_begin+ss_add+
9// ss_commit), same store the records already use
10//
11// THE CATALOG = the set of record CIDs held in ONE seg_store. Membership/order is tracked by a
12// single reserved index key whose value is the newline-joined list of member CIDs (one append per
13// NEW record). Because seg_store is content-addressed + last-writer-wins per key:
14// * a record is stored under its CID -> re-storing identical bytes is a no-op by content
15// * the index key (CAT_IDX) is re-committed -> the newest segment shadows the old list
16// so DEDUP and IDEMPOTENCE both fall out of the substrate, and ADDITIVE law #13 holds (a DUP writes
17// nothing; the prior record + index history stay on disk, never deleted).
18//
19// LAWS: #9 single responsibility (each cat_* does ONE thing); #10 idempotent (re-ingest == DUP,
20// count unchanged); #13 additive (dedup COLLAPSES to one record, never destroys history); #15 DRY
21// (composes R0 + the store, no copied boilerplate). No hardware/persistent-firmware writes (#26).
22//
23// API:
24// cat_ingest(prefix, bytes, n, out2) -> 0 ok / <0 err; out2[0]=status (CAT_NEW|CAT_DUP),
25// out2[1]=unused; the record's CID is written into `cidout`
26// via the cat_ingest_cid variant (see below).
27// cat_get(prefix, cid, ptrout, lenout) -> 1 found / 0 tombstoned / -1 absent (ss_hget semantics)
28// cat_list(prefix, out_cids, cap) -> count of CIDs written into out_cids[] (ptrs to NUL-term)
29// cat_count(prefix) -> number of records in the catalog (>=0)
30// license_tier: ORIGINAL
31import "nx_syscalls.nx"
32import "nx_canon_cid.nx"
33import "nx_seg_store.nx"
34import "nx_asset_record.nx"
35const K_MAGIC_4096: i64 = 4096
36
37// status codes (data-driven, no magic numbers buried at call sites -- law #11)
38func CAT_NEW() -> i64 { return 1 }
39func CAT_DUP() -> i64 { return 2 }
40
41// the ONE reserved index key (a CID-shaped key cannot collide with it: no real CID is this string).
42func cat_idx_key() -> *u8 { return "__nx_catalog_index__\x00" as *u8 }
43
44func cat_strlen(s: *u8) -> i64 {
45 var n: i64 = 0
46 while s[n] != (0 as u8) { n = n + 1 }
47 return n
48}
49
50// ---- the catalog index = newline-joined CID list, read from / written to the reserved key ----
51
52// Load the current index list into out_cids[] (each a fresh NUL-terminated copy); returns count.
53// Reads from a CONSISTENT snapshot handle (caller opens once, so list+membership agree). The index
54// value is the raw blob `cid\ncid\n...`; we split on '\n'. Absent index (fresh catalog) -> 0.
55func cat_idx_load(h: *i64, out_cids: *i64, cap: i64) -> i64 {
56 let pp: *i64 = sys_mmap(16) as *i64
57 let ll: *i64 = sys_mmap(16) as *i64
58 let r: i64 = ss_hget(h, cat_idx_key(), pp, ll)
59 if r != 1 { return 0 }
60 let b: *u8 = pp[0] as *u8
61 let sz: i64 = ll[0]
62 var cnt: i64 = 0
63 var i: i64 = 0
64 var ls: i64 = 0
65 while i < sz {
66 if b[i] == (10 as u8) {
67 let line_len: i64 = i - ls
68 if line_len > 0 {
69 if cnt < cap {
70 let name: *u8 = sys_mmap(line_len + 8)
71 var t: i64 = 0
72 while t < line_len { name[t] = b[ls + t]; t = t + 1 }
73 name[t] = 0 as u8
74 out_cids[cnt] = name as i64
75 cnt = cnt + 1
76 }
77 }
78 ls = i + 1
79 }
80 i = i + 1
81 }
82 return cnt
83}
84
85// Is `cid` already a member of the catalog index loaded in cids[0..n)? 1 yes / 0 no.
86func cat_idx_has(cids: *i64, n: i64, cid: *u8) -> i64 {
87 var i: i64 = 0
88 while i < n {
89 if ar_streq(cids[i] as *u8, cid) == 1 { return 1 }
90 i = i + 1
91 }
92 return 0
93}
94
95// ---- INGEST: dedup-by-CID, idempotent ----
96// Compute the record's CID; if already in the catalog -> status=CAT_DUP, write NOTHING (dedup +
97// idempotent). Otherwise commit the record under its CID AND the index key extended by this CID, in
98// ONE segment (atomic by the store's manifest-rename commit point). cidout receives the NUL-term CID
99// (caller buf >= 72). out2[0] = status. Returns 0 on success, <0 on a store error.
100func cat_ingest_cid(prefix: *u8, bytes: *u8, n: i64, cidout: *u8, out2: *i64) -> i64 {
101 ar_cid(bytes, n, cidout)
102 // snapshot the catalog ONCE: membership test + current index list are then mutually consistent.
103 let h: *i64 = ss_open(prefix)
104 let cap: i64 = K_MAGIC_4096
105 let cids: *i64 = sys_mmap(8 * cap) as *i64
106 var ncur: i64 = 0
107 if (h as i64) != 0 { ncur = cat_idx_load(h, cids, cap) }
108 // DUP: present already -> no write at all (content-addressed dedup, idempotent re-ingest).
109 if (h as i64) != 0 {
110 if cat_idx_has(cids, ncur, cidout) == 1 {
111 out2[0] = CAT_DUP()
112 return 0
113 }
114 }
115 // NEW: build the extended index blob = old list (ncur lines) + this CID + '\n'.
116 // size = sum of existing cid lens (+\n each) + new cid len (+\n).
117 var newsz: i64 = 0
118 var i: i64 = 0
119 while i < ncur { newsz = newsz + cat_strlen(cids[i] as *u8) + 1; i = i + 1 }
120 let clen: i64 = cat_strlen(cidout)
121 newsz = newsz + clen + 1
122 let idxb: *u8 = sys_mmap(newsz + 16)
123 var o: i64 = 0
124 i = 0
125 while i < ncur {
126 let cp: *u8 = cids[i] as *u8
127 var t: i64 = 0
128 while cp[t] != (0 as u8) { idxb[o] = cp[t]; o = o + 1; t = t + 1 }
129 idxb[o] = 10 as u8; o = o + 1
130 i = i + 1
131 }
132 var t2: i64 = 0
133 while cidout[t2] != (0 as u8) { idxb[o] = cidout[t2]; o = o + 1; t2 = t2 + 1 }
134 idxb[o] = 10 as u8; o = o + 1
135 // commit BOTH the record (keyed by its CID) and the new index (reserved key) in one segment.
136 let w: *i64 = ss_begin()
137 if ss_add(w, 1, cidout, bytes, n) != 0 { return 0 - 20 }
138 if ss_add(w, 1, cat_idx_key(), idxb, o) != 0 { return 0 - 21 }
139 let segid: i64 = sys_now_ms()
140 let rc: i64 = ss_commit(prefix, w, segid)
141 if rc != 0 { return rc }
142 out2[0] = CAT_NEW()
143 return 0
144}
145
146// Convenience: ingest without needing the CID back (status only).
147func cat_ingest(prefix: *u8, bytes: *u8, n: i64, out2: *i64) -> i64 {
148 let cid: *u8 = sys_mmap(128)
149 return cat_ingest_cid(prefix, bytes, n, cid, out2)
150}
151
152// ---- GET: retrieve a record's bytes by its CID ----
153// 1 found (ptrout/lenout set), 0 tombstoned, -1 absent. Reuses ar_get_by_cid (ss_open+ss_hget).
154func cat_get(prefix: *u8, cid: *u8, ptrout: *i64, lenout: *i64) -> i64 {
155 return ar_get_by_cid(prefix, cid, ptrout, lenout)
156}
157
158// ---- LIST: the set of member CIDs ----
159// Writes up to `cap` CID pointers (fresh NUL-term copies) into out_cids[]; returns the count.
160func cat_list(prefix: *u8, out_cids: *i64, cap: i64) -> i64 {
161 // ss_open_cached (seq905/962 class fix, seq1347 migration, 2026-07-30) -- ss_open reads the whole
162 // store into anon RAM and nothing frees it, so a repeated lister leaks it every call. SAFE: the
163 // handle is consumed by cat_idx_load and never held across a re-entrant store call.
164 let h: *i64 = ss_open_cached(prefix)
165 if (h as i64) == 0 { return 0 }
166 return cat_idx_load(h, out_cids, cap)
167}
168
169// ---- COUNT: number of records in the catalog ----
170func cat_count(prefix: *u8) -> i64 {
171 let h: *i64 = ss_open_cached(prefix) // seq1347: same class fix as cat_list
172 if (h as i64) == 0 { return 0 }
173 let cap: i64 = K_MAGIC_4096
174 let cids: *i64 = sys_mmap(8 * cap) as *i64
175 return cat_idx_load(h, cids, cap)
176}