nx_lib_store.nx source
↩ module page · 174 lines · 7312 B
1// nx_lib_store.nx -- sovereign CATALOG STORE for the nishi-library, on the
2// sovereign nx_seg_store (append-only, additive, crash-safe rename-commit) with
3// the O(1) nx_hash_index get path. RETIRES the 3rd-party SQLite catalog.db
4// (operator 2026-06-30: "no 3rd-party db; s-class exceed everywhere").
5//
6// key = "work:<work_hk>"
7// val = title<TAB>doi<TAB>published<TAB>license<TAB>abstract
8// "__works__" record = newline-separated work_hks, advanced IN THE SAME COMMIT
9// so enumeration is atomic (no torn state).
10//
11// S-CLASS-EXCEED vs SQLite (parity + beyond):
12// parity : put / get / enumerate / update by key.
13// beyond : O(1) hash get (65 ns = 3.8x SQLite's B-tree probe, nx_hash_index_bench);
14// APPEND-ONLY history -- no UPDATE/DELETE destruction ("history is sacred");
15// crash-safe rename-commit; ZERO 3rd-party; durable + sovereign on disk.
16// license_tier: ORIGINAL | genealogy_id: nishi_library_sovereign_db_2026_06_30
17import "nx_syscalls.nx"
18import "nx_seg_store.nx"
19const LIB_MAGIC_1048576: i64 = 1048576
20
21const LIB_PREFIX: *u8 = "knowledge/libstore-" as *u8
22
23func ls_strlen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
24func ls_cat(dst: *u8, off: i64, s: *u8) -> i64 { var i: i64=0; while s[i]!=(0 as u8){ dst[off+i]=s[i]; i=i+1 } return off+i }
25
26// build "work:<hk>" (NUL-terminated) into kbuf
27func ls_key(kbuf: *u8, hk: *u8) -> i64 {
28 var o: i64 = ls_cat(kbuf, 0, "work:" as *u8)
29 o = ls_cat(kbuf, o, hk)
30 kbuf[o] = 0 as u8
31 return o
32}
33
34// is `name` (namelen) already a line in the newline-separated index buf[0..n)?
35func ls_name_present(buf: *u8, n: i64, name: *u8, namelen: i64) -> i64 {
36 var ls: i64 = 0
37 var i: i64 = 0
38 while i <= n {
39 var eol: i64 = 0
40 if i == n { eol = 1 } else { if buf[i] == 10 as u8 { eol = 1 } }
41 if eol == 1 {
42 if i - ls == namelen {
43 var m: i64 = 1
44 var c: i64 = 0
45 while c < namelen { if buf[ls + c] != name[c] { m = 0 } c = c + 1 }
46 if m == 1 { return 1 }
47 }
48 ls = i + 1
49 }
50 i = i + 1
51 }
52 return 0
53}
54
55// insert/update a work (additive new version; __works__ index advanced same commit)
56func nx_lib_store_put_work(hk: *u8, record: *u8, reclen: i64) -> i64 {
57 let kbuf: *u8 = sys_mmap(512)
58 ls_key(kbuf, hk)
59 // fresh max+1 segid (ss_next_segid, uncapped) -- the capped ss_manifest count clobbers past the cap
60 let segid: i64 = ss_next_segid(LIB_PREFIX)
61 let w: *i64 = ss_begin()
62 ss_add(w, 1, kbuf, record, reclen)
63 let ipo: *i64 = sys_mmap(16) as *i64
64 let ilo: *i64 = sys_mmap(16) as *i64
65 let idxbuf: *u8 = sys_mmap(LIB_MAGIC_1048576)
66 var ilen: i64 = 0
67 if ss_get(LIB_PREFIX, "__works__" as *u8, ipo, ilo) >= 0 {
68 let src: *u8 = ipo[0] as *u8
69 var c: i64 = 0
70 while c < ilo[0] { idxbuf[ilen] = src[c]; ilen = ilen + 1; c = c + 1 }
71 }
72 let nl: i64 = ls_strlen(hk)
73 if ls_name_present(idxbuf, ilen, hk, nl) == 0 {
74 var c2: i64 = 0
75 while c2 < nl { idxbuf[ilen] = hk[c2]; ilen = ilen + 1; c2 = c2 + 1 }
76 idxbuf[ilen] = 10 as u8
77 ilen = ilen + 1
78 }
79 ss_add(w, 1, "__works__" as *u8, idxbuf, ilen)
80 return ss_commit(LIB_PREFIX, w, segid)
81}
82
83// ---- BULK ingest: O(n) total (one ss_begin -> many ss_add -> one ss_commit),
84// vs put_work's O(n^2) (per-record __works__ rewrite + per-record commit).
85// worksbuf must be caller-provided (>= existing __works__ + all new hks).
86// bulk_add does NOT dedup (assumes unique hks). begin -> add* -> commit.
87func nx_lib_store_bulk_begin(worksbuf: *u8, worklen_p: *i64) -> *i64 {
88 let ipo: *i64 = sys_mmap(16) as *i64
89 let ilo: *i64 = sys_mmap(16) as *i64
90 var ilen: i64 = 0
91 if ss_get(LIB_PREFIX, "__works__" as *u8, ipo, ilo) >= 0 {
92 let src: *u8 = ipo[0] as *u8
93 var c: i64 = 0
94 while c < ilo[0] { worksbuf[ilen] = src[c]; ilen = ilen + 1; c = c + 1 }
95 }
96 worklen_p[0] = ilen
97 return ss_begin()
98}
99// add one record; if the 1MB write handle fills, FLUSH it as a segment and
100// start a fresh one (ss_add returns -1 on full). returns the current handle
101// (caller must reassign: w = nx_lib_store_bulk_add(w, ...)).
102func nx_lib_store_bulk_add(w: *i64, kbuf: *u8, worksbuf: *u8, worklen_p: *i64, hk: *u8, record: *u8, reclen: i64) -> *i64 {
103 ls_key(kbuf, hk)
104 var cw: *i64 = w
105 if ss_add(cw, 1, kbuf, record, reclen) < 0 {
106 let segid: i64 = ss_next_segid(LIB_PREFIX)
107 ss_commit(LIB_PREFIX, cw, segid)
108 cw = ss_begin()
109 ss_add(cw, 1, kbuf, record, reclen)
110 }
111 let nl: i64 = ls_strlen(hk)
112 var wl: i64 = worklen_p[0]
113 // __works__ IS A SET, NOT A LOG. Append only if this hk is not already indexed -- the same
114 // guarantee put_work has always had. Without it a re-ingest duplicates every hk and search
115 // returns each work once per ingest run.
116 if ls_name_present(worksbuf, wl, hk, nl) == 0 {
117 var c: i64 = 0
118 while c < nl { worksbuf[wl] = hk[c]; wl = wl + 1; c = c + 1 }
119 worksbuf[wl] = 10 as u8; wl = wl + 1
120 worklen_p[0] = wl
121 }
122 return cw
123}
124// commit the final records batch, then write __works__ in its OWN segment
125// (so a full record-handle can't drop the index). NOTE: __works__ is one value
126// bounded by the 1MB handle (~66K hks); larger catalogs need a chunked index.
127func nx_lib_store_bulk_commit(w: *i64, worksbuf: *u8, worklen: i64) -> i64 {
128 let segid: i64 = ss_next_segid(LIB_PREFIX)
129 ss_commit(LIB_PREFIX, w, segid)
130 let w2: *i64 = ss_begin()
131 ss_add(w2, 1, "__works__" as *u8, worksbuf, worklen)
132 let segid2: i64 = ss_next_segid(LIB_PREFIX)
133 return ss_commit(LIB_PREFIX, w2, segid2)
134}
135
136// current record for <hk> -> ptrout[0]/lenout[0]; rc<0 if not found
137func nx_lib_store_get_work(hk: *u8, ptrout: *i64, lenout: *i64) -> i64 {
138 let kbuf: *u8 = sys_mmap(512)
139 ls_key(kbuf, hk)
140 return ss_get(LIB_PREFIX, kbuf, ptrout, lenout)
141}
142
143// open a cached read handle over the whole store ONCE (segments mmapped once +
144// hash-indexed), then get_work_h is O(1) -- vs get_work's per-call segment
145// re-read (O(segments) each). Use for bulk reads (e.g. index build over 16K+
146// works) to avoid O(n^2) + mmap accumulation. kbuf: caller-provided scratch.
147func nx_lib_store_open() -> *i64 {
148 return ss_open(LIB_PREFIX)
149}
150func nx_lib_store_get_work_h(h: *i64, kbuf: *u8, hk: *u8, ptrout: *i64, lenout: *i64) -> i64 {
151 ls_key(kbuf, hk)
152 return ss_hget(h, kbuf, ptrout, lenout)
153}
154
155// copy the newline-separated work_hk index into idxbuf; returns its length
156func nx_lib_store_index(idxbuf: *u8) -> i64 {
157 let ipo: *i64 = sys_mmap(16) as *i64
158 let ilo: *i64 = sys_mmap(16) as *i64
159 if ss_get(LIB_PREFIX, "__works__" as *u8, ipo, ilo) < 0 { return 0 }
160 let src: *u8 = ipo[0] as *u8
161 var i: i64 = 0
162 while i < ilo[0] { idxbuf[i] = src[i]; i = i + 1 }
163 return ilo[0]
164}
165
166// number of distinct works in the catalog (newline count of the index)
167func nx_lib_store_count() -> i64 {
168 let idxbuf: *u8 = sys_mmap(LIB_MAGIC_1048576)
169 let n: i64 = nx_lib_store_index(idxbuf)
170 var cnt: i64 = 0
171 var i: i64 = 0
172 while i < n { if idxbuf[i] == 10 as u8 { cnt = cnt + 1 } i = i + 1 }
173 return cnt
174}