nx_coindex_core.nx source
↩ module page · 181 lines · 8545 B
1// nx_coindex_core.nx -- CONFLICT-FREE co-edited index (07-15 operator: "we can't have conflicting
2// writes when we use you on parallel workstreams; we need SOTA orchestration everywhere"). The
3// root cause of the MEMORY.md / board merge-dance = many sessions HAND-EDIT the same file bytes.
4// The SOTA answer is append-and-derive (event-sourced / convergent): a session NEVER edits the
5// shared index; it APPENDS its own entry (keyed by a slug) to a journal -- atomic under the proven
6// flock floor (nx_framed_append / WMS R8, MEASURED exceed-git: concurrent lost 0/144 vs git 132/144)
7// -- and the index is DERIVED by a reducer that keeps the LAST entry per slug (last-write-wins). No
8// two sessions ever touch the same bytes, so there is nothing to conflict on, by construction.
9// Pure append + reduce funcs here (gate-locked); a CLI wires it to MEMORY.md / the board.
10// license_tier: ORIGINAL
11import "nx_syscalls.nx"
12import "nx_crashresume_census_core.nx" // ccz_read / ccz_slen / ccz_cat_str / ccz_cat_num
13
14// Per-entry byte budget for the derived COINDEX block (memindex-cap-treadmill
15// root fix). ~26 unique slugs * 500B ~= 13KB, safely under the 24.4KB read cap
16// with room for the curated sections. An entry over this keeps its slug +
17// leading pointer and trims the trailing prose to " ..." (topic file has full).
18const CI_ENTRY_CAP: i64 = 500
19
20// Buffer sizes, named per rule 11 (no magic numbers). Merged 2026-07-30 from the NAS build-root copy,
21// which had these named while lacking ci_split_at_budget -- neither copy was a superset of the other.
22const K_MAGIC_8192: i64 = 8192 // slug/line scratch
23const K_MAGIC_262144: i64 = 262144 // whole-journal read buffer
24const K_MAGIC_262143: i64 = 262143 // ...minus the NUL terminator
25
26// take the sidecar flock (blocking EX). fd or -1.
27func ci_lock(lockpath: *u8) -> i64 {
28 let fd: i64 = sys_openat_wr(lockpath, 420)
29 if fd < 0 { return 0 - 1 }
30 if sys_flock(fd, SYS_LOCK_EX) < 0 { sys_close(fd); return 0 - 1 }
31 return fd
32}
33func ci_unlock(fd: i64) -> i64 { if fd >= 0 { sys_flock(fd, SYS_LOCK_UN); sys_close(fd) } return 0 }
34
35// APPEND one entry `<slug> <entry>\n` as ONE write under the flock -> atomic vs concurrent appenders
36// (O_APPEND single-write atomicity + the lock makes the whole framed record atomic). slug must have
37// no spaces (kebab-case -- the reducer splits on the first space). Returns 0, or -1 on failure.
38func ci_append(journal: *u8, lockpath: *u8, slug: *u8, entry: *u8) -> i64 {
39 let lk: i64 = ci_lock(lockpath)
40 if lk < 0 { return 0 - 1 }
41 let fd: i64 = sys_openat_append(journal, 420)
42 if fd < 0 { ci_unlock(lk); return 0 - 1 }
43 let buf: *u8 = sys_mmap(K_MAGIC_8192)
44 var o: i64 = 0
45 o = ccz_cat_str(buf, o, slug)
46 o = ccz_cat_str(buf, o, " " as *u8)
47 o = ccz_cat_str(buf, o, entry)
48 o = ccz_cat_str(buf, o, "\n" as *u8)
49 sys_write(fd, buf, o)
50 sys_close(fd)
51 ci_unlock(lk)
52 return 0
53}
54
55// slug range of a line = [ls, first-space). If no space, slug = whole line.
56func ci_slug_end(buf: *u8, ls: i64, le: i64) -> i64 {
57 var i: i64 = ls
58 while i < le { if buf[i] == (32 as u8) { return i } i = i + 1 }
59 return le
60}
61
62// do two slug ranges match?
63func ci_slug_eq(buf: *u8, as_: i64, ae: i64, bs: i64, be: i64) -> i64 {
64 if ae - as_ != be - bs { return 0 }
65 var i: i64 = 0
66 let len: i64 = ae - as_
67 while i < len { if buf[as_+i] != buf[bs+i] { return 0 } i = i + 1 }
68 return 1
69}
70
71// REDUCE: emit the derived index = LAST entry per slug (last-write-wins), in last-occurrence order.
72// For each line, if its slug appears in a LATER line, skip it (superseded); else emit the whole line.
73// Writes into out (NUL-terminated), returns byte length. Deterministic; pure read of the journal.
74func ci_emit(journal: *u8, out: *u8, outcap: i64) -> i64 {
75 let buf: *u8 = sys_mmap(K_MAGIC_262144)
76 let n: i64 = ccz_read(journal, buf, K_MAGIC_262143)
77 if n <= 0 { out[0] = 0 as u8; return 0 }
78 var oo: i64 = 0
79 // iterate lines
80 var ls: i64 = 0
81 while ls < n {
82 var le: i64 = ls
83 var go: i64 = 1
84 while go == 1 { go = 0; if le < n { if buf[le] != (10 as u8) { le = le + 1; go = 1 } } }
85 if le > ls {
86 let se: i64 = ci_slug_end(buf, ls, le)
87 // does this slug reappear in a LATER line?
88 var superseded: i64 = 0
89 var ls2: i64 = le + 1
90 while ls2 < n {
91 var le2: i64 = ls2
92 var g2: i64 = 1
93 while g2 == 1 { g2 = 0; if le2 < n { if buf[le2] != (10 as u8) { le2 = le2 + 1; g2 = 1 } } }
94 if le2 > ls2 {
95 let se2: i64 = ci_slug_end(buf, ls2, le2)
96 if ci_slug_eq(buf, ls, se, ls2, se2) == 1 { superseded = 1; ls2 = n }
97 }
98 if ls2 < n { ls2 = le2 + 1 }
99 }
100 if superseded == 0 {
101 // BUDGET-ENFORCING EMIT (2026-07-16, memindex-cap-treadmill root fix):
102 // cap each entry to CI_ENTRY_CAP bytes so the whole derived block
103 // stays bounded no matter how verbose one session's append is. The
104 // ENTRY (slug + its primary pointer, which entries put up front) is
105 // preserved -- only the trailing prose is trimmed to " ..." (the
106 // topic file has the full text). No-loss holds: every unique slug is
107 // still emitted, so nx_coindex_gate's entry-count check is unaffected.
108 var linelen: i64 = le - ls
109 var elen: i64 = linelen
110 var trunc: i64 = 0
111 if linelen > CI_ENTRY_CAP { elen = CI_ENTRY_CAP; trunc = 1 }
112 // back up to the last ASCII space so we never split a word or a
113 // multi-byte UTF-8 glyph (★/arrows) mid-sequence.
114 if trunc == 1 {
115 var b: i64 = elen
116 var bstop: i64 = CI_ENTRY_CAP - 80
117 while b > bstop { if buf[ls + b] == (32 as u8) { elen = b; b = 0 } else { b = b - 1 } }
118 }
119 var i: i64 = ls
120 var c: i64 = 0
121 while c < elen { if oo < outcap - 8 { out[oo] = buf[i]; oo = oo + 1 } i = i + 1; c = c + 1 }
122 if trunc == 1 {
123 if oo < outcap - 6 { out[oo] = 32 as u8; out[oo+1] = 46 as u8; out[oo+2] = 46 as u8; out[oo+3] = 46 as u8; oo = oo + 4 }
124 }
125 if oo < outcap - 1 { out[oo] = 10 as u8; oo = oo + 1 }
126 }
127 }
128 ls = le + 1
129 }
130 out[oo] = 0 as u8
131 return oo
132}
133
134// SPLIT-AT-BUDGET (2026-07-16, the remaining memindex-cap-treadmill rail feature: TOTAL-size
135// budget). Input = a DERIVED block (ci_emit output: one entry per line, oldest->newest, trailing
136// newline). Returns the byte offset where the NEWEST suffix that fits `budget` bytes begins:
137// [0, split) = the EVICT set (oldest entries -> the overflow block), [split, bn) = LIVE. The
138// newest line is NEVER evicted even if it alone exceeds the budget (an index with zero live
139// entries is useless; CI_ENTRY_CAP keeps single lines far below any sane budget anyway).
140// budget <= 0 = unbudgeted -> 0 (nothing evicted). Pure function of its inputs (gate-locked).
141func ci_split_at_budget(block: *u8, bn: i64, budget: i64) -> i64 {
142 if budget <= 0 { return 0 }
143 if bn <= budget { return 0 }
144 var kept: i64 = 0
145 var nkept: i64 = 0
146 var split: i64 = bn
147 var pos: i64 = bn
148 var go: i64 = 1
149 while go == 1 {
150 go = 0
151 if pos > 0 {
152 // line ENDING at pos: back up to the previous newline (or buffer start)
153 var ls: i64 = pos - 1
154 var g2: i64 = 1
155 while g2 == 1 { g2 = 0; if ls > 0 { if block[ls-1] != (10 as u8) { ls = ls - 1; g2 = 1 } } }
156 let linesz: i64 = pos - ls
157 var fits: i64 = 0
158 if kept + linesz <= budget { fits = 1 }
159 if nkept == 0 { fits = 1 }
160 if fits == 1 {
161 split = ls
162 kept = kept + linesz
163 nkept = nkept + 1
164 pos = ls
165 go = 1
166 }
167 }
168 }
169 return split
170}
171
172// count entries (lines) in a journal -- for the no-loss check.
173func ci_count(journal: *u8) -> i64 {
174 let buf: *u8 = sys_mmap(K_MAGIC_262144)
175 let n: i64 = ccz_read(journal, buf, K_MAGIC_262143)
176 if n <= 0 { return 0 }
177 var c: i64 = 0
178 var i: i64 = 0
179 while i < n { if buf[i] == (10 as u8) { c = c + 1 } i = i + 1 }
180 return c
181}