nx_memindex_emit.nx source
↩ module page · 242 lines · 10449 B
1// nx_memindex_emit.nx -- splice the DERIVED index into a curated markdown file (the MEMORY.md
2// wiring of the conflict-free coindex rail). Sessions APPEND entries to the journal (nx_coindex
3// append -- never editing the shared file); THIS organ is the single writer that regenerates the
4// auto-derived section between whole-line markers:
5// <!-- COINDEX:BEGIN --> ...derived lines (last-write-wins per slug)... <!-- COINDEX:END -->
6// Everything OUTSIDE the markers is preserved BYTE-IDENTICAL (curated sections stay human).
7// FAIL-CLOSED: missing markers -> file untouched. ATOMIC: tmp + fsync + renameat. Serialized
8// under <journal>.lock.
9//
10// EVICTION (2026-07-31, debt 1785558752): the budget gate has always specified a SPLIT -- newest
11// entries stay live in the md block, oldest move into an overflow file COINDEX-OVFL block, nothing
12// is ever lost -- but that capability WAS NEVER IMPLEMENTED. This organ contained zero references
13// to the overflow file, so 4 gate teeth were testing a feature that did not exist, and the whole
14// memindex family read as broken logic when it was in fact an ABSENT CAPABILITY. Built here rather
15// than deleting the teeth (rule 25: build intelligence, never strip features).
16//
17// WHY EVICTION AND NOT REFUSAL: refusing protects the file but leaves the index stale forever, and
18// the harness drops everything past its hard load cliff SILENTLY and TAIL-FIRST. Moving the oldest
19// entries to an overflow file is NOT silent shrinking -- every entry still exists in exactly one of
20// the two blocks (additive, rule 13), and the budget is actually enforced instead of wished for.
21//
22// usage: nx_memindex_emit <journal> <md> [ovfl] [budget] -- legacy: <journal> <md> [budget]
23// The 3rd arg is disambiguated BY CONTENT: all-digits => legacy budget, otherwise the overflow
24// path. That keeps the pre-existing 4th-arg-is-budget contract working (rule 19).
25// With an overflow file the budget measures the DERIVED SECTION; without one it measures the
26// WHOLE composed file and the organ REFUSES rather than write over budget (unchanged behaviour).
27// license_tier: ORIGINAL expect_exit: 0
28import "nx_syscalls.nx"
29import "nx_coindex_core.nx"
30
31const MI_DEFAULT_BUDGET: i64 = 17100
32const K_1024: i64 = 1024
33const K_262144: i64 = 262144
34const K_262143: i64 = 262143
35const MI_NL: i64 = 10
36const MI_MODE: i64 = 420
37
38// module statics at TOP (forward-ref gotcha). They exist so mi_splice takes ZERO data args,
39// sidestepping the multi-arg helper miscompile class the budget gate documents.
40static s_path: i64 = 0
41static s_bmark: i64 = 0
42static s_emark: i64 = 0
43static s_content: i64 = 0
44static s_clen: i64 = 0
45static s_commit: i64 = 0
46
47func m_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
48// local decimal parse (the coindex lib has no atoi); 0 on any non-digit, which is also how a path
49// argument is told apart from a budget argument.
50func mi_atoi(s: *u8) -> i64 {
51 var v: i64 = 0
52 var i: i64 = 0
53 while s[i] != (0 as u8) { let c: i64 = s[i] as i64; if c < 48 { return 0 } if c > 57 { return 0 } v = v * 10 + (c - 48); i = i + 1 }
54 return v
55}
56func m_putn(v: i64) -> i64 { nxi_out(v); return 0 }
57
58// first index of needle in hay[0..n), or -1
59func m_find(hay: *u8, n: i64, needle: *u8) -> i64 {
60 let nn: i64 = ccz_slen(needle)
61 if nn == 0 { return 0 - 1 }
62 var i: i64 = 0
63 while i + nn <= n {
64 var k: i64 = 0
65 var ok: i64 = 1
66 while k < nn { if hay[i+k] != needle[k] { ok = 0; k = nn } k = k + 1 }
67 if ok == 1 { return i }
68 i = i + 1
69 }
70 return 0 - 1
71}
72
73// Splice s_content between the marker pair in s_path. Returns the COMPOSED size, -1 if the marker
74// pair is missing or misordered, -2 if the file is absent/empty. Writes ONLY when s_commit==1, so
75// a caller can SIZE a result before deciding to commit it -- that is what keeps the legacy
76// over-budget refusal non-destructive (we simply never rename).
77func mi_splice() -> i64 {
78 let path: *u8 = s_path as *u8
79 let buf: *u8 = sys_mmap(K_262144)
80 let n: i64 = ccz_read(path, buf, K_262143)
81 if n <= 0 { sys_munmap(buf, K_262144); return 0 - 2 }
82 let bpos: i64 = m_find(buf, n, s_bmark as *u8)
83 let epos: i64 = m_find(buf, n, s_emark as *u8)
84 var bad: i64 = 0
85 if bpos < 0 { bad = 1 }
86 if epos < 0 { bad = 1 }
87 if bad == 0 { if epos <= bpos { bad = 1 } }
88 if bad == 1 { sys_munmap(buf, K_262144); return 0 - 1 }
89 // prefix = through the END of the BEGIN-marker line (incl newline)
90 var pfx_end: i64 = bpos
91 var go: i64 = 1
92 while go == 1 { go = 0; if pfx_end < n { if buf[pfx_end] != (MI_NL as u8) { pfx_end = pfx_end + 1; go = 1 } } }
93 if pfx_end < n { pfx_end = pfx_end + 1 }
94 // suffix = from the START of the END-marker line
95 var sfx_start: i64 = epos
96 go = 1
97 while go == 1 { go = 0; if sfx_start > 0 { if buf[sfx_start-1] != (MI_NL as u8) { sfx_start = sfx_start - 1; go = 1 } } }
98 let clen: i64 = s_clen
99 let outn: i64 = pfx_end + clen + (n - sfx_start)
100 if s_commit == 0 { sys_munmap(buf, K_262144); return outn }
101 let tmpp: *u8 = sys_mmap(K_1024)
102 var to: i64 = 0
103 to = ccz_cat_str(tmpp, to, path)
104 to = ccz_cat_str(tmpp, to, ".cotmp" as *u8)
105 let tfd: i64 = sys_openat_wr(tmpp, MI_MODE)
106 if tfd < 0 { sys_munmap(tmpp, K_1024); sys_munmap(buf, K_262144); return 0 - 1 }
107 sys_write(tfd, buf, pfx_end)
108 if clen > 0 { sys_write(tfd, s_content as *u8, clen) }
109 sys_write(tfd, (buf as i64 + sfx_start) as *u8, n - sfx_start)
110 sys_fsync(tfd)
111 sys_close(tfd)
112 sys_renameat(tmpp, path)
113 sys_munmap(tmpp, K_1024)
114 sys_munmap(buf, K_262144)
115 return outn
116}
117
118// Offset in d[0..dn) where the KEPT (newest) tail begins; everything before it is evicted.
119// Walks line starts in order and takes the FIRST whose tail fits the budget, which is the earliest
120// one and therefore keeps the MOST entries live. ALWAYS keeps the final line even when it alone
121// exceeds the budget -- the newest entry is the one invariant an index cannot trade away.
122func mi_cut(d: *u8, dn: i64, budget: i64) -> i64 {
123 if dn <= 0 { return 0 }
124 if dn <= budget { return 0 }
125 var laststart: i64 = 0
126 var cut: i64 = 0 - 1
127 var i: i64 = 0
128 while i < dn {
129 if d[i] == (MI_NL as u8) {
130 let st: i64 = i + 1
131 if st < dn {
132 laststart = st
133 if cut < 0 { if dn - st <= budget { cut = st } }
134 }
135 }
136 i = i + 1
137 }
138 if cut < 0 { return laststart }
139 return cut
140}
141
142func main(argc: i64, argv: *i64) -> i64 {
143 if argc < 3 { m_puts("usage: nx_memindex_emit <journal> <md> [ovfl] [budget]\n" as *u8); return 2 }
144 let journal: *u8 = argv[1] as *u8
145 let mdp: *u8 = argv[2] as *u8
146 var ovp: i64 = 0
147 var budget: i64 = MI_DEFAULT_BUDGET
148 if argc >= 4 {
149 let a3: *u8 = argv[3] as *u8
150 let bq: i64 = mi_atoi(a3)
151 if bq > 0 { budget = bq }
152 if bq == 0 { ovp = a3 as i64 }
153 }
154 if argc >= 5 { let b4: i64 = mi_atoi(argv[4] as *u8); if b4 > 0 { budget = b4 } }
155
156 // serialize emitters on the journal lock (the same lock appenders take)
157 let lockp: *u8 = sys_mmap(K_1024)
158 var lo: i64 = 0
159 lo = ccz_cat_str(lockp, lo, journal)
160 lo = ccz_cat_str(lockp, lo, ".lock" as *u8)
161 let lk: i64 = ci_lock(lockp)
162
163 let derived: *u8 = sys_mmap(K_262144)
164 let dn: i64 = ci_emit(journal, derived, K_262143)
165
166 s_bmark = "<!-- COINDEX:BEGIN -->" as *u8 as i64
167 s_emark = "<!-- COINDEX:END -->" as *u8 as i64
168
169 // ---- LEGACY (no overflow file): whole-file budget, REFUSE rather than write over it ----
170 if ovp == 0 {
171 s_path = mdp as i64
172 s_content = derived as i64
173 s_clen = dn
174 s_commit = 0
175 let sz: i64 = mi_splice()
176 if sz == 0 - 2 { ci_unlock(lk); m_puts("MEMINDEX no-md-file\n" as *u8); return 0 }
177 if sz == 0 - 1 { ci_unlock(lk); m_puts("MEMINDEX NO-MARKERS (file untouched; add the COINDEX BEGIN/END marker lines once)\n" as *u8); return 0 }
178 if sz > budget {
179 ci_unlock(lk)
180 m_puts("MEMINDEX REFUSED-OVER-BUDGET out_bytes=" as *u8)
181 m_putn(sz)
182 m_puts(" budget=" as *u8)
183 m_putn(budget)
184 m_puts(" over_by=" as *u8)
185 m_putn(sz - budget)
186 m_puts(" -- INDEX LEFT UNTOUCHED. Pass an overflow file as the 3rd arg to EVICT the oldest entries instead of refusing (nothing is lost; they move to its COINDEX-OVFL block).\n" as *u8)
187 return 3
188 }
189 s_commit = 1
190 mi_splice()
191 ci_unlock(lk)
192 m_puts("MEMINDEX spliced entries_bytes=" as *u8)
193 m_putn(dn)
194 m_puts(" out_bytes=" as *u8)
195 m_putn(sz)
196 m_puts("\n" as *u8)
197 return 0
198 }
199
200 // ---- EVICTION: newest stay in the md block, oldest move to the overflow block ----
201 // FAIL-CLOSED FIRST: an overflow file with no marker pair means we evict NOTHING and the md
202 // gets the FULL block. A missing eviction target must never make an entry disappear.
203 s_path = ovp
204 s_bmark = "<!-- COINDEX-OVFL:BEGIN -->" as *u8 as i64
205 s_emark = "<!-- COINDEX-OVFL:END -->" as *u8 as i64
206 s_content = derived as i64
207 s_clen = 0
208 s_commit = 0
209 let ovsz: i64 = mi_splice()
210 var cut: i64 = 0
211 if ovsz >= 0 { cut = mi_cut(derived, dn, budget) }
212
213 s_path = mdp as i64
214 s_bmark = "<!-- COINDEX:BEGIN -->" as *u8 as i64
215 s_emark = "<!-- COINDEX:END -->" as *u8 as i64
216 s_content = (derived as i64) + cut
217 s_clen = dn - cut
218 s_commit = 1
219 let msz: i64 = mi_splice()
220 if msz == 0 - 2 { ci_unlock(lk); m_puts("MEMINDEX no-md-file\n" as *u8); return 0 }
221 if msz == 0 - 1 { ci_unlock(lk); m_puts("MEMINDEX NO-MARKERS (file untouched; add the COINDEX BEGIN/END marker lines once)\n" as *u8); return 0 }
222
223 if ovsz >= 0 {
224 s_path = ovp
225 s_bmark = "<!-- COINDEX-OVFL:BEGIN -->" as *u8 as i64
226 s_emark = "<!-- COINDEX-OVFL:END -->" as *u8 as i64
227 s_content = derived as i64
228 s_clen = cut
229 s_commit = 1
230 mi_splice()
231 }
232 ci_unlock(lk)
233 m_puts("MEMINDEX spliced live_bytes=" as *u8)
234 m_putn(dn - cut)
235 m_puts(" evicted_bytes=" as *u8)
236 m_putn(cut)
237 m_puts(" budget=" as *u8)
238 m_putn(budget)
239 if ovsz < 0 { m_puts(" ovfl=NO-MARKERS-NOTHING-EVICTED" as *u8) }
240 m_puts("\n" as *u8)
241 return 0
242}