nx_memindex_emit.nx source
↩ module page · 287 lines · 14919 B
1// nx_memindex_emit.nx -- splice the DERIVED index into a curated markdown file (the MEMORY.md
2// wiring of the conflict-free coindex rail). Sessions APPEND entries to the journal (nx_coindex
3// append -- never editing the shared file); THIS organ is the single writer that regenerates the
4// auto-derived section between whole-line markers:
5// <!-- COINDEX:BEGIN --> ...derived lines (last-write-wins per slug)... <!-- COINDEX:END -->
6// Everything OUTSIDE the markers is preserved BYTE-IDENTICAL (curated sections stay human).
7// FAIL-CLOSED: missing markers -> file untouched. ATOMIC: tmp + fsync + renameat. Serialized
8// under <journal>.lock.
9//
10// EVICTION (2026-07-31, debt 1785558752): the budget gate has always specified a SPLIT -- newest
11// entries stay live in the md block, oldest move into an overflow file COINDEX-OVFL block, nothing
12// is ever lost -- but that capability WAS NEVER IMPLEMENTED. This organ contained zero references
13// to the overflow file, so 4 gate teeth were testing a feature that did not exist, and the whole
14// memindex family read as broken logic when it was in fact an ABSENT CAPABILITY. Built here rather
15// than deleting the teeth (rule 25: build intelligence, never strip features).
16//
17// WHY EVICTION AND NOT REFUSAL: refusing protects the file but leaves the index stale forever, and
18// the harness drops everything past its hard load cliff SILENTLY and TAIL-FIRST. Moving the oldest
19// entries to an overflow file is NOT silent shrinking -- every entry still exists in exactly one of
20// the two blocks (additive, rule 13), and the budget is actually enforced instead of wished for.
21//
22// usage: nx_memindex_emit <journal> <md> [ovfl] [budget] -- legacy: <journal> <md> [budget]
23// The 3rd arg is disambiguated BY CONTENT: all-digits => legacy budget, otherwise the overflow
24// path. That keeps the pre-existing 4th-arg-is-budget contract working (rule 19).
25// With an overflow file the budget measures the DERIVED SECTION; without one it measures the
26// WHOLE composed file and the organ REFUSES rather than write over budget (unchanged behaviour).
27// license_tier: ORIGINAL expect_exit: 0
28import "nx_syscalls.nx"
29import "nx_coindex_core.nx"
30
31// ---- THE BUDGET IS MEASURED, NOT INHERITED FROM AN ADVISORY -----------------------------------
32// ⛔This was a bare `17100`, which is the HARNESS'S BUILT-IN ADVISORY ("compact to under 17.1KB").
33// That number is DISPROVEN: the cliff was measured by headless probe with fixed-width sentinels read
34// back from a live session at 25,000 B AND 200 LINES, whichever binds first. 17,100 evicts about five
35// of a day's work records to respect a figure nobody measured.
36// ★★★★★AN UNMEASURED BUDGET IS NOT CONSERVATIVE JUST BECAUSE IT IS SMALL -- here it silently
37// destroys the newest content, which is the direction nobody audits.
38// ★★★★★AN INSTRUCTION REPEATED ON EVERY EDIT IS OBEYED WITHOUT RE-EXAMINATION; ITS REPETITION IS
39// MISTAKEN FOR ITS AUTHORITY. It reached the DEFAULT of the single writer of MEMORY.md that way.
40//
41// SSOT is `memory/memcliff.conf` (cliff 25,000/200 - target 23,000/180 - warn 21,000), enforced by
42// nx_memcliff. That file is LAPTOP-LOCAL and this organ runs NAS-side, so it CANNOT read it: these
43// consts are a declared MIRROR, not a second ruler, and the 4th arg overrides them for any caller
44// that CAN read the SSOT. ⚠If the conf moves, this mirror is what goes stale -- pass the budget.
45// The reserve is DERIVED from the cliff rather than typed as a second independent number, so the two
46// can never drift into disagreeing about where the edge is.
47const MI_CLIFF_BYTES: i64 = 25000
48const MI_CLIFF_RESERVE: i64 = 2000
49const MI_DEFAULT_BUDGET: i64 = MI_CLIFF_BYTES - MI_CLIFF_RESERVE
50const K_1024: i64 = 1024
51const K_262144: i64 = 262144
52const K_262143: i64 = 262143
53const MI_NL: i64 = 10
54const MI_MODE: i64 = 420
55
56// module statics at TOP (forward-ref gotcha). They exist so mi_splice takes ZERO data args,
57// sidestepping the multi-arg helper miscompile class the budget gate documents.
58static s_path: i64 = 0
59static s_bmark: i64 = 0
60static s_emark: i64 = 0
61static s_content: i64 = 0
62static s_clen: i64 = 0
63static s_commit: i64 = 0
64
65func m_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
66// local decimal parse (the coindex lib has no atoi); 0 on any non-digit, which is also how a path
67// argument is told apart from a budget argument.
68func mi_atoi(s: *u8) -> i64 {
69 var v: i64 = 0
70 var i: i64 = 0
71 while s[i] != (0 as u8) { let c: i64 = s[i] as i64; if c < 48 { return 0 } if c > 57 { return 0 } v = v * 10 + (c - 48); i = i + 1 }
72 return v
73}
74func m_putn(v: i64) -> i64 { nxi_out(v); return 0 }
75
76// first index of needle in hay[0..n), or -1
77func m_find(hay: *u8, n: i64, needle: *u8) -> i64 {
78 let nn: i64 = ccz_slen(needle)
79 if nn == 0 { return 0 - 1 }
80 var i: i64 = 0
81 while i + nn <= n {
82 var k: i64 = 0
83 var ok: i64 = 1
84 while k < nn { if hay[i+k] != needle[k] { ok = 0; k = nn } k = k + 1 }
85 if ok == 1 { return i }
86 i = i + 1
87 }
88 return 0 - 1
89}
90
91// Splice s_content between the marker pair in s_path. Returns the COMPOSED size, -1 if the marker
92// pair is missing or misordered, -2 if the file is absent/empty. Writes ONLY when s_commit==1, so
93// a caller can SIZE a result before deciding to commit it -- that is what keeps the legacy
94// over-budget refusal non-destructive (we simply never rename).
95func mi_splice() -> i64 {
96 let path: *u8 = s_path as *u8
97 let buf: *u8 = sys_mmap(K_262144)
98 let n: i64 = ccz_read(path, buf, K_262143)
99 if n <= 0 { sys_munmap(buf, K_262144); return 0 - 2 }
100 let bpos: i64 = m_find(buf, n, s_bmark as *u8)
101 let epos: i64 = m_find(buf, n, s_emark as *u8)
102 var bad: i64 = 0
103 if bpos < 0 { bad = 1 }
104 if epos < 0 { bad = 1 }
105 if bad == 0 { if epos <= bpos { bad = 1 } }
106 if bad == 1 { sys_munmap(buf, K_262144); return 0 - 1 }
107 // prefix = through the END of the BEGIN-marker line (incl newline)
108 var pfx_end: i64 = bpos
109 var go: i64 = 1
110 while go == 1 { go = 0; if pfx_end < n { if buf[pfx_end] != (MI_NL as u8) { pfx_end = pfx_end + 1; go = 1 } } }
111 if pfx_end < n { pfx_end = pfx_end + 1 }
112 // suffix = from the START of the END-marker line
113 var sfx_start: i64 = epos
114 go = 1
115 while go == 1 { go = 0; if sfx_start > 0 { if buf[sfx_start-1] != (MI_NL as u8) { sfx_start = sfx_start - 1; go = 1 } } }
116 let clen: i64 = s_clen
117 let outn: i64 = pfx_end + clen + (n - sfx_start)
118 if s_commit == 0 { sys_munmap(buf, K_262144); return outn }
119 let tmpp: *u8 = sys_mmap(K_1024)
120 var to: i64 = 0
121 to = ccz_cat_str(tmpp, to, path)
122 to = ccz_cat_str(tmpp, to, ".cotmp" as *u8)
123 let tfd: i64 = sys_openat_wr(tmpp, MI_MODE)
124 // ★★★★★ONE FAILURE CODE SERVING TWO CAUSES IS A BUCKET NAMED FOR THE READER, NOT THE SUBJECT.
125 // This returned -1, the SAME code as "markers not found", so an unwritable directory was reported
126 // to the operator as "add the COINDEX BEGIN/END marker lines once" -- advice that cannot help and
127 // sends them to edit a file that is already correct. -3 and -4 are its own codes now.
128 if tfd < 0 { sys_munmap(tmpp, K_1024); sys_munmap(buf, K_262144); return 0 - 3 }
129 sys_write(tfd, buf, pfx_end)
130 if clen > 0 { sys_write(tfd, s_content as *u8, clen) }
131 sys_write(tfd, (buf as i64 + sfx_start) as *u8, n - sfx_start)
132 sys_fsync(tfd)
133 sys_close(tfd)
134 // The rename IS the atomic commit: if it fails, nothing landed and the md still holds the previous
135 // section. Unchecked, that is indistinguishable from a successful splice.
136 let rr: i64 = sys_renameat(tmpp, path)
137 sys_munmap(tmpp, K_1024)
138 sys_munmap(buf, K_262144)
139 if rr < 0 { return 0 - 4 }
140 return outn
141}
142
143// Offset in d[0..dn) where the KEPT (newest) tail begins; everything before it is evicted.
144// Walks line starts in order and takes the FIRST whose tail fits the budget, which is the earliest
145// one and therefore keeps the MOST entries live. ALWAYS keeps the final line even when it alone
146// exceeds the budget -- the newest entry is the one invariant an index cannot trade away.
147func mi_cut(d: *u8, dn: i64, budget: i64) -> i64 {
148 if dn <= 0 { return 0 }
149 if dn <= budget { return 0 }
150 var laststart: i64 = 0
151 var cut: i64 = 0 - 1
152 var i: i64 = 0
153 while i < dn {
154 if d[i] == (MI_NL as u8) {
155 let st: i64 = i + 1
156 if st < dn {
157 laststart = st
158 if cut < 0 { if dn - st <= budget { cut = st } }
159 }
160 }
161 i = i + 1
162 }
163 if cut < 0 { return laststart }
164 return cut
165}
166
167func main(argc: i64, argv: *i64) -> i64 {
168 if argc < 3 { m_puts("usage: nx_memindex_emit <journal> <md> [ovfl] [budget]\n" as *u8); return 2 }
169 let journal: *u8 = argv[1] as *u8
170 let mdp: *u8 = argv[2] as *u8
171 var ovp: i64 = 0
172 var budget: i64 = MI_DEFAULT_BUDGET
173 if argc >= 4 {
174 let a3: *u8 = argv[3] as *u8
175 let bq: i64 = mi_atoi(a3)
176 if bq > 0 { budget = bq }
177 if bq == 0 { ovp = a3 as i64 }
178 }
179 if argc >= 5 { let b4: i64 = mi_atoi(argv[4] as *u8); if b4 > 0 { budget = b4 } }
180
181 // serialize emitters on the journal lock (the same lock appenders take)
182 let lockp: *u8 = sys_mmap(K_1024)
183 var lo: i64 = 0
184 lo = ccz_cat_str(lockp, lo, journal)
185 lo = ccz_cat_str(lockp, lo, ".lock" as *u8)
186 let lk: i64 = ci_lock(lockp)
187
188 let derived: *u8 = sys_mmap(K_262144)
189 let dn: i64 = ci_emit(journal, derived, K_262143)
190
191 s_bmark = "<!-- COINDEX:BEGIN -->" as *u8 as i64
192 s_emark = "<!-- COINDEX:END -->" as *u8 as i64
193
194 // ---- LEGACY (no overflow file): whole-file budget, REFUSE rather than write over it ----
195 if ovp == 0 {
196 s_path = mdp as i64
197 s_content = derived as i64
198 s_clen = dn
199 s_commit = 0
200 let sz: i64 = mi_splice()
201 if sz == 0 - 2 { ci_unlock(lk); m_puts("MEMINDEX no-md-file\n" as *u8); return 0 }
202 if sz == 0 - 1 { ci_unlock(lk); m_puts("MEMINDEX NO-MARKERS (file untouched; add the COINDEX BEGIN/END marker lines once)\n" as *u8); return 0 }
203 if sz > budget {
204 ci_unlock(lk)
205 m_puts("MEMINDEX REFUSED-OVER-BUDGET out_bytes=" as *u8)
206 m_putn(sz)
207 m_puts(" budget=" as *u8)
208 m_putn(budget)
209 m_puts(" over_by=" as *u8)
210 m_putn(sz - budget)
211 // ★LOG WHAT HAPPENED, WHY IT MATTERS, AND WHAT TO DO. The WHY was dropped when this message
212 // gained its better remedy, and a refusal that states only its remedy reads as bureaucracy --
213 // the reader cannot tell whether ignoring it costs anything. It costs the tail of the index,
214 // silently, at the next session start.
215 m_puts(" -- INDEX LEFT UNTOUCHED. WHY IT MATTERS: the harness drops everything past MEMORY.md's 25,000 B / 200-line cliff SILENTLY and TAIL-FIRST, so an over-budget index loses its LAST lines at the next session start and nothing reports the loss. WHAT TO DO: pass an overflow file as the 3rd arg to EVICT the oldest entries instead of refusing (nothing is lost; they move to its COINDEX-OVFL block).\n" as *u8)
216 return 3
217 }
218 s_commit = 1
219 // ★★★★★★THE MEASURING CALL WAS CHECKED AND THE COMMITTING CALL'S RESULT WAS THROWN AWAY, so
220 // every failure on the ONLY call that actually writes was reported as `MEMINDEX spliced ...`
221 // with exit 0. A dry run that can fail loudly and a real run that cannot is the wrong way round.
222 let wr: i64 = mi_splice()
223 if wr < 0 {
224 ci_unlock(lk)
225 if wr == 0 - 3 { m_puts("MEMINDEX TMP-OPEN-FAILED on COMMIT (index UNTOUCHED) -- could not create <md>.cotmp; the directory is unwritable or full. NOT the missing-marker case: the markers were found.\n" as *u8); return 4 }
226 if wr == 0 - 4 { m_puts("MEMINDEX RENAME-FAILED on COMMIT (index UNTOUCHED, .cotmp left in place) -- the atomic replace did not land, so the file still holds the PREVIOUS derived section.\n" as *u8); return 5 }
227 m_puts("MEMINDEX COMMIT-FAILED (index UNTOUCHED)\n" as *u8)
228 return 6
229 }
230 ci_unlock(lk)
231 m_puts("MEMINDEX spliced entries_bytes=" as *u8)
232 m_putn(dn)
233 m_puts(" out_bytes=" as *u8)
234 m_putn(sz)
235 m_puts("\n" as *u8)
236 return 0
237 }
238
239 // ---- EVICTION: newest stay in the md block, oldest move to the overflow block ----
240 // FAIL-CLOSED FIRST: an overflow file with no marker pair means we evict NOTHING and the md
241 // gets the FULL block. A missing eviction target must never make an entry disappear.
242 s_path = ovp
243 s_bmark = "<!-- COINDEX-OVFL:BEGIN -->" as *u8 as i64
244 s_emark = "<!-- COINDEX-OVFL:END -->" as *u8 as i64
245 s_content = derived as i64
246 s_clen = 0
247 s_commit = 0
248 let ovsz: i64 = mi_splice()
249 var cut: i64 = 0
250 if ovsz >= 0 { cut = mi_cut(derived, dn, budget) }
251
252 s_path = mdp as i64
253 s_bmark = "<!-- COINDEX:BEGIN -->" as *u8 as i64
254 s_emark = "<!-- COINDEX:END -->" as *u8 as i64
255 s_content = (derived as i64) + cut
256 s_clen = dn - cut
257 s_commit = 1
258 let msz: i64 = mi_splice()
259 if msz == 0 - 2 { ci_unlock(lk); m_puts("MEMINDEX no-md-file\n" as *u8); return 0 }
260 if msz == 0 - 1 { ci_unlock(lk); m_puts("MEMINDEX NO-MARKERS (file untouched; add the COINDEX BEGIN/END marker lines once)\n" as *u8); return 0 }
261 // ★★★★★★NAME WHICH REASON. These two used to arrive as -1 and print NO-MARKERS, whose remedy is
262 // "add the marker lines once" -- a one-time edit to the md. The real remedies here are "the
263 // directory is unwritable/full" and "the atomic replace did not land". A BUCKET NAMED FOR HOW THE
264 // READER FAILED, RATHER THAN FOR WHAT THE SUBJECT IS, SENDS EVERY READER TO THE WRONG REMEDY.
265 if msz == 0 - 3 { ci_unlock(lk); m_puts("MEMINDEX TMP-OPEN-FAILED (index UNTOUCHED) -- could not create <md>.cotmp; the directory is unwritable or full. The markers WERE found.\n" as *u8); return 4 }
266 if msz == 0 - 4 { ci_unlock(lk); m_puts("MEMINDEX RENAME-FAILED (index UNTOUCHED, .cotmp left in place) -- the atomic replace did not land.\n" as *u8); return 5 }
267
268 if ovsz >= 0 {
269 s_path = ovp
270 s_bmark = "<!-- COINDEX-OVFL:BEGIN -->" as *u8 as i64
271 s_emark = "<!-- COINDEX-OVFL:END -->" as *u8 as i64
272 s_content = derived as i64
273 s_clen = cut
274 s_commit = 1
275 mi_splice()
276 }
277 ci_unlock(lk)
278 m_puts("MEMINDEX spliced live_bytes=" as *u8)
279 m_putn(dn - cut)
280 m_puts(" evicted_bytes=" as *u8)
281 m_putn(cut)
282 m_puts(" budget=" as *u8)
283 m_putn(budget)
284 if ovsz < 0 { m_puts(" ovfl=NO-MARKERS-NOTHING-EVICTED" as *u8) }
285 m_puts("\n" as *u8)
286 return 0
287}