nx_memindex_put.nx source
↩ module page · 387 lines · 18926 B
1// nx_memindex_put.nx -- the ATOMIC, SLUG-KEYED upsert for the HAND-CURATED half of the memory index
2// (F826 / debt seq273; the contention half of the class whose budget half shipped as F825).
3//
4// WHY THIS EXISTS. The COINDEX block is a conflict-free rail: sessions APPEND to a journal and one
5// writer splices the derived block. The curated pointer block below it had no such path, so every
6// session mutated the shared index with a read-modify-write Edit -- which raced siblings FIVE times in
7// one session and once produced a MANGLED CONCATENATED LINE. The defect is structural: a stale read
8// plus a blind overwrite. This organ removes both -- it re-reads the file INSIDE the lock and rewrites
9// exactly ONE slug-owned line, so a sibling's line can never be clobbered by a stale view.
10//
11// WHY NOT MAKE THE CURATED BLOCK JOURNAL-DERIVED TOO? Because it is genuinely EDITORIAL: the pointers
12// are grouped thematically (DEPLOY/MGMT, COORDINATION/PM, ...) by human judgment, and deriving it
13// per-slug would destroy that grouping. So authorship stays human; only the MUTATION becomes atomic.
14//
15// nx_memindex_put <journal> <md> <slug> <line> [ovfl]
16// Locks <journal>.lock -- the SAME lock nx_coindex append and nx_memindex_emit take -- so put, append
17// and splice are fully serialized against each other, not merely against other puts.
18//
19// ---- BUDGET + AUTO-EVICT (added 2026-07-20 for debt seq255 sev7 / F847; ADDITIVE, rule-19) --------
20// The upsert above kills the CLOBBER half of seq255. It does NOT kill the SCALE half, which the debt
21// row names precisely: "unbounded growth (20.4->21.2KB mid-session, over the 17.1KB soft target,
22// siblings add faster than anyone can hand-compact) + no session can safely compact". A WARNING does
23// not fix that -- warnings are addressed to a librarian who, by the row's own account, cannot keep up.
24// So with an [ovfl] file the zone becomes a true RAIL, symmetric with the COINDEX block: the budget is
25// DERIVED (whole-file target minus everything outside the zone, so it self-adjusts as the rest of the
26// file moves) and the OLDEST lines auto-evict into the overflow file's CURATED-OVFL block.
27// Orientation matters and is not arbitrary: this organ APPENDS new lines at the END of the block, so
28// the zone is oldest-first and ci_split_at_budget -- which keeps the newest SUFFIX -- is exactly the
29// right splitter, reused rather than re-derived.
30// NO-LOSS beats budget: if the overflow file is missing or markerless, NOTHING is evicted and the
31// organ says so. Omitting [ovfl] preserves the old behavior byte-for-byte (warn, never evict).
32// Structural lines (thematic groups, LAWS, DOCTRINE, the gotcha tail) live OUTSIDE the markers and are
33// unreachable here, so the round-4 injury -- silently dropping the file's own laws -- cannot recur.
34// Each managed line carries an invisible owner marker `<!--s:SLUG-->` at its end. Upsert = replace the
35// line carrying that marker IN PLACE (position preserved, so the editorial grouping survives), else
36// append just before the CURATED:END marker.
37// FAIL-CLOSED: missing/misordered CURATED markers -> file untouched and says so (never guesses where
38// to write). ATOMIC: tmp + fsync + renameat, so a concurrent reader sees whole-old or whole-new.
39// IDEMPOTENT: putting identical text twice leaves the file byte-identical.
40// It also REPORTS the resulting curated size against the whole-file target, so a writer learns it is
41// crowding the derived block at the moment it does so, instead of discovering it at the read cliff.
42// license_tier: ORIGINAL No hw writes (Rule 26). expect_exit: 0
43import "nx_syscalls.nx"
44import "nx_coindex_core.nx"
45
46const MP_BUF: i64 = 262144
47const MP_NL: i64 = 10
48const MP_TARGET: i64 = 17100
49// Floor for the managed zone when everything OUTSIDE it already exceeds the target: never shrink the
50// zone to nothing (an index with no live pointers is useless) -- clamp and say so LOUDLY instead.
51const MP_BLOCK_MIN: i64 = 2000
52// PER-ENTRY CAP (2026-07-20, found by DOGFOODING: I wrote a ~3.4KB pointer line, it alone exceeded
53// the whole derived zone budget, and the very next eviction threw it out). The COINDEX rail has had
54// CI_ENTRY_CAP for exactly this reason -- without a per-entry cap ONE verbose session consumes the
55// whole shared zone and starves every sibling. Derived, not dialled: the zone budget runs ~3-5KB and
56// 6-8 lanes want a slot at once => ~700B each. Detail belongs in the topic file; that is what the
57// link in the line is FOR. Over-cap lines are trimmed at a SPACE (never mid-UTF-8-glyph) + " ...".
58const MP_ENTRY_CAP: i64 = 700
59const MP_PATHCAP: i64 = 1024
60const MP_MSGCAP: i64 = 1024
61const MP_MODE: i64 = 420
62const MP_EXIT_USAGE: i64 = 2
63const MP_EXIT_MARKERS: i64 = 3
64const MP_EXIT_IO: i64 = 1
65
66// parse a decimal number from s (stops at the first non-digit); junk/empty -> 0
67func mp_parse_num(s: *u8) -> i64 {
68 var v: i64 = 0
69 var i: i64 = 0
70 var go: i64 = 1
71 while go == 1 {
72 go = 0
73 let c: i64 = s[i] as i64
74 if c >= 48 && c <= 57 { v = v * 10 + (c - 48); i = i + 1; go = 1 }
75 }
76 return v
77}
78
79func mp_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
80func mp_putn(v: i64) -> i64 { nxi_out(v); return 0 }
81
82// first index of needle in hay[0..n), or -1
83func mp_find(hay: *u8, n: i64, needle: *u8) -> i64 {
84 let nn: i64 = ccz_slen(needle)
85 if nn == 0 { return 0 - 1 }
86 var i: i64 = 0
87 while i + nn <= n {
88 var k: i64 = 0
89 var ok: i64 = 1
90 while k < nn { if hay[i+k] != needle[k] { ok = 0; k = nn } k = k + 1 }
91 if ok == 1 { return i }
92 i = i + 1
93 }
94 return 0 - 1
95}
96// does hay[a..b) contain needle?
97func mp_span_has(hay: *u8, a: i64, b: i64, needle: *u8) -> i64 {
98 let nn: i64 = ccz_slen(needle)
99 if nn == 0 { return 0 }
100 var i: i64 = a
101 while i + nn <= b {
102 var k: i64 = 0
103 var ok: i64 = 1
104 while k < nn { if hay[i+k] != needle[k] { ok = 0; k = nn } k = k + 1 }
105 if ok == 1 { return 1 }
106 i = i + 1
107 }
108 return 0
109}
110
111// Bank evicted lines into the overflow file's CURATED-OVFL block, ACCUMULATING them at the top.
112// The COINDEX rail can REPLACE its overflow block because it re-derives it from the journal every
113// run; the curated zone has no journal to replay, so these lines must be PREPENDED to what is already
114// banked -- replacing would destroy every previously evicted pointer. Atomic (tmp+fsync+rename).
115// Returns 1 on success, 0 if the overflow file is missing/markerless -- and the caller then evicts
116// NOTHING, because no-loss beats budget every time.
117// Every exit frees every buffer it took (2026-07-31, nx_mmapbal: was mmap=3 munmap=0). This ran on
118// EVERY put -- an organ sessions invoke repeatedly -- so the leak was per-call, which is exactly the
119// shape that grew the docportal mapping to 698GB. Restructured to ONE exit guarded by the file's own
120// `ok`-flag idiom rather than sprinkling munmaps down six branches: fix the structure, not the
121// symptom (rule 3). Behaviour is unchanged -- 1 only on a completed rename, 0 on every other path.
122func mp_ovfl_bank(ovp: *u8, lines: *u8, n: i64) -> i64 {
123 var rc: i64 = 0
124 var nb2p: i64 = 0
125 var tpp: i64 = 0
126 let ob: *u8 = sys_mmap(MP_BUF)
127 let on: i64 = ccz_read(ovp, ob, MP_BUF - 1)
128 var go: i64 = 1
129 if on <= 0 { go = 0 }
130 if go == 1 { if on >= MP_BUF - 1 { go = 0 } }
131 if go == 1 {
132 let bp: i64 = mp_find(ob, on, "<!-- CURATED-OVFL:BEGIN -->" as *u8)
133 let ep: i64 = mp_find(ob, on, "<!-- CURATED-OVFL:END -->" as *u8)
134 if bp < 0 { go = 0 }
135 if ep < 0 { go = 0 }
136 if go == 1 { if ep <= bp { go = 0 } }
137 if go == 1 {
138 var ins2: i64 = bp
139 var g: i64 = 1
140 while g == 1 { g = 0; if ins2 < on { if ob[ins2] != (MP_NL as u8) { ins2 = ins2 + 1; g = 1 } } }
141 if ins2 < on { ins2 = ins2 + 1 }
142 let nb2: *u8 = sys_mmap(MP_BUF)
143 nb2p = nb2 as i64
144 var o2: i64 = 0
145 var c2: i64 = 0
146 while c2 < ins2 { nb2[o2] = ob[c2]; o2 = o2 + 1; c2 = c2 + 1 }
147 var k2: i64 = 0
148 while k2 < n { if o2 < MP_BUF - 2 { nb2[o2] = lines[k2]; o2 = o2 + 1 } k2 = k2 + 1 }
149 var t3: i64 = ins2
150 while t3 < on { if o2 < MP_BUF - 2 { nb2[o2] = ob[t3]; o2 = o2 + 1 } t3 = t3 + 1 }
151 let tp: *u8 = sys_mmap(MP_PATHCAP)
152 tpp = tp as i64
153 var to2: i64 = ccz_cat_str(tp, 0, ovp)
154 to2 = ccz_cat_str(tp, to2, ".mptmp" as *u8)
155 tp[to2] = 0 as u8
156 let fd2: i64 = sys_openat_wr(tp, MP_MODE)
157 if fd2 >= 0 {
158 sys_write(fd2, nb2, o2)
159 sys_fsync(fd2)
160 sys_close(fd2)
161 sys_renameat(tp, ovp)
162 rc = 1
163 }
164 }
165 }
166 if tpp != 0 { sys_munmap(tpp as *u8, MP_PATHCAP) }
167 if nb2p != 0 { sys_munmap(nb2p as *u8, MP_BUF) }
168 sys_munmap(ob, MP_BUF)
169 return rc
170}
171
172func main(argc: i64, argv: *i64) -> i64 {
173 if argc < 5 { mp_puts("usage: nx_memindex_put <journal> <md> <slug> <line> [ovfl] [budget]\n" as *u8); return MP_EXIT_USAGE }
174 var ovflp: *u8 = 0 as *u8
175 if argc >= 6 { ovflp = argv[5] as *u8 }
176 let journal: *u8 = argv[1] as *u8
177 let mdp: *u8 = argv[2] as *u8
178 let slug: *u8 = argv[3] as *u8
179 // LINE SOURCE: a literal argv, or `@path` to read the line's FIRST LINE from a file. The file
180 // form is not a convenience -- curated pointer lines carry stars, em-dashes and [markdown](links),
181 // and pushing those through the PowerShell/wsl argv layers is a BANKED mangling class. nx_coindex
182 // append already solved it exactly this way, so both rail writers behave alike. CRLF tolerated.
183 let line: *u8 = sys_mmap(MP_BUF)
184 let a4: *u8 = argv[4] as *u8
185 if a4[0] == (64 as u8) {
186 let rn: i64 = ccz_read((a4 as i64 + 1) as *u8, line, MP_BUF - 4)
187 if rn <= 0 { mp_puts("MEMPUT line-file-unreadable\n" as *u8); return MP_EXIT_IO }
188 var tt: i64 = 0
189 var tg: i64 = 1
190 while tg == 1 {
191 tg = 0
192 if tt < rn {
193 if line[tt] == (MP_NL as u8) { line[tt] = 0 as u8 } else { if line[tt] == (13 as u8) { line[tt] = 0 as u8 } else { tt = tt + 1; tg = 1 } }
194 }
195 }
196 } else {
197 var li: i64 = 0
198 while a4[li] != (0 as u8) { if li < MP_BUF - 4 { line[li] = a4[li] } li = li + 1 }
199 if li > MP_BUF - 4 { li = MP_BUF - 4 }
200 line[li] = 0 as u8
201 }
202
203 // the owner marker: <!--s:SLUG-->
204 let mark: *u8 = sys_mmap(MP_PATHCAP)
205 var mo: i64 = ccz_cat_str(mark, 0, "<!--s:" as *u8)
206 mo = ccz_cat_str(mark, mo, slug)
207 mo = ccz_cat_str(mark, mo, "-->" as *u8)
208 mark[mo] = 0 as u8
209
210 // ENFORCE THE PER-ENTRY CAP before anything else sees the line. Back up to the last ASCII space
211 // so a multi-byte glyph (the index is full of stars and arrows) is never split mid-sequence --
212 // the same trim ci_emit uses for the derived block.
213 var linelen: i64 = 0
214 while line[linelen] != (0 as u8) { linelen = linelen + 1 }
215 if linelen > MP_ENTRY_CAP {
216 var cut: i64 = MP_ENTRY_CAP
217 var b: i64 = cut
218 var bstop: i64 = MP_ENTRY_CAP - 120
219 while b > bstop { if line[b] == (32 as u8) { cut = b; b = 0 } else { b = b - 1 } }
220 line[cut] = 32 as u8
221 line[cut+1] = 46 as u8
222 line[cut+2] = 46 as u8
223 line[cut+3] = 46 as u8
224 line[cut+4] = 0 as u8
225 mp_puts("MEMPUT ENTRY-TRIMMED from " as *u8)
226 mp_putn(linelen)
227 mp_puts("B to " as *u8)
228 mp_putn(cut + 4)
229 mp_puts("B (cap " as *u8)
230 mp_putn(MP_ENTRY_CAP)
231 mp_puts("B) -- the index is a POINTER list; put the detail in the linked topic file.\n" as *u8)
232 }
233
234 // serialize against nx_coindex append AND nx_memindex_emit (same lock file)
235 let lockp: *u8 = sys_mmap(MP_PATHCAP)
236 var lo: i64 = ccz_cat_str(lockp, 0, journal)
237 lo = ccz_cat_str(lockp, lo, ".lock" as *u8)
238 lockp[lo] = 0 as u8
239 let lk: i64 = ci_lock(lockp)
240
241 // READ INSIDE THE LOCK -- this is the half that kills the stale-read clobber
242 let md: *u8 = sys_mmap(MP_BUF)
243 let mn: i64 = ccz_read(mdp, md, MP_BUF - 1)
244 if mn <= 0 { ci_unlock(lk); mp_puts("MEMPUT no-md-file\n" as *u8); return MP_EXIT_IO }
245 if mn >= MP_BUF - 1 { ci_unlock(lk); mp_puts("MEMPUT REFUSED md exceeds read buffer (splicing a truncated view would drop the tail)\n" as *u8); return MP_EXIT_IO }
246
247 let bpos: i64 = mp_find(md, mn, "<!-- CURATED:BEGIN -->" as *u8)
248 let epos: i64 = mp_find(md, mn, "<!-- CURATED:END -->" as *u8)
249 var ok: i64 = 1
250 if bpos < 0 { ok = 0 }
251 if epos < 0 { ok = 0 }
252 if ok == 1 { if epos <= bpos { ok = 0 } }
253 if ok == 0 {
254 ci_unlock(lk)
255 mp_puts("MEMPUT NO-MARKERS (file untouched; add the CURATED BEGIN/END marker lines once)\n" as *u8)
256 return MP_EXIT_MARKERS
257 }
258 // block interior = [after the BEGIN line, start of the END line)
259 var ins: i64 = bpos
260 var g: i64 = 1
261 while g == 1 { g = 0; if ins < mn { if md[ins] != (MP_NL as u8) { ins = ins + 1; g = 1 } } }
262 if ins < mn { ins = ins + 1 }
263 var fin: i64 = epos
264 g = 1
265 while g == 1 { g = 0; if fin > 0 { if md[fin-1] != (MP_NL as u8) { fin = fin - 1; g = 1 } } }
266
267 let nb: *u8 = sys_mmap(MP_BUF)
268 var o: i64 = 0
269 var c: i64 = 0
270 while c < ins { nb[o] = md[c]; o = o + 1; c = c + 1 }
271
272 // walk the managed interior: replace the slug-owned line IN PLACE, copy every other line verbatim
273 // Offset (relative to the zone start) of the line THIS call writes, so eviction below can never
274 // throw out the very line we were asked to bank.
275 var self_rel: i64 = 0 - 1
276 var found: i64 = 0
277 var p: i64 = ins
278 while p < fin {
279 var le: i64 = p
280 var s: i64 = 1
281 while s == 1 { if le >= fin { s = 0 } else { if md[le] == (MP_NL as u8) { s = 0 } else { le = le + 1 } } }
282 if mp_span_has(md, p, le, mark) == 1 {
283 found = 1
284 self_rel = o - ins
285 o = ccz_cat_str(nb, o, line)
286 o = ccz_cat_str(nb, o, mark)
287 nb[o] = MP_NL as u8
288 o = o + 1
289 } else {
290 var t: i64 = p
291 while t < le { nb[o] = md[t]; o = o + 1; t = t + 1 }
292 nb[o] = MP_NL as u8
293 o = o + 1
294 }
295 p = le + 1
296 }
297 if found == 0 {
298 self_rel = o - ins
299 o = ccz_cat_str(nb, o, line)
300 o = ccz_cat_str(nb, o, mark)
301 nb[o] = MP_NL as u8
302 o = o + 1
303 }
304 let curated_end: i64 = o
305 var t2: i64 = fin
306 while t2 < mn { nb[o] = md[t2]; o = o + 1; t2 = t2 + 1 }
307
308 // ---- BUDGET + AUTO-EVICT (seq255 sev7 SCALE half) -------------------------------------------
309 // DERIVED, never assumed: the zone gets whatever the whole-file target leaves after everything
310 // OUTSIDE it (the COINDEX block, the structural lines, the laws). So it self-adjusts as the rest
311 // of the file moves, and the file is bounded from both sides -- the F825 law applied symmetrically
312 // to the other block instead of a second hardcoded number that would drift.
313 var managed: i64 = curated_end - ins
314 var evicted: i64 = 0
315 var budget: i64 = 0
316 if ovflp != (0 as *u8) {
317 let rest: i64 = o - managed
318 budget = MP_TARGET - rest
319 if budget < MP_BLOCK_MIN {
320 budget = MP_BLOCK_MIN
321 mp_puts("MEMPUT REST-OVER-BUDGET rest=" as *u8)
322 mp_putn(rest)
323 mp_puts(" target=" as *u8)
324 mp_putn(MP_TARGET)
325 mp_puts(" -- everything OUTSIDE the zone already exceeds the whole-file target; the zone cannot shrink enough to compensate. COMPACT THE STRUCTURAL SECTIONS (librarian action).\n" as *u8)
326 }
327 // EXPLICIT budget wins (rule 19, and it is what makes the property gate-testable on tiny
328 // fixtures -- the same reason nx_memindex_emit carries a 4-arg explicit form beside its
329 // derived one). A gate that can only exercise a 17KB path is a gate nobody runs.
330 if argc >= 7 { budget = mp_parse_num(argv[6] as *u8) }
331 // zone is oldest-first (new lines append at the END), so keep the newest SUFFIX -- which is
332 // precisely ci_split_at_budget's contract. Reused, not re-derived.
333 let zp: *u8 = (nb as i64 + ins) as *u8
334 var split: i64 = ci_split_at_budget(zp, managed, budget)
335 // NEVER EVICT THE LINE THIS CALL JUST WROTE. Found by DOGFOODING: an upsert preserves the
336 // line's ORIGINAL (older) position, so a freshly-refreshed pointer could sit inside the
337 // evict range and be thrown out by the same call that banked it -- incoherent, and it is
338 // exactly what happened to me. ci_split_at_budget already protects the NEWEST line; this
339 // protects the WRITTEN one, which is the line the caller actually cares about. Pulling the
340 // split back to the written line's start keeps it AND everything after it.
341 if self_rel >= 0 { if self_rel < split { split = self_rel } }
342 if split > 0 {
343 if mp_ovfl_bank(ovflp, zp, split) == 1 {
344 var s3: i64 = ins + split
345 var d3: i64 = ins
346 while s3 < o { nb[d3] = nb[s3]; d3 = d3 + 1; s3 = s3 + 1 }
347 o = d3
348 evicted = split
349 managed = managed - split
350 } else {
351 mp_puts("MEMPUT OVFL-UNAVAILABLE (no eviction; give the overflow file its CURATED-OVFL BEGIN/END marker lines once)\n" as *u8)
352 }
353 }
354 }
355
356 // atomic swap
357 let tmpp: *u8 = sys_mmap(MP_PATHCAP)
358 var to: i64 = ccz_cat_str(tmpp, 0, mdp)
359 to = ccz_cat_str(tmpp, to, ".mptmp" as *u8)
360 tmpp[to] = 0 as u8
361 let fd: i64 = sys_openat_wr(tmpp, MP_MODE)
362 if fd < 0 { ci_unlock(lk); mp_puts("MEMPUT cannot-open-tmp\n" as *u8); return MP_EXIT_IO }
363 sys_write(fd, nb, o)
364 sys_fsync(fd)
365 sys_close(fd)
366 sys_renameat(tmpp, mdp)
367 ci_unlock(lk)
368
369 mp_puts("MEMPUT slug=" as *u8)
370 mp_puts(slug)
371 if found == 1 { mp_puts(" action=REPLACED-IN-PLACE" as *u8) } else { mp_puts(" action=APPENDED" as *u8) }
372 mp_puts(" out_bytes=" as *u8)
373 mp_putn(o)
374 // tell the writer NOW if it is crowding the derived block, not at the read cliff.
375 // managed_bytes = the CURATED-marked block only; file_bytes = o (the whole index).
376 mp_puts(" managed_bytes=" as *u8)
377 mp_putn(managed)
378 if evicted > 0 { mp_puts(" evicted_bytes=" as *u8); mp_putn(evicted) }
379 if budget > 0 { mp_puts(" zone_budget=" as *u8); mp_putn(budget) }
380 mp_puts(" file_bytes=" as *u8)
381 mp_putn(o)
382 mp_puts(" target=" as *u8)
383 mp_putn(MP_TARGET)
384 if o > MP_TARGET { mp_puts(" WARN=FILE-OVER-TARGET-run-nx_memindex_emit-or-compact-the-curated-block" as *u8) }
385 mp_puts("\n" as *u8)
386 return 0
387}