nx_linemap.nx source
↩ module page · 465 lines · 21534 B
1// nx_linemap.nx -- OUT-OF-BAND expanded-line -> (file, source-line) map.
2//
3// WHY THIS EXISTS
4// nx_cc compiles PRE-EXPANDED source: nx_import.nx splices every imported file
5// inline into ONE flat buffer, so every line number the lexer produces is a line
6// in that flat stream. Consequences, both measured:
7// * a diagnostic can say WHERE the compiler was but never WHICH FILE the author
8// must open (nx_parse.nx says so in-source: "Mapping back to per-file lines is
9// a separate rung"; the /compare/lang matrix scores the 5W+H voice 24/25 and
10// names the missing point as the file name);
11// * nx_dwarf_line.nx -- a complete DWARF v5 .debug_line builder -- has no file
12// table to emit, so it sits built and unwired along with 11 sibling organs.
13// This module is the substrate both of those need. It is NOT itself the fix to
14// either; it is the thing they were both missing.
15//
16// DESIGN DECISION -- a SPAN TABLE, not C's `#line` directives
17// The C toolchain solves this IN BAND: cpp injects `#line N "file"` into the
18// expanded text and the compiler's lexer understands them. We reject that, and
19// the reasons are worth stating because the in-band design is the older and more
20// familiar one:
21// 1. EVERY consumer of the expanded buffer must then understand a second syntax
22// that is not NishiLang -- the lexer, and anything that ever reads the buffer
23// for any other purpose. One producer's convenience becomes every reader's
24// obligation.
25// 2. It is FORGEABLE FROM ORDINARY SOURCE. The expanded buffer carries string
26// literals verbatim; a literal containing a newline followed by
27// `#line 1 "victim.nx"` would relabel the real code after it. Diagnostics --
28// and later, debug info a debugger trusts -- could be made to point at an
29// innocent file BY WRITING A STRING. Putting metadata in a channel that also
30// carries attacker-writable text is a confusion of layers, and the estate has
31// already paid for that class once (the in-STRING `}` that closed a brace skip
32// mid-body, nx_import.nx skip_func_main, debt 1785894599).
33// 3. It perturbs the very bytes nx_diag_caret prints, so the snippet machinery
34// would have to learn to skip directives to avoid showing them to the author.
35// Clang (SourceManager) and rustc (SourceMap) both keep this mapping OUT of the
36// text, beside it. We match that known good.
37//
38// THE COST WE ACCEPT, STATED PLAINLY: an out-of-band map is a parallel structure
39// that can DESYNC from the buffer it describes -- it is only as true as its
40// producer. In-band directives cannot desync, because they ARE the buffer. That is
41// the real trade, and it is why (a) the producer emits a span at EVERY point where
42// output lines and source lines stop advancing in lockstep, and (b) lm_lookup
43// REFUSES rather than guesses when it is outside what it knows (see below).
44//
45// SHAPE
46// A span is (out_line, file_id, src_line): "expanded line `out_line` is line
47// `src_line` of file `file_id`, and the two advance together until the next span."
48// Spans are appended in strictly increasing out_line order because expansion is
49// sequential, so lookup walks to the last span at or before the query.
50//
51// IMPORTS: none, deliberately. nx_parse.nx uses sys_mmap without importing a syscall
52// layer ("sys_mmap comes from syscalls.nx (via ir.nx); no local copy") and this
53// module follows that convention, so it can never drag a SECOND syscall layer into
54// a closure that already has one -- the 37-file both-layers hazard (debt 1785523412).
55// Cost: this file does not compile standalone; its KAT lives in a witness probe that
56// imports the full chain, matching the nx_probe_*_live.nx convention.
57
58// Self-contained stderr helpers for the overflow announcement below. Local on purpose: this file is
59// imported by the compiler front-end, and a warning that drags in another module is a warning that gets
60// deleted the first time the closure is trimmed.
61func lm_ovf_puts(s: *u8) -> i64 {
62 var n: i64 = 0
63 while s[n] != (0 as u8) { n = n + 1 }
64 sys_write(2, s, n)
65 return 0
66}
67func lm_ovf_putn(v: i64) -> i64 {
68 let bb: *u8 = sys_mmap(32)
69 let t: *u8 = sys_mmap(32)
70 var m: i64 = v
71 var k: i64 = 0
72 if m == 0 { t[0] = 48; k = 1 }
73 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
74 var i: i64 = 0
75 while i < k { bb[i] = t[k - 1 - i]; i = i + 1 }
76 sys_write(2, bb, k)
77 return 0
78}
79
80const NX_LM_MAX_SPANS: i64 = 8192
81const NX_LM_MAX_FILES: i64 = 512
82const NX_LM_PATH_LEN: i64 = 1024
83const NX_LM_MAX_FUNCS: i64 = 4096
84const NX_LM_FNAME_LEN: i64 = 128
85
86struct LineMap {
87 span_out: *i64, // expanded line at which this span starts (1-based)
88 span_file: *i64, // file id, index into `files`
89 span_src: *i64, // line within that file at the span start (1-based)
90 n_spans: i64,
91 files: *u8, // NX_LM_MAX_FILES slots of NX_LM_PATH_LEN bytes
92 n_files: i64,
93 ovf_from: i64, // 0 = never overflowed; else the first out_line we stopped tracking
94 // FUNCTION SIDE TABLE (2026-08-06, DDR-002). Debug info needs each function's DEFINITION
95 // line. The obvious home is a field on struct Function -- and nx_parse.nx refuses that
96 // in-source: "its layout is load-bearing for the self-hosting bootstrap; never-brick says
97 // do not perturb what you do not have to." So it rides here instead, name-keyed, which is
98 // the same out-of-band discipline this module already uses for spans. One consequence
99 // worth stating: nx_linemap is now the SINGLE source of file/line truth for BOTH
100 // diagnostics and debug info, so the two can never disagree about where a thing is.
101 fn_names: *u8, // NX_LM_MAX_FUNCS slots of NX_LM_FNAME_LEN bytes
102 fn_line: *i64, // EXPANDED line of each function's definition
103 n_funcs: i64,
104}
105
106func lm_new() -> *LineMap {
107 // 128, NOT 64. The struct was 7 fields / 56 B and the function side table took it to
108 // 10 / 80 B -- a 64-byte allocation would have let the last three fields write past it,
109 // silently, into whatever mmap landed next. The ExpandCtx edit in nx_import.nx dodged
110 // this same trap only because it SPENT two reserved pads instead of growing.
111 // A STRUCT AND ITS ALLOCATION ARE ONE FACT WRITTEN IN TWO PLACES; GROWING ONE IS EDITING BOTH.
112 let raw: *u8 = sys_mmap(128)
113 let lm: *LineMap = raw as *LineMap
114 lm.span_out = sys_mmap(8 * NX_LM_MAX_SPANS) as *i64
115 lm.span_file = sys_mmap(8 * NX_LM_MAX_SPANS) as *i64
116 lm.span_src = sys_mmap(8 * NX_LM_MAX_SPANS) as *i64
117 lm.n_spans = 0
118 lm.files = sys_mmap(NX_LM_MAX_FILES * NX_LM_PATH_LEN)
119 lm.n_files = 0
120 lm.ovf_from = 0
121 lm.fn_names = sys_mmap(NX_LM_MAX_FUNCS * NX_LM_FNAME_LEN)
122 lm.fn_line = sys_mmap(8 * NX_LM_MAX_FUNCS) as *i64
123 lm.n_funcs = 0
124 return lm
125}
126
127// ---- local string helpers -------------------------------------------------
128// Deliberately lm_-prefixed rather than reusing cstr_len/cstr_eq/u8_copy from
129// nx_import.nx: both files land in the SAME expanded closure, and the compiler
130// now refuses duplicate definitions (2026-08-05). A shared-name helper here
131// would break every build that imports both.
132
133func lm_slen(s: *u8) -> i64 {
134 var n: i64 = 0
135 while s[n] != 0 { n = n + 1 }
136 return n
137}
138
139func lm_at(p: *u8, d: i64) -> *u8 {
140 let a: i64 = (p as i64) + d
141 return a as *u8
142}
143
144func lm_seq(a: *u8, b: *u8) -> i64 {
145 var i: i64 = 0
146 while a[i] != 0 {
147 if a[i] != b[i] { return 0 }
148 i = i + 1
149 }
150 if b[i] != 0 { return 0 }
151 return 1
152}
153
154func lm_copy(dst: *u8, src: *u8, n: i64) -> i64 {
155 var i: i64 = 0
156 while i < n { dst[i] = src[i]; i = i + 1 }
157 return n
158}
159
160// ---- files ----------------------------------------------------------------
161
162// Intern `path`, returning its id. Same path -> same id (the seen-set in
163// nx_import.nx already dedupes expansion, but a path can still be interned twice
164// when a span is re-emitted after a nested import returns). Returns -1 when full;
165// callers treat -1 as "do not record", never as file 0.
166func lm_intern_file(lm: *LineMap, path: *u8) -> i64 {
167 // NULL-SAFE BY CONTRACT. Mapping is opt-in, so the DEFAULT caller passes a null
168 // map on every build in the estate. Every entry point here must survive that --
169 // an organ whose disabled path segfaults is not disabled, it is armed.
170 if (lm as i64) == 0 { return 0 - 1 }
171 var i: i64 = 0
172 while i < lm.n_files {
173 let e: *u8 = lm_at(lm.files, i * NX_LM_PATH_LEN)
174 if lm_seq(e, path) == 1 { return i }
175 i = i + 1
176 }
177 if lm.n_files >= NX_LM_MAX_FILES { return 0 - 1 }
178 let slot: *u8 = lm_at(lm.files, lm.n_files * NX_LM_PATH_LEN)
179 var n: i64 = lm_slen(path)
180 if n > NX_LM_PATH_LEN - 1 { n = NX_LM_PATH_LEN - 1 }
181 lm_copy(slot, path, n)
182 slot[n] = 0
183 let id: i64 = lm.n_files
184 lm.n_files = lm.n_files + 1
185 return id
186}
187
188func lm_file_path(lm: *LineMap, id: i64) -> *u8 {
189 if (lm as i64) == 0 { return 0 as *u8 }
190 if id < 0 { return 0 as *u8 }
191 if id >= lm.n_files { return 0 as *u8 }
192 return lm_at(lm.files, id * NX_LM_PATH_LEN)
193}
194
195// Last path component. A diagnostic reads better as `nx_parse.nx:412` than as a
196// 90-character absolute path. The FULL path stays reachable via lm_file_path --
197// DWARF needs it, so this shortens for the human without discarding for the machine.
198func lm_basename(p: *u8) -> *u8 {
199 if (p as i64) == 0 { return p }
200 let n: i64 = lm_slen(p)
201 var last: i64 = 0 - 1
202 var i: i64 = 0
203 while i < n {
204 if p[i] == 0x2F { last = i }
205 if p[i] == 0x5C { last = i }
206 i = i + 1
207 }
208 if last < 0 { return p }
209 return lm_at(p, last + 1)
210}
211
212// ---- the ACTIVE map -------------------------------------------------------
213// The map has TWO consumers in different modules (nx_parse for diagnostics, the x86 emitter
214// for debug line info). Parking the handle in either consumer would make the other depend on
215// splice ORDER inside the flat expanded unit -- a dependency that is invisible until some
216// driver imports one without the other and the build breaks far from the cause.
217// It belongs to the module that owns the type.
218static lm_active: *LineMap
219
220// DEBUG-INFO SWITCH, DEFAULT OFF (DDR-002 section 6.1). The line map is ALWAYS on, because it
221// powers per-file DIAGNOSTICS and those cost nothing in the binary. Emitting .file/.loc is a
222// SEPARATE decision, because the moment nxasm learns to consume them every binary grows a
223// section header table -- and /compare/lang publishes a MEASURED exceed of a 168-byte minimal
224// static binary. Shipping debug info by default would trade a published win for an unpublished
225// one, silently. gcc defaults -g OFF for exactly this reason; we match the known good.
226// ★A DEFAULT THAT REGRESSES A PUBLISHED MEASUREMENT IS A RETRACTION NOBODY WROTE DOWN.
227static lm_debug: i64
228
229func lm_set_debug(v: i64) -> i64 {
230 lm_debug = v
231 return 0
232}
233
234func lm_debug_on() -> i64 {
235 return lm_debug
236}
237
238// ---- STATEMENT GRANULARITY (DDR-006, 2026-08-07) ------------------------------------------------
239//
240// Rung 1 gave one line per FUNCTION because struct Function has no line field and nx_parse.nx
241// refuses in-source to add one ("its layout is load-bearing for the self-hosting bootstrap").
242// struct Instr has the SAME problem from the other side: its comments record that widening it moved
243// the stride 128 -> 192 -> 256 and needed lockstep edits across nx_ir, nx_opt, nx_x86_regalloc and
244// nx_parse. So statement lines ride OUT OF BAND too -- the same discipline, for the same reason.
245//
246// WHY A POINTER KEY IS SOUND HERE, and this is the whole trick: alloc_instr hands out instructions
247// from a CONTIGUOUS per-function pool at `base + id * 256` (nx_ir.nx:337). An instruction pointer is
248// therefore already a dense identity, and (p >> 8) is its pool index -- no struct field, no parallel
249// array to thread through Function, no allocation the caller has to own.
250//
251// COST WHEN -g IS OFF: one predictable branch in alloc_instr. The 4 MB of tables is allocated
252// LAZILY on first stamp, so a normal build never touches it.
253const NX_LM_STMT_SLOTS: i64 = 262144
254static lm_stmt_key: *i64
255static lm_stmt_val: *i64
256static lm_stmt_cur_line: i64
257static lm_stmt_n: i64
258static lm_stmt_full: i64
259
260func lm_stmt_init() -> i64 {
261 if (lm_stmt_key as i64) != 0 { return 0 }
262 lm_stmt_key = sys_mmap(8 * NX_LM_STMT_SLOTS) as *i64
263 lm_stmt_val = sys_mmap(8 * NX_LM_STMT_SLOTS) as *i64
264 lm_stmt_n = 0
265 lm_stmt_full = 0
266 return 0
267}
268
269// The parser sets this at each statement boundary; every instruction the builders allocate while it
270// stands inherits it. Set to 0 to mean "no statement in scope" -- compiler-synthesised instructions
271// then carry NO line rather than inheriting a neighbouring statement, because attributing generated
272// code to a line the programmer did not write is exactly what sends a debugger to the wrong place.
273func lm_stmt_set_cur(line: i64) -> i64 { lm_stmt_cur_line = line; return 0 }
274func lm_stmt_cur() -> i64 { return lm_stmt_cur_line }
275func lm_stmt_count() -> i64 { return lm_stmt_n }
276func lm_stmt_overflowed() -> i64 { return lm_stmt_full }
277
278func lm_stmt_slot(p: i64) -> i64 { return (p >> 8) & (NX_LM_STMT_SLOTS - 1) }
279
280func lm_stmt_stamp(p: i64) -> i64 {
281 if p == 0 { return 0 }
282 if lm_stmt_cur_line <= 0 { return 0 }
283 lm_stmt_init()
284 var s: i64 = lm_stmt_slot(p)
285 var tries: i64 = 0
286 while tries < NX_LM_STMT_SLOTS {
287 let k: i64 = lm_stmt_key[s]
288 if k == 0 {
289 lm_stmt_key[s] = p
290 lm_stmt_val[s] = lm_stmt_cur_line
291 lm_stmt_n = lm_stmt_n + 1
292 return 1
293 }
294 if k == p { lm_stmt_val[s] = lm_stmt_cur_line; return 1 }
295 s = (s + 1) & (NX_LM_STMT_SLOTS - 1)
296 tries = tries + 1
297 }
298 // FULL: record the fact and drop the row. Never evict a live mapping to make room -- a wrong
299 // line is worse than a missing one, and lm_stmt_overflowed lets a gate SEE the truncation
300 // instead of silently shipping a partial table (the ovf_from field above exists for the same
301 // reason on spans).
302 lm_stmt_full = 1
303 return 0
304}
305
306func lm_stmt_lookup(p: i64) -> i64 {
307 if p == 0 { return 0 }
308 if (lm_stmt_key as i64) == 0 { return 0 }
309 var s: i64 = lm_stmt_slot(p)
310 var tries: i64 = 0
311 while tries < NX_LM_STMT_SLOTS {
312 let k: i64 = lm_stmt_key[s]
313 if k == 0 { return 0 }
314 if k == p { return lm_stmt_val[s] }
315 s = (s + 1) & (NX_LM_STMT_SLOTS - 1)
316 tries = tries + 1
317 }
318 return 0
319}
320
321func lm_set_active(lm: *LineMap) -> i64 {
322 lm_active = lm
323 return 0
324}
325
326func lm_get_active() -> *LineMap {
327 return lm_active
328}
329
330// ---- function definition lines --------------------------------------------
331
332// Record that function `name` is DEFINED at expanded line `out_line`.
333// NULL-SAFE: mapping is opt-in, so the default caller passes a null map on every build in
334// the estate. An organ whose disabled path faults is not disabled, it is armed.
335func lm_fn_record(lm: *LineMap, name: *u8, nlen: i64, out_line: i64) -> i64 {
336 if (lm as i64) == 0 { return 0 }
337 if nlen <= 0 { return 0 }
338 if lm.n_funcs >= NX_LM_MAX_FUNCS { return 0 }
339 var n: i64 = nlen
340 if n > NX_LM_FNAME_LEN - 1 { n = NX_LM_FNAME_LEN - 1 }
341 let slot: *u8 = lm_at(lm.fn_names, lm.n_funcs * NX_LM_FNAME_LEN)
342 lm_copy(slot, name, n)
343 slot[n] = 0
344 lm.fn_line[lm.n_funcs] = out_line
345 lm.n_funcs = lm.n_funcs + 1
346 return 1
347}
348
349// Expanded line of `name`'s definition, or 0 when unknown.
350// FIRST match wins ON PURPOSE: a forward declaration and its body share a name, and an
351// ADDRESS belongs to the body. Scanning forward and returning the first hit gives that for
352// free -- provided the recorder only ever records definitions, which is the caller's job.
353func lm_fn_lookup(lm: *LineMap, name: *u8, nlen: i64) -> i64 {
354 if (lm as i64) == 0 { return 0 }
355 if nlen <= 0 { return 0 }
356 var i: i64 = 0
357 while i < lm.n_funcs {
358 let e: *u8 = lm_at(lm.fn_names, i * NX_LM_FNAME_LEN)
359 var j: i64 = 0
360 var same: i64 = 1
361 while j < nlen {
362 if e[j] != name[j] { same = 0; j = nlen } else { j = j + 1 }
363 }
364 if same == 1 { if e[nlen] != 0 { same = 0 } }
365 if same == 1 { return lm.fn_line[i] }
366 i = i + 1
367 }
368 return 0
369}
370
371// ---- spans ----------------------------------------------------------------
372
373// Record that expanded line `out_line` is line `src_line` of `file_id`.
374// Returns 1 if recorded, 0 if dropped (overflow or unknown file).
375func lm_add_span(lm: *LineMap, out_line: i64, file_id: i64, src_line: i64) -> i64 {
376 if (lm as i64) == 0 { return 0 }
377 if file_id < 0 { return 0 }
378 // HOISTED LOCALS, DELIBERATELY (BISECT-D 86011501). These three arrays are indexed
379 // through LOCAL *i64 handles, never as `lm.span_out[i]` directly. The codebase's own
380 // known-good idiom does the same -- canonicalise_path indexes local *i64 handles --
381 // and the ONE place the expander indexes a pointer-typed struct field is ctx.out[pos],
382 // which is *u8, where an element-size scaling error is invisible by construction.
383 // HISTORY, KEPT DELIBERATELY (86011501, 2026-08-06). This plain field-indexed form was
384 // once believed to miscompile: nx_cc_equiv_gate went RED 9/10 on nx_jpeg_ascii_test, a
385 // 4-step bisect "isolated" it here, and hoisting the fields into local handles turned it
386 // GREEN. That conclusion was WRONG and was retracted (debt 1786067328). What actually
387 // moved was the TREE -- a sibling seat was rewriting runtime/nx_syscalls.nx every few
388 // seconds (measured oscillating 41870-46652 B), and the gate compiles baseline and
389 // challenger SEQUENTIALLY over the live tree with no stability check (debt 1786067358),
390 // so a mid-run change is reported as a challenger failure. It reproduced byte-identically
391 // on an immediate re-run, which reads as determinism and is not.
392 // THE STEP THAT FOUND THE TRUTH was restoring this exact shape and re-testing it: GREEN
393 // 10/10 + selfhost. A fix that is never un-applied is indistinguishable from a coincidence.
394 // The shape is fine. nx_probe_ptrfield_idx is the standing KAT that proves it.
395 if lm.n_spans > 0 {
396 let last: i64 = lm.n_spans - 1
397 if lm.span_out[last] == out_line {
398 lm.span_file[last] = file_id
399 lm.span_src[last] = src_line
400 return 1
401 }
402 }
403 if lm.n_spans >= NX_LM_MAX_SPANS {
404 // ANNOUNCE ONCE, AT THE MOMENT THE INFORMATION IS LOST. This table used to fill in SILENCE:
405 // lm_lookup then returns 0 for every line at or past ovf_from, so every later diagnostic prints
406 // an UNRESOLVED file name. MEASURED 2026-08-17 on an nx_browser build: errors came out as
407 // `error at cl2:187`, `error at .:670` and `error at <replacement-char>:661` -- which reads as a
408 // CORRUPTED COMPILER rather than an exhausted table, and cost real investigation time chasing a
409 // parser bug that was not there.
410 // *A CAP REACHED IN SILENCE BECOMES A MEASUREMENT NOBODY KNOWS IS PARTIAL* -- and this one
411 // degrades diagnostics precisely in the LARGEST translation units, where they matter most.
412 // Printed once (the ovf_from == 0 guard already existed and makes this exactly-once), so it
413 // cannot spam what is otherwise a hot path.
414 if lm.ovf_from == 0 {
415 lm.ovf_from = out_line
416 lm_ovf_puts("\nnx_linemap: span table FULL -- diagnostics past expanded line " as *u8)
417 lm_ovf_putn(out_line)
418 lm_ovf_puts(" cannot name their source file (spans=" as *u8)
419 lm_ovf_putn(NX_LM_MAX_SPANS)
420 lm_ovf_puts("). Line numbers below are EXPANDED-UNIT lines, not per-file lines.\n" as *u8)
421 }
422 return 0
423 }
424 lm.span_out[lm.n_spans] = out_line
425 lm.span_file[lm.n_spans] = file_id
426 lm.span_src[lm.n_spans] = src_line
427 lm.n_spans = lm.n_spans + 1
428 return 1
429}
430
431// Resolve an expanded line to (file_id, src_line). Returns 1 on success and 0 when
432// the answer is NOT KNOWN -- caller must fall back to the expanded line.
433//
434// REFUSING IS THE POINT. A wrong file name in a diagnostic is worse than no file
435// name: it sends the author to edit a file that is not the problem, and it is the
436// exact same failure class as a caret under the wrong line (which this compiler
437// shipped, caught, and corrected before promotion on 2026-08-05). Every path that
438// cannot be certain returns 0.
439func lm_lookup(lm: *LineMap, out_line: i64, o_file: *i64, o_src: *i64) -> i64 {
440 if (lm as i64) == 0 { return 0 }
441 if out_line <= 0 { return 0 }
442 if lm.n_spans <= 0 { return 0 }
443 if lm.ovf_from != 0 {
444 if out_line >= lm.ovf_from { return 0 }
445 }
446 // Same hoisting discipline as lm_add_span -- index local handles, never the fields.
447 let so: *i64 = lm.span_out
448 let sf: *i64 = lm.span_file
449 let ss: *i64 = lm.span_src
450 let n: i64 = lm.n_spans
451 var best: i64 = 0 - 1
452 var i: i64 = 0
453 while i < n {
454 if so[i] <= out_line {
455 best = i
456 i = i + 1
457 } else {
458 i = n
459 }
460 }
461 if best < 0 { return 0 }
462 *o_file = sf[best]
463 *o_src = ss[best] + (out_line - so[best])
464 return 1
465}