nx_tool_census.nx source
↩ module page · 408 lines · 16548 B
1// nx_tool_census.nx -- bits-up tool-census classification primitive.
2//
3// Composes [[nx_grep.nx]] (S1 of shell-replacement) + nx_string_ops.nx
4// to classify a substrate-file's HEADER bytes into one of:
5// AUTO_GENERATED (template-shape test corpus; excluded from gate)
6// HAND_WRITTEN_DECLARED (lineage declared: Composes / Supersedes /
7// Distinct / Canonical / genealogy_id)
8// HAND_WRITTEN_UNDECLARED (no lineage metadata)
9// TEST (file ending in "_test.nx" -- excluded from gate)
10// OFFC (file prefix "_offc" -- excluded from gate)
11//
12// FIRST STONE NOTE: this primitive ships the CLASSIFICATION LOGIC.
13// The directory-walker wrapper (reading every runtime/*.nx file +
14// aggregating counts) lands at S5 when nx_find ships. Until then
15// bench/nx_tool_census.sh remains the .sh harness; the .nx primitive
16// underneath is what does the substantive classification work.
17//
18// Per [[feedback-bits-up-shell-replace-linux-brother-tools]]: this
19// is the substrate's first cardinal-demonstrating audit primitive
20// where the LOGIC moves from grep/awk/sed bash plumbing into a
21// bits-up NishiLang function. When nx_find ships at S5, swapping
22// the .sh wrapper for an .nx executor closes the cardinal end-to-end
23// for this one tool.
24//
25// Composes:
26// [[NISHI_SHELL_REPLACEMENT_ROADMAP]] S1.5 (post-nx_grep audit primitive)
27// nx_grep.nx (S1 canonical pattern search; nx_grep_any for lineage detection)
28// nx_string_ops.nx (nx_str_ends_with for filename suffix matching;
29// nx_str_starts_with for offc-prefix detection)
30// [[feedback-no-tool-proliferation-consolidate-or-justify]] +
31// [[feedback-no-tool-proliferation-bit-level]] (this primitive's
32// classification IS the cardinal's enforcement primitive)
33// [[feedback-substrate-genealogy-dedupe-mapper]] (lineage-declared
34// classification is the genealogy's own truth source)
35
36// nx_safety_envelope:
37// intended_use: "substrate file-header classification:
38// AUTO_GENERATED / HAND_WRITTEN_DECLARED /
39// HAND_WRITTEN_UNDECLARED / TEST / OFFC;
40// composes nx_grep over header buffer"
41// sil_target: SIL2
42// evidence: [kat_autogen_detection,
43// kat_declared_lineage_detection,
44// kat_offc_test_filename_filter]
45// hazard_register: [bug-tape-pattern-substring-misclassified,
46// bug-tape-empty-header-handling]
47// verdict: NOT_YET_EVALUATED
48
49import "nx_syscalls.nx"
50import "nx_string_ops.nx"
51import "nx_grep.nx"
52import "nx_grep_rt.nx" // nx_grep_any lives here, not in nx_grep.nx
53import "nx_etg.nx"
54import "nx_dir.nx"
55
56// ===== Classification sealed enum =================================
57
58const NX_TOOL_CENSUS_NONE: i64 = 0
59const NX_TOOL_CENSUS_TEST: i64 = 1
60const NX_TOOL_CENSUS_OFFC: i64 = 2
61const NX_TOOL_CENSUS_AUTO_GENERATED: i64 = 3
62const NX_TOOL_CENSUS_HAND_WRITTEN_DECLARED: i64 = 4
63const NX_TOOL_CENSUS_HAND_WRITTEN_UNDECLARED: i64 = 5
64const NX_TOOL_CENSUS_N: i64 = 6
65
66func nx_tool_census_kind_is_valid(k: i64) -> i64 {
67 if k < 0 { return 0 }
68 if k >= NX_TOOL_CENSUS_N { return 0 }
69 return 1
70}
71
72func nx_tool_census_kind_name(k: i64) -> *u8 {
73 if k == NX_TOOL_CENSUS_NONE { return "NONE" }
74 if k == NX_TOOL_CENSUS_TEST { return "TEST" }
75 if k == NX_TOOL_CENSUS_OFFC { return "OFFC" }
76 if k == NX_TOOL_CENSUS_AUTO_GENERATED { return "AUTO_GENERATED" }
77 if k == NX_TOOL_CENSUS_HAND_WRITTEN_DECLARED { return "HAND_WRITTEN_DECLARED" }
78 if k == NX_TOOL_CENSUS_HAND_WRITTEN_UNDECLARED { return "HAND_WRITTEN_UNDECLARED" }
79 return "UNKNOWN"
80}
81
82// A kind contributes to the substrate-honesty gate iff it's a
83// HAND_WRITTEN_* (declared or undeclared). TEST + OFFC +
84// AUTO_GENERATED are excluded from the gate per the cardinal
85// (autogen test corpus has its own legitimacy; tests are caller-
86// authored; offc are compile-time probes).
87func nx_tool_census_kind_counts_toward_gate(k: i64) -> i64 {
88 if k == NX_TOOL_CENSUS_HAND_WRITTEN_DECLARED { return 1 }
89 if k == NX_TOOL_CENSUS_HAND_WRITTEN_UNDECLARED { return 1 }
90 return 0
91}
92
93// ===== Filename suffix / prefix matchers ==========================
94//
95// Test suffix: name ends with "_test.nx". OFFC prefix: name
96// starts with "_offc" (the substrate's compile-probe convention).
97
98func _census_name_is_test(name: *u8, n_name: i64) -> i64 {
99 let suffix: *u8 = "_test.nx"
100 return nx_str_ends_with(name, n_name, suffix, 8)
101}
102
103func _census_name_is_offc(name: *u8, n_name: i64) -> i64 {
104 let prefix: *u8 = "_offc"
105 return nx_str_starts_with(name, n_name, prefix, 5)
106}
107
108// ===== Header-content patterns =====================================
109//
110// Substrate convention: lineage declarations + AUTO-GENERATED
111// markers live in the file's header comment block (first ~4 KiB).
112// nx_grep_any composes the canonical substring match.
113
114// Returns 1 if the header content contains any AUTO-GENERATED marker.
115func _census_content_is_autogen(content: *u8, n_content: i64) -> i64 {
116 if nx_grep_any(content, n_content, "AUTO-GENERATED", 14) == 1 { return 1 }
117 if nx_grep_any(content, n_content, "auto-generated", 14) == 1 { return 1 }
118 if nx_grep_any(content, n_content, "AUTOGEN", 7) == 1 { return 1 }
119 return 0
120}
121
122// Returns 1 if the header content contains any lineage declaration:
123// Composes: Composes [[
124// Supersedes: Supersedes [[
125// Distinct from .* because (we use the literal "Distinct from"
126// + "because" check; full regex at S2)
127// Canonical: Canonical [[
128// genealogy_id: lineage_id:
129func _census_content_has_lineage(content: *u8, n_content: i64) -> i64 {
130 if nx_grep_any(content, n_content, "Composes:", 9) == 1 { return 1 }
131 if nx_grep_any(content, n_content, "Composes [[", 11) == 1 { return 1 }
132 if nx_grep_any(content, n_content, "Supersedes:", 11) == 1 { return 1 }
133 if nx_grep_any(content, n_content, "Supersedes [[", 13) == 1 { return 1 }
134 if nx_grep_any(content, n_content, "Canonical:", 10) == 1 { return 1 }
135 if nx_grep_any(content, n_content, "Canonical [[", 12) == 1 { return 1 }
136 if nx_grep_any(content, n_content, "genealogy_id:", 13) == 1 { return 1 }
137 if nx_grep_any(content, n_content, "lineage_id:", 11) == 1 { return 1 }
138 // "Distinct from X because" -- we conservatively require BOTH
139 // tokens; full regex tightening lands at nx_grep S2 (regex).
140 if nx_grep_any(content, n_content, "Distinct from", 13) == 1 {
141 if nx_grep_any(content, n_content, "because", 7) == 1 { return 1 }
142 }
143 return 0
144}
145
146// ===== Universal classifier =======================================
147//
148// Given (filename, filename_len, header_content, header_len)
149// returns one of NX_TOOL_CENSUS_* sealed-enum values. Order
150// reflects classification precedence:
151// 1. TEST (filename suffix wins)
152// 2. OFFC (filename prefix)
153// 3. AUTO_GENERATED (content marker)
154// 4. HAND_WRITTEN_DECLARED (lineage in content)
155// 5. HAND_WRITTEN_UNDECLARED (default for non-test/offc/autogen)
156
157func nx_tool_census_classify(
158 name: *u8, n_name: i64,
159 content: *u8, n_content: i64
160) -> i64 {
161 if _census_name_is_test(name, n_name) == 1 {
162 return NX_TOOL_CENSUS_TEST
163 }
164 if _census_name_is_offc(name, n_name) == 1 {
165 return NX_TOOL_CENSUS_OFFC
166 }
167 if _census_content_is_autogen(content, n_content) == 1 {
168 return NX_TOOL_CENSUS_AUTO_GENERATED
169 }
170 if _census_content_has_lineage(content, n_content) == 1 {
171 return NX_TOOL_CENSUS_HAND_WRITTEN_DECLARED
172 }
173 return NX_TOOL_CENSUS_HAND_WRITTEN_UNDECLARED
174}
175
176// ===== Aggregator =================================================
177//
178// NxToolCensusCounts struct: caller initializes to zero, then for
179// each classified file calls nx_tool_census_accumulate to bump the
180// appropriate count. Final stats derivable as needed.
181
182struct NxToolCensusCounts {
183 n_total: i64,
184 n_test: i64,
185 n_offc: i64,
186 n_auto_generated: i64,
187 n_hand_written: i64, // = declared + undeclared
188 n_declared: i64,
189 n_undeclared: i64,
190}
191
192func nx_tool_census_counts_init(c: *NxToolCensusCounts) {
193 c.n_total = 0
194 c.n_test = 0
195 c.n_offc = 0
196 c.n_auto_generated = 0
197 c.n_hand_written = 0
198 c.n_declared = 0
199 c.n_undeclared = 0
200}
201
202func nx_tool_census_accumulate(c: *NxToolCensusCounts, kind: i64) -> i64 {
203 if nx_tool_census_kind_is_valid(kind) != 1 { return -1 }
204 c.n_total = c.n_total + 1
205 if kind == NX_TOOL_CENSUS_TEST { c.n_test = c.n_test + 1 }
206 if kind == NX_TOOL_CENSUS_OFFC { c.n_offc = c.n_offc + 1 }
207 if kind == NX_TOOL_CENSUS_AUTO_GENERATED { c.n_auto_generated = c.n_auto_generated + 1 }
208 if kind == NX_TOOL_CENSUS_HAND_WRITTEN_DECLARED {
209 c.n_hand_written = c.n_hand_written + 1
210 c.n_declared = c.n_declared + 1
211 }
212 if kind == NX_TOOL_CENSUS_HAND_WRITTEN_UNDECLARED {
213 c.n_hand_written = c.n_hand_written + 1
214 c.n_undeclared = c.n_undeclared + 1
215 }
216 return 0
217}
218
219// Percent declared (of hand_written; 0..100). Returns 0 if
220// hand_written is zero (no division-by-zero; INCONCLUSIVE shape).
221func nx_tool_census_pct_declared(c: *NxToolCensusCounts) -> i64 {
222 if c.n_hand_written <= 0 { return 0 }
223 return (c.n_declared * 100) / c.n_hand_written
224}
225
226// ===== Path join =================================================
227//
228// Build "<dir>/<name>" into out buffer. Returns total bytes written
229// (or -1 if out_cap exhausted). Substrate-honest no silent overrun.
230
231func _census_path_join(
232 dir: *u8, n_dir: i64,
233 name: *u8, n_name: i64,
234 out: *u8, out_cap: i64
235) -> i64 {
236 if n_dir + 1 + n_name + 1 > out_cap { return -1 }
237 var i: i64 = 0
238 while i < n_dir {
239 out[i] = dir[i]
240 i = i + 1
241 }
242 out[n_dir] = 47 // '/'
243 var j: i64 = 0
244 while j < n_name {
245 out[n_dir + 1 + j] = name[j]
246 j = j + 1
247 }
248 out[n_dir + 1 + n_name] = 0 // NUL terminator for sys_openat
249 return n_dir + 1 + n_name
250}
251
252// ===== Walk directory + classify + accumulate ====================
253//
254// Opens dir_path, lists entries, for each regular file reads up to
255// 4 KiB of header + classifies via nx_tool_census_classify +
256// accumulates into counts. Composes nx_dir_list (LIST) +
257// sys_openat_rd / sys_read / sys_close (READ) +
258// nx_tool_census_classify (CLASSIFY) -- all bits-up NishiLang;
259// no bash/grep/xargs in the LOGIC path.
260//
261// Returns 0 on success; negative on syscall / dir error.
262//
263// This is the FIRST end-to-end bits-up substrate audit primitive.
264// Cardinal demonstrated: classification logic + directory walking
265// + per-file syscall I/O all in NishiLang. S6 will add recursive
266// descent + parallel walks; this S5 first stone is single-level.
267
268func nx_tool_census_walk_dir(
269 dir_path: *u8, n_dir_path: i64,
270 counts: *NxToolCensusCounts
271) -> i64 {
272 // 512 rows × 32B + 64KB name arena = 80KB total. Lifts the
273 // prior 64-row cap so substrate-wide audits (runtime/ ~1614
274 // primitives + docs/ ~278 .md files) classify in one pass.
275 // Substrate-honest: if a dir genuinely has >512 entries,
276 // result.verdict = NX_DIR_TRUNCATED + we accept TRUNCATED.
277 let rows_buf: *u8 = sys_mmap(512 * NX_DIR_ROW_BYTES + 64)
278 let rows: *NxDirRow = rows_buf as *NxDirRow
279 let name_arena: *u8 = sys_mmap(65536)
280 let result_buf: *u8 = sys_mmap(NX_DIR_RESULT_BYTES + 16)
281 let result: *NxDirResult = result_buf as *NxDirResult
282
283 let lc: i64 = nx_dir_list(dir_path, rows, 512, name_arena, 65536, 0, result)
284 if result.verdict != NX_DIR_OK {
285 if result.verdict != NX_DIR_TRUNCATED { return -10 }
286 // TRUNCATED is acceptable for the first stone; we just count
287 // what was filled and surface the verdict to the caller via
288 // counts.n_total + a downstream substrate-honest report.
289 }
290
291 // Per-file header buffer. 4 KiB is enough for lineage / autogen
292 // markers which live in the file's top comment block.
293 let hdr_buf: *u8 = sys_mmap(4096)
294 let path_buf: *u8 = sys_mmap(1024)
295
296 var i: i64 = 0
297 while i < result.n_filled {
298 let row: *NxDirRow = nx_dir_row_at(rows, i)
299 // Skip dotlike + non-regular files. Dirs / symlinks aren't
300 // classified at this layer (S6 recursive walker handles dirs).
301 if row.is_dotlike == 1 {
302 i = i + 1
303 continue
304 }
305 if row.dtype != NX_DT_REG {
306 i = i + 1
307 continue
308 }
309
310 // Build full path and open file.
311 let p_len: i64 = _census_path_join(
312 dir_path, n_dir_path,
313 row.name_ptr, row.name_len,
314 path_buf, 1024
315 )
316 if p_len < 0 {
317 i = i + 1
318 continue
319 }
320 let fd: i64 = sys_openat_rd(path_buf)
321 if fd < 0 {
322 // File unreadable -- classify as UNDECLARED so it counts
323 // toward the gate (substrate-honest: don't silently skip)
324 nx_tool_census_accumulate(counts, NX_TOOL_CENSUS_HAND_WRITTEN_UNDECLARED)
325 i = i + 1
326 continue
327 }
328
329 let n_read: i64 = sys_read(fd, hdr_buf, 4096)
330 sys_close(fd)
331 var content_len: i64 = n_read
332 if content_len < 0 { content_len = 0 }
333
334 let kind: i64 = nx_tool_census_classify(
335 row.name_ptr, row.name_len,
336 hdr_buf, content_len
337 )
338 nx_tool_census_accumulate(counts, kind)
339 i = i + 1
340 }
341 return 0
342}
343
344// ===== Ecosystem citizenship: ETG attestation + flow signal =======
345//
346// Per [[feedback-audits-are-ecosystem-citizens-not-islands]] cardinal:
347// every audit primitive's verdict must (1) emit an NxEtgEntry into
348// the substrate's attestation chain so ProvenanceLinked replay can
349// answer "what did the tool census report at time T on silicon S?",
350// and (2) populate at least one NxFlowState condition signal so the
351// substrate's flow verdict reflects audit-tier truth.
352
353// Map tool-census verdict to an ETG outcome:
354// pct >= 50 declared -> CONFIRMED (substrate-healthy)
355// 20 <= pct < 50 declared -> INCONCLUSIVE (mid-band; needs work)
356// pct < 20 declared -> FALSIFIED (proliferation pathology)
357func nx_tool_census_to_etg_outcome(counts: *NxToolCensusCounts) -> i64 {
358 if counts.n_hand_written <= 0 { return NX_ETG_OUTCOME_INCONCLUSIVE }
359 let pct: i64 = nx_tool_census_pct_declared(counts)
360 if pct >= 50 { return NX_ETG_OUTCOME_CONFIRMED }
361 if pct >= 20 { return NX_ETG_OUTCOME_INCONCLUSIVE }
362 return NX_ETG_OUTCOME_FALSIFIED
363}
364
365// Emit an NxEtgEntry attesting the tool-census verdict. The
366// attestation lets future audits (Byzantine N-of-M, conductor
367// cost-model, flow-state classifier) consume the substrate's
368// audit truth through one shared structure.
369//
370// Mapping:
371// probe_kind = NX_ETG_PROBE_AUDIT_TOOL_GENEALOGY
372// claim_source = NX_ETG_CLAIM_PRIOR_CALIBRATION
373// claim_value = n_hand_written (the denominator)
374// measurement_value = n_declared (the numerator)
375// outcome = derived from pct_declared band
376func nx_tool_census_attest(
377 counts: *NxToolCensusCounts,
378 entry: *NxEtgEntry,
379 silicon_serial: i64,
380 selector_version: i64,
381 timestamp_q14: i64
382) -> i64 {
383 let outcome: i64 = nx_tool_census_to_etg_outcome(counts)
384 return nx_etg_entry_init(
385 entry,
386 silicon_serial,
387 NX_ETG_PROBE_AUDIT_TOOL_GENEALOGY,
388 NX_ETG_CLAIM_PRIOR_CALIBRATION,
389 counts.n_hand_written,
390 counts.n_declared,
391 outcome,
392 selector_version,
393 timestamp_q14
394 )
395}
396
397// Flow signal: clear_goals (Csikszentmihalyi condition #1).
398// Substrate has clear goals iff substrate primitives have lineage
399// declared at gate threshold. Returns 1/0 for direct assignment
400// to NxFlowState.cond_clear_goals.
401func nx_tool_census_signal_clear_goals(counts: *NxToolCensusCounts) -> i64 {
402 if counts.n_hand_written <= 0 { return 0 }
403 let pct: i64 = nx_tool_census_pct_declared(counts)
404 // 50% declared is the substrate-honest threshold (matches
405 // CONFIRMED band in the ETG mapping).
406 if pct >= 50 { return 1 }
407 return 0
408}