nx_pipeline_recursive_walker.nx source
↩ module page · 261 lines · 10981 B
1// nx_pipeline_recursive_walker.nx -- V2.5 recursive directory walker
2// expanding pipeline classification from 24 hardcoded paths to all
3// substrate artifacts under operator-supplied roots.
4//
5// Per operator cardinal 2026-05-27: "lots and lots of parts of this on
6// the gitea" -- this walker discovers ALL .nx / .md / .sh files in the
7// substrate without operator maintaining a hardcoded list.
8//
9// COMPOSES (per "avoid duplicate primitives" cardinal):
10// nx_dir (sovereign getdents64 wrapper; NxDirRow + nx_dir_list)
11// wiki/nx_pipeline_walker (V2.0 P-1; per-file taxonomy + stage derivation)
12// wiki/nx_artifact_store (V2.0 P-2; data destination)
13// nx_syscalls (sys_mmap)
14//
15// COMPOSED BY:
16// wiki/nx_wiki_main (startup; replaces hardcoded 24-path list)
17//
18// V2.5 SCOPE:
19// - Recursive walk bounded by max-depth + max-entries
20// - Per-extension filter (.nx / .md / .sh; skip everything else)
21// - Skip dot-like entries (., .., .git, .DS_Store hidden)
22// - Skip _site/ subdirs (generated content; not source artifacts)
23// - Per-root call; caller iterates over roots
24//
25// V3+ SCOPE (TODO):
26// - Operator-configurable exclude patterns (.gitignore parser)
27// - Symlink loop detection
28// - Incremental walk (only re-process changed; content-hash diff)
29// - Per-file last-commit-hash via nx_git_walk
30//
31// Status: V2.5. 2026-05-27.
32
33import "nx_syscalls.nx"
34import "nx_dir.nx"
35import "wiki/nx_pipeline_walker.nx"
36import "wiki/nx_artifact_store.nx"
37import "nx_html_extract.nx"
38
39// ===== Sealed verdict surface (codes 3000-3009) =================================================
40const NX_PREC_OK: i64 = 0
41const NX_PREC_BAD_INPUT: i64 = 3000
42const NX_PREC_DIR_LIST_FAILED: i64 = 3001
43const NX_PREC_MAX_DEPTH_EXCEEDED: i64 = 3002
44const NX_PREC_STORE_FULL: i64 = 3003
45const NX_PREC_LOOP_BUDGET: i64 = 3004
46const NX_PREC_PATH_TOO_LONG: i64 = 3005
47
48// ===== Named constants (M7) =================================================
49const NX_PREC_MAX_DEPTH: i64 = 8
50const NX_PREC_MAX_ENTRIES_PER_DIR: i64 = 4096
51const NX_PREC_PATH_BUF_CAP: i64 = 1024
52const NX_PREC_NAME_ARENA_CAP: i64 = 65536 // 64 KB per dir-list batch
53const NX_PREC_LOOP_BUDGET_CAP: i64 = 100000
54
55// ===== Extension filter =================================================
56//
57// Returns 1 if filename ends with .nx, .md, or .sh; 0 otherwise.
58// Bounded; case-sensitive (substrate convention).
59
60func nx_prec_is_eligible_ext(name: *u8, name_n: i64) -> i64 {
61 if name_n < 3 { return 0 }
62 let suffix_off: i64 = name_n - 3
63 // .nx
64 if name[suffix_off] == (0x2E as u8) {
65 if name[suffix_off + 1] == (0x6E as u8) {
66 if name[suffix_off + 2] == (0x78 as u8) { return 1 }
67 }
68 // .md
69 if name[suffix_off + 1] == (0x6D as u8) {
70 if name[suffix_off + 2] == (0x64 as u8) { return 1 }
71 }
72 // .sh
73 if name[suffix_off + 1] == (0x73 as u8) {
74 if name[suffix_off + 2] == (0x68 as u8) { return 1 }
75 }
76 }
77 return 0
78}
79
80// ===== Skip-list: subdir names we don't recurse into =================================================
81//
82// V2.5: skip _site/, .git/, target/, build/, _backup, _old.
83// Conservative; operator can extend in future via config.
84
85func nx_prec_should_skip_subdir(name: *u8, name_n: i64) -> i64 {
86 if name_n < 1 { return 1 }
87 // dot-prefix (hidden / .git / .vscode etc)
88 if name[0] == (0x2E as u8) { return 1 }
89 // _site
90 if name_n == 5 {
91 if nx_he_match_ci(name, 0, 5, "_site" as *u8, 5) == 1 { return 1 }
92 }
93 // _backup (any prefix _backup or contains _backup)
94 if name_n >= 7 {
95 if nx_he_match_ci(name, 0, 7, "_backup" as *u8, 7) == 1 { return 1 }
96 }
97 // _old
98 if name_n == 4 {
99 if nx_he_match_ci(name, 0, 4, "_old" as *u8, 4) == 1 { return 1 }
100 }
101 // node_modules
102 if name_n == 12 {
103 if nx_he_match_ci(name, 0, 12, "node_modules" as *u8, 12) == 1 { return 1 }
104 }
105 // target / build (Rust/C build dirs)
106 if name_n == 6 {
107 if nx_he_match_ci(name, 0, 6, "target" as *u8, 6) == 1 { return 1 }
108 }
109 if name_n == 5 {
110 if nx_he_match_ci(name, 0, 5, "build" as *u8, 5) == 1 { return 1 }
111 }
112 return 0
113}
114
115// ===== Build full path: dir_path + "/" + name =================================================
116//
117// Caller-allocated out_path buffer (NX_PREC_PATH_BUF_CAP).
118// Returns bytes written (>= 0) or -verdict.
119
120func nx_prec_build_path(out_path: *u8, cap: i64,
121 dir_path: *u8, dir_path_n: i64,
122 name: *u8, name_n: i64) -> i64 {
123 if dir_path_n + 1 + name_n + 1 > cap { return 0 - NX_PREC_PATH_TOO_LONG }
124 var i: i64 = 0
125 while i < dir_path_n { out_path[i] = dir_path[i]; i = i + 1 }
126 out_path[dir_path_n] = 0x2F as u8 // '/'
127 var j: i64 = 0
128 while j < name_n { out_path[dir_path_n + 1 + j] = name[j]; j = j + 1 }
129 let total: i64 = dir_path_n + 1 + name_n
130 out_path[total] = 0 as u8 // NUL-terminate for sys_read_file
131 return total
132}
133
134// ===== Recursive walk one directory =================================================
135//
136// For each entry:
137// regular file + eligible extension → nx_pipeline_walker_walk_one
138// subdirectory + not skip-list → recurse (depth+1)
139//
140// Per Cardinal 14 graceful degradation: per-file errors don't abort walk.
141
142func nx_pipeline_recursive_walk(store: *NxArtifactStore,
143 dir_path: *u8, dir_path_n: i64,
144 depth: i64,
145 counters: *i64) -> i64 {
146 if store.valid != 1 { return 0 - NX_PREC_BAD_INPUT }
147 if dir_path_n < 1 { return 0 - NX_PREC_BAD_INPUT }
148 if dir_path_n > NX_PREC_PATH_BUF_CAP - 64 { return 0 - NX_PREC_PATH_TOO_LONG }
149 if depth > NX_PREC_MAX_DEPTH { return 0 - NX_PREC_MAX_DEPTH_EXCEEDED }
150 if (counters as i64) == 0 { return 0 - NX_PREC_BAD_INPUT }
151
152 // counters[0] = files walked; counters[1] = dirs visited; counters[2] = files skipped
153
154 // Allocate per-call scratch (could be hoisted; V2.5 acceptable per-call)
155 let rows_cap: i64 = NX_PREC_MAX_ENTRIES_PER_DIR
156 let rows: *NxDirRow = (sys_mmap(rows_cap * NX_DIR_ROW_BYTES)) as *NxDirRow
157 let name_arena: *u8 = sys_mmap(NX_PREC_NAME_ARENA_CAP)
158 let result: *NxDirResult = (sys_mmap(NX_DIR_RESULT_BYTES)) as *NxDirResult
159
160 counters[1] = counters[1] + 1
161
162 let list_rc: i64 = nx_dir_list(dir_path, rows, rows_cap,
163 name_arena, NX_PREC_NAME_ARENA_CAP,
164 0, // include_dotlike = false
165 result)
166 // Per Cardinal 14: directory list failures (perm denied, ENOENT) don't abort
167 // the entire walk; just skip this directory.
168 if list_rc != NX_DIR_OK { return NX_PREC_OK }
169 if result.verdict != NX_DIR_OK { return NX_PREC_OK }
170
171 let n: i64 = result.n_filled
172 if n < 1 { return NX_PREC_OK }
173
174 // Allocate path-build buffer
175 let path_buf: *u8 = sys_mmap(NX_PREC_PATH_BUF_CAP)
176
177 var i: i64 = 0
178 var iter: i64 = 0
179 while i < n {
180 if iter >= NX_PREC_LOOP_BUDGET_CAP { return 0 - NX_PREC_LOOP_BUDGET }
181 iter = iter + 1
182
183 let row: *NxDirRow = nx_dir_row_at(rows, i)
184 let name_ptr: *u8 = row.name_ptr
185 let name_len: i64 = row.name_len
186
187 if nx_dir_row_is_regular_file(row) == 1 {
188 if nx_prec_is_eligible_ext(name_ptr, name_len) == 1 {
189 let path_n: i64 = nx_prec_build_path(path_buf, NX_PREC_PATH_BUF_CAP,
190 dir_path, dir_path_n,
191 name_ptr, name_len)
192 if path_n > 0 {
193 let walk_rc: i64 = nx_pipeline_walker_walk_one(store, path_buf, path_n)
194 if walk_rc >= 0 { counters[0] = counters[0] + 1 }
195 if walk_rc < 0 { counters[2] = counters[2] + 1 }
196 // Store full -> bail entire recursion
197 if walk_rc == 0 - NX_ART_STORE_FULL { return 0 - NX_PREC_STORE_FULL }
198 }
199 }
200 }
201
202 if nx_dir_row_is_subdirectory(row) == 1 {
203 if row.is_dotlike == 0 {
204 if nx_prec_should_skip_subdir(name_ptr, name_len) == 0 {
205 let sub_path_n: i64 = nx_prec_build_path(path_buf, NX_PREC_PATH_BUF_CAP,
206 dir_path, dir_path_n,
207 name_ptr, name_len)
208 if sub_path_n > 0 {
209 let rec_rc: i64 = nx_pipeline_recursive_walk(store,
210 path_buf, sub_path_n,
211 depth + 1,
212 counters)
213 if rec_rc == 0 - NX_PREC_STORE_FULL { return rec_rc }
214 // Other errors: continue with next entry (graceful degradation)
215 }
216 }
217 }
218 }
219 i = i + 1
220 }
221 return NX_PREC_OK
222}
223
224// ===== Top-level: walk a list of roots =================================================
225//
226// Caller supplies array of NUL-terminated root path pointers + count.
227// Returns NX_PREC_OK on success; first STORE_FULL or LOOP_BUDGET on failure.
228
229func nx_pipeline_recursive_walk_roots(store: *NxArtifactStore,
230 roots: *i64, roots_count: i64,
231 out_counters: *i64) -> i64 {
232 if store.valid != 1 { return 0 - NX_PREC_BAD_INPUT }
233 if (roots as i64) == 0 { return 0 - NX_PREC_BAD_INPUT }
234 if roots_count < 1 { return NX_PREC_OK }
235 if (out_counters as i64) == 0 { return 0 - NX_PREC_BAD_INPUT }
236
237 // counters[0]=files walked; [1]=dirs; [2]=files skipped (errors)
238 out_counters[0] = 0
239 out_counters[1] = 0
240 out_counters[2] = 0
241
242 var i: i64 = 0
243 while i < roots_count {
244 let root_ptr: *u8 = roots[i] as *u8
245 if (root_ptr as i64) != 0 {
246 // Compute length (NUL-terminated)
247 var n: i64 = 0
248 while n < 1024 {
249 if root_ptr[n] == (0 as u8) { n = 1024 + 1 }
250 if n <= 1024 { if n < 1024 { n = n + 1 } }
251 }
252 let real_n: i64 = if n > 1024 then n - 1025 else n
253 let rc: i64 = nx_pipeline_recursive_walk(store, root_ptr, real_n,
254 0, out_counters)
255 // Continue with next root on per-root error (Cardinal 14)
256 if rc == 0 - NX_PREC_STORE_FULL { return rc }
257 }
258 i = i + 1
259 }
260 return NX_PREC_OK
261}