code wiki / _hdl_build / nx_tpl_site.nx
nx_tpl_site.nx source
↩ module page · 345 lines · 14231 B
1// nx_tpl_site.nx -- R-WHOLESITE of the SITE FACTORY: ingest a COMPLETE multi-page template/site as ONE
2// site-graph pattern ("pass it along wholesale"). Walks a site directory (root + one subdir level), and for
3// every page keeps the abstract structure (archetype + section sequence, via the R4 scanners) plus the NAV
4// GRAPH (internal href edges between the site's own pages). The whole site collapses to ONE content-addressed
5// `site:<hash>` record (canonical = byte-lex-sorted per-page arch\x1fseq lines + bucketed edge count) in the
6// SAME tplcorpus store -- so the corpus now holds three granularities: section BRICKS (part:), page PATTERNS
7// (pat:), and whole SITE GRAPHS (site:). Budget-guarded (max_pages trap), idempotent, structure-only.
8// usage: nx_tpl_site <site-dir> <corpus-prefix> [max_pages]
9// license_tier: ORIGINAL
10import "nx_tpl_ingest.nx"
11const TS_MAGIC_131072: i64 = 131072
12const TS_MAGIC_1024: i64 = 1024
13
14const TS_MAXPG: i64 = 64
15const TS_NAMEW: i64 = 96
16const TS_SEQW: i64 = 200
17const TS_RECCAP: i64 = 8192
18const TS_MAXE: i64 = 96
19
20// collect .html page names from dir (root + one subdir level, "sub/name"), stride TS_NAMEW. returns count.
21func ts_collect(dir: *u8, names: *u8, maxn: i64) -> i64 {
22 let dbuf: *u8 = sys_mmap(TS_MAGIC_131072)
23 let subs: *u8 = sys_mmap(8 * TS_NAMEW)
24 var nsub: i64 = 0
25 var cnt: i64 = 0
26 let fd: i64 = sys_openat_rd(dir)
27 if fd < 0 { return 0 - 1 }
28 var go: i64 = 1
29 while go == 1 {
30 let nr: i64 = sys_getdents64(fd, dbuf, TS_MAGIC_131072)
31 if nr <= 0 { go = 0 } else {
32 var off: i64 = 0
33 while off < nr {
34 let rec: *u8 = (dbuf as i64 + off) as *u8
35 let ty: i64 = dirent_type(rec)
36 let nm: *u8 = dirent_name(rec)
37 off = off + dirent_reclen(rec)
38 if ty == 4 {
39 if (nm[0] as i64) != 46 { if nsub < 8 {
40 let sd: *u8 = (subs as i64 + nsub * TS_NAMEW) as *u8
41 var k: i64 = 0
42 while nm[k] != (0 as u8) { sd[k] = nm[k]; k = k + 1 }
43 sd[k] = 0 as u8
44 nsub = nsub + 1
45 } }
46 } else {
47 if ti_suffix(nm, ".html" as *u8) == 1 { if cnt < maxn {
48 let d: *u8 = (names as i64 + cnt * TS_NAMEW) as *u8
49 var k2: i64 = 0
50 while nm[k2] != (0 as u8) { d[k2] = nm[k2]; k2 = k2 + 1 }
51 d[k2] = 0 as u8
52 cnt = cnt + 1
53 } }
54 }
55 }
56 }
57 }
58 sys_close(fd)
59 // one subdir level
60 let sp: *u8 = sys_mmap(TS_MAGIC_1024)
61 var si: i64 = 0
62 while si < nsub {
63 let sd2: *u8 = (subs as i64 + si * TS_NAMEW) as *u8
64 var po: i64 = ti_cat(sp, 0, dir)
65 po = ti_cat(sp, po, "/" as *u8)
66 po = ti_cat(sp, po, sd2)
67 sp[po] = 0 as u8
68 let fd2: i64 = sys_openat_rd(sp)
69 if fd2 >= 0 {
70 var go2: i64 = 1
71 while go2 == 1 {
72 let nr2: i64 = sys_getdents64(fd2, dbuf, TS_MAGIC_131072)
73 if nr2 <= 0 { go2 = 0 } else {
74 var off2: i64 = 0
75 while off2 < nr2 {
76 let rec2: *u8 = (dbuf as i64 + off2) as *u8
77 let ty2: i64 = dirent_type(rec2)
78 let nm2: *u8 = dirent_name(rec2)
79 off2 = off2 + dirent_reclen(rec2)
80 if ty2 != 4 { if ti_suffix(nm2, ".html" as *u8) == 1 { if cnt < maxn {
81 let d2: *u8 = (names as i64 + cnt * TS_NAMEW) as *u8
82 var o3: i64 = ti_cat(d2, 0, sd2)
83 o3 = ti_cat(d2, o3, "/" as *u8)
84 o3 = ti_cat(d2, o3, nm2)
85 d2[o3] = 0 as u8
86 cnt = cnt + 1
87 } } }
88 }
89 }
90 }
91 sys_close(fd2)
92 }
93 si = si + 1
94 }
95 return cnt
96}
97
98// byte-lex insertion sort of TS_NAMEW-stride strings (canonical page order)
99func ts_sort(names: *u8, n: i64) -> i64 {
100 let tmp: *u8 = sys_mmap(TS_NAMEW)
101 var i: i64 = 1
102 while i < n {
103 var j: i64 = i
104 var go: i64 = 1
105 while go == 1 {
106 if j <= 0 { go = 0 } else {
107 let a: *u8 = (names as i64 + (j - 1) * TS_NAMEW) as *u8
108 let b: *u8 = (names as i64 + j * TS_NAMEW) as *u8
109 var k: i64 = 0
110 var cmp: i64 = 0
111 var sc: i64 = 1
112 while sc == 1 {
113 let ca: i64 = a[k] as i64
114 let cb: i64 = b[k] as i64
115 if ca != cb { cmp = ca - cb; sc = 0 } else { if ca == 0 { sc = 0 } else { k = k + 1 } }
116 }
117 if cmp > 0 {
118 var t: i64 = 0
119 while t < TS_NAMEW { tmp[t] = a[t]; t = t + 1 }
120 t = 0
121 while t < TS_NAMEW { a[t] = b[t]; t = t + 1 }
122 t = 0
123 while t < TS_NAMEW { b[t] = tmp[t]; t = t + 1 }
124 j = j - 1
125 } else { go = 0 }
126 }
127 }
128 i = i + 1
129 }
130 return 0
131}
132
133func ts_nameeq(a: *u8, b: *u8) -> i64 { var i: i64=0; while 1==1 { if (a[i] as i64)!=(b[i] as i64) { return 0 } if a[i]==(0 as u8) { return 1 } i=i+1 } return 1 }
134// find a normalized href target among the site's pages; -1 = external/unknown
135func ts_page_idx(t: *u8, names: *u8, npg: i64) -> i64 {
136 var i: i64 = 0
137 while i < npg { if ts_nameeq(t, (names as i64 + i * TS_NAMEW) as *u8) == 1 { return i } i = i + 1 }
138 return 0 - 1
139}
140// scan page for internal nav edges: href="..." resolving to another collected page. returns edges added.
141func ts_edges(h: *u8, n: i64, from: i64, names: *u8, npg: i64, ef: *i64, et: *i64, ecnt: *i64) -> i64 {
142 let tgt: *u8 = sys_mmap(256)
143 var added: i64 = 0
144 var i: i64 = 0
145 while i + 8 <= n {
146 let hp: i64 = ti_find_ci(h, i, n, "href=\"" as *u8)
147 if hp < 0 { i = n } else {
148 var s: i64 = hp + 6
149 var o: i64 = 0
150 var sc: i64 = 1
151 while sc == 1 {
152 if s >= n { sc = 0 } else {
153 let c: i64 = h[s] as i64
154 if c == 34 { sc = 0 } else {
155 if c == 35 { sc = 0 } else {
156 if c == 63 { sc = 0 } else {
157 if o < 250 { tgt[o] = c as u8; o = o + 1 }
158 s = s + 1
159 }
160 }
161 }
162 }
163 }
164 tgt[o] = 0 as u8
165 // normalize: skip schemes/mailto; strip leading '/'; empty -> index.html
166 var ok: i64 = 1
167 if o >= 4 { if tgt[0]==(104 as u8) { if tgt[1]==(116 as u8) { ok = 0 } } } // http...
168 if o >= 7 { if tgt[0]==(109 as u8) { if tgt[1]==(97 as u8) { ok = 0 } } } // mailto...
169 var t0: i64 = 0
170 if ok == 1 {
171 if o > 0 { if tgt[0] == (47 as u8) { t0 = 1 } }
172 let tp: *u8 = (tgt as i64 + t0) as *u8
173 if tp[0] == (0 as u8) { tp[0] = 105 as u8 } // "" -> we compare against "index.html" below via literal
174 var ti2: i64 = ts_page_idx(tp, names, npg)
175 if tp[0] == (105 as u8) { if tp[1] == (0 as u8) { ti2 = ts_page_idx("index.html" as *u8, names, npg) } }
176 if ti2 >= 0 { if ti2 != from { if ecnt[0] < TS_MAXE {
177 ef[ecnt[0]] = from
178 et[ecnt[0]] = ti2
179 ecnt[0] = ecnt[0] + 1
180 added = added + 1
181 } } }
182 }
183 i = hp + 6
184 }
185 }
186 return added
187}
188
189func tp_bucket_site(v: i64) -> i64 {
190 if v == 0 { return 0 }
191 if v <= 6 { return 1 }
192 if v <= 20 { return 2 }
193 return 3
194}
195
196// THE WHOLE-SITE INGEST. counts: [0]stored [1]dups [2]pages [3]edges [4]trapped [5]segs. keyout = the site key.
197func ts_ingest(dir: *u8, prefix: *u8, max_pages: i64, counts: *i64, keyout: *u8) -> i64 {
198 var ci: i64 = 0
199 while ci < 6 { counts[ci] = 0; ci = ci + 1 }
200 keyout[0] = 0 as u8
201 let names: *u8 = sys_mmap(TS_MAXPG * TS_NAMEW)
202 var npg: i64 = ts_collect(dir, names, TS_MAXPG)
203 if npg < 0 { ti_puts("tpl-site: cannot open dir " as *u8); ti_puts(dir); ti_puts("\n" as *u8); return 0 - 1 }
204 if npg == 0 { return 0 }
205 if npg > max_pages { counts[4] = 1; npg = max_pages }
206 ts_sort(names, npg)
207 counts[2] = npg
208 // per-page: read, classify, seq, edges
209 let archs: *i64 = sys_mmap(TS_MAXPG * 8) as *i64
210 let seqs: *u8 = sys_mmap(TS_MAXPG * TS_SEQW)
211 let ef: *i64 = sys_mmap(TS_MAXE * 8) as *i64
212 let et: *i64 = sys_mmap(TS_MAXE * 8) as *i64
213 let ecnt: *i64 = sys_mmap(16) as *i64
214 ecnt[0] = 0
215 let path: *u8 = sys_mmap(TS_MAGIC_1024)
216 let seq: *u8 = sys_mmap(256)
217 let szp: *i64 = sys_mmap(16) as *i64
218 var p: i64 = 0
219 while p < npg {
220 let nm: *u8 = (names as i64 + p * TS_NAMEW) as *u8
221 var po: i64 = ti_cat(path, 0, dir)
222 po = ti_cat(path, po, "/" as *u8)
223 po = ti_cat(path, po, nm)
224 path[po] = 0 as u8
225 let raw: *u8 = ss_readall(path, szp)
226 let rn: i64 = szp[0]
227 archs[p] = 7
228 let sd: *u8 = (seqs as i64 + p * TS_SEQW) as *u8
229 sd[0] = 0 as u8
230 if rn >= 40 {
231 archs[p] = ti_classify(raw, rn)
232 ti_seq(raw, rn, seq)
233 var k: i64 = 0
234 while seq[k] != (0 as u8) { if k < TS_SEQW - 2 { sd[k] = seq[k] } k = k + 1 }
235 if k > TS_SEQW - 2 { k = TS_SEQW - 2 }
236 sd[k] = 0 as u8
237 ts_edges(raw, rn, p, names, npg, ef, et, ecnt)
238 }
239 p = p + 1
240 }
241 counts[3] = ecnt[0]
242 // canonical bytes -> content-addressed site key
243 let canon: *u8 = sys_mmap(TS_RECCAP)
244 let anb: *u8 = sys_mmap(32)
245 var co: i64 = 0
246 p = 0
247 while p < npg {
248 ti_arch_name(archs[p], anb)
249 co = ti_cat(canon, co, anb)
250 canon[co] = 31 as u8; co = co + 1
251 co = ti_cat(canon, co, (seqs as i64 + p * TS_SEQW) as *u8)
252 canon[co] = 10 as u8; co = co + 1
253 p = p + 1
254 }
255 co = ti_cat(canon, co, "E" as *u8)
256 co = ti_catn(canon, co, tp_bucket_site(ecnt[0]))
257 let hsh: i64 = ti_hash(canon, co)
258 var ko: i64 = ti_cat(keyout, 0, "site:" as *u8)
259 ko = ti_catn(keyout, ko, hsh)
260 keyout[ko] = 0 as u8
261 // dedup vs store
262 let hh: *i64 = ss_open(prefix)
263 let pbox: *i64 = sys_mmap(16) as *i64
264 let lbox: *i64 = sys_mmap(16) as *i64
265 if (hh as i64) != 0 { if ss_hget(hh, keyout, pbox, lbox) == 1 { counts[1] = 1; return 0 } }
266 // record
267 let rec: *u8 = sys_mmap(TS_RECCAP)
268 var ro: i64 = ti_cat(rec, 0, "pages|" as *u8)
269 ro = ti_catn(rec, ro, npg)
270 ro = ti_cat(rec, ro, "\n" as *u8)
271 p = 0
272 while p < npg {
273 ti_arch_name(archs[p], anb)
274 ro = ti_cat(rec, ro, "page|" as *u8)
275 ro = ti_cat(rec, ro, (names as i64 + p * TS_NAMEW) as *u8)
276 ro = ti_cat(rec, ro, "|" as *u8)
277 ro = ti_cat(rec, ro, anb)
278 ro = ti_cat(rec, ro, "|" as *u8)
279 ro = ti_cat(rec, ro, (seqs as i64 + p * TS_SEQW) as *u8)
280 ro = ti_cat(rec, ro, "\n" as *u8)
281 p = p + 1
282 }
283 var e: i64 = 0
284 while e < ecnt[0] {
285 if ro < TS_RECCAP - 260 {
286 ro = ti_cat(rec, ro, "nav|" as *u8)
287 ro = ti_cat(rec, ro, (names as i64 + ef[e] * TS_NAMEW) as *u8)
288 ro = ti_cat(rec, ro, "|" as *u8)
289 ro = ti_cat(rec, ro, (names as i64 + et[e] * TS_NAMEW) as *u8)
290 ro = ti_cat(rec, ro, "\n" as *u8)
291 }
292 e = e + 1
293 }
294 ro = ti_cat(rec, ro, "edges|" as *u8)
295 ro = ti_catn(rec, ro, ecnt[0])
296 ro = ti_cat(rec, ro, "\nsrc|" as *u8)
297 ro = ti_cat(rec, ro, dir)
298 ro = ti_cat(rec, ro, "\n" as *u8)
299 rec[ro] = 0 as u8
300 // fresh segid append
301 let segs: *i64 = sys_mmap(256 * 8) as *i64
302 let nseg: i64 = ss_manifest_file(prefix, "manifest.txt" as *u8, segs)
303 var segid: i64 = 1
304 var si: i64 = 0
305 // ROOT FIX seq1730 (id 1785450987): segs[] holds POINTERS to seg-<id> name strings
306 // (nx_seg_store.nx:1234 stores segs[cnt] = name as i64), NOT ids -- so segs[si] + 1 produced
307 // MMAP_ADDRESS+1 (~1.4e14), the pointer-shaped poison that PINS a plane forever: every later
308 // epoch id sorts BELOW it in supersede order and its rows are silently shadowed while rc=0.
309 // Parse the DIGITS, as nx_web_shard_compact.nx:55-58 already does on this SAME array.
310 while si < nseg {
311 let sg_nm: *u8 = segs[si] as *u8
312 var sg_v: i64 = 0
313 var sg_ci: i64 = 0
314 while sg_nm[sg_ci] != (0 as u8) { let sg_c: i64 = sg_nm[sg_ci] as i64; if sg_c >= 48 { if sg_c <= 57 { sg_v = sg_v * 10 + (sg_c - 48) } } sg_ci = sg_ci + 1 }
315 if sg_v >= segid { segid = sg_v + 1 }
316 si = si + 1
317 }
318 let w: *i64 = ss_begin()
319 if ss_add(w, 1, keyout, rec, ro) >= 0 {
320 if ss_commit(prefix, w, segid) == 0 { counts[5] = counts[5] + 1; counts[0] = 1 }
321 }
322 return counts[0]
323}
324
325func main(argc: i64, argv: *i64) -> i64 {
326 if argc < 3 { ti_puts("usage: nx_tpl_site <site-dir> <corpus-prefix> [max_pages]\n" as *u8); return 2 }
327 let dir: *u8 = argv[1] as *u8
328 let prefix: *u8 = argv[2] as *u8
329 var maxp: i64 = TS_MAXPG
330 if argc >= 4 { let mm: i64 = ti_atoi(argv[3] as *u8); if mm > 0 { maxp = mm } }
331 if maxp > TS_MAXPG { maxp = TS_MAXPG }
332 let counts: *i64 = sys_mmap(64) as *i64
333 let key: *u8 = sys_mmap(64)
334 let rc: i64 = ts_ingest(dir, prefix, maxp, counts, key)
335 ti_puts("TPL-SITE dir=" as *u8); ti_puts(dir)
336 ti_puts(" pages=" as *u8); ti_num(counts[2])
337 ti_puts(" edges=" as *u8); ti_num(counts[3])
338 ti_puts(" stored=" as *u8); ti_num(counts[0])
339 ti_puts(" dup=" as *u8); ti_num(counts[1])
340 ti_puts(" trapped=" as *u8); ti_num(counts[4])
341 ti_puts(" key=" as *u8); ti_puts(key)
342 ti_puts("\n" as *u8)
343 if rc < 0 { return 1 }
344 return 0
345}