code wiki / _hdl_build / nx_tpl_ingest.nx
nx_tpl_ingest.nx source
↩ module page · 423 lines · 21296 B
1// nx_tpl_ingest.nx -- R4 of the SITE FACTORY: consume the STRUCTURE/FLOW of open free web templates
2// (html5up / freewebsitetemplates / Figma-community exports / any HTML) into a sovereign, content-addressed
3// PATTERN CORPUS -- NOT their files or assets. For each template we extract only uncopyrightable IDEAS:
4// - the page ARCHETYPE (landing / ecommerce / social / media / resume / hub / brochure),
5// - the ORDERED SECTION SEQUENCE (information-architecture / layout flow, by first-occurrence offset),
6// - conversion-funnel signals + accessibility/responsive markers (feeds the award/ADA graders).
7// Each distinct (archetype, section-sequence) collapses to ONE corpus pattern via a deterministic key, so
8// "thousands of templates" become a COMPACT library of DISTINCT layout archetypes that R5 (nx_site_archetype)
9// selects + composes and nx_ui_judge/nx_ui_contrast measure. We emit OUR sovereign markup downstream; we never
10// redistribute a foreign template's bytes -- only the abstract structure (settled legal posture, site-factory R4).
11//
12// BUDGET-GUARDED BY CONSTRUCTION ([[feedback-substrate-generators-must-have-budget-guard]]): declares an upfront
13// ceiling on templates examined AND corpus bytes, TRAPS FAIL-FAST at the ceiling (never runs unbounded), is
14// resumable (skip-if-present, both in-run and cross-run), and reports its footprint. This is exactly the
15// unbounded-generation anti-pattern that once produced the 70 GB waste incident -- it cannot recur here.
16//
17// usage: nx_tpl_ingest <template-dir> [max_templates]
18// Composes the PROVEN writer mechanics of nx_corpus_ingest (fresh-segid batching, skip-if-present) over the
19// content-addressed seg_store. Pure sovereign string processing. license_tier: ORIGINAL
20import "nx_docportal_search_seg.nx"
21const TI_MAGIC_1125899906842597: i64 = 1125899906842597
22const TI_MAGIC_131072: i64 = 131072
23const TI_MAGIC_1024: i64 = 1024
24const TI_MAGIC_5000: i64 = 5000
25const TI_MAGIC_67108864: i64 = 67108864
26
27const TI_ARCH_LANDING: i64 = 1
28const TI_ARCH_ECOM: i64 = 2
29const TI_ARCH_SOCIAL: i64 = 3
30const TI_ARCH_MEDIA: i64 = 4
31const TI_ARCH_RESUME: i64 = 5
32const TI_ARCH_HUB: i64 = 6
33const TI_ARCH_BROCHURE: i64 = 7
34const TI_ARCH_PORTFOLIO: i64 = 8
35const TI_ARCH_GALLERY: i64 = 9
36const TI_ARCH_BLOG: i64 = 10
37
38const TI_RAWCAP: i64 = 2097152 // 2 MiB largest template we read (guard vs absurd files)
39const TI_MINDOC: i64 = 40 // below this = not a real template
40const TI_SEENCAP: i64 = 20000 // in-run dedup memory (bounded)
41const TI_KEYW: i64 = 48 // bytes per stored key slot
42const TI_RECCAP: i64 = 2048 // per-pattern record cap
43
44func ti_len(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
45func ti_lc(c: i64) -> i64 { if c >= 65 { if c <= 90 { return c + 32 } } return c }
46func ti_cat(d: *u8, o: i64, s: *u8) -> i64 { var i: i64 = 0; while s[i] != (0 as u8) { d[o + i] = s[i]; i = i + 1 } return o + i }
47func ti_catn(d: *u8, o: i64, v: i64) -> i64 {
48 var m: i64 = v
49 var oo: i64 = o
50 if m < 0 { d[oo] = 45 as u8; oo = oo + 1; m = 0 - m }
51 var nd: i64 = 1
52 var t: i64 = m
53 while t >= 10 { nd = nd + 1; t = t / 10 }
54 var i: i64 = nd - 1
55 while i >= 0 { d[oo + i] = (48 + (m % 10)) as u8; m = m / 10; i = i - 1 }
56 return oo + nd
57}
58func ti_num(v: i64) -> i64 { let b: *u8 = sys_mmap(28); let e: i64 = ti_catn(b, 0, v); sys_write(1, b, e); return 0 }
59func ti_puts(s: *u8) -> i64 { sys_write(1, s, ti_len(s)); return 0 }
60func ti_atoi(s: *u8) -> i64 { var v: i64 = 0; var i: i64 = 0; while s[i] != (0 as u8) { let c: i64 = s[i] as i64; if c >= 48 { if c <= 57 { v = v * 10 + (c - 48) } } i = i + 1 } return v }
61func ti_streq(a: *u8, b: *u8) -> i64 { var i: i64 = 0; while 1 == 1 { let ca: i64 = a[i] as i64; let cb: i64 = b[i] as i64; if ca != cb { return 0 } if ca == 0 { return 1 } i = i + 1 } return 1 }
62func ti_suffix(name: *u8, suf: *u8) -> i64 {
63 let nl: i64 = ti_len(name); let sl: i64 = ti_len(suf)
64 if nl < sl { return 0 }
65 var i: i64 = 0
66 while i < sl { if ti_lc(name[nl - sl + i] as i64) != ti_lc(suf[i] as i64) { return 0 } i = i + 1 }
67 return 1
68}
69
70// case-insensitive substring search (needle MUST be lowercase); -1 if absent
71func ti_find_ci(h: *u8, start: i64, n: i64, needle: *u8) -> i64 {
72 let nl: i64 = ti_len(needle)
73 if nl == 0 { return start }
74 var i: i64 = start
75 while i + nl <= n {
76 var j: i64 = 0
77 var ok: i64 = 1
78 while j < nl { if ti_lc(h[i + j] as i64) != (needle[j] as i64) { ok = 0; j = nl } else { j = j + 1 } }
79 if ok == 1 { return i }
80 i = i + 1
81 }
82 return 0 - 1
83}
84func ti_has(h: *u8, n: i64, needle: *u8) -> i64 { if ti_find_ci(h, 0, n, needle) >= 0 { return 1 } return 0 }
85func ti_count_ci(h: *u8, n: i64, needle: *u8) -> i64 {
86 let nl: i64 = ti_len(needle)
87 if nl == 0 { return 0 }
88 var c: i64 = 0
89 var i: i64 = 0
90 while i + nl <= n {
91 var j: i64 = 0
92 var ok: i64 = 1
93 while j < nl { if ti_lc(h[i + j] as i64) != (needle[j] as i64) { ok = 0; j = nl } else { j = j + 1 } }
94 if ok == 1 { c = c + 1; i = i + nl } else { i = i + 1 }
95 }
96 return c
97}
98
99// first offset (from `start`) of a call-to-action marker (min over several); returns n+1 if none present
100func ti_cta_off(h: *u8, start: i64, n: i64) -> i64 {
101 var best: i64 = n + 1
102 let o1: i64 = ti_find_ci(h, start, n, "get started" as *u8); if o1 >= 0 { if o1 < best { best = o1 } }
103 let o2: i64 = ti_find_ci(h, start, n, "sign up" as *u8); if o2 >= 0 { if o2 < best { best = o2 } }
104 let o3: i64 = ti_find_ci(h, start, n, "subscribe" as *u8); if o3 >= 0 { if o3 < best { best = o3 } }
105 let o4: i64 = ti_find_ci(h, start, n, "btn-primary" as *u8); if o4 >= 0 { if o4 < best { best = o4 } }
106 let o5: i64 = ti_find_ci(h, start, n, "call to action" as *u8); if o5 >= 0 { if o5 < best { best = o5 } }
107 return best
108}
109
110// section-sequence: the ordered layout flow (kinds present, sorted by first-occurrence offset), joined by '>'
111func ti_seq(h: *u8, n: i64, out: *u8) -> i64 {
112 let koff: *i64 = sys_mmap(16 * 8) as *i64
113 let ktok: *i64 = sys_mmap(16 * 8) as *i64
114 let mrk: *i64 = sys_mmap(16 * 8) as *i64
115 let idx: *i64 = sys_mmap(16 * 8) as *i64
116 ktok[0] = ("hd" as *u8) as i64; mrk[0] = ("<header" as *u8) as i64
117 ktok[1] = ("nav" as *u8) as i64; mrk[1] = ("<nav" as *u8) as i64
118 ktok[2] = ("hero" as *u8) as i64; mrk[2] = ("hero" as *u8) as i64
119 ktok[3] = ("feat" as *u8) as i64; mrk[3] = ("feature" as *u8) as i64
120 ktok[4] = ("proof" as *u8) as i64; mrk[4] = ("testimonial" as *u8) as i64
121 ktok[5] = ("price" as *u8) as i64; mrk[5] = ("pricing" as *u8) as i64
122 ktok[6] = ("gal" as *u8) as i64; mrk[6] = ("gallery" as *u8) as i64
123 ktok[7] = ("vid" as *u8) as i64; mrk[7] = ("<video" as *u8) as i64
124 ktok[8] = ("faq" as *u8) as i64; mrk[8] = ("faq" as *u8) as i64
125 ktok[9] = ("form" as *u8) as i64; mrk[9] = ("<form" as *u8) as i64
126 ktok[10] = ("ft" as *u8) as i64; mrk[10] = ("<footer" as *u8) as i64
127 // content-section scan starts AFTER nav/header so nav anchor links (href="#features") don't
128 // pollute the layout order; the structural landmarks header/nav/footer are located globally.
129 var cstart: i64 = 0
130 let navend: i64 = ti_find_ci(h, 0, n, "</nav>" as *u8)
131 if navend >= 0 { cstart = navend + 6 } else {
132 let hdend: i64 = ti_find_ci(h, 0, n, "</header>" as *u8)
133 if hdend >= 0 { cstart = hdend + 9 }
134 }
135 var nk: i64 = 11
136 var i: i64 = 0
137 while i < nk {
138 var st: i64 = cstart
139 if i == 0 { st = 0 }
140 if i == 1 { st = 0 }
141 if i == 10 { st = 0 }
142 let off_i: i64 = ti_find_ci(h, st, n, mrk[i] as *u8)
143 koff[i] = off_i
144 i = i + 1
145 }
146 // cta as an extra ordered kind (content signal -> scanned from cstart)
147 ktok[nk] = ("cta" as *u8) as i64
148 let ctao: i64 = ti_cta_off(h, cstart, n)
149 var ctastore: i64 = 0 - 1
150 if ctao <= n { ctastore = ctao }
151 koff[nk] = ctastore
152 nk = nk + 1
153 // present kinds
154 var m: i64 = 0
155 i = 0
156 while i < nk { if koff[i] >= 0 { idx[m] = i; m = m + 1 } i = i + 1 }
157 // insertion sort idx by koff ascending
158 i = 1
159 while i < m {
160 let cur: i64 = idx[i]
161 var j: i64 = i - 1
162 var go: i64 = 1
163 while go == 1 {
164 if j < 0 { go = 0 } else {
165 if koff[idx[j]] > koff[cur] { idx[j + 1] = idx[j]; j = j - 1 } else { go = 0 }
166 }
167 }
168 idx[j + 1] = cur
169 i = i + 1
170 }
171 var o: i64 = 0
172 i = 0
173 while i < m {
174 if i > 0 { out[o] = 62 as u8; o = o + 1 }
175 o = ti_cat(out, o, ktok[idx[i]] as *u8)
176 i = i + 1
177 }
178 out[o] = 0 as u8
179 return o
180}
181
182// archetype from structural signals (most-specific first; nested ifs, no else-if / no &&)
183func ti_classify(h: *u8, n: i64) -> i64 {
184 var ecom: i64 = 0
185 if ti_has(h, n, "add to cart" as *u8) == 1 { ecom = 1 }
186 if ti_has(h, n, "add-to-cart" as *u8) == 1 { ecom = 1 }
187 if ti_has(h, n, "checkout" as *u8) == 1 { ecom = 1 }
188 var social: i64 = 0
189 if ti_has(h, n, "news feed" as *u8) == 1 { social = 1 }
190 if ti_has(h, n, "activity feed" as *u8) == 1 { social = 1 }
191 if ti_has(h, n, "timeline" as *u8) == 1 { social = 1 }
192 var media: i64 = 0
193 if ti_has(h, n, "<video" as *u8) == 1 { media = 1 }
194 if ti_has(h, n, "video player" as *u8) == 1 { media = 1 }
195 if ti_has(h, n, "now playing" as *u8) == 1 { media = 1 }
196 var resume: i64 = 0
197 if ti_has(h, n, "work experience" as *u8) == 1 { resume = 1 }
198 if ti_has(h, n, "curriculum vitae" as *u8) == 1 { resume = 1 }
199 if ti_has(h, n, "resume" as *u8) == 1 { resume = 1 }
200 var blog: i64 = 0
201 if ti_has(h, n, "class=\"posts\"" as *u8) == 1 { blog = 1 }
202 if ti_has(h, n, "read more" as *u8) == 1 { blog = 1 }
203 if ti_has(h, n, ">blog<" as *u8) == 1 { blog = 1 }
204 var gallery: i64 = 0
205 if ti_has(h, n, "class=\"tiles\"" as *u8) == 1 { gallery = 1 }
206 if ti_has(h, n, "gallery" as *u8) == 1 { gallery = 1 }
207 var portfolio: i64 = 0
208 if ti_has(h, n, "portfolio" as *u8) == 1 { portfolio = 1 }
209 if ti_has(h, n, "id=\"work\"" as *u8) == 1 { if ti_has(h, n, "id=\"about\"" as *u8) == 1 { if ti_has(h, n, "id=\"contact\"" as *u8) == 1 { portfolio = 1 } } }
210 let hero: i64 = ti_has(h, n, "hero" as *u8)
211 let banner: i64 = ti_has(h, n, "banner" as *u8)
212 let spot: i64 = ti_has(h, n, "spotlight" as *u8)
213 let feats: i64 = ti_has(h, n, "features" as *u8)
214 let intro: i64 = ti_has(h, n, "intro" as *u8)
215 let navlinks: i64 = ti_count_ci(h, n, "<a " as *u8)
216 let paras: i64 = ti_count_ci(h, n, "<p" as *u8)
217 if ecom == 1 { return TI_ARCH_ECOM }
218 if social == 1 { return TI_ARCH_SOCIAL }
219 if media == 1 { return TI_ARCH_MEDIA }
220 if resume == 1 { return TI_ARCH_RESUME }
221 if blog == 1 { return TI_ARCH_BLOG }
222 if gallery == 1 { return TI_ARCH_GALLERY }
223 if portfolio == 1 { return TI_ARCH_PORTFOLIO }
224 var landing: i64 = 0
225 if hero == 1 { landing = 1 }
226 if banner == 1 { landing = 1 }
227 if spot == 1 { landing = 1 }
228 if feats == 1 { if intro == 1 { landing = 1 } }
229 if landing == 1 { return TI_ARCH_LANDING }
230 if navlinks >= 6 { if paras <= 3 { return TI_ARCH_HUB } }
231 return TI_ARCH_BROCHURE
232}
233func ti_arch_name(code: i64, out: *u8) -> i64 {
234 var s: *u8 = "brochure" as *u8
235 if code == TI_ARCH_LANDING { s = "landing" as *u8 }
236 if code == TI_ARCH_ECOM { s = "ecommerce" as *u8 }
237 if code == TI_ARCH_SOCIAL { s = "social" as *u8 }
238 if code == TI_ARCH_MEDIA { s = "media" as *u8 }
239 if code == TI_ARCH_RESUME { s = "resume" as *u8 }
240 if code == TI_ARCH_HUB { s = "hub" as *u8 }
241 if code == TI_ARCH_PORTFOLIO { s = "portfolio" as *u8 }
242 if code == TI_ARCH_GALLERY { s = "gallery" as *u8 }
243 if code == TI_ARCH_BLOG { s = "blog" as *u8 }
244 let o: i64 = ti_cat(out, 0, s)
245 out[o] = 0 as u8
246 return o
247}
248
249// stable positive content hash (same polynomial family as nx_corpus_ingest.ci_hash)
250func ti_hash(s: *u8, n: i64) -> i64 {
251 var h: i64 = TI_MAGIC_1125899906842597
252 var i: i64 = 0
253 while i < n { h = (h * 131) + (s[i] as i64); i = i + 1 }
254 if h < 0 { h = 0 - h }
255 return h & 0x7fffffffffffffff
256}
257// dedup key = "pat:" + hash( archetype-name \x1f section-sequence )
258func ti_mkkey(archcode: i64, seq: *u8, keyout: *u8) -> i64 {
259 let canon: *u8 = sys_mmap(512)
260 var o: i64 = ti_arch_name(archcode, canon)
261 canon[o] = 31 as u8; o = o + 1
262 o = ti_cat(canon, o, seq)
263 canon[o] = 0 as u8
264 let hsh: i64 = ti_hash(canon, o)
265 var ko: i64 = ti_cat(keyout, 0, "pat:" as *u8)
266 ko = ti_catn(keyout, ko, hsh)
267 keyout[ko] = 0 as u8
268 return ko
269}
270
271// portable, sovereign pattern record (line-oriented k|v; the abstract structure only)
272func ti_build_record(h: *u8, n: i64, archcode: i64, seq: *u8, src: *u8, out: *u8) -> i64 {
273 let anbuf: *u8 = sys_mmap(32)
274 ti_arch_name(archcode, anbuf)
275 var o: i64 = ti_cat(out, 0, "arch|" as *u8); o = ti_cat(out, o, anbuf)
276 o = ti_cat(out, o, "\nseq|" as *u8); o = ti_cat(out, o, seq)
277 o = ti_cat(out, o, "\nlinks|" as *u8); o = ti_catn(out, o, ti_count_ci(h, n, "<a " as *u8))
278 o = ti_cat(out, o, "\nforms|" as *u8); o = ti_catn(out, o, ti_count_ci(h, n, "<form" as *u8))
279 o = ti_cat(out, o, "\nsections|" as *u8); o = ti_catn(out, o, ti_count_ci(h, n, "<section" as *u8) + ti_count_ci(h, n, "<article" as *u8))
280 o = ti_cat(out, o, "\nheadings|" as *u8); o = ti_catn(out, o, ti_count_ci(h, n, "<h1" as *u8) + ti_count_ci(h, n, "<h2" as *u8) + ti_count_ci(h, n, "<h3" as *u8))
281 o = ti_cat(out, o, "\nimgs|" as *u8); o = ti_catn(out, o, ti_count_ci(h, n, "<img" as *u8))
282 o = ti_cat(out, o, "\nsearch|" as *u8); o = ti_catn(out, o, ti_has(h, n, "role=\"search\"" as *u8) + ti_has(h, n, "type=\"search\"" as *u8))
283 o = ti_cat(out, o, "\nmain|" as *u8); o = ti_catn(out, o, ti_has(h, n, "<main" as *u8))
284 o = ti_cat(out, o, "\nskip|" as *u8); o = ti_catn(out, o, ti_has(h, n, "skip" as *u8))
285 o = ti_cat(out, o, "\nlang|" as *u8); o = ti_catn(out, o, ti_has(h, n, "lang=" as *u8))
286 o = ti_cat(out, o, "\nresponsive|" as *u8); o = ti_catn(out, o, ti_has(h, n, "viewport" as *u8))
287 o = ti_cat(out, o, "\nsrc|" as *u8); o = ti_cat(out, o, src)
288 o = ti_cat(out, o, "\n" as *u8)
289 out[o] = 0 as u8
290 return o
291}
292
293// in-run seen-set membership (keys stored contiguously, TI_KEYW bytes each)
294func ti_seen(seenbuf: *u8, nseen: i64, key: *u8) -> i64 {
295 var i: i64 = 0
296 while i < nseen {
297 let a: *u8 = (seenbuf as i64 + i * TI_KEYW) as *u8
298 if ti_streq(a, key) == 1 { return 1 }
299 i = i + 1
300 }
301 return 0
302}
303
304// THE INGEST. counts: [0]stored [1]dups [2]skipped [3]segments [4]in [5]trapped [6]bytes
305func ti_run(tpldir: *u8, prefix: *u8, max_tpl: i64, max_bytes: i64, counts: *i64) -> i64 {
306 var ci: i64 = 0
307 while ci < 7 { counts[ci] = 0; ci = ci + 1 }
308 let segs: *i64 = sys_mmap(256 * 8) as *i64
309 let nseg: i64 = ss_manifest_file(prefix, "manifest.txt" as *u8, segs)
310 var segid: i64 = 1
311 var si: i64 = 0
312 // ROOT FIX seq1730 (id 1785450987): segs[] holds POINTERS to seg-<id> name strings
313 // (nx_seg_store.nx:1234 stores segs[cnt] = name as i64), NOT ids -- so segs[si] + 1 produced
314 // MMAP_ADDRESS+1 (~1.4e14), the pointer-shaped poison that PINS a plane forever: every later
315 // epoch id sorts BELOW it in supersede order and its rows are silently shadowed while rc=0.
316 // Parse the DIGITS, as nx_web_shard_compact.nx:55-58 already does on this SAME array.
317 while si < nseg {
318 let sg_nm: *u8 = segs[si] as *u8
319 var sg_v: i64 = 0
320 var sg_ci: i64 = 0
321 while sg_nm[sg_ci] != (0 as u8) { let sg_c: i64 = sg_nm[sg_ci] as i64; if sg_c >= 48 { if sg_c <= 57 { sg_v = sg_v * 10 + (sg_c - 48) } } sg_ci = sg_ci + 1 }
322 if sg_v >= segid { segid = sg_v + 1 }
323 si = si + 1
324 }
325 let hh: *i64 = ss_open(prefix)
326 let pbox: *i64 = sys_mmap(16) as *i64
327 let lbox: *i64 = sys_mmap(16) as *i64
328 let seenbuf: *u8 = sys_mmap(TI_SEENCAP * TI_KEYW)
329 var nseen: i64 = 0
330 let dbuf: *u8 = sys_mmap(TI_MAGIC_131072)
331 let path: *u8 = sys_mmap(TI_MAGIC_1024)
332 let key: *u8 = sys_mmap(64)
333 let seq: *u8 = sys_mmap(256)
334 let rec: *u8 = sys_mmap(TI_RECCAP)
335 let szp: *i64 = sys_mmap(16) as *i64
336 let fd: i64 = sys_openat_rd(tpldir)
337 if fd < 0 { ti_puts("tpl-ingest: cannot open dir " as *u8); ti_puts(tpldir); ti_puts("\n" as *u8); return 0 - 1 }
338 var w: *i64 = ss_begin()
339 var go: i64 = 1
340 while go == 1 {
341 let nr: i64 = sys_getdents64(fd, dbuf, TI_MAGIC_131072)
342 if nr <= 0 { go = 0 } else {
343 var off: i64 = 0
344 while off < nr {
345 let recd: *u8 = (dbuf as i64 + off) as *u8
346 let ty: i64 = dirent_type(recd)
347 let nm: *u8 = dirent_name(recd)
348 off = off + dirent_reclen(recd)
349 if ty != 4 {
350 var ishtml: i64 = 0
351 if ti_suffix(nm, ".html" as *u8) == 1 { ishtml = 1 }
352 if ti_suffix(nm, ".htm" as *u8) == 1 { ishtml = 1 }
353 if ishtml == 1 {
354 if counts[4] >= max_tpl { counts[5] = 1; go = 0; off = nr } else {
355 counts[4] = counts[4] + 1
356 var po: i64 = ti_cat(path, 0, tpldir)
357 po = ti_cat(path, po, "/" as *u8)
358 po = ti_cat(path, po, nm)
359 path[po] = 0 as u8
360 let raw: *u8 = ss_readall(path, szp)
361 var rn: i64 = szp[0]
362 if rn > TI_RAWCAP { rn = 0 }
363 if rn < TI_MINDOC { counts[2] = counts[2] + 1 } else {
364 if ti_find_ci(raw, 0, rn, "<" as *u8) < 0 { counts[2] = counts[2] + 1 } else {
365 let arch: i64 = ti_classify(raw, rn)
366 ti_seq(raw, rn, seq)
367 ti_mkkey(arch, seq, key)
368 var dup: i64 = 0
369 if ti_seen(seenbuf, nseen, key) == 1 { dup = 1 }
370 if dup == 0 { if (hh as i64) != 0 { if ss_hget(hh, key, pbox, lbox) == 1 { dup = 1 } } }
371 if dup == 1 { counts[1] = counts[1] + 1 } else {
372 let rl: i64 = ti_build_record(raw, rn, arch, seq, nm, rec)
373 if counts[6] + rl > max_bytes { counts[5] = 1; go = 0; off = nr } else {
374 if nseen < TI_SEENCAP {
375 let dst: *u8 = (seenbuf as i64 + nseen * TI_KEYW) as *u8
376 var kk: i64 = 0
377 while key[kk] != (0 as u8) { dst[kk] = key[kk]; kk = kk + 1 }
378 dst[kk] = 0 as u8
379 nseen = nseen + 1
380 }
381 let addr: i64 = ss_add(w, 1, key, rec, rl)
382 if addr < 0 {
383 if ss_commit(prefix, w, segid) == 0 { counts[3] = counts[3] + 1 }
384 segid = segid + 1
385 w = ss_begin()
386 let addr2: i64 = ss_add(w, 1, key, rec, rl)
387 if addr2 < 0 { counts[2] = counts[2] + 1 } else { counts[0] = counts[0] + 1; counts[6] = counts[6] + rl }
388 } else { counts[0] = counts[0] + 1; counts[6] = counts[6] + rl }
389 }
390 }
391 }
392 }
393 }
394 }
395 }
396 }
397 }
398 }
399 sys_close(fd)
400 if w[1] > 0 { if ss_commit(prefix, w, segid) == 0 { counts[3] = counts[3] + 1 } }
401 return counts[0]
402}
403
404func main(argc: i64, argv: *i64) -> i64 {
405 if argc < 2 { ti_puts("usage: nx_tpl_ingest <template-dir> [max_templates]\n" as *u8); return 2 }
406 let dir: *u8 = argv[1] as *u8
407 var maxt: i64 = TI_MAGIC_5000
408 if argc >= 3 { let mm: i64 = ti_atoi(argv[2] as *u8); if mm > 0 { maxt = mm } }
409 let counts: *i64 = sys_mmap(64) as *i64
410 let prefix: *u8 = "knowledge/index/tplcorpus/" as *u8
411 let rc: i64 = ti_run(dir, prefix, maxt, TI_MAGIC_67108864, counts)
412 ti_puts("TPL-INGEST dir=" as *u8); ti_puts(dir)
413 ti_puts(" in=" as *u8); ti_num(counts[4])
414 ti_puts(" stored=" as *u8); ti_num(counts[0])
415 ti_puts(" dups=" as *u8); ti_num(counts[1])
416 ti_puts(" skipped=" as *u8); ti_num(counts[2])
417 ti_puts(" segs=" as *u8); ti_num(counts[3])
418 ti_puts(" bytes=" as *u8); ti_num(counts[6])
419 ti_puts(" trapped=" as *u8); ti_num(counts[5])
420 ti_puts("\n" as *u8)
421 if rc < 0 { return 1 }
422 return 0
423}