code wiki / _hdl_build / nx_tpl_parts.nx
`nx_tpl_parts.nx` source

↩ module page · 240 lines · 11998 B
1// nx_tpl_parts.nx -- R-PARTS of the SITE FACTORY (the LEGO half of "ingest parts AND complete templates"):
2// extract SECTION-LEVEL patterns from template HTML into the pattern corpus as reusable BRICKS. For each
3// content section a template uses (hero/feat/proof/price/gal/vid/faq/form/cta) we keep only the abstract
4// STRUCTURE: the section kind + bucketed feature metrics (links/imgs/headings/forms/buttons) + its position.
5// Each distinct (kind, metric-buckets) collapses to ONE content-addressed `part:<hash>` record in the SAME
6// seg_store corpus that holds the page-level `pat:` records -- one corpus, two granularities, so composition
7// can lego from bricks as well as whole-page patterns. BUDGET-GUARDED by construction (max_parts + byte
8// ceiling, fail-fast trap; [[feedback-substrate-generators-must-have-budget-guard]]); idempotent (in-run +
9// cross-run dedup). We store structure only -- never a template's assets. license_tier: ORIGINAL
10//   usage: nx_tpl_parts <template-dir> <corpus-prefix> [max_parts]
11import "nx_tpl_ingest.nx"
12const TP_MAGIC_131072: i64 = 131072
13const TP_MAGIC_1024: i64 = 1024
14const TP_MAGIC_2000: i64 = 2000
15
16const TP_RECCAP: i64 = 1024
17const TP_MAXK:   i64 = 12
18
19func tp_bucket(v: i64) -> i64 {
  if v == 0 { return 0 }
  if v <= 2 { return 1 }
  if v <= 5 { return 2 }
  return 3
24}
25
26// scan content sections (after </nav>, like ti_seq): fill (tokptr, off) pairs sorted by offset. returns count.
27func tp_scan(h: *u8, n: i64, toks: *i64, offs: *i64) -> i64 {
  var cstart: i64 = 0
  let navend: i64 = ti_find_ci(h, 0, n, "</nav>" as *u8)
  if navend >= 0 { cstart = navend + 6 } else {
      let hdend: i64 = ti_find_ci(h, 0, n, "</header>" as *u8)
      if hdend >= 0 { cstart = hdend + 9 }
  }
  let mt: *i64 = sys_mmap(TP_MAXK * 8) as *i64
  let mm: *i64 = sys_mmap(TP_MAXK * 8) as *i64
  mt[0] = ("hero" as *u8) as i64;  mm[0] = ("hero" as *u8) as i64
  mt[1] = ("feat" as *u8) as i64;  mm[1] = ("feature" as *u8) as i64
  mt[2] = ("proof" as *u8) as i64; mm[2] = ("testimonial" as *u8) as i64
  mt[3] = ("price" as *u8) as i64; mm[3] = ("pricing" as *u8) as i64
  mt[4] = ("gal" as *u8) as i64;   mm[4] = ("gallery" as *u8) as i64
  mt[5] = ("vid" as *u8) as i64;   mm[5] = ("<video" as *u8) as i64
  mt[6] = ("faq" as *u8) as i64;   mm[6] = ("faq" as *u8) as i64
  mt[7] = ("form" as *u8) as i64;  mm[7] = ("<form" as *u8) as i64
  var cnt: i64 = 0
  var i: i64 = 0
  while i < 8 {
      let off_i: i64 = ti_find_ci(h, cstart, n, mm[i] as *u8)
      if off_i >= 0 { toks[cnt] = mt[i]; offs[cnt] = off_i; cnt = cnt + 1 }
      i = i + 1
  }
  let ctao: i64 = ti_cta_off(h, cstart, n)
  if ctao <= n { toks[cnt] = ("cta" as *u8) as i64; offs[cnt] = ctao; cnt = cnt + 1 }
  // insertion sort by offset
  i = 1
  while i < cnt {
      let ct: i64 = toks[i]
      let co: i64 = offs[i]
      var j: i64 = i - 1
      var go: i64 = 1
      while go == 1 {
          if j < 0 { go = 0 } else {
              if offs[j] > co { toks[j+1] = toks[j]; offs[j+1] = offs[j]; j = j - 1 } else { go = 0 }
          }
      }
      toks[j+1] = ct
      offs[j+1] = co
      i = i + 1
  }
  return cnt
70}
71
72// structural metrics of h[from..to): m[0]=links m[1]=imgs m[2]=headings m[3]=forms m[4]=buttons
73func tp_metrics(h: *u8, from: i64, to: i64, m: *i64) -> i64 {
  let seg: *u8 = (h as i64 + from) as *u8
  let len: i64 = to - from
  m[0] = ti_count_ci(seg, len, "<a " as *u8)
  m[1] = ti_count_ci(seg, len, "<img" as *u8)
  m[2] = ti_count_ci(seg, len, "<h2" as *u8) + ti_count_ci(seg, len, "<h3" as *u8)
  m[3] = ti_count_ci(seg, len, "<form" as *u8)
  m[4] = ti_count_ci(seg, len, "<button" as *u8) + ti_count_ci(seg, len, "btn" as *u8)
  return 0
82}
83
84// dedup key "part:" + hash(kind \x1f bucket-digits)
85func tp_mkkey(kind: *u8, m: *i64, keyout: *u8) -> i64 {
  let canon: *u8 = sys_mmap(64)
  var o: i64 = ti_cat(canon, 0, kind)
  canon[o] = 31 as u8; o = o + 1
  var i: i64 = 0
  while i < 5 { canon[o] = (48 + tp_bucket(m[i])) as u8; o = o + 1; i = i + 1 }
  canon[o] = 0 as u8
  let hsh: i64 = ti_hash(canon, o)
  var ko: i64 = ti_cat(keyout, 0, "part:" as *u8)
  ko = ti_catn(keyout, ko, hsh)
  keyout[ko] = 0 as u8
  return ko
97}
98
99// record: line-oriented k|v (abstract structure only)
100func tp_record(kind: *u8, m: *i64, pos: i64, src: *u8, out: *u8) -> i64 {
  var o: i64 = ti_cat(out, 0, "kind|" as *u8); o = ti_cat(out, o, kind)
  o = ti_cat(out, o, "\nlinks|" as *u8); o = ti_catn(out, o, m[0])
  o = ti_cat(out, o, "\nimgs|" as *u8); o = ti_catn(out, o, m[1])
  o = ti_cat(out, o, "\nheads|" as *u8); o = ti_catn(out, o, m[2])
  o = ti_cat(out, o, "\nforms|" as *u8); o = ti_catn(out, o, m[3])
  o = ti_cat(out, o, "\nbtns|" as *u8); o = ti_catn(out, o, m[4])
  o = ti_cat(out, o, "\npos|" as *u8); o = ti_catn(out, o, pos)
  o = ti_cat(out, o, "\nsrc|" as *u8); o = ti_cat(out, o, src)
  o = ti_cat(out, o, "\n" as *u8)
  out[o] = 0 as u8
  return o
112}
113
114// THE PARTS INGEST. counts: [0]stored [1]dups [2]skipped-files [3]segs [4]files [5]trapped [6]bytes [7]parts-seen
115func tp_run(tpldir: *u8, prefix: *u8, max_parts: i64, counts: *i64) -> i64 {
  var ci: i64 = 0
  while ci < 8 { counts[ci] = 0; ci = ci + 1 }
  let segs: *i64 = sys_mmap(256 * 8) as *i64
  let nseg: i64 = ss_manifest_file(prefix, "manifest.txt" as *u8, segs)
  var segid: i64 = 1
  var si: i64 = 0
  // ROOT FIX seq1730 (id 1785450987): segs[] holds POINTERS to seg-<id> name strings
  // (nx_seg_store.nx:1234 stores segs[cnt] = name as i64), NOT ids -- so segs[si] + 1 produced
  // MMAP_ADDRESS+1 (~1.4e14), the pointer-shaped poison that PINS a plane forever: every later
  // epoch id sorts BELOW it in supersede order and its rows are silently shadowed while rc=0.
  // Parse the DIGITS, as nx_web_shard_compact.nx:55-58 already does on this SAME array.
  while si < nseg {
      let sg_nm: *u8 = segs[si] as *u8
      var sg_v: i64 = 0
      var sg_ci: i64 = 0
      while sg_nm[sg_ci] != (0 as u8) { let sg_c: i64 = sg_nm[sg_ci] as i64; if sg_c >= 48 { if sg_c <= 57 { sg_v = sg_v * 10 + (sg_c - 48) } } sg_ci = sg_ci + 1 }
      if sg_v >= segid { segid = sg_v + 1 }
      si = si + 1
  }
  let hh: *i64 = ss_open(prefix)
  let pbox: *i64 = sys_mmap(16) as *i64
  let lbox: *i64 = sys_mmap(16) as *i64
  let seenbuf: *u8 = sys_mmap(TI_SEENCAP * TI_KEYW)
  var nseen: i64 = 0
  let dbuf: *u8 = sys_mmap(TP_MAGIC_131072)
  let path: *u8 = sys_mmap(TP_MAGIC_1024)
  let key:  *u8 = sys_mmap(64)
  let rec:  *u8 = sys_mmap(TP_RECCAP)
  let toks: *i64 = sys_mmap(TP_MAXK * 8) as *i64
  let offs: *i64 = sys_mmap(TP_MAXK * 8) as *i64
  let mets: *i64 = sys_mmap(8 * 8) as *i64
  let szp:  *i64 = sys_mmap(16) as *i64
  let fd: i64 = sys_openat_rd(tpldir)
  if fd < 0 { ti_puts("tpl-parts: cannot open dir " as *u8); ti_puts(tpldir); ti_puts("\n" as *u8); return 0 - 1 }
  var w: *i64 = ss_begin()
  var go: i64 = 1
  while go == 1 {
      let nr: i64 = sys_getdents64(fd, dbuf, TP_MAGIC_131072)
      if nr <= 0 { go = 0 } else {
          var off: i64 = 0
          while off < nr {
              let recd: *u8 = (dbuf as i64 + off) as *u8
              let ty: i64 = dirent_type(recd)
              let nm: *u8 = dirent_name(recd)
              off = off + dirent_reclen(recd)
              if ty != 4 {
                  var ishtml: i64 = 0
                  if ti_suffix(nm, ".html" as *u8) == 1 { ishtml = 1 }
                  if ti_suffix(nm, ".htm" as *u8) == 1 { ishtml = 1 }
                  if ishtml == 1 {
                      counts[4] = counts[4] + 1
                      var po: i64 = ti_cat(path, 0, tpldir)
                      po = ti_cat(path, po, "/" as *u8)
                      po = ti_cat(path, po, nm)
                      path[po] = 0 as u8
                      let raw: *u8 = ss_readall(path, szp)
                      var rn: i64 = szp[0]
                      if rn > TI_RAWCAP { rn = 0 }
                      if rn < TI_MINDOC { counts[2] = counts[2] + 1 } else {
                          if ti_find_ci(raw, 0, rn, "<" as *u8) < 0 { counts[2] = counts[2] + 1 } else {
                              let np: i64 = tp_scan(raw, rn, toks, offs)
                              var pi: i64 = 0
                              while pi < np {
                                  counts[7] = counts[7] + 1
                                  if counts[7] > max_parts { counts[5] = 1; pi = np; go = 0; off = nr } else {
                                      var to2: i64 = rn
                                      if pi + 1 < np { to2 = offs[pi + 1] }
                                      tp_metrics(raw, offs[pi], to2, mets)
                                      tp_mkkey(toks[pi] as *u8, mets, key)
                                      var dup: i64 = 0
                                      if ti_seen(seenbuf, nseen, key) == 1 { dup = 1 }
                                      if dup == 0 { if (hh as i64) != 0 { if ss_hget(hh, key, pbox, lbox) == 1 { dup = 1 } } }
                                      if dup == 1 { counts[1] = counts[1] + 1 } else {
                                          let rl: i64 = tp_record(toks[pi] as *u8, mets, pi, nm, rec)
                                          if nseen < TI_SEENCAP {
                                              let dst: *u8 = (seenbuf as i64 + nseen * TI_KEYW) as *u8
                                              var kk: i64 = 0
                                              while key[kk] != (0 as u8) { dst[kk] = key[kk]; kk = kk + 1 }
                                              dst[kk] = 0 as u8
                                              nseen = nseen + 1
                                          }
                                          let addr: i64 = ss_add(w, 1, key, rec, rl)
                                          if addr < 0 {
                                              if ss_commit(prefix, w, segid) == 0 { counts[3] = counts[3] + 1 }
                                              segid = segid + 1
                                              w = ss_begin()
                                              let addr2: i64 = ss_add(w, 1, key, rec, rl)
                                              if addr2 < 0 { counts[2] = counts[2] + 1 } else { counts[0] = counts[0] + 1; counts[6] = counts[6] + rl }
                                          } else { counts[0] = counts[0] + 1; counts[6] = counts[6] + rl }
                                      }
                                      pi = pi + 1
                                  }
                              }
                          }
                      }
                  }
              }
          }
      }
  }
  sys_close(fd)
  if w[1] > 0 { if ss_commit(prefix, w, segid) == 0 { counts[3] = counts[3] + 1 } }
  return counts[0]
219}
220
221func main(argc: i64, argv: *i64) -> i64 {
  if argc < 3 { ti_puts("usage: nx_tpl_parts <template-dir> <corpus-prefix> [max_parts]\n" as *u8); return 2 }
  let dir: *u8 = argv[1] as *u8
  let prefix: *u8 = argv[2] as *u8
  var maxp: i64 = TP_MAGIC_2000
  if argc >= 4 { let mm: i64 = ti_atoi(argv[3] as *u8); if mm > 0 { maxp = mm } }
  let counts: *i64 = sys_mmap(64) as *i64
  let rc: i64 = tp_run(dir, prefix, maxp, counts)
  ti_puts("TPL-PARTS dir=" as *u8); ti_puts(dir)
  ti_puts(" files=" as *u8); ti_num(counts[4])
  ti_puts(" parts_seen=" as *u8); ti_num(counts[7])
  ti_puts(" stored=" as *u8); ti_num(counts[0])
  ti_puts(" dups=" as *u8); ti_num(counts[1])
  ti_puts(" segs=" as *u8); ti_num(counts[3])
  ti_puts(" bytes=" as *u8); ti_num(counts[6])
  ti_puts(" trapped=" as *u8); ti_num(counts[5])
  ti_puts("\n" as *u8)
  if rc < 0 { return 1 }
  return 0
240}
nx_tpl_parts.nx source

`nx_tpl_parts.nx` source
