code wiki / _hdl_build / nx_tpl_fetch.nx
`nx_tpl_fetch.nx` source

1// nx_tpl_fetch.nx -- hybrid STEP 2 of the SITE FACTORY: fetch OPEN FREE web templates over SOVEREIGN TLS
2// (nx_https_fetch_follow_best: TLS-1.3 -> chrome-JA3 -> TLS-1.2 fallback, Mozilla CA trust) and save each page's
3// RAW HTML into a dir, so nx_tpl_ingest (R4) can extract its STRUCTURE into the pattern corpus. We keep bytes only
4// to extract abstract STRUCTURE/FLOW (uncopyrightable) -- never to redistribute the templates' files. BOUNDED +
5// POLITE by construction: fetches ONLY the explicit URLs passed (no unbounded crawl), each once, byte-capped.
6//   usage: nx_tpl_fetch <outdir> <url1> [url2 ...]
7// license_tier: ORIGINAL
8import "nx_x509_trust_store.nx"
9import "nx_trust_store_load_from_certdata.nx"
10import "nx_https_fetch_follow.nx"
11import "nx_syscalls.nx"
12const TF_MAGIC_4194304: i64 = 4194304
13const TF_MAGIC_1024: i64 = 1024
14
15const TF_OUTCAP: i64 = 4194304   // 4 MiB max per page
16
17func tf_w(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
18func tf_catn(d: *u8, o: i64, v: i64) -> i64 {
19    if v < 0 { d[o] = 45 as u8; return tf_catn(d, o + 1, 0 - v) }
20    // NEGATIVES (2026-08-07). Without this the `while m > 0` loop below never runs for a
21    // negative value and this function emits ZERO CHARACTERS, silently corrupting whatever
22    // format it is writing into. Handled AT THE SIGNATURE so it is independent of which
23    // cursor variable the body happens to use. Non-negative input is byte-identical (rule 19).
24    if v < 0 { d[o] = 45 as u8; return tf_catn(d, o + 1, 0 - v) }
25    var m: i64=v; var oo: i64=o
26    if m<0 { d[oo]=45 as u8; oo=oo+1; m=0-m }
27    var nd: i64=1; var t: i64=m
28    while t>=10 { nd=nd+1; t=t/10 }
29    var i: i64=nd-1
30    while i>=0 { d[oo+i]=(48+(m%10)) as u8; m=m/10; i=i-1 }
31    return oo+nd
32}
33func tf_num(v: i64) -> i64 { let b: *u8=sys_mmap(28); let e: i64=tf_catn(b,0,v); sys_write(1,b,e); return 0 }
34func tf_cat(d: *u8, o: i64, s: *u8) -> i64 { var i: i64=0; while s[i]!=(0 as u8){d[o+i]=s[i]; i=i+1} return o+i }
35
36func main(argc: i64, argv: *i64) -> i64 {
37    if argc < 3 { tf_w("usage: nx_tpl_fetch <outdir> <url1> [url2 ...]\n" as *u8); return 2 }
38    let outdir: *u8 = argv[1] as *u8
39    sys_mkdir(outdir, 0x1ed)
40    let r: i64 = nx_trust_store_load_from_certdata("data/mozilla_certdata.txt" as *u8, 512, TF_MAGIC_4194304)
41    if r <= 0 { tf_w("tpl-fetch: trust store load FAILED\n" as *u8); return 1 }
42    let store: *TrustStore = r as *TrustStore
43    let out: *u8 = sys_mmap(TF_OUTCAP)
44    let status: *i64 = sys_mmap(16) as *i64
45    let path: *u8 = sys_mmap(TF_MAGIC_1024)
46    var saved: i64 = 0
47    var i: i64 = 2
48    while i < argc {
49        let url: *u8 = argv[i] as *u8
50        status[0] = 0
51        let n: i64 = nx_https_fetch_follow_best(url, store, out, TF_OUTCAP, 6, status)
52        tf_w("FETCH " as *u8); tf_w(url); tf_w("  status=" as *u8); tf_num(status[0]); tf_w(" bytes=" as *u8); tf_num(n)
53        var ok: i64 = 0
54        if n > 0 { if status[0] >= 200 { if status[0] < 300 { ok = 1 } } }
55        if ok == 1 {
56            var po: i64 = tf_cat(path, 0, outdir)
57            po = tf_cat(path, po, "/tpl_" as *u8)
58            po = tf_catn(path, po, i - 2)
59            po = tf_cat(path, po, ".html" as *u8)
60            path[po] = 0 as u8
61            let fd: i64 = sys_openat_wr(path, 0x1a4)
62            if fd >= 0 { sys_write(fd, out, n); sys_close(fd); saved = saved + 1; tf_w("  SAVED " as *u8); tf_w(path) }
63        }
64        tf_w("\n" as *u8)
65        i = i + 1
66    }
67    tf_w("TPL-FETCH saved=" as *u8); tf_num(saved); tf_w("/" as *u8); tf_num(argc - 2); tf_w(" -> " as *u8); tf_w(outdir); tf_w("\n" as *u8)
68    return 0
69}
nx_tpl_fetch.nx source

`nx_tpl_fetch.nx` source
