code wiki / _hdl_build / nx_tpl_fetch.nx
nx_tpl_fetch.nx source
↩ module page · 63 lines · 3144 B
1// nx_tpl_fetch.nx -- hybrid STEP 2 of the SITE FACTORY: fetch OPEN FREE web templates over SOVEREIGN TLS
2// (nx_https_fetch_follow_best: TLS-1.3 -> chrome-JA3 -> TLS-1.2 fallback, Mozilla CA trust) and save each page's
3// RAW HTML into a dir, so nx_tpl_ingest (R4) can extract its STRUCTURE into the pattern corpus. We keep bytes only
4// to extract abstract STRUCTURE/FLOW (uncopyrightable) -- never to redistribute the templates' files. BOUNDED +
5// POLITE by construction: fetches ONLY the explicit URLs passed (no unbounded crawl), each once, byte-capped.
6// usage: nx_tpl_fetch <outdir> <url1> [url2 ...]
7// license_tier: ORIGINAL
8import "nx_x509_trust_store.nx"
9import "nx_trust_store_load_from_certdata.nx"
10import "nx_https_fetch_follow.nx"
11import "nx_syscalls.nx"
12const TF_MAGIC_4194304: i64 = 4194304
13const TF_MAGIC_1024: i64 = 1024
14
15const TF_OUTCAP: i64 = 4194304 // 4 MiB max per page
16
17func tf_w(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
18func tf_catn(d: *u8, o: i64, v: i64) -> i64 {
19 var m: i64=v; var oo: i64=o
20 if m<0 { d[oo]=45 as u8; oo=oo+1; m=0-m }
21 var nd: i64=1; var t: i64=m
22 while t>=10 { nd=nd+1; t=t/10 }
23 var i: i64=nd-1
24 while i>=0 { d[oo+i]=(48+(m%10)) as u8; m=m/10; i=i-1 }
25 return oo+nd
26}
27func tf_num(v: i64) -> i64 { let b: *u8=sys_mmap(28); let e: i64=tf_catn(b,0,v); sys_write(1,b,e); return 0 }
28func tf_cat(d: *u8, o: i64, s: *u8) -> i64 { var i: i64=0; while s[i]!=(0 as u8){d[o+i]=s[i]; i=i+1} return o+i }
29
30func main(argc: i64, argv: *i64) -> i64 {
31 if argc < 3 { tf_w("usage: nx_tpl_fetch <outdir> <url1> [url2 ...]\n" as *u8); return 2 }
32 let outdir: *u8 = argv[1] as *u8
33 sys_mkdir(outdir, 0x1ed)
34 let r: i64 = nx_trust_store_load_from_certdata("data/mozilla_certdata.txt" as *u8, 512, TF_MAGIC_4194304)
35 if r <= 0 { tf_w("tpl-fetch: trust store load FAILED\n" as *u8); return 1 }
36 let store: *TrustStore = r as *TrustStore
37 let out: *u8 = sys_mmap(TF_OUTCAP)
38 let status: *i64 = sys_mmap(16) as *i64
39 let path: *u8 = sys_mmap(TF_MAGIC_1024)
40 var saved: i64 = 0
41 var i: i64 = 2
42 while i < argc {
43 let url: *u8 = argv[i] as *u8
44 status[0] = 0
45 let n: i64 = nx_https_fetch_follow_best(url, store, out, TF_OUTCAP, 6, status)
46 tf_w("FETCH " as *u8); tf_w(url); tf_w(" status=" as *u8); tf_num(status[0]); tf_w(" bytes=" as *u8); tf_num(n)
47 var ok: i64 = 0
48 if n > 0 { if status[0] >= 200 { if status[0] < 300 { ok = 1 } } }
49 if ok == 1 {
50 var po: i64 = tf_cat(path, 0, outdir)
51 po = tf_cat(path, po, "/tpl_" as *u8)
52 po = tf_catn(path, po, i - 2)
53 po = tf_cat(path, po, ".html" as *u8)
54 path[po] = 0 as u8
55 let fd: i64 = sys_openat_wr(path, 0x1a4)
56 if fd >= 0 { sys_write(fd, out, n); sys_close(fd); saved = saved + 1; tf_w(" SAVED " as *u8); tf_w(path) }
57 }
58 tf_w("\n" as *u8)
59 i = i + 1
60 }
61 tf_w("TPL-FETCH saved=" as *u8); tf_num(saved); tf_w("/" as *u8); tf_num(argc - 2); tf_w(" -> " as *u8); tf_w(outdir); tf_w("\n" as *u8)
62 return 0
63}