nx_corpus_fetch.nx source

1// nx_corpus_fetch.nx -- Modelwright M1: sovereign corpus-at-scale builder. 2// 3// Fetches public-text pages via the sovereign fetcher (nx_https_fetch_follow), 4// strips HTML to plain text (nx_html_to_text), concatenates into one training 5// corpus, and writes knowledge/corpus/corpus.txt. This is rung M1 of training 6// Modelwright FROM SCRATCH: the toy LM trained on 21KB; this scales the corpus. 7// 8// 100% sovereign (nx_cc->nxasm, own TLS-1.3 + own HTML stripper). The only 9// non-Nishi inputs are the CA-root DATA (/tmp/mozilla_certdata.txt) + the 10// fetched public text itself -- exactly the "what we have to" DATA category. 11// 12// expect_exit: 0 license_tier: ORIGINAL 13// nx_safety_envelope: 14// intended_use: build a sovereign training corpus from public web text 15// sil_target: SIL1 16// verdict: NOT_YET_EVALUATED 17// genealogy_id: corpus_concat lineage_id: nx_corpus_fetch_v1 18 19import "nx_syscalls.nx" 20import "nx_x509_trust_store.nx" 21import "nx_trust_store_load_from_certdata.nx" 22import "nx_https_fetch_follow.nx" 23import "nx_html_to_text.nx" 24const K_MAGIC_8388608: i64 = 8388608 25const K_MAGIC_4194304: i64 = 4194304 26const K_MAGIC_33554432: i64 = 33554432 27 28func w(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 } 29func wn(v: i64) -> i64 { 30 let b: *u8=sys_mmap(28); var m: i64=v 31 if m<0{m=0-m;sys_write(1,"-" as *u8,1)} 32 let t: *u8=sys_mmap(28); var k: i64=0 33 if m==0{t[0]=48 as u8;k=1} 34 while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1} 35 var i: i64=0; while i<k{b[i]=t[k-1-i];i=i+1} 36 sys_write(1,b,k); return 0 37} 38 39// fetch url -> strip HTML -> append text (+ '\n' doc separator) into corpus[coff..]. 40// Returns text bytes appended (0 on any failure -- graceful, never aborts the run). 41func grab(url: *u8, store: *TrustStore, raw: *u8, txt: *u8, corpus: *u8, coff: *i64) -> i64 { 42 let st: *i64 = (sys_mmap(8)) as *i64 43 let n: i64 = nx_https_fetch_follow(url, store, raw, K_MAGIC_8388608, 6, st) 44 w(url); w(" status=" as *u8); wn(st[0]); w(" raw=" as *u8); wn(n) 45 if n <= 0 { w(" SKIP\n" as *u8); return 0 } 46 let tn: i64 = nx_html_to_text(raw, n, txt, K_MAGIC_8388608) 47 if tn <= 0 { w(" no-text\n" as *u8); return 0 } 48 var off: i64 = coff[0] 49 var i: i64 = 0 50 while i < tn { corpus[off+i] = txt[i]; i = i+1 } 51 off = off + tn 52 corpus[off] = 10 as u8 // '\n' document separator 53 off = off + 1 54 coff[0] = off 55 w(" text=" as *u8); wn(tn); w("\n" as *u8) 56 return tn 57} 58 59func main() -> i64 { 60 let r: i64 = nx_trust_store_load_from_certdata("/tmp/mozilla_certdata.txt" as *u8, 300, K_MAGIC_4194304) 61 if r <= 0 { w("STORE-FAIL (need /tmp/mozilla_certdata.txt)\n" as *u8); return 1 } 62 let store: *TrustStore = r as *TrustStore 63 64 let raw: *u8 = sys_mmap(K_MAGIC_8388608) // 8MB per-page raw HTML 65 let txt: *u8 = sys_mmap(K_MAGIC_8388608) // 8MB per-page stripped text 66 let corpus: *u8 = sys_mmap(K_MAGIC_33554432) // 32MB accumulated corpus 67 let coff: *i64 = (sys_mmap(8)) as *i64 68 coff[0] = 0 69 70 var got: i64 = 0 71 got = got + grab("https://en.wikipedia.org/wiki/Science" as *u8, store, raw, txt, corpus, coff) 72 got = got + grab("https://en.wikipedia.org/wiki/History" as *u8, store, raw, txt, corpus, coff) 73 got = got + grab("https://en.wikipedia.org/wiki/Mathematics" as *u8, store, raw, txt, corpus, coff) 74 got = got + grab("https://en.wikipedia.org/wiki/Technology" as *u8, store, raw, txt, corpus, coff) 75 got = got + grab("https://en.wikipedia.org/wiki/Economics" as *u8, store, raw, txt, corpus, coff) 76 got = got + grab("https://en.wikipedia.org/wiki/Biology" as *u8, store, raw, txt, corpus, coff) 77 got = got + grab("https://en.wikipedia.org/wiki/Music" as *u8, store, raw, txt, corpus, coff) 78 got = got + grab("https://en.wikipedia.org/wiki/Geography" as *u8, store, raw, txt, corpus, coff) 79 80 let total: i64 = coff[0] 81 let fd: i64 = sys_openat_wr("knowledge/corpus/corpus.txt" as *u8, 420) 82 if fd < 0 { w("OPEN-FAIL (mkdir knowledge/corpus first)\n" as *u8); return 1 } 83 var wr: i64 = 0 84 while wr < total { 85 let c: i64 = sys_write(fd, corpus + wr, total - wr) 86 if c <= 0 { break } 87 wr = wr + c 88 } 89 sys_close(fd) 90 91 w("CORPUS docs_text=" as *u8); wn(got) 92 w(" bytes=" as *u8); wn(total) 93 w(" written=" as *u8); wn(wr) 94 w(" -> knowledge/corpus/corpus.txt\n" as *u8) 95 if wr == total { return 0 } 96 return 1 97}