nx_fetch.nx source
↩ module page · 221 lines · 10643 B
1// nx_fetch.nx -- THE sovereign manifest-driven research fetcher. MERGES the N copy-pasted one-time
2// nx_*_research_fetch organs into ONE data-driven tool (rule 11 data-driven + rule 15 DRY): new research is a
3// MANIFEST ROW, never a new organ. Reads a TSV manifest (each line: url<TAB>name), and for each row:
4// 1. ensures the raw archive knowledge/fetched/<name>.raw (fetch over sovereign TLS-1.3 if absent; idempotent),
5// 2. strips it to clean plain text -> knowledge/library/<name>.txt (the dir the librarian nx_library_harvest_v2
6// walks for .txt) so it BANKS into the durable index, queryable via nx_search_cli.
7// Plaintext strip is purpose-fit for the index (drop tags, suppress <script>/<style>, collapse whitespace) --
8// NOT the browser's nx_html_to_text (which emits GUI link-markers wrong for a corpus). 100% sovereign.
9// usage: nx_fetch <manifest.tsv> expect_exit: 0 license_tier: ORIGINAL
10import "nx_syscalls.nx"
11import "nx_x509_trust_store.nx"
12import "nx_trust_store_load_from_certdata.nx"
13import "nx_https_fetch_follow.nx"
14const K_MAGIC_4096: i64 = 4096
15const K_MAGIC_4194304: i64 = 4194304
16const K_MAGIC_8388608: i64 = 8388608
17const K_MAGIC_1024: i64 = 1024
18
19func puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
20func putn(v: i64) -> i64 {
21 if v == 0 { sys_write(1, "0" as *u8, 1); return 0 }
22 var m: i64 = v
23 if m < 0 { sys_write(1, "-" as *u8, 1); m = 0 - m }
24 let d: *u8 = sys_mmap(24); var k: i64 = 0
25 while m > 0 { d[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
26 var j: i64 = k - 1
27 while j >= 0 { sys_write(1, ((d as i64)+j) as *u8, 1); j = j - 1 }
28 return 0
29}
30func have_file(path: *u8) -> i64 { let fd: i64 = sys_openat_rd(path); if fd < 0 { return 0 } sys_close(fd); return 1 }
31func scat(dst: *u8, o: i64, s: *u8) -> i64 { var i: i64 = 0; while s[i] != (0 as u8) { dst[o + i] = s[i]; i = i + 1 } return o + i }
32func bpath(dst: *u8, dir: *u8, name: *u8, suf: *u8) -> i64 {
33 var o: i64 = scat(dst, 0, dir); o = scat(dst, o, name); o = scat(dst, o, suf); dst[o] = 0 as u8; return o
34}
35func save_file(path: *u8, buf: *u8, n: i64) -> i64 {
36 let fd: i64 = sys_openat_wr(path, 0x1a4)
37 if fd < 0 { return 0 }
38 var off: i64 = 0
39 while off < n { let w: i64 = sys_write(fd, ((buf as i64) + off) as *u8, n - off); if w <= 0 { sys_close(fd); return 0 } off = off + w }
40 sys_close(fd); return 1
41}
42
43// case-insensitive compare src[off..off+litlen) == lit (lit lowercase)
44func tag_eq_ci(src: *u8, off: i64, n: i64, lit: *u8, litlen: i64) -> i64 {
45 if off + litlen > n { return 0 }
46 var i: i64 = 0
47 while i < litlen {
48 var c: i64 = src[off + i] as i64
49 if c >= 0x41 { if c <= 0x5a { c = c + 0x20 } }
50 if c != (lit[i] as i64) { return 0 }
51 i = i + 1
52 }
53 return 1
54}
55
56// tag name at src[off..] equals lit AND is terminated (space/>//tab/nl) -- so "p" won't match <param>.
57func tag_is(src: *u8, off: i64, n: i64, lit: *u8, litlen: i64) -> i64 {
58 if tag_eq_ci(src, off, n, lit, litlen) == 0 { return 0 }
59 let after: i64 = off + litlen
60 if after >= n { return 1 }
61 let c: i64 = src[after] as i64
62 if c == 0x20 { return 1 }
63 if c == 0x3e { return 1 }
64 if c == 0x2f { return 1 }
65 if c == 0x09 { return 1 }
66 if c == 0x0a { return 1 }
67 if c == 0x0d { return 1 }
68 return 0
69}
70
71// is this a BLOCK-level tag? Block boundaries are SENTENCE boundaries -- replacing them with a bare space (the
72// old behaviour) merged e.g. Wikipedia infobox fields into the intro paragraph as one unsplittable run, which
73// broke retrieve-then-read. Emitting a sentence break here is the general readability fix.
74func is_block_tag(src: *u8, off: i64, n: i64) -> i64 {
75 if tag_is(src, off, n, "br" as *u8, 2) == 1 { return 1 }
76 if tag_is(src, off, n, "p" as *u8, 1) == 1 { return 1 }
77 if tag_is(src, off, n, "div" as *u8, 3) == 1 { return 1 }
78 if tag_is(src, off, n, "td" as *u8, 2) == 1 { return 1 }
79 if tag_is(src, off, n, "tr" as *u8, 2) == 1 { return 1 }
80 if tag_is(src, off, n, "th" as *u8, 2) == 1 { return 1 }
81 if tag_is(src, off, n, "li" as *u8, 2) == 1 { return 1 }
82 if tag_is(src, off, n, "ul" as *u8, 2) == 1 { return 1 }
83 if tag_is(src, off, n, "ol" as *u8, 2) == 1 { return 1 }
84 if tag_is(src, off, n, "dd" as *u8, 2) == 1 { return 1 }
85 if tag_is(src, off, n, "dt" as *u8, 2) == 1 { return 1 }
86 if tag_is(src, off, n, "h1" as *u8, 2) == 1 { return 1 }
87 if tag_is(src, off, n, "h2" as *u8, 2) == 1 { return 1 }
88 if tag_is(src, off, n, "h3" as *u8, 2) == 1 { return 1 }
89 if tag_is(src, off, n, "h4" as *u8, 2) == 1 { return 1 }
90 if tag_is(src, off, n, "h5" as *u8, 2) == 1 { return 1 }
91 if tag_is(src, off, n, "h6" as *u8, 2) == 1 { return 1 }
92 if tag_is(src, off, n, "table" as *u8, 5) == 1 { return 1 }
93 if tag_is(src, off, n, "caption" as *u8, 7) == 1 { return 1 }
94 if tag_is(src, off, n, "section" as *u8, 7) == 1 { return 1 }
95 if tag_is(src, off, n, "article" as *u8, 7) == 1 { return 1 }
96 if tag_is(src, off, n, "blockquote" as *u8, 10) == 1 { return 1 }
97 if tag_is(src, off, n, "hr" as *u8, 2) == 1 { return 1 }
98 return 0
99}
100
101// minimal HTML -> plain text for the corpus: drop <...> tags, suppress <script>/<style>, BLOCK tags -> sentence
102// break (readability), collapse ws.
103func strip_html(src: *u8, n: i64, out: *u8, cap: i64) -> i64 {
104 var op: i64 = 0
105 var i: i64 = 0
106 var last_ws: i64 = 1
107 var suppress: i64 = 0
108 while i < n {
109 let c: i64 = src[i] as i64
110 if c == 0x3c {
111 var nameoff: i64 = i + 1
112 var isclose: i64 = 0
113 if nameoff < n { if (src[nameoff] as i64) == 0x2f { isclose = 1; nameoff = nameoff + 1 } }
114 var iss: i64 = 0
115 if tag_eq_ci(src, nameoff, n, "script" as *u8, 6) == 1 { iss = 1 }
116 if tag_eq_ci(src, nameoff, n, "style" as *u8, 5) == 1 { iss = 1 }
117 if iss == 1 { if isclose == 1 { suppress = 0 } else { suppress = 1 } }
118 let isblock: i64 = is_block_tag(src, nameoff, n)
119 i = i + 1
120 var g: i64 = 0
121 while g == 0 { if i >= n { g = 1 } else { if (src[i] as i64) == 0x3e { g = 1 } else { i = i + 1 } } }
122 if i < n { i = i + 1 }
123 if suppress == 0 {
124 if isblock == 1 {
125 // block boundary -> sentence break ". " (dropped as an empty sentence if it lands on ws)
126 if op < cap { if last_ws == 0 { out[op] = 0x20 as u8; op = op + 1 } }
127 if op < cap { out[op] = 0x2e as u8; op = op + 1 }
128 if op < cap { out[op] = 0x20 as u8; op = op + 1 }
129 last_ws = 1
130 } else {
131 if last_ws == 0 { if op < cap { out[op] = 0x20 as u8; op = op + 1; last_ws = 1 } }
132 }
133 }
134 } else {
135 if suppress == 1 { i = i + 1 }
136 else {
137 var isws: i64 = 0
138 if c == 0x20 { isws = 1 }
139 if c == 0x09 { isws = 1 }
140 if c == 0x0a { isws = 1 }
141 if c == 0x0d { isws = 1 }
142 if isws == 1 {
143 if last_ws == 0 { if op < cap { out[op] = 0x20 as u8; op = op + 1; last_ws = 1 } }
144 } else {
145 if op < cap { out[op] = c as u8; op = op + 1; last_ws = 0 }
146 }
147 i = i + 1
148 }
149 }
150 }
151 return op
152}
153
154// ensure raw archive (fetch if absent) -> strip -> library .txt. Returns 1 on a banked row.
155func fetch_to_lib(url: *u8, name: *u8, store: *TrustStore, out: *u8, cap: i64) -> i64 {
156 let rawpath: *u8 = sys_mmap(K_MAGIC_4096); bpath(rawpath, "knowledge/fetched/" as *u8, name, ".raw" as *u8)
157 let txtpath: *u8 = sys_mmap(K_MAGIC_4096); bpath(txtpath, "knowledge/library/" as *u8, name, ".txt" as *u8)
158 let lenbox: *i64 = sys_mmap(16) as *i64
159 var raw: *u8 = 0 as *u8
160 var n: i64 = 0
161 if have_file(rawpath) == 1 {
162 raw = sys_read_file(rawpath, lenbox); n = lenbox[0]
163 puts(name); puts(" [raw cached]")
164 } else {
165 let status: *i64 = sys_mmap(8) as *i64
166 n = nx_https_fetch_follow(url, store, out, cap, 6, status)
167 puts(name); puts(" status="); putn(status[0])
168 if n <= 0 { puts(" FETCH-FAIL\n"); return 0 }
169 save_file(rawpath, out, n)
170 raw = out
171 }
172 let txt: *u8 = sys_mmap(cap)
173 let tn: i64 = strip_html(raw, n, txt, cap)
174 if save_file(txtpath, txt, tn) == 0 { puts(" TXT-SAVE-FAIL\n"); return 0 }
175 puts(" raw="); putn(n); puts(" -> lib.txt="); putn(tn); puts(" BANKED\n")
176 return 1
177}
178
179func main(argc: i64, argv: *i64) -> i64 {
180 if argc < 2 { puts("usage: nx_fetch <manifest.tsv>\n" as *u8); return 1 }
181 let manifest: *u8 = argv[1] as *u8
182 let r: i64 = nx_trust_store_load_from_certdata("data/mozilla_certdata.txt" as *u8, 512, K_MAGIC_4194304)
183 if r <= 0 { puts("nx_fetch: certdata load failed\n" as *u8); return 1 }
184 let store: *TrustStore = r as *TrustStore
185 sys_mkdir("knowledge/fetched" as *u8, 0x1ed)
186 sys_mkdir("knowledge/library" as *u8, 0x1ed)
187 puts("CA roots="); putn(trust_store_count(store)); puts(" manifest="); puts(manifest); puts("\n")
188
189 let lb: *i64 = sys_mmap(16) as *i64
190 let mdata: *u8 = sys_read_file(manifest, lb)
191 if (mdata as i64) == 0 { puts("nx_fetch: cannot read manifest\n" as *u8); return 1 }
192 let mn: i64 = lb[0]
193 let cap: i64 = K_MAGIC_8388608
194 let out: *u8 = sys_mmap(cap)
195 let urlbuf: *u8 = sys_mmap(K_MAGIC_4096)
196 let namebuf: *u8 = sys_mmap(K_MAGIC_1024)
197
198 var pos: i64 = 0
199 var ok: i64 = 0
200 var nrows: i64 = 0
201 while pos < mn {
202 var eol: i64 = pos
203 var fe: i64 = 0
204 while fe == 0 { if eol >= mn { fe = 1 } else { if (mdata[eol] as i64) == 10 { fe = 1 } else { eol = eol + 1 } } }
205 var tab: i64 = pos
206 var ft: i64 = 0
207 while ft == 0 { if tab >= eol { ft = 1 } else { if (mdata[tab] as i64) == 9 { ft = 1 } else { tab = tab + 1 } } }
208 if tab < eol {
209 var a: i64 = 0
210 while pos + a < tab { urlbuf[a] = mdata[pos + a]; a = a + 1 }
211 urlbuf[a] = 0 as u8
212 var b: i64 = 0
213 while tab + 1 + b < eol { namebuf[b] = mdata[tab + 1 + b]; b = b + 1 }
214 namebuf[b] = 0 as u8
215 if b > 0 { ok = ok + fetch_to_lib(urlbuf, namebuf, store, out, cap); nrows = nrows + 1 }
216 }
217 pos = eol + 1
218 }
219 puts("BANKED-READY: "); putn(ok); puts(" / "); putn(nrows); puts(" rows -> knowledge/library/ (run nx_library_harvest_v2 to index)\n")
220 return 0
221}