code wiki / _hdl_build / nx_feed_extract.nx
nx_feed_extract.nx source
↩ module page · 299 lines · 16689 B
1// nx_feed_extract.nx -- RSS 2.0 + Atom feed extractor (grow-capabilities 2026-07-05, MEASURED: reddit's
2// /.rss returns 200 + a real Atom feed while its SPA shell is empty and .json is 403). Feeds are the
3// crawler-FRIENDLY non-JS content path a huge slice of the web exposes (news/blogs/forums/reddit) -- they
4// carry real item title + link + summary AND list fresh URLs (crawl frontier). This parses <entry> (Atom)
5// and <item> (RSS) into indexable text (title + stripped summary per item) + returns the item count. A
6// parser, not a JS VM. Pairs with nx_feed_discover (find the <link rel=alternate> feed URL in a shell).
7// license_tier: ORIGINAL
8import "nx_syscalls.nx"
9const K_MAGIC_65536: i64 = 65536
10const K_MAGIC_8192: i64 = 8192
11
12// PER-CALL SCRATCH -> LAZY STATICS (2026-08-19 lane F, MEASURED by nx_crawl_callee_probe on a real
13// 611 KB page: 272 kB/call leaked in nx_feed_extract alone = 4x64KiB + 2x8KiB own-VMA mmaps per call,
14// never freed -- THE dominant per-page allocator behind the crawler's +282 MB/min ingest growth
15// (debt 1787082132; wc_harvest+olh_scan explain only ~20 kB/page). nx_feed_item_at leaked 3x64KiB more.
16// All buffers are fully rewritten per use and the crawler's fetch children fork their own copies
17// (statics are per-process), so one shared set is SOUND. One-time cost ~470 KiB per process.
18static fe_tbuf_g: *u8
19static fe_sbuf_g: *u8
20static fe_dbuf1_g: *u8
21static fe_dbuf2_g: *u8
22static fe_strip_g: *u8
23static fe_tdec_g: *u8
24static fe_raw_g: *u8
25static fe_dc_g: *u8
26static fe_dc2_g: *u8
27func fe_scratch() -> i64 {
28 if (fe_tbuf_g as i64) == 0 {
29 fe_tbuf_g = sys_mmap(K_MAGIC_8192)
30 fe_sbuf_g = sys_mmap(K_MAGIC_65536)
31 fe_dbuf1_g = sys_mmap(K_MAGIC_65536)
32 fe_dbuf2_g = sys_mmap(K_MAGIC_65536)
33 fe_strip_g = sys_mmap(K_MAGIC_65536)
34 fe_tdec_g = sys_mmap(K_MAGIC_8192)
35 fe_raw_g = sys_mmap(K_MAGIC_65536)
36 fe_dc_g = sys_mmap(K_MAGIC_65536)
37 fe_dc2_g = sys_mmap(K_MAGIC_65536)
38 }
39 return 0
40}
41
42func fe_lc(c: i64) -> i64 { if c >= 65 { if c <= 90 { return c + 32 } } return c }
43// first index of ndl (len nlen) at/after `from` within hay[..hlen), or -1. case-insensitive.
44func fe_find(hay: *u8, from: i64, hlen: i64, ndl: *u8, nlen: i64) -> i64 {
45 var i: i64 = from
46 while i + nlen <= hlen {
47 var m: i64 = 1; var j: i64 = 0
48 while j < nlen { if fe_lc(hay[i+j] as i64) != fe_lc(ndl[j] as i64) { m = 0; j = nlen } else { j = j + 1 } }
49 if m == 1 { return i }
50 i = i + 1
51 }
52 return 0 - 1
53}
54// inner text of the FIRST <tag ...>...</tag> within [s, e); strips a wrapping CDATA. returns len or -1.
55func fe_tag_inner(xml: *u8, s: i64, e: i64, tag: *u8, tlen: i64, out: *u8, cap: i64) -> i64 {
56 // find "<tag" then the '>' that ends the open tag
57 let lt: *u8 = sys_mmap(64); lt[0] = 60 as u8 // '<'
58 var t: i64 = 0; while t < tlen { lt[1+t] = tag[t]; t = t + 1 }
59 let op: i64 = fe_find(xml, s, e, lt, tlen + 1)
60 if op < 0 { return 0 - 1 }
61 // char after "<tag" must be a boundary ('>' or ws or '/') so <title> != <titlex>
62 let bc: i64 = xml[op + 1 + tlen] as i64
63 if bc != 62 { if bc != 32 { if bc != 9 { if bc != 10 { if bc != 13 { if bc != 47 {
64 // not a clean tag boundary -> search again past this
65 return fe_tag_inner(xml, op + 1, e, tag, tlen, out, cap)
66 } } } } } }
67 var gt: i64 = op + 1 + tlen
68 var fnd: i64 = 0
69 while fnd == 0 { if gt >= e { return 0 - 1 } if xml[gt] == (62 as u8) { fnd = 1 } else { gt = gt + 1 } }
70 let istart: i64 = gt + 1
71 // find "</tag>"
72 let ct: *u8 = sys_mmap(64); ct[0] = 60 as u8; ct[1] = 47 as u8 // "</"
73 t = 0; while t < tlen { ct[2+t] = tag[t]; t = t + 1 }
74 let cl: i64 = fe_find(xml, istart, e, ct, tlen + 2)
75 if cl < 0 { return 0 - 1 }
76 var vs: i64 = istart; var ve: i64 = cl
77 // strip a wrapping CDATA: <![CDATA[ ... ]]>
78 let cd: *u8 = "<![CDATA[" as *u8
79 if ve - vs >= 11 {
80 var ism: i64 = 1; var z: i64 = 0
81 while z < 9 { if xml[vs+z] != cd[z] { ism = 0; z = 9 } else { z = z + 1 } }
82 if ism == 1 { vs = vs + 9; if ve - 3 >= vs { ve = ve - 3 } } // drop "]]>"
83 }
84 var o: i64 = 0; var p: i64 = vs
85 while p < ve { if o < cap - 1 { out[o] = xml[p]; o = o + 1 } p = p + 1 }
86 out[o] = 0 as u8
87 return o
88}
89// decode the common HTML/XML entities one pass: <>&"'' &#NN; -> chars.
90// (feed <content> is often HTML escaped INSIDE the XML, sometimes double-escaped -> caller runs this twice.)
91func fe_decode_entities(inb: *u8, inlen: i64, out: *u8, cap: i64) -> i64 {
92 var o: i64 = 0; var i: i64 = 0
93 while i < inlen {
94 if inb[i] == (38 as u8) { // '&'
95 var semi: i64 = 0 - 1; var s: i64 = i + 1
96 while s < inlen { if s - i > 10 { s = inlen } else { if inb[s] == (59 as u8) { semi = s; s = inlen } else { s = s + 1 } } }
97 if semi < 0 { if o < cap-1 { out[o] = 38 as u8; o = o + 1 } i = i + 1 }
98 else {
99 let nl: i64 = semi - i - 1
100 var emit: i64 = 0 - 1
101 if nl == 2 { if fe_lc(inb[i+1] as i64)==108 { if fe_lc(inb[i+2] as i64)==116 { emit = 60 } } // lt
102 if fe_lc(inb[i+1] as i64)==103 { if fe_lc(inb[i+2] as i64)==116 { emit = 62 } } } // gt
103 if nl == 3 { if fe_lc(inb[i+1] as i64)==97 { if fe_lc(inb[i+2] as i64)==109 { if fe_lc(inb[i+3] as i64)==112 { emit = 38 } } } } // amp
104 if nl == 4 { if fe_lc(inb[i+1] as i64)==113 { emit = 34 } // quot
105 if fe_lc(inb[i+1] as i64)==97 { if fe_lc(inb[i+2] as i64)==112 { emit = 39 } } // apos
106 if fe_lc(inb[i+1] as i64)==110 { emit = 32 } } // nbsp
107 if inb[i+1] == (35 as u8) { // numeric &#NN;
108 var v: i64 = 0; var p: i64 = i + 2
109 while p < semi { let d: i64 = inb[p] as i64; if d >= 48 { if d <= 57 { v = v*10 + (d-48) } } p = p + 1 }
110 if v > 0 { if v < 128 { emit = v } else { emit = 32 } }
111 }
112 if emit >= 0 { if o < cap-1 { out[o] = emit as u8; o = o + 1 } i = semi + 1 }
113 else { if o < cap-1 { out[o] = 38 as u8; o = o + 1 } i = i + 1 } // unknown entity: keep '&'
114 }
115 } else { if o < cap-1 { out[o] = inb[i]; o = o + 1 } i = i + 1 }
116 }
117 out[o] = 0 as u8
118 return o
119}
120// strip HTML tags + collapse whitespace from inb -> out (feed summaries are HTML). returns len.
121func fe_strip_tags(inb: *u8, inlen: i64, out: *u8, cap: i64) -> i64 {
122 var o: i64 = 0; var i: i64 = 0; var intag: i64 = 0; var lastsp: i64 = 0
123 while i < inlen {
124 let c: i64 = inb[i] as i64
125 if c == 60 { intag = 1 } // '<'
126 else { if c == 62 { intag = 0 } // '>'
127 else { if intag == 0 {
128 var ch: i64 = c
129 if ch == 9 { ch = 32 } if ch == 10 { ch = 32 } if ch == 13 { ch = 32 }
130 if ch == 32 { if lastsp == 0 { if o < cap-1 { out[o] = 32 as u8; o = o + 1 } lastsp = 1 } }
131 else { if o < cap-1 { out[o] = ch as u8; o = o + 1 } lastsp = 0 }
132 } } }
133 i = i + 1
134 }
135 out[o] = 0 as u8
136 return o
137}
138
139// FEED DISCOVERY: find the first <link rel="alternate" type="application/(rss|atom)+xml" href="..."> in a
140// page's <head> and copy its href -> out_url. Returns 1 if found. This is how the crawler turns a JS SPA
141// shell into real content: discover the feed link, then fetch+nx_feed_extract it. (attr order-independent.)
142func fe_link_href(html: *u8, ts: i64, te: i64, out: *u8, cap: i64) -> i64 {
143 // returns href value length within tag [ts,te), or -1
144 let key: *u8 = "href" as *u8
145 var i: i64 = ts
146 while i + 5 < te {
147 if fe_lc(html[i] as i64)==104 { if fe_lc(html[i+1] as i64)==114 { if fe_lc(html[i+2] as i64)==101 { if fe_lc(html[i+3] as i64)==102 {
148 var p: i64 = i + 4
149 while p < te { let c: i64 = html[p] as i64; if c==32 { p=p+1 } else { if c==9 { p=p+1 } else { p=te+9 } } }
150 var pp: i64 = i + 4
151 var st: i64 = 0
152 while st==0 { if pp>=te { st=1 } else { let c: i64=html[pp] as i64; if c==32 { pp=pp+1 } else { if c==9 { pp=pp+1 } else { st=1 } } } }
153 if pp < te { if html[pp]==(61 as u8) {
154 pp = pp + 1
155 var st2: i64 = 0
156 while st2==0 { if pp>=te { st2=1 } else { let c: i64=html[pp] as i64; if c==32 { pp=pp+1 } else { if c==9 { pp=pp+1 } else { if c==34 { st2=1 } else { if c==39 { st2=1 } else { st2=1 } } } } } }
157 if pp < te { let q: i64 = html[pp] as i64
158 var vs: i64 = 0; var ve: i64 = 0
159 if q==34 { pp=pp+1; vs=pp; while pp<te { if html[pp]==(34 as u8) { ve=pp; pp=te } else { pp=pp+1 } } }
160 else { if q==39 { pp=pp+1; vs=pp; while pp<te { if html[pp]==(39 as u8) { ve=pp; pp=te } else { pp=pp+1 } } }
161 else { vs=pp; while pp<te { let c: i64=html[pp] as i64; if c==32 { ve=pp; pp=te } else { if c==62 { ve=pp; pp=te } else { pp=pp+1 } } } if ve==0 { ve=te } } }
162 if ve > vs { var o: i64=0; while vs+o < ve { if o<cap-1 { out[o]=html[vs+o]; o=o+1 } } out[o]=0 as u8; return o }
163 }
164 } }
165 } } } }
166 i = i + 1
167 }
168 return 0 - 1
169}
170func nx_feed_discover(html: *u8, hlen: i64, out_url: *u8, cap: i64) -> i64 {
171 var i: i64 = 0
172 let rss: *u8 = "application/rss+xml" as *u8
173 let atom: *u8 = "application/atom+xml" as *u8
174 while i + 5 < hlen {
175 if html[i]==(60 as u8) { if fe_lc(html[i+1] as i64)==108 { if fe_lc(html[i+2] as i64)==105 { if fe_lc(html[i+3] as i64)==110 { if fe_lc(html[i+4] as i64)==107 { // "<link"
176 var te: i64 = i + 5; var fnd: i64 = 0
177 while fnd==0 { if te>=hlen { fnd=1 } else { if html[te]==(62 as u8) { fnd=1 } else { te=te+1 } } }
178 if fe_find(html, i, te, rss, 19) >= 0 { if fe_link_href(html, i+5, te, out_url, cap) > 0 { return 1 } }
179 if fe_find(html, i, te, atom, 20) >= 0 { if fe_link_href(html, i+5, te, out_url, cap) > 0 { return 1 } }
180 i = te
181 } } } } }
182 i = i + 1
183 }
184 return 0
185}
186// Per-item accessor: fill the idx-th item's decoded TITLE, article LINK, and clean SUMMARY. Returns 1 if the
187// item exists, 0 past the end. Link handles BOTH encodings: RSS <link>URL</link> (element text) and Atom
188// <link href="URL"/> (attribute). Lets the crawler index each feed item as its OWN doc with its OWN url --
189// specific-article retrieval + fresh frontier URLs. (Re-scans from start per idx; feeds are small.)
190func nx_feed_item_at(xml: *u8, xlen: i64, idx: i64,
191 out_title: *u8, tcap: i64, out_link: *u8, lcap: i64,
192 out_sum: *u8, scap: i64) -> i64 {
193 var itemtag: *u8 = "item" as *u8; var itlen: i64 = 4
194 if fe_find(xml, 0, xlen, "<entry" as *u8, 6) >= 0 { itemtag = "entry" as *u8; itlen = 5 }
195 let openlt: *u8 = sys_mmap(16); openlt[0] = 60 as u8
196 var z: i64 = 0; while z < itlen { openlt[1+z] = itemtag[z]; z = z + 1 }
197 let closelt: *u8 = sys_mmap(16); closelt[0] = 60 as u8; closelt[1] = 47 as u8
198 z = 0; while z < itlen { closelt[2+z] = itemtag[z]; z = z + 1 }
199 closelt[2+itlen] = 62 as u8
200 var pos: i64 = 0; var cur: i64 = 0
201 fe_scratch()
202 let raw: *u8 = fe_raw_g; let dc: *u8 = fe_dc_g; let dc2: *u8 = fe_dc2_g
203 while cur <= idx {
204 let es: i64 = fe_find(xml, pos, xlen, openlt, itlen + 1)
205 if es < 0 { return 0 }
206 let ee: i64 = fe_find(xml, es, xlen, closelt, itlen + 3)
207 if ee < 0 { return 0 }
208 if cur == idx {
209 // title (decode entities once)
210 let tl: i64 = fe_tag_inner(xml, es, ee, "title" as *u8, 5, raw, K_MAGIC_65536)
211 if tl > 0 { fe_decode_entities(raw, tl, out_title, tcap) } else { out_title[0] = 0 as u8 }
212 // link: RSS element text first; if empty/not-a-url, Atom href
213 out_link[0] = 0 as u8
214 let ll: i64 = fe_tag_inner(xml, es, ee, "link" as *u8, 4, out_link, lcap)
215 var haveurl: i64 = 0
216 if ll >= 7 { if out_link[0]==(104 as u8) { if out_link[1]==(116 as u8) { if out_link[2]==(116 as u8) { if out_link[3]==(112 as u8) { haveurl = 1 } } } } }
217 if haveurl == 0 {
218 out_link[0] = 0 as u8
219 let lp: i64 = fe_find(xml, es, ee, "<link" as *u8, 5)
220 if lp >= 0 {
221 var lte: i64 = lp + 5; var fd: i64 = 0
222 while fd == 0 { if lte >= ee { fd = 1 } else { if xml[lte]==(62 as u8) { fd = 1 } else { lte = lte + 1 } } }
223 fe_link_href(xml, lp + 5, lte, out_link, lcap)
224 }
225 }
226 // summary: content > summary > description ; decode x2 -> strip
227 var sl: i64 = fe_tag_inner(xml, es, ee, "content" as *u8, 7, raw, K_MAGIC_65536)
228 if sl < 0 { sl = fe_tag_inner(xml, es, ee, "summary" as *u8, 7, raw, K_MAGIC_65536) }
229 if sl < 0 { sl = fe_tag_inner(xml, es, ee, "description" as *u8, 11, raw, K_MAGIC_65536) }
230 if sl > 0 {
231 let d1: i64 = fe_decode_entities(raw, sl, dc, K_MAGIC_65536)
232 let d2: i64 = fe_decode_entities(dc, d1, dc2, K_MAGIC_65536)
233 fe_strip_tags(dc2, d2, out_sum, scap)
234 } else { out_sum[0] = 0 as u8 }
235 return 1
236 }
237 cur = cur + 1
238 pos = ee + itlen + 3
239 }
240 return 0
241}
242// Parse a feed -> indexable text: "TITLE. SUMMARY\n" per item. Returns item count.
243// Handles Atom <entry> and RSS <item>; summary from <content>|<summary>|<description>.
244func nx_feed_extract(xml: *u8, xlen: i64, out: *u8, out_cap: i64) -> i64 {
245 // choose the item element: Atom "entry" if the doc has <entry, else RSS "item"
246 var itemtag: *u8 = "item" as *u8; var itlen: i64 = 4
247 if fe_find(xml, 0, xlen, "<entry" as *u8, 6) >= 0 { itemtag = "entry" as *u8; itlen = 5 }
248 let openlt: *u8 = sys_mmap(16); openlt[0] = 60 as u8 // "<" + itemtag
249 var z: i64 = 0; while z < itlen { openlt[1+z] = itemtag[z]; z = z + 1 }
250 let closelt: *u8 = sys_mmap(16); closelt[0] = 60 as u8; closelt[1] = 47 as u8 // "</" + itemtag + ">"
251 z = 0; while z < itlen { closelt[2+z] = itemtag[z]; z = z + 1 }
252 closelt[2+itlen] = 62 as u8
253 var count: i64 = 0
254 var pos: i64 = 0
255 var ol: i64 = 0
256 fe_scratch()
257 let tbuf: *u8 = fe_tbuf_g
258 let sbuf: *u8 = fe_sbuf_g
259 let dbuf1: *u8 = fe_dbuf1_g
260 let dbuf2: *u8 = fe_dbuf2_g
261 let stripped: *u8 = fe_strip_g
262 let tdec: *u8 = fe_tdec_g
263 var go: i64 = 1
264 while go == 1 {
265 let es: i64 = fe_find(xml, pos, xlen, openlt, itlen + 1)
266 if es < 0 { go = 0 } else {
267 let ee: i64 = fe_find(xml, es, xlen, closelt, itlen + 3)
268 if ee < 0 { go = 0 } else {
269 // title
270 let tl: i64 = fe_tag_inner(xml, es, ee, "title" as *u8, 5, tbuf, K_MAGIC_8192)
271 // summary: content > summary > description
272 var sl: i64 = fe_tag_inner(xml, es, ee, "content" as *u8, 7, sbuf, K_MAGIC_65536)
273 if sl < 0 { sl = fe_tag_inner(xml, es, ee, "summary" as *u8, 7, sbuf, K_MAGIC_65536) }
274 if sl < 0 { sl = fe_tag_inner(xml, es, ee, "description" as *u8, 11, sbuf, K_MAGIC_65536) }
275 if tl > 0 {
276 // title: decode entities (titles are usually singly-escaped)
277 let tdl: i64 = fe_decode_entities(tbuf, tl, tdec, K_MAGIC_8192)
278 var k: i64 = 0
279 while k < tdl { if ol < out_cap-2 { out[ol] = tdec[k]; ol = ol + 1 } k = k + 1 }
280 if ol < out_cap-2 { out[ol] = 46 as u8; ol = ol + 1 } // '.'
281 if ol < out_cap-2 { out[ol] = 32 as u8; ol = ol + 1 }
282 if sl > 0 {
283 // summary is HTML-escaped inside XML (sometimes double): decode x2 -> strip tags -> clean text
284 let d1: i64 = fe_decode_entities(sbuf, sl, dbuf1, K_MAGIC_65536)
285 let d2: i64 = fe_decode_entities(dbuf1, d1, dbuf2, K_MAGIC_65536)
286 let spl: i64 = fe_strip_tags(dbuf2, d2, stripped, K_MAGIC_65536)
287 k = 0
288 while k < spl { if ol < out_cap-2 { out[ol] = stripped[k]; ol = ol + 1 } k = k + 1 }
289 }
290 if ol < out_cap-2 { out[ol] = 10 as u8; ol = ol + 1 } // '\n'
291 count = count + 1
292 }
293 pos = ee + itlen + 3
294 }
295 }
296 }
297 out[ol] = 0 as u8
298 return count
299}