code wiki / _hdl_build / nx_feed_extract.nx
nx_feed_extract.nx source
↩ module page · 267 lines · 15511 B
1// nx_feed_extract.nx -- RSS 2.0 + Atom feed extractor (grow-capabilities 2026-07-05, MEASURED: reddit's
2// /.rss returns 200 + a real Atom feed while its SPA shell is empty and .json is 403). Feeds are the
3// crawler-FRIENDLY non-JS content path a huge slice of the web exposes (news/blogs/forums/reddit) -- they
4// carry real item title + link + summary AND list fresh URLs (crawl frontier). This parses <entry> (Atom)
5// and <item> (RSS) into indexable text (title + stripped summary per item) + returns the item count. A
6// parser, not a JS VM. Pairs with nx_feed_discover (find the <link rel=alternate> feed URL in a shell).
7// license_tier: ORIGINAL
8import "nx_syscalls.nx"
9const K_MAGIC_65536: i64 = 65536
10const K_MAGIC_8192: i64 = 8192
11
12func fe_lc(c: i64) -> i64 { if c >= 65 { if c <= 90 { return c + 32 } } return c }
13// first index of ndl (len nlen) at/after `from` within hay[..hlen), or -1. case-insensitive.
14func fe_find(hay: *u8, from: i64, hlen: i64, ndl: *u8, nlen: i64) -> i64 {
15 var i: i64 = from
16 while i + nlen <= hlen {
17 var m: i64 = 1; var j: i64 = 0
18 while j < nlen { if fe_lc(hay[i+j] as i64) != fe_lc(ndl[j] as i64) { m = 0; j = nlen } else { j = j + 1 } }
19 if m == 1 { return i }
20 i = i + 1
21 }
22 return 0 - 1
23}
24// inner text of the FIRST <tag ...>...</tag> within [s, e); strips a wrapping CDATA. returns len or -1.
25func fe_tag_inner(xml: *u8, s: i64, e: i64, tag: *u8, tlen: i64, out: *u8, cap: i64) -> i64 {
26 // find "<tag" then the '>' that ends the open tag
27 let lt: *u8 = sys_mmap(64); lt[0] = 60 as u8 // '<'
28 var t: i64 = 0; while t < tlen { lt[1+t] = tag[t]; t = t + 1 }
29 let op: i64 = fe_find(xml, s, e, lt, tlen + 1)
30 if op < 0 { return 0 - 1 }
31 // char after "<tag" must be a boundary ('>' or ws or '/') so <title> != <titlex>
32 let bc: i64 = xml[op + 1 + tlen] as i64
33 if bc != 62 { if bc != 32 { if bc != 9 { if bc != 10 { if bc != 13 { if bc != 47 {
34 // not a clean tag boundary -> search again past this
35 return fe_tag_inner(xml, op + 1, e, tag, tlen, out, cap)
36 } } } } } }
37 var gt: i64 = op + 1 + tlen
38 var fnd: i64 = 0
39 while fnd == 0 { if gt >= e { return 0 - 1 } if xml[gt] == (62 as u8) { fnd = 1 } else { gt = gt + 1 } }
40 let istart: i64 = gt + 1
41 // find "</tag>"
42 let ct: *u8 = sys_mmap(64); ct[0] = 60 as u8; ct[1] = 47 as u8 // "</"
43 t = 0; while t < tlen { ct[2+t] = tag[t]; t = t + 1 }
44 let cl: i64 = fe_find(xml, istart, e, ct, tlen + 2)
45 if cl < 0 { return 0 - 1 }
46 var vs: i64 = istart; var ve: i64 = cl
47 // strip a wrapping CDATA: <![CDATA[ ... ]]>
48 let cd: *u8 = "<![CDATA[" as *u8
49 if ve - vs >= 11 {
50 var ism: i64 = 1; var z: i64 = 0
51 while z < 9 { if xml[vs+z] != cd[z] { ism = 0; z = 9 } else { z = z + 1 } }
52 if ism == 1 { vs = vs + 9; if ve - 3 >= vs { ve = ve - 3 } } // drop "]]>"
53 }
54 var o: i64 = 0; var p: i64 = vs
55 while p < ve { if o < cap - 1 { out[o] = xml[p]; o = o + 1 } p = p + 1 }
56 out[o] = 0 as u8
57 return o
58}
59// decode the common HTML/XML entities one pass: <>&"'' &#NN; -> chars.
60// (feed <content> is often HTML escaped INSIDE the XML, sometimes double-escaped -> caller runs this twice.)
61func fe_decode_entities(inb: *u8, inlen: i64, out: *u8, cap: i64) -> i64 {
62 var o: i64 = 0; var i: i64 = 0
63 while i < inlen {
64 if inb[i] == (38 as u8) { // '&'
65 var semi: i64 = 0 - 1; var s: i64 = i + 1
66 while s < inlen { if s - i > 10 { s = inlen } else { if inb[s] == (59 as u8) { semi = s; s = inlen } else { s = s + 1 } } }
67 if semi < 0 { if o < cap-1 { out[o] = 38 as u8; o = o + 1 } i = i + 1 }
68 else {
69 let nl: i64 = semi - i - 1
70 var emit: i64 = 0 - 1
71 if nl == 2 { if fe_lc(inb[i+1] as i64)==108 { if fe_lc(inb[i+2] as i64)==116 { emit = 60 } } // lt
72 if fe_lc(inb[i+1] as i64)==103 { if fe_lc(inb[i+2] as i64)==116 { emit = 62 } } } // gt
73 if nl == 3 { if fe_lc(inb[i+1] as i64)==97 { if fe_lc(inb[i+2] as i64)==109 { if fe_lc(inb[i+3] as i64)==112 { emit = 38 } } } } // amp
74 if nl == 4 { if fe_lc(inb[i+1] as i64)==113 { emit = 34 } // quot
75 if fe_lc(inb[i+1] as i64)==97 { if fe_lc(inb[i+2] as i64)==112 { emit = 39 } } // apos
76 if fe_lc(inb[i+1] as i64)==110 { emit = 32 } } // nbsp
77 if inb[i+1] == (35 as u8) { // numeric &#NN;
78 var v: i64 = 0; var p: i64 = i + 2
79 while p < semi { let d: i64 = inb[p] as i64; if d >= 48 { if d <= 57 { v = v*10 + (d-48) } } p = p + 1 }
80 if v > 0 { if v < 128 { emit = v } else { emit = 32 } }
81 }
82 if emit >= 0 { if o < cap-1 { out[o] = emit as u8; o = o + 1 } i = semi + 1 }
83 else { if o < cap-1 { out[o] = 38 as u8; o = o + 1 } i = i + 1 } // unknown entity: keep '&'
84 }
85 } else { if o < cap-1 { out[o] = inb[i]; o = o + 1 } i = i + 1 }
86 }
87 out[o] = 0 as u8
88 return o
89}
90// strip HTML tags + collapse whitespace from inb -> out (feed summaries are HTML). returns len.
91func fe_strip_tags(inb: *u8, inlen: i64, out: *u8, cap: i64) -> i64 {
92 var o: i64 = 0; var i: i64 = 0; var intag: i64 = 0; var lastsp: i64 = 0
93 while i < inlen {
94 let c: i64 = inb[i] as i64
95 if c == 60 { intag = 1 } // '<'
96 else { if c == 62 { intag = 0 } // '>'
97 else { if intag == 0 {
98 var ch: i64 = c
99 if ch == 9 { ch = 32 } if ch == 10 { ch = 32 } if ch == 13 { ch = 32 }
100 if ch == 32 { if lastsp == 0 { if o < cap-1 { out[o] = 32 as u8; o = o + 1 } lastsp = 1 } }
101 else { if o < cap-1 { out[o] = ch as u8; o = o + 1 } lastsp = 0 }
102 } } }
103 i = i + 1
104 }
105 out[o] = 0 as u8
106 return o
107}
108
109// FEED DISCOVERY: find the first <link rel="alternate" type="application/(rss|atom)+xml" href="..."> in a
110// page's <head> and copy its href -> out_url. Returns 1 if found. This is how the crawler turns a JS SPA
111// shell into real content: discover the feed link, then fetch+nx_feed_extract it. (attr order-independent.)
112func fe_link_href(html: *u8, ts: i64, te: i64, out: *u8, cap: i64) -> i64 {
113 // returns href value length within tag [ts,te), or -1
114 let key: *u8 = "href" as *u8
115 var i: i64 = ts
116 while i + 5 < te {
117 if fe_lc(html[i] as i64)==104 { if fe_lc(html[i+1] as i64)==114 { if fe_lc(html[i+2] as i64)==101 { if fe_lc(html[i+3] as i64)==102 {
118 var p: i64 = i + 4
119 while p < te { let c: i64 = html[p] as i64; if c==32 { p=p+1 } else { if c==9 { p=p+1 } else { p=te+9 } } }
120 var pp: i64 = i + 4
121 var st: i64 = 0
122 while st==0 { if pp>=te { st=1 } else { let c: i64=html[pp] as i64; if c==32 { pp=pp+1 } else { if c==9 { pp=pp+1 } else { st=1 } } } }
123 if pp < te { if html[pp]==(61 as u8) {
124 pp = pp + 1
125 var st2: i64 = 0
126 while st2==0 { if pp>=te { st2=1 } else { let c: i64=html[pp] as i64; if c==32 { pp=pp+1 } else { if c==9 { pp=pp+1 } else { if c==34 { st2=1 } else { if c==39 { st2=1 } else { st2=1 } } } } } }
127 if pp < te { let q: i64 = html[pp] as i64
128 var vs: i64 = 0; var ve: i64 = 0
129 if q==34 { pp=pp+1; vs=pp; while pp<te { if html[pp]==(34 as u8) { ve=pp; pp=te } else { pp=pp+1 } } }
130 else { if q==39 { pp=pp+1; vs=pp; while pp<te { if html[pp]==(39 as u8) { ve=pp; pp=te } else { pp=pp+1 } } }
131 else { vs=pp; while pp<te { let c: i64=html[pp] as i64; if c==32 { ve=pp; pp=te } else { if c==62 { ve=pp; pp=te } else { pp=pp+1 } } } if ve==0 { ve=te } } }
132 if ve > vs { var o: i64=0; while vs+o < ve { if o<cap-1 { out[o]=html[vs+o]; o=o+1 } } out[o]=0 as u8; return o }
133 }
134 } }
135 } } } }
136 i = i + 1
137 }
138 return 0 - 1
139}
140func nx_feed_discover(html: *u8, hlen: i64, out_url: *u8, cap: i64) -> i64 {
141 var i: i64 = 0
142 let rss: *u8 = "application/rss+xml" as *u8
143 let atom: *u8 = "application/atom+xml" as *u8
144 while i + 5 < hlen {
145 if html[i]==(60 as u8) { if fe_lc(html[i+1] as i64)==108 { if fe_lc(html[i+2] as i64)==105 { if fe_lc(html[i+3] as i64)==110 { if fe_lc(html[i+4] as i64)==107 { // "<link"
146 var te: i64 = i + 5; var fnd: i64 = 0
147 while fnd==0 { if te>=hlen { fnd=1 } else { if html[te]==(62 as u8) { fnd=1 } else { te=te+1 } } }
148 if fe_find(html, i, te, rss, 19) >= 0 { if fe_link_href(html, i+5, te, out_url, cap) > 0 { return 1 } }
149 if fe_find(html, i, te, atom, 20) >= 0 { if fe_link_href(html, i+5, te, out_url, cap) > 0 { return 1 } }
150 i = te
151 } } } } }
152 i = i + 1
153 }
154 return 0
155}
156// Per-item accessor: fill the idx-th item's decoded TITLE, article LINK, and clean SUMMARY. Returns 1 if the
157// item exists, 0 past the end. Link handles BOTH encodings: RSS <link>URL</link> (element text) and Atom
158// <link href="URL"/> (attribute). Lets the crawler index each feed item as its OWN doc with its OWN url --
159// specific-article retrieval + fresh frontier URLs. (Re-scans from start per idx; feeds are small.)
160func nx_feed_item_at(xml: *u8, xlen: i64, idx: i64,
161 out_title: *u8, tcap: i64, out_link: *u8, lcap: i64,
162 out_sum: *u8, scap: i64) -> i64 {
163 var itemtag: *u8 = "item" as *u8; var itlen: i64 = 4
164 if fe_find(xml, 0, xlen, "<entry" as *u8, 6) >= 0 { itemtag = "entry" as *u8; itlen = 5 }
165 let openlt: *u8 = sys_mmap(16); openlt[0] = 60 as u8
166 var z: i64 = 0; while z < itlen { openlt[1+z] = itemtag[z]; z = z + 1 }
167 let closelt: *u8 = sys_mmap(16); closelt[0] = 60 as u8; closelt[1] = 47 as u8
168 z = 0; while z < itlen { closelt[2+z] = itemtag[z]; z = z + 1 }
169 closelt[2+itlen] = 62 as u8
170 var pos: i64 = 0; var cur: i64 = 0
171 let raw: *u8 = sys_mmap(K_MAGIC_65536); let dc: *u8 = sys_mmap(K_MAGIC_65536); let dc2: *u8 = sys_mmap(K_MAGIC_65536)
172 while cur <= idx {
173 let es: i64 = fe_find(xml, pos, xlen, openlt, itlen + 1)
174 if es < 0 { return 0 }
175 let ee: i64 = fe_find(xml, es, xlen, closelt, itlen + 3)
176 if ee < 0 { return 0 }
177 if cur == idx {
178 // title (decode entities once)
179 let tl: i64 = fe_tag_inner(xml, es, ee, "title" as *u8, 5, raw, K_MAGIC_65536)
180 if tl > 0 { fe_decode_entities(raw, tl, out_title, tcap) } else { out_title[0] = 0 as u8 }
181 // link: RSS element text first; if empty/not-a-url, Atom href
182 out_link[0] = 0 as u8
183 let ll: i64 = fe_tag_inner(xml, es, ee, "link" as *u8, 4, out_link, lcap)
184 var haveurl: i64 = 0
185 if ll >= 7 { if out_link[0]==(104 as u8) { if out_link[1]==(116 as u8) { if out_link[2]==(116 as u8) { if out_link[3]==(112 as u8) { haveurl = 1 } } } } }
186 if haveurl == 0 {
187 out_link[0] = 0 as u8
188 let lp: i64 = fe_find(xml, es, ee, "<link" as *u8, 5)
189 if lp >= 0 {
190 var lte: i64 = lp + 5; var fd: i64 = 0
191 while fd == 0 { if lte >= ee { fd = 1 } else { if xml[lte]==(62 as u8) { fd = 1 } else { lte = lte + 1 } } }
192 fe_link_href(xml, lp + 5, lte, out_link, lcap)
193 }
194 }
195 // summary: content > summary > description ; decode x2 -> strip
196 var sl: i64 = fe_tag_inner(xml, es, ee, "content" as *u8, 7, raw, K_MAGIC_65536)
197 if sl < 0 { sl = fe_tag_inner(xml, es, ee, "summary" as *u8, 7, raw, K_MAGIC_65536) }
198 if sl < 0 { sl = fe_tag_inner(xml, es, ee, "description" as *u8, 11, raw, K_MAGIC_65536) }
199 if sl > 0 {
200 let d1: i64 = fe_decode_entities(raw, sl, dc, K_MAGIC_65536)
201 let d2: i64 = fe_decode_entities(dc, d1, dc2, K_MAGIC_65536)
202 fe_strip_tags(dc2, d2, out_sum, scap)
203 } else { out_sum[0] = 0 as u8 }
204 return 1
205 }
206 cur = cur + 1
207 pos = ee + itlen + 3
208 }
209 return 0
210}
211// Parse a feed -> indexable text: "TITLE. SUMMARY\n" per item. Returns item count.
212// Handles Atom <entry> and RSS <item>; summary from <content>|<summary>|<description>.
213func nx_feed_extract(xml: *u8, xlen: i64, out: *u8, out_cap: i64) -> i64 {
214 // choose the item element: Atom "entry" if the doc has <entry, else RSS "item"
215 var itemtag: *u8 = "item" as *u8; var itlen: i64 = 4
216 if fe_find(xml, 0, xlen, "<entry" as *u8, 6) >= 0 { itemtag = "entry" as *u8; itlen = 5 }
217 let openlt: *u8 = sys_mmap(16); openlt[0] = 60 as u8 // "<" + itemtag
218 var z: i64 = 0; while z < itlen { openlt[1+z] = itemtag[z]; z = z + 1 }
219 let closelt: *u8 = sys_mmap(16); closelt[0] = 60 as u8; closelt[1] = 47 as u8 // "</" + itemtag + ">"
220 z = 0; while z < itlen { closelt[2+z] = itemtag[z]; z = z + 1 }
221 closelt[2+itlen] = 62 as u8
222 var count: i64 = 0
223 var pos: i64 = 0
224 var ol: i64 = 0
225 let tbuf: *u8 = sys_mmap(K_MAGIC_8192)
226 let sbuf: *u8 = sys_mmap(K_MAGIC_65536)
227 let dbuf1: *u8 = sys_mmap(K_MAGIC_65536)
228 let dbuf2: *u8 = sys_mmap(K_MAGIC_65536)
229 let stripped: *u8 = sys_mmap(K_MAGIC_65536)
230 let tdec: *u8 = sys_mmap(K_MAGIC_8192)
231 var go: i64 = 1
232 while go == 1 {
233 let es: i64 = fe_find(xml, pos, xlen, openlt, itlen + 1)
234 if es < 0 { go = 0 } else {
235 let ee: i64 = fe_find(xml, es, xlen, closelt, itlen + 3)
236 if ee < 0 { go = 0 } else {
237 // title
238 let tl: i64 = fe_tag_inner(xml, es, ee, "title" as *u8, 5, tbuf, K_MAGIC_8192)
239 // summary: content > summary > description
240 var sl: i64 = fe_tag_inner(xml, es, ee, "content" as *u8, 7, sbuf, K_MAGIC_65536)
241 if sl < 0 { sl = fe_tag_inner(xml, es, ee, "summary" as *u8, 7, sbuf, K_MAGIC_65536) }
242 if sl < 0 { sl = fe_tag_inner(xml, es, ee, "description" as *u8, 11, sbuf, K_MAGIC_65536) }
243 if tl > 0 {
244 // title: decode entities (titles are usually singly-escaped)
245 let tdl: i64 = fe_decode_entities(tbuf, tl, tdec, K_MAGIC_8192)
246 var k: i64 = 0
247 while k < tdl { if ol < out_cap-2 { out[ol] = tdec[k]; ol = ol + 1 } k = k + 1 }
248 if ol < out_cap-2 { out[ol] = 46 as u8; ol = ol + 1 } // '.'
249 if ol < out_cap-2 { out[ol] = 32 as u8; ol = ol + 1 }
250 if sl > 0 {
251 // summary is HTML-escaped inside XML (sometimes double): decode x2 -> strip tags -> clean text
252 let d1: i64 = fe_decode_entities(sbuf, sl, dbuf1, K_MAGIC_65536)
253 let d2: i64 = fe_decode_entities(dbuf1, d1, dbuf2, K_MAGIC_65536)
254 let spl: i64 = fe_strip_tags(dbuf2, d2, stripped, K_MAGIC_65536)
255 k = 0
256 while k < spl { if ol < out_cap-2 { out[ol] = stripped[k]; ol = ol + 1 } k = k + 1 }
257 }
258 if ol < out_cap-2 { out[ol] = 10 as u8; ol = ol + 1 } // '\n'
259 count = count + 1
260 }
261 pos = ee + itlen + 3
262 }
263 }
264 }
265 out[ol] = 0 as u8
266 return count
267}