code wiki / _hdl_build / nx_feedfetch.nx
nx_feedfetch.nx source
↩ module page · 146 lines · 6452 B
1// nx_feedfetch.nx -- the missing CLI consumer that makes the RSS/Atom capability REACHABLE.
2//
3// WHY THIS EXISTS: nx_feed_extract has been in the tree since 2026-07-05 -- a real RSS 2.0 + Atom parser
4// (discover the feed link in a page shell, then parse <entry>/<item> into indexable text + a fresh-URL
5// crawl frontier). It is a LIBRARY with no main(), so nothing could ever call it: no deployed binary
6// referenced it, and the capability ruler scored rss-auto-acquire as GAP for that reason alone.
7// ★I previously wrote on a published page that this organ was "deploy work, not build work". THAT WAS
8// WRONG -- a library cannot be deployed into reachability; it needs an entry point. This is that entry
9// point, and the correction is published alongside it.
10//
11// Mirrors the proven nx_yt_probe shape (fetch over our own TLS -> hand the bytes to the extractor).
12// usage: nx_feedfetch <url>
13// Fetches the URL. If the body is already a feed, parses it. If it is a page shell, DISCOVERS the
14// <link rel=alternate> feed URL, resolves it against the origin, fetches that, and parses it.
15// Prints the discovered feed URL, the item count, and the extracted indexable text.
16// license_tier: ORIGINAL No hw writes (Rule 26). expect_exit: 0
17import "nx_syscalls.nx"
18import "nx_x509_trust_store.nx"
19import "nx_trust_store_load_from_certdata.nx"
20import "nx_https_fetch_follow.nx"
21import "nx_feed_extract.nx"
22
23const FF_CAP: i64 = 4194304 // page/feed body arena
24const FF_CERTDATA_CAP: i64 = 4194304 // trust-store parse arena
25const FF_URL_CAP: i64 = 4096
26const FF_OUT_CAP: i64 = 1048576 // extracted indexable text
27const FF_REDIRECTS: i64 = 6
28
29func fw(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
30func fnum(v: i64) -> i64 {
31 let b: *u8 = sys_mmap(28)
32 var m: i64 = v
33 if m < 0 { sys_write(1,"-" as *u8,1); m = 0 - m }
34 let t: *u8 = sys_mmap(28)
35 var k: i64 = 0
36 if m == 0 { t[0]=48 as u8; k=1 }
37 while m > 0 { t[k]=(48+(m%10)) as u8; m=m/10; k=k+1 }
38 var i: i64 = 0
39 while i < k { b[i]=t[k-1-i]; i=i+1 }
40 sys_write(1,b,k)
41 return 0
42}
43// does buf[0..n) contain lit? (bounded, used to tell a feed body from a page shell)
44func ff_has(buf: *u8, n: i64, lit: *u8) -> i64 {
45 var ll: i64 = 0
46 while lit[ll] != (0 as u8) { ll = ll + 1 }
47 if ll == 0 { return 0 }
48 var i: i64 = 0
49 while i + ll <= n {
50 var j: i64 = 0
51 var hit: i64 = 1
52 while j < ll { if buf[i+j] != lit[j] { hit = 0; j = ll } else { j = j + 1 } }
53 if hit == 1 { return 1 }
54 i = i + 1
55 }
56 return 0
57}
58// origin = "https://host" -- everything BEFORE the 3rd slash. Single forward pass, no rewinding.
59func ff_origin(url: *u8, out: *u8, cap: i64) -> i64 {
60 var i: i64 = 0
61 var slashes: i64 = 0
62 var o: i64 = 0
63 var go: i64 = 1
64 while go == 1 {
65 let c: i64 = url[i] as i64
66 if c == 0 { go = 0 }
67 else {
68 if c == 47 {
69 slashes = slashes + 1
70 if slashes == 3 { go = 0 }
71 }
72 if go == 1 {
73 if o < (cap - 1) { out[o] = url[i]; o = o + 1 }
74 i = i + 1
75 }
76 }
77 }
78 out[o] = 0 as u8
79 return 1
80}
81// resolve ref against origin: absolute stays, "/path" joins origin, else origin + "/" + ref
82func ff_resolve(origin: *u8, ref: *u8, out: *u8, cap: i64) -> i64 {
83 var o: i64 = 0
84 if ref[0] == (104 as u8) { if ref[1] == (116 as u8) { if ref[2] == (116 as u8) { if ref[3] == (112 as u8) {
85 var k: i64 = 0
86 while ref[k] != (0 as u8) { if o < (cap-1) { out[o]=ref[k]; o=o+1 } k=k+1 }
87 out[o] = 0 as u8
88 return 1
89 } } } }
90 var i: i64 = 0
91 while origin[i] != (0 as u8) { if o < (cap-1) { out[o]=origin[i]; o=o+1 } i=i+1 }
92 if ref[0] != (47 as u8) { if o < (cap-1) { out[o]=47 as u8; o=o+1 } }
93 var j: i64 = 0
94 while ref[j] != (0 as u8) { if o < (cap-1) { out[o]=ref[j]; o=o+1 } j=j+1 }
95 out[o] = 0 as u8
96 return 1
97}
98
99func main(argc: i64, argv: *i64) -> i64 {
100 if argc < 2 { fw("usage: nx_feedfetch <url>\n" as *u8); sys_exit(2); return 2 }
101 let url: *u8 = argv[1] as *u8
102 fw("=== nx_feedfetch === " as *u8); fw(url); fw("\n" as *u8)
103
104 let r: i64 = nx_trust_store_load_from_certdata("data/mozilla_certdata.txt" as *u8, 512, FF_CERTDATA_CAP)
105 if r <= 0 { fw("RED cannot load trust store (run from repo root)\n" as *u8); sys_exit(3); return 3 }
106 let store: *TrustStore = r as *TrustStore
107
108 let body: *u8 = sys_mmap(FF_CAP)
109 let st: *i64 = sys_mmap(8) as *i64
110 var n: i64 = nx_https_fetch_follow(url, store, body, FF_CAP, FF_REDIRECTS, st)
111 fw("fetch status=" as *u8); fnum(st[0]); fw(" bytes=" as *u8); fnum(n); fw("\n" as *u8)
112 if st[0] != 200 { fw("RED fetch not 200\n" as *u8); sys_exit(1); return 1 }
113 if n <= 0 { fw("RED empty body\n" as *u8); sys_exit(1); return 1 }
114
115 // Already a feed? RSS roots at <rss / <rdf, Atom at <feed.
116 var isfeed: i64 = 0
117 if ff_has(body, n, "<rss" as *u8) == 1 { isfeed = 1 }
118 if ff_has(body, n, "<feed" as *u8) == 1 { isfeed = 1 }
119 if ff_has(body, n, "<rdf:RDF" as *u8) == 1 { isfeed = 1 }
120
121 let feedurl: *u8 = sys_mmap(FF_URL_CAP)
122 if isfeed == 0 {
123 fw("body is a page shell -- discovering <link rel=alternate> feed\n" as *u8)
124 let ref: *u8 = sys_mmap(FF_URL_CAP)
125 let d: i64 = nx_feed_discover(body, n, ref, FF_URL_CAP)
126 if d <= 0 { fw("no feed link found in this page (conf 0)\n" as *u8); sys_exit(1); return 1 }
127 let origin: *u8 = sys_mmap(FF_URL_CAP)
128 ff_origin(url, origin, FF_URL_CAP)
129 ff_resolve(origin, ref, feedurl, FF_URL_CAP)
130 fw("discovered feed: " as *u8); fw(feedurl); fw("\n" as *u8)
131 n = nx_https_fetch_follow(feedurl, store, body, FF_CAP, FF_REDIRECTS, st)
132 fw("feed fetch status=" as *u8); fnum(st[0]); fw(" bytes=" as *u8); fnum(n); fw("\n" as *u8)
133 if st[0] != 200 { fw("RED feed fetch not 200\n" as *u8); sys_exit(1); return 1 }
134 } else {
135 fw("body IS a feed (no discovery needed)\n" as *u8)
136 }
137
138 let out: *u8 = sys_mmap(FF_OUT_CAP)
139 let items: i64 = nx_feed_extract(body, n, out, FF_OUT_CAP)
140 fw("items=" as *u8); fnum(items); fw("\n" as *u8)
141 if items <= 0 { fw("RED parsed zero items\n" as *u8); sys_exit(1); return 1 }
142 fw("--- extracted indexable text ---\n" as *u8)
143 fw(out)
144 fw("\nGREEN feed parsed\n" as *u8)
145 return 0
146}