nx_fetch_any.nx source
↩ module page · 201 lines · 9741 B
1// nx_fetch_any.nx -- EAT THE FETCH DEBT (operator 2026-07-14): the ONE intelligent fetch DISPATCHER the
2// researcher / census / adversary / critic route through, so they "use and switch and be intelligent about"
3// the ~200 fetch organs "as we gather information so we dont get errors and return nothing without informing
4// the user." Rule-15 DRY: this does NOT add a new fetcher -- it COMPOSES the proven primitives into ONE
5// classify->cascade->always-inform entry point.
6//
7// THE TWO SILENT FAILURES THIS KILLS (both proven live this session):
8// * SILENT-EMPTY : a handshake/host failure returned 0 and the researcher banked NOTHING, with no record.
9// * SILENT-BAD : nx_https_fetch_follow_best returns a Cloudflare 403 CHALLENGE PAGE with n>0, and the old
10// rf_fetch_bank banked that challenge HTML as if it were real research (garbage in the corpus).
11//
12// THE CONTRACT (what "informing the user" means mechanically):
13// nx_fetch_any ALWAYS writes a human-readable per-strategy attempts log into `att` (never left empty), and
14// returns the byte count of a VERIFIED-REAL page (2xx, non-challenge, >= FA_MIN_BYTES) or 0. On 0 the caller
15// MUST surface `att` (rf_fetch_bank banks it as <name>.fail) -- absence becomes DATA, never silence.
16//
17// THE CASCADE (each rung a real, existing, gated primitive):
18// S1 nx_https_fetch_follow_best -- sovereign TLS-1.3 (2 hellos) -> TLS-1.2, redirects, cookie-jar+age-gate
19// S2 nx_https_get_spoof -- real Firefox UA + browser Accept (+ cf_clearance cookie when supplied);
20// beats UA-gating and Cloudflare walls the raw sovereign JA3 does not.
21// (v2 rungs, wired next: nx_http_fetch plaintext:80 for HTTP-only hosts; nx_browser_fetch meta-refresh/JS.)
22//
23// 100% sovereign (own TLS, nx_cc->nxasm, no curl/wget/gcc). license_tier: ORIGINAL
24import "nx_syscalls.nx"
25import "nx_csprng.nx"
26import "nx_x509_trust_store.nx"
27import "nx_trust_store_load_from_certdata.nx"
28import "nx_https_fetch_follow.nx"
29import "nx_https_get_spoof.nx"
30const FA_MAGIC_4194304: i64 = 4194304
31const FA_MAGIC_8388608: i64 = 8388608
32const FA_MAGIC_8192: i64 = 8192
33
34// A real research page is essentially never smaller than this; below it = an error stub / empty / redirect husk.
35const FA_MIN_BYTES: i64 = 256
36
37func fa_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
38func fa_putn(v: i64) -> i64 {
39 let t: *u8 = sys_mmap(28); var m: i64 = v; var k: i64 = 0
40 if m < 0 { sys_write(1, "-" as *u8, 1); m = 0 - m }
41 if m == 0 { t[0] = 48 as u8; k = 1 }
42 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
43 let b: *u8 = sys_mmap(28); var i: i64 = 0
44 while i < k { b[i] = t[k - 1 - i]; i = i + 1 }
45 sys_write(1, b, k); return 0
46}
47func fa_slen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
48
49// substring search: index of needle[0..nl) in hay[0..hl), or -1.
50func fa_find(hay: *u8, hl: i64, needle: *u8, nl: i64) -> i64 {
51 if nl <= 0 { return 0 - 1 }
52 var i: i64 = 0
53 while i + nl <= hl {
54 var j: i64 = 0
55 var ok: i64 = 1
56 while j < nl { if hay[i + j] != needle[j] { ok = 0; j = nl } else { j = j + 1 } }
57 if ok == 1 { return i }
58 i = i + 1
59 }
60 return 0 - 1
61}
62func fa_has(hay: *u8, hl: i64, lit: *u8) -> i64 {
63 if fa_find(hay, hl, lit, fa_slen(lit)) >= 0 { return 1 }
64 return 0
65}
66
67// parse the HTTP status code out of a raw response (finds "HTTP/1." then the 3-digit code). 0 if none.
68func fa_status(buf: *u8, n: i64) -> i64 {
69 let p: i64 = fa_find(buf, n, "HTTP/1." as *u8, 7)
70 if p < 0 { return 0 }
71 let q: i64 = p + 9
72 if q + 3 > n { return 0 }
73 let d0: i64 = buf[q] as i64
74 let d1: i64 = buf[q + 1] as i64
75 let d2: i64 = buf[q + 2] as i64
76 if d0 < 48 { return 0 }
77 if d0 > 57 { return 0 }
78 return (d0 - 48) * 100 + (d1 - 48) * 10 + (d2 - 48)
79}
80
81// 1 iff the body is a bot-wall / JS-challenge / access-denied interstitial (NOT the content we asked for).
82func fa_challenge(buf: *u8, n: i64) -> i64 {
83 if fa_has(buf, n, "Just a moment" as *u8) == 1 { return 1 }
84 if fa_has(buf, n, "Checking your browser" as *u8) == 1 { return 1 }
85 if fa_has(buf, n, "cf-chl" as *u8) == 1 { return 1 }
86 if fa_has(buf, n, "challenge-platform" as *u8) == 1 { return 1 }
87 if fa_has(buf, n, "Attention Required" as *u8) == 1 { return 1 }
88 if fa_has(buf, n, "Enable JavaScript and cookies" as *u8) == 1 { return 1 }
89 return 0
90}
91
92// THE INTELLIGENCE: is this a REAL page worth banking? 2xx + big enough + not a challenge. This is what stops
93// the silent-BAD failure (banking a 403 challenge as research).
94func fa_is_real(status: i64, buf: *u8, n: i64) -> i64 {
95 if n < FA_MIN_BYTES { return 0 }
96 if status < 200 { return 0 }
97 if status >= 300 { return 0 }
98 if fa_challenge(buf, n) == 1 { return 0 }
99 return 1
100}
101
102// offset of the body in a raw HTTP response (past the CRLFCRLF); 0 if the buffer is already body-only.
103func fa_body_off(buf: *u8, n: i64) -> i64 {
104 if n < 12 { return 0 }
105 if fa_find(buf, 12, "HTTP/1." as *u8, 7) != 0 { return 0 } // doesn't start with HTTP/1. -> already body
106 let p: i64 = fa_find(buf, n, "\r\n\r\n" as *u8, 4)
107 if p < 0 { return 0 }
108 return p + 4
109}
110// shift the body to the front of buf (drop the HTTP headers) so callers bank clean content; returns body length.
111func fa_to_body(buf: *u8, n: i64) -> i64 {
112 let bo: i64 = fa_body_off(buf, n)
113 if bo <= 0 { return n }
114 var i: i64 = 0
115 while bo + i < n { buf[i] = buf[bo + i]; i = i + 1 }
116 return n - bo
117}
118
119// append a NUL-terminated string to the bounded attempts log; returns the new offset (always NUL-terminated).
120func fa_log(att: *u8, off: i64, cap: i64, s: *u8) -> i64 {
121 var o: i64 = off; var i: i64 = 0
122 while s[i] != (0 as u8) { if o < cap - 1 { att[o] = s[i]; o = o + 1 } i = i + 1 }
123 att[o] = 0 as u8
124 return o
125}
126func fa_logn(att: *u8, off: i64, cap: i64, v: i64) -> i64 {
127 let t: *u8 = sys_mmap(28); var m: i64 = v; var k: i64 = 0
128 if m < 0 { m = 0 - m }
129 if m == 0 { t[0] = 48 as u8; k = 1 }
130 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
131 var o: i64 = off; var i: i64 = 0
132 while i < k { if o < cap - 1 { att[o] = t[k - 1 - i]; o = o + 1 } i = i + 1 }
133 att[o] = 0 as u8
134 return o
135}
136
137// ---- THE DISPATCHER ----
138// url,store,out,out_cap,max_hops,out_status,att,acap -> bytes of a VERIFIED-REAL page (or 0). `att` is ALWAYS
139// populated with the per-strategy outcome so the caller can inform the user; `cookie`/`clen` let the caller pass
140// a cf_clearance for CF-walled hosts (0/0 = none, still spoofs a Firefox UA which beats plain UA-gating).
141func nx_fetch_any(url: *u8, store: *TrustStore, out: *u8, out_cap: i64, max_hops: i64, out_status: *i64,
142 cookie: *u8, clen: i64, att: *u8, acap: i64) -> i64 {
143 var ao: i64 = 0
144 att[0] = 0 as u8
145
146 // S1: sovereign TLS cascade (1.3 default -> 1.3 alt-hello -> 1.2) + redirect-follow + cookie-jar/age-gate.
147 out_status[0] = 0
148 let n1: i64 = nx_https_fetch_follow_best(url, store, out, out_cap, max_hops, out_status)
149 ao = fa_log(att, ao, acap, "S1 follow_best status=" as *u8)
150 ao = fa_logn(att, ao, acap, out_status[0])
151 ao = fa_log(att, ao, acap, " bytes=" as *u8)
152 ao = fa_logn(att, ao, acap, n1)
153 if fa_is_real(out_status[0], out, n1) == 1 {
154 ao = fa_log(att, ao, acap, " REAL -> OK\n" as *u8)
155 return n1
156 }
157 if n1 > 0 { if fa_challenge(out, n1) == 1 { ao = fa_log(att, ao, acap, " CHALLENGE-WALL" as *u8) } }
158 ao = fa_log(att, ao, acap, " rejected -> switch\n" as *u8)
159
160 // S2: browser spoof (real Firefox UA + Accept + optional cf_clearance) -> beats UA-gating + CF walls.
161 let cr: *u8 = sys_mmap(64)
162 let pk: *u8 = sys_mmap(64)
163 nx_csprng_fill(cr, 32)
164 nx_csprng_fill(pk, 32)
165 let now: i64 = sys_now_realtime_sec()
166 let n2: i64 = nx_https_get_spoof(url, cr, pk, store, now, cookie, clen, out, out_cap)
167 let st2: i64 = fa_status(out, n2)
168 out_status[0] = st2
169 ao = fa_log(att, ao, acap, "S2 spoof status=" as *u8)
170 ao = fa_logn(att, ao, acap, st2)
171 ao = fa_log(att, ao, acap, " bytes=" as *u8)
172 ao = fa_logn(att, ao, acap, n2)
173 if fa_is_real(st2, out, n2) == 1 {
174 ao = fa_log(att, ao, acap, " REAL -> OK\n" as *u8)
175 return n2
176 }
177 if n2 > 0 { if fa_challenge(out, n2) == 1 { ao = fa_log(att, ao, acap, " CHALLENGE-WALL" as *u8) } }
178 ao = fa_log(att, ao, acap, " rejected\n" as *u8)
179
180 ao = fa_log(att, ao, acap, "ALL-STRATEGIES-FAILED (no verified-real page; see per-strategy status above)\n" as *u8)
181 return 0
182}
183
184// CLI: nx_fetch_any <url> -- prove the cascade live (run against a CF-walled host AND a real page).
185func main(argc: i64, argv: *i64) -> i64 {
186 if argc < 2 { fa_puts("usage: nx_fetch_any <url>\n" as *u8); return 2 }
187 let url: *u8 = argv[1] as *u8
188 let r: i64 = nx_trust_store_load_from_certdata("data/mozilla_certdata.txt" as *u8, 512, FA_MAGIC_4194304)
189 if r <= 0 { fa_puts("STORE-FAIL: cannot load data/mozilla_certdata.txt\n" as *u8); return 1 }
190 let store: *TrustStore = r as *TrustStore
191 let out: *u8 = sys_mmap(FA_MAGIC_8388608)
192 let att: *u8 = sys_mmap(FA_MAGIC_8192)
193 let status: *i64 = sys_mmap(16) as *i64
194 let empty: *u8 = sys_mmap(8); empty[0] = 0 as u8
195 let n: i64 = nx_fetch_any(url, store, out, FA_MAGIC_8388608, 6, status, empty, 0, att, FA_MAGIC_8192)
196 fa_puts("=== nx_fetch_any attempts ===\n" as *u8)
197 fa_puts(att)
198 fa_puts("=== result: real_bytes=" as *u8); fa_putn(n)
199 fa_puts(" final_status=" as *u8); fa_putn(status[0]); fa_puts(" ===\n" as *u8)
200 return 0
201}