nx_fetch_any.nx source
↩ module page · 193 lines · 9487 B
1// nx_fetch_any.nx -- EAT THE FETCH DEBT (operator 2026-07-14): the ONE intelligent fetch DISPATCHER the
2// researcher / census / adversary / critic route through, so they "use and switch and be intelligent about"
3// the ~200 fetch organs "as we gather information so we dont get errors and return nothing without informing
4// the user." Rule-15 DRY: this does NOT add a new fetcher -- it COMPOSES the proven primitives into ONE
5// classify->cascade->always-inform entry point.
6//
7// THE TWO SILENT FAILURES THIS KILLS (both proven live this session):
8// * SILENT-EMPTY : a handshake/host failure returned 0 and the researcher banked NOTHING, with no record.
9// * SILENT-BAD : nx_https_fetch_follow_best returns a Cloudflare 403 CHALLENGE PAGE with n>0, and the old
10// rf_fetch_bank banked that challenge HTML as if it were real research (garbage in the corpus).
11//
12// THE CONTRACT (what "informing the user" means mechanically):
13// nx_fetch_any ALWAYS writes a human-readable per-strategy attempts log into `att` (never left empty), and
14// returns the byte count of a VERIFIED-REAL page (2xx, non-challenge, >= FA_MIN_BYTES) or 0. On 0 the caller
15// MUST surface `att` (rf_fetch_bank banks it as <name>.fail) -- absence becomes DATA, never silence.
16//
17// THE CASCADE (each rung a real, existing, gated primitive):
18// S1 nx_https_fetch_follow_best -- sovereign TLS-1.3 (2 hellos) -> TLS-1.2, redirects, cookie-jar+age-gate
19// S2 nx_https_get_spoof -- real Firefox UA + browser Accept (+ cf_clearance cookie when supplied);
20// beats UA-gating and Cloudflare walls the raw sovereign JA3 does not.
21// (v2 rungs, wired next: nx_http_fetch plaintext:80 for HTTP-only hosts; nx_browser_fetch meta-refresh/JS.)
22//
23// 100% sovereign (own TLS, nx_cc->nxasm, no curl/wget/gcc). license_tier: ORIGINAL
24import "nx_syscalls.nx"
25import "nx_http_response_parse.nx"
26import "nx_csprng.nx"
27import "nx_x509_trust_store.nx"
28import "nx_trust_store_load_from_certdata.nx"
29import "nx_https_fetch_follow.nx"
30import "nx_https_get_spoof.nx"
31const FA_MAGIC_4194304: i64 = 4194304
32const FA_MAGIC_8388608: i64 = 8388608
33const FA_MAGIC_8192: i64 = 8192
34
35// A real research page is essentially never smaller than this; below it = an error stub / empty / redirect husk.
36const FA_MIN_BYTES: i64 = 256
37
38func fa_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
39func fa_putn(v: i64) -> i64 {
40 let t: *u8 = sys_mmap(28); var m: i64 = v; var k: i64 = 0
41 if m < 0 { sys_write(1, "-" as *u8, 1); m = 0 - m }
42 if m == 0 { t[0] = 48 as u8; k = 1 }
43 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
44 let b: *u8 = sys_mmap(28); var i: i64 = 0
45 while i < k { b[i] = t[k - 1 - i]; i = i + 1 }
46 sys_write(1, b, k); return 0
47}
48func fa_slen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
49
50// substring search: index of needle[0..nl) in hay[0..hl), or -1.
51func fa_find(hay: *u8, hl: i64, needle: *u8, nl: i64) -> i64 {
52 if nl <= 0 { return 0 - 1 }
53 var i: i64 = 0
54 while i + nl <= hl {
55 var j: i64 = 0
56 var ok: i64 = 1
57 while j < nl { if hay[i + j] != needle[j] { ok = 0; j = nl } else { j = j + 1 } }
58 if ok == 1 { return i }
59 i = i + 1
60 }
61 return 0 - 1
62}
63func fa_has(hay: *u8, hl: i64, lit: *u8) -> i64 {
64 if fa_find(hay, hl, lit, fa_slen(lit)) >= 0 { return 1 }
65 return 0
66}
67
68// Raw response status uses the shared strict boundary; body text is never a status source.
69func fa_status(buf:*u8,n:i64)->i64{
70 let code:i64=nx_http_resp_status_code(buf,n);if code<0{return 0};return code
71}
72
73// 1 iff the body is a bot-wall / JS-challenge / access-denied interstitial (NOT the content we asked for).
74func fa_challenge(buf: *u8, n: i64) -> i64 {
75 if fa_has(buf, n, "Just a moment" as *u8) == 1 { return 1 }
76 if fa_has(buf, n, "Checking your browser" as *u8) == 1 { return 1 }
77 if fa_has(buf, n, "cf-chl" as *u8) == 1 { return 1 }
78 if fa_has(buf, n, "challenge-platform" as *u8) == 1 { return 1 }
79 if fa_has(buf, n, "Attention Required" as *u8) == 1 { return 1 }
80 if fa_has(buf, n, "Enable JavaScript and cookies" as *u8) == 1 { return 1 }
81 return 0
82}
83
84// THE INTELLIGENCE: is this a REAL page worth banking? 2xx + big enough + not a challenge. This is what stops
85// the silent-BAD failure (banking a 403 challenge as research).
86func fa_is_real(status: i64, buf: *u8, n: i64) -> i64 {
87 if n < FA_MIN_BYTES { return 0 }
88 if status < 200 { return 0 }
89 if status >= 300 { return 0 }
90 if fa_challenge(buf, n) == 1 { return 0 }
91 return 1
92}
93
94// offset of the body in a raw HTTP response (past the CRLFCRLF); 0 if the buffer is already body-only.
95func fa_body_off(buf: *u8, n: i64) -> i64 {
96 if n < 12 { return 0 }
97 if fa_find(buf, 12, "HTTP/1." as *u8, 7) != 0 { return 0 } // doesn't start with HTTP/1. -> already body
98 let p: i64 = fa_find(buf, n, "\r\n\r\n" as *u8, 4)
99 if p < 0 { return 0 }
100 return p + 4
101}
102// shift the body to the front of buf (drop the HTTP headers) so callers bank clean content; returns body length.
103func fa_to_body(buf: *u8, n: i64) -> i64 {
104 let bo: i64 = fa_body_off(buf, n)
105 if bo <= 0 { return n }
106 var i: i64 = 0
107 while bo + i < n { buf[i] = buf[bo + i]; i = i + 1 }
108 return n - bo
109}
110
111// append a NUL-terminated string to the bounded attempts log; returns the new offset (always NUL-terminated).
112func fa_log(att: *u8, off: i64, cap: i64, s: *u8) -> i64 {
113 var o: i64 = off; var i: i64 = 0
114 while s[i] != (0 as u8) { if o < cap - 1 { att[o] = s[i]; o = o + 1 } i = i + 1 }
115 att[o] = 0 as u8
116 return o
117}
118func fa_logn(att: *u8, off: i64, cap: i64, v: i64) -> i64 {
119 let t: *u8 = sys_mmap(28); var m: i64 = v; var k: i64 = 0
120 if m < 0 { m = 0 - m }
121 if m == 0 { t[0] = 48 as u8; k = 1 }
122 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
123 var o: i64 = off; var i: i64 = 0
124 while i < k { if o < cap - 1 { att[o] = t[k - 1 - i]; o = o + 1 } i = i + 1 }
125 att[o] = 0 as u8
126 return o
127}
128
129// ---- THE DISPATCHER ----
130// url,store,out,out_cap,max_hops,out_status,att,acap -> bytes of a VERIFIED-REAL page (or 0). `att` is ALWAYS
131// populated with the per-strategy outcome so the caller can inform the user; `cookie`/`clen` let the caller pass
132// a cf_clearance for CF-walled hosts (0/0 = none, still spoofs a Firefox UA which beats plain UA-gating).
133func nx_fetch_any(url: *u8, store: *TrustStore, out: *u8, out_cap: i64, max_hops: i64, out_status: *i64,
134 cookie: *u8, clen: i64, att: *u8, acap: i64) -> i64 {
135 var ao: i64 = 0
136 att[0] = 0 as u8
137
138 // S1: sovereign TLS cascade (1.3 default -> 1.3 alt-hello -> 1.2) + redirect-follow + cookie-jar/age-gate.
139 out_status[0] = 0
140 let n1: i64 = nx_https_fetch_follow_best(url, store, out, out_cap, max_hops, out_status)
141 ao = fa_log(att, ao, acap, "S1 follow_best status=" as *u8)
142 ao = fa_logn(att, ao, acap, out_status[0])
143 ao = fa_log(att, ao, acap, " bytes=" as *u8)
144 ao = fa_logn(att, ao, acap, n1)
145 if fa_is_real(out_status[0], out, n1) == 1 {
146 ao = fa_log(att, ao, acap, " REAL -> OK\n" as *u8)
147 return n1
148 }
149 if n1 > 0 { if fa_challenge(out, n1) == 1 { ao = fa_log(att, ao, acap, " CHALLENGE-WALL" as *u8) } }
150 ao = fa_log(att, ao, acap, " rejected -> switch\n" as *u8)
151
152 // S2: browser spoof (real Firefox UA + Accept + optional cf_clearance) -> beats UA-gating + CF walls.
153 let cr: *u8 = sys_mmap(64)
154 let pk: *u8 = sys_mmap(64)
155 nx_csprng_fill(cr, 32)
156 nx_csprng_fill(pk, 32)
157 let now: i64 = sys_now_realtime_sec()
158 let n2: i64 = nx_https_get_spoof(url, cr, pk, store, now, cookie, clen, out, out_cap)
159 let st2: i64 = fa_status(out, n2)
160 out_status[0] = st2
161 ao = fa_log(att, ao, acap, "S2 spoof status=" as *u8)
162 ao = fa_logn(att, ao, acap, st2)
163 ao = fa_log(att, ao, acap, " bytes=" as *u8)
164 ao = fa_logn(att, ao, acap, n2)
165 if fa_is_real(st2, out, n2) == 1 {
166 ao = fa_log(att, ao, acap, " REAL -> OK\n" as *u8)
167 return n2
168 }
169 if n2 > 0 { if fa_challenge(out, n2) == 1 { ao = fa_log(att, ao, acap, " CHALLENGE-WALL" as *u8) } }
170 ao = fa_log(att, ao, acap, " rejected\n" as *u8)
171
172 ao = fa_log(att, ao, acap, "ALL-STRATEGIES-FAILED (no verified-real page; see per-strategy status above)\n" as *u8)
173 return 0
174}
175
176// CLI: nx_fetch_any <url> -- prove the cascade live (run against a CF-walled host AND a real page).
177func main(argc: i64, argv: *i64) -> i64 {
178 if argc < 2 { fa_puts("usage: nx_fetch_any <url>\n" as *u8); return 2 }
179 let url: *u8 = argv[1] as *u8
180 let r: i64 = nx_trust_store_load_from_certdata("data/mozilla_certdata.txt" as *u8, 512, FA_MAGIC_4194304)
181 if r <= 0 { fa_puts("STORE-FAIL: cannot load data/mozilla_certdata.txt\n" as *u8); return 1 }
182 let store: *TrustStore = r as *TrustStore
183 let out: *u8 = sys_mmap(FA_MAGIC_8388608)
184 let att: *u8 = sys_mmap(FA_MAGIC_8192)
185 let status: *i64 = sys_mmap(16) as *i64
186 let empty: *u8 = sys_mmap(8); empty[0] = 0 as u8
187 let n: i64 = nx_fetch_any(url, store, out, FA_MAGIC_8388608, 6, status, empty, 0, att, FA_MAGIC_8192)
188 fa_puts("=== nx_fetch_any attempts ===\n" as *u8)
189 fa_puts(att)
190 fa_puts("=== result: real_bytes=" as *u8); fa_putn(n)
191 fa_puts(" final_status=" as *u8); fa_putn(status[0]); fa_puts(" ===\n" as *u8)
192 return 0
193}