nx_research_fetch.nx source
↩ module page · 113 lines · 5770 B
1// nx_research_fetch.nx -- the Nishi RESEARCHER's sovereign web-fetch for the SPEED / BEAT-CUDA arc. Reuses the
2// proven sovereign TLS-1.3 stack (nx_https_fetch_follow: url->connect->TLS1.3 handshake validated vs the
3// Mozilla CA store->GET->redirect-follow->dechunk) -- 100% sovereign (own TLS, nx_cc->nxasm, no curl/wget).
4// Fetches a CITED source on efficient LLM inference + cross-vendor (CUDA-alternative) GPU compute, saves the
5// raw body to knowledge/fetched/ for analysis. The only non-Nishi inputs are the CA-root DATA + the fetched
6// page (exactly the researcher's contract). license_tier: ORIGINAL
7import "nx_syscalls.nx"
8import "nx_x509_trust_store.nx"
9import "nx_trust_store_load_from_certdata.nx"
10import "nx_https_fetch_follow.nx"
11const K_MAGIC_5381: i64 = 5381
12const K_MAGIC_4194304: i64 = 4194304
13const K_MAGIC_2000: i64 = 2000
14const K_MAGIC_2048: i64 = 2048
15
16func rf_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
17func rf_putn(v: i64) -> i64 {
18 if v == 0 { sys_write(1, "0" as *u8, 1); return 0 }
19 var m: i64 = v
20 if m < 0 { sys_write(1, "-" as *u8, 1); m = 0 - m }
21 let d: *u8 = sys_mmap(24); var k: i64 = 0
22 while m > 0 { d[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
23 let o: *u8 = sys_mmap(24); var wi: i64 = 0
24 while wi < k { o[wi] = d[k - 1 - wi]; wi = wi + 1 }
25 sys_write(1, o, k)
26 return 0
27}
28
29// Build knowledge/fetched/<name>: caller name (argv[2]) if given (sanitized), else rf_<djb2hex(url)>.raw.
30// FIXES the srch_latest.raw clobber (2026-07-03): every distinct URL now maps to a stable distinct file
31// (idempotent -- re-fetching a URL overwrites only its own file), so parallel research fetches coexist.
32func rf_hexnib(v: i64) -> i64 { if v < 10 { return 48 + v } return 87 + v } // 0-9 then a-f
33func rf_build_path(url: *u8, argc: i64, argv: *i64, out: *u8) -> i64 {
34 let pre: *u8 = "knowledge/fetched/\x00"
35 var o: i64 = 0
36 while pre[o] != (0 as u8) { out[o] = pre[o]; o = o + 1 }
37 if argc >= 3 {
38 let name: *u8 = argv[2] as *u8
39 var i: i64 = 0
40 while name[i] != (0 as u8) {
41 let c: i64 = name[i] as i64
42 var ok: i64 = 0
43 if c >= 48 { if c <= 57 { ok = 1 } } // 0-9
44 if c >= 65 { if c <= 90 { ok = 1 } } // A-Z
45 if c >= 97 { if c <= 122 { ok = 1 } } // a-z
46 if c == 46 { ok = 1 } // .
47 if c == 95 { ok = 1 } // _
48 if c == 45 { ok = 1 } // -
49 if ok == 1 { out[o] = c as u8 } else { out[o] = 95 as u8 }
50 o = o + 1; i = i + 1
51 }
52 } else {
53 out[o] = 114 as u8; out[o+1] = 102 as u8; out[o+2] = 95 as u8; o = o + 3 // "rf_"
54 var h: i64 = K_MAGIC_5381
55 var i: i64 = 0
56 while url[i] != (0 as u8) { h = (((h << 5) + h) + (url[i] as i64)) & 0xFFFFFFFF; i = i + 1 }
57 var s: i64 = 28
58 while s >= 0 { out[o] = rf_hexnib((h >> s) & 0xF) as u8; o = o + 1; s = s - 4 }
59 let suf: *u8 = ".raw\x00"
60 var j: i64 = 0
61 while suf[j] != (0 as u8) { out[o] = suf[j]; o = o + 1; j = j + 1 }
62 }
63 out[o] = 0 as u8
64 return o
65}
66
67func main(argc: i64, argv: *i64) -> i64 {
68 // ARGV-DRIVEN (operator law: NO .txt staging-input files anywhere -- pass inputs as args). The URL is
69 // argv[1] (already NUL-terminated by the kernel). The Mozilla CA bundle is reference DATA, not a staging
70 // file, and is the researcher's one legitimate non-Nishi input.
71 if argc < 2 { rf_puts("usage: nx_research_fetch <url>\n"); return 3 }
72 let url: *u8 = argv[1] as *u8
73 let cpath: *u8 = "data/mozilla_certdata.txt\x00"
74 let r: i64 = nx_trust_store_load_from_certdata(cpath, 512, K_MAGIC_4194304)
75 if r <= 0 { rf_puts("RF: certdata load failed\n"); return 1 }
76 let store: *TrustStore = r as *TrustStore
77 rf_puts("CA roots="); rf_putn(trust_store_count(store)); rf_puts("\n")
78 rf_puts("FETCH "); rf_puts(url); rf_puts("\n")
79 let out: *u8 = sys_mmap(K_MAGIC_4194304)
80 let status: *i64 = sys_mmap(8) as *i64
81 // RETRY-WITH-BACKOFF 2026-07-08: arxiv (and most paper hosts) rate-limit
82 // rapid repeated fetches with HTTP 429 -- the reason a batch of 2025/2026
83 // papers "wasn't getting through". 429 clears in a few seconds, so back off
84 // (2s/4s/8s) and retry. got: 0=trying 1=ok 2=gave-up. Success = body AND
85 // status < 400.
86 var n: i64 = 0
87 var got: i64 = 0
88 var attempt: i64 = 0
89 var backoff_ms: i64 = K_MAGIC_2000
90 while got == 0 {
91 n = nx_https_fetch_follow_best(url, store, out, K_MAGIC_4194304, 6, status)
92 rf_puts("status="); rf_putn(status[0]); rf_puts(" body_bytes="); rf_putn(n); rf_puts("\n")
93 if n > 0 { if status[0] < 400 { got = 1 } }
94 if got == 0 {
95 attempt = attempt + 1
96 if attempt >= 4 { got = 2 } else {
97 rf_puts("RF: retry #"); rf_putn(attempt); rf_puts(" in "); rf_putn(backoff_ms); rf_puts("ms (rate-limit/transient)\n")
98 sys_sleep_ms(backoff_ms)
99 backoff_ms = backoff_ms * 2
100 }
101 }
102 }
103 if got != 1 { rf_puts("RF: fetch failed after retries (code "); rf_putn(n); rf_puts(")\n"); return 2 }
104
105 let opath: *u8 = sys_mmap(512)
106 rf_build_path(url, argc, argv, opath)
107 let fd: i64 = sys_openat_wr(opath, 0x1a4)
108 if fd < 0 { rf_puts("RF: save failed (dir missing?); first 2KB to stdout:\n"); var c: i64 = 0; while c < K_MAGIC_2048 { if c >= n { break } c = c + 1 } sys_write(1, out, c); rf_puts("\n"); return 0 }
109 sys_write(fd, out, n)
110 sys_close(fd)
111 rf_puts("SAVED "); rf_puts(opath); rf_puts(" ("); rf_putn(n); rf_puts(" bytes) -- grep it to cite\n")
112 return 0
113}