nx_onsite_frontier_fetch.nx source
↩ module page · 150 lines · 8494 B
1// nx_onsite_frontier_fetch.nx -- SOVEREIGN onsite-search S-CLASS FRONTIER research fetch.
2//
3// Rung S-RESEARCH-2 of the onsite-search arc: the 7-axis exceed census proved we are PARITY on the lexical
4// core + EXCEEDS on sovereignty, but BEHIND on the neural/semantic rung -- and that rung is what blocks a
5// real, full S-class. The operator's directive: find out, HARDWARE-RUNG-UP, what we need to build to close it.
6// Global Rule 4 (no assumptions): every rung + every new census cell must cite a REAL fetch, not memory.
7//
8// Reuses the team's OWN proven sovereign HTTPS stack (nx_tls13_client_session_run + nx_https_get_complete; no
9// browser/node/curl), validated against the real Mozilla CA store -- the IDENTICAL path proven live in
10// nx_onsite_research_fetch (the srch_* corpus, 2026-06-16). Fetches the fact-dense, canonical, parens/comma-free
11// open pages (en.wikipedia.org, static, NishiBot-allowed, no redirect) for the WHOLE hardware-up neural stack:
12//
13// SILICON / VECTOR-MATH KERNEL (the lowest rung -- what an embedding similarity actually computes):
14// /wiki/Dot_product -> srch_dotproduct.raw
15// /wiki/Cosine_similarity -> srch_cosine.raw
16// /wiki/Basic_Linear_Algebra_Subprograms -> srch_blas.raw (GEMM = the matmul kernel layer)
17// VECTOR REPRESENTATION (turning text into vectors -- the model rung):
18// /wiki/Word_embedding -> srch_embedding.raw
19// /wiki/Word2vec -> srch_word2vec.raw
20// /wiki/Large_language_model -> srch_llm.raw (embedding + cross-encoder rerank model)
21// ANN INDEX / SCALE (searching millions of vectors sub-linearly):
22// /wiki/Nearest_neighbor_search -> srch_ann.raw (approximate NN frame)
23// /wiki/Hierarchical_navigable_small_world -> srch_hnsw.raw (the dominant ANN graph index)
24// /wiki/Locality-sensitive_hashing -> srch_lsh.raw
25// /wiki/Vector_quantization -> srch_vq.raw (PQ = compress vectors for RAM budget)
26// /wiki/Vector_database -> srch_vectordb.raw
27// RETRIEVAL FRAME (hybrid fusion + evaluation):
28// /wiki/Information_retrieval -> srch_ir.raw
29//
30// expect_exit: 0
31// license_tier: ORIGINAL (clone of nx_onsite_research_fetch; only the URL list differs)
32
33import "nx_syscalls.nx"
34import "nx_csprng.nx"
35import "nx_x509_trust_store.nx"
36import "nx_trust_store_load_from_certdata.nx"
37import "nx_tls13_client_validate_certificate.nx"
38import "nx_tls13_client_session_run.nx"
39import "nx_https_url_for_fetch.nx"
40import "nx_https_url_connect.nx"
41import "nx_https_get.nx"
42import "nx_https_get_complete.nx"
43import "nx_http_response_parse.nx"
44const K_MAGIC_4194304: i64 = 4194304
45
46func sf_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
47func sf_putn(v: i64) -> i64 {
48 let bb: *u8 = sys_mmap(28); var m: i64 = v
49 if m < 0 { m = 0 - m; sys_write(1, "-" as *u8, 1) }
50 let t: *u8 = sys_mmap(28); var k: i64 = 0
51 if m == 0 { t[0] = 48 as u8; k = 1 }
52 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
53 var i: i64 = 0; while i < k { bb[i] = t[k - 1 - i]; i = i + 1 } sys_write(1, bb, k); return 0
54}
55func sf_strlen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
56
57// Fetch one page over the sovereign TLS1.3 client, save the raw response. Returns the HTTP status (200 ok).
58func fetch_page(store: *TrustStore, full_url: *u8, path: *u8, path_len: i64, out_path: *u8) -> i64 {
59 sf_puts("--- fetch "); sf_puts(full_url); sf_puts("\n")
60
61 let cr: *u8 = sys_mmap(32)
62 var i: i64 = 0
63 nx_csprng_fill(cr, 32) // CWE-330 (debt 1785970852): was the constant 0xC0..0xDF
64 let priv: *u8 = sys_mmap(32)
65 i = 0
66 nx_csprng_fill(priv, 32) // CWE-330: the X25519 scalar was the constant 0xA0..0xBF on EVERY session
67
68 let url_p: *NxUrl = nx_url_new()
69 let target_raw: *u8 = sys_mmap(32)
70 let target: *NxHttpsTarget = target_raw as *NxHttpsTarget
71 target.url = url_p
72 target.port = 0
73 if nx_https_url_for_fetch(full_url, target) != NX_HTTPS_URL_OK { return 0 - 41 }
74
75 let fd_p: *i64 = sys_mmap(16) as *i64
76 if nx_https_url_connect(target, full_url, sys_now_realtime_sec(), fd_p) != NX_HTTPS_CONNECT_OK { return 0 - 42 }
77 let fd: i64 = *fd_p
78
79 let val_ctx_raw: *u8 = sys_mmap(64)
80 let val_ctx: *TlsValidationContext = val_ctx_raw as *TlsValidationContext
81 val_ctx.store = store
82 val_ctx.sni_host = full_url + target.url.host_off
83 val_ctx.sni_host_len = target.url.host_len
84 val_ctx.now_epoch = sys_now_realtime_sec()
85
86 let sr: i64 = nx_tls13_client_session_run(
87 fd, full_url + target.url.host_off, target.url.host_len,
88 cr, priv, val_ctx
89 )
90 if sr <= 0 { sys_close(fd); return 0 - (200 + (0 - sr)) }
91
92 let session: *Tls13ClientSession = sr as *Tls13ClientSession
93 let buf: *u8 = sys_mmap(K_MAGIC_4194304)
94 let gc: i64 = nx_https_get_complete(
95 session, fd, path, path_len,
96 full_url + target.url.host_off, target.url.host_len,
97 buf, K_MAGIC_4194304
98 )
99 sys_close(fd)
100 if gc < 0 { return 0 - (100 + (0 - gc)) }
101
102 let rs: *i64 = sys_mmap(128) as *i64
103 nx_http_response_parse(buf, gc, rs)
104 let status: i64 = rs[1]
105
106 let ofd: i64 = sys_openat_wr(out_path, 0x1A4)
107 if ofd <= 0 { return 0 - 70 }
108 sys_write(ofd, buf, gc)
109 sys_close(ofd)
110
111 sf_puts(" ST="); sf_putn(status); sf_puts(" GC="); sf_putn(gc); sf_puts(" -> "); sf_puts(out_path); sf_puts("\n")
112 return status
113}
114
115func one(store: *TrustStore, full: *u8, path: *u8, out: *u8) -> i64 {
116 if fetch_page(store, full, path, sf_strlen(path), out) == 200 { return 1 }
117 return 0
118}
119
120func main() -> i64 {
121 let cpath: *u8 = "/tmp/mozilla_certdata.txt\x00"
122 let r: i64 = nx_trust_store_load_from_certdata(cpath, 512, K_MAGIC_4194304)
123 if r <= 0 { sf_puts("FRONTIER-FETCH: certdata load failed\n"); return 1 }
124 let store: *TrustStore = r as *TrustStore
125 let n: i64 = trust_store_count(store)
126 if n < 50 { sf_puts("FRONTIER-FETCH: too few CAs\n"); return 3 }
127 sf_puts("CA="); sf_putn(n); sf_puts("\n")
128
129 var ok: i64 = 0
130 // --- SILICON / VECTOR-MATH KERNEL ---
131 ok = ok + one(store, "https://en.wikipedia.org/wiki/Dot_product\x00", "/wiki/Dot_product\x00", "knowledge/fetched/srch_dotproduct.raw\x00" as *u8)
132 ok = ok + one(store, "https://en.wikipedia.org/wiki/Cosine_similarity\x00", "/wiki/Cosine_similarity\x00", "knowledge/fetched/srch_cosine.raw\x00" as *u8)
133 ok = ok + one(store, "https://en.wikipedia.org/wiki/Basic_Linear_Algebra_Subprograms\x00", "/wiki/Basic_Linear_Algebra_Subprograms\x00", "knowledge/fetched/srch_blas.raw\x00" as *u8)
134 // --- VECTOR REPRESENTATION ---
135 ok = ok + one(store, "https://en.wikipedia.org/wiki/Word_embedding\x00", "/wiki/Word_embedding\x00", "knowledge/fetched/srch_embedding.raw\x00" as *u8)
136 ok = ok + one(store, "https://en.wikipedia.org/wiki/Word2vec\x00", "/wiki/Word2vec\x00", "knowledge/fetched/srch_word2vec.raw\x00" as *u8)
137 ok = ok + one(store, "https://en.wikipedia.org/wiki/Large_language_model\x00", "/wiki/Large_language_model\x00", "knowledge/fetched/srch_llm.raw\x00" as *u8)
138 // --- ANN INDEX / SCALE ---
139 ok = ok + one(store, "https://en.wikipedia.org/wiki/Nearest_neighbor_search\x00", "/wiki/Nearest_neighbor_search\x00", "knowledge/fetched/srch_ann.raw\x00" as *u8)
140 ok = ok + one(store, "https://en.wikipedia.org/wiki/Hierarchical_navigable_small_world\x00", "/wiki/Hierarchical_navigable_small_world\x00", "knowledge/fetched/srch_hnsw.raw\x00" as *u8)
141 ok = ok + one(store, "https://en.wikipedia.org/wiki/Locality-sensitive_hashing\x00", "/wiki/Locality-sensitive_hashing\x00", "knowledge/fetched/srch_lsh.raw\x00" as *u8)
142 ok = ok + one(store, "https://en.wikipedia.org/wiki/Vector_quantization\x00", "/wiki/Vector_quantization\x00", "knowledge/fetched/srch_vq.raw\x00" as *u8)
143 ok = ok + one(store, "https://en.wikipedia.org/wiki/Vector_database\x00", "/wiki/Vector_database\x00", "knowledge/fetched/srch_vectordb.raw\x00" as *u8)
144 // --- RETRIEVAL FRAME ---
145 ok = ok + one(store, "https://en.wikipedia.org/wiki/Information_retrieval\x00", "/wiki/Information_retrieval\x00", "knowledge/fetched/srch_ir.raw\x00" as *u8)
146
147 sf_puts("FRONTIER-SOVEREIGN-FETCH-OK pages_200="); sf_putn(ok); sf_puts("/12\n")
148 if ok < 1 { return 51 }
149 return 0
150}