nx_semantic_research_fetch.nx source
↩ module page · 155 lines · 8777 B
1// nx_semantic_research_fetch.nx -- SOVEREIGN research fetch on the SEMANTIC-search frontier, to honestly test
2// the operator's doubt: "are we really exceed... doing semantic work?" Plain BM25/BM25F is LEXICAL (keyword);
3// world-class relevance is now SEMANTIC (embeddings / dense vectors / neural rerank), and the "partnership"
4// pattern (sovereign retrieval + an LLM doing the semantic step) is RAG. Fetches the canonical pages so the
5// reassessment is CITED, not asserted. Reuses the team's OWN TLS-1.3 stack (clone of nx_onsite_research_fetch).
6// /wiki/Semantic_search -> sem_search.raw
7// /wiki/Vector_database -> sem_vector.raw
8// /wiki/Retrieval-augmented_generation-> sem_rag.raw (retrieval + LLM = the partnership)
9// /wiki/Word_embedding -> sem_embed.raw
10// expect_exit: 0 license_tier: ORIGINAL
11import "nx_syscalls.nx"
12import "nx_x509_trust_store.nx"
13import "nx_trust_store_load_from_certdata.nx"
14import "nx_tls13_client_validate_certificate.nx"
15import "nx_tls13_client_session_run.nx"
16import "nx_https_url_for_fetch.nx"
17import "nx_https_url_connect.nx"
18import "nx_https_get.nx"
19import "nx_https_get_complete.nx"
20import "nx_http_response_parse.nx"
21import "nx_csprng.nx"
22const K_MAGIC_4194304: i64 = 4194304
23
24func sf_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
25func sf_putn(v: i64) -> i64 {
26 let bb: *u8 = sys_mmap(28); var m: i64 = v
27 if m < 0 { m = 0 - m; sys_write(1, "-" as *u8, 1) }
28 let t: *u8 = sys_mmap(28); var k: i64 = 0
29 if m == 0 { t[0] = 48 as u8; k = 1 }
30 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
31 var i: i64 = 0; while i < k { bb[i] = t[k - 1 - i]; i = i + 1 } sys_write(1, bb, k); return 0
32}
33func sf_strlen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
34
35func fetch_page(store: *TrustStore, full_url: *u8, path: *u8, path_len: i64, out_path: *u8) -> i64 {
36 sf_puts("--- fetch "); sf_puts(full_url); sf_puts("\n")
37 let cr: *u8 = sys_mmap(32)
38 var i: i64 = 0
39 nx_csprng_fill(cr, 32) // CWE-330 (debt 1785970852): was the constant 0xC0..0xDF
40 let priv: *u8 = sys_mmap(32)
41 i = 0
42 nx_csprng_fill(priv, 32) // CWE-330: the X25519 scalar was the constant 0xA0..0xBF on EVERY session
43 let url_p: *NxUrl = nx_url_new()
44 let target_raw: *u8 = sys_mmap(32)
45 let target: *NxHttpsTarget = target_raw as *NxHttpsTarget
46 target.url = url_p
47 target.port = 0
48 if nx_https_url_for_fetch(full_url, target) != NX_HTTPS_URL_OK { return 0 - 41 }
49 let fd_p: *i64 = sys_mmap(16) as *i64
50 if nx_https_url_connect(target, full_url, sys_now_realtime_sec(), fd_p) != NX_HTTPS_CONNECT_OK { return 0 - 42 }
51 let fd: i64 = *fd_p
52 let val_ctx_raw: *u8 = sys_mmap(64)
53 let val_ctx: *TlsValidationContext = val_ctx_raw as *TlsValidationContext
54 val_ctx.store = store
55 val_ctx.sni_host = full_url + target.url.host_off
56 val_ctx.sni_host_len = target.url.host_len
57 val_ctx.now_epoch = sys_now_realtime_sec()
58 let sr: i64 = nx_tls13_client_session_run(fd, full_url + target.url.host_off, target.url.host_len, cr, priv, val_ctx)
59 if sr <= 0 { sys_close(fd); return 0 - (200 + (0 - sr)) }
60 let session: *Tls13ClientSession = sr as *Tls13ClientSession
61 let buf: *u8 = sys_mmap(K_MAGIC_4194304)
62 let gc: i64 = nx_https_get_complete(session, fd, path, path_len, full_url + target.url.host_off, target.url.host_len, buf, K_MAGIC_4194304)
63 sys_close(fd)
64 if gc < 0 { return 0 - (100 + (0 - gc)) }
65 let rs: *i64 = sys_mmap(128) as *i64
66 nx_http_response_parse(buf, gc, rs)
67 let status: i64 = rs[1]
68 let ofd: i64 = sys_openat_wr(out_path, 0x1A4)
69 if ofd <= 0 { return 0 - 70 }
70 sys_write(ofd, buf, gc)
71 sys_close(ofd)
72 sf_puts(" ST="); sf_putn(status); sf_puts(" GC="); sf_putn(gc); sf_puts(" -> "); sf_puts(out_path); sf_puts("\n")
73 return status
74}
75
76func main() -> i64 {
77 let cpath: *u8 = "/tmp/mozilla_certdata.txt\x00"
78 let r: i64 = nx_trust_store_load_from_certdata(cpath, 512, K_MAGIC_4194304)
79 if r <= 0 { sf_puts("SEM-FETCH: certdata load failed\n"); return 1 }
80 let store: *TrustStore = r as *TrustStore
81 let n: i64 = trust_store_count(store)
82 if n < 50 { sf_puts("SEM-FETCH: too few CAs\n"); return 3 }
83 sf_puts("CA="); sf_putn(n); sf_puts("\n")
84 var ok: i64 = 0
85
86 let u1: *u8 = "https://en.wikipedia.org/wiki/Semantic_search\x00"
87 let p1: *u8 = "/wiki/Semantic_search\x00"
88 if fetch_page(store, u1, p1, sf_strlen(p1), "knowledge/fetched/sem_search.raw\x00" as *u8) == 200 { ok = ok + 1 }
89
90 let u2: *u8 = "https://en.wikipedia.org/wiki/Vector_database\x00"
91 let p2: *u8 = "/wiki/Vector_database\x00"
92 if fetch_page(store, u2, p2, sf_strlen(p2), "knowledge/fetched/sem_vector.raw\x00" as *u8) == 200 { ok = ok + 1 }
93
94 let u3: *u8 = "https://en.wikipedia.org/wiki/Retrieval-augmented_generation\x00"
95 let p3: *u8 = "/wiki/Retrieval-augmented_generation\x00"
96 if fetch_page(store, u3, p3, sf_strlen(p3), "knowledge/fetched/sem_rag.raw\x00" as *u8) == 200 { ok = ok + 1 }
97
98 let u4: *u8 = "https://en.wikipedia.org/wiki/Word_embedding\x00"
99 let p4: *u8 = "/wiki/Word_embedding\x00"
100 if fetch_page(store, u4, p4, sf_strlen(p4), "knowledge/fetched/sem_embed.raw\x00" as *u8) == 200 { ok = ok + 1 }
101
102 // --- neural-retrieval FRONTIER orientation (Wikipedia = orientation only; DEPTH comes via dynamic discovery) ---
103 let u5: *u8 = "https://en.wikipedia.org/wiki/Nearest_neighbor_search\x00"
104 let p5: *u8 = "/wiki/Nearest_neighbor_search\x00"
105 if fetch_page(store, u5, p5, sf_strlen(p5), "knowledge/fetched/sem_ann.raw\x00" as *u8) == 200 { ok = ok + 1 }
106
107 let u6: *u8 = "https://en.wikipedia.org/wiki/Transformer_(deep_learning_architecture)\x00"
108 let p6: *u8 = "/wiki/Transformer_(deep_learning_architecture)\x00"
109 if fetch_page(store, u6, p6, sf_strlen(p6), "knowledge/fetched/sem_transformer.raw\x00" as *u8) == 200 { ok = ok + 1 }
110
111 let u7: *u8 = "https://en.wikipedia.org/wiki/BERT_(language_model)\x00"
112 let p7: *u8 = "/wiki/BERT_(language_model)\x00"
113 if fetch_page(store, u7, p7, sf_strlen(p7), "knowledge/fetched/sem_bert.raw\x00" as *u8) == 200 { ok = ok + 1 }
114
115 let u8: *u8 = "https://en.wikipedia.org/wiki/Sentence_embedding\x00"
116 let p8: *u8 = "/wiki/Sentence_embedding\x00"
117 if fetch_page(store, u8, p8, sf_strlen(p8), "knowledge/fetched/sem_sentence.raw\x00" as *u8) == 200 { ok = ok + 1 }
118
119 let u9: *u8 = "https://en.wikipedia.org/wiki/Learning_to_rank\x00"
120 let p9: *u8 = "/wiki/Learning_to_rank\x00"
121 if fetch_page(store, u9, p9, sf_strlen(p9), "knowledge/fetched/sem_ltr.raw\x00" as *u8) == 200 { ok = ok + 1 }
122
123 let u10: *u8 = "https://en.wikipedia.org/wiki/Large_language_model\x00"
124 let p10: *u8 = "/wiki/Large_language_model\x00"
125 if fetch_page(store, u10, p10, sf_strlen(p10), "knowledge/fetched/sem_llm.raw\x00" as *u8) == 200 { ok = ok + 1 }
126
127 // --- the FRONTIER LIMIT path: how to build a sovereign no-float-TRAINED neural retriever ---
128 let u11: *u8 = "https://en.wikipedia.org/wiki/Backpropagation\x00"
129 let p11: *u8 = "/wiki/Backpropagation\x00"
130 if fetch_page(store, u11, p11, sf_strlen(p11), "knowledge/fetched/sem_backprop.raw\x00" as *u8) == 200 { ok = ok + 1 }
131
132 let u12: *u8 = "https://en.wikipedia.org/wiki/Stochastic_gradient_descent\x00"
133 let p12: *u8 = "/wiki/Stochastic_gradient_descent\x00"
134 if fetch_page(store, u12, p12, sf_strlen(p12), "knowledge/fetched/sem_sgd.raw\x00" as *u8) == 200 { ok = ok + 1 }
135
136 let u13: *u8 = "https://en.wikipedia.org/wiki/Fixed-point_arithmetic\x00"
137 let p13: *u8 = "/wiki/Fixed-point_arithmetic\x00"
138 if fetch_page(store, u13, p13, sf_strlen(p13), "knowledge/fetched/sem_fixedpoint.raw\x00" as *u8) == 200 { ok = ok + 1 }
139
140 let u14: *u8 = "https://en.wikipedia.org/wiki/Quantization_(signal_processing)\x00"
141 let p14: *u8 = "/wiki/Quantization_(signal_processing)\x00"
142 if fetch_page(store, u14, p14, sf_strlen(p14), "knowledge/fetched/sem_quant.raw\x00" as *u8) == 200 { ok = ok + 1 }
143
144 let u15: *u8 = "https://en.wikipedia.org/wiki/Knowledge_distillation\x00"
145 let p15: *u8 = "/wiki/Knowledge_distillation\x00"
146 if fetch_page(store, u15, p15, sf_strlen(p15), "knowledge/fetched/sem_distill.raw\x00" as *u8) == 200 { ok = ok + 1 }
147
148 let u16: *u8 = "https://en.wikipedia.org/wiki/Attention_(machine_learning)\x00"
149 let p16: *u8 = "/wiki/Attention_(machine_learning)\x00"
150 if fetch_page(store, u16, p16, sf_strlen(p16), "knowledge/fetched/sem_attention.raw\x00" as *u8) == 200 { ok = ok + 1 }
151
152 sf_puts("SEM-SOVEREIGN-FETCH-OK pages_200="); sf_putn(ok); sf_puts("/16\n")
153 if ok < 1 { return 51 }
154 return 0
155}