nx_search_extern.nx source
↩ module page · 114 lines · 6571 B
1// nx_search_extern.nx -- SOVEREIGN EXTERNAL-SEARCH FEDERATION (the #1 discovery gap). Queries OTHER search
2// systems -- arXiv (the research frontier) + GitHub (code frontier) -- over Nishi's OWN sovereign TLS 1.3
3// (no external SDK, no API key), parses their responses, and merges into one frontier list. This is what lets
4// the intake engine SEE where a field is moving (mature 'lightbulb' vs emergent 'photonic/quantum') instead of
5// relying on a hand-kept URL list. Composes nx_https_fetch_follow (proven live: arXiv 200 + GitHub 200).
6// Usage: nx_search_extern <term> [term ...] (default: photonic computing) license_tier: ORIGINAL
7import "nx_syscalls.nx"
8import "nx_x509_trust_store.nx"
9import "nx_trust_store_load_from_certdata.nx"
10import "nx_https_fetch_follow.nx"
11import "nx_net_polite.nx" // citizenship: pace + honor Retry-After + back off on 429/503 (don't get blocked/dried-up)
12const K_MAGIC_4194304: i64 = 4194304
13const K_MAGIC_1024: i64 = 1024
14
15func sp(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
16func sn(v: i64) -> i64 { if v==0 { sys_write(1,"0" as *u8,1); return 0 } var m: i64=v; if m<0 { sys_write(1,"-" as *u8,1); m=0-m } let t: *u8=sys_mmap(28); var k: i64=0; while m>0 { t[k]=(48+(m%10)) as u8; m=m/10; k=k+1 } while k>0 { k=k-1; sys_write(1,(((t as i64)+k) as *u8),1) } return 0 }
17func se_app(dst: *u8, off: i64, s: *u8) -> i64 { var i: i64=0; while s[i]!=(0 as u8) { dst[off+i]=s[i]; i=i+1 } return off+i }
18func se_strlen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8) { n=n+1 } return n }
19// count non-overlapping occurrences of needle in buf[0,n)
20func se_count(buf: *u8, n: i64, needle: *u8) -> i64 {
21 var nn: i64=0; while needle[nn]!=(0 as u8) { nn=nn+1 }
22 if nn==0 { return 0 }
23 var c: i64=0; var i: i64=0
24 while i+nn<=n {
25 var j: i64=0; var ok: i64=1
26 while j<nn { if buf[i+j]!=needle[j] { ok=0; j=nn } else { j=j+1 } }
27 if ok==1 { c=c+1; i=i+nn } else { i=i+1 }
28 }
29 return c
30}
31// print up to maxk spans that sit between openm and the byte `closeb`, skipping the first `skip` matches.
32func se_spans(buf: *u8, n: i64, openm: *u8, closeb: i64, maxk: i64, skip: i64) -> i64 {
33 let ol: i64=se_strlen(openm); if ol==0 { return 0 }
34 var found: i64=0; var printed: i64=0; var i: i64=0
35 while i+ol<=n {
36 if printed>=maxk { i=n } else {
37 var j: i64=0; var ok: i64=1
38 while j<ol { if buf[i+j]!=openm[j] { ok=0; j=ol } else { j=j+1 } }
39 if ok==1 {
40 found=found+1
41 if found>skip {
42 let start: i64=i+ol; var e: i64=start; var stop: i64=0
43 while stop==0 { if e>=n { stop=1 } else { if (buf[e] as i64)==closeb { stop=1 } else { e=e+1 } } }
44 sp(" - " as *u8); sys_write(1, ((buf as i64)+start) as *u8, e-start); sp("\n" as *u8)
45 printed=printed+1; i=e
46 } else { i=i+ol }
47 } else { i=i+1 }
48 }
49 }
50 return printed
51}
52// join argv[1..] with '+' into out (URL-safe-ish for arXiv/GitHub q); default "photonic+computing".
53func se_join(argc: i64, argv: *i64, out: *u8) -> i64 {
54 if argc<2 { return se_app(out,0,"photonic+computing" as *u8) }
55 var o: i64=0; var a: i64=1
56 while a<argc {
57 if a>1 { out[o]=43 as u8; o=o+1 }
58 let w: *u8=argv[a] as *u8; var k: i64=0
59 while w[k]!=(0 as u8) { let c: i64=w[k] as i64; if c==32 { out[o]=43 as u8 } else { out[o]=w[k] } o=o+1; k=k+1 }
60 a=a+1
61 }
62 out[o]=0 as u8; return o
63}
64// citizenship-composed fetch: try, and on a rate-limit response (429/503) honor Retry-After / back off and
65// retry up to POL_MAX_RETRY -- then honor the block instead of hammering (what gets a bot banned).
66func se_fetch(store: *TrustStore, url: *u8, out: *u8, cap: i64, label: *u8) -> i64 {
67 let status: *i64=sys_mmap(8) as *i64
68 var attempt: i64=0; var done: i64=0; var n: i64=0
69 while done==0 {
70 status[0]=0
71 n=nx_https_fetch_follow(url, store, out, cap, 6, status)
72 if status[0]==200 { done=1 } else {
73 if pol_should_retry(status[0])==1 { if attempt<POL_MAX_RETRY {
74 let waited: i64=pol_backoff_sleep(out, n, attempt)
75 sp(" "); sp(label); sp(": rate-limited status="); sn(status[0]); sp(" -> polite backoff "); sn(waited); sp("s (retry "); sn(attempt+1); sp(")\n" as *u8)
76 attempt=attempt+1
77 } else { done=1 } } else { done=1 }
78 }
79 }
80 sp(" "); sp(label); sp(": status="); sn(status[0]); sp(" bytes="); sn(n)
81 if attempt>0 { sp(" attempts="); sn(attempt+1) }
82 sp("\n" as *u8)
83 if status[0]==200 { return n }
84 return 0
85}
86
87func main(argc: i64, argv: *i64) -> i64 {
88 sp("=== nx_search_extern: sovereign federation of EXTERNAL search systems (arXiv + GitHub) ===\n" as *u8)
89 let r: i64=nx_trust_store_load_from_certdata("data/mozilla_certdata.txt" as *u8, 512, K_MAGIC_4194304)
90 if r<=0 { sp(" trust store load FAILED (data/mozilla_certdata.txt)\n" as *u8); sys_exit(1); return 1 }
91 let store: *TrustStore=r as *TrustStore
92 let q: *u8=sys_mmap(512); let qn: i64=se_join(argc, argv, q)
93 sp(" query: "); sys_write(1, q, qn); sp("\n" as *u8)
94 let cap: i64=K_MAGIC_4194304; let out: *u8=sys_mmap(cap)
95
96 // --- arXiv (research frontier) ---
97 let au: *u8=sys_mmap(K_MAGIC_1024); var ao: i64=0
98 ao=se_app(au,ao,"https://export.arxiv.org/api/query?search_query=all:" as *u8); ao=se_app(au,ao,q)
99 ao=se_app(au,ao,"&start=0&max_results=6&sortBy=submittedDate&sortOrder=descending" as *u8); au[ao]=0 as u8
100 let an: i64=se_fetch(store, au, out, cap, "arXiv" as *u8)
101 var aentries: i64=0
102 if an>0 { aentries=se_count(out, an, "<entry>" as *u8); sp(" entries="); sn(aentries); sp(" (newest first)\n" as *u8); se_spans(out, an, "<title>" as *u8, 60, 4, 1) }
103
104 // --- GitHub (code frontier) ---
105 let gu: *u8=sys_mmap(K_MAGIC_1024); var go: i64=0
106 go=se_app(gu,go,"https://api.github.com/search/repositories?q=" as *u8); go=se_app(gu,go,q)
107 go=se_app(gu,go,"&sort=updated&per_page=6" as *u8); gu[go]=0 as u8
108 let gn: i64=se_fetch(store, gu, out, cap, "GitHub" as *u8)
109 var grepos: i64=0
110 if gn>0 { grepos=se_count(out, gn, "\"full_name\"" as *u8); sp(" repos="); sn(grepos); sp(" (recently updated)\n" as *u8); se_spans(out, gn, "\"full_name\": \"" as *u8, 34, 4, 0) }
111
112 sp("\n FEDERATED frontier hits = "); sn(aentries+grepos); sp(" across 2 external systems (sovereign TLS, no API key)\n" as *u8)
113 sys_exit(0); return 0
114}