code wiki / (root) / nx_research_engine.nx

nx_research_engine.nx

buildroot/runtime/nx_research_engine.nx

18476 B351 linesdepth 23pulls 154 transitivereach 75 importersview sourcekind tooltopic research
docsdependenciesstructsconstsfunctions

about

nx_research_engine.nx -- the SHARED sovereign research fetch->extract->bank ENGINE. Rule-15 DRY: ONE copy of the fetch loop that was copy-pasted into every nx_*_research_fetch organ. Topic organs (nx_arith_research, ...) stay THIN: they COMPOSE this engine and declare their STRUCTURED sources (sections + purpose) -- no copy-pasted loop, no flat TSV. Functionality is a strict SUPERSET of the old fetchers: PRESERVED: idempotent have-skip, sovereign TLS-1.3 fetch, status print, GZIP-guard, raw archive, readable count, descriptive SECTIONS (rf_section). ADDED: HTML -> clean plain text -> knowledge/library/<name>.txt (so nx_library_harvest_v2 banks it). 100% sovereign (own TLS, nx_cc->nxasm, no curl/wget/gcc). license_tier: ORIGINAL

dependencies 8 imports · 71 importers

nx_syscalls.nx nx_itoa_lib.nx nx_x509_trust_store.nx nx_trust_store_load_from_certdata. nx_https_fetch_follow.nx nx_research_ka.nx nx_fetch_any.nx nx_gzip_wrap.nx nx_research_engine.nx nx_adversarial_research.nx nx_aitrain_research.nx nx_anatomy_research_fetch.nx nx_arch_research.nx nx_archive_probe.nx nx_archive_research.nx nx_arith_research.nx nx_auth_research.nx nx_avatar_cv_research_fetch.nx nx_beerqa_fetch.nx

diagram shows first 10 each side; +0 more imports, +61 more importers in the complete lists below.

imports: nx_syscalls.nxnx_itoa_lib.nxnx_x509_trust_store.nxnx_trust_store_load_from_certdata.nxnx_https_fetch_follow.nxnx_research_ka.nxnx_fetch_any.nxnx_gzip_wrap.nx

imported by: nx_adversarial_research.nxnx_aitrain_research.nxnx_anatomy_research_fetch.nxnx_arch_research.nxnx_archive_probe.nxnx_archive_research.nxnx_arith_research.nxnx_auth_research.nxnx_avatar_cv_research_fetch.nxnx_beerqa_fetch.nxnx_benchtools_research.nxnx_callui_research_fetch.nxnx_civic_fetch_test.nxnx_coaching_research.nxnx_comms_perf_fetch.nxnx_comms_rival_fetch.nxnx_companion_research_fetch.nxnx_court_ingest.nxnx_deepresearch_research.nxnx_deepresearch_research2.nxnx_depstack_research.nxnx_drbench_fetch.nxnx_emulator_research_fetch.nxnx_eval_landscape_research.nxnx_exefmt_research.nxnx_fetch_probe.nxnx_frontier_research_fetch.nxnx_gfx_research.nxnx_gpudriver_research.nxnx_hsx_research_fetch.nxnx_hub_research.nxnx_hwagnostic_research.nxnx_hwmodel_research_fetch.nxnx_infinigen_research_fetch.nxnx_infomgmt_research.nxnx_interpersonal_research.nxnx_ka_engine_test.nxnx_legaldocs_research.nxnx_livingviz_research.nxnx_llm_landscape_research.nxnx_mcbench_research_fetch.nxnx_media_archive_research.nxnx_media_serve_research_fetch.nxnx_mgmt_research.nxnx_neuraltex_research_fetch.nxnx_osboot_research_fetch.nxnx_oscompete_research.nxnx_photoreal_research_fetch.nxnx_physfid_research_fetch.nxnx_procpkg_research_fetch.nxnx_qabench_fetch.nxnx_reading_comfort_research_fetch.nxnx_real3d_research_fetch.nxnx_reliability_research.nxnx_render_sota_research_fetch.nxnx_research_engine_gate.nxnx_research_fetch_cli.nxnx_research_fetch_ext.nxnx_research_pool.nxnx_researcher_sota_research.nxnx_routing_research.nxnx_sovgpu_research_fetch.nxnx_steamgfx_research.nxnx_sunlight_readability_research_fetch.nxnx_swcompare_research.nxnx_vamx2_research_fetch.nxnx_vamx_research_fetch.nxnx_vidclass_research_fetch.nxnx_vqoe_research_fetch.nxnx_webviz_research.nxnx_wellbeing_research.nx

structs

none

consts

17const RF_GZ_MAX_OUT: i64 = 16777216 // 16 MiB inflated cap per document
18const RF_MAGIC_2026: i64 = 2026
19const RF_MAGIC_4194304: i64 = 4194304
20const RF_MAGIC_2048: i64 = 2048
21const RF_MAGIC_16384: i64 = 16384
22const RF_MAGIC_16383: i64 = 16383
23const RF_MAGIC_1024: i64 = 1024
24const RF_MAGIC_4096: i64 = 4096
25const RF_MAGIC_8192: i64 = 8192
122const RF_OEXCL: i64 = 193
123const RF_MODE: i64 = 420
124const RF_MAX_SLOTS: i64 = 4
125const RF_PER_FETCH_MB: i64 = 256
126const RF_STALE_SEC: i64 = 30
127const RF_WAIT_MS: i64 = 200
128const RF_WAIT_MAX: i64 = 300
129const RF_HOST_DELAY: i64 = 1 // seconds between fetches to the SAME host (polite per-host crawl-delay) -> svc-config
212const RF_CB_THRESHOLD: i64 = 3 // consecutive fails to TRIP the circuit breaker
213const RF_CB_COOLDOWN: i64 = 60 // seconds a tripped host is fast-failed (don't hammer a down site)

functions

27func rf_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
called by 74: mainmainmainmainmainmain+68 calls 1: sys_write
32func rf_putn(v: i64) -> i64 { nxi_out(v); return 0 }
called by 69: mainmainmainmainmainmain+63 calls 1: nxi_out
33func rf_have(path: *u8) -> i64 { let fd: i64 = sys_openat_rd(path); if fd < 0 { return 0 } sys_close(fd); return 1 }
called by 2: rf_fetch_bankrf_fetch_bank_paged_from calls 2: sys_openat_rdsys_close
34func rf_scat(dst: *u8, o: i64, s: *u8) -> i64 { var i: i64 = 0; while s[i] != (0 as u8) { dst[o + i] = s[i]; i = i + 1 } return o + i }
called by 1: rf_bpath
35func rf_bpath(dst: *u8, dir: *u8, name: *u8, suf: *u8) -> i64
called by 3: rf_fetch_bankmainrf_fetch_bank_paged_from calls 1: rf_scat
38func rf_save(path: *u8, buf: *u8, n: i64) -> i64
called by 1: rf_fetch_bank calls 3: sys_openat_wrsys_writesys_close
47func rf_tag_eq(src: *u8, off: i64, n: i64, lit: *u8, litlen: i64) -> i64
called by 1: rf_strip
60func rf_strip(src: *u8, n: i64, out: *u8, cap: i64) -> i64
called by 2: rf_fetch_bankge_tstrip calls 1: rf_tag_eq
103func rf_init() -> *TrustStore
called by 68: mainmainmainmainmainmain+62 calls 2: nx_trust_store_load_from_certdatasys_mkdir
112func rf_section(s: *u8) -> i64 { rf_puts(" == " as *u8); rf_puts(s); rf_puts(" ==\n" as *u8); return 0 }
called by 63: mainmainmainmainmainmain+57 calls 1: rf_puts
116func rf_is_gzip(buf: *u8, n: i64) -> i64 { if n >= 2 { if (buf[0] as i64) == 0x1f { if (buf[1] as i64) == 0x8b { return 1 } } } return 0 }
called by 3: rf_fetch_bankmainrf_fetch_bank_paged_from
130func rf_sleep_ms(ms: i64) -> i64 { sys_poll(0 as *u8, 0, ms); return 0 }
called by 8: FFFrf_slot_acquirerf_host_gaterf_fetch_bank+2 calls 1: sys_poll
131func rf_free_mb() -> i64
called by 2: rf_nslotsmain calls 4: sys_openat_rdsys_mmapsys_readsys_close
155func rf_nslots() -> i64
called by 2: rf_fetch_bankmain calls 1: rf_free_mb
162func rf_slotpath(dir: *u8, idx: i64, out: *u8) -> *u8
called by 3: rf_slot_acquirerf_slot_releasemain
178func rf_wint(fd: i64, v: i64) -> i64 { nxi_fd(fd, v); return 0 }
called by 2: rf_slot_lockrf_wfile calls 1: nxi_fd
179func rf_rint(path: *u8) -> i64 { let fd: i64=sys_openat_rd(path); if fd<0{return 0} let b: *u8=sys_mmap(64); let n: i64=sys_read(fd,b,63); sys_close(fd); var v: i64=0; var i: i64=0; while i<n { if b[i]>=(48 as u8){if b[i]<=(57 as u8){v=v*10+((b[i] as i64)-48)}} i=i+1 } return v }
called by 4: rf_slot_lockrf_host_gaterf_circuit_openrf_circuit_record calls 4: sys_openat_rdsys_mmapsys_readsys_close
181func rf_slot_lock(path: *u8) -> i64
called by 2: rf_slot_acquiremain calls 4: rf_wintsys_now_realtime_secrf_rintrf_unlink
192func rf_slot_acquire(dir: *u8, n: i64, idx_out: *i64) -> i64
called by 2: rf_fetch_bankmain calls 4: sys_mmaprf_slotpathrf_slot_lockrf_sleep_ms
207func rf_slot_release(dir: *u8, idx: i64, fd: i64) -> i64
called by 2: rf_fetch_bankmain calls 4: sys_mmaprf_slotpathsys_closerf_unlink
214func rf_wfile(path: *u8, v: i64) -> i64
called by 2: rf_host_gaterf_circuit_record calls 2: rf_wintsys_close
220func rf_host_key(url: *u8, suf: *u8, out: *u8) -> i64
called by 3: rf_host_gaterf_circuit_openrf_circuit_record
248func rf_host_gate(url: *u8) -> i64
called by 1: rf_fetch_bank calls 7: sys_mmaprf_host_keysys_mkdirrf_rintsys_now_realtime_secrf_sleep_ms+1
258func rf_circuit_open(url: *u8) -> i64
called by 1: rf_fetch_bank calls 4: sys_mmaprf_host_keyrf_rintsys_now_realtime_sec
265func rf_circuit_record(url: *u8, ok: i64) -> i64
called by 1: rf_fetch_bank calls 5: sys_mmaprf_host_keyrf_wfilerf_rintsys_now_realtime_sec
277func rf_fetch_bank(url: *u8, name: *u8, store: *TrustStore, out: *u8, cap: i64) -> i64
called by 65: mainmainFmainmainmain+59 calls 19: sys_mmaprf_bpathrf_haverf_putssys_read_filerf_circuit_open+13
351func main() -> i64 { return 0 }