nx_research_engine_admission_t68.nx
buildroot/runtime/nx_research_engine_admission_t68.nx
about
nx_research_engine.nx -- the SHARED sovereign research fetch->extract->bank ENGINE. Rule-15 DRY: ONE copy of the
fetch loop that was copy-pasted into every nx_*_research_fetch organ. Topic organs (nx_arith_research, ...) stay
THIN: they COMPOSE this engine and declare their STRUCTURED sources (sections + purpose) -- no copy-pasted loop,
no flat TSV. Functionality is a strict SUPERSET of the old fetchers:
PRESERVED: idempotent have-skip, sovereign TLS-1.3 fetch, status print, GZIP-guard, raw archive, readable count,
descriptive SECTIONS (rf_section).
ADDED: HTML -> clean plain text -> knowledge/library/<name>.txt (so nx_library_harvest_v2 banks it).
100% sovereign (own TLS, nx_cc->nxasm, no curl/wget/gcc). license_tier: ORIGINAL
dependencies 8 imports · 3 importers
imports: nx_syscalls.nxnx_itoa_lib.nxnx_x509_trust_store.nxnx_trust_store_load_from_certdata.nxnx_https_fetch_follow.nxnx_research_ka.nxnx_fetch_any.nxnx_gzip_wrap.nx
imported by: nx_research_engine_admission_t68_gate.nxnx_research_pool_admission_t68.nxnx_research_pool_admission_t68b.nx
structs
| none |
consts
| 17 | const RF_GZ_MAX_OUT: i64 = 16777216 // 16 MiB inflated cap per document |
| 18 | const RF_MAGIC_2026: i64 = 2026 |
| 19 | const RF_MAGIC_4194304: i64 = 4194304 |
| 20 | const RF_MAGIC_2048: i64 = 2048 |
| 21 | const RF_MAGIC_16384: i64 = 16384 |
| 22 | const RF_MAGIC_16383: i64 = 16383 |
| 23 | const RF_MAGIC_1024: i64 = 1024 |
| 24 | const RF_MAGIC_4096: i64 = 4096 |
| 25 | const RF_MAGIC_8192: i64 = 8192 |
| 121 | const RF_OEXCL: i64 = 193 |
| 122 | const RF_MODE: i64 = 420 |
| 123 | const RF_MAX_SLOTS: i64 = 4 |
| 124 | const RF_PER_FETCH_MB: i64 = 256 |
| 125 | const RF_STALE_SEC: i64 = 30 |
| 126 | const RF_WAIT_MS: i64 = 200 |
| 127 | const RF_WAIT_MAX: i64 = 300 |
| 128 | const RF_HOST_DELAY: i64 = 1 // seconds between fetches to the SAME host (polite per-host crawl-delay) -> svc-config |
| 182 | const RF_SLOT_MARKER_LEN: i64 = 9 |
| 183 | const RF_SLOT_LEGACY: i64 = 0 - 116 |
| 184 | const RF_SLOT_BUSY: i64 = 0 - 11 |
| 185 | const RF_SLOT_RDWR: i64 = 2 |
| 243 | const RF_CB_THRESHOLD: i64 = 3 // consecutive fails to TRIP the circuit breaker |
| 244 | const RF_CB_COOLDOWN: i64 = 60 // seconds a tripped host is fast-failed (don't hammer a down site) |
functions
| 27 | func rf_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 } |
| 32 | func rf_putn(v: i64) -> i64 { nxi_out(v); return 0 } |
| 33 | func rf_have(path: *u8) -> i64 { let fd: i64 = sys_openat_rd(path); if fd < 0 { return 0 } sys_close(fd); return 1 } |
| 34 | func rf_scat(dst: *u8, o: i64, s: *u8) -> i64 { var i: i64 = 0; while s[i] != (0 as u8) { dst[o + i] = s[i]; i = i + 1 } return o + i } |
| 35 | func rf_bpath(dst: *u8, dir: *u8, name: *u8, suf: *u8) -> i64 |
| 38 | func rf_save(path: *u8, buf: *u8, n: i64) -> i64 |
| 47 | func rf_tag_eq(src: *u8, off: i64, n: i64, lit: *u8, litlen: i64) -> i64 called by 1: rf_strip |
| 60 | func rf_strip(src: *u8, n: i64, out: *u8, cap: i64) -> i64 |
| 103 | func rf_init() -> *TrustStore |
| 112 | func rf_section(s: *u8) -> i64 { rf_puts(" == " as *u8); rf_puts(s); rf_puts(" ==\n" as *u8); return 0 } calls 1: rf_puts |
| 116 | func rf_is_gzip(buf: *u8, n: i64) -> i64 { if n >= 2 { if (buf[0] as i64) == 0x1f { if (buf[1] as i64) == 0x8b { return 1 } } } return 0 } |
| 129 | func rf_sleep_ms(ms: i64) -> i64 { sys_poll(0 as *u8, 0, ms); return 0 } |
| 130 | func rf_free_mb() -> i64 |
| 154 | func rf_nslots() -> i64 |
| 161 | func rf_slotpath(dir: *u8, idx: i64, out: *u8) -> *u8 |
| 177 | func rf_wint(fd: i64, v: i64) -> i64 { nxi_fd(fd, v); return 0 } |
| 178 | func rf_rint(path: *u8) -> i64 { let fd: i64=sys_openat_rd(path); if fd<0{return 0} let b: *u8=sys_mmap(64); let n: i64=sys_read(fd,b,63); sys_close(fd); var v: i64=0; var i: i64=0; while i<n { if b[i]>=(48 as u8){if b[i]<=(57 as u8){v=v*10+((b[i] as i64)-48)}} i=i+1 } return v } called by 4: rf_host_gaterf_circuit_openrf_circuit_recordmain calls 4: sys_openat_rdsys_mmapsys_readsys_close |
| 179 | func rf_unlink(path: *u8) -> i64 { return __syscall(263, AT_FDCWD, path as i64, 0, 0, 0, 0) } |
| 186 | func rf_slot_lock(path: *u8) -> i64 called by 4: rf_slot_acquiremainmainmain calls 6: sys_flocksys_closesys_writesys_mmapsys_readsys_munmap |
| 214 | func rf_slot_acquire(dir: *u8, n: i64, idx_out: *i64) -> i64 called by 4: rf_fetch_bankmainmainmain calls 5: sys_mmaprf_slotpathrf_slot_locksys_munmaprf_sleep_ms |
| 238 | func rf_slot_release(dir: *u8, idx: i64, fd: i64) -> i64 |
| 245 | func rf_wfile(path: *u8, v: i64) -> i64 |
| 251 | func rf_host_key(url: *u8, suf: *u8, out: *u8) -> i64 |
| 279 | func rf_host_gate(url: *u8) -> i64 called by 1: rf_fetch_bank calls 7: sys_mmaprf_host_keysys_mkdirrf_rintsys_now_realtime_secrf_sleep_ms+1 |
| 289 | func rf_circuit_open(url: *u8) -> i64 |
| 296 | func rf_circuit_record(url: *u8, ok: i64) -> i64 |
| 308 | func rf_fetch_bank(url: *u8, name: *u8, store: *TrustStore, out: *u8, cap: i64) -> i64 |
| 389 | func main() -> i64 { return 0 } |