nx_crawl_main.nx
buildroot/runtime/nx_crawl_main.nx
about
nx_crawl_main.nx -- the deployable Nishi crawl+serve daemon.
module: nishi-core.search.crawl_main
capability: APP_RUNNABLE (DAEMON)
One binary that does it all, ISP-safely:
STARTUP CRAWL: governed (nx_crawl_governor) polite-BFS (nx_robots +
crawl-delay + descriptive UA), URL-canonicalized (nx_urlcanon) so we never
re-fetch a variant, content-deduped (nx_simhash), indexed
(nx_search_inverted) and PERSISTED (nx_docstore) so it resumes.
SERVE: a live search site ranking the crawled corpus (nx_rank_fused:
tier + BM25 + RRF). Surfaces everything the hosts allow, by relevance.
ISP profile = slow ordinary browser: standard ports (80/443 -- here loopback
8088 for the demo), GET-only (never seed/upload like BitTorrent), sequential
concurrency=1, GLOBAL min-interval + request/byte budgets (governor). In the
networked env, change the seed host/port to real targets over DNS/TLS.
dependencies 14 imports · 0 importers
diagram shows first 10 each side; +4 more imports, +0 more importers in the complete lists below.
imports: fx.nxnx_str.nxnx_syscalls.nxnx_http_client.nxnx_http_server.nxnx_html_to_text.nxnx_simhash.nxnx_crawl_frontier.nxnx_robots.nxnx_urlcanon.nxnx_docstore.nxnx_net_governor.nxnx_http_get_ua.nxnx_rank_fused.nx
imported by: nobody (leaf or entry point)
call flow from main pre-order; caps 40 nodes / depth 6 declared; ↻ = already shown
structs
| none |
consts
| 33 | const NX_MAGIC_1048576: i64 = 1048576 |
| 34 | const NX_MAGIC_65536: i64 = 65536 |
| 35 | const NX_MAGIC_8192: i64 = 8192 |
| 37 | const NX_CM_CRAWL_PORT: i64 = 8088 |
| 38 | const NX_CM_SERVE_PORT: i64 = 8791 |
functions
| 40 | func nx_putc(c: i64) -> i64 { let b: *u8 = sys_mmap(1); b[0] = c; sys_write(1, b, 1); return 0 } |
| 41 | func nx_puts(s: *u8) -> i64 { sys_write(1, s, nx_str_len(s)); return 0 } |
| 42 | func nx_pi(n: i64) -> i64 |
| 49 | func bput(buf: *u8, p: *i64, cap: i64, s: *u8) -> i64 { let n: i64 = nx_str_len(s); var i: i64 = 0; while i < n { if p[0] < cap { buf[p[0]] = s[i]; p[0] = p[0]+1 } i = i+1 } return 0 } calls 1: nx_str_len |
| 50 | func nx_body_off(resp: *u8, n: i64) -> i64 |
| 58 | func nx_cm_fetch(addr: *u8, path: *u8, plen: i64, host: *u8, hlen: i64, ua: *u8, ualen: i64, out: *u8, cap: i64, out_status: *i64) -> i64 calls 1: nx_http_get_ua |
| 63 | func nx_cm_path(url: *u8, out: *u8, outcap: i64) -> i64 calls 1: nx_str_len |
| 80 | func main() -> i64 |