code wiki / (root) / nx_ingest_runner.nx

nx_ingest_runner.nx

buildroot/runtime/nx_ingest_runner.nx

4904 B148 linesdepth 6pulls 14 transitivereach 18 importersview sourcekind librarytopic ingest

docs dependencies structs consts functions

about

nx_ingest_runner.nx -- composition primitive for autonomous ingest. Wraps disk_budget / bloom / shard_writer / checkpoint / progress_watchdog behind one .offer(key, payload) call. Source-specific parsers feed records in; the runner handles budgeting, dedup, rotation, resume, stall detection. genealogy_id: lambda_architecture_kreps_2014 + map_reduce_dean_2004 lineage_id: composition

dependencies 8 imports · 18 importers

diagram shows first 10 each side; +0 more imports, +8 more importers in the complete lists below.

imports: nx_syscalls.nxnx_tier.nxnx_disk_budget.nxnx_bloom.nxnx_bloom_capacity.nxnx_shard_writer.nxnx_checkpoint.nxnx_progress_watchdog.nx

imported by: nx_coq_dir_ingest.nxnx_coq_stream_ingest.nxnx_coq_stream_ingest_test.nxnx_hol_dir_ingest.nxnx_hol_stream_ingest.nxnx_hol_stream_ingest_test.nxnx_ingest_runner_test.nxnx_isabelle_dir_ingest.nxnx_isabelle_stream_ingest.nxnx_isabelle_stream_ingest_test.nxnx_lean_dir_ingest.nxnx_lean_stream_ingest.nxnx_lean_stream_ingest_test.nxnx_mizar_dir_ingest.nxnx_mizar_stream_ingest.nxnx_mizar_stream_ingest_test.nxnx_rfc_dir_ingest.nxnx_rfc_stream_ingest.nx

structs

33	struct NxIngestRun

consts

26	const NX_OFFER_EMITTED: nx_int = 0
27	const NX_OFFER_DUPLICATE: nx_int = 1
28	const NX_OFFER_BUDGET_HALT: nx_int = -1
29	const NX_OFFER_WRITE_FAIL: nx_int = -2
31	const NX_INGEST_CKPT_EVERY: nx_int = 1000
48	const NX_INGEST_RUN_BYTES: nx_size = 96

functions

50	func nx_ingest_run_new( called by 12: main main main main main main+6 calls 6: sys_mmap nx_disk_budget_new nx_shard_writer_open nx_bloom_new_for_capacity nx_checkpoint_open nx_progress_watchdog_new
79	func nx_ingest_run_offer( called by 7: nx_coq_stream_offer_all nx_hol_stream_offer_all main nx_isabelle_stream_offer_all nx_lean_stream_offer_all nx_mizar_stream_offer_all+1 calls 7: nx_progress_watchdog_heartbeat bloom_contains nx_disk_budget_can_write bloom_insert nx_shard_writer_emit nx_disk_budget_account+1
117	func nx_ingest_run_is_stalled(r: *NxIngestRun) -> nx_int called by 1: main calls 1: nx_progress_watchdog_check
123	func nx_ingest_run_close(r: *NxIngestRun) -> nx_int called by 12: main main main main main main+6 calls 3: nx_checkpoint_set nx_checkpoint_close nx_shard_writer_close
130	func nx_ingest_run_n_offered(r: *NxIngestRun) -> nx_int { return r.n_offered } called by 1: main
131	func nx_ingest_run_n_emitted(r: *NxIngestRun) -> nx_int { return r.n_emitted } called by 6: main main main main main main
132	func nx_ingest_run_n_duplicate(r: *NxIngestRun) -> nx_int { return r.n_duplicate } called by 8: main main main main main main+2
133	func nx_ingest_run_n_budget_halt(r: *NxIngestRun) -> nx_int { return r.n_budget_halt } called by 2: main main
134	func nx_ingest_run_n_write_fail(r: *NxIngestRun) -> nx_int { return r.n_write_fail }
135	func nx_ingest_run_n_stalls(r: *NxIngestRun) -> nx_int { return r.n_stalls } called by 1: main
136	func nx_ingest_run_n_shards(r: *NxIngestRun) -> nx_int { return nx_shard_writer_n_shards(r.shard_writer) } called by 12: main main main main main main+6 calls 1: nx_shard_writer_n_shards
138	func nx_ingest_run_disk_used_bytes(r: *NxIngestRun) -> nx_size called by 3: main main main calls 1: nx_disk_budget_used
142	func nx_ingest_run_disk_pct_used(r: *NxIngestRun) -> nx_int called by 1: main calls 1: nx_disk_budget_pct_used
146	func nx_ingest_run_resume_from(r: *NxIngestRun) -> nx_int calls 1: nx_checkpoint_get