nx_bulk_ingest.nx source

1// nx_bulk_ingest.nx -- the POLITE-FETCH -> BULK-CATALOG HAND-OFF (operator 2026-06-05): "a hand off function 2// when its being polite that if we can download the whole catalog of research papers like arxiv that triggers 3// like we were doing with the nishi search engine and we grow and grow our library so we can self serve more." 4// 5// When the team's polite single-fetch (nx_pipeline_fetch) lands on a page that is actually a HARVESTABLE 6// REPOSITORY (a catalog/listing/OAI-PMH/sitemap), it HANDS OFF to bulk-catalog ingest: walk the catalog, 7// mirror each doc into the Library (content-addressed dedup), so over time the team SELF-SERVES from its own 8// Library and depends on the live web less. Two hard rules from the operator's prior laws: 9// - STAY POLITE: the bulk walk runs at the SAME one-IP polite floor (pf_polite_floor) -- never a blast. 10// - SERVE THE RAISED HANDS: prioritize repositories that fill the gaps the team raised its hand about. 11// Convergent (K dry rounds = catalog exhausted), resumable, never infinite. license_tier: ORIGINAL 12import "nx_pipeline_fetch.nx" 13import "nx_syscalls.nx" 14 15// hand-off decisions 16const BI_SINGLE: i64 = 0 // ordinary page -> normal single fetch 17const BI_BULK: i64 = 1 // harvestable repository + polite capacity -> bulk-catalog ingest 18const BI_DEFER: i64 = 2 // a repository, but no polite spare capacity right now -> come back later 19 20// thresholds (data-driven, not magic-in-logic): a listing page has many links that mostly share one URL shape. 21const BI_REPO_MIN_LINKS: i64 = 20 // fewer than this is not a catalog 22const BI_REPO_PATTERN_PMIL: i64 = 600 // >=60% of links share the catalog's URL pattern -> it's a listing 23const BI_MIN_CAPACITY_PMIL: i64 = 200 // need >=20% spare polite capacity before starting a bulk walk 24 25// REPOSITORY DETECTOR: is this fetched page actually a catalog we can harvest? A known bulk endpoint 26// (OAI-PMH / sitemap.xml / a list API) is definitive; otherwise it's the link-count x pattern-density signal. 27func bi_is_repository(link_count: i64, repeated_pattern_pmil: i64, known_bulk_endpoint: i64) -> i64 { 28 if known_bulk_endpoint == 1 { return 1 } 29 if link_count >= BI_REPO_MIN_LINKS { if repeated_pattern_pmil >= BI_REPO_PATTERN_PMIL { return 1 } } 30 return 0 31} 32 33// THE HAND-OFF TRIGGER: single page -> SINGLE; repository but no polite spare capacity -> DEFER (don't 34// overwhelm the host or our one IP); repository + capacity -> BULK. This is the function the polite fetcher 35// calls on every fetched page. 36func bi_handoff(is_repo: i64, polite_capacity_pmil: i64) -> i64 { 37 if is_repo == 0 { return BI_SINGLE } 38 if polite_capacity_pmil < BI_MIN_CAPACITY_PMIL { return BI_DEFER } 39 return BI_BULK 40} 41 42// PRIORITY: among detected repositories, harvest the ones that fill a RAISED-HAND assignment first (the team 43// flagged "I need research on X"; a repository matching X is worth more). assignment_match in {0,1}; 44// gap_severity 0..1000. Score = base + match-bonus + severity. (Feeds the work dispatcher's ordering.) 45func bi_priority(assignment_match: i64, gap_severity_pmil: i64) -> i64 { 46 var s: i64 = 100 47 if assignment_match == 1 { s = s + 1000 } // a raised-hand match dominates 48 s = s + gap_severity_pmil 49 return s 50} 51 52// CONTENT-ADDRESSED DEDUP: skip a doc already mirrored (same content hash present in the Library). Returns 1 53// = skip (already have it), 0 = mirror it. (Composes with the existing content-addressed blob store.) 54func bi_dedup_skip(content_hash: i64, seen: *i64, n_seen: i64) -> i64 { 55 var i: i64 = 0 56 while i < n_seen { if seen[i] == content_hash { return 1 } i = i + 1 } 57 return 0 58} 59 60// POLITE BULK-HARVEST wall-clock: the catalog is walked as a PIPELINE (fetch->extract->mirror) at the SAME 61// one-IP polite floor as ordinary fetch -- bulk does NOT mean fast/rude, it means CONTINUOUS. Reuses the 62// proven pf_team_pipeline so the harvest is polite-optimal by construction. 63func bi_harvest_wall_ms(n_docs: i64, pace: i64, extract_ms: i64, mirror_ms: i64) -> i64 { 64 return pf_team_pipeline(n_docs, pace, extract_ms, mirror_ms) 65} 66// the harvest is POLITE iff its wall never undercuts the irreducible polite floor (n paced fetches). 67func bi_harvest_is_polite(n_docs: i64, pace: i64, extract_ms: i64, mirror_ms: i64) -> i64 { 68 if bi_harvest_wall_ms(n_docs, pace, extract_ms, mirror_ms) >= pf_polite_floor(n_docs, pace) { return 1 } 69 return 0 70} 71 72// CONVERGENCE: the walk stops when K consecutive rounds add nothing new (catalog exhausted) OR a hard backstop 73// is hit -- never infinite. Returns 1 = done. 74func bi_converged(new_this_round: i64, dry_rounds: i64, max_dry: i64, rounds_done: i64, hard_cap: i64) -> i64 { 75 if rounds_done >= hard_cap { return 1 } // backstop: never run forever 76 if new_this_round > 0 { return 0 } // still finding new docs -> keep going 77 if dry_rounds >= max_dry { return 1 } // K dry rounds -> catalog exhausted 78 return 0 79} 80 81// LIBRARY SELF-SERVE GROWTH: after a harvest, how much of the team's information need can now be served from 82// the OWN Library (zero live fetch)? coverage = mirrored / needed, in permil. This is the whole point: it 83// goes UP with every harvest, so the team depends on the live web less and less. 84func bi_self_serve_pmil(mirrored_docs: i64, needed_docs: i64) -> i64 { 85 if needed_docs <= 0 { return 1000 } 86 if mirrored_docs >= needed_docs { return 1000 } 87 return (mirrored_docs * 1000) / needed_docs 88}