nx_pipeline_fetch.nx source

1// nx_pipeline_fetch.nx -- the team's LIVE-FETCH SCHEDULING POLICY, encoding the operator's architecture law 2// (2026-06-05): "we cant pull like you as we dont have lots of ip addresses to cycle through so we win in 3// parallelization through the PROCESS not like you and how you do it all at once on each step before moving on." 4// 5// The constraint: from ONE IP, fetches CANNOT parallelize -- hammering N requests at once from one address gets 6// rate-limited / bot-blocked. So the fetch stage is serial, paced at POLITE_MS (respect the host). Claude wins 7// breadth by fanning out across MANY IPs all-at-once per step (a BARRIER between steps). The team can't. 8// 9// The team's WIN = PIPELINE (process) parallelism: while source i is being EXTRACTED and source i-1 CATALOGED, 10// source i+1 is already FETCHING. The CPU stages overlap UNDER the unavoidable polite fetch pacing, so the only 11// real floor is the pacing itself + one item's tail -- NOT n x (fetch+extract+catalog) as a naive barrier would be. 12// 13// This module computes the wall-clock of three schedules so the policy is a MEASURED fact, not a claim: 14// pf_claude_barrier -- many IPs, fan-out-all-per-step (what Claude does; needs many IPs the team lacks) 15// pf_team_barrier -- one IP, naive: all fetches, THEN all extracts, THEN all catalogs (the WRONG team policy) 16// pf_team_pipeline -- one IP, process-parallel overlap (the RIGHT team policy: hide CPU under fetch pacing) 17// license_tier: ORIGINAL 18import "nx_syscalls.nx" 19 20func pf_max3(a: i64, b: i64, c: i64) -> i64 { 21 var m: i64 = a; if b > m { m = b } if c > m { m = c } return m 22} 23 24// CLAUDE: many IPs -> each STEP runs across all n sources in parallel (cost = the per-item stage time, once), 25// barrier between steps. wall = fetch_latency + extract + catalog. Fast -- but REQUIRES n distinct IPs/agents. 26// (fetch_latency here is the real network round-trip, NOT the polite pacing -- Claude doesn't pace one host.) 27func pf_claude_barrier(fetch_latency: i64, extract_ms: i64, catalog_ms: i64) -> i64 { 28 return fetch_latency + extract_ms + catalog_ms 29} 30 31// TEAM, NAIVE BARRIER (one IP): step1 = all n fetches SERIAL at the polite pace (can't parallelize one IP), 32// step2 = all n extracts, step3 = all n catalogs. wall = n*(pace + extract + catalog). The trap to avoid. 33func pf_team_barrier(n: i64, fetch_pace: i64, extract_ms: i64, catalog_ms: i64) -> i64 { 34 return n * (fetch_pace + extract_ms + catalog_ms) 35} 36 37// TEAM, PIPELINE (one IP, the WIN): items flow through fetch->extract->catalog continuously. The first item 38// pays the full latency (pace+extract+catalog); every subsequent item only adds the BOTTLENECK stage (the 39// slowest stage), because the other stages overlap with it. With polite single-IP pacing, fetch_pace is the 40// bottleneck, so extract+catalog hide entirely under it. wall = (pace+extract+catalog) + (n-1)*max(stages). 41func pf_team_pipeline(n: i64, fetch_pace: i64, extract_ms: i64, catalog_ms: i64) -> i64 { 42 if n <= 0 { return 0 } 43 let bottleneck: i64 = pf_max3(fetch_pace, extract_ms, catalog_ms) 44 return (fetch_pace + extract_ms + catalog_ms) + (n - 1) * bottleneck 45} 46 47// how much the PIPELINE saves over the NAIVE team barrier (the value of process-parallelism on one IP). 48func pf_pipeline_saving(n: i64, fetch_pace: i64, extract_ms: i64, catalog_ms: i64) -> i64 { 49 return pf_team_barrier(n, fetch_pace, extract_ms, catalog_ms) - pf_team_pipeline(n, fetch_pace, extract_ms, catalog_ms) 50} 51 52// the pipeline's wall is bounded BELOW by the polite fetch floor (n paced fetches) -- you cannot beat the pacing 53// from one IP, and the GOAL of the pipeline is to make the CPU stages free so this floor is all you pay. 54func pf_polite_floor(n: i64, fetch_pace: i64) -> i64 { return n * fetch_pace } 55 56// is the pipeline POLITE-OPTIMAL? -- i.e. did we successfully hide all CPU under the pacing, so wall is within 57// one item's tail of the irreducible polite floor? (the team's live-fetch is "right" iff this holds.) 58func pf_is_polite_optimal(n: i64, fetch_pace: i64, extract_ms: i64, catalog_ms: i64) -> i64 { 59 let wall: i64 = pf_team_pipeline(n, fetch_pace, extract_ms, catalog_ms) 60 let floor: i64 = pf_polite_floor(n, fetch_pace) 61 let tail: i64 = extract_ms + catalog_ms 62 if wall <= floor + tail { return 1 } // overlapped: only the floor + one tail remains 63 return 0 64}