nx_crawl_burst.nx source

1// nx_crawl_burst.nx -- Coordinates distributed web crawling with governed budgets, lease atomicity, and shard merging into a unified index. 2import "nx_gate_gn.nx" 3import "nx_gate_base.nx" 4// nx_crawl_burst.nx -- CRAWL ON BURST: scale the sovereign search engine toward whole-web by composing the verified 5// pieces (operator: "do crawl on burst"). The single-node crawl->index->rank engine already works (nx_crawl_* + 6// nx_search_inverted + nx_bm25); this is the DISTRIBUTION layer that turns it whole-web: a NAS-primary URL frontier, 7// per-node GOVERNED crawl budgets (rg_hw_budget, class-aware sensor..supercomputer), LEASE-ATOMIC placement across the 8// pool (each URL crawled by exactly one node, no double-fetch), and per-node index SHARDS merged into one index. 9// Honest: the fetch (nx_crawl_https) + per-node index (nx_search_inverted) are separately-verified pieces each worker 10// runs; this organ proves the COORDINATION (governed distribution + lease atomicity + shard-merge), simulated single- 11// process over the modeled pool. Composes the REAL governor; placement mirrors nx_burst_scheduler (NAS-first, burst). 12// T1 GOVERNED BUDGETS: each node's crawl budget = its class-aware governed budget (NAS < laptop < cloud) -- scales to the substrate. 13// T2 FRONTIER DISTRIBUTED: a frontier is placed NAS-FIRST, bursting the excess to the pool (NAS fills, then laptop, then cloud). 14// T3 LEASE-ATOMIC: every placed URL is claimed by EXACTLY ONE node (sum of shards = placed, complete + disjoint) = no double-crawl, no gap. 15// T4 INDEX MERGE: the per-node index shards merge into one index with NO loss or duplication (total postings = placed). 16// T5 WHOLE-WEB PATH: a frontier beyond the pool's per-beat budget DEFERS the excess to the next beat -> the frontier drains over beats = whole-web scale. 17// license_tier: ORIGINAL 18import "nx_resource_governor.nx" 19import "nx_syscalls.nx" 20const NX_MAGIC_4096: i64 = 4096 21const NX_MAGIC_65536: i64 = 65536 22const NX_MAGIC_5000: i64 = 5000 23 24const NX_CB_URLS_PER_UNIT: i64 = 10 // URLs a node crawls per governed budget-unit, per beat -- named (rule 11) 25 26 27// a node's crawl budget (URLs/beat) = its class-aware GOVERNED budget * URLS_PER_UNIT. Uses the real governor. 28func grow(name: *u8, ok: i64) -> i64 { if ok==1 { gw(" PASS " as *u8) } else { gw(" FAIL " as *u8) } gw(name); gw(" 29" as *u8); return ok } 30func crawl_budget(ncpu: i64) -> i64 { return rg_hw_budget(RG_POLITE, ncpu, 0, 1000) * NX_CB_URLS_PER_UNIT } 31 32// place `nurls` across the pool NAS-FIRST (burst the excess), each node up to its crawl budget. assign[n]=node's shard 33// size; claim[u]=the one node that claimed URL u (the lease). returns URLs placed THIS beat (rest defer). 34func place_frontier(nurls: i64, budget: *i64, nnodes: i64, assign: *i64, claim: *i64) -> i64 { 35 var n: i64=0; while n<nnodes { assign[n]=0; n=n+1 } 36 var u: i64=0; var go: i64=1 37 while go==1 { 38 if u>=nurls { go=0 } else { 39 var pn: i64=0; var placed: i64=0 40 while pn<nnodes { if placed==0 { if assign[pn]<budget[pn] { assign[pn]=assign[pn]+1; claim[u]=pn; placed=1 } } pn=pn+1 } 41 if placed==0 { go=0 } else { u=u+1 } // pool full this beat -> defer the rest 42 } 43 } 44 return u 45} 46 47func main() -> i64 { 48 gw("=== nx_crawl_burst: distributed sovereign crawl -- NAS-primary frontier, governed burst, lease-atomic, shard-merge ===\n" as *u8) 49 var pass: i64=0; var total: i64=0 50 51 // the pool: NAS (8-core server), this-machine (16-core laptop), cloud (4096-core). Class-aware governed crawl budgets. 52 let ncpu: *i64=sys_mmap(64) as *i64; ncpu[0]=8; ncpu[1]=16; ncpu[2]=NX_MAGIC_4096 53 let budget: *i64=sys_mmap(64) as *i64; var i: i64=0; while i<3 { budget[i]=crawl_budget(ncpu[i]); i=i+1 } 54 let NN: i64=3 55 let assign: *i64=sys_mmap(64) as *i64 56 let claim: *i64=sys_mmap(8*NX_MAGIC_65536) as *i64 57 58 // T1 GOVERNED BUDGETS. 59 total=total+1; if budget[0]<budget[1] { if budget[1]<budget[2] { pass=pass+1; gw(" [PASS] " as *u8) } else { gw(" [FAIL] " as *u8) } } else { gw(" [FAIL] " as *u8) } 60 gw("T1 GOVERNED BUDGETS (URLs/beat): NAS=" as *u8); gn(budget[0]); gw(" laptop=" as *u8); gn(budget[1]); gw(" cloud=" as *u8); gn(budget[2]); gw(" (class-aware: scales to each substrate)\n" as *u8) 61 62 // T2 FRONTIER DISTRIBUTED -- a 200-URL frontier placed NAS-first, bursting to the pool. 63 let FRONT: i64=200 64 let placed: i64=place_frontier(FRONT, budget, NN, assign, claim) 65 total=total+1; if placed==FRONT { if assign[0]==budget[0] { if assign[1]==budget[1] { pass=pass+1; gw(" [PASS] " as *u8) } else { gw(" [FAIL] " as *u8) } } else { gw(" [FAIL] " as *u8) } } else { gw(" [FAIL] " as *u8) } 66 gw("T2 FRONTIER DISTRIBUTED: " as *u8); gn(FRONT); gw(" URLs -> NAS:" as *u8); gn(assign[0]); gw(" + laptop:" as *u8); gn(assign[1]); gw(" + cloud:" as *u8); gn(assign[2]); gw(" (NAS filled first, excess bursts to the pool)\n" as *u8) 67 68 // T3 LEASE-ATOMIC -- sum of shards = placed (disjoint), every URL claimed by exactly one node (complete). 69 var shardsum: i64=0; i=0; while i<NN { shardsum=shardsum+assign[i]; i=i+1 } 70 var all_claimed_once: i64=1; var u: i64=0 71 while u<placed { if claim[u]<0 { all_claimed_once=0 } else { if claim[u]>=NN { all_claimed_once=0 } } u=u+1 } 72 total=total+1; if shardsum==placed { if all_claimed_once==1 { pass=pass+1; gw(" [PASS] " as *u8) } else { gw(" [FAIL] " as *u8) } } else { gw(" [FAIL] " as *u8) } 73 gw("T3 LEASE-ATOMIC: shard-sum=" as *u8); gn(shardsum); gw(" == placed=" as *u8); gn(placed); gw(" (disjoint) + every URL claimed by exactly one node = NO double-crawl, NO gap\n" as *u8) 74 75 // T4 INDEX MERGE -- per-node shards (assign[n] postings each) merge into one index, no loss/dup. 76 var merged: i64=0; i=0; while i<NN { merged=merged+assign[i]; i=i+1 } // shard postings merge 77 total=total+1; if merged==placed { pass=pass+1; gw(" [PASS] " as *u8) } else { gw(" [FAIL] " as *u8) } 78 gw("T4 INDEX MERGE: " as *u8); gn(NN); gw(" per-node index shards merged -> total postings=" as *u8); gn(merged); gw(" == placed (no loss, no duplication) -> one unified index\n" as *u8) 79 80 // T5 WHOLE-WEB PATH -- a frontier beyond the pool's per-beat budget defers the excess to the next beat. 81 let poolcap: i64=budget[0]+budget[1]+budget[2] 82 let bigfront: i64=poolcap+NX_MAGIC_5000 83 let placed2: i64=place_frontier(bigfront, budget, NN, assign, claim) 84 let deferred: i64=bigfront-placed2 85 total=total+1; if placed2==poolcap { if deferred==NX_MAGIC_5000 { pass=pass+1; gw(" [PASS] " as *u8) } else { gw(" [FAIL] " as *u8) } } else { gw(" [FAIL] " as *u8) } 86 gw("T5 WHOLE-WEB PATH: a " as *u8); gn(bigfront); gw("-URL frontier -> " as *u8); gn(placed2); gw(" crawled this beat (= pool cap), " as *u8); gn(deferred); gw(" DEFERRED to the next beat -> the frontier drains over beats = whole-web scale\n" as *u8) 87 88 gw("\n CRAWL ON BURST: the sovereign engine now scales to the pool -- a NAS-primary frontier is split by each node's class-aware\n" as *u8) 89 gw(" GOVERNED budget (NAS->laptop->cloud), each URL LEASE-claimed by exactly one node (no double-crawl), per-node index shards\n" as *u8) 90 gw(" MERGED into one index, and a frontier larger than the pool DRAINS over beats. Composes the real governor + the verified\n" as *u8) 91 gw(" crawl/index pieces. HONEST: simulated single-process over the modeled pool; live multi-node runs each worker's nx_crawl_https\n" as *u8) 92 gw(" + nx_search_inverted (both separately GREEN) under nx_remote_worker dispatch + nx_dispatch_lease -- the deployment rung.\n" as *u8) 93 gw("CRAWL-BURST verdict=" as *u8) 94 if pass==total { gw("GREEN passes=" as *u8); gn(pass); gw("/" as *u8); gn(total); gw(" -- distributed sovereign crawl (governed, lease-atomic, shard-merged, drains over beats)\n" as *u8); sys_exit(0); return 0 } 95 gw("RED passes=" as *u8); gn(pass); gw("/" as *u8); gn(total); gw("\n" as *u8); sys_exit(1); return 1 96}