nx_bulk_ingest_test.nx source

1// nx_bulk_ingest_test.nx -- proves the polite-fetch -> bulk-catalog hand-off on a realistic arXiv-style scenario: 2// the team's polite fetch lands on a 500-entry listing; it DETECTS the repository, hands off to BULK only when 3// it has polite spare capacity (else DEFER), harvests as a POLITE pipeline (never undercuts the floor), 4// content-dedups already-mirrored docs, GROWS the Library so self-serve coverage jumps, and CONVERGES when the 5// catalog is exhausted (never infinite). Exit 0 iff every property holds. 6import "nx_bulk_ingest.nx" 7import "nx_syscalls.nx" 8 9func tp(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 } 10func tn(v: i64) -> i64 { let bb: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m;sys_write(1,"-" as *u8,1)}; let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=48;k=1}; while m>0{t[k]=48+(m%10);m=m/10;k=k+1}; var i: i64=0; while i<k{bb[i]=t[k-1-i];i=i+1}; sys_write(1,bb,k); return 0 } 11 12func main() -> i64 { 13 tp("=== POLITE-FETCH -> BULK-CATALOG HAND-OFF (grow the Library, stay polite) ===\n" as *u8) 14 15 // (a) DETECT: an arXiv-style listing -- 500 links, 940permil share the /abs/ pattern -> a repository. 16 let listing_repo: i64 = bi_is_repository(500, 940, 0) 17 // an ordinary article page -- 12 links, low pattern density -> NOT a repository. 18 let article_repo: i64 = bi_is_repository(12, 200, 0) 19 // an OAI-PMH endpoint -> definitively a repository regardless of link shape. 20 let oai_repo: i64 = bi_is_repository(0, 0, 1) 21 tp(" detect: arxiv-listing=" as *u8); tn(listing_repo); tp(" article=" as *u8); tn(article_repo); tp(" OAI-PMH=" as *u8); tn(oai_repo); tp("\n" as *u8) 22 23 // (b) HAND-OFF: with 60% spare polite capacity -> BULK; if only 10% spare -> DEFER (don't overwhelm). 24 let ho_ok: i64 = bi_handoff(listing_repo, 600) 25 let ho_busy: i64 = bi_handoff(listing_repo, 100) 26 let ho_plain: i64 = bi_handoff(article_repo, 600) 27 tp(" handoff: capacity60%=" as *u8); tn(ho_ok); tp("(1=BULK) capacity10%=" as *u8); tn(ho_busy); tp("(2=DEFER) article=" as *u8); tn(ho_plain); tp("(0=SINGLE)\n" as *u8) 28 29 // (c) PRIORITY: a repository that fills a RAISED-HAND assignment outranks a generic one. 30 let pr_match: i64 = bi_priority(1, 500) 31 let pr_generic: i64 = bi_priority(0, 500) 32 tp(" priority: raised-hand-match=" as *u8); tn(pr_match); tp(" generic=" as *u8); tn(pr_generic); tp("\n" as *u8) 33 34 // (d) POLITE HARVEST: 500 docs, one IP, 800ms pace, extract 120ms, mirror 40ms. Bulk = CONTINUOUS, not rude. 35 let wall: i64 = bi_harvest_wall_ms(500, 800, 120, 40) 36 let polite: i64 = bi_harvest_is_polite(500, 800, 120, 40) 37 tp(" harvest 500 docs: wall=" as *u8); tn(wall); tp("ms stays-polite(>=floor)=" as *u8); tn(polite); tp("\n" as *u8) 38 39 // (e) DEDUP: 50 of the 500 already mirrored -> skipped (content-addressed). seen={hashes}. 40 let seen: *i64 = sys_mmap(8*4) as *i64; seen[0]=111; seen[1]=222; seen[2]=333; seen[3]=444 41 let skip_known: i64 = bi_dedup_skip(222, seen, 4) 42 let skip_new: i64 = bi_dedup_skip(999, seen, 4) 43 tp(" dedup: already-have-222=" as *u8); tn(skip_known); tp("(1=skip) new-999=" as *u8); tn(skip_new); tp("(0=mirror)\n" as *u8) 44 45 // (f) GROWTH: Library had 4 docs serving a 500-doc need (8permil); after harvesting 450 new -> 454/500. 46 let cov_before: i64 = bi_self_serve_pmil(4, 500) 47 let cov_after: i64 = bi_self_serve_pmil(454, 500) 48 tp(" self-serve coverage: before=" as *u8); tn(cov_before); tp("permil after-harvest=" as *u8); tn(cov_after); tp("permil\n" as *u8) 49 50 // (g) CONVERGENCE: still finding -> continue; K dry rounds -> done; hard cap -> done (never infinite). 51 let conv_more: i64 = bi_converged(30, 0, 2, 5, 1000) // found 30 new -> keep going 52 let conv_dry: i64 = bi_converged(0, 2, 2, 9, 1000) // 2 dry rounds -> catalog exhausted 53 let conv_cap: i64 = bi_converged(5, 0, 2, 1000, 1000) // hit hard cap -> stop anyway 54 tp(" converge: finding=" as *u8); tn(conv_more); tp("(0=go) 2-dry=" as *u8); tn(conv_dry); tp("(1=done) hard-cap=" as *u8); tn(conv_cap); tp("(1=done)\n" as *u8) 55 56 // verdict 57 var ok: i64 = 1 58 if listing_repo != 1 { ok = 0 } 59 if article_repo != 0 { ok = 0 } 60 if oai_repo != 1 { ok = 0 } 61 if ho_ok != BI_BULK { ok = 0 } 62 if ho_busy != BI_DEFER { ok = 0 } 63 if ho_plain != BI_SINGLE { ok = 0 } 64 if pr_match <= pr_generic { ok = 0 } 65 if polite != 1 { ok = 0 } 66 if skip_known != 1 { ok = 0 } 67 if skip_new != 0 { ok = 0 } 68 if cov_after <= cov_before { ok = 0 } 69 if conv_more != 0 { ok = 0 } 70 if conv_dry != 1 { ok = 0 } 71 if conv_cap != 1 { ok = 0 } 72 tp("----\n" as *u8) 73 if ok == 1 { 74 tp(" PROVEN: the team detects a harvestable repository, hands off to a POLITE bulk walk (never rude),\n" as *u8) 75 tp(" dedups what it already has, GROWS the Library 8permil -> 908permil self-serve, and CONVERGES (never infinite).\n" as *u8) 76 sys_exit(0); return 0 77 } 78 tp(" FAIL\n" as *u8); sys_exit(1); return 1 79}