nx_qabench_corpus_fetch.nx source
↩ module page · 42 lines · 2802 B
1// nx_qabench_corpus_fetch.nx -- GROW the PPMI corpus ~6x with rows DISJOINT from the eval subsets (eval =
2// squad/hotpot pages 0-2, musique pages 0-1; corpus chunks start BEYOND those). PPMI quality scales with
3// corpus; the ladder eval sets stay untouched = a clean corpus-scaling datapoint vs the 161/153/37 baseline.
4// Chunks of 5 pages (500 rows, ~3-6MB each, under the 16MB reader cap). Idempotent + atomic per chunk.
5// Run SOLO. expect_exit: 0 license_tier: ORIGINAL
6import "nx_research_fetch_ext.nx"
7
8func main() -> i64 {
9 let store: *TrustStore = rf_init()
10 if (store as i64) == 0 { rf_puts("qcorpus: trust store load failed\n" as *u8); return 1 }
11 rf_puts("CA roots="); rf_putn(trust_store_count(store)); rf_puts(" -- corpus growth (disjoint from eval): +1500 squad +1500 hotpot +1000 musique rows\n" as *u8)
12 let SQ: *u8 = "https://datasets-server.huggingface.co/rows?dataset=rajpurkar/squad&config=plain_text&split=validation" as *u8
13 let HP: *u8 = "https://datasets-server.huggingface.co/rows?dataset=hotpotqa/hotpot_qa&config=distractor&split=validation" as *u8
14 let MQ: *u8 = "https://datasets-server.huggingface.co/rows?dataset=dgslibisey/MuSiQue&config=default&split=validation" as *u8
15
16 rf_section("SQuAD corpus chunks (pages 3..17)" as *u8)
17 rf_fetch_bank_paged_from(SQ, "qab_squad_c0" as *u8, store, 100, 5, 3)
18 rf_fetch_bank_paged_from(SQ, "qab_squad_c1" as *u8, store, 100, 5, 8)
19 rf_fetch_bank_paged_from(SQ, "qab_squad_c2" as *u8, store, 100, 5, 13)
20
21 // R3f neural-reader DATA-SCALE lever (2026-07-10): +3000 SQuAD rows (pages 18..47; validation has ~105
22 // pages, eval uses 0-2, all chunks disjoint). Idempotent banked-skip -> re-runs cost nothing.
23 rf_section("SQuAD XL chunks for the neural reader (pages 18..47)" as *u8)
24 rf_fetch_bank_paged_from(SQ, "qab_squad_c3" as *u8, store, 100, 5, 18)
25 rf_fetch_bank_paged_from(SQ, "qab_squad_c4" as *u8, store, 100, 5, 23)
26 rf_fetch_bank_paged_from(SQ, "qab_squad_c5" as *u8, store, 100, 5, 28)
27 rf_fetch_bank_paged_from(SQ, "qab_squad_c6" as *u8, store, 100, 5, 33)
28 rf_fetch_bank_paged_from(SQ, "qab_squad_c7" as *u8, store, 100, 5, 38)
29 rf_fetch_bank_paged_from(SQ, "qab_squad_c8" as *u8, store, 100, 5, 43)
30
31 rf_section("HotpotQA corpus chunks (pages 3..17)" as *u8)
32 rf_fetch_bank_paged_from(HP, "qab_hotpot_c0" as *u8, store, 100, 5, 3)
33 rf_fetch_bank_paged_from(HP, "qab_hotpot_c1" as *u8, store, 100, 5, 8)
34 rf_fetch_bank_paged_from(HP, "qab_hotpot_c2" as *u8, store, 100, 5, 13)
35
36 rf_section("MuSiQue corpus chunks (pages 2..11)" as *u8)
37 rf_fetch_bank_paged_from(MQ, "qab_musique_c0" as *u8, store, 100, 5, 2)
38 rf_fetch_bank_paged_from(MQ, "qab_musique_c1" as *u8, store, 100, 5, 7)
39
40 rf_puts("QCORPUS done\n" as *u8)
41 return 0
42}