nx_swebv_research_fetch.nx source

1// nx_swebv_research_fetch.nx -- F787 rung-1: ingest the PUBLIC SWE-bench Verified 500-instance set 2// (contract plane knowledge/store/swebv-: 500 examples MEASURED, revision c104f840 pinned, 13-field 3// schema) as five /rows pages of 100 from the HuggingFace datasets-server, saved to 4// knowledge/fetched/swebv_rows_{000..400}.raw. Named *research_fetch* so the nx_job_run family gate 5// launches it async (each page ~1.5MB > the 64KB MCP envelope; the whole run > the 15s edge window). 6// Mirrors nx_supply_research_fetch (own TLS-1.3 nx_https_fetch_follow + Mozilla CA, idempotent 7// have-skip) with the ff_putn NO-WRITE bug FIXED (seq206 class: the template built digits then 8// returned without sys_write -- this copy reverses + writes them, rule 18 meaningful logging). 9// expect_exit: 0 license_tier: ORIGINAL No hw writes (Rule 26). 10import "nx_syscalls.nx" 11import "nx_x509_trust_store.nx" 12import "nx_trust_store_load_from_certdata.nx" 13import "nx_https_fetch_follow.nx" 14 15const SWF_CAP: i64 = 8388608 16const SWF_ROOTS_MAX: i64 = 512 17const SWF_CERTBUF: i64 = 4194304 18const SWF_HOPS: i64 = 6 19const SWF_MODE: i64 = 0x1a4 20const SWF_D0: i64 = 48 21const SWF_B10: i64 = 10 22const SWF_NUMB: i64 = 24 23 24func ff_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 } 25func ff_putn(v: i64) -> i64 { 26 if v == 0 { sys_write(1, "0" as *u8, 1); return 0 } 27 var m: i64 = v 28 if m < 0 { sys_write(1, "-" as *u8, 1); m = 0 - m } 29 let d: *u8 = sys_mmap(SWF_NUMB) 30 var k: i64 = 0 31 while m > 0 { d[k] = (SWF_D0 + (m % SWF_B10)) as u8; m = m / SWF_B10; k = k + 1 } 32 let o: *u8 = sys_mmap(SWF_NUMB) 33 var i: i64 = 0 34 while i < k { o[i] = d[k - 1 - i]; i = i + 1 } 35 sys_write(1, o, k) 36 return 0 37} 38func have_file(path: *u8) -> i64 { 39 let fd: i64 = sys_openat_rd(path) 40 if fd < 0 { return 0 } 41 sys_close(fd) 42 return 1 43} 44func fetch_save(url: *u8, opath: *u8, store: *TrustStore, out: *u8, cap: i64) -> i64 { 45 if have_file(opath) == 1 { ff_puts(opath); ff_puts(" [have-skip]\n" as *u8); return 1 } 46 let status: *i64 = sys_mmap(8) as *i64 47 let n: i64 = nx_https_fetch_follow(url, store, out, cap, SWF_HOPS, status) 48 ff_puts(url); ff_puts(" status=" as *u8); ff_putn(status[0]); ff_puts(" bytes=" as *u8); ff_putn(n) 49 if n <= 0 { ff_puts(" FETCH-FAIL\n" as *u8); return 0 } 50 var gz: i64 = 0 51 if n >= 2 { if out[0] == 0x1f as u8 { if out[1] == 0x8b as u8 { gz = 1 } } } 52 if gz == 1 { ff_puts(" [GZIP-skip]\n" as *u8); return 0 } 53 let fd: i64 = sys_openat_wr(opath, SWF_MODE) 54 if fd < 0 { ff_puts(" SAVE-FAIL\n" as *u8); return 0 } 55 sys_write(fd, out, n) 56 sys_close(fd) 57 ff_puts(" SAVED\n" as *u8) 58 return 1 59} 60 61func main() -> i64 { 62 let r: i64 = nx_trust_store_load_from_certdata("data/mozilla_certdata.txt" as *u8, SWF_ROOTS_MAX, SWF_CERTBUF) 63 if r <= 0 { ff_puts("SWEBV: certdata load failed\n" as *u8); return 1 } 64 let store: *TrustStore = r as *TrustStore 65 ff_puts("SWEBV ingest: CA roots=" as *u8); ff_putn(trust_store_count(store)); ff_puts("\n" as *u8) 66 let out: *u8 = sys_mmap(SWF_CAP) 67 var ok: i64 = 0 68 ok = ok + fetch_save("https://datasets-server.huggingface.co/rows?dataset=princeton-nlp/SWE-bench_Verified&config=default&split=test&offset=0&length=100" as *u8, "knowledge/fetched/swebv_rows_000.raw" as *u8, store, out, SWF_CAP) 69 ok = ok + fetch_save("https://datasets-server.huggingface.co/rows?dataset=princeton-nlp/SWE-bench_Verified&config=default&split=test&offset=100&length=100" as *u8, "knowledge/fetched/swebv_rows_100.raw" as *u8, store, out, SWF_CAP) 70 ok = ok + fetch_save("https://datasets-server.huggingface.co/rows?dataset=princeton-nlp/SWE-bench_Verified&config=default&split=test&offset=200&length=100" as *u8, "knowledge/fetched/swebv_rows_200.raw" as *u8, store, out, SWF_CAP) 71 ok = ok + fetch_save("https://datasets-server.huggingface.co/rows?dataset=princeton-nlp/SWE-bench_Verified&config=default&split=test&offset=300&length=100" as *u8, "knowledge/fetched/swebv_rows_300.raw" as *u8, store, out, SWF_CAP) 72 ok = ok + fetch_save("https://datasets-server.huggingface.co/rows?dataset=princeton-nlp/SWE-bench_Verified&config=default&split=test&offset=400&length=100" as *u8, "knowledge/fetched/swebv_rows_400.raw" as *u8, store, out, SWF_CAP) 73 ff_puts("SWEBV ingest done ok=" as *u8); ff_putn(ok); ff_puts("/5 (pages of 100, revision-pinned contract in knowledge/store/swebv-)\n" as *u8) 74 if ok == 5 { return 0 } 75 return 2 76}