nx_langintel_gh_fetch.nx source

1// nx_langintel_gh_fetch.nx v2 -- LangIntel FETCH stage, FULL-SCALE PAGINATION (no sampling): loops 2// GitHub pages (base_url + &page=N) forking nx_https_get each, accumulates every page body (sliced from 3// the first '['), and STOPS when a response has no Link rel="next". Writes the whole concatenation to a 4// .raw (the anchor-based ingest parser scans across page seams -- concatenated arrays are fine). 5// Composes nx_https_get via dep_run_capture. ORACLE/bench-only (F616). Rule 26. 6// argv: <base-url-with-per_page-but-no-page> <out-raw-path> expect_exit: 0 license_tier: ORACLE-FETCH 7import "nx_ingest_base.nx" 8import "nx_deploy_lib.nx" 9import "nx_syscalls.nx" 10 11const GF_STDERR: i64 = 2 12const GF_CAP: i64 = 16777216 13const GF_ACC: i64 = 16777216 14const GF_MODE: i64 = 420 15const GF_MAXPAGE: i64 = 60 16const GF_URLCAP: i64 = 2048 17const GF_ARGC_MIN: i64 = 3 18const GF_EXIT_USAGE: i64 = 2 19const GF_EXIT_IO: i64 = 1 20const GF_EXIT_RED: i64 = 3 21const GF_HTTPS_ELF: *u8 = "/volume1/homes/elderwesto/nishihost/nx_https_get.elf" as *u8 22const GF_TMP: *u8 = "/tmp/langintel_ghfetch.out" as *u8 23const GF_LBRACK: *u8 = "[" as *u8 24const GF_PAGEARG: *u8 = "&page=" as *u8 25const GF_NEXT: *u8 = "rel=\"next\"" as *u8 26 27func main(argc: i64, argv: *i64) -> i64 { 28 if argc < GF_ARGC_MIN { 29 ib_refuse("LANGINTEL-GH-FETCH" as *u8, "missing args <base-url> <out-raw-path>" as *u8, "seed a plan row: 1 TAB nx_langintel_gh_fetch TAB <github-api-url-with-per_page=100-no-page> TAB knowledge/fetched/<name>.raw" as *u8) 30 sys_exit(GF_EXIT_USAGE) 31 return GF_EXIT_USAGE 32 } 33 let base_url: *u8 = argv[1] as *u8 34 let out: *u8 = argv[2] as *u8 35 let acc: *u8 = sys_mmap(GF_ACC) 36 let buf: *u8 = sys_mmap(GF_CAP) 37 let urlb: *u8 = sys_mmap(GF_URLCAP) 38 let av: *i64 = sys_mmap(16) as *i64 39 var ao: i64 = 0 40 var page: i64 = 1 41 var pages: i64 = 0 42 var go: i64 = 1 43 while go == 1 { 44 if page > GF_MAXPAGE { go = 0 } 45 else { 46 var uo: i64 = ib_bput(urlb, 0, base_url) 47 uo = ib_bput(urlb, uo, GF_PAGEARG) 48 uo = ib_bputn(urlb, uo, page) 49 urlb[uo] = 0 as u8 50 av[0] = urlb as i64 51 let rc: i64 = dep_run_capture(GF_HTTPS_ELF, av, 1, GF_TMP) 52 let n: i64 = ib_read(GF_TMP, buf, GF_CAP) 53 if n <= 0 { go = 0 } 54 else { 55 let bs: i64 = ib_find(buf, n, GF_LBRACK, 0) 56 if bs < 0 { go = 0 } 57 else { 58 var j: i64 = bs 59 while j < n { if ao < GF_ACC - 2 { acc[ao] = buf[j]; ao = ao + 1 } j = j + 1 } 60 pages = pages + 1 61 let hasnext: i64 = ib_find(buf, bs, GF_NEXT, 0) 62 if hasnext < 0 { go = 0 } else { page = page + 1 } 63 } 64 } 65 } 66 } 67 if ao <= 0 { 68 ib_refuse("LANGINTEL-GH-FETCH" as *u8, "no data fetched across pages" as *u8, "check network + URL + X-RateLimit-Remaining; the sovereign fetch writes to /tmp" as *u8) 69 sys_exit(GF_EXIT_IO) 70 return GF_EXIT_IO 71 } 72 let fd: i64 = sys_openat_wr(out, GF_MODE) 73 if fd < 0 { 74 ib_refuse("LANGINTEL-GH-FETCH" as *u8, "cannot open out-raw for write" as *u8, "verify the out dir exists + is writable (knowledge/fetched/)" as *u8) 75 sys_exit(GF_EXIT_IO) 76 return GF_EXIT_IO 77 } 78 var w: i64 = 0 79 var gw: i64 = 1 80 while gw == 1 { 81 let base: i64 = acc as i64 82 let r: i64 = sys_write(fd, (base + w) as *u8, ao - w) 83 if r <= 0 { gw = 0 } else { w = w + r } 84 if w >= ao { gw = 0 } 85 } 86 sys_close(fd) 87 ib_werr("LANGINTEL-GH-FETCH OK verdict=GREEN pages=" as *u8) 88 ib_wn(GF_STDERR, pages) 89 ib_werr(" body_bytes=" as *u8) 90 ib_wn(GF_STDERR, w) 91 ib_werr(" out=" as *u8) 92 ib_werr(out) 93 ib_werr("\n" as *u8) 94 sys_exit(0) 95 return 0 96}