nx_library_cache.nx source

1// nx_library_cache.nx -- RESILIENCE: mirror external sources INTO the Nishi Library so that when sites 2// start blocking bot traffic we are not left without the information. The Researcher prefers the LOCAL 3// library; it fetches external only to FILL A GAP, then DOWNLOADS (mirrors) the bytes to disk so the 4// next read is sovereign. Over time we parse more and more of our OWN library and depend on the open 5// web less. Same extractor (nx_research_extract) parses local mirrors and live fetches identically. 6// 7// lc_mirror -- download a fetched source into the library (persist to disk) 8// lc_have_local -- is this source already mirrored? (no network needed) 9// lc_read_mirror -- read a mirrored source back for parsing 10// lc_route -- LOCAL (preferred) / EXTERNAL (fetch+mirror) / BLOCKED (no mirror, unreachable) 11// lc_should_mirror -- FLAG a source to pull into the library (citeable + not yet local) 12// lc_resilience_permil / lc_self_sufficient -- how bot-block-proof our knowledge is 13// license_tier: ORIGINAL 14 15import "nx_syscalls.nx" 16 17const LC_LOCAL: i64 = 1 // mirrored -> answer from the library, zero network 18const LC_EXTERNAL: i64 = 2 // not mirrored but reachable -> fetch THEN mirror 19const LC_BLOCKED: i64 = 3 // not mirrored AND unreachable -> we are starved (this is what mirroring prevents) 20 21// DOWNLOAD a fetched source into the library: write data[0..n) to `path`. Returns bytes written or <0. 22func lc_mirror(path: *u8, data: *u8, n: i64) -> i64 { 23 let fd: i64 = sys_openat_wr(path, 0x1a4) // 0644 24 if fd < 0 { return 0 - 1 } 25 let w: i64 = sys_write(fd, data, n) 26 sys_close(fd) 27 if w != n { return 0 - 2 } 28 return w 29} 30 31// is this source already in the library? open-for-read succeeds => yes. 32func lc_have_local(path: *u8) -> i64 { 33 let fd: i64 = sys_openat_rd(path) 34 if fd < 0 { return 0 } 35 sys_close(fd) 36 return 1 37} 38 39// read a mirrored source back into out[0..cap); returns bytes read or <0 if absent. 40func lc_read_mirror(path: *u8, out: *u8, cap: i64) -> i64 { 41 let fd: i64 = sys_openat_rd(path) 42 if fd < 0 { return 0 - 1 } 43 var off: i64 = 0; var keep: i64 = 1 44 while keep == 1 { 45 if off >= cap { keep = 0 } else { 46 let r: i64 = sys_read(fd, out + off, cap - off) 47 if r <= 0 { keep = 0 } else { off = off + r } 48 } 49 } 50 sys_close(fd) 51 return off 52} 53 54// where should the Researcher get this source? local-first; fetch only to fill a gap; else BLOCKED. 55func lc_route(have_local: i64, external_reachable: i64) -> i64 { 56 if have_local == 1 { return LC_LOCAL } 57 if external_reachable == 1 { return LC_EXTERNAL } 58 return LC_BLOCKED 59} 60 61// FLAG a source to download into the library: it is citeable/high-value and not yet mirrored. 62func lc_should_mirror(have_local: i64, is_citeable: i64) -> i64 { 63 if have_local == 1 { return 0 } 64 if is_citeable == 1 { return 1 } 65 return 0 66} 67 68// of the sources our research depends on, what fraction are mirrored (bot-block-proof)? permil. 69func lc_resilience_permil(total: i64, mirrored: i64) -> i64 { 70 if total <= 0 { return 0 } 71 return (mirrored * 1000) / total 72} 73 74// is the library self-sufficient for this research (every depended-on source has a local mirror)? 75func lc_self_sufficient(total: i64, mirrored: i64) -> i64 { 76 if total > 0 { if mirrored == total { return 1 } } 77 return 0 78}