nx_nist_grader.nx source

1// nx_nist_grader.nx -- reproducible, LIAR-KILLED ingestion grader for the NIST benchmark. 2// Runs a data-driven CONSUMER-PROBE battery IN-PROCESS against the live seg-store search 3// (dss_search_off_div) and a coverage proxy, then emits a JSON scorecard. Liar-killed BY CONSTRUCTION: 4// - each PASS requires the EXPECTED doc CID to appear in the REAL top-K search results (exact i64 CID 5// match -- no fuzzy self-grading, no asserting); 6// - the battery deliberately INCLUDES known-GAP consumers (CVE records, a constant's numeric value) 7// that NO ingested doc can satisfy (expect_cid=0), so consumability can NOT reach 1000 by testing 8// only easy discovery cases -- the gap stays visible until those asset TYPES are ingested. 9// Runs in nishihost CWD (the dp-<domain>-pub- seg store resolves there). Reuses the exact search the 10// public /search serves. license_tier: ORIGINAL | genealogy_id: nishi_nist_grader_2026_07_14 11// usage: nx_nist_grader [domain] (default nishifamily.com) 12import "nx_syscalls.nx" 13import "nx_docportal_search_seg.nx" 14const K_MAGIC_6693559182638029275: i64 = 6693559182638029275 15const K_MAGIC_5538705301442471477: i64 = 5538705301442471477 16const K_MAGIC_3119289749596666905: i64 = 3119289749596666905 17const K_MAGIC_2030190535770191367: i64 = 2030190535770191367 18const K_MAGIC_1282544026247849879: i64 = 1282544026247849879 19const K_MAGIC_1657827407921962443: i64 = 1657827407921962443 20 21func ng_slen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n } 22func ng_puts(s: *u8) -> i64 { sys_write(1, s, ng_slen(s)); return 0 } 23func ng_putn(v: i64) -> i64 { 24 let t: *u8 = sys_mmap(32) 25 var m: i64 = v 26 var k: i64 = 0 27 if m < 0 { sys_write(1, "-" as *u8, 1); m = 0 - m } 28 if m == 0 { t[0] = 48 as u8; k = 1 } 29 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 } 30 let b: *u8 = sys_mmap(32) 31 var i: i64 = 0 32 while i < k { b[i] = t[k - 1 - i]; i = i + 1 } 33 sys_write(1, b, k) 34 return 0 35} 36 37// run one query, return the 1-based rank of expect_cid in the top-K real results, or 0 if absent. 38func ng_rank(domain: *u8, q: *u8, expect_cid: i64, cids: *i64, scores: *i64, totp: *i64) -> i64 { 39 totp[0] = 0 40 let n: i64 = dss_search_off_div(domain, q, ng_slen(q), cids, scores, 10, 0, totp, 0) 41 var i: i64 = 0 42 while i < n { 43 if cids[i] == expect_cid { return i + 1 } 44 i = i + 1 45 } 46 return 0 47} 48 49// emit one probe record; kind=1 discovery (PASS iff expect_cid in top-10), kind=0 known-GAP (never pass). 50// returns 1 if passed, else 0. 51func ng_probe(domain: *u8, q: *u8, expect_cid: i64, kind: i64, label: *u8, cids: *i64, scores: *i64, totp: *i64, first: i64) -> i64 { 52 if first == 0 { ng_puts("," as *u8) } 53 var rank: i64 = 0 54 if expect_cid != 0 { rank = ng_rank(domain, q, expect_cid, cids, scores, totp) } 55 var pass: i64 = 0 56 if kind == 1 { if rank >= 1 { pass = 1 } } 57 ng_puts("{\"consumer\":\"" as *u8); ng_puts(label) 58 ng_puts("\",\"q\":\"" as *u8); ng_puts(q) 59 ng_puts("\",\"expect_cid\":" as *u8); ng_putn(expect_cid) 60 ng_puts(",\"rank\":" as *u8); ng_putn(rank) 61 ng_puts(",\"kind\":" as *u8); ng_putn(kind) 62 ng_puts(",\"pass\":" as *u8); ng_putn(pass) 63 ng_puts("}" as *u8) 64 return pass 65} 66 67func main(argc: i64, argv: *i64) -> i64 { 68 var domain: *u8 = "nishifamily.com" as *u8 69 if argc > 1 { domain = argv[1] as *u8 } 70 let cids: *i64 = sys_mmap(256) as *i64 71 let scores: *i64 = sys_mmap(256) as *i64 72 let totp: *i64 = sys_mmap(16) as *i64 73 74 // coverage proxy: how many corpus docs carry the NIST/data.nist.gov provenance (BM25-OR total). 75 totp[0] = 0 76 let covq: *u8 = "data.nist.gov NERDm DCAT Standard Reference public data" as *u8 77 dss_search_off_div(domain, covq, ng_slen(covq), cids, scores, 1, 0, totp, 0) 78 let cov: i64 = totp[0] 79 80 ng_puts("{\"grader\":\"nx_nist_grader\",\"v\":1,\"domain\":\"" as *u8); ng_puts(domain) 81 ng_puts("\",\"coverage_proxy_docs\":" as *u8); ng_putn(cov) 82 ng_puts(",\"coverage_denom_rmm\":1680,\"probes\":[" as *u8) 83 84 var passes: i64 = 0 85 var np: i64 = 0 86 // ---- DISCOVERY probes (topic-only queries; expect the exact ingested SRD doc in top-K) ---- 87 passes = passes + ng_probe(domain, "ionic liquid thermodynamic properties measurements database" as *u8, K_MAGIC_6693559182638029275, 1, "researcher" as *u8, cids, scores, totp, 1); np = np + 1 88 passes = passes + ng_probe(domain, "cryogenic material mechanical thermal properties for engineering" as *u8, K_MAGIC_5538705301442471477, 1, "library" as *u8, cids, scores, totp, 0); np = np + 1 89 passes = passes + ng_probe(domain, "structural ceramics material property reference data" as *u8, K_MAGIC_3119289749596666905, 1, "census" as *u8, cids, scores, totp, 0); np = np + 1 90 passes = passes + ng_probe(domain, "steam water ASME thermodynamic properties reference database" as *u8, K_MAGIC_2030190535770191367, 1, "search" as *u8, cids, scores, totp, 0); np = np + 1 91 passes = passes + ng_probe(domain, "thermophysical properties of hydrocarbon fluid mixtures" as *u8, K_MAGIC_1282544026247849879, 1, "researcher2" as *u8, cids, scores, totp, 0); np = np + 1 92 // ---- KNOWN-GAP probes (deep content in OTHER asset types; no ingested doc can answer -> stays a gap) ---- 93 passes = passes + ng_probe(domain, "sendmail debug command execute commands as root vulnerability" as *u8, K_MAGIC_1657827407921962443, 1, "adversary" as *u8, cids, scores, totp, 0); np = np + 1 94 ng_probe(domain, "numerical value of the electron mass in kilograms with uncertainty" as *u8, 0, 0, "critic(needs-CODATA-constants)" as *u8, cids, scores, totp, 0); np = np + 1 95 96 ng_puts("],\"discovery_passes\":" as *u8); ng_putn(passes) 97 ng_puts(",\"probes_total\":" as *u8); ng_putn(np) 98 ng_puts(",\"consumability_permille\":" as *u8); ng_putn(passes * 1000 / np) 99 ng_puts(",\"note\":\"liar-killed: PASS=exact expected CID in real top-K; 2 known-gap probes kept in denominator until NVD/constants ingested\"}" as *u8) 100 ng_puts("\n" as *u8) 101 return 0 102}