nx_nvd_ingest.nx source

1// nx_nvd_ingest.nx -- NAS-side API-WRAP ingester for NIST's National Vulnerability Database (NVD REST 2.0). 2// A DIFFERENT asset TYPE from the DCAT catalog pager (nx_nist_ingest): a JSON REST API with nested fields, 3// not HTML/DCAT. Proves the pipeline is NOT overfit to one shape. Fetches 4// services.nvd.nist.gov/rest/json/cves/2.0, splits vulnerabilities[] on {"cve":{"id":", prepends a 5// CVE-id/severity/description title, and dp_ingest's each into the domain's PUBLIC corpus so the CVE id, 6// CVSS severity, description, CPE and references are all BM25-searchable. BOUNDED + HONEST: logs 7// totalResults available vs the slice ingested (no silent truncation). Reuses nx_fetch_any (S1/S2) + 8// dp_ingest. Runs in nishihost CWD (trust store + seg store resolve). Anti-overfit: same fetch+ingest 9// spine, type-appropriate parse -- exactly the ontology pipeline's per-type extractor. 10// usage: nx_nvd_ingest <domain> <count> [startIndex] e.g. nx_nvd_ingest nishifamily.com 500 0 11// license_tier: ORIGINAL | genealogy_id: nishi_nvd_ingest_2026_07_14 12import "nx_syscalls.nx" 13import "nx_x509_trust_store.nx" 14import "nx_trust_store_load_from_certdata.nx" 15import "nx_https_fetch_follow.nx" 16import "nx_fetch_any.nx" 17import "nx_docportal_lib.nx" 18const K_MAGIC_2000: i64 = 2000 19const K_MAGIC_4194304: i64 = 4194304 20const K_MAGIC_16777216: i64 = 16777216 21const K_MAGIC_8192: i64 = 8192 22const K_MAGIC_786432: i64 = 786432 23 24func nv_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 } 25func nv_slen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n } 26func nv_putn(v: i64) -> i64 { 27 let t: *u8 = sys_mmap(32) 28 var m: i64 = v 29 var k: i64 = 0 30 if m < 0 { sys_write(1, "-" as *u8, 1); m = 0 - m } 31 if m == 0 { t[0] = 48 as u8; k = 1 } 32 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 } 33 let b: *u8 = sys_mmap(32) 34 var i: i64 = 0 35 while i < k { b[i] = t[k - 1 - i]; i = i + 1 } 36 sys_write(1, b, k) 37 return 0 38} 39func nv_find(hay: *u8, hl: i64, start: i64, needle: *u8, nl: i64) -> i64 { 40 var i: i64 = start 41 while i + nl <= hl { 42 var j: i64 = 0 43 while j < nl { if hay[i + j] != needle[j] { j = nl + 1 } else { j = j + 1 } } 44 if j == nl { return i } 45 i = i + 1 46 } 47 return 0 - 1 48} 49func nv_atoi(s: *u8) -> i64 { 50 var v: i64 = 0 51 var i: i64 = 0 52 var go: i64 = 1 53 while go == 1 { 54 let c: i64 = s[i] as i64 55 if c < 48 { go = 0 } else { if c > 57 { go = 0 } else { v = v * 10 + (c - 48); i = i + 1 } } 56 } 57 return v 58} 59// read a base-10 integer out of buf starting at pos (skips nothing; stops at first non-digit). 60func nv_readint(buf: *u8, hl: i64, pos: i64) -> i64 { 61 var v: i64 = 0 62 var i: i64 = pos 63 var go: i64 = 1 64 while go == 1 { 65 if i >= hl { go = 0 } else { 66 let c: i64 = buf[i] as i64 67 if c < 48 { go = 0 } else { if c > 57 { go = 0 } else { v = v * 10 + (c - 48); i = i + 1 } } 68 } 69 } 70 return v 71} 72func nv_scat(buf: *u8, o: i64, s: *u8) -> i64 { var i: i64 = 0; while s[i] != (0 as u8) { buf[o + i] = s[i]; i = i + 1 } return o + i } 73func nv_ncat(buf: *u8, o: i64, v: i64) -> i64 { 74 let t: *u8 = sys_mmap(32); var m: i64 = v; var k: i64 = 0 75 if m == 0 { t[0] = 48 as u8; k = 1 } 76 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 } 77 var i: i64 = 0 78 while i < k { buf[o + i] = t[k - 1 - i]; i = i + 1 } 79 return o + k 80} 81// copy src[from..to) into doc[dl..], bounded by doccap; return new dl. 82func nv_copy(doc: *u8, dl: i64, doccap: i64, src: *u8, from: i64, to: i64) -> i64 { 83 var i: i64 = from 84 var d: i64 = dl 85 while i < to { if d >= doccap - 1 { return d } doc[d] = src[i]; d = d + 1; i = i + 1 } 86 return d 87} 88// find `key` in src[start..limit), then copy its quoted value (up to the next ") into doc[dl..]; return new dl. 89func nv_field(doc: *u8, dl: i64, doccap: i64, src: *u8, start: i64, limit: i64, key: *u8, keylen: i64) -> i64 { 90 let p: i64 = nv_find(src, limit, start, key, keylen) 91 if p < 0 { return dl } 92 var i: i64 = p + keylen 93 var d: i64 = dl 94 var go: i64 = 1 95 while go == 1 { 96 if i >= limit { go = 0 } else { 97 if src[i] == (34 as u8) { go = 0 } else { if d < doccap - 1 { doc[d] = src[i]; d = d + 1 } i = i + 1 } 98 } 99 } 100 return d 101} 102 103func nv_ingest_record(domain: *u8, out: *u8, recstart: i64, recend: i64, doc: *u8, doccap: i64, cidp: *i64) -> i64 { 104 var dl: i64 = 0 105 dl = nv_field(doc, dl, doccap, out, recstart, recend, "\"id\":\"" as *u8, 6) // CVE-YYYY-NNNNN 106 dl = nv_scat(doc, dl, " [severity " as *u8) 107 dl = nv_field(doc, dl, doccap, out, recstart, recend, "\"baseSeverity\":\"" as *u8, 16) 108 dl = nv_scat(doc, dl, "] " as *u8) 109 dl = nv_field(doc, dl, doccap, out, recstart, recend, "\"value\":\"" as *u8, 9) // first (en) description 110 doc[dl] = 10 as u8; dl = dl + 1 111 dl = nv_scat(doc, dl, "NIST National Vulnerability Database (NVD) -- services.nvd.nist.gov (CVE / CVSS / CPE).\n" as *u8) 112 dl = nv_copy(doc, dl, doccap, out, recstart, recend) // raw record JSON (bounded) 113 if dl <= 0 { return 0 } 114 dp_ingest(domain, DP_VIS_PUBLIC, doc, dl, cidp) 115 return 1 116} 117 118func main(argc: i64, argv: *i64) -> i64 { 119 if argc < 3 { nv_puts("usage: nx_nvd_ingest <domain> <count> [startIndex]\n" as *u8); return 2 } 120 let domain: *u8 = argv[1] as *u8 121 var count: i64 = nv_atoi(argv[2] as *u8) 122 if count > K_MAGIC_2000 { count = K_MAGIC_2000 } 123 if count < 1 { count = 1 } 124 var start: i64 = 0 125 if argc > 3 { start = nv_atoi(argv[3] as *u8) } 126 127 let r: i64 = nx_trust_store_load_from_certdata("data/mozilla_certdata.txt" as *u8, 512, K_MAGIC_4194304) 128 if r <= 0 { nv_puts("STORE-FAIL: data/mozilla_certdata.txt\n" as *u8); return 1 } 129 let store: *TrustStore = r as *TrustStore 130 131 let cap: i64 = K_MAGIC_16777216 132 let out: *u8 = sys_mmap(cap) 133 let att: *u8 = sys_mmap(K_MAGIC_8192) 134 let status: *i64 = sys_mmap(16) as *i64 135 let empty: *u8 = sys_mmap(8); empty[0] = 0 as u8 136 let urlbuf: *u8 = sys_mmap(512) 137 let doccap: i64 = K_MAGIC_786432 138 let doc: *u8 = sys_mmap(doccap) 139 let cidp: *i64 = sys_mmap(16) as *i64 140 141 var o: i64 = 0 142 o = nv_scat(urlbuf, o, "https://services.nvd.nist.gov/rest/json/cves/2.0?resultsPerPage=" as *u8) 143 o = nv_ncat(urlbuf, o, count) 144 o = nv_scat(urlbuf, o, "&startIndex=" as *u8) 145 o = nv_ncat(urlbuf, o, start) 146 urlbuf[o] = 0 as u8 147 148 status[0] = 0 149 let n: i64 = nx_fetch_any(urlbuf, store, out, cap, 6, status, empty, 0, att, K_MAGIC_8192) 150 if n <= 0 { nv_puts("FETCH-FAIL (see cascade attempts):\n" as *u8); nv_puts(att); return 1 } 151 152 var total: i64 = 0 153 let trp: i64 = nv_find(out, n, 0, "\"totalResults\":" as *u8, 15) 154 if trp >= 0 { total = nv_readint(out, n, trp + 15) } 155 156 let rd: i64 = nv_find(out, n, 0, "\"vulnerabilities\"" as *u8, 17) 157 var ing: i64 = 0 158 if rd >= 0 { 159 var recstart: i64 = nv_find(out, n, rd, "{\"cve\":{\"id\":\"" as *u8, 14) 160 while recstart >= 0 { 161 let nextrec: i64 = nv_find(out, n, recstart + 14, "{\"cve\":{\"id\":\"" as *u8, 14) 162 var recend: i64 = nextrec 163 if recend < 0 { recend = n } 164 ing = ing + nv_ingest_record(domain, out, recstart, recend, doc, doccap, cidp) 165 recstart = nextrec 166 } 167 } else { nv_puts("NO-vulnerabilities-array (status=" as *u8); nv_putn(status[0]); nv_puts(")\n" as *u8) } 168 169 nv_puts("NVD-INGEST totalResults=" as *u8); nv_putn(total) 170 nv_puts(" startIndex=" as *u8); nv_putn(start) 171 nv_puts(" requested=" as *u8); nv_putn(count) 172 nv_puts(" ingested=" as *u8); nv_putn(ing) 173 nv_puts(" (bounded slice; remaining not silently dropped: totalResults-ingested still available via startIndex paging)\n" as *u8) 174 return 0 175}