code wiki / _hdl_build / nx_nist_ingest.nx
`nx_nist_ingest.nx` source

↩ module page · 175 lines · 7226 B
1// nx_nist_ingest.nx -- NAS-side STRUCTURED ingester for NIST's data.nist.gov RMM DCAT catalog.
2// Pages the RMM API (sovereign fetch cascade), splits ResultData[] into per-dataset NERDm records,
3// prepends a clean title line, and ingests each into the domain's PUBLIC seg_store corpus via
4// dp_ingest -- the IDENTICAL path the docportal admin ingest uses (auto .terms BM25 => instantly
5// searchable by nishi_search / researcher / census). Idempotent (segid=CID): re-ingest is a no-op.
6//
7// Runs in the tools-daemon (nishihost) CWD: data/mozilla_certdata.txt AND knowledge/store/ both resolve.
8// Reuses nx_fetch_any (S1 sovereign TLS cascade -> S2 Firefox-spoof) + dp_ingest. Nothing hand-rolled
9// that already exists. Anti-overfit: the record source is DCAT/schema.org (standard), so the same
10// splitter generalizes to any DCAT catalog (data.gov, EU portals).
11//
12// usage: nx_nist_ingest <domain> <pages> [size]     e.g.  nx_nist_ingest nishifamily.com 3 100
13// license_tier: ORIGINAL | genealogy_id: nishi_nist_ingest_2026_07_14
14import "nx_syscalls.nx"
15import "nx_x509_trust_store.nx"
16import "nx_trust_store_load_from_certdata.nx"
17import "nx_https_fetch_follow.nx"
18import "nx_fetch_any.nx"
19import "nx_docportal_lib.nx"
20const K_MAGIC_4194304: i64 = 4194304
21const K_MAGIC_8388608: i64 = 8388608
22const K_MAGIC_8192: i64 = 8192
23const K_MAGIC_524288: i64 = 524288
24
25func ni_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
26
27func ni_putn(v: i64) -> i64 {
28    let t: *u8 = sys_mmap(32)
29    var m: i64 = v
30    var k: i64 = 0
31    if m < 0 { sys_write(1, "-" as *u8, 1); m = 0 - m }
32    if m == 0 { t[0] = 48 as u8; k = 1 }
33    while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
34    let b: *u8 = sys_mmap(32)
35    var i: i64 = 0
36    while i < k { b[i] = t[k - 1 - i]; i = i + 1 }
37    sys_write(1, b, k)
38    return 0
39}
40
41// substring search in hay[0..hl) starting at `start`; returns index or -1.
42func ni_find(hay: *u8, hl: i64, start: i64, needle: *u8, nl: i64) -> i64 {
43    var i: i64 = start
44    while i + nl <= hl {
45        var j: i64 = 0
46        while j < nl {
47            if hay[i + j] != needle[j] { j = nl + 1 } else { j = j + 1 }
48        }
49        if j == nl { return i }
50        i = i + 1
51    }
52    return 0 - 1
53}
54
55// parse a base-10 integer prefix of s (stops at first non-digit / NUL).
56func ni_atoi(s: *u8) -> i64 {
57    var v: i64 = 0
58    var i: i64 = 0
59    var go: i64 = 1
60    while go == 1 {
61        let c: i64 = s[i] as i64
62        if c < 48 { go = 0 } else { if c > 57 { go = 0 } else { v = v * 10 + (c - 48); i = i + 1 } }
63    }
64    return v
65}
66
67// append NUL-terminated string s to buf at offset o; return new offset.
68func ni_scat(buf: *u8, o: i64, s: *u8) -> i64 {
69    var i: i64 = 0
70    while s[i] != (0 as u8) { buf[o + i] = s[i]; i = i + 1 }
71    return o + i
72}
73
74// append decimal v to buf at offset o; return new offset.
75func ni_ncat(buf: *u8, o: i64, v: i64) -> i64 {
76    let t: *u8 = sys_mmap(32)
77    var m: i64 = v
78    var k: i64 = 0
79    if m == 0 { t[0] = 48 as u8; k = 1 }
80    while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
81    var i: i64 = 0
82    while i < k { buf[o + i] = t[k - 1 - i]; i = i + 1 }
83    return o + k
84}
85
86func ni_build_url(buf: *u8, size: i64, page: i64) -> i64 {
87    var o: i64 = 0
88    o = ni_scat(buf, o, "https://data.nist.gov/rmm/records?size=" as *u8)
89    o = ni_ncat(buf, o, size)
90    o = ni_scat(buf, o, "&page=" as *u8)
91    o = ni_ncat(buf, o, page)
92    buf[o] = 0 as u8
93    return o
94}
95
96// Ingest one record chunk out[recstart..recend). Prepends a clean "<title>" line (extracted from the
97// record's "title":"..." field) + a provenance line, then the raw NERDm JSON, so the search title reads
98// well and every structured field is BM25-indexed. Returns 1 if ingested, 0 if skipped.
99func ni_ingest_record(domain: *u8, out: *u8, hl: i64, recstart: i64, recend: i64, doc: *u8, cidp: *i64) -> i64 {
100    var dl: i64 = 0
101    // title (search-first): "title":"  ...value...  "
102    let tpos: i64 = ni_find(out, recend, recstart, "\"title\":\"" as *u8, 9)
103    if tpos >= 0 {
104        var ti: i64 = tpos + 9
105        var go: i64 = 1
106        while go == 1 {
107            if ti >= recend { go = 0 } else {
108                if out[ti] == (34 as u8) { go = 0 } else { doc[dl] = out[ti]; dl = dl + 1; ti = ti + 1 }
109            }
110        }
111        doc[dl] = 10 as u8; dl = dl + 1
112    }
113    dl = ni_scat(doc, dl, "NIST -- National Institute of Standards and Technology, data.nist.gov (Standard Reference / public data, DCAT/NERDm).\n" as *u8)
114    // raw record JSON (every field: description, keyword, theme, DOI, landingPage)
115    var ri: i64 = recstart
116    while ri < recend { doc[dl] = out[ri]; dl = dl + 1; ri = ri + 1 }
117    if dl <= 0 { return 0 }
118    dp_ingest(domain, DP_VIS_PUBLIC, doc, dl, cidp)
119    return 1
120}
121
122func main(argc: i64, argv: *i64) -> i64 {
123    if argc < 3 {
124        ni_puts("usage: nx_nist_ingest <domain> <pages> [size]\n" as *u8)
125        return 2
126    }
127    let domain: *u8 = argv[1] as *u8
128    let pages: i64 = ni_atoi(argv[2] as *u8)
129    var size: i64 = 100
130    if argc > 3 { size = ni_atoi(argv[3] as *u8) }
131
132    let r: i64 = nx_trust_store_load_from_certdata("data/mozilla_certdata.txt" as *u8, 512, K_MAGIC_4194304)
133    if r <= 0 { ni_puts("STORE-FAIL: data/mozilla_certdata.txt not loadable (wrong CWD?)\n" as *u8); return 1 }
134    let store: *TrustStore = r as *TrustStore
135
136    let out: *u8 = sys_mmap(K_MAGIC_8388608)
137    let att: *u8 = sys_mmap(K_MAGIC_8192)
138    let status: *i64 = sys_mmap(16) as *i64
139    let empty: *u8 = sys_mmap(8); empty[0] = 0 as u8
140    let urlbuf: *u8 = sys_mmap(512)
141    let doc: *u8 = sys_mmap(K_MAGIC_524288)
142    let cidp: *i64 = sys_mmap(16) as *i64
143
144    var total: i64 = 0
145    var p: i64 = 1
146    while p <= pages {
147        ni_build_url(urlbuf, size, p)
148        status[0] = 0
149        let n: i64 = nx_fetch_any(urlbuf, store, out, K_MAGIC_8388608, 6, status, empty, 0, att, K_MAGIC_8192)
150        if n > 0 {
151            let rd: i64 = ni_find(out, n, 0, "ResultData" as *u8, 10)
152            if rd >= 0 {
153                var recstart: i64 = ni_find(out, n, rd, "{\"@context\"" as *u8, 11)
154                var pcount: i64 = 0
155                while recstart >= 0 {
156                    let nextrec: i64 = ni_find(out, n, recstart + 11, "{\"@context\"" as *u8, 11)
157                    var recend: i64 = nextrec
158                    if recend < 0 { recend = n }
159                    pcount = pcount + ni_ingest_record(domain, out, n, recstart, recend, doc, cidp)
160                    total = total + 1
161                    recstart = nextrec
162                }
163                ni_puts("page " as *u8); ni_putn(p); ni_puts(" status=" as *u8); ni_putn(status[0])
164                ni_puts(" records=" as *u8); ni_putn(pcount); ni_puts("\n" as *u8)
165            } else {
166                ni_puts("page " as *u8); ni_putn(p); ni_puts(" NO-ResultData status=" as *u8); ni_putn(status[0]); ni_puts("\n" as *u8)
167            }
168        } else {
169            ni_puts("page " as *u8); ni_putn(p); ni_puts(" FETCH-FAIL (see cascade attempts)\n" as *u8)
170        }
171        p = p + 1
172    }
173    ni_puts("TOTAL-INGESTED=" as *u8); ni_putn(total); ni_puts("\n" as *u8)
174    return 0
175}
nx_nist_ingest.nx source

`nx_nist_ingest.nx` source
