nx_lib_openalex.nx source
↩ module page · 304 lines · 14222 B
1// nx_lib_openalex.nx -- ingest REAL works from the OpenAlex API JSON into
2// nx_lib_store. Splits the "results":[...] array into work objects (string-
3// aware brace scan), then per work extracts id->hk, title, doi, publication_year
4// -> a TAB record. The sovereign end of the fetch->parse->store pipeline on
5// real academic data. license_tier: ORIGINAL
6import "nx_syscalls.nx"
7import "nx_lib_store.nx"
8const OA_WORKSBUF: i64 = 1048576
9
10// index just past the first occurrence of needle (NUL-terminated) in hay[start,end).
11func oa_find(hay: *u8, end: i64, start: i64, needle: *u8) -> i64 {
12 var nlen: i64 = 0; while needle[nlen] != (0 as u8) { nlen = nlen + 1 }
13 if nlen == 0 { return 0 - 1 }
14 if end < nlen { return 0 - 1 }
15 let last: i64 = end - nlen
16 var i: i64 = start
17 while i <= last {
18 var j: i64 = 0; var st: i64 = 0
19 while st == 0 { if j >= nlen { st = 2 } if st == 0 { if hay[i+j] != needle[j] { st = 1 } if st == 0 { j = j + 1 } } }
20 if st == 2 { return i + nlen }
21 i = i + 1
22 }
23 return 0 - 1
24}
25
26func oa_isws(c: i64) -> i64 { if c == 0x20 { return 1 } if c == 0x0a { return 1 } if c == 0x09 { return 1 } if c == 0x0d { return 1 } return 0 }
27func oa_hex(c: i64) -> i64 {
28 if c >= 0x30 { if c <= 0x39 { return c - 0x30 } }
29 if c >= 0x61 { if c <= 0x66 { return c - 0x61 + 10 } }
30 if c >= 0x41 { if c <= 0x46 { return c - 0x41 + 10 } }
31 return 0 - 1
32}
33// emit codepoint cp as UTF-8 into out at k; return new k.
34func oa_utf8(out: *u8, k: i64, maxlen: i64, cp: i64) -> i64 {
35 if cp < 0x80 { if k < maxlen { out[k] = cp as u8; return k + 1 } return k }
36 if cp < 0x800 { if k + 1 < maxlen { out[k] = (0xc0 | (cp >> 6)) as u8; out[k+1] = (0x80 | (cp & 0x3f)) as u8; return k + 2 } return k }
37 if k + 2 < maxlen { out[k] = (0xe0 | (cp >> 12)) as u8; out[k+1] = (0x80 | ((cp >> 6) & 0x3f)) as u8; out[k+2] = (0x80 | (cp & 0x3f)) as u8; return k + 3 }
38 return k
39}
40
41// copy the STRING value of key (quoted literal, e.g. "\"title\"") within [s,e). returns len or -1.
42func oa_str(hay: *u8, s: i64, e: i64, key: *u8, out: *u8, maxlen: i64) -> i64 {
43 let ka: i64 = oa_find(hay, e, s, key)
44 if ka < 0 { return 0 - 1 }
45 var p: i64 = ka
46 var go: i64 = 1
47 while go == 1 { if p >= e { return 0 - 1 } if (hay[p] as i64) == 0x3a { p = p + 1; go = 0 } else { p = p + 1 } }
48 go = 1
49 while go == 1 { if p >= e { return 0 - 1 } if oa_isws(hay[p] as i64) == 1 { p = p + 1 } else { go = 0 } }
50 if (hay[p] as i64) != 0x22 { return 0 - 1 }
51 p = p + 1
52 var k: i64 = 0
53 var run: i64 = 1
54 while run == 1 {
55 if p >= e { run = 0 }
56 else {
57 let c: i64 = hay[p] as i64
58 if c == 0x5c {
59 p = p + 1
60 if p < e {
61 let ec: i64 = hay[p] as i64
62 if ec == 0x75 {
63 p = p + 1
64 var cp: i64 = 0; var hn: i64 = 0; var hg: i64 = 1
65 while hg == 1 { if hn >= 4 { hg = 0 } else { if p >= e { hg = 0 } else { let hv: i64 = oa_hex(hay[p] as i64); if hv < 0 { hg = 0 } else { cp = cp * 16 + hv; p = p + 1; hn = hn + 1 } } } }
66 k = oa_utf8(out, k, maxlen, cp)
67 } else {
68 var o: i64 = ec; if ec == 0x6e { o = 0x0a } if ec == 0x74 { o = 0x09 } if ec == 0x72 { o = 0x0d }
69 if k < maxlen { out[k] = o as u8; k = k + 1 } p = p + 1
70 }
71 }
72 } else { if c == 0x22 { run = 0 } else { if k < maxlen { out[k] = c as u8; k = k + 1 } p = p + 1 } }
73 }
74 }
75 return k
76}
77
78// copy the NUMERIC value of key within [s,e) as decimal text into out. returns len or -1.
79func oa_num(hay: *u8, s: i64, e: i64, key: *u8, out: *u8, maxlen: i64) -> i64 {
80 let ka: i64 = oa_find(hay, e, s, key)
81 if ka < 0 { return 0 - 1 }
82 var p: i64 = ka
83 var go: i64 = 1
84 while go == 1 { if p >= e { return 0 - 1 } if (hay[p] as i64) == 0x3a { p = p + 1; go = 0 } else { p = p + 1 } }
85 go = 1
86 while go == 1 { if p >= e { return 0 - 1 } if oa_isws(hay[p] as i64) == 1 { p = p + 1 } else { go = 0 } }
87 var k: i64 = 0
88 var run: i64 = 1
89 while run == 1 {
90 if p >= e { run = 0 }
91 else { let c: i64 = hay[p] as i64; if c >= 0x30 { if c <= 0x39 { if k < maxlen { out[k] = c as u8; k = k + 1 } p = p + 1 } else { run = 0 } } else { run = 0 } }
92 }
93 if k == 0 { return 0 - 1 }
94 return k
95}
96
97// copy the last '/'-separated segment of src[0,srclen) into out. returns len.
98func oa_lastseg(src: *u8, srclen: i64, out: *u8, maxlen: i64) -> i64 {
99 var ls: i64 = 0
100 var i: i64 = 0
101 while i < srclen { if (src[i] as i64) == 0x2f { ls = i + 1 } i = i + 1 }
102 var k: i64 = 0
103 i = ls
104 while i < srclen { if k < maxlen { out[k] = src[i]; k = k + 1 } i = i + 1 }
105 return k
106}
107
108// collect author display_names from authorships[] within [s,e), joined ", ".
109// "author" (with closing quote) does not match "authorships", so we land on the
110// author field, then take the first display_name after it (the author, not an
111// institution). returns length.
112func oa_authors(hay: *u8, s: i64, e: i64, out: *u8, maxlen: i64) -> i64 {
113 let aa: i64 = oa_find(hay, e, s, "\"authorships\"" as *u8)
114 if aa < 0 { return 0 }
115 let nm: *u8 = sys_mmap(512)
116 var p: i64 = aa
117 var k: i64 = 0
118 var cnt: i64 = 0
119 var go: i64 = 1
120 while go == 1 {
121 if cnt >= 40 { go = 0 }
122 else {
123 let au: i64 = oa_find(hay, e, p, "\"author\"" as *u8)
124 if au < 0 { go = 0 }
125 else {
126 let nl: i64 = oa_str(hay, au, e, "\"display_name\"" as *u8, nm, 511)
127 if nl > 0 {
128 if k > 0 { if k + 2 < maxlen { out[k] = 0x2c as u8; out[k+1] = 0x20 as u8; k = k + 2 } }
129 var q: i64 = 0
130 while q < nl { if k < maxlen { out[k] = nm[q]; k = k + 1 } q = q + 1 }
131 cnt = cnt + 1
132 }
133 p = au + 8
134 }
135 }
136 }
137 return k
138}
139
140// reconstruct the abstract from OpenAlex's abstract_inverted_index
141// ({"word":[pos,...],...}) within [s,e): place each word at its position(s),
142// then join in order. returns length (0 if absent/null).
143func oa_abstract(hay: *u8, s: i64, e: i64, out: *u8, maxlen: i64) -> i64 {
144 let ii: i64 = oa_find(hay, e, s, "\"abstract_inverted_index\"" as *u8)
145 if ii < 0 { return 0 }
146 var p: i64 = ii
147 var g: i64 = 1
148 while g == 1 { if p >= e { return 0 } if (hay[p] as i64) == 0x3a { p = p + 1; g = 0 } else { p = p + 1 } }
149 g = 1
150 while g == 1 { if p >= e { return 0 } if oa_isws(hay[p] as i64) == 1 { p = p + 1 } else { g = 0 } }
151 if (hay[p] as i64) != 0x7b { return 0 } // null / not an object
152 p = p + 1
153
154 let MAXP: i64 = 6000
155 let posoff: *i64 = sys_mmap(MAXP * 8) as *i64
156 let poslen: *i64 = sys_mmap(MAXP * 8) as *i64
157 var z: i64 = 0
158 while z < MAXP { poslen[z] = 0; z = z + 1 }
159 let wbuf: *u8 = sys_mmap(262144)
160 var wp: i64 = 0
161 var maxp: i64 = 0 - 1
162
163 var run: i64 = 1
164 while run == 1 {
165 var sk: i64 = 1
166 while sk == 1 { if p >= e { sk = 0; run = 0 } else { let c: i64 = hay[p] as i64; if oa_isws(c) == 1 { p = p + 1 } else { if c == 0x2c { p = p + 1 } else { sk = 0 } } } }
167 if run == 1 {
168 let c0: i64 = hay[p] as i64
169 if c0 == 0x7d { run = 0 }
170 else { if c0 == 0x22 {
171 p = p + 1
172 let woff: i64 = wp
173 var kr: i64 = 1
174 while kr == 1 {
175 if p >= e { kr = 0; run = 0 }
176 else { let cc: i64 = hay[p] as i64
177 if cc == 0x5c { p = p + 1; if p < e { let ec: i64 = hay[p] as i64;
178 if ec == 0x75 { p = p + 1; var cpn: i64 = 0; var hn: i64 = 0; var hg: i64 = 1; while hg == 1 { if hn >= 4 { hg = 0 } else { if p >= e { hg = 0 } else { let hv: i64 = oa_hex(hay[p] as i64); if hv < 0 { hg = 0 } else { cpn = cpn * 16 + hv; p = p + 1; hn = hn + 1 } } } } wp = oa_utf8(wbuf, wp, 262143, cpn) }
179 else { var o: i64 = ec; if ec == 0x6e { o = 0x0a } if ec == 0x74 { o = 0x09 } if ec == 0x72 { o = 0x0d } if wp < 262143 { wbuf[wp] = o as u8; wp = wp + 1 } p = p + 1 } } }
180 else { if cc == 0x22 { kr = 0; p = p + 1 } else { if wp < 262143 { wbuf[wp] = cc as u8; wp = wp + 1 } p = p + 1 } } }
181 }
182 let wlen: i64 = wp - woff
183 var sb: i64 = 1
184 while sb == 1 { if p >= e { sb = 0; run = 0 } else { if (hay[p] as i64) == 0x5b { p = p + 1; sb = 0 } else { p = p + 1 } } }
185 var nr: i64 = 1
186 while nr == 1 {
187 if p >= e { nr = 0; run = 0 }
188 else { let d: i64 = hay[p] as i64
189 if d == 0x5d { nr = 0; p = p + 1 }
190 else { if d >= 0x30 { if d <= 0x39 {
191 var num: i64 = 0
192 var dr: i64 = 1
193 while dr == 1 { if p >= e { dr = 0 } else { let dd: i64 = hay[p] as i64; if dd >= 0x30 { if dd <= 0x39 { num = num * 10 + (dd - 0x30); p = p + 1 } else { dr = 0 } } else { dr = 0 } } }
194 if num < MAXP { posoff[num] = woff; poslen[num] = wlen; if num > maxp { maxp = num } }
195 } else { p = p + 1 } } else { p = p + 1 } }
196 }
197 }
198 } else { p = p + 1 } }
199 }
200 }
201
202 var k: i64 = 0
203 var pos: i64 = 0
204 while pos <= maxp {
205 if poslen[pos] > 0 {
206 var q: i64 = 0
207 while q < poslen[pos] { if k < maxlen { out[k] = wbuf[posoff[pos] + q]; k = k + 1 } q = q + 1 }
208 if k < maxlen { out[k] = 0x20 as u8; k = k + 1 }
209 }
210 pos = pos + 1
211 }
212 return k
213}
214
215// synthetic hk "W<seq>" when a work lacks an id (rare).
216func oa_dummy_hk(buf: *u8, seq: i64) -> i64 {
217 buf[0] = 87 as u8
218 var k: i64 = 1
219 var v: i64 = seq
220 if v == 0 { buf[1] = 48 as u8; return 2 }
221 let tmp: *u8 = sys_mmap(32); var t: i64 = 0
222 while v > 0 { tmp[t] = (48 + (v - (v/10)*10)) as u8; v = v / 10; t = t + 1 }
223 var j: i64 = t - 1
224 while j >= 0 { buf[k] = tmp[j]; k = k + 1; j = j - 1 }
225 return k
226}
227
228// ingest OpenAlex works JSON -> nx_lib_store. returns number of works stored.
229func nx_lib_openalex_ingest_buf(json: *u8, n: i64) -> i64 {
230 let rs: i64 = oa_find(json, n, 0, "\"results\"" as *u8)
231 if rs < 0 { return 0 }
232 // O(n) bulk path: one handle, one __works__ write, instead of a commit per record.
233 let worksbuf: *u8 = sys_mmap(OA_WORKSBUF)
234 let worklen_p: *i64 = sys_mmap(16) as *i64
235 let kbuf: *u8 = sys_mmap(512)
236 var bw: *i64 = nx_lib_store_bulk_begin(worksbuf, worklen_p)
237 var a: i64 = rs; var g0: i64 = 1
238 while g0 == 1 { if a >= n { return 0 } if (json[a] as i64) == 0x5b { a = a + 1; g0 = 0 } else { a = a + 1 } }
239
240 let idb: *u8 = sys_mmap(512)
241 let hkb: *u8 = sys_mmap(256)
242 let ttl: *u8 = sys_mmap(8192)
243 let doi: *u8 = sys_mmap(1024)
244 let yr: *u8 = sys_mmap(64)
245 let auth: *u8 = sys_mmap(16384)
246 let absb: *u8 = sys_mmap(65536)
247 let rec: *u8 = sys_mmap(131072)
248
249 var count: i64 = 0
250 var depth: i64 = 0
251 var instr: i64 = 0
252 var start: i64 = 0 - 1
253 var i: i64 = a
254 var run: i64 = 1
255 while run == 1 {
256 if i >= n { run = 0 }
257 else {
258 let c: i64 = json[i] as i64
259 if instr == 1 {
260 if c == 0x5c { i = i + 2 }
261 else { if c == 0x22 { instr = 0; i = i + 1 } else { i = i + 1 } }
262 } else {
263 if c == 0x22 { instr = 1; i = i + 1 }
264 else { if c == 0x7b { if depth == 0 { start = i } depth = depth + 1; i = i + 1 }
265 else { if c == 0x7d {
266 depth = depth - 1; i = i + 1
267 if depth == 0 { if start >= 0 {
268 let tl: i64 = oa_str(json, start, i, "\"title\"" as *u8, ttl, 8192)
269 if tl > 0 {
270 let il: i64 = oa_str(json, start, i, "\"id\"" as *u8, idb, 512)
271 var hl: i64 = 0
272 if il > 0 { hl = oa_lastseg(idb, il, hkb, 255) } else { hl = oa_dummy_hk(hkb, count) }
273 hkb[hl] = 0 as u8
274 var dl: i64 = oa_str(json, start, i, "\"doi\"" as *u8, doi, 1024)
275 if dl < 0 { dl = 0 }
276 let yl: i64 = oa_num(json, start, i, "\"publication_year\"" as *u8, yr, 64)
277 var o: i64 = 0
278 var q: i64 = 0
279 while q < tl { rec[o] = ttl[q]; o = o + 1; q = q + 1 }
280 rec[o] = 9 as u8; o = o + 1
281 q = 0; while q < dl { rec[o] = doi[q]; o = o + 1; q = q + 1 }
282 rec[o] = 9 as u8; o = o + 1
283 if yl > 0 { q = 0; while q < yl { rec[o] = yr[q]; o = o + 1; q = q + 1 } }
284 rec[o] = 9 as u8; o = o + 1 // -> license (empty)
285 rec[o] = 9 as u8; o = o + 1 // -> abstract
286 let abl: i64 = oa_abstract(json, start, i, absb, 60000)
287 q = 0; while q < abl { rec[o] = absb[q]; o = o + 1; q = q + 1 }
288 rec[o] = 9 as u8; o = o + 1 // -> authors
289 let al: i64 = oa_authors(json, start, i, auth, 16000)
290 q = 0; while q < al { rec[o] = auth[q]; o = o + 1; q = q + 1 }
291 bw = nx_lib_store_bulk_add(bw, kbuf, worksbuf, worklen_p, hkb, rec, o)
292 count = count + 1
293 }
294 start = 0 - 1
295 } }
296 }
297 else { if c == 0x5d { if depth == 0 { run = 0 } else { i = i + 1 } }
298 else { i = i + 1 } } } }
299 }
300 }
301 }
302 if count > 0 { nx_lib_store_bulk_commit(bw, worksbuf, worklen_p[0]) }
303 return count
304}