nx_lib_gbif.nx source
↩ module page · 121 lines · 5999 B
1// nx_lib_gbif.nx -- ingest the library's REAL data: GBIF backbone-taxonomy
2// JSONL (one flat taxon object per line) into nx_lib_store. Each taxon becomes
3// a searchable record: title=scientificName, doi=taxonID, published=year(from
4// authorship), license=rank, abstract=classification lineage, authors=authorship.
5// Reuses the oa_* JSON field extractors. license_tier: ORIGINAL
6import "nx_lib_openalex.nx"
7const K_MAGIC_1024: i64 = 1024
8const K_MAGIC_4096: i64 = 4096
9const K_MAGIC_16384: i64 = 16384
10const K_MAGIC_8388608: i64 = 8388608
11
12func gb_isdig(c: i64) -> i64 { if c >= 0x30 { if c <= 0x39 { return 1 } } return 0 }
13
14// first run of exactly 4 digits in src -> out (a year). returns 4 or 0.
15func gb_year(src: *u8, srclen: i64, out: *u8) -> i64 {
16 var i: i64 = 0
17 while i + 3 < srclen {
18 if gb_isdig(src[i] as i64) == 1 { if gb_isdig(src[i+1] as i64) == 1 { if gb_isdig(src[i+2] as i64) == 1 { if gb_isdig(src[i+3] as i64) == 1 {
19 var okb: i64 = 1; if i > 0 { if gb_isdig(src[i-1] as i64) == 1 { okb = 0 } }
20 var oka: i64 = 1; if i + 4 < srclen { if gb_isdig(src[i+4] as i64) == 1 { oka = 0 } }
21 if okb == 1 { if oka == 1 { out[0] = src[i]; out[1] = src[i+1]; out[2] = src[i+2]; out[3] = src[i+3]; return 4 } }
22 } } } }
23 i = i + 1
24 }
25 return 0
26}
27
28// append "<label>: <value> " to out at off if key is present. returns new off.
29func gb_seg(json: *u8, s: i64, e: i64, key: *u8, label: *u8, out: *u8, off: i64) -> i64 {
30 let tmp: *u8 = sys_mmap(512)
31 let v: i64 = oa_str(json, s, e, key, tmp, 511)
32 if v <= 0 { return off }
33 var o: i64 = off
34 var li: i64 = 0
35 while label[li] != (0 as u8) { out[o] = label[li]; o = o + 1; li = li + 1 }
36 out[o] = 0x3a as u8; o = o + 1; out[o] = 0x20 as u8; o = o + 1
37 var q: i64 = 0
38 while q < v { out[o] = tmp[q]; o = o + 1; q = q + 1 }
39 out[o] = 0x20 as u8; o = o + 1
40 return o
41}
42
43// ingest GBIF JSONL -> nx_lib_store (line = one taxon). returns count stored.
44func nx_lib_gbif_ingest_buf(json: *u8, n: i64) -> i64 {
45 let ttl: *u8 = sys_mmap(K_MAGIC_1024)
46 let doi: *u8 = sys_mmap(512)
47 let auth: *u8 = sys_mmap(512)
48 let rankb: *u8 = sys_mmap(128)
49 let absb: *u8 = sys_mmap(K_MAGIC_4096)
50 let keyb: *u8 = sys_mmap(64)
51 let yrb: *u8 = sys_mmap(16)
52 let hkb: *u8 = sys_mmap(128)
53 let rec: *u8 = sys_mmap(K_MAGIC_16384)
54 let worksbuf: *u8 = sys_mmap(K_MAGIC_8388608)
55 let kbuf: *u8 = sys_mmap(512)
56 let wlp: *i64 = sys_mmap(16) as *i64
57 var w: *i64 = nx_lib_store_bulk_begin(worksbuf, wlp)
58
59 var count: i64 = 0
60 var ls: i64 = 0
61 var i: i64 = 0
62 while i <= n {
63 var eol: i64 = 0
64 if i == n { eol = 1 } else { if json[i] == 10 as u8 { eol = 1 } }
65 if eol == 1 {
66 if i > ls + 5 {
67 let tl: i64 = oa_str(json, ls, i, "\"scientificName\"" as *u8, ttl, 1023)
68 if tl > 0 {
69 var kl: i64 = oa_num(json, ls, i, "\"key\"" as *u8, keyb, 63)
70 if kl <= 0 { kl = oa_num(json, ls, i, "\"usageKey\"" as *u8, keyb, 63) }
71 if kl <= 0 { kl = oa_num(json, ls, i, "\"speciesKey\"" as *u8, keyb, 63) }
72 if kl > 0 {
73 hkb[0] = 71 as u8; hkb[1] = 66 as u8; hkb[2] = 73 as u8; hkb[3] = 70 as u8; hkb[4] = 95 as u8
74 var hn: i64 = 5
75 var q: i64 = 0
76 while q < kl { hkb[hn] = keyb[q]; hn = hn + 1; q = q + 1 }
77 hkb[hn] = 0 as u8
78
79 var dl: i64 = oa_str(json, ls, i, "\"taxonID\"" as *u8, doi, 511)
80 if dl < 0 { dl = 0 }
81 var al: i64 = oa_str(json, ls, i, "\"authorship\"" as *u8, auth, 511)
82 if al < 0 { al = 0 }
83 var rl: i64 = oa_str(json, ls, i, "\"rank\"" as *u8, rankb, 127)
84 if rl < 0 { rl = 0 }
85 var yl: i64 = 0
86 if al > 0 { yl = gb_year(auth, al, yrb) }
87
88 var ao: i64 = 0
89 ao = gb_seg(json, ls, i, "\"kingdom\"" as *u8, "Kingdom" as *u8, absb, ao)
90 ao = gb_seg(json, ls, i, "\"phylum\"" as *u8, "Phylum" as *u8, absb, ao)
91 ao = gb_seg(json, ls, i, "\"class\"" as *u8, "Class" as *u8, absb, ao)
92 ao = gb_seg(json, ls, i, "\"order\"" as *u8, "Order" as *u8, absb, ao)
93 ao = gb_seg(json, ls, i, "\"family\"" as *u8, "Family" as *u8, absb, ao)
94 ao = gb_seg(json, ls, i, "\"genus\"" as *u8, "Genus" as *u8, absb, ao)
95 ao = gb_seg(json, ls, i, "\"publishedIn\"" as *u8, "Published in" as *u8, absb, ao)
96
97 // record: title \t doi \t year \t rank \t abstract \t authors
98 var o: i64 = 0
99 q = 0; while q < tl { rec[o] = ttl[q]; o = o + 1; q = q + 1 }
100 rec[o] = 9 as u8; o = o + 1
101 q = 0; while q < dl { rec[o] = doi[q]; o = o + 1; q = q + 1 }
102 rec[o] = 9 as u8; o = o + 1
103 q = 0; while q < yl { rec[o] = yrb[q]; o = o + 1; q = q + 1 }
104 rec[o] = 9 as u8; o = o + 1
105 q = 0; while q < rl { rec[o] = rankb[q]; o = o + 1; q = q + 1 }
106 rec[o] = 9 as u8; o = o + 1
107 q = 0; while q < ao { rec[o] = absb[q]; o = o + 1; q = q + 1 }
108 rec[o] = 9 as u8; o = o + 1
109 q = 0; while q < al { rec[o] = auth[q]; o = o + 1; q = q + 1 }
110 w = nx_lib_store_bulk_add(w, kbuf, worksbuf, wlp, hkb, rec, o)
111 count = count + 1
112 }
113 }
114 }
115 ls = i + 1
116 }
117 i = i + 1
118 }
119 nx_lib_store_bulk_commit(w, worksbuf, wlp[0])
120 return count
121}