nx_lib_gbif_bulk.nx source
↩ module page · 47 lines · 2370 B
1// nx_lib_gbif_bulk.nx -- SCALE: ingest the full GBIF bulk taxonomy (~16K taxa)
2// from disk into nx_lib_store, index it, and run full-text searches over the
3// whole real catalog. Proves the sovereign library at scale on real data.
4import "nx_syscalls.nx"
5import "nx_lib_gbif.nx"
6import "nx_lib_index.nx"
7const K_MAGIC_1048576: i64 = 1048576
8
9func bp_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
10func bp_wr(p: *u8, n: i64) -> i64 { sys_write(1, p, n); return 0 }
11func bp_putn(v: i64) -> i64 {
12 let bb: *u8 = sys_mmap(28); var m: i64 = v
13 let t: *u8 = sys_mmap(28); var k: i64 = 0
14 if m == 0 { t[0] = 48 as u8; k = 1 }
15 while m > 0 { t[k] = (48 + (m - (m/10)*10)) as u8; m = m / 10; k = k + 1 }
16 var i: i64 = 0; while i < k { bb[i] = t[k - 1 - i]; i = i + 1 } sys_write(1, bb, k); return 0
17}
18func bp_hits(out: *u8, n: i64) -> i64 {
19 if n <= 2 { bp_puts("0"); return 0 }
20 var c: i64 = 1; var i: i64 = 0
21 while i < n { if out[i] == 44 as u8 { c = c + 1 } i = i + 1 }
22 bp_putn(c); return 0
23}
24
25func main() -> i64 {
26 let lb: *i64 = sys_mmap(16) as *i64
27 let buf: *u8 = sys_read_file("knowledge/fetched/gbif_bulk.jsonl\x00" as *u8, lb)
28 if (buf as i64) == 0 { bp_puts("cannot read gbif_bulk.jsonl\n"); return 1 }
29 let n: i64 = lb[0]
30 bp_puts("gbif bulk bytes="); bp_putn(n); bp_puts("\n")
31
32 let count: i64 = nx_lib_gbif_ingest_buf(buf, n)
33 bp_puts("*** ingested REAL GBIF taxa: "); bp_putn(count); bp_puts(" ***\n")
34 if count == 0 { return 2 }
35
36 bp_puts("building full-text index over the whole catalog...\n")
37 nx_lib_index_build()
38 let out: *u8 = sys_mmap(K_MAGIC_1048576)
39
40 bp_puts("--- full-text search over ~16K real taxa (hits) ---\n")
41 bp_puts("q=Archaea -> "); let a1: i64 = nx_lib_index_search("Archaea" as *u8, 7, out); bp_hits(out, a1); bp_puts("\n")
42 bp_puts("q=Bacteria -> "); let a2: i64 = nx_lib_index_search("Bacteria" as *u8, 8, out); bp_hits(out, a2); bp_puts("\n")
43 bp_puts("q=Sulfolobales -> "); let a3: i64 = nx_lib_index_search("Sulfolobales" as *u8, 12, out); bp_hits(out, a3); bp_puts("\n")
44 bp_puts("q=Caldisphaera -> "); let a4: i64 = nx_lib_index_search("Caldisphaera" as *u8, 12, out); bp_wr(out, a4); bp_puts("\n")
45 bp_puts("--- end (REAL GBIF catalog @ scale, all sovereign) ---\n")
46 return 0
47}