nx_search_inverted_test.nx source
↩ module page · 154 lines · 7444 B
1// nx_search_inverted_test.nx -- smoke for inverted index.
2// Build against the 16,200-row GBIF bulk catalog, query "Plantae".
3
4import "syscalls.nx"
5import "nx_search_inverted.nx"
6
7func make_jsonl_path(out: *u8) -> i64 {
8 // "../../nishi-library/fixtures/gbif_bulk/ingested_bulk.jsonl"
9 out[0] = 0x2E; out[1] = 0x2E; out[2] = 0x2F
10 out[3] = 0x2E; out[4] = 0x2E; out[5] = 0x2F
11 out[6] = 0x6E; out[7] = 0x69; out[8] = 0x73; out[9] = 0x68
12 out[10] = 0x69; out[11] = 0x2D; out[12] = 0x6C; out[13] = 0x69
13 out[14] = 0x62; out[15] = 0x72; out[16] = 0x61; out[17] = 0x72
14 out[18] = 0x79; out[19] = 0x2F
15 out[20] = 0x66; out[21] = 0x69; out[22] = 0x78; out[23] = 0x74
16 out[24] = 0x75; out[25] = 0x72; out[26] = 0x65; out[27] = 0x73
17 out[28] = 0x2F
18 out[29] = 0x67; out[30] = 0x62; out[31] = 0x69; out[32] = 0x66
19 out[33] = 0x5F; out[34] = 0x62; out[35] = 0x75; out[36] = 0x6C; out[37] = 0x6B
20 out[38] = 0x2F
21 out[39] = 0x69; out[40] = 0x6E; out[41] = 0x67; out[42] = 0x65
22 out[43] = 0x73; out[44] = 0x74; out[45] = 0x65; out[46] = 0x64
23 out[47] = 0x5F; out[48] = 0x62; out[49] = 0x75; out[50] = 0x6C; out[51] = 0x6B
24 out[52] = 0x2E; out[53] = 0x6A; out[54] = 0x73; out[55] = 0x6F
25 out[56] = 0x6E; out[57] = 0x6C
26 out[58] = 0
27 return 58
28}
29
30func main() -> i64 {
31 let testbytes: *u8 = sys_mmap(8)
32 testbytes[0] = 0x70; testbytes[1] = 0x6C; testbytes[2] = 0x61; testbytes[3] = 0x6E
33 testbytes[4] = 0x74; testbytes[5] = 0x61; testbytes[6] = 0x65
34 let h0: i64 = nx_inv_hash_bytes_lower(testbytes, 7)
35 if h0 == 0 { return __syscall(93, 90, 0, 0, 0, 0, 0) }
36
37 // Phase 1: allocate index.
38 let idx0: *NxInvIndex = nx_inv_new(100)
39 if idx0 == 0 as *NxInvIndex { return __syscall(93, 100, 0, 0, 0, 0, 0) }
40 if idx0.vocab_capacity != 131072 { return __syscall(93, 101, 0, 0, 0, 0, 0) }
41
42 // Phase 2: PASS 1 only -- record token without writing posting yet.
43 let _ = nx_inv_add_posting(idx0, h0, 42)
44 if idx0.vocab_occupied != 1 { return __syscall(93, 110, 0, 0, 0, 0, 0) }
45 // After pass 1, postings_used is still 0 (pass 2 sets it via finalize).
46 if idx0.postings_used != 0 { return __syscall(93, 111, 0, 0, 0, 0, 0) }
47
48 // Phase 3: index a row directly (still pass-1).
49 let row: *u8 = sys_mmap(64)
50 row[0] = 0x70; row[1] = 0x6C; row[2] = 0x61; row[3] = 0x6E
51 row[4] = 0x74; row[5] = 0x61; row[6] = 0x65
52 row[7] = 0x20
53 row[8] = 0x73; row[9] = 0x6F; row[10] = 0x6C; row[11] = 0x61
54 row[12] = 0x6E; row[13] = 0x75; row[14] = 0x6D
55 let _2 = nx_inv_index_row(idx0, row, 15, 0)
56 if idx0.vocab_occupied != 2 { return __syscall(93, 120, 0, 0, 0, 0, 0) }
57
58 // Run finalize + pass 2 (emit) for this toy data
59 let fz: i64 = nx_inv_finalize_offsets(idx0)
60 if fz <= 0 { return __syscall(93, 125, 0, 0, 0, 0, 0) }
61 nx_inv_emit_row(idx0, row, 15, 0)
62
63 // Query "solanum" -> should find rowID 0
64 let q: *u8 = sys_mmap(16)
65 q[0] = 0x73; q[1] = 0x6F; q[2] = 0x6C; q[3] = 0x61
66 q[4] = 0x6E; q[5] = 0x75; q[6] = 0x6D
67 let rowids: *i64 = sys_mmap(64) as *i64
68 let r_raw: *u8 = sys_mmap(NX_INV_QUERY_RESULT_BYTES)
69 let r: *NxInvQueryResult = r_raw as *NxInvQueryResult
70 let qv: i64 = nx_inv_query_term(idx0, q, 7, rowids, 8, r)
71 if qv != NX_INV_OK { return __syscall(93, 130, 0, 0, 0, 0, 0) }
72 if r.postings_count != 1 { return __syscall(93, 131, 0, 0, 0, 0, 0) }
73 if r.n_rowids_filled != 1 { return __syscall(93, 132, 0, 0, 0, 0, 0) }
74 if rowids[0] != 0 { return __syscall(93, 133, 0, 0, 0, 0, 0) }
75
76 // Phase 4: full build against 16K-row catalog.
77 let path: *u8 = sys_mmap(64)
78 make_jsonl_path(path)
79 let idx: *NxInvIndex = nx_inv_build_from_jsonl(path)
80 if idx == 0 as *NxInvIndex { return __syscall(93, 200, 0, 0, 0, 0, 0) }
81 if idx.n_rows < 1000 { return __syscall(93, 201, 0, 0, 0, 0, 0) }
82 if idx.vocab_occupied < 50 { return __syscall(93, 202, 0, 0, 0, 0, 0) }
83 return 0
84}
85
86func main_disabled() -> i64 {
87 let path: *u8 = sys_mmap(64)
88 make_jsonl_path(path)
89
90 // Phase 1: allocate empty index, verify struct fields.
91 let idx0: *NxInvIndex = nx_inv_new(100)
92 if idx0 == 0 as *NxInvIndex { return __syscall(93, 100, 0, 0, 0, 0, 0) }
93 if idx0.vocab_capacity != 131072 { return __syscall(93, 101, 0, 0, 0, 0, 0) }
94 if idx0.vocab_occupied != 0 { return __syscall(93, 102, 0, 0, 0, 0, 0) }
95 if idx0.postings_used != 0 { return __syscall(93, 103, 0, 0, 0, 0, 0) }
96
97 // Phase 2: index a single hand-built row.
98 let row: *u8 = sys_mmap(64)
99 row[0] = 0x70; row[1] = 0x6C; row[2] = 0x61; row[3] = 0x6E
100 row[4] = 0x74; row[5] = 0x61; row[6] = 0x65 // "plantae"
101 row[7] = 0x20 // space
102 row[8] = 0x73; row[9] = 0x6F; row[10] = 0x6C; row[11] = 0x61 // "sola"
103 row[12] = 0x6E; row[13] = 0x75; row[14] = 0x6D // "num"
104 let _ = nx_inv_index_row(idx0, row, 15, 0)
105 if idx0.vocab_occupied != 2 { return __syscall(93, 110, 0, 0, 0, 0, 0) }
106 if idx0.postings_used != 8 { return __syscall(93, 111, 0, 0, 0, 0, 0) }
107
108 // Phase 3: query plantae -> 1 match (rowID 0).
109 let q0: *u8 = sys_mmap(16)
110 q0[0] = 0x70; q0[1] = 0x6C; q0[2] = 0x61; q0[3] = 0x6E
111 q0[4] = 0x74; q0[5] = 0x61; q0[6] = 0x65
112 let rowids0: *i64 = sys_mmap(64) as *i64
113 let r0_raw: *u8 = sys_mmap(NX_INV_QUERY_RESULT_BYTES)
114 let r0: *NxInvQueryResult = r0_raw as *NxInvQueryResult
115 let qv: i64 = nx_inv_query_term(idx0, q0, 7, rowids0, 8, r0)
116 if qv != NX_INV_OK { return __syscall(93, 120, 0, 0, 0, 0, 0) }
117 if r0.postings_count != 1 { return __syscall(93, 121, 0, 0, 0, 0, 0) }
118 if r0.n_rowids_filled != 1 { return __syscall(93, 122, 0, 0, 0, 0, 0) }
119
120 // Phase 4: full file build.
121 let idx: *NxInvIndex = nx_inv_build_from_jsonl(path)
122 if idx == 0 as *NxInvIndex { return __syscall(93, 1, 0, 0, 0, 0, 0) }
123 if idx.n_rows < 1000 { return __syscall(93, 2, 0, 0, 0, 0, 0) }
124 if idx.vocab_occupied < 50 { return __syscall(93, 3, 0, 0, 0, 0, 0) }
125 if idx.postings_used == 0 { return __syscall(93, 4, 0, 0, 0, 0, 0) }
126
127 // Query "plantae" (lowercase; tokenizer also lowercases input).
128 let q: *u8 = sys_mmap(16)
129 q[0] = 0x70; q[1] = 0x6C; q[2] = 0x61; q[3] = 0x6E
130 q[4] = 0x74; q[5] = 0x61; q[6] = 0x65 // "plantae"
131
132 let rowids: *i64 = sys_mmap(4096 * 8) as *i64
133 let r_raw: *u8 = sys_mmap(NX_INV_QUERY_RESULT_BYTES)
134 let r: *NxInvQueryResult = r_raw as *NxInvQueryResult
135
136 let v: i64 = nx_inv_query_term(idx, q, 7, rowids, 4096, r)
137 if v != NX_INV_OK { return __syscall(93, 10, 0, 0, 0, 0, 0) }
138 // Catalog has ~2700 Plantae rows; postings count must be substantial.
139 if r.postings_count < 100 { return __syscall(93, 11, 0, 0, 0, 0, 0) }
140
141 // Query non-existent term: returns NO_MATCHES.
142 let q2: *u8 = sys_mmap(32)
143 q2[0] = 0x7A; q2[1] = 0x7A; q2[2] = 0x7A; q2[3] = 0x7A
144 q2[4] = 0x7A; q2[5] = 0x6E; q2[6] = 0x6F; q2[7] = 0x6D
145 q2[8] = 0x61; q2[9] = 0x74; q2[10] = 0x63; q2[11] = 0x68
146 q2[12] = 0x77; q2[13] = 0x6F; q2[14] = 0x72; q2[15] = 0x64
147 let r2_raw: *u8 = sys_mmap(NX_INV_QUERY_RESULT_BYTES)
148 let r2: *NxInvQueryResult = r2_raw as *NxInvQueryResult
149 let v2: i64 = nx_inv_query_term(idx, q2, 16, rowids, 4096, r2)
150 if v2 != NX_INV_NO_MATCHES { return __syscall(93, 20, 0, 0, 0, 0, 0) }
151 if r2.postings_count != 0 { return __syscall(93, 21, 0, 0, 0, 0, 0) }
152
153 return 0
154}