nx_docstore.nx source
↩ module page · 92 lines · 3189 B
1// nx_docstore.nx -- persistent crawl content store (CAS/DV2.0 foundation).
2//
3// module: nishi-core.search.docstore
4// depends: nx_syscalls.nx, nx_str.nx
5// capability: CORE_IO
6// wired_status: FULLY_WIRED
7//
8// WHY: a real crawl must SURVIVE restarts and RESUME -- you cannot re-crawl the
9// web from scratch each run (and re-fetching hammers hosts -> blacklisting).
10// Each fetched doc is appended as a length-prefixed record; on restart the
11// store reloads and the inverted index is rebuilt, so the corpus persists and
12// the frontier can continue. Record: [u32 url_len][url][u32 text_len][text],
13// little-endian. The seed of the content-addressed store / DV2.0 ingest.
14
15import "nx_syscalls.nx"
16import "nx_str.nx"
17
18func _ds_put_u32(buf: *u8, off: i64, v: i64) -> i64 {
19 buf[off] = v & 0xff
20 buf[off + 1] = (v >> 8) & 0xff
21 buf[off + 2] = (v >> 16) & 0xff
22 buf[off + 3] = (v >> 24) & 0xff
23 return 0
24}
25func _ds_get_u32(buf: *u8, off: i64) -> i64 {
26 return (buf[off] as i64) | ((buf[off + 1] as i64) << 8) | ((buf[off + 2] as i64) << 16) | ((buf[off + 3] as i64) << 24)
27}
28
29// Truncate/create the store (begin a fresh crawl). mode 0644.
30func nx_docstore_reset(path: *u8) -> i64 {
31 let fd: i64 = sys_openat_wr(path, 0x1A4)
32 if fd < 0 { return 0 - 1 }
33 sys_close(fd)
34 return 0
35}
36
37// Append one crawled doc record. Returns 0, or -1 on open failure.
38func nx_docstore_append(path: *u8, url: *u8, ulen: i64, text: *u8, tlen: i64) -> i64 {
39 let fd: i64 = sys_openat_append(path, 0x1A4)
40 if fd < 0 { return 0 - 1 }
41 let rec: *u8 = sys_mmap(ulen + tlen + 16)
42 _ds_put_u32(rec, 0, ulen)
43 var p: i64 = 4
44 var i: i64 = 0
45 while i < ulen { rec[p + i] = url[i]; i = i + 1 }
46 p = p + ulen
47 _ds_put_u32(rec, p, tlen)
48 p = p + 4
49 i = 0
50 while i < tlen { rec[p + i] = text[i]; i = i + 1 }
51 p = p + tlen
52 sys_write(fd, rec, p)
53 sys_close(fd)
54 return 0
55}
56
57// Load all records: urls[]/ulens[]/texts[]/tlens[] point into the loaded file
58// buffer (use the lengths -- entries are NOT null-terminated). Returns count.
59func nx_docstore_load(path: *u8, urls: **u8, ulens: *i64, texts: **u8,
60 tlens: *i64, max: i64) -> i64 {
61 let lenbox: *i64 = sys_mmap(8) as *i64
62 let buf: *u8 = sys_read_file(path, lenbox)
63 if buf == 0 as *u8 { return 0 }
64 let total: i64 = lenbox[0]
65 var off: i64 = 0
66 var n: i64 = 0
67 var run: i64 = 1
68 while run == 1 {
69 run = 0
70 if n < max {
71 if off + 8 <= total {
72 let ul: i64 = _ds_get_u32(buf, off)
73 off = off + 4
74 if off + ul + 4 <= total {
75 urls[n] = ((buf as i64) + off) as *u8
76 ulens[n] = ul
77 off = off + ul
78 let tl: i64 = _ds_get_u32(buf, off)
79 off = off + 4
80 if off + tl <= total {
81 texts[n] = ((buf as i64) + off) as *u8
82 tlens[n] = tl
83 off = off + tl
84 n = n + 1
85 run = 1
86 }
87 }
88 }
89 }
90 }
91 return n
92}