nx_srcreg_lib.nx source
↩ module page · 293 lines · 11347 B
1// nx_srcreg_lib.nx -- THE ONE READER FOR THE RESEARCH SOURCE REGISTRY (knowledge/research_sources.conf).
2//
3// Operator 2026-09-06: "we need to measure and make sure we never have any unreachable data whether by search or api
4// we need to be able to gather it all" and "a regular cadence for all domains without smashing our nas resources".
5// The registry is the POPULATION every reach and coverage number on /compare/researchreach is measured over, so a
6// reader that silently drops a row it cannot parse under-reports coverage in the FLATTERING direction. This lib
7// refuses a bad row BY NAME instead, and its partition must sum to the rows seen.
8//
9// THE DATA (rows in knowledge/research_sources.conf, pipe-separated, NO pipe inside a field):
10// src|<key>|<kind>|<protocol>|<base-url>|<pace_ms>|<auth>|<licence>|<note>
11// kind forge | preprint | index | publisher | government | repository | registry
12// protocol oai-pmh | atom-api | rest-json | graphql | sitemap | html
13// pace_ms MINIMUM milliseconds between two requests to THIS host. Per-source and not global, because a shared
14// budget starves the slow hosts first.
15//
16// THE STATES, per row:
17// OK nine fields, a declared kind, a declared protocol, a positive integer pace
18// BAD-FIELDS fewer than nine fields -- the row cannot be read, and is NOT silently skipped
19// BAD-KIND a kind outside the declared vocabulary
20// BAD-PROTO a protocol outside the declared vocabulary
21// BAD-PACE a pace that is not all digits, or is zero or negative
22// A row belongs to exactly ONE bucket and rows = ok + bad_fields + bad_kind + bad_proto + bad_pace must SUM.
23// The vocabularies are CLOSED on purpose: an unknown protocol must REFUSE rather than default to the permissive
24// value, because a mistyped protocol that falls through to a default is a silent downgrade of a fail-closed row.
25//
26// COMPOSES, never re-implements: sys_read_file (sizes its buffer from the file and cannot short-read, so there is
27// no cap to reach in silence here).
28// license_tier: ORIGINAL No hw writes (Rule 26).
29import "nx_syscalls.nx"
30
31const SR_PIPE: i64 = 124
32const SR_NL: i64 = 10
33const SR_DIGIT0: i64 = 48
34const SR_DIGIT9: i64 = 57
35const SR_I64: i64 = 8
36const SR_NONE: i64 = 0 - 1
37const SR_ROW_TAG: *u8 = "src|"
38const SR_MIN_FIELDS: i64 = 9
39
40// field positions
41const SR_F_KEY: i64 = 1
42const SR_F_KIND: i64 = 2
43const SR_F_PROTO: i64 = 3
44const SR_F_URL: i64 = 4
45const SR_F_PACE: i64 = 5
46const SR_F_AUTH: i64 = 6
47const SR_F_LICENCE: i64 = 7
48const SR_F_NOTE: i64 = 8
49// OPTIONAL TENTH FIELD, added 2026-09-06 after the first live census. A base-url is a BASE and a probe treated it as
50// a TARGET: api.github.com answers 200 bare while huggingface.co/api returns 404 because the real call needs a
51// sub-path, so six NOT-FOUND rows in that census were the probe asking a url nobody serves rather than a source we
52// cannot reach. A row may append a probe path; absent, the base is probed as before, so every existing nine-field
53// row keeps working and SR_MIN_FIELDS is deliberately NOT raised.
54const SR_F_PROBE_PATH: i64 = 9
55
56// states
57const SR_S_OK: i64 = 0
58const SR_S_BAD_FIELDS: i64 = 1
59const SR_S_BAD_KIND: i64 = 2
60const SR_S_BAD_PROTO: i64 = 3
61const SR_S_BAD_PACE: i64 = 4
62
63// census slots
64const SR_C_ROWS: i64 = 0
65const SR_C_OK: i64 = 1
66const SR_C_BAD_FIELDS: i64 = 2
67const SR_C_BAD_KIND: i64 = 3
68const SR_C_BAD_PROTO: i64 = 4
69const SR_C_BAD_PACE: i64 = 5
70const SR_C_N: i64 = 6
71
72// exits
73const SR_EXIT_OK: i64 = 0
74const SR_EXIT_BAD: i64 = 1
75const SR_EXIT_USAGE: i64 = 2
76const SR_EXIT_NOREG: i64 = 3
77
78func sr_slen(s: *u8) -> i64 {
79 var n: i64 = 0
80 while s[n] != (0 as u8) { n = n + 1 }
81 return n
82}
83
84// a line ends at a newline, a NUL, or the end of the buffer
85func sr_line_end(buf: *u8, n: i64, p: i64) -> i64 {
86 var e: i64 = p
87 var go: i64 = 1
88 while go == 1 {
89 if e >= n { go = 0 } else {
90 if buf[e] == (SR_NL as u8) { go = 0 } else { if buf[e] == (0 as u8) { go = 0 } else { e = e + 1 } }
91 }
92 }
93 return e
94}
95
96func sr_line_starts(buf: *u8, p: i64, e: i64, tag: *u8) -> i64 {
97 var i: i64 = 0
98 while tag[i] != (0 as u8) {
99 if p + i >= e { return 0 }
100 if buf[p + i] != tag[i] { return 0 }
101 i = i + 1
102 }
103 return 1
104}
105
106// field k (0-based) of the line [p,e): its length, with off[0] = its start; SR_NONE when the line has no field k
107func sr_field(buf: *u8, p: i64, e: i64, k: i64, off: *i64) -> i64 {
108 var i: i64 = p
109 var f: i64 = 0
110 var s: i64 = p
111 while i < e {
112 if buf[i] == (SR_PIPE as u8) {
113 if f == k { off[0] = s; return i - s }
114 f = f + 1
115 s = i + 1
116 }
117 i = i + 1
118 }
119 if f == k { off[0] = s; return e - s }
120 return SR_NONE
121}
122
123// how many pipe-separated fields the line [p,e) carries
124func sr_field_count(buf: *u8, p: i64, e: i64) -> i64 {
125 var i: i64 = p
126 var f: i64 = 1
127 while i < e {
128 if buf[i] == (SR_PIPE as u8) { f = f + 1 }
129 i = i + 1
130 }
131 return f
132}
133
134// does the span [a,a+len) equal the NUL-terminated literal
135func sr_span_eq(buf: *u8, a: i64, len: i64, lit: *u8) -> i64 {
136 if len < 0 { return 0 }
137 if sr_slen(lit) != len { return 0 }
138 var i: i64 = 0
139 while i < len {
140 if buf[a + i] != lit[i] { return 0 }
141 i = i + 1
142 }
143 return 1
144}
145
146func sr_is_digit(b: u8) -> i64 {
147 if (b as i64) < SR_DIGIT0 { return 0 }
148 if (b as i64) > SR_DIGIT9 { return 0 }
149 return 1
150}
151
152// the span as a non-negative integer, or SR_NONE when it is empty or carries a non-digit
153func sr_atoi_span(buf: *u8, a: i64, len: i64) -> i64 {
154 if len <= 0 { return SR_NONE }
155 var v: i64 = 0
156 var i: i64 = 0
157 while i < len {
158 if sr_is_digit(buf[a + i]) == 0 { return SR_NONE }
159 v = v * 10 + ((buf[a + i] as i64) - SR_DIGIT0)
160 i = i + 1
161 }
162 return v
163}
164
165// THE CLOSED KIND VOCABULARY. An unknown kind returns SR_NONE and the row is refused by name.
166func sr_kind_id(buf: *u8, a: i64, len: i64) -> i64 {
167 if sr_span_eq(buf, a, len, "forge" as *u8) == 1 { return 0 }
168 if sr_span_eq(buf, a, len, "preprint" as *u8) == 1 { return 1 }
169 if sr_span_eq(buf, a, len, "index" as *u8) == 1 { return 2 }
170 if sr_span_eq(buf, a, len, "publisher" as *u8) == 1 { return 3 }
171 if sr_span_eq(buf, a, len, "government" as *u8) == 1 { return 4 }
172 if sr_span_eq(buf, a, len, "repository" as *u8) == 1 { return 5 }
173 if sr_span_eq(buf, a, len, "registry" as *u8) == 1 { return 6 }
174 return SR_NONE
175}
176
177// THE CLOSED PROTOCOL VOCABULARY. Deliberately closed: a mistyped protocol that fell through to a default would be
178// a silent downgrade, which is the defect class this estate has already measured in a routing table.
179func sr_proto_id(buf: *u8, a: i64, len: i64) -> i64 {
180 if sr_span_eq(buf, a, len, "oai-pmh" as *u8) == 1 { return 0 }
181 if sr_span_eq(buf, a, len, "atom-api" as *u8) == 1 { return 1 }
182 if sr_span_eq(buf, a, len, "rest-json" as *u8) == 1 { return 2 }
183 if sr_span_eq(buf, a, len, "graphql" as *u8) == 1 { return 3 }
184 if sr_span_eq(buf, a, len, "sitemap" as *u8) == 1 { return 4 }
185 if sr_span_eq(buf, a, len, "html" as *u8) == 1 { return 5 }
186 return SR_NONE
187}
188
189func sr_state_name(s: i64) -> *u8 {
190 if s == SR_S_OK { return "OK" as *u8 }
191 if s == SR_S_BAD_FIELDS { return "BAD-FIELDS" as *u8 }
192 if s == SR_S_BAD_KIND { return "BAD-KIND" as *u8 }
193 if s == SR_S_BAD_PROTO { return "BAD-PROTO" as *u8 }
194 if s == SR_S_BAD_PACE { return "BAD-PACE" as *u8 }
195 return "UNKNOWN" as *u8
196}
197
198func sr_kind_name(k: i64) -> *u8 {
199 if k == 0 { return "forge" as *u8 }
200 if k == 1 { return "preprint" as *u8 }
201 if k == 2 { return "index" as *u8 }
202 if k == 3 { return "publisher" as *u8 }
203 if k == 4 { return "government" as *u8 }
204 if k == 5 { return "repository" as *u8 }
205 if k == 6 { return "registry" as *u8 }
206 return "-" as *u8
207}
208
209// SR_LOAD -- read the registry whole. sys_read_file sizes its buffer from the file, so there is no cap to reach in
210// silence. Returns the buffer with out_len[0] set; out_len[0] <= 0 means unreadable.
211func sr_load(path: *u8, out_len: *i64) -> *u8 {
212 out_len[0] = 0
213 return sys_read_file(path, out_len)
214}
215
216// how many src| rows the buffer carries
217func sr_count_rows(buf: *u8, n: i64) -> i64 {
218 var i: i64 = 0
219 var c: i64 = 0
220 while i < n {
221 let e: i64 = sr_line_end(buf, n, i)
222 if sr_line_starts(buf, i, e, SR_ROW_TAG) == 1 { c = c + 1 }
223 i = e + 1
224 }
225 return c
226}
227
228// classify every src| row. off[] = line start, st[] = state, pace[] = pace_ms (SR_NONE when unparseable),
229// kind[] = kind id. Returns the number of rows written. c[] is the census and MUST sum.
230func sr_classify(buf: *u8, n: i64, off: *i64, st: *i64, pace: *i64, kind: *i64, c: *i64) -> i64 {
231 var s: i64 = 0
232 while s < SR_C_N { c[s] = 0; s = s + 1 }
233 let fo: *i64 = sys_mmap(SR_I64) as *i64
234 var i: i64 = 0
235 var r: i64 = 0
236 while i < n {
237 let e: i64 = sr_line_end(buf, n, i)
238 if sr_line_starts(buf, i, e, SR_ROW_TAG) == 1 {
239 off[r] = i
240 pace[r] = SR_NONE
241 kind[r] = SR_NONE
242 var state: i64 = SR_S_OK
243 if sr_field_count(buf, i, e) < SR_MIN_FIELDS {
244 state = SR_S_BAD_FIELDS
245 } else {
246 let kl: i64 = sr_field(buf, i, e, SR_F_KIND, fo)
247 let kid: i64 = sr_kind_id(buf, fo[0], kl)
248 kind[r] = kid
249 let pl: i64 = sr_field(buf, i, e, SR_F_PROTO, fo)
250 let pid: i64 = sr_proto_id(buf, fo[0], pl)
251 let cl: i64 = sr_field(buf, i, e, SR_F_PACE, fo)
252 let pv: i64 = sr_atoi_span(buf, fo[0], cl)
253 pace[r] = pv
254 if kid == SR_NONE { state = SR_S_BAD_KIND } else {
255 if pid == SR_NONE { state = SR_S_BAD_PROTO } else {
256 if pv == SR_NONE { state = SR_S_BAD_PACE } else {
257 if pv <= 0 { state = SR_S_BAD_PACE }
258 }
259 }
260 }
261 }
262 st[r] = state
263 c[SR_C_ROWS] = c[SR_C_ROWS] + 1
264 if state == SR_S_OK { c[SR_C_OK] = c[SR_C_OK] + 1 }
265 if state == SR_S_BAD_FIELDS { c[SR_C_BAD_FIELDS] = c[SR_C_BAD_FIELDS] + 1 }
266 if state == SR_S_BAD_KIND { c[SR_C_BAD_KIND] = c[SR_C_BAD_KIND] + 1 }
267 if state == SR_S_BAD_PROTO { c[SR_C_BAD_PROTO] = c[SR_C_BAD_PROTO] + 1 }
268 if state == SR_S_BAD_PACE { c[SR_C_BAD_PACE] = c[SR_C_BAD_PACE] + 1 }
269 r = r + 1
270 }
271 i = e + 1
272 }
273 return r
274}
275
276// the partition, so a caller can print it beside the row count and refuse if it does not reconcile
277func sr_partition_sum(c: *i64) -> i64 {
278 return c[SR_C_OK] + c[SR_C_BAD_FIELDS] + c[SR_C_BAD_KIND] + c[SR_C_BAD_PROTO] + c[SR_C_BAD_PACE]
279}
280
281func sr_verdict(c: *i64) -> i64 {
282 if c[SR_C_ROWS] <= 0 { return SR_EXIT_NOREG }
283 if sr_partition_sum(c) != c[SR_C_ROWS] { return SR_EXIT_BAD }
284 if c[SR_C_OK] == c[SR_C_ROWS] { return SR_EXIT_OK }
285 return SR_EXIT_BAD
286}
287
288func sr_verdict_name(v: i64) -> *u8 {
289 if v == SR_EXIT_OK { return "OK" as *u8 }
290 if v == SR_EXIT_BAD { return "REFUSED" as *u8 }
291 if v == SR_EXIT_NOREG { return "NO-REGISTRY" as *u8 }
292 return "UNKNOWN" as *u8
293}