nx_learnmine_lib.nx source
↩ module page · 435 lines · 17977 B
1// nx_learnmine_lib.nx -- LEARNING MINING: third-party how-to material as SUPPORTING GUIDES for the /compare rungs (operator
2// 2026-09-06: farm archive.org, YouTube, Vimeo, DeviantArt and the rest for videos, transcripts and text that power the
3// development rungs, so the boards carry third-party feedback and reference frames, not only the seat's own judgement).
4// The evidence law is the operator's: a workflow video is a LIAR-KILLER (it proves the field can do the thing and shows
5// how) or a REFERENCE FRAME (what the result looks like in that month); it is NEVER an oracle, and its month is its bar.
6// This lib is the FIRST rung of the lane: the archive.org advancedsearch listing (output=json) of one creator or query,
7// parsed from its own bytes into item rows and a census (media types, years, subjects) so a seat reads a 200-row digest
8// instead of a quarter-megabyte of JSON. Every later rung (item files, subtitle mirrors through nx_srt and nx_webvtt,
9// the YouTube caption track, the guide rows on <dom>.guides) composes this table; nothing here fetches -- the bytes come
10// from nx_research_fetch under provenance, and the organ only reads what was mirrored.
11// license_tier: ORIGINAL No hw writes (Rule 26).
12import "nx_syscalls.nx"
13
14const LM_QUOTE: i64 = 34
15const LM_BACKSLASH: i64 = 92
16const LM_LBRACE: i64 = 123
17const LM_RBRACE: i64 = 125
18const LM_LBRACKET: i64 = 91
19const LM_RBRACKET: i64 = 93
20const LM_COLON: i64 = 58
21const LM_COMMA: i64 = 44
22const LM_SPACE: i64 = 32
23const LM_NL: i64 = 10
24const LM_TAB: i64 = 9
25const LM_CR: i64 = 13
26const LM_PIPE: i64 = 124
27const LM_SLASH: i64 = 47
28const LM_SEMI: i64 = 59
29const LM_QMARK: i64 = 63
30const LM_UTF8_1: i64 = 128
31const LM_UTF8_2: i64 = 2048
32const LM_SURROGATE_LO: i64 = 55296
33const LM_SURROGATE_HI: i64 = 57344
34const LM_DATE_LEN: i64 = 10
35const LM_DESC_HEAD: i64 = 240
36const LM_FIELD_CAP: i64 = 65536
37const LM_SUBJ_CAP: i64 = 4096
38const LM_KEY_DOCS: *u8 = "\"docs\":"
39const LM_KEY_NUMFOUND: *u8 = "\"numFound\":"
40const LM_KEY_ID: *u8 = "\"identifier\":"
41const LM_KEY_TYPE: *u8 = "\"mediatype\":"
42const LM_KEY_DATE: *u8 = "\"date\":"
43const LM_KEY_TITLE: *u8 = "\"title\":"
44const LM_KEY_DESC: *u8 = "\"description\":"
45const LM_KEY_SUBJECT: *u8 = "\"subject\":"
46const LM_KEY_DOWNLOADS: *u8 = "\"downloads\":"
47const LM_TYPE_MOVIES: *u8 = "movies"
48const LM_TYPE_TEXTS: *u8 = "texts"
49const LM_TYPE_AUDIO: *u8 = "audio"
50const LM_TYPE_IMAGE: *u8 = "image"
51const LM_DASH: *u8 = "-"
52// item slots
53const LM_I_ID: i64 = 0
54const LM_I_TYPE: i64 = 1
55const LM_I_DATE: i64 = 2
56const LM_I_TITLE: i64 = 3
57const LM_I_DOWNLOADS: i64 = 4
58const LM_I_SUBJECTS: i64 = 5
59const LM_I_DESC: i64 = 6
60const LM_I_DESC_LEN: i64 = 7
61const LM_I_SLOTS: i64 = 8
62// listing slots
63const LM_L_ITEMS: i64 = 0
64const LM_L_NUMFOUND: i64 = 1
65const LM_L_MOVIES: i64 = 2
66const LM_L_TEXTS: i64 = 3
67const LM_L_AUDIO: i64 = 4
68const LM_L_IMAGE: i64 = 5
69const LM_L_OTHER: i64 = 6
70const LM_L_NEWEST: i64 = 7
71const LM_L_OLDEST: i64 = 8
72const LM_L_TRUNCATED_DESCS: i64 = 9
73const LM_L_DOWNLOADS: i64 = 10
74const LM_L_SLOTS: i64 = 12
75// results
76const LM_OK: i64 = 0
77const LM_ERR_UNREADABLE: i64 = 0 - 1
78const LM_ERR_NO_DOCS: i64 = 0 - 2
79const LM_ERR_MALFORMED: i64 = 0 - 3
80const LM_ERR_CAPACITY: i64 = 0 - 4
81
82func lm_len(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
83func lm_streq(a: *u8, b: *u8) -> i64 { var i: i64 = 0; while a[i] != (0 as u8) { if a[i] != b[i] { return 0 } i = i + 1 } if b[i] != (0 as u8) { return 0 } return 1 }
84func lm_cat(d: *u8, o: i64, s: *u8) -> i64 { var i: i64 = 0; while s[i] != (0 as u8) { d[o + i] = s[i]; i = i + 1 } return o + i }
85func lm_dup(s: *u8, n: i64) -> *u8 { let d: *u8 = sys_mmap(n + 1); var i: i64 = 0; while i < n { d[i] = s[i]; i = i + 1 } d[n] = 0 as u8; return d }
86func lm_putd(d: *u8, o: i64, v: i64) -> i64 {
87 if v == 0 { d[o] = 48 as u8; return o + 1 }
88 var x: i64 = v
89 var p: i64 = o
90 if x < 0 { d[p] = 45 as u8; p = p + 1; x = 0 - x }
91 let t: *u8 = sys_mmap(32)
92 var k: i64 = 0
93 while x > 0 { t[k] = (48 + (x % 10)) as u8; x = x / 10; k = k + 1 }
94 while k > 0 { k = k - 1; d[p] = t[k]; p = p + 1 }
95 return p
96}
97func lm_err_name(e: i64) -> *u8 {
98 if e == LM_OK { return "OK" as *u8 }
99 if e == LM_ERR_UNREADABLE { return "UNREADABLE" as *u8 }
100 if e == LM_ERR_NO_DOCS { return "NO-DOCS" as *u8 }
101 if e == LM_ERR_MALFORMED { return "MALFORMED" as *u8 }
102 if e == LM_ERR_CAPACITY { return "CAPACITY" as *u8 }
103 return "UNKNOWN-RC" as *u8
104}
105// the literal key (with its quotes and colon) at or after from, inside [from, end); -1 when absent
106func lm_find(b: *u8, from: i64, end: i64, key: *u8) -> i64 {
107 let kl: i64 = lm_len(key)
108 var i: i64 = from
109 while i + kl <= end {
110 var j: i64 = 0
111 while j < kl { if b[i + j] != key[j] { break } j = j + 1 }
112 if j == kl { return i + kl }
113 i = i + 1
114 }
115 return 0 - 1
116}
117func lm_skip_ws(b: *u8, p: i64, end: i64) -> i64 {
118 var i: i64 = p
119 while i < end {
120 let c: i64 = b[i] as i64
121 var ws: i64 = 0
122 if c == LM_SPACE { ws = 1 }
123 if c == LM_NL { ws = 1 }
124 if c == LM_TAB { ws = 1 }
125 if c == LM_CR { ws = 1 }
126 if ws == 0 { return i }
127 i = i + 1
128 }
129 return i
130}
131func lm_hexval(c: i64) -> i64 {
132 if c >= 48 { if c <= 57 { return c - 48 } }
133 if c >= 97 { if c <= 102 { return c - 87 } }
134 if c >= 65 { if c <= 70 { return c - 55 } }
135 return 0 - 1
136}
137// emit one code point as UTF-8 into out at o; a surrogate half becomes a question mark
138func lm_put_cp(out: *u8, o: i64, cp: i64) -> i64 {
139 if cp < LM_UTF8_1 { out[o] = cp as u8; return o + 1 }
140 if cp < LM_UTF8_2 { out[o] = (192 + cp / 64) as u8; out[o + 1] = (128 + cp % 64) as u8; return o + 2 }
141 if cp >= LM_SURROGATE_LO { if cp < LM_SURROGATE_HI { out[o] = LM_QMARK as u8; return o + 1 } }
142 out[o] = (224 + cp / 4096) as u8
143 out[o + 1] = (128 + (cp / 64) % 64) as u8
144 out[o + 2] = (128 + cp % 64) as u8
145 return o + 3
146}
147// a row byte: the format's own separators never leak into a field
148func lm_row_byte(c: i64) -> i64 {
149 if c == LM_PIPE { return LM_SLASH }
150 if c == LM_NL { return LM_SPACE }
151 if c == LM_CR { return LM_SPACE }
152 if c == LM_TAB { return LM_SPACE }
153 return c
154}
155// decode the JSON string whose opening quote is at p into out (sanitised, NUL-terminated, at most cap-1 bytes);
156// returns the position after the closing quote, or -1 when the string does not close inside end; outlen gets the
157// FULL decoded length (so a truncation is announced), out gets min(full, cap-1)
158func lm_str(b: *u8, p: i64, end: i64, out: *u8, cap: i64, outlen: *i64) -> i64 {
159 if (b[p] as i64) != LM_QUOTE { return 0 - 1 }
160 var i: i64 = p + 1
161 var o: i64 = 0
162 var full: i64 = 0
163 let tmp: *u8 = sys_mmap(8)
164 while i < end {
165 let c: i64 = b[i] as i64
166 if c == LM_QUOTE { out[o] = 0 as u8; outlen[0] = full; return i + 1 }
167 var w: i64 = 0
168 if c == LM_BACKSLASH {
169 if i + 1 >= end { return 0 - 1 }
170 let e: i64 = b[i + 1] as i64
171 i = i + 2
172 if e == 117 {
173 if i + 4 > end { return 0 - 1 }
174 var cp: i64 = 0
175 var k: i64 = 0
176 while k < 4 { let h: i64 = lm_hexval(b[i + k] as i64); if h < 0 { return 0 - 1 } cp = cp * 16 + h; k = k + 1 }
177 i = i + 4
178 w = lm_put_cp(tmp, 0, cp)
179 } else {
180 var d: i64 = e
181 if e == 110 { d = LM_NL }
182 if e == 116 { d = LM_TAB }
183 if e == 114 { d = LM_CR }
184 if e == 98 { d = LM_SPACE }
185 if e == 102 { d = LM_SPACE }
186 tmp[0] = d as u8
187 w = 1
188 }
189 } else {
190 tmp[0] = c as u8
191 w = 1
192 i = i + 1
193 }
194 var k2: i64 = 0
195 while k2 < w {
196 let rb: i64 = lm_row_byte(tmp[k2] as i64)
197 if o < cap - 1 { out[o] = rb as u8; o = o + 1 }
198 full = full + 1
199 k2 = k2 + 1
200 }
201 }
202 return 0 - 1
203}
204// the value after key inside [s, e): a string, or an array of strings joined by sep, or a number; out NUL-terminated.
205// returns the FULL decoded length (0 for absent or null), -1 when malformed
206func lm_value(b: *u8, s: i64, e: i64, key: *u8, out: *u8, cap: i64, sep: i64) -> i64 {
207 out[0] = 0 as u8
208 var p: i64 = lm_find(b, s, e, key)
209 if p < 0 { return 0 }
210 p = lm_skip_ws(b, p, e)
211 if p >= e { return 0 - 1 }
212 let c: i64 = b[p] as i64
213 let ol: *i64 = sys_mmap(16) as *i64
214 if c == LM_QUOTE {
215 if lm_str(b, p, e, out, cap, ol) < 0 { return 0 - 1 }
216 return ol[0]
217 }
218 if c == LM_LBRACKET {
219 var q: i64 = p + 1
220 var o: i64 = 0
221 var full: i64 = 0
222 var first: i64 = 1
223 let part: *u8 = sys_mmap(cap)
224 var go: i64 = 1
225 while go == 1 {
226 q = lm_skip_ws(b, q, e)
227 if q >= e { return 0 - 1 }
228 let d: i64 = b[q] as i64
229 if d == LM_RBRACKET { go = 0 }
230 else { if d == LM_COMMA { q = q + 1 }
231 else { if d == LM_QUOTE {
232 q = lm_str(b, q, e, part, cap, ol)
233 if q < 0 { return 0 - 1 }
234 if first == 0 { if o < cap - 2 { out[o] = sep as u8; out[o + 1] = LM_SPACE as u8; o = o + 2 } full = full + 2 }
235 first = 0
236 var k: i64 = 0
237 while part[k] != (0 as u8) { if o < cap - 1 { out[o] = part[k]; o = o + 1 } k = k + 1 }
238 full = full + ol[0]
239 } else { return 0 - 1 } } }
240 }
241 out[o] = 0 as u8
242 return full
243 }
244 if c == 110 { return 0 }
245 // a number: digits only into out
246 var o2: i64 = 0
247 var q2: i64 = p
248 while q2 < e {
249 let d2: i64 = b[q2] as i64
250 var dig: i64 = 0
251 if d2 >= 48 { if d2 <= 57 { dig = 1 } }
252 if d2 == 45 { dig = 1 }
253 if dig == 0 { break }
254 if o2 < cap - 1 { out[o2] = d2 as u8; o2 = o2 + 1 }
255 q2 = q2 + 1
256 }
257 out[o2] = 0 as u8
258 return o2
259}
260func lm_atoi(s: *u8) -> i64 {
261 var v: i64 = 0
262 var i: i64 = 0
263 var neg: i64 = 0
264 if (s[0] as i64) == 45 { neg = 1; i = 1 }
265 if s[i] == (0 as u8) { return 0 - 1 }
266 while s[i] != (0 as u8) { let c: i64 = s[i] as i64; if c < 48 { return 0 - 1 } if c > 57 { return 0 - 1 } v = v * 10 + (c - 48); i = i + 1 }
267 if neg == 1 { return 0 - v }
268 return v
269}
270// the end of the object whose opening brace is at s (strings respected, nesting counted); -1 when it never closes
271func lm_obj_end(b: *u8, s: i64, end: i64) -> i64 {
272 var depth: i64 = 0
273 var instr: i64 = 0
274 var i: i64 = s
275 while i < end {
276 let c: i64 = b[i] as i64
277 if instr == 1 {
278 if c == LM_BACKSLASH { i = i + 1 }
279 else { if c == LM_QUOTE { instr = 0 } }
280 } else {
281 if c == LM_QUOTE { instr = 1 }
282 else { if c == LM_LBRACE { depth = depth + 1 }
283 else { if c == LM_RBRACE { depth = depth - 1; if depth == 0 { return i + 1 } } } }
284 }
285 i = i + 1
286 }
287 return 0 - 1
288}
289// compare two YYYY-MM-DD prefixes; an empty date sorts as oldest
290func lm_date_cmp(a: *u8, c: *u8) -> i64 {
291 var i: i64 = 0
292 while i < LM_DATE_LEN {
293 let x: i64 = a[i] as i64
294 let y: i64 = c[i] as i64
295 if x != y { return x - y }
296 if x == 0 { return 0 }
297 i = i + 1
298 }
299 return 0
300}
301// ---- parse the listing: items (LM_I_SLOTS each) and the census in lst ----
302func lm_parse_listing(b: *u8, n: i64, items: *i64, cap: i64, lst: *i64) -> i64 {
303 var q: i64 = 0
304 while q < LM_L_SLOTS { lst[q] = 0; q = q + 1 }
305 lst[LM_L_NEWEST] = LM_DASH as i64
306 lst[LM_L_OLDEST] = LM_DASH as i64
307 let nf: *u8 = sys_mmap(64)
308 if lm_value(b, 0, n, LM_KEY_NUMFOUND, nf, 64, LM_SEMI) > 0 { lst[LM_L_NUMFOUND] = lm_atoi(nf) }
309 var p: i64 = lm_find(b, 0, n, LM_KEY_DOCS)
310 if p < 0 { return LM_ERR_NO_DOCS }
311 p = lm_skip_ws(b, p, n)
312 if p >= n { return LM_ERR_MALFORMED }
313 if (b[p] as i64) != LM_LBRACKET { return LM_ERR_MALFORMED }
314 p = p + 1
315 var count: i64 = 0
316 let f: *u8 = sys_mmap(LM_FIELD_CAP)
317 var go: i64 = 1
318 while go == 1 {
319 p = lm_skip_ws(b, p, n)
320 if p >= n { return LM_ERR_MALFORMED }
321 let c: i64 = b[p] as i64
322 if c == LM_RBRACKET { go = 0 }
323 else { if c == LM_COMMA { p = p + 1 }
324 else { if c == LM_LBRACE {
325 let e: i64 = lm_obj_end(b, p, n)
326 if e < 0 { return LM_ERR_MALFORMED }
327 if count >= cap { return LM_ERR_CAPACITY }
328 let base: i64 = count * LM_I_SLOTS
329 var l: i64 = lm_value(b, p, e, LM_KEY_ID, f, LM_FIELD_CAP, LM_SEMI)
330 if l < 0 { return LM_ERR_MALFORMED }
331 items[base + LM_I_ID] = (lm_dup(f, lm_len(f))) as i64
332 l = lm_value(b, p, e, LM_KEY_TYPE, f, LM_FIELD_CAP, LM_SEMI)
333 if l < 0 { return LM_ERR_MALFORMED }
334 items[base + LM_I_TYPE] = (lm_dup(f, lm_len(f))) as i64
335 l = lm_value(b, p, e, LM_KEY_DATE, f, LM_FIELD_CAP, LM_SEMI)
336 if l < 0 { return LM_ERR_MALFORMED }
337 var dl: i64 = lm_len(f)
338 if dl > LM_DATE_LEN { dl = LM_DATE_LEN }
339 items[base + LM_I_DATE] = (lm_dup(f, dl)) as i64
340 l = lm_value(b, p, e, LM_KEY_TITLE, f, LM_FIELD_CAP, LM_SEMI)
341 if l < 0 { return LM_ERR_MALFORMED }
342 items[base + LM_I_TITLE] = (lm_dup(f, lm_len(f))) as i64
343 l = lm_value(b, p, e, LM_KEY_DOWNLOADS, f, LM_FIELD_CAP, LM_SEMI)
344 if l < 0 { return LM_ERR_MALFORMED }
345 var dn: i64 = 0
346 if l > 0 { dn = lm_atoi(f) }
347 if dn < 0 { dn = 0 }
348 items[base + LM_I_DOWNLOADS] = dn
349 l = lm_value(b, p, e, LM_KEY_SUBJECT, f, LM_FIELD_CAP, LM_SEMI)
350 if l < 0 { return LM_ERR_MALFORMED }
351 items[base + LM_I_SUBJECTS] = (lm_dup(f, lm_len(f))) as i64
352 l = lm_value(b, p, e, LM_KEY_DESC, f, LM_DESC_HEAD + 1, LM_SPACE)
353 if l < 0 { return LM_ERR_MALFORMED }
354 items[base + LM_I_DESC] = (lm_dup(f, lm_len(f))) as i64
355 items[base + LM_I_DESC_LEN] = l
356 if l > LM_DESC_HEAD { lst[LM_L_TRUNCATED_DESCS] = lst[LM_L_TRUNCATED_DESCS] + 1 }
357 // census
358 let ty: *u8 = items[base + LM_I_TYPE] as *u8
359 if lm_streq(ty, LM_TYPE_MOVIES) == 1 { lst[LM_L_MOVIES] = lst[LM_L_MOVIES] + 1 }
360 else { if lm_streq(ty, LM_TYPE_TEXTS) == 1 { lst[LM_L_TEXTS] = lst[LM_L_TEXTS] + 1 }
361 else { if lm_streq(ty, LM_TYPE_AUDIO) == 1 { lst[LM_L_AUDIO] = lst[LM_L_AUDIO] + 1 }
362 else { if lm_streq(ty, LM_TYPE_IMAGE) == 1 { lst[LM_L_IMAGE] = lst[LM_L_IMAGE] + 1 }
363 else { lst[LM_L_OTHER] = lst[LM_L_OTHER] + 1 } } } }
364 lst[LM_L_DOWNLOADS] = lst[LM_L_DOWNLOADS] + dn
365 let dt: *u8 = items[base + LM_I_DATE] as *u8
366 if dt[0] != (0 as u8) {
367 if count == 0 { lst[LM_L_NEWEST] = dt as i64; lst[LM_L_OLDEST] = dt as i64 }
368 else {
369 if lm_date_cmp(dt, lst[LM_L_NEWEST] as *u8) > 0 { lst[LM_L_NEWEST] = dt as i64 }
370 if (lst[LM_L_OLDEST] as *u8)[0] == (0 as u8) { lst[LM_L_OLDEST] = dt as i64 }
371 else { if lm_date_cmp(dt, lst[LM_L_OLDEST] as *u8) < 0 { lst[LM_L_OLDEST] = dt as i64 } }
372 }
373 } else { if count == 0 { lst[LM_L_NEWEST] = ("" as *u8) as i64; lst[LM_L_OLDEST] = ("" as *u8) as i64 } }
374 count = count + 1
375 p = e
376 } else { return LM_ERR_MALFORMED } } }
377 }
378 lst[LM_L_ITEMS] = count
379 if count == 0 { return LM_ERR_NO_DOCS }
380 return LM_OK
381}
382// ---- subjects: split every item's joined subjects on the separator, count distinct, order by count desc ----
383// subj/counts hold up to cap distinct; returns the distinct count (a FLOOR when cap was reached: full[0]=1)
384func lm_subjects(items: *i64, n: i64, subj: *i64, counts: *i64, cap: i64, full: *i64) -> i64 {
385 var distinct: i64 = 0
386 full[0] = 0
387 let part: *u8 = sys_mmap(LM_FIELD_CAP)
388 var i: i64 = 0
389 while i < n {
390 let s: *u8 = items[i * LM_I_SLOTS + LM_I_SUBJECTS] as *u8
391 var k: i64 = 0
392 var st: i64 = 0
393 var go: i64 = 1
394 while go == 1 {
395 let c: i64 = s[k] as i64
396 var cut: i64 = 0
397 if c == 0 { cut = 1 }
398 if c == LM_SEMI { cut = 1 }
399 if cut == 1 {
400 var a: i64 = st
401 var z: i64 = k
402 while a < z { if (s[a] as i64) != LM_SPACE { break } a = a + 1 }
403 while z > a { if (s[z - 1] as i64) != LM_SPACE { break } z = z - 1 }
404 if z > a {
405 var j: i64 = 0
406 while j < z - a { part[j] = s[a + j]; j = j + 1 }
407 part[z - a] = 0 as u8
408 var found: i64 = 0 - 1
409 var d: i64 = 0
410 while d < distinct { if lm_streq(subj[d] as *u8, part) == 1 { found = d; break } d = d + 1 }
411 if found >= 0 { counts[found] = counts[found] + 1 }
412 else { if distinct < cap { subj[distinct] = (lm_dup(part, z - a)) as i64; counts[distinct] = 1; distinct = distinct + 1 } else { full[0] = 1 } }
413 }
414 st = k + 1
415 }
416 if c == 0 { go = 0 }
417 k = k + 1
418 }
419 i = i + 1
420 }
421 // order by count desc (stable insertion; the table is small)
422 var x: i64 = 1
423 while x < distinct {
424 var y: i64 = x
425 while y > 0 {
426 if counts[y] > counts[y - 1] {
427 let ts: i64 = subj[y]; subj[y] = subj[y - 1]; subj[y - 1] = ts
428 let tc: i64 = counts[y]; counts[y] = counts[y - 1]; counts[y - 1] = tc
429 } else { break }
430 y = y - 1
431 }
432 x = x + 1
433 }
434 return distinct
435}