code wiki / _hdl_build / nx_ingest_base.nx
nx_ingest_base.nx source
↩ module page · 206 lines · 8099 B
1// nx_ingest_base.nx -- LangIntel LI1: OO BASE CLASS for external-corpus ingest organs.
2// ib_read/ib_find/ib_field, ib_jstr (ws+escape-robust JSON string), ib_hash, ib_refuse (reason+fix),
3// ib_b* (buffer emit), ib_gh_issues (shared GitHub-issues parser -- now extracts REAL state so CLOSED
4// bugs = solved answer-keys are captured alongside OPEN gaps). Read-only external (Rule 26), bench-only (F616).
5import "nx_store_seed_lib.nx"
6import "nx_syscalls.nx"
7const IB_MAGIC_2048: i64 = 2048
8
9const IB_STDERR: i64 = 2
10const IB_D0: i64 = 48
11const IB_D9: i64 = 57
12const IB_B10: i64 = 10
13const IB_NUMB: i64 = 24
14const IB_Q: i64 = 34
15const IB_BSL: i64 = 92
16const IB_TAB: i64 = 9
17const IB_NL: i64 = 10
18const IB_COLON: i64 = 58
19const IB_SP: i64 = 32
20const IB_CR: i64 = 13
21const IB_HPRIME: i64 = 1000000007
22const IB_HMUL: i64 = 131
23const IB_GH_FCAP: i64 = 8388608
24const IB_GH_OCAP: i64 = 1048576
25const IB_GH_VCAP: i64 = 1024
26
27func ib_slen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
28func ib_wr(fd: i64, s: *u8) -> i64 { let n: i64 = ib_slen(s); sys_write(fd, s, n); return 0 }
29func ib_werr(s: *u8) -> i64 { ib_wr(IB_STDERR, s); return 0 }
30func ib_wn(fd: i64, v: i64) -> i64 {
31 if v == 0 { sys_write(fd, "0" as *u8, 1); return 0 }
32 var m: i64 = v
33 let d: *u8 = sys_mmap(IB_NUMB)
34 var k: i64 = 0
35 while m > 0 { d[k] = (IB_D0 + (m % IB_B10)) as u8; m = m / IB_B10; k = k + 1 }
36 let o: *u8 = sys_mmap(IB_NUMB)
37 var i: i64 = 0
38 while i < k { o[i] = d[k - 1 - i]; i = i + 1 }
39 sys_write(fd, o, k)
40 return 0
41}
42func ib_refuse(tag: *u8, reason: *u8, fix: *u8) -> i64 {
43 ib_werr(tag); ib_werr(" REFUSED verdict=RED reason=" as *u8); ib_werr(reason); ib_werr(" fix=" as *u8); ib_werr(fix); ib_werr("\n" as *u8)
44 return 0
45}
46func ib_hash(s: *u8, n: i64) -> i64 {
47 var h: i64 = 0
48 var i: i64 = 0
49 while i < n { h = (h * IB_HMUL + (s[i] as i64)) % IB_HPRIME; i = i + 1 }
50 return h
51}
52func ib_read(path: *u8, buf: *u8, cap: i64) -> i64 {
53 let fd: i64 = sys_openat_rd(path)
54 if fd < 0 { return 0 - 1 }
55 var n: i64 = 0
56 var go: i64 = 1
57 while go == 1 {
58 let base: i64 = buf as i64
59 let r: i64 = sys_read(fd, (base + n) as *u8, cap - n)
60 if r <= 0 { go = 0 } else { n = n + r }
61 if n >= cap { go = 0 }
62 }
63 sys_close(fd)
64 return n
65}
66func ib_find(buf: *u8, n: i64, needle: *u8, from: i64) -> i64 {
67 let m: i64 = ib_slen(needle)
68 if m == 0 { return 0 - 1 }
69 var i: i64 = from
70 while i + m <= n {
71 var j: i64 = 0
72 var ok: i64 = 1
73 while j < m { if buf[i+j] != needle[j] { ok = 0; j = m } else { j = j + 1 } }
74 if ok == 1 { if i > 0 { if buf[i-1] == (IB_BSL as u8) { ok = 0 } } }
75 if ok == 1 { return i }
76 i = i + 1
77 }
78 return 0 - 1
79}
80func ib_field(buf: *u8, n: i64, needle: *u8, from: i64, val: *u8, vcap: i64) -> i64 {
81 let p: i64 = ib_find(buf, n, needle, from)
82 if p < 0 { return 0 - 1 }
83 let vs: i64 = p + ib_slen(needle)
84 var i: i64 = vs
85 var k: i64 = 0
86 var go: i64 = 1
87 while go == 1 {
88 if i >= n { go = 0 }
89 else {
90 if buf[i] == (IB_BSL as u8) { if i + 1 < n { if k < vcap - 1 { val[k] = buf[i+1]; k = k + 1 } i = i + 2 } else { i = i + 1 } }
91 else { if buf[i] == (IB_Q as u8) { go = 0 } else { if k < vcap - 1 { val[k] = buf[i]; k = k + 1 } i = i + 1 } }
92 }
93 }
94 val[k] = 0 as u8
95 return i + 1
96}
97func ib_jstr(buf: *u8, n: i64, key: *u8, from: i64, val: *u8, vcap: i64) -> i64 {
98 let p: i64 = ib_find(buf, n, key, from)
99 if p < 0 { return 0 - 1 }
100 var i: i64 = p + ib_slen(key)
101 var g1: i64 = 1
102 while g1 == 1 {
103 if i >= n { return 0 - 1 }
104 let c: i64 = buf[i] as i64
105 if c == IB_COLON { i = i + 1 } else { if c == IB_SP { i = i + 1 } else { if c == IB_TAB { i = i + 1 } else { if c == IB_NL { i = i + 1 } else { if c == IB_CR { i = i + 1 } else { g1 = 0 } } } } }
106 }
107 if buf[i] != (IB_Q as u8) { return 0 - 1 }
108 i = i + 1
109 var k: i64 = 0
110 var g2: i64 = 1
111 while g2 == 1 {
112 if i >= n { g2 = 0 }
113 else {
114 if buf[i] == (IB_BSL as u8) { if i + 1 < n { if k < vcap - 1 { val[k] = buf[i+1]; k = k + 1 } i = i + 2 } else { i = i + 1 } }
115 else { if buf[i] == (IB_Q as u8) { g2 = 0 } else { if k < vcap - 1 { val[k] = buf[i]; k = k + 1 } i = i + 1 } }
116 }
117 }
118 val[k] = 0 as u8
119 return i + 1
120}
121func ib_alldigits(s: *u8) -> i64 {
122 var i: i64 = 0
123 var ok: i64 = 1
124 while s[i] != (0 as u8) { if (s[i] as i64) < IB_D0 { ok = 0 } if (s[i] as i64) > IB_D9 { ok = 0 } i = i + 1 }
125 if i == 0 { ok = 0 }
126 return ok
127}
128func ib_bput(buf: *u8, o: i64, s: *u8) -> i64 {
129 var oo: i64 = o
130 var j: i64 = 0
131 while s[j] != (0 as u8) { buf[oo] = s[j]; oo = oo + 1; j = j + 1 }
132 return oo
133}
134func ib_bputc(buf: *u8, o: i64, c: i64) -> i64 { buf[o] = c as u8; return o + 1 }
135func ib_bputn(buf: *u8, o: i64, v: i64) -> i64 {
136 if v == 0 { buf[o] = IB_D0 as u8; return o + 1 }
137 var m: i64 = v
138 let d: *u8 = sys_mmap(IB_NUMB)
139 var k: i64 = 0
140 while m > 0 { d[k] = (IB_D0 + (m % IB_B10)) as u8; m = m / IB_B10; k = k + 1 }
141 var oo: i64 = o
142 var i: i64 = k - 1
143 while i >= 0 { buf[oo] = d[i]; oo = oo + 1; i = i - 1 }
144 return oo
145}
146// SHARED GitHub-issues parser. anchor = issue-html_url field prefix. Extracts number+REAL state+title;
147// state=open (an unsolved GAP) or closed (a SOLVED answer-key). rows: number TAB state TAB flag TAB url TAB hash TAB title.
148func ib_gh_issues(raw_path: *u8, plane_prefix: *u8, anchor: *u8, url_base: *u8, label_needle: *u8, flag_yes: *u8, flag_no: *u8) -> i64 {
149 let fbuf: *u8 = sys_mmap(IB_GH_FCAP)
150 let n: i64 = ib_read(raw_path, fbuf, IB_GH_FCAP)
151 if n <= 0 { return 0 - 1 }
152 let out: *u8 = sys_mmap(IB_GH_OCAP)
153 let num: *u8 = sys_mmap(IB_GH_VCAP)
154 let title: *u8 = sys_mmap(IB_GH_VCAP)
155 let state: *u8 = sys_mmap(IB_GH_VCAP)
156 var o: i64 = 0
157 var rows: i64 = 0
158 var p: i64 = 0
159 var go: i64 = 1
160 while go == 1 {
161 let a: i64 = ib_find(fbuf, n, anchor, p)
162 if a < 0 { go = 0 }
163 else {
164 let q1: i64 = ib_field(fbuf, n, anchor, a, num, IB_GH_VCAP)
165 if q1 < 0 { go = 0 }
166 else {
167 if ib_alldigits(num) == 0 { p = q1 }
168 else {
169 let a2: i64 = ib_find(fbuf, n, anchor, q1)
170 var rend: i64 = n
171 if a2 >= 0 { rend = a2 }
172 let q2: i64 = ib_jstr(fbuf, rend, "\"title\"" as *u8, q1, title, IB_GH_VCAP)
173 state[0] = 0 as u8
174 let q3: i64 = ib_jstr(fbuf, rend, "\"state\"" as *u8, q1, state, IB_GH_VCAP)
175 if state[0] == (0 as u8) { state[0] = 63 as u8; state[1] = 0 as u8 }
176 var flagged: i64 = 0
177 let mp: i64 = ib_find(fbuf, rend, label_needle, a)
178 if mp >= 0 { flagged = 1 }
179 let tl: i64 = ib_slen(title)
180 let h: i64 = ib_hash(title, tl)
181 o = ib_bput(out, o, num)
182 o = ib_bputc(out, o, IB_TAB)
183 o = ib_bput(out, o, state)
184 o = ib_bputc(out, o, IB_TAB)
185 if flagged == 1 { o = ib_bput(out, o, flag_yes) } else { o = ib_bput(out, o, flag_no) }
186 o = ib_bputc(out, o, IB_TAB)
187 o = ib_bput(out, o, url_base)
188 o = ib_bput(out, o, num)
189 o = ib_bputc(out, o, IB_TAB)
190 o = ib_bputn(out, o, h)
191 o = ib_bputc(out, o, IB_TAB)
192 o = ib_bput(out, o, title)
193 o = ib_bputc(out, o, IB_NL)
194 rows = rows + 1
195 p = rend
196 if a2 < 0 { go = 0 }
197 if o > IB_GH_OCAP - IB_MAGIC_2048 { go = 0 }
198 }
199 }
200 }
201 }
202 if rows < 1 { return 0 }
203 let rc: i64 = sts_seed(plane_prefix, out, o)
204 if rc < 0 { return 0 - 2 }
205 return rows
206}