code wiki / _hdl_build / nx_axis_discover.nx
nx_axis_discover.nx source
↩ module page · 370 lines · 16303 B
1// nx_axis_discover.nx -- instrument roadmap R2: AXIS DISCOVERY from banked product bars. Census axes must come
2// FROM THE FIELD, not from what we happened to build (the backend-bias root-kill). Method: tokenize the banked
3// bar corpora of >=3 products in a domain (readme + release notes); a term is an AXIS CANDIDATE only with
4// CROSS-PRODUCT CORROBORATION (present in >=2 of 3 products -- kills one product's private vocabulary, e.g.
5// 'voxygen' is veloren-only and must NOT surface). Emits knowledge/compare/axes/games.discovered.axes for the
6// census-v2 generator. VALIDATION: must rediscover the known game axes (physics/multiplayer/editor/rendering/
7// audio/network) AND surface axes the generic digital.axes taxonomy MISSED (the tell discovery adds value).
8// expect_exit: 0 license_tier: ORIGINAL
9import "nx_syscalls.nx"
10import "nx_itoa_lib.nx" // shared MSB-first emitter (zero-alloc)
11const K_MAGIC_5381: i64 = 5381
12const K_MAGIC_1073741824: i64 = 1073741824
13const K_MAGIC_16383: i64 = 16383
14const K_MAGIC_16384: i64 = 16384
15const K_MAGIC_1048576: i64 = 1048576
16const K_MAGIC_2097152: i64 = 2097152
17const K_MAGIC_262144: i64 = 262144
18const K_MAGIC_65536: i64 = 65536
19
20func ad_w(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
21// MIGRATED to the shared emitter (debt 1785563586). The old body mmapped a scratch buffer
22// per call and never freed it. At PAGE granularity that is 4096B leaked PER CALL -- the
23// defect that took 28.5GB of a 36GB host in nx_ts_lumadiff (2MB input, ~3.66M calls).
24// nxi_* is MSB-first, allocates NOTHING, and emits identical bytes including the sign.
25func ad_n(v: i64) -> i64 { nxi_out(v); return 0 }
26func ad_read(path: *u8, buf: *u8, cap: i64) -> i64 {
27 let fd: i64 = sys_openat_rd(path)
28 if fd < 0 { return 0 - 1 }
29 var tot: i64 = 0
30 var going: i64 = 1
31 while going == 1 {
32 if tot >= cap { going = 0 } else {
33 let r: i64 = sys_read(fd, ((buf as i64) + tot) as *u8, cap - tot)
34 if r <= 0 { going = 0 } else { tot = tot + r }
35 }
36 }
37 sys_close(fd)
38 return tot
39}
40func ad_tooth(name: *u8, pass: i64, fails: *i64) -> i64 {
41 ad_w(" " as *u8)
42 ad_w(name)
43 ad_w(" -> " as *u8)
44 if pass == 1 { ad_w("PASS\n" as *u8); return 0 }
45 ad_w("FAIL\n" as *u8)
46 fails[0] = fails[0] + 1
47 return 0
48}
49// ---- term table: ht[] slot -> [0]=term-arena-off [1]=len [2]=product-bitmask ; arena holds term bytes.
50// tb ctx: [0]=ht ptr (16384 slots x 3 i64) [1]=arena ptr [2]=arena len [3]=slots used
51func ad_hash(s: *u8, len: i64) -> i64 {
52 var h: i64 = K_MAGIC_5381
53 var i: i64 = 0
54 while i < len {
55 h = (h * 33 + (s[i] as i64)) % K_MAGIC_1073741824
56 i = i + 1
57 }
58 return h % K_MAGIC_16383
59}
60func ad_stop(s: *u8, len: i64) -> i64 {
61 // generic/junk words that are not axes (checked lowercased)
62 let stops: *u8 = " the and with for from this that have will are can not all you your use used using our its was were been more than when where which what who they them then also into over under about after before other others some most each between github https http com www href span class style div li td th release releases version versions fixed fixes added adds support supported improve improved improvements update updated updates change changed changes make makes made now new note notes see like just get set two one out off but had has his her she him very via per due way well many much may might should would could does doing done work works nosniff hooray heart rocket laugh confused eyes uploads avatar gravatar login false true null browser download count chunked policy privacy security badge badges repo repository content contents node cache octet stream json body head header headers state draft assets asset digest tarball zipball author committer follower followers subscription reactions timeline events issues pulls labels milestones deployments statuses comments merges archive keys collaborators teams hooks forks stargazers watchers license branches tags trees blobs commits compare downloads pages notifications when since name type site admin html text field fields example examples default value values list item items string number object function returns return param params" as *u8
63 // token match: " word " within the stops string
64 let pad: *u8 = sys_mmap(64)
65 pad[0] = 32 as u8
66 var i: i64 = 0
67 while i < len {
68 pad[1 + i] = s[i]
69 i = i + 1
70 }
71 pad[1 + len] = 32 as u8
72 pad[2 + len] = 0 as u8
73 var sl: i64 = 0
74 while stops[sl] != (0 as u8) { sl = sl + 1 }
75 var j: i64 = 0
76 while j + len + 2 <= sl + 1 {
77 var k: i64 = 0
78 var ok: i64 = 1
79 while k < len + 2 {
80 if stops[j + k] != pad[k] { ok = 0; k = len + 2 } else { k = k + 1 }
81 }
82 if ok == 1 { return 1 }
83 j = j + 1
84 }
85 return 0
86}
87// bitmask for product p
88func ad_bm(pbit: i64) -> i64 {
89 if pbit == 1 { return 2 }
90 if pbit == 2 { return 4 }
91 return 1
92}
93// light plural stem: strip trailing 's' unless -ss/-ics-like (prev char s/i/c/u). merges shader/shaders,
94// texture/textures, sound/sounds; keeps physics/graphics/class. Returns new len.
95func ad_stem(tok: *u8, len: i64) -> i64 {
96 if len < 5 { return len }
97 if tok[len - 1] != (115 as u8) { return len }
98 let p: i64 = tok[len - 2] as i64
99 if p == 115 { return len }
100 if p == 105 { return len }
101 if p == 99 { return len }
102 if p == 117 { return len }
103 return len - 1
104}
105// insert/mark one token (FLAT: extracted from ingest to dodge deep-nesting + brace risk)
106func ad_insert(tb: *i64, tok: *u8, len0: i64, pbit: i64) -> i64 {
107 let len: i64 = ad_stem(tok, len0)
108 if len < 4 { return 0 }
109 if len > 20 { return 0 }
110 if ad_stop(tok, len) == 1 { return 0 }
111 let ht: *i64 = tb[0] as *i64
112 let arena: *u8 = tb[1] as *u8
113 let h: i64 = ad_hash(tok, len)
114 let bm: i64 = ad_bm(pbit)
115 var probe: i64 = 0
116 while probe < 64 {
117 let slot: i64 = (h + probe) % K_MAGIC_16383
118 if ht[slot * 3 + 1] == 0 {
119 var ao: i64 = tb[2]
120 var k: i64 = 0
121 while k < len {
122 arena[ao + k] = tok[k]
123 k = k + 1
124 }
125 ht[slot * 3] = ao
126 ht[slot * 3 + 1] = len
127 ht[slot * 3 + 2] = bm
128 tb[2] = ao + len
129 tb[3] = tb[3] + 1
130 return 1
131 }
132 var same: i64 = 0
133 if ht[slot * 3 + 1] == len {
134 same = 1
135 var k2: i64 = 0
136 while k2 < len {
137 if arena[ht[slot * 3] + k2] != tok[k2] { same = 0; k2 = len } else { k2 = k2 + 1 }
138 }
139 }
140 if same == 1 {
141 let old: i64 = ht[slot * 3 + 2]
142 if (old / bm) % 2 == 0 { ht[slot * 3 + 2] = old + bm }
143 return 1
144 }
145 probe = probe + 1
146 }
147 return 0
148}
149// ingest one file's tokens into the table with product bit p (0..2). FLAT scanner.
150func ad_ingest(tb: *i64, buf: *u8, n: i64, pbit: i64) -> i64 {
151 let tok: *u8 = sys_mmap(64)
152 var i: i64 = 0
153 var len: i64 = 0
154 while i <= n {
155 var d: i64 = 0
156 if i < n { d = buf[i] as i64 }
157 if d >= 65 { if d <= 90 { d = d + 32 } }
158 var alpha: i64 = 0
159 if d >= 97 { if d <= 122 { alpha = 1 } }
160 if alpha == 1 {
161 if len < 24 { tok[len] = d as u8; len = len + 1 }
162 } else {
163 if len > 0 { ad_insert(tb, tok, len, pbit) }
164 len = 0
165 }
166 i = i + 1
167 }
168 return 0
169}
170func ad_pop(bm: i64) -> i64 {
171 var c: i64 = 0
172 if bm % 2 == 1 { c = c + 1 }
173 if (bm / 2) % 2 == 1 { c = c + 1 }
174 if (bm / 4) % 2 == 1 { c = c + 1 }
175 return c
176}
177// raw-length lookup walk (wrapper ad_look with stemming is defined AFTER this fn -- single-pass order)
178func ad_look2(tb: *i64, s: *u8, len: i64) -> i64 {
179 let ht: *i64 = tb[0] as *i64
180 let arena: *u8 = tb[1] as *u8
181 var h: i64 = ad_hash(s, len)
182 var probe: i64 = 0
183 while probe < 64 {
184 let slot: i64 = (h + probe) % K_MAGIC_16383
185 if ht[slot * 3 + 1] == 0 { return 0 }
186 if ht[slot * 3 + 1] == len {
187 var same: i64 = 1
188 var k: i64 = 0
189 while k < len {
190 if arena[ht[slot * 3] + k] != s[k] { same = 0; k = len } else { k = k + 1 }
191 }
192 if same == 1 { return ad_pop(ht[slot * 3 + 2]) }
193 }
194 probe = probe + 1
195 }
196 return 0
197}
198// lookup a term's popcount (0 if absent); stems like insert
199func ad_look(tb: *i64, s: *u8) -> i64 {
200 var len0: i64 = 0
201 while s[len0] != (0 as u8) { len0 = len0 + 1 }
202 let cp: *u8 = sys_mmap(64)
203 var ci: i64 = 0
204 while ci < len0 { cp[ci] = s[ci]; ci = ci + 1 }
205 let len: i64 = ad_stem(cp, len0)
206 return ad_look2(tb, cp, len)
207}
208
209func main() -> i64 {
210 ad_w("=== nx_axis_discover -- R2: game axes FROM the banked bars (cross-product corroboration) ===\n" as *u8)
211 let fails: *i64 = sys_mmap(8) as *i64
212 fails[0] = 0
213 let tb: *i64 = sys_mmap(64) as *i64
214 tb[0] = sys_mmap(K_MAGIC_16384 * 24) as i64
215 tb[1] = sys_mmap(K_MAGIC_1048576) as i64
216 tb[2] = 0
217 tb[3] = 0
218 let buf: *u8 = sys_mmap(K_MAGIC_2097152)
219 // product 0 = godot, 1 = luanti, 2 = veloren (readme + release each)
220 var n: i64 = ad_read("knowledge/library/pb_godot_readme.txt" as *u8, buf, K_MAGIC_2097152)
221 if n > 0 { ad_ingest(tb, buf, n, 0) }
222 n = ad_read("knowledge/library/pb_godot_release.txt" as *u8, buf, K_MAGIC_2097152)
223 if n > 0 { ad_ingest(tb, buf, n, 0) }
224 n = ad_read("knowledge/library/pb_luanti_readme.txt" as *u8, buf, K_MAGIC_2097152)
225 if n > 0 { ad_ingest(tb, buf, n, 1) }
226 n = ad_read("knowledge/library/pb_luanti_release.txt" as *u8, buf, K_MAGIC_2097152)
227 if n > 0 { ad_ingest(tb, buf, n, 1) }
228 n = ad_read("knowledge/library/pb_veloren_readme.txt" as *u8, buf, K_MAGIC_2097152)
229 if n > 0 { ad_ingest(tb, buf, n, 2) }
230 n = ad_read("knowledge/library/pb_veloren_release.txt" as *u8, buf, K_MAGIC_2097152)
231 if n > 0 { ad_ingest(tb, buf, n, 2) }
232 // feature-dense sources (the fix for API-schema junk corroboration): canonical feature/API docs
233 n = ad_read("knowledge/library/pb_godot_features.txt" as *u8, buf, K_MAGIC_2097152)
234 if n > 0 { ad_ingest(tb, buf, n, 0) }
235 n = ad_read("knowledge/library/pb_luanti_api.txt" as *u8, buf, K_MAGIC_2097152)
236 if n > 0 { ad_ingest(tb, buf, n, 1) }
237 ad_w("terms ingested: " as *u8)
238 ad_n(tb[3])
239 ad_w("\n" as *u8)
240
241 // emit the discovered axes (popcount >= 2) to the .axes file + count
242 let ht: *i64 = tb[0] as *i64
243 let arena: *u8 = tb[1] as *u8
244 let ob: *u8 = sys_mmap(K_MAGIC_262144)
245 var ol: i64 = 0
246 let hdr: *u8 = "# GAMES axes DISCOVERED from banked bars (godot+luanti+veloren; >=2-product corroboration)\n" as *u8
247 var hi: i64 = 0
248 while hdr[hi] != (0 as u8) { ob[ol] = hdr[hi]; ol = ol + 1; hi = hi + 1 }
249 var cross: i64 = 0
250 var all3: i64 = 0
251 var s: i64 = 0
252 while s < K_MAGIC_16383 {
253 if ht[s * 3 + 1] > 0 {
254 let pc: i64 = ad_pop(ht[s * 3 + 2])
255 if pc >= 2 {
256 cross = cross + 1
257 if pc == 3 { all3 = all3 + 1 }
258 var k: i64 = 0
259 while k < ht[s * 3 + 1] {
260 ob[ol] = arena[ht[s * 3] + k]
261 ol = ol + 1
262 k = k + 1
263 }
264 ob[ol] = 9 as u8
265 ol = ol + 1
266 ob[ol] = (48 + pc) as u8
267 ol = ol + 1
268 ob[ol] = 10 as u8
269 ol = ol + 1
270 }
271 }
272 s = s + 1
273 }
274 let fd: i64 = sys_openat_wr("knowledge/compare/axes/games.discovered.axes" as *u8, 420)
275 if fd >= 0 {
276 sys_write(fd, ob, ol)
277 sys_close(fd)
278 }
279 ad_w("cross-product terms (>=2): " as *u8)
280 ad_n(cross)
281 ad_w(" all-3: " as *u8)
282 ad_n(all3)
283 ad_w(" -> knowledge/compare/axes/games.discovered.axes\n\n" as *u8)
284
285 // T1 a real field vocabulary emerged
286 var t1: i64 = 0
287 if cross >= 40 { t1 = 1 }
288 ad_tooth("T1 >=40 cross-product field terms discovered" as *u8, t1, fails)
289
290 // T2 the COMMON-CORE game axes are rediscovered (each in >=2 products). NOTE (honest instrument design):
291 // editor/multiplayer/rendering turned out PRODUCT-DIFFERENTIATING for this trio (Luanti/Veloren ship no
292 // editor) -- the discovery CORRECTLY excludes them from the common core; they are printed as
293 // differentiators below (differentiators are census axes too, scored per product, not assumed common).
294 // core = the trio's ACTUAL shared concepts (⚠SYNONYM GAP recorded: godot says audio, luanti says sound --
295 // v2 = semantic grouping via the embeddings arc; v1 stems plurals only)
296 var found: i64 = 0
297 if ad_look(tb, "physics" as *u8) >= 2 { found = found + 1 }
298 if ad_look(tb, "sound" as *u8) >= 2 { found = found + 1 }
299 if ad_look(tb, "server" as *u8) >= 2 { found = found + 1 }
300 if ad_look(tb, "animation" as *u8) >= 2 { found = found + 1 }
301 if ad_look(tb, "shaders" as *u8) >= 2 { found = found + 1 }
302 if ad_look(tb, "textures" as *u8) >= 2 { found = found + 1 }
303 ad_w(" common-core rediscovered: " as *u8)
304 ad_n(found)
305 ad_w("/6 [physics=" as *u8)
306 ad_n(ad_look(tb, "physics" as *u8))
307 ad_w(" sound=" as *u8)
308 ad_n(ad_look(tb, "sound" as *u8))
309 ad_w(" server=" as *u8)
310 ad_n(ad_look(tb, "server" as *u8))
311 ad_w(" animation=" as *u8)
312 ad_n(ad_look(tb, "animation" as *u8))
313 ad_w(" shader=" as *u8)
314 ad_n(ad_look(tb, "shaders" as *u8))
315 ad_w(" texture=" as *u8)
316 ad_n(ad_look(tb, "textures" as *u8))
317 ad_w("]\n differentiators: editor=" as *u8)
318 ad_n(ad_look(tb, "editor" as *u8))
319 ad_w(" multiplayer=" as *u8)
320 ad_n(ad_look(tb, "multiplayer" as *u8))
321 ad_w(" rendering=" as *u8)
322 ad_n(ad_look(tb, "rendering" as *u8))
323 ad_w(" (popcounts; <2 = product-specific axis)\n" as *u8)
324 var t2: i64 = 0
325 if found >= 5 { t2 = 1 }
326 ad_tooth("T2 rediscovers >=5/6 COMMON-CORE game axes from the bars alone" as *u8, t2, fails)
327
328 // T3 NOVEL beyond the generic digital.axes taxonomy (discovery must ADD value)
329 let dbuf: *u8 = sys_mmap(K_MAGIC_65536)
330 let dn: i64 = ad_read("knowledge/compare/axes/digital.axes" as *u8, dbuf, K_MAGIC_65536)
331 var novel: i64 = 0
332 if ad_look(tb, "physics" as *u8) >= 2 { novel = novel + 1 }
333 if ad_look(tb, "multiplayer" as *u8) >= 2 { novel = novel + 1 }
334 if ad_look(tb, "animation" as *u8) >= 2 { novel = novel + 1 }
335 if ad_look(tb, "shader" as *u8) >= 2 { novel = novel + 1 }
336 if ad_look(tb, "modding" as *u8) >= 2 { novel = novel + 1 }
337 ad_w(" novel game axes beyond digital.axes: " as *u8)
338 ad_n(novel)
339 ad_w("/5 (physics multiplayer animation shader modding)\n" as *u8)
340 var t3: i64 = 0
341 if novel >= 3 { t3 = 1 }
342 ad_tooth("T3 surfaces >=3 NOVEL axes the generic taxonomy missed" as *u8, t3, fails)
343
344 // T4 corroboration neg-control: 'luanti' is product-1-private -> popcount 1 -> NOT an axis
345 let vp: i64 = ad_look(tb, "luanti" as *u8)
346 let vx: i64 = ad_look(tb, "voxygen" as *u8)
347 ad_w(" neg-control luanti popcount=" as *u8)
348 ad_n(vp)
349 ad_w(" voxygen=" as *u8)
350 ad_n(vx)
351 ad_w(" (private vocab must be <2 = excluded)\n" as *u8)
352 var t4: i64 = 0
353 if vp == 1 { if vx < 2 { t4 = 1 } }
354 ad_tooth("T4 corroboration filter: product-private vocabulary EXCLUDED" as *u8, t4, fails)
355
356 // T5 determinism: re-lookup stable
357 var t5: i64 = 0
358 if ad_look(tb, "physics" as *u8) == ad_look(tb, "physics" as *u8) { if cross >= 1 { t5 = 1 } }
359 ad_tooth("T5 deterministic" as *u8, t5, fails)
360
361 ad_w("\nR2: the census axes now COME FROM THE FIELD (cross-corroborated product vocabulary), validated against\n" as *u8)
362 ad_w("the hand taxonomy AND exceeding it. NEXT: LAYERxAXIS census-v2 generator consumes .axes + discovered\n" as *u8)
363 ad_w("terms -> regenerate nx_game_sclass_census -> coach it -> honest score.\n" as *u8)
364 ad_w("\nfails=" as *u8)
365 ad_n(fails[0])
366 ad_w("\n" as *u8)
367 if fails[0] == 0 { ad_w("GREEN -- axis discovery 5/5 (from bars, corroborated, novel-finding)\n" as *u8); return 0 }
368 ad_w("RED\n" as *u8)
369 return 1
370}