code wiki / _hdl_build / nx_funcmine.nx
nx_funcmine.nx source
↩ module page · 237 lines · 10452 B
1// nx_funcmine.nx -- FUNCTION MINING for OO consolidation (operator 2026-07-20: "archive the functions
2// for functions mining... object oriented without losing functionalities"). Scans a dir's .nx files,
3// extracts every function BODY, FNV-1a hashes it, and reports IDENTICAL bodies appearing in 2+ places
4// = the TRUE function-level dedup the file-level detector can't see: a copy-pasted helper across N organs
5// is a base/shared-lib EXTRACTION target (rule 15: pattern in 3+ -> shared/), behaviour-preserving because
6// the body is byte-identical. Bucket-chained hash (O(files+funcs), not O(funcs^2)). Emits the mining
7// substrate for the SHARP2 archive.
8// argv: <dir>
9// Report: DUPFUNC hash=<h> copies=<n> : <file>:<func> | ... , then NX-FUNCMINE files=F funcs=N dupgroups=G dupfuncs=D
10// exit 0 = ran. Deterministic. license_tier: ORIGINAL No hw writes (Rule 26).
11import "nx_seat_drive_lib.nx"
12import "nx_deploy_lib.nx"
13import "nx_syscalls.nx"
14const FM_MAGIC_524288: i64 = 524288
15const FM_MAGIC_262144: i64 = 262144
16const FM_MAGIC_1024: i64 = 1024
17const FM_MAGIC_2048: i64 = 2048
18
19const FM_MAXREC: i64 = 60000
20const FM_NB: i64 = 65536
21const FM_NAMEBUF: i64 = 6291456
22const FM_FILECAP: i64 = 2097152
23const FM_MAXFILES: i64 = 12000
24const FM_FNV_OFF: i64 = 1469598103934665603
25const FM_FNV_PRIME: i64 = 1099511628211
26
27func fm_isid(c: i64) -> i64 {
28 if c >= 97 { if c <= 122 { return 1 } }
29 if c >= 65 { if c <= 90 { return 1 } }
30 if c >= 48 { if c <= 57 { return 1 } }
31 if c == 95 { return 1 }
32 return 0
33}
34func fm_ends_nx(nm: *u8) -> i64 {
35 var n: i64 = 0
36 while nm[n] != (0 as u8) { n = n + 1 }
37 if n < 3 { return 0 }
38 if nm[n-3] == (46 as u8) { if nm[n-2] == (110 as u8) { if nm[n-1] == (120 as u8) { return 1 } } }
39 return 0
40}
41func fm_fnv(buf: *u8, s: i64, e: i64) -> i64 {
42 var h: i64 = FM_FNV_OFF
43 var i: i64 = s
44 while i < e { h = h ^ (buf[i] as i64); h = h * FM_FNV_PRIME; i = i + 1 }
45 return h
46}
47
48func main(argc: i64, argv: *i64) -> i64 {
49 if argc < 2 { sd_w("usage: nx_funcmine <dir>\n" as *u8); sys_exit(2); return 2 }
50 let dir: *u8 = argv[1] as *u8
51 if argc >= 3 { let ofd: i64 = sys_openat_wr(argv[2] as *u8, 0x1a4); if ofd >= 0 { sys_dup3(ofd, 1, 0) } }
52
53 let fpbuf: *u8 = sys_mmap(FM_MAGIC_524288)
54 let fpoff: *i64 = sys_mmap(8 * FM_MAXFILES) as *i64
55 var fpb: i64 = 0
56 var nfiles: i64 = 0
57
58 let rhash: *i64 = sys_mmap(8 * FM_MAXREC) as *i64
59 let rfile: *i64 = sys_mmap(8 * FM_MAXREC) as *i64
60 let rnoff: *i64 = sys_mmap(8 * FM_MAXREC) as *i64
61 let rnlen: *i64 = sys_mmap(8 * FM_MAXREC) as *i64
62 let nbuf: *u8 = sys_mmap(FM_NAMEBUF)
63 var nb: i64 = 0
64 var nrec: i64 = 0
65
66 let src: *u8 = sys_mmap(FM_FILECAP)
67
68 // scan dir for .nx files
69 let dfd: i64 = sys_openat_rd(dir)
70 if dfd < 0 { sd_w("NX-FUNCMINE dir unreadable\n" as *u8); sys_exit(2); return 2 }
71 let dbuf: *u8 = sys_mmap(FM_MAGIC_262144)
72 var go: i64 = 1
73 while go == 1 {
74 let dn: i64 = sys_getdents64(dfd, dbuf, FM_MAGIC_262144)
75 if dn <= 0 { go = 0 } else {
76 var off: i64 = 0
77 while off < dn {
78 let rec: *u8 = ((dbuf as i64) + off) as *u8
79 let ty: i64 = dirent_type(rec)
80 let nm: *u8 = dirent_name(rec)
81 if ty != 4 { if fm_ends_nx(nm) == 1 { if nfiles < FM_MAXFILES {
82 // path = dir + "/" + nm, read it
83 let p: *u8 = sys_mmap(FM_MAGIC_1024)
84 var po: i64 = 0
85 var di: i64 = 0
86 while dir[di] != (0 as u8) { p[po] = dir[di]; po = po + 1; di = di + 1 }
87 p[po] = 47 as u8; po = po + 1
88 var mi: i64 = 0
89 while nm[mi] != (0 as u8) { p[po] = nm[mi]; po = po + 1; mi = mi + 1 }
90 p[po] = 0 as u8
91 // record file path for reporting
92 let thisfile: i64 = nfiles
93 fpoff[nfiles] = fpb
94 var cp: i64 = 0
95 while cp < po { fpbuf[fpb] = p[cp]; fpb = fpb + 1; cp = cp + 1 }
96 fpbuf[fpb] = 0 as u8; fpb = fpb + 1
97 nfiles = nfiles + 1
98 // read + extract funcs
99 let fd: i64 = sys_openat_rd(p)
100 if fd >= 0 {
101 var slen: i64 = 0
102 var r: i64 = sys_read(fd, src, FM_FILECAP - 1)
103 while r > 0 { slen = slen + r; if slen >= FM_FILECAP - 1 { r = 0 } else { r = sys_read(fd, src + slen, FM_FILECAP - 1 - slen) } }
104 sys_close(fd)
105 // find func starts (func NAME( at BOL)
106 let fstart: *i64 = sys_mmap(8 * FM_MAGIC_2048) as *i64
107 let fnoffs: *i64 = sys_mmap(8 * FM_MAGIC_2048) as *i64
108 let fnlens: *i64 = sys_mmap(8 * FM_MAGIC_2048) as *i64
109 var nf: i64 = 0
110 var i: i64 = 0
111 var bol: i64 = 1
112 while i < slen {
113 if bol == 1 {
114 var m: i64 = 1
115 if i + 5 >= slen { m = 0 }
116 if m == 1 { if src[i] != (102 as u8) { m = 0 } }
117 if m == 1 { if src[i+1] != (117 as u8) { m = 0 } }
118 if m == 1 { if src[i+2] != (110 as u8) { m = 0 } }
119 if m == 1 { if src[i+3] != (99 as u8) { m = 0 } }
120 if m == 1 { if src[i+4] != (32 as u8) { m = 0 } }
121 if m == 1 { if nf < FM_MAGIC_2048 {
122 let ns: i64 = i + 5
123 var ne: i64 = ns
124 var s2: i64 = 1
125 while s2 == 1 { if ne >= slen { s2 = 0 } else { if src[ne] == (40 as u8) { s2 = 0 } else { if fm_isid(src[ne] as i64) == 1 { ne = ne + 1 } else { s2 = 0 } } } }
126 if ne > ns { fstart[nf] = i; fnoffs[nf] = ns; fnlens[nf] = ne - ns; nf = nf + 1 }
127 } }
128 }
129 let c: i64 = src[i] as i64
130 if c == 10 { bol = 1 } else { bol = 0 }
131 i = i + 1
132 }
133 // hash each func body [fstart[k], fstart[k+1] or slen), record
134 var k: i64 = 0
135 while k < nf {
136 if nrec < FM_MAXREC {
137 let bs: i64 = fstart[k]
138 var be: i64 = slen
139 if k + 1 < nf { be = fstart[k+1] }
140 rhash[nrec] = fm_fnv(src, bs, be)
141 rfile[nrec] = thisfile
142 // copy func name into nbuf
143 let no: i64 = fnoffs[k]
144 let nl: i64 = fnlens[k]
145 if nb + nl + 1 < FM_NAMEBUF {
146 rnoff[nrec] = nb
147 rnlen[nrec] = nl
148 var w: i64 = 0
149 while w < nl { nbuf[nb] = src[no + w]; nb = nb + 1; w = w + 1 }
150 nrec = nrec + 1
151 }
152 }
153 k = k + 1
154 }
155 }
156 } } }
157 off = off + dirent_reclen(rec)
158 }
159 }
160 }
161 sys_close(dfd)
162
163 // bucket-chain by hash, report groups of >=2 with identical hash
164 let bhead: *i64 = sys_mmap(8 * FM_NB) as *i64
165 let rnext: *i64 = sys_mmap(8 * FM_MAXREC) as *i64
166 let done: *u8 = sys_mmap(FM_MAXREC)
167 var bi: i64 = 0
168 while bi < FM_NB { bhead[bi] = 0 - 1; bi = bi + 1 }
169 var ri: i64 = 0
170 while ri < nrec {
171 let b: i64 = rhash[ri] & (FM_NB - 1)
172 rnext[ri] = bhead[b]
173 bhead[b] = ri
174 done[ri] = 0 as u8
175 ri = ri + 1
176 }
177 var dupgroups: i64 = 0
178 var dupfuncs: i64 = 0
179 // iterate by BUCKET HEADS so the whole singly-linked chain is reachable (a tail record's rnext
180 // is -1 and can never see peers inserted after it -- the head-walk fixes that).
181 var b: i64 = 0
182 while b < FM_NB {
183 var x: i64 = bhead[b]
184 while x >= 0 {
185 if done[x] == (0 as u8) {
186 let h: i64 = rhash[x]
187 var cnt: i64 = 1
188 var y: i64 = rnext[x]
189 while y >= 0 { if done[y] == (0 as u8) { if rhash[y] == h { cnt = cnt + 1 } } y = rnext[y] }
190 if cnt >= 2 {
191 dupgroups = dupgroups + 1
192 dupfuncs = dupfuncs + cnt
193 sd_w("DUPFUNC copies=" as *u8)
194 let cb: *u8 = sys_mmap(24)
195 sd_num(cb, 0, cnt)
196 sd_w(cb)
197 sd_w(" : " as *u8)
198 sd_w(((fpbuf as i64) + fpoff[rfile[x]]) as *u8)
199 sd_w(":" as *u8)
200 sys_write(1, ((nbuf as i64) + rnoff[x]) as *u8, rnlen[x])
201 done[x] = 1 as u8
202 var z: i64 = rnext[x]
203 while z >= 0 {
204 if done[z] == (0 as u8) { if rhash[z] == h {
205 sd_w(" | " as *u8)
206 sd_w(((fpbuf as i64) + fpoff[rfile[z]]) as *u8)
207 sd_w(":" as *u8)
208 sys_write(1, ((nbuf as i64) + rnoff[z]) as *u8, rnlen[z])
209 done[z] = 1 as u8
210 } }
211 z = rnext[z]
212 }
213 sd_w("\n" as *u8)
214 } else { done[x] = 1 as u8 }
215 }
216 x = rnext[x]
217 }
218 b = b + 1
219 }
220
221 sd_w("NX-FUNCMINE files=" as *u8)
222 let ob: *u8 = sys_mmap(64)
223 sd_num(ob, 0, nfiles)
224 sd_w(ob)
225 sd_w(" funcs=" as *u8)
226 sd_num(ob, 0, nrec)
227 sd_w(ob)
228 sd_w(" dupgroups=" as *u8)
229 sd_num(ob, 0, dupgroups)
230 sd_w(ob)
231 sd_w(" dupfuncs=" as *u8)
232 sd_num(ob, 0, dupfuncs)
233 sd_w(ob)
234 sd_w("\n" as *u8)
235 sys_exit(0)
236 return 0
237}