code wiki / _hdl_build / nx_funcmine.nx
nx_funcmine.nx source
↩ module page · 209 lines · 9054 B
1// nx_funcmine.nx -- FUNCTION MINING for OO consolidation (operator 2026-07-20: "archive the functions
2// for functions mining... object oriented without losing functionalities"). Scans a dir's .nx files,
3// extracts every function BODY, FNV-1a hashes it, and reports IDENTICAL bodies appearing in 2+ places
4// = the TRUE function-level dedup the file-level detector can't see: a copy-pasted helper across N organs
5// is a base/shared-lib EXTRACTION target (rule 15: pattern in 3+ -> shared/), behaviour-preserving because
6// the body is byte-identical. Bucket-chained hash (O(files+funcs), not O(funcs^2)). Emits the mining
7// substrate for the SHARP2 archive.
8// argv: <dir>
9// Report: DUPFUNC hash=<h> copies=<n> : <file>:<func> | ... , then NX-FUNCMINE files=F funcs=N dupgroups=G dupfuncs=D
10// exit 0 = ran. Deterministic. license_tier: ORIGINAL No hw writes (Rule 26).
11import "nx_seat_drive_lib.nx"
12import "nx_deploy_lib.nx"
13import "nx_syscalls.nx"
14import "nx_funchead_lib.nx"
15const FM_MAGIC_524288: i64 = 524288
16const FM_MAGIC_262144: i64 = 262144
17const FM_MAGIC_1024: i64 = 1024
18const FM_MAGIC_2048: i64 = 2048
19
20const FM_MAXREC: i64 = 60000
21const FM_NB: i64 = 65536
22const FM_NAMEBUF: i64 = 6291456
23const FM_FILECAP: i64 = 2097152
24const FM_MAXFILES: i64 = 12000
25const FM_FNV_OFF: i64 = 1469598103934665603
26const FM_FNV_PRIME: i64 = 1099511628211
27
28// fm_isid retired 2026-09-14: the identifier alphabet lives in nx_funchead_lib (fh_isid) beside the one head scanner
29func fm_ends_nx(nm: *u8) -> i64 {
30 var n: i64 = 0
31 while nm[n] != (0 as u8) { n = n + 1 }
32 if n < 3 { return 0 }
33 if nm[n-3] == (46 as u8) { if nm[n-2] == (110 as u8) { if nm[n-1] == (120 as u8) { return 1 } } }
34 return 0
35}
36func fm_fnv(buf: *u8, s: i64, e: i64) -> i64 {
37 var h: i64 = FM_FNV_OFF
38 var i: i64 = s
39 while i < e { h = h ^ (buf[i] as i64); h = h * FM_FNV_PRIME; i = i + 1 }
40 return h
41}
42
43func main(argc: i64, argv: *i64) -> i64 {
44 if argc < 2 { sd_w("usage: nx_funcmine <dir>\n" as *u8); sys_exit(2); return 2 }
45 let dir: *u8 = argv[1] as *u8
46 if argc >= 3 { let ofd: i64 = sys_openat_wr(argv[2] as *u8, 0x1a4); if ofd >= 0 { sys_dup3(ofd, 1, 0) } }
47
48 let fpbuf: *u8 = sys_mmap(FM_MAGIC_524288)
49 let fpoff: *i64 = sys_mmap(8 * FM_MAXFILES) as *i64
50 var fpb: i64 = 0
51 var nfiles: i64 = 0
52
53 let rhash: *i64 = sys_mmap(8 * FM_MAXREC) as *i64
54 let rfile: *i64 = sys_mmap(8 * FM_MAXREC) as *i64
55 let rnoff: *i64 = sys_mmap(8 * FM_MAXREC) as *i64
56 let rnlen: *i64 = sys_mmap(8 * FM_MAXREC) as *i64
57 let nbuf: *u8 = sys_mmap(FM_NAMEBUF)
58 var nb: i64 = 0
59 var nrec: i64 = 0
60
61 let src: *u8 = sys_mmap(FM_FILECAP)
62
63 // scan dir for .nx files
64 let dfd: i64 = sys_openat_rd(dir)
65 if dfd < 0 { sd_w("NX-FUNCMINE dir unreadable\n" as *u8); sys_exit(2); return 2 }
66 let dbuf: *u8 = sys_mmap(FM_MAGIC_262144)
67 var go: i64 = 1
68 while go == 1 {
69 let dn: i64 = sys_getdents64(dfd, dbuf, FM_MAGIC_262144)
70 if dn <= 0 { go = 0 } else {
71 var off: i64 = 0
72 while off < dn {
73 let rec: *u8 = ((dbuf as i64) + off) as *u8
74 let ty: i64 = dirent_type(rec)
75 let nm: *u8 = dirent_name(rec)
76 if ty != 4 { if fm_ends_nx(nm) == 1 { if nfiles < FM_MAXFILES {
77 // path = dir + "/" + nm, read it
78 let p: *u8 = sys_mmap(FM_MAGIC_1024)
79 var po: i64 = 0
80 var di: i64 = 0
81 while dir[di] != (0 as u8) { p[po] = dir[di]; po = po + 1; di = di + 1 }
82 p[po] = 47 as u8; po = po + 1
83 var mi: i64 = 0
84 while nm[mi] != (0 as u8) { p[po] = nm[mi]; po = po + 1; mi = mi + 1 }
85 p[po] = 0 as u8
86 // record file path for reporting
87 let thisfile: i64 = nfiles
88 fpoff[nfiles] = fpb
89 var cp: i64 = 0
90 while cp < po { fpbuf[fpb] = p[cp]; fpb = fpb + 1; cp = cp + 1 }
91 fpbuf[fpb] = 0 as u8; fpb = fpb + 1
92 nfiles = nfiles + 1
93 // read + extract funcs
94 let fd: i64 = sys_openat_rd(p)
95 if fd >= 0 {
96 var slen: i64 = 0
97 var r: i64 = sys_read(fd, src, FM_FILECAP - 1)
98 while r > 0 { slen = slen + r; if slen >= FM_FILECAP - 1 { r = 0 } else { r = sys_read(fd, src + slen, FM_FILECAP - 1 - slen) } }
99 sys_close(fd)
100 // find func starts (func NAME( at BOL)
101 let fstart: *i64 = sys_mmap(8 * FM_MAGIC_2048) as *i64
102 let fnoffs: *i64 = sys_mmap(8 * FM_MAGIC_2048) as *i64
103 let fnlens: *i64 = sys_mmap(8 * FM_MAGIC_2048) as *i64
104 let nf: i64 = fh_scan(src, slen, fstart, fnoffs, fnlens, FM_MAGIC_2048) // ONE scanner, shared with the ingest's definition anchors (nx_funchead_lib)
105 // hash each func body [fstart[k], fstart[k+1] or slen), record
106 var k: i64 = 0
107 while k < nf {
108 if nrec < FM_MAXREC {
109 let bs: i64 = fstart[k]
110 var be: i64 = slen
111 if k + 1 < nf { be = fstart[k+1] }
112 rhash[nrec] = fm_fnv(src, bs, be)
113 rfile[nrec] = thisfile
114 // copy func name into nbuf
115 let no: i64 = fnoffs[k]
116 let nl: i64 = fnlens[k]
117 if nb + nl + 1 < FM_NAMEBUF {
118 rnoff[nrec] = nb
119 rnlen[nrec] = nl
120 var w: i64 = 0
121 while w < nl { nbuf[nb] = src[no + w]; nb = nb + 1; w = w + 1 }
122 nrec = nrec + 1
123 }
124 }
125 k = k + 1
126 }
127 }
128 } } }
129 off = off + dirent_reclen(rec)
130 }
131 }
132 }
133 sys_close(dfd)
134
135 // bucket-chain by hash, report groups of >=2 with identical hash
136 let bhead: *i64 = sys_mmap(8 * FM_NB) as *i64
137 let rnext: *i64 = sys_mmap(8 * FM_MAXREC) as *i64
138 let done: *u8 = sys_mmap(FM_MAXREC)
139 var bi: i64 = 0
140 while bi < FM_NB { bhead[bi] = 0 - 1; bi = bi + 1 }
141 var ri: i64 = 0
142 while ri < nrec {
143 let b: i64 = rhash[ri] & (FM_NB - 1)
144 rnext[ri] = bhead[b]
145 bhead[b] = ri
146 done[ri] = 0 as u8
147 ri = ri + 1
148 }
149 var dupgroups: i64 = 0
150 var dupfuncs: i64 = 0
151 // iterate by BUCKET HEADS so the whole singly-linked chain is reachable (a tail record's rnext
152 // is -1 and can never see peers inserted after it -- the head-walk fixes that).
153 var b: i64 = 0
154 while b < FM_NB {
155 var x: i64 = bhead[b]
156 while x >= 0 {
157 if done[x] == (0 as u8) {
158 let h: i64 = rhash[x]
159 var cnt: i64 = 1
160 var y: i64 = rnext[x]
161 while y >= 0 { if done[y] == (0 as u8) { if rhash[y] == h { cnt = cnt + 1 } } y = rnext[y] }
162 if cnt >= 2 {
163 dupgroups = dupgroups + 1
164 dupfuncs = dupfuncs + cnt
165 sd_w("DUPFUNC copies=" as *u8)
166 let cb: *u8 = sys_mmap(24)
167 sd_num(cb, 0, cnt)
168 sd_w(cb)
169 sd_w(" : " as *u8)
170 sd_w(((fpbuf as i64) + fpoff[rfile[x]]) as *u8)
171 sd_w(":" as *u8)
172 sys_write(1, ((nbuf as i64) + rnoff[x]) as *u8, rnlen[x])
173 done[x] = 1 as u8
174 var z: i64 = rnext[x]
175 while z >= 0 {
176 if done[z] == (0 as u8) { if rhash[z] == h {
177 sd_w(" | " as *u8)
178 sd_w(((fpbuf as i64) + fpoff[rfile[z]]) as *u8)
179 sd_w(":" as *u8)
180 sys_write(1, ((nbuf as i64) + rnoff[z]) as *u8, rnlen[z])
181 done[z] = 1 as u8
182 } }
183 z = rnext[z]
184 }
185 sd_w("\n" as *u8)
186 } else { done[x] = 1 as u8 }
187 }
188 x = rnext[x]
189 }
190 b = b + 1
191 }
192
193 sd_w("NX-FUNCMINE files=" as *u8)
194 let ob: *u8 = sys_mmap(64)
195 sd_num(ob, 0, nfiles)
196 sd_w(ob)
197 sd_w(" funcs=" as *u8)
198 sd_num(ob, 0, nrec)
199 sd_w(ob)
200 sd_w(" dupgroups=" as *u8)
201 sd_num(ob, 0, dupgroups)
202 sd_w(ob)
203 sd_w(" dupfuncs=" as *u8)
204 sd_num(ob, 0, dupfuncs)
205 sd_w(ob)
206 sd_w("\n" as *u8)
207 sys_exit(0)
208 return 0
209}