nx_mediafacts.nx source
↩ module page · 253 lines · 11272 B
1// nx_mediafacts.nx -- CLI for media fact extraction into the refcorpus measured-facts plane.
2//
3// nx_mediafacts probe <file>
4// print the closed-vocabulary facts for ONE file (measurement only, nothing journaled)
5// nx_mediafacts sweep <listfile> <source> <provenance> <license> [put]
6// measure every path in <listfile> (one path per line), aggregate per format into
7// DISTRIBUTION facts (count / min / median / max), print them; with the `put` verb each
8// row goes through rc_put_row IN-PROCESS -- the same admission walls the refcorpus gate
9// proves, incl. the k-anonymity floor: a bucket with fewer than 8 files is REFUSED, so a
10// "distribution" that could describe one creator's asset physically cannot be journaled.
11//
12// Composes: nx_mediafacts_lib (extraction core) + nx_refcorpus (admission + journal, the
13// import-with-main precedent proven by nx_refcorpus_gate). Bounded window read: only the first
14// MF_WINDOW bytes of a file are ever mapped -- facts live in headers, and a 500MB video costs
15// the same read as a thumbnail.
16// license_tier: ORIGINAL No hw writes (Rule 26). expect_exit: 0
17import "nx_syscalls.nx"
18import "nx_mediafacts_lib.nx"
19import "nx_refcorpus.nx"
20
21const MF_WINDOW: i64 = 1048576
22const MF_MAX_FILES: i64 = 4096
23const MF_PATH_MAX: i64 = 384
24
25func mfc_slen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
26func mfc_w(s: *u8) -> i64 { sys_write(1, s, mfc_slen(s)); return 0 }
27func mfc_e(s: *u8) -> i64 { sys_write(2, s, mfc_slen(s)); return 0 }
28func mfc_num(v: i64) -> i64 {
29 if v == 0 { sys_write(1, "0" as *u8, 1); return 0 }
30 var m: i64 = v
31 if m < 0 { sys_write(1, "-" as *u8, 1); m = 0 - m }
32 let t: *u8 = sys_mmap(32)
33 var k: i64 = 0
34 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
35 let o: *u8 = sys_mmap(32)
36 var i: i64 = 0
37 while i < k { o[i] = t[k - 1 - i]; i = i + 1 }
38 sys_write(1, o, k)
39 return 0
40}
41// render an i64 into a NUL-terminated decimal string (for rc_put_row values)
42func mfc_itoa(v: i64, out: *u8) -> i64 {
43 var m: i64 = v
44 var p: i64 = 0
45 if m < 0 { out[p] = 45 as u8; p = p + 1; m = 0 - m }
46 if m == 0 { out[p] = 48 as u8; out[p+1] = 0 as u8; return p + 1 }
47 let t: *u8 = sys_mmap(32)
48 var k: i64 = 0
49 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
50 var i: i64 = 0
51 while i < k { out[p] = t[k - 1 - i]; p = p + 1; i = i + 1 }
52 out[p] = 0 as u8
53 return p
54}
55
56func mfc_fmtname(f: i64) -> *u8 {
57 if f == MF_FMT_PNG { return "png" as *u8 }
58 if f == MF_FMT_GIF { return "gif" as *u8 }
59 if f == MF_FMT_JPEG { return "jpeg" as *u8 }
60 if f == MF_FMT_WEBP { return "webp" as *u8 }
61 if f == MF_FMT_WAV { return "wav" as *u8 }
62 return "unknown" as *u8
63}
64
65// bounded window read: at most MF_WINDOW bytes, returns bytes read or -1
66func mfc_read_window(path: *u8, buf: *u8) -> i64 {
67 let fd: i64 = sys_openat_rd(path)
68 if fd < 0 { return 0 - 1 }
69 var got: i64 = 0
70 var stop: i64 = 0
71 while stop == 0 {
72 if got >= MF_WINDOW { stop = 1 } else {
73 let k: i64 = sys_read(fd, ((buf as i64) + got) as *u8, MF_WINDOW - got)
74 if k <= 0 { stop = 1 } else { got = got + k }
75 }
76 }
77 sys_close(fd)
78 return got
79}
80
81func mfc_print_probe(facts: *i64) -> i64 {
82 mfc_w("MEDIAFACT format=" as *u8); mfc_w(mfc_fmtname(facts[0]))
83 if facts[1] >= 0 { mfc_w(" width_px=" as *u8); mfc_num(facts[1]) }
84 if facts[2] >= 0 { mfc_w(" height_px=" as *u8); mfc_num(facts[2]) }
85 if facts[3] >= 0 { mfc_w(" exif_present=" as *u8); mfc_num(facts[3]) }
86 if facts[4] >= 0 { mfc_w(" channels=" as *u8); mfc_num(facts[4]) }
87 if facts[5] >= 0 { mfc_w(" sample_rate_hz=" as *u8); mfc_num(facts[5]) }
88 if facts[6] >= 0 { mfc_w(" bits_per_sample=" as *u8); mfc_num(facts[6]) }
89 if facts[7] >= 0 { mfc_w(" duration_ms=" as *u8); mfc_num(facts[7]) }
90 mfc_w("\n" as *u8)
91 return 0
92}
93
94// one distribution row: print, and in put mode journal through the admission wall.
95// key = <fmt>_<metric>; n = bucket count. Returns rc_put_row's code (0 in print-only mode).
96func mfc_dist_row(fmt: i64, metric: *u8, val: i64, unit: *u8, cnt: i64, src: *u8, prov: *u8, lic: *u8, do_put: i64) -> i64 {
97 let key: *u8 = sys_mmap(160)
98 var p: i64 = 0
99 let fn: *u8 = mfc_fmtname(fmt)
100 var i: i64 = 0
101 while fn[i] != (0 as u8) { key[p] = fn[i]; p = p + 1; i = i + 1 }
102 key[p] = 95 as u8; p = p + 1
103 i = 0
104 while metric[i] != (0 as u8) { key[p] = metric[i]; p = p + 1; i = i + 1 }
105 key[p] = 0 as u8
106 mfc_w("MEDIAFACT-DIST " as *u8); mfc_w(key); mfc_w("=" as *u8); mfc_num(val)
107 mfc_w(" unit=" as *u8); mfc_w(unit); mfc_w(" n=" as *u8); mfc_num(cnt); mfc_w("\n" as *u8)
108 if do_put == 0 { return 0 }
109 let vs: *u8 = sys_mmap(32)
110 mfc_itoa(val, vs)
111 return rc_put_row(src, key, vs, unit, cnt, prov, lic)
112}
113
114// emit the distribution rows for one format bucket from its parallel sample arrays
115func mfc_bucket(fmt: i64, cnt: i64, ws: *i64, hs: *i64, rates: *i64, chans: *i64, src: *u8, prov: *u8, lic: *u8, do_put: i64, wrote: *i64) -> i64 {
116 if cnt <= 0 { return 0 }
117 var r: i64 = 0
118 r = mfc_dist_row(fmt, "files_measured" as *u8, cnt, "files" as *u8, cnt, src, prov, lic, do_put)
119 if r == 0 { wrote[0] = wrote[0] + do_put }
120 if fmt == MF_FMT_WAV {
121 if rates[0] >= 0 {
122 r = mfc_dist_row(fmt, "sample_rate_hz_median" as *u8, mf_median(rates, cnt), "hz" as *u8, cnt, src, prov, lic, do_put)
123 if r == 0 { wrote[0] = wrote[0] + do_put }
124 r = mfc_dist_row(fmt, "channels_median" as *u8, mf_median(chans, cnt), "channels" as *u8, cnt, src, prov, lic, do_put)
125 if r == 0 { wrote[0] = wrote[0] + do_put }
126 }
127 return 0
128 }
129 // image bucket: mf_median SORTS in place, so it must run BEFORE ws[0]/ws[cnt-1] are read as
130 // min/max -- reading them pre-sort would report the first and last FILE, a plausible number
131 // that is not the measurement (the exact failure class the fact plane exists to prevent).
132 let wmed: i64 = mf_median(ws, cnt)
133 let hmed: i64 = mf_median(hs, cnt)
134 r = mfc_dist_row(fmt, "width_px_min" as *u8, ws[0], "px" as *u8, cnt, src, prov, lic, do_put)
135 if r == 0 { wrote[0] = wrote[0] + do_put }
136 r = mfc_dist_row(fmt, "width_px_median" as *u8, wmed, "px" as *u8, cnt, src, prov, lic, do_put)
137 if r == 0 { wrote[0] = wrote[0] + do_put }
138 r = mfc_dist_row(fmt, "width_px_max" as *u8, ws[cnt-1], "px" as *u8, cnt, src, prov, lic, do_put)
139 if r == 0 { wrote[0] = wrote[0] + do_put }
140 r = mfc_dist_row(fmt, "height_px_min" as *u8, hs[0], "px" as *u8, cnt, src, prov, lic, do_put)
141 if r == 0 { wrote[0] = wrote[0] + do_put }
142 r = mfc_dist_row(fmt, "height_px_median" as *u8, hmed, "px" as *u8, cnt, src, prov, lic, do_put)
143 if r == 0 { wrote[0] = wrote[0] + do_put }
144 r = mfc_dist_row(fmt, "height_px_max" as *u8, hs[cnt-1], "px" as *u8, cnt, src, prov, lic, do_put)
145 if r == 0 { wrote[0] = wrote[0] + do_put }
146 return 0
147}
148
149func main(argc: i64, argv: *i64) -> i64 {
150 if argc < 3 {
151 mfc_e("usage: nx_mediafacts probe <file>\n" as *u8)
152 mfc_e(" nx_mediafacts sweep <listfile> <source> <provenance> <license> [put]\n" as *u8)
153 mfc_e(" facts only: computed integers + fixed labels; no file byte reaches the output.\n" as *u8)
154 sys_exit(2)
155 return 2
156 }
157 let verb: *u8 = argv[1] as *u8
158 let buf: *u8 = sys_mmap(MF_WINDOW)
159
160 if verb[0] == (112 as u8) { // probe
161 let n: i64 = mfc_read_window(argv[2] as *u8, buf)
162 if n < 0 { mfc_e("ERROR: cannot open file\n" as *u8); sys_exit(1); return 1 }
163 let facts: *i64 = sys_mmap(MF_N_SLOTS * 8) as *i64
164 mf_probe(buf, n, facts)
165 mfc_print_probe(facts)
166 sys_exit(0)
167 return 0
168 }
169
170 // sweep
171 if argc < 6 { mfc_e("sweep needs <listfile> <source> <provenance> <license> [put]\n" as *u8); sys_exit(2); return 2 }
172 let src: *u8 = argv[3] as *u8
173 let prov: *u8 = argv[4] as *u8
174 let lic: *u8 = argv[5] as *u8
175 var do_put: i64 = 0
176 if argc >= 7 { let pv: *u8 = argv[6] as *u8; if pv[0] == (112 as u8) { do_put = 1 } }
177
178 let lp: *i64 = sys_mmap(16) as *i64
179 let lst: *u8 = sys_read_file(argv[2] as *u8, lp)
180 if (lst as i64) == 0 { mfc_e("ERROR: cannot read listfile\n" as *u8); sys_exit(1); return 1 }
181 let ln: i64 = lp[0]
182
183 // per-format parallel sample arrays (widths/heights sorted later; wav rate+channels)
184 let ws: *i64 = sys_mmap(6 * MF_MAX_FILES * 8) as *i64
185 let hs: *i64 = sys_mmap(6 * MF_MAX_FILES * 8) as *i64
186 let rates: *i64 = sys_mmap(MF_MAX_FILES * 8) as *i64
187 let chans: *i64 = sys_mmap(MF_MAX_FILES * 8) as *i64
188 let cnt: *i64 = sys_mmap(6 * 8) as *i64
189 var ci: i64 = 0
190 while ci < 6 { cnt[ci] = 0; ci = ci + 1 }
191
192 let path: *u8 = sys_mmap(MF_PATH_MAX + 8)
193 let facts: *i64 = sys_mmap(MF_N_SLOTS * 8) as *i64
194 var pos: i64 = 0
195 var files: i64 = 0
196 var skipped: i64 = 0
197 while pos < ln {
198 var pl: i64 = 0
199 var eol: i64 = 0
200 while eol == 0 {
201 if pos >= ln { eol = 1 } else {
202 let c: i64 = lst[pos] as i64 & 0xff
203 pos = pos + 1
204 if c == 10 { eol = 1 } else {
205 if c != 13 { if pl < MF_PATH_MAX { path[pl] = c as u8; pl = pl + 1 } }
206 }
207 }
208 }
209 path[pl] = 0 as u8
210 if pl > 0 { if files < MF_MAX_FILES {
211 let n2: i64 = mfc_read_window(path, buf)
212 if n2 <= 0 { skipped = skipped + 1 } else {
213 mf_probe(buf, n2, facts)
214 let f: i64 = facts[0]
215 files = files + 1
216 if f != MF_FMT_UNKNOWN {
217 let k: i64 = cnt[f]
218 if f == MF_FMT_WAV {
219 if facts[5] >= 0 { if k < MF_MAX_FILES { rates[k] = facts[5]; chans[k] = facts[4]; cnt[f] = k + 1 } }
220 } else {
221 if facts[1] >= 0 { if facts[2] >= 0 { if k < MF_MAX_FILES {
222 ws[f * MF_MAX_FILES + k] = facts[1]
223 hs[f * MF_MAX_FILES + k] = facts[2]
224 cnt[f] = k + 1
225 } } }
226 }
227 }
228 }
229 } }
230 }
231
232 mfc_w("MEDIAFACTS-SWEEP files=" as *u8); mfc_num(files)
233 mfc_w(" unreadable=" as *u8); mfc_num(skipped); mfc_w("\n" as *u8)
234
235 let wrote: *i64 = sys_mmap(16) as *i64
236 wrote[0] = 0
237 var fmt: i64 = 1
238 while fmt <= 5 {
239 if fmt == MF_FMT_WAV {
240 mfc_bucket(fmt, cnt[fmt], ws, hs, rates, chans, src, prov, lic, do_put, wrote)
241 } else {
242 mfc_bucket(fmt, cnt[fmt], ((ws as i64) + fmt * MF_MAX_FILES * 8) as *i64, ((hs as i64) + fmt * MF_MAX_FILES * 8) as *i64, rates, chans, src, prov, lic, do_put, wrote)
243 }
244 fmt = fmt + 1
245 }
246
247 if do_put == 1 {
248 mfc_w("MEDIAFACTS-PUT rows_admitted=" as *u8); mfc_num(wrote[0]); mfc_w("\n" as *u8)
249 if wrote[0] == 0 { mfc_e("MEDIAFACTS-REFUSED-ALL nothing met the admission walls (bucket under the k=8 floor?)\n" as *u8); sys_exit(3); return 3 }
250 }
251 sys_exit(0)
252 return 0
253}