nx_media_grab.nx source
↩ module page · 214 lines · 13833 B
1// nx_media_grab.nx -- "nishi grab -> gallery" (operator 2026-07-08). Deep-harvests a page's media (nx_media_
2// deepcrawl: follows css/js/iframes + reconstructs JS-built URLs, no page-JS execution), DOWNLOADS each file
3// over sovereign TLS, content-addresses it (sha256), dedups, and adds it to an ISOLATED grabs gallery -- a
4// growing masonry of everything you've grabbed, kept SEPARATE from the gen-img/LoRA corpus on the NAS so web
5// grabs never pollute the training-favorite set. Regenerates a self-contained gallery page you can open.
6// usage: nx_media_grab <url>
7// files: knowledge/media/grabs/<sha16>.<ext> manifest: knowledge/status/grabs_manifest.tsv
8// gallery: knowledge/status/grabs_gallery.html (open in any browser; local previews render)
9// license_tier: ORIGINAL
10import "nx_syscalls.nx"
11import "nx_x509_trust_store.nx"
12import "nx_trust_store_load_from_certdata.nx"
13import "nx_https_fetch_follow.nx"
14import "nx_media_harvest.nx"
15import "nx_media_deepcrawl.nx"
16import "nx_sha256.nx"
17const GR_MAGIC_1089: i64 = 1089
18const GR_MAGIC_8388608: i64 = 8388608
19const GR_MAGIC_16777216: i64 = 16777216
20const GR_MAGIC_1024: i64 = 1024
21const GR_MAGIC_4096: i64 = 4096
22const GR_MAGIC_4194304: i64 = 4194304
23const GR_MAGIC_5120: i64 = 5120
24
25const GR_DLBUF: i64 = 67108864 // 64MB per-file
26const GR_MAN: *u8 = "knowledge/status/grabs_manifest.tsv" as *u8
27const GR_DIR: *u8 = "knowledge/media/grabs" as *u8
28const GR_GAL: *u8 = "knowledge/status/grabs_gallery.html" as *u8
29
30func gr_w(fd: i64, s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(fd,s,n); return 0 }
31func gr_n(fd: i64, v0: i64) -> i64 { var v: i64=v0; if v<0 { sys_write(fd,"-" as *u8,1); v=0-v } let b: *u8=sys_mmap(24); var k: i64=0; if v==0{b[0]=48 as u8;k=1} while v>0{b[k]=(48+(v%10)) as u8;v=v/10;k=k+1} let o: *u8=sys_mmap(24); var j: i64=0; while j<k{o[j]=b[k-1-j];j=j+1} sys_write(fd,o,k); return 0 }
32func gr_slen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
33func gr_mkdir(path: *u8) -> i64 { return __syscall(258, 0-100, path as i64, 0x1ff, 0, 0, 0) }
34func gr_ap(buf: *u8, off: i64, s: *u8) -> i64 { var i: i64=0; var w: i64=off; while s[i]!=(0 as u8){ buf[w]=s[i]; w=w+1; i=i+1 } return w }
35func gr_apr(buf: *u8, off: i64, ptr: *u8, len: i64) -> i64 { var i: i64=0; var w: i64=off; while i<len { buf[w]=ptr[i]; w=w+1; i=i+1 } return w }
36
37func gr_write_file(path: *u8, buf: *u8, n: i64) -> i64 {
38 let fd: i64 = __syscall(257, 0-100, path as i64, 577, 420, 0, 0)
39 if fd < 0 { return 0 - 1 }
40 var done: i64 = 0
41 while done < n { let w: i64 = sys_write(fd, ((buf as i64)+done) as *u8, n-done); if w <= 0 { done = n } else { done = done + w } }
42 __syscall(3, fd, 0, 0, 0, 0, 0)
43 return n
44}
45func gr_append(path: *u8, buf: *u8, n: i64) -> i64 {
46 let fd: i64 = __syscall(257, 0-100, path as i64, GR_MAGIC_1089, 420, 0, 0) // O_WRONLY|O_CREAT|O_APPEND
47 if fd < 0 { return 0 - 1 }
48 var done: i64 = 0
49 while done < n { let w: i64 = sys_write(fd, ((buf as i64)+done) as *u8, n-done); if w <= 0 { done = n } else { done = done + w } }
50 __syscall(3, fd, 0, 0, 0, 0, 0)
51 return n
52}
53// read whole file into buf (cap), return length or 0
54func gr_read_file(path: *u8, buf: *u8, cap: i64) -> i64 {
55 let fd: i64 = sys_openat_rd(path)
56 if fd < 0 { return 0 }
57 var tot: i64 = 0
58 while tot < cap { let r: i64 = sys_read(fd, ((buf as i64)+tot) as *u8, cap-tot); if r <= 0 { tot = tot } if r <= 0 { sys_close(fd); return tot } tot = tot + r }
59 sys_close(fd); return tot
60}
61// hex of 32-byte digest -> out (64 chars + NUL)
62func gr_hex(d: *u8, out: *u8) -> i64 { let hx: *u8="0123456789abcdef" as *u8; var i: i64=0; while i<32 { let b: i64=d[i] as i64; out[i*2]=hx[(b>>4)&15]; out[i*2+1]=hx[b&15]; i=i+1 } out[64]=0 as u8; return 64 }
63// extension (incl '.') from url path -> out; default ".bin". returns len.
64func gr_ext(url: *u8, ulen: i64, out: *u8) -> i64 {
65 var end: i64 = ulen; var i: i64 = 0
66 while i < ulen { let c: i64=url[i] as i64; if c==63 { if end==ulen { end=i } } if c==35 { if end==ulen { end=i } } i=i+1 }
67 var dot: i64 = 0 - 1; var j: i64 = 0
68 while j < end { if url[j]==(46 as u8) { dot=j } if url[j]==(47 as u8) { dot = 0 - 1 } j=j+1 }
69 if dot < 0 { out[0]=46 as u8; out[1]=98; out[2]=105; out[3]=110; out[4]=0 as u8; return 4 } // ".bin"
70 var o: i64 = 0; var k: i64 = dot
71 while k < end { if o < 12 { out[o]=url[k]; o=o+1 } k=k+1 }
72 out[o]=0 as u8; return o
73}
74// is hex64 present in manifest buf (line-start match)? (dedup)
75func gr_seen(man: *u8, mlen: i64, hex: *u8) -> i64 {
76 var i: i64 = 0
77 while i + 64 <= mlen {
78 var atstart: i64 = 0
79 if i == 0 { atstart = 1 } else { if man[i-1]==(10 as u8) { atstart = 1 } }
80 if atstart == 1 {
81 var eq: i64 = 1; var k: i64 = 0
82 while k < 64 { if man[i+k] != hex[k] { eq=0; k=64 } else { k=k+1 } }
83 if eq == 1 { return 1 }
84 }
85 i = i + 1
86 }
87 return 0
88}
89
90// extract tab-field #idx (0-based) of line man[ls..le) into out (NUL-term); returns len.
91func gr_field(man: *u8, ls: i64, le: i64, idx: i64, out: *u8) -> i64 {
92 var f: i64 = 0; var s: i64 = ls; var i: i64 = ls
93 while i <= le {
94 var brk: i64 = 0
95 if i == le { brk = 1 } else { if man[i]==(9 as u8) { brk = 1 } }
96 if brk == 1 { if f == idx { var o: i64=0; var k: i64=s; while k<i { out[o]=man[k]; o=o+1; k=k+1 } out[o]=0 as u8; return o } f=f+1; s=i+1 }
97 i = i + 1
98 }
99 out[0]=0 as u8; return 0
100}
101
102func gr_streq(a: *u8, b: *u8) -> i64 { var i: i64=0; while a[i]!=(0 as u8){ if a[i]!=b[i]{return 0} i=i+1 } if b[i]==(0 as u8){return 1} return 0 }
103// regenerate the masonry grabs gallery from the full manifest
104func gr_emit_gallery() -> i64 {
105 let man: *u8 = sys_mmap(GR_MAGIC_8388608)
106 let mlen: i64 = gr_read_file(GR_MAN, man, GR_MAGIC_8388608)
107 let H: *u8 = sys_mmap(GR_MAGIC_16777216)
108 var o: i64 = 0
109 o = gr_ap(H, o, "<!DOCTYPE html><html lang='en'><head><meta charset='utf-8'><meta name='viewport' content='width=device-width,initial-scale=1'><meta name='color-scheme' content='dark'><title>Nishi Grabs</title><style>" as *u8)
110 o = gr_ap(H, o, "body{margin:0;background:#0f1014;color:#e9e9f0;font:14px/1.5 ui-sans-serif,system-ui,Segoe UI,Roboto,Arial,sans-serif}" as *u8)
111 o = gr_ap(H, o, ".hd{padding:18px 20px;border-bottom:1px solid #23252e;position:sticky;top:0;background:#0f1014e6;backdrop-filter:blur(6px)}.hd h1{margin:0;font-size:1.12rem}.hd p{margin:4px 0 0;color:#8a8f9c;font-size:.85rem}" as *u8)
112 o = gr_ap(H, o, ".masonry{column-width:230px;column-gap:12px;padding:14px}.card{break-inside:avoid;margin:0 0 12px;background:#181a22;border:1px solid #262933;border-radius:11px;overflow:hidden}" as *u8)
113 o = gr_ap(H, o, ".card a.p{display:block}.card img,.card video,.card audio{width:100%;display:block;background:#0b0c10}.meta{padding:8px 10px}.k{display:inline-block;font-size:.64rem;color:#0d0f14;background:#7bb0ff;border-radius:5px;padding:1px 7px;margin-bottom:5px;text-transform:uppercase;letter-spacing:.04em}" as *u8)
114 o = gr_ap(H, o, ".gif .k{background:#ffcf6b}.video .k{background:#8fe08f}.audio .k{background:#e08fd0}.stream .k{background:#e0918c}.u{font-size:.71rem;color:#9aa2b2;word-break:break-all;line-height:1.3}</style></head><body>" as *u8)
115 o = gr_ap(H, o, "<div class='hd'><h1>Nishi Grabs</h1><p>Media grabbed from the web \xe2\x80\x94 sovereign, content-addressed, isolated from the gen-img gallery.</p></div><div class='masonry'>" as *u8)
116
117 let fkind: *u8 = sys_mmap(64); let ffile: *u8 = sys_mmap(GR_MAGIC_1024); let furl: *u8 = sys_mmap(GR_MAGIC_4096)
118 var count: i64 = 0
119 // iterate lines
120 var ls: i64 = 0; var i: i64 = 0
121 while i <= mlen {
122 var brk: i64 = 0
123 if i == mlen { brk = 1 } else { if man[i]==(10 as u8) { brk = 1 } }
124 if brk == 1 {
125 if i > ls {
126 gr_field(man, ls, i, 2, fkind) // kind label
127 gr_field(man, ls, i, 3, ffile) // filename
128 gr_field(man, ls, i, 1, furl) // source url
129 let fl: i64 = gr_slen(ffile)
130 if fl > 0 {
131 o = gr_ap(H, o, "<div class='card " as *u8); o = gr_ap(H, o, fkind); o = gr_ap(H, o, "'>" as *u8)
132 // preview by kind
133 if gr_streq(fkind, "video" as *u8)==1 { o = gr_ap(H, o, "<video controls preload='metadata' src='../media/grabs/" as *u8); o = gr_ap(H, o, ffile); o = gr_ap(H, o, "'></video>" as *u8) }
134 else { if gr_streq(fkind, "audio" as *u8)==1 { o = gr_ap(H, o, "<audio controls preload='metadata' src='../media/grabs/" as *u8); o = gr_ap(H, o, ffile); o = gr_ap(H, o, "'></audio>" as *u8) }
135 else { o = gr_ap(H, o, "<a class='p' href='../media/grabs/" as *u8); o = gr_ap(H, o, ffile); o = gr_ap(H, o, "' target='_blank'><img loading='lazy' src='../media/grabs/" as *u8); o = gr_ap(H, o, ffile); o = gr_ap(H, o, "' alt=''></a>" as *u8) } }
136 o = gr_ap(H, o, "<div class='meta'><span class='k'>" as *u8); o = gr_ap(H, o, fkind); o = gr_ap(H, o, "</span><div class='u'>" as *u8); o = gr_ap(H, o, furl); o = gr_ap(H, o, "</div></div></div>" as *u8)
137 count = count + 1
138 }
139 }
140 ls = i + 1
141 }
142 i = i + 1
143 }
144 o = gr_ap(H, o, "</div></body></html>" as *u8)
145 gr_write_file(GR_GAL, H, o)
146 return count
147}
148
149func main(argc: i64, argv: *i64) -> i64 {
150 if argc < 2 { gr_w(1, "usage: nx_media_grab <url>\n" as *u8); return 1 }
151 let arg: *u8 = argv[1] as *u8
152 let al: i64 = gr_slen(arg)
153 let url: *u8 = sys_mmap(GR_MAGIC_4096)
154 var has_scheme: i64 = 0
155 if al >= 4 { if arg[0]==(104 as u8) { if arg[1]==(116 as u8) { if arg[2]==(116 as u8) { if arg[3]==(112 as u8) { has_scheme = 1 } } } } }
156 if has_scheme == 1 { var i: i64=0; while i<al { url[i]=arg[i]; i=i+1 } url[al]=0 as u8 }
157 else { var o: i64 = gr_ap(url, 0, "https://" as *u8); var i2: i64=0; while i2<al { url[o]=arg[i2]; o=o+1; i2=i2+1 } url[o]=0 as u8 }
158 let ul: i64 = gr_slen(url)
159
160 let r: i64 = nx_trust_store_load_from_certdata("data/mozilla_certdata.txt" as *u8, 512, GR_MAGIC_4194304)
161 if r <= 0 { gr_w(1, "trust store load failed\n" as *u8); return 2 }
162 let store: *TrustStore = r as *TrustStore
163
164 gr_w(1, "nishi grab (deep crawl -> gallery): " as *u8); gr_w(1, url); gr_w(1, "\n" as *u8)
165 let M: *MediaSet = nx_mediaset_new()
166 nx_media_deepcrawl(url, ul, store, M, 28, 2, 1, "knowledge/status/media_page_raw.html" as *u8)
167 gr_w(1, " media discovered=" as *u8); gr_n(1, M.cnt); gr_w(1, " -- downloading + content-hashing...\n" as *u8)
168
169 gr_mkdir("knowledge/media" as *u8); gr_mkdir(GR_DIR)
170 let man: *u8 = sys_mmap(GR_MAGIC_8388608)
171 var mlen: i64 = gr_read_file(GR_MAN, man, GR_MAGIC_8388608) // existing manifest for dedup
172 let fbuf: *u8 = sys_mmap(GR_DLBUF)
173 let fst: *i64 = sys_mmap(16) as *i64
174 let dg: *u8 = sys_mmap(32); let hex: *u8 = sys_mmap(72); let ext: *u8 = sys_mmap(16)
175 let ub: *u8 = sys_mmap(GR_MAGIC_4096); let fname: *u8 = sys_mmap(128); let path: *u8 = sys_mmap(GR_MAGIC_1024); let line: *u8 = sys_mmap(GR_MAGIC_5120)
176 var grabbed: i64 = 0; var dup: i64 = 0
177
178 var mi: i64 = 0
179 while mi < M.cnt {
180 let bp: *u8 = ((M.buf as i64)+M.off[mi]) as *u8
181 var z: i64=0; while z < M.len[mi] { ub[z]=bp[z]; z=z+1 } ub[M.len[mi]]=0 as u8
182 if M.kind[mi] == MH_KIND_STREAM { gr_w(1, " skip stream " as *u8); sys_write(1, bp, M.len[mi]); gr_w(1, "\n" as *u8) }
183 else {
184 let n: i64 = nx_https_fetch_follow_best(ub, store, fbuf, GR_DLBUF, 6, fst)
185 if fst[0]==200 { if n>0 {
186 sha256_digest(fbuf, n, dg); gr_hex(dg, hex)
187 if gr_seen(man, mlen, hex)==1 { dup = dup + 1 }
188 else {
189 gr_ext(ub, M.len[mi], ext)
190 // fname = <hex16><ext>
191 var fo: i64=0; while fo<16 { fname[fo]=hex[fo]; fo=fo+1 } var eo: i64=0; while ext[eo]!=(0 as u8){ fname[fo]=ext[eo]; fo=fo+1; eo=eo+1 } fname[fo]=0 as u8
192 var po: i64 = gr_ap(path, 0, GR_DIR); path[po]=47 as u8; po=po+1; var pk: i64=0; while fname[pk]!=(0 as u8){ path[po]=fname[pk]; po=po+1; pk=pk+1 } path[po]=0 as u8
193 gr_write_file(path, fbuf, n)
194 // manifest line: hex64 \t url \t kind \t fname \n
195 var lo: i64 = gr_apr(line, 0, hex, 64); line[lo]=9 as u8; lo=lo+1
196 lo = gr_apr(line, lo, ub, M.len[mi]); line[lo]=9 as u8; lo=lo+1
197 lo = gr_ap(line, lo, nx_media_kind_label(M.kind[mi])); line[lo]=9 as u8; lo=lo+1
198 lo = gr_ap(line, lo, fname); line[lo]=10 as u8; lo=lo+1
199 gr_append(GR_MAN, line, lo)
200 // reflect into in-memory manifest so within-run dups dedup
201 var ci: i64=0; while ci<lo { man[mlen]=line[ci]; mlen=mlen+1; ci=ci+1 }
202 gr_w(1, " grabbed " as *u8); gr_w(1, fname); gr_w(1, " (" as *u8); gr_n(1, n); gr_w(1, " B) <- " as *u8); sys_write(1, bp, M.len[mi]); gr_w(1, "\n" as *u8)
203 grabbed = grabbed + 1
204 }
205 } } else { gr_w(1, " skip status=" as *u8); gr_n(1, fst[0]); gr_w(1, " " as *u8); sys_write(1, bp, M.len[mi]); gr_w(1, "\n" as *u8) }
206 }
207 mi = mi + 1
208 }
209
210 let total: i64 = gr_emit_gallery()
211 gr_w(1, " grabbed " as *u8); gr_n(1, grabbed); gr_w(1, " new (" as *u8); gr_n(1, dup); gr_w(1, " already had); gallery now " as *u8); gr_n(1, total); gr_w(1, " items\n" as *u8)
212 gr_w(1, " gallery -> knowledge/status/grabs_gallery.html\n open: C:\\Users\\elder\\nishi-core\\nxc2\\knowledge\\status\\grabs_gallery.html\n" as *u8)
213 return 0
214}