nx_media_import.nx source

1// nx_media_import.nx -- GENERAL POLITE media importer: pull images from ANY site (Shein/e-commerce/art/galleries) 2// into the family library, RESPECTING the host's rules. Operator 2026-07-03: "import politely images from places 3// like shein etc to get state of the art." The differentiator vs a crude scraper is POLITENESS BY CONSTRUCTION: 4// 1. fetch the host's /robots.txt and HONOR it (nx_robots_allowed) -- a Disallow => we DO NOT fetch (refuse). 5// 2. send an HONEST User-Agent identifying us + a contact/purpose. 6// 3. pace requests to the host's Crawl-delay (nx_robots_crawl_delay), default a courteous floor. 7// Then it reuses the PROVEN engine: mg_gallery_to_pages (Chrome-JA3 fetch that beats anti-bot CDNs + multi-strategy 8// nx_img_harvest + download-integrity + cbx.json manifest) and lands the result in the caller's PRIVATE reader area 9// (rl_add_personal) so it shows up in the zoom reader as a browsable image collection. Composes; adds only the 10// politeness gate. CLI: nx_media_import <page_url> <slug> <ukey> (forked by the daemon on an "Import" POST). 11// license_tier: ORIGINAL 12import "nx_syscalls.nx" 13import "nx_x509_trust_store.nx" 14import "nx_trust_store_load_from_certdata.nx" 15import "nx_https_fetch_follow.nx" // nx_https_fetch_follow_chrome 16import "nx_robots.nx" // nx_robots_allowed / nx_robots_crawl_delay 17import "nx_manga_get.nx" // mg_gallery_to_pages (the download engine) + MG constants 18import "nx_reader_library.nx" // rl_personal_slug_dir / rl_add_personal 19const MI_MAGIC_1000000: i64 = 1000000 20const MI_MAGIC_1024: i64 = 1024 21const MI_MAGIC_9309: i64 = 9309 22const MI_MAGIC_4194304: i64 = 4194304 23 24// Honest, identifying UA (RFC 9309 politeness: a real product token the host can allow/deny + a contact). 25const MI_UA: *u8 = "NishiMediaImporter/1.0 (+https://nishifamily.com; personal family library import)" 26const MI_UA_LEN: i64 = 78 27const MI_POLITE_FLOOR_MS: i64 = 1000 // courteous default spacing if the host sets no Crawl-delay 28const MI_ROBOTS_CAP: i64 = 262144 29 30func mi_w(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 } 31func mi_slen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n } 32 33// Parse "scheme://host[/path...]" -> host copied to host_out (NUL-term), path start index into url (or -1 => "/"). 34// Returns host length, or -1 on malformed. 35func mi_split(url: *u8, host_out: *u8, path_idx_out: *i64) -> i64 { 36 var i: i64 = 0 37 // find "://" 38 var found: i64 = 0 - 1 39 while url[i] != (0 as u8) { 40 if (url[i] as i64)==58 { if (url[i+1] as i64)==47 { if (url[i+2] as i64)==47 { found = i + 3; } } } 41 if found >= 0 { break } 42 i = i + 1 43 } 44 if found < 0 { return 0 - 1 } 45 var h: i64 = found 46 var ho: i64 = 0 47 while url[h] != (0 as u8) { 48 let c: i64 = url[h] as i64 49 if c == 47 { break } // '/' ends host 50 host_out[ho] = url[h]; ho = ho + 1; h = h + 1 51 } 52 host_out[ho] = 0 as u8 53 if url[h] == (0 as u8) { path_idx_out[0] = 0 - 1 } else { path_idx_out[0] = h } 54 return ho 55} 56 57// courteous busy-wait pacing (ms). No sleep syscall dependency assumption -> nanosleep via syscall 35 if available, 58// else a bounded spin. We use nanosleep (clock is fine; determinism not required for a live network importer). 59func mi_pace_ms(ms: i64) -> i64 { 60 if ms <= 0 { return 0 } 61 let ts: *i64 = sys_mmap(16) as *i64 62 ts[0] = ms / 1000 // tv_sec 63 ts[1] = (ms % 1000) * MI_MAGIC_1000000 // tv_nsec 64 __syscall(35, ts as i64, 0, 0, 0, 0, 0) // nanosleep(req, NULL) 65 return 0 66} 67 68// The polite import. Returns pages written (>=0), or negative: -1 malformed url, -2 robots DISALLOW (refused), -3 no pages. 69func mi_import(page_url: *u8, ukey: *u8, slug: *u8, store: *TrustStore) -> i64 { 70 let host: *u8 = sys_mmap(512) 71 let pidx_p: *i64 = sys_mmap(8) as *i64 72 let hlen: i64 = mi_split(page_url, host, pidx_p) 73 if hlen < 0 { mi_w("MEDIA-IMPORT: malformed url\n" as *u8); return 0 - 1 } 74 // the request path (default "/") 75 var path: *u8 = "/" as *u8 76 var plen: i64 = 1 77 if pidx_p[0] >= 0 { path = ((page_url as i64) + pidx_p[0]) as *u8; plen = mi_slen(path) } 78 79 // ---- POLITENESS 1: fetch + honor robots.txt ---- 80 let rob_url: *u8 = sys_mmap(MI_MAGIC_1024) 81 var o: i64 = 0 82 let pre: *u8 = "https://" as *u8; var pi: i64 = 0; while pre[pi]!=(0 as u8){ rob_url[o]=pre[pi]; o=o+1; pi=pi+1 } 83 var hi: i64 = 0; while host[hi]!=(0 as u8){ rob_url[o]=host[hi]; o=o+1; hi=hi+1 } 84 let rp: *u8 = "/robots.txt" as *u8; var ri: i64 = 0; while rp[ri]!=(0 as u8){ rob_url[o]=rp[ri]; o=o+1; ri=ri+1 } 85 rob_url[o] = 0 as u8 86 87 let robots: *u8 = sys_mmap(MI_ROBOTS_CAP) 88 let rstatus: *i64 = sys_mmap(8) as *i64 89 let rlen: i64 = nx_https_fetch_follow_chrome(rob_url, store, robots, MI_ROBOTS_CAP, 4, rstatus) 90 var rob_bytes: i64 = 0 91 if rlen > 0 { if rstatus[0] == 200 { rob_bytes = rlen } } // 404/other => treat as no robots.txt (full allow per RFC MI_MAGIC_9309) 92 93 if nx_robots_allowed(robots, rob_bytes, MI_UA, MI_UA_LEN, path, plen) != 1 { 94 mi_w("MEDIA-IMPORT: REFUSED by robots.txt (polite) -> " as *u8); mi_w(page_url); mi_w("\n" as *u8) 95 return 0 - 2 96 } 97 // pacing: host Crawl-delay (seconds) or the courteous floor 98 var pace_ms: i64 = MI_POLITE_FLOOR_MS 99 let cd: i64 = nx_robots_crawl_delay(robots, rob_bytes, MI_UA, MI_UA_LEN) 100 if cd > 0 { pace_ms = cd * 1000 } 101 mi_pace_ms(pace_ms) // pause before the first real request 102 103 // ---- reuse the PROVEN engine (Chrome fetch + img harvest + integrity + cbx) into the PRIVATE area ---- 104 let dest: *u8 = sys_mmap(MI_MAGIC_1024) 105 rl_personal_slug_dir(ukey, slug, dest) 106 let title_out: *u8 = sys_mmap(512); title_out[0] = 0 as u8 107 let pages: i64 = mg_gallery_to_pages(page_url, dest, "" as *u8, store, title_out) 108 if pages <= 0 { mi_w("MEDIA-IMPORT: no images harvested\n" as *u8); return 0 - 3 } 109 var teff: *u8 = title_out 110 if title_out[0] == (0 as u8) { teff = slug } 111 rl_add_personal(ukey, slug, teff, mi_slen(teff)) 112 mi_w("MEDIA-IMPORT OK slug=" as *u8); mi_w(slug); mi_w(" pages=" as *u8) 113 // print pages 114 let b: *u8=sys_mmap(24); var m: i64=pages; var k: i64=0; if m==0{b[0]=48 as u8;k=1} let t: *u8=sys_mmap(24); while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1} var z: i64=0; while z<k{b[z]=t[k-1-z];z=z+1} sys_write(1,b,k) 115 mi_w(" -> personal area (robots-honored, paced)\n" as *u8) 116 return pages 117} 118 119func main(argc: i64, argv: *i64) -> i64 { 120 if argc < 4 { mi_w("usage: nx_media_import <page_url> <slug> <ukey>\n" as *u8); return 2 } 121 let r: i64 = nx_trust_store_load_from_certdata("data/mozilla_certdata.txt" as *u8, 512, MI_MAGIC_4194304) 122 if r <= 0 { mi_w("MEDIA-IMPORT: certdata load failed\n" as *u8); return 3 } 123 let store: *TrustStore = r as *TrustStore 124 let rc: i64 = mi_import(argv[1] as *u8, argv[3] as *u8, argv[2] as *u8, store) 125 if rc == 0 - 2 { return 4 } // robots refusal -> distinct exit for the daemon 126 if rc < 0 { return 1 } 127 return 0 128}