code wiki / _hdl_build / nx_cleanwatch_serve.nx
`nx_cleanwatch_serve.nx` source

↩ module page · 189 lines · 14079 B
1// nx_cleanwatch_serve.nx -- HANDS-ON, REAL: the operator enters ANY site; we fetch it over our own sovereign TLS,
2// harvest its ACTUAL resource URLs, and run nx_web_filter over them -- showing the real ad/tracker/miner/popunder
3// requests THAT page makes that we'd block. Real evidence on real sites, not a canned list. Bind 0.0.0.0 -> WSL2
4// forwards Windows localhost (same path as nx_browser_view_serve). Run from nxc2 root (certdata is relative).
5//   Honest scope: static fetch shows resources in the delivered HTML; URLs the page builds via JS at runtime need
6//   the JS-exec sniffer layer (noted in the report). Run: nx_cleanwatch_serve [port]   license_tier: ORIGINAL
7import "nx_syscalls.nx"
8import "nx_http_server.nx"
9import "nx_x509_trust_store.nx"
10import "nx_trust_store_load_from_certdata.nx"
11import "nx_https_fetch_follow.nx"
12import "nx_media_signal.nx"
13import "nx_web_filter.nx"
14import "nx_video_sniff.nx"     // vs_extract_scripts + (transitively) js_run_source_keep / js_pending_url
15const K_MAGIC_1048576: i64 = 1048576
16const K_MAGIC_4194304: i64 = 4194304
17const K_MAGIC_262144: i64 = 262144
18const K_MAGIC_4096: i64 = 4096
19const K_MAGIC_4095: i64 = 4095
20const K_MAGIC_8100: i64 = 8100
21const K_MAGIC_16384: i64 = 16384
22const K_MAGIC_65536: i64 = 65536
23
24func cw_slen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
25func cw_put(dst: *u8, off: i64, s: *u8) -> i64 { var i: i64=0; while s[i]!=(0 as u8){dst[off]=s[i];off=off+1;i=i+1} return off }
26func cw_putn(dst: *u8, off: i64, v: i64) -> i64 { var m: i64=v; if m==0 { dst[off]=48 as u8; return off+1 } let t: *u8=sys_mmap(24); var k: i64=0; while m>0 { t[k]=(48+(m%10)) as u8; m=m/10; k=k+1 } var j: i64=0; while j<k { dst[off]=t[k-1-j]; off=off+1; j=j+1 } return off }
27func cw_datoi(s: *u8) -> i64 { var v: i64=0; var i: i64=0; while (s[i]&0xff)>=48 { if (s[i]&0xff)>57 { i=i } else { v=v*10+((s[i]&0xff)-48); i=i+1 } if (s[i]&0xff)<48 { i=i } } return v }
28func cw_hexval(c: i64) -> i64 { if c>=48 { if c<=57 { return c-48 } } if c>=97 { if c<=102 { return c-87 } } if c>=65 { if c<=70 { return c-55 } } return 0 }
29func cw_dstarts(p: *u8, plen: i64, pre: *u8) -> i64 { var i: i64=0; while pre[i]!=(0 as u8) { if i>=plen { return 0 } if (p[i]&0xff)!=(pre[i]&0xff) { return 0 } i=i+1 } return 1 }
30// HTML-escape s into dst (prevent the displayed URL from injecting markup)
31func cw_esc(dst: *u8, off: i64, s: *u8, sl: i64) -> i64 {
  var i: i64=0
  while i<sl { let c: i64=s[i]&0xff
      if c==60 { off=cw_put(dst,off,"&lt;" as *u8) } else { if c==62 { off=cw_put(dst,off,"&gt;" as *u8) } else { if c==38 { off=cw_put(dst,off,"&amp;" as *u8) } else { if c==34 { off=cw_put(dst,off,"&quot;" as *u8) } else { dst[off]=c as u8; off=off+1 } } } }
      i=i+1 }
  return off
37}
38// parse ?url= (param-boundary aware) + percent-decode into out; returns length
39func cw_query_url(q: *u8, qlen: i64, out: *u8, cap: i64) -> i64 {
  var start: i64 = 0-1; var si: i64=0
  while si<qlen { var isp: i64=0; if si==0 { isp=1 } else { let pc: i64=q[si-1]&0xff; if pc==38 { isp=1 } if pc==63 { isp=1 } }
      if isp==1 { if (si+4)<=qlen { if (q[si]&0xff)==117 { if (q[si+1]&0xff)==114 { if (q[si+2]&0xff)==108 { if (q[si+3]&0xff)==61 { start=si; si=qlen } } } } } } si=si+1 }
  if start<0 { out[0]=0 as u8; return 0 }
  var i: i64=start+4; var o: i64=0; var go: i64=1
  while go==1 { if i>=qlen { go=0 } else { let c: i64=q[i]&0xff
      if c==38 { go=0 } else { if c==37 { if (i+2)<qlen { let hi: i64=cw_hexval(q[i+1]&0xff); let lo: i64=cw_hexval(q[i+2]&0xff); if o<(cap-1) { out[o]=(hi*16+lo) as u8; o=o+1 } i=i+3 } else { i=i+1 } } else { if c==43 { if o<(cap-1){out[o]=32 as u8;o=o+1} i=i+1 } else { if o<(cap-1){out[o]=c as u8;o=o+1} i=i+1 } } } } }
  out[o]=0 as u8; return o
48}
49// is byte a URL-terminator?
50func cw_delim(c: i64) -> i64 { if c==34{return 1} if c==39{return 1} if c==32{return 1} if c==60{return 1} if c==62{return 1} if c==41{return 1} if c==125{return 1} if c==93{return 1} if c==92{return 1} if c==10{return 1} if c==9{return 1} if c==13{return 1} return 0 }
51func cw_contains(hay: *u8, hl: i64, needle: *u8, nl: i64) -> i64 { if nl==0 {return 1} if nl>hl {return 0} var i: i64=0; let last: i64=hl-nl; while i<=last { var j: i64=0; var m: i64=1; while j<nl { if (hay[i+j]&0xff)!=(needle[j]&0xff){m=0;j=nl} else {j=j+1} } if m==1 {return 1} i=i+1 } return 0 }
52
53func send_html(cfd: i64, body: *u8, blen: i64) -> i64 {
  let resp: *u8 = sys_mmap(blen + 512)
  var o: i64 = cw_put(resp, 0, "HTTP/1.1 200 OK\r\nContent-Type: text/html; charset=utf-8\r\nContent-Length: " as *u8)
  o = cw_putn(resp, o, blen); o = cw_put(resp, o, "\r\nConnection: close\r\n\r\n" as *u8)
  var i: i64=0; while i<blen { resp[o]=body[i]; o=o+1; i=i+1 }
  return nx_http_server_send_response(cfd, resp, o)
59}
60func cw_head(page: *u8, o: i64) -> i64 {
  return cw_put(page, o, "<!doctype html><html><head><meta charset=utf-8><title>Nishi Clean-Watch</title><style>body{font-family:system-ui,sans-serif;max-width:900px;margin:2rem auto;padding:0 1rem;background:#0f1115;color:#e8e8ea;line-height:1.5}h1{margin:0 0 .2rem}.sub{color:#9aa;font-size:.92rem}.card{background:#171a21;border:1px solid #2a2f3a;border-radius:10px;padding:1rem 1.2rem;margin:1.2rem 0}ul{list-style:none;padding:0}li{padding:.25rem 0;border-bottom:1px solid #21252e;font-size:.9rem}code{color:#bcd;word-break:break-all}.blk{color:#ff6b6b}.ok{color:#51cf66}.cat{color:#889;font-size:.82rem}input{background:#0c0e12;color:#eee;border:1px solid #333;border-radius:6px;padding:.5rem;width:60%}button{background:#2b6cff;color:#fff;border:0;border-radius:6px;padding:.55rem 1rem;cursor:pointer}a{color:#7ab7ff}.tot{font-weight:600;margin:.6rem 0}video{width:100%;border-radius:8px;background:#000}</style></head><body>" as *u8)
62}
63
64func cw_home(page: *u8) -> i64 {
  var o: i64 = cw_head(page, 0)
  o = cw_put(page, o, "<h1>Nishi Clean-Watch</h1><div class=sub>Enter any site you browse to. We fetch it over our own sovereign TLS and run the <code>nx_web_filter</code> engine over its <b>actual</b> resources &mdash; showing the real ad / tracker / crypto-miner / popunder requests that page makes that we'd block to keep you safe.</div>" as *u8)
  o = cw_put(page, o, "<div class=card><form action=/analyze><input name=url placeholder=\"https://example.com\" autofocus> <button>Check what we&#39;d block</button></form></div>" as *u8)
  o = cw_put(page, o, "<div class=card><h2>Clean player</h2><div class=sub>Clean-watch on a legal test stream (the video, nothing around it):</div><video controls preload=metadata src=\"https://commondatastorage.googleapis.com/gtv-videos-bucket/sample/BigBuckBunny.mp4\"></video></div>" as *u8)
  o = cw_put(page, o, "</body></html>" as *u8)
  return o
71}
72
73func cw_analyze(cfd: i64, t: *i64, store: *TrustStore, url: *u8, ulen: i64) -> i64 {
  let page: *u8 = sys_mmap(K_MAGIC_1048576)
  var o: i64 = cw_head(page, 0)
  o = cw_put(page, o, "<h1>Protection report</h1><div class=sub>for <code>" as *u8); o = cw_esc(page, o, url, ulen); o = cw_put(page, o, "</code></div>" as *u8)
  // fetch the page over sovereign TLS
  let cap: i64 = K_MAGIC_4194304
  let html: *u8 = sys_mmap(cap)
  let status: *i64 = sys_mmap(8) as *i64
  let hn: i64 = nx_https_fetch_follow(url, store, html, cap, 6, status)
  if hn <= 0 {
      o = cw_put(page, o, "<div class=card><b class=blk>Fetch failed</b> (status "); o = cw_putn(page, o, status[0]); o = cw_put(page, o, ") &mdash; the site may block non-browser clients or need JS. <a href=/>&larr; try another</a></div></body></html>" as *u8)
      return send_html(cfd, page, o)
  }
  // harvest every http(s):// URL, filter each, render blocked ones (deduped)
  let seen: *u8 = sys_mmap(K_MAGIC_262144); var seen_o: i64 = 0
  let ub: *u8 = sys_mmap(K_MAGIC_4096)
  var total: i64 = 0; var blk: i64 = 0
  let cat: *i64 = sys_mmap(8) as *i64
  o = cw_put(page, o, "<div class=card><ul>" as *u8)
  var i: i64 = 0
  while i < hn {
      var hit: i64 = 0
      if cw_dstarts(((html as i64)+i) as *u8, hn-i, "https://" as *u8) == 1 { hit = 8 }
      if hit == 0 { if cw_dstarts(((html as i64)+i) as *u8, hn-i, "http://" as *u8) == 1 { hit = 7 } }
      if hit > 0 {
          var k: i64 = 0; var j: i64 = i
          var go: i64 = 1
          while go == 1 { if j>=hn { go=0 } else { let c: i64=html[j]&0xff; if cw_delim(c)==1 { go=0 } else { if k<K_MAGIC_4095 { ub[k]=c as u8; k=k+1 } j=j+1 } } }
          ub[k]=0 as u8
          if k > 10 {
              total = total + 1
              if wf_should_block(t, ub, k, cat) == 1 {
                  if cw_contains(seen, seen_o, ub, k) == 0 {
                      if (seen_o+k+1) < K_MAGIC_262144 { var a: i64=0; while a<k { seen[seen_o]=ub[a]; seen_o=seen_o+1; a=a+1 } seen[seen_o]=10 as u8; seen_o=seen_o+1 }
                      blk = blk + 1
                      o = cw_put(page, o, "<li><b class=blk>BLOCK</b> <span class=cat>[" as *u8); o = cw_put(page, o, wf_cat_name(cat[0])); o = cw_put(page, o, "]</span> <code>" as *u8); o = cw_esc(page, o, ub, k); o = cw_put(page, o, "</code></li>" as *u8)
                  }
              }
          }
          i = j
      } else { i = i + 1 }
  }
  // PASS 2: JS-EXEC -- run the page's inline scripts on our engine, filter the fetch/XHR URLs they BUILD at
  // runtime (the ones not present as literals in the static HTML). Fetches PEND (no real network); we read them.
  let scripts: *u8 = sys_mmap(K_MAGIC_262144)
  let slen: i64 = vs_extract_scripts(html, hn, scripts, K_MAGIC_262144)
  if slen > 0 {
      let outbox: *i64 = sys_mmap(16) as *i64
      if js_run_source_keep(scripts, slen, outbox) == 0 {
          let genv: *i64 = (outbox[1]) as *i64
          if (genv as i64) != 0 {
              var ji: i64 = 0
              var jgo: i64 = 1
              while jgo == 1 {
                  if ji >= K_MAGIC_4096 { jgo = 0 }
                  else {
                      let jul: i64 = js_pending_url(genv, ji, ub, K_MAGIC_4095)
                      if jul == 0 { jgo = 0 }
                      else {
                          total = total + 1
                          if wf_should_block(t, ub, jul, cat) == 1 {
                              if cw_contains(seen, seen_o, ub, jul) == 0 {
                                  if (seen_o+jul+1) < K_MAGIC_262144 { var a2: i64=0; while a2<jul { seen[seen_o]=ub[a2]; seen_o=seen_o+1; a2=a2+1 } seen[seen_o]=10 as u8; seen_o=seen_o+1 }
                                  blk = blk + 1
                                  o = cw_put(page, o, "<li><b class=blk>BLOCK</b> <span class=cat>[js&middot;" as *u8); o = cw_put(page, o, wf_cat_name(cat[0])); o = cw_put(page, o, "]</span> <code>" as *u8); o = cw_esc(page, o, ub, jul); o = cw_put(page, o, "</code></li>" as *u8)
                              }
                          }
                          ji = ji + 1
                      }
                  }
              }
          }
      }
  }
  o = cw_put(page, o, "</ul>" as *u8)
  if blk == 0 { o = cw_put(page, o, "<div class=tot><b class=ok>No known malicious/ad requests found</b> in the delivered HTML.</div>" as *u8) }
  else { o = cw_put(page, o, "<div class=tot>Would <span class=blk>BLOCK "); o = cw_putn(page, o, blk); o = cw_put(page, o, "</span> requests (of " as *u8); o = cw_putn(page, o, total); o = cw_put(page, o, " external URLs scanned) to keep you safe.</div>" as *u8) }
  o = cw_put(page, o, "<div class=sub>Fetched " as *u8); o = cw_putn(page, o, hn); o = cw_put(page, o, " bytes. Note: this scans resources in the delivered HTML; URLs the page builds via JavaScript at runtime need the JS-exec sniffer layer (also ours). <a href=/>&larr; test another site</a></div></div></body></html>" as *u8)
  return send_html(cfd, page, o)
152}
153
154func main(argc: i64, argv: *i64) -> i64 {
  var port: i64 = K_MAGIC_8100
  if argc >= 2 { port = cw_datoi(argv[1] as *u8) }
  let r: i64 = nx_trust_store_load_from_certdata("data/mozilla_certdata.txt" as *u8, 512, K_MAGIC_4194304)
  if r <= 0 { sys_write(2, "CLEANWATCH certdata load failed (run from nxc2 root)\n" as *u8, 52); sys_exit(1); return 1 }
  let store: *TrustStore = r as *TrustStore
  let t: *i64 = wf_new(); wf_seed(t)
  let addr: *u8 = sys_mmap(16)
  nx_http_server_addr_any(addr, port)
  let vb: *i64 = sys_mmap(8) as *i64
  let lfd: i64 = nx_http_server_listen(addr, 64, vb)
  if lfd < 0 { sys_write(2, "CLEANWATCH listen failed\n" as *u8, 25); sys_exit(1); return 1 }
  let banner: *u8 = sys_mmap(96); var bo: i64 = cw_put(banner, 0, "nx_cleanwatch_serve: http://localhost:" as *u8); bo = cw_putn(banner, bo, port); bo = cw_put(banner, bo, "/\n" as *u8); sys_write(1, banner, bo)
  var running: i64 = 1
  while running == 1 {
      let cfd: i64 = nx_http_server_accept_one(lfd, vb)
      if cfd >= 0 {
          let req: *u8 = sys_mmap(K_MAGIC_16384)
          let om: *i64=sys_mmap(8) as *i64; let opo: *i64=sys_mmap(8) as *i64; let opl: *i64=sys_mmap(8) as *i64; let ocl: *i64=sys_mmap(8) as *i64; let obo: *i64=sys_mmap(8) as *i64; let orn: *i64=sys_mmap(8) as *i64
          let rc: i64 = nx_http_server_read_request(cfd, req, K_MAGIC_16384, om, opo, opl, ocl, obo, orn)
          if rc == NXS_OK {
              let path: *u8 = ((req as i64)+opo[0]) as *u8
              let plen: i64 = opl[0]
              if cw_dstarts(path, plen, "/analyze" as *u8) == 1 {
                  let ub: *u8 = sys_mmap(K_MAGIC_4096)
                  let un: i64 = cw_query_url(path, plen, ub, K_MAGIC_4096)
                  if un > 0 { cw_analyze(cfd, t, store, ub, un) }
                  else { let p2: *u8 = sys_mmap(K_MAGIC_4096); let n2: i64 = cw_home(p2); send_html(cfd, p2, n2) }
              } else {
                  let p: *u8 = sys_mmap(K_MAGIC_65536); let n: i64 = cw_home(p); send_html(cfd, p, n)
              }
          } else { sys_close(cfd) }
      }
  }
  return 0
189}
nx_cleanwatch_serve.nx source

`nx_cleanwatch_serve.nx` source
