nx_clean_serve.nx source

1// nx_clean_serve.nx -- the ETHICAL clean-serve policy engine: the win-win-win transform for the Nishi search 2// engine's neutral, safe delivery of hostile sites. The problem with strip-and-block (ad_strip_junk) is that it 3// nukes EVERY script -> the user is safe but the SITE earns nothing (all ads die too). The problem with letting 4// ads through is malvertising (redirect / popunder / cryptojack / tabnab / fingerprint). The middle path is a 5// THIRD verdict -- SANITIZE: an ad is KEPT as an inert, script-free static creative (image + a click link with 6// rel="noopener noreferrer nofollow") so the SITE still earns the impression + click, while every attack vector 7// (which ALL require active content -- JS, unsandboxed frames, beacons, auto-redirect) is removed. Result: 8// USER safe (no active content survives) + gets the content native HTML5; 9// SITE monetizes (its safe static ads render + click through); 10// SEARCH neutral (a difficult site is served cleanly, not deranked). 11// Composes nx_web_filter's data-driven category verdict. Pure core: the archive daemon imports it for /visit and 12// /preserve; the gate tests it headless with an adversary corpus. Emits a machine-readable SAFETY RECEIPT. 13// license_tier: ORIGINAL 14import "nx_syscalls.nx" 15import "nx_web_filter.nx" // wf_new/wf_seed/wf_should_block + WF_AD/WF_TRACK/WF_MAL/WF_MINER/WF_POPUP/WF_CONTENT 16const CS_MAGIC_4096: i64 = 4096 17const CS_MAGIC_4095: i64 = 4095 18 19// --- clean-serve verdict (a superset of web_filter's BLOCK/ALLOW: the AD case graduates to SANITIZE) --- 20const CS_ALLOW: i64 = 0 // content / first-party / unknown -> pass through 21const CS_BLOCK: i64 = 1 // an attack vector (tracker/miner/malware/popunder) -> remove entirely 22const CS_SANITIZE: i64 = 2 // an ad -> keep as an inert static creative (site monetizes, user stays safe) 23 24// --- safety receipt: a flat i64 counters buffer (indices below). The honest, adversary/critic-checkable output. --- 25const CR_SCRIPTS: i64 = 0 // <script> blocks removed 26const CR_IFRAMES: i64 = 1 // active third-party frames removed 27const CR_HANDLERS: i64 = 2 // inline on*= event handlers stripped 28const CR_REDIRECTS: i64 = 3 // meta-refresh + javascript: exec/redirect vectors neutralized 29const CR_MINER: i64 = 4 // crypto-miner request URLs neutralized 30const CR_POPUP: i64 = 5 // popunder request URLs neutralized 31const CR_MAL: i64 = 6 // malware/scam request URLs neutralized 32const CR_TRACK: i64 = 7 // tracker/beacon request URLs neutralized 33const CR_ADS_KEPT: i64 = 8 // ad creatives PRESERVED as safe static (the site's monetization, intact) 34const CR_BLOCKED: i64 = 9 // total attack request URLs neutralized (sum of miner+popup+mal+track) 35const CR_N: i64 = 10 36 37// --- small helpers (mirror the proven ad_*/cw_* idioms; kept local so this is a self-contained core) --- 38func cs_slen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n } 39func cs_put(dst: *u8, off: i64, s: *u8) -> i64 { var i: i64=0; while s[i]!=(0 as u8){dst[off]=s[i];off=off+1;i=i+1} return off } 40func cs_putn(dst: *u8, off: i64, v: i64) -> i64 { var m: i64=v; if m==0 { dst[off]=48 as u8; return off+1 } let t: *u8=sys_mmap(24); var k: i64=0; while m>0 { t[k]=(48+(m%10)) as u8; m=m/10; k=k+1 } var j: i64=0; while j<k { dst[off]=t[k-1-j]; off=off+1; j=j+1 } return off } 41func cs_lc(c: i64) -> i64 { if c>=65 { if c<=90 { return c+32 } } return c } 42func cs_ci_at(src: *u8, slen: i64, pos: i64, lit: *u8, litlen: i64) -> i64 { if pos+litlen>slen { return 0 } var i: i64=0; while i<litlen { if cs_lc(src[pos+i]&0xff)!=cs_lc(lit[i]&0xff) { return 0 } i=i+1 } return 1 } 43func cs_skip_block(src: *u8, slen: i64, from: i64, close: *u8, closelen: i64) -> i64 { 44 var j: i64=from; var found: i64=0 45 while found==0 { if j>=slen { found=1 } else { if cs_ci_at(src, slen, j, close, closelen)==1 { j=j+closelen; found=1 } else { j=j+1 } } } 46 return j 47} 48// a real HTML tag is never this long; bounding cs_tag_end (and the cs_tag_attr/cs_within that scan up to its result) 49// to it stops an unclosed-quote tag from scanning to EOF on EVERY occurrence -- the O(n)-per-tag blowup that made a 50// tag-dense page (a live-cam roomlist) take ~40s to clean. 16 KiB comfortably covers even a big inline data: URI. 51const CS_MAX_TAG: i64 = 16384 52// index of this tag's closing '>' (quote-aware so a '>' inside an attribute value doesn't end it early) 53func cs_tag_end(src: *u8, slen: i64, from: i64) -> i64 { 54 var j: i64 = from; var q: i64 = 0 55 let lim: i64 = from + CS_MAX_TAG 56 while j < slen { 57 if j >= lim { return j } 58 let c: i64 = src[j]&0xff 59 if q==0 { if c==34 { q=1 } else { if c==39 { q=2 } else { if c==62 { return j } } } } 60 else { if q==1 { if c==34 { q=0 } } else { if c==39 { q=0 } } } 61 j = j+1 62 } 63 return slen-1 64} 65// case-insensitive "does [from,to) contain lit" 66func cs_within(src: *u8, from: i64, to: i64, lit: *u8, litlen: i64) -> i64 { 67 var i: i64 = from 68 while i <= to-litlen { if cs_ci_at(src, to, i, lit, litlen)==1 { return 1 } i=i+1 } 69 return 0 70} 71// is byte a delimiter that ends a bare <a> tag name (so <article> is NOT treated as <a>) 72func cs_a_delim(c: i64) -> i64 { if c==32 {return 1} if c==9 {return 1} if c==10 {return 1} if c==13 {return 1} if c==62 {return 1} return 0 } 73// is byte a URL terminator (for harvesting request URLs from the raw HTML) 74func cs_url_delim(c: i64) -> i64 { if c==34{return 1} if c==39{return 1} if c==32{return 1} if c==60{return 1} if c==62{return 1} if c==41{return 1} if c==125{return 1} if c==93{return 1} if c==92{return 1} if c==10{return 1} if c==9{return 1} if c==13{return 1} return 0 } 75// extract the value of attribute `name` from within a tag span [from,to) (quoted or bare, space-delimited attr) 76func cs_tag_attr(src: *u8, from: i64, to: i64, name: *u8, namelen: i64, out: *u8, cap: i64) -> i64 { 77 var i: i64 = from+1 78 while i < to { 79 var bnd: i64 = 0 80 let pc: i64 = src[i-1]&0xff 81 if pc==32 { bnd=1 } else { if pc==9 { bnd=1 } else { if pc==10 { bnd=1 } else { if pc==13 { bnd=1 } } } } 82 if bnd==1 { if cs_ci_at(src, to, i, name, namelen)==1 { 83 let after: i64 = i+namelen 84 if after<to { if (src[after]&0xff)==61 { 85 var vi: i64 = after+1; var qc: i64 = 0 86 if vi<to { let c0: i64=src[vi]&0xff; if c0==34 { qc=34; vi=vi+1 } else { if c0==39 { qc=39; vi=vi+1 } } } 87 var o: i64=0; var go: i64=1 88 while go==1 { if vi>=to { go=0 } else { let c: i64=src[vi]&0xff 89 if qc!=0 { if c==qc { go=0 } else { if o<(cap-1){out[o]=c as u8;o=o+1} vi=vi+1 } } 90 else { if c==32 { go=0 } else { if c==62 { go=0 } else { if o<(cap-1){out[o]=c as u8;o=o+1} vi=vi+1 } } } } } 91 out[o]=0 as u8; return o 92 } } 93 } } 94 i=i+1 95 } 96 out[0]=0 as u8; return 0 97} 98 99// --- allocate + zero a fresh receipt buffer --- 100func cs_rc_new() -> *i64 { let rc: *i64 = sys_mmap(8*CR_N) as *i64; var z: i64=0; while z<CR_N { rc[z]=0; z=z+1 } return rc } 101 102// --- the 3-way verdict: compose the web-filter category into BLOCK / SANITIZE / ALLOW --- 103func cs_verdict(t: *i64, url: *u8, ul: i64, out_cat: *i64) -> i64 { 104 let blk: i64 = wf_should_block(t, url, ul, out_cat) 105 if blk==0 { return CS_ALLOW } 106 if out_cat[0]==WF_AD { return CS_SANITIZE } // an ad -> sanitize+keep, not blanket-block 107 return CS_BLOCK // tracker / miner / malware / popunder -> remove 108} 109 110// The receipt's attack TALLY is evidence, not correctness. On a URL-dense page (a live-cam roomlist carries 111// thousands of thumbnail/link URLs) verdict-checking EVERY one against the O(table) filter is the dominant cost and 112// made large pages take ~40s. Bound the verdict calls: the transform still neutralizes EVERY attack regardless (the 113// script/iframe/handler stripping is verdict-independent); only the receipt COUNTS are capped past the budget. 114const CS_VERDICT_BUDGET: i64 = 400 115// --- scan the raw HTML for request URLs, tally the attacks we neutralize (evidence for the receipt) --- 116func cs_scan_urls(src: *u8, slen: i64, t: *i64, rc: *i64) -> i64 { 117 let ub: *u8 = sys_mmap(CS_MAGIC_4096) 118 let cat: *i64 = sys_mmap(8) as *i64 119 var vc: i64 = 0 120 var i: i64 = 0 121 while i < slen { 122 var hit: i64 = 0 123 if cs_ci_at(src, slen, i, "https://" as *u8, 8)==1 { hit=1 } 124 if hit==0 { if cs_ci_at(src, slen, i, "http://" as *u8, 7)==1 { hit=1 } } 125 if hit==1 { 126 var k: i64 = 0; var j: i64 = i; var go: i64 = 1 127 while go==1 { if j>=slen { go=0 } else { let c: i64=src[j]&0xff; if cs_url_delim(c)==1 { go=0 } else { if k<CS_MAGIC_4095 { ub[k]=c as u8; k=k+1 } j=j+1 } } } 128 ub[k]=0 as u8 129 if k>10 { if vc < CS_VERDICT_BUDGET { 130 vc = vc + 1 131 let v: i64 = cs_verdict(t, ub, k, cat) 132 if v==CS_BLOCK { 133 rc[CR_BLOCKED]=rc[CR_BLOCKED]+1 134 if cat[0]==WF_MINER { rc[CR_MINER]=rc[CR_MINER]+1 } 135 if cat[0]==WF_POPUP { rc[CR_POPUP]=rc[CR_POPUP]+1 } 136 if cat[0]==WF_MAL { rc[CR_MAL]=rc[CR_MAL]+1 } 137 if cat[0]==WF_TRACK { rc[CR_TRACK]=rc[CR_TRACK]+1 } 138 } 139 } } 140 i = j 141 } else { i = i + 1 } 142 } 143 return 0 144} 145 146// --- THE TRANSFORM: neutralize all active/attack content, KEEP+sanitize ad creatives, pass content. --- 147// Removes: <script>/<iframe>/<noscript>/<embed>/<object> blocks, <meta refresh>, inline on*= handlers, and 148// javascript: URIs in attribute values. Rewrites <a ...> to carry rel="noopener noreferrer nofollow" (kills 149// reverse-tabnabbing + referrer leak) while keeping the click, so the ad still monetizes. Everything else -- 150// text, images, native <video>, the article DOM -- passes through byte-for-byte. Counts into the receipt `rc`. 151func cs_clean_into(out: *u8, off: i64, cap: i64, src: *u8, slen: i64, t: *i64, rc: *i64) -> i64 { 152 var o: i64 = off; var i: i64 = 0; var vc: i64 = 0 153 let av: *u8 = sys_mmap(CS_MAGIC_4096); let acat: *i64 = sys_mmap(8) as *i64 // reused per-tag scratch -- allocated ONCE, not re-mmap'd per <a>/<img> (per-tag mmap = VMA growth = the large-page O(n^2) slowdown) 154 while i < slen { 155 var handled: i64 = 0 156 if cs_ci_at(src, slen, i, "<script" as *u8, 7)==1 { i=cs_skip_block(src, slen, i+7, "</script>" as *u8, 9); rc[CR_SCRIPTS]=rc[CR_SCRIPTS]+1; handled=1 } 157 if handled==0 { if cs_ci_at(src, slen, i, "<iframe" as *u8, 7)==1 { i=cs_skip_block(src, slen, i+7, "</iframe>" as *u8, 9); rc[CR_IFRAMES]=rc[CR_IFRAMES]+1; handled=1 } } 158 if handled==0 { if cs_ci_at(src, slen, i, "<noscript" as *u8, 9)==1 { i=cs_skip_block(src, slen, i+9, "</noscript>" as *u8, 11); handled=1 } } 159 if handled==0 { if cs_ci_at(src, slen, i, "<embed" as *u8, 6)==1 { i=cs_skip_block(src, slen, i+6, ">" as *u8, 1); rc[CR_IFRAMES]=rc[CR_IFRAMES]+1; handled=1 } } 160 if handled==0 { if cs_ci_at(src, slen, i, "<object" as *u8, 7)==1 { i=cs_skip_block(src, slen, i+7, "</object>" as *u8, 9); rc[CR_IFRAMES]=rc[CR_IFRAMES]+1; handled=1 } } 161 // <meta http-equiv=refresh ...> -> drop (auto-redirect / meta-refresh malvertising). Other metas kept. 162 if handled==0 { if cs_ci_at(src, slen, i, "<meta" as *u8, 5)==1 { 163 let te: i64 = cs_tag_end(src, slen, i) 164 if cs_within(src, i, te, "refresh" as *u8, 7)==1 { i=te+1; rc[CR_REDIRECTS]=rc[CR_REDIRECTS]+1; handled=1 } 165 } } 166 // <a ...> -> KEEP but harden: inject rel; the rest of the tag copies normally (so the on*= stripper + the 167 // javascript: neutralizer below still scrub it). If the href is an ad, it counts as a preserved safe ad. 168 if handled==0 { if cs_ci_at(src, slen, i, "<a" as *u8, 2)==1 { 169 let nc: i64 = src[i+2]&0xff 170 if cs_a_delim(nc)==1 { 171 let te: i64 = cs_tag_end(src, slen, i) 172 let al: i64 = cs_tag_attr(src, i, te, "href" as *u8, 4, av, CS_MAGIC_4096) 173 if al>0 { if vc < CS_VERDICT_BUDGET { vc = vc + 1; if cs_verdict(t, av, al, acat)==CS_SANITIZE { rc[CR_ADS_KEPT]=rc[CR_ADS_KEPT]+1 } } } 174 o = cs_put(out, o, "<a rel=\"noopener noreferrer nofollow\"" as *u8) 175 i = i + 2 176 handled = 1 177 } 178 } } 179 // <img ...> -> KEPT (a static image is inherently safe). Count it if its src is an ad creative. No consume. 180 if handled==0 { if cs_ci_at(src, slen, i, "<img" as *u8, 4)==1 { 181 let te: i64 = cs_tag_end(src, slen, i) 182 let al: i64 = cs_tag_attr(src, i, te, "src" as *u8, 3, av, CS_MAGIC_4096) 183 if al>0 { if vc < CS_VERDICT_BUDGET { vc = vc + 1; if cs_verdict(t, av, al, acat)==CS_SANITIZE { rc[CR_ADS_KEPT]=rc[CR_ADS_KEPT]+1 } } } 184 } } 185 // inline event handler " on...=..." -> strip (drops onclick/onload/onerror/onmouseover redirect+exec vectors) 186 if handled==0 { if cs_ci_at(src, slen, i, " on" as *u8, 3)==1 { 187 var j: i64=i+3; var ish: i64=0 188 while j<slen { if (j-i)>64 { j=slen } else { if (src[j]&0xff)==61 { ish=1; j=slen } else { if (src[j]&0xff)==32 { j=slen } else { if (src[j]&0xff)==62 { j=slen } else { j=j+1 } } } } } 189 if ish==1 { 190 var k: i64=i+3 191 var f1: i64=1; while f1==1 { if k>=slen { f1=0 } else { if (src[k]&0xff)==61 { k=k+1; f1=0 } else { k=k+1 } } } 192 if k<slen { let qc: i64=src[k]&0xff 193 if qc==34 { k=k+1; var fq: i64=1; while fq==1 { if k>=slen { fq=0 } else { if (src[k]&0xff)==34 { k=k+1; fq=0 } else { k=k+1 } } } } else { if qc==39 { k=k+1; var fs: i64=1; while fs==1 { if k>=slen { fs=0 } else { if (src[k]&0xff)==39 { k=k+1; fs=0 } else { k=k+1 } } } } else { var fb: i64=1; while fb==1 { if k>=slen { fb=0 } else { let cb: i64=src[k]&0xff; if cb==32 { fb=0 } else { if cb==62 { fb=0 } else { k=k+1 } } } } } } 194 } 195 i=k; rc[CR_HANDLERS]=rc[CR_HANDLERS]+1; handled=1 196 } 197 } } 198 // javascript: URI in an attribute value ( ="javascript:.." / ='javascript:.. / =javascript:.. ) -> neutralize 199 if handled==0 { if i>0 { 200 let pc: i64 = src[i-1]&0xff 201 var atv: i64 = 0 202 if pc==34 { atv=1 } else { if pc==39 { atv=1 } else { if pc==61 { atv=1 } } } 203 if atv==1 { if cs_ci_at(src, slen, i, "javascript:" as *u8, 11)==1 { if o<(cap-1){out[o]=35 as u8;o=o+1} i=i+11; rc[CR_REDIRECTS]=rc[CR_REDIRECTS]+1; handled=1 } } 204 } } 205 if handled==0 { if o<(cap-1) { out[o]=src[i]; o=o+1 } i=i+1 } 206 } 207 return o 208} 209 210// --- convenience: run the full policy (scan for evidence + transform) in one call. `rc` must be cs_rc_new()'d. --- 211func cs_clean_page(src: *u8, slen: i64, t: *i64, out: *u8, cap: i64, rc: *i64) -> i64 { 212 cs_scan_urls(src, slen, t, rc) 213 let o: i64 = cs_clean_into(out, 0, cap, src, slen, t, rc) 214 out[o]=0 as u8 215 return o 216} 217 218// --- machine-readable safety receipt (the win-win-win evidence, for the clean-view chrome + census/critic) --- 219func cs_receipt_json(rc: *i64, out: *u8, cap: i64) -> i64 { 220 var o: i64 = 0 221 o = cs_put(out, o, "{\"safe\":true,\"attacks_neutralized\":{\"scripts\":" as *u8); o = cs_putn(out, o, rc[CR_SCRIPTS]) 222 o = cs_put(out, o, ",\"iframes\":" as *u8); o = cs_putn(out, o, rc[CR_IFRAMES]) 223 o = cs_put(out, o, ",\"inline_handlers\":" as *u8); o = cs_putn(out, o, rc[CR_HANDLERS]) 224 o = cs_put(out, o, ",\"redirects\":" as *u8); o = cs_putn(out, o, rc[CR_REDIRECTS]) 225 o = cs_put(out, o, ",\"miners\":" as *u8); o = cs_putn(out, o, rc[CR_MINER]) 226 o = cs_put(out, o, ",\"popunders\":" as *u8); o = cs_putn(out, o, rc[CR_POPUP]) 227 o = cs_put(out, o, ",\"malware\":" as *u8); o = cs_putn(out, o, rc[CR_MAL]) 228 o = cs_put(out, o, ",\"trackers\":" as *u8); o = cs_putn(out, o, rc[CR_TRACK]) 229 o = cs_put(out, o, "},\"ads_preserved_safe\":" as *u8); o = cs_putn(out, o, rc[CR_ADS_KEPT]) 230 o = cs_put(out, o, "}" as *u8) 231 out[o]=0 as u8 232 return o 233}