code wiki / _hdl_build / nx_web_purge_slop.nx
nx_web_purge_slop.nx source
↩ module page · 184 lines · 9819 B
1// nx_web_purge_slop.nx -- SCOPED SLOP PURGE for the web shard (2026-08-04, debt 1785895889):
2// every url:-keyed doc already ingested is re-judged against the MEASURED bad classes ONLY --
3// tag/category listing farms (the ruler's SERP pollution) + the interstitial content markers the
4// live crawler refuses (verification/challenge walls). An offender's pol:<cid> row is banked as
5// "2" (AI_BLOG bit only -- the SEARCH bit cleared), which the query-time owner-consent check
6// enforces IMMEDIATELY and REVERSIBLY (re-bank "1" to restore); no doc row is deleted (rule 13:
7// additive-only, history sacred). Default is a DRY RUN; pass `commit` to write.
8// usage: nx_web_purge_slop <domain> [commit] [maxrows]
9// SINGLE-WRITER CAUTION: run while the crawl beat is idle (searchcrawl defers during compact; this
10// organ takes seconds) -- segid is claimed at COMMIT time via ss_next_segid to shrink the race window.
11// license_tier: ORIGINAL
12import "nx_corpus_ingest.nx" // dss_*/ss_* + ci_hash (the shard machinery)
13const K_MAGIC_20000: i64 = 20000
14
15func ps_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
16func ps_num(v: i64) -> i64 {
17 let bb: *u8 = sys_mmap(28); var m: i64 = v
18 if m < 0 { sys_write(1, "-" as *u8, 1); m = 0 - m }
19 let t: *u8 = sys_mmap(28); var k: i64 = 0
20 if m == 0 { t[0] = 48 as u8; k = 1 }
21 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
22 var i: i64 = 0; while i < k { bb[i] = t[k - 1 - i]; i = i + 1 }
23 sys_write(1, bb, k); return 0
24}
25// case-sensitive substring in hay[0..hn) (the markers are literal page bytes)
26func ps_hassub(hay: *u8, hn: i64, lit: *u8) -> i64 {
27 var ll: i64 = 0
28 while lit[ll] != (0 as u8) { ll = ll + 1 }
29 if ll == 0 { return 0 }
30 var i: i64 = 0
31 while i + ll <= hn {
32 var m: i64 = 1
33 var k: i64 = 0
34 while k < ll { if hay[i + k] != lit[k] { m = 0; k = ll } else { k = k + 1 } }
35 if m == 1 { return 1 }
36 i = i + 1
37 }
38 return 0
39}
40// SCOPED slop classes -- deliberately NARROWER than olh_urlok (measured 2026-08-04: retroactive
41// full-policy application flagged 30,218 docs = 19% of the corpus, including REAL content the
42// frontier policy merely declines to FETCH -- et.wiktionary interwiki pages, query-string URLs.
43// Admission-for-fetch and keep-what-we-have are DIFFERENT judgments; rule 25 says better data,
44// never fewer options). Purged here: only the tag/category listing-farm classes whose SERP
45// pollution the ruler measured. NUL-terminated url string.
46func ps_is_slop_url(u: *u8, ul: i64) -> i64 {
47 if ps_hassub(u, ul, "/tag/" as *u8) == 1 { return 1 }
48 if ps_hassub(u, ul, "/tags/" as *u8) == 1 { return 1 }
49 if ps_hassub(u, ul, "/category/" as *u8) == 1 { return 1 }
50 return 0
51}
52// the verification/challenge markers the live crawler already refuses (nx_web_crawl_step): a doc
53// whose stored TEXT carries one in its head was ingested through the pre-guard era.
54func ps_is_interstitial(doc: *u8, dn: i64) -> i64 {
55 var scan: i64 = dn
56 if scan > 600 { scan = 600 }
57 if ps_hassub(doc, scan, "Please wait for verification" as *u8) == 1 { return 1 }
58 if ps_hassub(doc, scan, "Just a moment" as *u8) == 1 { return 1 }
59 if ps_hassub(doc, scan, "Attention Required! | Cloudflare" as *u8) == 1 { return 1 }
60 if ps_hassub(doc, scan, "Checking your browser" as *u8) == 1 { return 1 }
61 return 1 - 1
62}
63func ps_atoi(s: *u8) -> i64 { var v: i64 = 0; var i: i64 = 0; while s[i] != (0 as u8) { let c: i64 = s[i] as i64; if c >= 48 { if c <= 57 { v = v * 10 + (c - 48) } } i = i + 1 } return v }
64
65func main(argc: i64, argv: *i64) -> i64 {
66 if argc < 2 { ps_puts("usage: nx_web_purge_slop <domain> [commit] [maxrows]\n" as *u8); return 1 }
67 let domain: *u8 = argv[1] as *u8
68 var docommit: i64 = 0
69 if argc >= 3 {
70 let a2: *u8 = argv[2] as *u8
71 if a2[0] == (99 as u8) { docommit = 1 } // "commit"
72 }
73 var maxrows: i64 = K_MAGIC_20000
74 if argc >= 4 { maxrows = ps_atoi(argv[3] as *u8) }
75 let prefix: *u8 = sys_mmap(512)
76 dss_prefix(domain, prefix)
77 let h: *i64 = ss_open(prefix)
78 if (h as i64) == 0 { ps_puts("PURGE FAIL: shard absent\n" as *u8); return 2 }
79 let w: *i64 = ss_begin()
80 let segsbox: *i64 = sys_mmap(16) as *i64
81 segsbox[0] = 0
82 let segidbox: *i64 = sys_mmap(16) as *i64
83 segidbox[0] = ss_next_segid(prefix)
84 let pp: *i64 = sys_mmap(16) as *i64
85 let pl: *i64 = sys_mmap(16) as *i64
86 let dp: *i64 = sys_mmap(16) as *i64
87 let dl: *i64 = sys_mmap(16) as *i64
88 let kbuf: *u8 = sys_mmap(64)
89 let dkey: *u8 = sys_mmap(64)
90 let pkey: *u8 = sys_mmap(64)
91 var nurl: i64 = 0
92 var nslop: i64 = 0
93 var nintr: i64 = 0
94 var nalready: i64 = 0
95 var nbanked: i64 = 0
96 var shown: i64 = 0
97 let ns: i64 = h[0]
98 var s: i64 = 0
99 while s < ns {
100 let kb: *u8 = h[1 + 8 * s] as *u8
101 if h[2 + 8 * s] >= 8 {
102 let m9: i64 = ss_r32(kb, 4)
103 var e9: i64 = 0
104 while e9 < m9 {
105 let eo: i64 = 8 + 4 * m9 + ss_r32(kb, 8 + 4 * e9)
106 if (kb[eo] as i64) == 1 {
107 let kl9: i64 = ss_r32(kb, eo + 1)
108 if kl9 >= 5 { if kl9 < 60 {
109 // "url:" prefix
110 if kb[eo + 5] == (117 as u8) { if kb[eo + 6] == (114 as u8) { if kb[eo + 7] == (108 as u8) { if kb[eo + 8] == (58 as u8) {
111 var c: i64 = 0
112 while c < kl9 { kbuf[c] = kb[eo + 5 + c]; c = c + 1 }
113 kbuf[kl9] = 0 as u8
114 if ss_hget(h, kbuf, pp, pl) == 1 { if pl[0] > 0 {
115 nurl = nurl + 1
116 // cid = digits after "url:"
117 var cid: i64 = 0
118 var ki: i64 = 4
119 while ki < kl9 { let kc: i64 = kbuf[ki] as i64; if kc >= 48 { if kc <= 57 { cid = cid * 10 + (kc - 48) } } ki = ki + 1 }
120 var bad: i64 = 0
121 if ps_is_slop_url(pp[0] as *u8, pl[0]) == 1 { bad = 1 }
122 if bad == 0 {
123 dss_mkkey(cid, dkey)
124 if ss_hget(h, dkey, dp, dl) == 1 { if dl[0] > 0 {
125 if ps_is_interstitial(dp[0] as *u8, dl[0]) == 1 { bad = 2 }
126 } }
127 }
128 if bad > 0 {
129 // already excluded? (pol row present with the search bit clear)
130 dss_mkpolkey(cid, pkey)
131 var skip: i64 = 0
132 if ss_hget(h, pkey, pp, pl) == 1 { if pl[0] > 0 {
133 var fv: i64 = 0
134 let fp: *u8 = pp[0] as *u8
135 var fi: i64 = 0
136 while fi < pl[0] { let ch: i64 = fp[fi] as i64; if ch >= 48 { if ch <= 57 { fv = fv * 10 + (ch - 48) } } fi = fi + 1 }
137 if (fv & 1) == 0 { skip = 1 }
138 } }
139 if skip == 1 { nalready = nalready + 1 } else {
140 if bad == 1 { nslop = nslop + 1 } else { nintr = nintr + 1 }
141 if nbanked < maxrows {
142 nbanked = nbanked + 1
143 if docommit == 1 {
144 if ss_add(w, 1, pkey, "2" as *u8, 1) < 0 {
145 if ss_commit(prefix, w, segidbox[0]) == 0 { segsbox[0] = segsbox[0] + 1 }
146 segidbox[0] = segidbox[0] + 1
147 w[1] = 0
148 ss_add(w, 1, pkey, "2" as *u8, 1)
149 }
150 }
151 if shown < 25 {
152 shown = shown + 1
153 if bad == 1 { ps_puts(" [slop-url] " as *u8) } else { ps_puts(" [interstitial] " as *u8) }
154 // reprint the url (refetch the row: pp/pl were reused by the pol probe)
155 if ss_hget(h, kbuf, pp, pl) == 1 { sys_write(1, pp[0] as *u8, pl[0]) }
156 ps_puts("\n" as *u8)
157 }
158 }
159 }
160 }
161 } }
162 } } } }
163 } }
164 }
165 e9 = e9 + 1
166 }
167 }
168 s = s + 1
169 }
170 if docommit == 1 { if w[1] > 0 {
171 segidbox[0] = ss_next_segid(prefix)
172 if ss_commit(prefix, w, segidbox[0]) == 0 { segsbox[0] = segsbox[0] + 1 }
173 } }
174 ps_puts("PURGE " as *u8)
175 if docommit == 1 { ps_puts("COMMIT" as *u8) } else { ps_puts("DRY-RUN" as *u8) }
176 ps_puts(": urls=" as *u8); ps_num(nurl)
177 ps_puts(" slop_url=" as *u8); ps_num(nslop)
178 ps_puts(" interstitial=" as *u8); ps_num(nintr)
179 ps_puts(" already_excluded=" as *u8); ps_num(nalready)
180 ps_puts(" banked=" as *u8); ps_num(nbanked)
181 ps_puts(" segs=" as *u8); ps_num(segsbox[0])
182 ps_puts("\n" as *u8)
183 return 0
184}