nx_crawl_pace.nx source

1// nx_crawl_pace.nx -- SOTA polite crawler pacing (operator 2026-07-05: "pacing is a great callout on all the 2// systems... get to state of the art"). Per-host adaptive rate limiting: a min interval between hits to the 3// same host, HONOR Retry-After on 429/503, EXPONENTIAL backoff on repeated throttles (R9 2026-08-24: a success 4// HALVES the backoff instead of erasing it -- pace_decay -- so one lucky 200 cannot re-trip a host's limit), and 5// robots crawl-delay override. State persists across invocations (each nx_url_index run is a fresh process) in 6// a fixed hash-table file, so serial crawls stay polite and don't get IP-banned (a ban = LOST reach, not just 7// rudeness -- reddit 429'd us on rapid feed hits). A shared organ any fetcher can call: pace_before(host) 8// before a request, pace_after(host,status,retry_after) after. license_tier: ORIGINAL 9import "nx_syscalls.nx" 10const PACE_MAGIC_5381: i64 = 5381 11const PACE_MAGIC_1024: i64 = 1024 12const PACE_MAGIC_1000000: i64 = 1000000 13 14// --- policy config (v1 surface; a later rung reads svc-config per Cardinal 11) --- 15const PACE_BASE_MS: i64 = 1000 // polite default: >= 1s between requests to the same host 16const PACE_MAX_BACKOFF_MS: i64 = 300000 // cap any single backoff at 5 min 17const PACE_WAIT_CAP_MS: i64 = 60000 // never block ONE fetch > 60s (caller may defer beyond) 18const PACE_SLOTS: i64 = 4096 19const PACE_REC: i64 = 40 // host_hash(8) | next_allowed_ms(8) | consec_throttle(8) | crawl_delay_ms(8) | flags(8) 20const PACE_TBL_BYTES: i64 = 163840 // PACE_SLOTS * PACE_REC 21const PACE_REC_V1: i64 = 32 // the pre-R12 record; on-disk tables in that layout are MIGRATED, never discarded 22const PACE_TBL_BYTES_V1: i64 = 131072 23const PACE_F_OKSEEN: i64 = 1 // flags bit 0: this host has returned a 2xx at least once 24// Byte offsets of each field WITHIN one PACE_REC, derived from the layout PACE_REC already declares 25// above (host_hash | next_allowed_ms | consec_throttle | crawl_delay_ms | flags), every field an i64. 26// The pre-existing accessors below still spell these as raw 8/16/24/32; these names exist so new code 27// stops adding to that pile, and so a reader of the R14 rule can see WHICH field it reads. 28const PACE_OFF_HASH: i64 = 0 29const PACE_OFF_NEXT_ALLOWED: i64 = 8 30const PACE_OFF_CONSEC: i64 = 16 31const PACE_OFF_CRAWL_DELAY: i64 = 24 32const PACE_OFF_FLAGS: i64 = 32 33const PACE_TBL_MODE: i64 = 0x1A4 // 0o644 file MODE for sys_openat_wr (flags O_CREAT|O_WRONLY|O_TRUNC 34 // are hardcoded inside it). Passing flags here made the file 0o1101 35 // (no owner-write) -> 2nd open EACCES; a real mode fixes re-writes. 36 37// --- little-endian i64 load/store into the table buffer --- 38func pace_ld(tbl: *u8, off: i64) -> i64 { var v: i64=0; var i: i64=0; while i<8 { v = v | ((tbl[off+i] as i64) << (i*8)); i=i+1 } return v } 39func pace_st(tbl: *u8, off: i64, val: i64) -> i64 { var i: i64=0; while i<8 { tbl[off+i] = ((val >> (i*8)) & 0xff) as u8; i=i+1 } return 0 } 40 41// stable positive host hash (djb2, masked) 42func pace_hash(h: *u8, n: i64) -> i64 { 43 var x: i64 = PACE_MAGIC_5381 44 var i: i64 = 0 45 while i < n { x = (((x << 5) + x) + (h[i] as i64)) & 0x7fffffffffffffff; i = i + 1 } 46 if x == 0 { x = 1 } // 0 reserved for "empty slot" 47 return x 48} 49 50// --- persistence: read-modify-write a fixed table file (serial-crawler safe) --- 51// R13b (2026-08-25): fill a CALLER-OWNED buffer. pace_load_tbl() below is exactly this plus the mmap, so 52// there is ONE loader and the two cannot drift. A caller that reloads on a loop (the crawler reloads once 53// per BATCH) then owns exactly one buffer for the whole run instead of one per reload. 54// ZEROING IS LOAD-BEARING AND IS THE WHOLE REASON THIS IS NOT A FREE SPLIT: sys_mmap hands back zero-filled 55// pages, so the original could rely on dst starting clean, but a REUSED buffer does not. Two paths below 56// leave bytes unwritten -- the no-file-at-all path writes nothing, and the v1 migration writes only 57// PACE_REC_V1 of each PACE_REC record -- so without this a reload would inherit the PREVIOUS load's bytes 58// in exactly the fields (flags, and every slot the new table dropped) that decide a deferral. 59// The v2 path overwrites all PACE_TBL_BYTES and so does not need it; it is unconditional anyway because a 60// loader whose postcondition is "dst is fully defined" cannot be misused, and the cost is one memset per 61// BATCH, not per row. 62func pace_load_tbl_into(dst: *u8) -> i64 { 63 var z: i64 = 0 64 while z < PACE_TBL_BYTES { dst[z] = 0 as u8; z = z + 1 } 65 // NEVER hand back the file's own mapping (which is what the caller would get from sys_read_file): 66 // pace_save_tbl's O_TRUNC would then wipe the buffer mid-write. Copy in, decouple. 67 let lp: *i64 = sys_mmap(16) as *i64 68 // v2 first. Its presence means the migration already happened; v1 is then only history. 69 let src2: *u8 = sys_read_file("knowledge/status/pace_crawl2.tbl" as *u8, lp) 70 if (src2 as i64) != 0 { if lp[0] >= PACE_TBL_BYTES { 71 var j: i64 = 0 72 while j < PACE_TBL_BYTES { dst[j] = src2[j]; j = j + 1 } 73 return 0 74 } } 75 // No v2 yet -> seed from the v1 (32-byte) table exactly once. v1 is left ON DISK untouched, so a 76 // still-running old binary keeps working against it and cannot reach anything we own (rule 13: 77 // additive only -- the migration reads history, it never deletes it). 78 let src: *u8 = sys_read_file("knowledge/status/pace_crawl.tbl" as *u8, lp) 79 if (src as i64) != 0 { if lp[0] >= PACE_TBL_BYTES { 80 var i: i64 = 0 81 while i < PACE_TBL_BYTES { dst[i] = src[i]; i = i + 1 } 82 } else { if lp[0] >= PACE_TBL_BYTES_V1 { 83 // R12 MIGRATION: the on-disk table is the 32-byte layout. Re-lay it into 40-byte records 84 // rather than ignoring it. Discarding it would silently drop every in-flight backoff and 85 // every adopted Crawl-delay -- i.e. a politeness change that begins by being impolite. 86 // flags starts 0 (ok_seen unknown), which is the SAFE direction: an unproven host's 403 is 87 // read as a refusal, never as a throttle. 88 var s: i64 = 0 89 while s < PACE_SLOTS { 90 var f: i64 = 0 91 while f < PACE_REC_V1 { dst[s * PACE_REC + f] = src[s * PACE_REC_V1 + f]; f = f + 1 } 92 s = s + 1 93 } 94 } } } 95 return 0 96} 97// The original 0-arg loader, byte-for-byte in meaning: allocate, then delegate. Every existing caller and 98// every gate tooth is unchanged, and there is still exactly ONE piece of load logic in the estate. 99func pace_load_tbl() -> *u8 { 100 let dst: *u8 = sys_mmap(PACE_TBL_BYTES) // anon, zero-filled 101 pace_load_tbl_into(dst) 102 return dst 103} 104// R16 (2026-09-01, /compare/mediaingest, debt 1788270007): stderr emitters so a failed PERSIST can 105// announce itself. Lengths are DERIVED by scanning to NUL, never hand-counted beside the literal -- a 106// hand-counted length is a second copy of the string's shape and the two drift silently. 107// stderr and not stdout on purpose: callers on this path emit machine-read output on fd 1, and a 108// diagnostic written there would corrupt a parser that is not the subject of this change. 109func pace_ew(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(2, s, n); return 0 } 110func pace_en(v: i64) -> i64 { 111 let t: *u8 = sys_mmap(32) 112 var m: i64 = v 113 if m < 0 { pace_ew("-" as *u8); m = 0 - m } 114 var k: i64 = 0 115 if m == 0 { t[0] = 48 as u8; k = 1 } 116 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 } 117 let b: *u8 = sys_mmap(32) 118 var j: i64 = 0 119 while j < k { b[j] = t[k - 1 - j]; j = j + 1 } 120 sys_write(2, b, k) 121 return 0 122} 123 124// PERSIST THE TABLE. THE RETURN CONTRACT IS UNCHANGED (0 ok, -1 fail) so no caller's behaviour moves. 125// 126// WHY THE ANNOUNCE EXISTS. Both callers of this function DISCARD its return value: pace_after_tbl ends 127// `pace_save_tbl(tbl); return delay` and pace_set_crawl_delay ends `pace_save_tbl(tbl); return 0` -- 128// success, unconditionally. So before this change a failed save was INDISTINGUISHABLE FROM A SUCCESSFUL 129// ONE at every call site, and nx_crawl_pace could not tell RECORDED from FAILED-TO-RECORD. That is the 130// same fail-open shape R15 closed on the SLOT path, arriving through the SAVE path: a host whose backoff 131// never persisted reads exactly like a host that was never throttled, so the crawler re-hits it at base 132// delay -- the operator-facing symptom this whole domain was admitted for. 133// 134// MEASURED 2026-09-01 by nx_pacetbl_probe (which scans all PACE_SLOTS records directly instead of going 135// through pace_slot, so it separates write-did-not-land from read-cannot-find-it): nx_crawl_pace_gate's 136// two fixture hosts are ABSENT FROM EVERY SLOT after their writes, on a table that is FULL with empty=0. 137// The write genuinely does not land. WHICH failure it is -- open refused, or a short write -- was 138// unobservable from outside, and five candidate mechanisms were refuted by control before that could 139// even be established. 140// 141// This does not FIX the failure; it makes the failure NAME ITSELF, so the next run tells the reader 142// which syscall refused and with what, instead of costing them the same investigation again. A fix that 143// guessed between the remaining candidates would be a guess shipped as a repair. 144func pace_save_tbl(tbl: *u8) -> i64 { 145 let fd: i64 = sys_openat_wr("knowledge/status/pace_crawl2.tbl" as *u8, PACE_TBL_MODE) 146 if fd < 0 { 147 pace_ew("PACE-PERSIST-FAILED stage=open rc=" as *u8); pace_en(fd) 148 pace_ew(" path=knowledge/status/pace_crawl2.tbl bytes_owed=" as *u8); pace_en(PACE_TBL_BYTES) 149 pace_ew(" -- the caller has ALREADY been told this succeeded; its backoff is NOT recorded\n" as *u8) 150 return 0 - 1 151 } 152 var w: i64 = 0 // loop: a single write() need not flush 128KB in one go 153 while w < PACE_TBL_BYTES { 154 let k: i64 = sys_write(fd, (tbl as i64 + w) as *u8, PACE_TBL_BYTES - w) 155 if k <= 0 { 156 pace_ew("PACE-PERSIST-FAILED stage=write rc=" as *u8); pace_en(k) 157 pace_ew(" wrote=" as *u8); pace_en(w) 158 pace_ew(" of=" as *u8); pace_en(PACE_TBL_BYTES) 159 // A short write here leaves the table TRUNCATED on disk, which is strictly worse than not 160 // having written at all: the next load reads fewer than PACE_TBL_BYTES and silently starts 161 // from a zeroed table, dropping every in-flight backoff in the estate. Say so. 162 pace_ew(" -- TABLE IS NOW TRUNCATED ON DISK; the next load will read a partial file\n" as *u8) 163 sys_close(fd) 164 return 0 - 1 165 } 166 w = w + k 167 } 168 sys_close(fd) 169 return 0 170} 171// find the slot for hh (existing match or first empty), linear probe. -1 if full. 172func pace_slot(tbl: *u8, hh: i64) -> i64 { 173 var s: i64 = hh % PACE_SLOTS 174 var tries: i64 = 0 175 while tries < PACE_SLOTS { 176 let stored: i64 = pace_ld(tbl, s * PACE_REC) 177 if stored == 0 { return s } 178 if stored == hh { return s } 179 s = (s + 1) % PACE_SLOTS 180 tries = tries + 1 181 } 182 return 0 - 1 183} 184 185// R15 (2026-08-30, /compare/mediaingest R0): the WRITE-side slot finder. A READ that misses a full table is 186// correctly "no state" -- an absent host is not throttled -- so pace_slot returns -1 for reads. But a WRITE that 187// returns -1 SILENTLY DROPS the record: the pacer fails OPEN exactly when it is busiest. MEASURED 2026-08-30: the 188// nishihost pace table had reached 4096/4096 REAL hosts (4066 distinct hashes, 3815 plausible next_allowed epochs), 189// so pace_slot returned -1 and every NEW host got no pacing at all. An open-addressed table with no eviction cannot 190// grow, so a full table MUST evict. The victim is the slot whose next_allowed_ms is furthest in the PAST: that 191// host's backoff has most-expired, so it is the one least likely to still need pacing, and re-inserting it later 192// costs at most one un-paced request. This never returns -1 -- a write always lands -- and on a NON-full table it 193// is byte-identical to pace_slot (returns the found/empty slot), so the crawler's healthy table is unaffected. 194func pace_slot_or_evict(tbl: *u8, hh: i64) -> i64 { 195 let s: i64 = pace_slot(tbl, hh) 196 if s >= 0 { return s } 197 var victim: i64 = 0 198 var best: i64 = pace_ld(tbl, PACE_OFF_NEXT_ALLOWED) 199 var i: i64 = 1 200 while i < PACE_SLOTS { 201 let na: i64 = pace_ld(tbl, i * PACE_REC + PACE_OFF_NEXT_ALLOWED) 202 if na < best { best = na; victim = i } 203 i = i + 1 204 } 205 // clear the victim's whole record so the incoming host starts clean (no inherited backoff, crawl_delay or flags) 206 var f: i64 = 0 207 while f < PACE_REC { tbl[victim * PACE_REC + f] = 0 as u8; f = f + 1 } 208 return victim 209} 210 211// --- PURE POLICY (deterministic, gate-tested): post-fetch delay before the host may be hit again --- 212// status 429/503 -> honor Retry-After (s), else exponential base*2^consec (capped); success -> reset to base. 213func pace_delay_after(base_ms: i64, crawl_delay_ms: i64, consec_in: i64, 214 status: i64, retry_after_s: i64, out_consec: *i64) -> i64 { 215 return pace_delay_after2(base_ms, crawl_delay_ms, consec_in, status, retry_after_s, 0, out_consec) 216} 217// R12: the same policy, plus ok_seen (1 = this host has already served us a 2xx). The 6-arg form 218// above is preserved EXACTLY (ok_seen=0) so every existing caller and gate tooth is byte-identical. 219func pace_delay_after2(base_ms: i64, crawl_delay_ms: i64, consec_in: i64, 220 status: i64, retry_after_s: i64, ok_seen: i64, out_consec: *i64) -> i64 { 221 var eff: i64 = base_ms 222 if crawl_delay_ms > eff { eff = crawl_delay_ms } 223 var throttle: i64 = 0 224 if status == 429 { throttle = 1 } 225 if status == 503 { throttle = 1 } 226 // A 403 from a host that has ALREADY served us is a rate limit wearing a 403 (measured on 227 // loc.gov: 2 pages served, then 403 on every one after, six observations agreeing). A 403 from a 228 // host that never served us is a real refusal, and backing off cannot fix a refusal. 229 if status == 403 { if ok_seen == 1 { throttle = 1 } } 230 if throttle == 1 { 231 out_consec[0] = consec_in + 1 232 var delay: i64 = 0 233 if retry_after_s > 0 { delay = retry_after_s * 1000 } 234 else { 235 var mult: i64 = 1; var k: i64 = 0 236 while k < consec_in { mult = mult * 2; if mult >= PACE_MAGIC_1024 { mult = PACE_MAGIC_1024; k = consec_in } else { k = k + 1 } } 237 delay = eff * mult 238 } 239 if delay < eff { delay = eff } 240 if delay > PACE_MAX_BACKOFF_MS { delay = PACE_MAX_BACKOFF_MS } 241 return delay 242 } 243 // R9 (2026-08-24, /compare/webscraping contract pace_decay): a success DECAYS the backoff instead of erasing it. 244 // One 200 after a 429 used to reset to base, so the very next burst re-tripped the limit; now each success takes 245 // one halving off the backoff (consec - 1) and a host earns its way back to base over as many successes as it 246 // took throttles. A host that was never throttled is byte-identical to before (consec 0 -> base). 247 let nc: i64 = pace_decay(consec_in) 248 out_consec[0] = nc 249 if nc == 0 { return eff } 250 var mult2: i64 = 1 251 var k2: i64 = 0 252 while k2 < nc { mult2 = mult2 * 2; if mult2 >= PACE_MAGIC_1024 { mult2 = PACE_MAGIC_1024; k2 = nc } else { k2 = k2 + 1 } } 253 var d2: i64 = eff * mult2 254 if d2 > PACE_MAX_BACKOFF_MS { d2 = PACE_MAX_BACKOFF_MS } 255 return d2 256} 257// R9: the decay step, pure and gate-tested -- one success takes one halving off the backoff, never below zero 258func pace_decay(consec_in: i64) -> i64 { if consec_in <= 0 { return 0 } return consec_in - 1 } 259// forget a host's throttle history and next-allowed stamp (gate self-isolation; an operator reset after a fixed 260// client bug). The robots crawl-delay is KEPT: it is the host's number, not ours to forget. 261func pace_reset_host(host: *u8, host_len: i64) -> i64 { 262 let tbl: *u8 = pace_load_tbl() 263 let hh: i64 = pace_hash(host, host_len) 264 let s: i64 = pace_slot(tbl, hh) 265 if s < 0 { return 0 - 1 } 266 let off: i64 = s * PACE_REC 267 if pace_ld(tbl, off) != hh { return 0 } 268 pace_st(tbl, off + 8, 0) 269 pace_st(tbl, off + 16, 0) 270 pace_st(tbl, off + 32, 0) 271 pace_save_tbl(tbl) 272 return 1 273} 274// wait (ms) required before hitting the host now, given its next-allowed time. capped. 275func pace_wait_ms(now_ms: i64, next_allowed_ms: i64) -> i64 { 276 var w: i64 = next_allowed_ms - now_ms 277 if w <= 0 { return 0 } 278 if w > PACE_WAIT_CAP_MS { return PACE_WAIT_CAP_MS } 279 return w 280} 281 282// --- top-level: call BEFORE a request to a host (loads state, sleeps the polite interval) --- 283func pace_before(host: *u8, host_len: i64) -> i64 { 284 return pace_before_tbl(pace_load_tbl(), host, host_len) 285} 286// R15 (2026-08-30, /compare/mediaingest R0): the same wait against an ALREADY-LOADED table. The 0-arg form above 287// mmaps 160 KB per call and never unmaps it (R13b measured that for the crawler); the capture path calls the pacer 288// once per FETCH, and a 1000-segment HLS download through the 0-arg form would leak ~320 MB of touched anon pages. 289// nx_paced_fetch owns ONE process-lifetime buffer and reloads it in place before each decision instead. The two 290// forms cannot disagree: the 0-arg form loads and delegates, so there is still exactly one wait computation. 291func pace_before_tbl(tbl: *u8, host: *u8, host_len: i64) -> i64 { 292 if (tbl as i64) == 0 { return 0 } 293 let hh: i64 = pace_hash(host, host_len) 294 let s: i64 = pace_slot(tbl, hh) 295 if s < 0 { return 0 } 296 let off: i64 = s * PACE_REC 297 if pace_ld(tbl, off) != hh { return 0 } // no prior state -> no wait 298 let w: i64 = pace_wait_ms(sys_now_realtime_ms(), pace_ld(tbl, off + 8)) 299 if w > 0 { sys_sleep_ms(w) } 300 return w 301} 302// --- THE DEFERRAL PREDICATE THIS FILE'S OWN CONTRACT HAS ALWAYS PROMISED AND NO CALLER EVER HAD --- 303// PACE_WAIT_CAP_MS above reads "never block ONE fetch > 60s (caller may defer beyond)" -- but there has 304// never been a verb a caller could ASK, so the only production caller sleeps instead, and it sleeps IN 305// THE PARENT INSIDE ITS FORK LOOP. MEASURED 2026-08-25: books.google.com fetched at ~62s intervals, 41 306// consecutive times, while an 8-way pool sat idle behind it; crawl throughput fell 280.7 -> 72.3 docs/hr. 307// 308// IT ALSO BREAKS A SELF-SUSTAINING LOCK. pace_delay_after2 can set a 300s backoff, but pace_wait_ms CAPS 309// THE WAIT AT 60s -- so the crawler re-hits the host after 60s, earns another 429, and consec can never 310// decay (pace_decay requires a success). The backoff is computed and then never actually served. 311// DEFERRING IS WHAT SERVES IT. 312// 313// Returns 1 = DEFER this host now (leave the url PENDING -- it costs nothing and returns next run) 314// 0 = proceed (ready now, or only an ordinary politeness gap worth sleeping through) 315// 316// THE DISCRIMINATOR IS consec_throttle, NOT A DURATION, and that choice is load-bearing: an ordinary 317// politeness gap and a throttle backoff can be the SAME number of milliseconds while meaning completely 318// different things. A caller that deferred on duration would also defer normal politeness and fetch one 319// url per host per run -- slower than the defect it was written to fix. consec > 0 is the only signal 320// that means "this host has actively refused us", which is the one case where waiting is wasted. 321// R13b (2026-08-25): THE SAME PREDICATE AGAINST AN ALREADY-LOADED TABLE -- this is the form the crawler 322// calls, and the split is not cosmetic. pace_load_tbl() mmaps 160 KB, reads the 160 KB table file and 323// copies it byte-by-byte, and the buffer is NEVER unmapped. The crawler asks this question once per 324// CANDIDATE ROW at batch selection, and its pull window is WC_MAXPEND = 2048 rows, so the 0-arg form 325// would cost up to 2048 x (160 KB anon mmap + 160 KB read + 163,840-iteration copy) = ~335 MB of leaked 326// anon VMA and ~335 MB of read I/O in the crawler PARENT, every run, every 20 minutes. 327// NEVER ALLOCATE IN A HOT LOOP -- PASS THE BUFFER IN. The caller reloads once per BATCH, which is also 328// exactly the freshness granularity that matters: pace_after rewrites the table during the reap of the 329// previous batch, so a per-batch reload sees every update a per-row reload would have seen. 330func pace_should_defer_tbl(tbl: *u8, host: *u8, host_len: i64) -> i64 { 331 if (tbl as i64) == 0 { return 0 } 332 let hh: i64 = pace_hash(host, host_len) 333 let s: i64 = pace_slot(tbl, hh) 334 if s < 0 { return 0 } 335 let off: i64 = s * PACE_REC 336 if pace_ld(tbl, off) != hh { return 0 } // no prior state 337 // ---- R14 (2026-08-25): DEFER A HOST THAT PUBLISHED A CRAWL-DELAY LONGER THAN ONE FETCH CAN WAIT. 338 // WHY THIS IS A CORRECTNESS FIX AND NOT A TUNING KNOB: pace_before honours an interval by SLEEPING, 339 // and pace_wait_ms caps that sleep at PACE_WAIT_CAP_MS. So for any published Crawl-delay ABOVE that 340 // cap the sleep is truncated and we hit the host EARLY -- a host asking 120 s was fetched at 60 s. 341 // The crawler's old answer was worse still: nx_web_crawl_step clamped the published number DOWN to 342 // 30 s before it ever reached this table, so the host was fetched at 30 s. Both are the same defect, 343 // which is that a delay we cannot SLEEP through was converted into a delay we ignored. 344 // The threshold is DERIVED, not chosen: it is exactly the largest interval pace_before is capable of 345 // honouring in line. At or below it, sleeping is correct and nothing changes. Above it, sleeping 346 // CANNOT be correct, and the only compliant move left is to not fetch now. 347 // WHY IT DOES NOT RE-INTRODUCE THE DURATION-DEFERRAL TRAP the 2-arg contract above warns about: the 348 // discriminator is still not a raw wait. It is "this host PUBLISHED a delay we are structurally 349 // unable to sleep through", which is false for every host that never published one -- ordinary 350 // politeness (PACE_BASE_MS = 1 s) can never reach it, so a normal host is byte-identical to before. 351 // COST OF DEFERRING: none. The frontier row is persisted and simply returns on a later step, by 352 // which time the interval HAS elapsed and this test yields 0. The host is crawled at its own pace 353 // rather than not at all. 354 if pace_ld(tbl, off + PACE_OFF_CRAWL_DELAY) > PACE_WAIT_CAP_MS { 355 if pace_wait_ms(sys_now_realtime_ms(), pace_ld(tbl, off + PACE_OFF_NEXT_ALLOWED)) > 0 { return 1 } 356 } 357 if pace_ld(tbl, off + 16) <= 0 { return 0 } // not throttled 358 if pace_wait_ms(sys_now_realtime_ms(), pace_ld(tbl, off + 8)) <= 0 { return 0 } // backoff served 359 return 1 360} 361// The 2-arg form the contract above describes, preserved EXACTLY for any caller that asks the question 362// once and does not have a table in hand. It loads and delegates, so the two forms cannot disagree about 363// what "defer" means -- there is one predicate, and the gate pins them equal on the same host. 364func pace_should_defer(host: *u8, host_len: i64) -> i64 { 365 return pace_should_defer_tbl(pace_load_tbl(), host, host_len) 366} 367 368// call AFTER the response: update next-allowed + backoff. returns the delay applied (ms). 369func pace_after(host: *u8, host_len: i64, status: i64, retry_after_s: i64) -> i64 { 370 return pace_after_tbl(pace_load_tbl(), host, host_len, status, retry_after_s) 371} 372// R15 (2026-08-30): the same update against an ALREADY-LOADED table -- see pace_before_tbl. It still SAVES the 373// table (persistence is the contract), so a caller's buffer is only a read cache: the disk is the truth. 374func pace_after_tbl(tbl: *u8, host: *u8, host_len: i64, status: i64, retry_after_s: i64) -> i64 { 375 if (tbl as i64) == 0 { return 0 - 1 } 376 let hh: i64 = pace_hash(host, host_len) 377 let s: i64 = pace_slot_or_evict(tbl, hh) // R15: a WRITE always lands; a full table evicts the oldest host rather than failing open 378 if s < 0 { return 0 - 1 } 379 let off: i64 = s * PACE_REC 380 var consec_in: i64 = 0 381 var crawl_delay: i64 = 0 382 var flags: i64 = 0 383 if pace_ld(tbl, off) == hh { 384 consec_in = pace_ld(tbl, off + 16) 385 crawl_delay = pace_ld(tbl, off + 24) 386 flags = pace_ld(tbl, off + 32) 387 } 388 var ok_seen: i64 = 0 389 if (flags & PACE_F_OKSEEN) != 0 { ok_seen = 1 } 390 let oc: *i64 = sys_mmap(16) as *i64 391 let delay: i64 = pace_delay_after2(PACE_BASE_MS, crawl_delay, consec_in, status, retry_after_s, ok_seen, oc) 392 // Record the 2xx AFTER the decision, never before: setting the bit first would make the very 393 // first 2xx-then-403 pair look like it already had history, and the discrimination would be a 394 // tautology that fires on every 403. 395 if status >= 200 { if status < 300 { flags = flags | PACE_F_OKSEEN } } 396 pace_st(tbl, off, hh) 397 pace_st(tbl, off + 8, sys_now_realtime_ms() + delay) 398 pace_st(tbl, off + 16, oc[0]) 399 pace_st(tbl, off + 24, crawl_delay) 400 pace_st(tbl, off + 32, flags) 401 pace_save_tbl(tbl) 402 return delay 403} 404// robots.txt crawl-delay for a host (ms): recorded so pace_delay_after uses max(base, crawl_delay). 405func pace_set_crawl_delay(host: *u8, host_len: i64, cd_ms: i64) -> i64 { 406 let tbl: *u8 = pace_load_tbl() 407 let hh: i64 = pace_hash(host, host_len) 408 let s: i64 = pace_slot_or_evict(tbl, hh) // R15: recording a host's crawl-delay is a WRITE -- it must land even on a full table 409 if s < 0 { return 0 - 1 } 410 let off: i64 = s * PACE_REC 411 pace_st(tbl, off, hh) 412 if pace_ld(tbl, off + 8) == 0 { pace_st(tbl, off + 8, 0) } 413 pace_st(tbl, off + 24, cd_ms) 414 pace_save_tbl(tbl) 415 return 0 416} 417// parse "Retry-After: <seconds>" from response headers (integer-seconds form). 0 if absent/date-form. 418func pace_retry_after(resp: *u8, n: i64) -> i64 { 419 let key: *u8 = "retry-after:" as *u8 420 var i: i64 = 0 421 while i + 12 < n { 422 var m: i64 = 1; var j: i64 = 0 423 while j < 12 { let c: i64 = resp[i+j] as i64; var lc: i64 = c; if c >= 65 { if c <= 90 { lc = c + 32 } } if lc != (key[j] as i64) { m = 0; j = 12 } else { j = j + 1 } } 424 if m == 1 { 425 var p: i64 = i + 12 426 while p < n { let c: i64 = resp[p] as i64; if c == 32 { p = p + 1 } else { if c == 9 { p = p + 1 } else { p = n + PACE_MAGIC_1000000 } } } 427 var q: i64 = i + 12 428 var st: i64 = 0 429 while st == 0 { if q >= n { st = 1 } else { let c: i64 = resp[q] as i64; if c == 32 { q = q + 1 } else { if c == 9 { q = q + 1 } else { st = 1 } } } } 430 var v: i64 = 0; var got: i64 = 0 431 while q < n { let d: i64 = resp[q] as i64; if d >= 48 { if d <= 57 { v = v*10 + (d - 48); got = 1; q = q + 1 } else { q = n } } else { q = n } } 432 if got == 1 { return v } 433 return 0 434 } 435 i = i + 1 436 } 437 return 0 438} 439 440func main() -> i64 { return 0 }