code wiki / _hdl_build / nx_crawl_pace_gate.nx
`nx_crawl_pace_gate.nx` source

↩ module page · 316 lines · 23182 B
1// nx_crawl_pace_gate.nx -- deterministic gate for the pacing POLICY (no network): honor-Retry-After,
2// exponential backoff, reset-on-success, crawl-delay override, wait clamp, Retry-After parse, persistence.
3import "nx_syscalls.nx"
4import "nx_crawl_pace.nx"
5import "nx_gate_verdict.nx"
6
7// R13 fixture constants (rule 11). Each is DERIVED from a policy constant rather than picked, so the
8// fixtures cannot silently drift out of the regime they were chosen to sit in if the policy is retuned.
9const R13_BACKOFF_S: i64 = PACE_MAX_BACKOFF_MS / PACE_BASE_MS      // 300 s = the LONGEST backoff the
  // policy can set. The throttled fixture uses the maximum on purpose: a short backoff could elapse
  // while the gate is running on a loaded box, which would turn a POLICY tooth into a TIMING tooth.
12const R13_POLITE_CD_MS: i64 = PACE_WAIT_CAP_MS                     // the polite fixture's published
  // Crawl-delay. CHANGED 2026-08-25 FROM PACE_WAIT_CAP_MS * 2, AND THE NEG-CONTROL IT SERVES IS NOT
  // WEAKENED BY IT -- read this before concluding otherwise.
  // The *2 was never the subject of the tooth. It was a CONVENIENCE: doubling the cap was simply an
  // easy way to manufacture an outstanding wait long enough to clamp to PACE_WAIT_CAP_MS, so that the
  // polite and throttled fixtures would be indistinguishable by DURATION. That property is the tooth.
  // R14 then made the discarded detail load-bearing: a host that PUBLISHED a delay longer than one
  // fetch can sleep through must now defer, because sleeping is structurally incapable of honouring
  // it. So the old fixture had quietly become a host that SHOULD defer, being asserted not to.
  // The fix keeps the tooth's property and removes the confound: the published delay now sits exactly
  // AT the cap (a wholly realistic 60 s Crawl-delay, and R14's boundary is strictly greater-than, so
  // it does not fire), while the long outstanding wait is manufactured explicitly below via
  // R13_LONG_WAIT_MS. Duration-identical, consec-different, and no longer entangled with the delay.
25const R13_LONG_WAIT_MS: i64 = PACE_WAIT_CAP_MS * 2                 // pushed into next_allowed BY HAND so
  // the polite fixture's wait still clamps to exactly PACE_WAIT_CAP_MS. Written to the record rather
  // than derived from a published delay precisely so the two facts stay independent.
28const R13_DIRTY_BYTE: i64 = 0xff                                   // all-ones fill for the reused-buffer
  // test: the pace table's empty state is all-zero, so 0xff is maximally distinguishable from both a
  // correctly zeroed buffer and any real record.
31
32func gw(s: *u8) -> i64 { var k: i64=0; while s[k]!=(0 as u8){k=k+1} sys_write(1,s,k); return 0 }
33func gn(v: i64) -> i64 { if v==0 { sys_write(1,"0" as *u8,1); return 0 } var m: i64=v; if m<0 { sys_write(1,"-" as *u8,1); m=0-m } let t: *u8=sys_mmap(24); var k: i64=0; while m>0 { t[k]=(48+(m%10)) as u8; m=m/10; k=k+1 } let o: *u8=sys_mmap(24); var q: i64=k-1; var x: i64=0; while q>=0 { o[x]=t[q]; x=x+1; q=q-1 } sys_write(1,o,x); return 0 }
34func chk(name: *u8, got: i64, want: i64, p: *i64, t: *i64) -> i64 {
  t[0]=t[0]+1
  if got == want { p[0]=p[0]+1; gw("  ok   " as *u8) } else { gw("  FAIL " as *u8) }
  gw(name); gw("  got=" as *u8); gn(got); gw(" want=" as *u8); gn(want); gw("\n" as *u8)
  return 0
39}
40
41func main() -> i64 {
  gw("=== NX-CRAWL-PACE policy gate (SOTA semantics) ===\n" as *u8)
  let p: *i64=sys_mmap(16) as *i64; p[0]=0
  let t: *i64=sys_mmap(16) as *i64; t[0]=0
  let oc: *i64=sys_mmap(16) as *i64
46
  // success -> base interval, consec reset
  chk("200 success -> base" as *u8, pace_delay_after(1000, 0, 0, 200, 0, oc), 1000, p, t)
  chk("200 resets consec" as *u8, oc[0], 0, p, t)
  // 429 no Retry-After -> exponential base*2^consec
  chk("429 consec0 -> base*1" as *u8, pace_delay_after(1000, 0, 0, 429, 0, oc), 1000, p, t)
  chk("429 consec0 -> consec=1" as *u8, oc[0], 1, p, t)
  chk("429 consec1 -> base*2" as *u8, pace_delay_after(1000, 0, 1, 429, 0, oc), 2000, p, t)
  chk("429 consec3 -> base*8" as *u8, pace_delay_after(1000, 0, 3, 429, 0, oc), 8000, p, t)
  // 429 with Retry-After honored (seconds -> ms)
  chk("429 Retry-After=120 -> 120000" as *u8, pace_delay_after(1000, 0, 0, 429, 120, oc), 120000, p, t)
  // backoff cap
  chk("429 consec20 -> capped 300000" as *u8, pace_delay_after(1000, 0, 20, 429, 0, oc), 300000, p, t)
  // 503 also throttles
  chk("503 consec0 -> base*1" as *u8, pace_delay_after(1000, 0, 0, 503, 0, oc), 1000, p, t)
  // crawl-delay override (max of base, crawl_delay)
  chk("crawl_delay 5000 on success" as *u8, pace_delay_after(1000, 5000, 0, 200, 0, oc), 5000, p, t)
  chk("crawl_delay 5000 on 429" as *u8, pace_delay_after(1000, 5000, 0, 429, 0, oc), 5000, p, t)
  // wait clamp
  chk("wait now1000 next3000 -> 2000" as *u8, pace_wait_ms(1000, 3000), 2000, p, t)
  chk("wait past -> 0" as *u8, pace_wait_ms(5000, 3000), 0, p, t)
  chk("wait huge -> capped 60000" as *u8, pace_wait_ms(0, 999999999), 60000, p, t)
  // Retry-After header parse
  let hdr: *u8 = "HTTP/1.1 429 Too Many Requests\r\nRetry-After: 42\r\nServer: x\r\n\r\n" as *u8
  var hl: i64=0; while hdr[hl]!=(0 as u8){hl=hl+1}
  chk("parse Retry-After: 42" as *u8, pace_retry_after(hdr, hl), 42, p, t)
  let hdr2: *u8 = "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n" as *u8
  var hl2: i64=0; while hdr2[hl2]!=(0 as u8){hl2=hl2+1}
  chk("parse no Retry-After -> 0" as *u8, pace_retry_after(hdr2, hl2), 0, p, t)
75
  // persistence round-trip: set crawl-delay for a host, reload, verify it stuck
  let host: *u8 = "gate.example.test" as *u8; var hnl: i64=0; while host[hnl]!=(0 as u8){hnl=hnl+1}
  pace_set_crawl_delay(host, hnl, 7000)
  let tbl: *u8 = pace_load_tbl()
  let hh: i64 = pace_hash(host, hnl)
  let s: i64 = pace_slot(tbl, hh)
  var cdval: i64 = 0 - 1
  if s >= 0 { cdval = pace_ld(tbl, s*PACE_REC + 24) }
  chk("persist crawl_delay round-trip" as *u8, cdval, 7000, p, t)
  // SELF-ISOLATION (2026-08-02): this gate MUTATES a PERSISTENT host table, so without a reset the
  // consec counter accumulates across runs and the gate passes only on a machine that never ran it
  // (measured: got=3 want=1 on the 3rd run). A re-runnable gate must establish its own precondition:
  // a 200 resets consec to 0 by policy, so the following 429 must land on exactly 1 every time.
  // R9 (2026-08-24): a 200 no longer RESETS consec, it decays it by one, so a persisted streak from an earlier run
  // would leak into this tooth. The reset verb is the gate's precondition now, and it is a real verb, not a fixture.
  pace_reset_host(host, hnl)
  // pace_after 429 persists incremented consec (2nd file-write in the process -> guards the EACCES bug)
  pace_after(host, hnl, 429, 0)
  let tbl2: *u8 = pace_load_tbl()
  let s2: i64 = pace_slot(tbl2, hh)
  var consecval: i64 = 0 - 1
  if s2 >= 0 { consecval = pace_ld(tbl2, s2*PACE_REC + 16) }
  chk("persist 429 consec=1 (2nd write)" as *u8, consecval, 1, p, t)
  // and next-allowed got stamped (nonzero)
  var nextval: i64 = 0
  if s2 >= 0 { nextval = pace_ld(tbl2, s2*PACE_REC + 8) }
  var nz: i64 = 0; if nextval > 0 { nz = 1 }
  chk("persist next-allowed stamped" as *u8, nz, 1, p, t)
104
  // ================= R12: 403-AFTER-200 IS A RATE SIGNAL =================
  // Measured on loc.gov (2026-08-06, debt 1786031862): two pages served, then 403 on everything
  // after, six independent observations agreeing. The lever there is PACING, not anti-bot capability
  // -- but the pacer could not see it, because it only ever recognised 429/503.
  // Both directions are tested, because a rule that throttles EVERY 403 would be just as wrong: it
  // would slow every genuinely-forbidden path and dress a real access failure up as patience.
  chk("403 with NO prior 2xx -> NOT throttled (a refusal is not a rate limit)" as *u8,
      pace_delay_after2(1000, 0, 0, 403, 0, 0, oc), 1000, p, t)
  chk("403 with no prior 2xx -> consec stays 0" as *u8, oc[0], 0, p, t)
  chk("403 AFTER a 2xx -> throttled base*1" as *u8,
      pace_delay_after2(1000, 0, 0, 403, 0, 1, oc), 1000, p, t)
  chk("403 AFTER a 2xx -> consec INCREMENTS to 1" as *u8, oc[0], 1, p, t)
  chk("403 AFTER a 2xx, consec2 -> base*4 (real backoff, not a flat delay)" as *u8,
      pace_delay_after2(1000, 0, 2, 403, 0, 1, oc), 4000, p, t)
  chk("403 AFTER a 2xx honours Retry-After" as *u8,
      pace_delay_after2(1000, 0, 0, 403, 30, 1, oc), 30000, p, t)
  // API STABILITY: the 6-arg form must still mean exactly what it meant before (ok_seen=0).
  chk("legacy 6-arg 403 == ok_seen=0 form" as *u8,
      pace_delay_after(1000, 0, 0, 403, 0, oc), pace_delay_after2(1000, 0, 0, 403, 0, 0, oc), p, t)
  // REGRESSION CONTROL: ok_seen must not perturb the rules that already existed.
  chk("429 unchanged with ok_seen=1" as *u8, pace_delay_after2(1000, 0, 1, 429, 0, 1, oc), 2000, p, t)
  chk("429 unchanged with ok_seen=0" as *u8, pace_delay_after2(1000, 0, 1, 429, 0, 0, oc), 2000, p, t)
  // R9 DECAY (2026-08-24): a success takes ONE halving off the backoff; it no longer erases the streak.
  chk("R9: 200 after 5 throttles decays to consec 4 = base*16 (not base)" as *u8, pace_delay_after2(1000, 0, 5, 200, 0, 1, oc), 16000, p, t)
  chk("R9: ...and the persisted consec becomes 4" as *u8, oc[0], 4, p, t)
  chk("R9: 200 after ONE throttle is back at base (consec 1 -> 0)" as *u8, pace_delay_after2(1000, 0, 1, 200, 0, 0, oc), 1000, p, t)
  chk("R9: ...consec 0" as *u8, oc[0], 0, p, t)
  chk("R9: a never-throttled host is byte-identical (consec 0, 200 -> base)" as *u8, pace_delay_after2(1000, 0, 0, 200, 0, 1, oc), 1000, p, t)
  chk("R9: pace_decay is the pure step: 3 -> 2, 0 -> 0" as *u8, pace_decay(3) * 10 + pace_decay(0), 20, p, t)
  chk("R9: decay caps at the backoff ceiling (consec 20 -> 19 -> still 300000)" as *u8, pace_delay_after2(1000, 0, 20, 200, 0, 1, oc), 300000, p, t)
135
  // PERSISTED end-to-end: a 2xx sets the ok_seen bit, and the NEXT 403 is therefore throttled.
  // This is the whole loc.gov sequence, replayed against the real table.
  let h4: *u8 = "r12.example.test" as *u8; var h4l: i64=0; while h4[h4l]!=(0 as u8){h4l=h4l+1}
  pace_after(h4, h4l, 200, 0)                      // serve one page -> ok_seen
  let tb4: *u8 = pace_load_tbl()
  let s4: i64 = pace_slot(tb4, pace_hash(h4, h4l))
  var fl4: i64 = 0
  if s4 >= 0 { fl4 = pace_ld(tb4, s4*PACE_REC + 32) & PACE_F_OKSEEN }
  chk("a 2xx sets the persisted ok_seen bit" as *u8, fl4, PACE_F_OKSEEN, p, t)
  pace_after(h4, h4l, 403, 0)                      // ...then the host 403s us
  let tb5: *u8 = pace_load_tbl()
  let s5: i64 = pace_slot(tb5, pace_hash(h4, h4l))
  var cs5: i64 = 0 - 1
  if s5 >= 0 { cs5 = pace_ld(tb5, s5*PACE_REC + 16) }
  chk("the following 403 IS counted as a throttle (consec=1)" as *u8, cs5, 1, p, t)
  // NEGATIVE CONTROL on a host that never served us: its 403 must NOT count as a throttle.
  let h6: *u8 = "r12neg.example.test" as *u8; var h6l: i64=0; while h6[h6l]!=(0 as u8){h6l=h6l+1}
  pace_after(h6, h6l, 403, 0)
  let tb6: *u8 = pace_load_tbl()
  let s6: i64 = pace_slot(tb6, pace_hash(h6, h6l))
  var cs6: i64 = 0 - 1
  if s6 >= 0 { cs6 = pace_ld(tb6, s6*PACE_REC + 16) }
  chk("NEG: a 403 from a host that never served us stays consec=0" as *u8, cs6, 0, p, t)
159
  // ================= R13: THE DEFERRAL PREDICATE (2026-08-25) =================
  // pace_should_defer is what lets a backoff actually ELAPSE. PACE_WAIT_CAP_MS caps a WAIT at 60 s while
  // pace_delay_after2 can set 300 s, so before this the crawler re-hit a throttled host at 60 s, earned
  // another 429, and consec could never decay (pace_decay needs a success). DEFERRING SERVES THE BACKOFF.
  // MEASURED 2026-08-25: books.google.com fetched at ~62 s intervals 41 consecutive times while an 8-way
  // pool sat idle behind it; crawl throughput fell 280.7 -> 72.3 docs/hour.
  // FIXTURE A -- a THROTTLED host. Retry-After 300 s deliberately, so the backoff cannot elapse mid-gate
  // and leave the tooth deciding on timing instead of on policy.
  let dh1: *u8 = "r13throttled.example.test" as *u8; var dh1l: i64=0; while dh1[dh1l]!=(0 as u8){dh1l=dh1l+1}
  pace_reset_host(dh1, dh1l)
  pace_after(dh1, dh1l, 429, R13_BACKOFF_S)
  // FIXTURE B -- a merely-POLITE host: never throttled, but publishing a TWO-MINUTE Crawl-delay.
  let dh2: *u8 = "r13polite.example.test" as *u8; var dh2l: i64=0; while dh2[dh2l]!=(0 as u8){dh2l=dh2l+1}
  pace_reset_host(dh2, dh2l)
  pace_set_crawl_delay(dh2, dh2l, R13_POLITE_CD_MS)
  pace_after(dh2, dh2l, 200, 0)
  // Manufacture the long outstanding wait EXPLICITLY (see R13_LONG_WAIT_MS). pace_after has just set
  // next_allowed to now + 60 s, which would decay below the clamp within milliseconds on a loaded box
  // and turn a POLICY tooth into a TIMING tooth -- the same trap R13_BACKOFF_S was chosen to avoid.
  let dfx: *u8 = pace_load_tbl()
  let dfs: i64 = pace_slot(dfx, pace_hash(dh2, dh2l))
  if dfs >= 0 { pace_st(dfx, dfs*PACE_REC + PACE_OFF_NEXT_ALLOWED, sys_now_realtime_ms() + R13_LONG_WAIT_MS) }
  pace_save_tbl(dfx)
  // ASSERT THE FIXTURE REACHED THE CONDITION BEFORE ASSERTING THE OUTCOME. Both hosts must have a LARGE
  // wait outstanding RIGHT NOW -- and because pace_wait_ms clamps at PACE_WAIT_CAP_MS, both clamp to the
  // SAME 60 s. That is the point: the two fixtures are indistinguishable by DURATION, so the verdicts
  // below can only be explained by consec_throttle.
  let dtb: *u8 = pace_load_tbl()
  let ds1: i64 = pace_slot(dtb, pace_hash(dh1, dh1l))
  let ds2: i64 = pace_slot(dtb, pace_hash(dh2, dh2l))
  var dw1: i64 = 0
  var dw2: i64 = 0
  var dc1: i64 = 0
  var dc2: i64 = 0
  if ds1 >= 0 { dw1 = pace_wait_ms(sys_now_realtime_ms(), pace_ld(dtb, ds1*PACE_REC + 8)); dc1 = pace_ld(dtb, ds1*PACE_REC + 16) }
  if ds2 >= 0 { dw2 = pace_wait_ms(sys_now_realtime_ms(), pace_ld(dtb, ds2*PACE_REC + 8)); dc2 = pace_ld(dtb, ds2*PACE_REC + 16) }
  chk("R13 fixture: THROTTLED host has a wait outstanding (clamped to the 60s cap)" as *u8, dw1, PACE_WAIT_CAP_MS, p, t)
  chk("R13 fixture: POLITE host has an IDENTICAL outstanding wait -- only consec differs" as *u8, dw2, PACE_WAIT_CAP_MS, p, t)
  chk("R13 fixture: throttled host consec=1" as *u8, dc1, 1, p, t)
  chk("R13 fixture: polite host consec=0 (a published Crawl-delay is not a throttle)" as *u8, dc2, 0, p, t)
  chk("R13 a THROTTLED host DEFERS (1)" as *u8, pace_should_defer(dh1, dh1l), 1, p, t)
  // THE MANDATORY NEG-CONTROL. If deferral keyed on DURATION instead of consec, this host -- same 60 s
  // outstanding wait, zero throttles -- would also defer, and the crawler would fetch ONE url per host
  // per run: slower than the defect this rung was written to fix.
  chk("R13 neg-control-POLITE-HOST-SAME-WAIT-MUST-NOT-DEFER (0)" as *u8, pace_should_defer(dh2, dh2l), 0, p, t)
  // the length is DERIVED, never hand-counted beside the literal: a typed-out length is a second copy
  // of the string's shape and the two drift silently (change the host, forget the number, and the
  // predicate is asked about a PREFIX while still compiling and still appearing to work).
  let dh3: *u8 = "r13unknown.example.test" as *u8; var dh3l: i64=0; while dh3[dh3l]!=(0 as u8){dh3l=dh3l+1}
  chk("R13 neg-control-UNKNOWN-HOST-never-deferred (deferral is not a default)" as *u8, pace_should_defer(dh3, dh3l), 0, p, t)
  // consec > 0 but the backoff has ALREADY ELAPSED -> proceed. This branch is what lets a host come BACK:
  // without it one throttle would defer forever, pace_decay could never run (it needs a success), and the
  // deferral would become the permanent lock it exists to break.
  let dtb2: *u8 = pace_load_tbl()
  let ds3: i64 = pace_slot(dtb2, pace_hash(dh1, dh1l))
  if ds3 >= 0 { pace_st(dtb2, ds3*PACE_REC + 8, 1) }
  pace_save_tbl(dtb2)
  chk("R13 consec>0 but backoff ALREADY SERVED -> proceed (0), so a throttled host can come back" as *u8, pace_should_defer(dh1, dh1l), 0, p, t)
  let dtb3: *u8 = pace_load_tbl()
  let ds4: i64 = pace_slot(dtb3, pace_hash(dh1, dh1l))
  var dc4: i64 = 0
  if ds4 >= 0 { dc4 = pace_ld(dtb3, ds4*PACE_REC + 16) }
  chk("R13 ...and that host STILL carries consec=1 (the tooth above is not a no-history pass)" as *u8, dc4, 1, p, t)
223
  // ================= R13b: ONE PREDICATE, TWO ENTRY POINTS =================
  pace_reset_host(dh1, dh1l)
  pace_after(dh1, dh1l, 429, R13_BACKOFF_S)
  let etb: *u8 = pace_load_tbl()
  chk("R13b wrapper == table-form on a THROTTLED host" as *u8, pace_should_defer(dh1, dh1l), pace_should_defer_tbl(etb, dh1, dh1l), p, t)
  chk("R13b wrapper == table-form on a POLITE host" as *u8, pace_should_defer(dh2, dh2l), pace_should_defer_tbl(etb, dh2, dh2l), p, t)
  chk("R13b ...and that first equality is 1-vs-1, not 0-vs-0 (an equality of two zeroes proves nothing)" as *u8, pace_should_defer_tbl(etb, dh1, dh1l), 1, p, t)
  chk("R13b neg-control-NULL-TABLE-defers-nothing (a failed load must not blockade the crawl)" as *u8, pace_should_defer_tbl(0 as *u8, dh1, dh1l), 0, p, t)
232
  // ================= R14: PUBLISHED CRAWL-DELAY DEFERRAL (2026-08-25) =================
  // WHAT IT FIXES. pace_before honours an interval by SLEEPING, and pace_wait_ms clamps that sleep at
  // PACE_WAIT_CAP_MS. So for any published Crawl-delay ABOVE the clamp the sleep is truncated and the
  // host is hit EARLY -- a host asking 120 s was fetched at 60 s. Upstream it was worse: the crawler
  // rewrote the published number DOWN to 30 s before it ever reached this table, so the host was hit
  // at 30 s. Both are one defect: a delay we could not SLEEP through became a delay we ignored.
  // Crawl-delay is not in RFC 9309, but ignoring it is named by CDN verified-bot policy as a breach.
  // FIXTURE C -- a host publishing ONE MILLISECOND past the clamp. Chosen at the boundary on purpose:
  // it proves the rule fires on strictly-greater-than and not on some vaguer notion of "a long delay".
  let dh4: *u8 = "r14slowhost.example.test" as *u8; var dh4l: i64=0; while dh4[dh4l]!=(0 as u8){dh4l=dh4l+1}
  pace_reset_host(dh4, dh4l)
  pace_set_crawl_delay(dh4, dh4l, PACE_WAIT_CAP_MS + 1)
  pace_after(dh4, dh4l, 200, 0)
  let rtb: *u8 = pace_load_tbl()
  let rs4: i64 = pace_slot(rtb, pace_hash(dh4, dh4l))
  if rs4 >= 0 { pace_st(rtb, rs4*PACE_REC + PACE_OFF_NEXT_ALLOWED, sys_now_realtime_ms() + R13_LONG_WAIT_MS) }
  pace_save_tbl(rtb)
  // ASSERT THE FIXTURE REACHED THE CONDITION BEFORE ASSERTING THE OUTCOME.
  let rtb2: *u8 = pace_load_tbl()
  let rs5: i64 = pace_slot(rtb2, pace_hash(dh4, dh4l))
  var rcd: i64 = 0
  var rcc: i64 = 0
  var rww: i64 = 0
  if rs5 >= 0 {
      rcd = pace_ld(rtb2, rs5*PACE_REC + PACE_OFF_CRAWL_DELAY)
      rcc = pace_ld(rtb2, rs5*PACE_REC + PACE_OFF_CONSEC)
      rww = pace_wait_ms(sys_now_realtime_ms(), pace_ld(rtb2, rs5*PACE_REC + PACE_OFF_NEXT_ALLOWED))
  }
  chk("R14 fixture: the slow host really published cap+1 ms" as *u8, rcd, PACE_WAIT_CAP_MS + 1, p, t)
  chk("R14 fixture: it was NEVER throttled (consec=0) -- so no tooth below can pass via the R13 path" as *u8, rcc, 0, p, t)
  chk("R14 fixture: its outstanding wait clamps to the SAME 60 s as the polite host" as *u8, rww, PACE_WAIT_CAP_MS, p, t)
  // THE RULE.
  chk("R14 a host publishing a delay we CANNOT SLEEP THROUGH defers (1) instead of being hit early" as *u8,
      pace_should_defer(dh4, dh4l), 1, p, t)
  // NEG-CONTROL 1 -- THE BOUNDARY. The polite fixture publishes EXACTLY the cap, has an IDENTICAL
  // clamped wait and an identical consec=0, and must NOT defer: at the cap, sleeping still honours the
  // host, so deferring would cost coverage for nothing. This is the tooth an over-broad implementation
  // ("defer whenever a wait is long") cannot pass, and it is the same property R13's neg-control held.
  chk("R14 neg-control-AT-THE-CAP-MUST-NOT-DEFER (strictly greater-than, and sleeping still honours it)" as *u8,
      pace_should_defer(dh2, dh2l), 0, p, t)
  // NEG-CONTROL 2 -- NOT STARVATION. Once the host's own interval HAS elapsed we fetch. Without this,
  // R14 would silently convert "crawl this host slowly" into "never crawl this host", which is a
  // coverage loss wearing a compliance badge.
  let rtb3: *u8 = pace_load_tbl()
  let rs6: i64 = pace_slot(rtb3, pace_hash(dh4, dh4l))
  if rs6 >= 0 { pace_st(rtb3, rs6*PACE_REC + PACE_OFF_NEXT_ALLOWED, 1) }
  pace_save_tbl(rtb3)
  chk("R14 neg-control-INTERVAL-ELAPSED-PROCEEDS (0): the host is crawled at ITS pace, never starved" as *u8,
      pace_should_defer(dh4, dh4l), 0, p, t)
  let rtb4: *u8 = pace_load_tbl()
  let rs7: i64 = pace_slot(rtb4, pace_hash(dh4, dh4l))
  var rcd2: i64 = 0
  if rs7 >= 0 { rcd2 = pace_ld(rtb4, rs7*PACE_REC + PACE_OFF_CRAWL_DELAY) }
  chk("R14 ...and it STILL publishes cap+1 (the tooth above is not a forgotten-delay pass)" as *u8,
      rcd2, PACE_WAIT_CAP_MS + 1, p, t)
288
  // ================= R13b: THE REUSED-BUFFER CONTRACT =================
  // pace_load_tbl_into fills a CALLER-OWNED buffer so the crawler holds ONE 160 KB buffer per RUN instead
  // of one per candidate row (WC_MAXPEND=2048 rows x 160 KB = ~335 MB of leaked anon VMA and read I/O per
  // run, in the parent). That is only sound if a reload FULLY DEFINES the buffer: sys_mmap hands back zero
  // pages, a REUSED buffer does not, and two load paths leave bytes unwritten (no table on disk at all,
  // and the v1 migration, which writes only PACE_REC_V1 of each PACE_REC record). A surviving stale byte
  // there is a stale flags or consec field, i.e. a WRONG DEFERRAL.
  let dirty: *u8 = sys_mmap(PACE_TBL_BYTES)
  var dz: i64 = 0
  while dz < PACE_TBL_BYTES { dirty[dz] = R13_DIRTY_BYTE as u8; dz = dz + 1 }
  let clean: *u8 = pace_load_tbl()
  var predif: i64 = 0
  var dq: i64 = 0
  while dq < PACE_TBL_BYTES { if dirty[dq] != clean[dq] { predif = 1; dq = PACE_TBL_BYTES } else { dq = dq + 1 } }
  chk("R13b fixture: the dirty buffer DIFFERS from a fresh load before reload (tooth is non-vacuous)" as *u8, predif, 1, p, t)
  pace_load_tbl_into(dirty)
  var samed: i64 = 1
  var dr: i64 = 0
  while dr < PACE_TBL_BYTES { if dirty[dr] != clean[dr] { samed = 0; dr = PACE_TBL_BYTES } else { dr = dr + 1 } }
  chk("R13b reload into a REUSED buffer == a fresh load, byte for byte (no stale byte survives)" as *u8, samed, 1, p, t)
309
  gw("=== NX-CRAWL-PACE " as *u8); gn(p[0]); gw("/" as *u8); gn(t[0])
  if p[0] == t[0] { gw(" GREEN\n" as *u8) } else { gw(" RED\n" as *u8) }
  let ctr: *i64 = gv_ctr()
  ctr[0] = p[0]
  ctr[1] = t[0]
  return gv_verdict("CRAWL-PACE-GATE" as *u8, ctr, "politeness policy: Retry-After honored, exponential backoff, decay-on-success (R9: one success halves the backoff), crawl-delay persisted" as *u8)
316}
nx_crawl_pace_gate.nx source

`nx_crawl_pace_gate.nx` source
