nx_paced_fetch.nx source
↩ module page · 133 lines · 6513 B
1// nx_paced_fetch.nx -- THE ONE PACING HOOK FOR THE CAPTURE PATH (/compare/mediaingest R0; the vault's watch
2// contract symbol is vc_pace_hook in nx_vault_capture, which delegates here so every capture verb shares one hook).
3//
4// WHY (measured 2026-08-30, corpus_complete=1): nx_crawl_pace -- Retry-After honoured, exponential backoff, decay on
5// success, a table persisted across processes -- was imported by the crawl lane and nx_4chan and by NONE of the
6// capture verbs (nx_vault_capture, nx_mvault_fetch, nx_hls_get, nx_media_grab), so every capture hit a host as fast
7// as its loop ran. That is the rate-limit the operator meets on Harvestr, unmanaged here by construction.
8//
9// THIS IS NOT A SECOND PACING RULER. Every decision below is nx_crawl_pace's own arithmetic (pace_*_tbl). The lib
10// adds only what the capture verbs lacked: URL to host, the before/after pair around ONE fetch, a BURST form for
11// segment streams, and ONE process-lifetime table buffer. Spend-gate 2026-08-30 listed the incumbents and they are
12// policies, not this hook: nx_retry_policy (per-call retry maths) and the builder-generated _cn_fetch/_pe_crawl/
13// _wb_pace governors (per-organ min-gap copies -- the duplicate-ruler class this composition exists to stop).
14//
15// THE BURST FORM (pf_before_burst): segments of ONE stream are not separate acts of politeness. The default
16// interval is PACE_BASE_MS = 1 s per host, which on a 1000-segment HLS VOD is a 17-minute floor for no reason a
17// host cares about. The burst form pays the wait ONLY once the host has actively throttled us (pace_should_defer:
18// consec_throttle > 0, or a published Crawl-delay longer than one fetch can sleep through) -- the exact signal the
19// pacer's own contract names as the only one that means refused-us. A healthy CDN streams at full speed; a 429
20// still lands as backoff on the very next segment. pf_after is called on EVERY response either way, so the
21// table always carries the truth.
22//
23// MEMORY: pace_before/pace_after mmap a 160 KB table per call and never unmap it (nx_crawl_pace R13b measured that
24// for the crawler). Called once per FETCH by a 1000-segment download that is ~320 MB of touched anon pages. This
25// lib owns ONE buffer for the process and reloads it IN PLACE (pace_load_tbl_into) before every decision, so a
26// write by another process, or by our own pf_after, is always seen and nothing leaks. license_tier: ORIGINAL
27import "nx_syscalls.nx"
28import "nx_crawl_pace.nx"
29
30const PF_HOST_CAP: i64 = 512 // the host scratch buffer; DNS names are <= 253 octets so this is headroom, not a guess
31const PF_SLASH: i64 = 47
32const PF_COLON: i64 = 58
33const PF_QMARK: i64 = 63
34const PF_HASH: i64 = 35
35const PF_NO_HOST: i64 = 0 - 1 // the url carried no scheme://host -- nothing to pace on, and the caller is TOLD, never silently unpaced
36
37static pf_tbl_g: *u8 // ONE process-lifetime pace table (PACE_TBL_BYTES), reloaded in place before every decision
38static pf_host_g: *u8 // ONE host scratch buffer -- never allocate in a per-fetch loop
39
40// host of "scheme://host[:port]/path?q#f" -> out (NUL-terminated); returns its length, 0 when there is no "://".
41func pf_host_of(url: *u8, out: *u8, cap: i64) -> i64 {
42 var i: i64 = 0
43 var start: i64 = 0 - 1
44 var scan: i64 = 1
45 while scan == 1 {
46 let c: i64 = url[i] as i64
47 if c == 0 { scan = 0 }
48 if scan == 1 {
49 if c == PF_COLON {
50 let c1: i64 = url[i + 1] as i64
51 if c1 == PF_SLASH {
52 let c2: i64 = url[i + 2] as i64
53 if c2 == PF_SLASH { start = i + 3; scan = 0 }
54 }
55 }
56 }
57 if scan == 1 { i = i + 1 }
58 }
59 if start < 0 { out[0] = 0 as u8; return 0 }
60 var n: i64 = 0
61 var p: i64 = start
62 var go: i64 = 1
63 while go == 1 {
64 let d: i64 = url[p] as i64
65 if d == 0 { go = 0 }
66 if d == PF_SLASH { go = 0 }
67 if d == PF_COLON { go = 0 }
68 if d == PF_QMARK { go = 0 }
69 if d == PF_HASH { go = 0 }
70 if go == 1 {
71 if n < cap - 1 { out[n] = d as u8; n = n + 1 }
72 p = p + 1
73 }
74 }
75 out[n] = 0 as u8
76 return n
77}
78
79// the process's one table, reloaded IN PLACE from disk so every decision sees the latest persisted state
80func pf_tbl() -> *u8 {
81 if (pf_tbl_g as i64) == 0 { pf_tbl_g = sys_mmap(PACE_TBL_BYTES) }
82 pace_load_tbl_into(pf_tbl_g)
83 return pf_tbl_g
84}
85func pf_host() -> *u8 {
86 if (pf_host_g as i64) == 0 { pf_host_g = sys_mmap(PF_HOST_CAP) }
87 return pf_host_g
88}
89
90// BEFORE one request: the polite wait for the url's host (sleeps it). Returns ms waited; PF_NO_HOST when the url
91// has no host. This is the form for page and album fetches -- one document per act.
92func pf_before(url: *u8) -> i64 {
93 let h: *u8 = pf_host()
94 let n: i64 = pf_host_of(url, h, PF_HOST_CAP)
95 if n <= 0 { return PF_NO_HOST }
96 return pace_before_tbl(pf_tbl(), h, n)
97}
98
99// BEFORE one segment of a stream: wait ONLY when the host has actively throttled us; a healthy host streams.
100func pf_before_burst(url: *u8) -> i64 {
101 let h: *u8 = pf_host()
102 let n: i64 = pf_host_of(url, h, PF_HOST_CAP)
103 if n <= 0 { return PF_NO_HOST }
104 let t: *u8 = pf_tbl()
105 if pace_should_defer_tbl(t, h, n) != 1 { return 0 }
106 return pace_before_tbl(t, h, n)
107}
108
109// AFTER one response: feed the pacer the REAL status. retry_after_s is the Retry-After seconds when the caller
110// holds the response headers (pace_retry_after over the raw response), else 0 -- never a guessed number.
111// Returns the delay the pacer applied (ms); PF_NO_HOST when the url has no host.
112func pf_after(url: *u8, status: i64, retry_after_s: i64) -> i64 {
113 let h: *u8 = pf_host()
114 let n: i64 = pf_host_of(url, h, PF_HOST_CAP)
115 if n <= 0 { return PF_NO_HOST }
116 return pace_after_tbl(pf_tbl(), h, n, status, retry_after_s)
117}
118
119// the deferral predicate by url: 1 = this host is actively throttling us (leave the item pending), else 0
120func pf_should_defer(url: *u8) -> i64 {
121 let h: *u8 = pf_host()
122 let n: i64 = pf_host_of(url, h, PF_HOST_CAP)
123 if n <= 0 { return 0 }
124 return pace_should_defer_tbl(pf_tbl(), h, n)
125}
126
127// gate self-isolation and operator reset: forget a host's throttle history (its robots crawl-delay is kept by design)
128func pf_reset(url: *u8) -> i64 {
129 let h: *u8 = pf_host()
130 let n: i64 = pf_host_of(url, h, PF_HOST_CAP)
131 if n <= 0 { return PF_NO_HOST }
132 return pace_reset_host(h, n)
133}