`nx_net_polite.nx` source

1// nx_net_polite.nx -- INTERNET CITIZENSHIP decisions (pure, deterministic, gate-able): how the harvester
2// paces itself + backs off so it is NOT flagged/blocked/rate-banned (which would dry up our sources). This is
3// the DECISION layer -- the caller (nx_search_extern) composes these around the TLS fetch. Keeps the logic
4// pure so it gates without touching the network. Named thresholds (rule 11, no magic numbers). license_tier: ORIGINAL
5import "nx_syscalls.nx"
6
7const POL_MAX_RETRY:        i64 = 3    // give up after this many retries (then honor the block, don't hammer)
8const POL_BACKOFF_BASE_SEC: i64 = 2    // exponential backoff base
9const POL_BACKOFF_CAP_SEC:  i64 = 60   // never sleep longer than this per retry
10// per-source minimum request intervals the caller passes to pol_wait_sec (source TOU): arXiv ~1/3s, GitHub 10/min
11const POL_ARXIV_MIN_SEC:    i64 = 3
12const POL_GITHUB_MIN_SEC:   i64 = 6
13
14// seconds to sleep before the next request to a source so we honor its min interval (0 if enough time passed
15// or first request). now/last are epoch seconds.
16func pol_wait_sec(now: i64, last: i64, min_interval: i64) -> i64 {
17    if last<=0 { return 0 }
18    let elapsed: i64 = now - last
19    if elapsed >= min_interval { return 0 }
20    return min_interval - elapsed
21}
22// exponential backoff (base * 2^attempt), capped. attempt 0 -> base.
23func pol_backoff_sec(attempt: i64, base: i64, cap: i64) -> i64 {
24    var s: i64 = base; var i: i64 = 0
25    while i < attempt { s = s*2; if s>cap { s=cap; i=attempt } else { i=i+1 } }
26    if s>cap { s=cap }
27    return s
28}
29// which statuses mean "you are being rate-limited / try later" (retry w/ backoff) vs a hard error (give up)
30func pol_should_retry(status: i64) -> i64 { if status==429 { return 1 } if status==503 { return 1 } return 0 }
31
32func pol_digit(c: i64) -> i64 { if c>=48 { if c<=57 { return 1 } } return 0 }
33func pol_find_after(buf: *u8, n: i64, needle: *u8) -> i64 {
34    var nn: i64=0; while needle[nn]!=(0 as u8) { nn=nn+1 }
35    if nn==0 { return 0-1 }
36    var i: i64=0
37    while i+nn<=n { var j: i64=0; var ok: i64=1; while j<nn { if buf[i+j]!=needle[j] { ok=0; j=nn } else { j=j+1 } } if ok==1 { return i+nn } i=i+1 }
38    return 0-1
39}
40// parse the numeric Retry-After header value (delta-seconds form) from a raw response; -1 if absent.
41func pol_retry_after(resp: *u8, n: i64) -> i64 {
42    let p: i64=pol_find_after(resp, n, "Retry-After:" as *u8); if p<0 { return 0-1 }
43    var lim: i64=p+16; if lim>n { lim=n }
44    var ds: i64=0-1; var i: i64=p
45    while i<lim { if ds<0 { if pol_digit(resp[i] as i64)==1 { ds=i } } i=i+1 }
46    if ds<0 { return 0-1 }
47    var v: i64=0; var j: i64=ds
48    while j<n { if pol_digit(resp[j] as i64)==1 { v=v*10+((resp[j] as i64)-48); j=j+1 } else { j=n } }
49    return v
50}
51// the actual sleep-before-request (pacing), given a per-source last-fetch epoch. returns the new "last" (now).
52func pol_pace(last: i64, min_interval: i64) -> i64 {
53    let now: i64 = sys_now_realtime_sec()
54    let w: i64 = pol_wait_sec(now, last, min_interval)
55    if w>0 { sys_sleep_ms(w*1000) }
56    return sys_now_realtime_sec()
57}
58// the backoff sleep after a rate-limit response: honor Retry-After if present, else exponential backoff.
59func pol_backoff_sleep(resp: *u8, n: i64, attempt: i64) -> i64 {
60    var wait: i64 = pol_retry_after(resp, n)
61    if wait<0 { wait = pol_backoff_sec(attempt, POL_BACKOFF_BASE_SEC, POL_BACKOFF_CAP_SEC) }
62    if wait>POL_BACKOFF_CAP_SEC { wait=POL_BACKOFF_CAP_SEC }
63    if wait>0 { sys_sleep_ms(wait*1000) }
64    return wait
65}
nx_net_polite.nx source

`nx_net_polite.nx` source
