nx_freshness.nx source
↩ module page · 29 lines · 1788 B
1// nx_freshness.nx -- LIB: adaptive RE-CRAWL scheduling = index freshness without wasting crawl budget. Per URL the
2// scheduler adapts the re-crawl interval to observed change: a page that CHANGED since last crawl is crawled SOONER
3// (halve the interval); a page that DID NOT change BACKS OFF (double it), bounded [MIN,MAX]. Important pages (high
4// PageRank priority) get a shorter EFFECTIVE interval so the head of the web stays fresh. This keeps a billion-page
5// index current on a finite crawler -- the freshness half of "a bigger BETTER index". Built + gated NOW; runs over the
6// live corpus on the NAS (the scheduler drives the crawler; last-content-hash from the ingested doc). No float.
7// license_tier: ORIGINAL
8import "nx_syscalls.nx"
9const FR_MAGIC_2000: i64 = 2000
10
11const FR_MIN: i64 = 3600 // 1h floor (seconds)
12const FR_MAX: i64 = 2592000 // 30d ceiling
13
14// after a re-crawl: changed -> halve (crawl sooner), unchanged -> double (back off); clamped to [MIN,MAX].
15func fr_update_interval(interval: i64, changed: i64) -> i64 {
16 var ni: i64 = interval
17 if changed == 1 { ni = interval / 2 } else { ni = interval * 2 }
18 if ni < FR_MIN { ni = FR_MIN }
19 if ni > FR_MAX { ni = FR_MAX }
20 return ni
21}
22// is a url due for re-crawl at `now`? (now >= last_crawl + interval)
23func fr_due(last_crawl: i64, interval: i64, now: i64) -> i64 { if now >= last_crawl + interval { return 1 } return 0 }
24// priority-adjusted interval: priority_permille 0..1000 (e.g. from PageRank) -> higher = shorter effective interval.
25func fr_effective_interval(interval: i64, priority_permille: i64) -> i64 {
26 var eff: i64 = interval * (FR_MAGIC_2000 - priority_permille) / FR_MAGIC_2000 // priority 1000 -> half; 0 -> full
27 if eff < FR_MIN { eff = FR_MIN }
28 return eff
29}