code wiki / _hdl_build / nx_research_freshness.nx
nx_research_freshness.nx source
↩ module page · 191 lines · 11032 B
1// nx_research_freshness.nx -- AUTONOMOUS FRESHNESS REVIEW on a cadence. Previously-researched veins get
2// re-checked: is the content MOVING (changed since last review) or STATIC? Maintains freshness_state.txt
3// (<path> <fp> <stable> <reviews>) over the registered veins. Each review cycle re-fingerprints the source and
4// compares to the last-recorded fingerprint:
5// - new source -> BASELINE (first sight)
6// - fingerprint changed -> MOVING (reset stable=0; flag for re-mine -> the vein is active)
7// - fingerprint unchanged -> STATIC (stable++); once stable >= RF_CONST_LIMIT -> CONSTANT (it has hit its
8// limit / is a constant -> recommend BACKING OFF the review cadence)
9// The monthly/other CADENCE is how often this organ is RUN (cron/schedule); CONSTANT veins are flagged so the
10// scheduler can review them less. v1 compares the on-disk .raw fingerprint (detects movement whenever the corpus
11// is re-fetched fresh); production active-refetch needs source-URL persistence (the named next gap). Offline,
12// deterministic, gateable. expect_exit: 0 license_tier: ORIGINAL
13import "nx_search_inverted_persist.nx"
14import "nx_itoa_lib.nx" // shared MSB-first emitter (zero-alloc)
15
16const RF_CAP: i64 = 65536
17const RF_CONST_LIMIT: i64 = 3 // consecutive STATIC reviews before a vein is declared a CONSTANT (settled)
18
19func rf_puts(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
20// MIGRATED to the shared emitter (debt 1785563586). The old body mmapped a scratch buffer
21// per call and never freed it. At PAGE granularity that is 4096B leaked PER CALL -- the
22// defect that took 28.5GB of a 36GB host in nx_ts_lumadiff (2MB input, ~3.66M calls).
23// nxi_* is MSB-first, allocates NOTHING, and emits identical bytes including the sign.
24func rf_num(v: i64) -> i64 { nxi_out(v); return 0 }
25func rf_cat(dst: *u8, off: i64, s: *u8) -> i64 { var i: i64=0; while s[i]!=(0 as u8){dst[off+i]=s[i];i=i+1} return off+i }
26func rf_catbuf(dst: *u8, off: i64, src: *u8, n: i64) -> i64 { var i: i64=0; while i<n { dst[off+i]=src[i]; i=i+1 } return off+n }
27func rf_catnum(dst: *u8, off: i64, v: i64) -> i64 {
28 if v == 0 { dst[off]=48 as u8; return off+1 }
29 let tmp: *u8=sys_mmap(28); var m: i64=v; var k: i64=0
30 while m>0 { tmp[k]=(48+(m%10)) as u8; m=m/10; k=k+1 }
31 var i: i64=0; while i<k { dst[off+i]=tmp[k-1-i]; i=i+1 }
32 return off+k
33}
34func rf_field_end(buf: *u8, i: i64, end: i64) -> i64 {
35 var p: i64=i; var stop: i64=0
36 while stop==0 { if p>=end { stop=1 } else { if buf[p]==(32 as u8) { stop=1 } else { if buf[p]==(10 as u8) { stop=1 } else { p=p+1 } } } }
37 return p
38}
39func rf_parse_dec(buf: *u8, start: i64, fend: i64) -> i64 {
40 var v: i64=0; var p: i64=start
41 while p < fend { let c: i64=buf[p] as i64; if c>=48 { if c<=57 { v=v*10+(c-48) } } p=p+1 }
42 return v
43}
44func rf_streq_rng(buf: *u8, a_off: i64, a_len: i64, b: *u8, b_len: i64) -> i64 {
45 if a_len != b_len { return 0 }
46 var i: i64=0
47 while i < a_len { if buf[a_off+i] != b[i] { return 0 } i=i+1 }
48 return 1
49}
50
51// full-document FNV-1a 64-bit (same as the grow organ -- distinct docs -> distinct fingerprints)
52func rf_fp(buf: *u8, len: i64) -> i64 {
53 var h: i64 = NX_INV_FNV1A_OFFSET_BASIS
54 var i: i64 = 0
55 while i < len { let c: i64=buf[i] as i64; h = h ^ c; h = h * NX_INV_FNV1A_PRIME; i=i+1 }
56 if h < 0 { h = 0 - h }
57 return h
58}
59
60func main() -> i64 {
61 rf_puts("=== nx_research_freshness: autonomous moving/static/constant review of researched veins ===\n" as *u8)
62 let reg_path: *u8 = "knowledge/index/content_registry.txt" as *u8
63 let st_path: *u8 = "knowledge/index/freshness_state.txt" as *u8
64 let rep_path: *u8 = "knowledge/index/freshness_report.log" as *u8
65
66 let rbox: *i64 = sys_mmap(16) as *i64
67 let reg: *u8 = sys_read_file(reg_path, rbox)
68 if reg == 0 as *u8 { rf_puts(" NO registry yet (run nx_research_grow first)\n" as *u8); sys_exit(1); return 1 }
69 let rlen: i64 = rbox[0]
70
71 // ---- load prior freshness state: parallel arrays keyed by path ----
72 let s_poff: *i64 = sys_mmap(8*RF_CAP) as *i64 // path off into state buffer
73 let s_plen: *i64 = sys_mmap(8*RF_CAP) as *i64
74 let s_fp: *i64 = sys_mmap(8*RF_CAP) as *i64
75 let s_stab: *i64 = sys_mmap(8*RF_CAP) as *i64
76 let s_rev: *i64 = sys_mmap(8*RF_CAP) as *i64
77 var ns: i64 = 0
78 let sbox: *i64 = sys_mmap(16) as *i64
79 let stbuf: *u8 = sys_read_file(st_path, sbox)
80 var slen: i64 = 0
81 if stbuf != 0 as *u8 {
82 slen = sbox[0]
83 var i: i64 = 0
84 while i < slen {
85 let f1: i64 = rf_field_end(stbuf, i, slen); let po: i64=i; let pl: i64=f1-i // path
86 var j: i64 = f1; if j<slen { if stbuf[j]==(32 as u8) { j=j+1 } }
87 let f2: i64 = rf_field_end(stbuf, j, slen); let fp: i64 = rf_parse_dec(stbuf, j, f2)
88 var j2: i64 = f2; if j2<slen { if stbuf[j2]==(32 as u8) { j2=j2+1 } }
89 let f3: i64 = rf_field_end(stbuf, j2, slen); let stab: i64 = rf_parse_dec(stbuf, j2, f3)
90 var j3: i64 = f3; if j3<slen { if stbuf[j3]==(32 as u8) { j3=j3+1 } }
91 let f4: i64 = rf_field_end(stbuf, j3, slen); let rev: i64 = rf_parse_dec(stbuf, j3, f4)
92 var e: i64 = f4; var st: i64=0
93 while st==0 { if e>=slen { st=1 } else { if stbuf[e]==(10 as u8) { st=1 } else { e=e+1 } } }
94 if e<slen { e=e+1 }
95 if pl > 0 { if ns < RF_CAP { s_poff[ns]=po; s_plen[ns]=pl; s_fp[ns]=fp; s_stab[ns]=stab; s_rev[ns]=rev; ns=ns+1 } }
96 i = e
97 }
98 }
99
100 // ---- collect distinct canonical paths from the registry (the veins to review) ----
101 // (registry lines: <fp> <topic> <path>; we review each distinct path once)
102 let rep_fd: i64 = sys_openat_wr(rep_path, 0x1a4)
103 let nst_fd: i64 = sys_openat_wr("knowledge/index/freshness_state.next" as *u8, 0x1a4)
104 let pbuf: *u8 = sys_mmap(RF_CAP)
105 let dbox: *i64 = sys_mmap(16) as *i64
106 var baseline: i64=0; var moving: i64=0; var stat: i64=0; var constant: i64=0
107 var i: i64 = 0
108 while i < rlen {
109 // skip fp + topic fields, get path field
110 let f1: i64 = rf_field_end(reg, i, rlen)
111 var j: i64 = f1; if j<rlen { if reg[j]==(32 as u8) { j=j+1 } }
112 let f2: i64 = rf_field_end(reg, j, rlen)
113 var p: i64 = f2; if p<rlen { if reg[p]==(32 as u8) { p=p+1 } }
114 let f3: i64 = rf_field_end(reg, p, rlen); let poff: i64=p; let pl: i64=f3-p
115 var e: i64 = f3; var st: i64=0
116 while st==0 { if e>=rlen { st=1 } else { if reg[e]==(10 as u8) { st=1 } else { e=e+1 } } }
117 if e<rlen { e=e+1 }
118
119 if pl > 0 {
120 // dedup: only review a path the FIRST time it appears in the registry this scan
121 var earlier: i64 = 0
122 var q: i64 = 0
123 while q < i {
124 // re-derive earlier line's path is costly; instead check against state-or-next via a simple marker:
125 q = i // (no-op; dedup handled by the next-state write being keyed; see note)
126 }
127 // build NUL-terminated path
128 var po2: i64 = rf_catbuf(pbuf, 0, ((reg as i64)+poff) as *u8, pl); pbuf[po2]=0 as u8
129 // find in prior state
130 var sidx: i64 = 0-1
131 var k: i64 = 0
132 while k < ns { if rf_streq_rng(stbuf, s_poff[k], s_plen[k], pbuf, pl)==1 { sidx=k } k=k+1 }
133 // skip if we've already emitted this path to next-state this run (dedup via prior-emit check on state arrays appended)
134 var already: i64 = 0
135 if sidx >= 0 { if s_rev[sidx] < 0 { already=1 } } // we mark emitted by negating rev (see below)
136 if already == 0 {
137 var cur_fp: i64 = 0
138 var nstab: i64 = 0; var nrev: i64 = 1; var status: *u8 = "BASELINE" as *u8
139 var settled: i64 = 0
140 if sidx >= 0 { if s_stab[sidx] >= RF_CONST_LIMIT { settled = 1 } }
141 if settled == 1 {
142 // INCREMENTAL: a settled CONSTANT carries forward WITHOUT reading the file (constants are
143 // reviewed rarely -- the operator's "won't move after hitting their limit"). Avoids the
144 // per-run read of every doc, so freshness is fast once the corpus settles.
145 let prev_rev0: i64 = s_rev[sidx]
146 cur_fp = s_fp[sidx]; nstab = s_stab[sidx]; nrev = prev_rev0 + 1
147 constant = constant + 1; status = "CONSTANT" as *u8
148 s_rev[sidx] = 0 - prev_rev0 - 1
149 } else {
150 let buf: *u8 = sys_read_file(pbuf, dbox)
151 if buf != 0 as *u8 { cur_fp = rf_fp(buf, dbox[0]) }
152 if sidx >= 0 {
153 let prev_rev: i64 = s_rev[sidx]
154 nrev = prev_rev + 1
155 if s_fp[sidx] != cur_fp { nstab = 0; moving=moving+1; status="MOVING" as *u8 }
156 else {
157 nstab = s_stab[sidx] + 1
158 if nstab >= RF_CONST_LIMIT { constant=constant+1; status="CONSTANT" as *u8 } else { stat=stat+1; status="STATIC" as *u8 }
159 }
160 s_rev[sidx] = 0 - prev_rev - 1 // mark emitted (negative) to dedup repeat registry lines
161 } else { baseline=baseline+1 }
162 }
163 // write next-state line: <path> <fp> <stab> <rev>
164 let nl: *u8 = sys_mmap(RF_CAP+128); var lp: i64=0
165 lp=rf_catbuf(nl,lp,pbuf,pl); nl[lp]=32 as u8; lp=lp+1
166 lp=rf_catnum(nl,lp,cur_fp); nl[lp]=32 as u8; lp=lp+1
167 lp=rf_catnum(nl,lp,nstab); nl[lp]=32 as u8; lp=lp+1
168 lp=rf_catnum(nl,lp,nrev); nl[lp]=10 as u8; lp=lp+1
169 if nst_fd >= 0 { sys_write(nst_fd, nl, lp) }
170 // report line
171 let rl: *u8 = sys_mmap(RF_CAP+128); var rp2: i64=0
172 rp2=rf_cat(rl,rp2,status); rl[rp2]=32 as u8; rp2=rp2+1
173 rp2=rf_cat(rl,rp2,"stable=" as *u8); rp2=rf_catnum(rl,rp2,nstab)
174 rp2=rf_cat(rl,rp2," rev=" as *u8); rp2=rf_catnum(rl,rp2,nrev)
175 rp2=rf_cat(rl,rp2," " as *u8); rp2=rf_catbuf(rl,rp2,pbuf,pl); rl[rp2]=10 as u8; rp2=rp2+1
176 if rep_fd >= 0 { sys_write(rep_fd, rl, rp2) }
177 }
178 }
179 i = e
180 }
181 if rep_fd >= 0 { sys_close(rep_fd) }
182 if nst_fd >= 0 { sys_close(nst_fd) }
183 // promote next-state -> state (atomic-ish rename)
184 sys_renameat("knowledge/index/freshness_state.next" as *u8, st_path)
185
186 rf_puts(" reviewed: baseline="); rf_num(baseline); rf_puts(" moving="); rf_num(moving)
187 rf_puts(" static="); rf_num(stat); rf_puts(" constant="); rf_num(constant); rf_puts("\n" as *u8)
188 rf_puts(" (MOVING veins -> re-mine; CONSTANT veins -> back off cadence) -> knowledge/index/freshness_report.log\n" as *u8)
189 rf_puts(" FRESHNESS-OK\n" as *u8)
190 sys_exit(0); return 0
191}