nx_entity_pin_lib.nx source
↩ module page · 319 lines · 14065 B
1// nx_entity_pin_lib.nx -- ENTITY PIN PROJECTION (S11, 2026-09-17): the decision core nx_entity_pin and its gate share.
2//
3// WHY. Operator REJECT 2026-09-16 ("i just tried julia kyoka on nishifamily.com/search and it still isnt very good
4// results"): the canonical entity page (the javdatabase idol page) was in the ranker's candidate set but says JULIA,
5// so a token scorer put albums and three results pages above it. The estate ALREADY resolves the alias: nx_entity_dossier
6// picks the canonical landing out of the same search and writes knowledge/status/dossier-<slug>.txt with the card. What
7// was missing is the wire: this lib projects those resolutions into ONE served table the ranker reads,
8// <status-prefix>entitypin.tsv rows <query>\t<cid>\t<url>\t<jp>\t<authority>\t<asof>\n
9// (query = the entity name lowercased, single-spaced; cid = the landing document; a truncate-write that keeps every
10// other row). The ranker (nx_docportal_search_seg, S11) loads the table once and reloads it when its size or mtime
11// moves, and pins the cid to the top band for that exact query. NO third-party format, NO plane read per query.
12//
13// CONTRACT. ep_project(prefix, names, nnames, counts): for each name, read <prefix>dossier-<slug>.txt; a missing file is
14// MISSING (counts[EP_C_MISSING]), found:false is UNRESOLVED (counts[EP_C_UNRESOLVED]), otherwise the row is upserted
15// (counts[EP_C_PINNED]); counts[EP_C_ROWS] is the table's row count after the write. Every outcome prints a receipt
16// line naming the entity, so a caller can never mistake "nothing pinned" for "pinned". license_tier: ORIGINAL
17import "nx_syscalls.nx"
18
19const EP_DOSSIER_PREFIX: *u8 = "dossier-"
20const EP_DOSSIER_EXT: *u8 = ".txt"
21const EP_TSV_NAME: *u8 = "entitypin.tsv"
22const EP_PATH_CAP: i64 = 1024
23const EP_FIELD_CAP: i64 = 512
24const EP_LINE_CAP: i64 = 2048 // one row: six fields, the url the longest (EP_FIELD_CAP), never more
25const EP_STAT_BYTES: i64 = 256
26const EP_STAT_SIZE_OFF: i64 = 48
27const EP_READ_CHUNK: i64 = 65536
28const EP_FILE_MODE: i64 = 420 // 0644
29const EP_C_ROWS: i64 = 0
30const EP_C_PINNED: i64 = 1
31const EP_C_MISSING: i64 = 2
32const EP_C_UNRESOLVED: i64 = 3
33const EP_C_N: i64 = 4
34const EP_CH_TAB: i64 = 9
35const EP_CH_NL: i64 = 10
36const EP_CH_CR: i64 = 13
37const EP_CH_SPACE: i64 = 32
38const EP_CH_QUOTE: i64 = 34
39const EP_CH_MINUS: i64 = 45
40const EP_CH_ZERO: i64 = 48
41const EP_CH_NINE: i64 = 57
42const EP_CH_A: i64 = 65
43const EP_CH_Z: i64 = 90
44const EP_CH_LA: i64 = 97
45const EP_CH_LZ: i64 = 122
46const EP_CH_UPPER_TO_LOWER: i64 = 32
47const EP_TEN: i64 = 10
48
49func ep_slen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
50func ep_cat(d: *u8, o: i64, s: *u8) -> i64 { var i: i64 = 0; var p: i64 = o; while s[i] != (0 as u8) { d[p] = s[i]; p = p + 1; i = i + 1 } return p }
51func ep_catn(d: *u8, o: i64, v: i64) -> i64 {
52 var m: i64 = v
53 var p: i64 = o
54 if m < 0 { d[p] = EP_CH_MINUS as u8; p = p + 1; m = 0 - m }
55 let t: *u8 = sys_mmap(32)
56 var k: i64 = 0
57 if m == 0 { t[0] = EP_CH_ZERO as u8; k = 1 }
58 while m > 0 { t[k] = (EP_CH_ZERO + (m % EP_TEN)) as u8; m = m / EP_TEN; k = k + 1 }
59 while k > 0 { k = k - 1; d[p] = t[k]; p = p + 1 }
60 sys_munmap(t, 32)
61 return p
62}
63func ep_lower(c: i64) -> i64 { if c >= EP_CH_A { if c <= EP_CH_Z { return c + EP_CH_UPPER_TO_LOWER } } return c }
64func ep_alnum(c: i64) -> i64 {
65 if c >= EP_CH_ZERO { if c <= EP_CH_NINE { return 1 } }
66 if c >= EP_CH_LA { if c <= EP_CH_LZ { return 1 } }
67 return 0
68}
69// the served key: lowercase, every run of non-alphanumerics one space, no leading or trailing space
70func ep_norm(name: *u8, dst: *u8, cap: i64) -> i64 {
71 var o: i64 = 0
72 var pend: i64 = 0
73 var i: i64 = 0
74 while name[i] != (0 as u8) {
75 let c: i64 = ep_lower(name[i] as i64)
76 if ep_alnum(c) == 1 {
77 if pend == 1 { if o > 0 { if o < cap - 1 { dst[o] = EP_CH_SPACE as u8; o = o + 1 } } pend = 0 }
78 if o < cap - 1 { dst[o] = c as u8; o = o + 1 }
79 } else { pend = 1 }
80 i = i + 1
81 }
82 dst[o] = 0 as u8
83 return o
84}
85// the dossier's file slug: the same key with '-' for the separator (nx_entity_dossier's own rule)
86func ep_slug(name: *u8, dst: *u8, cap: i64) -> i64 {
87 let n: i64 = ep_norm(name, dst, cap)
88 var i: i64 = 0
89 while i < n { if dst[i] as i64 == EP_CH_SPACE { dst[i] = EP_CH_MINUS as u8 } i = i + 1 }
90 return n
91}
92// whole-file read sized from the file, freed by the caller with ep_free; returns 0 with lenout=0 when absent
93func ep_read(path: *u8, lenout: *i64) -> *u8 {
94 lenout[0] = 0
95 let fd: i64 = sys_openat_rd(path)
96 if fd < 0 { return 0 as *u8 }
97 let sb: *u8 = sys_mmap(EP_STAT_BYTES)
98 var sz: i64 = 0
99 if sys_fstat_fd(fd, sb) >= 0 { let sp: *i64 = ((sb as i64) + EP_STAT_SIZE_OFF) as *i64; sz = sp[0] }
100 sys_munmap(sb, EP_STAT_BYTES)
101 let cap: i64 = sz + 1
102 let buf: *u8 = sys_mmap(cap)
103 var got: i64 = 0
104 var r: i64 = 1
105 while r > 0 {
106 var want: i64 = cap - got
107 if want > EP_READ_CHUNK { want = EP_READ_CHUNK }
108 if want <= 0 { r = 0 } else {
109 r = sys_read(fd, ((buf as i64) + got) as *u8, want)
110 if r > 0 { got = got + r }
111 }
112 }
113 sys_close(fd)
114 buf[got] = 0 as u8
115 lenout[0] = got
116 return buf
117}
118func ep_free(buf: *u8, n: i64) -> i64 { if (buf as i64) != 0 { sys_munmap(buf, n + 1) } return 0 }
119// first offset of needle in buf[0..n), or -1
120func ep_find(buf: *u8, n: i64, needle: *u8) -> i64 {
121 let m: i64 = ep_slen(needle)
122 if m == 0 { return 0 - 1 }
123 var i: i64 = 0
124 while i + m <= n {
125 var j: i64 = 0
126 var same: i64 = 1
127 while j < m { if buf[i + j] != needle[j] { same = 0; j = m } else { j = j + 1 } }
128 if same == 1 { return i }
129 i = i + 1
130 }
131 return 0 - 1
132}
133// digits at off (an optional leading quote is skipped); -1 when there are none
134func ep_num_at(buf: *u8, n: i64, off: i64) -> i64 {
135 var i: i64 = off
136 if i < n { if buf[i] as i64 == EP_CH_QUOTE { i = i + 1 } }
137 var v: i64 = 0
138 var any: i64 = 0
139 var go: i64 = 1
140 while go == 1 {
141 if i >= n { go = 0 } else {
142 let c: i64 = buf[i] as i64
143 if c >= EP_CH_ZERO { if c <= EP_CH_NINE { v = v * EP_TEN + (c - EP_CH_ZERO); any = 1; i = i + 1 } else { go = 0 } } else { go = 0 }
144 }
145 }
146 if any == 0 { return 0 - 1 }
147 return v
148}
149// the string at off up to the closing quote, tabs and newlines replaced by spaces (a row field can carry neither)
150func ep_str_at(buf: *u8, n: i64, off: i64, dst: *u8, cap: i64) -> i64 {
151 var i: i64 = off
152 var o: i64 = 0
153 var go: i64 = 1
154 while go == 1 {
155 if i >= n { go = 0 } else {
156 let c: i64 = buf[i] as i64
157 if c == EP_CH_QUOTE { go = 0 } else {
158 var w: i64 = c
159 if c == EP_CH_TAB { w = EP_CH_SPACE }
160 if c == EP_CH_NL { w = EP_CH_SPACE }
161 if c == EP_CH_CR { w = EP_CH_SPACE }
162 if o < cap - 1 { dst[o] = w as u8; o = o + 1 }
163 i = i + 1
164 }
165 }
166 }
167 dst[o] = 0 as u8
168 return o
169}
170func ep_puts(s: *u8) -> i64 { sys_write(1, s, ep_slen(s)); return 0 }
171func ep_putn(v: i64) -> i64 { let t: *u8 = sys_mmap(32); let k: i64 = ep_catn(t, 0, v); sys_write(1, t, k); sys_munmap(t, 32); return 0 }
172// does the row at buf[off..] (a full line) start with key followed by a TAB?
173func ep_row_has_key(buf: *u8, n: i64, off: i64, key: *u8) -> i64 {
174 var j: i64 = 0
175 while key[j] != (0 as u8) { if off + j >= n { return 0 } if buf[off + j] != key[j] { return 0 } j = j + 1 }
176 if off + j >= n { return 0 }
177 if buf[off + j] as i64 == EP_CH_TAB { return 1 }
178 return 0
179}
180// one row from a dossier: writes the row text into rowbuf (ending in NL) and returns its length; 0 = unresolved
181func ep_row_from_dossier(dbuf: *u8, dn: i64, key: *u8, asof: i64, rowbuf: *u8) -> i64 {
182 if ep_find(dbuf, dn, "\"found\":true" as *u8) < 0 { return 0 }
183 let oc: i64 = ep_find(dbuf, dn, "\"landing_cid\":" as *u8)
184 if oc < 0 { return 0 }
185 let cid: i64 = ep_num_at(dbuf, dn, oc + ep_slen("\"landing_cid\":" as *u8))
186 if cid <= 0 { return 0 }
187 let url: *u8 = sys_mmap(EP_FIELD_CAP)
188 let jp: *u8 = sys_mmap(EP_FIELD_CAP)
189 url[0] = 0 as u8
190 jp[0] = 0 as u8
191 let ou: i64 = ep_find(dbuf, dn, "\"canonical_url\":\"" as *u8)
192 if ou >= 0 { ep_str_at(dbuf, dn, ou + ep_slen("\"canonical_url\":\"" as *u8), url, EP_FIELD_CAP) }
193 let oj: i64 = ep_find(dbuf, dn, "\"jp\":\"" as *u8)
194 if oj >= 0 { ep_str_at(dbuf, dn, oj + ep_slen("\"jp\":\"" as *u8), jp, EP_FIELD_CAP) }
195 var auth: i64 = 0
196 let oa: i64 = ep_find(dbuf, dn, "\"authority_total\":" as *u8)
197 if oa >= 0 { auth = ep_num_at(dbuf, dn, oa + ep_slen("\"authority_total\":" as *u8)); if auth < 0 { auth = 0 } }
198 var o: i64 = 0
199 o = ep_cat(rowbuf, o, key)
200 rowbuf[o] = EP_CH_TAB as u8; o = o + 1
201 o = ep_catn(rowbuf, o, cid)
202 rowbuf[o] = EP_CH_TAB as u8; o = o + 1
203 o = ep_cat(rowbuf, o, url)
204 rowbuf[o] = EP_CH_TAB as u8; o = o + 1
205 o = ep_cat(rowbuf, o, jp)
206 rowbuf[o] = EP_CH_TAB as u8; o = o + 1
207 o = ep_catn(rowbuf, o, auth)
208 rowbuf[o] = EP_CH_TAB as u8; o = o + 1
209 o = ep_catn(rowbuf, o, asof)
210 rowbuf[o] = EP_CH_NL as u8; o = o + 1
211 rowbuf[o] = 0 as u8
212 sys_munmap(url, EP_FIELD_CAP)
213 sys_munmap(jp, EP_FIELD_CAP)
214 return o
215}
216// THE PROJECTION. names[0..nnames) -> upserted rows in <prefix>entitypin.tsv; counts as documented in the header.
217func ep_project(prefix: *u8, names: *i64, nnames: i64, counts: *i64) -> i64 {
218 var c: i64 = 0
219 while c < EP_C_N { counts[c] = 0; c = c + 1 }
220 let asof: i64 = sys_now_realtime_sec() // wall-clock epoch seconds: one machine writes it and another judges it
221 // the new rows and their keys
222 let rows: *u8 = sys_mmap((nnames + 1) * EP_LINE_CAP)
223 let keys: *u8 = sys_mmap((nnames + 1) * EP_FIELD_CAP)
224 var nrows: i64 = 0
225 let path: *u8 = sys_mmap(EP_PATH_CAP)
226 let slug: *u8 = sys_mmap(EP_FIELD_CAP)
227 let lenp: *i64 = sys_mmap(8) as *i64
228 var i: i64 = 0
229 while i < nnames {
230 let name: *u8 = names[i] as *u8
231 let key: *u8 = ((keys as i64) + nrows * EP_FIELD_CAP) as *u8
232 ep_norm(name, key, EP_FIELD_CAP)
233 ep_slug(name, slug, EP_FIELD_CAP)
234 var o: i64 = ep_cat(path, 0, prefix)
235 o = ep_cat(path, o, EP_DOSSIER_PREFIX)
236 o = ep_cat(path, o, slug)
237 o = ep_cat(path, o, EP_DOSSIER_EXT)
238 path[o] = 0 as u8
239 let dbuf: *u8 = ep_read(path, lenp)
240 let dn: i64 = lenp[0]
241 if (dbuf as i64) == 0 {
242 counts[EP_C_MISSING] = counts[EP_C_MISSING] + 1
243 ep_puts("MISSING name=" as *u8); ep_puts(key); ep_puts(" dossier=" as *u8); ep_puts(path); ep_puts("\n" as *u8)
244 } else {
245 let row: *u8 = ((rows as i64) + nrows * EP_LINE_CAP) as *u8
246 let rl: i64 = ep_row_from_dossier(dbuf, dn, key, asof, row)
247 if rl == 0 {
248 counts[EP_C_UNRESOLVED] = counts[EP_C_UNRESOLVED] + 1
249 ep_puts("UNRESOLVED name=" as *u8); ep_puts(key); ep_puts(" dossier=" as *u8); ep_puts(path); ep_puts("\n" as *u8)
250 } else {
251 counts[EP_C_PINNED] = counts[EP_C_PINNED] + 1
252 ep_puts("PIN " as *u8); sys_write(1, row, rl)
253 nrows = nrows + 1
254 }
255 ep_free(dbuf, dn)
256 }
257 i = i + 1
258 }
259 // the table: every existing row whose key is not being replaced, then the new rows; truncate-written
260 var o2: i64 = ep_cat(path, 0, prefix)
261 o2 = ep_cat(path, o2, EP_TSV_NAME)
262 path[o2] = 0 as u8
263 let old: *u8 = ep_read(path, lenp)
264 let on: i64 = lenp[0]
265 let outcap: i64 = on + (nrows + 1) * EP_LINE_CAP + 1
266 let out: *u8 = sys_mmap(outcap)
267 var w: i64 = 0
268 var total: i64 = 0
269 var p: i64 = 0
270 while p < on {
271 // the line [p, q): q is the NL offset, or on when the last line is unterminated (a flag, never a sentinel cursor)
272 var q: i64 = p
273 var scanning: i64 = 1
274 while scanning == 1 { if q >= on { scanning = 0 } else { if old[q] as i64 == EP_CH_NL { scanning = 0 } else { q = q + 1 } } }
275 // q is the NL offset (or on when the last line is unterminated); the flag ends the scan, the cursor is never clobbered
276 var replaced: i64 = 0
277 var r: i64 = 0
278 while r < nrows { if ep_row_has_key(old, on, p, ((keys as i64) + r * EP_FIELD_CAP) as *u8) == 1 { replaced = 1 } r = r + 1 }
279 let linelen: i64 = q - p
280 if replaced == 0 { if linelen > 0 {
281 var k: i64 = 0
282 while k < linelen { out[w + k] = old[p + k]; k = k + 1 }
283 w = w + linelen
284 out[w] = EP_CH_NL as u8; w = w + 1
285 total = total + 1
286 } }
287 p = q + 1
288 }
289 var r2: i64 = 0
290 while r2 < nrows {
291 let row: *u8 = ((rows as i64) + r2 * EP_LINE_CAP) as *u8
292 let rl: i64 = ep_slen(row)
293 var k2: i64 = 0
294 while k2 < rl { out[w + k2] = row[k2]; k2 = k2 + 1 }
295 w = w + rl
296 total = total + 1
297 r2 = r2 + 1
298 }
299 let fd: i64 = sys_openat_wr(path, EP_FILE_MODE)
300 if fd >= 0 {
301 var wr: i64 = 0
302 var bad: i64 = 0
303 while wr < w { if bad == 1 { wr = w } else { let k3: i64 = sys_write(fd, ((out as i64) + wr) as *u8, w - wr); if k3 <= 0 { bad = 1 } else { wr = wr + k3 } } }
304 sys_close(fd)
305 }
306 counts[EP_C_ROWS] = total
307 ep_puts("ENTITYPIN rows=" as *u8); ep_putn(counts[EP_C_ROWS])
308 ep_puts(" pinned=" as *u8); ep_putn(counts[EP_C_PINNED])
309 ep_puts(" missing=" as *u8); ep_putn(counts[EP_C_MISSING])
310 ep_puts(" unresolved=" as *u8); ep_putn(counts[EP_C_UNRESOLVED])
311 ep_puts(" table=" as *u8); ep_puts(path); ep_puts("\n" as *u8)
312 ep_free(old, on)
313 sys_munmap(out, outcap)
314 sys_munmap(rows, (nnames + 1) * EP_LINE_CAP)
315 sys_munmap(keys, (nnames + 1) * EP_FIELD_CAP)
316 sys_munmap(path, EP_PATH_CAP)
317 sys_munmap(slug, EP_FIELD_CAP)
318 return counts[EP_C_PINNED]
319}