nx_reviewmine_lib.nx source
↩ module page · 1270 lines · 60434 B
1// nx_reviewmine_lib.nx -- REVIEW MINER, THE SHARED DECISION CORE (/compare/reviewmine, contract symbols
2// rm_steam_parse_page, rm_row_emit, rm_seen_has, rm_mine_rank). LIB: pure parsing, the journal row codec, the
3// dedupe set, the tokenizer and the complaint-vs-praise ranker. NO network here: the fetch I/O lives in the thin
4// program nx_steam_reviews so this closure stays nx_syscalls-only and every branch is gate-provable with planted
5// pages -- the same lib/program split nx_source_health uses, and the one that keeps a gate honest.
6//
7// WHY (operator 2026-09-05): "a review miner that can help us address opportunities from amazon to ebay to steam"
8// as part of fetch and ingest. ONE decision core, per-source adapters: Steam first because the estate is doing
9// game dev; the row codec, dedupe set and ranker are source-agnostic so the Amazon Reviews 2023 shape
10// (rating, title, text, helpful_vote, verified_purchase, timestamp) lands as a second adapter, not a second miner.
11//
12// THE ROW (one line, pipe-separated, text LAST and escaped so a review can never split a row):
13// r|recid|steamid|lang|up|pt_review|pt_forever|ts_c|ts_u|votes_up|votes_funny|wvs_permil|comments|flags|
14// games_owned|num_reviews|textlen|text
15// escapes inside text: backslash -> \\ pipe -> \p LF -> \n CR -> \r (nothing else is touched)
16//
17// THE RANKER (rm_mine_rank): per-term DOCUMENT frequency in negative vs positive reviews, smoothed, as a permil
18// ratio ratio = (neg_df+1) * 1000 * (pos_docs+2) / ((pos_df+1) * (neg_docs+2)). A term with equal share in both
19// classes reads ~1000; a term that appears only in complaints reads high. It is the classic feature-based
20// summarisation question (Hu and Liu 2004: which features do customers complain about) answered with document
21// counts a machine can recompute, never a sentiment lexicon. Integer arithmetic throughout, no float.
22// license_tier: ORIGINAL
23import "nx_syscalls.nx"
24
25const RM_Q: i64 = 34
26const RM_BS: i64 = 92
27const RM_PIPE: i64 = 124
28const RM_LF: i64 = 10
29const RM_CR: i64 = 13
30const RM_COLON: i64 = 58
31const RM_SP: i64 = 32
32const RM_TAB: i64 = 9
33const RM_MINUS: i64 = 45
34const RM_DOT: i64 = 46
35const RM_ZERO: i64 = 48
36const RM_NINE: i64 = 57
37const RM_LOWER_A: i64 = 97
38const RM_LOWER_Z: i64 = 122
39const RM_UPPER_A: i64 = 65
40const RM_UPPER_Z: i64 = 90
41const RM_LOWER_F: i64 = 102
42const RM_UPPER_F: i64 = 70
43const RM_CASE_DELTA: i64 = 32
44const RM_PCT: i64 = 37
45const RM_HIGH_BYTE: i64 = 128
46const RM_CH_n: i64 = 110
47const RM_CH_r: i64 = 114
48const RM_CH_t: i64 = 116
49const RM_CH_b: i64 = 98
50const RM_CH_f: i64 = 102
51const RM_CH_u: i64 = 117
52const RM_CH_p: i64 = 112
53const RM_CH_SLASH: i64 = 47
54const RM_PERMIL: i64 = 1000
55const RM_DECIMAL: i64 = 10
56const RM_HEX: i64 = 16
57const RM_NUMBUF: i64 = 32
58const RM_TRUE_LEN: i64 = 4
59const RM_FALSE_LEN: i64 = 5
60const RM_UTF16_HI_LO: i64 = 55296 // 0xD800
61const RM_UTF16_HI_HI: i64 = 56319 // 0xDBFF
62const RM_UTF16_LO_LO: i64 = 56320 // 0xDC00
63const RM_UTF16_LO_HI: i64 = 57343 // 0xDFFF
64const RM_CP_1BYTE_MAX: i64 = 127
65const RM_CP_2BYTE_MAX: i64 = 2047
66const RM_CP_3BYTE_MAX: i64 = 65535
67const RM_SURROGATE_BASE: i64 = 65536
68const RM_SURROGATE_SHIFT: i64 = 1024
69const RM_LBRACE: i64 = 123
70const RM_RBRACE: i64 = 125
71const RM_APOS: i64 = 39
72const RM_UNDERSCORE: i64 = 95
73const RM_TILDE: i64 = 126
74const RM_SEMICOLON: i64 = 59
75const RM_KNUTH_MULT: i64 = 2654435761
76const RM_CH_BACKSPACE: i64 = 8
77const RM_CH_FORMFEED: i64 = 12
78const RM_TOK_MIN: i64 = 3
79const RM_TOK_MAX: i64 = 32
80const RM_MODE_DIR: i64 = 493 // 0755
81const RM_MODE_FILE: i64 = 420 // 0644
82
83// ---- the decoded review record: *i64 slots, named ----
84const RM_F_RECID: i64 = 0
85const RM_F_STEAMID: i64 = 1
86const RM_F_UP: i64 = 2
87const RM_F_PT_REVIEW: i64 = 3
88const RM_F_PT_FOREVER: i64 = 4
89const RM_F_TS_C: i64 = 5
90const RM_F_TS_U: i64 = 6
91const RM_F_VOTES_UP: i64 = 7
92const RM_F_VOTES_FUNNY: i64 = 8
93const RM_F_WVS: i64 = 9
94const RM_F_COMMENTS: i64 = 10
95const RM_F_FLAGS: i64 = 11
96const RM_F_GAMES_OWNED: i64 = 12
97const RM_F_NUM_REVIEWS: i64 = 13
98const RM_F_TEXTLEN: i64 = 14
99const RM_F_N: i64 = 16
100const RM_REC_BYTES: i64 = 128 // RM_F_N * 8
101const RM_FLAG_STEAM_PURCHASE: i64 = 1
102const RM_FLAG_FREE: i64 = 2
103const RM_FLAG_EARLY_ACCESS: i64 = 4
104const RM_FLAG_DECK: i64 = 8
105const RM_FLAG_REFUNDED: i64 = 16
106const RM_LANG_CAP: i64 = 32
107// Steam caps a written review at 8,000 characters and UTF-8 spends at most 4 bytes per character: the text
108// ceiling is DERIVED from those two published facts, not guessed. Every byte may escape to two in the row.
109const RM_TEXT_CAP: i64 = 32768
110const RM_ROW_CAP: i64 = 66048 // RM_TEXT_CAP * 2 + 512 header
111const RM_ROW_HDR_FIELDS: i64 = 17 // fields before text
112
113// ---- page summary: *i64 slots ----
114const RM_S_SUCCESS: i64 = 0
115const RM_S_NUM: i64 = 1
116const RM_S_SCORE: i64 = 2
117const RM_S_POS: i64 = 3
118const RM_S_NEG: i64 = 4
119const RM_S_TOTAL: i64 = 5
120const RM_S_N: i64 = 8
121const RM_CURSOR_CAP: i64 = 256
122
123// ---- widths and shapes the code below spells out, named so the count and its meaning travel together (rule 11) ----
124const RM_I64_BYTES: i64 = 8 // sizeof i64: every *i64 table is slots * RM_I64_BYTES
125const RM_BITS_PER_BYTE: i64 = 8 // the shift between adjacent bytes in rm_ld8 / rm_st8 (same value, different meaning)
126const RM_BYTE_MASK: i64 = 255
127const RM_I64_PAIR: i64 = 16 // two i64 slots: the sys_read_file length cell plus one spare
128const RM_NUMBUF_SLOTS: i64 = 2 // rm_numbuf serves two in-flight numbers
129const RM_TOK_BUF_SPARE: i64 = 2 // the token buffer holds RM_TOK_MAX bytes plus a NUL plus the over-long sentinel slot
130const RM_STR_TAIL: i64 = 2 // a copied string carries its NUL plus one spare byte
131const RM_GROW_FACTOR: i64 = 2 // the open-addressed seen set doubles, and grows at half full
132const RM_KEY_QUOTES: i64 = 2 // the two quote bytes around a JSON key
133const RM_ESC_LEN: i64 = 2 // a row escape is a backslash plus one byte
134const RM_ROW_PREFIX_LEN: i64 = 2 // every row opens with the two bytes r and pipe
135const RM_ROWF_LANG: i64 = 2 // the third row field is lang (the rec slots skip it)
136const RM_PCT_ESC_LEN: i64 = 3 // a percent escape is three bytes
137const RM_NIBBLE_SHIFT: i64 = 4
138const RM_NIBBLE_MASK: i64 = 15
139const RM_HEX4_DIGITS: i64 = 4 // a JSON unicode escape carries four hex digits
140const RM_UESC_PREFIX: i64 = 2 // the backslash and the u before them
141const RM_UESC_LEN: i64 = 6 // the whole escape
142const RM_PERMIL_DIGITS: i64 = 3 // a quoted decimal keeps three fractional digits (permil)
143const RM_DJB2_SEED: i64 = 5381 // Bernstein hash
144const RM_DJB2_SHIFT: i64 = 5
145const RM_UTF8_LEAD2: i64 = 192 // 110xxxxx
146const RM_UTF8_LEAD3: i64 = 224 // 1110xxxx
147const RM_UTF8_LEAD4: i64 = 240 // 11110xxx
148const RM_UTF8_CONT: i64 = 128 // 10xxxxxx
149const RM_UTF8_CONT_MASK: i64 = 63 // the six payload bits of a continuation byte
150const RM_UTF8_SHIFT1: i64 = 6 // payload bits carried per continuation byte
151const RM_UTF8_SHIFT2: i64 = 12
152const RM_UTF8_SHIFT3: i64 = 18
153const RM_UTF8_LEN2: i64 = 2
154const RM_UTF8_LEN3: i64 = 3
155const RM_UTF8_LEN4: i64 = 4
156const RM_UTF8_B2: i64 = 2 // byte offsets inside one encoded sequence (the second and third continuation slots)
157const RM_UTF8_B3: i64 = 3
158const RM_TOK_STATE_APOS: i64 = 2 // rm_next_token: an apostrophe neither extends nor ends a token
159const RM_BAND_UNDER_1H: i64 = 0 // rm_band_of: the four playtime bands, in order
160const RM_BAND_1_10H: i64 = 1
161const RM_BAND_10_100H: i64 = 2
162const RM_BAND_OVER_100H: i64 = 3
163
164func rm_slen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
165func rm_w(s: *u8) -> i64 { sys_write(1, s, rm_slen(s)); return 0 }
166func rm_wb(b: *u8, n: i64) -> i64 { if n > 0 { sys_write(1, b, n) } return 0 }
167static rm_num_g: *u8
168static rm_ep_g: *i64
169static rm_tok_g: *u8
170func rm_numbuf() -> *u8 { if (rm_num_g as i64) == 0 { rm_num_g = sys_mmap(RM_NUMBUF * RM_NUMBUF_SLOTS); rm_ep_g = sys_mmap(RM_I64_PAIR) as *i64; rm_tok_g = sys_mmap(RM_TOK_MAX + RM_TOK_BUF_SPARE) } return rm_num_g }
171func rm_itoa(v: i64, out: *u8) -> i64 {
172 var m: i64 = v
173 var o: i64 = 0
174 if m < 0 { out[0] = RM_MINUS as u8; o = 1; m = 0 - m }
175 let t: *u8 = rm_numbuf()
176 var k: i64 = 0
177 if m == 0 { t[0] = RM_ZERO as u8; k = 1 }
178 while m > 0 { t[k] = (RM_ZERO + (m % RM_DECIMAL)) as u8; m = m / RM_DECIMAL; k = k + 1 }
179 while k > 0 { k = k - 1; out[o] = t[k]; o = o + 1 }
180 return o
181}
182func rm_wn(v: i64) -> i64 { rm_numbuf(); let t: *u8 = (rm_num_g as i64 + RM_NUMBUF) as *u8; let k: i64 = rm_itoa(v, t); sys_write(1, t, k); return 0 }
183func rm_cat(d: *u8, o: i64, s: *u8) -> i64 { var i: i64 = 0; var p: i64 = o; while s[i] != (0 as u8) { d[p] = s[i]; p = p + 1; i = i + 1 } return p }
184func rm_catn(d: *u8, o: i64, s: *u8, n: i64) -> i64 { var i: i64 = 0; var p: i64 = o; while i < n { d[p] = s[i]; p = p + 1; i = i + 1 } return p }
185func rm_cati(d: *u8, o: i64, v: i64) -> i64 { return o + rm_itoa(v, (d as i64 + o) as *u8) }
186func rm_streq(a: *u8, b: *u8) -> i64 {
187 var i: i64 = 0
188 while a[i] != (0 as u8) { if a[i] != b[i] { return 0 } i = i + 1 }
189 if b[i] != (0 as u8) { return 0 }
190 return 1
191}
192func rm_atoi(s: *u8) -> i64 {
193 var v: i64 = 0; var i: i64 = 0
194 while s[i] != (0 as u8) { let c: i64 = s[i] as i64; if c >= RM_ZERO { if c <= RM_NINE { v = v * RM_DECIMAL + (c - RM_ZERO) } } i = i + 1 }
195 return v
196}
197func rm_hash(b: *u8, n: i64) -> i64 {
198 var x: i64 = RM_DJB2_SEED; var i: i64 = 0
199 while i < n { x = (((x << RM_DJB2_SHIFT) + x) + (b[i] as i64)) & 0x7fffffffffffffff; i = i + 1 }
200 if x == 0 { x = 1 }
201 return x
202}
203
204// ---- JSON field reading over a byte span (RFC 8259 strings; numbers; true/false) -------------------------
205// exact-key match: "<key>" then optional ws, ':', optional ws. Returns the offset of the VALUE's first byte
206// within [from, to), or -1. Bounded by `to` so a per-review key (num_reviews lives in BOTH query_summary and
207// author) is never read from a neighbouring object.
208func rm_key(buf: *u8, n: i64, key: *u8, from: i64, to: i64) -> i64 {
209 let kl: i64 = rm_slen(key)
210 var lim: i64 = to
211 if lim > n { lim = n }
212 var i: i64 = from
213 if i < 0 { i = 0 }
214 while i + kl + RM_KEY_QUOTES < lim {
215 if (buf[i] as i64) == RM_Q {
216 if (buf[i + 1 + kl] as i64) == RM_Q {
217 var same: i64 = 1
218 var j: i64 = 0
219 while j < kl { if buf[i + 1 + j] != key[j] { same = 0 } j = j + 1 }
220 if same == 1 {
221 var p: i64 = i + kl + RM_KEY_QUOTES
222 var ws: i64 = 1
223 while ws == 1 { if p >= lim { ws = 0 } else { let c: i64 = buf[p] as i64; if c == RM_SP { p = p + 1 } else { if c == RM_TAB { p = p + 1 } else { if c == RM_LF { p = p + 1 } else { if c == RM_CR { p = p + 1 } else { ws = 0 } } } } } }
224 if p < lim { if (buf[p] as i64) == RM_COLON {
225 p = p + 1
226 ws = 1
227 while ws == 1 { if p >= lim { ws = 0 } else { let c2: i64 = buf[p] as i64; if c2 == RM_SP { p = p + 1 } else { if c2 == RM_TAB { p = p + 1 } else { if c2 == RM_LF { p = p + 1 } else { if c2 == RM_CR { p = p + 1 } else { ws = 0 } } } } } }
228 if p < lim { return p }
229 return 0 - 1
230 } }
231 }
232 }
233 }
234 i = i + 1
235 }
236 return 0 - 1
237}
238func rm_hexval(c: i64) -> i64 {
239 if c >= RM_ZERO { if c <= RM_NINE { return c - RM_ZERO } }
240 if c >= RM_LOWER_A { if c <= RM_LOWER_F { return c - RM_LOWER_A + RM_DECIMAL } }
241 if c >= RM_UPPER_A { if c <= RM_UPPER_F { return c - RM_UPPER_A + RM_DECIMAL } }
242 return 0 - 1
243}
244func rm_utf8_put(cp: i64, out: *u8, o: i64, cap: i64) -> i64 {
245 if cp <= RM_CP_1BYTE_MAX { if o + 1 > cap { return 0 - 1 } out[o] = cp as u8; return o + 1 }
246 if cp <= RM_CP_2BYTE_MAX { if o + RM_UTF8_LEN2 > cap { return 0 - 1 } out[o] = (RM_UTF8_LEAD2 + (cp >> RM_UTF8_SHIFT1)) as u8; out[o + 1] = (RM_UTF8_CONT + (cp & RM_UTF8_CONT_MASK)) as u8; return o + RM_UTF8_LEN2 }
247 if cp <= RM_CP_3BYTE_MAX { if o + RM_UTF8_LEN3 > cap { return 0 - 1 } out[o] = (RM_UTF8_LEAD3 + (cp >> RM_UTF8_SHIFT2)) as u8; out[o + 1] = (RM_UTF8_CONT + ((cp >> RM_UTF8_SHIFT1) & RM_UTF8_CONT_MASK)) as u8; out[o + RM_UTF8_B2] = (RM_UTF8_CONT + (cp & RM_UTF8_CONT_MASK)) as u8; return o + RM_UTF8_LEN3 }
248 if o + RM_UTF8_LEN4 > cap { return 0 - 1 }
249 out[o] = (RM_UTF8_LEAD4 + (cp >> RM_UTF8_SHIFT3)) as u8; out[o + 1] = (RM_UTF8_CONT + ((cp >> RM_UTF8_SHIFT2) & RM_UTF8_CONT_MASK)) as u8; out[o + RM_UTF8_B2] = (RM_UTF8_CONT + ((cp >> RM_UTF8_SHIFT1) & RM_UTF8_CONT_MASK)) as u8; out[o + RM_UTF8_B3] = (RM_UTF8_CONT + (cp & RM_UTF8_CONT_MASK)) as u8
250 return o + RM_UTF8_LEN4
251}
252// read 4 hex digits at buf[at..at+4) -> code unit, or -1
253func rm_hex4(buf: *u8, n: i64, at: i64) -> i64 {
254 if at + RM_HEX4_DIGITS > n { return 0 - 1 }
255 var v: i64 = 0; var h: i64 = 0
256 while h < RM_HEX4_DIGITS { let d: i64 = rm_hexval(buf[at + h] as i64); if d < 0 { return 0 - 1 } v = v * RM_HEX + d; h = h + 1 }
257 return v
258}
259// decode the JSON string whose OPENING quote is at buf[at] into out (UTF-8, all escapes incl. \uXXXX and
260// surrogate pairs). Returns the decoded byte length; end_out[0] = offset just past the closing quote.
261// -1 malformed or over cap (the caller announces; nothing is silently cut).
262func rm_str(buf: *u8, n: i64, at: i64, out: *u8, cap: i64, end_out: *i64) -> i64 {
263 end_out[0] = 0 - 1
264 if at < 0 { return 0 - 1 }
265 if at >= n { return 0 - 1 }
266 if (buf[at] as i64) != RM_Q { return 0 - 1 }
267 var i: i64 = at + 1
268 var o: i64 = 0
269 var done: i64 = 0
270 while done == 0 {
271 if i >= n { return 0 - 1 }
272 let c: i64 = buf[i] as i64
273 if c == RM_Q { done = 1; i = i + 1 }
274 else {
275 if c == RM_BS {
276 i = i + 1
277 if i >= n { return 0 - 1 }
278 let e: i64 = buf[i] as i64
279 var put: i64 = 0 - 1
280 if e == RM_Q { put = RM_Q }
281 if e == RM_BS { put = RM_BS }
282 if e == RM_CH_SLASH { put = RM_CH_SLASH }
283 if e == RM_CH_n { put = RM_LF }
284 if e == RM_CH_r { put = RM_CR }
285 if e == RM_CH_t { put = RM_TAB }
286 if e == RM_CH_b { put = RM_CH_BACKSPACE }
287 if e == RM_CH_f { put = RM_CH_FORMFEED }
288 if e == RM_CH_u {
289 var cp: i64 = rm_hex4(buf, n, i + 1)
290 if cp < 0 { return 0 - 1 }
291 i = i + RM_HEX4_DIGITS + 1
292 if cp >= RM_UTF16_HI_LO { if cp <= RM_UTF16_HI_HI {
293 // high surrogate: a low surrogate MUST follow as \uDC00..\uDFFF
294 if i + 1 < n { if (buf[i] as i64) == RM_BS { if (buf[i + 1] as i64) == RM_CH_u {
295 let lo: i64 = rm_hex4(buf, n, i + RM_UESC_PREFIX)
296 if lo >= RM_UTF16_LO_LO { if lo <= RM_UTF16_LO_HI {
297 cp = RM_SURROGATE_BASE + ((cp - RM_UTF16_HI_LO) * RM_SURROGATE_SHIFT) + (lo - RM_UTF16_LO_LO)
298 i = i + RM_UESC_LEN
299 } }
300 } } }
301 } }
302 o = rm_utf8_put(cp, out, o, cap)
303 if o < 0 { return 0 - 1 }
304 } else {
305 if put < 0 { return 0 - 1 }
306 if o + 1 > cap { return 0 - 1 }
307 out[o] = put as u8; o = o + 1
308 i = i + 1
309 }
310 } else {
311 if o + 1 > cap { return 0 - 1 }
312 out[o] = c as u8; o = o + 1
313 i = i + 1
314 }
315 }
316 }
317 end_out[0] = i
318 return o
319}
320// integer at buf[at] (optional '-', digits; a quoted integer "123" is accepted too). No digit -> 0.
321func rm_int(buf: *u8, n: i64, at: i64) -> i64 {
322 if at < 0 { return 0 }
323 var i: i64 = at
324 if i < n { if (buf[i] as i64) == RM_Q { i = i + 1 } }
325 var neg: i64 = 0
326 if i < n { if (buf[i] as i64) == RM_MINUS { neg = 1; i = i + 1 } }
327 var v: i64 = 0
328 var go: i64 = 1
329 while go == 1 {
330 if i >= n { go = 0 } else { let c: i64 = buf[i] as i64; if c >= RM_ZERO { if c <= RM_NINE { v = v * RM_DECIMAL + (c - RM_ZERO); i = i + 1 } else { go = 0 } } else { go = 0 } }
331 }
332 if neg == 1 { return 0 - v }
333 return v
334}
335// true -> 1, false -> 0, anything else -> -1
336func rm_bool(buf: *u8, n: i64, at: i64) -> i64 {
337 if at < 0 { return 0 - 1 }
338 if at + RM_TRUE_LEN <= n { if (buf[at] as i64) == RM_CH_t { return 1 } }
339 if at + RM_FALSE_LEN <= n { if (buf[at] as i64) == RM_CH_f { return 0 } }
340 return 0 - 1
341}
342// decimal fraction -> permil (0.5 -> 500, "0.523809552" -> 523, 1 -> 1000). Steam sends this field as a bare
343// number OR a quoted string; both accepted. Extra fractional digits are TRUNCATED (declared: not rounded).
344func rm_permil(buf: *u8, n: i64, at: i64) -> i64 {
345 if at < 0 { return 0 }
346 var i: i64 = at
347 if i < n { if (buf[i] as i64) == RM_Q { i = i + 1 } }
348 var ip: i64 = 0
349 var go: i64 = 1
350 while go == 1 { if i >= n { go = 0 } else { let c: i64 = buf[i] as i64; if c >= RM_ZERO { if c <= RM_NINE { ip = ip * RM_DECIMAL + (c - RM_ZERO); i = i + 1 } else { go = 0 } } else { go = 0 } } }
351 var frac: i64 = 0
352 var nd: i64 = 0
353 if i < n { if (buf[i] as i64) == RM_DOT {
354 i = i + 1
355 go = 1
356 while go == 1 { if i >= n { go = 0 } else { let c2: i64 = buf[i] as i64; if c2 >= RM_ZERO { if c2 <= RM_NINE { if nd < RM_PERMIL_DIGITS { frac = frac * RM_DECIMAL + (c2 - RM_ZERO); nd = nd + 1 } i = i + 1 } else { go = 0 } } else { go = 0 } } }
357 } }
358 while nd < RM_PERMIL_DIGITS { frac = frac * RM_DECIMAL; nd = nd + 1 }
359 return ip * RM_PERMIL + frac
360}
361// literal substring search from `from`; -1 if absent
362func rm_find(buf: *u8, n: i64, lit: *u8, from: i64) -> i64 {
363 let ll: i64 = rm_slen(lit)
364 if ll == 0 { return 0 - 1 }
365 var i: i64 = from
366 if i < 0 { i = 0 }
367 while i + ll <= n {
368 var same: i64 = 1
369 var j: i64 = 0
370 while j < ll { if buf[i + j] != lit[j] { same = 0; j = ll } j = j + 1 }
371 if same == 1 { return i }
372 i = i + 1
373 }
374 return 0 - 1
375}
376// percent-encode every byte outside [A-Za-z0-9-_.~] (RFC 3986 unreserved). Returns the encoded length, -1 over cap.
377func rm_urlenc(src: *u8, n: i64, dst: *u8, cap: i64) -> i64 {
378 var i: i64 = 0; var o: i64 = 0
379 while i < n {
380 let c: i64 = src[i] as i64
381 var keep: i64 = 0
382 if c >= RM_ZERO { if c <= RM_NINE { keep = 1 } }
383 if c >= RM_LOWER_A { if c <= RM_LOWER_Z { keep = 1 } }
384 if c >= RM_UPPER_A { if c <= RM_UPPER_Z { keep = 1 } }
385 if c == RM_MINUS { keep = 1 }
386 if c == RM_UNDERSCORE { keep = 1 }
387 if c == RM_DOT { keep = 1 }
388 if c == RM_TILDE { keep = 1 }
389 if keep == 1 { if o + 1 > cap { return 0 - 1 } dst[o] = c as u8; o = o + 1 }
390 else {
391 if o + RM_PCT_ESC_LEN > cap { return 0 - 1 }
392 dst[o] = RM_PCT as u8
393 let hi: i64 = c >> RM_NIBBLE_SHIFT
394 let lo: i64 = c & RM_NIBBLE_MASK
395 if hi < RM_DECIMAL { dst[o + 1] = (RM_ZERO + hi) as u8 } else { dst[o + 1] = (RM_UPPER_A + hi - RM_DECIMAL) as u8 }
396 if lo < RM_DECIMAL { dst[o + RM_PCT_ESC_LEN - 1] = (RM_ZERO + lo) as u8 } else { dst[o + RM_PCT_ESC_LEN - 1] = (RM_UPPER_A + lo - RM_DECIMAL) as u8 }
397 o = o + RM_PCT_ESC_LEN
398 }
399 i = i + 1
400 }
401 return o
402}
403
404// ---- STEAM appreviews page (store.steampowered.com/appreviews/<appid>?json=1) --------------------------------
405// query_summary -> out[RM_S_*]; returns 1 when the summary object was found, 0 otherwise (a page without it is not a page).
406func rm_steam_summary(buf: *u8, n: i64, out: *i64) -> i64 {
407 var k: i64 = 0
408 while k < RM_S_N { out[k] = 0; k = k + 1 }
409 out[RM_S_SUCCESS] = rm_int(buf, n, rm_key(buf, n, "success" as *u8, 0, n))
410 let qs: i64 = rm_key(buf, n, "query_summary" as *u8, 0, n)
411 if qs < 0 { return 0 }
412 // the summary object ends at its closing brace; bound the per-field reads to it
413 var e: i64 = qs
414 var depth: i64 = 0
415 var go: i64 = 1
416 while go == 1 { if e >= n { go = 0 } else { let c: i64 = buf[e] as i64; if c == RM_LBRACE { depth = depth + 1 } if c == RM_RBRACE { depth = depth - 1; if depth == 0 { go = 0 } } e = e + 1 } }
417 out[RM_S_NUM] = rm_int(buf, n, rm_key(buf, n, "num_reviews" as *u8, qs, e))
418 out[RM_S_SCORE] = rm_int(buf, n, rm_key(buf, n, "review_score" as *u8, qs, e))
419 out[RM_S_POS] = rm_int(buf, n, rm_key(buf, n, "total_positive" as *u8, qs, e))
420 out[RM_S_NEG] = rm_int(buf, n, rm_key(buf, n, "total_negative" as *u8, qs, e))
421 out[RM_S_TOTAL] = rm_int(buf, n, rm_key(buf, n, "total_reviews" as *u8, qs, e))
422 return 1
423}
424// the top-level "cursor" (it sits AFTER the reviews array on the wire, so search from the end of the last
425// review; a review body containing the word cursor cannot be mistaken for it because the key form is exact and
426// the value must be a string). Returns its decoded length into out, -1 absent.
427func rm_steam_cursor(buf: *u8, n: i64, out: *u8, cap: i64) -> i64 {
428 // scan every "cursor" key; the top-level one is the LAST match whose value is a string
429 var best: i64 = 0 - 1
430 var from: i64 = 0
431 var go: i64 = 1
432 while go == 1 {
433 let p: i64 = rm_key(buf, n, "cursor" as *u8, from, n)
434 if p < 0 { go = 0 } else { if (buf[p] as i64) == RM_Q { best = p } from = p + 1 }
435 }
436 if best < 0 { return 0 - 1 }
437 rm_numbuf()
438 return rm_str(buf, n, best, out, cap, rm_ep_g)
439}
440// offset of the next review record (its "recommendationid" key) at or after `from`; -1 none
441func rm_steam_next(buf: *u8, n: i64, from: i64) -> i64 {
442 let p: i64 = rm_key(buf, n, "recommendationid" as *u8, from, n)
443 if p < 0 { return 0 - 1 }
444 return p
445}
446// decode ONE review whose recommendationid VALUE starts at `at`; the record spans [at, end). rec = *i64[RM_F_N],
447// lang = *u8[RM_LANG_CAP], text = *u8[RM_TEXT_CAP]. Returns 1 decoded, 0 when the text could not be decoded
448// (malformed or over the cap -- announced by the caller, never silently truncated).
449func rm_steam_review(buf: *u8, n: i64, at: i64, end: i64, rec: *i64, lang: *u8, text: *u8) -> i64 {
450 var k: i64 = 0
451 while k < RM_F_N { rec[k] = 0; k = k + 1 }
452 lang[0] = 0 as u8
453 text[0] = 0 as u8
454 rec[RM_F_RECID] = rm_int(buf, n, at)
455 rec[RM_F_STEAMID] = rm_int(buf, n, rm_key(buf, n, "steamid" as *u8, at, end))
456 rec[RM_F_GAMES_OWNED] = rm_int(buf, n, rm_key(buf, n, "num_games_owned" as *u8, at, end))
457 rec[RM_F_NUM_REVIEWS] = rm_int(buf, n, rm_key(buf, n, "num_reviews" as *u8, at, end))
458 rec[RM_F_PT_FOREVER] = rm_int(buf, n, rm_key(buf, n, "playtime_forever" as *u8, at, end))
459 rec[RM_F_PT_REVIEW] = rm_int(buf, n, rm_key(buf, n, "playtime_at_review" as *u8, at, end))
460 rec[RM_F_TS_C] = rm_int(buf, n, rm_key(buf, n, "timestamp_created" as *u8, at, end))
461 rec[RM_F_TS_U] = rm_int(buf, n, rm_key(buf, n, "timestamp_updated" as *u8, at, end))
462 rec[RM_F_VOTES_UP] = rm_int(buf, n, rm_key(buf, n, "votes_up" as *u8, at, end))
463 rec[RM_F_VOTES_FUNNY] = rm_int(buf, n, rm_key(buf, n, "votes_funny" as *u8, at, end))
464 rec[RM_F_WVS] = rm_permil(buf, n, rm_key(buf, n, "weighted_vote_score" as *u8, at, end))
465 rec[RM_F_COMMENTS] = rm_int(buf, n, rm_key(buf, n, "comment_count" as *u8, at, end))
466 var up: i64 = rm_bool(buf, n, rm_key(buf, n, "voted_up" as *u8, at, end))
467 if up < 0 { up = 0 }
468 rec[RM_F_UP] = up
469 var flags: i64 = 0
470 if rm_bool(buf, n, rm_key(buf, n, "steam_purchase" as *u8, at, end)) == 1 { flags = flags + RM_FLAG_STEAM_PURCHASE }
471 if rm_bool(buf, n, rm_key(buf, n, "received_for_free" as *u8, at, end)) == 1 { flags = flags + RM_FLAG_FREE }
472 if rm_bool(buf, n, rm_key(buf, n, "written_during_early_access" as *u8, at, end)) == 1 { flags = flags + RM_FLAG_EARLY_ACCESS }
473 if rm_bool(buf, n, rm_key(buf, n, "primarily_steam_deck" as *u8, at, end)) == 1 { flags = flags + RM_FLAG_DECK }
474 if rm_bool(buf, n, rm_key(buf, n, "refunded" as *u8, at, end)) == 1 { flags = flags + RM_FLAG_REFUNDED }
475 rec[RM_F_FLAGS] = flags
476 rm_numbuf()
477 let ep: *i64 = rm_ep_g
478 let lp: i64 = rm_key(buf, n, "language" as *u8, at, end)
479 var ll: i64 = 0
480 if lp >= 0 { ll = rm_str(buf, n, lp, lang, RM_LANG_CAP - 1, ep) }
481 if ll < 0 { ll = 0 }
482 lang[ll] = 0 as u8
483 let tp: i64 = rm_key(buf, n, "review" as *u8, at, end)
484 if tp < 0 { return 0 }
485 let tl: i64 = rm_str(buf, n, tp, text, RM_TEXT_CAP - 1, ep)
486 if tl < 0 { return 0 }
487 text[tl] = 0 as u8
488 rec[RM_F_TEXTLEN] = tl
489 return 1
490}
491
492// ---- the journal row codec ----------------------------------------------------------------------------------
493// emit one row into `row` (cap RM_ROW_CAP); returns its length (LF included), -1 when it would not fit.
494func rm_row_emit(rec: *i64, lang: *u8, text: *u8, row: *u8, cap: i64) -> i64 {
495 var o: i64 = 0
496 row[o] = RM_CH_r as u8; o = o + 1
497 var f: i64 = 0
498 while f < RM_F_TEXTLEN {
499 row[o] = RM_PIPE as u8; o = o + 1
500 o = rm_cati(row, o, rec[f])
501 if f == RM_F_STEAMID { row[o] = RM_PIPE as u8; o = o + 1; o = rm_cat(row, o, lang) }
502 f = f + 1
503 if o + RM_NUMBUF >= cap { return 0 - 1 }
504 }
505 row[o] = RM_PIPE as u8; o = o + 1
506 o = rm_cati(row, o, rec[RM_F_TEXTLEN])
507 row[o] = RM_PIPE as u8; o = o + 1
508 let tl: i64 = rec[RM_F_TEXTLEN]
509 var i: i64 = 0
510 while i < tl {
511 if o + RM_ESC_LEN + 1 >= cap { return 0 - 1 }
512 let c: i64 = text[i] as i64
513 if c == RM_BS { row[o] = RM_BS as u8; row[o + 1] = RM_BS as u8; o = o + RM_ESC_LEN }
514 else { if c == RM_PIPE { row[o] = RM_BS as u8; row[o + 1] = RM_CH_p as u8; o = o + RM_ESC_LEN }
515 else { if c == RM_LF { row[o] = RM_BS as u8; row[o + 1] = RM_CH_n as u8; o = o + RM_ESC_LEN }
516 else { if c == RM_CR { row[o] = RM_BS as u8; row[o + 1] = RM_CH_r as u8; o = o + RM_ESC_LEN }
517 else { row[o] = c as u8; o = o + 1 } } } }
518 i = i + 1
519 }
520 row[o] = RM_LF as u8; o = o + 1
521 return o
522}
523// parse one row (row[0..len), LF optional) back into rec/lang/text. Returns 1 ok, 0 malformed.
524func rm_row_parse(row: *u8, len: i64, rec: *i64, lang: *u8, text: *u8) -> i64 {
525 var k: i64 = 0
526 while k < RM_F_N { rec[k] = 0; k = k + 1 }
527 lang[0] = 0 as u8; text[0] = 0 as u8
528 if len < RM_ROW_PREFIX_LEN { return 0 }
529 if (row[0] as i64) != RM_CH_r { return 0 }
530 if (row[1] as i64) != RM_PIPE { return 0 }
531 var i: i64 = RM_ROW_PREFIX_LEN
532 var field: i64 = 0
533 // fields 0..16: recid steamid lang up pt_review pt_forever ts_c ts_u votes_up votes_funny wvs comments flags games_owned num_reviews textlen
534 var slot: i64 = 0
535 while field < RM_ROW_HDR_FIELDS - 1 {
536 // read until the next pipe
537 var s: i64 = i
538 var go: i64 = 1
539 while go == 1 { if i >= len { return 0 } else { if (row[i] as i64) == RM_PIPE { go = 0 } else { i = i + 1 } } }
540 if field == RM_ROWF_LANG {
541 var ll: i64 = i - s
542 if ll > RM_LANG_CAP - 1 { ll = RM_LANG_CAP - 1 }
543 rm_catn(lang, 0, (row as i64 + s) as *u8, ll)
544 lang[ll] = 0 as u8
545 } else {
546 var neg: i64 = 0
547 var v: i64 = 0
548 var p: i64 = s
549 if p < i { if (row[p] as i64) == RM_MINUS { neg = 1; p = p + 1 } }
550 while p < i { let c: i64 = row[p] as i64; if c >= RM_ZERO { if c <= RM_NINE { v = v * RM_DECIMAL + (c - RM_ZERO) } } p = p + 1 }
551 if neg == 1 { v = 0 - v }
552 rec[slot] = v
553 slot = slot + 1
554 }
555 i = i + 1
556 field = field + 1
557 }
558 // text: unescape to the end of the row (LF excluded)
559 var o: i64 = 0
560 while i < len {
561 let c: i64 = row[i] as i64
562 if c == RM_LF { i = len }
563 else {
564 if c == RM_BS {
565 if i + 1 < len {
566 let e: i64 = row[i + 1] as i64
567 var put: i64 = e
568 if e == RM_CH_p { put = RM_PIPE }
569 if e == RM_CH_n { put = RM_LF }
570 if e == RM_CH_r { put = RM_CR }
571 if o < RM_TEXT_CAP - 1 { text[o] = put as u8; o = o + 1 }
572 i = i + RM_ESC_LEN
573 } else { i = i + 1 }
574 } else {
575 if o < RM_TEXT_CAP - 1 { text[o] = c as u8; o = o + 1 }
576 i = i + 1
577 }
578 }
579 }
580 text[o] = 0 as u8
581 // rec[RM_F_TEXTLEN] was parsed from the header; the decoded length must agree or the row is corrupt
582 if rec[RM_F_TEXTLEN] != o { return 0 }
583 return 1
584}
585// end (exclusive, LF excluded) of the line starting at `from`; returns the LF offset or n
586func rm_line_end(buf: *u8, n: i64, from: i64) -> i64 {
587 var e: i64 = from
588 while e < n { if (buf[e] as i64) == RM_LF { return e } e = e + 1 }
589 return n
590}
591
592// ---- files and directories ----------------------------------------------------------------------------------
593func rm_mkdir(path: *u8) -> i64 { sys_mkdir(path, RM_MODE_DIR); return 0 }
594// mkdir -p over a NUL-terminated path (each '/' prefix); the final component too
595func rm_mkdirp(path: *u8) -> i64 {
596 let n: i64 = rm_slen(path)
597 let tmp: *u8 = sys_mmap(n + RM_STR_TAIL)
598 var i: i64 = 0
599 while i < n {
600 if (path[i] as i64) == RM_CH_SLASH { if i > 0 { rm_catn(tmp, 0, path, i); tmp[i] = 0 as u8; rm_mkdir(tmp) } }
601 i = i + 1
602 }
603 rm_mkdir(path)
604 return 0
605}
606func rm_write_all(fd: i64, b: *u8, n: i64) -> i64 {
607 var w: i64 = 0
608 while w < n { let k: i64 = sys_write(fd, (b as i64 + w) as *u8, n - w); if k <= 0 { return 0 - 1 } w = w + k }
609 return n
610}
611// truncate-write a whole file; returns bytes or -1
612func rm_file_put(path: *u8, b: *u8, n: i64) -> i64 {
613 let fd: i64 = sys_openat_wr(path, RM_MODE_FILE)
614 if fd < 0 { return 0 - 1 }
615 let r: i64 = rm_write_all(fd, b, n)
616 sys_close(fd)
617 return r
618}
619// append one buffer under O_APPEND; returns bytes or -1
620func rm_file_append(path: *u8, b: *u8, n: i64) -> i64 {
621 let fd: i64 = sys_openat_append(path, RM_MODE_FILE)
622 if fd < 0 { return 0 - 1 }
623 let r: i64 = rm_write_all(fd, b, n)
624 sys_close(fd)
625 return r
626}
627// ---- APP IDENTITY (intelmine IM27): the platform's own name for an appid, and the banked .name file the proposer reads ----
628// The proposer used to fall back to the appid string when no census row named the title, so a NAS-side run (the census
629// is laptop-side until IM23) published rows whose name column was a number. The store's appdetails endpoint answers
630// {"<appid>":{"success":true,"data":{"type":..,"name":..}}}; rm_json_name takes the FIRST "name" key of that body through
631// the one JSON decoder (rm_key + rm_str, so \u escapes and surrogate pairs land as UTF-8), and rm_name_file_load reads the
632// banked one-line file back. -1 in both is a NAMED absence the caller announces; nothing is silently cut.
633const RM_NAME_CAP: i64 = 256 // a store title; the banked file is one line, name LF
634func rm_json_name(buf: *u8, n: i64, out: *u8, cap: i64) -> i64 {
635 let at: i64 = rm_key(buf, n, "name" as *u8, 0, n)
636 if at < 0 { return 0 - 1 }
637 let ep: *i64 = sys_mmap(RM_I64_PAIR) as *i64
638 let len: i64 = rm_str(buf, n, at, out, cap - 1, ep)
639 if len < 0 { return 0 - 1 }
640 out[len] = 0 as u8
641 return len
642}
643func rm_name_file_load(path: *u8, out: *u8, cap: i64) -> i64 {
644 let lp: *i64 = sys_mmap(RM_I64_PAIR) as *i64
645 lp[0] = 0
646 let b: *u8 = sys_read_file(path, lp)
647 if (b as i64) == 0 { return 0 - 1 }
648 let n: i64 = lp[0]
649 var len: i64 = rm_line_end(b, n, 0)
650 if len > cap - 1 { len = cap - 1 }
651 if len <= 0 { return 0 - 1 }
652 rm_catn(out, 0, b, len)
653 out[len] = 0 as u8
654 return len
655}
656func rm_ld8(b: *u8, off: i64) -> i64 { var v: i64 = 0; var i: i64 = 0; while i < RM_I64_BYTES { v = v | ((b[off + i] as i64) << (i * RM_BITS_PER_BYTE)); i = i + 1 } return v }
657func rm_st8(b: *u8, off: i64, val: i64) -> i64 { var i: i64 = 0; while i < RM_I64_BYTES { b[off + i] = ((val >> (i * RM_BITS_PER_BYTE)) & RM_BYTE_MASK) as u8; i = i + 1 } return 0 }
658
659// ---- the SEEN set: recids already journaled (one file of 8-byte little-endian ints; an in-memory open-addressed
660// set built from it). Idempotency by construction: a re-fetch of the same page appends nothing. -----------------
661static rm_seen_tbl: *i64
662static rm_seen_slots: i64
663static rm_seen_count: i64
664const RM_SEEN_MIN_SLOTS: i64 = 1024
665func rm_seen_slot_of(id: i64) -> i64 {
666 var h: i64 = (id * RM_KNUTH_MULT) & 0x7fffffffffffffff
667 var s: i64 = h % rm_seen_slots
668 var tries: i64 = 0
669 while tries < rm_seen_slots {
670 let v: i64 = rm_seen_tbl[s]
671 if v == 0 { return s }
672 if v == id { return s }
673 s = (s + 1) % rm_seen_slots
674 tries = tries + 1
675 }
676 return 0 - 1
677}
678func rm_seen_insert_mem(id: i64) -> i64 {
679 if id == 0 { return 0 }
680 let s: i64 = rm_seen_slot_of(id)
681 if s < 0 { return 0 - 1 }
682 if rm_seen_tbl[s] == id { return 0 }
683 rm_seen_tbl[s] = id
684 rm_seen_count = rm_seen_count + 1
685 return 1
686}
687func rm_seen_grow() -> i64 {
688 let old: *i64 = rm_seen_tbl
689 let oldn: i64 = rm_seen_slots
690 rm_seen_slots = oldn * RM_GROW_FACTOR
691 rm_seen_tbl = sys_mmap(rm_seen_slots * RM_I64_BYTES) as *i64
692 rm_seen_count = 0
693 var i: i64 = 0
694 while i < oldn { if old[i] != 0 { rm_seen_insert_mem(old[i]) } i = i + 1 }
695 return 0
696}
697// load <path> (absent = empty set). Returns the number of ids loaded.
698func rm_seen_load(path: *u8) -> i64 {
699 let lp: *i64 = sys_mmap(RM_I64_PAIR) as *i64
700 lp[0] = 0
701 let b: *u8 = sys_read_file(path, lp)
702 var cnt: i64 = 0
703 if (b as i64) != 0 { cnt = lp[0] / RM_I64_BYTES }
704 var slots: i64 = RM_SEEN_MIN_SLOTS
705 while slots < (cnt + 1) * RM_GROW_FACTOR { slots = slots * RM_GROW_FACTOR }
706 rm_seen_slots = slots
707 rm_seen_tbl = sys_mmap(slots * RM_I64_BYTES) as *i64
708 rm_seen_count = 0
709 var i: i64 = 0
710 while i < cnt { rm_seen_insert_mem(rm_ld8(b, i * RM_I64_BYTES)); i = i + 1 }
711 return cnt
712}
713func rm_seen_has(id: i64) -> i64 {
714 if (rm_seen_tbl as i64) == 0 { return 0 }
715 let s: i64 = rm_seen_slot_of(id)
716 if s < 0 { return 0 }
717 if rm_seen_tbl[s] == id { return 1 }
718 return 0
719}
720func rm_seen_count_get() -> i64 { return rm_seen_count }
721// add + persist (append 8 bytes). Returns 1 added, 0 already present, -1 persist failed.
722func rm_seen_add(id: i64, path: *u8) -> i64 {
723 if (rm_seen_tbl as i64) == 0 { rm_seen_load(path) }
724 if rm_seen_count * RM_GROW_FACTOR >= rm_seen_slots { rm_seen_grow() }
725 let r: i64 = rm_seen_insert_mem(id)
726 if r != 1 { return 0 }
727 let b: *u8 = sys_mmap(RM_I64_BYTES)
728 rm_st8(b, 0, id)
729 if rm_file_append(path, b, RM_I64_BYTES) < 0 { return 0 - 1 }
730 return 1
731}
732
733// ---- ingest one page: decode every review, journal the unseen ones. Counters out: [decoded, new, dup, undecodable]
734const RM_I_DECODED: i64 = 0
735const RM_I_NEW: i64 = 1
736const RM_I_DUP: i64 = 2
737const RM_I_UNDECODABLE: i64 = 3
738const RM_I_N: i64 = 4
739static rm_rec_g: *i64
740static rm_lang_g: *u8
741static rm_text_g: *u8
742static rm_row_g: *u8
743func rm_scratch() -> i64 {
744 if (rm_rec_g as i64) == 0 { rm_rec_g = sys_mmap(RM_REC_BYTES) as *i64; rm_lang_g = sys_mmap(RM_LANG_CAP); rm_text_g = sys_mmap(RM_TEXT_CAP); rm_row_g = sys_mmap(RM_ROW_CAP) }
745 return 0
746}
747func rm_steam_ingest_page(buf: *u8, n: i64, journal: *u8, seen: *u8, out: *i64) -> i64 {
748 rm_scratch()
749 var k: i64 = 0
750 while k < RM_I_N { out[k] = 0; k = k + 1 }
751 if (rm_seen_tbl as i64) == 0 { rm_seen_load(seen) }
752 var at: i64 = rm_steam_next(buf, n, 0)
753 while at >= 0 {
754 var end: i64 = rm_steam_next(buf, n, at + 1)
755 if end < 0 { end = n }
756 if rm_steam_review(buf, n, at, end, rm_rec_g, rm_lang_g, rm_text_g) == 1 {
757 out[RM_I_DECODED] = out[RM_I_DECODED] + 1
758 let id: i64 = rm_rec_g[RM_F_RECID]
759 if rm_seen_has(id) == 1 { out[RM_I_DUP] = out[RM_I_DUP] + 1 }
760 else {
761 let rl: i64 = rm_row_emit(rm_rec_g, rm_lang_g, rm_text_g, rm_row_g, RM_ROW_CAP)
762 if rl < 0 { out[RM_I_UNDECODABLE] = out[RM_I_UNDECODABLE] + 1 }
763 else {
764 if rm_file_append(journal, rm_row_g, rl) < 0 { return 0 - 1 }
765 rm_seen_add(id, seen)
766 out[RM_I_NEW] = out[RM_I_NEW] + 1
767 }
768 }
769 } else { out[RM_I_UNDECODABLE] = out[RM_I_UNDECODABLE] + 1 }
770 if end >= n { at = 0 - 1 } else { at = end }
771 }
772 return out[RM_I_DECODED]
773}
774
775// ---- STATS over a journal: partitions that SUM ---------------------------------------------------------------
776const RM_ST_ROWS: i64 = 0
777const RM_ST_POS: i64 = 1
778const RM_ST_NEG: i64 = 2
779const RM_ST_EA: i64 = 3
780const RM_ST_FREE: i64 = 4
781const RM_ST_DECK: i64 = 5
782const RM_ST_REFUNDED: i64 = 6
783const RM_ST_PT_LT1H: i64 = 7
784const RM_ST_PT_1_10H: i64 = 8
785const RM_ST_PT_10_100H: i64 = 9
786const RM_ST_PT_GT100H: i64 = 10
787const RM_ST_STEAM_PURCHASE: i64 = 11
788const RM_ST_MALFORMED: i64 = 12
789const RM_ST_TEXT_BYTES: i64 = 13
790const RM_ST_N: i64 = 16
791const RM_MIN_PER_HOUR: i64 = 60
792const RM_PT_BAND_1: i64 = 60 // minutes: under one hour
793const RM_PT_BAND_2: i64 = 600 // ten hours
794const RM_PT_BAND_3: i64 = 6000 // one hundred hours
795const RM_LANG_SLOTS: i64 = 64
796static rm_lang_tbl: *u8
797static rm_lang_cnt: *i64
798static rm_lang_n: i64
799func rm_lang_bump(lang: *u8) -> i64 {
800 if (rm_lang_tbl as i64) == 0 { rm_lang_tbl = sys_mmap(RM_LANG_SLOTS * RM_LANG_CAP); rm_lang_cnt = sys_mmap(RM_LANG_SLOTS * RM_I64_BYTES) as *i64; rm_lang_n = 0 }
801 var i: i64 = 0
802 while i < rm_lang_n { if rm_streq((rm_lang_tbl as i64 + i * RM_LANG_CAP) as *u8, lang) == 1 { rm_lang_cnt[i] = rm_lang_cnt[i] + 1; return i } i = i + 1 }
803 if rm_lang_n >= RM_LANG_SLOTS { return 0 - 1 }
804 let d: *u8 = (rm_lang_tbl as i64 + rm_lang_n * RM_LANG_CAP) as *u8
805 let o: i64 = rm_cat(d, 0, lang)
806 d[o] = 0 as u8
807 rm_lang_cnt[rm_lang_n] = 1
808 rm_lang_n = rm_lang_n + 1
809 return rm_lang_n - 1
810}
811func rm_lang_count() -> i64 { return rm_lang_n }
812func rm_lang_at(i: i64) -> *u8 { return (rm_lang_tbl as i64 + i * RM_LANG_CAP) as *u8 }
813func rm_lang_n_at(i: i64) -> i64 { return rm_lang_cnt[i] }
814// returns rows read; out[RM_ST_*]. The language table is rebuilt per call.
815func rm_stats(journal: *u8, out: *i64) -> i64 {
816 rm_scratch()
817 var k: i64 = 0
818 while k < RM_ST_N { out[k] = 0; k = k + 1 }
819 rm_lang_n = 0
820 let lp: *i64 = sys_mmap(RM_I64_PAIR) as *i64
821 lp[0] = 0
822 let b: *u8 = sys_read_file(journal, lp)
823 if (b as i64) == 0 { return 0 }
824 let n: i64 = lp[0]
825 var i: i64 = 0
826 while i < n {
827 let e: i64 = rm_line_end(b, n, i)
828 if e > i {
829 if rm_row_parse((b as i64 + i) as *u8, e - i, rm_rec_g, rm_lang_g, rm_text_g) == 1 {
830 out[RM_ST_ROWS] = out[RM_ST_ROWS] + 1
831 if rm_rec_g[RM_F_UP] == 1 { out[RM_ST_POS] = out[RM_ST_POS] + 1 } else { out[RM_ST_NEG] = out[RM_ST_NEG] + 1 }
832 let fl: i64 = rm_rec_g[RM_F_FLAGS]
833 if (fl & RM_FLAG_EARLY_ACCESS) != 0 { out[RM_ST_EA] = out[RM_ST_EA] + 1 }
834 if (fl & RM_FLAG_FREE) != 0 { out[RM_ST_FREE] = out[RM_ST_FREE] + 1 }
835 if (fl & RM_FLAG_DECK) != 0 { out[RM_ST_DECK] = out[RM_ST_DECK] + 1 }
836 if (fl & RM_FLAG_REFUNDED) != 0 { out[RM_ST_REFUNDED] = out[RM_ST_REFUNDED] + 1 }
837 if (fl & RM_FLAG_STEAM_PURCHASE) != 0 { out[RM_ST_STEAM_PURCHASE] = out[RM_ST_STEAM_PURCHASE] + 1 }
838 let pt: i64 = rm_rec_g[RM_F_PT_REVIEW]
839 if pt < RM_PT_BAND_1 { out[RM_ST_PT_LT1H] = out[RM_ST_PT_LT1H] + 1 }
840 else { if pt < RM_PT_BAND_2 { out[RM_ST_PT_1_10H] = out[RM_ST_PT_1_10H] + 1 }
841 else { if pt < RM_PT_BAND_3 { out[RM_ST_PT_10_100H] = out[RM_ST_PT_10_100H] + 1 }
842 else { out[RM_ST_PT_GT100H] = out[RM_ST_PT_GT100H] + 1 } } }
843 out[RM_ST_TEXT_BYTES] = out[RM_ST_TEXT_BYTES] + rm_rec_g[RM_F_TEXTLEN]
844 rm_lang_bump(rm_lang_g)
845 } else { out[RM_ST_MALFORMED] = out[RM_ST_MALFORMED] + 1 }
846 }
847 i = e + 1
848 }
849 return out[RM_ST_ROWS]
850}
851
852// ---- the MINER: vocabulary table with per-class document frequency -------------------------------------------
853const RM_VOCAB_SLOTS: i64 = 262144 // power of two; a review corpus of one title stays far under it -- and when
854 // it does not, rm_mine reports coverage_complete=0 rather than dropping terms silently
855const RM_VOCAB_ARENA: i64 = 8650752 // RM_VOCAB_SLOTS * (RM_TOK_MAX + 1)
856const RM_MODE_COMPLAINT: i64 = 0
857const RM_MODE_PRAISE: i64 = 1
858const RM_LAPLACE: i64 = 1
859const RM_CLASS_PRIOR: i64 = 2
860static rm_v_hash: *i64
861static rm_v_off: *i64
862static rm_v_len: *i64
863static rm_v_neg: *i64
864static rm_v_pos: *i64
865static rm_v_stop: *i64
866static rm_v_lastdoc: *i64
867static rm_v_arena: *u8
868static rm_v_arena_used: i64
869static rm_v_n: i64
870static rm_v_full: i64
871static rm_m_docs: i64
872static rm_m_neg_docs: i64
873static rm_m_pos_docs: i64
874static rm_m_skipped_lang: i64
875static rm_m_malformed: i64
876// ---- the RUBRIC (operator 2026-09-05): exceeds / meets / does-not-meet expectations, with FEATURES and DEFECTS split
877// out from the vote, and VALUE cues, every vocabulary a DATA conf (knowledge/reviewmine/lexicon_<class>.conf) ----
878const RM_LEX_DEFECT: i64 = 1
879const RM_LEX_FEATURE: i64 = 2
880const RM_LEX_EXCEED: i64 = 4
881const RM_LEX_NOTMEET: i64 = 8
882const RM_LEX_VALUE: i64 = 16
883const RM_LEX_DEMAND: i64 = 32
884const RM_TIER_EXCEEDS: i64 = 0
885const RM_TIER_MEETS: i64 = 1
886const RM_TIER_MIXED: i64 = 2 // a positive vote carrying an explicit not-met cue (refund, disappointed...): declared, never folded into MEETS
887const RM_TIER_DNM: i64 = 3
888const RM_TIER_N: i64 = 4
889const RM_RANK_DEFECT: i64 = 0
890const RM_RANK_FEATURE: i64 = 1
891// Steam refunds a purchase played under two hours (the store's published refund rule): a negative review inside that
892// window is a refund in waiting, so the window is reported as its own signal. Published number, not a tuning knob.
893const RM_REFUND_WINDOW_MIN: i64 = 120
894const RM_BAND_N: i64 = 4
895const RM_DOC_SLOT_CAP: i64 = 4096 // distinct terms one review can contribute; a longer review is counted truncated=1, never dropped
896static rm_v_lex: *i64
897static rm_v_tier: *i64 // RM_VOCAB_SLOTS * RM_TIER_N document counts
898static rm_v_dem: *i64 // FEATURE terms co-occurring with a DEMAND cue
899static rm_r_tier: *i64 // docs per tier
900static rm_r_value: *i64 // docs with a VALUE cue per tier
901static rm_r_band_docs: *i64
902static rm_r_band_pos: *i64
903static rm_r_rw_docs: i64
904static rm_r_rw_pos: i64
905static rm_r_docs: i64
906static rm_r_truncated: i64
907static rm_r_slots: *i64 // per-doc distinct slot list
908func rm_vocab_reset() -> i64 {
909 if (rm_v_hash as i64) == 0 {
910 rm_v_hash = sys_mmap(RM_VOCAB_SLOTS * RM_I64_BYTES) as *i64; rm_v_off = sys_mmap(RM_VOCAB_SLOTS * RM_I64_BYTES) as *i64; rm_v_len = sys_mmap(RM_VOCAB_SLOTS * RM_I64_BYTES) as *i64
911 rm_v_neg = sys_mmap(RM_VOCAB_SLOTS * RM_I64_BYTES) as *i64; rm_v_pos = sys_mmap(RM_VOCAB_SLOTS * RM_I64_BYTES) as *i64; rm_v_stop = sys_mmap(RM_VOCAB_SLOTS * RM_I64_BYTES) as *i64
912 rm_v_lastdoc = sys_mmap(RM_VOCAB_SLOTS * RM_I64_BYTES) as *i64; rm_v_arena = sys_mmap(RM_VOCAB_ARENA)
913 } else {
914 var i: i64 = 0
915 while i < RM_VOCAB_SLOTS { rm_v_hash[i] = 0; rm_v_neg[i] = 0; rm_v_pos[i] = 0; rm_v_stop[i] = 0; rm_v_lastdoc[i] = 0; i = i + 1 }
916 }
917 rm_v_arena_used = 0; rm_v_n = 0; rm_v_full = 0
918 rm_m_docs = 0; rm_m_neg_docs = 0; rm_m_pos_docs = 0; rm_m_skipped_lang = 0; rm_m_malformed = 0
919 if (rm_v_lex as i64) == 0 {
920 rm_v_lex = sys_mmap(RM_VOCAB_SLOTS * RM_I64_BYTES) as *i64; rm_v_tier = sys_mmap(RM_VOCAB_SLOTS * RM_TIER_N * RM_I64_BYTES) as *i64
921 rm_v_dem = sys_mmap(RM_VOCAB_SLOTS * RM_I64_BYTES) as *i64; rm_r_tier = sys_mmap(RM_TIER_N * RM_I64_BYTES) as *i64; rm_r_value = sys_mmap(RM_TIER_N * RM_I64_BYTES) as *i64
922 rm_r_band_docs = sys_mmap(RM_BAND_N * RM_I64_BYTES) as *i64; rm_r_band_pos = sys_mmap(RM_BAND_N * RM_I64_BYTES) as *i64; rm_r_slots = sys_mmap(RM_DOC_SLOT_CAP * RM_I64_BYTES) as *i64
923 } else {
924 var j: i64 = 0
925 while j < RM_VOCAB_SLOTS { rm_v_lex[j] = 0; rm_v_dem[j] = 0; var t: i64 = 0; while t < RM_TIER_N { rm_v_tier[j * RM_TIER_N + t] = 0; t = t + 1 } j = j + 1 }
926 }
927 var q: i64 = 0
928 while q < RM_TIER_N { rm_r_tier[q] = 0; rm_r_value[q] = 0; rm_r_band_docs[q] = 0; rm_r_band_pos[q] = 0; q = q + 1 }
929 rm_r_rw_docs = 0; rm_r_rw_pos = 0; rm_r_docs = 0; rm_r_truncated = 0
930 return 0
931}
932// slot for a token (insert when absent); -1 when the table is full (announced via rm_v_full)
933func rm_vocab_slot(tok: *u8, n: i64) -> i64 {
934 let h: i64 = rm_hash(tok, n)
935 var s: i64 = h % RM_VOCAB_SLOTS
936 var tries: i64 = 0
937 while tries < RM_VOCAB_SLOTS {
938 if rm_v_hash[s] == 0 {
939 if rm_v_arena_used + n + 1 > RM_VOCAB_ARENA { rm_v_full = 1; return 0 - 1 }
940 rm_v_hash[s] = h
941 rm_v_off[s] = rm_v_arena_used
942 rm_v_len[s] = n
943 rm_catn(rm_v_arena, rm_v_arena_used, tok, n)
944 rm_v_arena[rm_v_arena_used + n] = 0 as u8
945 rm_v_arena_used = rm_v_arena_used + n + 1
946 rm_v_n = rm_v_n + 1
947 return s
948 }
949 if rm_v_hash[s] == h { if rm_v_len[s] == n {
950 var same: i64 = 1
951 var j: i64 = 0
952 while j < n { if rm_v_arena[rm_v_off[s] + j] != tok[j] { same = 0 } j = j + 1 }
953 if same == 1 { return s }
954 } }
955 s = (s + 1) % RM_VOCAB_SLOTS
956 tries = tries + 1
957 }
958 rm_v_full = 1
959 return 0 - 1
960}
961func rm_vocab_tok(slot: i64) -> *u8 { return (rm_v_arena as i64 + rm_v_off[slot]) as *u8 }
962func rm_vocab_neg(slot: i64) -> i64 { return rm_v_neg[slot] }
963func rm_vocab_pos(slot: i64) -> i64 { return rm_v_pos[slot] }
964func rm_vocab_stop(slot: i64) -> i64 { return rm_v_stop[slot] } // 1 when the slot is a loaded stopword (a phrase boundary for n-gram miners)
965func rm_vocab_size() -> i64 { return rm_v_n }
966func rm_mine_docs() -> i64 { return rm_m_docs }
967func rm_mine_neg_docs() -> i64 { return rm_m_neg_docs }
968func rm_mine_pos_docs() -> i64 { return rm_m_pos_docs }
969func rm_mine_skipped_lang() -> i64 { return rm_m_skipped_lang }
970func rm_mine_malformed() -> i64 { return rm_m_malformed }
971func rm_mine_coverage_complete() -> i64 { if rm_v_full == 1 { return 0 } return 1 }
972// stopwords: one token per line (comments start with a semicolon, so the file can explain itself). Returns the count.
973func rm_stopwords_load(path: *u8) -> i64 {
974 let lp: *i64 = sys_mmap(RM_I64_PAIR) as *i64
975 lp[0] = 0
976 let b: *u8 = sys_read_file(path, lp)
977 if (b as i64) == 0 { return 0 }
978 let n: i64 = lp[0]
979 var i: i64 = 0
980 var cnt: i64 = 0
981 while i < n {
982 let e: i64 = rm_line_end(b, n, i)
983 var s: i64 = i
984 var t: i64 = e
985 if t > s { if (b[t - 1] as i64) == RM_CR { t = t - 1 } }
986 if t > s { if (b[s] as i64) != RM_SEMICOLON {
987 let len: i64 = t - s
988 if len >= 1 { if len <= RM_TOK_MAX {
989 let sl: i64 = rm_vocab_slot((b as i64 + s) as *u8, len)
990 if sl >= 0 { rm_v_stop[sl] = 1; cnt = cnt + 1 }
991 } }
992 } }
993 i = e + 1
994 }
995 return cnt
996}
997// tokenize one review text and bump document frequency for its class (docid distinguishes documents so a term
998// counts once per review). Token bytes: ASCII letters lowercased, digits, and any byte >= 128 (so non-Latin
999// reviews tokenise as UTF-8 byte runs); everything else delimits. Length RM_TOK_MIN..RM_TOK_MAX.
1000// THE ONE TOKENIZER (miner and rubric compose it). Token bytes: ASCII letters lowercased, digits, and any byte >= 128
1001// (so non-Latin reviews tokenise as UTF-8 byte runs); an apostrophe joins (dont, cant) without becoming a byte;
1002// everything else delimits. Returns the next token's length in RM_TOK_MIN..RM_TOK_MAX written to tok, advancing
1003// ip[0]; 0 at end of text. Over-long runs are skipped, never truncated into a different word.
1004func rm_next_token(text: *u8, n: i64, ip: *i64, tok: *u8) -> i64 {
1005 var i: i64 = ip[0]
1006 var tl: i64 = 0
1007 while i <= n {
1008 var c: i64 = 0
1009 if i < n { c = text[i] as i64 }
1010 var is_tok: i64 = 0
1011 if c >= RM_LOWER_A { if c <= RM_LOWER_Z { is_tok = 1 } }
1012 if c >= RM_UPPER_A { if c <= RM_UPPER_Z { is_tok = 1; c = c + RM_CASE_DELTA } }
1013 if c >= RM_ZERO { if c <= RM_NINE { is_tok = 1 } }
1014 if c >= RM_HIGH_BYTE { is_tok = 1 }
1015 if c == RM_APOS { is_tok = RM_TOK_STATE_APOS }
1016 i = i + 1
1017 if is_tok == 1 { if tl < RM_TOK_MAX { tok[tl] = c as u8; tl = tl + 1 } else { tl = RM_TOK_MAX + 1 } }
1018 else { if is_tok == RM_TOK_STATE_APOS { } else {
1019 if tl >= RM_TOK_MIN { if tl <= RM_TOK_MAX { ip[0] = i; return tl } }
1020 tl = 0
1021 } }
1022 }
1023 ip[0] = i
1024 return 0
1025}
1026// bump per-class document frequency for one review's terms (a term counts once per review via lastdoc)
1027func rm_mine_doc(text: *u8, n: i64, up: i64, docid: i64) -> i64 {
1028 rm_numbuf()
1029 let tok: *u8 = rm_tok_g
1030 let ip: *i64 = rm_ep_g
1031 ip[0] = 0
1032 var terms: i64 = 0
1033 var tl: i64 = rm_next_token(text, n, ip, tok)
1034 while tl > 0 {
1035 let s: i64 = rm_vocab_slot(tok, tl)
1036 if s >= 0 { if rm_v_stop[s] == 0 { if rm_v_lastdoc[s] != docid {
1037 rm_v_lastdoc[s] = docid
1038 if up == 1 { rm_v_pos[s] = rm_v_pos[s] + 1 } else { rm_v_neg[s] = rm_v_neg[s] + 1 }
1039 terms = terms + 1
1040 } } }
1041 tl = rm_next_token(text, n, ip, tok)
1042 }
1043 return terms
1044}
1045// mine a journal: lang = "all" or one Steam language code. Returns documents mined.
1046func rm_mine(journal: *u8, lang: *u8) -> i64 {
1047 rm_scratch()
1048 let lp: *i64 = sys_mmap(RM_I64_PAIR) as *i64
1049 lp[0] = 0
1050 let b: *u8 = sys_read_file(journal, lp)
1051 if (b as i64) == 0 { return 0 }
1052 let n: i64 = lp[0]
1053 let all: i64 = rm_streq(lang, "all" as *u8)
1054 var i: i64 = 0
1055 while i < n {
1056 let e: i64 = rm_line_end(b, n, i)
1057 if e > i {
1058 if rm_row_parse((b as i64 + i) as *u8, e - i, rm_rec_g, rm_lang_g, rm_text_g) == 1 {
1059 var take: i64 = all
1060 if take == 0 { take = rm_streq(rm_lang_g, lang) }
1061 if take == 1 {
1062 rm_m_docs = rm_m_docs + 1
1063 if rm_rec_g[RM_F_UP] == 1 { rm_m_pos_docs = rm_m_pos_docs + 1 } else { rm_m_neg_docs = rm_m_neg_docs + 1 }
1064 rm_mine_doc(rm_text_g, rm_rec_g[RM_F_TEXTLEN], rm_rec_g[RM_F_UP], rm_m_docs)
1065 } else { rm_m_skipped_lang = rm_m_skipped_lang + 1 }
1066 } else { rm_m_malformed = rm_m_malformed + 1 }
1067 }
1068 i = e + 1
1069 }
1070 return rm_m_docs
1071}
1072// smoothed class-share ratio in permil for a slot. COMPLAINT: neg share over pos share; PRAISE: the inverse.
1073func rm_vocab_ratio(slot: i64, mode: i64) -> i64 {
1074 let neg: i64 = rm_v_neg[slot] + RM_LAPLACE
1075 let pos: i64 = rm_v_pos[slot] + RM_LAPLACE
1076 let nd: i64 = rm_m_neg_docs + RM_CLASS_PRIOR
1077 let pd: i64 = rm_m_pos_docs + RM_CLASS_PRIOR
1078 if mode == RM_MODE_COMPLAINT { return (neg * RM_PERMIL * pd) / (pos * nd) }
1079 return (pos * RM_PERMIL * nd) / (neg * pd)
1080}
1081// top-k slots by ratio for the mode, requiring the mode's own class df >= min_support. out = *i64[k]. Returns the
1082// number filled (may be < k). Ties break toward the higher class count, then the earlier slot (deterministic).
1083func rm_mine_rank(mode: i64, k: i64, min_support: i64, out: *i64) -> i64 {
1084 var filled: i64 = 0
1085 let taken: *i64 = sys_mmap(RM_VOCAB_SLOTS * RM_I64_BYTES) as *i64
1086 var r: i64 = 0
1087 while r < k {
1088 var best: i64 = 0 - 1
1089 var best_ratio: i64 = 0 - 1
1090 var best_cnt: i64 = 0 - 1
1091 var s: i64 = 0
1092 while s < RM_VOCAB_SLOTS {
1093 if rm_v_hash[s] != 0 { if taken[s] == 0 { if rm_v_stop[s] == 0 {
1094 var cnt: i64 = rm_v_neg[s]
1095 if mode == RM_MODE_PRAISE { cnt = rm_v_pos[s] }
1096 if cnt >= min_support {
1097 let ratio: i64 = rm_vocab_ratio(s, mode)
1098 var better: i64 = 0
1099 if ratio > best_ratio { better = 1 }
1100 if ratio == best_ratio { if cnt > best_cnt { better = 1 } }
1101 if better == 1 { best = s; best_ratio = ratio; best_cnt = cnt }
1102 }
1103 } } }
1104 s = s + 1
1105 }
1106 if best < 0 { r = k } else { taken[best] = 1; out[filled] = best; filled = filled + 1; r = r + 1 }
1107 }
1108 return filled
1109}
1110
1111// ---- lexicons: one token per line (semicolon comments), each conf a CLASS bit on the vocab slot ----
1112func rm_lexicon_load(path: *u8, bit: i64) -> i64 {
1113 let lp: *i64 = sys_mmap(RM_I64_PAIR) as *i64
1114 lp[0] = 0
1115 let b: *u8 = sys_read_file(path, lp)
1116 if (b as i64) == 0 { return 0 }
1117 let n: i64 = lp[0]
1118 var i: i64 = 0
1119 var cnt: i64 = 0
1120 while i < n {
1121 let e: i64 = rm_line_end(b, n, i)
1122 var t: i64 = e
1123 if t > i { if (b[t - 1] as i64) == RM_CR { t = t - 1 } }
1124 if t > i { if (b[i] as i64) != RM_SEMICOLON {
1125 let len: i64 = t - i
1126 if len >= 1 { if len <= RM_TOK_MAX {
1127 let sl: i64 = rm_vocab_slot((b as i64 + i) as *u8, len)
1128 if sl >= 0 { rm_v_lex[sl] = rm_v_lex[sl] | bit; cnt = cnt + 1 }
1129 } }
1130 } }
1131 i = e + 1
1132 }
1133 return cnt
1134}
1135func rm_band_of(pt: i64) -> i64 {
1136 if pt < RM_PT_BAND_1 { return RM_BAND_UNDER_1H }
1137 if pt < RM_PT_BAND_2 { return RM_BAND_1_10H }
1138 if pt < RM_PT_BAND_3 { return RM_BAND_10_100H }
1139 return RM_BAND_OVER_100H
1140}
1141// classify one review and bump the per-term tier counts. Returns the tier.
1142// TIER RULE (pre-declared, data-driven): the vote decides MEETS vs DOES-NOT-MEET; an EXCEED cue lifts a positive vote to
1143// EXCEEDS; a NOT-MET cue on a positive vote makes it MIXED (never silently MEETS). A negative vote is DNM regardless of
1144// praise words, because the reviewer's own verdict outranks a lexicon.
1145func rm_rubric_doc(text: *u8, n: i64, up: i64, pt: i64, docid: i64) -> i64 {
1146 rm_numbuf()
1147 let tok: *u8 = rm_tok_g
1148 let ip: *i64 = rm_ep_g
1149 ip[0] = 0
1150 var nslots: i64 = 0
1151 var hits: i64 = 0 // OR of lexicon bits seen in this doc
1152 var tl: i64 = rm_next_token(text, n, ip, tok)
1153 while tl > 0 {
1154 let s: i64 = rm_vocab_slot(tok, tl)
1155 if s >= 0 { if rm_v_stop[s] == 0 { if rm_v_lastdoc[s] != docid {
1156 rm_v_lastdoc[s] = docid
1157 if nslots < RM_DOC_SLOT_CAP { rm_r_slots[nslots] = s; nslots = nslots + 1 } else { rm_r_truncated = 1 }
1158 hits = hits | rm_v_lex[s]
1159 } } }
1160 tl = rm_next_token(text, n, ip, tok)
1161 }
1162 var tier: i64 = RM_TIER_DNM
1163 if up == 1 {
1164 tier = RM_TIER_MEETS
1165 if (hits & RM_LEX_EXCEED) != 0 { tier = RM_TIER_EXCEEDS }
1166 if (hits & RM_LEX_NOTMEET) != 0 { tier = RM_TIER_MIXED }
1167 }
1168 rm_r_tier[tier] = rm_r_tier[tier] + 1
1169 rm_r_docs = rm_r_docs + 1
1170 if (hits & RM_LEX_VALUE) != 0 { rm_r_value[tier] = rm_r_value[tier] + 1 }
1171 let band: i64 = rm_band_of(pt)
1172 rm_r_band_docs[band] = rm_r_band_docs[band] + 1
1173 if up == 1 { rm_r_band_pos[band] = rm_r_band_pos[band] + 1 }
1174 if pt < RM_REFUND_WINDOW_MIN { rm_r_rw_docs = rm_r_rw_docs + 1; if up == 1 { rm_r_rw_pos = rm_r_rw_pos + 1 } }
1175 var demand: i64 = 0
1176 if (hits & RM_LEX_DEMAND) != 0 { demand = 1 }
1177 var k: i64 = 0
1178 while k < nslots {
1179 let sl: i64 = rm_r_slots[k]
1180 rm_v_tier[sl * RM_TIER_N + tier] = rm_v_tier[sl * RM_TIER_N + tier] + 1
1181 if demand == 1 { if (rm_v_lex[sl] & RM_LEX_FEATURE) != 0 { rm_v_dem[sl] = rm_v_dem[sl] + 1 } }
1182 k = k + 1
1183 }
1184 return tier
1185}
1186// run the rubric over a journal (lang = all | code). Call rm_vocab_reset, load stopwords + lexicons first.
1187func rm_rubric(journal: *u8, lang: *u8) -> i64 {
1188 rm_scratch()
1189 let lp: *i64 = sys_mmap(RM_I64_PAIR) as *i64
1190 lp[0] = 0
1191 let b: *u8 = sys_read_file(journal, lp)
1192 if (b as i64) == 0 { return 0 }
1193 let n: i64 = lp[0]
1194 let all: i64 = rm_streq(lang, "all" as *u8)
1195 var i: i64 = 0
1196 var docid: i64 = 0
1197 while i < n {
1198 let e: i64 = rm_line_end(b, n, i)
1199 if e > i {
1200 if rm_row_parse((b as i64 + i) as *u8, e - i, rm_rec_g, rm_lang_g, rm_text_g) == 1 {
1201 var take: i64 = all
1202 if take == 0 { take = rm_streq(rm_lang_g, lang) }
1203 if take == 1 {
1204 docid = docid + 1
1205 rm_rubric_doc(rm_text_g, rm_rec_g[RM_F_TEXTLEN], rm_rec_g[RM_F_UP], rm_rec_g[RM_F_PT_REVIEW], docid)
1206 } else { rm_m_skipped_lang = rm_m_skipped_lang + 1 }
1207 } else { rm_m_malformed = rm_m_malformed + 1 }
1208 }
1209 i = e + 1
1210 }
1211 return rm_r_docs
1212}
1213func rm_rubric_docs() -> i64 { return rm_r_docs }
1214func rm_rubric_tier(t: i64) -> i64 { return rm_r_tier[t] }
1215func rm_rubric_value(t: i64) -> i64 { return rm_r_value[t] }
1216func rm_rubric_band_docs(b: i64) -> i64 { return rm_r_band_docs[b] }
1217func rm_rubric_band_pos(b: i64) -> i64 { return rm_r_band_pos[b] }
1218func rm_rubric_rw_docs() -> i64 { return rm_r_rw_docs }
1219func rm_rubric_rw_pos() -> i64 { return rm_r_rw_pos }
1220func rm_rubric_truncated() -> i64 { return rm_r_truncated }
1221func rm_vocab_tier(slot: i64, t: i64) -> i64 { return rm_v_tier[slot * RM_TIER_N + t] }
1222func rm_vocab_dem(slot: i64) -> i64 { return rm_v_dem[slot] }
1223func rm_vocab_lex(slot: i64) -> i64 { return rm_v_lex[slot] }
1224func rm_vocab_all_tiers(slot: i64) -> i64 { var t: i64 = 0; var v: i64 = 0; while t < RM_TIER_N { v = v + rm_v_tier[slot * RM_TIER_N + t]; t = t + 1 } return v }
1225func rm_tier_name(t: i64) -> *u8 {
1226 if t == RM_TIER_EXCEEDS { return "EXCEEDS" as *u8 }
1227 if t == RM_TIER_MEETS { return "MEETS" as *u8 }
1228 if t == RM_TIER_MIXED { return "MIXED" as *u8 }
1229 return "DOES-NOT-MEET" as *u8
1230}
1231// the DNM lift of a term in permil: (df_dnm / dnm_docs) / (df_all / docs). 1000 = proportional; higher = concentrated in
1232// does-not-meet reviews, i.e. a defect that drives the negative verdict. Smoothed by one document per class.
1233func rm_vocab_dnm_lift(slot: i64) -> i64 {
1234 let dnm: i64 = rm_v_tier[slot * RM_TIER_N + RM_TIER_DNM] + RM_LAPLACE
1235 let all: i64 = rm_vocab_all_tiers(slot) + RM_LAPLACE
1236 let dd: i64 = rm_r_tier[RM_TIER_DNM] + RM_CLASS_PRIOR
1237 let d: i64 = rm_r_docs + RM_CLASS_PRIOR
1238 return (dnm * RM_PERMIL * d) / (all * dd)
1239}
1240// top-k lexicon terms: DEFECT ranked by dnm+mixed document count then lift; FEATURE ranked by all-tier document count then demand.
1241func rm_rubric_rank(kind: i64, k: i64, min_support: i64, out: *i64) -> i64 {
1242 var want: i64 = RM_LEX_DEFECT
1243 if kind == RM_RANK_FEATURE { want = RM_LEX_FEATURE }
1244 let taken: *i64 = sys_mmap(RM_VOCAB_SLOTS * RM_I64_BYTES) as *i64
1245 var filled: i64 = 0
1246 var r: i64 = 0
1247 while r < k {
1248 var best: i64 = 0 - 1
1249 var best_a: i64 = 0 - 1
1250 var best_b: i64 = 0 - 1
1251 var s: i64 = 0
1252 while s < RM_VOCAB_SLOTS {
1253 if rm_v_hash[s] != 0 { if taken[s] == 0 { if (rm_v_lex[s] & want) != 0 {
1254 var a: i64 = 0
1255 var bsc: i64 = 0
1256 if kind == RM_RANK_DEFECT { a = rm_v_tier[s * RM_TIER_N + RM_TIER_DNM] + rm_v_tier[s * RM_TIER_N + RM_TIER_MIXED]; bsc = rm_vocab_dnm_lift(s) }
1257 else { a = rm_vocab_all_tiers(s); bsc = rm_v_dem[s] }
1258 if a >= min_support {
1259 var better: i64 = 0
1260 if a > best_a { better = 1 }
1261 if a == best_a { if bsc > best_b { better = 1 } }
1262 if better == 1 { best = s; best_a = a; best_b = bsc }
1263 }
1264 } } }
1265 s = s + 1
1266 }
1267 if best < 0 { r = k } else { taken[best] = 1; out[filled] = best; filled = filled + 1; r = r + 1 }
1268 }
1269 return filled
1270}