nx_steam_reviews_gate.nx source
↩ module page · 447 lines · 28728 B
1// nx_steam_reviews_gate.nx -- GATE for the review miner's decision core (/compare/intelmine R0-R3: the Steam page
2// reader, the journal row codec, the dedupe set, the stats partitions and the complaint-vs-praise ranker). No network:
3// the page is PLANTED at runtime (built byte by byte so the fixture carries real JSON escapes -- a quoted quote, a
4// backslash, a newline, a pipe, an e-acute and a surrogate pair -- without any of them in a source literal), journaled
5// into /tmp/nx_steam_reviews_gate/, read back, mined. Every number the teeth check is derived from the planted
6// inputs and printed beside its expectation so an outside adjudicator can recompute it.
7// license_tier: ORIGINAL
8import "nx_syscalls.nx"
9import "nx_reviewmine_lib.nx"
10import "nx_gate_verdict.nx"
11
12const G_DIR: *u8 = "/tmp/nx_steam_reviews_gate"
13const G_PAGE: *u8 = "/tmp/nx_steam_reviews_gate/page.json"
14const G_JRNL: *u8 = "/tmp/nx_steam_reviews_gate/fixture.jrnl"
15const G_SEEN: *u8 = "/tmp/nx_steam_reviews_gate/fixture.seen"
16const G_CORPUS: *u8 = "/tmp/nx_steam_reviews_gate/corpus.jrnl"
17const G_STOP: *u8 = "/tmp/nx_steam_reviews_gate/stop.conf"
18const G_CORPUS2: *u8 = "/tmp/nx_steam_reviews_gate/rubric.jrnl"
19const G_LEX_DEFECT: *u8 = "/tmp/nx_steam_reviews_gate/lex_defect.conf"
20const G_LEX_FEATURE: *u8 = "/tmp/nx_steam_reviews_gate/lex_feature.conf"
21const G_LEX_EXCEED: *u8 = "/tmp/nx_steam_reviews_gate/lex_exceed.conf"
22const G_LEX_NOTMEET: *u8 = "/tmp/nx_steam_reviews_gate/lex_notmeet.conf"
23const G_LEX_VALUE: *u8 = "/tmp/nx_steam_reviews_gate/lex_value.conf"
24const G_LEX_DEMAND: *u8 = "/tmp/nx_steam_reviews_gate/lex_demand.conf"
25const G_RUBRIC_DOCS: i64 = 7
26const G_BROKEN_LIFT: i64 = 1800 // (2+1)*1000*(7+2) / ((2+1)*(3+2))
27const G_CAP: i64 = 65536
28const G_TOTAL: i64 = 211103
29const G_POS: i64 = 208637
30const G_NEG: i64 = 2466
31const G_SCORE: i64 = 9
32const G_NUM: i64 = 3
33const G_R2_VOTES_UP: i64 = 7
34const G_R2_WVS: i64 = 523 // "0.523809552192687988" truncated to permil
35const G_R2_PT: i64 = 30 // minutes -> the under-one-hour band
36const G_R1_PT: i64 = 2381 // minutes -> the 10-100 h band
37const G_R3_PT: i64 = 9000 // minutes -> the over-100 h band
38const G_CURSOR_LEN: i64 = 20
39const G_CURSOR_ENC_LEN: i64 = 24 // two escapes of three bytes each replace two single bytes
40const G_TOP: i64 = 8
41const G_MIN_SUPPORT: i64 = 1
42const G_CRASH_RATIO: i64 = 4000 // (3+1)*1000*(3+2) / ((0+1)*(3+2))
43const G_GAME_RATIO: i64 = 1000 // equal share in both classes
44const G_MUSIC_COMPLAINT_RATIO: i64 = 250
45const G_UTF8_C3: i64 = 195
46const G_UTF8_A9: i64 = 169
47const G_SMALL_CAP: i64 = 4
48// IM27: the platform's own title (appdetails body) and the banked .name file
49const G_NAME: *u8 = "/tmp/nx_steam_reviews_gate/620.name"
50const G_NAME_ABSENT: *u8 = "/tmp/nx_steam_reviews_gate/absent.name"
51const G_NAME_EMPTY: *u8 = "/tmp/nx_steam_reviews_gate/empty.name"
52const G_NAME_LEN: i64 = 16 // Caf + e-acute (2 bytes) + space + quote + Portal + quote + space + 2
53
54func g_q(b: *u8, o: i64) -> i64 { b[o] = RM_Q as u8; return o + 1 }
55func g_bs(b: *u8, o: i64) -> i64 { b[o] = RM_BS as u8; return o + 1 }
56func g_s(b: *u8, o: i64, s: *u8) -> i64 { return rm_cat(b, o, s) }
57// "key": (quoted key + colon)
58func g_k(b: *u8, o: i64, k: *u8) -> i64 { var p: i64 = g_q(b, o); p = g_s(b, p, k); p = g_q(b, p); b[p] = RM_COLON as u8; return p + 1 }
59func g_kv_s(b: *u8, o: i64, k: *u8, v: *u8) -> i64 { var p: i64 = g_k(b, o, k); p = g_q(b, p); p = g_s(b, p, v); p = g_q(b, p); b[p] = 44 as u8; return p + 1 }
60func g_kv_raw(b: *u8, o: i64, k: *u8, v: *u8) -> i64 { var p: i64 = g_k(b, o, k); p = g_s(b, p, v); b[p] = 44 as u8; return p + 1 }
61func g_review(b: *u8, o: i64, recid: *u8, steamid: *u8, pt: *u8, lang: *u8, textbuilder: i64, up: *u8, votes: *u8, wvs: *u8, free: *u8, ea: *u8, deck: *u8, last: i64) -> i64 {
62 var p: i64 = o
63 b[p] = 123 as u8; p = p + 1
64 p = g_kv_s(b, p, "recommendationid" as *u8, recid)
65 p = g_k(b, p, "author" as *u8); b[p] = 123 as u8; p = p + 1
66 p = g_kv_s(b, p, "steamid" as *u8, steamid)
67 p = g_kv_raw(b, p, "num_games_owned" as *u8, "561" as *u8)
68 p = g_kv_raw(b, p, "num_reviews" as *u8, "14" as *u8)
69 p = g_kv_raw(b, p, "playtime_forever" as *u8, pt)
70 p = g_kv_raw(b, p, "playtime_last_two_weeks" as *u8, "266" as *u8)
71 p = g_kv_raw(b, p, "playtime_at_review" as *u8, pt)
72 p = g_k(b, p, "last_played" as *u8); p = g_s(b, p, "1788580422" as *u8)
73 b[p] = 125 as u8; p = p + 1; b[p] = 44 as u8; p = p + 1
74 p = g_kv_s(b, p, "language" as *u8, lang)
75 p = g_k(b, p, "review" as *u8); p = g_q(b, p)
76 if textbuilder == 1 { p = g_s(b, p, "im quite fond of wheatley" as *u8) }
77 if textbuilder == 2 {
78 // crash on start, newline, said quoted broken, pipe, cafe with e-acute, backslash, end -- as JSON escapes on the wire
79 p = g_s(b, p, "crash on start" as *u8); p = g_bs(b, p); p = g_s(b, p, "nsaid " as *u8)
80 p = g_bs(b, p); p = g_q(b, p); p = g_s(b, p, "broken" as *u8); p = g_bs(b, p); p = g_q(b, p)
81 p = g_s(b, p, " | caf" as *u8); p = g_bs(b, p); p = g_s(b, p, "u00e9 " as *u8); p = g_bs(b, p); p = g_bs(b, p); p = g_s(b, p, " end" as *u8)
82 }
83 if textbuilder == 3 {
84 // cyrillic da, a space, and the grinning-face emoji written as a UTF-16 surrogate pair
85 p = g_bs(b, p); p = g_s(b, p, "u0434" as *u8); p = g_bs(b, p); p = g_s(b, p, "u0430 " as *u8)
86 p = g_bs(b, p); p = g_s(b, p, "ud83d" as *u8); p = g_bs(b, p); p = g_s(b, p, "ude00" as *u8)
87 }
88 if textbuilder == 4 {
89 // MALFORMED on purpose: an escape RFC 8259 does not define (backslash q). A string that merely never
90 // closes would be closed by the NEXT quote on the page and decode as junk, which is not the case under test.
91 p = g_s(b, p, "bad " as *u8); p = g_bs(b, p); p = g_s(b, p, "q escape" as *u8)
92 }
93 p = g_q(b, p); b[p] = 44 as u8; p = p + 1
94 p = g_kv_raw(b, p, "timestamp_created" as *u8, "1788580450" as *u8)
95 p = g_kv_raw(b, p, "timestamp_updated" as *u8, "1788580450" as *u8)
96 p = g_kv_raw(b, p, "voted_up" as *u8, up)
97 p = g_kv_raw(b, p, "votes_up" as *u8, votes)
98 p = g_kv_raw(b, p, "votes_funny" as *u8, "0" as *u8)
99 p = g_kv_raw(b, p, "weighted_vote_score" as *u8, wvs)
100 p = g_kv_raw(b, p, "comment_count" as *u8, "0" as *u8)
101 p = g_kv_raw(b, p, "steam_purchase" as *u8, "true" as *u8)
102 p = g_kv_raw(b, p, "received_for_free" as *u8, free)
103 p = g_kv_raw(b, p, "refunded" as *u8, "false" as *u8)
104 p = g_kv_raw(b, p, "written_during_early_access" as *u8, ea)
105 p = g_k(b, p, "primarily_steam_deck" as *u8); p = g_s(b, p, deck)
106 b[p] = 125 as u8; p = p + 1
107 if last == 0 { b[p] = 44 as u8; p = p + 1 }
108 return p
109}
110func g_head(b: *u8, total: *u8, num: *u8) -> i64 {
111 var p: i64 = 0
112 b[p] = 123 as u8; p = p + 1
113 p = g_kv_raw(b, p, "success" as *u8, "1" as *u8)
114 p = g_k(b, p, "query_summary" as *u8); b[p] = 123 as u8; p = p + 1
115 p = g_kv_raw(b, p, "num_reviews" as *u8, num)
116 p = g_kv_raw(b, p, "review_score" as *u8, "9" as *u8)
117 p = g_kv_s(b, p, "review_score_desc" as *u8, "Overwhelmingly Positive" as *u8)
118 p = g_kv_raw(b, p, "total_positive" as *u8, "208637" as *u8)
119 p = g_kv_raw(b, p, "total_negative" as *u8, "2466" as *u8)
120 p = g_k(b, p, "total_reviews" as *u8); p = g_s(b, p, total)
121 b[p] = 125 as u8; p = p + 1; b[p] = 44 as u8; p = p + 1
122 p = g_k(b, p, "reviews" as *u8); b[p] = 91 as u8; p = p + 1
123 return p
124}
125func g_tail(b: *u8, o: i64, cursor: *u8) -> i64 {
126 var p: i64 = o
127 b[p] = 93 as u8; p = p + 1; b[p] = 44 as u8; p = p + 1
128 p = g_k(b, p, "cursor" as *u8); p = g_q(b, p); p = g_s(b, p, cursor); p = g_q(b, p)
129 b[p] = 125 as u8; p = p + 1
130 return p
131}
132func g_build_page(b: *u8) -> i64 {
133 var p: i64 = g_head(b, "211103" as *u8, "3" as *u8)
134 p = g_review(b, p, "234501535" as *u8, "76561197983101720" as *u8, "2381" as *u8, "english" as *u8, 1, "true" as *u8, "0" as *u8, "0.5" as *u8, "false" as *u8, "false" as *u8, "false" as *u8, 0)
135 var w: i64 = p
136 w = g_q(b, w); w = g_s(b, w, "0.523809552192687988" as *u8); w = g_q(b, w)
137 b[w] = 0 as u8
138 let wvs2: *u8 = (b as i64 + p) as *u8
139 // the quoted decimal literal is assembled in the page buffer past p and referenced from there; move it aside
140 let hold: *u8 = sys_mmap(64)
141 rm_cat(hold, 0, wvs2)
142 p = g_review(b, p, "234500550" as *u8, "76561199014742033" as *u8, "30" as *u8, "english" as *u8, 2, "false" as *u8, "7" as *u8, hold, "true" as *u8, "true" as *u8, "false" as *u8, 0)
143 p = g_review(b, p, "234499012" as *u8, "76561199761010013" as *u8, "9000" as *u8, "russian" as *u8, 3, "true" as *u8, "0" as *u8, "0.5" as *u8, "false" as *u8, "false" as *u8, "true" as *u8, 1)
144 p = g_tail(b, p, "AoJ4zrzht6ADdLzF/gY=" as *u8)
145 return p
146}
147func g_build_empty(b: *u8) -> i64 {
148 var p: i64 = g_head(b, "0" as *u8, "0" as *u8)
149 p = g_tail(b, p, "*" as *u8)
150 return p
151}
152func g_build_bad(b: *u8) -> i64 {
153 var p: i64 = g_head(b, "1" as *u8, "1" as *u8)
154 p = g_review(b, p, "900000001" as *u8, "76561190000000001" as *u8, "5" as *u8, "english" as *u8, 4, "true" as *u8, "0" as *u8, "0.5" as *u8, "false" as *u8, "false" as *u8, "false" as *u8, 1)
155 p = g_tail(b, p, "*" as *u8)
156 return p
157}
158// expected decoded text of review 2, built without escapes in any literal
159func g_expect2(b: *u8) -> i64 {
160 var p: i64 = g_s(b, 0, "crash on start" as *u8)
161 b[p] = RM_LF as u8; p = p + 1
162 p = g_s(b, p, "said " as *u8); p = g_q(b, p); p = g_s(b, p, "broken" as *u8); p = g_q(b, p)
163 p = g_s(b, p, " | caf" as *u8); b[p] = G_UTF8_C3 as u8; b[p + 1] = G_UTF8_A9 as u8; p = p + 2
164 b[p] = RM_SP as u8; p = p + 1; p = g_bs(b, p); p = g_s(b, p, " end" as *u8)
165 b[p] = 0 as u8
166 return p
167}
168func g_expect3(b: *u8) -> i64 {
169 b[0] = 208 as u8; b[1] = 180 as u8; b[2] = 208 as u8; b[3] = 176 as u8; b[4] = RM_SP as u8
170 b[5] = 240 as u8; b[6] = 159 as u8; b[7] = 152 as u8; b[8] = 128 as u8; b[9] = 0 as u8
171 return 9
172}
173func g_bytes_eq(a: *u8, b: *u8, n: i64) -> i64 { var i: i64 = 0; while i < n { if a[i] != b[i] { return 0 } i = i + 1 } return 1 }
174func g_file_size(path: *u8) -> i64 { let lp: *i64 = sys_mmap(16) as *i64; lp[0] = 0; let b: *u8 = sys_read_file(path, lp); if (b as i64) == 0 { return 0 } return lp[0] }
175// a planted corpus row: text -> journal (voted_up as given)
176func g_corpus_row(recid: i64, up: i64, text: *u8) -> i64 {
177 let rec: *i64 = sys_mmap(RM_REC_BYTES) as *i64
178 var k: i64 = 0
179 while k < RM_F_N { rec[k] = 0; k = k + 1 }
180 rec[RM_F_RECID] = recid; rec[RM_F_UP] = up; rec[RM_F_TEXTLEN] = rm_slen(text)
181 let row: *u8 = sys_mmap(RM_ROW_CAP)
182 let n: i64 = rm_row_emit(rec, "english" as *u8, text, row, RM_ROW_CAP)
183 return rm_file_append(G_CORPUS, row, n)
184}
185func g_slot_of(tok: *u8) -> i64 { return rm_vocab_slot(tok, rm_slen(tok)) }
186func g_corpus2_row(recid: i64, up: i64, pt: i64, text: *u8) -> i64 {
187 let rec: *i64 = sys_mmap(RM_REC_BYTES) as *i64
188 var k: i64 = 0
189 while k < RM_F_N { rec[k] = 0; k = k + 1 }
190 rec[RM_F_RECID] = recid; rec[RM_F_UP] = up; rec[RM_F_PT_REVIEW] = pt; rec[RM_F_TEXTLEN] = rm_slen(text)
191 let row: *u8 = sys_mmap(RM_ROW_CAP)
192 let n: i64 = rm_row_emit(rec, "english" as *u8, text, row, RM_ROW_CAP)
193 return rm_file_append(G_CORPUS2, row, n)
194}
195func g_put(path: *u8, s: *u8) -> i64 { return rm_file_put(path, s, rm_slen(s)) }
196
197func main() -> i64 {
198 gv_head("=== nx_steam_reviews_gate -- review miner decision core (intelmine R0-R3) ===" as *u8)
199 let c: *i64 = gv_ctr()
200 // ---- setup: scratch dir, empty journals (idempotent: every run starts from nothing) ----
201 rm_mkdirp(G_DIR)
202 let empty: *u8 = sys_mmap(8)
203 rm_file_put(G_JRNL, empty, 0); rm_file_put(G_SEEN, empty, 0); rm_file_put(G_CORPUS, empty, 0)
204 let page: *u8 = sys_mmap(G_CAP)
205 let pn: i64 = g_build_page(page)
206 rm_file_put(G_PAGE, page, pn)
207 gv_check("fixture-reached-the-condition-page-has-bytes" as *u8, (pn > 500) as i64, c)
208
209 // ---- summary ----
210 let sum: *i64 = sys_mmap(RM_S_N * 8) as *i64
211 gv_check("summary-object-found" as *u8, rm_steam_summary(page, pn, sum), c)
212 gv_check_eq("summary-total-reviews" as *u8, sum[RM_S_TOTAL], G_TOTAL, c)
213 gv_check_eq("summary-total-positive" as *u8, sum[RM_S_POS], G_POS, c)
214 gv_check_eq("summary-total-negative" as *u8, sum[RM_S_NEG], G_NEG, c)
215 gv_check_eq("summary-review-score" as *u8, sum[RM_S_SCORE], G_SCORE, c)
216 gv_check_eq("summary-num-reviews-on-page" as *u8, sum[RM_S_NUM], G_NUM, c)
217
218 // ---- IM27: the platform's own title from an appdetails body (rm_json_name) and the banked .name file ----
219 let ad: *u8 = "{\x22620\x22:{\x22success\x22:true,\x22data\x22:{\x22type\x22:\x22game\x22,\x22name\x22:\x22Caf\x5cu00e9 \x5c\x22Portal\x5c\x22 2\x22,\x22steam_appid\x22:620}}}" as *u8
220 let nm: *u8 = sys_mmap(RM_NAME_CAP)
221 let nl: i64 = rm_json_name(ad, rm_slen(ad), nm, RM_NAME_CAP)
222 gv_check_eq("appdetails-name-decoded-length-utf8" as *u8, nl, G_NAME_LEN, c)
223 gv_check("appdetails-name-decodes-unicode-and-escaped-quotes" as *u8, rm_streq(nm, "Caf\xc3\xa9 \x22Portal\x22 2" as *u8), c)
224 let nod: *u8 = "{\x22620\x22:{\x22success\x22:false}}" as *u8
225 gv_check_eq("neg-control-appdetails-without-name-key-minus-one" as *u8, rm_json_name(nod, rm_slen(nod), nm, RM_NAME_CAP), 0 - 1, c)
226 let ovr: *u8 = sys_mmap(G_SMALL_CAP)
227 gv_check_eq("neg-control-appdetails-name-over-cap-minus-one-never-cut" as *u8, rm_json_name(ad, rm_slen(ad), ovr, G_SMALL_CAP), 0 - 1, c)
228 let nline: *u8 = "Caf\xc3\xa9 \x22Portal\x22 2\n" as *u8
229 rm_file_put(G_NAME, nline, rm_slen(nline))
230 let nb: *u8 = sys_mmap(RM_NAME_CAP)
231 gv_check_eq("name-file-roundtrip-length" as *u8, rm_name_file_load(G_NAME, nb, RM_NAME_CAP), G_NAME_LEN, c)
232 gv_check("name-file-roundtrip-bytes" as *u8, rm_streq(nb, nm), c)
233 sys_unlinkat(G_NAME_ABSENT)
234 gv_check_eq("neg-control-name-file-absent-minus-one" as *u8, rm_name_file_load(G_NAME_ABSENT, nb, RM_NAME_CAP), 0 - 1, c)
235 rm_file_put(G_NAME_EMPTY, nline, 0)
236 gv_check_eq("neg-control-name-file-empty-minus-one" as *u8, rm_name_file_load(G_NAME_EMPTY, nb, RM_NAME_CAP), 0 - 1, c)
237
238 // ---- cursor + url encoding ----
239 let cur: *u8 = sys_mmap(RM_CURSOR_CAP)
240 let cl: i64 = rm_steam_cursor(page, pn, cur, RM_CURSOR_CAP - 1)
241 gv_check_eq("cursor-decoded-length" as *u8, cl, G_CURSOR_LEN, c)
242 let enc: *u8 = sys_mmap(RM_CURSOR_CAP)
243 let el: i64 = rm_urlenc(cur, cl, enc, RM_CURSOR_CAP - 1)
244 gv_check_eq("cursor-url-encoded-length" as *u8, el, G_CURSOR_ENC_LEN, c)
245 enc[el] = 0 as u8
246 gv_check("cursor-url-encoded-slash-and-equals-escaped" as *u8, rm_streq(enc, "AoJ4zrzht6ADdLzF%2FgY%3D" as *u8), c)
247 let plain: *u8 = "abc-_.~09"
248 let enc2: *u8 = sys_mmap(64)
249 let el2: i64 = rm_urlenc(plain, rm_slen(plain), enc2, 63)
250 enc2[el2] = 0 as u8
251 gv_check("neg-control-unreserved-bytes-pass-through-unencoded" as *u8, rm_streq(enc2, plain), c)
252
253 // ---- ingest: three decoded, three new ----
254 let cnt: *i64 = sys_mmap(RM_I_N * 8) as *i64
255 let dec: i64 = rm_steam_ingest_page(page, pn, G_JRNL, G_SEEN, cnt)
256 gv_check_eq("ingest-decoded-three" as *u8, dec, G_NUM, c)
257 gv_check_eq("ingest-new-three" as *u8, cnt[RM_I_NEW], G_NUM, c)
258 gv_check_eq("ingest-undecodable-zero" as *u8, cnt[RM_I_UNDECODABLE], 0, c)
259 let jsize1: i64 = g_file_size(G_JRNL)
260 gv_check("journal-written" as *u8, (jsize1 > 0) as i64, c)
261
262 // ---- review 2 field decode (the escaped, negative, quoted-decimal one) ----
263 let rec: *i64 = sys_mmap(RM_REC_BYTES) as *i64
264 let lang: *u8 = sys_mmap(RM_LANG_CAP)
265 let text: *u8 = sys_mmap(RM_TEXT_CAP)
266 let r1: i64 = rm_steam_next(page, pn, 0)
267 let r2: i64 = rm_steam_next(page, pn, r1 + 1)
268 let r3: i64 = rm_steam_next(page, pn, r2 + 1)
269 gv_check("fixture-reached-the-condition-three-records-located" as *u8, (r3 > r2) as i64, c)
270 let ok2: i64 = rm_steam_review(page, pn, r2, r3, rec, lang, text)
271 gv_check("review2-decoded" as *u8, ok2, c)
272 gv_check_eq("review2-voted-up-is-negative" as *u8, rec[RM_F_UP], 0, c)
273 gv_check_eq("review2-votes-up" as *u8, rec[RM_F_VOTES_UP], G_R2_VOTES_UP, c)
274 gv_check_eq("review2-quoted-decimal-weighted-vote-permil" as *u8, rec[RM_F_WVS], G_R2_WVS, c)
275 gv_check_eq("review2-flags-free-and-early-access" as *u8, rec[RM_F_FLAGS], RM_FLAG_STEAM_PURCHASE + RM_FLAG_FREE + RM_FLAG_EARLY_ACCESS, c)
276 gv_check_eq("review2-playtime-at-review" as *u8, rec[RM_F_PT_REVIEW], G_R2_PT, c)
277 let ex2: *u8 = sys_mmap(256)
278 let ex2n: i64 = g_expect2(ex2)
279 gv_check_eq("review2-text-length-after-unescape" as *u8, rec[RM_F_TEXTLEN], ex2n, c)
280 gv_check("review2-text-bytes-exact-quote-backslash-newline-pipe-u00e9" as *u8, g_bytes_eq(text, ex2, ex2n), c)
281 gv_check("review2-language-english" as *u8, rm_streq(lang, "english" as *u8), c)
282 // review 3: surrogate pair -> 4-byte UTF-8, deck flag, russian
283 let ok3: i64 = rm_steam_review(page, pn, r3, pn, rec, lang, text)
284 gv_check("review3-decoded" as *u8, ok3, c)
285 let ex3: *u8 = sys_mmap(64)
286 let ex3n: i64 = g_expect3(ex3)
287 gv_check_eq("review3-text-length-surrogate-pair-is-four-bytes" as *u8, rec[RM_F_TEXTLEN], ex3n, c)
288 gv_check("review3-text-bytes-exact-cyrillic-plus-emoji" as *u8, g_bytes_eq(text, ex3, ex3n), c)
289 gv_check_eq("review3-flag-steam-deck" as *u8, rec[RM_F_FLAGS] & RM_FLAG_DECK, RM_FLAG_DECK, c)
290
291 // ---- journal round trip via stats: partitions SUM ----
292 let st: *i64 = sys_mmap(RM_ST_N * 8) as *i64
293 let rows: i64 = rm_stats(G_JRNL, st)
294 gv_check_eq("stats-rows-three" as *u8, rows, G_NUM, c)
295 gv_check_eq("stats-malformed-zero" as *u8, st[RM_ST_MALFORMED], 0, c)
296 gv_check_eq("stats-positive-two" as *u8, st[RM_ST_POS], 2, c)
297 gv_check_eq("stats-negative-one" as *u8, st[RM_ST_NEG], 1, c)
298 gv_check_eq("stats-playtime-bands-sum-to-rows" as *u8, st[RM_ST_PT_LT1H] + st[RM_ST_PT_1_10H] + st[RM_ST_PT_10_100H] + st[RM_ST_PT_GT100H], rows, c)
299 gv_check_eq("stats-playtime-under-one-hour" as *u8, st[RM_ST_PT_LT1H], 1, c)
300 gv_check_eq("stats-playtime-over-hundred-hours" as *u8, st[RM_ST_PT_GT100H], 1, c)
301 var lsum: i64 = 0
302 var li: i64 = 0
303 while li < rm_lang_count() { lsum = lsum + rm_lang_n_at(li); li = li + 1 }
304 gv_check_eq("stats-language-partition-sums-to-rows" as *u8, lsum, rows, c)
305 gv_check_eq("stats-two-languages" as *u8, rm_lang_count(), 2, c)
306
307 // ---- idempotency: the same page again adds nothing ----
308 let dec2: i64 = rm_steam_ingest_page(page, pn, G_JRNL, G_SEEN, cnt)
309 gv_check_eq("reingest-decoded-three" as *u8, dec2, G_NUM, c)
310 gv_check_eq("reingest-new-zero" as *u8, cnt[RM_I_NEW], 0, c)
311 gv_check_eq("reingest-dup-three" as *u8, cnt[RM_I_DUP], G_NUM, c)
312 gv_check_eq("reingest-journal-size-unchanged" as *u8, g_file_size(G_JRNL), jsize1, c)
313 gv_check_eq("seen-set-holds-three" as *u8, rm_seen_count_get(), G_NUM, c)
314
315 // ---- neg-controls: an empty page is not an error; a malformed review is counted, never journaled ----
316 let ep: *u8 = sys_mmap(G_CAP)
317 let en: i64 = g_build_empty(ep)
318 let dec3: i64 = rm_steam_ingest_page(ep, en, G_JRNL, G_SEEN, cnt)
319 gv_check_eq("neg-control-empty-page-decodes-zero-not-error" as *u8, dec3, 0, c)
320 let bp: *u8 = sys_mmap(G_CAP)
321 let bn: i64 = g_build_bad(bp)
322 rm_steam_ingest_page(bp, bn, G_JRNL, G_SEEN, cnt)
323 gv_check_eq("neg-control-invalid-escape-review-is-undecodable" as *u8, cnt[RM_I_UNDECODABLE], 1, c)
324 gv_check_eq("neg-control-invalid-escape-review-adds-no-row" as *u8, cnt[RM_I_NEW], 0, c)
325 gv_check_eq("neg-control-journal-size-still-unchanged" as *u8, g_file_size(G_JRNL), jsize1, c)
326 let small: *u8 = sys_mmap(16)
327 let se: *i64 = sys_mmap(8) as *i64
328 let over: i64 = rm_str(page, pn, rm_key(page, pn, "review_score_desc" as *u8, 0, pn), small, G_SMALL_CAP, se)
329 gv_check_eq("neg-control-string-over-cap-refuses-minus-one-not-a-cut" as *u8, over, 0 - 1, c)
330
331 // ---- the ranker on a planted corpus: three complaints say crash, three praises say music, all say game ----
332 g_corpus_row(1, 0, "the game will crash on load" as *u8)
333 g_corpus_row(2, 0, "crash and the save is gone in this game" as *u8)
334 g_corpus_row(3, 0, "game crash again" as *u8)
335 g_corpus_row(4, 1, "the music in this game is lovely" as *u8)
336 g_corpus_row(5, 1, "music and art carry the game" as *u8)
337 g_corpus_row(6, 1, "great game great music" as *u8)
338 rm_file_put(G_STOP, "; planted stopwords\nthe\nand\nthis\n" as *u8, 33)
339 rm_vocab_reset()
340 let sw: i64 = rm_stopwords_load(G_STOP)
341 gv_check_eq("stopwords-loaded-three" as *u8, sw, 3, c)
342 let docs: i64 = rm_mine(G_CORPUS, "english" as *u8)
343 gv_check_eq("mine-docs-six" as *u8, docs, 6, c)
344 gv_check_eq("mine-neg-docs-three" as *u8, rm_mine_neg_docs(), 3, c)
345 gv_check_eq("mine-pos-docs-three" as *u8, rm_mine_pos_docs(), 3, c)
346 let s_crash: i64 = g_slot_of("crash" as *u8)
347 let s_game: i64 = g_slot_of("game" as *u8)
348 let s_music: i64 = g_slot_of("music" as *u8)
349 let s_the: i64 = g_slot_of("the" as *u8)
350 gv_check_eq("crash-counted-once-per-document-not-per-occurrence" as *u8, rm_vocab_neg(s_crash), 3, c)
351 gv_check_eq("game-in-every-document" as *u8, rm_vocab_neg(s_game) + rm_vocab_pos(s_game), 6, c)
352 gv_check_eq("stopword-never-counted" as *u8, rm_vocab_neg(s_the) + rm_vocab_pos(s_the), 0, c)
353 gv_check_eq("complaint-ratio-crash" as *u8, rm_vocab_ratio(s_crash, RM_MODE_COMPLAINT), G_CRASH_RATIO, c)
354 gv_check_eq("complaint-ratio-game-equal-share-reads-1000" as *u8, rm_vocab_ratio(s_game, RM_MODE_COMPLAINT), G_GAME_RATIO, c)
355 gv_check_eq("complaint-ratio-music-below-parity" as *u8, rm_vocab_ratio(s_music, RM_MODE_COMPLAINT), G_MUSIC_COMPLAINT_RATIO, c)
356 let top: *i64 = sys_mmap(G_TOP * 8) as *i64
357 let got: i64 = rm_mine_rank(RM_MODE_COMPLAINT, G_TOP, G_MIN_SUPPORT, top)
358 gv_check("complaint-rank-returned-terms" as *u8, (got > 0) as i64, c)
359 gv_check("complaint-rank-first-is-crash" as *u8, rm_streq(rm_vocab_tok(top[0]), "crash" as *u8), c)
360 let got2: i64 = rm_mine_rank(RM_MODE_PRAISE, G_TOP, G_MIN_SUPPORT, top)
361 gv_check("praise-rank-first-is-music" as *u8, rm_streq(rm_vocab_tok(top[0]), "music" as *u8), c)
362 gv_check_eq("praise-ratio-music-mirrors-complaint-ratio-crash" as *u8, rm_vocab_ratio(s_music, RM_MODE_PRAISE), G_CRASH_RATIO, c)
363 // min_support binds: with support 4 nothing qualifies (no term reaches 4 documents in one class)
364 let got3: i64 = rm_mine_rank(RM_MODE_COMPLAINT, G_TOP, 4, top)
365 gv_check_eq("neg-control-min-support-above-any-count-ranks-nothing" as *u8, got3, 0, c)
366 gv_check_eq("mine-coverage-complete" as *u8, rm_mine_coverage_complete(), 1, c)
367 gv_check("praise-rank-returned-terms" as *u8, (got2 > 0) as i64, c)
368
369 // ---- the RUBRIC on a second planted corpus: 7 reviews, every tier represented, lexicons planted as data ----
370 rm_file_put(G_CORPUS2, empty, 0)
371 g_corpus2_row(11, 1, 3000, "amazing masterpiece the building is great" as *u8)
372 g_corpus2_row(12, 1, 500, "good fun but crash on load, refund if it keeps crashing" as *u8)
373 g_corpus2_row(13, 1, 200, "solid game with coop" as *u8)
374 g_corpus2_row(14, 0, 30, "crash every hour, not worth the price, refund" as *u8)
375 g_corpus2_row(15, 0, 800, "servers broken, needs dedicated servers" as *u8)
376 g_corpus2_row(16, 1, 1000, "great game needs more content" as *u8)
377 g_corpus2_row(17, 0, 90, "amazing masterpiece but broken" as *u8)
378 g_put(G_LEX_DEFECT, "crash\ncrashing\nbroken\n" as *u8)
379 g_put(G_LEX_FEATURE, "building\ncoop\nservers\ncontent\n" as *u8)
380 g_put(G_LEX_EXCEED, "amazing\nmasterpiece\n" as *u8)
381 g_put(G_LEX_NOTMEET, "refund\n" as *u8)
382 g_put(G_LEX_VALUE, "worth\nprice\nrefund\n" as *u8)
383 g_put(G_LEX_DEMAND, "needs\nneed\nshould\n" as *u8)
384 rm_vocab_reset()
385 rm_stopwords_load(G_STOP)
386 var lex: i64 = 0
387 lex = lex + rm_lexicon_load(G_LEX_DEFECT, RM_LEX_DEFECT)
388 lex = lex + rm_lexicon_load(G_LEX_FEATURE, RM_LEX_FEATURE)
389 lex = lex + rm_lexicon_load(G_LEX_EXCEED, RM_LEX_EXCEED)
390 lex = lex + rm_lexicon_load(G_LEX_NOTMEET, RM_LEX_NOTMEET)
391 lex = lex + rm_lexicon_load(G_LEX_VALUE, RM_LEX_VALUE)
392 lex = lex + rm_lexicon_load(G_LEX_DEMAND, RM_LEX_DEMAND)
393 gv_check_eq("rubric-lexicons-loaded-sixteen-terms" as *u8, lex, 16, c)
394 let rdocs: i64 = rm_rubric(G_CORPUS2, "english" as *u8)
395 gv_check_eq("rubric-docs-seven" as *u8, rdocs, G_RUBRIC_DOCS, c)
396 gv_check_eq("rubric-tier-exceeds" as *u8, rm_rubric_tier(RM_TIER_EXCEEDS), 1, c)
397 gv_check_eq("rubric-tier-meets" as *u8, rm_rubric_tier(RM_TIER_MEETS), 2, c)
398 gv_check_eq("rubric-tier-mixed-positive-vote-with-refund-cue" as *u8, rm_rubric_tier(RM_TIER_MIXED), 1, c)
399 gv_check_eq("rubric-tier-does-not-meet" as *u8, rm_rubric_tier(RM_TIER_DNM), 3, c)
400 gv_check_eq("rubric-tiers-partition-sums-to-docs" as *u8, rm_rubric_tier(0) + rm_rubric_tier(1) + rm_rubric_tier(2) + rm_rubric_tier(3), rdocs, c)
401 gv_check_eq("neg-control-praise-words-never-lift-a-negative-vote" as *u8, rm_vocab_tier(g_slot_of("masterpiece" as *u8), RM_TIER_DNM), 1, c)
402 gv_check_eq("rubric-refund-window-docs-under-120min" as *u8, rm_rubric_rw_docs(), 2, c)
403 gv_check_eq("rubric-refund-window-positive-zero" as *u8, rm_rubric_rw_pos(), 0, c)
404 gv_check_eq("rubric-value-cue-in-does-not-meet" as *u8, rm_rubric_value(RM_TIER_DNM), 1, c)
405 gv_check_eq("rubric-value-cue-in-mixed" as *u8, rm_rubric_value(RM_TIER_MIXED), 1, c)
406 gv_check_eq("rubric-bands-partition-sums-to-docs" as *u8, rm_rubric_band_docs(0) + rm_rubric_band_docs(1) + rm_rubric_band_docs(2) + rm_rubric_band_docs(3), rdocs, c)
407 let s_crash2: i64 = g_slot_of("crash" as *u8)
408 let s_broken: i64 = g_slot_of("broken" as *u8)
409 let s_servers: i64 = g_slot_of("servers" as *u8)
410 let s_coop: i64 = g_slot_of("coop" as *u8)
411 gv_check_eq("defect-crash-in-does-not-meet" as *u8, rm_vocab_tier(s_crash2, RM_TIER_DNM), 1, c)
412 gv_check_eq("defect-crash-in-mixed" as *u8, rm_vocab_tier(s_crash2, RM_TIER_MIXED), 1, c)
413 gv_check_eq("defect-broken-dnm-lift" as *u8, rm_vocab_dnm_lift(s_broken), G_BROKEN_LIFT, c)
414 gv_check_eq("feature-servers-demanded-via-needs-cue" as *u8, rm_vocab_dem(s_servers), 1, c)
415 gv_check_eq("feature-coop-praised-not-demanded" as *u8, rm_vocab_dem(s_coop), 0, c)
416 gv_check_eq("feature-coop-in-meets" as *u8, rm_vocab_tier(s_coop, RM_TIER_MEETS), 1, c)
417 let rtop: *i64 = sys_mmap(G_TOP * 8) as *i64
418 let ndef: i64 = rm_rubric_rank(RM_RANK_DEFECT, G_TOP, 1, rtop)
419 gv_check_eq("defect-rank-count-three-terms-present-crash-crashing-broken" as *u8, ndef, 3, c)
420 gv_check("defect-rank-first-is-broken-by-dnm-count-then-lift" as *u8, rm_streq(rm_vocab_tok(rtop[0]), "broken" as *u8), c)
421 let nfeat: i64 = rm_rubric_rank(RM_RANK_FEATURE, G_TOP, 1, rtop)
422 gv_check_eq("feature-rank-count-four-terms-present" as *u8, nfeat, 4, c)
423 gv_check_eq("feature-rank-first-carries-a-demand" as *u8, rm_vocab_dem(rtop[0]), 1, c)
424 gv_check_eq("neg-control-defect-rank-min-support-three-ranks-nothing" as *u8, rm_rubric_rank(RM_RANK_DEFECT, G_TOP, 3, rtop), 0, c)
425 gv_check_eq("rubric-no-doc-slot-truncation" as *u8, rm_rubric_truncated(), 0, c)
426 // tokenizer: an over-long run is skipped, never cut into a different word; the next word still tokenises
427 let long: *u8 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa okay" as *u8
428 let ipb: *i64 = sys_mmap(8) as *i64
429 ipb[0] = 0
430 let tk: *u8 = sys_mmap(RM_TOK_MAX + 2)
431 let tl1: i64 = rm_next_token(long, rm_slen(long), ipb, tk)
432 gv_check_eq("tokenizer-skips-over-long-run-returns-next-word" as *u8, tl1, 4, c)
433 let tl2: i64 = rm_next_token(long, rm_slen(long), ipb, tk)
434 gv_check_eq("tokenizer-end-of-text-returns-zero" as *u8, tl2, 0, c)
435
436 gv_values_head()
437 gv_kv("page_bytes" as *u8, pn)
438 gv_kv("journal_bytes" as *u8, jsize1)
439 gv_kv("vocab_size" as *u8, rm_vocab_size())
440 gv_kv("complaint_terms_ranked" as *u8, got)
441 gv_kv("praise_terms_ranked" as *u8, got2)
442 gv_kv("rubric_docs" as *u8, rdocs)
443 gv_kv("rubric_lexicon_terms" as *u8, lex)
444 gv_kv("rubric_defects_ranked" as *u8, ndef)
445 gv_kv("rubric_features_ranked" as *u8, nfeat)
446 return gv_verdict("nx_steam_reviews_gate" as *u8, c, "the page reader, row codec, dedupe set, stats partitions and ranker are proven on a runtime-planted page carrying real JSON escapes and on a planted six-document corpus, with the empty page, the invalid escape, the over-cap string, an unreachable min_support and a negative vote carrying praise words as the controls; no network, all scratch under /tmp" as *u8)
447}