nx_steam_reviews.nx source
↩ module page · 633 lines · 32749 B
1// nx_steam_reviews.nx -- STEAM REVIEW ADAPTER: the thin I/O program over nx_reviewmine_lib (/compare/reviewmine
2// R0 sr_fetch_pages, R1 rm_steam_ingest_page, R2 rm_stats, R3 rm_mine_rank). Every decision lives in the lib and
3// is gate-proven with planted pages; THIS file only fetches, banks, prints.
4//
5// SOURCE: GET store.steampowered.com/appreviews/<appid>?json=1 (Steamworks doc "User Reviews - Get List"): filter
6// recent|updated|all, language, cursor (pass * first, then the returned cursor URL-ENCODED), num_per_page up to 100,
7// purchase_type all, review_type all; the default excludes off-topic review bombs (filter_offtopic_activity) and
8// this adapter keeps that default so the mined population matches the store's own score population.
9//
10// WHAT IT BANKS (knowledge/reviews/steam/):
11// raw/<appid>/p<N>.json every page body exactly as received (provenance: the bytes we read)
12// <appid>.jrnl one row per review, append-only, dedupe by recommendationid (lib row grammar)
13// <appid>.seen the recommendationids already journaled (8-byte LE ints)
14// <appid>.summary appid|total_reviews|total_positive|total_negative|review_score|fetched_epoch|rows
15//
16// PACING: every request goes through nx_paced_fetch (pf_before/pf_after) -- the ONE pacer the capture path shares
17// (mediaingest R0) -- plus a named inter-page delay from knowledge/reviewmine/steam_page_delay_ms.conf. A 429 is
18// fed to the pacer with its Retry-After and the page is retried ONCE after the pacer's wait; any other non-200
19// stops the run and says why. A truncated response is never ingested; it is announced and the run stops.
20// license_tier: ORIGINAL
21import "nx_syscalls.nx"
22import "nx_x509_trust_store.nx"
23import "nx_trust_store_load_from_certdata.nx"
24import "nx_https_fetch_follow.nx"
25import "nx_paced_fetch.nx"
26import "nx_reviewmine_lib.nx"
27
28const SR_CERTDATA: *u8 = "data/mozilla_certdata.txt"
29const SR_CERT_MAXCERTS: i64 = 512
30const SR_CERT_BUFCAP: i64 = 4194304 // 4 MiB scratch for the CA bundle parse (matches nx_source_health_probe)
31// response reserve: 100 reviews x RM_TEXT_CAP (32 KiB, the derived per-review ceiling) is 3.2 MiB of text; JSON
32// escaping can double it. 8 MiB holds the worst page; a response that fills it is announced TRUNCATED, never ingested.
33const SR_RESP_CAP: i64 = 8388608
34const SR_REDIRECTS: i64 = 6
35const SR_PER_PAGE: i64 = 100
36const SR_HTTP_OK: i64 = 200
37const SR_HTTP_TOOMANY: i64 = 429
38const SR_BASE: *u8 = "knowledge/reviews/steam"
39const SR_RAWDIR: *u8 = "knowledge/reviews/steam/raw"
40const SR_STOPWORDS: *u8 = "knowledge/reviewmine/stopwords.conf"
41const SR_DELAY_CONF: *u8 = "knowledge/reviewmine/steam_page_delay_ms.conf"
42const SR_LEX_DEFECT: *u8 = "knowledge/reviewmine/lexicon_defect.conf"
43const SR_LEX_FEATURE: *u8 = "knowledge/reviewmine/lexicon_feature.conf"
44const SR_LEX_EXCEED: *u8 = "knowledge/reviewmine/lexicon_exceed.conf"
45const SR_LEX_NOTMEET: *u8 = "knowledge/reviewmine/lexicon_notmeet.conf"
46const SR_LEX_VALUE: *u8 = "knowledge/reviewmine/lexicon_value.conf"
47const SR_LEX_DEMAND: *u8 = "knowledge/reviewmine/lexicon_demand.conf"
48// the only published pacing figure for this endpoint is the community downloader's "about 10 reviews per second"
49// (download-steam-reviews README) -- one 100-review page per 10 s at the limit. 1500 ms between pages sits well
50// inside it; the conf row overrides. DATA, named for its purpose, not a tuning knob in code.
51const SR_DEFAULT_DELAY_MS: i64 = 1500
52const SR_PATH_CAP: i64 = 512
53const SR_URL_CAP: i64 = 1024
54const SR_DEFAULT_TOP: i64 = 25
55const SR_DEFAULT_MIN_SUPPORT: i64 = 3
56const SR_STATUS_SLOT: i64 = 8
57const SR_CRLF_LEN: i64 = 2 // one CR LF
58const SR_CRLF2_LEN: i64 = 4 // the blank line that ends an HTTP head
59const SR_MAX_PAGE_ARG: i64 = 100000 // an upper bound on the max_pages argument itself, so a typo cannot loop forever
60const SR_EXIT_USAGE: i64 = 2 // bad verb, bad appid, unreadable page file
61const SR_EXIT_FAIL: i64 = 3 // the trust store could not be loaded
62const SR_CURSOR_FIRST_PAGE: i64 = 42 // the byte * -- Steam's cursor for the first page (getreviews doc)
63const SR_STOP_CAP: i64 = 64 // the stop-reason word
64const SR_ARGC_MIN: i64 = 3 // <verb> <appid>
65const SR_ARG_VERB: i64 = 1 // argv positions
66const SR_ARG_APPID: i64 = 2
67const SR_ARG_FETCH_PAGES: i64 = 3
68const SR_ARG_FETCH_LANG: i64 = 4
69const SR_ARG_FETCH_FULL: i64 = 5
70const SR_ARG_INGEST_FILE: i64 = 3
71const SR_ARG_LANG: i64 = 3 // mine and rubric
72const SR_ARG_TOP: i64 = 4
73const SR_ARG_MIN_SUPPORT: i64 = 5
74// IM27: the platform's own title (Storefront appdetails, filters=basic keeps the body small); banked beside the pages
75const SR_APPDETAILS_URL: *u8 = "https://store.steampowered.com/api/appdetails?filters=basic&appids="
76const SR_APPDETAILS_RAW: *u8 = "appdetails.json"
77
78func sr_stop(stop: *u8, s: *u8) -> i64 { let o: i64 = rm_cat(stop, 0, s); stop[o] = 0 as u8; return o }
79func sr_body_start(buf: *u8, n: i64) -> i64 {
80 var i: i64 = 0
81 while i + SR_CRLF2_LEN <= n {
82 if (buf[i] as i64) == RM_CR { if (buf[i + 1] as i64) == RM_LF { if (buf[i + SR_CRLF_LEN] as i64) == RM_CR { if (buf[i + SR_CRLF_LEN + 1] as i64) == RM_LF { return i + SR_CRLF2_LEN } } } }
83 i = i + 1
84 }
85 return 0
86}
87// Retry-After: <seconds> in the response head (case-insensitive key match on the wire form); 0 when absent
88func sr_retry_after(buf: *u8, n: i64, body: i64) -> i64 {
89 var i: i64 = 0
90 let key: *u8 = "retry-after:"
91 let kl: i64 = rm_slen(key)
92 while i + kl < body {
93 var same: i64 = 1
94 var j: i64 = 0
95 while j < kl {
96 var c: i64 = buf[i + j] as i64
97 if c >= RM_UPPER_A { if c <= RM_UPPER_Z { c = c + RM_CASE_DELTA } }
98 if c != (key[j] as i64) { same = 0 }
99 j = j + 1
100 }
101 if same == 1 { return rm_int(buf, n, i + kl + 1) }
102 i = i + 1
103 }
104 return 0
105}
106func sr_conf_int(path: *u8, dflt: i64) -> i64 {
107 let lp: *i64 = sys_mmap(RM_I64_PAIR) as *i64
108 lp[0] = 0
109 let b: *u8 = sys_read_file(path, lp)
110 if (b as i64) == 0 { return dflt }
111 if lp[0] <= 0 { return dflt }
112 let v: i64 = rm_int(b, lp[0], 0)
113 if v <= 0 { return dflt }
114 return v
115}
116func sr_path(out: *u8, appid: *u8, suffix: *u8) -> i64 {
117 var o: i64 = rm_cat(out, 0, SR_BASE)
118 out[o] = RM_CH_SLASH as u8; o = o + 1
119 o = rm_cat(out, o, appid)
120 o = rm_cat(out, o, suffix)
121 out[o] = 0 as u8
122 return o
123}
124func sr_rawpath(out: *u8, appid: *u8, page: i64) -> i64 {
125 var o: i64 = rm_cat(out, 0, SR_RAWDIR)
126 out[o] = RM_CH_SLASH as u8; o = o + 1
127 o = rm_cat(out, o, appid)
128 out[o] = RM_CH_SLASH as u8; o = o + 1
129 o = rm_cat(out, o, "p" as *u8)
130 o = rm_cati(out, o, page)
131 o = rm_cat(out, o, ".json" as *u8)
132 out[o] = 0 as u8
133 return o
134}
135func sr_url(out: *u8, appid: *u8, lang: *u8, cursor: *u8, clen: i64) -> i64 {
136 var o: i64 = rm_cat(out, 0, "https://store.steampowered.com/appreviews/" as *u8)
137 o = rm_cat(out, o, appid)
138 o = rm_cat(out, o, "?json=1&filter=recent&purchase_type=all&review_type=all&num_per_page=" as *u8)
139 o = rm_cati(out, o, SR_PER_PAGE)
140 o = rm_cat(out, o, "&language=" as *u8)
141 o = rm_cat(out, o, lang)
142 o = rm_cat(out, o, "&cursor=" as *u8)
143 let e: i64 = rm_urlenc(cursor, clen, (out as i64 + o) as *u8, SR_URL_CAP - o - 1)
144 if e < 0 { return 0 - 1 }
145 o = o + e
146 out[o] = 0 as u8
147 return o
148}
149func sr_usage() -> i64 {
150 rm_w("usage: nx_steam_reviews fetch <appid> [max_pages(0=until-empty)] [language=all] [full]\n" as *u8)
151 rm_w(" nx_steam_reviews ingest <appid> <pagefile.json> offline: journal a banked page\n" as *u8)
152 rm_w(" nx_steam_reviews stats <appid> partitions over the journal (they sum)\n" as *u8)
153 rm_w(" nx_steam_reviews mine <appid> [language=english] [top=25] [min_support=3]\n" as *u8)
154 rm_w(" nx_steam_reviews rubric <appid> [language=english] [top=25] [min_support=3] exceeds/meets/mixed/does-not-meet + defects + features + value\n" as *u8)
155 rm_w(" nx_steam_reviews name <appid> the platform's own title (appdetails) -> <appid>.name, read by the proposer when no census names it\n" as *u8)
156 rm_w(" fetch stops after the first page that adds nothing new unless `full` is given (incremental by default)\n" as *u8)
157 return SR_EXIT_USAGE
158}
159func sr_print_ingest(cnt: *i64) -> i64 {
160 rm_w(" decoded=" as *u8); rm_wn(cnt[RM_I_DECODED])
161 rm_w(" new=" as *u8); rm_wn(cnt[RM_I_NEW])
162 rm_w(" dup=" as *u8); rm_wn(cnt[RM_I_DUP])
163 rm_w(" undecodable=" as *u8); rm_wn(cnt[RM_I_UNDECODABLE])
164 return 0
165}
166func sr_write_summary(appid: *u8, sum: *i64, rows: i64) -> i64 {
167 let p: *u8 = sys_mmap(SR_PATH_CAP)
168 sr_path(p, appid, ".summary" as *u8)
169 let b: *u8 = sys_mmap(SR_PATH_CAP)
170 var o: i64 = rm_cat(b, 0, appid)
171 b[o] = RM_PIPE as u8; o = o + 1; o = rm_cati(b, o, sum[RM_S_TOTAL])
172 b[o] = RM_PIPE as u8; o = o + 1; o = rm_cati(b, o, sum[RM_S_POS])
173 b[o] = RM_PIPE as u8; o = o + 1; o = rm_cati(b, o, sum[RM_S_NEG])
174 b[o] = RM_PIPE as u8; o = o + 1; o = rm_cati(b, o, sum[RM_S_SCORE])
175 b[o] = RM_PIPE as u8; o = o + 1; o = rm_cati(b, o, sys_now_realtime_sec())
176 b[o] = RM_PIPE as u8; o = o + 1; o = rm_cati(b, o, rows)
177 b[o] = RM_LF as u8; o = o + 1
178 return rm_file_put(p, b, o)
179}
180
181// IM27: the platform's own name for the appid. One paced GET of the Storefront appdetails body, banked as
182// raw/<appid>/appdetails.json (provenance), the decoded name written to <appid>.name (one line) for the proposer's
183// census miss. Every stop names its reason; an unknown or delisted appid answers success:false with no name key.
184func sr_name(appid: *u8) -> i64 {
185 rm_mkdirp(SR_BASE)
186 let rawdir: *u8 = sys_mmap(SR_PATH_CAP)
187 var ro: i64 = rm_cat(rawdir, 0, SR_RAWDIR)
188 rawdir[ro] = RM_CH_SLASH as u8; ro = ro + 1
189 ro = rm_cat(rawdir, ro, appid)
190 rawdir[ro] = 0 as u8
191 rm_mkdirp(rawdir)
192 let tr: i64 = nx_trust_store_load_from_certdata(SR_CERTDATA, SR_CERT_MAXCERTS, SR_CERT_BUFCAP)
193 if tr <= 0 { rm_w("STEAM-APP-NAME REFUSED reason=trust-store-load-failed path=" as *u8); rm_w(SR_CERTDATA); rm_w("\n" as *u8); return SR_EXIT_FAIL }
194 let store: *TrustStore = tr as *TrustStore
195 let buf: *u8 = sys_mmap(SR_RESP_CAP)
196 let stp: *i64 = sys_mmap(SR_STATUS_SLOT) as *i64
197 let url: *u8 = sys_mmap(SR_URL_CAP)
198 var uo: i64 = rm_cat(url, 0, SR_APPDETAILS_URL)
199 uo = rm_cat(url, uo, appid)
200 url[uo] = 0 as u8
201 let waited: i64 = pf_before(url)
202 stp[0] = 0
203 let n: i64 = nx_https_fetch_follow_best(url, store, buf, SR_RESP_CAP, SR_REDIRECTS, stp)
204 let status: i64 = stp[0]
205 var body: i64 = 0
206 if n > 0 { body = sr_body_start(buf, n) }
207 var ra: i64 = 0
208 if status == SR_HTTP_TOOMANY { ra = sr_retry_after(buf, n, body) }
209 pf_after(url, status, ra)
210 rm_w("STEAM-APP-NAME appid=" as *u8); rm_w(appid)
211 rm_w(" status=" as *u8); rm_wn(status)
212 rm_w(" bytes=" as *u8); rm_wn(n)
213 rm_w(" waited_ms=" as *u8); rm_wn(waited)
214 if status != SR_HTTP_OK { rm_w(" action=STOP reason=http-status\n" as *u8); return 1 }
215 if n >= SR_RESP_CAP - 1 { rm_w(" action=STOP reason=TRUNCATED-response-not-read\n" as *u8); return 1 }
216 let bp: *u8 = (buf as i64 + body) as *u8
217 let bn: i64 = n - body
218 let rawp: *u8 = sys_mmap(SR_PATH_CAP)
219 var po: i64 = rm_cat(rawp, 0, rawdir)
220 rawp[po] = RM_CH_SLASH as u8; po = po + 1
221 po = rm_cat(rawp, po, SR_APPDETAILS_RAW)
222 rawp[po] = 0 as u8
223 let banked: i64 = rm_file_put(rawp, bp, bn)
224 rm_w(" banked=" as *u8); rm_wn(banked)
225 let title: *u8 = sys_mmap(RM_NAME_CAP)
226 let len: i64 = rm_json_name(bp, bn, title, RM_NAME_CAP)
227 if len < 0 { rm_w(" name=ABSENT action=STOP reason=no-name-key (an unknown or delisted appid answers success:false with no data)\n" as *u8); return 1 }
228 let np: *u8 = sys_mmap(SR_PATH_CAP)
229 sr_path(np, appid, ".name" as *u8)
230 let line: *u8 = sys_mmap(RM_NAME_CAP + 1)
231 var lo: i64 = rm_cat(line, 0, title)
232 line[lo] = RM_LF as u8; lo = lo + 1
233 let wrote: i64 = rm_file_put(np, line, lo)
234 rm_w(" name=" as *u8); rm_w(title)
235 rm_w(" file=" as *u8); rm_w(np)
236 rm_w(" wrote=" as *u8); rm_wn(wrote)
237 rm_w("\n" as *u8)
238 if wrote < 0 { return 1 }
239 return 0
240}
241
242func sr_fetch(appid: *u8, max_pages: i64, lang: *u8, full: i64) -> i64 {
243 rm_mkdirp(SR_BASE)
244 let rawdir: *u8 = sys_mmap(SR_PATH_CAP)
245 var ro: i64 = rm_cat(rawdir, 0, SR_RAWDIR)
246 rawdir[ro] = RM_CH_SLASH as u8; ro = ro + 1
247 ro = rm_cat(rawdir, ro, appid)
248 rawdir[ro] = 0 as u8
249 rm_mkdirp(rawdir)
250 let journal: *u8 = sys_mmap(SR_PATH_CAP)
251 let seen: *u8 = sys_mmap(SR_PATH_CAP)
252 sr_path(journal, appid, ".jrnl" as *u8)
253 sr_path(seen, appid, ".seen" as *u8)
254 let known: i64 = rm_seen_load(seen)
255
256 let tr: i64 = nx_trust_store_load_from_certdata(SR_CERTDATA, SR_CERT_MAXCERTS, SR_CERT_BUFCAP)
257 if tr <= 0 { rm_w("STEAM-REVIEWS-FETCH REFUSED reason=trust-store-load-failed path=" as *u8); rm_w(SR_CERTDATA); rm_w("\n" as *u8); return SR_EXIT_FAIL }
258 let store: *TrustStore = tr as *TrustStore
259 let buf: *u8 = sys_mmap(SR_RESP_CAP)
260 let stp: *i64 = sys_mmap(SR_STATUS_SLOT) as *i64
261 let url: *u8 = sys_mmap(SR_URL_CAP)
262 let cursor: *u8 = sys_mmap(RM_CURSOR_CAP)
263 let prev_cursor: *u8 = sys_mmap(RM_CURSOR_CAP)
264 let rawp: *u8 = sys_mmap(SR_PATH_CAP)
265 let sum: *i64 = sys_mmap(RM_S_N * RM_I64_BYTES) as *i64
266 let last_sum: *i64 = sys_mmap(RM_S_N * RM_I64_BYTES) as *i64
267 let cnt: *i64 = sys_mmap(RM_I_N * RM_I64_BYTES) as *i64
268 let delay_ms: i64 = sr_conf_int(SR_DELAY_CONF, SR_DEFAULT_DELAY_MS)
269 cursor[0] = SR_CURSOR_FIRST_PAGE as u8
270 var clen: i64 = 1
271 prev_cursor[0] = 0 as u8
272 var plen: i64 = 0
273 var page: i64 = 0
274 var http_ok: i64 = 0
275 var decoded: i64 = 0
276 var newrows: i64 = 0
277 var dups: i64 = 0
278 var undec: i64 = 0
279 var have_sum: i64 = 0
280 var retried: i64 = 0
281 let stop: *u8 = sys_mmap(SR_STOP_CAP)
282 sr_stop(stop, "until-empty" as *u8)
283 var go: i64 = 1
284 while go == 1 {
285 if max_pages > 0 { if page >= max_pages { sr_stop(stop, "max_pages" as *u8); go = 0 } }
286 if go == 1 {
287 if sr_url(url, appid, lang, cursor, clen) < 0 { sr_stop(stop, "cursor-too-long" as *u8); go = 0 }
288 }
289 if go == 1 {
290 let waited: i64 = pf_before(url)
291 if page > 0 { sys_sleep_ms(delay_ms) }
292 stp[0] = 0
293 let n: i64 = nx_https_fetch_follow_best(url, store, buf, SR_RESP_CAP, SR_REDIRECTS, stp) // the full reach ladder (minimal 1.3 hello -> Chrome hello -> 1.2), the one nx_research_fetch rides
294 let status: i64 = stp[0]
295 var body: i64 = 0
296 if n > 0 { body = sr_body_start(buf, n) }
297 var ra: i64 = 0
298 if status == SR_HTTP_TOOMANY { ra = sr_retry_after(buf, n, body) }
299 pf_after(url, status, ra)
300 page = page + 1
301 rm_w("PAGE n=" as *u8); rm_wn(page)
302 rm_w(" status=" as *u8); rm_wn(status)
303 rm_w(" bytes=" as *u8); rm_wn(n)
304 rm_w(" waited_ms=" as *u8); rm_wn(waited)
305 if status == SR_HTTP_TOOMANY {
306 rm_w(" retry_after_s=" as *u8); rm_wn(ra)
307 if retried == 0 { rm_w(" action=RETRY-ONCE-AFTER-PACER-WAIT\n" as *u8); retried = 1; page = page - 1 }
308 else { rm_w(" action=STOP\n" as *u8); sr_stop(stop, "rate-limited-twice" as *u8); go = 0 }
309 } else {
310 if status != SR_HTTP_OK { rm_w(" action=STOP reason=http-status\n" as *u8); sr_stop(stop, "http-status" as *u8); go = 0 }
311 else {
312 if n >= SR_RESP_CAP - 1 { rm_w(" action=STOP reason=TRUNCATED-response-not-ingested\n" as *u8); sr_stop(stop, "truncated" as *u8); go = 0 }
313 else {
314 http_ok = http_ok + 1
315 retried = 0
316 let bp: *u8 = (buf as i64 + body) as *u8
317 let bn: i64 = n - body
318 // bank the page bytes first (provenance), then decode
319 sr_rawpath(rawp, appid, page)
320 let banked: i64 = rm_file_put(rawp, bp, bn)
321 if rm_steam_summary(bp, bn, sum) == 1 {
322 if sum[RM_S_TOTAL] > 0 { var k: i64 = 0; while k < RM_S_N { last_sum[k] = sum[k]; k = k + 1 } have_sum = 1 }
323 }
324 let dec: i64 = rm_steam_ingest_page(bp, bn, journal, seen, cnt)
325 if dec < 0 { rm_w(" action=STOP reason=journal-append-failed\n" as *u8); sr_stop(stop, "journal-append-failed" as *u8); go = 0 }
326 else {
327 decoded = decoded + cnt[RM_I_DECODED]; newrows = newrows + cnt[RM_I_NEW]; dups = dups + cnt[RM_I_DUP]; undec = undec + cnt[RM_I_UNDECODABLE]
328 sr_print_ingest(cnt)
329 rm_w(" banked=" as *u8); rm_wn(banked)
330 let nc: i64 = rm_steam_cursor(bp, bn, cursor, RM_CURSOR_CAP - 1)
331 if nc < 0 { rm_w(" cursor=ABSENT action=STOP\n" as *u8); sr_stop(stop, "cursor-absent" as *u8); go = 0 }
332 else {
333 cursor[nc] = 0 as u8
334 clen = nc
335 rm_w(" cursor=" as *u8); rm_wb(cursor, nc); rm_w("\n" as *u8)
336 if cnt[RM_I_DECODED] == 0 { sr_stop(stop, "empty-page" as *u8); go = 0 }
337 if go == 1 { if full == 0 { if cnt[RM_I_NEW] == 0 { sr_stop(stop, "no-new-rows-incremental" as *u8); go = 0 } } }
338 if go == 1 { if nc == plen { if rm_streq(cursor, prev_cursor) == 1 { sr_stop(stop, "cursor-repeated" as *u8); go = 0 } } }
339 rm_catn(prev_cursor, 0, cursor, nc); prev_cursor[nc] = 0 as u8; plen = nc
340 }
341 }
342 }
343 }
344 }
345 }
346 }
347 let rows: i64 = rm_seen_count_get()
348 if have_sum == 1 { sr_write_summary(appid, last_sum, rows) }
349 rm_w("STEAM-REVIEWS-FETCH appid=" as *u8); rm_w(appid)
350 rm_w(" lang=" as *u8); rm_w(lang)
351 rm_w(" pages=" as *u8); rm_wn(page)
352 rm_w(" http_ok=" as *u8); rm_wn(http_ok)
353 rm_w(" decoded=" as *u8); rm_wn(decoded)
354 rm_w(" new=" as *u8); rm_wn(newrows)
355 rm_w(" dup=" as *u8); rm_wn(dups)
356 rm_w(" undecodable=" as *u8); rm_wn(undec)
357 rm_w(" known_before=" as *u8); rm_wn(known)
358 rm_w(" rows_now=" as *u8); rm_wn(rows)
359 if have_sum == 1 {
360 rm_w(" total_reviews=" as *u8); rm_wn(last_sum[RM_S_TOTAL])
361 rm_w(" positive=" as *u8); rm_wn(last_sum[RM_S_POS])
362 rm_w(" negative=" as *u8); rm_wn(last_sum[RM_S_NEG])
363 rm_w(" score=" as *u8); rm_wn(last_sum[RM_S_SCORE])
364 } else { rm_w(" total_reviews=UNKNOWN" as *u8) }
365 rm_w(" stop=" as *u8); rm_w(stop)
366 rm_w(" journal=" as *u8); rm_w(journal)
367 rm_w("\n" as *u8)
368 if http_ok == 0 { return 1 }
369 return 0
370}
371
372func sr_ingest(appid: *u8, file: *u8) -> i64 {
373 rm_mkdirp(SR_BASE)
374 let journal: *u8 = sys_mmap(SR_PATH_CAP)
375 let seen: *u8 = sys_mmap(SR_PATH_CAP)
376 sr_path(journal, appid, ".jrnl" as *u8)
377 sr_path(seen, appid, ".seen" as *u8)
378 let lp: *i64 = sys_mmap(RM_I64_PAIR) as *i64
379 lp[0] = 0
380 let b: *u8 = sys_read_file(file, lp)
381 if (b as i64) == 0 { rm_w("STEAM-REVIEWS-INGEST REFUSED reason=cannot-read file=" as *u8); rm_w(file); rm_w("\n" as *u8); return SR_EXIT_USAGE }
382 let n: i64 = lp[0]
383 var body: i64 = 0
384 if n > SR_CRLF2_LEN { if (b[0] as i64) != RM_LBRACE { body = sr_body_start(b, n) } }
385 let cnt: *i64 = sys_mmap(RM_I_N * RM_I64_BYTES) as *i64
386 rm_seen_load(seen)
387 let dec: i64 = rm_steam_ingest_page((b as i64 + body) as *u8, n - body, journal, seen, cnt)
388 rm_w("STEAM-REVIEWS-INGEST appid=" as *u8); rm_w(appid); rm_w(" file=" as *u8); rm_w(file); rm_w(" bytes=" as *u8); rm_wn(n - body)
389 sr_print_ingest(cnt)
390 rm_w(" rows_now=" as *u8); rm_wn(rm_seen_count_get()); rm_w("\n" as *u8)
391 if dec < 0 { return 1 }
392 return 0
393}
394
395func sr_stats(appid: *u8) -> i64 {
396 let journal: *u8 = sys_mmap(SR_PATH_CAP)
397 sr_path(journal, appid, ".jrnl" as *u8)
398 let st: *i64 = sys_mmap(RM_ST_N * RM_I64_BYTES) as *i64
399 let rows: i64 = rm_stats(journal, st)
400 rm_w("STEAM-REVIEWS-STATS appid=" as *u8); rm_w(appid)
401 rm_w(" rows=" as *u8); rm_wn(rows)
402 rm_w(" malformed=" as *u8); rm_wn(st[RM_ST_MALFORMED])
403 rm_w(" positive=" as *u8); rm_wn(st[RM_ST_POS])
404 rm_w(" negative=" as *u8); rm_wn(st[RM_ST_NEG])
405 var pp: i64 = 0
406 if rows > 0 { pp = (st[RM_ST_POS] * RM_PERMIL) / rows }
407 rm_w(" positive_permil=" as *u8); rm_wn(pp)
408 rm_w(" text_bytes=" as *u8); rm_wn(st[RM_ST_TEXT_BYTES])
409 rm_w("\n" as *u8)
410 rm_w(" PLAYTIME-AT-REVIEW lt1h=" as *u8); rm_wn(st[RM_ST_PT_LT1H])
411 rm_w(" 1-10h=" as *u8); rm_wn(st[RM_ST_PT_1_10H])
412 rm_w(" 10-100h=" as *u8); rm_wn(st[RM_ST_PT_10_100H])
413 rm_w(" gt100h=" as *u8); rm_wn(st[RM_ST_PT_GT100H])
414 rm_w(" sum=" as *u8); rm_wn(st[RM_ST_PT_LT1H] + st[RM_ST_PT_1_10H] + st[RM_ST_PT_10_100H] + st[RM_ST_PT_GT100H])
415 rm_w(" of=" as *u8); rm_wn(rows); rm_w("\n" as *u8)
416 rm_w(" FLAGS steam_purchase=" as *u8); rm_wn(st[RM_ST_STEAM_PURCHASE])
417 rm_w(" received_free=" as *u8); rm_wn(st[RM_ST_FREE])
418 rm_w(" early_access=" as *u8); rm_wn(st[RM_ST_EA])
419 rm_w(" steam_deck=" as *u8); rm_wn(st[RM_ST_DECK])
420 rm_w(" refunded=" as *u8); rm_wn(st[RM_ST_REFUNDED])
421 rm_w(" (flags overlap; not a partition)\n" as *u8)
422 let ln: i64 = rm_lang_count()
423 var i: i64 = 0
424 var lsum: i64 = 0
425 rm_w(" LANGUAGES n=" as *u8); rm_wn(ln); rm_w("\n" as *u8)
426 while i < ln {
427 rm_w(" lang=" as *u8); rm_w(rm_lang_at(i)); rm_w(" rows=" as *u8); rm_wn(rm_lang_n_at(i)); rm_w("\n" as *u8)
428 lsum = lsum + rm_lang_n_at(i)
429 i = i + 1
430 }
431 rm_w(" LANGUAGES sum=" as *u8); rm_wn(lsum); rm_w(" of=" as *u8); rm_wn(rows)
432 if lsum == rows { rm_w(" partition=SUMS\n" as *u8) } else { rm_w(" partition=LEAK\n" as *u8) }
433 if rows == 0 { return 1 }
434 return 0
435}
436
437func sr_mine(appid: *u8, lang: *u8, top: i64, min_support: i64) -> i64 {
438 let journal: *u8 = sys_mmap(SR_PATH_CAP)
439 sr_path(journal, appid, ".jrnl" as *u8)
440 rm_vocab_reset()
441 let sw: i64 = rm_stopwords_load(SR_STOPWORDS)
442 if sw == 0 { rm_w(" NOTE stopwords conf absent or empty at " as *u8); rm_w(SR_STOPWORDS); rm_w(" -- function words will rank; add the conf\n" as *u8) }
443 let docs: i64 = rm_mine(journal, lang)
444 rm_w("STEAM-REVIEWS-MINE appid=" as *u8); rm_w(appid)
445 rm_w(" lang=" as *u8); rm_w(lang)
446 rm_w(" docs=" as *u8); rm_wn(docs)
447 rm_w(" neg_docs=" as *u8); rm_wn(rm_mine_neg_docs())
448 rm_w(" pos_docs=" as *u8); rm_wn(rm_mine_pos_docs())
449 rm_w(" skipped_other_lang=" as *u8); rm_wn(rm_mine_skipped_lang())
450 rm_w(" malformed=" as *u8); rm_wn(rm_mine_malformed())
451 rm_w(" vocab=" as *u8); rm_wn(rm_vocab_size())
452 rm_w(" stopwords=" as *u8); rm_wn(sw)
453 rm_w(" min_support=" as *u8); rm_wn(min_support)
454 rm_w(" coverage_complete=" as *u8); rm_wn(rm_mine_coverage_complete())
455 rm_w("\n" as *u8)
456 if docs == 0 { rm_w(" NO-DOCS nothing to mine for this language; run fetch first or pass all\n" as *u8); return 1 }
457 let out: *i64 = sys_mmap((top + 1) * RM_I64_BYTES) as *i64
458 var mode: i64 = RM_MODE_COMPLAINT
459 while mode <= RM_MODE_PRAISE {
460 let got: i64 = rm_mine_rank(mode, top, min_support, out)
461 var r: i64 = 0
462 while r < got {
463 let s: i64 = out[r]
464 if mode == RM_MODE_COMPLAINT { rm_w("COMPLAINT rank=" as *u8) } else { rm_w("PRAISE rank=" as *u8) }
465 rm_wn(r + 1)
466 rm_w(" term=" as *u8); rm_w(rm_vocab_tok(s))
467 rm_w(" neg_docs=" as *u8); rm_wn(rm_vocab_neg(s))
468 rm_w(" pos_docs=" as *u8); rm_wn(rm_vocab_pos(s))
469 rm_w(" ratio_permil=" as *u8); rm_wn(rm_vocab_ratio(s, mode))
470 rm_w("\n" as *u8)
471 r = r + 1
472 }
473 mode = mode + 1
474 }
475 return 0
476}
477
478func sr_lex(path: *u8, bit: i64, label: *u8) -> i64 {
479 let n: i64 = rm_lexicon_load(path, bit)
480 rm_w(" LEXICON " as *u8); rm_w(label); rm_w(" terms=" as *u8); rm_wn(n); rm_w(" conf=" as *u8); rm_w(path)
481 if n == 0 { rm_w(" ABSENT-OR-EMPTY -- this class cannot fire" as *u8) }
482 rm_w("\n" as *u8)
483 return n
484}
485func sr_rubric(appid: *u8, lang: *u8, top: i64, min_support: i64) -> i64 {
486 let journal: *u8 = sys_mmap(SR_PATH_CAP)
487 sr_path(journal, appid, ".jrnl" as *u8)
488 rm_vocab_reset()
489 let sw: i64 = rm_stopwords_load(SR_STOPWORDS)
490 var lexn: i64 = 0
491 lexn = lexn + sr_lex(SR_LEX_DEFECT, RM_LEX_DEFECT, "defect" as *u8)
492 lexn = lexn + sr_lex(SR_LEX_FEATURE, RM_LEX_FEATURE, "feature" as *u8)
493 lexn = lexn + sr_lex(SR_LEX_EXCEED, RM_LEX_EXCEED, "exceed" as *u8)
494 lexn = lexn + sr_lex(SR_LEX_NOTMEET, RM_LEX_NOTMEET, "notmeet" as *u8)
495 lexn = lexn + sr_lex(SR_LEX_VALUE, RM_LEX_VALUE, "value" as *u8)
496 lexn = lexn + sr_lex(SR_LEX_DEMAND, RM_LEX_DEMAND, "demand" as *u8)
497 let docs: i64 = rm_rubric(journal, lang)
498 let pricep: *u8 = sys_mmap(SR_PATH_CAP)
499 sr_path(pricep, appid, ".price" as *u8)
500 let price: i64 = sr_conf_int(pricep, 0)
501 rm_w("STEAM-REVIEWS-RUBRIC appid=" as *u8); rm_w(appid)
502 rm_w(" lang=" as *u8); rm_w(lang)
503 rm_w(" docs=" as *u8); rm_wn(docs)
504 rm_w(" exceeds=" as *u8); rm_wn(rm_rubric_tier(RM_TIER_EXCEEDS))
505 rm_w(" meets=" as *u8); rm_wn(rm_rubric_tier(RM_TIER_MEETS))
506 rm_w(" mixed=" as *u8); rm_wn(rm_rubric_tier(RM_TIER_MIXED))
507 rm_w(" does_not_meet=" as *u8); rm_wn(rm_rubric_tier(RM_TIER_DNM))
508 var tsum: i64 = 0
509 var tt: i64 = 0
510 while tt < RM_TIER_N { tsum = tsum + rm_rubric_tier(tt); tt = tt + 1 }
511 rm_w(" sum=" as *u8); rm_wn(tsum)
512 if tsum == docs { rm_w(" partition=SUMS" as *u8) } else { rm_w(" partition=LEAK" as *u8) }
513 rm_w(" stopwords=" as *u8); rm_wn(sw)
514 rm_w(" lexicon_terms=" as *u8); rm_wn(lexn)
515 rm_w(" skipped_other_lang=" as *u8); rm_wn(rm_mine_skipped_lang())
516 rm_w(" malformed=" as *u8); rm_wn(rm_mine_malformed())
517 rm_w(" doc_slot_truncated=" as *u8); rm_wn(rm_rubric_truncated())
518 rm_w(" coverage_complete=" as *u8); rm_wn(rm_mine_coverage_complete())
519 rm_w("\n" as *u8)
520 if docs == 0 { rm_w(" NO-DOCS nothing to grade for this language; run fetch first or pass all\n" as *u8); return 1 }
521 var t: i64 = 0
522 while t < RM_TIER_N {
523 rm_w(" TIER " as *u8); rm_w(rm_tier_name(t))
524 rm_w(" docs=" as *u8); rm_wn(rm_rubric_tier(t))
525 rm_w(" permil=" as *u8); rm_wn((rm_rubric_tier(t) * RM_PERMIL) / docs)
526 rm_w(" value_mentions=" as *u8); rm_wn(rm_rubric_value(t))
527 rm_w("\n" as *u8)
528 t = t + 1
529 }
530 rm_w(" VALUE price_cents=" as *u8)
531 if price > 0 { rm_wn(price) } else { rm_w("UNKNOWN(write knowledge/reviews/steam/" as *u8); rm_w(appid); rm_w(".price)" as *u8) }
532 rm_w(" refund_window_docs(pt_lt_120min)=" as *u8); rm_wn(rm_rubric_rw_docs())
533 var rwp: i64 = 0
534 if rm_rubric_rw_docs() > 0 { rwp = (rm_rubric_rw_pos() * RM_PERMIL) / rm_rubric_rw_docs() }
535 rm_w(" refund_window_positive_permil=" as *u8); rm_wn(rwp)
536 let pos_all: i64 = rm_rubric_tier(RM_TIER_EXCEEDS) + rm_rubric_tier(RM_TIER_MEETS) + rm_rubric_tier(RM_TIER_MIXED)
537 rm_w(" overall_positive_permil=" as *u8); rm_wn((pos_all * RM_PERMIL) / docs)
538 rm_w("\n" as *u8)
539 var b: i64 = 0
540 var bsum: i64 = 0
541 while b < RM_BAND_N {
542 rm_w(" BAND " as *u8)
543 if b == RM_BAND_UNDER_1H { rm_w("lt1h" as *u8) } else { if b == RM_BAND_1_10H { rm_w("1-10h" as *u8) } else { if b == RM_BAND_10_100H { rm_w("10-100h" as *u8) } else { rm_w("gt100h" as *u8) } } }
544 rm_w(" docs=" as *u8); rm_wn(rm_rubric_band_docs(b))
545 var bp: i64 = 0
546 if rm_rubric_band_docs(b) > 0 { bp = (rm_rubric_band_pos(b) * RM_PERMIL) / rm_rubric_band_docs(b) }
547 rm_w(" positive_permil=" as *u8); rm_wn(bp)
548 rm_w("\n" as *u8)
549 bsum = bsum + rm_rubric_band_docs(b)
550 b = b + 1
551 }
552 rm_w(" BANDS sum=" as *u8); rm_wn(bsum); rm_w(" of=" as *u8); rm_wn(docs)
553 if bsum == docs { rm_w(" partition=SUMS\n" as *u8) } else { rm_w(" partition=LEAK\n" as *u8) }
554 let out: *i64 = sys_mmap((top + 1) * RM_I64_BYTES) as *i64
555 let nd: i64 = rm_rubric_rank(RM_RANK_DEFECT, top, min_support, out)
556 var r: i64 = 0
557 while r < nd {
558 let sl: i64 = out[r]
559 rm_w("DEFECT rank=" as *u8); rm_wn(r + 1)
560 rm_w(" term=" as *u8); rm_w(rm_vocab_tok(sl))
561 rm_w(" does_not_meet=" as *u8); rm_wn(rm_vocab_tier(sl, RM_TIER_DNM))
562 rm_w(" mixed=" as *u8); rm_wn(rm_vocab_tier(sl, RM_TIER_MIXED))
563 rm_w(" meets=" as *u8); rm_wn(rm_vocab_tier(sl, RM_TIER_MEETS))
564 rm_w(" exceeds=" as *u8); rm_wn(rm_vocab_tier(sl, RM_TIER_EXCEEDS))
565 rm_w(" all=" as *u8); rm_wn(rm_vocab_all_tiers(sl))
566 rm_w(" dnm_lift_permil=" as *u8); rm_wn(rm_vocab_dnm_lift(sl))
567 rm_w("\n" as *u8)
568 r = r + 1
569 }
570 let nf: i64 = rm_rubric_rank(RM_RANK_FEATURE, top, min_support, out)
571 r = 0
572 while r < nf {
573 let sl: i64 = out[r]
574 rm_w("FEATURE rank=" as *u8); rm_wn(r + 1)
575 rm_w(" term=" as *u8); rm_w(rm_vocab_tok(sl))
576 rm_w(" praised=" as *u8); rm_wn(rm_vocab_tier(sl, RM_TIER_EXCEEDS) + rm_vocab_tier(sl, RM_TIER_MEETS))
577 rm_w(" mixed=" as *u8); rm_wn(rm_vocab_tier(sl, RM_TIER_MIXED))
578 rm_w(" complained=" as *u8); rm_wn(rm_vocab_tier(sl, RM_TIER_DNM))
579 rm_w(" demanded=" as *u8); rm_wn(rm_vocab_dem(sl))
580 rm_w(" all=" as *u8); rm_wn(rm_vocab_all_tiers(sl))
581 rm_w("\n" as *u8)
582 r = r + 1
583 }
584 rm_w(" RANKED defects=" as *u8); rm_wn(nd); rm_w(" features=" as *u8); rm_wn(nf); rm_w(" min_support=" as *u8); rm_wn(min_support); rm_w("\n" as *u8)
585 return 0
586}
587
588func main(argc: i64, argv: *i64) -> i64 {
589 if argc < SR_ARGC_MIN { let u: i64 = sr_usage(); sys_exit(u); return u }
590 let verb: *u8 = argv[SR_ARG_VERB] as *u8
591 let appid: *u8 = argv[SR_ARG_APPID] as *u8
592 if rm_atoi(appid) <= 0 { rm_w("REFUSED appid must be a positive integer\n" as *u8); sys_exit(SR_EXIT_USAGE); return SR_EXIT_USAGE }
593 if rm_streq(verb, "fetch" as *u8) == 1 {
594 var mp: i64 = 0
595 var lang: *u8 = "all" as *u8
596 var full: i64 = 0
597 if argc > SR_ARG_FETCH_PAGES { mp = rm_atoi(argv[SR_ARG_FETCH_PAGES] as *u8); if mp > SR_MAX_PAGE_ARG { mp = SR_MAX_PAGE_ARG } }
598 if argc > SR_ARG_FETCH_LANG { lang = argv[SR_ARG_FETCH_LANG] as *u8 }
599 if argc > SR_ARG_FETCH_FULL { if rm_streq(argv[SR_ARG_FETCH_FULL] as *u8, "full" as *u8) == 1 { full = 1 } }
600 let rc: i64 = sr_fetch(appid, mp, lang, full)
601 sys_exit(rc); return rc
602 }
603 if rm_streq(verb, "ingest" as *u8) == 1 {
604 if argc <= SR_ARG_INGEST_FILE { let u: i64 = sr_usage(); sys_exit(u); return u }
605 let rc: i64 = sr_ingest(appid, argv[SR_ARG_INGEST_FILE] as *u8)
606 sys_exit(rc); return rc
607 }
608 if rm_streq(verb, "stats" as *u8) == 1 { let rc: i64 = sr_stats(appid); sys_exit(rc); return rc }
609 if rm_streq(verb, "name" as *u8) == 1 { let rc: i64 = sr_name(appid); sys_exit(rc); return rc }
610 if rm_streq(verb, "rubric" as *u8) == 1 {
611 var lang: *u8 = "english" as *u8
612 var top: i64 = SR_DEFAULT_TOP
613 var ms: i64 = SR_DEFAULT_MIN_SUPPORT
614 if argc > SR_ARG_LANG { lang = argv[SR_ARG_LANG] as *u8 }
615 if argc > SR_ARG_TOP { top = rm_atoi(argv[SR_ARG_TOP] as *u8); if top <= 0 { top = SR_DEFAULT_TOP } }
616 if argc > SR_ARG_MIN_SUPPORT { ms = rm_atoi(argv[SR_ARG_MIN_SUPPORT] as *u8); if ms <= 0 { ms = SR_DEFAULT_MIN_SUPPORT } }
617 let rc: i64 = sr_rubric(appid, lang, top, ms)
618 sys_exit(rc); return rc
619 }
620 if rm_streq(verb, "mine" as *u8) == 1 {
621 var lang: *u8 = "english" as *u8
622 var top: i64 = SR_DEFAULT_TOP
623 var ms: i64 = SR_DEFAULT_MIN_SUPPORT
624 if argc > SR_ARG_LANG { lang = argv[SR_ARG_LANG] as *u8 }
625 if argc > SR_ARG_TOP { top = rm_atoi(argv[SR_ARG_TOP] as *u8); if top <= 0 { top = SR_DEFAULT_TOP } }
626 if argc > SR_ARG_MIN_SUPPORT { ms = rm_atoi(argv[SR_ARG_MIN_SUPPORT] as *u8); if ms <= 0 { ms = SR_DEFAULT_MIN_SUPPORT } }
627 let rc: i64 = sr_mine(appid, lang, top, ms)
628 sys_exit(rc); return rc
629 }
630 let u2: i64 = sr_usage()
631 sys_exit(u2)
632 return u2
633}