code wiki / (root) / nx_reviewmine_lib.nx

nx_reviewmine_lib.nx

buildroot/runtime/nx_reviewmine_lib.nx

60434 B1270 linesdepth 2pulls 2 transitivereach 25 importersview sourcekind library
docsdependenciesstructsconstsfunctions

about

nx_reviewmine_lib.nx -- REVIEW MINER, THE SHARED DECISION CORE (/compare/reviewmine, contract symbols rm_steam_parse_page, rm_row_emit, rm_seen_has, rm_mine_rank). LIB: pure parsing, the journal row codec, the dedupe set, the tokenizer and the complaint-vs-praise ranker. NO network here: the fetch I/O lives in the thin program nx_steam_reviews so this closure stays nx_syscalls-only and every branch is gate-provable with planted pages -- the same lib/program split nx_source_health uses, and the one that keeps a gate honest. WHY (operator 2026-09-05): "a review miner that can help us address opportunities from amazon to ebay to steam" as part of fetch and ingest. ONE decision core, per-source adapters: Steam first because the estate is doing game dev; the row codec, dedupe set and ranker are source-agnostic so the Amazon Reviews 2023 shape (rating, title, text, helpful_vote, verified_purchase, timestamp) lands as a second adapter, not a second miner. THE ROW (one line, pipe-separated, text LAST and escaped so a review can never split a row): r|recid|steamid|lang|up|pt_review|pt_forever|ts_c|ts_u|votes_up|votes_funny|wvs_permil|comments|flags| games_owned|num_reviews|textlen|text escapes inside text: backslash -> \\ pipe -> \p LF -> \n CR -> \r (nothing else is touched) THE RANKER (rm_mine_rank): per-term DOCUMENT frequency in negative vs positive reviews, smoothed, as a permil ratio ratio = (neg_df+1) * 1000 * (pos_docs+2) / ((pos_df+1) * (neg_docs+2)). A term with equal share in both classes reads ~1000; a term that appears only in complaints reads high. It is the classic feature-based summarisation question (Hu and Liu 2004: which features do customers complain about) answered with document counts a machine can recompute, never a sentiment lexicon. Integer arithmetic throughout, no float. license_tier: ORIGINAL

dependencies 1 imports · 25 importers

nx_syscalls.nx nx_reviewmine_lib.nx nx_absa_bench.nx nx_absa_bench_gate.nx nx_absa_lib.nx nx_absa_seq.nx nx_absa_seq_gate.nx nx_absa_seqextract.nx nx_depparse.nx nx_depparse_eval.nx nx_depparse_gate.nx nx_embfeat_gate.nx

diagram shows first 10 each side; +0 more imports, +15 more importers in the complete lists below.

imports: nx_syscalls.nx

imported by: nx_absa_bench.nxnx_absa_bench_gate.nxnx_absa_lib.nxnx_absa_seq.nxnx_absa_seq_gate.nxnx_absa_seqextract.nxnx_depparse.nxnx_depparse_eval.nxnx_depparse_gate.nxnx_embfeat_gate.nxnx_embfeat_lib.nxnx_installed_census.nxnx_installed_census_gate.nxnx_installed_census_lib.nxnx_intelmine_lib.nxnx_intelmine_propose.nxnx_intelmine_propose_gate.nxnx_postag.nxnx_postag_eval.nxnx_postag_gate.nxnx_reviewmine_semantic_t55.nxnx_steam_reviews.nxnx_steam_reviews_gate.nxnx_wordclust.nxnx_wordclust_gate.nx

structs

none

consts

25const RM_Q: i64 = 34
26const RM_BS: i64 = 92
27const RM_PIPE: i64 = 124
28const RM_LF: i64 = 10
29const RM_CR: i64 = 13
30const RM_COLON: i64 = 58
31const RM_SP: i64 = 32
32const RM_TAB: i64 = 9
33const RM_MINUS: i64 = 45
34const RM_DOT: i64 = 46
35const RM_ZERO: i64 = 48
36const RM_NINE: i64 = 57
37const RM_LOWER_A: i64 = 97
38const RM_LOWER_Z: i64 = 122
39const RM_UPPER_A: i64 = 65
40const RM_UPPER_Z: i64 = 90
41const RM_LOWER_F: i64 = 102
42const RM_UPPER_F: i64 = 70
43const RM_CASE_DELTA: i64 = 32
44const RM_PCT: i64 = 37
45const RM_HIGH_BYTE: i64 = 128
46const RM_CH_n: i64 = 110
47const RM_CH_r: i64 = 114
48const RM_CH_t: i64 = 116
49const RM_CH_b: i64 = 98
50const RM_CH_f: i64 = 102
51const RM_CH_u: i64 = 117
52const RM_CH_p: i64 = 112
53const RM_CH_SLASH: i64 = 47
54const RM_PERMIL: i64 = 1000
55const RM_DECIMAL: i64 = 10
56const RM_HEX: i64 = 16
57const RM_NUMBUF: i64 = 32
58const RM_TRUE_LEN: i64 = 4
59const RM_FALSE_LEN: i64 = 5
60const RM_UTF16_HI_LO: i64 = 55296 // 0xD800
61const RM_UTF16_HI_HI: i64 = 56319 // 0xDBFF
62const RM_UTF16_LO_LO: i64 = 56320 // 0xDC00
63const RM_UTF16_LO_HI: i64 = 57343 // 0xDFFF
64const RM_CP_1BYTE_MAX: i64 = 127
65const RM_CP_2BYTE_MAX: i64 = 2047
66const RM_CP_3BYTE_MAX: i64 = 65535
67const RM_SURROGATE_BASE: i64 = 65536
68const RM_SURROGATE_SHIFT: i64 = 1024
69const RM_LBRACE: i64 = 123
70const RM_RBRACE: i64 = 125
71const RM_APOS: i64 = 39
72const RM_UNDERSCORE: i64 = 95
73const RM_TILDE: i64 = 126
74const RM_SEMICOLON: i64 = 59
75const RM_KNUTH_MULT: i64 = 2654435761
76const RM_CH_BACKSPACE: i64 = 8
77const RM_CH_FORMFEED: i64 = 12
78const RM_TOK_MIN: i64 = 3
79const RM_TOK_MAX: i64 = 32
80const RM_MODE_DIR: i64 = 493 // 0755
81const RM_MODE_FILE: i64 = 420 // 0644
84const RM_F_RECID: i64 = 0
85const RM_F_STEAMID: i64 = 1
86const RM_F_UP: i64 = 2
87const RM_F_PT_REVIEW: i64 = 3
88const RM_F_PT_FOREVER: i64 = 4
89const RM_F_TS_C: i64 = 5
90const RM_F_TS_U: i64 = 6
91const RM_F_VOTES_UP: i64 = 7
92const RM_F_VOTES_FUNNY: i64 = 8
93const RM_F_WVS: i64 = 9
94const RM_F_COMMENTS: i64 = 10
95const RM_F_FLAGS: i64 = 11
96const RM_F_GAMES_OWNED: i64 = 12
97const RM_F_NUM_REVIEWS: i64 = 13
98const RM_F_TEXTLEN: i64 = 14
99const RM_F_N: i64 = 16
100const RM_REC_BYTES: i64 = 128 // RM_F_N * 8
101const RM_FLAG_STEAM_PURCHASE: i64 = 1
102const RM_FLAG_FREE: i64 = 2
103const RM_FLAG_EARLY_ACCESS: i64 = 4
104const RM_FLAG_DECK: i64 = 8
105const RM_FLAG_REFUNDED: i64 = 16
106const RM_LANG_CAP: i64 = 32
109const RM_TEXT_CAP: i64 = 32768
110const RM_ROW_CAP: i64 = 66048 // RM_TEXT_CAP * 2 + 512 header
111const RM_ROW_HDR_FIELDS: i64 = 17 // fields before text
114const RM_S_SUCCESS: i64 = 0
115const RM_S_NUM: i64 = 1
116const RM_S_SCORE: i64 = 2
117const RM_S_POS: i64 = 3
118const RM_S_NEG: i64 = 4
119const RM_S_TOTAL: i64 = 5
120const RM_S_N: i64 = 8
121const RM_CURSOR_CAP: i64 = 256
124const RM_I64_BYTES: i64 = 8 // sizeof i64: every *i64 table is slots * RM_I64_BYTES
125const RM_BITS_PER_BYTE: i64 = 8 // the shift between adjacent bytes in rm_ld8 / rm_st8 (same value, different meaning)
126const RM_BYTE_MASK: i64 = 255
127const RM_I64_PAIR: i64 = 16 // two i64 slots: the sys_read_file length cell plus one spare
128const RM_NUMBUF_SLOTS: i64 = 2 // rm_numbuf serves two in-flight numbers
129const RM_TOK_BUF_SPARE: i64 = 2 // the token buffer holds RM_TOK_MAX bytes plus a NUL plus the over-long sentinel slot
130const RM_STR_TAIL: i64 = 2 // a copied string carries its NUL plus one spare byte
131const RM_GROW_FACTOR: i64 = 2 // the open-addressed seen set doubles, and grows at half full
132const RM_KEY_QUOTES: i64 = 2 // the two quote bytes around a JSON key
133const RM_ESC_LEN: i64 = 2 // a row escape is a backslash plus one byte
134const RM_ROW_PREFIX_LEN: i64 = 2 // every row opens with the two bytes r and pipe
135const RM_ROWF_LANG: i64 = 2 // the third row field is lang (the rec slots skip it)
136const RM_PCT_ESC_LEN: i64 = 3 // a percent escape is three bytes
137const RM_NIBBLE_SHIFT: i64 = 4
138const RM_NIBBLE_MASK: i64 = 15
139const RM_HEX4_DIGITS: i64 = 4 // a JSON unicode escape carries four hex digits
140const RM_UESC_PREFIX: i64 = 2 // the backslash and the u before them
141const RM_UESC_LEN: i64 = 6 // the whole escape
142const RM_PERMIL_DIGITS: i64 = 3 // a quoted decimal keeps three fractional digits (permil)
143const RM_DJB2_SEED: i64 = 5381 // Bernstein hash
144const RM_DJB2_SHIFT: i64 = 5
145const RM_UTF8_LEAD2: i64 = 192 // 110xxxxx
146const RM_UTF8_LEAD3: i64 = 224 // 1110xxxx
147const RM_UTF8_LEAD4: i64 = 240 // 11110xxx
148const RM_UTF8_CONT: i64 = 128 // 10xxxxxx
149const RM_UTF8_CONT_MASK: i64 = 63 // the six payload bits of a continuation byte
150const RM_UTF8_SHIFT1: i64 = 6 // payload bits carried per continuation byte
151const RM_UTF8_SHIFT2: i64 = 12
152const RM_UTF8_SHIFT3: i64 = 18
153const RM_UTF8_LEN2: i64 = 2
154const RM_UTF8_LEN3: i64 = 3
155const RM_UTF8_LEN4: i64 = 4
156const RM_UTF8_B2: i64 = 2 // byte offsets inside one encoded sequence (the second and third continuation slots)
157const RM_UTF8_B3: i64 = 3
158const RM_TOK_STATE_APOS: i64 = 2 // rm_next_token: an apostrophe neither extends nor ends a token
159const RM_BAND_UNDER_1H: i64 = 0 // rm_band_of: the four playtime bands, in order
160const RM_BAND_1_10H: i64 = 1
161const RM_BAND_10_100H: i64 = 2
162const RM_BAND_OVER_100H: i64 = 3
633const RM_NAME_CAP: i64 = 256 // a store title; the banked file is one line, name LF
664const RM_SEEN_MIN_SLOTS: i64 = 1024
734const RM_I_DECODED: i64 = 0
735const RM_I_NEW: i64 = 1
736const RM_I_DUP: i64 = 2
737const RM_I_UNDECODABLE: i64 = 3
738const RM_I_N: i64 = 4
776const RM_ST_ROWS: i64 = 0
777const RM_ST_POS: i64 = 1
778const RM_ST_NEG: i64 = 2
779const RM_ST_EA: i64 = 3
780const RM_ST_FREE: i64 = 4
781const RM_ST_DECK: i64 = 5
782const RM_ST_REFUNDED: i64 = 6
783const RM_ST_PT_LT1H: i64 = 7
784const RM_ST_PT_1_10H: i64 = 8
785const RM_ST_PT_10_100H: i64 = 9
786const RM_ST_PT_GT100H: i64 = 10
787const RM_ST_STEAM_PURCHASE: i64 = 11
788const RM_ST_MALFORMED: i64 = 12
789const RM_ST_TEXT_BYTES: i64 = 13
790const RM_ST_N: i64 = 16
791const RM_MIN_PER_HOUR: i64 = 60
792const RM_PT_BAND_1: i64 = 60 // minutes: under one hour
793const RM_PT_BAND_2: i64 = 600 // ten hours
794const RM_PT_BAND_3: i64 = 6000 // one hundred hours
795const RM_LANG_SLOTS: i64 = 64
853const RM_VOCAB_SLOTS: i64 = 262144 // power of two; a review corpus of one title stays far under it -- and when
855const RM_VOCAB_ARENA: i64 = 8650752 // RM_VOCAB_SLOTS * (RM_TOK_MAX + 1)
856const RM_MODE_COMPLAINT: i64 = 0
857const RM_MODE_PRAISE: i64 = 1
858const RM_LAPLACE: i64 = 1
859const RM_CLASS_PRIOR: i64 = 2
878const RM_LEX_DEFECT: i64 = 1
879const RM_LEX_FEATURE: i64 = 2
880const RM_LEX_EXCEED: i64 = 4
881const RM_LEX_NOTMEET: i64 = 8
882const RM_LEX_VALUE: i64 = 16
883const RM_LEX_DEMAND: i64 = 32
884const RM_TIER_EXCEEDS: i64 = 0
885const RM_TIER_MEETS: i64 = 1
886const RM_TIER_MIXED: i64 = 2 // a positive vote carrying an explicit not-met cue (refund, disappointed...): declared, never folded into MEETS
887const RM_TIER_DNM: i64 = 3
888const RM_TIER_N: i64 = 4
889const RM_RANK_DEFECT: i64 = 0
890const RM_RANK_FEATURE: i64 = 1
893const RM_REFUND_WINDOW_MIN: i64 = 120
894const RM_BAND_N: i64 = 4
895const RM_DOC_SLOT_CAP: i64 = 4096 // distinct terms one review can contribute; a longer review is counted truncated=1, never dropped

functions

164func rm_slen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
165func rm_w(s: *u8) -> i64 { sys_write(1, s, rm_slen(s)); return 0 }
166func rm_wb(b: *u8, n: i64) -> i64 { if n > 0 { sys_write(1, b, n) } return 0 }
called by 2: ip_emitsr_fetch calls 1: sys_write
170func rm_numbuf() -> *u8 { if (rm_num_g as i64) == 0 { rm_num_g = sys_mmap(RM_NUMBUF * RM_NUMBUF_SLOTS); rm_ep_g = sys_mmap(RM_I64_PAIR) as *i64; rm_tok_g = sys_mmap(RM_TOK_MAX + RM_TOK_BUF_SPARE) } return rm_num_g }
171func rm_itoa(v: i64, out: *u8) -> i64
called by 2: rm_wnrm_cati calls 1: rm_numbuf
182func rm_wn(v: i64) -> i64 { rm_numbuf(); let t: *u8 = (rm_num_g as i64 + RM_NUMBUF) as *u8; let k: i64 = rm_itoa(v, t); sys_write(1, t, k); return 0 }
183func rm_cat(d: *u8, o: i64, s: *u8) -> i64 { var i: i64 = 0; var p: i64 = o; while s[i] != (0 as u8) { d[p] = s[i]; p = p + 1; i = i + 1 } return p }
184func rm_catn(d: *u8, o: i64, s: *u8, n: i64) -> i64 { var i: i64 = 0; var p: i64 = o; while i < n { d[p] = s[i]; p = p + 1; i = i + 1 } return p }
185func rm_cati(d: *u8, o: i64, v: i64) -> i64 { return o + rm_itoa(v, (d as i64 + o) as *u8) }
186func rm_streq(a: *u8, b: *u8) -> i64
192func rm_atoi(s: *u8) -> i64
197func rm_hash(b: *u8, n: i64) -> i64
208func rm_key(buf: *u8, n: i64, key: *u8, from: i64, to: i64) -> i64
238func rm_hexval(c: i64) -> i64
called by 1: rm_hex4
244func rm_utf8_put(cp: i64, out: *u8, o: i64, cap: i64) -> i64
called by 1: rm_str
253func rm_hex4(buf: *u8, n: i64, at: i64) -> i64
called by 1: rm_str calls 1: rm_hexval
262func rm_str(buf: *u8, n: i64, at: i64, out: *u8, cap: i64, end_out: *i64) -> i64
321func rm_int(buf: *u8, n: i64, at: i64) -> i64
336func rm_bool(buf: *u8, n: i64, at: i64) -> i64
called by 1: rm_steam_review
344func rm_permil(buf: *u8, n: i64, at: i64) -> i64
called by 1: rm_steam_review
362func rm_find(buf: *u8, n: i64, lit: *u8, from: i64) -> i64
377func rm_urlenc(src: *u8, n: i64, dst: *u8, cap: i64) -> i64
called by 1: main
406func rm_steam_summary(buf: *u8, n: i64, out: *i64) -> i64
called by 2: sr_fetchmain calls 2: rm_intrm_key
427func rm_steam_cursor(buf: *u8, n: i64, out: *u8, cap: i64) -> i64
called by 2: sr_fetchmain calls 3: rm_keyrm_numbufrm_str
441func rm_steam_next(buf: *u8, n: i64, from: i64) -> i64
called by 2: rm_steam_ingest_pagemain calls 1: rm_key
449func rm_steam_review(buf: *u8, n: i64, at: i64, end: i64, rec: *i64, lang: *u8, text: *u8) -> i64
494func rm_row_emit(rec: *i64, lang: *u8, text: *u8, row: *u8, cap: i64) -> i64
524func rm_row_parse(row: *u8, len: i64, rec: *i64, lang: *u8, text: *u8) -> i64
called by 3: rm_statsrm_minerm_rubric calls 1: rm_catn
586func rm_line_end(buf: *u8, n: i64, from: i64) -> i64
593func rm_mkdir(path: *u8) -> i64 { sys_mkdir(path, RM_MODE_DIR); return 0 }
called by 1: rm_mkdirp calls 1: sys_mkdir
595func rm_mkdirp(path: *u8) -> i64
606func rm_write_all(fd: i64, b: *u8, n: i64) -> i64
called by 2: rm_file_putrm_file_append calls 1: sys_write
612func rm_file_put(path: *u8, b: *u8, n: i64) -> i64
620func rm_file_append(path: *u8, b: *u8, n: i64) -> i64
634func rm_json_name(buf: *u8, n: i64, out: *u8, cap: i64) -> i64
called by 2: sr_namemain calls 3: rm_keysys_mmaprm_str
643func rm_name_file_load(path: *u8, out: *u8, cap: i64) -> i64
656func rm_ld8(b: *u8, off: i64) -> i64 { var v: i64 = 0; var i: i64 = 0; while i < RM_I64_BYTES { v = v | ((b[off + i] as i64) << (i * RM_BITS_PER_BYTE)); i = i + 1 } return v }
called by 1: rm_seen_load
657func rm_st8(b: *u8, off: i64, val: i64) -> i64 { var i: i64 = 0; while i < RM_I64_BYTES { b[off + i] = ((val >> (i * RM_BITS_PER_BYTE)) & RM_BYTE_MASK) as u8; i = i + 1 } return 0 }
called by 1: rm_seen_add
665func rm_seen_slot_of(id: i64) -> i64
678func rm_seen_insert_mem(id: i64) -> i64
687func rm_seen_grow() -> i64
698func rm_seen_load(path: *u8) -> i64
713func rm_seen_has(id: i64) -> i64
720func rm_seen_count_get() -> i64 { return rm_seen_count }
722func rm_seen_add(id: i64, path: *u8) -> i64
743func rm_scratch() -> i64
747func rm_steam_ingest_page(buf: *u8, n: i64, journal: *u8, seen: *u8, out: *i64) -> i64
799func rm_lang_bump(lang: *u8) -> i64
called by 1: rm_stats calls 3: sys_mmaprm_streqrm_cat
811func rm_lang_count() -> i64 { return rm_lang_n }
called by 2: sr_statsmain
812func rm_lang_at(i: i64) -> *u8 { return (rm_lang_tbl as i64 + i * RM_LANG_CAP) as *u8 }
called by 1: sr_stats
813func rm_lang_n_at(i: i64) -> i64 { return rm_lang_cnt[i] }
called by 2: sr_statsmain
815func rm_stats(journal: *u8, out: *i64) -> i64
908func rm_vocab_reset() -> i64
933func rm_vocab_slot(tok: *u8, n: i64) -> i64
961func rm_vocab_tok(slot: i64) -> *u8 { return (rm_v_arena as i64 + rm_v_off[slot]) as *u8 }
962func rm_vocab_neg(slot: i64) -> i64 { return rm_v_neg[slot] }
called by 2: sr_minemain
963func rm_vocab_pos(slot: i64) -> i64 { return rm_v_pos[slot] }
called by 2: sr_minemain
964func rm_vocab_stop(slot: i64) -> i64 { return rm_v_stop[slot] } // 1 when the slot is a loaded stopword (a phrase boundary for n-gram miners)
called by 1: ab_ng_tokens
965func rm_vocab_size() -> i64 { return rm_v_n }
966func rm_mine_docs() -> i64 { return rm_m_docs }
967func rm_mine_neg_docs() -> i64 { return rm_m_neg_docs }
called by 2: sr_minemain
968func rm_mine_pos_docs() -> i64 { return rm_m_pos_docs }
called by 2: sr_minemain
969func rm_mine_skipped_lang() -> i64 { return rm_m_skipped_lang }
called by 2: sr_minesr_rubric
970func rm_mine_malformed() -> i64 { return rm_m_malformed }
called by 2: sr_minesr_rubric
971func rm_mine_coverage_complete() -> i64 { if rm_v_full == 1 { return 0 } return 1 }
973func rm_stopwords_load(path: *u8) -> i64
1004func rm_next_token(text: *u8, n: i64, ip: *i64, tok: *u8) -> i64
1027func rm_mine_doc(text: *u8, n: i64, up: i64, docid: i64) -> i64
1046func rm_mine(journal: *u8, lang: *u8) -> i64
1073func rm_vocab_ratio(slot: i64, mode: i64) -> i64
1083func rm_mine_rank(mode: i64, k: i64, min_support: i64, out: *i64) -> i64
1112func rm_lexicon_load(path: *u8, bit: i64) -> i64
1135func rm_band_of(pt: i64) -> i64
called by 1: rm_rubric_doc
1145func rm_rubric_doc(text: *u8, n: i64, up: i64, pt: i64, docid: i64) -> i64
1187func rm_rubric(journal: *u8, lang: *u8) -> i64
1213func rm_rubric_docs() -> i64 { return rm_r_docs }
1214func rm_rubric_tier(t: i64) -> i64 { return rm_r_tier[t] }
1215func rm_rubric_value(t: i64) -> i64 { return rm_r_value[t] }
called by 2: sr_rubricmain
1216func rm_rubric_band_docs(b: i64) -> i64 { return rm_r_band_docs[b] }
called by 2: sr_rubricmain
1217func rm_rubric_band_pos(b: i64) -> i64 { return rm_r_band_pos[b] }
called by 1: sr_rubric
1218func rm_rubric_rw_docs() -> i64 { return rm_r_rw_docs }
called by 2: sr_rubricmain
1219func rm_rubric_rw_pos() -> i64 { return rm_r_rw_pos }
called by 2: sr_rubricmain
1220func rm_rubric_truncated() -> i64 { return rm_r_truncated }
called by 2: sr_rubricmain
1221func rm_vocab_tier(slot: i64, t: i64) -> i64 { return rm_v_tier[slot * RM_TIER_N + t] }
1222func rm_vocab_dem(slot: i64) -> i64 { return rm_v_dem[slot] }
1223func rm_vocab_lex(slot: i64) -> i64 { return rm_v_lex[slot] }
called by 1: ab_lex_cues
1224func rm_vocab_all_tiers(slot: i64) -> i64 { var t: i64 = 0; var v: i64 = 0; while t < RM_TIER_N { v = v + rm_v_tier[slot * RM_TIER_N + t]; t = t + 1 } return v }
1225func rm_tier_name(t: i64) -> *u8
called by 1: sr_rubric
1233func rm_vocab_dnm_lift(slot: i64) -> i64
1241func rm_rubric_rank(kind: i64, k: i64, min_support: i64, out: *i64) -> i64