nx_survey_sentiment.nx source
↩ module page · 194 lines · 12111 B
1// nx_survey_sentiment.nx -- sovereign open-text SENTIMENT for survey free-text answers.
2// A faithful INTEGER reimplementation of lexicon-based sentiment (VADER-style): a valence lexicon +
3// negation flip + intensifier boost + the VADER compound normalization compound = S / sqrt(S^2 + alpha).
4// EVIDENCE (banked 2024-26, knowledge/library/swc_svy_sent.txt): "VADER", "lexicon-based", "polarity",
5// "Opinion Mining" -- the method is grounded; this adds emotional VALENCE beside the frequency themes so the
6// admin sees not just WHICH words recur but whether the written feedback is warming or cooling. Admin-only:
7// like the theme extractor it reads the free text that is WALLED from every public page (never leaks).
8//
9// All arithmetic is integer fixed-point: valences carry MILLIS (AFINN -4..+4 scale x1000), compound is
10// PERMILLE (-1000..+1000). Deterministic + KAT-exact (gate nx_survey_sentiment_gate). No float, no LLM.
11// Imports quality (which brings stats+engine+seg_store) so the serve layer gets the whole survey lib
12// through one import; st_isqrt (stats) is used for the compound normalization.
13import "nx_survey_quality.nx"
14const SN_MAGIC_4096: i64 = 4096
15
16// -- VADER constants, grounded, kept as named config (rule 11: no magic numbers) --
17const SN_BINCR: i64 = 293 // booster/intensifier increment 0.293, applied sign-preserving to |v|
18const SN_NSCALAR: i64 = 0 - 740 // negation scalar -0.74 (flips + dampens a valence in a negation window)
19const SN_NWIN: i64 = 3 // negation look-back window (words) -- VADER's tri-gram
20const SN_ALPHA: i64 = 15000000 // normalization alpha = 15 in millis^2 (15 * 1000 * 1000)
21const SN_POS_TH: i64 = 50 // compound >= +0.05 (+50 permille) => positive (VADER standard)
22const SN_NEG_TH: i64 = 0 - 50 // compound <= -0.05 (-50 permille) => negative
23const SN_MAXLEX: i64 = 400 // lexicon entry bound
24
25// Embedded default lexicon = the bootstrap default in the config hierarchy (rule 17). One entry per line:
26// "<word> <kind> <valence_millis>". kind: w = sentiment word (valence used) · n = negation (valence 0) ·
27// i = intensifier (valence = +boost for amplifiers, -boost for downtoners). Curated affect + welfare/
28// feedback domain seed (~130 entries); a knowledge/survey/sentiment_lexicon.tsv override is the named rung.
29const SN_LEX: *u8 = "love w 3000\nwonderful w 3000\nexcellent w 3000\nfantastic w 3000\namazing w 3000\nperfect w 3000\nawesome w 3000\ngrateful w 2000\nthankful w 2000\nthank w 2000\nthanks w 2000\nappreciate w 2000\nappreciated w 2000\nhappy w 2000\nglad w 2000\npleased w 2000\ngood w 1900\ngreat w 3000\nhelpful w 2000\nhelped w 1900\nhelp w 1000\nsupport w 1900\nsupported w 2000\nsupportive w 2000\ncaring w 2000\nkind w 2000\nkindness w 2000\ngenerous w 2000\nwelcoming w 1900\nfriendly w 1900\nsafe w 1700\nsecure w 1500\ncomfortable w 1500\nhopeful w 1700\nhope w 1400\nblessed w 2200\nrelief w 1900\nrelieved w 2000\nbetter w 1700\nimproved w 1900\nimproving w 1700\nimprovement w 1500\nsatisfied w 1900\nenjoy w 1900\nenjoyed w 1900\nnice w 1500\ngreatly w 1400\neasy w 1200\ncomforting w 1900\nencouraged w 1800\nencouraging w 1800\nwarm w 1200\npositive w 1400\nbad w 0 - 2100\nterrible w 0 - 3000\nawful w 0 - 3000\nhorrible w 0 - 3000\nworst w 0 - 3000\nhate w 0 - 2700\ndifficult w 0 - 1800\nhard w 0 - 1400\nstruggle w 0 - 2000\nstruggling w 0 - 2000\nstruggles w 0 - 2000\nworried w 0 - 2000\nworry w 0 - 1800\nworrying w 0 - 1800\nstress w 0 - 1800\nstressed w 0 - 2000\nstressful w 0 - 1800\nanxious w 0 - 2000\nafraid w 0 - 1900\nscared w 0 - 2000\nangry w 0 - 2300\nupset w 0 - 1800\nfrustrated w 0 - 2100\nfrustrating w 0 - 2100\ndisappointed w 0 - 2000\ndisappointing w 0 - 2000\nsad w 0 - 1800\nlonely w 0 - 1900\nalone w 0 - 1300\npoor w 0 - 1400\nsick w 0 - 1600\nill w 0 - 1500\npain w 0 - 1900\npainful w 0 - 1900\nhungry w 0 - 2000\ntired w 0 - 1200\nexhausted w 0 - 2000\noverwhelmed w 0 - 2100\nconfused w 0 - 1200\nconfusing w 0 - 1300\nlost w 0 - 1200\nbehind w 0 - 1200\nexpensive w 0 - 1200\nunaffordable w 0 - 2200\nbroken w 0 - 1800\nfail w 0 - 2000\nfailed w 0 - 2000\nfailing w 0 - 2000\nproblem w 0 - 1200\nproblems w 0 - 1300\nconcern w 0 - 1000\nconcerned w 0 - 1200\nconcerns w 0 - 1100\nhurt w 0 - 1900\nsuffering w 0 - 2800\ndesperate w 0 - 2900\ncrisis w 0 - 2700\nhomeless w 0 - 2600\neviction w 0 - 2600\nevicted w 0 - 2700\ndebt w 0 - 1600\nunemployed w 0 - 1800\ndepressed w 0 - 2500\ndepressing w 0 - 2200\nhopeless w 0 - 2600\nnegative w 0 - 1400\nnot n 0\nno n 0\nnever n 0\nnone n 0\nnothing n 0\nneither n 0\nnobody n 0\ncannot n 0\ncant n 0\ndont n 0\ndoesnt n 0\ndidnt n 0\nisnt n 0\nwasnt n 0\narent n 0\nwont n 0\nwouldnt n 0\ncouldnt n 0\nshouldnt n 0\nwithout n 0\nhardly n 0\nbarely n 0\nvery i 293\nreally i 293\nextremely i 293\nso i 293\ntoo i 293\nabsolutely i 293\ncompletely i 293\ntotally i 293\nincredibly i 293\nespecially i 293\nslightly i 0 - 293\nsomewhat i 0 - 293\nkinda i 0 - 293\n" as *u8
30
31// parse an integer that may be written "0 - N" (NishiLang-literal-safe negative) or "N".
32func sn_pval(s: *u8) -> i64 {
33 var i: i64 = 0
34 var neg: i64 = 0
35 // leading "0 - " marks a negative valence
36 if s[0] == (48 as u8) { if s[1] == (32 as u8) { if s[2] == (45 as u8) { neg = 1; i = 3
37 while s[i] == (32 as u8) { i = i + 1 } } } }
38 var v: i64 = 0
39 while s[i] != (0 as u8) {
40 if s[i] >= (48 as u8) { if s[i] <= (57 as u8) { v = v * 10 + (s[i] as i64 - 48) } }
41 i = i + 1
42 }
43 if neg == 1 { return 0 - v }
44 return v
45}
46
47// LOAD the lexicon string into parallel arrays: words[k] (ptr), kinds[k] (119='w' 110='n' 105='i'),
48// vals[k] (millis). Returns entry count, or -1 on overflow. cap = SN_MAXLEX sized by the caller.
49func sn_load(lex: *u8, words: *i64, kinds: *i64, vals: *i64, cap: i64) -> i64 {
50 let pool: *u8 = sys_mmap(cap * 40 + SN_MAGIC_4096)
51 var pooloff: i64 = 0
52 var n: i64 = 0
53 var i: i64 = 0
54 let ll: i64 = se_slen(lex)
55 while i < ll {
56 // one line: word <sp> kind <sp> value
57 var ws: i64 = i
58 while i < ll { if lex[i] == (32 as u8) { break } if lex[i] == (10 as u8) { break } i = i + 1 }
59 let we: i64 = i
60 if we > ws { if lex[i] == (32 as u8) {
61 // kind char
62 let kc: i64 = lex[i + 1] as i64
63 i = i + 2
64 while i < ll { if lex[i] == (32 as u8) { break } i = i + 1 }
65 // value = rest of line
66 let vs: i64 = i + 1
67 var ve: i64 = vs
68 while ve < ll { if lex[ve] == (10 as u8) { break } ve = ve + 1 }
69 if n >= cap { return 0 - 1 }
70 // copy word to pool, null-terminated
71 let dst: *u8 = (pool as i64 + pooloff) as *u8
72 var x: i64 = 0
73 while ws + x < we { dst[x] = lex[ws + x]; x = x + 1 }
74 dst[x] = 0 as u8
75 pooloff = pooloff + (we - ws) + 1
76 // copy value substring, null-terminated, then parse
77 let vb: *u8 = sys_mmap(24)
78 var y: i64 = 0
79 while vs + y < ve { if y < 23 { vb[y] = lex[vs + y] } y = y + 1 }
80 vb[y] = 0 as u8
81 words[n] = dst as i64
82 kinds[n] = kc
83 vals[n] = sn_pval(vb)
84 n = n + 1
85 i = ve + 1
86 } else { i = i + 1 } } else { i = i + 1 }
87 }
88 return n
89}
90
91// lexicon lookup: returns index or -1. (linear; lexicon is a few hundred entries, answers are short)
92func sn_find(words: *i64, n: i64, tok: *u8) -> i64 {
93 var k: i64 = 0
94 while k < n { if se_seq(words[k] as *u8, tok) == 1 { return k } k = k + 1 }
95 return 0 - 1
96}
97
98// SCORE one text answer -> compound sentiment in PERMILLE (-1000..+1000). VADER-style:
99// each sentiment word contributes its valence; an intensifier immediately before amplifies (sign-preserving);
100// a negation within the previous SN_NWIN words flips+dampens (x -0.74). Sum S -> S/sqrt(S^2+alpha).
101func sn_score(words: *i64, kinds: *i64, vals: *i64, nlex: i64, text: *u8, tlen: i64) -> i64 {
102 let tbl: *u8 = sys_mmap(272)
103 ss_tok_table(tbl)
104 let tok: *u8 = sys_mmap(64)
105 let pos: *i64 = sys_mmap(16) as *i64
106 pos[0] = 0
107 var s: i64 = 0
108 var negctr: i64 = 0 // words remaining in an active negation window
109 var intens: i64 = 0 // pending intensifier boost (millis of scalar), 0 = none
110 var go: i64 = 1
111 while go == 1 {
112 let tl: i64 = ss_tok_next2(text, tlen, pos, tok, tbl)
113 if tl < 0 { go = 0 } else {
114 // lowercase the token (lexicon is lowercase) -- robust whatever the tokenizer emits
115 var lc: i64 = 0
116 while lc < tl { if tok[lc] >= (65 as u8) { if tok[lc] <= (90 as u8) { tok[lc] = (tok[lc] as i64 + 32) as u8 } } lc = lc + 1 }
117 let ix: i64 = sn_find(words, nlex, tok)
118 if ix < 0 {
119 if negctr > 0 { negctr = negctr - 1 }
120 intens = 0
121 } else {
122 let kc: i64 = kinds[ix]
123 if kc == 110 { // negation -> open a window (do NOT reset by itself)
124 negctr = SN_NWIN
125 intens = 0
126 } else { if kc == 105 { // intensifier -> pending boost for the NEXT sentiment word
127 intens = vals[ix]
128 if negctr > 0 { negctr = negctr - 1 }
129 } else { // sentiment word
130 var v: i64 = vals[ix]
131 if intens != 0 { v = v + (v * intens) / 1000 }
132 if negctr > 0 { v = (v * SN_NSCALAR) / 1000 }
133 s = s + v
134 intens = 0
135 if negctr > 0 { negctr = negctr - 1 }
136 } }
137 }
138 }
139 }
140 if s == 0 { return 0 }
141 let denom: i64 = st_isqrt(s * s + SN_ALPHA)
142 if denom == 0 { return 0 }
143 return (1000 * s) / denom
144}
145
146// classify a compound permille -> +1 positive / 0 neutral / -1 negative (VADER standard thresholds)
147func sn_class(compound: i64) -> i64 {
148 if compound >= SN_POS_TH { return 1 }
149 if compound <= SN_NEG_TH { return 0 - 1 }
150 return 0
151}
152
153// AGGREGATE sentiment across every effective ballot's answer to qid. out[0..4] =
154// {n_scored, mean_compound_permille, n_positive, n_neutral, n_negative}. Returns n_scored.
155// Only non-empty text answers are scored (a skipped optional text question contributes nothing).
156// AGGREGATE over ballot arrays already in hand (bv/bl/nb = what se_ballots yields) -- the insights path
157// reuses the ballots it already loaded rather than re-walking the store. out[0..4] =
158// {n_scored, mean_compound_permille, n_positive, n_neutral, n_negative}. Returns n_scored.
159func sn_aggregate_over(bv: *i64, bl: *i64, nb: i64, qid: *u8, words: *i64, kinds: *i64, vals: *i64, nlex: i64, out: *i64) -> i64 {
160 var n: i64 = 0
161 var sum: i64 = 0
162 var pos: i64 = 0
163 var neu: i64 = 0
164 var neg: i64 = 0
165 let ab: *u8 = sys_mmap(SN_MAGIC_4096)
166 var i: i64 = 0
167 while i < nb {
168 let al: i64 = se_answer_of(bv[i] as *u8, bl[i], qid, ab, SN_MAGIC_4096)
169 if al > 0 {
170 let c: i64 = sn_score(words, kinds, vals, nlex, ab, al)
171 sum = sum + c
172 let cl: i64 = sn_class(c)
173 if cl > 0 { pos = pos + 1 } else { if cl < 0 { neg = neg + 1 } else { neu = neu + 1 } }
174 n = n + 1
175 }
176 i = i + 1
177 }
178 out[0] = n
179 if n > 0 { out[1] = sum / n } else { out[1] = 0 }
180 out[2] = pos
181 out[3] = neu
182 out[4] = neg
183 return n
184}
185
186// store-reading wrapper: walk the ballots for <id> then aggregate qid's answers (used by the gate + any
187// standalone caller). The insights path uses sn_aggregate_over directly on ballots it already holds.
188func sn_aggregate(prefix: *u8, id: *u8, qid: *u8, words: *i64, kinds: *i64, vals: *i64, nlex: i64, out: *i64) -> i64 {
189 let bk: *i64 = sys_mmap(8 * 512) as *i64
190 let bv: *i64 = sys_mmap(8 * 512) as *i64
191 let bl: *i64 = sys_mmap(8 * 512) as *i64
192 let nb: i64 = se_ballots(prefix, id, bk, bv, bl, 512)
193 return sn_aggregate_over(bv, bl, nb, qid, words, kinds, vals, nlex, out)
194}