nx_absa_bench_gate.nx source

1// nx_absa_bench_gate.nx -- GATE for the SemEval-2014 SB1 scorer in nx_absa_lib (intelmine IM17), driven 2// IN-PROCESS on a planted gold set under /tmp/nx_absa_bench_gate. The load-bearing proof is the F1 arithmetic 3// (ab_prf) checked at exact permil values including the ZERO-PREDICTION control the rung's done-rule names; the 4// gold parser is proven to dedupe distinct terms and to survive CRLF; the normaliser is proven case- and 5// whitespace-insensitive; a multi-word gold term is proven UNMATCHABLE by a unigram (the honest weakness); and 6// the full ab_eval pipeline is proven non-vacuous on the planted gold. Every fixture asserts its own condition 7// before any outcome is judged. No network. license_tier: ORIGINAL No hw writes (Rule 26). 8import "nx_syscalls.nx" 9import "nx_gatekit_lib.nx" 10import "nx_reviewmine_lib.nx" 11import "nx_absa_lib.nx" 12import "nx_gate_verdict.nx" 13 14const G_ROOT: *u8 = "/tmp/nx_absa_bench_gate" 15const G_GOLD: *u8 = "/tmp/nx_absa_bench_gate/gold.seg" 16const G_GOLDCRLF: *u8 = "/tmp/nx_absa_bench_gate/goldcrlf.seg" 17const G_I64: i64 = 8 18const G_LINES: i64 = 12 // four three-line records 19const G_NG: i64 = 3 // distinct terms: food, service, hard disk (food appears twice) 20const G_LF: i64 = 10 21const G_CR: i64 = 13 22const G_CRLF_CAP: i64 = 512 23// subtask 2 fixtures: three planted lexicons and a directory with none 24const G_LEXDIR: *u8 = "/tmp/nx_absa_bench_gate/lex/" 25const G_NOLEXDIR: *u8 = "/tmp/nx_absa_bench_gate/nolex/" 26const G_LEX_DEFECT: *u8 = "/tmp/nx_absa_bench_gate/lex/lexicon_defect.conf" 27const G_LEX_NOTMEET: *u8 = "/tmp/nx_absa_bench_gate/lex/lexicon_notmeet.conf" 28const G_LEX_EXCEED: *u8 = "/tmp/nx_absa_bench_gate/lex/lexicon_exceed.conf" 29const G_ABSENT_GOLD: *u8 = "/tmp/nx_absa_bench_gate/absent.seg" 30const G_RECORDS: i64 = 4 31const G_LEX_TERMS: i64 = 5 // slow, noisy (defect) + refund (not met) + great, fast (exceed) 32// n-gram fixtures: the four records plus a fifth that repeats the multi-word term, so "hard disk" is a frequent bigram 33// at support two while "hard" and "disk" never occur on their own (p-support zero: fragments to prune) 34const G_GOLD_NG: *u8 = "/tmp/nx_absa_bench_gate/gold_ng.seg" 35const G_ROWS_NG: *u8 = "The $T$ was great .\nfood\n1\nThe $T$ is slow .\nservice\n-1\nThe $T$ is noisy .\nhard disk\n0\nGreat $T$ here .\nfood\n1\nThe $T$ is loud .\nhard disk\n-1\n" 36const G_NG_LINES: i64 = 15 37const G_NG_RECORDS: i64 = 5 38const G_NG_MINSUP: i64 = 2 // pins support at the bigram's frequency 39const G_NG_MINSUP_ABOVE: i64 = 3 // one above it: the control that must find no frequent bigram 40const G_NG_CANDS: i64 = 3 // hard disk, food, great 41const G_NG_PRUNED: i64 = 2 // hard, disk 42const G_TRAIN: *u8 = "/tmp/nx_absa_bench_gate/train.seg" 43const G_TEST: *u8 = "/tmp/nx_absa_bench_gate/test.seg" 44const G_TRAIN_ROWS: *u8 = "The $T$ was slow .\nfood\n1\nThe $T$ is rude .\nservice\n-1\nThe $T$ is noisy .\nhard disk\n0\nThe $T$ is nice .\nkeyboard\n1\n" 45const G_TEST_ROWS: *u8 = "The $T$ was great .\nfood\n1\nThe $T$ is noisy .\nhard disk\n0\nThe $T$ broke .\nmouse\n-1\nI like the keyboard and $T$ .\nfood\n1\n" 46const G_D_DICT: i64 = 4 47const G_D_RECORDS: i64 = 4 48const G_D_NG: i64 = 3 49const G_D_PRED: i64 = 3 50const G_D_INTER: i64 = 2 51// four records; "food" repeats so the gold set must dedupe it; "hard disk" is a multi-word term a unigram cannot match 52const G_ROWS: *u8 = "The $T$ was great .\nfood\n1\nThe $T$ is slow .\nservice\n-1\nThe $T$ is noisy .\nhard disk\n0\nGreat $T$ here .\nfood\n1\n" 53 54func g_lines(path: *u8) -> i64 { 55 let lp: *i64 = sys_mmap(G_I64) as *i64 56 lp[0] = 0 57 let b: *u8 = sys_read_file(path, lp) 58 if (b as i64) == 0 { return 0 } 59 var n: i64 = 0 60 var i: i64 = 0 61 while i < lp[0] { if b[i] == (G_LF as u8) { n = n + 1 } i = i + 1 } 62 return n 63} 64// write a CRLF twin of a NUL-terminated LF source into out, return its byte length 65func g_crlf(src: *u8, out: *u8) -> i64 { 66 var i: i64 = 0 67 var o: i64 = 0 68 while src[i] != (0 as u8) { 69 if (src[i] as i64) == G_LF { out[o] = G_CR as u8; o = o + 1 } 70 out[o] = src[i]; o = o + 1 71 i = i + 1 72 } 73 out[o] = 0 as u8 74 return o 75} 76 77func main() -> i64 { 78 gv_head("=== nx_absa_bench_gate -- the SemEval-2014 SB1 scorer, in-process on a planted gold set under /tmp ===" as *u8) 79 let c: *i64 = gv_ctr() 80 gk_mkdir(G_ROOT) 81 gk_write(G_GOLD, G_ROWS) 82 let cbuf: *u8 = sys_mmap(G_CRLF_CAP) 83 let clen: i64 = g_crlf(G_ROWS, cbuf) 84 gk_write(G_GOLDCRLF, cbuf) 85 86 // ---- fixtures reached their condition ---- 87 gv_need("fixture-root-created" as *u8, gk_exists(G_ROOT), c) 88 gv_check_eq("fixture-reached-the-condition: gold file has twelve lines" as *u8, g_lines(G_GOLD), G_LINES, c) 89 gv_check("fixture-reached-the-condition: crlf twin is longer than the lf source" as *u8, (gk_size(G_GOLDCRLF) > gk_size(G_GOLD)) as i64, c) 90 91 // ---- F1 arithmetic (the load-bearing proof), from counts, in permil ---- 92 let p: *i64 = sys_mmap(AB_PRF_N * G_I64) as *i64 93 ab_prf(2, 3, 4, p) 94 gv_check_eq("prf-precision 2/3 = 666 permil" as *u8, p[AB_P], 666, c) 95 gv_check_eq("prf-recall 2/4 = 500 permil" as *u8, p[AB_R], 500, c) 96 gv_check_eq("prf-f1 2*2/(3+4) = 571 permil" as *u8, p[AB_F1], 571, c) 97 ab_prf(1, 4, 2, p) 98 gv_check_eq("prf-f1 2*1/(4+2) = 333 permil" as *u8, p[AB_F1], 333, c) 99 ab_prf(4, 4, 4, p) 100 gv_check_eq("prf-perfect-precision" as *u8, p[AB_P], 1000, c) 101 gv_check_eq("prf-perfect-recall" as *u8, p[AB_R], 1000, c) 102 gv_check_eq("prf-perfect-f1" as *u8, p[AB_F1], 1000, c) 103 ab_prf(0, 0, 4, p) 104 gv_check_eq("neg-control-zero-prediction-precision-is-zero-not-a-divide" as *u8, p[AB_P], 0, c) 105 gv_check_eq("neg-control-zero-prediction-recall-is-zero" as *u8, p[AB_R], 0, c) 106 gv_check_eq("neg-control-zero-prediction-f1-is-zero" as *u8, p[AB_F1], 0, c) 107 ab_prf(0, 3, 0, p) 108 gv_check_eq("neg-control-no-gold-f1-is-zero" as *u8, p[AB_F1], 0, c) 109 110 // ---- gold parser: distinct-term dedupe, multi-word term stored, CRLF survived ---- 111 let ng: i64 = ab_gold_load(G_GOLD) 112 gv_check_eq("gold-load-dedupes-to-three-distinct-terms" as *u8, ng, G_NG, c) 113 gv_check("gold-contains-a-single-word-term" as *u8, ab_set_has_raw("food" as *u8, 4), c) 114 gv_check("gold-normaliser-is-case-and-whitespace-insensitive" as *u8, ab_set_has_raw(" HARD DISK " as *u8, 15), c) 115 gv_check("neg-control-a-unigram-never-matches-a-multiword-gold-term" as *u8, (ab_set_has_raw("hard" as *u8, 4) == 0) as i64, c) 116 gv_check("neg-control-absent-term-not-in-gold" as *u8, (ab_set_has_raw("keyboard" as *u8, 8) == 0) as i64, c) 117 let ngc: i64 = ab_gold_load(G_GOLDCRLF) 118 gv_check_eq("gold-load-survives-crlf-and-still-dedupes-to-three" as *u8, ngc, G_NG, c) 119 gv_check("neg-control-crlf-term-has-no-trailing-cr" as *u8, ab_set_has_raw("service" as *u8, 7), c) 120 121 // ---- full pipeline: non-vacuous on the planted gold ---- 122 let out: *i64 = sys_mmap(AB_O_N * G_I64) as *i64 123 ab_eval(G_GOLD, 0, out) 124 gv_check_eq("eval-gold-terms-is-three" as *u8, out[AB_O_NG], G_NG, c) 125 gv_check_eq("eval-k-defaults-to-gold-size" as *u8, out[AB_O_K], G_NG, c) 126 gv_check("eval-predicts-at-least-one-term" as *u8, (out[AB_O_NS] > 0) as i64, c) 127 gv_check("eval-matches-at-least-one-gold-term" as *u8, (out[AB_O_INTER] > 0) as i64, c) 128 gv_check("eval-f1-is-positive-on-a-corpus-that-shares-terms-with-its-gold" as *u8, (out[AB_O_F1] > 0) as i64, c) 129 gv_check("eval-matched-never-exceeds-predicted" as *u8, (out[AB_O_INTER] <= out[AB_O_NS]) as i64, c) 130 gv_check("eval-matched-never-exceeds-gold" as *u8, (out[AB_O_INTER] <= out[AB_O_NG]) as i64, c) 131 132 // ---- subtask 2: polarity accuracy on the same planted gold, lexicons planted under /tmp ---- 133 gk_mkdir(G_LEXDIR) 134 gk_rm(G_ABSENT_GOLD) 135 gk_write(G_LEX_DEFECT, "slow\nnoisy\n" as *u8) 136 gk_write(G_LEX_NOTMEET, "refund\n" as *u8) 137 gk_write(G_LEX_EXCEED, "great\nfast\n" as *u8) 138 gv_check("fixture-reached-the-condition: three lexicons planted" as *u8, gk_exists(G_LEX_DEFECT) * gk_exists(G_LEX_NOTMEET) * gk_exists(G_LEX_EXCEED), c) 139 gv_check("fixture-reached-the-condition: absent gold has no file" as *u8, (gk_exists(G_ABSENT_GOLD) == 0) as i64, c) 140 let pq: *i64 = sys_mmap(AB_Q_N * G_I64) as *i64 141 ab_polarity(G_GOLD, G_LEXDIR, pq) 142 gv_check_eq("sb2-records-four" as *u8, pq[AB_Q_RECORDS], G_RECORDS, c) 143 gv_check_eq("sb2-gold-positive-two" as *u8, pq[AB_Q_POS], 2, c) 144 gv_check_eq("sb2-gold-negative-one" as *u8, pq[AB_Q_NEG], 1, c) 145 gv_check_eq("sb2-gold-neutral-one" as *u8, pq[AB_Q_NEU], 1, c) 146 gv_check_eq("sb2-gold-other-zero" as *u8, pq[AB_Q_OTHER], 0, c) 147 gv_check_eq("sb2-class-partition-sums-to-records" as *u8, pq[AB_Q_POS] + pq[AB_Q_NEG] + pq[AB_Q_NEU] + pq[AB_Q_OTHER], pq[AB_Q_RECORDS], c) 148 gv_check_eq("sb2-majority-class-is-positive" as *u8, pq[AB_Q_MAJ_CLASS], AB_POL_POS, c) 149 gv_check_eq("sb2-majority-accuracy 2/4 = 500 permil" as *u8, pq[AB_Q_MAJ_ACC], 500, c) 150 gv_check_eq("sb2-lexicon-terms-loaded-five" as *u8, pq[AB_Q_LEX_TERMS], G_LEX_TERMS, c) 151 gv_check_eq("sb2-lexicon-accuracy 3/4 = 750 permil (noisy reads the neutral record as negative: the honest miss)" as *u8, pq[AB_Q_LEX_ACC], 750, c) 152 gv_check_eq("sb2-lexicon-predicted-positive-two" as *u8, pq[AB_Q_LEX_PRED_POS], 2, c) 153 gv_check_eq("sb2-lexicon-predicted-negative-two" as *u8, pq[AB_Q_LEX_PRED_NEG], 2, c) 154 gv_check_eq("sb2-lexicon-predicted-neutral-zero" as *u8, pq[AB_Q_LEX_PRED_NEU], 0, c) 155 gv_check_eq("sb2-prediction-partition-sums-to-records" as *u8, pq[AB_Q_LEX_PRED_POS] + pq[AB_Q_LEX_PRED_NEG] + pq[AB_Q_LEX_PRED_NEU], pq[AB_Q_RECORDS], c) 156 ab_polarity(G_GOLD, G_NOLEXDIR, pq) 157 gv_check_eq("neg-control-no-lexicon-loads-zero-terms" as *u8, pq[AB_Q_LEX_TERMS], 0, c) 158 gv_check_eq("neg-control-no-lexicon-predicts-every-record-neutral" as *u8, pq[AB_Q_LEX_PRED_NEU], G_RECORDS, c) 159 gv_check_eq("neg-control-no-lexicon-accuracy-is-the-neutral-share 1/4 = 250" as *u8, pq[AB_Q_LEX_ACC], 250, c) 160 ab_polarity(G_ABSENT_GOLD, G_LEXDIR, pq) 161 gv_check_eq("neg-control-absent-gold-zero-records" as *u8, pq[AB_Q_RECORDS], 0, c) 162 gv_check_eq("neg-control-absent-gold-majority-accuracy-zero-not-a-divide" as *u8, pq[AB_Q_MAJ_ACC], 0, c) 163 gv_check_eq("neg-control-absent-gold-lexicon-accuracy-zero" as *u8, pq[AB_Q_LEX_ACC], 0, c) 164 ab_polarity(G_GOLD, G_LEXDIR, pq) 165 166 // ---- the frequent n-gram arm (Hu and Liu candidates with p-support pruning) on a gold with a repeated bigram ---- 167 gk_write(G_GOLD_NG, G_ROWS_NG) 168 gv_check_eq("fixture-reached-the-condition: n-gram gold has fifteen lines" as *u8, g_lines(G_GOLD_NG), G_NG_LINES, c) 169 let u2: *i64 = sys_mmap(AB_O_N * G_I64) as *i64 170 ab_eval(G_GOLD_NG, 0, u2) 171 let g: *i64 = sys_mmap(AB_G_N * G_I64) as *i64 172 ab_ngram_eval(G_GOLD_NG, 0, G_NG_MINSUP, g) 173 let ngf1: i64 = g[AB_G_F1] 174 gv_check_eq("ngram-records-five" as *u8, g[AB_G_RECORDS], G_NG_RECORDS, c) 175 gv_check_eq("ngram-min-support-pinned-two" as *u8, g[AB_G_MINSUP], G_NG_MINSUP, c) 176 gv_check_eq("ngram-frequent-bigram-hard-disk-found" as *u8, g[AB_G_BIGRAMS], 1, c) 177 gv_check_eq("ngram-psupport-prunes-the-two-fragments-hard-and-disk" as *u8, g[AB_G_PRUNED], G_NG_PRUNED, c) 178 gv_check("ngram-candidates-at-least-two-including-the-frequent-bigram" as *u8, (g[AB_G_CAND] >= 2) as i64, c) 179 gv_check_eq("ngram-predicted-equals-candidates" as *u8, g[AB_G_NS], g[AB_G_CAND], c) 180 gv_check_eq("ngram-multiword-gold-term-now-matched" as *u8, g[AB_G_MATCHED_BIGRAMS], 1, c) 181 gv_check_eq("ngram-matched-food-and-hard-disk-two" as *u8, g[AB_G_INTER], 2, c) 182 gv_check_eq("ngram-f1-recomputed-from-its-own-counts (matched 2, gold 3)" as *u8, g[AB_G_F1], 2 * g[AB_G_INTER] * AB_PERMIL / (g[AB_G_NS] + G_NG), c) 183 gv_check("ngram-arm-beats-the-unigram-arm-on-the-same-multiword-gold" as *u8, (g[AB_G_F1] > u2[AB_O_F1]) as i64, c) 184 ab_ngram_eval(G_GOLD_NG, 0, G_NG_MINSUP_ABOVE, g) 185 gv_check_eq("neg-control-support-above-the-bigram-frequency-finds-no-frequent-bigram" as *u8, g[AB_G_BIGRAMS], 0, c) 186 gv_check_eq("neg-control-nothing-pruned-without-a-frequent-bigram" as *u8, g[AB_G_PRUNED], 0, c) 187 gv_check_eq("neg-control-no-candidate-reaches-support-three-so-nothing-is-predicted" as *u8, g[AB_G_NS], 0, c) 188 gv_check_eq("neg-control-ngram-zero-prediction-f1-zero" as *u8, g[AB_G_F1], 0, c) 189 ab_ngram_eval(G_GOLD_NG, 0, 0, g) 190 gv_check_eq("ngram-derived-support-floors-at-one-on-five-records" as *u8, g[AB_G_MINSUP], AB_NG_MINSUP_FLOOR, c) 191 gv_check("ngram-derived-support-keeps-at-least-one-bigram-as-a-candidate" as *u8, (g[AB_G_BIGRAMS] >= 1) as i64, c) 192 193 // ---- the supervised training-dictionary tagger (rung IM28 first cut) on a planted train and test ---- 194 gk_write(G_TRAIN, G_TRAIN_ROWS) 195 gk_write(G_TEST, G_TEST_ROWS) 196 gv_check("fixture-reached-the-condition: train and test seg planted" as *u8, gk_exists(G_TRAIN) * gk_exists(G_TEST), c) 197 let d: *i64 = sys_mmap(AB_D_N * G_I64) as *i64 198 ab_dict_eval(G_TRAIN, G_TEST, d) 199 let dictf1: i64 = d[AB_D_F1] 200 gv_check_eq("dict-learns-four-training-terms" as *u8, d[AB_D_DICT], G_D_DICT, c) 201 gv_check_eq("dict-test-records-four" as *u8, d[AB_D_RECORDS], G_D_RECORDS, c) 202 gv_check_eq("dict-test-gold-terms-three" as *u8, d[AB_D_NG], G_D_NG, c) 203 gv_check_eq("dict-predicts-three: food, hard disk, keyboard (service unseen in the test corpus is not predicted)" as *u8, d[AB_D_NS], G_D_PRED, c) 204 gv_check_eq("dict-matches-food-and-hard-disk-two (keyboard is a false positive, mouse a miss, so a multi-word bigram matched)" as *u8, d[AB_D_INTER], G_D_INTER, c) 205 gv_check_eq("dict-precision 2/3 = 666 permil (keyboard predicted but not gold)" as *u8, d[AB_D_P], 666, c) 206 gv_check_eq("dict-recall 2/3 = 666 permil (mouse missed: absent from the training dictionary)" as *u8, d[AB_D_R], 666, c) 207 gv_check_eq("dict-f1-recomputed-from-its-own-counts" as *u8, d[AB_D_F1], 2 * d[AB_D_INTER] * AB_PERMIL / (d[AB_D_NS] + d[AB_D_NG]), c) 208 209 gv_values_head() 210 gv_kv("dict_f1_permil" as *u8, dictf1) 211 gv_kv("ngram_f1_permil_at_support_two" as *u8, ngf1) 212 gv_kv("unigram_f1_permil_same_gold" as *u8, u2[AB_O_F1]) 213 gv_kv("sb2_majority_acc_permil" as *u8, pq[AB_Q_MAJ_ACC]) 214 gv_kv("sb2_lexicon_acc_permil" as *u8, pq[AB_Q_LEX_ACC]) 215 gv_kv("gold_terms" as *u8, out[AB_O_NG]) 216 gv_kv("predicted" as *u8, out[AB_O_NS]) 217 gv_kv("matched" as *u8, out[AB_O_INTER]) 218 gv_kv("f1_permil" as *u8, out[AB_O_F1]) 219 gv_kv("vocab" as *u8, out[AB_O_VOCAB]) 220 return gv_verdict("nx_absa_bench_gate" as *u8, c, "the SemEval-2014 SB1 F1 scorer proven in-process on a planted gold set under /tmp: the F1 arithmetic at exact permil values with a zero-prediction and a no-gold control, distinct-term dedupe, CRLF tolerance, a case/whitespace-insensitive normaliser, the unigram-cannot-match-multiword honest weakness, a non-vacuous end-to-end eval, and the subtask-2 polarity arms (majority and lexicon) on planted lexicons with a no-lexicon control, and the frequent n-gram arm with p-support pruning on a gold whose multi-word term repeats, with a support-above-frequency control, and the supervised training-dictionary tagger with a false-positive and a miss planted so precision and recall are each below one; every fixture asserts its own condition first" as *u8) 221}