nx_absa_bench.nx source
↩ module page · 125 lines · 7380 B
1// nx_absa_bench.nx -- score the estate's unigram frequency baseline for aspect-term extraction against the
2// SemEval-2014 Task 4 subtask-1 gold (the pre-declared oracle for intelmine, rung IM17). ONE call:
3// nx_absa_bench <gold.seg> [k] [domain-label]
4// It composes nx_absa_lib (the scorer) which composes nx_reviewmine_lib (the ONE tokenizer and frequency
5// ranker). It prints P, R and F1 in permil with the counts, then the PUBLISHED winning F1s read from the
6// mirrored, pinned paper (knowledge/fetched/cmp_intelmine_semeval14_paper.pdf, Table 4) so the gap is visible:
7// this is a weak unigram baseline, not a SemEval submission, and the gap is the un-landed rm_aspects rung.
8// license_tier: ORIGINAL No hw writes (Rule 26).
9import "nx_syscalls.nx"
10import "nx_reviewmine_lib.nx"
11import "nx_absa_lib.nx"
12
13const BENCH_ARG_GOLD: i64 = 1
14const BENCH_ARG_K: i64 = 2
15const BENCH_ARG_LABEL: i64 = 3
16const BENCH_ARG_TRAIN: i64 = 4 // the TRAIN split for the supervised dictionary arm (optional)
17const BENCH_K_DEFAULT: i64 = 0 // 0 -> K = |G| (balanced baseline)
18const BENCH_EXIT_USAGE: i64 = 2
19const BENCH_EXIT_NOGOLD: i64 = 3
20// published SemEval-2014 Task 4 SB1 F1 in permil, read from the mirrored paper Table 4 (winners and the
21// organisers' dictionary baseline) -- CONTEXT the reader compares against, never a threshold this tool enforces
22const BENCH_WIN_REST: i64 = 840 // DLIREC 84.01 (restaurants, best)
23const BENCH_WIN_LAP: i64 = 745 // IHS RD 74.55 (laptops, best)
24const BENCH_BASE_REST: i64 = 471 // organisers dictionary baseline 47.15 (restaurants)
25const BENCH_BASE_LAP: i64 = 356 // organisers dictionary baseline 35.64 (laptops)
26// subtask 2 (aspect term polarity) accuracies in permil, Table 6 of the same paper, truncated like the SB1 figures
27const BENCH_SB2_WIN_REST: i64 = 809 // DCU 80.95 (restaurants, best)
28const BENCH_SB2_WIN_LAP: i64 = 704 // DCU and NRC-Canada 70.48 (laptops, best)
29const BENCH_SB2_BASE_REST: i64 = 642 // organisers baseline 64.28
30const BENCH_SB2_BASE_LAP: i64 = 513 // organisers baseline 51.37
31const BENCH_SB2_MAJ_REST: i64 = 641 // majority 64.19
32const BENCH_SB2_MAJ_LAP: i64 = 521 // majority 52.14
33
34func main(argc: i64, argv: *i64) -> i64 {
35 if argc <= BENCH_ARG_GOLD {
36 rm_w("usage: nx_absa_bench <gold.seg> [k] [domain-label] (k<=0 -> K=|G|, a balanced baseline)\n" as *u8)
37 sys_exit(BENCH_EXIT_USAGE); return BENCH_EXIT_USAGE
38 }
39 let gold: *u8 = argv[BENCH_ARG_GOLD] as *u8
40 var k: i64 = BENCH_K_DEFAULT
41 if argc > BENCH_ARG_K { k = rm_atoi(argv[BENCH_ARG_K] as *u8) }
42 var label: *u8 = gold
43 if argc > BENCH_ARG_LABEL { label = argv[BENCH_ARG_LABEL] as *u8 }
44
45 let out: *i64 = sys_mmap(AB_O_N * RM_I64_BYTES) as *i64
46 ab_eval(gold, k, out)
47 if out[AB_O_NG] <= 0 {
48 rm_w("ABSA-BENCH REFUSED no gold terms loaded from " as *u8); rm_w(gold)
49 rm_w(" -- check the path and the three-line .seg format\n" as *u8)
50 sys_exit(BENCH_EXIT_NOGOLD); return BENCH_EXIT_NOGOLD
51 }
52 rm_w("ABSA-BENCH domain=" as *u8); rm_w(label)
53 rm_w(" gold_terms=" as *u8); rm_wn(out[AB_O_NG])
54 rm_w(" predicted=" as *u8); rm_wn(out[AB_O_NS])
55 rm_w(" matched=" as *u8); rm_wn(out[AB_O_INTER])
56 rm_w(" precision_permil=" as *u8); rm_wn(out[AB_O_P])
57 rm_w(" recall_permil=" as *u8); rm_wn(out[AB_O_R])
58 rm_w(" f1_permil=" as *u8); rm_wn(out[AB_O_F1])
59 rm_w(" k=" as *u8); rm_wn(out[AB_O_K])
60 rm_w(" vocab=" as *u8); rm_wn(out[AB_O_VOCAB])
61 rm_w(" metric=unigram-df-baseline-vs-semeval2014-sb1-exact-term-set\n" as *u8)
62 rm_w(" BAR published SemEval-2014 Task 4 SB1 F1 (permil): restaurants winner DLIREC=" as *u8); rm_wn(BENCH_WIN_REST)
63 rm_w(" laptops winner IHS_RD=" as *u8); rm_wn(BENCH_WIN_LAP)
64 rm_w(" organisers dictionary baseline restaurants=" as *u8); rm_wn(BENCH_BASE_REST)
65 rm_w(" laptops=" as *u8); rm_wn(BENCH_BASE_LAP)
66 rm_w(" [@semeval14]\n" as *u8)
67 let g: *i64 = sys_mmap(AB_G_N * RM_I64_BYTES) as *i64
68 ab_ngram_eval(gold, k, 0, g)
69 rm_w("ABSA-BENCH-NGRAM domain=" as *u8); rm_w(label)
70 rm_w(" gold_terms=" as *u8); rm_wn(g[AB_G_NG])
71 rm_w(" predicted=" as *u8); rm_wn(g[AB_G_NS])
72 rm_w(" matched=" as *u8); rm_wn(g[AB_G_INTER])
73 rm_w(" matched_bigrams=" as *u8); rm_wn(g[AB_G_MATCHED_BIGRAMS])
74 rm_w(" precision_permil=" as *u8); rm_wn(g[AB_G_P])
75 rm_w(" recall_permil=" as *u8); rm_wn(g[AB_G_R])
76 rm_w(" f1_permil=" as *u8); rm_wn(g[AB_G_F1])
77 rm_w(" k=" as *u8); rm_wn(g[AB_G_K])
78 rm_w(" candidates=" as *u8); rm_wn(g[AB_G_CAND])
79 rm_w(" frequent_bigrams=" as *u8); rm_wn(g[AB_G_BIGRAMS])
80 rm_w(" pruned_unigrams=" as *u8); rm_wn(g[AB_G_PRUNED])
81 rm_w(" min_support=" as *u8); rm_wn(g[AB_G_MINSUP])
82 rm_w(" records=" as *u8); rm_wn(g[AB_G_RECORDS])
83 rm_w(" truncated=" as *u8); rm_wn(g[AB_G_TRUNC])
84 rm_w(" metric=hu-liu-frequent-ngram-with-psupport-pruning-vs-semeval2014-sb1-exact-term-set [@hu-liu-2004]\n" as *u8)
85 let pq: *i64 = sys_mmap(AB_Q_N * RM_I64_BYTES) as *i64
86 ab_polarity(gold, AB_LEX_DIR_DEFAULT, pq)
87 rm_w("ABSA-BENCH-SB2 domain=" as *u8); rm_w(label)
88 rm_w(" records=" as *u8); rm_wn(pq[AB_Q_RECORDS])
89 rm_w(" positive=" as *u8); rm_wn(pq[AB_Q_POS])
90 rm_w(" negative=" as *u8); rm_wn(pq[AB_Q_NEG])
91 rm_w(" neutral=" as *u8); rm_wn(pq[AB_Q_NEU])
92 rm_w(" other=" as *u8); rm_wn(pq[AB_Q_OTHER])
93 rm_w(" majority=" as *u8); rm_w(ab_pol_name(pq[AB_Q_MAJ_CLASS]))
94 rm_w(" majority_acc_permil=" as *u8); rm_wn(pq[AB_Q_MAJ_ACC])
95 rm_w(" lexicon_terms=" as *u8); rm_wn(pq[AB_Q_LEX_TERMS])
96 rm_w(" lexicon_acc_permil=" as *u8); rm_wn(pq[AB_Q_LEX_ACC])
97 rm_w(" lexicon_pred_positive=" as *u8); rm_wn(pq[AB_Q_LEX_PRED_POS])
98 rm_w(" lexicon_pred_negative=" as *u8); rm_wn(pq[AB_Q_LEX_PRED_NEG])
99 rm_w(" lexicon_pred_neutral=" as *u8); rm_wn(pq[AB_Q_LEX_PRED_NEU])
100 rm_w(" metric=accuracy-over-gold-aspect-terms-semeval2014-sb2\n" as *u8)
101 rm_w(" BAR published SemEval-2014 Task 4 SB2 accuracy (permil): restaurants winner DCU=" as *u8); rm_wn(BENCH_SB2_WIN_REST)
102 rm_w(" laptops winners DCU and NRC-Canada=" as *u8); rm_wn(BENCH_SB2_WIN_LAP)
103 rm_w(" organisers baseline restaurants=" as *u8); rm_wn(BENCH_SB2_BASE_REST)
104 rm_w(" laptops=" as *u8); rm_wn(BENCH_SB2_BASE_LAP)
105 rm_w(" majority restaurants=" as *u8); rm_wn(BENCH_SB2_MAJ_REST)
106 rm_w(" laptops=" as *u8); rm_wn(BENCH_SB2_MAJ_LAP)
107 rm_w(" [@semeval14-paper]\n" as *u8)
108 if argc > BENCH_ARG_TRAIN {
109 let dtrain: *u8 = argv[BENCH_ARG_TRAIN] as *u8
110 let dq: *i64 = sys_mmap(AB_D_N * RM_I64_BYTES) as *i64
111 ab_dict_eval(dtrain, gold, dq)
112 rm_w("ABSA-BENCH-DICT domain=" as *u8); rm_w(label)
113 rm_w(" gold_terms=" as *u8); rm_wn(dq[AB_D_NG])
114 rm_w(" predicted=" as *u8); rm_wn(dq[AB_D_NS])
115 rm_w(" matched=" as *u8); rm_wn(dq[AB_D_INTER])
116 rm_w(" precision_permil=" as *u8); rm_wn(dq[AB_D_P])
117 rm_w(" recall_permil=" as *u8); rm_wn(dq[AB_D_R])
118 rm_w(" f1_permil=" as *u8); rm_wn(dq[AB_D_F1])
119 rm_w(" dict_terms=" as *u8); rm_wn(dq[AB_D_DICT])
120 rm_w(" records=" as *u8); rm_wn(dq[AB_D_RECORDS])
121 rm_w(" metric=supervised-training-dictionary-tagger-vs-semeval2014-sb1-exact-term-set [@semeval14]\n" as *u8)
122 }
123 sys_exit(0)
124 return 0
125}