nx_absa_seqextract.nx source
↩ module page · 163 lines · 9424 B
1// nx_absa_seqextract.nx -- rung IM28: train the averaged structured perceptron (nx_absa_seq) on a SemEval TRAIN
2// split and score aspect-term extraction on the TEST split. ONE call:
3// nx_absa_seqextract <train.seg> <test.seg> [label]
4// Prints P, R and F1 in permil with the counts beside the published winners, so the distance to SOTA is visible.
5// license_tier: ORIGINAL No hw writes (Rule 26).
6import "nx_syscalls.nx"
7import "nx_reviewmine_lib.nx"
8import "nx_absa_lib.nx"
9import "nx_absa_seq.nx"
10
11const SE_ARG_TRAIN: i64 = 1
12const SE_ARG_TEST: i64 = 2
13const SE_ARG_LABEL: i64 = 3
14const SE_ARG_TREEBANK: i64 = 4 // optional: a CoNLL-U treebank -> POS and parse features on
15const SE_ARG_MODE: i64 = 5 // optional: parse -> add the parser's features on top of the tagger (off by default,
16 // measured to lower F1 with the 5-epoch parser); noparse -> the default, spelled out
17const SE_MODE_PARSE: *u8 = "parse"
18const SE_MODE_PARSE_LEN: i64 = 5
19const SE_MODE_NOPARSE: *u8 = "noparse"
20const SE_MODE_NOPARSE_LEN: i64 = 7
21const SE_MODE_SHUFFLE: *u8 = "shuffle" // per-epoch shuffle of the training groups (off by default: measured a non-result)
22const SE_MODE_SHUFFLE_LEN: i64 = 7
23const SE_MODE_CLUSTERS: *u8 = "clusters" // PPMI word-cluster features from the estate's own model (off until measured)
24const SE_MODE_CLUSTERS_LEN: i64 = 8
25const SE_CLUST_MODEL: *u8 = "knowledge/index/semppmi_v1.bin" // the ONE PPMI model nx_semppmi_build maintains
26const SE_MODE_CRF: *u8 = "crf" // the conditional log-likelihood objective (off until measured)
27const SE_MODE_CRF_LEN: i64 = 3
28const SE_MODE_EMBED: *u8 = "embed" // dense embedding coordinate features from the estate's own table (off until measured)
29const SE_MODE_EMBED_LEN: i64 = 5
30const SE_EMB_MODEL: *u8 = "knowledge/index/embed_v1.bin" // the ONE embedding table nx_embed_train writes
31const SE_EXIT_USAGE: i64 = 2
32const SE_EXIT_NOGOLD: i64 = 3
33const SE_WIN_REST: i64 = 840 // DLIREC 84.01
34const SE_WIN_LAP: i64 = 745 // IHS RD 74.55
35
36func main(argc: i64, argv: *i64) -> i64 {
37 if argc <= SE_ARG_TEST {
38 rm_w("usage: nx_absa_seqextract <train.seg> <test.seg> [label]\n" as *u8)
39 sys_exit(SE_EXIT_USAGE); return SE_EXIT_USAGE
40 }
41 let train: *u8 = argv[SE_ARG_TRAIN] as *u8
42 let test: *u8 = argv[SE_ARG_TEST] as *u8
43 var label: *u8 = test
44 if argc > SE_ARG_LABEL { label = argv[SE_ARG_LABEL] as *u8 }
45 if argc > SE_ARG_TREEBANK { sp_set_treebank(argv[SE_ARG_TREEBANK] as *u8) }
46 // every argument from the fifth on is a mode token: parse, noparse, shuffle (unknown tokens are ignored, announced below)
47 var ai: i64 = SE_ARG_MODE
48 while ai < argc {
49 let m: *u8 = argv[ai] as *u8
50 var same: i64 = 1
51 var mi: i64 = 0
52 while mi < SE_MODE_NOPARSE_LEN { if m[mi] != SE_MODE_NOPARSE[mi] { same = 0 } mi = mi + 1 }
53 if m[SE_MODE_NOPARSE_LEN] != 0 { same = 0 }
54 if same == 1 { sp_set_parse(0) }
55 var samep: i64 = 1
56 var mp: i64 = 0
57 while mp < SE_MODE_PARSE_LEN { if m[mp] != SE_MODE_PARSE[mp] { samep = 0 } mp = mp + 1 }
58 if m[SE_MODE_PARSE_LEN] != 0 { samep = 0 }
59 if samep == 1 { sp_set_parse(1) }
60 var sames: i64 = 1
61 var ms: i64 = 0
62 while ms < SE_MODE_SHUFFLE_LEN { if m[ms] != SE_MODE_SHUFFLE[ms] { sames = 0 } ms = ms + 1 }
63 if m[SE_MODE_SHUFFLE_LEN] != 0 { sames = 0 }
64 if sames == 1 { sp_set_shuffle(1) }
65 var samec: i64 = 1
66 var mc: i64 = 0
67 while mc < SE_MODE_CLUSTERS_LEN { if m[mc] != SE_MODE_CLUSTERS[mc] { samec = 0 } mc = mc + 1 }
68 if m[SE_MODE_CLUSTERS_LEN] != 0 { samec = 0 }
69 if samec == 1 { sp_set_clusters(SE_CLUST_MODEL as *u8) }
70 var samef: i64 = 1
71 var mf: i64 = 0
72 while mf < SE_MODE_CRF_LEN { if m[mf] != SE_MODE_CRF[mf] { samef = 0 } mf = mf + 1 }
73 if m[SE_MODE_CRF_LEN] != 0 { samef = 0 }
74 if samef == 1 { sp_set_crf(1) }
75 var samee: i64 = 1
76 var me: i64 = 0
77 while me < SE_MODE_EMBED_LEN { if m[me] != SE_MODE_EMBED[me] { samee = 0 } me = me + 1 }
78 if m[SE_MODE_EMBED_LEN] != 0 { samee = 0 }
79 if samee == 1 { sp_set_embed(SE_CLUST_MODEL as *u8, SE_EMB_MODEL as *u8) }
80 ai = ai + 1
81 }
82 let out: *i64 = sys_mmap(SP_O_N * RM_I64_BYTES) as *i64
83 sp_eval(train, test, out)
84 if out[SP_O_NG] <= 0 {
85 rm_w("SEQEXTRACT REFUSED no gold terms in " as *u8); rm_w(test); rm_w("\n" as *u8)
86 sys_exit(SE_EXIT_NOGOLD); return SE_EXIT_NOGOLD
87 }
88 rm_w("ABSA-SEQEXTRACT domain=" as *u8); rm_w(label)
89 rm_w(" gold_terms=" as *u8); rm_wn(out[SP_O_NG])
90 rm_w(" predicted=" as *u8); rm_wn(out[SP_O_NS])
91 rm_w(" matched=" as *u8); rm_wn(out[SP_O_INTER])
92 rm_w(" precision_permil=" as *u8); rm_wn(out[SP_O_P])
93 rm_w(" recall_permil=" as *u8); rm_wn(out[SP_O_R])
94 rm_w(" f1_permil=" as *u8); rm_wn(out[SP_O_F1])
95 rm_w(" dict_terms=" as *u8); rm_wn(out[SP_O_DICT])
96 rm_w(" train_records=" as *u8); rm_wn(out[SP_O_TRAINREC])
97 rm_w(" train_skipped=" as *u8); rm_wn(out[SP_O_TRAINSKIP])
98 rm_w(" train_sentences=" as *u8); rm_wn(out[SP_O_TRAINSENT])
99 rm_w(" test_records=" as *u8); rm_wn(out[SP_O_TESTREC])
100 rm_w(" epochs=" as *u8); rm_wn(out[SP_O_EPOCHS])
101 rm_w(" last_epoch_updates=" as *u8); rm_wn(out[SP_O_UPDATES])
102 rm_w(" pos_features=" as *u8); rm_wn(out[SP_O_POS])
103 rm_w(" pos_train_tokens=" as *u8); rm_wn(out[SP_O_POS_TOK])
104 rm_w(" parse_features=" as *u8); rm_wn(out[SP_O_PARSE])
105 rm_w(" parse_train_sentences=" as *u8); rm_wn(out[SP_O_PARSE_SENT])
106 rm_w(" train_groups=" as *u8); rm_wn(out[SP_O_GROUPS])
107 rm_w(" shuffle=" as *u8); rm_wn(out[SP_O_SHUF])
108 rm_w(" clusters=" as *u8); rm_wn(out[SP_O_CLUST])
109 rm_w(" cluster_vocab=" as *u8); rm_wn(out[SP_O_CLUST_VOCAB])
110 rm_w(" cluster_inmodel=" as *u8); rm_wn(out[SP_O_CLUST_INMODEL])
111 rm_w(" crf=" as *u8); rm_wn(out[SP_O_CRF])
112 rm_w(" crf_nll_first=" as *u8); rm_wn(out[SP_O_CRF_NLL_FIRST])
113 rm_w(" crf_nll_last=" as *u8); rm_wn(out[SP_O_CRF_NLL_LAST])
114 rm_w(" crf_margdev=" as *u8); rm_wn(out[SP_O_CRF_MARGDEV])
115 rm_w(" crf_negnll=" as *u8); rm_wn(out[SP_O_CRF_NEGNLL])
116 rm_w(" embed=" as *u8); rm_wn(out[SP_O_EMB])
117 rm_w(" embed_dim=" as *u8); rm_wn(out[SP_O_EMB_DIM])
118 rm_w(" embed_lookups=" as *u8); rm_wn(out[SP_O_EMB_LOOKUPS])
119 rm_w(" embed_hits=" as *u8); rm_wn(out[SP_O_EMB_HITS])
120 rm_w(" metric=averaged-structured-perceptron-bio-viterbi-vs-semeval2014-sb1-exact-term-set [@semeval14-paper]\n" as *u8)
121 rm_w("ABSA-SEQEXTRACT-UNION domain=" as *u8); rm_w(label)
122 rm_w(" gold_terms=" as *u8); rm_wn(out[SP_O_NG])
123 rm_w(" predicted=" as *u8); rm_wn(out[SP_O_U_NS])
124 rm_w(" matched=" as *u8); rm_wn(out[SP_O_U_INTER])
125 rm_w(" precision_permil=" as *u8); rm_wn(out[SP_O_U_P])
126 rm_w(" recall_permil=" as *u8); rm_wn(out[SP_O_U_R])
127 rm_w(" f1_permil=" as *u8); rm_wn(out[SP_O_U_F1])
128 rm_w(" dict_majority=" as *u8); rm_wn(out[SP_O_DICT_MAJ])
129 rm_w(" metric=perceptron-UNION-majority-rule-dictionary-vs-semeval2014-sb1-exact-term-set [@semeval14-paper]\n" as *u8)
130 rm_w("ABSA-SEQEXTRACT-OCC domain=" as *u8); rm_w(label)
131 rm_w(" gold_occurrences=" as *u8); rm_wn(out[SP_O_OC_NG])
132 rm_w(" predicted=" as *u8); rm_wn(out[SP_O_OC_NS])
133 rm_w(" matched=" as *u8); rm_wn(out[SP_O_OC_TP])
134 rm_w(" precision_permil=" as *u8); rm_wn(out[SP_O_OC_P])
135 rm_w(" recall_permil=" as *u8); rm_wn(out[SP_O_OC_R])
136 rm_w(" f1_permil=" as *u8); rm_wn(out[SP_O_OC_F1])
137 rm_w(" test_sentences=" as *u8); rm_wn(out[SP_O_TESTSENT])
138 rm_w(" metric=averaged-structured-perceptron-PER-OCCURRENCE-the-paper-metric-semeval2014-sb1 [@semeval14-paper]\n" as *u8)
139 rm_w("ABSA-SEQEXTRACT-UNION-OCC domain=" as *u8); rm_w(label)
140 rm_w(" gold_occurrences=" as *u8); rm_wn(out[SP_O_OC_NG])
141 rm_w(" predicted=" as *u8); rm_wn(out[SP_O_OCU_NS])
142 rm_w(" matched=" as *u8); rm_wn(out[SP_O_OCU_TP])
143 rm_w(" precision_permil=" as *u8); rm_wn(out[SP_O_OCU_P])
144 rm_w(" recall_permil=" as *u8); rm_wn(out[SP_O_OCU_R])
145 rm_w(" f1_permil=" as *u8); rm_wn(out[SP_O_OCU_F1])
146 rm_w(" metric=perceptron-UNION-dictionary-PER-OCCURRENCE-the-paper-metric-semeval2014-sb1 [@semeval14-paper]\n" as *u8)
147 rm_w("ABSA-SEQEXTRACT-MISSES domain=" as *u8); rm_w(label)
148 rm_w(" gold_occurrences=" as *u8); rm_wn(out[SP_O_OC_NG])
149 rm_w(" perceptron_matched=" as *u8); rm_wn(out[SP_O_OC_TP])
150 rm_w(" untokenisable=" as *u8); rm_wn(out[SP_O_EC_UNTOK])
151 rm_w(" unseen_single=" as *u8); rm_wn(out[SP_O_EC_UNSEEN1])
152 rm_w(" unseen_multi=" as *u8); rm_wn(out[SP_O_EC_UNSEENM])
153 rm_w(" seen_single=" as *u8); rm_wn(out[SP_O_EC_SEEN1])
154 rm_w(" seen_multi=" as *u8); rm_wn(out[SP_O_EC_SEENM])
155 rm_w(" parts_sum=" as *u8); rm_wn(out[SP_O_EC_UNTOK] + out[SP_O_EC_UNSEEN1] + out[SP_O_EC_UNSEENM] + out[SP_O_EC_SEEN1] + out[SP_O_EC_SEENM])
156 rm_w(" union_missed=" as *u8); rm_wn(out[SP_O_EC_UNIONMISS])
157 rm_w(" metric=perceptron-misses-partitioned-by-cause-the-parts-must-sum-to-gold-minus-matched\n" as *u8)
158 rm_w(" BAR published SemEval-2014 Task 4 SB1 F1 (permil): restaurants winner DLIREC=" as *u8); rm_wn(SE_WIN_REST)
159 rm_w(" laptops winner IHS_RD=" as *u8); rm_wn(SE_WIN_LAP)
160 rm_w(" [@semeval14]\n" as *u8)
161 sys_exit(0)
162 return 0
163}