nx_absa_seq.nx source

1// nx_absa_seq.nx -- rung IM28: a SUPERVISED SEQUENCE model for aspect-term extraction (SemEval-2014 Task 4 SB1), 2// the step that closes the gap from the dictionary baseline (~46 F1) toward the CRF winners (74-84 F1). It is an 3// AVERAGED STRUCTURED PERCEPTRON with Viterbi BIO decoding -- integer weights only, so it runs in the nofloat 4// estate; the CRF winners add a probabilistic objective and POS/parse features, which we do not have a tagger for, 5// so this is the honest reachable model, not a claim of parity. Trained on the TRAIN split, decoded on the TEST 6// split, scored by the same set-overlap F1 (nx_absa_lib) the other arms use so the numbers are comparable. 7// 8// FEATURES per token (the hashing trick, one big integer weight table): word, 3-char prefix and suffix, previous 9// and next word, a has-digit shape, dictionary membership (the supervised dictionary signal folded in as a 10// feature), and a bias. TRANSITIONS between BIO tags, with I only reachable from B or I. Averaging is the standard 11// lazy scheme (a weight's average is its running sum plus its current value times the time since it last changed); 12// for the Viterbi argmax the common divisor is dropped. license_tier: ORIGINAL No hw writes (Rule 26). LIB. 13import "nx_syscalls.nx" 14import "nx_reviewmine_lib.nx" 15import "nx_absa_lib.nx" 16import "nx_postag.nx" 17import "nx_depparse.nx" 18import "nx_wordclust.nx" 19import "nx_logadd_lib.nx" 20import "nx_embfeat_lib.nx" 21 22const SP_W: i64 = 1048576 // 2^20 feature-weight slots 23const SP_WMASK: i64 = 1048575 24const SP_NTAGS: i64 = 3 25const SP_O: i64 = 0 26const SP_B: i64 = 1 27const SP_I: i64 = 2 28const SP_NTRANS: i64 = 9 // SP_NTAGS * SP_NTAGS 29const SP_EPOCHS: i64 = 16 // 8 left a third of sentences still updating; doubled for convergence 30const SP_TOK_CAP: i64 = 256 // tokens per sentence 31const SP_TOKBUF: i64 = 16384 32const SP_FEATBUF: i64 = 128 33const SP_NFEAT: i64 = 53 // 8 base + prev2, next2, prev+cur, cur+next, 4-char suffix, dictionary reliability, 5 POS, 5 parse, 5 cluster, 24 embedding 34// DENSE EMBEDDING features (a MODE, off by default): the estate's own trained word vectors (nx_embed_train's embed_v1.bin, 35// 24 Q10 coordinates per word, row index shared with the PPMI vocabulary) read through nx_embfeat_lib, each coordinate 36// quantised to a signed bucket at half-RMS steps derived from the table. One hashed family per coordinate; a word the 37// vocabulary does not know spells '@' at every coordinate and degrades to the base features. 38const SP_PFX_EMB: i64 = 108 // 'l' embedding coordinate families (the payload carries the coordinate index) 39const SP_EMB_BASE: i64 = 29 // first embedding feature index 40const SP_EMB_DIMS: i64 = 24 // the table's coordinate count; a table of another width refuses the mode 41// WORD-CLUSTER features (the winners' other lever, name lists and Brown clusters from in-domain text): when a PPMI model is 42// set (sp_set_clusters), nx_wordclust clusters the training vocabulary by PPMI cosine at three granularities and every 43// token reads its cluster id at all three plus its neighbours' at the middle one -- five hashed families. A word the model 44// does not know spells '@', so an unclustered token degrades to the base features, never to garbage. 45const SP_PFX_CL0: i64 = 90 // 'Z' this token's cluster at the coarsest level 46const SP_PFX_CL1: i64 = 97 // 'a' this token's cluster at the middle level 47const SP_PFX_CL2: i64 = 105 // 'i' this token's cluster at the finest level 48const SP_PFX_CLP: i64 = 106 // 'j' previous token's cluster, middle level 49const SP_PFX_CLN: i64 = 107 // 'k' next token's cluster, middle level 50const SP_CLUST_BASE: i64 = 24 // first cluster feature index 51const SP_CL_MID: i64 = 1 // the middle granularity's level index 52// CRF OBJECTIVE (a MODE, off by default): the same hashed features and transition table, the same lazy averaging and the 53// same Viterbi decoder, trained on the conditional log-likelihood instead of the perceptron's mistake-driven votes. In 54// this mode a weight is Q10 nats (nx_logadd_lib's scale), forward-backward runs in the log domain, and every merged 55// sentence takes one stochastic-gradient step w += eta * (empirical - expected) with the expected counts read off the 56// token and pair marginals. The step is DERIVED, not tuned: eta_0 = 1/4 nat per unit of gradient (a full-confidence error 57// moves a weight a quarter nat, so four such errors in one epoch reach one nat, the perceptron's single vote in this 58// scale), decayed as eta_0 / (1 + epoch) -- the Robbins-Monro schedule every SGD text derives -- and the averaging already 59// in place plays the regulariser's part. Neither constant is moved against the test split. 60const SP_CRF_ETA_NUM: i64 = 1 61const SP_CRF_ETA_DEN: i64 = 4 62const SP_NPAIR: i64 = 9 // SP_NTAGS * SP_NTAGS pair marginals per position 63const SP_CRF_NLL_SLACK: i64 = 8 // Q10 units a chain of log-adds may lose: a sentence NLL below -slack is impossible and counted 64// DEPENDENCY features (DLIREC's lever): when a treebank is set the parser (nx_depparse) is trained on it after the tagger 65// and every sentence is parsed after tagging; this token's relation, its head word, its head's tag, relation+own tag and 66// the relations of its leftmost and rightmost dependents enter as their own families. sp_set_parse(0) keeps the tagger 67// and drops the parser, the ablation switch that separates the two levers on one binary. 68const SP_PFX_REL: i64 = 82 // 'R' relation to the head 69const SP_PFX_HW: i64 = 72 // 'H' head word ('^' for the root) 70const SP_PFX_HT: i64 = 71 // 'G' head tag ('$' for the root) 71const SP_PFX_RELT: i64 = 74 // 'J' relation + own tag 72const SP_PFX_CHR: i64 = 75 // 'K' relations of the leftmost and rightmost dependents 73const SP_PARSE_BASE: i64 = 19 // first parse feature index 74// PART-OF-SPEECH features (the winners' missing lever): when a treebank is set (sp_set_treebank), nx_postag is trained on 75// it once and every sentence is tagged after tokenising; the tag of this token, the previous and the next, and the two 76// tag bigrams enter the weight table as their own families. With no treebank every POS byte reads '^' (tag -1), so the 77// model degrades to the 14-feature form rather than to garbage. 78const SP_PFX_POS: i64 = 84 // 'T' tag of this token 79const SP_PFX_POSP: i64 = 85 // 'U' previous token's tag 80const SP_PFX_POSN: i64 = 86 // 'V' next token's tag 81const SP_PFX_POSPC: i64 = 88 // 'X' previous+current tag bigram 82const SP_PFX_POSCN: i64 = 89 // 'Y' current+next tag bigram 83const SP_POS_BASE: i64 = 14 // first POS feature index 84// DICTIONARY RELIABILITY: the training dictionary is every term that was EVER an aspect, including words that are mostly 85// not aspects in the very training data (laptops: union precision 466 from that). The train split already knows each 86// term's reliability -- occurrences as a gold aspect over occurrences at all -- so it is (a) a GRADED feature the 87// perceptron weighs instead of a binary in-dictionary bit, and (b) the MAJORITY RULE for the union: a dictionary hit is 88// admitted only if the term was an aspect in at least half its training occurrences. A derived bar, not a tuned one. 89const SP_BUCKETS: i64 = 4 // reliability quantised to fifths: 0..4 of SP_BUCKETS 90const SP_PFX_E: i64 = 101 // 'e' dictionary reliability bucket 91const SP_CH_DASH: i64 = 45 // '-' the bucket of a token not in the dictionary 92const SP_MAJ_NUM: i64 = 2 // gold * 2 >= seen <=> an aspect in at least half its occurrences 93const SP_AFF: i64 = 3 // affix length 94const SP_AFF4: i64 = 4 // the longer suffix (a second affix family, not a replacement) 95const SP_NEG: i64 = 0 - 1000000000 // Viterbi -infinity 96const SP_GRAM_CAP: i64 = 512 97// feature prefix bytes (distinguish the feature families in the shared hash space) 98const SP_PFX_W: i64 = 119 // 'w' 99const SP_PFX_P: i64 = 112 // 'p' prefix 100const SP_PFX_S: i64 = 115 // 's' suffix 101const SP_PFX_PW: i64 = 80 // 'P' previous word 102const SP_PFX_NW: i64 = 78 // 'N' next word 103const SP_PFX_SH: i64 = 104 // 'h' shape 104const SP_PFX_D: i64 = 100 // 'd' dictionary membership 105const SP_PFX_B: i64 = 98 // 'b' bias 106const SP_PFX_PW2: i64 = 81 // 'Q' word two back 107const SP_PFX_NW2: i64 = 77 // 'M' word two ahead 108const SP_PFX_PC: i64 = 99 // 'c' previous+current conjunction 109const SP_PFX_CN: i64 = 67 // 'C' current+next conjunction 110const SP_PFX_S4: i64 = 83 // 'S' 4-char suffix 111const SP_CH_DIGIT0: i64 = 48 112const SP_CH_DIGIT9: i64 = 57 113const SP_CH_ONE: i64 = 49 114const SP_CH_ZERO: i64 = 48 115const SP_CH_CARET: i64 = 94 // '^' BOS 116const SP_CH_DOLLAR2: i64 = 36 // '$' EOS 117// out[] indices 118const SP_O_INTER: i64 = 0 119const SP_O_NS: i64 = 1 120const SP_O_NG: i64 = 2 121const SP_O_P: i64 = 3 122const SP_O_R: i64 = 4 123const SP_O_F1: i64 = 5 124const SP_O_DICT: i64 = 6 125const SP_O_TRAINREC: i64 = 7 // train records used (term found and labelled) 126const SP_O_TRAINSKIP: i64 = 8 // train records skipped (term not a contiguous token run) 127const SP_O_TESTREC: i64 = 9 128const SP_O_EPOCHS: i64 = 10 129const SP_O_UPDATES: i64 = 11 // perceptron updates in the last epoch 130const SP_O_TRAINSENT: i64 = 12 // MERGED training sentences (consecutive same-sentence records folded into one) 131// the UNION arm: perceptron predictions unioned with the training-dictionary tagger's, scored as a SECOND line so the 132// perceptron-only number above stays comparable across runs 133const SP_O_U_INTER: i64 = 13 134const SP_O_U_NS: i64 = 14 135const SP_O_U_P: i64 = 15 136const SP_O_U_R: i64 = 16 137const SP_O_U_F1: i64 = 17 138// the PAPER's metric: per-OCCURRENCE (SemEval-2014 Task 4 eq. 1-2 compares the sets of aspect-term annotations 139// across the test sentences, so a term that appears in forty sentences is forty gold items). The distinct-set lines 140// above count each term once and so under-score a model that gets frequent terms right; both are published. 141const SP_O_OC_TP: i64 = 18 142const SP_O_OC_NS: i64 = 19 143const SP_O_OC_NG: i64 = 20 144const SP_O_OC_P: i64 = 21 145const SP_O_OC_R: i64 = 22 146const SP_O_OC_F1: i64 = 23 147const SP_O_OCU_TP: i64 = 24 148const SP_O_OCU_NS: i64 = 25 149const SP_O_OCU_P: i64 = 26 150const SP_O_OCU_R: i64 = 27 151const SP_O_OCU_F1: i64 = 28 152const SP_O_TESTSENT: i64 = 29 // merged test sentences 153const SP_O_DICT_MAJ: i64 = 30 // dictionary terms passing the majority rule (an aspect in >= half their training occurrences) 154const SP_O_POS: i64 = 31 // 1 when a treebank was set and the tagger trained, else 0 155const SP_O_POS_TOK: i64 = 32 // tagger training tokens (0 without a treebank) 156const SP_O_PARSE: i64 = 33 // 1 when the parser trained and every sentence was parsed, else 0 157const SP_O_PARSE_SENT: i64 = 34 // parser training sentences (0 without a treebank or with the parse switch off) 158const SP_O_GROUPS: i64 = 35 // merged-sentence groups the first pass recorded (the shuffle's unit; counts unusable ones too) 159const SP_O_SHUF: i64 = 36 // 1 when epochs 1 and later walked the shuffled order, 0 in file order (the default) 160const SP_O_CLUST: i64 = 37 // 1 when a PPMI model was set, loaded and the vocabulary clustered, else 0 161const SP_O_CLUST_VOCAB: i64 = 38 // training vocabulary words the clusterer holds 162const SP_O_CLUST_INMODEL: i64 = 39 // of those, words the PPMI model knows 163const SP_O_CRF: i64 = 40 // 1 when the CRF objective trained the weights, 0 for the perceptron (the default) 164const SP_O_CRF_NLL_FIRST: i64 = 41 // epoch-0 total negative log-likelihood, Q10 (0 in perceptron mode) 165const SP_O_CRF_NLL_LAST: i64 = 42 // last-epoch total negative log-likelihood, Q10 166const SP_O_CRF_MARGDEV: i64 = 43 // largest |sum of a token's marginals - S| seen, Q10 (a partition witness) 167const SP_O_CRF_NEGNLL: i64 = 44 // sentences whose partition read below their gold path score beyond the slack (must be 0) 168const SP_O_EMB: i64 = 45 // 1 when the embedding table was set, loaded at the declared width and its features were on 169const SP_O_EMB_DIM: i64 = 46 // the loaded table's coordinate count (0 when none) 170const SP_O_EMB_LOOKUPS: i64 = 47 // training-pass token lookups against the vocabulary 171const SP_O_EMB_HITS: i64 = 48 // of those, tokens the vocabulary knows (coverage numerator) 172// THE ERROR CENSUS (an instrument, not a feature): every gold occurrence the perceptron arm missed, partitioned by WHY it 173// could be missed -- the term is not a contiguous token run of its sentence (a tokenisation ceiling no model can cross), or 174// it is a single-token or multi-token term the training dictionary has never seen (generalisation), or one it has seen 175// (a modelling miss the union arm exists to catch). The five parts sum to gold_occurrences - matched by construction. 176const SP_O_EC_UNTOK: i64 = 49 // missed and not tokenisable in its own sentence 177const SP_O_EC_UNSEEN1: i64 = 50 // missed, single-token, absent from the training dictionary 178const SP_O_EC_UNSEENM: i64 = 51 // missed, multi-token, absent from the training dictionary 179const SP_O_EC_SEEN1: i64 = 52 // missed, single-token, present in the training dictionary 180const SP_O_EC_SEENM: i64 = 53 // missed, multi-token, present in the training dictionary 181const SP_O_EC_UNIONMISS: i64 = 54 // gold occurrences the UNION arm missed too 182const SP_O_N: i64 = 55 183const SP_CH_SPACE: i64 = 32 184// PER-EPOCH SHUFFLE (the tagger's lesson applied here): a structured perceptron trained sixteen epochs in file order learns 185// the file's order. Epoch 0 walks the file in order and records where every merged-sentence group starts; every later epoch 186// walks the groups in a fresh DETERMINISTIC permutation (nx_postag's pt_permute, seeded by the epoch), so two runs still 187// agree and the gate's determinism tooth holds. 188const SP_GROUP_CAP: i64 = 65536 // groups the shuffle can hold (SemEval restaurants train is 2018) 189// per-sentence span lists (small, deduped, byte-compared): header = [buf, offs, lens, count, used] 190const SL_BUF: i64 = 0 191const SL_OFFS: i64 = 1 192const SL_LENS: i64 = 2 193const SL_COUNT: i64 = 3 194const SL_USED: i64 = 4 195const SL_FIELDS: i64 = 5 196const SL_CAP: i64 = 64 // spans per sentence 197const SL_BUFCAP: i64 = 8192 198const SP_CTR_N: i64 = 4 // first-pass flag + records used + records skipped + merged sentences 199 200static sp_w: *i64 201static sp_wsum: *i64 202static sp_wtime: *i64 203static sp_tw: *i64 204static sp_twsum: *i64 205static sp_twtime: *i64 206static sp_clock: i64 207static sp_T: i64 208static sp_featbuf: *u8 209static sp_tokbuf: *u8 210static sp_offs: *i64 211static sp_lens: *i64 212static sp_fhs: *i64 // SP_TOK_CAP * SP_NFEAT base feature hashes for the current sentence 213static sp_dp: *i64 214static sp_bp: *i64 215static sp_gold: *i64 216static sp_pred: *i64 217static sp_gram: *u8 218// the record MERGE: a .seg emits ONE record per aspect, so a sentence with three aspects appears three times, each 219// labelling the other two as O -- contradictory supervision a sequence model can never converge on (measured: 2915 220// of 3607 sentences still updating in epoch 8, F1 294 with P 796 / R 180). Consecutive records that reconstruct to the 221// SAME sentence are folded into ONE example carrying every aspect. 222static sp_prevsent: *u8 223static sp_prevlen: i64 224static sp_prevcnt: i64 225static sp_goldacc: *i64 226static sp_pending: i64 227// per-sentence lists for the occurrence metric: this sentence's gold terms, predicted spans, union spans 228static sp_lg: *i64 229static sp_lp: *i64 230static sp_lu: *i64 231static sp_normbuf: *u8 232// per-dictionary-term counters, parallel to the dictionary set's slots: sentences the term appears in / times it is gold 233static sp_dseen: *i64 234static sp_dgold: *i64 235// POS: the treebank path (0 = none), the trained flag, and this sentence's tags (parallel to sp_offs/sp_lens) 236static sp_treebank: *u8 237static sp_pos_on: i64 238static sp_postags: *i64 239// PARSE: the switch (default on whenever a treebank is set), the trained flag, and this sentence's heads and relations 240static sp_parse_want: i64 241static sp_parse_on: i64 242static sp_heads: *i64 243static sp_rels: *i64 244// the shuffle: byte offset of every merged-sentence group's first record, the per-epoch order, and the group trainer's 245// scratch (allocated once; a group is trained thousands of times per run) 246static sp_gstarts: *i64 247static sp_gorder: *i64 248static sp_ngroups: i64 249static sp_grbuf: *u8 250static sp_grp: *i64 251static sp_grr: *i64 252// the shuffle switch: OFF by default. MEASURED 2026-09-06 on one binary against its pre-declared rule (union F1 must rise on 253// both domains): restaurants 683 -> 679, laptops 532 -> 536 -- one down, one up, both inside the ten-permil band, so file 254// order stays the default and the shuffle is a mode (sp_set_shuffle(1), the CLI's shuffle token) 255static sp_shuffle_want: i64 256 257// add (1) or drop (0) the per-epoch shuffle of the merged-sentence groups; off unless asked 258func sp_set_shuffle(on: i64) -> i64 { sp_shuffle_want = on; return 0 } 259// CLUSTERS: the PPMI model path (0 = none, the default) and the trained flag 260static sp_clust_model: *u8 261static sp_clust_on: i64 262// set (or clear with 0) the PPMI model whose clusters feed the five cluster families; read by sp_eval 263func sp_set_clusters(path: *u8) -> i64 { sp_clust_model = path; return 0 } 264// one pass over the TRAIN split adding every token to the clusterer (frequencies seed the medoids) 265func sp_clust_vocab(train: *u8) -> i64 { 266 let lp: *i64 = sys_mmap(RM_I64_PAIR) as *i64 267 lp[0] = 0 268 let b: *u8 = sys_read_file(train, lp) 269 if (b as i64) == 0 { return 0 } 270 let n: i64 = lp[0] 271 let rbuf: *u8 = sys_mmap(AB_RECON_CAP) 272 let rp: *i64 = sys_mmap(RM_I64_PAIR) as *i64 273 let r: *i64 = sys_mmap(AB_REC_N * RM_I64_BYTES) as *i64 274 rp[0] = 0 275 var added: i64 = 0 276 while ab_rec_next(b, n, rp, r) == 1 { 277 let rl: i64 = ab_reconstruct((b as i64 + r[0]) as *u8, r[1], (b as i64 + r[2]) as *u8, r[3], rbuf, AB_RECON_CAP) 278 let cnt: i64 = sp_tokenize(rbuf, rl) 279 var i: i64 = 0 280 while i < cnt { wc_add((sp_tokbuf as i64 + sp_offs[i]) as *u8, sp_lens[i]); added = added + 1; i = i + 1 } 281 } 282 return added 283} 284 285// CRF: the mode flag, the epoch the step size decays on, the log-domain lattices and the marginals 286static sp_crf_want: i64 287static sp_crf_on: i64 288static sp_crf_epoch: i64 289static sp_em: *i64 // emission cache SP_TOK_CAP * SP_NTAGS for the current sentence 290static sp_alpha: *i64 // forward lattice, Q10 log domain 291static sp_beta: *i64 // backward lattice 292static sp_marg: *i64 // Q10 token marginals SP_TOK_CAP * SP_NTAGS 293static sp_pair: *i64 // Q10 pair marginals SP_TOK_CAP * SP_NPAIR (row i: previous tag * SP_NTAGS + tag) 294static sp_crf_nll: i64 // the current epoch's summed NLL, Q10 295static sp_crf_margdev: i64 // largest |sum of marginals - S| seen this run 296static sp_crf_negnll: i64 // sentences with NLL < -SP_CRF_NLL_SLACK (impossible, counted) 297// add (1) or drop (0) the CRF objective; off unless asked 298func sp_set_crf(on: i64) -> i64 { sp_crf_want = on; return 0 } 299// EMBEDDINGS: the PPMI vocabulary and the embedding table (0 = none, the default) and the loaded flag 300static sp_emb_ppmi: *u8 301static sp_emb_path: *u8 302static sp_emb_on: i64 303static sp_fmask: *i64 // SP_TOK_CAP * SP_NFEAT: 1 when the slot carries a feature, 0 when the slot is absent 304 // (an absent feature casts no vote in scoring or updates -- so a mode that is off 305 // leaves the model byte-identical to the model without the mode) 306// set (or clear with 0, 0) the vocabulary and table whose coordinates feed the embedding families; read by sp_eval 307func sp_set_embed(ppmi: *u8, emb: *u8) -> i64 { sp_emb_ppmi = ppmi; sp_emb_path = emb; return 0 } 308 309// set (or clear with 0) the CoNLL-U treebank the POS tagger trains on; read by sp_eval. The parser stays OFF by default: 310// MEASURED 2026-09-06 on one binary, full SemEval splits, the parse features from the 5-epoch parser (UAS 804) LOWERED the 311// paper-metric union F1 (restaurants 687 -> 683, laptops 528 -> 521), so the default is the arm that measured best and the 312// parser is a mode (sp_set_parse(1), the CLI's parse argument) until a stronger parser earns the default back 313func sp_set_treebank(path: *u8) -> i64 { sp_treebank = path; sp_parse_want = 0; return 0 } 314// drop (0) or add (1) the parser on top of the tagger: the ablation switch; sp_set_treebank resets it to off 315func sp_set_parse(on: i64) -> i64 { sp_parse_want = on; return 0 } 316// tag, then parse, the tokens currently in sp_tokbuf; without a trained tagger every tag reads -1, without a trained 317// parser every head and relation reads -1 318func sp_tag_sentence(cnt: i64) -> i64 { 319 var i: i64 = 0 320 while i < cnt { sp_postags[i] = PT_TAG_NONE; sp_heads[i] = DP_NONE; sp_rels[i] = DP_NONE; i = i + 1 } 321 if sp_pos_on == 1 { pt_tag_stream(sp_tokbuf, sp_offs, sp_lens, cnt, sp_postags) } 322 if sp_parse_on == 1 { dp_parse_stream(sp_tokbuf, sp_offs, sp_lens, sp_postags, cnt, sp_heads, sp_rels) } 323 return 0 324} 325 326func sl_new() -> *i64 { 327 let h: *i64 = sys_mmap(SL_FIELDS * RM_I64_BYTES) as *i64 328 h[SL_BUF] = sys_mmap(SL_BUFCAP) as i64 329 h[SL_OFFS] = sys_mmap(SL_CAP * RM_I64_BYTES) as i64 330 h[SL_LENS] = sys_mmap(SL_CAP * RM_I64_BYTES) as i64 331 h[SL_COUNT] = 0 332 h[SL_USED] = 0 333 return h 334} 335func sl_reset(h: *i64) -> i64 { h[SL_COUNT] = 0; h[SL_USED] = 0; return 0 } 336func sl_has(h: *i64, s: *u8, n: i64) -> i64 { 337 let buf: *u8 = h[SL_BUF] as *u8 338 let offs: *i64 = h[SL_OFFS] as *i64 339 let lens: *i64 = h[SL_LENS] as *i64 340 var i: i64 = 0 341 while i < h[SL_COUNT] { 342 if lens[i] == n { 343 var e: i64 = 0 344 var same: i64 = 1 345 while e < n { if buf[offs[i] + e] != s[e] { same = 0; e = n } else { e = e + 1 } } 346 if same == 1 { return 1 } 347 } 348 i = i + 1 349 } 350 return 0 351} 352// add if absent (dedupe within the sentence); a full list drops silently but COUNTS it, so a cap is never a lie 353func sl_add(h: *i64, s: *u8, n: i64) -> i64 { 354 if n <= 0 { return 0 } 355 if sl_has(h, s, n) == 1 { return 0 } 356 if h[SL_COUNT] >= SL_CAP { return 0 } 357 if h[SL_USED] + n > SL_BUFCAP { return 0 } 358 let buf: *u8 = h[SL_BUF] as *u8 359 let offs: *i64 = h[SL_OFFS] as *i64 360 let lens: *i64 = h[SL_LENS] as *i64 361 rm_catn(buf, h[SL_USED], s, n) 362 offs[h[SL_COUNT]] = h[SL_USED] 363 lens[h[SL_COUNT]] = n 364 h[SL_USED] = h[SL_USED] + n 365 h[SL_COUNT] = h[SL_COUNT] + 1 366 return 1 367} 368// how many entries of a are also in b 369func sl_inter(a: *i64, b: *i64) -> i64 { 370 let buf: *u8 = a[SL_BUF] as *u8 371 let offs: *i64 = a[SL_OFFS] as *i64 372 let lens: *i64 = a[SL_LENS] as *i64 373 var k: i64 = 0 374 var i: i64 = 0 375 while i < a[SL_COUNT] { if sl_has(b, (buf as i64 + offs[i]) as *u8, lens[i]) == 1 { k = k + 1 } i = i + 1 } 376 return k 377} 378 379func sp_reset() -> i64 { 380 if (sp_w as i64) == 0 { 381 sp_w = sys_mmap(SP_W * RM_I64_BYTES) as *i64 382 sp_wsum = sys_mmap(SP_W * RM_I64_BYTES) as *i64 383 sp_wtime = sys_mmap(SP_W * RM_I64_BYTES) as *i64 384 sp_tw = sys_mmap(SP_NTRANS * RM_I64_BYTES) as *i64 385 sp_twsum = sys_mmap(SP_NTRANS * RM_I64_BYTES) as *i64 386 sp_twtime = sys_mmap(SP_NTRANS * RM_I64_BYTES) as *i64 387 sp_featbuf = sys_mmap(SP_FEATBUF) 388 sp_tokbuf = sys_mmap(SP_TOKBUF) 389 sp_offs = sys_mmap(SP_TOK_CAP * RM_I64_BYTES) as *i64 390 sp_lens = sys_mmap(SP_TOK_CAP * RM_I64_BYTES) as *i64 391 sp_fhs = sys_mmap(SP_TOK_CAP * SP_NFEAT * RM_I64_BYTES) as *i64 392 sp_dp = sys_mmap(SP_TOK_CAP * SP_NTAGS * RM_I64_BYTES) as *i64 393 sp_bp = sys_mmap(SP_TOK_CAP * SP_NTAGS * RM_I64_BYTES) as *i64 394 sp_gold = sys_mmap(SP_TOK_CAP * RM_I64_BYTES) as *i64 395 sp_pred = sys_mmap(SP_TOK_CAP * RM_I64_BYTES) as *i64 396 sp_gram = sys_mmap(SP_GRAM_CAP) 397 sp_prevsent = sys_mmap(AB_RECON_CAP) 398 sp_goldacc = sys_mmap(SP_TOK_CAP * RM_I64_BYTES) as *i64 399 sp_lg = sl_new() 400 sp_lp = sl_new() 401 sp_lu = sl_new() 402 sp_normbuf = sys_mmap(AB_TERM_MAX) 403 sp_dseen = sys_mmap(AB_SET_SLOTS * RM_I64_BYTES) as *i64 404 sp_dgold = sys_mmap(AB_SET_SLOTS * RM_I64_BYTES) as *i64 405 sp_postags = sys_mmap(SP_TOK_CAP * RM_I64_BYTES) as *i64 406 sp_heads = sys_mmap(SP_TOK_CAP * RM_I64_BYTES) as *i64 407 sp_rels = sys_mmap(SP_TOK_CAP * RM_I64_BYTES) as *i64 408 sp_gstarts = sys_mmap(SP_GROUP_CAP * RM_I64_BYTES) as *i64 409 sp_gorder = sys_mmap(SP_GROUP_CAP * RM_I64_BYTES) as *i64 410 sp_grbuf = sys_mmap(AB_RECON_CAP) 411 sp_grp = sys_mmap(RM_I64_PAIR) as *i64 412 sp_grr = sys_mmap(AB_REC_N * RM_I64_BYTES) as *i64 413 sp_em = sys_mmap(SP_TOK_CAP * SP_NTAGS * RM_I64_BYTES) as *i64 414 sp_alpha = sys_mmap(SP_TOK_CAP * SP_NTAGS * RM_I64_BYTES) as *i64 415 sp_beta = sys_mmap(SP_TOK_CAP * SP_NTAGS * RM_I64_BYTES) as *i64 416 sp_marg = sys_mmap(SP_TOK_CAP * SP_NTAGS * RM_I64_BYTES) as *i64 417 sp_pair = sys_mmap(SP_TOK_CAP * SP_NPAIR * RM_I64_BYTES) as *i64 418 sp_fmask = sys_mmap(SP_TOK_CAP * SP_NFEAT * RM_I64_BYTES) as *i64 419 } else { 420 var i: i64 = 0 421 while i < SP_W { sp_w[i] = 0; sp_wsum[i] = 0; sp_wtime[i] = 0; i = i + 1 } 422 } 423 var d: i64 = 0 424 while d < AB_SET_SLOTS { sp_dseen[d] = 0; sp_dgold[d] = 0; d = d + 1 } 425 var t: i64 = 0 426 while t < SP_NTRANS { sp_tw[t] = 0; sp_twsum[t] = 0; sp_twtime[t] = 0; t = t + 1 } 427 sp_clock = 0 428 sp_T = 0 429 sp_prevlen = 0 430 sp_prevcnt = 0 431 sp_pending = 0 432 return 0 433} 434func sp_acc_reset() -> i64 { 435 var i: i64 = 0 436 while i < SP_TOK_CAP { sp_goldacc[i] = SP_O; i = i + 1 } 437 return 0 438} 439// fold this record's labels (sp_gold) into the accumulated example: a B or I never reverts to O 440func sp_acc_or(cnt: i64) -> i64 { 441 var i: i64 = 0 442 while i < cnt { if sp_gold[i] != SP_O { sp_goldacc[i] = sp_gold[i] } i = i + 1 } 443 return 0 444} 445func sp_same_sent(rbuf: *u8, rl: i64) -> i64 { 446 if rl != sp_prevlen { return 0 } 447 var i: i64 = 0 448 while i < rl { if rbuf[i] != sp_prevsent[i] { return 0 } i = i + 1 } 449 return 1 450} 451// train ONE merged example: the tokens in sp_tokbuf (still the previous sentence) with the accumulated gold 452func sp_flush(cnt: i64, dic: *i64) -> i64 { 453 var i: i64 = 0 454 while i < cnt { sp_gold[i] = sp_goldacc[i]; i = i + 1 } 455 i = 0 456 while i < cnt { sp_feats(cnt, i, dic); i = i + 1 } 457 if sp_crf_on == 1 { 458 // CRF: partition and marginals under the current weights, the mistake count under the same weights (the receipt's 459 // last_epoch_updates keeps its meaning), then one gradient step 460 let logz: i64 = sp_crf_fb(cnt) 461 let nll: i64 = logz - sp_crf_gold_score(cnt) 462 sp_crf_nll = sp_crf_nll + nll 463 if nll < (0 - SP_CRF_NLL_SLACK) { sp_crf_negnll = sp_crf_negnll + 1 } 464 sp_viterbi(cnt, 0) 465 var d: i64 = 0 466 i = 0 467 while i < cnt { if sp_pred[i] != sp_gold[i] { d = 1 } i = i + 1 } 468 sp_crf_update(cnt) 469 sp_clock = sp_clock + 1 470 return d 471 } 472 sp_viterbi(cnt, 0) 473 let u: i64 = sp_update(cnt) 474 sp_clock = sp_clock + 1 475 return u 476} 477// tokenise a reconstructed sentence into sp_tokbuf, filling sp_offs/sp_lens; returns the token count (capped) 478func sp_tokenize(sent: *u8, n: i64) -> i64 { 479 let ip: *i64 = sys_mmap(RM_I64_PAIR) as *i64 480 ip[0] = 0 481 let tok: *u8 = sys_mmap(RM_TOK_MAX + RM_TOK_BUF_SPARE) 482 var cnt: i64 = 0 483 var used: i64 = 0 484 var tl: i64 = rm_next_token(sent, n, ip, tok) 485 while tl > 0 { 486 if cnt < SP_TOK_CAP { if used + tl < SP_TOKBUF { 487 sp_offs[cnt] = used 488 sp_lens[cnt] = tl 489 rm_catn(sp_tokbuf, used, tok, tl) 490 used = used + tl 491 cnt = cnt + 1 492 } } 493 tl = rm_next_token(sent, n, ip, tok) 494 } 495 return cnt 496} 497// the base hash of one feature: prefix byte then the span 498func sp_fh(pfx: i64, s: *u8, n: i64) -> i64 { 499 sp_featbuf[0] = pfx as u8 500 var i: i64 = 0 501 while i < n { if i + 1 < SP_FEATBUF { sp_featbuf[i + 1] = s[i] } i = i + 1 } 502 return rm_hash(sp_featbuf, n + 1) 503} 504func sp_has_digit(s: *u8, n: i64) -> i64 { 505 var i: i64 = 0 506 while i < n { let c: i64 = s[i] as i64; if c >= SP_CH_DIGIT0 { if c <= SP_CH_DIGIT9 { return 1 } } i = i + 1 } 507 return 0 508} 509// compute the SP_NFEAT base feature hashes for position i into sp_fhs[i*SP_NFEAT ..] 510func sp_feats(cnt: i64, i: i64, dic: *i64) -> i64 { 511 let base: i64 = i * SP_NFEAT 512 let toff: i64 = sp_offs[i] 513 let tlen: i64 = sp_lens[i] 514 let tp: *u8 = (sp_tokbuf as i64 + toff) as *u8 515 sp_fhs[base + 0] = sp_fh(SP_PFX_W, tp, tlen) 516 var af: i64 = SP_AFF 517 if tlen < af { af = tlen } 518 sp_fhs[base + 1] = sp_fh(SP_PFX_P, tp, af) 519 sp_fhs[base + 2] = sp_fh(SP_PFX_S, (sp_tokbuf as i64 + toff + tlen - af) as *u8, af) 520 if i > 0 { sp_fhs[base + 3] = sp_fh(SP_PFX_PW, (sp_tokbuf as i64 + sp_offs[i - 1]) as *u8, sp_lens[i - 1]) } 521 else { let bos: *u8 = sys_mmap(2); bos[0] = SP_CH_CARET as u8; sp_fhs[base + 3] = sp_fh(SP_PFX_PW, bos, 1) } 522 if i < cnt - 1 { sp_fhs[base + 4] = sp_fh(SP_PFX_NW, (sp_tokbuf as i64 + sp_offs[i + 1]) as *u8, sp_lens[i + 1]) } 523 else { let eos: *u8 = sys_mmap(2); eos[0] = SP_CH_DOLLAR2 as u8; sp_fhs[base + 4] = sp_fh(SP_PFX_NW, eos, 1) } 524 let sh: *u8 = sys_mmap(2) 525 if sp_has_digit(tp, tlen) == 1 { sh[0] = SP_CH_ONE as u8 } else { sh[0] = SP_CH_ZERO as u8 } 526 sp_fhs[base + 5] = sp_fh(SP_PFX_SH, sh, 1) 527 var dslot: i64 = hs_slot_raw(dic, tp, tlen) 528 if dslot < 0 { if i < cnt - 1 { 529 var go: i64 = rm_catn(sp_gram, 0, tp, tlen) 530 sp_gram[go] = AB_NG_JOIN as u8; go = go + 1 531 go = rm_catn(sp_gram, go, (sp_tokbuf as i64 + sp_offs[i + 1]) as *u8, sp_lens[i + 1]) 532 dslot = hs_slot_raw(dic, sp_gram, go) 533 } } 534 var indict: i64 = 0 535 if dslot >= 0 { indict = 1 } 536 let dch: *u8 = sys_mmap(2) 537 if indict == 1 { dch[0] = SP_CH_ONE as u8 } else { dch[0] = SP_CH_ZERO as u8 } 538 sp_fhs[base + 6] = sp_fh(SP_PFX_D, dch, 1) 539 // the graded reliability of that dictionary term in the TRAIN split (bucket 0..SP_BUCKETS), '-' when not in it 540 let ech: *u8 = sys_mmap(2) 541 ech[0] = SP_CH_DASH as u8 542 if dslot >= 0 { 543 var bucket: i64 = 0 544 if sp_dseen[dslot] > 0 { bucket = (sp_dgold[dslot] * SP_BUCKETS) / sp_dseen[dslot] } 545 if bucket > SP_BUCKETS { bucket = SP_BUCKETS } 546 ech[0] = (SP_CH_ZERO + bucket) as u8 547 } 548 sp_fhs[base + 13] = sp_fh(SP_PFX_E, ech, 1) 549 sp_fhs[base + 7] = sp_fh(SP_PFX_B, sp_featbuf, 0) 550 // wider context: two back, two ahead (BOS/EOS markers reused when the window runs off the sentence) 551 let bos2: *u8 = sys_mmap(2); bos2[0] = SP_CH_CARET as u8 552 let eos2: *u8 = sys_mmap(2); eos2[0] = SP_CH_DOLLAR2 as u8 553 if i > 1 { sp_fhs[base + 8] = sp_fh(SP_PFX_PW2, (sp_tokbuf as i64 + sp_offs[i - 2]) as *u8, sp_lens[i - 2]) } 554 else { sp_fhs[base + 8] = sp_fh(SP_PFX_PW2, bos2, 1) } 555 if i < cnt - 2 { sp_fhs[base + 9] = sp_fh(SP_PFX_NW2, (sp_tokbuf as i64 + sp_offs[i + 2]) as *u8, sp_lens[i + 2]) } 556 else { sp_fhs[base + 9] = sp_fh(SP_PFX_NW2, eos2, 1) } 557 // conjunctions: previous+current and current+next, joined through sp_gram 558 var cj: i64 = 0 559 if i > 0 { cj = rm_catn(sp_gram, 0, (sp_tokbuf as i64 + sp_offs[i - 1]) as *u8, sp_lens[i - 1]) } else { cj = rm_catn(sp_gram, 0, bos2, 1) } 560 sp_gram[cj] = AB_NG_JOIN as u8; cj = cj + 1 561 cj = rm_catn(sp_gram, cj, tp, tlen) 562 sp_fhs[base + 10] = sp_fh(SP_PFX_PC, sp_gram, cj) 563 cj = rm_catn(sp_gram, 0, tp, tlen) 564 sp_gram[cj] = AB_NG_JOIN as u8; cj = cj + 1 565 if i < cnt - 1 { cj = rm_catn(sp_gram, cj, (sp_tokbuf as i64 + sp_offs[i + 1]) as *u8, sp_lens[i + 1]) } else { cj = rm_catn(sp_gram, cj, eos2, 1) } 566 sp_fhs[base + 11] = sp_fh(SP_PFX_CN, sp_gram, cj) 567 // the 4-char suffix as its own family (the 3-char one stays) 568 var af4: i64 = SP_AFF4 569 if tlen < af4 { af4 = tlen } 570 sp_fhs[base + 12] = sp_fh(SP_PFX_S4, (sp_tokbuf as i64 + toff + tlen - af4) as *u8, af4) 571 // POS of this, the previous and the next token, and the two tag bigrams (tag -1 encodes as '^') 572 let pb: *u8 = sys_mmap(4) 573 var pc: i64 = PT_TAG_NONE 574 var pp: i64 = PT_TAG_NONE 575 var pn: i64 = PT_TAG_NONE 576 if sp_pos_on == 1 { 577 pc = sp_postags[i] 578 if i > 0 { pp = sp_postags[i - 1] } 579 if i < cnt - 1 { pn = sp_postags[i + 1] } 580 } 581 pb[0] = (SP_CH_CARET + pc + 1) as u8 582 sp_fhs[base + SP_POS_BASE + 0] = sp_fh(SP_PFX_POS, pb, 1) 583 pb[0] = (SP_CH_CARET + pp + 1) as u8 584 sp_fhs[base + SP_POS_BASE + 1] = sp_fh(SP_PFX_POSP, pb, 1) 585 pb[0] = (SP_CH_CARET + pn + 1) as u8 586 sp_fhs[base + SP_POS_BASE + 2] = sp_fh(SP_PFX_POSN, pb, 1) 587 pb[0] = (SP_CH_CARET + pp + 1) as u8 588 pb[1] = (SP_CH_CARET + pc + 1) as u8 589 sp_fhs[base + SP_POS_BASE + 3] = sp_fh(SP_PFX_POSPC, pb, 2) 590 pb[0] = (SP_CH_CARET + pc + 1) as u8 591 pb[1] = (SP_CH_CARET + pn + 1) as u8 592 sp_fhs[base + SP_POS_BASE + 4] = sp_fh(SP_PFX_POSCN, pb, 2) 593 // DEPENDENCY features: relation, head word, head tag, relation+own tag, the relations of the outermost dependents 594 // (every byte reads '@' or '^' when no parser ran, so the no-parser model degrades to the 19-feature form) 595 var rb: i64 = DP_CH_AT 596 var hidx: i64 = DP_NONE 597 var lchild: i64 = DP_NONE 598 var rchild: i64 = DP_NONE 599 if sp_parse_on == 1 { 600 if sp_rels[i] >= 0 { rb = DP_CH_A + sp_rels[i] } 601 hidx = sp_heads[i] 602 var j: i64 = 0 603 while j < cnt { if sp_heads[j] == i { if lchild < 0 { lchild = j } rchild = j } j = j + 1 } 604 } 605 pb[0] = rb as u8 606 sp_fhs[base + SP_PARSE_BASE + 0] = sp_fh(SP_PFX_REL, pb, 1) 607 if hidx >= 0 { sp_fhs[base + SP_PARSE_BASE + 1] = sp_fh(SP_PFX_HW, (sp_tokbuf as i64 + sp_offs[hidx]) as *u8, sp_lens[hidx]) } 608 else { pb[0] = SP_CH_CARET as u8; sp_fhs[base + SP_PARSE_BASE + 1] = sp_fh(SP_PFX_HW, pb, 1) } 609 if hidx >= 0 { pb[0] = (SP_CH_CARET + sp_postags[hidx] + 1) as u8 } else { pb[0] = SP_CH_DOLLAR2 as u8 } 610 sp_fhs[base + SP_PARSE_BASE + 2] = sp_fh(SP_PFX_HT, pb, 1) 611 pb[0] = rb as u8 612 pb[1] = (SP_CH_CARET + pc + 1) as u8 613 sp_fhs[base + SP_PARSE_BASE + 3] = sp_fh(SP_PFX_RELT, pb, 2) 614 var lrb: i64 = DP_CH_AT 615 var rrb: i64 = DP_CH_AT 616 if lchild >= 0 { if sp_rels[lchild] >= 0 { lrb = DP_CH_A + sp_rels[lchild] } } 617 if rchild >= 0 { if sp_rels[rchild] >= 0 { rrb = DP_CH_A + sp_rels[rchild] } } 618 pb[0] = lrb as u8 619 pb[1] = rrb as u8 620 sp_fhs[base + SP_PARSE_BASE + 4] = sp_fh(SP_PFX_CHR, pb, 2) 621 // WORD CLUSTERS: this token at three granularities, its neighbours at the middle one ('@' when unclustered) 622 var cl0: i64 = WC_NONE 623 var cl1: i64 = WC_NONE 624 var cl2: i64 = WC_NONE 625 var clp: i64 = WC_NONE 626 var cln: i64 = WC_NONE 627 if sp_clust_on == 1 { 628 cl0 = wc_cluster(tp, tlen, 0) 629 cl1 = wc_cluster(tp, tlen, SP_CL_MID) 630 cl2 = wc_cluster(tp, tlen, 2) 631 if i > 0 { clp = wc_cluster((sp_tokbuf as i64 + sp_offs[i - 1]) as *u8, sp_lens[i - 1], SP_CL_MID) } 632 if i < cnt - 1 { cln = wc_cluster((sp_tokbuf as i64 + sp_offs[i + 1]) as *u8, sp_lens[i + 1], SP_CL_MID) } 633 } 634 sp_fhs[base + SP_CLUST_BASE + 0] = sp_fh(SP_PFX_CL0, pb, wc_spell(cl0, pb)) 635 sp_fhs[base + SP_CLUST_BASE + 1] = sp_fh(SP_PFX_CL1, pb, wc_spell(cl1, pb)) 636 sp_fhs[base + SP_CLUST_BASE + 2] = sp_fh(SP_PFX_CL2, pb, wc_spell(cl2, pb)) 637 if i > 0 { sp_fhs[base + SP_CLUST_BASE + 3] = sp_fh(SP_PFX_CLP, pb, wc_spell(clp, pb)) } 638 else { pb[0] = SP_CH_CARET as u8; sp_fhs[base + SP_CLUST_BASE + 3] = sp_fh(SP_PFX_CLP, pb, 1) } 639 if i < cnt - 1 { sp_fhs[base + SP_CLUST_BASE + 4] = sp_fh(SP_PFX_CLN, pb, wc_spell(cln, pb)) } 640 else { pb[0] = SP_CH_DOLLAR2 as u8; sp_fhs[base + SP_CLUST_BASE + 4] = sp_fh(SP_PFX_CLN, pb, 1) } 641 // every slot below the embedding block always carries a feature 642 var mk: i64 = 0 643 while mk < SP_EMB_BASE { sp_fmask[base + mk] = 1; mk = mk + 1 } 644 // DENSE EMBEDDING: this token's coordinates as buckets, one family per coordinate. With the mode off every embedding 645 // slot is ABSENT (masked out, no vote), so the default model is the model without the mode; with the mode on an 646 // unknown word casts ONE marker vote (its first slot spells '@') and leaves the other slots absent, never twenty-four 647 // copies of the same ignorance 648 var ew: i64 = EF_NONE 649 if sp_emb_on == 1 { ew = ef_wid(tp, tlen) } 650 var ed: i64 = 0 651 while ed < SP_EMB_DIMS { 652 var on: i64 = 0 653 if sp_emb_on == 1 { if ew >= 0 { on = 1 } else { if ed == 0 { on = 1 } } } 654 if on == 1 { sp_fhs[base + SP_EMB_BASE + ed] = sp_fh(SP_PFX_EMB, pb, ef_spell(ew, ed, pb)) } else { sp_fhs[base + SP_EMB_BASE + ed] = 0 } 655 sp_fmask[base + SP_EMB_BASE + ed] = on 656 ed = ed + 1 657 } 658 return SP_NFEAT 659} 660func sp_w_at(idx: i64, useavg: i64) -> i64 { 661 if useavg == 1 { return sp_wsum[idx] + sp_w[idx] * (sp_T - sp_wtime[idx]) } 662 return sp_w[idx] 663} 664func sp_bump(idx: i64, delta: i64) -> i64 { 665 sp_wsum[idx] = sp_wsum[idx] + sp_w[idx] * (sp_clock - sp_wtime[idx]) 666 sp_wtime[idx] = sp_clock 667 sp_w[idx] = sp_w[idx] + delta 668 return 0 669} 670func sp_tw_at(pt: i64, t: i64, useavg: i64) -> i64 { 671 let idx: i64 = pt * SP_NTAGS + t 672 if useavg == 1 { return sp_twsum[idx] + sp_tw[idx] * (sp_T - sp_twtime[idx]) } 673 return sp_tw[idx] 674} 675func sp_tbump(pt: i64, t: i64, delta: i64) -> i64 { 676 let idx: i64 = pt * SP_NTAGS + t 677 sp_twsum[idx] = sp_twsum[idx] + sp_tw[idx] * (sp_clock - sp_twtime[idx]) 678 sp_twtime[idx] = sp_clock 679 sp_tw[idx] = sp_tw[idx] + delta 680 return 0 681} 682func sp_emission(i: i64, tag: i64, useavg: i64) -> i64 { 683 let base: i64 = i * SP_NFEAT 684 var s: i64 = 0 685 var k: i64 = 0 686 while k < SP_NFEAT { if sp_fmask[base + k] == 1 { s = s + sp_w_at(((sp_fhs[base + k] * SP_NTAGS + tag) & SP_WMASK), useavg) } k = k + 1 } 687 return s 688} 689// Viterbi over the current sentence (features already in sp_fhs); writes the best tags into sp_pred 690func sp_viterbi(cnt: i64, useavg: i64) -> i64 { 691 if cnt <= 0 { return 0 } 692 var t: i64 = 0 693 while t < SP_NTAGS { 694 var e: i64 = sp_emission(0, t, useavg) 695 if t == SP_I { e = SP_NEG } // a sentence cannot start with I 696 sp_dp[t] = e 697 sp_bp[t] = 0 - 1 698 t = t + 1 699 } 700 var i: i64 = 1 701 while i < cnt { 702 var t2: i64 = 0 703 while t2 < SP_NTAGS { 704 let em: i64 = sp_emission(i, t2, useavg) 705 var best: i64 = SP_NEG 706 var bpt: i64 = 0 707 var pt: i64 = 0 708 while pt < SP_NTAGS { 709 var ok: i64 = 1 710 if t2 == SP_I { if pt == SP_O { ok = 0 } } 711 if ok == 1 { 712 let sc: i64 = sp_dp[(i - 1) * SP_NTAGS + pt] + sp_tw_at(pt, t2, useavg) + em 713 if sc > best { best = sc; bpt = pt } 714 } 715 pt = pt + 1 716 } 717 sp_dp[i * SP_NTAGS + t2] = best 718 sp_bp[i * SP_NTAGS + t2] = bpt 719 t2 = t2 + 1 720 } 721 i = i + 1 722 } 723 var bestt: i64 = 0 724 var bestv: i64 = sp_dp[(cnt - 1) * SP_NTAGS + 0] 725 var tt: i64 = 1 726 while tt < SP_NTAGS { let v: i64 = sp_dp[(cnt - 1) * SP_NTAGS + tt]; if v > bestv { bestv = v; bestt = tt } tt = tt + 1 } 727 var j: i64 = cnt - 1 728 var cur: i64 = bestt 729 while j >= 0 { sp_pred[j] = cur; cur = sp_bp[j * SP_NTAGS + cur]; if cur < 0 { cur = SP_O } j = j - 1 } 730 return 0 731} 732// build gold BIO for one record whose aspect term is [term,tlen]; returns 1 if the term was found and labelled 733func sp_gold_bio(cnt: i64, term: *u8, tlen: i64) -> i64 { 734 var i: i64 = 0 735 while i < cnt { sp_gold[i] = SP_O; i = i + 1 } 736 // tokenise the term into a small list by walking it the same way 737 let ip: *i64 = sys_mmap(RM_I64_PAIR) as *i64 738 ip[0] = 0 739 let tbuf: *u8 = sys_mmap(RM_TOK_MAX + RM_TOK_BUF_SPARE) 740 let toffs: *i64 = sys_mmap(SP_TOK_CAP * RM_I64_BYTES) as *i64 741 let tlens: *i64 = sys_mmap(SP_TOK_CAP * RM_I64_BYTES) as *i64 742 let tstore: *u8 = sys_mmap(SP_TOKBUF) 743 var tc: i64 = 0 744 var used: i64 = 0 745 var tl: i64 = rm_next_token(term, tlen, ip, tbuf) 746 while tl > 0 { 747 if tc < SP_TOK_CAP { if used + tl < SP_TOKBUF { toffs[tc] = used; tlens[tc] = tl; rm_catn(tstore, used, tbuf, tl); used = used + tl; tc = tc + 1 } } 748 tl = rm_next_token(term, tlen, ip, tbuf) 749 } 750 if tc == 0 { return 0 } 751 // find a contiguous run in the sentence tokens equal to the term tokens 752 var start: i64 = 0 753 while start + tc <= cnt { 754 var ok: i64 = 1 755 var k: i64 = 0 756 while k < tc { 757 if tlens[k] != sp_lens[start + k] { ok = 0 } else { 758 var e: i64 = 0 759 while e < tlens[k] { if tstore[toffs[k] + e] != sp_tokbuf[sp_offs[start + k] + e] { ok = 0 } e = e + 1 } 760 } 761 k = k + 1 762 } 763 if ok == 1 { 764 sp_gold[start] = SP_B 765 var m: i64 = 1 766 while m < tc { sp_gold[start + m] = SP_I; m = m + 1 } 767 return 1 768 } 769 start = start + 1 770 } 771 return 0 772} 773// one perceptron update comparing sp_pred to sp_gold over the current sentence; returns 1 if any tag differed 774func sp_update(cnt: i64) -> i64 { 775 var diff: i64 = 0 776 var i: i64 = 0 777 while i < cnt { if sp_pred[i] != sp_gold[i] { diff = 1 } i = i + 1 } 778 if diff == 0 { return 0 } 779 i = 0 780 while i < cnt { 781 if sp_pred[i] != sp_gold[i] { 782 let base: i64 = i * SP_NFEAT 783 var k: i64 = 0 784 while k < SP_NFEAT { 785 if sp_fmask[base + k] == 1 { 786 sp_bump(((sp_fhs[base + k] * SP_NTAGS + sp_gold[i]) & SP_WMASK), 1) 787 sp_bump(((sp_fhs[base + k] * SP_NTAGS + sp_pred[i]) & SP_WMASK), 0 - 1) 788 } 789 k = k + 1 790 } 791 } 792 if i > 0 { 793 if (sp_gold[i - 1] != sp_pred[i - 1]) | (sp_gold[i] != sp_pred[i]) { 794 sp_tbump(sp_gold[i - 1], sp_gold[i], 1) 795 sp_tbump(sp_pred[i - 1], sp_pred[i], 0 - 1) 796 } 797 } 798 i = i + 1 799 } 800 return 1 801} 802// CRF forward-backward over the current sentence (features in sp_fhs, weights read raw): fills sp_em, sp_alpha, sp_beta, 803// the token marginals sp_marg and the pair marginals sp_pair (Q10), and returns log Z (Q10). The hard constraints the 804// decoder applies (no I at the start, no O -> I) are the same masks here, so the CRF never puts mass on a path Viterbi 805// could not return. 806func sp_crf_fb(cnt: i64) -> i64 { 807 var i: i64 = 0 808 while i < cnt { 809 var t: i64 = 0 810 while t < SP_NTAGS { sp_em[i * SP_NTAGS + t] = sp_emission(i, t, 0); t = t + 1 } 811 i = i + 1 812 } 813 var t0: i64 = 0 814 while t0 < SP_NTAGS { 815 var a0: i64 = sp_em[t0] 816 if t0 == SP_I { a0 = LA_NEG_INF } 817 sp_alpha[t0] = a0 818 t0 = t0 + 1 819 } 820 i = 1 821 while i < cnt { 822 var t2: i64 = 0 823 while t2 < SP_NTAGS { 824 var acc: i64 = LA_NEG_INF 825 var pt: i64 = 0 826 while pt < SP_NTAGS { 827 var ok: i64 = 1 828 if t2 == SP_I { if pt == SP_O { ok = 0 } } 829 if ok == 1 { 830 let ap: i64 = sp_alpha[(i - 1) * SP_NTAGS + pt] 831 if la_is_neg_inf(ap) == 0 { acc = la_lse(acc, ap + sp_tw_at(pt, t2, 0)) } 832 } 833 pt = pt + 1 834 } 835 var v: i64 = LA_NEG_INF 836 if la_is_neg_inf(acc) == 0 { v = acc + sp_em[i * SP_NTAGS + t2] } 837 sp_alpha[i * SP_NTAGS + t2] = v 838 t2 = t2 + 1 839 } 840 i = i + 1 841 } 842 var tl: i64 = 0 843 while tl < SP_NTAGS { sp_beta[(cnt - 1) * SP_NTAGS + tl] = 0; tl = tl + 1 } 844 i = cnt - 2 845 while i >= 0 { 846 var t3: i64 = 0 847 while t3 < SP_NTAGS { 848 var acc2: i64 = LA_NEG_INF 849 var nt: i64 = 0 850 while nt < SP_NTAGS { 851 var ok2: i64 = 1 852 if nt == SP_I { if t3 == SP_O { ok2 = 0 } } 853 if ok2 == 1 { 854 let bn: i64 = sp_beta[(i + 1) * SP_NTAGS + nt] 855 if la_is_neg_inf(bn) == 0 { acc2 = la_lse(acc2, sp_tw_at(t3, nt, 0) + sp_em[(i + 1) * SP_NTAGS + nt] + bn) } 856 } 857 nt = nt + 1 858 } 859 sp_beta[i * SP_NTAGS + t3] = acc2 860 t3 = t3 + 1 861 } 862 i = i - 1 863 } 864 var logz: i64 = LA_NEG_INF 865 var tz: i64 = 0 866 while tz < SP_NTAGS { logz = la_lse(logz, sp_alpha[(cnt - 1) * SP_NTAGS + tz]); tz = tz + 1 } 867 i = 0 868 while i < cnt { 869 var sum: i64 = 0 870 var tm: i64 = 0 871 while tm < SP_NTAGS { 872 var m: i64 = 0 873 let a2: i64 = sp_alpha[i * SP_NTAGS + tm] 874 let b2: i64 = sp_beta[i * SP_NTAGS + tm] 875 if (la_is_neg_inf(a2) == 0) & (la_is_neg_inf(b2) == 0) { m = la_exp_neg(a2 + b2 - logz) } 876 sp_marg[i * SP_NTAGS + tm] = m 877 sum = sum + m 878 tm = tm + 1 879 } 880 var dev: i64 = sum - LA_S 881 if dev < 0 { dev = 0 - dev } 882 if dev > sp_crf_margdev { sp_crf_margdev = dev } 883 if i > 0 { 884 var pp: i64 = 0 885 while pp < SP_NTAGS { 886 var tt: i64 = 0 887 while tt < SP_NTAGS { 888 var pm: i64 = 0 889 var ok3: i64 = 1 890 if tt == SP_I { if pp == SP_O { ok3 = 0 } } 891 if ok3 == 1 { 892 let ap2: i64 = sp_alpha[(i - 1) * SP_NTAGS + pp] 893 let bt: i64 = sp_beta[i * SP_NTAGS + tt] 894 if (la_is_neg_inf(ap2) == 0) & (la_is_neg_inf(bt) == 0) { pm = la_exp_neg(ap2 + sp_tw_at(pp, tt, 0) + sp_em[i * SP_NTAGS + tt] + bt - logz) } 895 } 896 sp_pair[i * SP_NPAIR + pp * SP_NTAGS + tt] = pm 897 tt = tt + 1 898 } 899 pp = pp + 1 900 } 901 } 902 i = i + 1 903 } 904 return logz 905} 906// the gold path's score under the current weights (emissions from the cache sp_crf_fb filled), Q10 907func sp_crf_gold_score(cnt: i64) -> i64 { 908 var s: i64 = 0 909 var i: i64 = 0 910 while i < cnt { 911 s = s + sp_em[i * SP_NTAGS + sp_gold[i]] 912 if i > 0 { s = s + sp_tw_at(sp_gold[i - 1], sp_gold[i], 0) } 913 i = i + 1 914 } 915 return s 916} 917// one stochastic-gradient step: w += eta * (empirical - expected), the expected counts from the marginals sp_crf_fb left 918func sp_crf_update(cnt: i64) -> i64 { 919 let den: i64 = SP_CRF_ETA_DEN * (1 + sp_crf_epoch) 920 var i: i64 = 0 921 while i < cnt { 922 let base: i64 = i * SP_NFEAT 923 var t: i64 = 0 924 while t < SP_NTAGS { 925 var target: i64 = 0 926 if sp_gold[i] == t { target = LA_S } 927 let delta: i64 = ((target - sp_marg[i * SP_NTAGS + t]) * SP_CRF_ETA_NUM) / den 928 if delta != 0 { 929 var k: i64 = 0 930 while k < SP_NFEAT { if sp_fmask[base + k] == 1 { sp_bump(((sp_fhs[base + k] * SP_NTAGS + t) & SP_WMASK), delta) } k = k + 1 } 931 } 932 t = t + 1 933 } 934 if i > 0 { 935 var pt: i64 = 0 936 while pt < SP_NTAGS { 937 var t2: i64 = 0 938 while t2 < SP_NTAGS { 939 var tg: i64 = 0 940 if (sp_gold[i - 1] == pt) & (sp_gold[i] == t2) { tg = LA_S } 941 let d2: i64 = ((tg - sp_pair[i * SP_NPAIR + pt * SP_NTAGS + t2]) * SP_CRF_ETA_NUM) / den 942 if d2 != 0 { sp_tbump(pt, t2, d2) } 943 t2 = t2 + 1 944 } 945 pt = pt + 1 946 } 947 } 948 i = i + 1 949 } 950 return 1 951} 952// extract predicted spans from sp_pred into a per-sentence LIST (the occurrence metric's unit) 953func sp_extract_list(cnt: i64, lst: *i64) -> i64 { 954 var i: i64 = 0 955 while i < cnt { 956 if sp_pred[i] == SP_B { 957 var go: i64 = rm_catn(sp_gram, 0, (sp_tokbuf as i64 + sp_offs[i]) as *u8, sp_lens[i]) 958 var j: i64 = i + 1 959 while j < cnt { if sp_pred[j] == SP_I { 960 sp_gram[go] = AB_NG_JOIN as u8; go = go + 1 961 go = rm_catn(sp_gram, go, (sp_tokbuf as i64 + sp_offs[j]) as *u8, sp_lens[j]) 962 j = j + 1 963 } else { j = cnt } } 964 sl_add(lst, sp_gram, go) 965 var e: i64 = i + 1 966 while e < cnt { if sp_pred[e] == SP_I { e = e + 1 } else { i = e; e = cnt } } 967 if e >= cnt { i = cnt } 968 } else { i = i + 1 } 969 } 970 return 0 971} 972// a dictionary term passes the MAJORITY RULE when it was a gold aspect in at least half its training occurrences 973func sp_dict_majority(slot: i64) -> i64 { 974 if slot < 0 { return 0 } 975 if sp_dgold[slot] * SP_MAJ_NUM >= sp_dseen[slot] { return 1 } 976 return 0 977} 978// the dictionary arm's hits over the current tokens, into a per-sentence list. majority=1 admits only terms that pass 979// the majority rule; majority=0 admits every dictionary hit (the census pass that COUNTS occurrences) 980func sp_dict_list(cnt: i64, dic: *i64, lst: *i64, majority: i64) -> i64 { 981 var u: i64 = 0 982 while u < cnt { 983 let up: *u8 = (sp_tokbuf as i64 + sp_offs[u]) as *u8 984 var s1: i64 = hs_slot_raw(dic, up, sp_lens[u]) 985 if s1 >= 0 { if (majority == 0) | (sp_dict_majority(s1) == 1) { sl_add(lst, up, sp_lens[u]) } } 986 if u + 1 < cnt { 987 var g2: i64 = rm_catn(sp_gram, 0, up, sp_lens[u]) 988 sp_gram[g2] = AB_NG_JOIN as u8; g2 = g2 + 1 989 g2 = rm_catn(sp_gram, g2, (sp_tokbuf as i64 + sp_offs[u + 1]) as *u8, sp_lens[u + 1]) 990 var s2: i64 = hs_slot_raw(dic, sp_gram, g2) 991 if s2 >= 0 { if (majority == 0) | (sp_dict_majority(s2) == 1) { sl_add(lst, sp_gram, g2) } } 992 if u + 2 < cnt { 993 sp_gram[g2] = AB_NG_JOIN as u8; g2 = g2 + 1 994 g2 = rm_catn(sp_gram, g2, (sp_tokbuf as i64 + sp_offs[u + 2]) as *u8, sp_lens[u + 2]) 995 var s3: i64 = hs_slot_raw(dic, sp_gram, g2) 996 if s3 >= 0 { if (majority == 0) | (sp_dict_majority(s3) == 1) { sl_add(lst, sp_gram, g2) } } 997 } 998 } 999 u = u + 1 1000 } 1001 return 0 1002} 1003// one pass over the TRAIN split counting, per dictionary term, the sentences it appears in and the times it is gold 1004// (both deduped within a sentence); returns the number of dictionary terms that pass the majority rule 1005func sp_dict_stats(train: *u8, dic: *i64) -> i64 { 1006 let lp: *i64 = sys_mmap(RM_I64_PAIR) as *i64 1007 lp[0] = 0 1008 let b: *u8 = sys_read_file(train, lp) 1009 if (b as i64) == 0 { return 0 } 1010 let n: i64 = lp[0] 1011 let rbuf: *u8 = sys_mmap(AB_RECON_CAP) 1012 let rp: *i64 = sys_mmap(RM_I64_PAIR) as *i64 1013 let r: *i64 = sys_mmap(AB_REC_N * RM_I64_BYTES) as *i64 1014 rp[0] = 0 1015 var pend: i64 = 0 1016 sp_prevlen = 0 1017 sp_prevcnt = 0 1018 while ab_rec_next(b, n, rp, r) == 1 { 1019 let rl: i64 = ab_reconstruct((b as i64 + r[0]) as *u8, r[1], (b as i64 + r[2]) as *u8, r[3], rbuf, AB_RECON_CAP) 1020 if pend == 1 { if sp_same_sent(rbuf, rl) == 0 { 1021 sp_dict_stats_flush(dic) 1022 pend = 0 1023 } } 1024 if pend == 0 { 1025 let cnt: i64 = sp_tokenize(rbuf, rl) 1026 var cpy: i64 = 0 1027 while cpy < rl { sp_prevsent[cpy] = rbuf[cpy]; cpy = cpy + 1 } 1028 sp_prevlen = rl 1029 sp_prevcnt = cnt 1030 sl_reset(sp_lg) 1031 sl_reset(sp_lu) 1032 if cnt > 0 { sp_dict_list(cnt, dic, sp_lu, 0) } 1033 pend = 1 1034 } 1035 let gl: i64 = ab_norm((b as i64 + r[2]) as *u8, r[3], sp_normbuf, AB_TERM_MAX) 1036 sl_add(sp_lg, sp_normbuf, gl) 1037 } 1038 if pend == 1 { sp_dict_stats_flush(dic) } 1039 // census: how many dictionary terms pass the majority rule 1040 let hash: *i64 = dic[HS_HASH] as *i64 1041 var maj: i64 = 0 1042 var s: i64 = 0 1043 while s < AB_SET_SLOTS { if hash[s] != 0 { if sp_dict_majority(s) == 1 { maj = maj + 1 } } s = s + 1 } 1044 return maj 1045} 1046func sp_dict_stats_flush(dic: *i64) -> i64 { 1047 let ubuf: *u8 = sp_lu[SL_BUF] as *u8 1048 let uoffs: *i64 = sp_lu[SL_OFFS] as *i64 1049 let ulens: *i64 = sp_lu[SL_LENS] as *i64 1050 var i: i64 = 0 1051 while i < sp_lu[SL_COUNT] { 1052 let s: i64 = hs_slot_raw(dic, (ubuf as i64 + uoffs[i]) as *u8, ulens[i]) 1053 if s >= 0 { sp_dseen[s] = sp_dseen[s] + 1 } 1054 i = i + 1 1055 } 1056 let gbuf: *u8 = sp_lg[SL_BUF] as *u8 1057 let goffs: *i64 = sp_lg[SL_OFFS] as *i64 1058 let glens: *i64 = sp_lg[SL_LENS] as *i64 1059 i = 0 1060 while i < sp_lg[SL_COUNT] { 1061 let s: i64 = hs_slot_raw(dic, (gbuf as i64 + goffs[i]) as *u8, glens[i]) 1062 if s >= 0 { sp_dgold[s] = sp_dgold[s] + 1 } 1063 i = i + 1 1064 } 1065 return 0 1066} 1067// extract predicted aspect-term spans from sp_pred into the predicted set (deduped, normalised) 1068func sp_extract(cnt: i64, pred_set: *i64) -> i64 { 1069 var i: i64 = 0 1070 while i < cnt { 1071 if sp_pred[i] == SP_B { 1072 var go: i64 = rm_catn(sp_gram, 0, (sp_tokbuf as i64 + sp_offs[i]) as *u8, sp_lens[i]) 1073 var j: i64 = i + 1 1074 while j < cnt { if sp_pred[j] == SP_I { 1075 sp_gram[go] = AB_NG_JOIN as u8; go = go + 1 1076 go = rm_catn(sp_gram, go, (sp_tokbuf as i64 + sp_offs[j]) as *u8, sp_lens[j]) 1077 j = j + 1 1078 } else { j = cnt } } 1079 if hs_has_raw(pred_set, sp_gram, go) == 0 { hs_add_raw(pred_set, sp_gram, go) } 1080 // advance past this span 1081 var e: i64 = i + 1 1082 while e < cnt { if sp_pred[e] == SP_I { e = e + 1 } else { i = e; e = cnt } } 1083 if e >= cnt { i = cnt } 1084 } else { i = i + 1 } 1085 } 1086 return 0 1087} 1088// one pass over the train file: reconstruct, tokenise, gold BIO, decode with current weights, update. Returns updates. 1089func sp_train_pass(train: *u8, dic: *i64, first: *i64) -> i64 { 1090 let lp: *i64 = sys_mmap(RM_I64_PAIR) as *i64 1091 lp[0] = 0 1092 let b: *u8 = sys_read_file(train, lp) 1093 if (b as i64) == 0 { return 0 } 1094 let n: i64 = lp[0] 1095 let rbuf: *u8 = sys_mmap(AB_RECON_CAP) 1096 let rp: *i64 = sys_mmap(RM_I64_PAIR) as *i64 1097 let r: *i64 = sys_mmap(AB_REC_N * RM_I64_BYTES) as *i64 1098 rp[0] = 0 1099 var updates: i64 = 0 1100 sp_prevlen = 0 1101 sp_prevcnt = 0 1102 sp_pending = 0 1103 if first[0] == 1 { sp_ngroups = 0 } 1104 var cur: i64 = rp[0] 1105 while ab_rec_next(b, n, rp, r) == 1 { 1106 let rl: i64 = ab_reconstruct((b as i64 + r[0]) as *u8, r[1], (b as i64 + r[2]) as *u8, r[3], rbuf, AB_RECON_CAP) 1107 if sp_pending == 1 { if sp_same_sent(rbuf, rl) == 0 { 1108 // a NEW sentence: train the accumulated one first -- sp_tokbuf still holds its tokens 1109 updates = updates + sp_flush(sp_prevcnt, dic) 1110 if first[0] == 1 { first[3] = first[3] + 1 } 1111 sp_pending = 0 1112 } } 1113 var cnt: i64 = sp_prevcnt 1114 if sp_pending == 0 { 1115 // a group begins at the record just read: remember where it starts (first pass only) for the shuffled epochs 1116 if first[0] == 1 { if sp_same_sent(rbuf, rl) == 0 { if sp_ngroups < SP_GROUP_CAP { sp_gstarts[sp_ngroups] = cur; sp_ngroups = sp_ngroups + 1 } } } 1117 cnt = sp_tokenize(rbuf, rl) 1118 sp_tag_sentence(cnt) 1119 var cpy: i64 = 0 1120 while cpy < rl { sp_prevsent[cpy] = rbuf[cpy]; cpy = cpy + 1 } 1121 sp_prevlen = rl 1122 sp_prevcnt = cnt 1123 sp_acc_reset() 1124 } 1125 if cnt > 0 { 1126 if sp_gold_bio(cnt, (b as i64 + r[2]) as *u8, r[3]) == 1 { 1127 if first[0] == 1 { first[1] = first[1] + 1 } // count usable train records once 1128 sp_acc_or(cnt) 1129 sp_pending = 1 1130 } else { if first[0] == 1 { first[2] = first[2] + 1 } } 1131 } 1132 cur = rp[0] 1133 } 1134 if sp_pending == 1 { 1135 updates = updates + sp_flush(sp_prevcnt, dic) 1136 if first[0] == 1 { first[3] = first[3] + 1 } 1137 sp_pending = 0 1138 } 1139 return updates 1140} 1141// train ONE merged-sentence group whose first record starts at byte offset start: tokenise and tag once, fold every 1142// consecutive same-sentence record's gold into the accumulator, flush once. Returns the update count (0 when the group 1143// carries no usable record). The file is already in memory (b, n). 1144func sp_train_group(b: *u8, n: i64, start: i64, dic: *i64) -> i64 { 1145 sp_grp[0] = start 1146 if ab_rec_next(b, n, sp_grp, sp_grr) != 1 { return 0 } 1147 let rl: i64 = ab_reconstruct((b as i64 + sp_grr[0]) as *u8, sp_grr[1], (b as i64 + sp_grr[2]) as *u8, sp_grr[3], sp_grbuf, AB_RECON_CAP) 1148 let cnt: i64 = sp_tokenize(sp_grbuf, rl) 1149 sp_tag_sentence(cnt) 1150 var cpy: i64 = 0 1151 while cpy < rl { sp_prevsent[cpy] = sp_grbuf[cpy]; cpy = cpy + 1 } 1152 sp_prevlen = rl 1153 sp_prevcnt = cnt 1154 sp_acc_reset() 1155 var any: i64 = 0 1156 if cnt > 0 { if sp_gold_bio(cnt, (b as i64 + sp_grr[2]) as *u8, sp_grr[3]) == 1 { sp_acc_or(cnt); any = 1 } } 1157 var more: i64 = 1 1158 while more == 1 { 1159 if ab_rec_next(b, n, sp_grp, sp_grr) == 1 { 1160 let rl2: i64 = ab_reconstruct((b as i64 + sp_grr[0]) as *u8, sp_grr[1], (b as i64 + sp_grr[2]) as *u8, sp_grr[3], sp_grbuf, AB_RECON_CAP) 1161 if sp_same_sent(sp_grbuf, rl2) == 1 { 1162 if cnt > 0 { if sp_gold_bio(cnt, (b as i64 + sp_grr[2]) as *u8, sp_grr[3]) == 1 { sp_acc_or(cnt); any = 1 } } 1163 } else { more = 0 } 1164 } else { more = 0 } 1165 } 1166 if any == 1 { return sp_flush(cnt, dic) } 1167 return 0 1168} 1169// train the perceptron and evaluate on the test split. out is SP_O_N wide. 1170func sp_eval(train: *u8, test: *u8, out: *i64) -> i64 { 1171 var q: i64 = 0 1172 while q < SP_O_N { out[q] = 0; q = q + 1 } 1173 let ng: i64 = ab_gold_load(test) // gold set G in the global ab_set 1174 out[SP_O_NG] = ng 1175 if ng <= 0 { return 0 } 1176 sp_reset() 1177 // the POS tagger: trained once on the treebank set by sp_set_treebank (none -> every POS byte reads '^') 1178 sp_pos_on = 0 1179 if (sp_treebank as i64) != 0 { 1180 let pto: *i64 = sys_mmap(PT_O_N * RM_I64_BYTES) as *i64 1181 var pq: i64 = 0 1182 while pq < PT_O_N { pto[pq] = 0; pq = pq + 1 } 1183 pt_reset() 1184 if pt_train(sp_treebank, pto) >= 0 { sp_pos_on = 1; out[SP_O_POS_TOK] = pto[PT_O_TRAINTOK] } 1185 } 1186 out[SP_O_POS] = sp_pos_on 1187 // the parser: trained once on the same treebank, reading the tagger's tags, unless the parse switch is off 1188 sp_parse_on = 0 1189 if (sp_pos_on == 1) & (sp_parse_want == 1) { 1190 let dpo: *i64 = sys_mmap(DP_O_N * RM_I64_BYTES) as *i64 1191 var dq: i64 = 0 1192 while dq < DP_O_N { dpo[dq] = 0; dq = dq + 1 } 1193 dp_reset() 1194 if dp_train(sp_treebank, dpo) >= 0 { sp_parse_on = 1; out[SP_O_PARSE_SENT] = dpo[DP_O_TRAINSENT] } 1195 } 1196 out[SP_O_PARSE] = sp_parse_on 1197 // the clusters: the training vocabulary clustered once by the PPMI model set by sp_set_clusters (none -> every 1198 // cluster byte reads '@') 1199 sp_clust_on = 0 1200 if (sp_clust_model as i64) != 0 { 1201 wc_reset() 1202 if wc_load_model(sp_clust_model) == 1 { 1203 sp_clust_vocab(train) 1204 let wco: *i64 = sys_mmap(WC_O_N * RM_I64_BYTES) as *i64 1205 if wc_build(wco) == 1 { 1206 sp_clust_on = 1 1207 out[SP_O_CLUST_VOCAB] = wco[WC_O_VOCAB] 1208 out[SP_O_CLUST_INMODEL] = wco[WC_O_INMODEL] 1209 } 1210 } 1211 } 1212 out[SP_O_CLUST] = sp_clust_on 1213 // the objective: the perceptron unless the CRF mode is set; the log-domain tables are built once, idempotently 1214 sp_crf_on = sp_crf_want 1215 sp_crf_epoch = 0 1216 sp_crf_nll = 0 1217 sp_crf_margdev = 0 1218 sp_crf_negnll = 0 1219 if sp_crf_on == 1 { la_init() } 1220 // the embeddings: the table set by sp_set_embed, loaded through the ONE reader and admitted only at the declared width 1221 sp_emb_on = 0 1222 ef_reset() 1223 if (sp_emb_path as i64) != 0 { 1224 if ef_load(sp_emb_ppmi, sp_emb_path) == 1 { if ef_dim_of() == SP_EMB_DIMS { sp_emb_on = 1 } } 1225 } 1226 out[SP_O_EMB] = sp_emb_on 1227 out[SP_O_EMB_DIM] = ef_dim_of() 1228 let dic: *i64 = hs_new() 1229 out[SP_O_DICT] = ab_dict_load(train, dic) 1230 out[SP_O_DICT_MAJ] = sp_dict_stats(train, dic) 1231 let ctr: *i64 = sys_mmap(SP_CTR_N * RM_I64_BYTES) as *i64 1232 ctr[0] = 1; ctr[1] = 0; ctr[2] = 0; ctr[3] = 0 // first-pass flag + used + skipped + merged sentences 1233 var ep: i64 = 0 1234 var lastupd: i64 = 0 1235 // epoch 0 in file order (it records the groups); the later epochs walk the file again by default, or the groups in 1236 // a fresh deterministic order when the shuffle mode is set 1237 lastupd = sp_train_pass(train, dic, ctr) 1238 ctr[0] = 0 1239 out[SP_O_CRF_NLL_FIRST] = sp_crf_nll 1240 out[SP_O_GROUPS] = sp_ngroups 1241 out[SP_O_SHUF] = sp_shuffle_want 1242 if sp_shuffle_want == 1 { 1243 let glp: *i64 = sys_mmap(RM_I64_PAIR) as *i64 1244 glp[0] = 0 1245 let gb: *u8 = sys_read_file(train, glp) 1246 if (gb as i64) != 0 { 1247 let gn: i64 = glp[0] 1248 ep = 1 1249 while ep < SP_EPOCHS { 1250 pt_permute(sp_gorder, sp_ngroups, ep) 1251 sp_crf_epoch = ep 1252 sp_crf_nll = 0 1253 lastupd = 0 1254 var gi: i64 = 0 1255 while gi < sp_ngroups { lastupd = lastupd + sp_train_group(gb, gn, sp_gstarts[sp_gorder[gi]], dic); gi = gi + 1 } 1256 ep = ep + 1 1257 } 1258 } 1259 } else { 1260 ep = 1 1261 while ep < SP_EPOCHS { 1262 sp_crf_epoch = ep 1263 sp_crf_nll = 0 1264 lastupd = sp_train_pass(train, dic, ctr) 1265 ep = ep + 1 1266 } 1267 } 1268 out[SP_O_CRF] = sp_crf_on 1269 out[SP_O_CRF_NLL_LAST] = sp_crf_nll 1270 out[SP_O_CRF_MARGDEV] = sp_crf_margdev 1271 out[SP_O_CRF_NEGNLL] = sp_crf_negnll 1272 out[SP_O_EMB_LOOKUPS] = ef_lookup_count() 1273 out[SP_O_EMB_HITS] = ef_hit_count() 1274 out[SP_O_TRAINREC] = ctr[1] 1275 out[SP_O_TRAINSKIP] = ctr[2] 1276 out[SP_O_TRAINSENT] = ctr[3] 1277 out[SP_O_EPOCHS] = SP_EPOCHS 1278 out[SP_O_UPDATES] = lastupd 1279 sp_T = sp_clock 1280 if sp_T <= 0 { sp_T = 1 } 1281 // decode the test split with the averaged weights, collect predicted spans; the UNION set also carries the 1282 // training-dictionary tagger's n<=3-gram hits over the same tokens 1283 let pred_set: *i64 = hs_new() 1284 hs_reset(pred_set) 1285 let union_set: *i64 = hs_new() 1286 hs_reset(union_set) 1287 let lp: *i64 = sys_mmap(RM_I64_PAIR) as *i64 1288 lp[0] = 0 1289 let b: *u8 = sys_read_file(test, lp) 1290 if (b as i64) == 0 { return 0 } 1291 let n: i64 = lp[0] 1292 let rbuf: *u8 = sys_mmap(AB_RECON_CAP) 1293 let rp: *i64 = sys_mmap(RM_I64_PAIR) as *i64 1294 let r: *i64 = sys_mmap(AB_REC_N * RM_I64_BYTES) as *i64 1295 rp[0] = 0 1296 // test records of ONE sentence are consecutive (one per aspect): decode each sentence ONCE, collect its gold terms, 1297 // and score per OCCURRENCE (the paper's unit) beside the distinct-term sets 1298 var doc: i64 = 0 1299 var oc_tp: i64 = 0 1300 var oc_ns: i64 = 0 1301 var oc_ng: i64 = 0 1302 var ocu_tp: i64 = 0 1303 var ocu_ns: i64 = 0 1304 var tsent: i64 = 0 1305 var tpend: i64 = 0 1306 var ec_untok: i64 = 0 1307 var ec_unseen1: i64 = 0 1308 var ec_unseenm: i64 = 0 1309 var ec_seen1: i64 = 0 1310 var ec_seenm: i64 = 0 1311 var ec_unionmiss: i64 = 0 1312 sp_prevlen = 0 1313 sp_prevcnt = 0 1314 while ab_rec_next(b, n, rp, r) == 1 { 1315 doc = doc + 1 1316 let rl: i64 = ab_reconstruct((b as i64 + r[0]) as *u8, r[1], (b as i64 + r[2]) as *u8, r[3], rbuf, AB_RECON_CAP) 1317 if tpend == 1 { if sp_same_sent(rbuf, rl) == 0 { 1318 oc_ng = oc_ng + sp_lg[SL_COUNT] 1319 oc_ns = oc_ns + sp_lp[SL_COUNT] 1320 ocu_ns = ocu_ns + sp_lu[SL_COUNT] 1321 oc_tp = oc_tp + sl_inter(sp_lp, sp_lg) 1322 ocu_tp = ocu_tp + sl_inter(sp_lu, sp_lg) 1323 tsent = tsent + 1 1324 tpend = 0 1325 } } 1326 if tpend == 0 { 1327 let cnt: i64 = sp_tokenize(rbuf, rl) 1328 sp_tag_sentence(cnt) 1329 var cpy: i64 = 0 1330 while cpy < rl { sp_prevsent[cpy] = rbuf[cpy]; cpy = cpy + 1 } 1331 sp_prevlen = rl 1332 sp_prevcnt = cnt 1333 sl_reset(sp_lg) 1334 sl_reset(sp_lp) 1335 sl_reset(sp_lu) 1336 if cnt > 0 { 1337 var i: i64 = 0 1338 while i < cnt { sp_feats(cnt, i, dic); i = i + 1 } 1339 sp_viterbi(cnt, 1) 1340 sp_extract(cnt, pred_set) 1341 sp_extract_list(cnt, sp_lp) 1342 sp_extract_list(cnt, sp_lu) 1343 sp_dict_list(cnt, dic, sp_lu, 1) 1344 // fold this sentence's union spans into the corpus-level union SET (the distinct-term line) 1345 let ubuf: *u8 = sp_lu[SL_BUF] as *u8 1346 let uoffs: *i64 = sp_lu[SL_OFFS] as *i64 1347 let ulens: *i64 = sp_lu[SL_LENS] as *i64 1348 var q2: i64 = 0 1349 while q2 < sp_lu[SL_COUNT] { 1350 if hs_has_raw(union_set, (ubuf as i64 + uoffs[q2]) as *u8, ulens[q2]) == 0 { hs_add_raw(union_set, (ubuf as i64 + uoffs[q2]) as *u8, ulens[q2]) } 1351 q2 = q2 + 1 1352 } 1353 } 1354 tpend = 1 1355 } 1356 // this record's gold term, normalised the way the gold set is, into the sentence's gold list 1357 let gl: i64 = ab_norm((b as i64 + r[2]) as *u8, r[3], sp_normbuf, AB_TERM_MAX) 1358 // a term repeated as the gold of one sentence is ONE gold occurrence to the scorer (the list dedupes), so the census 1359 // counts it once too: a duplicate record is skipped, and the five parts equal gold minus matched by construction 1360 let dupgold: i64 = sl_has(sp_lg, sp_normbuf, gl) 1361 sl_add(sp_lg, sp_normbuf, gl) 1362 // the error census: this occurrence against the sentence's decoded lists (decoded when the sentence began), with 1363 // the same predicate the scorer uses, then the reason it could be missed 1364 if (dupgold == 0) & (sl_has(sp_lp, sp_normbuf, gl) == 0) { 1365 var multi: i64 = 0 1366 var q3: i64 = 0 1367 while q3 < gl { if sp_normbuf[q3] == (AB_NG_JOIN as u8) { multi = 1 } q3 = q3 + 1 } 1368 if sp_gold_bio(sp_prevcnt, (b as i64 + r[2]) as *u8, r[3]) == 0 { ec_untok = ec_untok + 1 } 1369 else { 1370 if hs_has_raw(dic, (b as i64 + r[2]) as *u8, r[3]) == 1 { 1371 if multi == 1 { ec_seenm = ec_seenm + 1 } else { ec_seen1 = ec_seen1 + 1 } 1372 } else { 1373 if multi == 1 { ec_unseenm = ec_unseenm + 1 } else { ec_unseen1 = ec_unseen1 + 1 } 1374 } 1375 } 1376 } 1377 if (dupgold == 0) & (sl_has(sp_lu, sp_normbuf, gl) == 0) { ec_unionmiss = ec_unionmiss + 1 } 1378 } 1379 out[SP_O_EC_UNTOK] = ec_untok 1380 out[SP_O_EC_UNSEEN1] = ec_unseen1 1381 out[SP_O_EC_UNSEENM] = ec_unseenm 1382 out[SP_O_EC_SEEN1] = ec_seen1 1383 out[SP_O_EC_SEENM] = ec_seenm 1384 out[SP_O_EC_UNIONMISS] = ec_unionmiss 1385 if tpend == 1 { 1386 oc_ng = oc_ng + sp_lg[SL_COUNT] 1387 oc_ns = oc_ns + sp_lp[SL_COUNT] 1388 ocu_ns = ocu_ns + sp_lu[SL_COUNT] 1389 oc_tp = oc_tp + sl_inter(sp_lp, sp_lg) 1390 ocu_tp = ocu_tp + sl_inter(sp_lu, sp_lg) 1391 tsent = tsent + 1 1392 tpend = 0 1393 } 1394 out[SP_O_TESTREC] = doc 1395 out[SP_O_TESTSENT] = tsent 1396 out[SP_O_OC_TP] = oc_tp 1397 out[SP_O_OC_NS] = oc_ns 1398 out[SP_O_OC_NG] = oc_ng 1399 let ocprf: *i64 = sys_mmap(AB_PRF_N * RM_I64_BYTES) as *i64 1400 ab_prf(oc_tp, oc_ns, oc_ng, ocprf) 1401 out[SP_O_OC_P] = ocprf[AB_P] 1402 out[SP_O_OC_R] = ocprf[AB_R] 1403 out[SP_O_OC_F1] = ocprf[AB_F1] 1404 out[SP_O_OCU_TP] = ocu_tp 1405 out[SP_O_OCU_NS] = ocu_ns 1406 let ocuprf: *i64 = sys_mmap(AB_PRF_N * RM_I64_BYTES) as *i64 1407 ab_prf(ocu_tp, ocu_ns, oc_ng, ocuprf) 1408 out[SP_O_OCU_P] = ocuprf[AB_P] 1409 out[SP_O_OCU_R] = ocuprf[AB_R] 1410 out[SP_O_OCU_F1] = ocuprf[AB_F1] 1411 out[SP_O_NS] = hs_count(pred_set) 1412 // score the union arm 1413 let uhash: *i64 = union_set[HS_HASH] as *i64 1414 let uoff: *i64 = union_set[HS_OFF] as *i64 1415 let ulen: *i64 = union_set[HS_LEN] as *i64 1416 let uarena: *u8 = union_set[HS_ARENA] as *u8 1417 var uinter: i64 = 0 1418 var uslot: i64 = 0 1419 while uslot < AB_SET_SLOTS { 1420 if uhash[uslot] != 0 { if ab_set_has_raw((uarena as i64 + uoff[uslot]) as *u8, ulen[uslot]) == 1 { uinter = uinter + 1 } } 1421 uslot = uslot + 1 1422 } 1423 out[SP_O_U_INTER] = uinter 1424 out[SP_O_U_NS] = hs_count(union_set) 1425 let uprf: *i64 = sys_mmap(AB_PRF_N * RM_I64_BYTES) as *i64 1426 ab_prf(uinter, hs_count(union_set), ng, uprf) 1427 out[SP_O_U_P] = uprf[AB_P] 1428 out[SP_O_U_R] = uprf[AB_R] 1429 out[SP_O_U_F1] = uprf[AB_F1] 1430 let phash: *i64 = pred_set[HS_HASH] as *i64 1431 let poff: *i64 = pred_set[HS_OFF] as *i64 1432 let plen: *i64 = pred_set[HS_LEN] as *i64 1433 let parena: *u8 = pred_set[HS_ARENA] as *u8 1434 var inter: i64 = 0 1435 var slot: i64 = 0 1436 while slot < AB_SET_SLOTS { 1437 if phash[slot] != 0 { if ab_set_has_raw((parena as i64 + poff[slot]) as *u8, plen[slot]) == 1 { inter = inter + 1 } } 1438 slot = slot + 1 1439 } 1440 out[SP_O_INTER] = inter 1441 let prf: *i64 = sys_mmap(AB_PRF_N * RM_I64_BYTES) as *i64 1442 ab_prf(inter, hs_count(pred_set), ng, prf) 1443 out[SP_O_P] = prf[AB_P] 1444 out[SP_O_R] = prf[AB_R] 1445 out[SP_O_F1] = prf[AB_F1] 1446 return 0 1447}