nx_postag.nx
buildroot/runtime/nx_postag.nx
about
nx_postag.nx -- a PART-OF-SPEECH TAGGER for the estate (rung IM28's next lever: every SemEval-2014 winner used POS
features, and nx_capsearch over 7,433 organs found no tagger and no treebank here). A greedy left-to-right AVERAGED
PERCEPTRON over the 17 Universal POS tags, integer weights only (nofloat), trained on a CoNLL-U treebank (UD English
EWT, CC BY-SA 4.0, mirrored under knowledge/fetched/ with its LICENSE beside it -- a DATA asset, never code).
FEATURES per token: bias, word, 3-char suffix, first char, previous predicted tag, tag two back, previous tag+word,
previous word and its suffix, next word and its suffix, has-digit -- the standard set. Words are lowercased at train
and tag time so the tagger and the aspect model see one vocabulary. Averaging is the same lazy scheme as nx_absa_seq.
pt_tag_stream tags ANY token stream (offsets+lengths into one buffer), which is how the aspect model consumes it.
license_tier: ORIGINAL No hw writes (Rule 26). LIB.
dependencies 2 imports · 6 importers
imports: nx_syscalls.nxnx_reviewmine_lib.nx
imported by: nx_absa_seq.nxnx_depparse.nxnx_depparse_eval.nxnx_depparse_gate.nxnx_postag_eval.nxnx_postag_gate.nx
structs
| none |
consts
| 13 | const PT_NTAGS: i64 = 17 |
| 14 | const PT_W: i64 = 1048576 // 2^20 weight slots shared by (feature, tag) pairs |
| 15 | const PT_WMASK: i64 = 1048575 |
| 16 | const PT_EPOCHS: i64 = 8 // 5 left 8003 updates in the last epoch on EWT; raised with the shuffle below |
| 17 | const PT_NFEAT: i64 = 15 // the 12 base + the tag bigram, the word two back and the word two ahead |
| 22 | const PT_SENT_CAP: i64 = 131072 // sentence starts the shuffle can hold (EWT train is 12544) |
| 23 | const PT_LCG_A: i64 = 1103515245 |
| 24 | const PT_LCG_C: i64 = 12345 |
| 25 | const PT_LCG_M: i64 = 2147483648 |
| 26 | const PT_SEED: i64 = 20260906 |
| 27 | const PT_TOK_CAP: i64 = 256 |
| 28 | const PT_TOKBUF: i64 = 16384 |
| 29 | const PT_FEATBUF: i64 = 128 |
| 30 | const PT_AFF: i64 = 3 |
| 31 | const PT_LINE_CAP: i64 = 4096 |
| 32 | const PT_COLS_FORM: i64 = 1 |
| 33 | const PT_COLS_UPOS: i64 = 3 |
| 34 | const PT_COLS_HEAD: i64 = 6 // HEAD: the 1-based id of the head token, 0 = root (read for the parser) |
| 35 | const PT_COLS_DEPREL: i64 = 7 // DEPREL: the relation; the base name before any ':' subtype is kept |
| 36 | const PT_COLON: i64 = 58 |
| 37 | const PT_RELBUF: i64 = 4096 // base relation names of one sentence, back to back |
| 38 | const PT_HEAD_NONE: i64 = 0 - 1 // HEAD column absent or not a number |
| 39 | const PT_TAB: i64 = 9 |
| 40 | const PT_LF: i64 = 10 |
| 41 | const PT_CR: i64 = 13 |
| 42 | const PT_HASH: i64 = 35 // '#' comment line |
| 43 | const PT_DASH: i64 = 45 |
| 44 | const PT_DOT: i64 = 46 |
| 45 | const PT_CH_DIGIT0: i64 = 48 |
| 46 | const PT_CH_DIGIT9: i64 = 57 |
| 47 | const PT_CH_UPPER_A: i64 = 65 |
| 48 | const PT_CH_UPPER_Z: i64 = 90 |
| 49 | const PT_CASE_DELTA: i64 = 32 |
| 50 | const PT_TAG_NONE: i64 = 0 - 1 |
| 52 | const PT_PFX_B: i64 = 98 // 'b' bias |
| 53 | const PT_PFX_W: i64 = 119 // 'w' word |
| 54 | const PT_PFX_S: i64 = 115 // 's' suffix |
| 55 | const PT_PFX_F: i64 = 102 // 'f' first char |
| 56 | const PT_PFX_T1: i64 = 116 // 't' previous tag |
| 57 | const PT_PFX_T2: i64 = 84 // 'T' tag two back |
| 58 | const PT_PFX_TW: i64 = 117 // 'u' previous tag + word |
| 59 | const PT_PFX_PW: i64 = 80 // 'P' previous word |
| 60 | const PT_PFX_PS: i64 = 112 // 'p' previous word suffix |
| 61 | const PT_PFX_NW: i64 = 78 // 'N' next word |
| 62 | const PT_PFX_NS: i64 = 110 // 'n' next word suffix |
| 63 | const PT_PFX_D: i64 = 100 // 'd' has digit |
| 64 | const PT_PFX_TT: i64 = 103 // 'g' tag bigram (two back, previous) |
| 65 | const PT_PFX_PW2: i64 = 81 // 'Q' word two back |
| 66 | const PT_PFX_NW2: i64 = 77 // 'M' word two ahead |
| 67 | const PT_CH_CARET: i64 = 94 |
| 68 | const PT_CH_DOLLAR: i64 = 36 |
| 70 | const PT_TAGS: *u8 = "ADJ ADP ADV AUX CCONJ DET INTJ NOUN NUM PART PRON PROPN PUNCT SCONJ SYM VERB X" |
| 71 | const PT_TAG_NOUN: i64 = 7 |
| 72 | const PT_TAG_PROPN: i64 = 11 |
| 74 | const PT_O_TOKENS: i64 = 0 |
| 75 | const PT_O_CORRECT: i64 = 1 |
| 76 | const PT_O_ACC: i64 = 2 // permil |
| 77 | const PT_O_TRAINSENT: i64 = 3 |
| 78 | const PT_O_TRAINTOK: i64 = 4 |
| 79 | const PT_O_TESTSENT: i64 = 5 |
| 80 | const PT_O_UPDATES: i64 = 6 // updates in the last epoch |
| 81 | const PT_O_UNKTAG: i64 = 7 // gold tags not in the 17 (a corpus dialect check) |
| 82 | const PT_O_N: i64 = 8 |
| 83 | const PT_PERMIL: i64 = 1000 |
functions
| 111 | func pt_reset() -> i64 |
| 140 | func pt_tag_id(s: *u8, n: i64) -> i64 |
| 158 | func pt_tag_name(id: i64, out: *u8) -> i64 called by 1: main |
| 171 | func pt_fh(pfx: i64, s: *u8, n: i64) -> i64 |
| 177 | func pt_has_digit(s: *u8, n: i64) -> i64 called by 1: pt_feats |
| 183 | func pt_feats(cnt: i64, i: i64) -> i64 |
| 240 | func pt_permute(order: *i64, n: i64, epoch: i64) -> i64 |
| 256 | func pt_shuffle(n: i64, epoch: i64) -> i64 { return pt_permute(pt_order, n, epoch) } |
| 257 | func pt_w_at(idx: i64, useavg: i64) -> i64 called by 1: pt_score |
| 261 | func pt_bump(idx: i64, delta: i64) -> i64 called by 1: pt_tag_loaded |
| 267 | func pt_score(tag: i64, useavg: i64) -> i64 |
| 274 | func pt_tag_loaded(cnt: i64, train: i64) -> i64 |
| 299 | func pt_read_sentence(b: *u8, n: i64, ip: *i64, unk: *i64) -> i64 |
| 387 | func pt_train(path: *u8, out: *i64) -> i64 |
| 437 | func pt_test(path: *u8, out: *i64) -> i64 |
| 465 | func pt_eval(train: *u8, test: *u8, out: *i64) -> i64 |
| 473 | func pt_tag_stream(buf: *u8, offs: *i64, lens: *i64, cnt: i64, tags_out: *i64) -> i64 |