nx_postag.nx source
↩ module page · 493 lines · 20951 B
1// nx_postag.nx -- a PART-OF-SPEECH TAGGER for the estate (rung IM28's next lever: every SemEval-2014 winner used POS
2// features, and nx_capsearch over 7,433 organs found no tagger and no treebank here). A greedy left-to-right AVERAGED
3// PERCEPTRON over the 17 Universal POS tags, integer weights only (nofloat), trained on a CoNLL-U treebank (UD English
4// EWT, CC BY-SA 4.0, mirrored under knowledge/fetched/ with its LICENSE beside it -- a DATA asset, never code).
5// FEATURES per token: bias, word, 3-char suffix, first char, previous predicted tag, tag two back, previous tag+word,
6// previous word and its suffix, next word and its suffix, has-digit -- the standard set. Words are lowercased at train
7// and tag time so the tagger and the aspect model see one vocabulary. Averaging is the same lazy scheme as nx_absa_seq.
8// pt_tag_stream tags ANY token stream (offsets+lengths into one buffer), which is how the aspect model consumes it.
9// license_tier: ORIGINAL No hw writes (Rule 26). LIB.
10import "nx_syscalls.nx"
11import "nx_reviewmine_lib.nx"
12
13const PT_NTAGS: i64 = 17
14const PT_W: i64 = 1048576 // 2^20 weight slots shared by (feature, tag) pairs
15const PT_WMASK: i64 = 1048575
16const PT_EPOCHS: i64 = 8 // 5 left 8003 updates in the last epoch on EWT; raised with the shuffle below
17const PT_NFEAT: i64 = 15 // the 12 base + the tag bigram, the word two back and the word two ahead
18// PER-EPOCH SHUFFLE: an averaged perceptron trained in file order learns the file's order; the standard remedy is a
19// different sentence order every epoch. The order is a DETERMINISTIC permutation (Fisher-Yates driven by a small
20// linear congruential generator seeded by the epoch), so two runs still give identical weights and the gate's
21// determinism tooth holds. Sentence starts are recorded during the first pass; epoch 0 stays in file order.
22const PT_SENT_CAP: i64 = 131072 // sentence starts the shuffle can hold (EWT train is 12544)
23const PT_LCG_A: i64 = 1103515245
24const PT_LCG_C: i64 = 12345
25const PT_LCG_M: i64 = 2147483648
26const PT_SEED: i64 = 20260906
27const PT_TOK_CAP: i64 = 256
28const PT_TOKBUF: i64 = 16384
29const PT_FEATBUF: i64 = 128
30const PT_AFF: i64 = 3
31const PT_LINE_CAP: i64 = 4096
32const PT_COLS_FORM: i64 = 1
33const PT_COLS_UPOS: i64 = 3
34const PT_COLS_HEAD: i64 = 6 // HEAD: the 1-based id of the head token, 0 = root (read for the parser)
35const PT_COLS_DEPREL: i64 = 7 // DEPREL: the relation; the base name before any ':' subtype is kept
36const PT_COLON: i64 = 58
37const PT_RELBUF: i64 = 4096 // base relation names of one sentence, back to back
38const PT_HEAD_NONE: i64 = 0 - 1 // HEAD column absent or not a number
39const PT_TAB: i64 = 9
40const PT_LF: i64 = 10
41const PT_CR: i64 = 13
42const PT_HASH: i64 = 35 // '#' comment line
43const PT_DASH: i64 = 45
44const PT_DOT: i64 = 46
45const PT_CH_DIGIT0: i64 = 48
46const PT_CH_DIGIT9: i64 = 57
47const PT_CH_UPPER_A: i64 = 65
48const PT_CH_UPPER_Z: i64 = 90
49const PT_CASE_DELTA: i64 = 32
50const PT_TAG_NONE: i64 = 0 - 1
51// feature family prefix bytes
52const PT_PFX_B: i64 = 98 // 'b' bias
53const PT_PFX_W: i64 = 119 // 'w' word
54const PT_PFX_S: i64 = 115 // 's' suffix
55const PT_PFX_F: i64 = 102 // 'f' first char
56const PT_PFX_T1: i64 = 116 // 't' previous tag
57const PT_PFX_T2: i64 = 84 // 'T' tag two back
58const PT_PFX_TW: i64 = 117 // 'u' previous tag + word
59const PT_PFX_PW: i64 = 80 // 'P' previous word
60const PT_PFX_PS: i64 = 112 // 'p' previous word suffix
61const PT_PFX_NW: i64 = 78 // 'N' next word
62const PT_PFX_NS: i64 = 110 // 'n' next word suffix
63const PT_PFX_D: i64 = 100 // 'd' has digit
64const PT_PFX_TT: i64 = 103 // 'g' tag bigram (two back, previous)
65const PT_PFX_PW2: i64 = 81 // 'Q' word two back
66const PT_PFX_NW2: i64 = 77 // 'M' word two ahead
67const PT_CH_CARET: i64 = 94
68const PT_CH_DOLLAR: i64 = 36
69// the 17 Universal POS tags, in a fixed order; the id is the index
70const PT_TAGS: *u8 = "ADJ ADP ADV AUX CCONJ DET INTJ NOUN NUM PART PRON PROPN PUNCT SCONJ SYM VERB X"
71const PT_TAG_NOUN: i64 = 7
72const PT_TAG_PROPN: i64 = 11
73// out[] of pt_eval
74const PT_O_TOKENS: i64 = 0
75const PT_O_CORRECT: i64 = 1
76const PT_O_ACC: i64 = 2 // permil
77const PT_O_TRAINSENT: i64 = 3
78const PT_O_TRAINTOK: i64 = 4
79const PT_O_TESTSENT: i64 = 5
80const PT_O_UPDATES: i64 = 6 // updates in the last epoch
81const PT_O_UNKTAG: i64 = 7 // gold tags not in the 17 (a corpus dialect check)
82const PT_O_N: i64 = 8
83const PT_PERMIL: i64 = 1000
84
85static pt_w: *i64
86static pt_wsum: *i64
87static pt_wtime: *i64
88static pt_clock: i64
89static pt_T: i64
90static pt_featbuf: *u8
91static pt_tokbuf: *u8
92static pt_offs: *i64
93static pt_lens: *i64
94static pt_gold: *i64
95static pt_pred: *i64
96static pt_fhs: *i64
97static pt_line: *u8
98static pt_ready: i64 // 1 once trained; pt_tag_stream on an untrained tagger tags everything NOUN? no: tag 0 by ties
99// the dependency columns of the sentence last read (the parser's gold): HEAD as the 1-based id (0 root, -1 absent) and the
100// base DEPREL name as a span into pt_relbuf. The tagger ignores them; one reader serves both organs.
101static pt_heads: *i64
102static pt_reloffs: *i64
103static pt_rellens: *i64
104static pt_relbuf: *u8
105static pt_relused: i64
106// the shuffle: sentence start offsets of the training file and the per-epoch order
107static pt_starts: *i64
108static pt_order: *i64
109static pt_nstarts: i64
110
111func pt_reset() -> i64 {
112 if (pt_w as i64) == 0 {
113 pt_w = sys_mmap(PT_W * RM_I64_BYTES) as *i64
114 pt_wsum = sys_mmap(PT_W * RM_I64_BYTES) as *i64
115 pt_wtime = sys_mmap(PT_W * RM_I64_BYTES) as *i64
116 pt_featbuf = sys_mmap(PT_FEATBUF)
117 pt_tokbuf = sys_mmap(PT_TOKBUF)
118 pt_offs = sys_mmap(PT_TOK_CAP * RM_I64_BYTES) as *i64
119 pt_lens = sys_mmap(PT_TOK_CAP * RM_I64_BYTES) as *i64
120 pt_gold = sys_mmap(PT_TOK_CAP * RM_I64_BYTES) as *i64
121 pt_pred = sys_mmap(PT_TOK_CAP * RM_I64_BYTES) as *i64
122 pt_fhs = sys_mmap(PT_NFEAT * RM_I64_BYTES) as *i64
123 pt_line = sys_mmap(PT_LINE_CAP)
124 pt_heads = sys_mmap(PT_TOK_CAP * RM_I64_BYTES) as *i64
125 pt_reloffs = sys_mmap(PT_TOK_CAP * RM_I64_BYTES) as *i64
126 pt_rellens = sys_mmap(PT_TOK_CAP * RM_I64_BYTES) as *i64
127 pt_relbuf = sys_mmap(PT_RELBUF)
128 pt_starts = sys_mmap(PT_SENT_CAP * RM_I64_BYTES) as *i64
129 pt_order = sys_mmap(PT_SENT_CAP * RM_I64_BYTES) as *i64
130 } else {
131 var i: i64 = 0
132 while i < PT_W { pt_w[i] = 0; pt_wsum[i] = 0; pt_wtime[i] = 0; i = i + 1 }
133 }
134 pt_clock = 0
135 pt_T = 0
136 pt_ready = 0
137 return 0
138}
139// tag id of a UPOS string (or -1)
140func pt_tag_id(s: *u8, n: i64) -> i64 {
141 var id: i64 = 0
142 var p: i64 = 0
143 while PT_TAGS[p] != 0 {
144 var q: i64 = p
145 while (PT_TAGS[q] != 0) & (PT_TAGS[q] != 32) { q = q + 1 }
146 if q - p == n {
147 var same: i64 = 1
148 var k: i64 = 0
149 while k < n { if PT_TAGS[p + k] != s[k] { same = 0 } k = k + 1 }
150 if same == 1 { return id }
151 }
152 id = id + 1
153 if PT_TAGS[q] == 0 { p = q } else { p = q + 1 }
154 }
155 return PT_TAG_NONE
156}
157// the name of a tag id, written into out (returns its length); ids outside 0..16 write "?"
158func pt_tag_name(id: i64, out: *u8) -> i64 {
159 var cur: i64 = 0
160 var p: i64 = 0
161 while PT_TAGS[p] != 0 {
162 var q: i64 = p
163 while (PT_TAGS[q] != 0) & (PT_TAGS[q] != 32) { q = q + 1 }
164 if cur == id { var k: i64 = 0; while k < q - p { out[k] = PT_TAGS[p + k]; k = k + 1 } return q - p }
165 cur = cur + 1
166 if PT_TAGS[q] == 0 { p = q } else { p = q + 1 }
167 }
168 out[0] = 63
169 return 1
170}
171func pt_fh(pfx: i64, s: *u8, n: i64) -> i64 {
172 pt_featbuf[0] = pfx as u8
173 var i: i64 = 0
174 while i < n { if i + 1 < PT_FEATBUF { pt_featbuf[i + 1] = s[i] } i = i + 1 }
175 return rm_hash(pt_featbuf, n + 1)
176}
177func pt_has_digit(s: *u8, n: i64) -> i64 {
178 var i: i64 = 0
179 while i < n { let c: i64 = s[i] as i64; if c >= PT_CH_DIGIT0 { if c <= PT_CH_DIGIT9 { return 1 } } i = i + 1 }
180 return 0
181}
182// features for position i over the stream in pt_tokbuf/pt_offs/pt_lens with predicted tags pt_pred[0..i-1]
183func pt_feats(cnt: i64, i: i64) -> i64 {
184 let tp: *u8 = (pt_tokbuf as i64 + pt_offs[i]) as *u8
185 let tl: i64 = pt_lens[i]
186 var af: i64 = PT_AFF
187 if tl < af { af = tl }
188 pt_fhs[0] = pt_fh(PT_PFX_B, pt_featbuf, 0)
189 pt_fhs[1] = pt_fh(PT_PFX_W, tp, tl)
190 pt_fhs[2] = pt_fh(PT_PFX_S, (pt_tokbuf as i64 + pt_offs[i] + tl - af) as *u8, af)
191 pt_fhs[3] = pt_fh(PT_PFX_F, tp, 1)
192 let tb: *u8 = sys_mmap(4)
193 var t1: i64 = PT_TAG_NONE
194 var t2: i64 = PT_TAG_NONE
195 if i > 0 { t1 = pt_pred[i - 1] }
196 if i > 1 { t2 = pt_pred[i - 2] }
197 tb[0] = (PT_CH_CARET + t1 + 1) as u8
198 pt_fhs[4] = pt_fh(PT_PFX_T1, tb, 1)
199 tb[0] = (PT_CH_CARET + t2 + 1) as u8
200 pt_fhs[5] = pt_fh(PT_PFX_T2, tb, 1)
201 // previous tag + current word: prefix byte, tag byte, then the word
202 pt_featbuf[0] = PT_PFX_TW as u8
203 pt_featbuf[1] = (PT_CH_CARET + t1 + 1) as u8
204 var k: i64 = 0
205 while k < tl { if k + 2 < PT_FEATBUF { pt_featbuf[k + 2] = tp[k] } k = k + 1 }
206 pt_fhs[6] = rm_hash(pt_featbuf, tl + 2)
207 let bos: *u8 = sys_mmap(2); bos[0] = PT_CH_CARET as u8
208 let eos: *u8 = sys_mmap(2); eos[0] = PT_CH_DOLLAR as u8
209 if i > 0 {
210 let pp: *u8 = (pt_tokbuf as i64 + pt_offs[i - 1]) as *u8
211 let pl: i64 = pt_lens[i - 1]
212 var pa: i64 = PT_AFF
213 if pl < pa { pa = pl }
214 pt_fhs[7] = pt_fh(PT_PFX_PW, pp, pl)
215 pt_fhs[8] = pt_fh(PT_PFX_PS, (pt_tokbuf as i64 + pt_offs[i - 1] + pl - pa) as *u8, pa)
216 } else { pt_fhs[7] = pt_fh(PT_PFX_PW, bos, 1); pt_fhs[8] = pt_fh(PT_PFX_PS, bos, 1) }
217 if i < cnt - 1 {
218 let np: *u8 = (pt_tokbuf as i64 + pt_offs[i + 1]) as *u8
219 let nl: i64 = pt_lens[i + 1]
220 var na: i64 = PT_AFF
221 if nl < na { na = nl }
222 pt_fhs[9] = pt_fh(PT_PFX_NW, np, nl)
223 pt_fhs[10] = pt_fh(PT_PFX_NS, (pt_tokbuf as i64 + pt_offs[i + 1] + nl - na) as *u8, na)
224 } else { pt_fhs[9] = pt_fh(PT_PFX_NW, eos, 1); pt_fhs[10] = pt_fh(PT_PFX_NS, eos, 1) }
225 let dg: *u8 = sys_mmap(2)
226 dg[0] = (PT_CH_DIGIT0 + pt_has_digit(tp, tl)) as u8
227 pt_fhs[11] = pt_fh(PT_PFX_D, dg, 1)
228 // the tag bigram (two back, previous) and the words two back and two ahead
229 tb[0] = (PT_CH_CARET + t2 + 1) as u8
230 tb[1] = (PT_CH_CARET + t1 + 1) as u8
231 pt_fhs[12] = pt_fh(PT_PFX_TT, tb, 2)
232 if i > 1 { pt_fhs[13] = pt_fh(PT_PFX_PW2, (pt_tokbuf as i64 + pt_offs[i - 2]) as *u8, pt_lens[i - 2]) }
233 else { pt_fhs[13] = pt_fh(PT_PFX_PW2, bos, 1) }
234 if i < cnt - 2 { pt_fhs[14] = pt_fh(PT_PFX_NW2, (pt_tokbuf as i64 + pt_offs[i + 2]) as *u8, pt_lens[i + 2]) }
235 else { pt_fhs[14] = pt_fh(PT_PFX_NW2, eos, 1) }
236 return PT_NFEAT
237}
238// a deterministic permutation of 0..n-1 into a caller-supplied buffer (Fisher-Yates over a small LCG seeded by the
239// epoch): the ONE shuffle both the tagger and the aspect model train with, so their epochs agree on what a shuffle is
240func pt_permute(order: *i64, n: i64, epoch: i64) -> i64 {
241 var i: i64 = 0
242 while i < n { order[i] = i; i = i + 1 }
243 var x: i64 = (PT_SEED + epoch) % PT_LCG_M
244 var k: i64 = n - 1
245 while k > 0 {
246 x = (x * PT_LCG_A + PT_LCG_C) % PT_LCG_M
247 let j: i64 = x % (k + 1)
248 let t: i64 = order[k]
249 order[k] = order[j]
250 order[j] = t
251 k = k - 1
252 }
253 return n
254}
255// the tagger's own order buffer through the shared permutation
256func pt_shuffle(n: i64, epoch: i64) -> i64 { return pt_permute(pt_order, n, epoch) }
257func pt_w_at(idx: i64, useavg: i64) -> i64 {
258 if useavg == 1 { return pt_wsum[idx] + pt_w[idx] * (pt_T - pt_wtime[idx]) }
259 return pt_w[idx]
260}
261func pt_bump(idx: i64, delta: i64) -> i64 {
262 pt_wsum[idx] = pt_wsum[idx] + pt_w[idx] * (pt_clock - pt_wtime[idx])
263 pt_wtime[idx] = pt_clock
264 pt_w[idx] = pt_w[idx] + delta
265 return 0
266}
267func pt_score(tag: i64, useavg: i64) -> i64 {
268 var s: i64 = 0
269 var k: i64 = 0
270 while k < PT_NFEAT { s = s + pt_w_at(((pt_fhs[k] * PT_NTAGS + tag) & PT_WMASK), useavg); k = k + 1 }
271 return s
272}
273// greedy tagging of the loaded stream into pt_pred; with train=1, updates against pt_gold. Returns updates.
274func pt_tag_loaded(cnt: i64, train: i64) -> i64 {
275 var updates: i64 = 0
276 var i: i64 = 0
277 while i < cnt {
278 pt_feats(cnt, i)
279 var best: i64 = 0
280 var bestv: i64 = pt_score(0, 1 - train)
281 var t: i64 = 1
282 while t < PT_NTAGS { let v: i64 = pt_score(t, 1 - train); if v > bestv { bestv = v; best = t } t = t + 1 }
283 pt_pred[i] = best
284 if train == 1 { if pt_gold[i] >= 0 { if best != pt_gold[i] {
285 var k: i64 = 0
286 while k < PT_NFEAT {
287 pt_bump(((pt_fhs[k] * PT_NTAGS + pt_gold[i]) & PT_WMASK), 1)
288 pt_bump(((pt_fhs[k] * PT_NTAGS + best) & PT_WMASK), 0 - 1)
289 k = k + 1
290 }
291 updates = updates + 1
292 } } }
293 i = i + 1
294 }
295 return updates
296}
297// load one CoNLL-U sentence starting at *ip (lowercased FORM into pt_tokbuf, UPOS id into pt_gold); returns the
298// token count, 0 at end of file. Multiword ranges (1-2) and empty nodes (1.1) are skipped like every UD reader does.
299func pt_read_sentence(b: *u8, n: i64, ip: *i64, unk: *i64) -> i64 {
300 var cnt: i64 = 0
301 var used: i64 = 0
302 var p: i64 = ip[0]
303 var seen_token: i64 = 0
304 while p < n {
305 var e: i64 = p
306 while (e < n) & (b[e] != PT_LF as u8) { e = e + 1 }
307 var ll: i64 = e - p
308 if ll > 0 { if b[p + ll - 1] == PT_CR as u8 { ll = ll - 1 } }
309 if ll == 0 {
310 p = e + 1
311 if seen_token == 1 { ip[0] = p; return cnt }
312 } else { if b[p] == PT_HASH as u8 { p = e + 1 } else {
313 // split the 10 tab columns; keep FORM (col 1) and UPOS (col 3)
314 var col: i64 = 0
315 var cs: i64 = p
316 var q: i64 = p
317 var idok: i64 = 1
318 var form_s: i64 = 0
319 var form_l: i64 = 0
320 var upos_s: i64 = 0
321 var upos_l: i64 = 0
322 var head_s: i64 = 0
323 var head_l: i64 = 0
324 var rel_s: i64 = 0
325 var rel_l: i64 = 0
326 while q <= p + ll {
327 if (q == p + ll) | (b[q] == PT_TAB as u8) {
328 if col == 0 {
329 var z: i64 = cs
330 while z < q { if (b[z] == PT_DASH as u8) | (b[z] == PT_DOT as u8) { idok = 0 } z = z + 1 }
331 }
332 if col == PT_COLS_FORM { form_s = cs; form_l = q - cs }
333 if col == PT_COLS_UPOS { upos_s = cs; upos_l = q - cs }
334 if col == PT_COLS_HEAD { head_s = cs; head_l = q - cs }
335 if col == PT_COLS_DEPREL { rel_s = cs; rel_l = q - cs }
336 col = col + 1
337 cs = q + 1
338 }
339 q = q + 1
340 }
341 if (idok == 1) & (form_l > 0) & (cnt < PT_TOK_CAP) & (used + form_l < PT_TOKBUF) {
342 // HEAD: digits only, else absent; DEPREL: the base name up to ':' (a subtype never changes the base)
343 var hv: i64 = 0
344 var hok: i64 = 0
345 var hz: i64 = 0
346 if head_l > 0 { hok = 1 }
347 while hz < head_l {
348 let hc: i64 = b[head_s + hz] as i64
349 if (hc >= PT_CH_DIGIT0) & (hc <= PT_CH_DIGIT9) { hv = hv * 10 + (hc - PT_CH_DIGIT0) } else { hok = 0 }
350 hz = hz + 1
351 }
352 if hok == 1 { pt_heads[cnt] = hv } else { pt_heads[cnt] = PT_HEAD_NONE }
353 var rb: i64 = 0
354 while rb < rel_l { if b[rel_s + rb] == PT_COLON as u8 { rel_l = rb } else { rb = rb + 1 } }
355 if cnt == 0 { pt_relused = 0 }
356 pt_reloffs[cnt] = pt_relused
357 pt_rellens[cnt] = 0
358 if pt_relused + rel_l < PT_RELBUF {
359 rm_catn(pt_relbuf, pt_relused, (b as i64 + rel_s) as *u8, rel_l)
360 pt_rellens[cnt] = rel_l
361 pt_relused = pt_relused + rel_l
362 }
363 // lowercase the form into the token buffer
364 var k: i64 = 0
365 while k < form_l {
366 var c: i64 = b[form_s + k] as i64
367 if (c >= PT_CH_UPPER_A) & (c <= PT_CH_UPPER_Z) { c = c + PT_CASE_DELTA }
368 pt_tokbuf[used + k] = c as u8
369 k = k + 1
370 }
371 pt_offs[cnt] = used
372 pt_lens[cnt] = form_l
373 let tid: i64 = pt_tag_id((b as i64 + upos_s) as *u8, upos_l)
374 if tid < 0 { unk[0] = unk[0] + 1 }
375 pt_gold[cnt] = tid
376 used = used + form_l
377 cnt = cnt + 1
378 seen_token = 1
379 }
380 p = e + 1
381 } }
382 }
383 ip[0] = p
384 return cnt
385}
386// train on a CoNLL-U file for PT_EPOCHS; returns updates in the last epoch and fills counters
387func pt_train(path: *u8, out: *i64) -> i64 {
388 let lp: *i64 = sys_mmap(RM_I64_PAIR) as *i64
389 lp[0] = 0
390 let b: *u8 = sys_read_file(path, lp)
391 if (b as i64) == 0 { return 0 - 1 }
392 let n: i64 = lp[0]
393 let ip: *i64 = sys_mmap(RM_I64_PAIR) as *i64
394 let unk: *i64 = sys_mmap(RM_I64_PAIR) as *i64
395 unk[0] = 0
396 var ep: i64 = 0
397 var last: i64 = 0
398 var sents: i64 = 0
399 var toks: i64 = 0
400 pt_nstarts = 0
401 // epoch 0 in file order, recording every sentence start; later epochs walk a fresh deterministic permutation
402 ip[0] = 0
403 var start: i64 = 0
404 var cnt: i64 = pt_read_sentence(b, n, ip, unk)
405 while cnt > 0 {
406 if pt_nstarts < PT_SENT_CAP { pt_starts[pt_nstarts] = start; pt_nstarts = pt_nstarts + 1 }
407 last = last + pt_tag_loaded(cnt, 1)
408 pt_clock = pt_clock + 1
409 sents = sents + 1
410 toks = toks + cnt
411 start = ip[0]
412 cnt = pt_read_sentence(b, n, ip, unk)
413 }
414 ep = 1
415 while ep < PT_EPOCHS {
416 last = 0
417 pt_shuffle(pt_nstarts, ep)
418 var si: i64 = 0
419 while si < pt_nstarts {
420 ip[0] = pt_starts[pt_order[si]]
421 let c2: i64 = pt_read_sentence(b, n, ip, unk)
422 if c2 > 0 { last = last + pt_tag_loaded(c2, 1); pt_clock = pt_clock + 1 }
423 si = si + 1
424 }
425 ep = ep + 1
426 }
427 pt_T = pt_clock
428 if pt_T <= 0 { pt_T = 1 }
429 pt_ready = 1
430 out[PT_O_TRAINSENT] = sents
431 out[PT_O_TRAINTOK] = toks
432 out[PT_O_UPDATES] = last
433 out[PT_O_UNKTAG] = unk[0]
434 return last
435}
436// token accuracy on a CoNLL-U file with the averaged weights
437func pt_test(path: *u8, out: *i64) -> i64 {
438 let lp: *i64 = sys_mmap(RM_I64_PAIR) as *i64
439 lp[0] = 0
440 let b: *u8 = sys_read_file(path, lp)
441 if (b as i64) == 0 { return 0 - 1 }
442 let n: i64 = lp[0]
443 let ip: *i64 = sys_mmap(RM_I64_PAIR) as *i64
444 let unk: *i64 = sys_mmap(RM_I64_PAIR) as *i64
445 unk[0] = 0
446 ip[0] = 0
447 var toks: i64 = 0
448 var correct: i64 = 0
449 var sents: i64 = 0
450 var cnt: i64 = pt_read_sentence(b, n, ip, unk)
451 while cnt > 0 {
452 pt_tag_loaded(cnt, 0)
453 var i: i64 = 0
454 while i < cnt { if pt_gold[i] >= 0 { toks = toks + 1; if pt_pred[i] == pt_gold[i] { correct = correct + 1 } } i = i + 1 }
455 sents = sents + 1
456 cnt = pt_read_sentence(b, n, ip, unk)
457 }
458 out[PT_O_TOKENS] = toks
459 out[PT_O_CORRECT] = correct
460 if toks > 0 { out[PT_O_ACC] = correct * PT_PERMIL / toks } else { out[PT_O_ACC] = 0 }
461 out[PT_O_TESTSENT] = sents
462 return 0
463}
464// train then test; out is PT_O_N wide
465func pt_eval(train: *u8, test: *u8, out: *i64) -> i64 {
466 var q: i64 = 0
467 while q < PT_O_N { out[q] = 0; q = q + 1 }
468 pt_reset()
469 if pt_train(train, out) < 0 { return 0 - 1 }
470 return pt_test(test, out)
471}
472// tag an arbitrary token stream (offsets+lengths into buf) with the trained tagger; tags written to tags_out
473func pt_tag_stream(buf: *u8, offs: *i64, lens: *i64, cnt: i64, tags_out: *i64) -> i64 {
474 if cnt <= 0 { return 0 }
475 var c: i64 = cnt
476 if c > PT_TOK_CAP { c = PT_TOK_CAP }
477 var used: i64 = 0
478 var i: i64 = 0
479 while i < c {
480 var l: i64 = lens[i]
481 if used + l >= PT_TOKBUF { l = 0 }
482 rm_catn(pt_tokbuf, used, (buf as i64 + offs[i]) as *u8, l)
483 pt_offs[i] = used
484 pt_lens[i] = l
485 pt_gold[i] = PT_TAG_NONE
486 used = used + l
487 i = i + 1
488 }
489 pt_tag_loaded(c, 0)
490 i = 0
491 while i < c { tags_out[i] = pt_pred[i]; i = i + 1 }
492 return c
493}