nx_depparse.nx source
↩ module page · 536 lines · 21393 B
1// nx_depparse.nx -- a DEPENDENCY PARSER for the estate (rung IM28's parse-feature lever: DLIREC, the SemEval-2014
2// restaurants winner, took its exceed from dependency features, and nx_capsearch over 7,437 organs found no parser
3// here -- every hit was the software-dependency homonym). ARC-STANDARD transition parsing (SHIFT, LEFT-ARC(rel),
4// RIGHT-ARC(rel)) with a greedy AVERAGED PERCEPTRON over the actions, integer weights only (nofloat), trained by the
5// STATIC ORACLE on a CoNLL-U treebank read by nx_postag's reader (HEAD and DEPREL columns), with POS tags from
6// nx_postag's own predictions on the training sentences, so train and test see ONE tag generator. Relation labels are
7// the UD base names (before ':'), interned in first-seen order at training. Non-projective training sentences (the
8// static oracle cannot reach them) are SKIPPED and counted. A single-root constraint: the root attaches only once the
9// buffer is empty. dp_parse_stream parses ANY tagged token stream, which is how the aspect model consumes it.
10// FEATURES (Zhang and Nivre 2011's core): word and tag of s0, s1, b0, b1, b2, their word+tag pairs, tag pairs and
11// triples across the stack top and buffer front, the tags of s0's and s1's leftmost and rightmost children, word pairs
12// s0+b0 and s1+s0, the bucketed distance s1..s0 with their tags, and the relation of s0's rightmost child.
13// license_tier: ORIGINAL No hw writes (Rule 26). LIB.
14import "nx_syscalls.nx"
15import "nx_reviewmine_lib.nx"
16import "nx_postag.nx"
17
18const DP_W: i64 = 2097152 // 2^21 weight slots shared by (feature, action) pairs
19const DP_WMASK: i64 = 2097151
20const DP_EPOCHS: i64 = 12 // 5 left 17777 updates in the last epoch on EWT (UAS 804); raised toward convergence
21const DP_NFEAT: i64 = 24
22const DP_NREL: i64 = 48 // label table cap (UD v2 has 37 universal relations)
23const DP_RELNAME: i64 = 16 // bytes per interned relation name
24const DP_ACT_SHIFT: i64 = 0 // LEFT-ARC(r) = 1 + r ; RIGHT-ARC(r) = 1 + DP_NREL + r
25const DP_NACT: i64 = 97 // 1 + 2 * DP_NREL
26const DP_NONE: i64 = 0 - 1
27const DP_TOK_CAP: i64 = 256 // same as the reader's
28const DP_NODE_CAP: i64 = 258 // tokens + the virtual ROOT + one spare
29const DP_FEATBUF: i64 = 192
30const DP_DIST_CAP: i64 = 6 // distance buckets 1..5 and 6-or-more
31const DP_CH_CARET: i64 = 94 // '^' encodes tag -1
32const DP_CH_DOLLAR: i64 = 36 // '$' the ROOT node's tag byte
33const DP_CH_AT: i64 = 64 // '@' no such node (child absent, buffer exhausted)
34const DP_CH_A: i64 = 65 // 'A' + rel id encodes a relation as one byte
35const DP_CH_ZERO: i64 = 48
36const DP_SEP: i64 = 31 // unit separator between two words in one feature
37const DP_PERMIL: i64 = 1000
38// feature family prefix bytes
39const DP_PFX_B: i64 = 98 // 'b' bias
40const DP_PFX_S0W: i64 = 65
41const DP_PFX_S0P: i64 = 66
42const DP_PFX_S0WP: i64 = 67
43const DP_PFX_S1W: i64 = 68
44const DP_PFX_S1P: i64 = 69
45const DP_PFX_S1WP: i64 = 70
46const DP_PFX_B0W: i64 = 71
47const DP_PFX_B0P: i64 = 72
48const DP_PFX_B0WP: i64 = 73
49const DP_PFX_B1P: i64 = 74
50const DP_PFX_B2P: i64 = 75
51const DP_PFX_S0PB0P: i64 = 76
52const DP_PFX_S1PS0P: i64 = 77
53const DP_PFX_S1PS0PB0P: i64 = 78
54const DP_PFX_S0PB0PB1P: i64 = 79
55const DP_PFX_S1PS0PS0LP: i64 = 80
56const DP_PFX_S1PS0PS0RP: i64 = 81
57const DP_PFX_S1PS1RPS0P: i64 = 82
58const DP_PFX_S1PS1LPS0P: i64 = 83
59const DP_PFX_S0WB0W: i64 = 84
60const DP_PFX_S1WS0W: i64 = 85
61const DP_PFX_DIST: i64 = 86
62const DP_PFX_S0RR: i64 = 87
63// out[] of dp_eval
64const DP_O_TOKENS: i64 = 0
65const DP_O_UAS_CORRECT: i64 = 1
66const DP_O_LAS_CORRECT: i64 = 2
67const DP_O_UAS: i64 = 3 // permil
68const DP_O_LAS: i64 = 4 // permil, on the base relation names
69const DP_O_TRAINSENT: i64 = 5
70const DP_O_TRAINTOK: i64 = 6
71const DP_O_NONPROJ: i64 = 7 // training sentences the static oracle could not reach (skipped)
72const DP_O_TESTSENT: i64 = 8
73const DP_O_UPDATES: i64 = 9 // updates in the last epoch
74const DP_O_NREL: i64 = 10 // relation labels interned
75const DP_O_UNUSABLE: i64 = 11 // training sentences with an absent HEAD or an overflowing label table
76const DP_O_N: i64 = 12
77
78static dp_w: *i64
79static dp_wsum: *i64
80static dp_wtime: *i64
81static dp_clock: i64
82static dp_T: i64
83static dp_featbuf: *u8
84static dp_fhs: *i64
85// the sentence being parsed: tokens as spans into one buffer, a tag per token, cnt tokens; node cnt is ROOT
86static dp_buf: *u8
87static dp_offs: *i64
88static dp_lens: *i64
89static dp_tags: *i64
90static dp_cnt: i64
91// gold (training and scoring): head index (cnt = ROOT), relation id, gold child counts
92static dp_gh: *i64
93static dp_gr: *i64
94static dp_gnc: *i64
95// the parser state
96static dp_heads: *i64
97static dp_rels: *i64
98static dp_lc: *i64
99static dp_rc: *i64
100static dp_att: *i64
101static dp_st: *i64
102static dp_sp: i64
103static dp_bi: i64
104// the relation label table
105static dp_relnames: *u8
106static dp_rellens: *i64
107static dp_nrel: i64
108static dp_ready: i64
109static dp_rootw: *u8
110static dp_eosw: *u8
111
112func dp_reset() -> i64 {
113 if (dp_w as i64) == 0 {
114 dp_w = sys_mmap(DP_W * RM_I64_BYTES) as *i64
115 dp_wsum = sys_mmap(DP_W * RM_I64_BYTES) as *i64
116 dp_wtime = sys_mmap(DP_W * RM_I64_BYTES) as *i64
117 dp_featbuf = sys_mmap(DP_FEATBUF)
118 dp_fhs = sys_mmap(DP_NFEAT * RM_I64_BYTES) as *i64
119 dp_gh = sys_mmap(DP_NODE_CAP * RM_I64_BYTES) as *i64
120 dp_gr = sys_mmap(DP_NODE_CAP * RM_I64_BYTES) as *i64
121 dp_gnc = sys_mmap(DP_NODE_CAP * RM_I64_BYTES) as *i64
122 dp_heads = sys_mmap(DP_NODE_CAP * RM_I64_BYTES) as *i64
123 dp_rels = sys_mmap(DP_NODE_CAP * RM_I64_BYTES) as *i64
124 dp_lc = sys_mmap(DP_NODE_CAP * RM_I64_BYTES) as *i64
125 dp_rc = sys_mmap(DP_NODE_CAP * RM_I64_BYTES) as *i64
126 dp_att = sys_mmap(DP_NODE_CAP * RM_I64_BYTES) as *i64
127 dp_st = sys_mmap(DP_NODE_CAP * RM_I64_BYTES) as *i64
128 dp_relnames = sys_mmap(DP_NREL * DP_RELNAME)
129 dp_rellens = sys_mmap(DP_NREL * RM_I64_BYTES) as *i64
130 dp_rootw = sys_mmap(2)
131 dp_rootw[0] = DP_CH_CARET as u8
132 dp_eosw = sys_mmap(2)
133 dp_eosw[0] = DP_CH_DOLLAR as u8
134 } else {
135 var i: i64 = 0
136 while i < DP_W { dp_w[i] = 0; dp_wsum[i] = 0; dp_wtime[i] = 0; i = i + 1 }
137 }
138 dp_nrel = 0
139 dp_clock = 0
140 dp_T = 0
141 dp_ready = 0
142 dp_cnt = 0
143 return 0
144}
145// the id of a base relation name; intern=1 adds an unseen name (first-seen order) when the table has room
146func dp_rel_id(s: *u8, n: i64, intern: i64) -> i64 {
147 var r: i64 = 0
148 while r < dp_nrel {
149 if dp_rellens[r] == n {
150 var same: i64 = 1
151 var k: i64 = 0
152 while k < n { if dp_relnames[r * DP_RELNAME + k] != s[k] { same = 0 } k = k + 1 }
153 if same == 1 { return r }
154 }
155 r = r + 1
156 }
157 if (intern == 1) & (dp_nrel < DP_NREL) & (n > 0) & (n <= DP_RELNAME) {
158 var k2: i64 = 0
159 while k2 < n { dp_relnames[dp_nrel * DP_RELNAME + k2] = s[k2]; k2 = k2 + 1 }
160 dp_rellens[dp_nrel] = n
161 dp_nrel = dp_nrel + 1
162 return dp_nrel - 1
163 }
164 return DP_NONE
165}
166// the name of a relation id written into out (returns its length); unknown ids write "?"
167func dp_rel_name(id: i64, out: *u8) -> i64 {
168 if (id < 0) | (id >= dp_nrel) { out[0] = 63; return 1 }
169 var k: i64 = 0
170 while k < dp_rellens[id] { out[k] = dp_relnames[id * DP_RELNAME + k]; k = k + 1 }
171 return dp_rellens[id]
172}
173// node access: real tokens 0..cnt-1, ROOT = cnt, absent = -1
174func dp_wp(i: i64) -> *u8 {
175 if i < 0 { return dp_eosw }
176 if i >= dp_cnt { return dp_rootw }
177 return (dp_buf as i64 + dp_offs[i]) as *u8
178}
179func dp_wl(i: i64) -> i64 {
180 if i < 0 { return 1 }
181 if i >= dp_cnt { return 1 }
182 return dp_lens[i]
183}
184func dp_tb(i: i64) -> i64 {
185 if i < 0 { return DP_CH_AT }
186 if i >= dp_cnt { return DP_CH_DOLLAR }
187 return DP_CH_CARET + dp_tags[i] + 1
188}
189func dp_lcof(i: i64) -> i64 { if i < 0 { return DP_NONE } return dp_lc[i] }
190func dp_rcof(i: i64) -> i64 { if i < 0 { return DP_NONE } return dp_rc[i] }
191func dp_relbyte(i: i64) -> i64 {
192 if i < 0 { return DP_CH_AT }
193 if dp_rels[i] < 0 { return DP_CH_AT }
194 return DP_CH_A + dp_rels[i]
195}
196// feature hashing helpers, every one through the single scratch buffer
197func dp_f0(pfx: i64) -> i64 { dp_featbuf[0] = pfx as u8; return rm_hash(dp_featbuf, 1) }
198func dp_ft1(pfx: i64, a: i64) -> i64 { dp_featbuf[0] = pfx as u8; dp_featbuf[1] = a as u8; return rm_hash(dp_featbuf, 2) }
199func dp_ft2(pfx: i64, a: i64, b: i64) -> i64 { dp_featbuf[0] = pfx as u8; dp_featbuf[1] = a as u8; dp_featbuf[2] = b as u8; return rm_hash(dp_featbuf, 3) }
200func dp_ft3(pfx: i64, a: i64, b: i64, c: i64) -> i64 { dp_featbuf[0] = pfx as u8; dp_featbuf[1] = a as u8; dp_featbuf[2] = b as u8; dp_featbuf[3] = c as u8; return rm_hash(dp_featbuf, 4) }
201func dp_fw(pfx: i64, i: i64) -> i64 {
202 dp_featbuf[0] = pfx as u8
203 let s: *u8 = dp_wp(i)
204 var n: i64 = dp_wl(i)
205 if n + 1 > DP_FEATBUF { n = DP_FEATBUF - 1 }
206 var k: i64 = 0
207 while k < n { dp_featbuf[k + 1] = s[k]; k = k + 1 }
208 return rm_hash(dp_featbuf, n + 1)
209}
210func dp_fwt(pfx: i64, i: i64) -> i64 {
211 dp_featbuf[0] = pfx as u8
212 dp_featbuf[1] = dp_tb(i) as u8
213 let s: *u8 = dp_wp(i)
214 var n: i64 = dp_wl(i)
215 if n + 2 > DP_FEATBUF { n = DP_FEATBUF - 2 }
216 var k: i64 = 0
217 while k < n { dp_featbuf[k + 2] = s[k]; k = k + 1 }
218 return rm_hash(dp_featbuf, n + 2)
219}
220func dp_fww(pfx: i64, i: i64, j: i64) -> i64 {
221 dp_featbuf[0] = pfx as u8
222 var o: i64 = 1
223 let s: *u8 = dp_wp(i)
224 var n: i64 = dp_wl(i)
225 var k: i64 = 0
226 while k < n { if o < DP_FEATBUF - 1 { dp_featbuf[o] = s[k]; o = o + 1 } k = k + 1 }
227 if o < DP_FEATBUF - 1 { dp_featbuf[o] = DP_SEP as u8; o = o + 1 }
228 let s2: *u8 = dp_wp(j)
229 var n2: i64 = dp_wl(j)
230 k = 0
231 while k < n2 { if o < DP_FEATBUF - 1 { dp_featbuf[o] = s2[k]; o = o + 1 } k = k + 1 }
232 return rm_hash(dp_featbuf, o)
233}
234// the DP_NFEAT feature hashes of the current state into dp_fhs
235func dp_feats() -> i64 {
236 var s0: i64 = DP_NONE
237 var s1: i64 = DP_NONE
238 if dp_sp >= 0 { s0 = dp_st[dp_sp] }
239 if dp_sp >= 1 { s1 = dp_st[dp_sp - 1] }
240 var b0: i64 = DP_NONE
241 var b1: i64 = DP_NONE
242 var b2: i64 = DP_NONE
243 if dp_bi < dp_cnt { b0 = dp_bi }
244 if dp_bi + 1 < dp_cnt { b1 = dp_bi + 1 }
245 if dp_bi + 2 < dp_cnt { b2 = dp_bi + 2 }
246 dp_fhs[0] = dp_f0(DP_PFX_B)
247 dp_fhs[1] = dp_fw(DP_PFX_S0W, s0)
248 dp_fhs[2] = dp_ft1(DP_PFX_S0P, dp_tb(s0))
249 dp_fhs[3] = dp_fwt(DP_PFX_S0WP, s0)
250 dp_fhs[4] = dp_fw(DP_PFX_S1W, s1)
251 dp_fhs[5] = dp_ft1(DP_PFX_S1P, dp_tb(s1))
252 dp_fhs[6] = dp_fwt(DP_PFX_S1WP, s1)
253 dp_fhs[7] = dp_fw(DP_PFX_B0W, b0)
254 dp_fhs[8] = dp_ft1(DP_PFX_B0P, dp_tb(b0))
255 dp_fhs[9] = dp_fwt(DP_PFX_B0WP, b0)
256 dp_fhs[10] = dp_ft1(DP_PFX_B1P, dp_tb(b1))
257 dp_fhs[11] = dp_ft1(DP_PFX_B2P, dp_tb(b2))
258 dp_fhs[12] = dp_ft2(DP_PFX_S0PB0P, dp_tb(s0), dp_tb(b0))
259 dp_fhs[13] = dp_ft2(DP_PFX_S1PS0P, dp_tb(s1), dp_tb(s0))
260 dp_fhs[14] = dp_ft3(DP_PFX_S1PS0PB0P, dp_tb(s1), dp_tb(s0), dp_tb(b0))
261 dp_fhs[15] = dp_ft3(DP_PFX_S0PB0PB1P, dp_tb(s0), dp_tb(b0), dp_tb(b1))
262 dp_fhs[16] = dp_ft3(DP_PFX_S1PS0PS0LP, dp_tb(s1), dp_tb(s0), dp_tb(dp_lcof(s0)))
263 dp_fhs[17] = dp_ft3(DP_PFX_S1PS0PS0RP, dp_tb(s1), dp_tb(s0), dp_tb(dp_rcof(s0)))
264 dp_fhs[18] = dp_ft3(DP_PFX_S1PS1RPS0P, dp_tb(s1), dp_tb(dp_rcof(s1)), dp_tb(s0))
265 dp_fhs[19] = dp_ft3(DP_PFX_S1PS1LPS0P, dp_tb(s1), dp_tb(dp_lcof(s1)), dp_tb(s0))
266 dp_fhs[20] = dp_fww(DP_PFX_S0WB0W, s0, b0)
267 dp_fhs[21] = dp_fww(DP_PFX_S1WS0W, s1, s0)
268 var d: i64 = 0
269 if (s0 >= 0) & (s1 >= 0) { if s1 < dp_cnt { d = s0 - s1 } }
270 if d > DP_DIST_CAP { d = DP_DIST_CAP }
271 dp_fhs[22] = dp_ft3(DP_PFX_DIST, DP_CH_ZERO + d, dp_tb(s1), dp_tb(s0))
272 dp_fhs[23] = dp_ft2(DP_PFX_S0RR, dp_relbyte(dp_rcof(s0)), dp_tb(s0))
273 return DP_NFEAT
274}
275func dp_w_at(idx: i64, useavg: i64) -> i64 {
276 if useavg == 1 { return dp_wsum[idx] + dp_w[idx] * (dp_T - dp_wtime[idx]) }
277 return dp_w[idx]
278}
279func dp_bump(idx: i64, delta: i64) -> i64 {
280 dp_wsum[idx] = dp_wsum[idx] + dp_w[idx] * (dp_clock - dp_wtime[idx])
281 dp_wtime[idx] = dp_clock
282 dp_w[idx] = dp_w[idx] + delta
283 return 0
284}
285func dp_score(act: i64, useavg: i64) -> i64 {
286 var s: i64 = 0
287 var k: i64 = 0
288 while k < DP_NFEAT { s = s + dp_w_at(((dp_fhs[k] * DP_NACT + act) & DP_WMASK), useavg); k = k + 1 }
289 return s
290}
291// is the action legal in the current state (only interned relations; the single-root constraint on RIGHT-ARC to ROOT)
292func dp_valid(act: i64) -> i64 {
293 if act == DP_ACT_SHIFT { if dp_bi < dp_cnt { return 1 } return 0 }
294 let r: i64 = act - 1
295 if r < DP_NREL {
296 if r >= dp_nrel { return 0 }
297 if dp_sp >= 2 { return 1 }
298 return 0
299 }
300 let r2: i64 = r - DP_NREL
301 if r2 >= dp_nrel { return 0 }
302 if dp_sp >= 2 { return 1 }
303 if dp_sp == 1 { if dp_bi >= dp_cnt { return 1 } }
304 return 0
305}
306// the best-scoring legal action, SHIFT winning ties (so an untrained parser is a right-branching chain, never a crash)
307func dp_predict(useavg: i64) -> i64 {
308 var best: i64 = DP_NONE
309 var bestv: i64 = 0
310 var act: i64 = 0
311 while act < DP_NACT {
312 if dp_valid(act) == 1 {
313 let v: i64 = dp_score(act, useavg)
314 if best < 0 { best = act; bestv = v } else { if v > bestv { best = act; bestv = v } }
315 }
316 act = act + 1
317 }
318 return best
319}
320func dp_apply(act: i64) -> i64 {
321 if act == DP_ACT_SHIFT {
322 dp_sp = dp_sp + 1
323 dp_st[dp_sp] = dp_bi
324 dp_bi = dp_bi + 1
325 return 0
326 }
327 let s0: i64 = dp_st[dp_sp]
328 let s1: i64 = dp_st[dp_sp - 1]
329 let r: i64 = act - 1
330 if r < DP_NREL {
331 dp_heads[s1] = s0
332 dp_rels[s1] = r
333 if dp_lc[s0] < 0 { dp_lc[s0] = s1 } else { if s1 < dp_lc[s0] { dp_lc[s0] = s1 } }
334 dp_att[s0] = dp_att[s0] + 1
335 dp_st[dp_sp - 1] = s0
336 dp_sp = dp_sp - 1
337 return 0
338 }
339 let r2: i64 = r - DP_NREL
340 dp_heads[s0] = s1
341 dp_rels[s0] = r2
342 if dp_rc[s1] < 0 { dp_rc[s1] = s0 } else { if s0 > dp_rc[s1] { dp_rc[s1] = s0 } }
343 dp_att[s1] = dp_att[s1] + 1
344 dp_sp = dp_sp - 1
345 return 0
346}
347// the static oracle: the gold action in this state, or -1 when the gold tree cannot be reached (non-projective)
348func dp_oracle() -> i64 {
349 let s0: i64 = dp_st[dp_sp]
350 if dp_sp >= 2 {
351 let s1: i64 = dp_st[dp_sp - 1]
352 if dp_gh[s1] == s0 { return 1 + dp_gr[s1] }
353 }
354 if dp_sp >= 1 {
355 let s1b: i64 = dp_st[dp_sp - 1]
356 if dp_gh[s0] == s1b { if dp_att[s0] == dp_gnc[s0] { if (dp_sp >= 2) | (dp_bi >= dp_cnt) { return 1 + DP_NREL + dp_gr[s0] } } }
357 }
358 if dp_bi < dp_cnt { return DP_ACT_SHIFT }
359 return DP_NONE
360}
361func dp_init_state() -> i64 {
362 dp_sp = 0
363 dp_st[0] = dp_cnt
364 dp_bi = 0
365 var i: i64 = 0
366 while i <= dp_cnt { dp_heads[i] = DP_NONE; dp_rels[i] = DP_NONE; dp_lc[i] = DP_NONE; dp_rc[i] = DP_NONE; dp_att[i] = 0; i = i + 1 }
367 return 0
368}
369func dp_update(gold: i64, pred: i64) -> i64 {
370 var k: i64 = 0
371 while k < DP_NFEAT {
372 dp_bump(((dp_fhs[k] * DP_NACT + gold) & DP_WMASK), 1)
373 dp_bump(((dp_fhs[k] * DP_NACT + pred) & DP_WMASK), 0 - 1)
374 k = k + 1
375 }
376 return 0
377}
378// parse the loaded sentence; train=1 follows the static oracle and updates on every wrong prediction. Returns the
379// update count, or -1 when training and the oracle could not reach the gold tree (the sentence is then skipped).
380func dp_parse_loaded(train: i64) -> i64 {
381 dp_init_state()
382 var upd: i64 = 0
383 var stuck: i64 = 0
384 while ((dp_bi < dp_cnt) | (dp_sp > 0)) & (stuck == 0) {
385 dp_feats()
386 var g: i64 = DP_NONE
387 if train == 1 { g = dp_oracle(); if g < 0 { stuck = 1 } }
388 if stuck == 0 {
389 let p: i64 = dp_predict(1 - train)
390 if p < 0 { stuck = 1 } else {
391 if train == 1 {
392 if p != g { dp_update(g, p); upd = upd + 1 }
393 dp_apply(g)
394 } else { dp_apply(p) }
395 }
396 }
397 }
398 if stuck == 1 { return DP_NONE }
399 return upd
400}
401// gold from the reader's last sentence into dp_gh/dp_gr/dp_gnc; returns 1 when usable (every HEAD present and in
402// range, every relation known or interned), else 0. intern=1 adds unseen relation names.
403func dp_load_gold(cnt: i64, intern: i64) -> i64 {
404 var ok: i64 = 1
405 var i: i64 = 0
406 while i <= cnt { dp_gnc[i] = 0; i = i + 1 }
407 i = 0
408 while i < cnt {
409 let h: i64 = pt_heads[i]
410 if h < 0 { ok = 0; dp_gh[i] = DP_NONE }
411 else { if h == 0 { dp_gh[i] = cnt } else { if h - 1 < cnt { dp_gh[i] = h - 1 } else { ok = 0; dp_gh[i] = DP_NONE } } }
412 dp_gr[i] = dp_rel_id((pt_relbuf as i64 + pt_reloffs[i]) as *u8, pt_rellens[i], intern)
413 if (intern == 1) & (dp_gr[i] < 0) { ok = 0 }
414 if dp_gh[i] >= 0 { dp_gnc[dp_gh[i]] = dp_gnc[dp_gh[i]] + 1 }
415 i = i + 1
416 }
417 return ok
418}
419// point the parser at the reader's sentence and tag it with the trained tagger (gold tags when no tagger is ready)
420func dp_bind_reader(cnt: i64) -> i64 {
421 dp_buf = pt_tokbuf
422 dp_offs = pt_offs
423 dp_lens = pt_lens
424 dp_cnt = cnt
425 if pt_ready == 1 { pt_tag_loaded(cnt, 0); dp_tags = pt_pred } else { dp_tags = pt_gold }
426 return 0
427}
428// train on a CoNLL-U file for DP_EPOCHS with the static oracle; fills the counters
429func dp_train(path: *u8, out: *i64) -> i64 {
430 let lp: *i64 = sys_mmap(RM_I64_PAIR) as *i64
431 lp[0] = 0
432 let b: *u8 = sys_read_file(path, lp)
433 if (b as i64) == 0 { return 0 - 1 }
434 let n: i64 = lp[0]
435 let ip: *i64 = sys_mmap(RM_I64_PAIR) as *i64
436 let unk: *i64 = sys_mmap(RM_I64_PAIR) as *i64
437 unk[0] = 0
438 var ep: i64 = 0
439 var last: i64 = 0
440 var sents: i64 = 0
441 var toks: i64 = 0
442 var nonproj: i64 = 0
443 var unusable: i64 = 0
444 while ep < DP_EPOCHS {
445 ip[0] = 0
446 last = 0
447 var cnt: i64 = pt_read_sentence(b, n, ip, unk)
448 while cnt > 0 {
449 dp_bind_reader(cnt)
450 if dp_load_gold(cnt, 1) == 1 {
451 let r: i64 = dp_parse_loaded(1)
452 if r < 0 { if ep == 0 { nonproj = nonproj + 1 } } else { last = last + r }
453 dp_clock = dp_clock + 1
454 } else { if ep == 0 { unusable = unusable + 1 } }
455 if ep == 0 { sents = sents + 1; toks = toks + cnt }
456 cnt = pt_read_sentence(b, n, ip, unk)
457 }
458 ep = ep + 1
459 }
460 dp_T = dp_clock
461 if dp_T <= 0 { dp_T = 1 }
462 dp_ready = 1
463 out[DP_O_TRAINSENT] = sents
464 out[DP_O_TRAINTOK] = toks
465 out[DP_O_NONPROJ] = nonproj
466 out[DP_O_UNUSABLE] = unusable
467 out[DP_O_UPDATES] = last
468 out[DP_O_NREL] = dp_nrel
469 return last
470}
471// attachment scores on a CoNLL-U file with the averaged weights: UAS over every token with a HEAD, LAS on base names
472func dp_test(path: *u8, out: *i64) -> i64 {
473 let lp: *i64 = sys_mmap(RM_I64_PAIR) as *i64
474 lp[0] = 0
475 let b: *u8 = sys_read_file(path, lp)
476 if (b as i64) == 0 { return 0 - 1 }
477 let n: i64 = lp[0]
478 let ip: *i64 = sys_mmap(RM_I64_PAIR) as *i64
479 let unk: *i64 = sys_mmap(RM_I64_PAIR) as *i64
480 unk[0] = 0
481 ip[0] = 0
482 var toks: i64 = 0
483 var uas: i64 = 0
484 var las: i64 = 0
485 var sents: i64 = 0
486 var cnt: i64 = pt_read_sentence(b, n, ip, unk)
487 while cnt > 0 {
488 dp_bind_reader(cnt)
489 dp_load_gold(cnt, 0)
490 dp_parse_loaded(0)
491 var i: i64 = 0
492 while i < cnt {
493 if dp_gh[i] >= 0 {
494 toks = toks + 1
495 if dp_heads[i] == dp_gh[i] { uas = uas + 1; if (dp_gr[i] >= 0) & (dp_rels[i] == dp_gr[i]) { las = las + 1 } }
496 }
497 i = i + 1
498 }
499 sents = sents + 1
500 cnt = pt_read_sentence(b, n, ip, unk)
501 }
502 out[DP_O_TOKENS] = toks
503 out[DP_O_UAS_CORRECT] = uas
504 out[DP_O_LAS_CORRECT] = las
505 if toks > 0 { out[DP_O_UAS] = uas * DP_PERMIL / toks; out[DP_O_LAS] = las * DP_PERMIL / toks } else { out[DP_O_UAS] = 0; out[DP_O_LAS] = 0 }
506 out[DP_O_TESTSENT] = sents
507 return 0
508}
509// train then test (the tagger must already be trained by the caller, or gold tags are used); out is DP_O_N wide
510func dp_eval(train: *u8, test: *u8, out: *i64) -> i64 {
511 var q: i64 = 0
512 while q < DP_O_N { out[q] = 0; q = q + 1 }
513 dp_reset()
514 if dp_train(train, out) < 0 { return 0 - 1 }
515 return dp_test(test, out)
516}
517// parse an arbitrary tagged token stream with the trained parser: heads_out[i] is the 0-based head index or -1 for the
518// root, rels_out[i] the relation id (dp_rel_name spells it); returns the token count parsed
519func dp_parse_stream(buf: *u8, offs: *i64, lens: *i64, tags: *i64, cnt: i64, heads_out: *i64, rels_out: *i64) -> i64 {
520 if cnt <= 0 { return 0 }
521 var c: i64 = cnt
522 if c > DP_TOK_CAP { c = DP_TOK_CAP }
523 dp_buf = buf
524 dp_offs = offs
525 dp_lens = lens
526 dp_tags = tags
527 dp_cnt = c
528 dp_parse_loaded(0)
529 var i: i64 = 0
530 while i < c {
531 if dp_heads[i] >= c { heads_out[i] = DP_NONE } else { heads_out[i] = dp_heads[i] }
532 rels_out[i] = dp_rels[i]
533 i = i + 1
534 }
535 return c
536}