nx_postag_gate.nx source

1// nx_postag_gate.nx -- GATE for the POS tagger (nx_postag), driven IN-PROCESS on a planted CoNLL-U mini-treebank under 2// /tmp/nx_postag_gate. Proves: the CoNLL-U reader (comment lines skipped, a multiword range row skipped, a decimal 3// empty-node row skipped, CRLF tolerated, forms lowercased), the tag table (all 17 UPOS names round-trip to their ids 4// and an unknown tag reads -1), that the tagger LEARNS (a train where the same words carry the same tags is tagged 5// perfectly at test), is deterministic, tags an arbitrary token stream through pt_tag_stream with the same answers, 6// and that an untrained tagger is not a learned one (its accuracy on the same test is far below). Every fixture 7// asserts its own condition first. No network. license_tier: ORIGINAL No hw writes (Rule 26). 8import "nx_syscalls.nx" 9import "nx_gatekit_lib.nx" 10import "nx_reviewmine_lib.nx" 11import "nx_postag.nx" 12import "nx_gate_verdict.nx" 13 14const G_ROOT: *u8 = "/tmp/nx_postag_gate" 15const G_TRAIN: *u8 = "/tmp/nx_postag_gate/train.conllu" 16const G_TEST: *u8 = "/tmp/nx_postag_gate/test.conllu" 17const G_I64: i64 = 8 18// 4 training sentences; the multiword row "3-4" and the empty node "2.1" must be skipped; one comment line; CRLF on one row 19const G_TRAIN_ROWS: *u8 = "# sent_id = 1\n1\tThe\tthe\tDET\tDT\t_\t2\tdet\t_\t_\n2\tdog\tdog\tNOUN\tNN\t_\t3\tnsubj\t_\t_\n3\truns\trun\tVERB\tVBZ\t_\t0\troot\t_\t_\n4\t.\t.\tPUNCT\t.\t_\t3\tpunct\t_\t_\n\n# sent_id = 2\n1\tA\ta\tDET\tDT\t_\t2\tdet\t_\t_\r\n2\tcat\tcat\tNOUN\tNN\t_\t3\tnsubj\t_\t_\n3\tsleeps\tsleep\tVERB\tVBZ\t_\t0\troot\t_\t_\n3-4\tdoesn't\t_\t_\t_\t_\t_\t_\t_\t_\n4\t.\t.\tPUNCT\t.\t_\t3\tpunct\t_\t_\n\n1\tThe\tthe\tDET\tDT\t_\t2\tdet\t_\t_\n2\tbig\tbig\tADJ\tJJ\t_\t3\tamod\t_\t_\n2.1\tempty\t_\t_\t_\t_\t_\t_\t_\t_\n3\tdog\tdog\tNOUN\tNN\t_\t4\tnsubj\t_\t_\n4\tsleeps\tsleep\tVERB\tVBZ\t_\t0\troot\t_\t_\n5\t.\t.\tPUNCT\t.\t_\t4\tpunct\t_\t_\n\n1\tA\ta\tDET\tDT\t_\t3\tdet\t_\t_\n2\tsmall\tsmall\tADJ\tJJ\t_\t3\tamod\t_\t_\n3\tcat\tcat\tNOUN\tNN\t_\t4\tnsubj\t_\t_\n4\truns\trun\tVERB\tVBZ\t_\t0\troot\t_\t_\n5\t.\t.\tPUNCT\t.\t_\t4\tpunct\t_\t_\n\n1\tThe\tthe\tDET\tDT\t_\t2\tdet\t_\t_\n2\tcat\tcat\tNOUN\tNN\t_\t3\tnsubj\t_\t_\n3\truns\trun\tVERB\tVBZ\t_\t0\troot\t_\t_\n" 20// the fifth sentence ends WITHOUT punctuation: otherwise end-of-sentence is perfectly predictive of PUNCT in this tiny 21// treebank and a verb at the end of an unpunctuated stream is a context the tagger has never seen (measured: it read 22// PUNCT for "runs" until this sentence was planted) -- the fixture must show the pattern the stream control tests 23const G_TRAIN_SENTS: i64 = 5 24const G_TRAIN_TOKS: i64 = 21 // 4+4+5+5+3, the range row and the empty node excluded 25// test: the same words, one new order; 10 tokens all seen in training with consistent tags 26const G_TEST_ROWS: *u8 = "1\tThe\tthe\tDET\tDT\t_\t3\tdet\t_\t_\n2\tsmall\tsmall\tADJ\tJJ\t_\t3\tamod\t_\t_\n3\tdog\tdog\tNOUN\tNN\t_\t4\tnsubj\t_\t_\n4\tsleeps\tsleep\tVERB\tVBZ\t_\t0\troot\t_\t_\n5\t.\t.\tPUNCT\t.\t_\t4\tpunct\t_\t_\n\n1\tA\ta\tDET\tDT\t_\t3\tdet\t_\t_\n2\tbig\tbig\tADJ\tJJ\t_\t3\tamod\t_\t_\n3\tcat\tcat\tNOUN\tNN\t_\t4\tnsubj\t_\t_\n4\truns\trun\tVERB\tVBZ\t_\t0\troot\t_\t_\n5\t.\t.\tPUNCT\t.\t_\t4\tpunct\t_\t_\n" 27const G_TEST_TOKS: i64 = 10 28const G_TEST_SENTS: i64 = 2 29const G_STREAM_TOKS: i64 = 3 30 31func main() -> i64 { 32 gv_head("=== nx_postag_gate -- the POS tagger reads CoNLL-U, learns, and tags a token stream ===" as *u8) 33 let c: *i64 = gv_ctr() 34 gk_mkdir(G_ROOT) 35 gk_write(G_TRAIN, G_TRAIN_ROWS) 36 gk_write(G_TEST, G_TEST_ROWS) 37 gv_check("fixture-reached-the-condition: train and test treebanks planted" as *u8, gk_exists(G_TRAIN) * gk_exists(G_TEST), c) 38 39 // the tag table: every UPOS name round-trips, an unknown one reads -1 40 gv_check_eq("tag-table-ADJ-is-id-0" as *u8, pt_tag_id("ADJ" as *u8, 3), 0, c) 41 gv_check_eq("tag-table-NOUN-is-id-7" as *u8, pt_tag_id("NOUN" as *u8, 4), PT_TAG_NOUN, c) 42 gv_check_eq("tag-table-PROPN-is-id-11" as *u8, pt_tag_id("PROPN" as *u8, 5), PT_TAG_PROPN, c) 43 gv_check_eq("tag-table-X-is-the-last-id-16" as *u8, pt_tag_id("X" as *u8, 1), PT_NTAGS - 1, c) 44 gv_check_eq("neg-control-tag-table-unknown-tag-reads-minus-one" as *u8, pt_tag_id("ZZZ" as *u8, 3), PT_TAG_NONE, c) 45 let nb: *u8 = sys_mmap(16) 46 gv_check_eq("tag-name-round-trips (VERB -> id -> VERB, 4 chars)" as *u8, pt_tag_name(pt_tag_id("VERB" as *u8, 4), nb), 4, c) 47 48 // learn 49 let o: *i64 = sys_mmap(PT_O_N * G_I64) as *i64 50 pt_eval(G_TRAIN, G_TEST, o) 51 gv_check_eq("conllu-reader-training-sentences-equal-the-planted-count (five)" as *u8, o[PT_O_TRAINSENT], G_TRAIN_SENTS, c) 52 gv_check_eq("conllu-reader-training-tokens-equal-the-planted-count (twenty-one: range row and empty node skipped, comment skipped, CRLF tolerated)" as *u8, o[PT_O_TRAINTOK], G_TRAIN_TOKS, c) 53 gv_check_eq("conllu-reader-no-unknown-gold-tags" as *u8, o[PT_O_UNKTAG], 0, c) 54 gv_check_eq("test-two-sentences-ten-tokens" as *u8, o[PT_O_TOKENS], G_TEST_TOKS, c) 55 gv_check_eq("test-sentences-two" as *u8, o[PT_O_TESTSENT], G_TEST_SENTS, c) 56 gv_check_eq("tagger-LEARNS: every test token tagged correctly (accuracy 1000 permil)" as *u8, o[PT_O_ACC], PT_PERMIL, c) 57 gv_check_eq("tagger-converged: no updates in the last epoch" as *u8, o[PT_O_UPDATES], 0, c) 58 59 // determinism 60 let o2: *i64 = sys_mmap(PT_O_N * G_I64) as *i64 61 pt_eval(G_TRAIN, G_TEST, o2) 62 gv_check_eq("tagger-is-deterministic: same accuracy on a second run" as *u8, o2[PT_O_ACC], o[PT_O_ACC], c) 63 64 // the stream entry point the aspect model uses: lowercased tokens in one buffer -> the same tags 65 let sb: *u8 = sys_mmap(64) 66 let so: *i64 = sys_mmap(8 * G_I64) as *i64 67 let sl: *i64 = sys_mmap(8 * G_I64) as *i64 68 let st: *i64 = sys_mmap(8 * G_I64) as *i64 69 var w: i64 = rm_catn(sb, 0, "the" as *u8, 3); so[0] = 0; sl[0] = 3 70 so[1] = w; sl[1] = 3; w = rm_catn(sb, w, "dog" as *u8, 3) 71 so[2] = w; sl[2] = 4; w = rm_catn(sb, w, "runs" as *u8, 4) 72 let tagged: i64 = pt_tag_stream(sb, so, sl, G_STREAM_TOKS, st) 73 gv_check_eq("stream-tagging-returns-the-token-count" as *u8, tagged, G_STREAM_TOKS, c) 74 gv_check_eq("stream-tagging-the-is-DET" as *u8, st[0], pt_tag_id("DET" as *u8, 3), c) 75 gv_check_eq("stream-tagging-dog-is-NOUN" as *u8, st[1], PT_TAG_NOUN, c) 76 gv_check_eq("stream-tagging-runs-is-VERB" as *u8, st[2], pt_tag_id("VERB" as *u8, 4), c) 77 78 // neg-control: an UNTRAINED tagger (weights zero) ties every tag and picks id 0 (ADJ) for every token, so its 79 // accuracy on the same test is the ADJ share of the gold: 2 of 10 tokens, never 1000 80 pt_reset() 81 let ne: *i64 = sys_mmap(PT_O_N * G_I64) as *i64 82 var qq: i64 = 0 83 while qq < PT_O_N { ne[qq] = 0; qq = qq + 1 } 84 pt_T = 1 85 pt_test(G_TEST, ne) 86 gv_check_eq("neg-control-untrained-tagger-scores-only-the-first-tag's-share (200 permil, not 1000)" as *u8, ne[PT_O_ACC], 200, c) 87 88 gv_values_head() 89 gv_kv("test_accuracy_permil" as *u8, o[PT_O_ACC]) 90 gv_kv("train_tokens" as *u8, o[PT_O_TRAINTOK]) 91 gv_kv("train_sentences" as *u8, o[PT_O_TRAINSENT]) 92 gv_kv("last_epoch_updates" as *u8, o[PT_O_UPDATES]) 93 gv_kv("untrained_accuracy_permil" as *u8, ne[PT_O_ACC]) 94 return gv_verdict("nx_postag_gate" as *u8, c, "the POS tagger proven on a planted CoNLL-U mini-treebank: the reader skips comments, multiword ranges and empty nodes and tolerates CRLF, the 17-tag table round-trips with an unknown tag refused, the tagger learns to perfect accuracy and converges, is deterministic, tags an arbitrary token stream identically, and an untrained tagger scores only the first tag's share; every fixture asserts its own condition first" as *u8) 95}