`nx_absa_lib.nx` source

↩ module page · 883 lines · 35526 B
1// nx_absa_lib.nx -- the SCORING MATH for SemEval-2014 Task 4 subtask 1 (aspect term extraction), as a shared lib
2// so the CLI (nx_absa_bench) and the gate (nx_absa_bench_gate) share ONE definition of P, R and F1 and one gold
3// parser. The metric is the paper's own (Pontiki et al. 2014, eq. 1-2): P=|S and G|/|S|, R=|S and G|/|G|,
4// F1=2PR/(P+R), which for a set overlap is exactly 2*inter/(|S|+|G|) -- computed from counts so there is no
5// double rounding. Everything is permil (1000 = 1.00). Nothing here is a threshold: the lib reports numbers, the
6// gate says what a correct scorer must return, and the CLI prints the number beside the published winners.
7//
8// THE GOLD FORMAT (ABSA-PyTorch .seg mirror of the official test set): three lines per record --
9//   line 0 mod 3: the sentence with the aspect term replaced by the placeholder $T$
10//   line 1 mod 3: the aspect term (single or multi word, e.g. "hard disk", "delivery times")
11//   line 2 mod 3: the polarity (1 / 0 / -1)
12// The gold aspect-term SET G is the distinct normalised terms across the term lines (footnote 3 of the paper
13// counts occurrences; this is the distinct-term variant, DECLARED, a related but not identical metric).
14//
15// THE PREDICTOR is a UNIGRAM DOCUMENT-FREQUENCY baseline over the test sentences: it composes the ONE tokenizer
16// (rm_next_token via rm_mine_doc) and the ONE frequency ranker (rm_mine_rank) from nx_reviewmine_lib, so no
17// second tokenizer or ranker exists. It is a weak proxy for Hu-Liu (2004) frequent-noun-phrase mining -- our
18// miner does not yet extract multi-word noun phrases (rung rm_aspects), so a multi-word gold term is UNMATCHABLE
19// by a unigram prediction under exact-set match. That is the honest reason this baseline sits far below the
20// supervised systems, and the gap IS the un-landed rung. license_tier: ORIGINAL   No hw writes (Rule 26).   LIB.
21import "nx_syscalls.nx"
22import "nx_reviewmine_lib.nx"
23
24const AB_PERMIL: i64 = 1000
25const AB_P: i64 = 0
26const AB_R: i64 = 1
27const AB_F1: i64 = 2
28const AB_PRF_N: i64 = 3
29// out[] indices for ab_eval
30const AB_O_INTER: i64 = 0
31const AB_O_NS: i64 = 1
32const AB_O_NG: i64 = 2
33const AB_O_P: i64 = 3
34const AB_O_R: i64 = 4
35const AB_O_F1: i64 = 5
36const AB_O_K: i64 = 6
37const AB_O_VOCAB: i64 = 7
38const AB_O_N: i64 = 8
39
40const AB_SET_SLOTS: i64 = 65536          // power of two; a SemEval domain has ~1-2k distinct aspect terms
41const AB_SET_ARENA: i64 = 4194304        // AB_SET_SLOTS * avg term bytes, generous (multi-word terms allowed)
42const AB_TERM_MAX: i64 = 256             // longest aspect term stored
43const AB_RECORD_LINES: i64 = 3           // the .seg record is three lines
44const AB_TERM_LINE: i64 = 1              // lineno mod 3 == 1 is the aspect-term line
45const AB_SENT_LINE: i64 = 0             // lineno mod 3 == 0 is the sentence line
46const AB_MIN_SUPPORT: i64 = 1            // a candidate term must appear in at least one sentence
47const AB_ARENA_TAIL: i64 = 1            // the NUL after a stored term
48
49const AB_STOPWORDS: *u8 = "knowledge/reviewmine/stopwords.conf"
50
51static ab_hash: *i64
52static ab_off: *i64
53static ab_len: *i64
54static ab_arena: *u8
55static ab_used: i64
56static ab_cnt: i64
57static ab_nbuf: *u8
58
59// normalise a raw term span into out: lowercase ASCII, trim ends, collapse internal whitespace to one space,
60// strip a trailing CR (a gold file may arrive CRLF from any host). Returns the normalised length.
61func ab_norm(src: *u8, n: i64, out: *u8, cap: i64) -> i64 {
  var m: i64 = n
  if m > 0 { if (src[m - 1] as i64) == RM_CR { m = m - 1 } }
  var o: i64 = 0
  var pend_sp: i64 = 0
  var started: i64 = 0
  var i: i64 = 0
  while i < m {
      var c: i64 = src[i] as i64
      if c >= RM_UPPER_A { if c <= RM_UPPER_Z { c = c + RM_CASE_DELTA } }
      if c == RM_SP { if started == 1 { pend_sp = 1 } }
      else { if c == RM_TAB { if started == 1 { pend_sp = 1 } }
      else {
          if pend_sp == 1 { if o < cap - 1 { out[o] = RM_SP as u8; o = o + 1 } pend_sp = 0 }
          if o < cap - 1 { out[o] = c as u8; o = o + 1 }
          started = 1
      } }
      i = i + 1
  }
  out[o] = 0 as u8
  return o
82}
83
84func ab_set_reset() -> i64 {
  if (ab_hash as i64) == 0 {
      ab_hash = sys_mmap(AB_SET_SLOTS * RM_I64_BYTES) as *i64
      ab_off = sys_mmap(AB_SET_SLOTS * RM_I64_BYTES) as *i64
      ab_len = sys_mmap(AB_SET_SLOTS * RM_I64_BYTES) as *i64
      ab_arena = sys_mmap(AB_SET_ARENA)
      ab_nbuf = sys_mmap(AB_TERM_MAX)
  } else {
      var i: i64 = 0
      while i < AB_SET_SLOTS { ab_hash[i] = 0; i = i + 1 }
  }
  ab_used = 0
  ab_cnt = 0
  return 0
98}
99
100// insert an ALREADY-normalised term (length n). Returns 1 if newly added, 0 if a duplicate or the arena is full.
101func ab_set_add_norm(s: *u8, n: i64) -> i64 {
  if n <= 0 { return 0 }
  let h: i64 = rm_hash(s, n)
  var slot: i64 = h % AB_SET_SLOTS
  var tries: i64 = 0
  while tries < AB_SET_SLOTS {
      if ab_hash[slot] == 0 {
          if ab_used + n + AB_ARENA_TAIL > AB_SET_ARENA { return 0 }
          ab_hash[slot] = h
          ab_off[slot] = ab_used
          ab_len[slot] = n
          rm_catn(ab_arena, ab_used, s, n)
          ab_arena[ab_used + n] = 0 as u8
          ab_used = ab_used + n + AB_ARENA_TAIL
          ab_cnt = ab_cnt + 1
          return 1
      }
      if ab_hash[slot] == h { if ab_len[slot] == n {
          var same: i64 = 1
          var j: i64 = 0
          while j < n { if ab_arena[ab_off[slot] + j] != s[j] { same = 0 } j = j + 1 }
          if same == 1 { return 0 }
      } }
      slot = (slot + 1) % AB_SET_SLOTS
      tries = tries + 1
  }
  return 0
128}
129
130func ab_set_add_raw(src: *u8, n: i64) -> i64 {
  let m: i64 = ab_norm(src, n, ab_nbuf, AB_TERM_MAX)
  return ab_set_add_norm(ab_nbuf, m)
133}
134
135// membership of a raw (un-normalised) term span in the gold set
136func ab_set_has_raw(src: *u8, n: i64) -> i64 {
  let m: i64 = ab_norm(src, n, ab_nbuf, AB_TERM_MAX)
  if m <= 0 { return 0 }
  let h: i64 = rm_hash(ab_nbuf, m)
  var slot: i64 = h % AB_SET_SLOTS
  var tries: i64 = 0
  while tries < AB_SET_SLOTS {
      if ab_hash[slot] == 0 { return 0 }
      if ab_hash[slot] == h { if ab_len[slot] == m {
          var same: i64 = 1
          var j: i64 = 0
          while j < m { if ab_arena[ab_off[slot] + j] != ab_nbuf[j] { same = 0 } j = j + 1 }
          if same == 1 { return 1 }
      } }
      slot = (slot + 1) % AB_SET_SLOTS
      tries = tries + 1
  }
  return 0
154}
155
156func ab_set_count() -> i64 { return ab_cnt }
157
158// the paper's F1 for a set overlap, from counts, in permil. Zero-prediction (ns==0) and no-gold (ng==0) both
159// give F1=0 rather than a divide -- the zero-prediction control the rung's done-rule names.
160func ab_prf(inter: i64, ns: i64, ng: i64, out: *i64) -> i64 {
  if ns <= 0 { out[AB_P] = 0 } else { out[AB_P] = inter * AB_PERMIL / ns }
  if ng <= 0 { out[AB_R] = 0 } else { out[AB_R] = inter * AB_PERMIL / ng }
  let denom: i64 = ns + ng
  if denom <= 0 { out[AB_F1] = 0 } else { out[AB_F1] = (2 * inter * AB_PERMIL) / denom }
  return 0
166}
167
168// load the gold aspect-term SET from a .seg file. Returns |G| (distinct normalised terms).
169func ab_gold_load(path: *u8) -> i64 {
  ab_set_reset()
  let lp: *i64 = sys_mmap(RM_I64_PAIR) as *i64
  lp[0] = 0
  let b: *u8 = sys_read_file(path, lp)
  if (b as i64) == 0 { return 0 }
  let n: i64 = lp[0]
  var i: i64 = 0
  var lineno: i64 = 0
  while i < n {
      let e: i64 = rm_line_end(b, n, i)
      if (lineno % AB_RECORD_LINES) == AB_TERM_LINE { if e > i { ab_set_add_raw((b as i64 + i) as *u8, e - i) } }
      i = e + 1
      lineno = lineno + 1
  }
  return ab_set_count()
185}
186
187const AB_RECON_CAP: i64 = 8192           // a reconstructed review sentence
188const AB_DOLLAR: i64 = 36                 // '$'
189const AB_UPPER_T: i64 = 84               // 'T'  -- the $T$ placeholder the .seg masks the aspect term with
190const AB_PLACEHOLDER_LEN: i64 = 3         // the three bytes of $T$
191
192// rebuild the real review sentence by substituting the aspect term back into the FIRST $T$ placeholder, so the
193// term actually appears in the corpus the frequency baseline mines (the .seg masks it out otherwise). Trailing
194// CR is stripped from both spans. No placeholder -> the sentence is copied as-is. Returns the byte length.
195func ab_reconstruct(sent: *u8, sl: i64, term: *u8, tl: i64, out: *u8, cap: i64) -> i64 {
  var m: i64 = sl
  if m > 0 { if (sent[m - 1] as i64) == RM_CR { m = m - 1 } }
  var tm: i64 = tl
  if tm > 0 { if (term[tm - 1] as i64) == RM_CR { tm = tm - 1 } }
  var pos: i64 = 0 - 1
  var found: i64 = 0
  var i: i64 = 0
  while i + AB_PLACEHOLDER_LEN <= m {
      if found == 0 {
          if (sent[i] as i64) == AB_DOLLAR { if (sent[i + 1] as i64) == AB_UPPER_T { if (sent[i + 2] as i64) == AB_DOLLAR { pos = i; found = 1 } } }
      }
      i = i + 1
  }
  var o: i64 = 0
  if pos < 0 {
      var j: i64 = 0
      while j < m { if o < cap - 1 { out[o] = sent[j]; o = o + 1 } j = j + 1 }
  } else {
      var a: i64 = 0
      while a < pos { if o < cap - 1 { out[o] = sent[a]; o = o + 1 } a = a + 1 }
      var t: i64 = 0
      while t < tm { if o < cap - 1 { out[o] = term[t]; o = o + 1 } t = t + 1 }
      var z: i64 = pos + AB_PLACEHOLDER_LEN
      while z < m { if o < cap - 1 { out[o] = sent[z]; o = o + 1 } z = z + 1 }
  }
  out[o] = 0 as u8
  return o
223}
224
225// full pipeline: load the gold set, predict the top-K frequent unigrams over the reconstructed sentences, score.
226// k<=0 means K = |G| (a balanced baseline that predicts as many terms as the gold has). out is AB_O_N wide.
227func ab_eval(path: *u8, k: i64, out: *i64) -> i64 {
  var q: i64 = 0
  while q < AB_O_N { out[q] = 0; q = q + 1 }
  let ng: i64 = ab_gold_load(path)
  out[AB_O_NG] = ng
  var kk: i64 = k
  if kk <= 0 { kk = ng }
  out[AB_O_K] = kk
  if ng <= 0 { return 0 }
  let lp: *i64 = sys_mmap(RM_I64_PAIR) as *i64
  lp[0] = 0
  let b: *u8 = sys_read_file(path, lp)
  if (b as i64) == 0 { return 0 }
  let n: i64 = lp[0]
  rm_vocab_reset()
  let sw: i64 = rm_stopwords_load(AB_STOPWORDS)
  let rbuf: *u8 = sys_mmap(AB_RECON_CAP)
  var i: i64 = 0
  var docid: i64 = 0
  var more: i64 = 1
  while more == 1 {
      if i >= n { more = 0 } else {
          let e0: i64 = rm_line_end(b, n, i)
          let s0: i64 = i
          let l0: i64 = e0 - i
          i = e0 + 1
          var l1: i64 = 0
          var s1: i64 = i
          if i < n {
              let e1: i64 = rm_line_end(b, n, i)
              s1 = i
              l1 = e1 - i
              i = e1 + 1
              if i < n { let e2: i64 = rm_line_end(b, n, i); i = e2 + 1 }   // skip the polarity line
          }
          docid = docid + 1
          let rl: i64 = ab_reconstruct((b as i64 + s0) as *u8, l0, (b as i64 + s1) as *u8, l1, rbuf, AB_RECON_CAP)
          if rl > 0 { rm_mine_doc(rbuf, rl, 1, docid) }
      }
  }
  out[AB_O_VOCAB] = rm_vocab_size() - sw   // content terms only; stopword slots do not count
  // top-kk slots by frequency: every sentence was up=1, so rm_mine_rank in PRAISE mode ranks by document count
  let slots: *i64 = sys_mmap((kk + 1) * RM_I64_BYTES) as *i64
  let ns: i64 = rm_mine_rank(RM_MODE_PRAISE, kk, AB_MIN_SUPPORT, slots)
  var inter: i64 = 0
  var r: i64 = 0
  while r < ns {
      let tk: *u8 = rm_vocab_tok(slots[r])
      let tl: i64 = rm_slen(tk)
      if ab_set_has_raw(tk, tl) == 1 { inter = inter + 1 }
      r = r + 1
  }
  out[AB_O_INTER] = inter
  out[AB_O_NS] = ns
  let prf: *i64 = sys_mmap(AB_PRF_N * RM_I64_BYTES) as *i64
  ab_prf(inter, ns, ng, prf)
  out[AB_O_P] = prf[AB_P]
  out[AB_O_R] = prf[AB_R]
  out[AB_O_F1] = prf[AB_F1]
  return 0
287}
288
289// ---- subtask 2: aspect term polarity (SB2), scored as ACCURACY over the gold aspect terms, two arms: the MAJORITY
290// class (the paper's own majority baseline, so the arm is comparable in kind) and the estate's LEXICON arm -- the
291// DEFECT and NOT-MET cue lexicons vote negative, the EXCEED lexicon votes positive, a tie or no cue is neutral; the
292// cues are read through the ONE tokenizer and the lexicon bits rm_lexicon_load sets on the vocab, so no second
293// lexicon reader exists. A gold polarity the .seg does not define is OTHER and is never counted correct.
294const AB_POL_POS: i64 = 0
295const AB_POL_NEG: i64 = 1
296const AB_POL_NEU: i64 = 2
297const AB_POL_OTHER: i64 = 3
298const AB_POL_N: i64 = 4
299const AB_Q_RECORDS: i64 = 0
300const AB_Q_POS: i64 = 1
301const AB_Q_NEG: i64 = 2
302const AB_Q_NEU: i64 = 3
303const AB_Q_OTHER: i64 = 4
304const AB_Q_MAJ_CLASS: i64 = 5
305const AB_Q_MAJ_ACC: i64 = 6
306const AB_Q_LEX_ACC: i64 = 7
307const AB_Q_LEX_PRED_POS: i64 = 8
308const AB_Q_LEX_PRED_NEG: i64 = 9
309const AB_Q_LEX_PRED_NEU: i64 = 10
310const AB_Q_LEX_TERMS: i64 = 11           // lexicon terms loaded; 0 means the lexicon arm had no cues and says so
311const AB_Q_N: i64 = 12
312const AB_LEX_DIR_DEFAULT: *u8 = "knowledge/reviewmine/"
313const AB_LEX_DEFECT_FILE: *u8 = "lexicon_defect.conf"
314const AB_LEX_NOTMEET_FILE: *u8 = "lexicon_notmeet.conf"
315const AB_LEX_EXCEED_FILE: *u8 = "lexicon_exceed.conf"
316const AB_PATH_CAP: i64 = 512
317const AB_CH_ONE: i64 = 49                // '1'
318const AB_POL_LEN_ONE: i64 = 1            // "1" and "0" are one byte
319const AB_POL_LEN_NEG: i64 = 2            // "-1" is two bytes
320
321func ab_pol_class(p: *u8, n: i64) -> i64 {
  var m: i64 = n
  if m > 0 { if (p[m - 1] as i64) == RM_CR { m = m - 1 } }
  if m == AB_POL_LEN_ONE {
      if (p[0] as i64) == AB_CH_ONE { return AB_POL_POS }
      if (p[0] as i64) == RM_ZERO { return AB_POL_NEU }
      return AB_POL_OTHER
  }
  if m == AB_POL_LEN_NEG { if (p[0] as i64) == RM_MINUS { if (p[1] as i64) == AB_CH_ONE { return AB_POL_NEG } } }
  return AB_POL_OTHER
331}
332func ab_pol_name(c: i64) -> *u8 {
  if c == AB_POL_POS { return "positive" as *u8 }
  if c == AB_POL_NEG { return "negative" as *u8 }
  if c == AB_POL_NEU { return "neutral" as *u8 }
  return "other" as *u8
337}
338func ab_lex_path(dir: *u8, file: *u8, out: *u8) -> i64 {
  var o: i64 = rm_cat(out, 0, dir)
  o = rm_cat(out, o, file)
  out[o] = 0 as u8
  return o
343}
344// cues[0] = exceed cues, cues[1] = defect-or-not-met cues in one reconstructed sentence
345func ab_lex_cues(text: *u8, n: i64, tok: *u8, ip: *i64, cues: *i64) -> i64 {
  cues[0] = 0
  cues[1] = 0
  let negmask: i64 = RM_LEX_DEFECT + RM_LEX_NOTMEET
  ip[0] = 0
  var tl: i64 = rm_next_token(text, n, ip, tok)
  while tl > 0 {
      let s: i64 = rm_vocab_slot(tok, tl)
      if s >= 0 {
          let lex: i64 = rm_vocab_lex(s)
          if (lex & negmask) != 0 { cues[1] = cues[1] + 1 }
          if (lex & RM_LEX_EXCEED) != 0 { cues[0] = cues[0] + 1 }
      }
      tl = rm_next_token(text, n, ip, tok)
  }
  return 0
361}
362func ab_polarity(path: *u8, lexdir: *u8, out: *i64) -> i64 {
  var q: i64 = 0
  while q < AB_Q_N { out[q] = 0; q = q + 1 }
  let lp: *i64 = sys_mmap(RM_I64_PAIR) as *i64
  lp[0] = 0
  let b: *u8 = sys_read_file(path, lp)
  if (b as i64) == 0 { return 0 }
  let n: i64 = lp[0]
  rm_vocab_reset()
  let pb: *u8 = sys_mmap(AB_PATH_CAP)
  var terms: i64 = 0
  ab_lex_path(lexdir, AB_LEX_DEFECT_FILE, pb)
  terms = terms + rm_lexicon_load(pb, RM_LEX_DEFECT)
  ab_lex_path(lexdir, AB_LEX_NOTMEET_FILE, pb)
  terms = terms + rm_lexicon_load(pb, RM_LEX_NOTMEET)
  ab_lex_path(lexdir, AB_LEX_EXCEED_FILE, pb)
  terms = terms + rm_lexicon_load(pb, RM_LEX_EXCEED)
  out[AB_Q_LEX_TERMS] = terms
  let rbuf: *u8 = sys_mmap(AB_RECON_CAP)
  let tok: *u8 = sys_mmap(RM_TOK_MAX + RM_TOK_BUF_SPARE)
  let ip: *i64 = sys_mmap(RM_I64_PAIR) as *i64
  let cues: *i64 = sys_mmap(RM_I64_PAIR) as *i64
  let cls: *i64 = sys_mmap(AB_POL_N * RM_I64_BYTES) as *i64
  var k: i64 = 0
  while k < AB_POL_N { cls[k] = 0; k = k + 1 }
  var correct: i64 = 0
  var i: i64 = 0
  var more: i64 = 1
  while more == 1 {
      if i >= n { more = 0 } else {
          let e0: i64 = rm_line_end(b, n, i)
          let s0: i64 = i
          let l0: i64 = e0 - i
          i = e0 + 1
          var s1: i64 = i
          var l1: i64 = 0
          var s2: i64 = i
          var l2: i64 = 0
          if i < n {
              let e1: i64 = rm_line_end(b, n, i)
              s1 = i
              l1 = e1 - i
              i = e1 + 1
              if i < n { let e2: i64 = rm_line_end(b, n, i); s2 = i; l2 = e2 - i; i = e2 + 1 }
          }
          let g: i64 = ab_pol_class((b as i64 + s2) as *u8, l2)
          cls[g] = cls[g] + 1
          let rl: i64 = ab_reconstruct((b as i64 + s0) as *u8, l0, (b as i64 + s1) as *u8, l1, rbuf, AB_RECON_CAP)
          ab_lex_cues(rbuf, rl, tok, ip, cues)
          var pred: i64 = AB_POL_NEU
          if cues[0] > cues[1] { pred = AB_POL_POS }
          if cues[1] > cues[0] { pred = AB_POL_NEG }
          if pred == AB_POL_POS { out[AB_Q_LEX_PRED_POS] = out[AB_Q_LEX_PRED_POS] + 1 }
          if pred == AB_POL_NEG { out[AB_Q_LEX_PRED_NEG] = out[AB_Q_LEX_PRED_NEG] + 1 }
          if pred == AB_POL_NEU { out[AB_Q_LEX_PRED_NEU] = out[AB_Q_LEX_PRED_NEU] + 1 }
          if pred == g { correct = correct + 1 }
      }
  }
  let records: i64 = cls[AB_POL_POS] + cls[AB_POL_NEG] + cls[AB_POL_NEU] + cls[AB_POL_OTHER]
  out[AB_Q_RECORDS] = records
  out[AB_Q_POS] = cls[AB_POL_POS]
  out[AB_Q_NEG] = cls[AB_POL_NEG]
  out[AB_Q_NEU] = cls[AB_POL_NEU]
  out[AB_Q_OTHER] = cls[AB_POL_OTHER]
  var maj: i64 = AB_POL_POS
  if cls[AB_POL_NEG] > cls[maj] { maj = AB_POL_NEG }
  if cls[AB_POL_NEU] > cls[maj] { maj = AB_POL_NEU }
  out[AB_Q_MAJ_CLASS] = maj
  if records > 0 {
      out[AB_Q_MAJ_ACC] = cls[maj] * AB_PERMIL / records
      out[AB_Q_LEX_ACC] = correct * AB_PERMIL / records
  }
  return 0
435}
436
437// ---- rung rm_aspects, first cut: FREQUENT N-GRAM CANDIDATES with Hu and Liu (2004) redundancy pruning ----
438// Candidates are adjacent token pairs (bigrams) beside unigrams, counted once per record over the reconstructed
439// sentences through the ONE tokenizer and stored in the ONE vocab (a bigram is the string "left right" in the same
440// arena, so no second hash exists); a stopword ends a phrase. Redundancy pruning is Hu and Liu's p-support: a
441// unigram that sits inside a frequent bigram and rarely occurs on its own is a fragment, not an aspect. Minimum
442// support is derived from the corpus (Hu and Liu: one percent of sentences) unless the caller pins it. The unigram
443// arm above stays the runnable incumbent; this arm prints beside it, never instead of it.
444const AB_NG_MINSUP_PERMIL: i64 = 10      // Hu and Liu 2004: a frequent feature appears in at least 1 percent of sentences
445const AB_NG_MINSUP_FLOOR: i64 = 1
446const AB_NG_MIN_PSUPPORT: i64 = 3        // Hu and Liu 2004: a subset phrase with p-support below 3 is redundant
447const AB_NG_JOIN: i64 = 32               // the space between the two words of a stored bigram
448const AB_NG_MAX_TOKENS: i64 = 512        // tokens one record carries into the n-gram pass; more is counted truncated
449const AB_NG_BG_CAP: i64 = 66             // RM_TOK_MAX * 2 + a space + a NUL
450const AB_NG_STOP: i64 = 0 - 1            // a stopword in the token list: a phrase boundary
451const AB_REC_N: i64 = 6                  // s0 l0 s1 l1 s2 l2
452const AB_G_INTER: i64 = 0
453const AB_G_NS: i64 = 1
454const AB_G_NG: i64 = 2
455const AB_G_P: i64 = 3
456const AB_G_R: i64 = 4
457const AB_G_F1: i64 = 5
458const AB_G_K: i64 = 6
459const AB_G_CAND: i64 = 7                 // candidates after support and pruning
460const AB_G_BIGRAMS: i64 = 8              // frequent bigrams among the candidates
461const AB_G_PRUNED: i64 = 9               // unigrams pruned by p-support
462const AB_G_MINSUP: i64 = 10
463const AB_G_MATCHED_BIGRAMS: i64 = 11
464const AB_G_TRUNC: i64 = 12               // records whose token list overflowed
465const AB_G_RECORDS: i64 = 13
466const AB_G_N: i64 = 14
467
468static ab_df: *i64
469static ab_last: *i64
470static ab_ps: *i64
471static ab_pslast: *i64
472static ab_isbig: *i64
473static ab_sel: *i64
474static ab_inbig: *i64
475static ab_toks: *i64
476static ab_cov: *i64
477static ab_bg: *u8
478
479func ab_ng_reset() -> i64 {
  if (ab_df as i64) == 0 {
      ab_df = sys_mmap(RM_VOCAB_SLOTS * RM_I64_BYTES) as *i64
      ab_last = sys_mmap(RM_VOCAB_SLOTS * RM_I64_BYTES) as *i64
      ab_ps = sys_mmap(RM_VOCAB_SLOTS * RM_I64_BYTES) as *i64
      ab_pslast = sys_mmap(RM_VOCAB_SLOTS * RM_I64_BYTES) as *i64
      ab_isbig = sys_mmap(RM_VOCAB_SLOTS * RM_I64_BYTES) as *i64
      ab_sel = sys_mmap(RM_VOCAB_SLOTS * RM_I64_BYTES) as *i64
      ab_inbig = sys_mmap(RM_VOCAB_SLOTS * RM_I64_BYTES) as *i64
      ab_toks = sys_mmap(AB_NG_MAX_TOKENS * RM_I64_BYTES) as *i64
      ab_cov = sys_mmap(AB_NG_MAX_TOKENS * RM_I64_BYTES) as *i64
      ab_bg = sys_mmap(AB_NG_BG_CAP)
  } else {
      var i: i64 = 0
      while i < RM_VOCAB_SLOTS { ab_df[i] = 0; ab_last[i] = 0; ab_ps[i] = 0; ab_pslast[i] = 0; ab_isbig[i] = 0; ab_sel[i] = 0; ab_inbig[i] = 0; i = i + 1 }
  }
  return 0
496}
497// one record's tokens as vocab slots into ab_toks (a stopword becomes AB_NG_STOP); returns the count, trunc[0]=1 on overflow
498func ab_ng_tokens(text: *u8, n: i64, tok: *u8, ip: *i64, trunc: *i64) -> i64 {
  ip[0] = 0
  var cnt: i64 = 0
  var tl: i64 = rm_next_token(text, n, ip, tok)
  while tl > 0 {
      if cnt < AB_NG_MAX_TOKENS {
          let s: i64 = rm_vocab_slot(tok, tl)
          var v: i64 = AB_NG_STOP
          if s >= 0 { if rm_vocab_stop(s) == 0 { v = s } }
          ab_toks[cnt] = v
          cnt = cnt + 1
      } else { trunc[0] = 1 }
      tl = rm_next_token(text, n, ip, tok)
  }
  return cnt
513}
514// the vocab slot of the bigram "left right" (inserted when absent), -1 when the vocab is full
515func ab_ng_bigram(left: i64, right: i64) -> i64 {
  var o: i64 = rm_cat(ab_bg, 0, rm_vocab_tok(left))
  ab_bg[o] = AB_NG_JOIN as u8
  o = o + 1
  o = rm_cat(ab_bg, o, rm_vocab_tok(right))
  ab_bg[o] = 0 as u8
  return rm_vocab_slot(ab_bg, o)
522}
523func ab_ng_bump(slot: i64, doc: i64) -> i64 { if ab_last[slot] != doc { ab_last[slot] = doc; ab_df[slot] = ab_df[slot] + 1 } return 0 }
524func ab_ng_psbump(slot: i64, doc: i64) -> i64 { if ab_pslast[slot] != doc { ab_pslast[slot] = doc; ab_ps[slot] = ab_ps[slot] + 1 } return 0 }
525// walk one three-line record from ip[0]; r = s0 l0 s1 l1 s2 l2; returns 1, or 0 at the end
526func ab_rec_next(b: *u8, n: i64, ip: *i64, r: *i64) -> i64 {
  var i: i64 = ip[0]
  if i >= n { return 0 }
  let e0: i64 = rm_line_end(b, n, i)
  r[0] = i; r[1] = e0 - i; i = e0 + 1
  r[2] = i; r[3] = 0; r[4] = i; r[5] = 0
  if i < n {
      let e1: i64 = rm_line_end(b, n, i)
      r[2] = i; r[3] = e1 - i; i = e1 + 1
      if i < n { let e2: i64 = rm_line_end(b, n, i); r[4] = i; r[5] = e2 - i; i = e2 + 1 }
  }
  ip[0] = i
  return 1
539}
540// the frequent n-gram arm: k<=0 -> K=|G|; minsup_pin<=0 -> derived from the record count. out is AB_G_N wide.
541func ab_ngram_eval(path: *u8, k: i64, minsup_pin: i64, out: *i64) -> i64 {
  var q: i64 = 0
  while q < AB_G_N { out[q] = 0; q = q + 1 }
  let ng: i64 = ab_gold_load(path)
  out[AB_G_NG] = ng
  var kk: i64 = k
  if kk <= 0 { kk = ng }
  out[AB_G_K] = kk
  if ng <= 0 { return 0 }
  let lp: *i64 = sys_mmap(RM_I64_PAIR) as *i64
  lp[0] = 0
  let b: *u8 = sys_read_file(path, lp)
  if (b as i64) == 0 { return 0 }
  let n: i64 = lp[0]
  rm_vocab_reset()
  rm_stopwords_load(AB_STOPWORDS)
  ab_ng_reset()
  let rbuf: *u8 = sys_mmap(AB_RECON_CAP)
  let tok: *u8 = sys_mmap(RM_TOK_MAX + RM_TOK_BUF_SPARE)
  let ip: *i64 = sys_mmap(RM_I64_PAIR) as *i64
  let rp: *i64 = sys_mmap(RM_I64_PAIR) as *i64
  let r: *i64 = sys_mmap(AB_REC_N * RM_I64_BYTES) as *i64
  let trunc: *i64 = sys_mmap(RM_I64_PAIR) as *i64
  rp[0] = 0
  var doc: i64 = 0
  while ab_rec_next(b, n, rp, r) == 1 {
      doc = doc + 1
      let rl: i64 = ab_reconstruct((b as i64 + r[0]) as *u8, r[1], (b as i64 + r[2]) as *u8, r[3], rbuf, AB_RECON_CAP)
      trunc[0] = 0
      let cnt: i64 = ab_ng_tokens(rbuf, rl, tok, ip, trunc)
      if trunc[0] == 1 { out[AB_G_TRUNC] = out[AB_G_TRUNC] + 1 }
      var i: i64 = 0
      while i < cnt {
          let s: i64 = ab_toks[i]
          if s >= 0 {
              ab_ng_bump(s, doc)
              if i + 1 < cnt { let t: i64 = ab_toks[i + 1]; if t >= 0 {
                  let bs: i64 = ab_ng_bigram(s, t)
                  if bs >= 0 { ab_isbig[bs] = 1; ab_ng_bump(bs, doc) }
              } }
          }
          i = i + 1
      }
  }
  out[AB_G_RECORDS] = doc
  var minsup: i64 = minsup_pin
  if minsup <= 0 { minsup = doc * AB_NG_MINSUP_PERMIL / AB_PERMIL; if minsup < AB_NG_MINSUP_FLOOR { minsup = AB_NG_MINSUP_FLOOR } }
  out[AB_G_MINSUP] = minsup
  var s2: i64 = 0
  var nbig: i64 = 0
  while s2 < RM_VOCAB_SLOTS { if ab_isbig[s2] == 1 { if ab_df[s2] >= minsup { ab_sel[s2] = 1; nbig = nbig + 1 } } s2 = s2 + 1 }
  out[AB_G_BIGRAMS] = nbig
  rp[0] = 0
  doc = 0
  while ab_rec_next(b, n, rp, r) == 1 {
      doc = doc + 1
      let rl2: i64 = ab_reconstruct((b as i64 + r[0]) as *u8, r[1], (b as i64 + r[2]) as *u8, r[3], rbuf, AB_RECON_CAP)
      trunc[0] = 0
      let cnt2: i64 = ab_ng_tokens(rbuf, rl2, tok, ip, trunc)
      var j: i64 = 0
      while j < cnt2 { ab_cov[j] = 0; j = j + 1 }
      j = 0
      while j + 1 < cnt2 {
          let s: i64 = ab_toks[j]
          let t: i64 = ab_toks[j + 1]
          if s >= 0 { if t >= 0 { let bs: i64 = ab_ng_bigram(s, t); if bs >= 0 { if ab_sel[bs] == 1 { ab_cov[j] = 1; ab_cov[j + 1] = 1; ab_inbig[s] = 1; ab_inbig[t] = 1 } } } }
          j = j + 1
      }
      j = 0
      while j < cnt2 { let s: i64 = ab_toks[j]; if s >= 0 { if ab_cov[j] == 0 { ab_ng_psbump(s, doc) } } j = j + 1 }
  }
  let cand: *i64 = sys_mmap((rm_vocab_size() + 1) * RM_I64_BYTES) as *i64
  var nc: i64 = 0
  var pruned: i64 = 0
  var s3: i64 = 0
  while s3 < RM_VOCAB_SLOTS {
      if ab_df[s3] >= minsup {
          if ab_isbig[s3] == 1 { cand[nc] = s3; nc = nc + 1 }
          else {
              if ab_inbig[s3] == 0 { cand[nc] = s3; nc = nc + 1 }
              else { if ab_ps[s3] >= AB_NG_MIN_PSUPPORT { cand[nc] = s3; nc = nc + 1 } else { pruned = pruned + 1 } }
          }
      }
      s3 = s3 + 1
  }
  out[AB_G_CAND] = nc
  out[AB_G_PRUNED] = pruned
  var inter: i64 = 0
  var mbig: i64 = 0
  var ns: i64 = 0
  var round: i64 = 0
  while round < kk {
      var best: i64 = 0 - 1
      var bi: i64 = 0 - 1
      var cc: i64 = 0
      while cc < nc {
          let s: i64 = cand[cc]
          if s >= 0 {
              var better: i64 = 0
              if best < 0 { better = 1 }
              else {
                  if ab_df[s] > ab_df[best] { better = 1 }
                  if ab_df[s] == ab_df[best] { if ab_isbig[s] > ab_isbig[best] { better = 1 } }
              }
              if better == 1 { best = s; bi = cc }
          }
          cc = cc + 1
      }
      if best < 0 { round = kk } else {
          cand[bi] = 0 - 1
          ns = ns + 1
          let tk: *u8 = rm_vocab_tok(best)
          if ab_set_has_raw(tk, rm_slen(tk)) == 1 { inter = inter + 1; if ab_isbig[best] == 1 { mbig = mbig + 1 } }
          round = round + 1
      }
  }
  out[AB_G_INTER] = inter
  out[AB_G_NS] = ns
  out[AB_G_MATCHED_BIGRAMS] = mbig
  let prf: *i64 = sys_mmap(AB_PRF_N * RM_I64_BYTES) as *i64
  ab_prf(inter, ns, ng, prf)
  out[AB_G_P] = prf[AB_P]
  out[AB_G_R] = prf[AB_R]
  out[AB_G_F1] = prf[AB_F1]
  return 0
666}
667
668// ---- rung IM28 first cut: the SUPERVISED training-DICTIONARY tagger (the organisers' own SB1 baseline) ----
669const HS_HASH: i64 = 0
670const HS_OFF: i64 = 1
671const HS_LEN: i64 = 2
672const HS_ARENA: i64 = 3
673const HS_USED: i64 = 4
674const HS_CNT: i64 = 5
675const HS_FIELDS: i64 = 6
676const AB_GRAM_CAP: i64 = 512
677const AB_D_INTER: i64 = 0
678const AB_D_NS: i64 = 1
679const AB_D_NG: i64 = 2
680const AB_D_P: i64 = 3
681const AB_D_R: i64 = 4
682const AB_D_F1: i64 = 5
683const AB_D_DICT: i64 = 6
684const AB_D_RECORDS: i64 = 7
685const AB_D_N: i64 = 8
686
687func hs_new() -> *i64 {
  let h: *i64 = sys_mmap(HS_FIELDS * RM_I64_BYTES) as *i64
  h[HS_HASH] = sys_mmap(AB_SET_SLOTS * RM_I64_BYTES) as i64
  h[HS_OFF] = sys_mmap(AB_SET_SLOTS * RM_I64_BYTES) as i64
  h[HS_LEN] = sys_mmap(AB_SET_SLOTS * RM_I64_BYTES) as i64
  h[HS_ARENA] = sys_mmap(AB_SET_ARENA) as i64
  h[HS_USED] = 0
  h[HS_CNT] = 0
  return h
696}
697func hs_reset(h: *i64) -> i64 {
  let hash: *i64 = h[HS_HASH] as *i64
  var i: i64 = 0
  while i < AB_SET_SLOTS { hash[i] = 0; i = i + 1 }
  h[HS_USED] = 0
  h[HS_CNT] = 0
  return 0
704}
705func hs_add_norm(h: *i64, s: *u8, n: i64) -> i64 {
  if n <= 0 { return 0 }
  let hash: *i64 = h[HS_HASH] as *i64
  let off: *i64 = h[HS_OFF] as *i64
  let len: *i64 = h[HS_LEN] as *i64
  let arena: *u8 = h[HS_ARENA] as *u8
  let hh: i64 = rm_hash(s, n)
  var slot: i64 = hh % AB_SET_SLOTS
  var tries: i64 = 0
  while tries < AB_SET_SLOTS {
      if hash[slot] == 0 {
          if h[HS_USED] + n + AB_ARENA_TAIL > AB_SET_ARENA { return 0 }
          hash[slot] = hh
          off[slot] = h[HS_USED]
          len[slot] = n
          rm_catn(arena, h[HS_USED], s, n)
          arena[h[HS_USED] + n] = 0 as u8
          h[HS_USED] = h[HS_USED] + n + AB_ARENA_TAIL
          h[HS_CNT] = h[HS_CNT] + 1
          return 1
      }
      if hash[slot] == hh { if len[slot] == n {
          var same: i64 = 1
          var j: i64 = 0
          while j < n { if arena[off[slot] + j] != s[j] { same = 0 } j = j + 1 }
          if same == 1 { return 0 }
      } }
      slot = (slot + 1) % AB_SET_SLOTS
      tries = tries + 1
  }
  return 0
736}
737func hs_add_raw(h: *i64, s: *u8, n: i64) -> i64 { let m: i64 = ab_norm(s, n, ab_nbuf, AB_TERM_MAX); return hs_add_norm(h, ab_nbuf, m) }
738func hs_has_raw(h: *i64, s: *u8, n: i64) -> i64 {
  let m: i64 = ab_norm(s, n, ab_nbuf, AB_TERM_MAX)
  if m <= 0 { return 0 }
  let hash: *i64 = h[HS_HASH] as *i64
  let off: *i64 = h[HS_OFF] as *i64
  let len: *i64 = h[HS_LEN] as *i64
  let arena: *u8 = h[HS_ARENA] as *u8
  let hh: i64 = rm_hash(ab_nbuf, m)
  var slot: i64 = hh % AB_SET_SLOTS
  var tries: i64 = 0
  while tries < AB_SET_SLOTS {
      if hash[slot] == 0 { return 0 }
      if hash[slot] == hh { if len[slot] == m {
          var same: i64 = 1
          var j: i64 = 0
          while j < m { if arena[off[slot] + j] != ab_nbuf[j] { same = 0 } j = j + 1 }
          if same == 1 { return 1 }
      } }
      slot = (slot + 1) % AB_SET_SLOTS
      tries = tries + 1
  }
  return 0
760}
761func hs_count(h: *i64) -> i64 { return h[HS_CNT] }
762// the SLOT a term occupies (or -1): lets a caller keep per-term counters in arrays parallel to the set
763func hs_slot_raw(h: *i64, s: *u8, n: i64) -> i64 {
  let m: i64 = ab_norm(s, n, ab_nbuf, AB_TERM_MAX)
  if m <= 0 { return 0 - 1 }
  let hash: *i64 = h[HS_HASH] as *i64
  let off: *i64 = h[HS_OFF] as *i64
  let len: *i64 = h[HS_LEN] as *i64
  let arena: *u8 = h[HS_ARENA] as *u8
  let hh: i64 = rm_hash(ab_nbuf, m)
  var slot: i64 = hh % AB_SET_SLOTS
  var tries: i64 = 0
  while tries < AB_SET_SLOTS {
      if hash[slot] == 0 { return 0 - 1 }
      if hash[slot] == hh { if len[slot] == m {
          var same: i64 = 1
          var j: i64 = 0
          while j < m { if arena[off[slot] + j] != ab_nbuf[j] { same = 0 } j = j + 1 }
          if same == 1 { return slot }
      } }
      slot = (slot + 1) % AB_SET_SLOTS
      tries = tries + 1
  }
  return 0 - 1
785}
786func ab_dict_load(path: *u8, h: *i64) -> i64 {
  hs_reset(h)
  let lp: *i64 = sys_mmap(RM_I64_PAIR) as *i64
  lp[0] = 0
  let b: *u8 = sys_read_file(path, lp)
  if (b as i64) == 0 { return 0 }
  let n: i64 = lp[0]
  var i: i64 = 0
  var lineno: i64 = 0
  while i < n {
      let e: i64 = rm_line_end(b, n, i)
      if (lineno % AB_RECORD_LINES) == AB_TERM_LINE { if e > i { hs_add_raw(h, (b as i64 + i) as *u8, e - i) } }
      i = e + 1
      lineno = lineno + 1
  }
  return hs_count(h)
802}
803func ab_dict_try(dic: *i64, pred: *i64, g: *u8, n: i64) -> i64 {
  if hs_has_raw(dic, g, n) == 1 { if hs_has_raw(pred, g, n) == 0 { hs_add_raw(pred, g, n) } }
  return 0
806}
807func ab_dict_eval(train: *u8, test: *u8, out: *i64) -> i64 {
  var q: i64 = 0
  while q < AB_D_N { out[q] = 0; q = q + 1 }
  let ng: i64 = ab_gold_load(test)
  out[AB_D_NG] = ng
  if ng <= 0 { return 0 }
  let dic: *i64 = hs_new()
  out[AB_D_DICT] = ab_dict_load(train, dic)
  let pred: *i64 = hs_new()
  hs_reset(pred)
  let lp: *i64 = sys_mmap(RM_I64_PAIR) as *i64
  lp[0] = 0
  let b: *u8 = sys_read_file(test, lp)
  if (b as i64) == 0 { return 0 }
  let n: i64 = lp[0]
  let rbuf: *u8 = sys_mmap(AB_RECON_CAP)
  let tok: *u8 = sys_mmap(RM_TOK_MAX + RM_TOK_BUF_SPARE)
  let ip: *i64 = sys_mmap(RM_I64_PAIR) as *i64
  let p1: *u8 = sys_mmap(RM_TOK_MAX + RM_TOK_BUF_SPARE)
  let p2: *u8 = sys_mmap(RM_TOK_MAX + RM_TOK_BUF_SPARE)
  let gram: *u8 = sys_mmap(AB_GRAM_CAP)
  let rp: *i64 = sys_mmap(RM_I64_PAIR) as *i64
  let r: *i64 = sys_mmap(AB_REC_N * RM_I64_BYTES) as *i64
  rp[0] = 0
  var doc: i64 = 0
  while ab_rec_next(b, n, rp, r) == 1 {
      doc = doc + 1
      let rl: i64 = ab_reconstruct((b as i64 + r[0]) as *u8, r[1], (b as i64 + r[2]) as *u8, r[3], rbuf, AB_RECON_CAP)
      var l1: i64 = 0
      var l2: i64 = 0
      ip[0] = 0
      var tl: i64 = rm_next_token(rbuf, rl, ip, tok)
      while tl > 0 {
          ab_dict_try(dic, pred, tok, tl)
          if l1 > 0 {
              var go: i64 = rm_catn(gram, 0, p1, l1)
              gram[go] = AB_NG_JOIN as u8; go = go + 1
              go = rm_catn(gram, go, tok, tl)
              ab_dict_try(dic, pred, gram, go)
              if l2 > 0 {
                  var to: i64 = rm_catn(gram, 0, p2, l2)
                  gram[to] = AB_NG_JOIN as u8; to = to + 1
                  to = rm_catn(gram, to, p1, l1)
                  gram[to] = AB_NG_JOIN as u8; to = to + 1
                  to = rm_catn(gram, to, tok, tl)
                  ab_dict_try(dic, pred, gram, to)
              }
          }
          l2 = l1
          var w2: i64 = 0
          while w2 < l1 { p2[w2] = p1[w2]; w2 = w2 + 1 }
          l1 = tl
          var w1: i64 = 0
          while w1 < tl { p1[w1] = tok[w1]; w1 = w1 + 1 }
          tl = rm_next_token(rbuf, rl, ip, tok)
      }
  }
  out[AB_D_RECORDS] = doc
  out[AB_D_NS] = hs_count(pred)
  let phash: *i64 = pred[HS_HASH] as *i64
  let poff: *i64 = pred[HS_OFF] as *i64
  let plen: *i64 = pred[HS_LEN] as *i64
  let parena: *u8 = pred[HS_ARENA] as *u8
  var inter: i64 = 0
  var slot: i64 = 0
  while slot < AB_SET_SLOTS {
      if phash[slot] != 0 { if ab_set_has_raw((parena as i64 + poff[slot]) as *u8, plen[slot]) == 1 { inter = inter + 1 } }
      slot = slot + 1
  }
  out[AB_D_INTER] = inter
  let prf: *i64 = sys_mmap(AB_PRF_N * RM_I64_BYTES) as *i64
  ab_prf(inter, hs_count(pred), ng, prf)
  out[AB_D_P] = prf[AB_P]
  out[AB_D_R] = prf[AB_R]
  out[AB_D_F1] = prf[AB_F1]
  return 0
883}
nx_absa_lib.nx source

`nx_absa_lib.nx` source
