nx_wordclust.nx
buildroot/runtime/nx_wordclust.nx
about
nx_wordclust.nx -- WORD CLUSTERS from the estate's own PPMI model, the sovereign stand-in for the Brown clusters every
SemEval-2014 winner fed its tagger (DLIREC's largest single gains were name lists and word clusters from in-domain text).
The estate has no external corpus licensed for that, but it has knowledge/index/semppmi_v1.bin (built by nx_semppmi_build
from its own corpus) and ONE loader and cosine for it, nx_ppmi_lib -- composed here, never re-implemented.
K-MEDOIDS over PPMI cosine at three granularities (32 / 128 / 256 clusters, the Brown prefix-length idea): medoids seeded
by the caller's word frequencies (the training split's vocabulary), a few deterministic assignment/recentre rounds, and a
word not seen at build time is assigned on demand to its nearest medoid (a cluster is a function of the word, never of
the split it came from). Integer arithmetic only. A word the model does not know reads -1 at every level.
license_tier: ORIGINAL No hw writes (Rule 26). LIB.
dependencies 3 imports · 3 importers
imports: nx_syscalls.nxnx_reviewmine_lib.nxnx_ppmi_lib.nx
imported by: nx_absa_seq.nxnx_embfeat_gate.nxnx_wordclust_gate.nx
structs
| none |
consts
| 14 | const WC_G_BYTES: i64 = 1024 // the PPMI lib's g-block (its contract: a zeroed block of at least 632 bytes) |
| 15 | const WC_VOCAB_CAP: i64 = 32768 // distinct words the clusterer holds |
| 16 | const WC_WORD_MAX: i64 = 64 |
| 17 | const WC_ARENA: i64 = 1048576 |
| 18 | const WC_HASH_SLOTS: i64 = 131072 // open addressing, power of two, >= 4x the vocabulary cap |
| 19 | const WC_HASH_MASK: i64 = 131071 |
| 20 | const WC_LEVELS: i64 = 3 |
| 21 | const WC_K0: i64 = 32 |
| 22 | const WC_K1: i64 = 128 |
| 23 | const WC_K2: i64 = 256 |
| 24 | const WC_KMAX: i64 = 256 |
| 25 | const WC_ITERS: i64 = 3 |
| 26 | const WC_MEDOID_SAMPLE: i64 = 48 // members considered when a cluster recentres (bounds the quadratic step) |
| 27 | const WC_NONE: i64 = 0 - 1 |
| 28 | const WC_CH_ZERO: i64 = 48 |
| 29 | const WC_ID_RADIX: i64 = 64 // a cluster id is spelled as two bytes: id / 64 and id mod 64, offset from '0' |
| 31 | const WC_E_OFF: i64 = 0 |
| 32 | const WC_E_LEN: i64 = 1 |
| 33 | const WC_E_WID: i64 = 2 |
| 34 | const WC_E_FREQ: i64 = 3 |
| 35 | const WC_E_C0: i64 = 4 |
| 36 | const WC_E_C1: i64 = 5 |
| 37 | const WC_E_C2: i64 = 6 |
| 38 | const WC_E_N: i64 = 7 |
| 40 | const WC_O_VOCAB: i64 = 0 // words added |
| 41 | const WC_O_INMODEL: i64 = 1 // words the model knows (wid >= 0) |
| 42 | const WC_O_K0: i64 = 2 // medoids actually used at each level |
| 43 | const WC_O_K1: i64 = 3 |
| 44 | const WC_O_K2: i64 = 4 |
| 45 | const WC_O_N: i64 = 5 |
| 47 | const WC_FX_HDR: i64 = 32 |
| 48 | const WC_FX_VAL: i64 = 1000 |
| 49 | const WC_MAGIC_N: i64 = 78 |
| 50 | const WC_MAGIC_X: i64 = 88 |
| 51 | const WC_MAGIC_P: i64 = 80 |
| 52 | const WC_MAGIC_M: i64 = 77 |
| 53 | const WC_MAGIC_I: i64 = 73 |
| 54 | const WC_MAGIC_1: i64 = 49 |
functions
| 75 | func wc_set_k(k0: i64, k1: i64, k2: i64) -> i64 { wc_k0 = k0; wc_k1 = k1; wc_k2 = k2; return 0 } |
| 77 | func wc_reset() -> i64 |
| 100 | func wc_load_model(path: *u8) -> i64 |
| 104 | func wc_lc(c: i64) -> i64 { if c >= 65 { if c <= 90 { return c + 32 } } return c } |
| 106 | func wc_find(s: *u8, n: i64) -> i64 |
| 130 | func wc_add(s: *u8, n: i64) -> i64 |
| 156 | func wc_ecol(e: i64, col: i64) -> i64 { return wc_ent[e * WC_E_N + col] } |
| 157 | func wc_cluster_col(level: i64) -> i64 { if level == 0 { return WC_E_C0 } if level == 1 { return WC_E_C1 } return WC_E_C2 } |
| 158 | func wc_k_of(level: i64) -> i64 |
| 167 | func wc_nearest(level: i64, wid: i64) -> i64 |
| 182 | func wc_sort_by_freq() -> i64 |
| 213 | func wc_build_level(level: i64, ninmodel: i64) -> i64 |
| 270 | func wc_build(out: *i64) -> i64 |
| 285 | func wc_cluster(s: *u8, n: i64, level: i64) -> i64 |
| 300 | func wc_spell(id: i64, out: *u8) -> i64 |
| 310 | func wc_plant_model(path: *u8, buf: *u8, offs: *i64, lens: *i64, ctx: *i64, nw: i64) -> i64 |