nx_dr_densejudge_cli.nx source

1// nx_dr_densejudge_cli.nx -- score insights under THREE representations in one call (DR-13): 2// lexical containment, sparse-PPMI late interaction, and TRAINED DENSE late interaction. 3// Holding the document, the insights and the code constant makes the representations directly 4// comparable -- which is the only way to test the §14 diagnosis (the ceiling is in the features). 5// nx_dr_densejudge <ppmi.bin> <embed.bin> <document> <insight_1> [insight_2 ...] 6// license_tier: ORIGINAL No hw writes (Rule 26). expect_exit: 0 7import "nx_dr_densejudge.nx" 8import "nx_itoa_lib.nx" // shared MSB-first emitter (zero-alloc) 9import "nx_ppmi_lib.nx" 10import "nx_dr_run.nx" 11import "nx_dr_verify.nx" 12const K_MAGIC_16384: i64 = 16384 13const K_MAGIC_2048: i64 = 2048 14 15func dc_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 } 16func dc_q() -> i64 { let b: *u8 = sys_mmap(1); b[0] = 34 as u8; sys_write(1, b, 1); return 0 } 17func dc_key(name: *u8) -> i64 { dc_q(); dc_puts(name); dc_q(); dc_puts(":" as *u8); return 0 } 18// MIGRATED to the shared emitter (debt 1785563586). The old body mmapped a scratch buffer 19// per call and never freed it. At PAGE granularity that is 4096B leaked PER CALL -- the 20// defect that took 28.5GB of a 36GB host in nx_ts_lumadiff (2MB input, ~3.66M calls). 21// nxi_* is MSB-first, allocates NOTHING, and emits identical bytes including the sign. 22func dc_num(v: i64) -> i64 { nxi_out(v); return 0 } 23 24func main(argc: i64, argv: *i64) -> i64 { 25 if argc < 5 { dc_puts("{" as *u8); dc_key("error" as *u8); dc_q(); dc_puts("usage: nx_dr_densejudge <ppmi.bin> <embed.bin> <document> <insight>..." as *u8); dc_q(); dc_puts("}\n" as *u8); sys_exit(2); return 2 } 26 let g: *i64 = sys_mmap(128 * 8) as *i64 27 if ppl_load(g, argv[1] as *u8) == 0 { dc_puts("{" as *u8); dc_key("error" as *u8); dc_q(); dc_puts("ppmi model load failed" as *u8); dc_q(); dc_puts("}\n" as *u8); sys_exit(1); return 1 } 28 if dj_load_embed(g, argv[2] as *u8) == 0 { dc_puts("{" as *u8); dc_key("error" as *u8); dc_q(); dc_puts("embed model load failed" as *u8); dc_q(); dc_puts("}\n" as *u8); sys_exit(1); return 1 } 29 dj_build_mean(g) 30 31 let doc: *u8 = argv[3] as *u8 32 let dlen: i64 = drr_strlen(doc) 33 let lex_doc: *i64 = sys_mmap(K_MAGIC_16384 * 8) as *i64 34 let nld: i64 = drr_tokenize(doc, dlen, lex_doc, K_MAGIC_16384) 35 let ids_doc: *i64 = sys_mmap(K_MAGIC_16384 * 8) as *i64 36 let nid: i64 = ppl_tokenize_ids(g, doc, dlen, ids_doc, K_MAGIC_16384) 37 38 let n: i64 = argc - 4 39 var k: i64 = 0 40 dc_puts("{" as *u8) 41 dc_key("tool" as *u8); dc_q(); dc_puts("nx_dr_densejudge" as *u8); dc_q(); dc_puts("," as *u8) 42 dc_key("vocab" as *u8); dc_num(g[71]); dc_puts("," as *u8) 43 dc_key("embed_dim" as *u8); dc_num(g[82]); dc_puts("," as *u8) 44 dc_key("insights" as *u8); dc_puts("[" as *u8) 45 while k < n { 46 if k > 0 { dc_puts("," as *u8) } 47 let s: *u8 = argv[4 + k] as *u8 48 let slen: i64 = drr_strlen(s) 49 let li: *i64 = sys_mmap(K_MAGIC_2048 * 8) as *i64 50 let nli: i64 = drr_tokenize(s, slen, li, K_MAGIC_2048) 51 let lex: i64 = dv_entail(li, nli, lex_doc, nld) 52 let si: *i64 = sys_mmap(K_MAGIC_2048 * 8) as *i64 53 let nsi: i64 = ppl_tokenize_ids(g, s, slen, si, K_MAGIC_2048) 54 let sparse: i64 = ppl_maxsim(g, si, nsi, ids_doc, nid) 55 let dense: i64 = dj_maxsim(g, si, nsi, ids_doc, nid) 56 let centered: i64 = dj_maxsim_centered(g, si, nsi, ids_doc, nid) 57 dc_puts("{" as *u8) 58 dc_key("idx" as *u8); dc_num(k); dc_puts("," as *u8) 59 dc_key("lexical_permil" as *u8); dc_num(lex); dc_puts("," as *u8) 60 dc_key("sparse_permil" as *u8); dc_num(sparse); dc_puts("," as *u8) 61 dc_key("dense_permil" as *u8); dc_num(dense); dc_puts("," as *u8) 62 dc_key("centered_permil" as *u8); dc_num(centered) 63 dc_puts("}" as *u8) 64 k = k + 1 65 } 66 dc_puts("]," as *u8) 67 dc_key("note" as *u8); dc_q(); dc_puts("same document, same insights, same code -- only the REPRESENTATION differs" as *u8); dc_q() 68 dc_puts("}\n" as *u8) 69 sys_exit(0); return 0 70}