nx_context_packer.nx source

1// nx_context_packer.nx -- THE sovereign "push a document into an LLM natively" CLI: read a text file (a book's 2// extracted chapters, a research doc, anything), pack it into an LLM context that fits a TOKEN BUDGET (with a 10% 3// safety margin), and write an LLM-ready context file. Greedy in-order line packing + an honest truncation marker 4// when the document exceeds the budget. No 3rd-party tokenizer/SDK -- pure Nishi. Composes nx_token_counter. 5// usage: nx_context_packer <infile> <budget_tokens> <outfile> license_tier: ORIGINAL 6import "nx_token_counter.nx" 7const K_MAGIC_65536: i64 = 65536 8 9func cp_slen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n } 10func cp_w(s: *u8) -> i64 { sys_write(1, s, cp_slen(s)); return 0 } 11func cp_atoi(s: *u8) -> i64 { var v: i64=0; var i: i64=0; while s[i]!=(0 as u8){ let c: i64=s[i] as i64; if c>=48 { if c<=57 { v=v*10+(c-48) } } i=i+1 } return v } 12func cp_wn(v: i64) -> i64 { 13 if v == 0 { cp_w("0" as *u8); return 0 } 14 let t: *u8 = sys_mmap(32); var k: i64 = 0; var m: i64 = v 15 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 } 16 let o: *u8 = sys_mmap(32); var j: i64 = 0 17 while k > 0 { k = k - 1; o[j] = t[k]; j = j + 1 } 18 sys_write(1, o, j); return 0 19} 20 21func main(argc: i64, argv: *i64) -> i64 { 22 cp_w("=== NISHI CONTEXT PACKER: document -> token-budgeted LLM context (sovereign, no 3rd-party) ===\n" as *u8) 23 if argc < 4 { 24 cp_w("usage: nx_context_packer <infile> <budget_tokens> <outfile>\n" as *u8) 25 sys_exit(2); return 2 26 } 27 let infile: *u8 = argv[1] as *u8 28 let budget: i64 = cp_atoi(argv[2] as *u8) 29 let outfile: *u8 = argv[3] as *u8 30 if budget <= 0 { cp_w("ERROR: budget must be > 0\n" as *u8); sys_exit(2); return 2 } 31 let szp: *i64 = sys_mmap(16) as *i64 32 let buf: *u8 = sys_read_file(infile, szp) 33 if (buf as i64) == 0 { cp_w("ERROR: cannot read infile\n" as *u8); sys_exit(1); return 1 } 34 let n: i64 = szp[0] 35 let cap: i64 = (budget * 9) / 10 // 10% margin: room for the wrapper + the LLM's response 36 let bodycap: i64 = budget * 8 + K_MAGIC_65536 37 let body: *u8 = sys_mmap(bodycap) 38 let incl: *i64 = sys_mmap(16) as *i64; let omit: *i64 = sys_mmap(16) as *i64 39 let used: i64 = pack_text(buf, n, cap, body, bodycap, incl, omit) 40 let fd: i64 = sys_openat_wr(outfile, 0x1a4) 41 if fd < 0 { cp_w("ERROR: cannot write outfile\n" as *u8); sys_exit(1); return 1 } 42 let h: *u8 = "=== DOCUMENT CONTEXT (sovereign Nishi pack, token-budgeted) ===\n\n" as *u8 43 sys_write(fd, h, cp_slen(h)) 44 sys_write(fd, body, cp_slen(body)) 45 sys_close(fd) 46 cp_w("packed: budget=" as *u8); cp_wn(budget); cp_w(" used~" as *u8); cp_wn(used) 47 cp_w(" tokens lines_kept=" as *u8); cp_wn(incl[0]); cp_w(" omitted=" as *u8); cp_wn(omit[0]) 48 cp_w(" -> " as *u8); cp_w(outfile); cp_w("\n" as *u8) 49 sys_exit(0); return 0 50}