nx_f32_llm_special_tokens.nx source

1// nx_f32_llm_special_tokens.nx -- read BOS/EOS/PAD token ids from GGUF. 2// 3// Composes with nx_f32_llm_run: the caller passes the eos_token_id 4// arg. Pre-this-brick, callers had to hardcode it. Post: read it 5// from the GGUF and pass it through. Production runs no longer 6// loop forever on models that emit EOS to terminate generation. 7// 8// Metadata keys (GGUF v3, gpt2-style tokenizers + Llama tokenizers): 9// tokenizer.ggml.bos_token_id (u32) beginning-of-sequence 10// tokenizer.ggml.eos_token_id (u32) end-of-sequence 11// tokenizer.ggml.padding_token_id (u32) optional padding token 12// 13// All three return -1 if not present in metadata. 14// 15// genealogy_id: ggml_tokenizer_special_token_keys 16// lineage_id: substrate_f32_llm_special_tokens_v1 17 18import "nx_syscalls.nx" 19import "nx_tier.nx" 20import "nx_gguf.nx" 21import "nx_gguf_meta.nx" 22 23func _flst_read_u32_key(buf: *u8, len: i64, hdr: *NxGgufHeader, 24 key: *u8, key_len: nx_int) -> nx_int { 25 let off_out: *i64 = sys_mmap(8) as *i64 26 let typ_out: *i64 = sys_mmap(8) as *i64 27 let v: nx_int = nx_gguf_meta_find(buf, len, hdr, key, key_len, off_out, typ_out) 28 if v != NX_GMETA_OK { return -1 } 29 if typ_out[0] != 4 { return -1 } // u32 30 return nx_gguf_meta_read_u32(buf, off_out[0]) as nx_int 31} 32 33// Read tokenizer.ggml.bos_token_id. Returns -1 if absent. 34 35func nx_f32_llm_read_bos(buf: *u8, len: i64, hdr: *NxGgufHeader) -> nx_int { 36 let key: *u8 = sys_mmap(28) 37 key[0]=0x74 as u8; key[1]=0x6f as u8; key[2]=0x6b as u8; key[3]=0x65 as u8 38 key[4]=0x6e as u8; key[5]=0x69 as u8; key[6]=0x7a as u8; key[7]=0x65 as u8 39 key[8]=0x72 as u8; key[9]=0x2e as u8; key[10]=0x67 as u8; key[11]=0x67 as u8 40 key[12]=0x6d as u8; key[13]=0x6c as u8; key[14]=0x2e as u8; key[15]=0x62 as u8 41 key[16]=0x6f as u8; key[17]=0x73 as u8; key[18]=0x5f as u8; key[19]=0x74 as u8 42 key[20]=0x6f as u8; key[21]=0x6b as u8; key[22]=0x65 as u8; key[23]=0x6e as u8 43 key[24]=0x5f as u8; key[25]=0x69 as u8; key[26]=0x64 as u8; key[27]=0 as u8 44 return _flst_read_u32_key(buf, len, hdr, key, 27) 45} 46 47// Read tokenizer.ggml.eos_token_id. Returns -1 if absent. 48 49func nx_f32_llm_read_eos(buf: *u8, len: i64, hdr: *NxGgufHeader) -> nx_int { 50 let key: *u8 = sys_mmap(28) 51 key[0]=0x74 as u8; key[1]=0x6f as u8; key[2]=0x6b as u8; key[3]=0x65 as u8 52 key[4]=0x6e as u8; key[5]=0x69 as u8; key[6]=0x7a as u8; key[7]=0x65 as u8 53 key[8]=0x72 as u8; key[9]=0x2e as u8; key[10]=0x67 as u8; key[11]=0x67 as u8 54 key[12]=0x6d as u8; key[13]=0x6c as u8; key[14]=0x2e as u8; key[15]=0x65 as u8 55 key[16]=0x6f as u8; key[17]=0x73 as u8; key[18]=0x5f as u8; key[19]=0x74 as u8 56 key[20]=0x6f as u8; key[21]=0x6b as u8; key[22]=0x65 as u8; key[23]=0x6e as u8 57 key[24]=0x5f as u8; key[25]=0x69 as u8; key[26]=0x64 as u8; key[27]=0 as u8 58 return _flst_read_u32_key(buf, len, hdr, key, 27) 59} 60 61// Read tokenizer.ggml.padding_token_id. Returns -1 if absent. 62 63func nx_f32_llm_read_pad(buf: *u8, len: i64, hdr: *NxGgufHeader) -> nx_int { 64 let key: *u8 = sys_mmap(32) 65 key[0]=0x74 as u8; key[1]=0x6f as u8; key[2]=0x6b as u8; key[3]=0x65 as u8 66 key[4]=0x6e as u8; key[5]=0x69 as u8; key[6]=0x7a as u8; key[7]=0x65 as u8 67 key[8]=0x72 as u8; key[9]=0x2e as u8; key[10]=0x67 as u8; key[11]=0x67 as u8 68 key[12]=0x6d as u8; key[13]=0x6c as u8; key[14]=0x2e as u8; key[15]=0x70 as u8 69 key[16]=0x61 as u8; key[17]=0x64 as u8; key[18]=0x64 as u8; key[19]=0x69 as u8 70 key[20]=0x6e as u8; key[21]=0x67 as u8; key[22]=0x5f as u8; key[23]=0x74 as u8 71 key[24]=0x6f as u8; key[25]=0x6b as u8; key[26]=0x65 as u8; key[27]=0x6e as u8 72 key[28]=0x5f as u8; key[29]=0x69 as u8; key[30]=0x64 as u8; key[31]=0 as u8 73 return _flst_read_u32_key(buf, len, hdr, key, 31) 74}