code wiki / (root) / nx_nofloat_llm.nx

nx_nofloat_llm.nx

buildroot/runtime/nx_nofloat_llm.nx

107654 B1835 linesdepth 7pulls 18 transitivereach 83 importersview sourcekind librarytopic nofloat
docsdependenciesstructsconstsfunctions

about

nx_nofloat_llm.nx -- CANONICAL sovereign no-float LLM library (ONE source of truth; no copy-paste debt). Every integer-Q16 building block for running a real GGUF transformer (Qwen2.5) in deterministic no-float: - dequant of every quant the model uses (F32 / Q5_0 / Q8_0 / Q4_K / Q6_K) -> Q16 - by-name tensor load + window-dequant (for embeddings / LM-head rows) - fixed-point transcendentals (exp, sigmoid, SiLU, sin, cos) - the ONE matmul: full-precision accumulate-then-shift (the correct, non-underflowing version) - RMSNorm, RoPE, attention sublayer, FFN sublayer, the lazy N-layer stack Gates IMPORT this instead of re-defining (CLAUDE.md #15 DRY). One canonical, correct implementation: a precision/convention fix is made HERE, once, not in 25 copies. No `main` (pure library). No hw writes (Rule 26). license_tier: ORIGINAL

dependencies 8 imports · 34 importers

nx_syscalls.nx nx_tier.nx nx_le.nx nx_tensor.nx nx_gguf.nx nx_gguf_load.nx nx_thread_pool.nx nx_vecmath.nx nx_nofloat_llm.nx nx_coder_swap_gate.nx nx_gguf_dequant_kat_gate.nx nx_gpu_export.nx nx_gpu_serve_gate.nx nx_llm_capcheck.nx nx_llm_loopb_gate.nx nx_llm_ppl_bench_gate.nx nx_nofloat_arch.nx nx_nofloat_arch_config_gate.nx nx_nofloat_arch_moe_gate.nx

diagram shows first 10 each side; +0 more imports, +24 more importers in the complete lists below.

imports: nx_syscalls.nxnx_tier.nxnx_le.nxnx_tensor.nxnx_gguf.nxnx_gguf_load.nxnx_thread_pool.nxnx_vecmath.nx

imported by: nx_coder_swap_gate.nxnx_gguf_dequant_kat_gate.nxnx_gpu_export.nxnx_gpu_serve_gate.nxnx_llm_capcheck.nxnx_llm_loopb_gate.nxnx_llm_ppl_bench_gate.nxnx_nofloat_arch.nxnx_nofloat_arch_config_gate.nxnx_nofloat_arch_moe_gate.nxnx_nofloat_mla.nxnx_nofloat_moe.nxnx_nofloat_moe_real_gate.nxnx_nofloat_olmoe.nxnx_nofloat_olmoe_forward_gate.nxnx_nofloat_olmoe_gate.nxnx_nofloat_q4k.nxnx_nofloat_q4k_gate.nxnx_nofloat_qwen_diag_gate.nxnx_nofloat_qwen_dqprobe_gate.nxnx_nofloat_qwen_fastgen_gate.nxnx_nofloat_qwen_fastgen_i32_gate.nxnx_nofloat_qwen_fastgen_i8_gate.nxnx_nofloat_qwen_fidelity_gate.nxnx_nofloat_qwen_forward_gate.nxnx_nofloat_qwen_gen_gate.nxnx_nofloat_qwen_kvgen_gate.nxnx_nofloat_qwen_rope_h2h_gate.nxnx_nofloat_qwen_stack_gate.nxnx_nofloat_qwen_text_gate.nxnx_nofloat_qwen_wload_gate.nxnx_nofloat_serve_core.nxnx_nofloat_w12a12_gate.nxnx_q4k_simd2_gate.nx

structs

none

consts

22const Q16: i64 = 65536
23const LOG2E: i64 = 94548
24const PC0: i64 = 65536
25const PC1: i64 = 45426
26const PC2: i64 = 15743
27const PC3: i64 = 4367
28const LN_BASE_Q16: i64 = 905421 // ln(1e6), Qwen2.5 rope base
29const HALF_PI: i64 = 102944
30const PI: i64 = 205887
31const THREE_HALF_PI: i64 = 308831
32const TWO_PI: i64 = 411775
400const NF_MADD_W: i64 = 16 // lanes one __i16x16_madd consumes
401const NF_R4_STRIDE: i64 = 64 // four madds per step, one per accumulator
402const NF_ACC_BYTES: i64 = 32 // one int32x8 accumulator
403const NF_ACC4_I64: i64 = 16 // four accumulators as i64 slots to zero (128 B)
411const NF_PREFILL_M: i64 = 32
420const NF_I8_DOT_DEFAULT: i64 = 1
425const NF_QUANT_SERIAL_DEFAULT: i64 = 0
426const NF_MMARENA_SLOTS: i64 = 32 // ctx slots in g_nf_mmarena (128 B each): the ceiling on bands and on quantisation tasks
430const NF_ATTN_SERIAL_DEFAULT: i64 = 0
431const NF_I8_R4M_DEFAULT: i64 = 0 // MEASURED 2026-09-17 on the decode shape, 32 rows: single-accumulator 451 us per row, four-accumulator 448 (1006 permil, a wash), so the batched kernel is not bound by the accumulator chain either; the simpler task stays served, the four-accumulator task is the gate's bit-identical control, and the bottleneck is the per-madd instruction count the compiler emits (R0r-b)
432const NF_MMCTX_M: i64 = 10 // band ctx slot: activation row count
433const NF_MMCTX_OUT: i64 = 11 // band ctx slot: out_dim, the dst row stride
434const NF_I8_R4_DEFAULT: i64 = 0 // MEASURED 2026-09-17 on the decode shape (1536 x 8960): incumbent 33.7 GB/s, four-accumulator 27.3 GB/s -- decode is DRAM-bound and the extra accumulator traffic costs 19 percent, so the incumbent stays the served path; the r4 task is kept as the gate's bit-identical control
1267const NF_W_QMAX: i64 = 4095
1268const NF_X_QMAX: i64 = 4095
1269const NF_I32_ACC_MAX: i64 = 2147483647
1270const NF_MADD_PER_LANE: i64 = 8
1271const NF_CHUNK_K: i64 = 1024

functions

34func qmul(a: i64, b: i64) -> i64 { return (a*b) >> 16 }
called by 19: silusin_qcos_qrope_applyrmsnorm_gamma_rowattn_sublayer+13
35func cpy(d: *i64, s: *i64, n: i64) -> i64 { var i: i64=0; while i<n { d[i]=s[i]; i=i+1 } return 0 }
called by 11: run_stackrun_stack_prefill_kvdecode_step_kvdecode_step_kv_cachednf_dequant_all_layers_i32decode_step_kv_cached_i32+5
36func q_i8(buf: *u8, off: i64) -> i64 { let v: i64=nx_le_read_u8(buf, off); if v>=128 { return v-256 } return v }
called by 2: q8_0_blockq6k_block calls 1: nx_le_read_u8
39func f32_to_q16(bits: i64) -> i64 { let s: i64=(bits>>31)&1; let e: i64=(bits>>23)&255; let mant: i64=bits&8388607; var v: i64=0; if e==0 { v=0 } else { if e==255 { v=2147483647 } else { let m: i64=8388608+mant; let ee: i64=e-134; if ee>=0 { v=m<<ee } else { v=m>>(0-ee) } } } if s==1 { v=0-v } return v }
called by 3: nac_f32_intdequant_to_q16dequant_row
40func f16_to_q16(h: i64) -> i64 { let s: i64=(h>>15)&1; let e: i64=(h>>10)&31; let mant: i64=h&1023; var v: i64=0; if e==0 { v=mant>>8 } else { if e==31 { v=2147483647 } else { let m: i64=1024+mant; let ee: i64=e-9; if ee>=0 { v=m<<ee } else { v=m>>(0-ee) } } } if s==1 { v=0-v } return v }
called by 7: q5_0_blockq8_0_blockq4_0_blockq4_1_blockq5_1_blockdequant_to_q16+1
43func q5_0_block(buf: *u8, off: i64, n: i64, out: *i64) -> i64 { let d: i64=f16_to_q16(nx_le_read_u16(buf, off)); let qh: i64=nx_le_read_u32(buf, off+2); let qo: i64=off+6; var j: i64=0; while j<16 { let qs: i64=nx_le_read_u8(buf, qo+j); let xh0: i64=((qh>>j)<<4)&16; let xh1: i64=(qh>>(j+12))&16; if j<n { out[j]=d*(((qs&15)|xh0)-16) } if j+16<n { out[j+16]=d*(((qs>>4)|xh1)-16) } j=j+1 } return 0 }
called by 2: dequant_to_q16dequant_row calls 4: f16_to_q16nx_le_read_u16nx_le_read_u32nx_le_read_u8
44func q8_0_block(buf: *u8, off: i64, n: i64, out: *i64) -> i64 { let d: i64=f16_to_q16(nx_le_read_u16(buf, off)); var j: i64=0; while j<32 { if j<n { out[j]=d*q_i8(buf, off+2+j) } j=j+1 } return 0 }
called by 2: dequant_to_q16dequant_row calls 3: f16_to_q16nx_le_read_u16q_i8
48func q6k_block(buf: *u8, so: i64, n: i64, out: *i64) -> i64
called by 2: dequant_to_q16dequant_row calls 4: _gguf_f16_to_q24nx_le_read_u16nx_le_read_u8q_i8
67func q4_0_block(buf: *u8, off: i64, n: i64, out: *i64) -> i64 { let d: i64=f16_to_q16(nx_le_read_u16(buf, off)); let qo: i64=off+2; var j: i64=0; while j<16 { let qs: i64=nx_le_read_u8(buf, qo+j); if j<n { out[j]=d*((qs&15)-8) } if j+16<n { out[j+16]=d*((qs>>4)-8) } j=j+1 } return 0 }
called by 3: maindequant_to_q16dequant_row calls 3: f16_to_q16nx_le_read_u16nx_le_read_u8
69func q4_1_block(buf: *u8, off: i64, n: i64, out: *i64) -> i64 { let d: i64=f16_to_q16(nx_le_read_u16(buf, off)); let m: i64=f16_to_q16(nx_le_read_u16(buf, off+2)); let qo: i64=off+4; var j: i64=0; while j<16 { let qs: i64=nx_le_read_u8(buf, qo+j); if j<n { out[j]=d*(qs&15)+m } if j+16<n { out[j+16]=d*(qs>>4)+m } j=j+1 } return 0 }
called by 3: maindequant_to_q16dequant_row calls 3: f16_to_q16nx_le_read_u16nx_le_read_u8
71func q5_1_block(buf: *u8, off: i64, n: i64, out: *i64) -> i64 { let d: i64=f16_to_q16(nx_le_read_u16(buf, off)); let m: i64=f16_to_q16(nx_le_read_u16(buf, off+2)); let qh: i64=nx_le_read_u32(buf, off+4); let qo: i64=off+8; var j: i64=0; while j<16 { let qs: i64=nx_le_read_u8(buf, qo+j); let xh0: i64=((qh>>j)<<4)&16; let xh1: i64=(qh>>(j+12))&16; if j<n { out[j]=d*((qs&15)|xh0)+m } if j+16<n { out[j+16]=d*((qs>>4)|xh1)+m } j=j+1 } return 0 }
called by 3: maindequant_to_q16dequant_row calls 4: f16_to_q16nx_le_read_u16nx_le_read_u32nx_le_read_u8
73func qk_scm(buf: *u8, sco: i64, is: i64, box: *i64) -> i64
called by 1: q5k_block calls 1: nx_le_read_u8
88func q5k_block(buf: *u8, so: i64, n: i64, out: *i64) -> i64
called by 3: maindequant_to_q16dequant_row calls 5: _gguf_f16_to_q24nx_le_read_u16sys_mmapqk_scmnx_le_read_u8
122func q3k_sc(buf: *u8, sco: i64, is: i64) -> i64
called by 1: q3k_block calls 1: nx_le_read_u8
137func q3k_block(buf: *u8, so: i64, n: i64, out: *i64) -> i64
called by 3: maindequant_to_q16dequant_row calls 4: _gguf_f16_to_q24nx_le_read_u16q3k_scnx_le_read_u8
180func q2k_block(buf: *u8, so: i64, n: i64, out: *i64) -> i64
called by 3: maindequant_to_q16dequant_row calls 3: _gguf_f16_to_q24nx_le_read_u16nx_le_read_u8
222func nf_type_stride(ty: i64, vb: *i64) -> i64
called by 4: mainnf_expert_byteoffmr_val_byteoffnsv_load_meta
240func nf_expert_byteoff(ty: i64, voff: i64) -> i64
called by 1: nolmoe_moe calls 2: sys_mmapnf_type_stride
246func dequant_to_q16(buf: *u8, base: i64, gt: i64, n: i64, out: *i64) -> i64
called by 7: mainload_named_q16mainnolmoe_moeof_load_blkmain+1 calls 14: f32_to_q16nx_le_read_u32f16_to_q16nx_le_read_u16nx_gguf_dequant_q4_k_q14q6k_block+8
263func load_named_q16(buf: *u8, hdr: *NxGgufHeader, name: *u8, nlen: i64, out: *i64, want: i64) -> i64
called by 21: mainmainmainmainmainmain+15 calls 4: nx_gguf_find_tensornx_gguf_tensor_atnx_gguf_tensor_n_valuesdequant_to_q16
275func dequant_row(buf: *u8, base: i64, gt: i64, v: i64, ne: i64, out: *i64, tmp: *i64) -> i64
called by 29: mainge2_gpu_embedmainmaingdx_generatemain+23 calls 14: f32_to_q16nx_le_read_u32f16_to_q16nx_le_read_u16q8_0_blockq5_0_block+8
297func build_name(out: *u8, L: i64, suffix: *u8) -> i64
called by 4: load_blkof_load_blkof_expbase_blknq_slot_fill calls 1: sys_mmap
305func load_blk(buf: *u8, hdr: *NxGgufHeader, nm: *u8, L: i64, suffix: *u8, out: *i64, want: i64) -> i64 { let len: i64=build_name(nm, L, suffix); return load_named_q16(buf, hdr, nm, len, out, want) }
called by 4: _nfld_taskrun_stacknq_load_vecmain calls 2: build_nameload_named_q16
308func isqrt(v: i64) -> i64 { return vm_isqrt(v) }
called by 5: mainnac_read_configag_scalermsnorm_gamma_rowrmsnorm_gamma_row_q24 calls 1: vm_isqrt
309func fx_exp(x: i64) -> i64 { var xm: i64=0-x; if x>0 { xm=0 } let ym: i64=(xm*LOG2E)>>16; let yi: i64=ym>>16; let yf: i64=ym-(yi<<16); let g: i64=Q16-yf; var t: i64=PC3; t=PC2+((g*t)>>16); t=PC1+((g*t)>>16); t=PC0+((g*t)>>16); t=t>>1; if yi>=31 { return 0 } return t>>yi }
called by 14: sigmoidrope_freqsrope_freqs_baseattn_sublayerdecode_step_kvdecode_step_kv_cached+8
310func sigmoid(x: i64) -> i64 { if x>=0 { let ex: i64=fx_exp(0-x); return (Q16*Q16)/(Q16+ex) } let ex: i64=fx_exp(x); let sp: i64=(Q16*Q16)/(Q16+ex); return Q16-sp }
called by 1: silu calls 1: fx_exp
311func silu(x: i64) -> i64 { return qmul(x, sigmoid(x)) }
called by 12: ffn_sublayerdecode_step_kvdecode_step_kv_cacheddecode_step_kv_cached_i32decode_step_kv_cached_i8_nf_silu_rows_task+6 calls 2: qmulsigmoid
312func sin_q(x: i64) -> i64 { let x2: i64=qmul(x,x); let x3: i64=qmul(x2,x); let x5: i64=qmul(x3,x2); let x7: i64=qmul(x5,x2); let x9: i64=qmul(x7,x2); return x - x3/6 + x5/120 - x7/5040 + x9/362880 }
called by 1: sin_full calls 1: qmul
313func cos_q(x: i64) -> i64 { let x2: i64=qmul(x,x); let x4: i64=qmul(x2,x2); let x6: i64=qmul(x4,x2); let x8: i64=qmul(x6,x2); return Q16 - x2/2 + x4/24 - x6/720 + x8/40320 }
called by 1: cos_full calls 1: qmul
314func reduce2pi(a: i64) -> i64 { var t: i64=a; while t<0 { t=t+TWO_PI } while t>=TWO_PI { t=t-TWO_PI } return t }
called by 2: sin_fullcos_full
315func sin_full(a: i64) -> i64 { let t: i64=reduce2pi(a); if t<HALF_PI { return sin_q(t) } if t<PI { return sin_q(PI-t) } if t<THREE_HALF_PI { return 0-sin_q(t-PI) } return 0-sin_q(TWO_PI-t) }
called by 1: rope_apply calls 2: reduce2pisin_q
316func cos_full(a: i64) -> i64 { let t: i64=reduce2pi(a); if t<HALF_PI { return cos_q(t) } if t<PI { return 0-cos_q(PI-t) } if t<THREE_HALF_PI { return 0-cos_q(t-PI) } return cos_q(TWO_PI-t) }
called by 1: rope_apply calls 2: reduce2picos_q
320func rope_apply(v: *i64, hd: i64, pos: i64, freqs: *i64) -> i64 { let np: i64=hd/2; var i: i64=0; while i<np { let ang: i64=pos*freqs[i]; let c: i64=cos_full(ang); let s: i64=sin_full(ang); let a: i64=v[i]; let b: i64=v[i+np]; v[i]=qmul(a,c)-qmul(b,s); v[i+np]=qmul(a,s)+qmul(b,c); i=i+1 } return 0 }
called by 9: attn_sublayerdecode_step_kvdecode_step_kv_cacheddecode_step_kv_cached_i32nf_prefill_batcheddecode_step_kv_cached_i8+3 calls 3: cos_fullsin_fullqmul
322func rope_freqs(freqs: *i64, hd: i64) -> i64 { let np: i64=hd/2; var i: i64=0; while i<np { freqs[i]=fx_exp(0-((i*LN_BASE_Q16)/np)); i=i+1 } return 0 }
called by 18: mainmainmainmainmainmain+12 calls 1: fx_exp
327func fx_ln_int(x: i64) -> i64
called by 2: nac_read_configmain
352func rope_freqs_base(freqs: *i64, hd: i64, ln_base_q16: i64) -> i64 { let np: i64=hd/2; var i: i64=0; while i<np { freqs[i]=fx_exp(0-((i*ln_base_q16)/np)); i=i+1 } return 0 }
called by 2: mainmain calls 1: fx_exp
355func mm_out_in(inp: *i64, W: *i64, dst: *i64, T: i64, in_dim: i64, out_dim: i64, rev: i64) -> i64
called by 11: attn_sublayerffn_sublayernmla_downprojnmla_upprojnmla_forward_latentnmla_forward_fullkv+5
365func rmsnorm_gamma_row(x: *i64, gamma: *i64, xoff: i64, D: i64, out: *i64, ooff: i64) -> i64
called by 8: mainnmoe_block_forwardmb_ref_tokennolmoe_attnnolmoe_moemain+2 calls 2: isqrtqmul
374func rmsnorm_gamma_row_q24(x: *i64, gamma: *i64, xoff: i64, D: i64, out: *i64, ooff: i64) -> i64
called by 23: mainmaingdx_generatemainmainmain+17 calls 1: isqrt
381func mm_q24in(inp: *i64, W: *i64, dst: *i64, T: i64, in_dim: i64, out_dim: i64, rev: i64) -> i64
called by 2: attn_sublayerffn_sublayer
467func nf_pool() -> *NxThreadPool
called by 19: mm_poolnf_load_layer_poolhead_argmax_poolnf_dequant_head_allhead_argmax_cachedmm_pool_i32+13 calls 2: nx_pool_newsys_mmap
492func nf_hsum_sx(acc: *u8) -> i64
called by 4: _nfmm_i8_task_nfmm_i8_task4_nfmm_i8_task_m_nfmm_i8_task_m4
499func _nfmm_task(ctx_i: i64) -> i64
512func mm_pool(inp: *i64, W: *i64, dst: *i64, T: i64, in_dim: i64, out_dim: i64, shift: i64) -> i64
called by 3: attn_sublayerdecode_step_kvdecode_step_kv_cached calls 4: nf_poolnx_pool_n_completednx_pool_submitnx_pool_wait
534func _nfld_task(ctx_i: i64) -> i64
calls 1: load_blk
540func nf_ld1(p: *NxThreadPool, i: i64, buf: *u8, hdr: *NxGgufHeader, L: i64, sfx: *u8, out: i64, want: i64) -> i64
called by 1: nf_load_layer_pool calls 1: nx_pool_submit
547func nf_load_layer_pool(buf: *u8, hdr: *NxGgufHeader, L: i64, wb: *i64, ne: i64, qd: i64, kvd: i64, fd: i64) -> i64
called by 6: run_stack_prefill_kvdecode_step_kvnf_dequant_all_layersnf_dequant_all_layers_i32nf_dequant_all_layers_i8main calls 4: nf_poolnx_pool_n_completednf_ld1nx_pool_wait
566func _nfhd_task(ctx_i: i64) -> i64
calls 1: dequant_row
588func head_argmax_pool(hp: *i64) -> i64
called by 3: mainmainmain calls 6: nf_poolsys_mmapnx_pool_n_completednx_pool_submitnx_pool_waitsys_munmap
637func attn_sublayer(x: *i64, gamma: *i64, Wq: *i64, Wk: *i64, Wv: *i64, Wo: *i64, bq: *i64, bk: *i64, bv: *i64, freqs: *i64, xn: *i64, Q: *i64, K: *i64, V: *i64, concat: *i64, sc: *i64, at: *i64, proj: *i64, out: *i64, cfg: *i64, rev: i64) -> i64
called by 3: run_stackrun_stack_prefill_kvmain calls 7: rmsnorm_gamma_row_q24mm_poolmm_q24inrope_applyqmulfx_exp+1
673func ffn_sublayer(x: *i64, gamma: *i64, Wg: *i64, Wu: *i64, Wd: *i64, xn: *i64, gate: *i64, up: *i64, hbuf: *i64, proj: *i64, out: *i64, cfg: *i64, rev: i64) -> i64
called by 3: run_stackrun_stack_prefill_kvmain calls 5: rmsnorm_gamma_row_q24mm_q24inqmulsilumm_out_in
684func run_stack(buf: *u8, hdr: *NxGgufHeader, x: *i64, out: *i64, wb: *i64, sb: *i64, nm: *u8, freqs: *i64, cfgA: *i64, cfgF: *i64, N: i64, rev: i64) -> i64
called by 8: mainmainmainmainmainmain+2 calls 4: cpyload_blkattn_sublayerffn_sublayer
722func run_stack_prefill_kv(buf: *u8, hdr: *NxGgufHeader, x: *i64, out: *i64, wb: *i64, sb: *i64, nm: *u8, freqs: *i64, kvc: *i64, cfgA: *i64, cfgF: *i64, N: i64) -> i64
called by 5: mainmainmainmainmain calls 4: cpynf_load_layer_poolattn_sublayerffn_sublayer
750func decode_step_kv(buf: *u8, hdr: *NxGgufHeader, x1: *i64, out1: *i64, wb: *i64, sb: *i64, nm: *u8, freqs: *i64, kvc: *i64, pos: i64, cfgA: *i64, cfgF: *i64, N: i64) -> i64
called by 2: mainmain calls 8: cpynf_load_layer_poolrmsnorm_gamma_row_q24mm_poolrope_applyqmul+2
809func nf_alloc_layer_wb(ne: i64, qd: i64, kvd: i64, fd: i64) -> *i64
called by 3: nf_dequant_all_layersnf_dequant_all_layers_i32nf_dequant_all_layers_i8 calls 1: sys_mmap
818func nf_dequant_all_layers(buf: *u8, hdr: *NxGgufHeader, wcache: *i64, N: i64, ne: i64, qd: i64, kvd: i64, fd: i64) -> i64
called by 1: main calls 2: nf_alloc_layer_wbnf_load_layer_pool
831func decode_step_kv_cached(buf: *u8, hdr: *NxGgufHeader, x1: *i64, out1: *i64, wcache: *i64, sb: *i64, nm: *u8, freqs: *i64, kvc: *i64, pos: i64, cfgA: *i64, cfgF: *i64, N: i64) -> i64
called by 1: main calls 7: cpyrmsnorm_gamma_row_q24mm_poolrope_applyqmulfx_exp+1
885func _nfhdq_task(ctx_i: i64) -> i64
calls 1: dequant_row
894func nf_dequant_head_all(buf: *u8, oh_base: i64, oh_ty: i64, vocab: i64, ne: i64) -> *i64
called by 1: main calls 5: nf_poolsys_mmapnx_pool_n_completednx_pool_submitnx_pool_wait
919func _nfhac_task(ctx_i: i64) -> i64
937func head_argmax_cached(hcp: *i64) -> i64
called by 1: main calls 4: nf_poolnx_pool_n_completednx_pool_submitnx_pool_wait
977func nf_narrow_i32(src: *i64, dst: *i32, cnt: i64) -> i64
called by 1: nf_dequant_all_layers_i32
990func _nfmm_i32_task(ctx_i: i64) -> i64
1004func mm_pool_i32(inp: *i64, W: *i32, dst: *i64, T: i64, in_dim: i64, out_dim: i64, shift: i64) -> i64
called by 1: decode_step_kv_cached_i32 calls 4: nf_poolnx_pool_n_completednx_pool_submitnx_pool_wait
1026func nf_alloc_layer_wb_i32(ne: i64, qd: i64, kvd: i64, fd: i64) -> *i64
called by 1: nf_dequant_all_layers_i32 calls 1: sys_mmap
1035func nf_dequant_all_layers_i32(buf: *u8, hdr: *NxGgufHeader, wc32: *i64, N: i64, ne: i64, qd: i64, kvd: i64, fd: i64) -> i64
called by 2: mainnsv_build_caches calls 5: nf_alloc_layer_wbnf_load_layer_poolnf_alloc_layer_wb_i32cpynf_narrow_i32
1061func decode_step_kv_cached_i32(buf: *u8, hdr: *NxGgufHeader, x1: *i64, out1: *i64, wcache: *i64, sb: *i64, nm: *u8, freqs: *i64, kvc: *i64, pos: i64, cfgA: *i64, cfgF: *i64, N: i64) -> i64
called by 2: mainnsv_step calls 7: cpyrmsnorm_gamma_row_q24mm_pool_i32rope_applyqmulfx_exp+1
1115func _nfhdq_i32_task(ctx_i: i64) -> i64
calls 1: dequant_row
1123func nf_dequant_head_all_i32(buf: *u8, oh_base: i64, oh_ty: i64, vocab: i64, ne: i64) -> *i32
called by 6: mainmainmainmainmainnsv_build_caches calls 6: nf_poolsys_mmapnx_pool_n_completednx_pool_submitnx_pool_waitsys_munmap
1160func _nfhac_i32_task(ctx_i: i64) -> i64
1179func _nfhlg_task(ctx_i: i64) -> i64
1193func head_logits_cached_i32(hlp: *i64) -> i64
called by 3: gdx_pickmainnsv_next_token calls 4: nf_poolnx_pool_n_completednx_pool_submitnx_pool_wait
1217func head_argmax_cached_i32(hcp: *i64) -> i64
called by 6: mainmaingdx_pickmainmainnsv_next_token calls 4: nf_poolnx_pool_n_completednx_pool_submitnx_pool_wait
1272func nf_pack2(buf: *u8, idx: i64, val: i64) -> i64 { buf[idx*2]=(val) as u8; buf[idx*2+1]=(val>>8) as u8; return 0 }
called by 7: nf_quant_w_i8mm_pool_i8nf_quant_rownq8_dot_row_scalarmq8_gemmmq_fused_dequant_gemm+1
1275func nf_qround(v: i64, s: i64, qmax: i64) -> i64
called by 4: nf_quant_w_i8mm_pool_i8nf_quant_rowmain
1282func nf_quant_w_i8(Wsrc: *i64, Wi8: *u8, sw: *i64, out_dim: i64, in_dim: i64) -> i64
called by 3: nf_dequant_all_layers_i8nq_slot_from_q16wg_quant_weights calls 2: nf_pack2nf_qround
1295func nf_alloc_layer_wb_i8(ne: i64, qd: i64, kvd: i64, fd: i64) -> *i64
called by 1: nf_dequant_all_layers_i8 calls 1: sys_mmap
1309func nf_dequant_all_layers_i8(buf: *u8, hdr: *NxGgufHeader, wc8: *i64, N: i64, ne: i64, qd: i64, kvd: i64, fd: i64) -> i64
called by 4: mainmainmainnsv_build_caches calls 5: nf_alloc_layer_wbnf_load_layer_poolnf_alloc_layer_wb_i8cpynf_quant_w_i8
1332func _nfmm_i8_task(ctx_i: i64) -> i64
calls 1: nf_hsum_sx
1356func _nfmm_i8_task4(ctx_i: i64) -> i64
calls 1: nf_hsum_sx
1393func mm_pool_i8(x: *i64, Wi8: *u8, sw: *i64, dst: *i64, in_dim: i64, out_dim: i64, shift: i64) -> i64
called by 4: decode_step_kv_cached_i8nq_mmmainmain calls 8: nf_poolsys_mmapnf_pack2nf_qroundnx_pool_n_completednx_pool_submit+2
1423func _nfmm_i8_task_m(ctx_i: i64) -> i64
calls 1: nf_hsum_sx
1460func _nfmm_i8_task_m4(ctx_i: i64) -> i64
calls 1: nf_hsum_sx
1503func mm_pool_i8_m(x: *i64, M: i64, Wi8: *u8, sw: *i64, dst: *i64, in_dim: i64, out_dim: i64, shift: i64) -> i64
called by 2: nf_prefill_batchedmain calls 2: nf_quant_rows_poolmm_pool_i8_mq
1508func nf_quant_row(xr: *i64, in_dim: i64, xq: *u8, sxm: *i64, m: i64) -> i64
called by 2: _nf_quant_rows_tasknf_quant_rows_pool calls 2: nf_pack2nf_qround
1516func _nf_quant_rows_task(ctx_i: i64) -> i64
calls 1: nf_quant_row
1524func nf_quant_rows_pool(x: *i64, M: i64, in_dim: i64) -> i64
called by 3: mm_pool_i8_mnf_prefill_batchedmain calls 8: sys_munmapsys_mmapsys_now_usnf_quant_rownf_poolnx_pool_n_completed+2
1556func mm_pool_i8_mq(M: i64, Wi8: *u8, sw: *i64, dst: *i64, in_dim: i64, out_dim: i64, shift: i64) -> i64
called by 2: mm_pool_i8_mnf_prefill_batched calls 5: nf_poolsys_now_usnx_pool_n_completednx_pool_submitnx_pool_wait
1587func nf_prefill_batched(buf: *u8, hdr: *NxGgufHeader, xT: *i64, out1: *i64, wc8: *i64, sb: *i64, nm: *u8, freqs: *i64, kvc: *i64, pos0: i64, T: i64, cfgA: *i64, cfgF: *i64, N: i64) -> i64
called by 1: nsv_prefill_i8_batched calls 9: cpysys_now_usrmsnorm_gamma_row_q24nf_quant_rows_poolmm_pool_i8_mqrope_apply+3
1666func decode_step_kv_cached_i8(buf: *u8, hdr: *NxGgufHeader, x1: *i64, out1: *i64, wc8: *i64, sb: *i64, nm: *u8, freqs: *i64, kvc: *i64, pos: i64, cfgA: *i64, cfgF: *i64, N: i64) -> i64
called by 4: mainmainmainnsv_step calls 7: cpyrmsnorm_gamma_row_q24mm_pool_i8rope_applynf_attn_kv_coreqmul+1
1714func nf_attn_head(Q: *i64, Kc: *i64, Vc: *i64, concat: *i64, hh: i64, cnt: i64, hd: i64, kvd: i64, group: i64, scale: i64, sc: *i64, at: *i64) -> i64
called by 3: _nf_attn_pairs_tasknf_attn_kv_corenf_attn_kv_block calls 2: qmulfx_exp
1727func nf_attn_scratch(cntmax: i64) -> i64
called by 1: nf_attn_pairs_pool calls 2: sys_munmapsys_mmap
1734func _nf_attn_pairs_task(ctx_i: i64) -> i64
calls 1: nf_attn_head
1748func nf_attn_pairs_pool(Q: *i64, Kc: *i64, Vc: *i64, concat: *i64, nh: i64, pos0: i64, T: i64, hd: i64, kvd: i64, group: i64, scale: i64, qd: i64) -> i64
called by 2: nf_attn_kv_corenf_attn_kv_block calls 5: nf_poolnf_attn_scratchnx_pool_n_completednx_pool_submitnx_pool_wait
1775func nf_attn_kv_core(Q: *i64, K: *i64, V: *i64, kvc: *i64, L: i64, pos: i64, cfgA: *i64, sc: *i64, at: *i64, concat: *i64) -> i64
called by 2: decode_step_kv_cached_i8decode_step_kv_q4k calls 2: nf_attn_headnf_attn_pairs_pool
1790func nf_attn_kv_block(Q: *i64, K: *i64, V: *i64, kvc: *i64, L: i64, pos0: i64, T: i64, cfgA: *i64, sc: *i64, at: *i64, concat: *i64) -> i64
called by 1: nf_prefill_batched calls 2: nf_attn_headnf_attn_pairs_pool
1809func _nf_silu_rows_task(ctx_i: i64) -> i64
calls 2: qmulsilu
1816func nf_silu_rows_pool(gate: *i64, up: *i64, hbuf: *i64, T: i64, fd: i64) -> i64
called by 1: nf_prefill_batched calls 4: nf_poolnx_pool_n_completednx_pool_submitnx_pool_wait