code wiki / (root) / nx_nofloat_autograd.nx

nx_nofloat_autograd.nx

buildroot/runtime/nx_nofloat_autograd.nx

39001 B774 linesdepth 2pulls 2 transitivereach 47 importersview sourcekind librarytopic nofloat
docsdependenciesstructsconstsfunctions

about

nx_nofloat_autograd.nx -- NO-FLOAT (integer Q16 fixed-point) reverse-mode TENSOR autograd: the genuine MISSING GENERATION in the no-float lineage (genealogy DeepMind ladder, operator first law = don't lie). HONEST landscape this fills (verified by reading the organs, 2026-06-22): - GENERAL reverse-mode autograd ALREADY EXISTS, but on the SOFTWARE-FLOAT tower: nx_autograd / nx_autograd_tensor / nx_tgrad_core all build on nx_f32 (IEEE-754 emulated). Float, not no-float. - NO-FLOAT (integer Q16) training ALREADY EXISTS, but only nx_nn_train = a SINGLE linear layer with a HAND-CODED analytic gradient. No general graph, no composition. - What was genuinely missing = a GENERAL reverse-mode autograd in the SAME integer Q16 tower the no-float Qwen inference pipeline actually uses (nx_nofloat_llm: Q16=65536, qmul=(a*b)>>16, accumulate-then-shift matmul). This file is that bridge: a tape of arbitrary ops + ONE reverse sweep, so an MLP (matvec->vadd->relu->matvec->vadd->mse) trains by the SAME loop that will train the transformer -- all deterministic integer arithmetic (no nx_f32, no FPU, no GPU, no third-party AD). TAPE: stride-7 nodes {op, ai, bi, rows, cols, valp, gradp}; valp/gradp index a per-build BUMP ARENA of Q16 cells (st[1] counter; no allocator -> deterministic). Forward EAGER (Q16), backward ONE reverse sweep (nodes built in topological order, so reverse index order visits parents before children). Backward identities (Q16) = the proven float identities with nx_f32_mul->qmul, nx_f32_add->integer +: matvec y=W*x: dW[i][j] += qmul(gy[i], x[j]); dx[j] += qmul(W[i][j], gy[i]) vadd y=a+b: ga += gy ; gb += gy relu: gx += gy iff the INPUT cell was > 0 mse L=(1/n)sum(p-t)^2: dp_i += qmul(gL, (2*(p_i-t_i))/n) Determinism EXCEED-AXIS: integer add is EXACTLY associative -> bit-identical training independent of order (the f32 tower is only bit-identical because it pins ONE order; here it is structural). genealogy_id: linnainmaa_1970_reverse_mode_ad + rumelhart_1986_backprop, ported to fixed-point Q16 lineage_id: sovereign_nofloat_tensor_tape_autograd_v1 license_tier: ORIGINAL No `main` (pure library; proof lives in nx_nofloat_autograd_gate.nx).

dependencies 1 imports · 47 importers

nx_syscalls.nx nx_nofloat_autograd.nx nx_blockfloat_block_gate.nx nx_blockfloat_generate_gate.nx nx_blockfloat_lm_gate.nx nx_blockfloat_mha_gate.nx nx_blockfloat_stack_gate.nx nx_ng_avatar.nx nx_ng_avatar_rig.nx nx_ng_gen3d.nx nx_ng_gsplat.nx nx_ng_hashenc.nx

diagram shows first 10 each side; +0 more imports, +37 more importers in the complete lists below.

imports: nx_syscalls.nx

imported by: nx_blockfloat_block_gate.nxnx_blockfloat_generate_gate.nxnx_blockfloat_lm_gate.nxnx_blockfloat_mha_gate.nxnx_blockfloat_stack_gate.nxnx_ng_avatar.nxnx_ng_avatar_rig.nxnx_ng_gen3d.nxnx_ng_gsplat.nxnx_ng_hashenc.nxnx_ng_radiance.nxnx_nofloat_adamw32_gate.nxnx_nofloat_adamw_gate.nxnx_nofloat_attn_gate.nxnx_nofloat_autograd_ddp_gate.nxnx_nofloat_autograd_gate.nxnx_nofloat_block_gate.nxnx_nofloat_bpe_lm_gate.nxnx_nofloat_corpus_gate.nxnx_nofloat_deepstack_gate.nxnx_nofloat_distill_gate.nxnx_nofloat_generalize_gate.nxnx_nofloat_generate_gate.nxnx_nofloat_grammar_gate.nxnx_nofloat_incontext_gate.nxnx_nofloat_induction_gate.nxnx_nofloat_kvcache_gate.nxnx_nofloat_kvcache_int8_gate.nxnx_nofloat_landing_gate.nxnx_nofloat_lm_gate.nxnx_nofloat_multihead_gate.nxnx_nofloat_prose_gate.nxnx_nofloat_quant_gate.nxnx_nofloat_rope_gate.nxnx_nofloat_sample_gate.nxnx_nofloat_scale2_gate.nxnx_nofloat_scale_batch_gate.nxnx_nofloat_scale_ffn_gate.nxnx_nofloat_scale_gate.nxnx_nofloat_scale_wide_gate.nxnx_nofloat_train_exceed.nxnx_nofloat_vs_float_gate.nxnx_nofloat_weighttie_gate.nxnx_nofloat_xformer_gate.nxnx_reader_quantize_gate.nxnx_reader_span_gate.nxnx_reader_squad_gate.nx

structs

none

consts

29const NFA_MAGIC_5040: i64 = 5040
30const NFA_MAGIC_362880: i64 = 362880
31const NFA_MAGIC_40320: i64 = 40320
32const NFA_MAGIC_45426: i64 = 45426
34const NFA_LEAF: i64 = 0
35const NFA_MATVEC: i64 = 1
36const NFA_VADD: i64 = 2
37const NFA_RELU: i64 = 3
38const NFA_MSE: i64 = 4
39const NFA_SOFTMAX: i64 = 5 // attention nonlinearity
40const NFA_SILU: i64 = 6 // FFN activation (x*sigmoid(x))
41const NFA_RMSNORM: i64 = 7 // transformer normalization
42const NFA_MATMUL: i64 = 8 // C[m,p] = A[m,k] . B[k,p]
43const NFA_MATMUL_NT: i64 = 9 // S[m,p] = A[m,k] . B[p,k]^T (the Q.K^T attention contraction)
44const NFA_CMUL: i64 = 10 // elementwise scale by a Q16 constant (bi holds the constant, not a node)
45const NFA_SOFTMAX_ROWS: i64 = 11 // per-row softmax (bi=1 -> CAUSAL: row i normalizes over j<=i only)
46const NFA_ROPE: i64 = 12 // rotary position embedding on [T,hd] (parameter-free; pos = row index)
47const NFA_HADAMARD: i64 = 13 // elementwise vector multiply a (*) b (the SwiGLU gate)
48const NFA_RMSNORM_ROWS: i64 = 14 // PER-ROW (per-token) RMSNorm over the cols of an [T,d] node
49const NFA_EMBED: i64 = 15 // gather rows of E[V,dm] by integer token ids -> X[T,dm] (bi = ids ptr)
50const NFA_SOFTCE_ROWS: i64 = 16 // fused per-row softmax cross-entropy over targets (bi = target ids ptr)
51const NFA_SLICE_COLS: i64 = 17 // extract cols [c0,c0+w) of an [T,C] node -> [T,w] (bi = c0); for multi-head
52const NFA_CONCAT_COLS: i64 = 18 // concat two [T,*] nodes column-wise -> [T, wa+wb]; for multi-head
53const NFA_Q16: i64 = 65536 // 1.0 in Q16 fixed-point (matches nx_nofloat_llm)
58const NFA_LOG2E: i64 = 94548
59const NFA_PC0: i64 = 65536
60const NFA_PC1: i64 = 45426
61const NFA_PC2: i64 = 15743
62const NFA_PC3: i64 = 4367
64const NFA_HALF_PI: i64 = 102944
65const NFA_PI: i64 = 205887
66const NFA_3HALF_PI: i64 = 308831
67const NFA_TWO_PI: i64 = 411775
68const NFA_LN_BASE: i64 = 905421

functions

71func nfa_qmul(a: i64, b: i64) -> i64 { return (a * b) >> 16 }
74func nfa_isqrt(v: i64) -> i64 { if v<=0 { return 0 } if v<4 { return 1 } var x: i64=v; var y: i64=(x+1)>>1; var go: i64=1; while go==1 { if y<x { x=y; y=(x+v/x)>>1 } else { go=0 } } return x }
76func nfa_fxexp(x: i64) -> i64 { var xm: i64=0-x; if x>0 { xm=0 } let ym: i64=(xm*NFA_LOG2E)>>16; let yi: i64=ym>>16; let yf: i64=ym-(yi<<16); let g: i64=NFA_Q16-yf; var t: i64=NFA_PC3; t=NFA_PC2+((g*t)>>16); t=NFA_PC1+((g*t)>>16); t=NFA_PC0+((g*t)>>16); t=t>>1; if yi>=31 { return 0 } return t>>yi }
77func nfa_sigmoid(x: i64) -> i64 { if x>=0 { let ex: i64=nfa_fxexp(0-x); return (NFA_Q16*NFA_Q16)/(NFA_Q16+ex) } let ex: i64=nfa_fxexp(x); let sp: i64=(NFA_Q16*NFA_Q16)/(NFA_Q16+ex); return NFA_Q16-sp }
called by 2: nfa_silufnfa_silud calls 1: nfa_fxexp
78func nfa_siluf(x: i64) -> i64 { return nfa_qmul(x, nfa_sigmoid(x)) } // silu(x) = x*sigmoid(x)
79func nfa_silud(x: i64) -> i64 { let s: i64=nfa_sigmoid(x); return s + nfa_qmul(x, nfa_qmul(s, NFA_Q16 - s)) } // silu'(x) = s + x*s*(1-s)
called by 1: nfa_backward calls 2: nfa_sigmoidnfa_qmul
81func nfa_sinq(x: i64) -> i64 { let x2: i64=nfa_qmul(x,x); let x3: i64=nfa_qmul(x2,x); let x5: i64=nfa_qmul(x3,x2); let x7: i64=nfa_qmul(x5,x2); let x9: i64=nfa_qmul(x7,x2); return x - x3/6 + x5/120 - x7/NFA_MAGIC_5040 + x9/NFA_MAGIC_362880 }
called by 1: nfa_sinf calls 1: nfa_qmul
82func nfa_cosq(x: i64) -> i64 { let x2: i64=nfa_qmul(x,x); let x4: i64=nfa_qmul(x2,x2); let x6: i64=nfa_qmul(x4,x2); let x8: i64=nfa_qmul(x6,x2); return NFA_Q16 - x2/2 + x4/24 - x6/720 + x8/NFA_MAGIC_40320 }
called by 1: nfa_cosf calls 1: nfa_qmul
83func nfa_reduce2pi(a: i64) -> i64 { var t: i64=a; while t<0 { t=t+NFA_TWO_PI } while t>=NFA_TWO_PI { t=t-NFA_TWO_PI } return t }
called by 2: nfa_sinfnfa_cosf
84func nfa_sinf(a: i64) -> i64 { let t: i64=nfa_reduce2pi(a); if t<NFA_HALF_PI { return nfa_sinq(t) } if t<NFA_PI { return nfa_sinq(NFA_PI-t) } if t<NFA_3HALF_PI { return 0-nfa_sinq(t-NFA_PI) } return 0-nfa_sinq(NFA_TWO_PI-t) }
85func nfa_cosf(a: i64) -> i64 { let t: i64=nfa_reduce2pi(a); if t<NFA_HALF_PI { return nfa_cosq(t) } if t<NFA_PI { return 0-nfa_cosq(NFA_PI-t) } if t<NFA_3HALF_PI { return 0-nfa_cosq(t-NFA_PI) } return nfa_cosq(NFA_TWO_PI-t) }
89func nfa_ln(x: i64) -> i64
called by 1: nfa_softce_rows calls 1: nfa_qmul
105func nfa_new(tape: *i64, st: *i64, op: i64, ai: i64, bi: i64, rows: i64, cols: i64) -> i64
117func nfa_leaf(tape: *i64, vals: *i64, st: *i64, rows: i64, cols: i64, src: *i64, soff: i64) -> i64
127func nfa_matvec(tape: *i64, vals: *i64, st: *i64, aW: i64, bx: i64) -> i64
146func nfa_vadd(tape: *i64, vals: *i64, st: *i64, a: i64, b: i64) -> i64
160func nfa_relu(tape: *i64, vals: *i64, st: *i64, a: i64) -> i64
179func nfa_mse(tape: *i64, vals: *i64, st: *i64, pred: i64, target: i64) -> i64
198func nfa_softmax(tape: *i64, vals: *i64, st: *i64, a: i64) -> i64
called by 1: xf_loss calls 2: nfa_newnfa_fxexp
212func nfa_silu(tape: *i64, vals: *i64, st: *i64, a: i64) -> i64
221func nfa_rmsnorm(tape: *i64, vals: *i64, st: *i64, a: i64) -> i64
called by 1: xf_loss calls 3: nfa_newnfa_qmulnfa_isqrt
236func nfa_matmul(tape: *i64, vals: *i64, st: *i64, a: i64, b: i64) -> i64
254func nfa_matmul_nt(tape: *i64, vals: *i64, st: *i64, a: i64, b: i64) -> i64
272func nfa_cmul(tape: *i64, vals: *i64, st: *i64, a: i64, c_q: i64) -> i64
281func nfa_softmax_rows(tape: *i64, vals: *i64, st: *i64, a: i64, causal: i64) -> i64
305func nfa_rope(tape: *i64, vals: *i64, st: *i64, a: i64) -> i64
326func nfa_hadamard(tape: *i64, vals: *i64, st: *i64, a: i64, b: i64) -> i64
335func nfa_rmsnorm_rows(tape: *i64, vals: *i64, st: *i64, a: i64) -> i64
354func nfa_embed(tape: *i64, vals: *i64, st: *i64, e_node: i64, ids: *i64, T: i64) -> i64
369func nfa_softce_rows(tape: *i64, vals: *i64, st: *i64, logits: i64, tgt: *i64) -> i64
389func nfa_slice_cols(tape: *i64, vals: *i64, st: *i64, a: i64, c0: i64, w: i64) -> i64
called by 3: mha_fwdmha_fwdmain calls 1: nfa_new
398func nfa_concat_cols(tape: *i64, vals: *i64, st: *i64, a: i64, b: i64) -> i64
called by 3: mha_fwdmha_fwdmain calls 1: nfa_new
413func nfa_val(tape: *i64, vals: *i64, k: i64, c: i64) -> i64 { return vals[tape[7*k+5] + c] }
414func nfa_grad(tape: *i64, grads: *i64, k: i64, c: i64) -> i64 { return grads[tape[7*k+6] + c] }
417func nfa_backward(tape: *i64, vals: *i64, grads: *i64, n: i64, root: i64) -> i64
727func nfa_sgd(w: *i64, g: *i64, n: i64, lr_q: i64) -> i64
738func nfa_sgd_ef(w: *i64, g: *i64, resid: *i64, n: i64, lr_q: i64) -> i64
called by 1: rq_train
754func nfa_adamw(w: *i64, g: *i64, m: *i64, v: *i64, n: i64, lr: i64, b1: i64, b2: i64, eps: i64, wd: i64, t: i64) -> i64