nx_nofloat_autograd_ddp_gate.nx source
↩ module page · 170 lines · 7944 B
1// nx_nofloat_autograd_ddp_gate.nx -- DEEPER R4->LLM: bit-reproducible data-parallel training driven by the
2// REAL no-float autograd tape (vs the linear/SGD stand-in in nx_nofloat_ddp_train_gate).
3// (knowledge/research/2026-06-23-fabric-applies-to-room-and-elsewhere.md ยง4.)
4//
5// Each of D workers builds the SAME SSE loss graph on its OWN sample shard via the general Q16 autograd
6// (nfa_leaf/nfa_matvec/nfa_mse/nfa_vadd -> one nfa_backward reverse sweep -> nfa_grad), extracts its W
7// gradient, then R4's `ring_allreduce` sums the per-worker gradients and `nfa_sgd` applies the update. THE
8// EXCEED: the updated weights are BYTE-IDENTICAL across D in {1,2,4,8} AND equal to a single-process
9// full-batch step -- because the autograd is exact integer (associative) AND the all-reduce is order-
10// independent. Real trainer components, not a stand-in. Float DDP/NCCL cannot guarantee this.
11//
12// Composes the verified nx_nofloat_autograd (gate B/C pattern) + nx_fabric_collective.ring_allreduce.
13// HONEST SCOPE: a linear W[1x8] model trained by the GENERAL tape (same engine the Q16 transformer uses);
14// N logical workers in one box. license_tier: ORIGINAL expect_exit: 0
15import "nx_nofloat_autograd.nx"
16import "nx_fabric_collective.nx"
17
18// SSE loss over sample range [s0,s1): leaf W[1xV] -> per-sample (leaf x[Vx1] -> matvec -> mse vs y) -> sum.
19// Mirrors gB_build (nx_nofloat_autograd_gate) generalized to width V + a sample range. outW[0] = the W node.
20func dda_build(tape: *i64, vals: *i64, st: *i64, Wp: *i64, X: *i64, Y: *i64, s0: i64, s1: i64, V: i64, outW: *i64) -> i64 {
21 st[0] = 0; st[1] = 0
22 let nW: i64 = nfa_leaf(tape, vals, st, 1, V, Wp, 0)
23 outW[0] = nW
24 var root: i64 = 0 - 1
25 var t: i64 = s0
26 while t < s1 {
27 let nx: i64 = nfa_leaf(tape, vals, st, V, 1, X, t * V)
28 let np: i64 = nfa_matvec(tape, vals, st, nW, nx)
29 let ny: i64 = nfa_leaf(tape, vals, st, 1, 1, Y, t)
30 let nm: i64 = nfa_mse(tape, vals, st, np, ny)
31 if root < 0 { root = nm } else { root = nfa_vadd(tape, vals, st, root, nm) }
32 t = t + 1
33 }
34 return root
35}
36
37// ONE data-parallel step over D workers: per-worker shard gradient -> ring_allreduce -> nfa_sgd. Fills outWp.
38func dda_step(D: i64, S: i64, V: i64, Wp: *i64, X: *i64, Y: *i64, tape: *i64, vals: *i64, grads: *i64, st: *i64, lr: i64, outWp: *i64) -> i64 {
39 let chunk: i64 = S / D
40 let parts: *i64 = sys_mmap(8 * D * V) as *i64
41 let total: *i64 = sys_mmap(8 * V) as *i64
42 let outW: *i64 = sys_mmap(8) as *i64
43 var w: i64 = 0
44 while w < D {
45 let s0: i64 = w * chunk
46 let s1: i64 = (w + 1) * chunk
47 let root: i64 = dda_build(tape, vals, st, Wp, X, Y, s0, s1, V, outW)
48 nfa_backward(tape, vals, grads, st[0], root)
49 var c: i64 = 0
50 while c < V { parts[w * V + c] = nfa_grad(tape, grads, outW[0], c); c = c + 1 }
51 w = w + 1
52 }
53 ring_allreduce(parts, D, V, total) // R4: sum the per-worker gradients
54 var i: i64 = 0
55 while i < V { outWp[i] = Wp[i]; i = i + 1 }
56 nfa_sgd(outWp, total, V, lr)
57 return 0
58}
59
60// single-process full-batch step (ground truth the DDP step must match byte-for-byte).
61func dda_single(S: i64, V: i64, Wp: *i64, X: *i64, Y: *i64, tape: *i64, vals: *i64, grads: *i64, st: *i64, lr: i64, outWp: *i64) -> i64 {
62 let outW: *i64 = sys_mmap(8) as *i64
63 let g: *i64 = sys_mmap(8 * V) as *i64
64 let root: i64 = dda_build(tape, vals, st, Wp, X, Y, 0, S, V, outW)
65 nfa_backward(tape, vals, grads, st[0], root)
66 var c: i64 = 0
67 while c < V { g[c] = nfa_grad(tape, grads, outW[0], c); c = c + 1 }
68 var i: i64 = 0
69 while i < V { outWp[i] = Wp[i]; i = i + 1 }
70 nfa_sgd(outWp, g, V, lr)
71 return 0
72}
73
74func ap(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
75func an(v: i64) -> i64 {
76 let b: *u8 = sys_mmap(28); var m: i64 = v
77 if m < 0 { m = 0 - m; sys_write(1, "-" as *u8, 1) }
78 let t: *u8 = sys_mmap(28); var k: i64 = 0
79 if m == 0 { t[0] = 48 as u8; k = 1 }
80 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
81 var i: i64 = 0
82 while i < k { b[i] = t[k - 1 - i]; i = i + 1 }
83 sys_write(1, b, k); return 0
84}
85func achk(name: *u8, ok: i64) -> i64 {
86 if ok == 1 { ap(" PASS " as *u8); ap(name); ap("\n" as *u8); return 1 }
87 ap(" FAIL " as *u8); ap(name); ap("\n" as *u8); return 0
88}
89func aeq(a: *i64, b: *i64, V: i64) -> i64 { var i: i64 = 0; while i < V { if a[i] != b[i] { return 0 } i = i + 1 } return 1 }
90
91func main() -> i64 {
92 let V: i64 = 8
93 let S: i64 = 8
94 let LR: i64 = 2048 // Q16 learning rate (0.03125), as gate B
95
96 let tape: *i64 = sys_mmap(8 * 7 * 600) as *i64
97 let vals: *i64 = sys_mmap(8 * 6000) as *i64
98 let grads: *i64 = sys_mmap(8 * 6000) as *i64
99 let st: *i64 = sys_mmap(8 * 4) as *i64
100
101 // data: target W*, X (S*V Q16), Y[s] = matvec(W*, x_s) (the accumulate-shift convention)
102 let Wt: *i64 = sys_mmap(8 * V) as *i64
103 var v: i64 = 0
104 while v < V { Wt[v] = ((v % 3) - 1) * 32768; v = v + 1 } // pattern -0.5, 0, +0.5 ... Q16
105 let X: *i64 = sys_mmap(8 * S * V) as *i64
106 var s: i64 = 0
107 while s < S {
108 v = 0
109 while v < V { X[s * V + v] = (((s * 3 + v * 5 + 1) % 7) + 1) << 13; v = v + 1 } // ~0.125..0.875 Q16
110 s = s + 1
111 }
112 let Y: *i64 = sys_mmap(8 * S) as *i64
113 s = 0
114 while s < S {
115 var acc: i64 = 0
116 v = 0
117 while v < V { acc = acc + Wt[v] * X[s * V + v]; v = v + 1 }
118 Y[s] = acc >> 16
119 s = s + 1
120 }
121 let W0: *i64 = sys_mmap(8 * V) as *i64
122 v = 0
123 while v < V { W0[v] = 0; v = v + 1 } // learn from zero (one step)
124
125 let d1: *i64 = sys_mmap(8 * V) as *i64
126 let d2: *i64 = sys_mmap(8 * V) as *i64
127 let d4: *i64 = sys_mmap(8 * V) as *i64
128 let d8: *i64 = sys_mmap(8 * V) as *i64
129 let ws: *i64 = sys_mmap(8 * V) as *i64
130
131 ap("nx_nofloat_autograd_ddp -- data-parallel REAL autograd training step via R4 ring_allreduce\n" as *u8)
132
133 dda_step(1, S, V, W0, X, Y, tape, vals, grads, st, LR, d1)
134 dda_step(2, S, V, W0, X, Y, tape, vals, grads, st, LR, d2)
135 dda_step(4, S, V, W0, X, Y, tape, vals, grads, st, LR, d4)
136 dda_step(8, S, V, W0, X, Y, tape, vals, grads, st, LR, d8)
137 dda_single(S, V, W0, X, Y, tape, vals, grads, st, LR, ws)
138
139 ap(" W' (DDP D=8) = " as *u8); v = 0
140 while v < V { an(d8[v]); ap(" " as *u8); v = v + 1 }
141 ap("\n W' (1-proc) = " as *u8); v = 0
142 while v < V { an(ws[v]); ap(" " as *u8); v = v + 1 }
143 ap("\n" as *u8)
144
145 var pass: i64 = 0
146 var tot: i64 = 0
147
148 var k1: i64 = 0
149 if aeq(d1, d2, V) == 1 { if aeq(d1, d4, V) == 1 { if aeq(d1, d8, V) == 1 { k1 = 1 } } }
150 pass = pass + achk("1 REAL-autograd weights BYTE-IDENTICAL across D=1,2,4,8 (worker-count-invariant)" as *u8, k1); tot = tot + 1
151
152 var k2: i64 = 0; if aeq(d8, ws, V) == 1 { if aeq(d1, ws, V) == 1 { k2 = 1 } }
153 pass = pass + achk("2 DDP == single-process full-batch (ring_allreduce correct over autograd grads)" as *u8, k2); tot = tot + 1
154
155 var k3: i64 = 0; if aeq(d1, W0, V) == 0 { k3 = 1 }
156 pass = pass + achk("3 weights changed from W0 (the autograd produced a real gradient)" as *u8, k3); tot = tot + 1
157
158 ap("---- nx_nofloat_autograd_ddp: passed " as *u8); an(pass); ap(" / " as *u8); an(tot); ap("\n" as *u8)
159 if pass == tot {
160 let lfd: i64 = sys_openat_append("knowledge/status/nofloat_autograd_ddp.log" as *u8, 420)
161 if lfd >= 0 {
162 sys_write(lfd, "NOFLOAT-AUTOGRAD-DDP real-tape grads + ring_allreduce: weights worker-count-invariant + == single-proc verdict=GREEN\n" as *u8, 113)
163 sys_close(lfd)
164 }
165 ap("AUTOGRAD-DDP GREEN -- the REAL no-float autograd trains data-parallel with identical, correct weights\n" as *u8)
166 sys_exit(0)
167 }
168 sys_exit(1)
169 return 0
170}