nx_nofloat_autograd_ddp_gate.nx source
↩ module page · 168 lines · 8194 B
1// nx_nofloat_autograd_ddp_gate.nx -- DEEPER R4->LLM: bit-reproducible data-parallel training driven by the
2// REAL no-float autograd tape (vs the linear/SGD stand-in in nx_nofloat_ddp_train_gate).
3// (knowledge/research/2026-06-23-fabric-applies-to-room-and-elsewhere.md ยง4.)
4//
5// Each of D workers builds the SAME SSE loss graph on its OWN sample shard via the general Q16 autograd
6// (nfa_leaf/nfa_matvec/nfa_mse/nfa_vadd -> one nfa_backward reverse sweep -> nfa_grad), extracts its W
7// gradient, then R4's `ring_allreduce` sums the per-worker gradients and `nfa_sgd` applies the update. THE
8// EXCEED: the updated weights are BYTE-IDENTICAL across D in {1,2,4,8} AND equal to a single-process
9// full-batch step -- because the autograd is exact integer (associative) AND the all-reduce is order-
10// independent. Real trainer components, not a stand-in. Float DDP/NCCL cannot guarantee this.
11//
12// Composes the verified nx_nofloat_autograd (gate B/C pattern) + nx_fabric_collective.ring_allreduce.
13// HONEST SCOPE: a linear W[1x8] model trained by the GENERAL tape (same engine the Q16 transformer uses);
14// N logical workers in one box. license_tier: ORIGINAL expect_exit: 0
15import "nx_nofloat_autograd.nx"
16import "nx_fabric_collective.nx"
17// D001 MIGRATION 2026-08-25: this gate rolled its own verdict, so /api/promote refused it -- correctly. A gate
18// whose EXIT CODE does not carry its verdict can print FAIL and still exit 0, silently blessing every failure
19// it finds. The hand-rolled pass/tot pair is the same defect one layer down: declared and executed tooth counts
20// are two numbers that can drift, and gv_ctr makes them one by construction.
21import "nx_gate_verdict.nx"
22
23// SSE loss over sample range [s0,s1): leaf W[1xV] -> per-sample (leaf x[Vx1] -> matvec -> mse vs y) -> sum.
24// Mirrors gB_build (nx_nofloat_autograd_gate) generalized to width V + a sample range. outW[0] = the W node.
25func dda_build(tape: *i64, vals: *i64, st: *i64, Wp: *i64, X: *i64, Y: *i64, s0: i64, s1: i64, V: i64, outW: *i64) -> i64 {
26 st[0] = 0; st[1] = 0
27 let nW: i64 = nfa_leaf(tape, vals, st, 1, V, Wp, 0)
28 outW[0] = nW
29 var root: i64 = 0 - 1
30 var t: i64 = s0
31 while t < s1 {
32 let nx: i64 = nfa_leaf(tape, vals, st, V, 1, X, t * V)
33 let np: i64 = nfa_matvec(tape, vals, st, nW, nx)
34 let ny: i64 = nfa_leaf(tape, vals, st, 1, 1, Y, t)
35 let nm: i64 = nfa_mse(tape, vals, st, np, ny)
36 if root < 0 { root = nm } else { root = nfa_vadd(tape, vals, st, root, nm) }
37 t = t + 1
38 }
39 return root
40}
41
42// ONE data-parallel step over D workers: per-worker shard gradient -> ring_allreduce -> nfa_sgd. Fills outWp.
43func dda_step(D: i64, S: i64, V: i64, Wp: *i64, X: *i64, Y: *i64, tape: *i64, vals: *i64, grads: *i64, st: *i64, lr: i64, outWp: *i64) -> i64 {
44 let chunk: i64 = S / D
45 let parts: *i64 = sys_mmap(8 * D * V) as *i64
46 let total: *i64 = sys_mmap(8 * V) as *i64
47 let outW: *i64 = sys_mmap(8) as *i64
48 var w: i64 = 0
49 while w < D {
50 let s0: i64 = w * chunk
51 let s1: i64 = (w + 1) * chunk
52 let root: i64 = dda_build(tape, vals, st, Wp, X, Y, s0, s1, V, outW)
53 nfa_backward(tape, vals, grads, st[0], root)
54 var c: i64 = 0
55 while c < V { parts[w * V + c] = nfa_grad(tape, grads, outW[0], c); c = c + 1 }
56 w = w + 1
57 }
58 ring_allreduce(parts, D, V, total) // R4: sum the per-worker gradients
59 var i: i64 = 0
60 while i < V { outWp[i] = Wp[i]; i = i + 1 }
61 nfa_sgd(outWp, total, V, lr)
62 return 0
63}
64
65// single-process full-batch step (ground truth the DDP step must match byte-for-byte).
66func dda_single(S: i64, V: i64, Wp: *i64, X: *i64, Y: *i64, tape: *i64, vals: *i64, grads: *i64, st: *i64, lr: i64, outWp: *i64) -> i64 {
67 let outW: *i64 = sys_mmap(8) as *i64
68 let g: *i64 = sys_mmap(8 * V) as *i64
69 let root: i64 = dda_build(tape, vals, st, Wp, X, Y, 0, S, V, outW)
70 nfa_backward(tape, vals, grads, st[0], root)
71 var c: i64 = 0
72 while c < V { g[c] = nfa_grad(tape, grads, outW[0], c); c = c + 1 }
73 var i: i64 = 0
74 while i < V { outWp[i] = Wp[i]; i = i + 1 }
75 nfa_sgd(outWp, g, V, lr)
76 return 0
77}
78
79func ap(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
80func an(v: i64) -> i64 {
81 let b: *u8 = sys_mmap(28); var m: i64 = v
82 if m < 0 { m = 0 - m; sys_write(1, "-" as *u8, 1) }
83 let t: *u8 = sys_mmap(28); var k: i64 = 0
84 if m == 0 { t[0] = 48 as u8; k = 1 }
85 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
86 var i: i64 = 0
87 while i < k { b[i] = t[k - 1 - i]; i = i + 1 }
88 sys_write(1, b, k); return 0
89}
90func aeq(a: *i64, b: *i64, V: i64) -> i64 { var i: i64 = 0; while i < V { if a[i] != b[i] { return 0 } i = i + 1 } return 1 }
91
92func main() -> i64 {
93 let V: i64 = 8
94 let S: i64 = 8
95 let LR: i64 = 2048 // Q16 learning rate (0.03125), as gate B
96
97 let tape: *i64 = sys_mmap(8 * 7 * 600) as *i64
98 let vals: *i64 = sys_mmap(8 * 6000) as *i64
99 let grads: *i64 = sys_mmap(8 * 6000) as *i64
100 let st: *i64 = sys_mmap(8 * 4) as *i64
101
102 // data: target W*, X (S*V Q16), Y[s] = matvec(W*, x_s) (the accumulate-shift convention)
103 let Wt: *i64 = sys_mmap(8 * V) as *i64
104 var v: i64 = 0
105 while v < V { Wt[v] = ((v % 3) - 1) * 32768; v = v + 1 } // pattern -0.5, 0, +0.5 ... Q16
106 let X: *i64 = sys_mmap(8 * S * V) as *i64
107 var s: i64 = 0
108 while s < S {
109 v = 0
110 while v < V { X[s * V + v] = (((s * 3 + v * 5 + 1) % 7) + 1) << 13; v = v + 1 } // ~0.125..0.875 Q16
111 s = s + 1
112 }
113 let Y: *i64 = sys_mmap(8 * S) as *i64
114 s = 0
115 while s < S {
116 var acc: i64 = 0
117 v = 0
118 while v < V { acc = acc + Wt[v] * X[s * V + v]; v = v + 1 }
119 Y[s] = acc >> 16
120 s = s + 1
121 }
122 let W0: *i64 = sys_mmap(8 * V) as *i64
123 v = 0
124 while v < V { W0[v] = 0; v = v + 1 } // learn from zero (one step)
125
126 let d1: *i64 = sys_mmap(8 * V) as *i64
127 let d2: *i64 = sys_mmap(8 * V) as *i64
128 let d4: *i64 = sys_mmap(8 * V) as *i64
129 let d8: *i64 = sys_mmap(8 * V) as *i64
130 let ws: *i64 = sys_mmap(8 * V) as *i64
131
132 ap("nx_nofloat_autograd_ddp -- data-parallel REAL autograd training step via R4 ring_allreduce\n" as *u8)
133
134 dda_step(1, S, V, W0, X, Y, tape, vals, grads, st, LR, d1)
135 dda_step(2, S, V, W0, X, Y, tape, vals, grads, st, LR, d2)
136 dda_step(4, S, V, W0, X, Y, tape, vals, grads, st, LR, d4)
137 dda_step(8, S, V, W0, X, Y, tape, vals, grads, st, LR, d8)
138 dda_single(S, V, W0, X, Y, tape, vals, grads, st, LR, ws)
139
140 ap(" W' (DDP D=8) = " as *u8); v = 0
141 while v < V { an(d8[v]); ap(" " as *u8); v = v + 1 }
142 ap("\n W' (1-proc) = " as *u8); v = 0
143 while v < V { an(ws[v]); ap(" " as *u8); v = v + 1 }
144 ap("\n" as *u8)
145
146 let ctr: *i64 = gv_ctr()
147
148 var k1: i64 = 0
149 if aeq(d1, d2, V) == 1 { if aeq(d1, d4, V) == 1 { if aeq(d1, d8, V) == 1 { k1 = 1 } } }
150 gv_check("1 REAL-autograd weights BYTE-IDENTICAL across D=1,2,4,8 (worker-count-invariant)" as *u8, k1, ctr)
151
152 var k2: i64 = 0; if aeq(d8, ws, V) == 1 { if aeq(d1, ws, V) == 1 { k2 = 1 } }
153 gv_check("2 DDP == single-process full-batch (ring_allreduce correct over autograd grads)" as *u8, k2, ctr)
154
155 var k3: i64 = 0; if aeq(d1, W0, V) == 0 { k3 = 1 }
156 gv_check("neg-control-3 weights CHANGED from W0 -- a tape that produced no gradient would leave them equal and this tooth is the only thing that would notice" as *u8, k3, ctr)
157
158 let rc: i64 = gv_verdict("NOFLOAT-AUTOGRAD-DDP-GATE" as *u8, ctr, "the REAL no-float autograd trains data-parallel: weights byte-identical across D=1,2,4,8, equal to the single-process full batch, and genuinely moved off W0" as *u8)
159 if rc == 0 {
160 let lfd: i64 = sys_openat_append("knowledge/status/nofloat_autograd_ddp.log" as *u8, 420)
161 if lfd >= 0 {
162 sys_write(lfd, "NOFLOAT-AUTOGRAD-DDP real-tape grads + ring_allreduce: weights worker-count-invariant + == single-proc verdict=GREEN\n" as *u8, 113)
163 sys_close(lfd)
164 }
165 }
166 sys_exit(rc)
167 return rc
168}