nx_nofloat_blockfloat_layer_gate.nx source
↩ module page · 147 lines · 7526 B
1// nx_nofloat_blockfloat_layer_gate.nx -- BLOCK-FLOAT weights through a real LINEAR LAYER (W.x + bias -> ReLU),
2// the rung after the bare GEMM: a genuine weight MATRIX (OUT x IN, mixed-magnitude rows) quantized block-float
3// per-row-per-block, run through bias + activation. Proves the dynamic-range win + determinism survive a real
4// layer (the core of FFN + attention projections).
5// 1 block-float layer (computed on mantissas) == dequant-then-layer, BYTE-EXACT
6// 2 EXCEED (MEASURED): block-float layer error vs full-precision < per-tensor INT8 layer error
7// 3 DETERMINISTIC: the block-float layer run twice == bit-identical
8// 4 per-tensor scaling ZEROED a row's small block; block-float kept it (the super-weight fix, in a layer)
9// expect_exit: 0 license_tier: ORIGINAL
10import "nx_syscalls.nx"
11import "nx_gate_verdict.nx"
12
13func bf_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
14func bf_putn(v: i64) -> i64 {
15 let b: *u8 = sys_mmap(28); var m: i64 = v
16 if m < 0 { m = 0 - m; sys_write(1, "-" as *u8, 1) }
17 let t: *u8 = sys_mmap(28); var k: i64 = 0
18 if m == 0 { t[0] = 48 as u8; k = 1 }
19 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
20 var i: i64 = 0
21 while i < k { b[i] = t[k - 1 - i]; i = i + 1 }
22 sys_write(1, b, k); return 0
23}
24func bf_chk(name: *u8, ok: i64) -> i64 {
25 if ok == 1 { bf_puts(" PASS " as *u8); bf_puts(name); bf_puts("\n" as *u8); return 1 }
26 bf_puts(" FAIL " as *u8); bf_puts(name); bf_puts("\n" as *u8); return 0
27}
28func bf_bitlen(x: i64) -> i64 { var b: i64 = 0; var m: i64 = x; while m > 0 { m = m >> 1; b = b + 1 } return b }
29func bf_absdiff(a: i64, b: i64) -> i64 { if a >= b { return a - b } return b - a }
30func bf_relu(x: i64) -> i64 { if x < 0 { return 0 } return x }
31func bf_scale(W: *i64, off: i64, B: i64, M: i64) -> i64 {
32 var amax: i64 = 0; var i: i64 = 0
33 while i < B { if W[off + i] > amax { amax = W[off + i] } i = i + 1 }
34 var e: i64 = bf_bitlen(amax) - M
35 if e < 0 { e = 0 }
36 return e
37}
38
39// full-precision linear layer: y[k] = relu( sum_i W[k][i]*x[i] + bias[k] ).
40func fp_layer(W: *i64, x: *i64, bias: *i64, OUT: i64, IN: i64, out: *i64) -> i64 {
41 var k: i64 = 0
42 while k < OUT {
43 var z: i64 = 0; var i: i64 = 0
44 while i < IN { z = z + W[k * IN + i] * x[i]; i = i + 1 }
45 out[k] = bf_relu(z + bias[k]); k = k + 1
46 }
47 return 0
48}
49
50// block-float linear layer: per row, block-float matmul on mantissas + per-block scales, + bias, ReLU.
51func bf_layer(q_W: *i64, e_bf: *i64, x: *i64, bias: *i64, OUT: i64, IN: i64, B: i64, out: *i64) -> i64 {
52 let NB: i64 = IN / B
53 var k: i64 = 0
54 while k < OUT {
55 var z: i64 = 0
56 var blk: i64 = 0
57 while blk < NB {
58 let off: i64 = blk * B
59 var bsum: i64 = 0; var i: i64 = 0
60 while i < B { bsum = bsum + x[off + i] * q_W[k * IN + off + i]; i = i + 1 }
61 z = z + bsum * (1 << e_bf[k * NB + blk])
62 blk = blk + 1
63 }
64 out[k] = bf_relu(z + bias[k]); k = k + 1
65 }
66 return 0
67}
68
69func main() -> i64 {
70 let OUT: i64 = 2; let IN: i64 = 8; let B: i64 = 4; let MB: i64 = 4
71 let NB: i64 = IN / B
72 bf_puts("=== BLOCK-FLOAT weights through a real LINEAR LAYER (W.x + bias -> ReLU) vs per-tensor INT8 ===\n" as *u8)
73
74 let W: *i64 = sys_mmap(8 * OUT * IN) as *i64
75 // row 0: small block + large block ; row 1: large block + small block (mixed-magnitude matrix)
76 W[0]=3; W[1]=5; W[2]=7; W[3]=9; W[4]=100; W[5]=200; W[6]=150; W[7]=120
77 W[8]=80; W[9]=160; W[10]=120; W[11]=100; W[12]=2; W[13]=4; W[14]=6; W[15]=8
78 let x: *i64 = sys_mmap(8 * IN) as *i64
79 x[0]=2; x[1]=1; x[2]=3; x[3]=1; x[4]=1; x[5]=1; x[6]=1; x[7]=1
80 let bias: *i64 = sys_mmap(8 * OUT) as *i64
81 bias[0]=10; bias[1]=20
82
83 // ---- quantize W: block-float (per-row-per-block) ----
84 let q_W: *i64 = sys_mmap(8 * OUT * IN) as *i64
85 let W_bf: *i64 = sys_mmap(8 * OUT * IN) as *i64
86 let e_bf: *i64 = sys_mmap(8 * OUT * NB) as *i64
87 var k: i64 = 0
88 while k < OUT {
89 var blk: i64 = 0
90 while blk < NB {
91 let off: i64 = k * IN + blk * B
92 let e: i64 = bf_scale(W, off, B, MB)
93 e_bf[k * NB + blk] = e
94 var i: i64 = 0
95 while i < B { q_W[off + i] = W[off + i] >> e; W_bf[off + i] = q_W[off + i] << e; i = i + 1 }
96 blk = blk + 1
97 }
98 k = k + 1
99 }
100 // ---- per-tensor: one global scale over the whole matrix ----
101 let W_pt: *i64 = sys_mmap(8 * OUT * IN) as *i64
102 let eg: i64 = bf_scale(W, 0, OUT * IN, MB)
103 var g: i64 = 0
104 while g < OUT * IN { W_pt[g] = (W[g] >> eg) << eg; g = g + 1 }
105
106 // ---- run the layer four ways ----
107 let y_ref: *i64 = sys_mmap(8 * OUT) as *i64; fp_layer(W, x, bias, OUT, IN, y_ref)
108 let y_bf: *i64 = sys_mmap(8 * OUT) as *i64; bf_layer(q_W, e_bf, x, bias, OUT, IN, B, y_bf)
109 let y_deq: *i64 = sys_mmap(8 * OUT) as *i64; fp_layer(W_bf, x, bias, OUT, IN, y_deq)
110 let y_pt: *i64 = sys_mmap(8 * OUT) as *i64; fp_layer(W_pt, x, bias, OUT, IN, y_pt)
111
112 var err_bf: i64 = 0; var err_pt: i64 = 0; var t1: i64 = 1; var o: i64 = 0
113 while o < OUT {
114 err_bf = err_bf + bf_absdiff(y_bf[o], y_ref[o])
115 err_pt = err_pt + bf_absdiff(y_pt[o], y_ref[o])
116 if y_bf[o] != y_deq[o] { t1 = 0 }
117 o = o + 1
118 }
119 bf_puts(" y_ref=[" as *u8); bf_putn(y_ref[0]); bf_puts(" " as *u8); bf_putn(y_ref[1]); bf_puts("] y_bf=[" as *u8); bf_putn(y_bf[0]); bf_puts(" " as *u8); bf_putn(y_bf[1]); bf_puts("] y_pt=[" as *u8); bf_putn(y_pt[0]); bf_puts(" " as *u8); bf_putn(y_pt[1]); bf_puts("]\n" as *u8)
120 bf_puts(" layer error: BLOCK-FLOAT=" as *u8); bf_putn(err_bf); bf_puts(" PER-TENSOR=" as *u8); bf_putn(err_pt); bf_puts("\n" as *u8)
121
122 let y_bf2: *i64 = sys_mmap(8 * OUT) as *i64; bf_layer(q_W, e_bf, x, bias, OUT, IN, B, y_bf2)
123 var det_ok: i64 = 1; var d: i64 = 0
124 while d < OUT { if y_bf2[d] != y_bf[d] { det_ok = 0 } d = d + 1 }
125
126 // row-0 small block (cols 0..3): block-float kept it, per-tensor zeroed it
127 var t4: i64 = 1; var s: i64 = 0
128 while s < B { if W_bf[s] != W[s] { t4 = 0 } if W_pt[s] != 0 { t4 = 0 } s = s + 1 }
129
130 var pass: i64 = 0; var total: i64 = 0
131 total = total + 1; pass = pass + bf_chk("T1 block-float layer == dequant-then-layer (byte-exact)" as *u8, t1)
132 var t2: i64 = 0; if err_bf < err_pt { t2 = 1 }
133 total = total + 1; pass = pass + bf_chk("T2 EXCEED: block-float layer error < per-tensor (dynamic-range win)" as *u8, t2)
134 total = total + 1; pass = pass + bf_chk("T3 DETERMINISTIC: block-float layer twice == bit-identical" as *u8, det_ok)
135 total = total + 1; pass = pass + bf_chk("T4 per-tensor ZEROED a row's small block; block-float kept it" as *u8, t4)
136
137 bf_puts("NX-NOFLOAT-BLOCKFLOAT-LAYER-GATE " as *u8); bf_putn(pass); bf_puts(" / " as *u8); bf_putn(total)
138 // MIGRATED onto nx_gate_verdict by nx_gate_dry_apply (D001, minimal form): every check
139 // row above is untouched, so the PASS/FAIL vector cannot change; only the hand-rolled
140 // verdict emission is replaced by the ONE shared base class. Proven by nx_gate_migrate verify.
141 let ctr__dry: *i64 = gv_ctr()
142 ctr__dry[0] = pass
143 ctr__dry[1] = total
144 let rc__dry: i64 = gv_verdict("NOFLOAT-BLOCKFLOAT-LAYER-GATE" as *u8, ctr__dry, "block-float through a real linear layer: FP8 range + bit-exact determinism, on mantissas)" as *u8)
145 sys_exit(rc__dry)
146 return rc__dry
147}