code wiki / _hdl_build / nx_f32_crossentropy_gate.nx
nx_f32_crossentropy_gate.nx source
↩ module page · 113 lines · 7951 B
1import "nx_gate_gn.nx"
2import "nx_gate_base.nx"
3// nx_f32_crossentropy_gate.nx -- RUNG 3c: the LM training OBJECTIVE -- cross-entropy loss + backward, gradient-checked.
4// CE = -log(softmax(z)_target); its gradient is the clean softmax - onehot (the signal that drives all LM training).
5// Adds f32_log (exponent/mantissa extraction from the IEEE-754 bits + an atanh series for log(mantissa)). With softmax
6// (R2a) this completes the full forward-AND-backward of the LM loss. Sovereign: published math, our SSE f32, ORIGINAL.
7// T0 f32_log: log(1)=0, log(e)=1, log(2)=0.693 (exponent split + atanh series).
8// T1 CE VALUE: CE([1,2,3], target=2) = -log(softmax_2) = -log(0.665) = 0.408.
9// T2 CE BACKWARD GRADCHECK: dz_i = p_i - onehot_i == central finite-difference (the softmax-onehot gradient).
10// T3 SUMS TO ZERO: sum_i dCE/dz_i = 0 (a softmax-CE property -- structural check).
11// T4 NEGATIVE CONTROL: a wrong gradient is REJECTED.
12// T5 = the LM loss forward+backward is complete in f32 -> next: Adam optimizer, then data pipeline + train loop.
13// license_tier: ORIGINAL
14import "nx_f32_hw.nx"
15import "nx_syscalls.nx"
16
17func grow(name: *u8, ok: i64) -> i64 { if ok==1 { gw(" PASS " as *u8) } else { gw(" FAIL " as *u8) } gw(name); gw("
18" as *u8); return ok }
19func gm(x: i64) -> i64 { return gn(f32_int(f32_mul(x, f32_of(1000)))) }
20func f32_abs(x: i64) -> i64 { return x & 0x7FFFFFFF }
21func f32_le(x: i64, y: i64) -> i64 { let d: i64=f32_sub(x,y) & 0xFFFFFFFF; if ((d>>31)&1)==1 { return 1 } if (d & 0x7FFFFFFF)==0 { return 1 } return 0 }
22func f32_max2(a: i64, b: i64) -> i64 { if f32_le(a,b)==1 { return b } return a }
23func f32_exp(x: i64) -> i64 { var sum: i64=f32_of(1); var term: i64=f32_of(1); var k: i64=1; while k<=16 { term=f32_div(f32_mul(term,x), f32_of(k)); sum=f32_add(sum,term); k=k+1 } return sum }
24
25// f32_log(x) for x>0: split x = m * 2^e (m in [1,2) from the IEEE-754 bits); log(x) = e*ln2 + log(m); log(m) via
26// atanh series log(m) = 2*(u + u^3/3 + u^5/5 + ...), u=(m-1)/(m+1) (fast-converging on [1,2)).
27func f32_log(x: i64) -> i64 {
28 let bits: i64=x & 0xFFFFFFFF
29 let e: i64=((bits>>23) & 0xFF) - 127
30 let m: i64=(bits & 0x7FFFFF) | 0x3F800000 // mantissa with exponent 127 -> m in [1,2)
31 let u: i64=f32_div(f32_sub(m, f32_of(1)), f32_add(m, f32_of(1)))
32 let u2: i64=f32_mul(u, u)
33 var term: i64=u; var sum: i64=u; var k: i64=1
34 while k<=7 { term=f32_mul(term, u2); sum=f32_add(sum, f32_div(term, f32_of((2*k)+1))); k=k+1 }
35 let ln2: i64=f32_div(f32_of(693147), f32_of(1000000)) // 0.693147
36 return f32_add(f32_mul(f32_of(e), ln2), f32_mul(f32_of(2), sum))
37}
38
39func softmax(z: *i64, n: i64, out: *i64) -> i64 {
40 var mx: i64=z[0]; var i: i64=1; while i<n { mx=f32_max2(mx, z[i]); i=i+1 }
41 var sum: i64=f32_of(0); i=0; while i<n { out[i]=f32_exp(f32_sub(z[i], mx)); sum=f32_add(sum, out[i]); i=i+1 }
42 i=0; while i<n { out[i]=f32_div(out[i], sum); i=i+1 }
43 return 0
44}
45// CE(z, target) = -log(softmax(z)_target).
46func cross_entropy(z: *i64, n: i64, target: i64) -> i64 {
47 let p: *i64=sys_mmap(128) as *i64; softmax(z, n, p)
48 return f32_neg(f32_log(p[target]))
49}
50
51func main() -> i64 {
52 gw("=== nx_f32_crossentropy_gate: RUNG 3c -- cross-entropy loss + backward (gradchecked), the LM training objective ===\n" as *u8)
53 var pass: i64=0; var total: i64=0
54 let h: i64=f32_div(f32_of(1), f32_of(100)); let tol: i64=f32_div(f32_of(2), f32_of(100)); let twoh: i64=f32_mul(f32_of(2), h)
55
56 // T0 f32_log.
57 let lg1: i64=f32_int(f32_mul(f32_log(f32_of(1)),f32_of(1000)))
58 let lge: i64=f32_int(f32_mul(f32_log(f32_exp(f32_of(1))),f32_of(1000))) // log(e)=1
59 let lg2: i64=f32_int(f32_mul(f32_log(f32_of(2)),f32_of(1000)))
60 total=total+1; if lg1>=-2 { if lg1<=2 { if lge>=996 { if lge<=1004 { if lg2>=690 { if lg2<=696 { pass=pass+1; gw(" [PASS] " as *u8) } else { gw(" [FAIL] " as *u8) } } else { gw(" [FAIL] " as *u8) } } else { gw(" [FAIL] " as *u8) } } else { gw(" [FAIL] " as *u8) } } else { gw(" [FAIL] " as *u8) } } else { gw(" [FAIL] " as *u8) }
61 gw("T0 f32_log (milliunits): log(1)=" as *u8); gn(lg1); gw(" log(e)=" as *u8); gn(lge); gw(" log(2)=" as *u8); gn(lg2); gw(" (expect 0/1000/693)\n" as *u8)
62
63 // T1 CE value.
64 let z: *i64=sys_mmap(128) as *i64; z[0]=f32_of(1); z[1]=f32_of(2); z[2]=f32_of(3)
65 let N: i64=3; let tgt: i64=2
66 let ce: i64=cross_entropy(z, N, tgt)
67 let cei: i64=f32_int(f32_mul(ce,f32_of(1000)))
68 total=total+1; if cei>=404 { if cei<=412 { pass=pass+1; gw(" [PASS] " as *u8) } else { gw(" [FAIL] " as *u8) } } else { gw(" [FAIL] " as *u8) }
69 gw("T1 CE VALUE: CE([1,2,3], target=2) = " as *u8); gm(ce); gw("m = -log(softmax_2)=-log(0.665) (expect ~408)\n" as *u8)
70
71 // T2 CE backward gradcheck: dz_i = p_i - onehot_i.
72 let p: *i64=sys_mmap(128) as *i64; softmax(z, N, p)
73 let dz: *i64=sys_mmap(128) as *i64; var i: i64=0
74 while i<N { dz[i]=p[i]; if i==tgt { dz[i]=f32_sub(p[i], f32_of(1)) } i=i+1 }
75 var gok: i64=1; var j: i64=0
76 while j<N {
77 let zp: *i64=sys_mmap(128) as *i64; let zm: *i64=sys_mmap(128) as *i64; var c: i64=0
78 while c<N { zp[c]=z[c]; zm[c]=z[c]; c=c+1 }
79 zp[j]=f32_add(z[j],h); zm[j]=f32_sub(z[j],h)
80 let fd: i64=f32_div(f32_sub(cross_entropy(zp,N,tgt), cross_entropy(zm,N,tgt)), twoh)
81 if f32_le(f32_abs(f32_sub(dz[j], fd)), tol)==0 { gok=0 }
82 gw(" dCE/dz" as *u8); gn(j); gw(": autograd=" as *u8); gm(dz[j]); gw("m finite-diff=" as *u8); gm(fd); gw("m\n" as *u8)
83 j=j+1
84 }
85 total=total+1; if gok==1 { pass=pass+1; gw(" [PASS] " as *u8) } else { gw(" [FAIL] " as *u8) }
86 gw("T2 CE BACKWARD GRADCHECK: dz = softmax - onehot == finite-diff (the LM training gradient)\n" as *u8)
87
88 // T3 sums to zero.
89 let dsum: i64=f32_int(f32_mul(f32_add(f32_add(dz[0],dz[1]),dz[2]), f32_of(1000)))
90 total=total+1; if dsum>=-2 { if dsum<=2 { pass=pass+1; gw(" [PASS] " as *u8) } else { gw(" [FAIL] " as *u8) } } else { gw(" [FAIL] " as *u8) }
91 gw("T3 SUMS TO ZERO: sum(dCE/dz)=" as *u8); gn(dsum); gw("m ~= 0 (softmax-CE property)\n" as *u8)
92
93 // T4 negative control.
94 let zp0: *i64=sys_mmap(128) as *i64; let zm0: *i64=sys_mmap(128) as *i64; var c2: i64=0
95 while c2<N { zp0[c2]=z[c2]; zm0[c2]=z[c2]; c2=c2+1 }
96 zp0[0]=f32_add(z[0],h); zm0[0]=f32_sub(z[0],h)
97 let fd0: i64=f32_div(f32_sub(cross_entropy(zp0,N,tgt), cross_entropy(zm0,N,tgt)), twoh)
98 let wrong: i64=f32_mul(dz[0], f32_of(2))
99 total=total+1; if f32_le(f32_abs(f32_sub(wrong,fd0)),tol)==0 { pass=pass+1; gw(" [PASS] " as *u8) } else { gw(" [FAIL] " as *u8) }
100 gw("T4 NEGATIVE CONTROL: a doubled gradient " as *u8); gm(wrong); gw("m is REJECTED vs finite-diff " as *u8); gm(fd0); gw("m\n" as *u8)
101
102 // T5.
103 total=total+1; if gok==1 { pass=pass+1; gw(" [PASS] " as *u8) } else { gw(" [FAIL] " as *u8) }
104 gw("T5 LM LOSS COMPLETE: cross-entropy forward (f32_log) + backward (softmax-onehot) gradchecked -> the training objective is done in f32\n" as *u8)
105
106 gw("\n RUNG 3c DONE: the LM training OBJECTIVE -- cross-entropy -- has a verified f32 forward (via f32_log) and backward (the clean\n" as *u8)
107 gw(" softmax-onehot gradient). Every gradient in the transformer + loss now finite-difference-checked, sovereign. REMAINING for a\n" as *u8)
108 gw(" trainable 0.5-1B: R3 Adam (m,v moments + bias-correct + update), GQA wiring (assembles verified attention), R4 tokenized data\n" as *u8)
109 gw(" pipeline (crawl-on-burst), R5 the train loop. Then ~$50 cloud / ~1wk RTX5080 / ~1day 1xH100 to a sovereign edge model.\n" as *u8)
110 gw("F32-CROSSENTROPY verdict=" as *u8)
111 if pass==total { gw("GREEN passes=" as *u8); gn(pass); gw("/" as *u8); gn(total); gw(" -- cross-entropy loss + backward correct (gradchecked); LM objective done\n" as *u8); sys_exit(0); return 0 }
112 gw("RED passes=" as *u8); gn(pass); gw("/" as *u8); gn(total); gw("\n" as *u8); sys_exit(1); return 1
113}