nx_nofloat_distill_gate.nx source

1// nx_nofloat_distill_gate.nx -- R3 (cheap quality) from the sovereign-researcher roadmap 2// (knowledge/research/2026-06-23-nofloat-affordable-land-roadmap.md, grounded in nfs_hinton_distill.raw, 3// arXiv 1503.02531): DISTILLATION -- a student trained on a teacher's SOFT output distribution inherits the 4// full distribution ("dark knowledge": relative probabilities of the non-top classes), which one-hot HARD 5// labels throw away. Pure integer Q16; trains student logits W via softmax_rows(W) -> mse(target) (existing 6// gradcheck-verified ops, NO lib change -> zero regression risk to the other gates). 7// T1 the distilled student MATCHES the teacher's soft distribution (low total L1). 8// T2 (the point) distilled L1-to-teacher << hard-label student L1-to-teacher (soft targets > hard labels). 9// expect_exit: 0 Sovereign: nx_nofloat_autograd + nx_syscalls. 10import "nx_nofloat_autograd.nx" 11import "nx_syscalls.nx" 12import "nx_gate_emit_lib.nx" 13const Q16: i64 = 65536 14 15 16func g_abs(v: i64) -> i64 { if v<0 { return 0-v } return v } 17 18// train student logits Wp[C*V] so softmax_rows(Wp) matches target[C*V]; returns nothing (Wp updated in place). 19func train_student(tape: *i64, vals: *i64, grads: *i64, st: *i64, Wp: *i64, target: *i64, C: i64, V: i64, gb: *i64, steps: i64) -> i64 { 20 var ep: i64=0 21 while ep < steps { 22 st[0]=0; st[1]=0 23 let nW: i64 = nfa_leaf(tape,vals,st,C,V,Wp,0) 24 let nP: i64 = nfa_softmax_rows(tape,vals,st,nW,0) // causal=0 -> full per-row softmax 25 let nT: i64 = nfa_leaf(tape,vals,st,C,V,target,0) 26 let loss: i64 = nfa_mse(tape,vals,st,nP,nT) 27 nfa_backward(tape,vals,grads,st[0],loss) 28 var i: i64=0; while i<C*V { gb[i]=nfa_grad(tape,grads,nW,i); i=i+1 } 29 nfa_sgd(Wp, gb, C*V, 8192) 30 ep=ep+1 31 } 32 return 0 33} 34// total L1 distance between softmax_rows(Wp) and the teacher T (both [C,V], Q16). Lower = closer to teacher. 35func l1_to_teacher(tape: *i64, vals: *i64, st: *i64, Wp: *i64, T: *i64, C: i64, V: i64) -> i64 { 36 st[0]=0; st[1]=0 37 let nW: i64 = nfa_leaf(tape,vals,st,C,V,Wp,0) 38 let nP: i64 = nfa_softmax_rows(tape,vals,st,nW,0) 39 let off: i64 = tape[7*nP+5] 40 var s: i64=0; var i: i64=0 41 while i<C*V { s = s + g_abs(vals[off+i]-T[i]); i=i+1 } 42 return s 43} 44 45func main() -> i64 { 46 g_puts("nx_nofloat_distill gate (DISTILLATION: soft targets transfer the full distribution vs hard labels)\n" as *u8) 47 let C: i64=4; let V: i64=4 48 let tape: *i64 = sys_mmap(256*7*8) as *i64 49 let vals: *i64 = sys_mmap(4096*8) as *i64 50 let grads: *i64 = sys_mmap(4096*8) as *i64 51 let st: *i64 = sys_mmap(2*8) as *i64 52 let gb: *i64 = sys_mmap(64*8) as *i64 53 54 // teacher T: soft per-context distributions (each row sums to Q16). Row 3 is UNIFORM = pure dark knowledge. 55 let T: *i64 = sys_mmap(C*V*8) as *i64 56 T[0]=45875; T[1]=13107; T[2]=6554; T[3]=0 // 0.70 0.20 0.10 0.00 57 T[4]=6554; T[5]=39322; T[6]=13107; T[7]=6553 // 0.10 0.60 0.20 0.10 58 T[8]=0; T[9]=6554; T[10]=52428; T[11]=6554 // 0.00 0.10 0.80 0.10 59 T[12]=16384;T[13]=16384;T[14]=16384;T[15]=16384 // 0.25 0.25 0.25 0.25 (uniform) 60 // hard labels H = one-hot(argmax row of T) 61 let H: *i64 = sys_mmap(C*V*8) as *i64 62 var z: i64=0; while z<C*V { H[z]=0; z=z+1 } 63 H[0]=Q16; H[4+1]=Q16; H[8+2]=Q16; H[12+0]=Q16 // argmax rows: 0,1,2,0 64 65 // distilled student (soft targets = T) 66 let Wd: *i64 = sys_mmap(C*V*8) as *i64; var i: i64=0; while i<C*V { Wd[i]=((i%5)-2)*4096; i=i+1 } 67 train_student(tape,vals,grads,st,Wd,T,C,V,gb,5000) 68 let ld: i64 = l1_to_teacher(tape,vals,st,Wd,T,C,V) 69 70 // hard-label student (one-hot targets = H), measured against the SAME soft teacher T 71 let Wh: *i64 = sys_mmap(C*V*8) as *i64; i=0; while i<C*V { Wh[i]=((i%5)-2)*4096; i=i+1 } 72 train_student(tape,vals,grads,st,Wh,H,C,V,gb,5000) 73 let lh: i64 = l1_to_teacher(tape,vals,st,Wh,T,C,V) 74 75 g_puts(" [measure] L1(student softmax vs soft teacher T): distilled=" as *u8); g_pn(ld); g_puts(" hard-label=" as *u8); g_pn(lh); g_puts(" (Q16; lower=closer; total mass=" as *u8); g_pn(C*Q16); g_puts(")\n" as *u8) 76 77 var pass: i64=0; var total: i64=0 78 var t1: i64=0; if ld < 26214 { t1=1 } // distilled total L1 < 0.4 (Q16) across all C rows = matches T 79 pass=pass+g_check("T1: distilled student MATCHES the teacher's soft distribution (low L1)" as *u8, t1); total=total+1 80 var t2: i64=0; if ld*2 < lh { t2=1 } // distilled at least 2x closer than hard-label 81 pass=pass+g_check("T2: soft targets >> hard labels -- distilled L1 << hard L1 (transfers the dark knowledge)" as *u8, t2); total=total+1 82 83 var okall: i64=0; if pass==total { okall=1 } 84 if okall==1 { 85 let logf: i64 = sys_openat_append("knowledge/status/nofloat_distill.log" as *u8, 420) 86 if logf >= 0 { let x0: i64=sys_write(logf,"NOFLOATDISTILL soft-vs-hard target distillation measured\n" as *u8,56); sys_close(logf) } 87 } 88 g_puts("---- distill gate: passed " as *u8); g_pn(pass); g_puts(" / " as *u8); g_pn(total); g_puts(" ----\n" as *u8) 89 if okall==1 { g_puts("verdict=GREEN (distillation: soft targets transfer the teacher's full distribution; R3 cheap-quality win)\n" as *u8); sys_exit(0); return 0 } 90 g_puts("verdict=RED\n" as *u8); sys_exit(1); return 1 91}