nx_nofloat_distill_gate.nx source

1// nx_nofloat_distill_gate.nx -- R3 (cheap quality) from the sovereign-researcher roadmap 2// (knowledge/research/2026-06-23-nofloat-affordable-land-roadmap.md, grounded in nfs_hinton_distill.raw, 3// arXiv 1503.02531): DISTILLATION -- a student trained on a teacher's SOFT output distribution inherits the 4// full distribution ("dark knowledge": relative probabilities of the non-top classes), which one-hot HARD 5// labels throw away. Pure integer Q16; trains student logits W via softmax_rows(W) -> mse(target) (existing 6// gradcheck-verified ops, NO lib change -> zero regression risk to the other gates). 7// T1 the distilled student MATCHES the teacher's soft distribution (low total L1). 8// T2 (the point) distilled L1-to-teacher << hard-label student L1-to-teacher (soft targets > hard labels). 9// expect_exit: 0 Sovereign: nx_nofloat_autograd + nx_syscalls. 10import "nx_nofloat_autograd.nx" 11import "nx_syscalls.nx" 12import "nx_gate_emit_lib.nx" 13import "nx_gate_verdict.nx" 14const Q16: i64 = 65536 15 16 17func g_abs(v: i64) -> i64 { if v<0 { return 0-v } return v } 18 19// train student logits Wp[C*V] so softmax_rows(Wp) matches target[C*V]; returns nothing (Wp updated in place). 20func train_student(tape: *i64, vals: *i64, grads: *i64, st: *i64, Wp: *i64, target: *i64, C: i64, V: i64, gb: *i64, steps: i64) -> i64 { 21 var ep: i64=0 22 while ep < steps { 23 st[0]=0; st[1]=0 24 let nW: i64 = nfa_leaf(tape,vals,st,C,V,Wp,0) 25 let nP: i64 = nfa_softmax_rows(tape,vals,st,nW,0) // causal=0 -> full per-row softmax 26 let nT: i64 = nfa_leaf(tape,vals,st,C,V,target,0) 27 let loss: i64 = nfa_mse(tape,vals,st,nP,nT) 28 nfa_backward(tape,vals,grads,st[0],loss) 29 var i: i64=0; while i<C*V { gb[i]=nfa_grad(tape,grads,nW,i); i=i+1 } 30 nfa_sgd(Wp, gb, C*V, 8192) 31 ep=ep+1 32 } 33 return 0 34} 35// total L1 distance between softmax_rows(Wp) and the teacher T (both [C,V], Q16). Lower = closer to teacher. 36func l1_to_teacher(tape: *i64, vals: *i64, st: *i64, Wp: *i64, T: *i64, C: i64, V: i64) -> i64 { 37 st[0]=0; st[1]=0 38 let nW: i64 = nfa_leaf(tape,vals,st,C,V,Wp,0) 39 let nP: i64 = nfa_softmax_rows(tape,vals,st,nW,0) 40 let off: i64 = tape[7*nP+5] 41 var s: i64=0; var i: i64=0 42 while i<C*V { s = s + g_abs(vals[off+i]-T[i]); i=i+1 } 43 return s 44} 45 46func main() -> i64 { 47 g_puts("nx_nofloat_distill gate (DISTILLATION: soft targets transfer the full distribution vs hard labels)\n" as *u8) 48 let C: i64=4; let V: i64=4 49 let tape: *i64 = sys_mmap(256*7*8) as *i64 50 let vals: *i64 = sys_mmap(4096*8) as *i64 51 let grads: *i64 = sys_mmap(4096*8) as *i64 52 let st: *i64 = sys_mmap(2*8) as *i64 53 let gb: *i64 = sys_mmap(64*8) as *i64 54 55 // teacher T: soft per-context distributions (each row sums to Q16). Row 3 is UNIFORM = pure dark knowledge. 56 let T: *i64 = sys_mmap(C*V*8) as *i64 57 T[0]=45875; T[1]=13107; T[2]=6554; T[3]=0 // 0.70 0.20 0.10 0.00 58 T[4]=6554; T[5]=39322; T[6]=13107; T[7]=6553 // 0.10 0.60 0.20 0.10 59 T[8]=0; T[9]=6554; T[10]=52428; T[11]=6554 // 0.00 0.10 0.80 0.10 60 T[12]=16384;T[13]=16384;T[14]=16384;T[15]=16384 // 0.25 0.25 0.25 0.25 (uniform) 61 // hard labels H = one-hot(argmax row of T) 62 let H: *i64 = sys_mmap(C*V*8) as *i64 63 var z: i64=0; while z<C*V { H[z]=0; z=z+1 } 64 H[0]=Q16; H[4+1]=Q16; H[8+2]=Q16; H[12+0]=Q16 // argmax rows: 0,1,2,0 65 66 // distilled student (soft targets = T) 67 let Wd: *i64 = sys_mmap(C*V*8) as *i64; var i: i64=0; while i<C*V { Wd[i]=((i%5)-2)*4096; i=i+1 } 68 train_student(tape,vals,grads,st,Wd,T,C,V,gb,5000) 69 let ld: i64 = l1_to_teacher(tape,vals,st,Wd,T,C,V) 70 71 // hard-label student (one-hot targets = H), measured against the SAME soft teacher T 72 let Wh: *i64 = sys_mmap(C*V*8) as *i64; i=0; while i<C*V { Wh[i]=((i%5)-2)*4096; i=i+1 } 73 train_student(tape,vals,grads,st,Wh,H,C,V,gb,5000) 74 let lh: i64 = l1_to_teacher(tape,vals,st,Wh,T,C,V) 75 76 g_puts(" [measure] L1(student softmax vs soft teacher T): distilled=" as *u8); g_pn(ld); g_puts(" hard-label=" as *u8); g_pn(lh); g_puts(" (Q16; lower=closer; total mass=" as *u8); g_pn(C*Q16); g_puts(")\n" as *u8) 77 78 var pass: i64=0; var total: i64=0 79 var t1: i64=0; if ld < 26214 { t1=1 } // distilled total L1 < 0.4 (Q16) across all C rows = matches T 80 pass=pass+g_check("T1: distilled student MATCHES the teacher's soft distribution (low L1)" as *u8, t1); total=total+1 81 var t2: i64=0; if ld*2 < lh { t2=1 } // distilled at least 2x closer than hard-label 82 pass=pass+g_check("T2: soft targets >> hard labels -- distilled L1 << hard L1 (transfers the dark knowledge)" as *u8, t2); total=total+1 83 84 var okall: i64=0; if pass==total { okall=1 } 85 if okall==1 { 86 let logf: i64 = sys_openat_append("knowledge/status/nofloat_distill.log" as *u8, 420) 87 if logf >= 0 { let x0: i64=sys_write(logf,"NOFLOATDISTILL soft-vs-hard target distillation measured\n" as *u8,56); sys_close(logf) } 88 } 89 g_puts("---- distill gate: passed " as *u8); g_pn(pass); g_puts(" / " as *u8); g_pn(total); g_puts(" ----\n" as *u8) 90 // MIGRATED onto nx_gate_verdict by nx_gate_dry_apply (D001, minimal form): every check 91 // row above is untouched, so the PASS/FAIL vector cannot change; only the hand-rolled 92 // verdict emission is replaced by the ONE shared base class. Proven by nx_gate_migrate verify. 93 let ctr__dry: *i64 = gv_ctr() 94 ctr__dry[0] = pass 95 ctr__dry[1] = total 96 let rc__dry: i64 = gv_verdict("NOFLOAT-DISTILL-GATE" as *u8, ctr__dry, "distillation: soft targets transfer the teacher's full distribution; R3 cheap-quality win)" as *u8) 97 sys_exit(rc__dry) 98 return rc__dry 99}