nx_intfp_dpo_gradcheck_gate.nx source

1// nx_intfp_dpo_gradcheck_gate.nx -- DPO preference alignment (the RLHF/DPO axis) in Q20 INTEGER, gradchecked, no 2// float. Uses the just-built fp_log (softplus) + fp_exp (sigmoid). DPO math is clean in integer: for the SAME x the 3// logsumexp cancels, so margin = beta*[(l_c - l_r) - (lref_c - lref_r)] (pure logit differences). Loss = 4// softplus(-margin) = log(1+exp(-margin)); dL/dmargin = -sigmoid(-margin). Policy W trainable, reference Wref 5// FROZEN. Gradcheck the policy W. license_tier: ORIGINAL 6import "nx_syscalls.nx" 7 8func w(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 } 9func wn(v: i64) -> i64 { if v==0 { sys_write(1,"0" as *u8,1); return 0 } var m: i64=v; if m<0{sys_write(1,"-" as *u8,1);m=0-m} let t: *u8=sys_mmap(24); var k: i64=0; while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1} let o: *u8=sys_mmap(24); var q: i64=k-1; var i: i64=0; while q>=0{o[i]=t[q];i=i+1;q=q-1} sys_write(1,o,i); return 0 } 10func iabs(v: i64) -> i64 { if v<0 { return 0-v } return v } 11 12const S: i64 = 1048576 13const T: i64 = 3 // preference pairs 14const DM: i64 = 4 15const V: i64 = 5 // vocab 16const BETA: i64 = 1048576 // beta = 1.0 17 18func fp_exp(xq: i64) -> i64 { let y: i64=(xq*1512776)/S; var yi: i64=0; if y>=0 { yi=y/S } else { yi=0-(((0-y)+S-1)/S) } let yf: i64=y-yi*S; var p: i64=10085; p=58197+(p*yf)/S; p=251882+(p*yf)/S; p=726817+(p*yf)/S; p=S+(p*yf)/S; if yi>=0 { if yi>=31 { return 2000000000 } return p*(1<<yi) } let k: i64=0-yi; if k>=31 { return 0 } return p/(1<<k) } 19// natural log via ln(m)=2*atanh((m-1)/(m+1)) (fast: s<=1/3 for m in [1,2)); ln(x)=e*ln2+ln(m). 20func fp_log(xq: i64) -> i64 { if xq<=0 { return 0-2000000000 } var e: i64=0; var t: i64=xq; while t>=2*S { t=t/2; e=e+1 } while t<S { t=t*2; e=e-1 } let s: i64=((t-S)*S)/(t+S); let s2: i64=(s*s)/S; let poly: i64=S+s2/3+((s2*s2)/S)/5; let logm: i64=(2*s*poly)/S; return e*726817+logm } 21func sigmoid(z: i64) -> i64 { let e: i64=fp_exp(0-z); return (S*S)/(S+e) } 22// stable softplus(z) = max(z,0) + log(1+exp(-|z|)) 23func softplus(z: i64) -> i64 { var mx: i64=0; if z>0 { mx=z } let e: i64=fp_exp(0-iabs(z)); return mx + fp_log(S+e) } 24 25// slots: 0x 1W 2Wref 3ch 4rj | 5margin 26func dpo_loss(P: *i64) -> i64 { 27 let x: *i64=P[0] as *i64; let W: *i64=P[1] as *i64; let Wref: *i64=P[2] as *i64; let ch: *i64=P[3] as *i64; let rj: *i64=P[4] as *i64; let mg: *i64=P[5] as *i64 28 var L: i64=0; var t: i64=0 29 while t<T { 30 let c: i64=ch[t]; let r: i64=rj[t] 31 // policy logits l_c, l_r ; ref logits 32 var lc: i64=0; var lr: i64=0; var rc: i64=0; var rr: i64=0; var i: i64=0 33 while i<DM { lc=lc+x[t*DM+i]*W[i*V+c]; lr=lr+x[t*DM+i]*W[i*V+r]; rc=rc+x[t*DM+i]*Wref[i*V+c]; rr=rr+x[t*DM+i]*Wref[i*V+r]; i=i+1 } 34 lc=lc/S; lr=lr/S; rc=rc/S; rr=rr/S 35 let m: i64=(BETA*((lc-lr)-(rc-rr)))/S // margin (Q20) 36 mg[t]=m 37 L=L+softplus(0-m) // loss += softplus(-margin) 38 t=t+1 39 } 40 return L 41} 42// backward: dW[i,j] += dLdm[t]*beta*x[t,i]*(onehot_c - onehot_r); dLdm = -sigmoid(-margin) 43func dpo_bwd(P: *i64, dW: *i64) -> i64 { 44 let x: *i64=P[0] as *i64; let ch: *i64=P[3] as *i64; let rj: *i64=P[4] as *i64; let mg: *i64=P[5] as *i64 45 var z: i64=0; while z<DM*V { dW[z]=0; z=z+1 } 46 var t: i64=0 47 while t<T { 48 let c: i64=ch[t]; let r: i64=rj[t] 49 let dLdm: i64=0-sigmoid(0-mg[t]) // -sigmoid(-margin), Q20 50 let g: i64=(dLdm*BETA)/S // dLdm*beta (Q20) 51 var i: i64=0 52 while i<DM { let gx: i64=(g*x[t*DM+i])/S; dW[i*V+c]=dW[i*V+c]+gx; dW[i*V+r]=dW[i*V+r]-gx; i=i+1 } 53 t=t+1 54 } 55 return 0 56} 57func gcheck(name: *u8, P: *i64, Wt: *i64, dW: *i64, ncell: i64) -> i64 { 58 let DELTA: i64=52428; let TOLP: i64=100 // delta=0.05: softplus signal must clear the fp_log/fp_exp truncation noise (L~1.8M) 59 var maxabs: i64=1; var q: i64=0; while q<ncell { if iabs(dW[q])>maxabs { maxabs=iabs(dW[q]) } q=q+1 } 60 var npass: i64=0; var worst: i64=0; var i: i64=0 61 while i<ncell { 62 let save: i64=Wt[i] 63 Wt[i]=save+DELTA; let Lp: i64=dpo_loss(P) 64 Wt[i]=save-DELTA; let Lm: i64=dpo_loss(P) 65 Wt[i]=save; let dd: i64=dpo_loss(P) 66 let num: i64=((Lp-Lm)*S)/(2*DELTA); let rel: i64=(iabs(num-dW[i])*1000)/maxabs 67 if rel<=TOLP { npass=npass+1 } else { w(" " as *u8); w(name); w(" cell " as *u8); wn(i); w(" ana=" as *u8); wn(dW[i]); w(" num=" as *u8); wn(num); w(" rel=" as *u8); wn(rel); w("\n" as *u8) } 68 if rel>worst { worst=rel } 69 i=i+1 70 } 71 w(" " as *u8); w(name); w(": " as *u8); wn(npass); w("/" as *u8); wn(ncell); w(" (worst=" as *u8); wn(worst); w("permil)\n" as *u8) 72 return npass 73} 74func main() -> i64 { 75 w("=== nx_intfp_dpo_gradcheck: DPO preference alignment (RLHF axis) in Q20 INTEGER, no float ===\n\n" as *u8) 76 let P: *i64=sys_mmap(6*8) as *i64 77 P[0]=sys_mmap(T*DM*8); P[1]=sys_mmap(DM*V*8); P[2]=sys_mmap(DM*V*8); P[3]=sys_mmap(T*8); P[4]=sys_mmap(T*8); P[5]=sys_mmap(T*8) 78 let x: *i64=P[0] as *i64; let W: *i64=P[1] as *i64; let Wref: *i64=P[2] as *i64; let ch: *i64=P[3] as *i64; let rj: *i64=P[4] as *i64 79 var i: i64=0; while i<T*DM { x[i]=((((i*5+2)%11)-5)*S)/10; i=i+1 } 80 i=0; while i<DM*V { W[i]=((((i*7+3)%13)-6)*S)/14; Wref[i]=((((i*7+3)%13)-6)*S)/16; i=i+1 } // policy near ref 81 ch[0]=2; rj[0]=0; ch[1]=4; rj[1]=1; ch[2]=1; rj[2]=3 82 let L0: i64=dpo_loss(P); let dW: *i64=sys_mmap(DM*V*8) as *i64; dpo_bwd(P, dW) 83 w(" DPO loss_q20=" as *u8); wn(L0); w(" (softplus(-margin) over " as *u8); wn(T); w(" preference pairs; policy trainable, ref FROZEN)\n gradcheck:\n" as *u8) 84 let pw: i64=gcheck("W (policy)" as *u8, P, W, dW, DM*V) 85 w("\n DPO gradcheck: " as *u8); wn(pw); w("/" as *u8); wn(DM*V); w(" cells correct\n" as *u8) 86 w("NX-INTFP-DPO verdict=" as *u8) 87 if pw==DM*V { w("GREEN " as *u8); wn(pw); w("/" as *u8); wn(DM*V); w(" -- integer DPO preference-alignment gradient PROVEN (softplus via fp_log, -sigmoid(-m) weight), no float. modelwright RLHF/DPO axis: closing.\n" as *u8) } 88 else { w("RED " as *u8); wn(pw); w("/" as *u8); wn(DM*V); w(" -- DPO Q-scaling bug\n" as *u8) } 89 return 0 90}