code wiki / _hdl_build / nx_intfp_lora_gradcheck_gate.nx
nx_intfp_lora_gradcheck_gate.nx source
↩ module page · 81 lines · 5952 B
1// nx_intfp_lora_gradcheck_gate.nx -- LoRA / parameter-efficient fine-tuning in Q20 INTEGER, gradchecked, no float.
2// modelwright census marks LoRA ABSENT; this closes it. Base weight W is FROZEN; a low-rank adapter dW = A@B
3// (A[DM,r], B[r,DM], rank r << DM) is the only trainable part -> tiny # of trained params. y = x@W + (x@A)@B.
4// Train ONLY A,B. Gradcheck A and B (the adapter). Composes proven integer matmuls. license_tier: ORIGINAL
5import "nx_syscalls.nx"
6
7func w(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
8func wn(v: i64) -> i64 { if v==0 { sys_write(1,"0" as *u8,1); return 0 } var m: i64=v; if m<0{sys_write(1,"-" as *u8,1);m=0-m} let t: *u8=sys_mmap(24); var k: i64=0; while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1} let o: *u8=sys_mmap(24); var q: i64=k-1; var i: i64=0; while q>=0{o[i]=t[q];i=i+1;q=q-1} sys_write(1,o,i); return 0 }
9func iabs(v: i64) -> i64 { if v<0 { return 0-v } return v }
10
11const S: i64 = 1048576 // Q20
12const T: i64 = 3
13const DM: i64 = 6
14const R: i64 = 2 // LoRA rank (<< DM)
15
16// slots: 0x 1W 2A 3B 4Y | 5xw 6h 7yl 8y | 9dA 10dB
17func lora_fwd(P: *i64) -> i64 {
18 let x: *i64=P[0] as *i64; let W: *i64=P[1] as *i64; let A: *i64=P[2] as *i64; let B: *i64=P[3] as *i64; let Y: *i64=P[4] as *i64
19 let xw: *i64=P[5] as *i64; let h: *i64=P[6] as *i64; let yl: *i64=P[7] as *i64; let y: *i64=P[8] as *i64
20 // xw = x@W (frozen base)
21 var t: i64=0; while t<T { var j: i64=0; while j<DM { var acc: i64=0; var i: i64=0; while i<DM { acc=acc+x[t*DM+i]*W[i*DM+j]; i=i+1 } xw[t*DM+j]=acc/S; j=j+1 } t=t+1 }
22 // h = x@A [T,R]
23 t=0; while t<T { var k: i64=0; while k<R { var acc: i64=0; var i: i64=0; while i<DM { acc=acc+x[t*DM+i]*A[i*R+k]; i=i+1 } h[t*R+k]=acc/S; k=k+1 } t=t+1 }
24 // yl = h@B [T,DM] ; y = xw + yl ; L = sum y^2
25 var Lq: i64=0; t=0
26 while t<T { var j: i64=0; while j<DM { var acc: i64=0; var k: i64=0; while k<R { acc=acc+h[t*R+k]*B[k*DM+j]; k=k+1 } let yv: i64=xw[t*DM+j]+acc/S; y[t*DM+j]=yv; yl[t*DM+j]=acc/S; Lq=Lq+yv*yv; j=j+1 } t=t+1 }
27 return Lq
28}
29func lora_bwd(P: *i64) -> i64 {
30 let x: *i64=P[0] as *i64; let B: *i64=P[3] as *i64; let h: *i64=P[6] as *i64; let y: *i64=P[8] as *i64
31 let dA: *i64=P[9] as *i64; let dB: *i64=P[10] as *i64
32 let dy: *i64=sys_mmap(T*DM*8) as *i64; let dh: *i64=sys_mmap(T*R*8) as *i64
33 var t: i64=0; while t<T { var j: i64=0; while j<DM { dy[t*DM+j]=2*y[t*DM+j]; j=j+1 } t=t+1 }
34 // dB[k,j] = sum_t h[t,k] dy[t,j]
35 var k: i64=0; while k<R { var j: i64=0; while j<DM { var acc: i64=0; t=0; while t<T { acc=acc+(h[t*R+k]*dy[t*DM+j])/S; t=t+1 } dB[k*DM+j]=acc; j=j+1 } k=k+1 }
36 // dh[t,k] = sum_j dy[t,j] B[k,j]
37 t=0; while t<T { k=0; while k<R { var acc: i64=0; var j: i64=0; while j<DM { acc=acc+(dy[t*DM+j]*B[k*DM+j])/S; j=j+1 } dh[t*R+k]=acc; k=k+1 } t=t+1 }
38 // dA[i,k] = sum_t x[t,i] dh[t,k]
39 var i: i64=0; while i<DM { k=0; while k<R { var acc: i64=0; t=0; while t<T { acc=acc+(x[t*DM+i]*dh[t*R+k])/S; t=t+1 } dA[i*R+k]=acc; k=k+1 } i=i+1 }
40 return 0
41}
42func gcheck(name: *u8, P: *i64, Wt: *i64, dW: *i64, ncell: i64) -> i64 {
43 let DELTA: i64=2097; let TOLP: i64=70
44 var maxabs: i64=1; var q: i64=0; while q<ncell { if iabs(dW[q])>maxabs { maxabs=iabs(dW[q]) } q=q+1 }
45 var npass: i64=0; var worst: i64=0; var i: i64=0
46 while i<ncell {
47 let save: i64=Wt[i]
48 Wt[i]=save+DELTA; let Lp: i64=lora_fwd(P)
49 Wt[i]=save-DELTA; let Lm: i64=lora_fwd(P)
50 Wt[i]=save; let dd: i64=lora_fwd(P)
51 let num: i64=(Lp-Lm)/(2*DELTA); let rel: i64=(iabs(num-dW[i])*1000)/maxabs
52 if rel<=TOLP { npass=npass+1 } else { w(" " as *u8); w(name); w(" cell " as *u8); wn(i); w(" ana=" as *u8); wn(dW[i]); w(" num=" as *u8); wn(num); w(" rel=" as *u8); wn(rel); w("\n" as *u8) }
53 if rel>worst { worst=rel }
54 i=i+1
55 }
56 w(" " as *u8); w(name); w(": " as *u8); wn(npass); w("/" as *u8); wn(ncell); w(" (worst=" as *u8); wn(worst); w("permil of max|grad|=" as *u8); wn(maxabs); w(")\n" as *u8)
57 return npass
58}
59func main() -> i64 {
60 w("=== nx_intfp_lora_gradcheck: LoRA parameter-efficient fine-tuning (rank " as *u8); wn(R); w(") in Q20 INTEGER, no float ===\n\n" as *u8)
61 let P: *i64=sys_mmap(11*8) as *i64
62 P[0]=sys_mmap(T*DM*8); P[1]=sys_mmap(DM*DM*8); P[2]=sys_mmap(DM*R*8); P[3]=sys_mmap(R*DM*8); P[4]=sys_mmap(T*DM*8)
63 P[5]=sys_mmap(T*DM*8); P[6]=sys_mmap(T*R*8); P[7]=sys_mmap(T*DM*8); P[8]=sys_mmap(T*DM*8)
64 P[9]=sys_mmap(DM*R*8); P[10]=sys_mmap(R*DM*8)
65 let x: *i64=P[0] as *i64; let W: *i64=P[1] as *i64; let A: *i64=P[2] as *i64; let B: *i64=P[3] as *i64
66 var i: i64=0; while i<T*DM { x[i]=((((i*5+2)%11)-5)*S)/10; i=i+1 }
67 i=0; while i<DM*DM { W[i]=((((i*7+3)%13)-6)*S)/12; i=i+1 } // FROZEN base
68 i=0; while i<DM*R { A[i]=((((i*7+1)%11)-5)*S)/16; i=i+1 } // LoRA A (trainable)
69 i=0; while i<R*DM { B[i]=((((i*3+2)%11)-5)*S)/16; i=i+1 } // LoRA B (trainable)
70 let L0: i64=lora_fwd(P); lora_bwd(P)
71 let trained: i64=DM*R+R*DM; let total: i64=DM*DM+DM*R+R*DM
72 w(" params: base W frozen (" as *u8); wn(DM*DM); w(") + LoRA A,B trainable (" as *u8); wn(trained); w(") = " as *u8); wn((trained*1000)/total); w("permil of full trained (PEFT) L_q32=" as *u8); wn(L0); w("\n gradcheck:\n" as *u8)
73 let pa: i64=gcheck("A (lora)" as *u8, P, A, P[9] as *i64, DM*R)
74 let pb: i64=gcheck("B (lora)" as *u8, P, B, P[10] as *i64, R*DM)
75 let tot: i64=pa+pb; let want: i64=DM*R+R*DM
76 w("\n LoRA gradcheck: " as *u8); wn(tot); w("/" as *u8); wn(want); w(" cells correct\n" as *u8)
77 w("NX-INTFP-LORA verdict=" as *u8)
78 if tot==want { w("GREEN " as *u8); wn(tot); w("/" as *u8); wn(want); w(" -- integer LoRA/PEFT PROVEN (frozen base + low-rank adapter grads), no float. modelwright LoRA axis: ABSENT->present.\n" as *u8) }
79 else { w("RED " as *u8); wn(tot); w("/" as *u8); wn(want); w(" -- adapter Q-scaling bug\n" as *u8) }
80 return 0
81}