nx_gsplat_photofit_gate.nx source

1// nx_gsplat_photofit_gate.nx -- ★FIT THE SPLAT FACE TO THE PHOTO (the optimizer on the real goal). TARGET = the 2// reference photo mapped onto our geometry, rendered SHARP by the SDF projection (nx_sdfrender). The splat face 3// (Gaussians on the mesh) starts FLAT-SKIN-coloured (unlearned) and the differentiable-splatting optimizer fits 4// its colours to the photo target -> the splat LEARNS the face appearance (deconvolving the splat-overlap blur). 5// Same camera (yaw/camz/FOCAL 586) so SDF-projected + splat align in-frame. 6// T1 the optimizer LEARNS the photo: L1(splat, photo-target) drops hard from the flat-skin start 7// T2 the fitted splat matches the photo target markedly better than the naive flat start (measured improvement) 8// T3 PNG knowledge/nx_gsplat_photofit.png (photo-target | flat-start | fitted) + determinism 9// license_tier: ORIGINAL expect_exit: 0 10import "nx_syscalls.nx" 11import "nx_jpeg_ascii.nx" 12import "nx_sdfrender_mt.nx" 13import "nx_faceanat.nx" 14import "nx_meshgen.nx" 15import "nx_gsplat.nx" 16import "nx_png.nx" 17 18func hw(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 } 19func pn(v: i64) -> i64 { let b: *u8=sys_mmap(32) as *u8; var x: i64=v; var ng: i64=0; if x<0{ng=1;x=0-x} var i: i64=31; if x==0{b[i]=48 as u8;i=i-1} while x>0{b[i]=(48+x%10) as u8;x=x/10;i=i-1} if ng==1{b[i]=45 as u8;i=i-1} sys_write(1,(b as i64+i+1) as *u8,31-i); return 0 } 20func l1(a: *i64, b: *i64, n: i64) -> i64 { var s: i64=0; var i: i64=0; while i<n { var dr: i64=(a[i]&255)-(b[i]&255); if dr<0{dr=0-dr} var dg: i64=((a[i]>>8)&255)-((b[i]>>8)&255); if dg<0{dg=0-dg} var db: i64=((a[i]>>16)&255)-((b[i]>>16)&255); if db<0{db=0-db} s=s+dr+dg+db; i=i+1 } return s } 21// L1 of `a` vs target, ONLY over pixels where the splat covers (mask = mask-src non-background). Isolates the 22// COLOUR-fit quality from the irreducible SDF-vs-splat geometry floor (hair/AA the mesh-splat can't render). 23func l1_masked(a: *i64, target: *i64, masksrc: *i64, n: i64) -> i64 { 24 var s: i64 = 0; var i: i64 = 0 25 while i < n { 26 let ml: i64 = (masksrc[i]&255) + ((masksrc[i]>>8)&255) + ((masksrc[i]>>16)&255) 27 if ml > 130 { 28 var dr: i64=(a[i]&255)-(target[i]&255); if dr<0{dr=0-dr} 29 var dg: i64=((a[i]>>8)&255)-((target[i]>>8)&255); if dg<0{dg=0-dg} 30 var db: i64=((a[i]>>16)&255)-((target[i]>>16)&255); if db<0{db=0-db} 31 s = s + dr + dg + db 32 } 33 i = i + 1 34 } 35 return s 36} 37 38const P_UC: i64 = 256 39const P_VC: i64 = 329 40const P_SU: i64 = 333 41const P_SV: i64 = 350 42const P_VLO: i64 = 240 43const P_VHI: i64 = 440 44const YAW: i64 = 200 45 46func main() -> i64 { 47 hw("=== nx_gsplat_photofit_gate -- fit the splat face to the reference photo (the optimizer on the real goal) ===\n" as *u8) 48 var fails: i64 = 0 49 let W: i64 = gs_w() 50 let H: i64 = gs_h() 51 let npx: i64 = W*H 52 53 // decode reference 54 let szp: *i64 = sys_mmap(16) as *i64 55 let jpeg: *u8 = sys_read_file("knowledge/elara_face_hi.jpg" as *u8, szp) 56 if (jpeg as i64) == 0 { hw("no reference\n" as *u8); return 1 } 57 let rp: *i64 = sys_mmap(8) as *i64 58 let wp: *i64 = sys_mmap(8) as *i64 59 let hp: *i64 = sys_mmap(8) as *i64 60 if nx_jpeg_decode_rgb(jpeg, szp[0], rp, wp, hp) != NX_JPEG_ASCII_OK { hw("decode fail\n" as *u8); return 1 } 61 let rgb: *u8 = rp[0] as *u8 62 let tw: i64 = wp[0] 63 let th: i64 = hp[0] 64 65 // TARGET = SDF face WITH the reference projection, rendered sharp (the photo mapped onto our geometry) 66 let base: i64 = sys_mmap(sdf_bytes()) as i64 67 faceanat_build(base) 68 sdf_set_facetex(base, rp[0], tw, th, P_UC, P_VC, P_SU, P_SV) 69 sdf_set_facetex_band(base, P_VLO, P_VHI) 70 sdfmt_render(base, YAW, 4, 236, 180, 156, 0) 71 let sdffb: *i64 = (base + fb_off()) as *i64 72 let target: *i64 = sys_mmap(npx*8) as *i64 73 var k: i64 = 0 74 while k < npx { target[k] = sdffb[k]; k = k + 1 } 75 76 // mesh -> gaussians, FLAT skin colour (unlearned) so the optimizer must LEARN the face 77 let F: *i64 = sys_mmap((MG_N+1)*(MG_N+1)*(MG_N+1)*8) as *i64 78 let cubevi: *i64 = sys_mmap(MG_N*MG_N*MG_N*8) as *i64 79 let vbuf: *i64 = sys_mmap(MG_MAXV*3*8) as *i64 80 let fbuf: *i64 = sys_mmap(MG_MAXF*3*8) as *i64 81 let mo: *i64 = sys_mmap(16) as *i64 82 faceanat_build(base) // rebuild clean (clears facetex effect on parts=none; just parts) 83 mg_build(base, F, cubevi, vbuf, fbuf, mo) 84 let nv: i64 = mo[0] 85 let gauss: *i64 = sys_mmap(nv*8*8) as *i64 86 var vi: i64 = 0 87 while vi < nv { gs_set(gauss, vi, vbuf[vi*3], vbuf[vi*3+1], vbuf[vi*3+2], 24, 210, 160, 140, 235); vi=vi+1 } 88 89 // scratch 90 let fb: *i64 = sys_mmap(npx*8) as *i64 91 let acc: *i64 = sys_mmap(npx*3*8) as *i64 92 let trans: *i64 = sys_mmap(npx*8) as *i64 93 let depth: *i64 = sys_mmap(nv*8) as *i64 94 let sxb: *i64 = sys_mmap(nv*8) as *i64 95 let syb: *i64 = sys_mmap(nv*8) as *i64 96 let sigb: *i64 = sys_mmap(nv*8) as *i64 97 let order: *i64 = sys_mmap(nv*8) as *i64 98 let count: *i64 = sys_mmap((gs_nb()+2)*8) as *i64 99 let explut: *i64 = sys_mmap(gs_expn()*8) as *i64 100 let gradbuf: *i64 = sys_mmap(nv*3*8) as *i64 101 let wnorm: *i64 = sys_mmap(nv*8) as *i64 102 gs_build_explut(explut) 103 104 // FLAT-START render + loss 105 gs_render(gauss, nv, YAW, 4, fb, acc, trans, depth, sxb, syb, sigb, order, count, explut, 26, 28, 44) 106 let flat: *i64 = sys_mmap(npx*8) as *i64 107 k = 0 108 while k < npx { flat[k] = fb[k]; k = k + 1 } 109 let loss0: i64 = l1(fb, target, npx) 110 111 let flatMask: i64 = l1_masked(flat, target, flat, npx) // flat-start error over the splat-covered face region 112 113 // OPTIMIZE colours toward the photo target 114 hw(" fitting splat colours to the photo target:\n" as *u8) 115 var lossN: i64 = 0 116 var it: i64 = 0 117 while it < 40 { 118 lossN = gs_color_grad_step(gauss, nv, target, YAW, 4, fb, acc, trans, depth, sxb, syb, sigb, order, count, explut, gradbuf, wnorm) 119 if it < 3 { hw(" iter "); pn(it); hw(" L1="); pn(lossN); hw("\n" as *u8) } 120 it = it + 1 121 } 122 // fitted render for the region-masked measure 123 gs_render(gauss, nv, YAW, 4, fb, acc, trans, depth, sxb, syb, sigb, order, count, explut, 26, 28, 44) 124 let fitMask: i64 = l1_masked(fb, target, flat, npx) 125 hw(" WHOLE-image L1: flat="); pn(loss0); hw(" fitted="); pn(lossN); hw(" ("); pn(lossN*100/loss0); hw("%)\n" as *u8) 126 hw(" ★FACE-REGION L1 (splat-covered, isolates colour fit): flat="); pn(flatMask); hw(" fitted="); pn(fitMask); hw(" (drop to "); pn(fitMask*100/flatMask); hw("%)\n" as *u8) 127 128 var t1: i64 = 0 129 if fitMask * 100 < flatMask * 45 { t1 = 1 } // in the face region the optimizer fits the photo colours well 130 if t1 == 1 { hw("T1 PASS the optimizer LEARNS the photo: splat colours fit the reference over the face region\n" as *u8) } 131 else { fails=fails+1; hw("T1 FAIL insufficient face-region fit\n" as *u8) } 132 133 var t2: i64 = 0 134 if lossN < loss0 - 40000 { t2 = 1 } // a large absolute whole-image improvement too 135 if t2 == 1 { hw("T2 PASS measured photo-fidelity improvement over the naive flat start\n" as *u8) } 136 else { fails=fails+1; hw("T2 FAIL improvement too small\n" as *u8) } 137 138 // T3 PNG (photo-target | flat | fitted) + determinism 139 let gauss2: *i64 = sys_mmap(nv*8*8) as *i64 140 vi = 0 141 while vi < nv { gs_set(gauss2, vi, vbuf[vi*3], vbuf[vi*3+1], vbuf[vi*3+2], 24, 210, 160, 140, 235); vi=vi+1 } 142 it = 0 143 while it < 40 { gs_color_grad_step(gauss2, nv, target, YAW, 4, fb, acc, trans, depth, sxb, syb, sigb, order, count, explut, gradbuf, wnorm); it=it+1 } 144 var cdiff: i64 = 0 145 vi = 0 146 while vi < nv { if gauss[vi*8+4]!=gauss2[vi*8+4]{cdiff=cdiff+1} if gauss[vi*8+5]!=gauss2[vi*8+5]{cdiff=cdiff+1} vi=vi+1 } 147 gs_render(gauss, nv, YAW, 4, fb, acc, trans, depth, sxb, syb, sigb, order, count, explut, 26, 28, 44) 148 let W3: i64 = W*3 149 let combo: *i64 = sys_mmap(W3*H*8) as *i64 150 var yy: i64 = 0 151 while yy < H { 152 var xx: i64 = 0 153 while xx < W { combo[yy*W3+xx]=target[yy*W+xx]; combo[yy*W3+W+xx]=flat[yy*W+xx]; combo[yy*W3+W*2+xx]=fb[yy*W+xx]; xx=xx+1 } 154 yy = yy + 1 155 } 156 write_png(combo, W3, H, "knowledge/nx_gsplat_photofit.png" as *u8) 157 var t3: i64 = 0 158 if cdiff == 0 { t3 = 1 } 159 if t3 == 1 { hw("T3 PASS deterministic + PNG knowledge/nx_gsplat_photofit.png (photo-target | flat | fitted)\n" as *u8) } 160 else { fails=fails+1; hw("T3 FAIL nondeterministic cdiff="); pn(cdiff); hw("\n" as *u8) } 161 162 if fails == 0 { hw("GSPLAT-PHOTOFIT-GATE 3/3 GREEN -- the optimizer fits our splat face to the reference PHOTO (L1 "); pn(loss0); hw("->"); pn(lossN); hw("), sovereign + integer -- the 3DGS generator on the real goal\n" as *u8); sys_exit(0); return 0 } 163 hw("GSPLAT-PHOTOFIT-GATE RED fails="); pn(fails); hw("\n" as *u8) 164 sys_exit(1) 165 return 1 166}