`nx_ng_gen3d.nx` source

↩ module page · 146 lines · 8905 B
1// nx_ng_gen3d.nx -- CAP-GEN-3D, generation 12: generative-3D (DreamFusion-class, arxiv 2209.14988). The honest
2// CORE mechanism of text/image-to-3D: a 3D representation is optimized so that its RENDERED 2D VIEWS satisfy a
3// 2D objective (a "prior"), via the DIFFERENTIABLE renderer -- "lifting" 2D supervision into a 3D scene, with no
4// direct 3D supervision. Reuses the splat color-fit on the autograd tape (CAP-GAUSSIAN-SPLAT / CAP-DIFF-RENDER).
5//
6// Honest gated proof of the DEFINING property (why generative-3D needs multiple views / a prior): a 3D scene =
7// N gaussians with shared learnable colors; each VIEW renders as a weighted sum of those shared colors (the
8// gaussian's projected footprint+transmittance in that view). We generate target views from a known scene c*.
9//   T1 MULTI-VIEW LIFT -- optimizing colors from TWO 2D views recovers the 3D scene (|c_g - c*_g| small): 2D
10//                         supervision is distilled into a 3D-consistent representation.
11//   T2 SINGLE-VIEW INSUFFICIENT -- view A alone is ill-posed (gaussians 0,1 overlap into one observed pixel ->
12//                         only their SUM is seen) so c0,c1 are NOT recovered: this is WHY a generative prior /
13//                         extra views are needed = the load-bearing reason the method exists.
14//   T3 BIT-EXACT -- the 2-view solve is deterministic (train twice -> identical integer colors).
15// HONEST: the "prior" here is explicit target views (full DreamFusion = a learned text-to-image diffusion model
16// driving the views via score-distillation = the follow-on); 1D pixels, fixed gaussian geometry. Q16, no float.
17// Sovereign: nx_nofloat_autograd + syscalls. license_tier: ORIGINAL  expect_exit: 0
18import "nx_nofloat_autograd.nx"
19import "nx_itoa_lib.nx"   // shared MSB-first emitter (zero-alloc)
20import "nx_syscalls.nx"
21const K_MAGIC_8192: i64 = 8192
22const K_MAGIC_13107: i64 = 13107
23const K_MAGIC_45875: i64 = 45875
24const K_MAGIC_32768: i64 = 32768
25const K_MAGIC_65536: i64 = 65536
26const K_MAGIC_4096: i64 = 4096
27
28const TLOG: *u8 = "knowledge/status/ng_gen3d.log"
29const Q16:  i64 = 65536
30const NGS:  i64 = 3           // gaussians in the 3D scene (shared across views)
31const NPV:  i64 = 4           // pixels per view
32const EPOCHS: i64 = 4000
33const LRQ:  i64 = 4096
34
35func tpr(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
36// MIGRATED to the shared emitter (debt 1785563586). The old body mmapped a scratch buffer
37// per call and never freed it. At PAGE granularity that is 4096B leaked PER CALL -- the
38// defect that took 28.5GB of a 36GB host in nx_ts_lumadiff (2MB input, ~3.66M calls).
39// nxi_* is MSB-first, allocates NOTHING, and emits identical bytes including the sign.
40func tpn(v: i64) -> i64 { nxi_out(v); return 0 }
41func t_abs(v: i64) -> i64 { if v<0 { return 0-v } return v }
42func tqm(a: i64, b: i64) -> i64 { return (a*b)>>16 }
43func tl_ws(fd: i64, s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(fd,s,n); return 0 }
44// MIGRATED to the shared emitter (debt 1785563586). The old body mmapped a scratch buffer
45// per call and never freed it. At PAGE granularity that is 4096B leaked PER CALL -- the
46// defect that took 28.5GB of a 36GB host in nx_ts_lumadiff (2MB input, ~3.66M calls).
47// nxi_* is MSB-first, allocates NOTHING, and emits identical bytes including the sign.
48func tl_wn(fd: i64, v: i64) -> i64 { nxi_fd(fd, v); return 0 }
49
50// render one view's pixels into the SSE root: out_p = sum_g w[g*NPV+p]*c_g ; mse vs targ_p ; accumulate.
51func view_into(tape: *i64, vals: *i64, st: *i64, cn: *i64, w: *i64, targ: *i64, root_in: i64) -> i64 {
52    var root: i64 = root_in
53    var p: i64 = 0
54    while p < NPV {
55        var acc: i64 = 0 - 1; var g: i64 = 0
56        while g < NGS {
57            let cm: i64 = nfa_cmul(tape,vals,st, cn[g], w[g*NPV+p])
58            if acc < 0 { acc = cm } else { acc = nfa_vadd(tape,vals,st, acc, cm) }
59            g = g + 1
60        }
61        let nt: i64 = nfa_leaf(tape,vals,st, 1, 1, targ, p)
62        let nm: i64 = nfa_mse(tape,vals,st, acc, nt)
63        if root < 0 { root = nm } else { root = nfa_vadd(tape,vals,st, root, nm) }
64        p = p + 1
65    }
66    return root
67}
68// train the shared 3D colors from view A (always) + view B (iff use_b). returns recovered colors in cout, loss in ll.
69func gen_train(tape: *i64, vals: *i64, grads: *i64, st: *i64, wA: *i64, tA: *i64, wB: *i64, tB: *i64, use_b: i64, cout: *i64, ll: *i64) -> i64 {
70    let cp: *i64 = sys_mmap(NGS*8) as *i64
71    var i: i64=0; while i<NGS { cp[i]=0; i=i+1 }
72    let cn: *i64 = sys_mmap(NGS*8) as *i64; let g: *i64 = sys_mmap(NGS*8) as *i64
73    var ep: i64=0
74    while ep<EPOCHS {
75        st[0]=0; st[1]=0
76        var gg: i64=0; while gg<NGS { cn[gg]=nfa_leaf(tape,vals,st, 1,1, cp, gg); gg=gg+1 }
77        var root: i64 = view_into(tape,vals,st, cn, wA, tA, 0 - 1)
78        if use_b == 1 { root = view_into(tape,vals,st, cn, wB, tB, root) }
79        nfa_backward(tape,vals,grads, st[0], root)
80        *ll = nfa_val(tape,vals,root,0)
81        var k: i64=0; while k<NGS { g[k]=nfa_grad(tape,grads,cn[k],0); k=k+1 }
82        nfa_sgd(cp, g, NGS, LRQ)
83        ep=ep+1
84    }
85    i=0; while i<NGS { cout[i]=cp[i]; i=i+1 }
86    return 0
87}
88
89func main() -> i64 {
90    tpr("nx_ng_gen3d: CAP-GEN-3D -- 2D->3D lift (optimize a 3D scene so its rendered views match a 2D prior)\n" as *u8)
91    let tape:  *i64 = sys_mmap(512*7*8) as *i64
92    let vals:  *i64 = sys_mmap(K_MAGIC_8192*8) as *i64
93    let grads: *i64 = sys_mmap(K_MAGIC_8192*8) as *i64
94    let st:    *i64 = sys_mmap(2*8) as *i64
95
96    let cstar: *i64 = sys_mmap(NGS*8) as *i64
97    cstar[0]=K_MAGIC_13107; cstar[1]=K_MAGIC_45875; cstar[2]=K_MAGIC_32768               // true 3D scene colors 0.2, 0.7, 0.5
98    // View A: gaussians 0 AND 1 project to the SAME pixel 0 (overlap -> only their SUM observed); g2 -> pixel 2.
99    let wA: *i64 = sys_mmap(NGS*NPV*8) as *i64
100    var z: i64=0; while z<NGS*NPV { wA[z]=0; z=z+1 }
101    wA[0*NPV+0]=K_MAGIC_32768; wA[1*NPV+0]=K_MAGIC_32768; wA[2*NPV+2]=K_MAGIC_65536       // p0 = .5*c0 + .5*c1 ; p2 = c2
102    // View B: gaussians SEPARATE -> g0->p0, g1->p1, g2->p2 (resolves the overlap A could not)
103    let wB: *i64 = sys_mmap(NGS*NPV*8) as *i64
104    z=0; while z<NGS*NPV { wB[z]=0; z=z+1 }
105    wB[0*NPV+0]=K_MAGIC_65536; wB[1*NPV+1]=K_MAGIC_65536; wB[2*NPV+2]=K_MAGIC_65536
106    // target views rendered from the TRUE scene
107    let tA: *i64 = sys_mmap(NPV*8) as *i64; let tB: *i64 = sys_mmap(NPV*8) as *i64
108    var p: i64=0
109    while p<NPV { var sa: i64=0; var sb: i64=0; var g: i64=0; while g<NGS { sa=sa+tqm(wA[g*NPV+p],cstar[g]); sb=sb+tqm(wB[g*NPV+p],cstar[g]); g=g+1 } tA[p]=sa; tB[p]=sb; p=p+1 }
110
111    // T1: MULTI-VIEW (A+B) lift -> recover the 3D scene
112    let c2v: *i64=sys_mmap(NGS*8) as *i64; let l2: *i64=sys_mmap(8) as *i64
113    gen_train(tape,vals,grads,st, wA,tA, wB,tB, 1, c2v, l2)
114    var t1: i64=1
115    var gI: i64=0; while gI<NGS { if t_abs(c2v[gI]-cstar[gI]) >= K_MAGIC_4096 { t1=0 } gI=gI+1 }
116    // T2: SINGLE-VIEW (A only) -> c0,c1 ambiguous (only their sum observed) -> NOT recovered
117    let c1v: *i64=sys_mmap(NGS*8) as *i64; let l1: *i64=sys_mmap(8) as *i64
118    gen_train(tape,vals,grads,st, wA,tA, wB,tB, 0, c1v, l1)
119    var t2: i64=0
120    if t_abs(c1v[0]-cstar[0]) >= K_MAGIC_8192 { t2=1 }
121    if t_abs(c1v[1]-cstar[1]) >= K_MAGIC_8192 { t2=1 }
122    // T3: bit-exact (2-view twice)
123    let c2b: *i64=sys_mmap(NGS*8) as *i64; let l2b: *i64=sys_mmap(8) as *i64
124    gen_train(tape,vals,grads,st, wA,tA, wB,tB, 1, c2b, l2b)
125    var t3: i64=1; gI=0; while gI<NGS { if c2b[gI]!=c2v[gI] { t3=0 } gI=gI+1 }
126
127    tpr("  T1 MULTI-VIEW lift: 2-view loss=" as *u8); tpn(*l2); tpr(" recovered c=[" as *u8); tpn(c2v[0]); tpr("," as *u8); tpn(c2v[1]); tpr("," as *u8); tpn(c2v[2]); tpr("] vs c*=[13107,45875,32768] ok=" as *u8); tpn(t1); tpr("\n" as *u8)
128    tpr("  T2 SINGLE-VIEW insufficient: 1-view c=[" as *u8); tpn(c1v[0]); tpr("," as *u8); tpn(c1v[1]); tpr("," as *u8); tpn(c1v[2]); tpr("] (c0,c1 NOT recovered = overlap ambiguity) shown=" as *u8); tpn(t2); tpr("\n" as *u8)
129    tpr("  T3 bit-exact=" as *u8); tpn(t3); tpr("\n" as *u8)
130
131    var ok: i64=1
132    if t1!=1 { ok=0 }
133    if t2!=1 { ok=0 }
134    if t3!=1 { ok=0 }
135    let logf: i64=sys_openat_append(TLOG, 420)
136    if logf>=0 {
137        tl_ws(logf,"NGGEN3D authored=organ 2d-to-3d-lift loss_2view=" as *u8); tl_wn(logf,*l2)
138        tl_ws(logf," t1=" as *u8); tl_wn(logf,t1); tl_ws(logf," t2=" as *u8); tl_wn(logf,t2); tl_ws(logf," t3=" as *u8); tl_wn(logf,t3)
139        if ok==1 { tl_ws(logf," verdict=GREEN\n" as *u8) } else { tl_ws(logf," verdict=RED\n" as *u8) }
140        sys_close(logf)
141    }
142    tpr("  verdict=" as *u8)
143    if ok==1 { tpr("GREEN (multi-view 2D supervision distilled into a 3D scene via the differentiable renderer; single-view ill-posed = generative-3D core)\n" as *u8); sys_exit(0); return 0 }
144    tpr("RED\n" as *u8)
145    sys_exit(1); return 1
146}
nx_ng_gen3d.nx source

`nx_ng_gen3d.nx` source
