code wiki / _hdl_build / nx_text2motion_gate.nx
nx_text2motion_gate.nx source
↩ module page · 182 lines · 10318 B
1// nx_text2motion_gate.nx -- TEXT-CONDITIONED neural motion, proven with the honest ML criterion (GENERALIZATION
2// to held-out captions -- a keyword if-else cannot generalize to sentences it never saw). Chain: caption ->
3// LEARNED perceptron encoder -> motion class -> class-conditioned generated tokens -> VQ decode -> rig -> video.
4// T1 the encoder LEARNS: perceptron training mistakes -> 0 (separated the training set)
5// T2 it GENERALIZES: 6/6 HELD-OUT captions (novel word combinations, never trained as sentences) classify
6// correctly -- AND the UNTRAINED (zero) encoder gets ~1/6, proving the LEARNING did it (not the vocab)
7// T3 full chain: a held-out caption -> class -> generated motion tokens -> decoded poses MOVE -> APNG rendered
8// T4 TEXT STEERS: two different captions -> different classes -> different token sequences
9// Emits knowledge/synth_text2motion.png (APNG of the motion generated FROM a sentence). license_tier: ORIGINAL
10// expect_exit: 0
11import "nx_syscalls.nx"
12import "nx_skeleton.nx"
13import "nx_figure_render.nx"
14import "nx_motion_neural.nx"
15import "nx_text2motion.nx"
16import "nx_apng.nx"
17
18func hw(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
19func pn(v: i64) -> i64 {
20 let b: *u8 = sys_mmap(32) as *u8
21 var x: i64 = v; var neg: i64 = 0
22 if x < 0 { neg = 1; x = 0 - x }
23 var i: i64 = 31
24 if x == 0 { b[i] = 48 as u8; i = i - 1 }
25 while x > 0 { b[i] = (48 + x % 10) as u8; x = x / 10; i = i - 1 }
26 if neg == 1 { b[i] = 45 as u8; i = i - 1 }
27 sys_write(1, (b as i64 + i + 1) as *u8, 31 - i)
28 return 0
29}
30func find4(buf: *u8, n: i64, a: i64, b: i64, c: i64, d: i64) -> i64 {
31 var i: i64 = 0
32 while i + 4 <= n { if (buf[i]&0xff)==a { if (buf[i+1]&0xff)==b { if (buf[i+2]&0xff)==c { if (buf[i+3]&0xff)==d { return i } } } } i = i + 1 }
33 return 0 - 1
34}
35const GW: i64 = 128
36const GH: i64 = 96
37
38func main() -> i64 {
39 hw("=== nx_text2motion_gate -- TEXT -> LEARNED encoder -> motion class -> generated video ===\n" as *u8)
40 let sk: i64 = sys_mmap(sk_bytes()) as i64
41 rig_build(sk)
42 // motion side: corpus -> learned VQ codebook -> tokens (reuse nx_motion_neural)
43 let corpus: *i64 = sys_mmap(MN_N * MN_D * 8) as *i64
44 mn_build_corpus(sk, corpus)
45 let cb: *i64 = sys_mmap(MN_K * MN_D * 8) as *i64
46 mn_init_cb(corpus, cb)
47 mn_train(corpus, cb, 15)
48 let tokens: *i64 = sys_mmap(MN_N * 8) as *i64
49 mn_tokenize(corpus, cb, tokens)
50
51 // ---- training set: captions -> class labels (words are class-indicative; combos are the data) ----
52 let caps: *i64 = sys_mmap(32 * 8) as *i64
53 let labs: *i64 = sys_mmap(32 * 8) as *i64
54 var nc: i64 = 0
55 caps[nc]="wave at them" as *u8 as i64; labs[nc]=0; nc=nc+1
56 caps[nc]="say hello with a wave" as *u8 as i64; labs[nc]=0; nc=nc+1
57 caps[nc]="wave hi" as *u8 as i64; labs[nc]=0; nc=nc+1
58 caps[nc]="cheer for the win" as *u8 as i64; labs[nc]=1; nc=nc+1
59 caps[nc]="celebrate excited" as *u8 as i64; labs[nc]=1; nc=nc+1
60 caps[nc]="cheer and celebrate" as *u8 as i64; labs[nc]=1; nc=nc+1
61 caps[nc]="sway to the rhythm" as *u8 as i64; labs[nc]=2; nc=nc+1
62 caps[nc]="dance and groove" as *u8 as i64; labs[nc]=2; nc=nc+1
63 caps[nc]="sway dance" as *u8 as i64; labs[nc]=2; nc=nc+1
64 caps[nc]="kick and march" as *u8 as i64; labs[nc]=3; nc=nc+1
65 caps[nc]="march step" as *u8 as i64; labs[nc]=3; nc=nc+1
66 caps[nc]="kick leg" as *u8 as i64; labs[nc]=3; nc=nc+1
67 caps[nc]="look around" as *u8 as i64; labs[nc]=4; nc=nc+1
68 caps[nc]="search and scan" as *u8 as i64; labs[nc]=4; nc=nc+1
69 caps[nc]="look and search" as *u8 as i64; labs[nc]=4; nc=nc+1
70 caps[nc]="stretch and reach" as *u8 as i64; labs[nc]=5; nc=nc+1
71 caps[nc]="circle the arm" as *u8 as i64; labs[nc]=5; nc=nc+1
72 caps[nc]="arm stretch" as *u8 as i64; labs[nc]=5; nc=nc+1
73 caps[nc]="clap for them" as *u8 as i64; labs[nc]=6; nc=nc+1
74 caps[nc]="clap and applaud" as *u8 as i64; labs[nc]=6; nc=nc+1
75 caps[nc]="applaud" as *u8 as i64; labs[nc]=6; nc=nc+1
76 caps[nc]="punch forward" as *u8 as i64; labs[nc]=7; nc=nc+1
77 caps[nc]="jab and punch" as *u8 as i64; labs[nc]=7; nc=nc+1
78 caps[nc]="fight jab" as *u8 as i64; labs[nc]=7; nc=nc+1
79 caps[nc]="take a bow" as *u8 as i64; labs[nc]=8; nc=nc+1
80 caps[nc]="bow to them" as *u8 as i64; labs[nc]=8; nc=nc+1
81 caps[nc]="bow" as *u8 as i64; labs[nc]=8; nc=nc+1
82 caps[nc]="jump and jack" as *u8 as i64; labs[nc]=9; nc=nc+1
83 caps[nc]="do a jump jack" as *u8 as i64; labs[nc]=9; nc=nc+1
84 caps[nc]="jack jump" as *u8 as i64; labs[nc]=9; nc=nc+1
85
86 let W: *i64 = sys_mmap(T2M_CLASSES * T2M_VOCAB * 8) as *i64
87 let mistakes: i64 = t2m_train(W, caps, labs, nc, 40)
88 hw(" perceptron trained on " as *u8); pn(nc); hw(" captions -> last-epoch mistakes=" as *u8); pn(mistakes); hw("\n" as *u8)
89
90 var fails: i64 = 0
91 if mistakes == 0 { hw("T1 PASS encoder LEARNED (perceptron separated the training set)\n" as *u8) }
92 else { fails = fails + 1; hw("T1 FAIL still " as *u8); pn(mistakes); hw(" mistakes\n" as *u8) }
93
94 // ---- held-out captions (novel combinations of trained words; never trained as sentences) ----
95 let hc: *i64 = sys_mmap(16 * 8) as *i64
96 let hl: *i64 = sys_mmap(16 * 8) as *i64
97 var nh: i64 = 0
98 hc[nh]="hello wave" as *u8 as i64; hl[nh]=0; nh=nh+1
99 hc[nh]="excited celebration cheer" as *u8 as i64; hl[nh]=1; nh=nh+1
100 hc[nh]="groove and dance" as *u8 as i64; hl[nh]=2; nh=nh+1
101 hc[nh]="march the leg" as *u8 as i64; hl[nh]=3; nh=nh+1
102 hc[nh]="scan around" as *u8 as i64; hl[nh]=4; nh=nh+1
103 hc[nh]="reach and circle" as *u8 as i64; hl[nh]=5; nh=nh+1
104 hc[nh]="applaud and clap" as *u8 as i64; hl[nh]=6; nh=nh+1
105 hc[nh]="punch jab" as *u8 as i64; hl[nh]=7; nh=nh+1
106 hc[nh]="a polite bow" as *u8 as i64; hl[nh]=8; nh=nh+1
107 hc[nh]="jack and jump" as *u8 as i64; hl[nh]=9; nh=nh+1
108 // trained accuracy on held-out
109 var correct: i64 = 0
110 var h: i64 = 0
111 while h < nh {
112 let pc: i64 = t2m_encode(W, hc[h] as *u8)
113 hw(" '" as *u8); hw(hc[h] as *u8); hw("' -> class " as *u8); pn(pc); hw(" (want " as *u8); pn(hl[h]); hw(")" as *u8)
114 if pc == hl[h] { correct = correct + 1; hw(" ok\n" as *u8) } else { hw(" MISS\n" as *u8) }
115 h = h + 1
116 }
117 // untrained (zero) encoder accuracy -- the contrast proving LEARNING did it
118 let W0: *i64 = sys_mmap(T2M_CLASSES * T2M_VOCAB * 8) as *i64
119 var zi: i64 = 0
120 while zi < T2M_CLASSES * T2M_VOCAB { W0[zi] = 0; zi = zi + 1 }
121 var correct0: i64 = 0
122 h = 0
123 while h < nh { if t2m_encode(W0, hc[h] as *u8) == hl[h] { correct0 = correct0 + 1 } h = h + 1 }
124 hw(" held-out accuracy: TRAINED " as *u8); pn(correct); hw("/" as *u8); pn(nh); hw(" vs UNTRAINED " as *u8); pn(correct0); hw("/" as *u8); pn(nh); hw("\n" as *u8)
125 if correct == nh { if correct0 < nh { hw("T2 PASS GENERALIZES (trained " as *u8); pn(correct); hw("/" as *u8); pn(nh); hw(" held-out vs untrained " as *u8); pn(correct0); hw(" -> learning, not lookup)\n" as *u8) } else { fails = fails + 1; hw("T2 FAIL untrained also perfect (vocab too easy)\n" as *u8) } }
126 else { fails = fails + 1; hw("T2 FAIL only " as *u8); pn(correct); hw("/" as *u8); pn(nh); hw(" held-out correct\n" as *u8) }
127
128 // ---- T4 text steers: two captions -> classes -> class-conditioned token sequences ----
129 let clsA: i64 = t2m_encode(W, "hello wave" as *u8) // -> wave (class 0)
130 let clsB: i64 = t2m_encode(W, "march the leg" as *u8) // -> kick (class 3)
131 let trA: *i64 = sys_mmap(MN_K * MN_K * 8) as *i64
132 let trB: *i64 = sys_mmap(MN_K * MN_K * 8) as *i64
133 t2m_class_bigram(tokens, clsA, trA)
134 t2m_class_bigram(tokens, clsB, trB)
135 let stA: *i64 = sys_mmap(8) as *i64; stA[0] = 424242
136 let stB: *i64 = sys_mmap(8) as *i64; stB[0] = 424242
137 let genA: *i64 = sys_mmap(MN_FR * 8) as *i64
138 let genB: *i64 = sys_mmap(MN_FR * 8) as *i64
139 mn_generate(trA, tokens[clsA * MN_FR], stA, genA, MN_FR)
140 mn_generate(trB, tokens[clsB * MN_FR], stB, genB, MN_FR)
141 var steer: i64 = 0
142 var gi: i64 = 0
143 while gi < MN_FR { if genA[gi] != genB[gi] { steer = steer + 1 } gi = gi + 1 }
144 if clsA != clsB { if steer > 0 { hw("T4 PASS TEXT STEERS ('hello wave'->cls" as *u8); pn(clsA); hw(" vs 'march the leg'->cls" as *u8); pn(clsB); hw("; " as *u8); pn(steer); hw("/16 tokens differ)\n" as *u8) } else { fails = fails + 1; hw("T4 FAIL same sequence\n" as *u8) } }
145 else { fails = fails + 1; hw("T4 FAIL captions mapped to same class\n" as *u8) }
146
147 // ---- T3 full chain: render genA (the motion FROM the sentence 'hello wave') ----
148 let fb: *i64 = sys_mmap(GW * GH * 8) as *i64
149 let zb: *i64 = sys_mmap(GW * GH * 8) as *i64
150 let apbuf: *u8 = sys_mmap(8 * 1024 * 1024)
151 let seq: *i64 = sys_mmap(8) as *i64; seq[0] = 0
152 let vec: *i64 = sys_mmap(MN_D * 8) as *i64
153 var moved: i64 = 0
154 var o: i64 = apng_open(apbuf, GW, GH, MN_FR)
155 var gf: i64 = 0
156 while gf < MN_FR {
157 let tk: i64 = genA[gf]
158 var d: i64 = 0
159 while d < MN_D { vec[d] = cb[tk * MN_D + d]; d = d + 1 }
160 pose_apply(sk, vec)
161 sk_update(sk)
162 rig_draw(sk, fb, zb, GW, GH, md_deg(15), 86)
163 if gf > 0 { if genA[gf] != genA[gf-1] { moved = moved + 1 } }
164 var first: i64 = 0
165 if gf == 0 { first = 1 }
166 o = apng_frame(apbuf, o, seq, fb, GW, GH, 8, first)
167 gf = gf + 1
168 }
169 o = apng_close(apbuf, o)
170 let fd: i64 = sys_openat_wr("knowledge/synth_text2motion.png" as *u8, 0x1a4)
171 if fd >= 0 { sys_write(fd, apbuf, o); sys_close(fd) }
172 hw(" wrote knowledge/synth_text2motion.png bytes=" as *u8); pn(o); hw("\n" as *u8)
173 var t3: i64 = 1
174 if find4(apbuf, o, 97, 99, 84, 76) < 0 { t3 = 0 }
175 if moved < 1 { t3 = 0 }
176 if t3 == 1 { hw("T3 PASS full chain: sentence -> class -> tokens -> decoded MOTION -> valid APNG (" as *u8); pn(moved); hw(" pose changes)\n" as *u8) }
177 else { fails = fails + 1; hw("T3 FAIL chain/render\n" as *u8) }
178
179 if fails == 0 { hw("TEXT2MOTION-GATE 4/4 verdict=GREEN -- LEARNED text encoder (generalizes to held-out) drives generated motion -> video, sovereign\n" as *u8); sys_exit(0); return 0 }
180 hw("TEXT2MOTION-GATE RED fails=" as *u8); pn(fails); hw("\n" as *u8)
181 sys_exit(1); return 1
182}