nx_video_pose_coder.nx source
↩ module page · 308 lines · 15624 B
1// nx_video_pose_coder.nx -- CV-based behavioral pose extraction from video.
2//
3// module: nishi-core.perception.video_pose_coder
4// depends: nishi-core.perception.profile + nishi-core.perception.perceptual_dataset +
5// nishi-core.perception.provenance_curve_store + nishi-core.io.syscalls
6// disk_kb: 7
7// capability: PERCEPTION
8// wired_status: PARTIAL_WIRED
9//
10// MISSING_CAPABILITIES:
11// - VIDEO_DECODE (decode .mp4/.mov/.webm frames into RGB tensors;
12// queued, composes with nx_video_codec_pmesh arc in
13// NISHI_BITS_UP_EXCEED_INDUSTRY)
14// - NEURAL_NET_INFERENCE (run pose-estimation network on frame tensor;
15// queued, depends on nx_tensor + nx_conv2d + nx_transformer_stack
16// primitives from AI-port arc)
17// - POSE_KEYPOINT_DETECTION (per-frame keypoint extraction; DeepLabCut /
18// SLEAP / OpenPose architecture family, re-derived bits-up per
19// [[feedback-bits-up-exceed-never-match]] — never adopt closed weights)
20// - TEMPORAL_TRACKING (associate keypoints across frames into per-subject
21// trajectories; Hungarian-algorithm-style assignment)
22// - BEHAVIORAL_METRIC_AGGREGATION (compute orientation_latency_ms /
23// approach_distance_cm / etc. from keypoint trajectories)
24// - PER_SUBJECT_FINE_TUNING (learn individual subject's body
25// proportions for higher accuracy; user-ratified Stage 1 SUPERVISED)
26//
27// license_tier: PUBLIC_NISHI_SUBSTRATE
28// genealogy_id: feedback-substrate-does-heavy-lifting-user-is-partner-not-gate_2026 +
29// feedback-substrate-primitives-meta-not-one-off_2026 +
30// feedback-multi-species-perceptual-substrate-not-anthropocentric_2026 +
31// feedback-self-surfacing-intelligence-staged-autonomy +
32// mathis_2018_deeplabcut +
33// pereira_2022_sleap +
34// cao_2017_openpose +
35// feedback-bits-up-exceed-never-match
36//
37// Per cardinal [[feedback-substrate-does-heavy-lifting-user-is-partner-not-gate]]:
38// User mounts camera on tripod + records. THIS primitive extracts every
39// behavioral metric from the resulting video automatically. User only
40// ratifies ambiguous frames where confidence is low; substrate doesn't
41// require user to blind-code from scratch.
42//
43// Bits-up sovereignty constraint per
44// [[feedback-bits-up-exceed-never-match]]:
45// DeepLabCut + SLEAP + OpenPose are the academic references genealogy
46// cites; substrate must NOT adopt closed weights or non-commercial-licensed
47// pretrained models. Substrate trains its own weights from
48// nx_perceptual_dataset behavioral entries (Customer Zero = founder's
49// Vizsla footage as first training set) and ships sovereign weights
50// alongside source.
51//
52// Reuse set served by this single primitive:
53// - Dog perceptual test rig behavioral coding (user's Vizslas, Tier 1+)
54// - Dog toy production-line QC (per-unit acceptance test with dog
55// interaction)
56// - Livestock acoustic welfare monitoring (correlate vocalizations with
57// body posture)
58// - Wildlife conservation (population counts + behavior studies)
59// - Working-dog training (assess training response quality)
60// - Beekeeping (waggle-dance direction/distance decoding)
61// - Aquaculture (fish stress behavior signaling)
62// - Veterinary diagnostics (lameness, gait analysis)
63// - Human applications when caller declares NX_PERCEPT_HUMAN profile
64// (physical therapy compliance, sleep studies)
65
66import "nx_syscalls.nx"
67import "nx_perceptual_profile.nx"
68import "nx_perceptual_dataset.nx"
69import "nx_provenance_curve_store.nx"
70
71// ===== Subject kind sealed enum ===================================
72//
73// Different species have different body topologies requiring different
74// keypoint sets. Substrate dispatches the appropriate pose model.
75
76const NX_POSE_SUBJECT_DOG: i64 = 1
77const NX_POSE_SUBJECT_CAT: i64 = 2
78const NX_POSE_SUBJECT_CATTLE: i64 = 3
79const NX_POSE_SUBJECT_SHEEP: i64 = 4
80const NX_POSE_SUBJECT_GOAT: i64 = 5
81const NX_POSE_SUBJECT_PIG: i64 = 6
82const NX_POSE_SUBJECT_HORSE: i64 = 7
83const NX_POSE_SUBJECT_POULTRY: i64 = 8
84const NX_POSE_SUBJECT_SONGBIRD: i64 = 9
85const NX_POSE_SUBJECT_RAPTOR: i64 = 10
86const NX_POSE_SUBJECT_WATERFOWL: i64 = 11
87const NX_POSE_SUBJECT_WILDLIFE_MAMMAL: i64 = 12
88const NX_POSE_SUBJECT_FISH: i64 = 13
89const NX_POSE_SUBJECT_BEE: i64 = 14
90const NX_POSE_SUBJECT_HUMAN: i64 = 15
91const NX_POSE_SUBJECT_MULTIPLE_DOGS: i64 = 16 // multi-subject tracking
92
93func nx_pose_subject_name(s: i64) -> *u8 {
94 if s == NX_POSE_SUBJECT_DOG { return "DOG" }
95 if s == NX_POSE_SUBJECT_CAT { return "CAT" }
96 if s == NX_POSE_SUBJECT_CATTLE { return "CATTLE" }
97 if s == NX_POSE_SUBJECT_SHEEP { return "SHEEP" }
98 if s == NX_POSE_SUBJECT_GOAT { return "GOAT" }
99 if s == NX_POSE_SUBJECT_PIG { return "PIG" }
100 if s == NX_POSE_SUBJECT_HORSE { return "HORSE" }
101 if s == NX_POSE_SUBJECT_POULTRY { return "POULTRY" }
102 if s == NX_POSE_SUBJECT_SONGBIRD { return "SONGBIRD" }
103 if s == NX_POSE_SUBJECT_RAPTOR { return "RAPTOR" }
104 if s == NX_POSE_SUBJECT_WATERFOWL { return "WATERFOWL" }
105 if s == NX_POSE_SUBJECT_WILDLIFE_MAMMAL { return "WILDLIFE_MAMMAL" }
106 if s == NX_POSE_SUBJECT_FISH { return "FISH" }
107 if s == NX_POSE_SUBJECT_BEE { return "BEE" }
108 if s == NX_POSE_SUBJECT_HUMAN { return "HUMAN" }
109 if s == NX_POSE_SUBJECT_MULTIPLE_DOGS { return "MULTIPLE_DOGS" }
110 return "UNKNOWN_SUBJECT"
111}
112
113// ===== Behavioral pose class sealed enum ==========================
114//
115// Substrate classifies each frame's pose into one of these. Composes
116// with temporal tracker to compute durations, transitions, latencies.
117
118const NX_POSE_CLASS_UNKNOWN: i64 = 0
119const NX_POSE_CLASS_NEUTRAL_STANDING: i64 = 1
120const NX_POSE_CLASS_NEUTRAL_LYING: i64 = 2
121const NX_POSE_CLASS_ORIENTING_TO_TARGET: i64 = 3 // head-turned, body still
122const NX_POSE_CLASS_APPROACHING_TARGET: i64 = 4 // body moving toward
123const NX_POSE_CLASS_AT_TARGET: i64 = 5 // within proximity threshold
124const NX_POSE_CLASS_RETREATING: i64 = 6 // moving away
125const NX_POSE_CLASS_FREEZING: i64 = 7 // sudden stillness (alert)
126const NX_POSE_CLASS_PLAY_BOW: i64 = 8 // dog-specific signal
127const NX_POSE_CLASS_PREY_POINT: i64 = 9 // upland bird breed pointing
128const NX_POSE_CLASS_HEAD_COCK: i64 = 10 // characteristic tilt
129const NX_POSE_CLASS_VOCALIZING: i64 = 11 // mouth-open + audio correlate
130const NX_POSE_CLASS_TAIL_HIGH_WAG: i64 = 12 // arousal positive
131const NX_POSE_CLASS_TAIL_TUCKED: i64 = 13 // fear / submission
132const NX_POSE_CLASS_EAR_FORWARD_ALERT: i64 = 14
133const NX_POSE_CLASS_EAR_PINNED: i64 = 15 // fear / aggression
134const NX_POSE_CLASS_GAZE_AT_TARGET: i64 = 16
135
136func nx_pose_class_name(c: i64) -> *u8 {
137 if c == NX_POSE_CLASS_UNKNOWN { return "UNKNOWN" }
138 if c == NX_POSE_CLASS_NEUTRAL_STANDING { return "NEUTRAL_STANDING" }
139 if c == NX_POSE_CLASS_NEUTRAL_LYING { return "NEUTRAL_LYING" }
140 if c == NX_POSE_CLASS_ORIENTING_TO_TARGET { return "ORIENTING_TO_TARGET" }
141 if c == NX_POSE_CLASS_APPROACHING_TARGET { return "APPROACHING_TARGET" }
142 if c == NX_POSE_CLASS_AT_TARGET { return "AT_TARGET" }
143 if c == NX_POSE_CLASS_RETREATING { return "RETREATING" }
144 if c == NX_POSE_CLASS_FREEZING { return "FREEZING" }
145 if c == NX_POSE_CLASS_PLAY_BOW { return "PLAY_BOW" }
146 if c == NX_POSE_CLASS_PREY_POINT { return "PREY_POINT" }
147 if c == NX_POSE_CLASS_HEAD_COCK { return "HEAD_COCK" }
148 if c == NX_POSE_CLASS_VOCALIZING { return "VOCALIZING" }
149 if c == NX_POSE_CLASS_TAIL_HIGH_WAG { return "TAIL_HIGH_WAG" }
150 if c == NX_POSE_CLASS_TAIL_TUCKED { return "TAIL_TUCKED" }
151 if c == NX_POSE_CLASS_EAR_FORWARD_ALERT { return "EAR_FORWARD_ALERT" }
152 if c == NX_POSE_CLASS_EAR_PINNED { return "EAR_PINNED" }
153 if c == NX_POSE_CLASS_GAZE_AT_TARGET { return "GAZE_AT_TARGET" }
154 return "UNKNOWN_CLASS"
155}
156
157// ===== Extraction verdicts ========================================
158
159const NX_POSE_OK: i64 = 0
160const NX_POSE_FAIL_NO_SUBJECT_DETECTED: i64 = 1
161const NX_POSE_FAIL_SUBJECT_OBSCURED: i64 = 2
162const NX_POSE_FAIL_MULTIPLE_SUBJECTS_AMBIGUOUS: i64 = 3
163const NX_POSE_FAIL_LOW_CONFIDENCE: i64 = 4 // below ratification threshold
164const NX_POSE_FAIL_BAD_VIDEO: i64 = 5 // decode error
165const NX_POSE_FAIL_WRONG_SUBJECT_KIND: i64 = 6 // model mismatch
166const NX_POSE_FAIL_DEPENDENCY_MISSING: i64 = 7 // PARTIAL_WIRED default
167
168func nx_pose_verdict_name(v: i64) -> *u8 {
169 if v == NX_POSE_OK { return "OK" }
170 if v == NX_POSE_FAIL_NO_SUBJECT_DETECTED { return "FAIL_NO_SUBJECT_DETECTED" }
171 if v == NX_POSE_FAIL_SUBJECT_OBSCURED { return "FAIL_SUBJECT_OBSCURED" }
172 if v == NX_POSE_FAIL_MULTIPLE_SUBJECTS_AMBIGUOUS { return "FAIL_MULTIPLE_SUBJECTS_AMBIGUOUS" }
173 if v == NX_POSE_FAIL_LOW_CONFIDENCE { return "FAIL_LOW_CONFIDENCE" }
174 if v == NX_POSE_FAIL_BAD_VIDEO { return "FAIL_BAD_VIDEO" }
175 if v == NX_POSE_FAIL_WRONG_SUBJECT_KIND { return "FAIL_WRONG_SUBJECT_KIND" }
176 if v == NX_POSE_FAIL_DEPENDENCY_MISSING { return "FAIL_DEPENDENCY_MISSING" }
177 return "UNKNOWN_POSE_VERDICT"
178}
179
180// ===== Per-frame pose struct ======================================
181//
182// Bundled per [[feedback-nishilang-16-arg-function-limit]]. One per
183// detected subject per frame; multi-subject tracking produces multiple
184// per frame.
185
186struct NxPoseFrame {
187 frame_index: i64
188 timestamp_ms: i64
189 subject_id: i64 // 1+ for tracked subject; 0 = no subject
190 bbox_x_px: i64
191 bbox_y_px: i64
192 bbox_w_px: i64
193 bbox_h_px: i64
194 keypoints_detected: i64 // count out of expected
195 pose_class: i64 // NX_POSE_CLASS_*
196 confidence_score: i64 // 0..100
197 distance_to_target_cm: i64 // -1 if target not configured
198 ear_position_score: i64 // 0..100, forward to pinned
199 tail_position_score: i64
200}
201
202// ===== Aggregated session metrics struct ==========================
203//
204// Per-session aggregate computed by nx_pose_coder_aggregate_session.
205// Mirrors NxPdsBehavioralMetrics in nx_perceptual_dataset so the
206// aggregated result can write_trial directly into the dataset.
207
208struct NxPoseSessionMetrics {
209 orientation_latency_ms: i64
210 approach_distance_cm: i64
211 approach_latency_ms: i64
212 time_on_target_s_x10: i64
213 ear_position_score: i64
214 tail_position_score: i64
215 head_cock_count: i64
216 vocalization_flag: i64
217 freeze_duration_ms: i64
218 gaze_directed_at_target_pct: i64
219}
220
221// ===== Top-level entry stubs ======================================
222
223// nx_pose_coder_extract_frame -- extract pose from a single decoded frame.
224// frame_buf must be RGB8 width*height*3 bytes; substrate routes to
225// appropriate species model per subject_kind.
226
227func nx_pose_coder_extract_frame(frame_buf_ptr: *u8, frame_buf_len: i64,
228 width_px: i64, height_px: i64,
229 subject_kind: i64,
230 result_ptr: *NxPoseFrame) -> i64 {
231 if frame_buf_len <= 0 { return NX_POSE_FAIL_BAD_VIDEO }
232 if width_px <= 0 { return NX_POSE_FAIL_BAD_VIDEO }
233 if height_px <= 0 { return NX_POSE_FAIL_BAD_VIDEO }
234 if subject_kind < NX_POSE_SUBJECT_DOG { return NX_POSE_FAIL_WRONG_SUBJECT_KIND }
235 if subject_kind > NX_POSE_SUBJECT_MULTIPLE_DOGS { return NX_POSE_FAIL_WRONG_SUBJECT_KIND }
236 if frame_buf_len != width_px * height_px * 3 { return NX_POSE_FAIL_BAD_VIDEO }
237 // PARTIAL_WIRED: NN inference + keypoint detection queued.
238 return NX_POSE_FAIL_DEPENDENCY_MISSING
239}
240
241// nx_pose_coder_aggregate_session -- given a sequence of per-frame poses
242// (typically 60s × 60fps = 3600 frames), compute aggregate session
243// behavioral metrics ready for nx_pds_write_trial.
244
245func nx_pose_coder_aggregate_session(frames_ptr: *NxPoseFrame, frames_len: i64,
246 target_position_x_px: i64, target_position_y_px: i64,
247 proximity_threshold_cm: i64,
248 metrics_out_ptr: *NxPoseSessionMetrics) -> i64 {
249 if frames_len <= 0 { return NX_POSE_FAIL_NO_SUBJECT_DETECTED }
250 if proximity_threshold_cm <= 0 { return NX_POSE_FAIL_BAD_VIDEO }
251 // PARTIAL_WIRED: temporal-tracking + aggregation queued.
252 return NX_POSE_FAIL_DEPENDENCY_MISSING
253}
254
255// nx_pose_coder_classify_behavior -- given a short window of frames
256// (e.g., 30 frames = 500ms at 60fps), classify the dominant pose class.
257// Used for transitions like ORIENTING_TO_TARGET → APPROACHING_TARGET.
258
259func nx_pose_coder_classify_behavior(window_ptr: *NxPoseFrame, window_len: i64,
260 subject_kind: i64) -> i64 {
261 if window_len <= 0 { return NX_POSE_CLASS_UNKNOWN }
262 if subject_kind < NX_POSE_SUBJECT_DOG { return NX_POSE_CLASS_UNKNOWN }
263 if subject_kind > NX_POSE_SUBJECT_MULTIPLE_DOGS { return NX_POSE_CLASS_UNKNOWN }
264 return NX_POSE_CLASS_UNKNOWN
265}
266
267// nx_pose_coder_user_ratify_frame -- user reviews a low-confidence frame
268// and provides the correct pose class. Per
269// [[feedback-self-surfacing-intelligence-staged-autonomy]] STAGE 1
270// SUPERVISED: substrate surfaces ambiguous frames; user ratifies; ratified
271// examples feed back to per-subject fine-tuning.
272
273func nx_pose_coder_user_ratify_frame(frame_index: i64,
274 corrected_class: i64,
275 subject_id: i64) -> i64 {
276 if corrected_class < NX_POSE_CLASS_UNKNOWN { return 0 }
277 if corrected_class > NX_POSE_CLASS_GAZE_AT_TARGET { return 0 }
278 if subject_id < 0 { return 0 }
279 return 1
280}
281
282// nx_pose_coder_register_subject_template -- per-subject fine-tuning
283// shot. User uploads a few labeled frames of their specific dog; substrate
284// fine-tunes the per-subject head; accuracy on that subject improves over
285// time. Composes with nx_provenance_curve_store (template versioned per
286// subject serial).
287
288func nx_pose_coder_register_subject_template(subject_id: i64,
289 template_frames_ptr: *NxPoseFrame,
290 template_frames_len: i64) -> i64 {
291 if subject_id <= 0 { return 0 }
292 if template_frames_len <= 0 { return 0 }
293 return 0
294}
295
296// nx_pose_coder_get_confidence_threshold -- inspector: below what
297// confidence does substrate request user ratification? Default 70%;
298// caller can override per use case.
299
300func nx_pose_coder_get_confidence_threshold() -> i64 {
301 return 70
302}
303
304// nx_pose_coder_get_last_verdict -- inspector.
305
306func nx_pose_coder_get_last_verdict() -> i64 {
307 return NX_POSE_FAIL_DEPENDENCY_MISSING
308}