code wiki / (root) / nx_modality_router.nx

nx_modality_router.nx

buildroot/runtime/nx_modality_router.nx

17099 B448 linesdepth 2pulls 2 transitivereach 2 importersview sourcekind library
docsdependenciesstructsconstsfunctions

about

nx_modality_router.nx -- H10 multi-modal model sharing (bits-up). Per NISHI_ELDER_AI_OFF_DOCKER_2026_05_20.md ยง2.1 H10: one shared backbone trunk serves multiple modality-specific heads concurrently -- text response, image prompt conditioning, audio embed, vision token embed. Stable Diffusion shares one UNet across N LoRA + ControlNet adaptations; Llava + Flamingo share one LLM trunk across text + vision modalities; the operator's "multiused the llm for response and image gen" cardinal IS this pattern. V1 mechanics: two registries -- backbones (one trunk weight reference each) + modality heads (small per-modality projection layers that attach to a backbone). Registering a head ACQUIRES the named backbone (bumps refcount); unregistering RELEASES it. A backbone with refcount > 0 cannot be evicted. resolve_backbone_ for_modality() walks the modality -> backbone link in one call. Composition path: - register_backbone(id, weights_data, hidden_dim, n_layers) ONCE per loaded base model - register_modality(id, backbone_id, head_data, kind) per modality the consumer wants to serve; this acquires the backbone - resolve_backbone_for_modality(modality_id) at request-time gives the consumer the backbone pointer + the head pointer in one call, so the forward-pass kernel can run trunk + head in sequence without a second lookup - unregister_modality on shutdown releases the backbone; evict_backbone only succeeds when no modality references it Pure substrate logic. No Linux features. Composes with shipped H9 nx_lora_pool (modality head's "head_data" pointer can itself be an adapter_id resolved through the LoRA pool, letting one backbone serve N modalities each with M fine-tuned adapter variants). V1 honest scope: - Sealed-enum modality kinds: TEXT, IMAGE, AUDIO, EMBED. A V2 could add VIDEO + PROPRIOCEPT etc.; the sealed-enum is

dependencies 1 imports · 2 importers

nx_syscalls.nx nx_modality_router.nx nx_hackers_algo_compose_test.nx nx_modality_router_test.nx

imports: nx_syscalls.nx

imported by: nx_hackers_algo_compose_test.nxnx_modality_router_test.nx

structs

126struct NxBackbone
137struct NxModalityHead
147struct NxModalityRouter

consts

83const NX_MODALITY_MAX_BACKBONES: i64 = 16
84const NX_MODALITY_MAX_HEADS: i64 = 64
87const NX_MODALITY_TEXT: i64 = 0
88const NX_MODALITY_IMAGE: i64 = 1
89const NX_MODALITY_AUDIO: i64 = 2
90const NX_MODALITY_EMBED: i64 = 3
91const NX_MODALITY_N_KINDS: i64 = 4
100const NX_MOD_OK: i64 = 0
101const NX_MOD_BAD_INPUT: i64 = 1
102const NX_MOD_NOT_FOUND: i64 = 2
103const NX_MOD_DUPLICATE: i64 = 3
104const NX_MOD_FULL: i64 = 4
105const NX_MOD_IN_USE: i64 = 5
106const NX_MOD_BAD_KIND: i64 = 6
107const NX_MOD_BACKBONE_NOT_FOUND: i64 = 7
108const NX_MOD_TAMPER: i64 = 8
109const NX_MOD_N_VERDICTS: i64 = 9
118const NX_BACKBONE_CANARY_PRE: i64 = 0x4261636B626F6E65 // "Backbone"
119const NX_BACKBONE_CANARY_POST: i64 = 0x426F6E654531446F // "BoneE1Do"
120const NX_MOD_HEAD_CANARY_PRE: i64 = 0x4D6F6448656164A1 // "MoHHead!"
121const NX_MOD_HEAD_CANARY_POST: i64 = 0x4865616445314432 // "HeadE1D2"
122const NX_MOD_ROUTER_CANARY_PRE: i64 = 0x4D6F64526F75740A // "ModRout."
123const NX_MOD_ROUTER_CANARY_POST:i64 = 0x526F75744531443B // "RoutE1D;"

functions

93func nx_modality_kind_is_valid(k: i64) -> i64
111func nx_mod_verdict_is_valid(v: i64) -> i64
called by 1: main
159func nx_backbone_is_valid(b: *NxBackbone) -> i64
170func nx_modality_head_is_valid(h: *NxModalityHead) -> i64
179func nx_modality_router_is_valid(r: *NxModalityRouter) -> i64
195func nx_modality_router_new(max_backbones: i64, max_heads: i64) -> *NxModalityRouter
called by 2: mainmain calls 1: sys_mmap
225func nx_mod__find_backbone(r: *NxModalityRouter, backbone_id: i64) -> i64
238func nx_mod__find_head(r: *NxModalityRouter, modality_id: i64) -> i64
252func nx_modality_register_backbone(r: *NxModalityRouter, backbone_id: i64, weights_data: *i64, weights_size: i64, hidden_dim: i64, n_layers: i64) -> i64
280func nx_modality_register_head(r: *NxModalityRouter, modality_id: i64, backbone_id: i64, head_data: *i64, head_size: i64, kind: i64) -> i64
313func nx_modality_lookup_backbone(r: *NxModalityRouter, backbone_id: i64) -> *NxBackbone
320func nx_modality_lookup_head(r: *NxModalityRouter, modality_id: i64) -> *NxModalityHead
330func nx_modality_resolve_backbone(r: *NxModalityRouter, modality_id: i64) -> *NxBackbone
342func nx_modality_unregister_head(r: *NxModalityRouter, modality_id: i64) -> i64
369func nx_modality_evict_backbone(r: *NxModalityRouter, backbone_id: i64) -> i64
388func nx_modality_n_backbones(r: *NxModalityRouter) -> i64
called by 1: main calls 1: nx_modality_router_is_valid
393func nx_modality_n_heads(r: *NxModalityRouter) -> i64
called by 1: main calls 1: nx_modality_router_is_valid
398func nx_modality_backbone_refcount(r: *NxModalityRouter, backbone_id: i64) -> i64
410func nx_modality_count_heads_for_backbone(r: *NxModalityRouter, backbone_id: i64) -> i64
425func nx_modality_head_kind(h: *NxModalityHead) -> i64
called by 1: main calls 1: nx_modality_head_is_valid
430func nx_modality_head_backbone_id(h: *NxModalityHead) -> i64
called by 1: main calls 1: nx_modality_head_is_valid
435func nx_modality_backbone_id(b: *NxBackbone) -> i64
called by 1: main calls 1: nx_backbone_is_valid
440func nx_modality_backbone_hidden_dim(b: *NxBackbone) -> i64
called by 1: main calls 1: nx_backbone_is_valid
445func nx_modality_backbone_n_layers(b: *NxBackbone) -> i64
called by 1: main calls 1: nx_backbone_is_valid