nx_mvault_ingest.nx source

1// nx_mvault_ingest.nx -- vault classify-at-the-door: shallow faceted split, 2// two FULLY ORTHOGONAL axes so everything splits gen-vs-real. 3// 4// BROWSE MODEL (operator 2026-07-17, "i dont want to walk a huge tree ... 5// gifs together, stories together, pages/warc together, video together, 6// major sections gen vs real ... 3d models and audiobooks ... split as gen 7// or real ... origin/source as a VISIBLE TAG i can see as i browse"): 8// MAJOR axis = provenance CLASS: gen (AI) vs real (downloaded/captured). 9// GROUP axis = media TYPE: image / gif / video / audio / story / page(WARC) 10// / manga / model3d / audiobook -- "gifs together, ...". 11// VISIBLE TAG = the SOURCE string (tumblr/reddit/redgifs/cam/...), a badge 12// + filter, NOT a folder. 13// CLASS and TYPE are independent inputs (a gen 3D model and a real audiobook 14// both classify correctly); source is passthrough metadata. 15// 16// THE SPLIT FIX: download sources MIX types, so TYPE is keyed on the ACTUAL 17// bytes (nx_media_sniff) -- never on the source. A type_hint covers the types 18// that are not byte-sniffable (manga pages, audiobooks, some 3D). 19// 20// SOTA STORAGE: this feeds nx_mvault_record -> canonical NXR1 CID records on 21// the seg-store (NOT tsv); the str mappers below are the record field values. 22// Classify-at-the-door MLS stamp keeps private content in the TS index. 23// 24// NOTE: nx_vault.nx = the SECRETS vault; distinct nx_mvault_/mv_ prefix. 25// Composes: nx_media_sniff.nx (nx_sniff), nx_media_pool.nx (NX_MEDIA_TYPE_*). 26// license_tier: ORIGINAL 27 28import "nx_media_pool.nx" 29import "nx_media_sniff.nx" 30 31// ===== MAJOR axis: provenance class ===== 32const MV_CLASS_UNKNOWN: i64 = 0 33const MV_CLASS_GEN: i64 = 1 34const MV_CLASS_REAL: i64 = 2 35 36// ===== GROUP axis: media type (additive-only; codes stable forever) ===== 37const MV_TYPE_UNKNOWN: i64 = 0 38const MV_TYPE_IMAGE: i64 = 1 39const MV_TYPE_GIF: i64 = 2 40const MV_TYPE_VIDEO: i64 = 3 41const MV_TYPE_AUDIO: i64 = 4 42const MV_TYPE_STORY: i64 = 5 // text 43const MV_TYPE_PAGE: i64 = 6 // archived web page (WARC/HTML) 44const MV_TYPE_MANGA: i64 = 7 // multi-page manga/comic (image-sequence work) 45const MV_TYPE_MODEL3D: i64 = 8 // 3D model / mesh 46const MV_TYPE_AUDIOBOOK: i64 = 9 // long-form narrated audio 47const MV_TYPE_BOOK: i64 = 10 // ebook (epub/pdf) / text book 48 49// ===== provenance input (-> class) ===== 50const MV_PROV_UNKNOWN: i64 = 0 51const MV_PROV_GEN: i64 = 1 52const MV_PROV_REAL: i64 = 2 53 54// ===== type hint (-> for the non-byte-sniffable types; NONE = sniff) ===== 55const MV_HINT_NONE: i64 = 0 56const MV_HINT_MANGA: i64 = 1 57const MV_HINT_AUDIOBOOK: i64 = 2 58const MV_HINT_MODEL3D: i64 = 3 59 60// ===== MLS sensitivity (Bell-LaPadula); PROVISIONAL -- reconcile with the 61// librarian/info-finding canonical U/C/S/TS lib when it lands. 62const MV_LVL_U: i64 = 0 63const MV_LVL_C: i64 = 1 64const MV_LVL_S: i64 = 2 65const MV_LVL_TS: i64 = 3 66 67// class from provenance (orthogonal to type). 68func mv_class(prov: i64) -> i64 { 69 if prov == MV_PROV_GEN { return MV_CLASS_GEN } 70 if prov == MV_PROV_REAL { return MV_CLASS_REAL } 71 return MV_CLASS_UNKNOWN 72} 73 74// glTF binary magic ("glTF") -- the one 3D format with a clean magic. 75func mv_is_glb(b: *u8, n: i64) -> i64 { 76 if n < 4 { return 0 } 77 if b[0] != (0x67 as u8) { return 0 } // g 78 if b[1] != (0x6C as u8) { return 0 } // l 79 if b[2] != (0x54 as u8) { return 0 } // T 80 if b[3] != (0x46 as u8) { return 0 } // F 81 return 1 82} 83 84// type group from a hint (wins) or the sniffed content type. 85func mv_type(hint: i64, mtype: i64) -> i64 { 86 if hint == MV_HINT_MANGA { return MV_TYPE_MANGA } 87 if hint == MV_HINT_AUDIOBOOK { return MV_TYPE_AUDIOBOOK } 88 if hint == MV_HINT_MODEL3D { return MV_TYPE_MODEL3D } 89 if mtype == NX_MEDIA_TYPE_IMAGE { return MV_TYPE_IMAGE } 90 if mtype == NX_MEDIA_TYPE_GIF { return MV_TYPE_GIF } 91 if mtype == NX_MEDIA_TYPE_VIDEO { return MV_TYPE_VIDEO } 92 if mtype == NX_MEDIA_TYPE_AUDIO { return MV_TYPE_AUDIO } 93 if mtype == NX_MEDIA_TYPE_TEXT { return MV_TYPE_STORY } 94 if mtype == NX_MEDIA_TYPE_HTML { return MV_TYPE_PAGE } 95 // Added 2026-07-30. These five codes previously had NO mapping, which is 96 // the reason MV_TYPE_BOOK and MV_TYPE_MODEL3D were declared but 97 // structurally unreachable from bytes: the sniffer had no code to return 98 // and this function had no branch to receive one. WARC folds into PAGE -- 99 // an archived web capture IS the page type this axis already declares. 100 if mtype == NX_MEDIA_TYPE_MODEL3D { return MV_TYPE_MODEL3D } 101 if mtype == NX_MEDIA_TYPE_BOOK { return MV_TYPE_BOOK } 102 if mtype == NX_MEDIA_TYPE_AUDIOBOOK { return MV_TYPE_AUDIOBOOK } 103 if mtype == NX_MEDIA_TYPE_MANGA { return MV_TYPE_MANGA } 104 if mtype == NX_MEDIA_TYPE_WARC { return MV_TYPE_PAGE } 105 // NX_MEDIA_TYPE_ARCHIVE deliberately falls through to UNKNOWN: a bare 106 // zip/rar with no distinguishing extension is a container, not media. 107 return MV_TYPE_UNKNOWN 108} 109 110// Sniff raw bytes then group -- the per-item split for a MIXED source. 111func mv_type_bytes(hint: i64, b: *u8, n: i64, 112 mime: *u8, mlen: i64, path: *u8, plen: i64) -> i64 { 113 if hint != MV_HINT_NONE { return mv_type(hint, NX_MEDIA_TYPE_UNKNOWN) } 114 if mv_is_glb(b, n) == 1 { return MV_TYPE_MODEL3D } 115 let mtype: i64 = nx_sniff(b, n, mime, mlen, path, plen) 116 return mv_type(MV_HINT_NONE, mtype) 117} 118 119// Classify-at-the-door MLS level. NO-WRITE-DOWN: default TS; U only when the 120// item is explicitly flagged approved-for-public-release. 121func mv_level(public_release: i64) -> i64 { 122 if public_release == 1 { return MV_LVL_U } 123 return MV_LVL_TS 124} 125 126// ===== canonical string values for the NXR1 record fields ===== 127func mv_class_str(c: i64) -> *u8 { 128 let g: *u8 = "gen" as *u8 129 let r: *u8 = "real" as *u8 130 let u: *u8 = "unknown" as *u8 131 if c == MV_CLASS_GEN { return g } 132 if c == MV_CLASS_REAL { return r } 133 return u 134} 135 136func mv_type_str(t: i64) -> *u8 { 137 let s_img: *u8 = "image" as *u8 138 let s_gif: *u8 = "gif" as *u8 139 let s_vid: *u8 = "video" as *u8 140 let s_aud: *u8 = "audio" as *u8 141 let s_sto: *u8 = "story" as *u8 142 let s_pag: *u8 = "page" as *u8 143 let s_man: *u8 = "manga" as *u8 144 let s_mod: *u8 = "model3d" as *u8 145 let s_abk: *u8 = "audiobook" as *u8 146 let s_bok: *u8 = "book" as *u8 147 let s_unk: *u8 = "unknown" as *u8 148 if t == MV_TYPE_IMAGE { return s_img } 149 if t == MV_TYPE_GIF { return s_gif } 150 if t == MV_TYPE_VIDEO { return s_vid } 151 if t == MV_TYPE_AUDIO { return s_aud } 152 if t == MV_TYPE_STORY { return s_sto } 153 if t == MV_TYPE_PAGE { return s_pag } 154 if t == MV_TYPE_MANGA { return s_man } 155 if t == MV_TYPE_MODEL3D { return s_mod } 156 if t == MV_TYPE_AUDIOBOOK { return s_abk } 157 if t == MV_TYPE_BOOK { return s_bok } 158 return s_unk 159} 160 161func mv_level_str(l: i64) -> *u8 { 162 let s_u: *u8 = "U" as *u8 163 let s_c: *u8 = "C" as *u8 164 let s_s: *u8 = "S" as *u8 165 let s_ts: *u8 = "TS" as *u8 166 if l == MV_LVL_U { return s_u } 167 if l == MV_LVL_C { return s_c } 168 if l == MV_LVL_S { return s_s } 169 return s_ts 170}