nx_media_ingest_pipeline.nx source
↩ module page · 103 lines · 5893 B
1// nx_media_ingest_pipeline.nx -- KEYSTONE (operator 2026-07-14): analysis/derivation-on-INGEST for the gallery.
2// The root cause of "newly-found media has no thumbnail and doesnt play" is that the derivation organs exist but
3// are DISJOINT -- nothing runs them when media lands. This COMPOSES the existing organs (rule-15, no rebuild) into
4// ONE idempotent, resumable pass so media gets its thumbnail + duration/keyframe index at ingest time (not lazily
5// on first play, and never for harvested video). Run modes: default = BACKFILL the whole corpus; the torrent/
6// harvest ingest path can call this same organ after each drop (the per-item hook).
7//
8// ALWAYS-INFORM: every tool is resolved NAS-first (/volume1/ai/galx) with a dev fallback, and a MISSING tool or an
9// unwired stage is REPORTED explicitly -- never a silent skip (operator: "dont return nothing without informing").
10// Usage: nx_media_ingest_pipeline [max_videos] (default = all). license_tier: ORIGINAL
11import "nx_syscalls.nx"
12const GALX_MAGIC_1024: i64 = 1024
13const GALX_MAGIC_4096: i64 = 4096
14
15const GALX_DIR_NAS: *u8 = "/volume1/ai/galx"
16const GALX_DIR_DEV: *u8 = "/mnt/c/Users/elder/nishi-core/nxc2/runtime/_hdl_build"
17
18func mp_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
19func mp_putn(v: i64) -> i64 {
20 let t: *u8 = sys_mmap(28); var m: i64 = v; var k: i64 = 0
21 if m < 0 { sys_write(1, "-" as *u8, 1); m = 0 - m }
22 if m == 0 { t[0] = 48 as u8; k = 1 }
23 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
24 let b: *u8 = sys_mmap(28); var i: i64 = 0
25 while i < k { b[i] = t[k - 1 - i]; i = i + 1 }
26 sys_write(1, b, k); return 0
27}
28func mp_cat(dst: *u8, off: i64, s: *u8) -> i64 { var i: i64 = 0; while s[i] != (0 as u8) { dst[off + i] = s[i]; i = i + 1 } return off + i }
29func mp_exists(path: *u8) -> i64 { let fd: i64 = sys_openat_rd(path); if fd < 0 { return 0 } sys_close(fd); return 1 }
30
31// build <dir>/<name> into out (NUL-terminated); returns length.
32func mp_join(dir: *u8, name: *u8, out: *u8) -> i64 {
33 var o: i64 = mp_cat(out, 0, dir)
34 out[o] = 47 as u8; o = o + 1
35 o = mp_cat(out, o, name); out[o] = 0 as u8
36 return o
37}
38// resolve <name> NAS-first then dev; writes the found path into out; 1 if found, 0 if MISSING (report + skip).
39func mp_resolve(name: *u8, out: *u8) -> i64 {
40 mp_join(GALX_DIR_NAS, name, out)
41 if mp_exists(out) == 1 { return 1 }
42 mp_join(GALX_DIR_DEV, name, out)
43 if mp_exists(out) == 1 { return 1 }
44 return 0
45}
46// fork+exec elf with up to 3 trailing args (0 = arg-list ends); wait; return child exit code.
47func mp_run(elf: *u8, a1: *u8, a2: *u8, a3: *u8) -> i64 {
48 let pid: i64 = sys_fork()
49 if pid == 0 {
50 let av: *i64 = sys_mmap(64) as *i64
51 av[0] = elf as i64; av[1] = a1 as i64; av[2] = a2 as i64; av[3] = a3 as i64; av[4] = 0
52 let ev: *i64 = sys_mmap(8) as *i64; ev[0] = 0
53 sys_execve(elf, av, ev); sys_exit(127)
54 }
55 let st: *i64 = sys_mmap(16) as *i64
56 sys_wait4(pid, st, 0)
57 return (st[0] >> 8) & 0xff
58}
59
60func main(argc: i64, argv: *i64) -> i64 {
61 var maxs: *u8 = "1000000" as *u8
62 if argc >= 2 { maxs = argv[1] as *u8 }
63 mp_puts("=== nx_media_ingest_pipeline (analysis-on-ingest; idempotent; resumable) ===\n" as *u8)
64
65 // ---- STAGE A: image thumbnails -- prewarm thumbs/<cid>.jpg for every cid lacking one (cold grid fix) ----
66 var okA: i64 = 0
67 let prewarm: *u8 = sys_mmap(GALX_MAGIC_4096)
68 let thumbelf: *u8 = sys_mmap(GALX_MAGIC_4096)
69 if mp_resolve("nx_galx_thumb_prewarm.elf" as *u8, prewarm) == 1 {
70 if mp_resolve("nx_galx_thumb.elf" as *u8, thumbelf) == 1 {
71 mp_puts("[A] image thumbnails: prewarming missing thumbs...\n" as *u8)
72 let rc: i64 = mp_run(prewarm, "knowledge/status/galx_cid_paths.tsv" as *u8, "thumbs" as *u8, thumbelf)
73 mp_puts(" thumb_prewarm exit=" as *u8); mp_putn(rc); mp_puts("\n" as *u8)
74 okA = 1
75 } else { mp_puts("[A] MISSING nx_galx_thumb.elf under /volume1/ai/galx -> deploy it (thumbnails NOT prewarmed)\n" as *u8) }
76 } else { mp_puts("[A] MISSING nx_galx_thumb_prewarm.elf -> deploy it (thumbnails NOT prewarmed)\n" as *u8) }
77
78 // ---- STAGE B: video duration/keyframe/markers -- NXVI backfill for every recording lacking a v2 index ----
79 var okB: i64 = 0
80 let backfill: *u8 = sys_mmap(GALX_MAGIC_4096)
81 if mp_resolve("nx_galx_idx_backfill.elf" as *u8, backfill) == 1 {
82 mp_puts("[B] video NXVI (duration/seek/markers): backfilling...\n" as *u8)
83 let rc2: i64 = mp_run(backfill, maxs, 0 as *u8, 0 as *u8)
84 mp_puts(" idx_backfill exit=" as *u8); mp_putn(rc2); mp_puts("\n" as *u8)
85 okB = 1
86 } else { mp_puts("[B] MISSING nx_galx_idx_backfill.elf -> deploy it (durations NOT built)\n" as *u8) }
87
88 // ---- STAGE C: video posters -- HONEST GAP: harvested video has no -poster.jpg and there is no keyframe->JPEG
89 // extractor yet, so /vidthumb 404s = blank tile. Reported, not silently skipped. Next rung = nx_galx_vidposter.
90 var okC: i64 = 0
91 let poster: *u8 = sys_mmap(GALX_MAGIC_1024)
92 if mp_resolve("nx_cam_poster_batch.elf" as *u8, poster) == 1 {
93 mp_puts("[C] video posters: generating missing -poster.jpg siblings (idempotent skip-existing)...\n" as *u8)
94 let rc3: i64 = mp_run(poster, "0" as *u8, maxs, 0 as *u8)
95 mp_puts(" cam_poster_batch exit=" as *u8); mp_putn(rc3); mp_puts("\n" as *u8)
96 okC = 1
97 } else { mp_puts("[C] MISSING nx_cam_poster_batch.elf -> deploy it (video tiles stay blank)\n" as *u8) }
98
99 mp_puts("=== pipeline done: thumbs_stage=" as *u8); mp_putn(okA)
100 mp_puts(" nxvi_stage=" as *u8); mp_putn(okB); mp_puts(" posters=" as *u8); mp_putn(okC); mp_puts(" ===\n" as *u8)
101 if okA == 1 { if okB == 1 { sys_exit(0); return 0 } }
102 sys_exit(1); return 1
103}