nx_ingest.nx source
↩ module page · 138 lines · 7564 B
1// nx_ingest.nx -- MLIB-004 (CALLOUT-012). The INGESTION SOURCE-ADAPTER FRAMEWORK.
2// DATA-DRIVEN + GROWABLE: reads knowledge/registry/media_sources.tsv (a new endpoint = a row),
3// builds source_id -> drm_policy. The uniform adapter contract per item:
4// 1. POLICY: look up the item's source policy; if not "drm-free" -> REFUSE (no DRM
5// circumvention -- the hard line, gate RED-fails any DRM source ingest).
6// 2. FINGERPRINT: content CID via nx_canon_cid cid_of (= MLIB-003 dedup key).
7// 3. IDEMPOTENT INGEST (rule 10): if the fingerprint was already seen -> DEDUP-skip, else ingest.
8// Live external fetch (Humble/Fanatical/... over their APIs) rides the sovereign HTTP stack once
9// the external-HTTPS TLS-FIN gap clears (X-CENSUS-003); here the contract+policy+dedup is proven
10// on a deterministic local fixture. Self-validating gate (INGESTGATE): two-pass idempotency +
11// DRM-refused hard line + content-dedup. license_tier: ORIGINAL
12import "nx_syscalls.nx"
13import "nx_canon_cid.nx"
14const IN_MAGIC_20000: i64 = 20000
15
16const IN_MAXS: i64 = 64
17const IN_REG: *u8 = "knowledge/registry/media_sources.tsv"
18
19func in_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
20func in_putn(v: i64) -> i64 { let bb: *u8 = sys_mmap(28); var m: i64=v; if m<0 {m=0-m}; let t: *u8 = sys_mmap(28); var k: i64=0; if m==0 {t[0]=48;k=1}; while m>0 {t[k]=48+(m%10); m=m/10; k=k+1}; var i: i64=0; while i<k {bb[i]=t[k-1-i]; i=i+1}; sys_write(1, bb, k); return 0 }
21func in_strlen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
22
23func in_read_file(path: *u8, buf: *u8, cap: i64) -> i64 {
24 let fd: i64 = sys_openat_rd(path)
25 if fd < 0 { return 0 - 1 }
26 var tot: i64 = 0
27 var r: i64 = 1
28 while r > 0 { let dst: *u8 = ((buf as i64) + tot) as *u8; r = sys_read(fd, dst, cap - tot); if r > 0 { tot = tot + r } }
29 sys_close(fd)
30 return tot
31}
32func in_scan(buf: *u8, n: i64, start: i64, delim: i64) -> i64 { var i: i64 = start; var s: i64 = 1; while s == 1 { if i >= n { s = 0 } else { if buf[i] == (delim as u8) { s = 0 } else { i = i + 1 } } } return i }
33
34// parse media_sources.tsv -> sid[] (source_id), spol[] (drm_policy). returns count.
35func in_parse_reg(buf: *u8, blen: i64, sid: *i64, spol: *i64) -> i64 {
36 var c: i64 = 0
37 var p: i64 = 0
38 while p < blen {
39 if buf[p] == (35 as u8) { let e: i64 = in_scan(buf, blen, p, 10); p = e + 1 }
40 else { if buf[p] == (10 as u8) { p = p + 1 }
41 else {
42 let a0: i64 = p
43 let t1: i64 = in_scan(buf, blen, a0, 9); buf[t1] = 0 as u8 // source_id
44 let a1: i64 = t1 + 1
45 let t2: i64 = in_scan(buf, blen, a1, 9); buf[t2] = 0 as u8 // kind (skip)
46 let a2: i64 = t2 + 1
47 let t3: i64 = in_scan(buf, blen, a2, 9); buf[t3] = 0 as u8 // drm_policy
48 let t4: i64 = in_scan(buf, blen, t3 + 1, 10); buf[t4] = 0 as u8 // status (skip)
49 if c < IN_MAXS { sid[c] = (buf as i64) + a0; spol[c] = (buf as i64) + a2; c = c + 1 }
50 p = t4 + 1
51 } }
52 }
53 return c
54}
55
56// policy string for a source id (or 0 if unknown source).
57func in_policy_of(sid: *i64, spol: *i64, ns: i64, src: *u8) -> i64 {
58 var i: i64 = 0
59 while i < ns { if cc_cmp((sid[i]) as *u8, src) == 0 { return spol[i] } i = i + 1 }
60 return 0
61}
62
63func in_seen(seen: *i64, ns: i64, fp: *u8) -> i64 { var i: i64 = 0; while i < ns { if cc_cmp((seen[i]) as *u8, fp) == 0 { return 1 } i = i + 1 } return 0 }
64
65// classify one item: 2=REFUSE(policy), 1=DEDUP, 0=INGEST. mutates seen/nseen on ingest.
66func in_classify(src: *u8, content: *u8, sid: *i64, spol: *i64, ns: i64, seen: *i64, nseen: *i64) -> i64 {
67 let pol: i64 = in_policy_of(sid, spol, ns, src)
68 if pol == 0 { return 2 }
69 if cc_cmp(pol as *u8, "drm-free" as *u8) != 0 { return 2 }
70 let fp: *u8 = sys_mmap(80)
71 cid_of(content, in_strlen(content), fp)
72 if in_seen(seen, nseen[0], fp) == 1 { return 1 }
73 seen[nseen[0]] = (fp as *u8) as i64
74 nseen[0] = nseen[0] + 1
75 return 0
76}
77
78// run one pass over all items; counts[0]=ingested counts[1]=deduped counts[2]=refused.
79func in_run(N: i64, srcA: *i64, conA: *i64, sid: *i64, spol: *i64, ns: i64, seen: *i64, nseen: *i64, counts: *i64, verbose: i64) -> i64 {
80 counts[0] = 0; counts[1] = 0; counts[2] = 0
81 var i: i64 = 0
82 while i < N {
83 let cls: i64 = in_classify((srcA[i]) as *u8, (conA[i]) as *u8, sid, spol, ns, seen, nseen)
84 if cls == 0 { counts[0] = counts[0] + 1; if verbose == 1 { in_puts("INGEST src=" as *u8); in_puts((srcA[i]) as *u8); in_puts("\n" as *u8) } }
85 if cls == 1 { counts[1] = counts[1] + 1; if verbose == 1 { in_puts("DEDUP src=" as *u8); in_puts((srcA[i]) as *u8); in_puts(" (content already ingested)\n" as *u8) } }
86 if cls == 2 { counts[2] = counts[2] + 1; if verbose == 1 { in_puts("REFUSE src=" as *u8); in_puts((srcA[i]) as *u8); in_puts(" (not drm-free -- no circumvention)\n" as *u8) } }
87 i = i + 1
88 }
89 return 0
90}
91
92func main() -> i64 {
93 let rb: *u8 = sys_mmap(IN_MAGIC_20000)
94 let rbn: i64 = in_read_file(IN_REG, rb, IN_MAGIC_20000)
95 if rbn <= 0 { in_puts("INGESTGATE verdict=RED reason=registry-unreadable\n" as *u8); return 1 }
96 let sid: *i64 = sys_mmap(8 * IN_MAXS) as *i64
97 let spol: *i64 = sys_mmap(8 * IN_MAXS) as *i64
98 let ns: i64 = in_parse_reg(rb, rbn, sid, spol)
99
100 // deterministic local fixture: (source, content). idx1 = same content as idx0 (dup);
101 // drmstore items must be REFUSED by policy.
102 let N: i64 = 8
103 let srcA: *i64 = sys_mmap(8 * N) as *i64
104 let conA: *i64 = sys_mmap(8 * N) as *i64
105 srcA[0]=("humble" as *u8) as i64; conA[0]=("AAA" as *u8) as i64
106 srcA[1]=("humble" as *u8) as i64; conA[1]=("AAA" as *u8) as i64
107 srcA[2]=("fanatical" as *u8) as i64; conA[2]=("BBB" as *u8) as i64
108 srcA[3]=("gog" as *u8) as i64; conA[3]=("CCC" as *u8) as i64
109 srcA[4]=("bandcamp" as *u8) as i64; conA[4]=("DDD" as *u8) as i64
110 srcA[5]=("drivethru" as *u8) as i64; conA[5]=("EEE" as *u8) as i64
111 srcA[6]=("drmstore" as *u8) as i64; conA[6]=("FFF" as *u8) as i64
112 srcA[7]=("drmstore" as *u8) as i64; conA[7]=("GGG" as *u8) as i64
113
114 let seen: *i64 = sys_mmap(8 * 128) as *i64
115 let nseen: *i64 = sys_mmap(8) as *i64
116 nseen[0] = 0
117 let c1: *i64 = sys_mmap(8 * 4) as *i64
118 let c2: *i64 = sys_mmap(8 * 4) as *i64
119
120 in_puts("-- pass 1 (cold) --\n" as *u8)
121 in_run(N, srcA, conA, sid, spol, ns, seen, nseen, c1, 1)
122 in_run(N, srcA, conA, sid, spol, ns, seen, nseen, c2, 0) // pass 2 (warm) -- idempotency
123
124 in_puts("INGESTGATE sources=" as *u8); in_putn(ns)
125 in_puts(" pass1[ingested=" as *u8); in_putn(c1[0]); in_puts(" deduped=" as *u8); in_putn(c1[1]); in_puts(" refused=" as *u8); in_putn(c1[2])
126 in_puts("] pass2[ingested=" as *u8); in_putn(c2[0]); in_puts(" deduped=" as *u8); in_putn(c2[1]); in_puts(" refused=" as *u8); in_putn(c2[2]); in_puts("]" as *u8)
127
128 var ok: i64 = 1
129 if c1[0] != 5 { ok = 0 } // 6 drm-free items, 5 unique contents
130 if c1[1] != 1 { ok = 0 } // 1 content-dup deduped
131 if c1[2] != 2 { ok = 0 } // 2 DRM items refused (hard line)
132 if c2[0] != 0 { ok = 0 } // idempotent: nothing new on pass 2
133 if c2[1] != 6 { ok = 0 } // all 6 drm-free now dedup-skipped
134 if c2[2] != 2 { ok = 0 } // DRM still refused
135 if ok == 1 { in_puts(" verdict=GREEN\n" as *u8); return 0 }
136 in_puts(" verdict=RED\n" as *u8)
137 return 1
138}