_cdc_fixture_gen.nx source

1// _cdc_fixture_gen.nx -- LOCAL INSTRUMENT (2026-09-06): a seeded pseudo-random fixture generator and a chunk-length 2// census for the content-defined chunker, so a dedupe accept rule is measured on ENTROPY-BEARING bytes. 3// 4// WHY: the first 64 MiB dedupe measurement used a generated PNG whose IDAT is a stored deflate block of a period-3 5// byte pattern (1 MiB gzips to 5,678 B). A gear fingerprint over period-3 content takes three values, none hit the 6// mask, so EVERY cut was forced at max (1339 x 48,402 + 28,970 = 64,839,248 exactly) -- fixed-size chunking wearing 7// a content-defined name -- and a one-byte insertion shifted every later cut, so reuse stopped at the insertion 8// point (661 of 1340 = the offset-preserving prefix, 49.3 percent = 32,000,000 / 64,839,248). That is a property of 9// the DATA every chunker shares; the accept rule needs a fixture whose cuts are content-defined. 10// 11// verbs: 12// gen <out> <bytes> <seed> xorshift64* stream, deterministic for (bytes, seed) 13// insert <in> <out> <offset> <byte> copy with ONE byte inserted at offset (the dedupe workload) 14// chunks <file> <min> <avg> <max> cut with nx_cdc_lib at the door's own parameters; prints count, min/max/avg 15// length, how many cuts were FORCED at max, and the gzip-free entropy proxy 16// (distinct bytes in the first block) 17// exit: 0 ok | 2 usage | 1 io 18import "nx_syscalls.nx" 19import "nx_cdc_lib.nx" 20const FG_BLOCK: i64 = 1048576 21const FG_EXIT_USAGE: i64 = 2 22const FG_EXIT_IO: i64 = 1 23const FG_MODE_644: i64 = 420 24const FG_XS_A: i64 = 12 25const FG_XS_B: i64 = 25 26const FG_XS_C: i64 = 27 27const FG_XS_MUL: i64 = 2685821657736338717 28 29func fg_slen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n } 30func fg_puts(s: *u8) -> i64 { return sys_write(1, s, fg_slen(s)) } 31func fg_putn(v: i64) -> i64 { 32 let b: *u8 = sys_mmap(32) 33 var n: i64 = v 34 var neg: i64 = 0 35 if n < 0 { neg = 1; n = 0 - n } 36 var k: i64 = 0 37 if n == 0 { b[0] = 48 as u8; k = 1 } 38 let t: *u8 = sys_mmap(32) 39 var m: i64 = 0 40 while n > 0 { t[m] = (48 + (n % 10)) as u8; n = n / 10; m = m + 1 } 41 var o: i64 = 0 42 if neg == 1 { b[0] = 45 as u8; o = 1 } 43 while m > 0 { m = m - 1; b[o] = t[m]; o = o + 1 } 44 if o == 0 { o = k } 45 return sys_write(1, b, o) 46} 47func fg_parse(s: *u8) -> i64 { 48 var v: i64 = 0 49 var i: i64 = 0 50 while s[i] != (0 as u8) { let c: i64 = s[i] as i64; if c >= 48 { if c <= 57 { v = v * 10 + (c - 48) } } i = i + 1 } 51 return v 52} 53// xorshift64*: state must be non-zero; the high byte of the multiplied state is the output byte 54func fg_next(st: *i64) -> i64 { 55 var x: i64 = st[0] 56 x = x ^ (x >> FG_XS_A) 57 x = x ^ (x << FG_XS_B) 58 x = x ^ (x >> FG_XS_C) 59 st[0] = x 60 let y: i64 = x * FG_XS_MUL 61 return (y >> 56) & 255 62} 63func fg_write_all(fd: i64, b: *u8, n: i64) -> i64 { 64 var w: i64 = 0 65 while w < n { let r: i64 = sys_write(fd, ((b as i64) + w) as *u8, n - w); if r <= 0 { return 0 } w = w + r } 66 return 1 67} 68func v_gen(out: *u8, bytes: i64, seed: i64) -> i64 { 69 let st: *i64 = sys_mmap(16) as *i64 70 st[0] = seed 71 if st[0] == 0 { st[0] = 1 } 72 let fd: i64 = sys_openat_wr(out, FG_MODE_644) 73 if fd < 0 { fg_puts("FG-IO cannot open output\n" as *u8); return FG_EXIT_IO } 74 let blk: *u8 = sys_mmap(FG_BLOCK) 75 var left: i64 = bytes 76 while left > 0 { 77 var n: i64 = left 78 if n > FG_BLOCK { n = FG_BLOCK } 79 var i: i64 = 0 80 while i < n { blk[i] = fg_next(st) as u8; i = i + 1 } 81 if fg_write_all(fd, blk, n) == 0 { sys_close(fd); fg_puts("FG-IO short write\n" as *u8); return FG_EXIT_IO } 82 left = left - n 83 } 84 sys_close(fd) 85 fg_puts("FG-GEN bytes=" as *u8); fg_putn(bytes); fg_puts(" seed=" as *u8); fg_putn(seed); fg_puts(" out=" as *u8); fg_puts(out); fg_puts("\n" as *u8) 86 return 0 87} 88func v_insert(inp: *u8, out: *u8, off: i64, byte: i64) -> i64 { 89 let lp: *i64 = sys_mmap(16) as *i64 90 let b: *u8 = sys_read_file(inp, lp) 91 if (b as i64) == 0 { fg_puts("FG-IO cannot read input\n" as *u8); return FG_EXIT_IO } 92 let n: i64 = lp[0] 93 if off > n { fg_puts("FG-USAGE offset past end\n" as *u8); return FG_EXIT_USAGE } 94 let fd: i64 = sys_openat_wr(out, FG_MODE_644) 95 if fd < 0 { fg_puts("FG-IO cannot open output\n" as *u8); return FG_EXIT_IO } 96 let one: *u8 = sys_mmap(8) 97 one[0] = byte as u8 98 var ok: i64 = fg_write_all(fd, b, off) 99 if ok == 1 { ok = fg_write_all(fd, one, 1) } 100 if ok == 1 { ok = fg_write_all(fd, ((b as i64) + off) as *u8, n - off) } 101 sys_close(fd) 102 if ok == 0 { fg_puts("FG-IO short write\n" as *u8); return FG_EXIT_IO } 103 fg_puts("FG-INSERT in_bytes=" as *u8); fg_putn(n); fg_puts(" out_bytes=" as *u8); fg_putn(n + 1); fg_puts(" at=" as *u8); fg_putn(off); fg_puts("\n" as *u8) 104 return 0 105} 106func v_chunks(path: *u8, mn: i64, avg: i64, mx: i64) -> i64 { 107 let lp: *i64 = sys_mmap(16) as *i64 108 let b: *u8 = sys_read_file(path, lp) 109 if (b as i64) == 0 { fg_puts("FG-IO cannot read file\n" as *u8); return FG_EXIT_IO } 110 let n: i64 = lp[0] 111 let tbl: *u8 = sys_mmap(CDC_GEAR_BYTES) 112 cdc_gear_table(tbl) 113 let cap: i64 = n / mn + 2 114 let offs: *i64 = sys_mmap(cap * 8) as *i64 115 let count: i64 = cdc_plan(tbl, b, n, mn, avg, mx, offs, cap) 116 if count < 0 { fg_puts("FG-CDC refused parameters\n" as *u8); return FG_EXIT_USAGE } 117 var lmin: i64 = n 118 var lmax: i64 = 0 119 var forced: i64 = 0 120 var k: i64 = 0 121 while k < count { 122 let l: i64 = offs[k + 1] - offs[k] 123 if l < lmin { lmin = l } 124 if l > lmax { lmax = l } 125 if l == mx { forced = forced + 1 } 126 k = k + 1 127 } 128 // entropy proxy: distinct byte values in the first block (256 = entropy-bearing, a handful = periodic) 129 let seen: *u8 = sys_mmap(256) 130 var d: i64 = 0 131 var i: i64 = 0 132 var lim: i64 = n 133 if lim > FG_BLOCK { lim = FG_BLOCK } 134 while i < lim { let c: i64 = b[i] as i64; if seen[c] == (0 as u8) { seen[c] = 1 as u8; d = d + 1 } i = i + 1 } 135 fg_puts("FG-CHUNKS bytes=" as *u8); fg_putn(n) 136 fg_puts(" min=" as *u8); fg_putn(mn); fg_puts(" avg=" as *u8); fg_putn(avg); fg_puts(" max=" as *u8); fg_putn(mx) 137 fg_puts(" count=" as *u8); fg_putn(count) 138 fg_puts(" len_min=" as *u8); fg_putn(lmin); fg_puts(" len_max=" as *u8); fg_putn(lmax) 139 var mean: i64 = 0 140 if count > 0 { mean = n / count } 141 fg_puts(" len_mean=" as *u8); fg_putn(mean) 142 fg_puts(" forced_at_max=" as *u8); fg_putn(forced); fg_puts("/" as *u8); fg_putn(count) 143 fg_puts(" distinct_bytes_first_block=" as *u8); fg_putn(d) 144 fg_puts("\n" as *u8) 145 return 0 146} 147func main(argc: i64, argv: *i64) -> i64 { 148 if argc < 2 { fg_puts("usage: _cdc_fixture_gen gen <out> <bytes> <seed> | insert <in> <out> <offset> <byte> | chunks <file> <min> <avg> <max>\n" as *u8); return FG_EXIT_USAGE } 149 let verb: *u8 = argv[1] as *u8 150 if verb[0] == (103 as u8) { if argc < 5 { fg_puts("usage: gen <out> <bytes> <seed>\n" as *u8); return FG_EXIT_USAGE } return v_gen(argv[2] as *u8, fg_parse(argv[3] as *u8), fg_parse(argv[4] as *u8)) } 151 if verb[0] == (105 as u8) { if argc < 6 { fg_puts("usage: insert <in> <out> <offset> <byte>\n" as *u8); return FG_EXIT_USAGE } return v_insert(argv[2] as *u8, argv[3] as *u8, fg_parse(argv[4] as *u8), fg_parse(argv[5] as *u8)) } 152 if verb[0] == (99 as u8) { if argc < 6 { fg_puts("usage: chunks <file> <min> <avg> <max>\n" as *u8); return FG_EXIT_USAGE } return v_chunks(argv[2] as *u8, fg_parse(argv[3] as *u8), fg_parse(argv[4] as *u8), fg_parse(argv[5] as *u8)) } 153 fg_puts("usage: gen | insert | chunks\n" as *u8) 154 return FG_EXIT_USAGE 155}