code wiki / _hdl_build / nx_seg_columnar.nx
nx_seg_columnar.nx source
↩ module page · 39 lines · 1867 B
1// nx_seg_columnar.nx -- CAP-SEG-COLUMNAR: columnar layout + compression for analytic scans (Parquet/ORC-class).
2// Instead of row records, values of one field are stored CONTIGUOUSLY (a column chunk), so an analytic scan reads
3// just that column -- no row deserialization. Each column is compressed: RLE (col_rle) for runs, DICTIONARY
4// (col_dict) for low-cardinality. Both are exactly invertible (lossless round-trip). A LIBRARY (no main -> run _gate).
5// license_tier: ORIGINAL
6import "nx_syscalls.nx"
7
8// RLE: encode vals[0..n) as (count,value) pairs into out (2 i64 per run); returns run count.
9func col_rle_encode(vals: *i64, n: i64, out: *i64) -> i64 {
10 var i: i64=0; var p: i64=0
11 while i < n {
12 var c: i64=1
13 var go: i64=1
14 while go == 1 { if i+c < n { if vals[i+c]==vals[i] { c=c+1 } else { go=0 } } else { go=0 } }
15 out[p*2]=c; out[p*2+1]=vals[i]; p=p+1; i=i+c
16 }
17 return p
18}
19// RLE: expand `np` (count,value) pairs into out; returns value count.
20func col_rle_decode(pairs: *i64, np: i64, out: *i64) -> i64 {
21 var o: i64=0; var i: i64=0
22 while i < np { let c: i64=pairs[i*2]; let v: i64=pairs[i*2+1]; var k: i64=0; while k<c { out[o]=v; o=o+1; k=k+1 } i=i+1 }
23 return o
24}
25// DICTIONARY: distinct values -> dict[]; per-row id -> ids[]. Returns dictionary size (cardinality).
26func col_dict_encode(vals: *i64, n: i64, dict: *i64, ids: *i64) -> i64 {
27 var dn: i64=0; var i: i64=0
28 while i < n {
29 var j: i64=0; var found: i64=0-1
30 while j < dn { if dict[j]==vals[i] { found=j } j=j+1 }
31 if found < 0 { dict[dn]=vals[i]; found=dn; dn=dn+1 }
32 ids[i]=found; i=i+1
33 }
34 return dn
35}
36// DICTIONARY: reconstruct vals from dict + ids.
37func col_dict_decode(dict: *i64, ids: *i64, n: i64, out: *i64) -> i64 {
38 var i: i64=0; while i < n { out[i]=dict[ids[i]]; i=i+1 } return n
39}