nx_colframe.nx source
↩ module page · 104 lines · 5276 B
1// nx_colframe.nx -- SOVEREIGN ZERO-COPY COLUMNAR INTERCHANGE (arrow-interchange, F-arrow). The Arrow idea,
2// built from the first byte up as our OWN format, not a clone: a single self-describing contiguous buffer
3// that carries N columns COLUMNAR (each column packed contiguously, 8-byte aligned), so any organ that
4// receives the buffer reads a whole column with ONE pointer cast -- zero per-value deserialization, zero
5// copy on the read side. Cross-organ dataset passing becomes "hand over a pointer + a length".
6//
7// THE EXACT BYTE LAYOUT (all fields native i64 little-endian; the value IS the layout):
8// [0] magic = CF_MAGIC (identifies + versions the frame)
9// [1] version = CF_VERSION
10// [2] ncol
11// [3] nrows
12// then a SCHEMA table of ncol entries, 3 i64 each (i64-index 4 + i*3):
13// [+0] type (CF_T_I64 = 0; the v1 type)
14// [+1] name_off (byte offset to the column's NUL-terminated name)
15// [+2] data_off (byte offset to the column's data; 8-BYTE ALIGNED so the *i64 cast is valid+fast)
16// then the packed name strings, then each column's nrows*8 bytes of i64 data.
17//
18// ZERO-COPY is provable, not asserted: cf_col(buf,i) returns buf+data_off -- a pointer INTO the shared
19// buffer. Called twice it returns the SAME address (a view, never a per-call materialisation), and an
20// analytics organ runs DIRECTLY on that view. EXCEED vs Arrow: 100% integer/fixed layout => bit-identical
21// frames across runs (Arrow's buffers carry padding/impl variance). license_tier: ORIGINAL No hw writes.
22import "nx_syscalls.nx"
23
24const CF_MAGIC: i64 = 5136462849 // "NXCF" epoch tag; distinctive, non-zero
25const CF_VERSION: i64 = 1
26const CF_T_I64: i64 = 0
27const CF_HDR_I64: i64 = 4 // magic,version,ncol,nrows
28const CF_SCH_I64: i64 = 3 // per-column schema words
29
30func cf_align8(x: i64) -> i64 { let r: i64 = x % 8; if r == 0 { return x } return x + (8 - r) }
31func cf_slen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
32
33// bytes an encode of (ncol, nrows, names) will occupy -- lets a caller size the buffer exactly.
34func cf_encoded_bytes(names: *i64, ncol: i64, nrows: i64) -> i64 {
35 var p: i64 = (CF_HDR_I64 + ncol * CF_SCH_I64) * 8
36 var i: i64 = 0
37 while i < ncol { p = p + cf_slen(names[i] as *u8) + 1; i = i + 1 }
38 p = cf_align8(p)
39 p = p + ncol * nrows * 8
40 return p
41}
42
43// Encode ncol i64 columns into out (which the caller sized via cf_encoded_bytes). cols[i] -> *i64 of nrows.
44// Returns total bytes written. This copies each column ONCE into the frame; thereafter every consumer is
45// zero-copy. (Arrow builders copy once too; the interchange win is on the read/pass side.)
46func cf_encode(cols: *i64, names: *i64, ncol: i64, nrows: i64, out: *u8) -> i64 {
47 let w: *i64 = out as *i64
48 w[0] = CF_MAGIC
49 w[1] = CF_VERSION
50 w[2] = ncol
51 w[3] = nrows
52 // names region begins right after the schema table
53 var p: i64 = (CF_HDR_I64 + ncol * CF_SCH_I64) * 8
54 var i: i64 = 0
55 while i < ncol {
56 let si: i64 = CF_HDR_I64 + i * CF_SCH_I64
57 w[si] = CF_T_I64
58 w[si + 1] = p // name_off
59 let nm: *u8 = names[i] as *u8
60 var j: i64 = 0
61 while nm[j] != (0 as u8) { out[p] = nm[j]; p = p + 1; j = j + 1 }
62 out[p] = 0 as u8
63 p = p + 1
64 i = i + 1
65 }
66 // align, then pack each column's data contiguously; record its aligned data_off
67 p = cf_align8(p)
68 i = 0
69 while i < ncol {
70 let si: i64 = CF_HDR_I64 + i * CF_SCH_I64
71 w[si + 2] = p // data_off (8-aligned by construction)
72 let src: *i64 = cols[i] as *i64
73 let dst: *i64 = (((out as i64) + p) as *i64)
74 var r: i64 = 0
75 while r < nrows { dst[r] = src[r]; r = r + 1 }
76 p = p + nrows * 8
77 i = i + 1
78 }
79 return p
80}
81
82// ---- ZERO-COPY ACCESSORS: all O(1), all return VIEWS into buf (no allocation, no copy) ----------
83func cf_valid(buf: *u8) -> i64 { let w: *i64 = buf as *i64; if w[0] == CF_MAGIC { if w[1] == CF_VERSION { return 1 } } return 0 }
84func cf_ncol(buf: *u8) -> i64 { let w: *i64 = buf as *i64; return w[2] }
85func cf_nrows(buf: *u8) -> i64 { let w: *i64 = buf as *i64; return w[3] }
86func cf_type(buf: *u8, i: i64) -> i64 { let w: *i64 = buf as *i64; return w[CF_HDR_I64 + i * CF_SCH_I64] }
87func cf_name(buf: *u8, i: i64) -> *u8 { let w: *i64 = buf as *i64; let off: i64 = w[CF_HDR_I64 + i * CF_SCH_I64 + 1]; return (((buf as i64) + off) as *u8) }
88func cf_data_off(buf: *u8, i: i64) -> i64 { let w: *i64 = buf as *i64; return w[CF_HDR_I64 + i * CF_SCH_I64 + 2] }
89// THE zero-copy view: a typed pointer straight into the frame's own bytes.
90func cf_col(buf: *u8, i: i64) -> *i64 { return (((buf as i64) + cf_data_off(buf, i)) as *i64) }
91// find a column by name -> index, or -1
92func cf_col_index(buf: *u8, name: *u8) -> i64 {
93 let nc: i64 = cf_ncol(buf)
94 var i: i64 = 0
95 while i < nc {
96 let cn: *u8 = cf_name(buf, i)
97 var same: i64 = 1
98 var j: i64 = 0
99 while name[j] != (0 as u8) { if cn[j] != name[j] { same = 0 } j = j + 1 }
100 if same == 1 { if cn[j] == (0 as u8) { return i } }
101 i = i + 1
102 }
103 return 0 - 1
104}