nx_record_tsv.nx source
↩ module page · 142 lines · 6483 B
1// nx_record_tsv.nx -- the TSV <-> NXR1 COMPATIBILITY BRIDGE. This is what makes seq1326 rung 2 (adoption)
2// possible WITHOUT a flag day.
3//
4// THE ADOPTION PROBLEM: every plane's decoder today splits on TAB and counts columns by position. Changing
5// the physical record format breaks all of them at once, so "migrate the plane" has meant "rewrite every
6// reader in the same commit" -- which is why the retirement never happened and 289 files are still TSV.
7//
8// THE FIX is the same one Iceberg uses for format evolution: separate the PHYSICAL encoding from the
9// LOGICAL view. Store NXR1 (typed, id-addressed, delimiter-free). Render a TSV line ON DEMAND for any
10// reader that still wants one. New writers and new readers use the typed API and get all four structural
11// properties; legacy readers keep working byte-for-byte (rule 19) until they are migrated one at a time.
12//
13// ★THE COMPATIBILITY VIEW MUST REFUSE TO LIE. If a stored record holds a value TSV cannot represent -- one
14// containing a TAB or a NEWLINE -- rtv_to_tsv REFUSES with a negative code instead of emitting a line that
15// would silently parse as extra columns or an extra row. A lossy view that fails loudly is a bridge; a
16// lossy view that fails silently is the corruption class we are retiring.
17//
18// COLUMN <-> FIELD ID: column i (0-based) maps to field_id i+1. 1-based so id 0 stays reserved, and STABLE:
19// once a column has an id, reordering or dropping columns never re-points an existing id.
20//
21// EMPTY vs ABSENT: a TSV empty column round-trips as a PRESENT field of length 0, which is a real semantic
22// gain -- NXR1 can distinguish "known to be empty" from "not recorded". The reverse direction cannot:
23// rendering collapses absent and empty to the same empty column, because TSV has no way to say otherwise.
24// That asymmetry is inherent to TSV and is documented here rather than hidden.
25// license_tier: ORIGINAL No hw writes (Rule 26).
26import "nx_record.nx"
27
28const RTV_TAB: i64 = 9
29const RTV_NL: i64 = 10
30const RTV_MINUS: i64 = 45
31const RTV_ZERO: i64 = 48
32const RTV_NINE: i64 = 57
33const RTV_BASE10: i64 = 10
34const RTV_NUMBUF: i64 = 32
35const RTV_OUTS: i64 = 32
36const RTV_ERR_TAB_IN_VALUE: i64 = 0 - 1 // record holds a byte TSV cannot represent
37const RTV_ERR_CAP: i64 = 0 - 2 // rendered line would exceed the caller buffer
38const RTV_MAXCOL: i64 = 512
39
40// parse a decimal integer from p[0..n). Leading '-' honoured; non-digits are skipped rather than trusted,
41// which matches how the existing flat readers behave -- the point of this bridge is to move data, not to
42// retroactively reject rows the old format already accepted.
43func rtv_atoi(p: *u8, n: i64) -> i64 {
44 var v: i64 = 0
45 var i: i64 = 0
46 var neg: i64 = 0
47 if n > 0 { if p[0] == (RTV_MINUS as u8) { neg = 1; i = 1 } }
48 while i < n {
49 let c: i64 = p[i]
50 if c >= RTV_ZERO { if c <= RTV_NINE { v = v * RTV_BASE10 + (c - RTV_ZERO) } }
51 i = i + 1
52 }
53 if neg == 1 { return 0 - v }
54 return v
55}
56
57// append the decimal form of v at out[off]; returns the new offset.
58func rtv_itoa(out: *u8, off: i64, v: i64) -> i64 {
59 var m: i64 = v
60 var o: i64 = off
61 if m < 0 { m = 0 - m; out[o] = RTV_MINUS as u8; o = o + 1 }
62 let t: *u8 = sys_mmap(RTV_NUMBUF)
63 var k: i64 = 0
64 if m == 0 { t[0] = RTV_ZERO as u8; k = 1 }
65 while m > 0 { t[k] = (RTV_ZERO + (m % RTV_BASE10)) as u8; m = m / RTV_BASE10; k = k + 1 }
66 var i: i64 = 0
67 while i < k { out[o + i] = t[k - 1 - i]; i = i + 1 }
68 sys_munmap(t, RTV_NUMBUF)
69 return o + k
70}
71
72// TSV LINE -> NXR1 RECORD. types[i] gives column i's NXR type (NXR_T_I64 renders/parses as an integer,
73// anything else is stored as raw bytes). Columns past ntypes default to NXR_T_STR. Returns record length.
74func rtv_to_nxr(line: *u8, n: i64, types: *u8, ntypes: i64, out: *u8) -> i64 {
75 var o: i64 = nxr_init(out)
76 var col: i64 = 0
77 var start: i64 = 0
78 var i: i64 = 0
79 while i <= n {
80 var atend: i64 = 0
81 if i == n { atend = 1 } else { if line[i] == (RTV_TAB as u8) { atend = 1 } }
82 if atend == 1 {
83 let flen: i64 = i - start
84 var ty: i64 = NXR_T_STR
85 if col < ntypes { ty = types[col] }
86 let fp: *u8 = (line as i64 + start) as *u8
87 if ty == NXR_T_I64 {
88 o = nxr_add_i64(out, o, col + 1, rtv_atoi(fp, flen))
89 } else {
90 o = nxr_add(out, o, col + 1, ty, fp, flen)
91 }
92 col = col + 1
93 start = i + 1
94 }
95 i = i + 1
96 }
97 return o
98}
99
100// NXR1 RECORD -> TSV LINE, for legacy readers. ncols columns are emitted (field ids 1..ncols).
101// REFUSES (negative) rather than emitting a line that would mis-parse. Returns line length.
102func rtv_to_tsv(rec: *u8, n: i64, ncols: i64, out: *u8, outcap: i64) -> i64 {
103 let outs: *i64 = sys_mmap(RTV_OUTS) as *i64
104 var o: i64 = 0
105 var col: i64 = 0
106 var err: i64 = 0
107 while col < ncols {
108 if col > 0 {
109 if o + 1 > outcap { err = RTV_ERR_CAP; col = ncols } else { out[o] = RTV_TAB as u8; o = o + 1 }
110 }
111 if err == 0 {
112 if nxr_find(rec, n, col + 1, outs) == NXR_FOUND {
113 if outs[2] == NXR_T_I64 {
114 if o + RTV_NUMBUF > outcap { err = RTV_ERR_CAP; col = ncols } else {
115 o = rtv_itoa(out, o, nxr_r64(rec, outs[0]))
116 }
117 } else {
118 let vo: i64 = outs[0]
119 let vl: i64 = outs[1]
120 if o + vl > outcap { err = RTV_ERR_CAP; col = ncols } else {
121 var k: i64 = 0
122 while k < vl {
123 let c: i64 = rec[vo + k]
124 // THE REFUSAL: this byte cannot survive a TSV line. Fail loud, never emit.
125 if c == RTV_TAB { err = RTV_ERR_TAB_IN_VALUE; k = vl; col = ncols } else {
126 if c == RTV_NL { err = RTV_ERR_TAB_IN_VALUE; k = vl; col = ncols } else {
127 out[o + k] = c as u8
128 k = k + 1
129 }
130 }
131 }
132 if err == 0 { o = o + vl }
133 }
134 }
135 }
136 }
137 if err == 0 { col = col + 1 }
138 }
139 sys_munmap(outs as *u8, RTV_OUTS)
140 if err != 0 { return err }
141 return o
142}