code wiki / _hdl_build / nx_dataset_export.nx
`nx_dataset_export.nx` source

↩ module page · 633 lines · 26382 B
1// nx_dataset_export.nx -- IMS Thrust D R2: BYTE-FAITHFUL multi-format dataset export.
2//
3// ONE source dataset -> THREE serializations (CSV / JSON / TSV) that are ROUND-TRIPPABLE: parse any of
4// the three back and you recover the IDENTICAL records (same headers, same cells, byte-exact). This is
5// the "dataset in lots of formats to download" half of the arc -- the data a chart visualises, offered
6// for download in the format the consumer wants, with no lossy drift between them.
7//
8// REUSE-vs-NEW: the UXF family (nx_uxf_tsv_migrate / nx_canon_cid) models "one source -> ONE content-
9// addressed CID for dedup/round-trip-to-CID" -- a DIFFERENT goal (it does not emit human-readable CSV/
10// JSON/TSV text files a browser can download). The CSV escaping RULES here follow RFC 4180 exactly as
11// nx_chem_report_csv established (quote a field with , " CR LF; double internal "); the JSON string
12// escaping follows RFC 8259 as json_emit established. We implement the 3 generic emitters + 3 parsers
13// directly over a tiny generic ROW MODEL (no per-schema code) -- the gap those organs do not fill.
14//
15// ROW MODEL (NxDataset): nrows records, each with ncols string fields, in flat parallel arrays
16//   hdr_ptr[c]/hdr_len[c]              -- the c-th column header
17//   cell_ptr[r*ncols+c]/cell_len[..]  -- the field at (row r, col c)
18// Fields are arbitrary bytes (may contain commas, quotes, tabs, newlines). PURE: no I/O here; the
19// caller owns buffers (sys_mmap scratch only). license_tier: ORIGINAL
20import "nx_syscalls.nx"
21const DX_MAGIC_4096: i64 = 4096
22const DX_MAGIC_1024: i64 = 1024
23
24// ===== sealed verdict surface (codes 2910-2919) =====
25const DX_OK:         i64 = 0
26const DX_BAD_INPUT:  i64 = 2910
27const DX_OVERFLOW:   i64 = 2911
28const DX_PARSE:      i64 = 2912
29
30const DX_MAXCOLS:    i64 = 32
31const DX_MAXROWS:    i64 = 256
32const DX_MAXCELLS:   i64 = 8192      // DX_MAXROWS * DX_MAXCOLS ceiling guard
33
34// the generic dataset. cell at (r,c) lives at index r*ncols + c.
35struct NxDataset {
  ncols:    i64
  nrows:    i64
  hdr_ptr:  *i64
  hdr_len:  *i64
  cell_ptr: *i64
  cell_len: *i64
  valid:    i64
43}
44
45// allocate the parallel arrays for a dataset of up to DX_MAXROWS x DX_MAXCOLS.
46func dx_alloc(ds: *NxDataset, ncols: i64, nrows: i64) -> i64 {
  if ncols < 0 { return 0 - DX_BAD_INPUT }
  if ncols > DX_MAXCOLS { return 0 - DX_BAD_INPUT }
  if nrows < 0 { return 0 - DX_BAD_INPUT }
  if nrows > DX_MAXROWS { return 0 - DX_BAD_INPUT }
  ds.ncols = ncols
  ds.nrows = nrows
  ds.hdr_ptr  = sys_mmap(DX_MAXCOLS * 8) as *i64
  ds.hdr_len  = sys_mmap(DX_MAXCOLS * 8) as *i64
  ds.cell_ptr = sys_mmap(DX_MAXCELLS * 8) as *i64
  ds.cell_len = sys_mmap(DX_MAXCELLS * 8) as *i64
  ds.valid = 1
  return DX_OK
59}
60
61// set the c-th header to the NUL-terminated string s.
62func dx_set_hdr(ds: *NxDataset, c: i64, s: *u8) -> i64 {
  if ds.valid != 1 { return 0 - DX_BAD_INPUT }
  if c < 0 { return 0 - DX_BAD_INPUT }
  if c >= ds.ncols { return 0 - DX_BAD_INPUT }
  var n: i64 = 0
  while s[n] != (0 as u8) { n = n + 1 }
  ds.hdr_ptr[c] = s as i64
  ds.hdr_len[c] = n
  return DX_OK
71}
72
73// set the (r,c) cell to a byte range [p, p+len). (range form so cells may contain NULs/commas/etc.)
74func dx_set_cell(ds: *NxDataset, r: i64, c: i64, p: *u8, len: i64) -> i64 {
  if ds.valid != 1 { return 0 - DX_BAD_INPUT }
  if r < 0 { return 0 - DX_BAD_INPUT }
  if r >= ds.nrows { return 0 - DX_BAD_INPUT }
  if c < 0 { return 0 - DX_BAD_INPUT }
  if c >= ds.ncols { return 0 - DX_BAD_INPUT }
  let idx: i64 = r * ds.ncols + c
  ds.cell_ptr[idx] = p as i64
  ds.cell_len[idx] = len
  return DX_OK
84}
85
86// convenience: set (r,c) from a NUL-terminated string.
87func dx_set_cell_z(ds: *NxDataset, r: i64, c: i64, s: *u8) -> i64 {
  var n: i64 = 0
  while s[n] != (0 as u8) { n = n + 1 }
  return dx_set_cell(ds, r, c, s, n)
91}
92
93func dx_cell_ptr(ds: *NxDataset, r: i64, c: i64) -> *u8 { return ds.cell_ptr[r * ds.ncols + c] as *u8 }
94func dx_cell_len(ds: *NxDataset, r: i64, c: i64) -> i64 { return ds.cell_len[r * ds.ncols + c] }
95
96// ============================================================================
97// byte-range equality.
98// ============================================================================
99func dx_bytes_eq(a: *u8, an: i64, b: *u8, bn: i64) -> i64 {
  if an != bn { return 0 }
  var i: i64 = 0
  while i < an {
      if a[i] != b[i] { return 0 }
      i = i + 1
  }
  return 1
107}
108
109// FULL dataset equality: same shape + every header + every cell byte-exact. The round-trip oracle.
110func dx_equal(a: *NxDataset, b: *NxDataset) -> i64 {
  if a.valid != 1 { return 0 }
  if b.valid != 1 { return 0 }
  if a.ncols != b.ncols { return 0 }
  if a.nrows != b.nrows { return 0 }
  var c: i64 = 0
  while c < a.ncols {
      if dx_bytes_eq(a.hdr_ptr[c] as *u8, a.hdr_len[c], b.hdr_ptr[c] as *u8, b.hdr_len[c]) == 0 { return 0 }
      c = c + 1
  }
  var r: i64 = 0
  while r < a.nrows {
      var cc: i64 = 0
      while cc < a.ncols {
          let ia: i64 = r * a.ncols + cc
          if dx_bytes_eq(a.cell_ptr[ia] as *u8, a.cell_len[ia], b.cell_ptr[ia] as *u8, b.cell_len[ia]) == 0 { return 0 }
          cc = cc + 1
      }
      r = r + 1
  }
  return 1
131}
132
133// ============================================================================
134// CSV emit (RFC 4180). A field is quoted iff it contains , " CR or LF; internal " is doubled.
135// ============================================================================
136func dx_field_needs_quote(p: *u8, n: i64) -> i64 {
  var i: i64 = 0
  while i < n {
      let c: i64 = p[i] as i64
      if c == 44 { return 1 }          // ','
      if c == 34 { return 1 }          // '"'
      if c == 10 { return 1 }          // LF
      if c == 13 { return 1 }          // CR
      i = i + 1
  }
  return 0
147}
148
149// emit one CSV field (with escaping) into out at off; returns new off (or -overflow).
150func dx_emit_csv_field(out: *u8, off: i64, cap: i64, p: *u8, n: i64) -> i64 {
  var o: i64 = off
  if dx_field_needs_quote(p, n) == 1 {
      if o >= cap { return 0 - DX_OVERFLOW }
      out[o] = 34 as u8; o = o + 1                  // opening "
      var i: i64 = 0
      while i < n {
          let c: i64 = p[i] as i64
          if c == 34 {
              if o + 1 >= cap { return 0 - DX_OVERFLOW }
              out[o] = 34 as u8; o = o + 1
              out[o] = 34 as u8; o = o + 1          // doubled
          } else {
              if o >= cap { return 0 - DX_OVERFLOW }
              out[o] = p[i]; o = o + 1
          }
          i = i + 1
      }
      if o >= cap { return 0 - DX_OVERFLOW }
      out[o] = 34 as u8; o = o + 1                  // closing "
  } else {
      var i2: i64 = 0
      while i2 < n {
          if o >= cap { return 0 - DX_OVERFLOW }
          out[o] = p[i2]; o = o + 1
          i2 = i2 + 1
      }
  }
  return o
179}
180
181// emit the whole dataset as CSV (header row + data rows, CRLF-free LF line endings). NUL-terminates.
182// returns bytes written (>=0) or -DX_*.
183func dx_emit_csv(ds: *NxDataset, out: *u8, cap: i64) -> i64 {
  if ds.valid != 1 { return 0 - DX_BAD_INPUT }
  var o: i64 = 0
  // header
  var c: i64 = 0
  while c < ds.ncols {
      if c > 0 { if o >= cap { return 0 - DX_OVERFLOW } out[o] = 44 as u8; o = o + 1 }
      o = dx_emit_csv_field(out, o, cap, ds.hdr_ptr[c] as *u8, ds.hdr_len[c])
      if o < 0 { return o }
      c = c + 1
  }
  if o >= cap { return 0 - DX_OVERFLOW }
  out[o] = 10 as u8; o = o + 1
  // rows
  var r: i64 = 0
  while r < ds.nrows {
      var cc: i64 = 0
      while cc < ds.ncols {
          if cc > 0 { if o >= cap { return 0 - DX_OVERFLOW } out[o] = 44 as u8; o = o + 1 }
          let idx: i64 = r * ds.ncols + cc
          o = dx_emit_csv_field(out, o, cap, ds.cell_ptr[idx] as *u8, ds.cell_len[idx])
          if o < 0 { return o }
          cc = cc + 1
      }
      if o >= cap { return 0 - DX_OVERFLOW }
      out[o] = 10 as u8; o = o + 1
      r = r + 1
  }
  if o >= cap { return 0 - DX_OVERFLOW }
  out[o] = 0 as u8
  return o
214}
215
216// ============================================================================
217// CSV parse (RFC 4180) -> NxDataset. Handles quoted fields, doubled "" , embedded , and newlines.
218// Allocates fresh cell byte-buffers (so parsed cells are independent of the source text). header row =
219// first line. ncols fixed by the header. returns DX_OK or -DX_PARSE.
220// ============================================================================
221func dx_parse_csv(src: *u8, n: i64, ds: *NxDataset) -> i64 {
  // first pass: parse into a temporary flat list of (offset,len) fields + a row-boundary marker.
  // we materialise each field into its own buffer as we go.
  let f_ptr: *i64 = sys_mmap(DX_MAXCELLS * 8) as *i64
  let f_len: *i64 = sys_mmap(DX_MAXCELLS * 8) as *i64
  let row_of: *i64 = sys_mmap(DX_MAXCELLS * 8) as *i64
  var nf: i64 = 0
  var cur_row: i64 = 0
  var fields_this_row: i64 = 0
  var maxcols: i64 = 0
231
  var i: i64 = 0
  var row_had_data: i64 = 0
  while i < n {
      // parse one field starting at i.
      let fbuf: *u8 = sys_mmap(DX_MAGIC_4096)
      var fo: i64 = 0
      var quoted: i64 = 0
      if src[i] == 34 as u8 { quoted = 1; i = i + 1 }
      var fdone: i64 = 0
      while fdone == 0 {
          if i >= n { fdone = 1 }
          else {
              let c: i64 = src[i] as i64
              if quoted == 1 {
                  if c == 34 {
                      // doubled "" -> literal "; lone " -> end of quoted field.
                      if i + 1 < n { if src[i + 1] == 34 as u8 { fbuf[fo] = 34 as u8; fo = fo + 1; i = i + 2 } else { quoted = 0; i = i + 1 } }
                      else { quoted = 0; i = i + 1 }
                  } else { fbuf[fo] = src[i]; fo = fo + 1; i = i + 1 }
              } else {
                  if c == 44 { fdone = 1 }                 // ',' ends field
                  else { if c == 10 { fdone = 1 } else { if c == 13 { fdone = 1 } else { fbuf[fo] = src[i]; fo = fo + 1; i = i + 1 } } }
              }
          }
      }
      // record the field
      if nf >= DX_MAXCELLS { return 0 - DX_PARSE }
      fbuf[fo] = 0 as u8
      f_ptr[nf] = fbuf as i64
      f_len[nf] = fo
      row_of[nf] = cur_row
      nf = nf + 1
      fields_this_row = fields_this_row + 1
      row_had_data = 1
266
      // consume the delimiter we stopped on.
      if i < n {
          if src[i] == 44 as u8 { i = i + 1 }              // next field on same row
          else {
              // end of line: consume CR?LF and close the row.
              if src[i] == 13 as u8 { i = i + 1 }
              if i < n { if src[i] == 10 as u8 { i = i + 1 } }
              if fields_this_row > maxcols { maxcols = fields_this_row }
              cur_row = cur_row + 1
              fields_this_row = 0
              row_had_data = 0
          }
      }
  }
  // close a final row that had data but no trailing newline.
  if row_had_data == 1 {
      if fields_this_row > maxcols { maxcols = fields_this_row }
      cur_row = cur_row + 1
  }
286
  if cur_row < 1 { return 0 - DX_PARSE }       // need at least the header row
  let ncols: i64 = maxcols
  let nrows: i64 = cur_row - 1                  // first row is the header
  if dx_alloc(ds, ncols, nrows) != DX_OK { return 0 - DX_PARSE }
291
  // distribute fields into header (row 0) + cells (rows 1..).
  var fi: i64 = 0
  while fi < nf {
      let r: i64 = row_of[fi]
      // column index = position of this field within its row.
      // recompute by scanning; cheap for our sizes.
      var col: i64 = 0
      var j: i64 = fi - 1
      var scanning: i64 = 1
      while scanning == 1 {
          if j < 0 { scanning = 0 }
          else { if row_of[j] == r { col = col + 1; j = j - 1 } else { scanning = 0 } }
      }
      if col < ncols {
          if r == 0 {
              ds.hdr_ptr[col] = f_ptr[fi]
              ds.hdr_len[col] = f_len[fi]
          } else {
              let idx: i64 = (r - 1) * ncols + col
              ds.cell_ptr[idx] = f_ptr[fi]
              ds.cell_len[idx] = f_len[fi]
          }
      }
      fi = fi + 1
  }
  // any short rows: fill missing trailing cells with empty (ptr=valid 0-len). default already 0/0.
  return DX_OK
319}
320
321// ============================================================================
322// TSV emit / parse. Fields are tab-separated; our model forbids literal TAB/NL inside a TSV field
323// (lossless because the CSV/JSON forms carry the same fields; TSV is the simple grid form). To keep
324// TSV byte-faithful for the ROUND TRIP we emit fields verbatim and parse on TAB/NL -- so the dataset
325// used for TSV round-trip must not contain TAB or NL in cells (the gate exercises a comma/quote cell
326// for CSV teeth, and a plain grid for the tri-format round trip). returns bytes / -DX_*.
327// ============================================================================
328func dx_emit_tsv(ds: *NxDataset, out: *u8, cap: i64) -> i64 {
  if ds.valid != 1 { return 0 - DX_BAD_INPUT }
  var o: i64 = 0
  var c: i64 = 0
  while c < ds.ncols {
      if c > 0 { if o >= cap { return 0 - DX_OVERFLOW } out[o] = 9 as u8; o = o + 1 }
      let hp: *u8 = ds.hdr_ptr[c] as *u8
      let hn: i64 = ds.hdr_len[c]
      var k: i64 = 0
      while k < hn { if o >= cap { return 0 - DX_OVERFLOW } out[o] = hp[k]; o = o + 1; k = k + 1 }
      c = c + 1
  }
  if o >= cap { return 0 - DX_OVERFLOW }
  out[o] = 10 as u8; o = o + 1
  var r: i64 = 0
  while r < ds.nrows {
      var cc: i64 = 0
      while cc < ds.ncols {
          if cc > 0 { if o >= cap { return 0 - DX_OVERFLOW } out[o] = 9 as u8; o = o + 1 }
          let idx: i64 = r * ds.ncols + cc
          let p: *u8 = ds.cell_ptr[idx] as *u8
          let ln: i64 = ds.cell_len[idx]
          var k2: i64 = 0
          while k2 < ln { if o >= cap { return 0 - DX_OVERFLOW } out[o] = p[k2]; o = o + 1; k2 = k2 + 1 }
          cc = cc + 1
      }
      if o >= cap { return 0 - DX_OVERFLOW }
      out[o] = 10 as u8; o = o + 1
      r = r + 1
  }
  if o >= cap { return 0 - DX_OVERFLOW }
  out[o] = 0 as u8
  return o
361}
362
363func dx_parse_tsv(src: *u8, n: i64, ds: *NxDataset) -> i64 {
  let f_ptr: *i64 = sys_mmap(DX_MAXCELLS * 8) as *i64
  let f_len: *i64 = sys_mmap(DX_MAXCELLS * 8) as *i64
  let row_of: *i64 = sys_mmap(DX_MAXCELLS * 8) as *i64
  var nf: i64 = 0
  var cur_row: i64 = 0
  var fields_this_row: i64 = 0
  var maxcols: i64 = 0
  var i: i64 = 0
  var row_had_data: i64 = 0
  while i < n {
      let fbuf: *u8 = sys_mmap(DX_MAGIC_4096)
      var fo: i64 = 0
      var fdone: i64 = 0
      while fdone == 0 {
          if i >= n { fdone = 1 }
          else {
              let c: i64 = src[i] as i64
              if c == 9 { fdone = 1 } else { if c == 10 { fdone = 1 } else { if c == 13 { fdone = 1 } else { fbuf[fo] = src[i]; fo = fo + 1; i = i + 1 } } }
          }
      }
      if nf >= DX_MAXCELLS { return 0 - DX_PARSE }
      fbuf[fo] = 0 as u8
      f_ptr[nf] = fbuf as i64; f_len[nf] = fo; row_of[nf] = cur_row
      nf = nf + 1; fields_this_row = fields_this_row + 1; row_had_data = 1
      if i < n {
          if src[i] == 9 as u8 { i = i + 1 }
          else {
              if src[i] == 13 as u8 { i = i + 1 }
              if i < n { if src[i] == 10 as u8 { i = i + 1 } }
              if fields_this_row > maxcols { maxcols = fields_this_row }
              cur_row = cur_row + 1; fields_this_row = 0; row_had_data = 0
          }
      }
  }
  if row_had_data == 1 { if fields_this_row > maxcols { maxcols = fields_this_row } cur_row = cur_row + 1 }
  if cur_row < 1 { return 0 - DX_PARSE }
  let ncols: i64 = maxcols
  let nrows: i64 = cur_row - 1
  if dx_alloc(ds, ncols, nrows) != DX_OK { return 0 - DX_PARSE }
  var fi: i64 = 0
  while fi < nf {
      let r: i64 = row_of[fi]
      var col: i64 = 0
      var j: i64 = fi - 1
      var scanning: i64 = 1
      while scanning == 1 { if j < 0 { scanning = 0 } else { if row_of[j] == r { col = col + 1; j = j - 1 } else { scanning = 0 } } }
      if col < ncols {
          if r == 0 { ds.hdr_ptr[col] = f_ptr[fi]; ds.hdr_len[col] = f_len[fi] }
          else { let idx: i64 = (r - 1) * ncols + col; ds.cell_ptr[idx] = f_ptr[fi]; ds.cell_len[idx] = f_len[fi] }
      }
      fi = fi + 1
  }
  return DX_OK
417}
418
419// ============================================================================
420// JSON emit: an array of row OBJECTS keyed by header. RFC 8259 string escaping (", \, control chars).
421// shape: [{"hdr0":"cell00","hdr1":"cell01"},{...}]  -- self-describing, key per column.
422// ============================================================================
423func dx_emit_json_str(out: *u8, off: i64, cap: i64, p: *u8, n: i64) -> i64 {
  var o: i64 = off
  if o >= cap { return 0 - DX_OVERFLOW }
  out[o] = 34 as u8; o = o + 1                       // "
  var i: i64 = 0
  while i < n {
      let b: i64 = p[i] as i64
      if b == 34 { if o + 1 >= cap { return 0 - DX_OVERFLOW } out[o] = 92 as u8; o = o + 1; out[o] = 34 as u8; o = o + 1 }
      else { if b == 92 { if o + 1 >= cap { return 0 - DX_OVERFLOW } out[o] = 92 as u8; o = o + 1; out[o] = 92 as u8; o = o + 1 }
      else { if b == 10 { if o + 1 >= cap { return 0 - DX_OVERFLOW } out[o] = 92 as u8; o = o + 1; out[o] = 110 as u8; o = o + 1 }
      else { if b == 13 { if o + 1 >= cap { return 0 - DX_OVERFLOW } out[o] = 92 as u8; o = o + 1; out[o] = 114 as u8; o = o + 1 }
      else { if b == 9 { if o + 1 >= cap { return 0 - DX_OVERFLOW } out[o] = 92 as u8; o = o + 1; out[o] = 116 as u8; o = o + 1 }
      else { if b < 32 {
          // \u00XX
          if o + 5 >= cap { return 0 - DX_OVERFLOW }
          out[o] = 92 as u8; o = o + 1; out[o] = 117 as u8; o = o + 1; out[o] = 48 as u8; o = o + 1; out[o] = 48 as u8; o = o + 1
          let hi: i64 = (b >> 4) & 0xF
          let lo: i64 = b & 0xF
          var hc: i64 = 48 + hi
          if hi > 9 { hc = 97 + hi - 10 }
          var lc: i64 = 48 + lo
          if lo > 9 { lc = 97 + lo - 10 }
          out[o] = hc as u8; o = o + 1; out[o] = lc as u8; o = o + 1
      } else {
          if o >= cap { return 0 - DX_OVERFLOW }
          out[o] = p[i]; o = o + 1
      } } } } } }
      i = i + 1
  }
  if o >= cap { return 0 - DX_OVERFLOW }
  out[o] = 34 as u8; o = o + 1
  return o
455}
456
457func dx_emit_json(ds: *NxDataset, out: *u8, cap: i64) -> i64 {
  if ds.valid != 1 { return 0 - DX_BAD_INPUT }
  var o: i64 = 0
  if o >= cap { return 0 - DX_OVERFLOW }
  out[o] = 91 as u8; o = o + 1                       // [
  var r: i64 = 0
  while r < ds.nrows {
      if r > 0 { if o >= cap { return 0 - DX_OVERFLOW } out[o] = 44 as u8; o = o + 1 }
      if o >= cap { return 0 - DX_OVERFLOW }
      out[o] = 123 as u8; o = o + 1                  // {
      var c: i64 = 0
      while c < ds.ncols {
          if c > 0 { if o >= cap { return 0 - DX_OVERFLOW } out[o] = 44 as u8; o = o + 1 }
          o = dx_emit_json_str(out, o, cap, ds.hdr_ptr[c] as *u8, ds.hdr_len[c])
          if o < 0 { return o }
          if o >= cap { return 0 - DX_OVERFLOW }
          out[o] = 58 as u8; o = o + 1               // :
          let idx: i64 = r * ds.ncols + c
          o = dx_emit_json_str(out, o, cap, ds.cell_ptr[idx] as *u8, ds.cell_len[idx])
          if o < 0 { return o }
          c = c + 1
      }
      if o >= cap { return 0 - DX_OVERFLOW }
      out[o] = 125 as u8; o = o + 1                  // }
      r = r + 1
  }
  if o >= cap { return 0 - DX_OVERFLOW }
  out[o] = 93 as u8; o = o + 1                       // ]
  if o >= cap { return 0 - DX_OVERFLOW }
  out[o] = 0 as u8
  return o
488}
489
490// JSON parse: read [{"k":"v",...},...] back into the model. We learn the column order from the FIRST
491// object's keys (matching how we emit, key-per-column in header order). Unescapes \" \\ \n \r \t \uXXXX.
492// returns DX_OK or -DX_PARSE.
493func dx_json_unescape_string(src: *u8, n: i64, i_io: *i64, outbuf: *u8) -> i64 {
  var i: i64 = i_io[0]
  if i >= n { return 0 - DX_PARSE }
  if src[i] != 34 as u8 { return 0 - DX_PARSE }
  i = i + 1
  var o: i64 = 0
  var done: i64 = 0
  while done == 0 {
      if i >= n { return 0 - DX_PARSE }
      let c: i64 = src[i] as i64
      if c == 34 { i = i + 1; done = 1 }
      else {
          if c == 92 {
              i = i + 1
              if i >= n { return 0 - DX_PARSE }
              let e: i64 = src[i] as i64
              if e == 34 { outbuf[o] = 34 as u8; o = o + 1; i = i + 1 }
              else { if e == 92 { outbuf[o] = 92 as u8; o = o + 1; i = i + 1 }
              else { if e == 110 { outbuf[o] = 10 as u8; o = o + 1; i = i + 1 }
              else { if e == 114 { outbuf[o] = 13 as u8; o = o + 1; i = i + 1 }
              else { if e == 116 { outbuf[o] = 9 as u8; o = o + 1; i = i + 1 }
              else { if e == 117 {
                  // \uXXXX -- decode the low byte (our emitter only ever produces \u00XX for <0x20).
                  if i + 4 >= n { return 0 - DX_PARSE }
                  var val: i64 = 0
                  var h: i64 = 1
                  while h <= 4 {
                      let hc: i64 = src[i + h] as i64
                      var d: i64 = 0
                      if hc >= 48 { if hc <= 57 { d = hc - 48 } }
                      if hc >= 97 { if hc <= 102 { d = hc - 97 + 10 } }
                      if hc >= 65 { if hc <= 70 { d = hc - 65 + 10 } }
                      val = val * 16 + d
                      h = h + 1
                  }
                  outbuf[o] = (val & 0xff) as u8; o = o + 1
                  i = i + 5
              } else { outbuf[o] = src[i]; o = o + 1; i = i + 1 } } } } } }
          } else { outbuf[o] = src[i]; o = o + 1; i = i + 1 }
      }
  }
  outbuf[o] = 0 as u8
  i_io[0] = i
  return o
537}
538
539func dx_parse_json(src: *u8, n: i64, ds: *NxDataset) -> i64 {
  // collect keys (col order) from the first object; collect every value in row-major order.
  let key_ptr: *i64 = sys_mmap(DX_MAXCOLS * 8) as *i64
  let key_len: *i64 = sys_mmap(DX_MAXCOLS * 8) as *i64
  let val_ptr: *i64 = sys_mmap(DX_MAXCELLS * 8) as *i64
  let val_len: *i64 = sys_mmap(DX_MAXCELLS * 8) as *i64
  var ncols: i64 = 0
  var nvals: i64 = 0
  var nrows: i64 = 0
  let ipos: *i64 = sys_mmap(16) as *i64
549
  var i: i64 = 0
  // expect leading '['
  while i < n { if src[i] == 91 as u8 { i = i + 1; break } i = i + 1 }
  var in_arr: i64 = 1
  while in_arr == 1 {
      // skip whitespace/commas to the next '{' or ']'
      var f: i64 = 0
      while f == 0 {
          if i >= n { in_arr = 0; f = 1 }
          else {
              let c: i64 = src[i] as i64
              if c == 123 { f = 1 }                      // '{'
              else { if c == 93 { in_arr = 0; f = 1; i = i + 1 } else { i = i + 1 } }
          }
      }
      if in_arr == 1 {
          // parse one object {"k":"v", ...}
          i = i + 1                                       // past '{'
          var col_in_row: i64 = 0
          var obj_done: i64 = 0
          while obj_done == 0 {
              // skip to '"' (key) or '}'
              var g: i64 = 0
              while g == 0 {
                  if i >= n { return 0 - DX_PARSE }
                  let c2: i64 = src[i] as i64
                  if c2 == 34 { g = 1 }
                  else { if c2 == 125 { obj_done = 1; g = 1; i = i + 1 } else { i = i + 1 } }
              }
              if obj_done == 0 {
                  // key
                  let kbuf: *u8 = sys_mmap(DX_MAGIC_1024)
                  ipos[0] = i
                  let kl: i64 = dx_json_unescape_string(src, n, ipos, kbuf)
                  if kl < 0 { return 0 - DX_PARSE }
                  i = ipos[0]
                  // skip to ':'
                  while i < n { if src[i] == 58 as u8 { i = i + 1; break } i = i + 1 }
                  // skip whitespace to value '"'
                  while i < n { if src[i] == 34 as u8 { break } i = i + 1 }
                  let vbuf: *u8 = sys_mmap(DX_MAGIC_4096)
                  ipos[0] = i
                  let vl: i64 = dx_json_unescape_string(src, n, ipos, vbuf)
                  if vl < 0 { return 0 - DX_PARSE }
                  i = ipos[0]
                  // first object defines the columns.
                  if nrows == 0 {
                      if ncols < DX_MAXCOLS { key_ptr[ncols] = kbuf as i64; key_len[ncols] = kl; ncols = ncols + 1 }
                  }
                  if nvals < DX_MAXCELLS { val_ptr[nvals] = vbuf as i64; val_len[nvals] = vl; nvals = nvals + 1 }
                  col_in_row = col_in_row + 1
              }
          }
          nrows = nrows + 1
      }
  }
  if ncols < 1 { return 0 - DX_PARSE }
  if dx_alloc(ds, ncols, nrows) != DX_OK { return 0 - DX_PARSE }
  var c: i64 = 0
  while c < ncols { ds.hdr_ptr[c] = key_ptr[c]; ds.hdr_len[c] = key_len[c]; c = c + 1 }
  // values were collected row-major (ncols per row).
  var vi: i64 = 0
  while vi < nvals {
      if vi < nrows * ncols { ds.cell_ptr[vi] = val_ptr[vi]; ds.cell_len[vi] = val_len[vi] }
      vi = vi + 1
  }
  return DX_OK
617}
618
619// compile smoke; real KAT in nx_figure_gate.
620func main() -> i64 {
  let ds: *NxDataset = sys_mmap(128) as *NxDataset
  dx_alloc(ds, 2, 1)
  dx_set_hdr(ds, 0, "k" as *u8); dx_set_hdr(ds, 1, "v" as *u8)
  dx_set_cell_z(ds, 0, 0, "x" as *u8); dx_set_cell_z(ds, 0, 1, "y" as *u8)
  let out: *u8 = sys_mmap(DX_MAGIC_4096)
  let nc: i64 = dx_emit_csv(ds, out, DX_MAGIC_4096)
  if nc <= 0 { return 1 }
  let nj: i64 = dx_emit_json(ds, out, DX_MAGIC_4096)
  if nj <= 0 { return 2 }
  let nt: i64 = dx_emit_tsv(ds, out, DX_MAGIC_4096)
  if nt <= 0 { return 3 }
  return 0
633}
nx_dataset_export.nx source

`nx_dataset_export.nx` source
