code wiki / (root) / nx_bright_prep.nx
`nx_bright_prep.nx` source

↩ module page · 346 lines · 15543 B
1// nx_bright_prep.nx -- BRIGHT (huggingface xlangai/BRIGHT, cc-by-4.0) reshaped into the BEIR layout nx_beir_eval reads
2// (search R0d be_bright).
3//
4// usage: nx_bright_prep <split> <examples.parquet> <documents.parquet> <outdir>
5// Writes, through the sovereign parquet reader (nx_parquet_lib):
6//   <outdir>/corpus.tsv        doc id TAB content            (the documents file, id and content, via pq_tsv)
7//   <outdir>/queries.tsv       query id TAB query            (the examples file)
8//   <outdir>/qrels/test.tsv    the BEIR header line, then query id TAB doc id TAB 1 for every gold_ids element
9//   <outdir>/excluded.tsv      query id TAB doc id for every excluded_ids element: BRIGHT lists the query's own
10//                              source documents there, so a retriever must not be credited for finding them
11//   <outdir>/../LICENSE        the dataset, its source and its licence beside the bytes (a data asset carries its
12//                              licence in the same directory, the estate's rule for foreign data)
13// Every file is staged as .tmp and renamed into place. Tabs, newlines and carriage returns inside a cell become
14// spaces. A row with a null id or query is COUNTED and skipped; a gold or excluded element whose row has no id is
15// COUNTED as an orphan. The known-answer test is the dataset card's num_examples per split (search.refs brightds26):
16// queries must equal it and corpus rows must equal the documents count. The work lives in bp_run so the gate drives
17// it in-process; main prints the receipt. Exit 0 OK, 1 REFUSED, 2 usage.
18// Measured 2026-09-14 on eleven splits: every count agrees with the card, orphans 0 (search.plan 1789433620).
19// license_tier: ORIGINAL   No hw writes (Rule 26).
20import "nx_syscalls.nx"
21import "nx_parquet_lib.nx"
22
23const BP_OK: i64 = 0
24const BP_REFUSED: i64 = 1
25const BP_USAGE: i64 = 2
26const BP_STDOUT: i64 = 1
27const BP_MODE_DIR: i64 = 0x1ed
28const BP_PATH_CAP: i64 = 1024
29const BP_SLASH: i64 = 47
30const BP_TMP: *u8 = ".tmp"
31const BP_F_CORPUS: *u8 = "/corpus.tsv"
32const BP_F_QUERIES: *u8 = "/queries.tsv"
33const BP_D_QRELS: *u8 = "/qrels"
34const BP_F_QRELS: *u8 = "/qrels/test.tsv"
35const BP_F_EXCLUDED: *u8 = "/excluded.tsv"
36const BP_F_LICENSE: *u8 = "/LICENSE"
37const BP_COL_ID: *u8 = "id"
38const BP_COL_CONTENT: *u8 = "content"
39const BP_COL_QUERY: *u8 = "query"
40const BP_COL_GOLD: *u8 = "gold_ids"
41const BP_COL_EXCL: *u8 = "excluded_ids"
42const BP_H1: *u8 = "query-id"
43const BP_H2: *u8 = "corpus-id"
44const BP_H3: *u8 = "score"
45const BP_SCORE: *u8 = "1"
46const BP_LICENSE_TEXT: *u8 = "BRIGHT: A Realistic and Challenging Benchmark for Reasoning-Intensive Retrieval. Source: huggingface.co/datasets/xlangai/BRIGHT (the dataset card is pinned as search.refs brightds26, read 2026-09-14). Licence: Creative Commons Attribution 4.0 (cc-by-4.0), as declared on that card. The files under this directory are derived from the documents and examples parquet files of that dataset by the estate's own reader (nx_parquet_lib) and prep organ (nx_bright_prep), reshaped into the BEIR layout nx_beir_eval reads (corpus.tsv, queries.tsv, qrels/test.tsv, excluded.tsv per split); content is unchanged apart from tabs, newlines and carriage returns inside a cell becoming spaces. This file is written by nx_bright_prep on every run."
47// the receipt slots bp_run fills
48const BR_QUERIES: i64 = 0
49const BR_NULLQ: i64 = 1
50const BR_QDECL: i64 = 2
51const BR_QRELS: i64 = 3
52const BR_EXCL: i64 = 4
53const BR_ORPHANS: i64 = 5
54const BR_CORPUS_DECL: i64 = 6
55const BR_CORPUS_ROWS: i64 = 7
56const BR_CORPUS_WRITTEN: i64 = 8
57const BR_CORPUS_NULL: i64 = 9
58const BR_CORPUS_BYTES: i64 = 10
59const BR_STAGE: i64 = 11       // which stage refused (BR_ST_*), 0 when none
60const BR_WHAT: i64 = 12        // the reader's pq[PQ_WHAT] at the refusal
61const BR_SLOTS: i64 = 13
62// stages, named so a refusal says where
63const BR_ST_NONE: i64 = 0
64const BR_ST_LICENSE: i64 = 1
65const BR_ST_DOCS_OPEN: i64 = 2
66const BR_ST_DOCS_COLUMNS: i64 = 3
67const BR_ST_CORPUS: i64 = 4
68const BR_ST_EX_OPEN: i64 = 5
69const BR_ST_EX_COLUMNS: i64 = 6
70const BR_ST_OUTPUTS: i64 = 7
71const BR_ST_EX_READ: i64 = 8
72const BR_ST_ROWS: i64 = 9
73const BR_ST_FINISH: i64 = 10
74
75func bp_slen(s: *u8) -> i64 {
  var n: i64 = 0
  while (s[n] & PQ_BYTE) as i64 != 0 { n = n + 1 }
  return n
79}
80// dst = base + tail, NUL-terminated; returns the length or PQ_E_CAPACITY
81func bp_path(dst: *u8, base: *u8, tail: *u8) -> i64 {
  let bl: i64 = bp_slen(base)
  let tl: i64 = bp_slen(tail)
  if bl + tl + 1 > BP_PATH_CAP { return PQ_E_CAPACITY }
  var i: i64 = 0
  while i < bl { dst[i] = base[i]; i = i + 1 }
  var j: i64 = 0
  while j < tl { dst[bl + j] = tail[j]; j = j + 1 }
  dst[bl + tl] = 0 as u8
  return bl + tl
91}
92// the parent directory of a path (cut at the last slash) into dst; 0 when there is none
93func bp_parent(dst: *u8, path: *u8) -> i64 {
  let n: i64 = bp_slen(path)
  var cut: i64 = PQ_NONE
  var i: i64 = 0
  while i < n { if (path[i] & PQ_BYTE) as i64 == BP_SLASH { cut = i } i = i + 1 }
  if cut <= 0 { return 0 }
  var k: i64 = 0
  while k < cut { dst[k] = path[k]; k = k + 1 }
  dst[cut] = 0 as u8
  return cut
103}
104// open <path>.tmp for writing; tmp receives the staged name; returns the fd or a refusal
105func bp_open_tmp(path: *u8, tmp: *u8) -> i64 {
  if bp_path(tmp, path, BP_TMP) < 0 { return PQ_E_CAPACITY }
  let fd: i64 = sys_openat_wr(tmp, MODE_0644)
  if fd < 0 { return PQ_E_OPEN }
  return fd
110}
111func bp_finish(w: *i64, fd: i64, tmp: *u8, path: *u8) -> i64 {
  wb_flush(w)
  sys_close(fd)
  if w[WB_ERR] != 0 { return PQ_E_CAPACITY }
  if sys_renameat(tmp, path) != 0 { return PQ_E_OPEN }
  return 0
117}
118func bp_stage_name(st: i64) -> *u8 {
  if st == BR_ST_LICENSE { return "LICENSE" as *u8 }
  if st == BR_ST_DOCS_OPEN { return "documents-open" as *u8 }
  if st == BR_ST_DOCS_COLUMNS { return "documents-columns (id, content)" as *u8 }
  if st == BR_ST_CORPUS { return "corpus.tsv" as *u8 }
  if st == BR_ST_EX_OPEN { return "examples-open" as *u8 }
  if st == BR_ST_EX_COLUMNS { return "examples-columns (id, query, gold_ids, excluded_ids)" as *u8 }
  if st == BR_ST_OUTPUTS { return "output-files" as *u8 }
  if st == BR_ST_EX_READ { return "examples-read" as *u8 }
  if st == BR_ST_ROWS { return "examples-id-and-query-row-counts" as *u8 }
  if st == BR_ST_FINISH { return "finish (rename into place)" as *u8 }
  return "none" as *u8
130}
131// the whole job: fills st[BR_*]; returns 0 or a refusal code with st[BR_STAGE] naming the stage
132func bp_run(exf: *u8, docf: *u8, outdir: *u8, st: *i64) -> i64 {
  var s0: i64 = 0
  while s0 < BR_SLOTS { st[s0] = 0; s0 = s0 + 1 }
  st[BR_WHAT] = PQ_NONE
  let p1: *u8 = sys_mmap(BP_PATH_CAP)
  let p2: *u8 = sys_mmap(BP_PATH_CAP)
  let p3: *u8 = sys_mmap(BP_PATH_CAP)
  let t1: *u8 = sys_mmap(BP_PATH_CAP)
  let t2: *u8 = sys_mmap(BP_PATH_CAP)
  let t3: *u8 = sys_mmap(BP_PATH_CAP)
  let parent: *u8 = sys_mmap(BP_PATH_CAP)
  let pq: *i64 = sys_mmap(PQ_SLOTS * PQ_I64) as *i64
  // directories (an existing one answers EEXIST, which is the state wanted)
  if bp_parent(parent, outdir) > 0 { sys_mkdir(parent, BP_MODE_DIR) }
  sys_mkdir(outdir, BP_MODE_DIR)
  bp_path(p1, outdir, BP_D_QRELS)
  sys_mkdir(p1, BP_MODE_DIR)
  // the licence beside the bytes
  if bp_parent(parent, outdir) > 0 {
      bp_path(p1, parent, BP_F_LICENSE)
      let lfd: i64 = bp_open_tmp(p1, t1)
      if lfd < 0 { st[BR_STAGE] = BR_ST_LICENSE; return lfd }
      let lw: *i64 = wb_new(lfd)
      wb_puts(lw, BP_LICENSE_TEXT)
      wb_byte(lw, PQ_NL)
      let lrc: i64 = bp_finish(lw, lfd, t1, p1)
      if lrc < 0 { st[BR_STAGE] = BR_ST_LICENSE; return lrc }
  }
  // corpus.tsv from the documents file
  let rcd: i64 = pq_open(pq, docf)
  if rcd < 0 { st[BR_STAGE] = BR_ST_DOCS_OPEN; st[BR_WHAT] = pq[PQ_WHAT]; return rcd }
  let lid: i64 = pq_leaf_named(pq, BP_COL_ID)
  let lct: i64 = pq_leaf_named(pq, BP_COL_CONTENT)
  if lid < 0 || lct < 0 { st[BR_STAGE] = BR_ST_DOCS_COLUMNS; st[BR_WHAT] = lid; return PQ_E_ARG }
  let stats: *i64 = sys_mmap(PT_SLOTS * PQ_I64) as *i64
  bp_path(p1, outdir, BP_F_CORPUS)
  let rct: i64 = pq_tsv(pq, lid, lct, p1, stats)
  st[BR_CORPUS_ROWS] = stats[PT_ROWS]
  st[BR_CORPUS_WRITTEN] = stats[PT_WRITTEN]
  st[BR_CORPUS_NULL] = stats[PT_NULLROWS]
  st[BR_CORPUS_BYTES] = stats[PT_BYTES]
  st[BR_CORPUS_DECL] = pq[PQ_NROWS]
  if rct < 0 { st[BR_STAGE] = BR_ST_CORPUS; st[BR_WHAT] = pq[PQ_WHAT]; return rct }
  // the examples file
  let pqe: *i64 = sys_mmap(PQ_SLOTS * PQ_I64) as *i64
  let rce: i64 = pq_open(pqe, exf)
  if rce < 0 { st[BR_STAGE] = BR_ST_EX_OPEN; st[BR_WHAT] = pqe[PQ_WHAT]; return rce }
  st[BR_QDECL] = pqe[PQ_NROWS]
  let eid: i64 = pq_leaf_named(pqe, BP_COL_ID)
  let eq: i64 = pq_leaf_named(pqe, BP_COL_QUERY)
  let eg: i64 = pq_leaf_named(pqe, BP_COL_GOLD)
  let ex: i64 = pq_leaf_named(pqe, BP_COL_EXCL)
  if eid < 0 || eq < 0 || eg < 0 || ex < 0 { st[BR_STAGE] = BR_ST_EX_COLUMNS; st[BR_WHAT] = eid; return PQ_E_ARG }
  bp_path(p1, outdir, BP_F_QUERIES)
  bp_path(p2, outdir, BP_F_QRELS)
  bp_path(p3, outdir, BP_F_EXCLUDED)
  let fq: i64 = bp_open_tmp(p1, t1)
  if fq < 0 { st[BR_STAGE] = BR_ST_OUTPUTS; return fq }
  let fr: i64 = bp_open_tmp(p2, t2)
  if fr < 0 { st[BR_STAGE] = BR_ST_OUTPUTS; return fr }
  let fx: i64 = bp_open_tmp(p3, t3)
  if fx < 0 { st[BR_STAGE] = BR_ST_OUTPUTS; return fx }
  let wq: *i64 = wb_new(fq)
  let wr: *i64 = wb_new(fr)
  let wx: *i64 = wb_new(fx)
  wb_puts(wr, BP_H1)
  wb_byte(wr, PQ_TAB)
  wb_puts(wr, BP_H2)
  wb_byte(wr, PQ_TAB)
  wb_puts(wr, BP_H3)
  wb_byte(wr, PQ_NL)
  var r: i64 = 0
  while r < pqe[PQ_NRG] {
      let pid: *i64 = pr_alloc(pqe, r, eid)
      let pqy: *i64 = pr_alloc(pqe, r, eq)
      let pg: *i64 = pr_alloc(pqe, r, eg)
      let px: *i64 = pr_alloc(pqe, r, ex)
      let r1: i64 = pq_read_column(pqe, r, eid, pid)
      if r1 < 0 { st[BR_STAGE] = BR_ST_EX_READ; st[BR_WHAT] = pqe[PQ_WHAT]; return r1 }
      let r2: i64 = pq_read_column(pqe, r, eq, pqy)
      if r2 < 0 { st[BR_STAGE] = BR_ST_EX_READ; st[BR_WHAT] = pqe[PQ_WHAT]; return r2 }
      let r3: i64 = pq_read_column(pqe, r, eg, pg)
      if r3 < 0 { st[BR_STAGE] = BR_ST_EX_READ; st[BR_WHAT] = pqe[PQ_WHAT]; return r3 }
      let r4: i64 = pq_read_column(pqe, r, ex, px)
      if r4 < 0 { st[BR_STAGE] = BR_ST_EX_READ; st[BR_WHAT] = pqe[PQ_WHAT]; return r4 }
      if pid[PR_N] != pqy[PR_N] { st[BR_STAGE] = BR_ST_ROWS; return PQ_E_LEVELS }
      let ida: *u8 = pid[PR_ARENA] as *u8
      let ioff: *i64 = pid[PR_OFF] as *i64
      let ilen: *i64 = pid[PR_LEN] as *i64
      let qa: *u8 = pqy[PR_ARENA] as *u8
      let qoff: *i64 = pqy[PR_OFF] as *i64
      let qlen: *i64 = pqy[PR_LEN] as *i64
      var i: i64 = 0
      while i < pid[PR_N] {
          if ioff[i] >= 0 && qoff[i] >= 0 {
              wb_cell(wq, ida, ioff[i], ilen[i])
              wb_byte(wq, PQ_TAB)
              wb_cell(wq, qa, qoff[i], qlen[i])
              wb_byte(wq, PQ_NL)
              st[BR_QUERIES] = st[BR_QUERIES] + 1
          } else { st[BR_NULLQ] = st[BR_NULLQ] + 1 }
          i = i + 1
      }
      let ga: *u8 = pg[PR_ARENA] as *u8
      let goff: *i64 = pg[PR_OFF] as *i64
      let glen: *i64 = pg[PR_LEN] as *i64
      let grow: *i64 = pg[PR_ROW] as *i64
      var e: i64 = 0
      while e < pg[PR_N] {
          if goff[e] >= 0 {
              let row: i64 = grow[e]
              if row >= 0 && row < pid[PR_N] && ioff[row] >= 0 {
                  wb_cell(wr, ida, ioff[row], ilen[row])
                  wb_byte(wr, PQ_TAB)
                  wb_cell(wr, ga, goff[e], glen[e])
                  wb_byte(wr, PQ_TAB)
                  wb_puts(wr, BP_SCORE)
                  wb_byte(wr, PQ_NL)
                  st[BR_QRELS] = st[BR_QRELS] + 1
              } else { st[BR_ORPHANS] = st[BR_ORPHANS] + 1 }
          }
          e = e + 1
      }
      let xa: *u8 = px[PR_ARENA] as *u8
      let xoff: *i64 = px[PR_OFF] as *i64
      let xlen: *i64 = px[PR_LEN] as *i64
      let xrow: *i64 = px[PR_ROW] as *i64
      var e2: i64 = 0
      while e2 < px[PR_N] {
          if xoff[e2] >= 0 {
              let row2: i64 = xrow[e2]
              if row2 >= 0 && row2 < pid[PR_N] && ioff[row2] >= 0 {
                  wb_cell(wx, ida, ioff[row2], ilen[row2])
                  wb_byte(wx, PQ_TAB)
                  wb_cell(wx, xa, xoff[e2], xlen[e2])
                  wb_byte(wx, PQ_NL)
                  st[BR_EXCL] = st[BR_EXCL] + 1
              } else { st[BR_ORPHANS] = st[BR_ORPHANS] + 1 }
          }
          e2 = e2 + 1
      }
      pr_free(pid)
      pr_free(pqy)
      pr_free(pg)
      pr_free(px)
      r = r + 1
  }
  let f1: i64 = bp_finish(wq, fq, t1, p1)
  if f1 < 0 { st[BR_STAGE] = BR_ST_FINISH; return f1 }
  let f2: i64 = bp_finish(wr, fr, t2, p2)
  if f2 < 0 { st[BR_STAGE] = BR_ST_FINISH; return f2 }
  let f3: i64 = bp_finish(wx, fx, t3, p3)
  if f3 < 0 { st[BR_STAGE] = BR_ST_FINISH; return f3 }
  return 0
286}
287// the receipt's verdict: queries equal the file's declared rows, corpus written equals the documents file's rows,
288// no orphan
289func bp_verdict(st: *i64) -> i64 {
  if st[BR_QUERIES] != st[BR_QDECL] { return 0 }
  if st[BR_CORPUS_WRITTEN] != st[BR_CORPUS_DECL] { return 0 }
  if st[BR_ORPHANS] != 0 { return 0 }
  return 1
294}
295
296func main(argc: i64, argv: *i64) -> i64 {
  let w: *i64 = wb_new(BP_STDOUT)
  if argc < 5 {
      wb_puts(w, "usage: nx_bright_prep <split> <examples.parquet> <documents.parquet> <outdir>" as *u8)
      wb_byte(w, PQ_NL)
      wb_flush(w)
      sys_exit(BP_USAGE)
      return BP_USAGE
  }
  let split: *u8 = argv[1] as *u8
  let st: *i64 = sys_mmap(BR_SLOTS * PQ_I64) as *i64
  let rc: i64 = bp_run(argv[2] as *u8, argv[3] as *u8, argv[4] as *u8, st)
  wb_puts(w, "NX-BRIGHT-PREP split=" as *u8)
  wb_puts(w, split)
  wb_byte(w, PQ_SPACE)
  wb_kv(w, "queries" as *u8, st[BR_QUERIES])
  wb_kv(w, "null_queries" as *u8, st[BR_NULLQ])
  wb_kv(w, "queries_declared" as *u8, st[BR_QDECL])
  wb_kv(w, "qrels" as *u8, st[BR_QRELS])
  wb_kv(w, "excluded" as *u8, st[BR_EXCL])
  wb_kv(w, "orphans" as *u8, st[BR_ORPHANS])
  wb_kv(w, "corpus_declared" as *u8, st[BR_CORPUS_DECL])
  wb_kv(w, "corpus_rows" as *u8, st[BR_CORPUS_ROWS])
  wb_kv(w, "corpus_written" as *u8, st[BR_CORPUS_WRITTEN])
  wb_kv(w, "corpus_null_rows" as *u8, st[BR_CORPUS_NULL])
  wb_kv(w, "corpus_bytes" as *u8, st[BR_CORPUS_BYTES])
  wb_puts(w, "outdir=" as *u8)
  wb_puts(w, argv[4] as *u8)
  wb_byte(w, PQ_SPACE)
  if rc < 0 {
      wb_puts(w, "REFUSED " as *u8)
      wb_puts(w, bp_stage_name(st[BR_STAGE]))
      wb_byte(w, PQ_SPACE)
      wb_puts(w, pq_errname(rc))
      wb_byte(w, PQ_SPACE)
      wb_kv(w, "code" as *u8, rc)
      wb_kv(w, "what" as *u8, st[BR_WHAT])
      wb_puts(w, "verdict=REFUSED" as *u8)
      wb_byte(w, PQ_NL)
      wb_flush(w)
      sys_exit(BP_REFUSED)
      return BP_REFUSED
  }
  let ok: i64 = bp_verdict(st)
  if ok == 1 { wb_puts(w, "verdict=OK" as *u8) } else { wb_puts(w, "verdict=REFUSED (counts disagree with the files' own declarations)" as *u8) }
  wb_byte(w, PQ_NL)
  wb_flush(w)
  if ok == 1 { sys_exit(BP_OK); return BP_OK }
  sys_exit(BP_REFUSED)
  return BP_REFUSED
346}
nx_bright_prep.nx source

`nx_bright_prep.nx` source
