nx_reader_rows_tsv.nx source
↩ module page · 71 lines · 3158 B
1// nx_reader_rows_tsv.nx -- dump the first N rows of reader_rows.bin (the real SQuAD q/ctx/gold triples the
2// neural reader trained on) as TSV to stdout: q<TAB>ctx<TAB>gold, one row per line. Tabs/newlines inside the
3// text are replaced with spaces so the TSV is clean for a benchmark harness (the Qwen-as-reader F1 eval).
4// reader_rows.bin format: [magic 8B "NXRR1"] then per row [qlen i64][q][clen i64][ctx][alen i64][gold].
5// usage: nx_reader_rows_tsv [N] (default N=40) license_tier: ORIGINAL Sovereign: nx_syscalls.
6import "nx_syscalls.nx"
7const K_MAGIC_16777216: i64 = 16777216
8const K_MAGIC_100000: i64 = 100000
9
10func rt_putc(c: i64) -> i64 { let b: *u8 = sys_mmap(1); b[0] = c as u8; sys_write(1, b, 1); return 0 }
11func rt_pn(v: i64) -> i64 { let b: *u8=sys_mmap(28); var x: i64=v; if x<0{b[0]=45;sys_write(1,b,1);x=0-x} if x==0{b[0]=48;sys_write(1,b,1);return 0} var d: i64=0; var y: i64=x; while y>0{d=d+1;y=y/10} var i: i64=d-1; y=x; while i>=0{b[i]=(48+(y%10)) as u8;y=y/10;i=i-1} sys_write(1,b,d); return 0 }
12
13// write bytes [p, p+n) with TAB(9)/newline(10/13) -> space so the field stays on one TSV cell
14func rt_field(p: *u8, n: i64) -> i64 {
15 var i: i64 = 0
16 while i < n {
17 var c: i64 = p[i] as i64
18 if c == 9 { c = 32 }
19 if c == 10 { c = 32 }
20 if c == 13 { c = 32 }
21 rt_putc(c)
22 i = i + 1
23 }
24 return 0
25}
26
27func main(argc: i64, argv: *i64) -> i64 {
28 var N: i64 = 40
29 if argc >= 2 {
30 let a: *u8 = argv[1] as *u8
31 var v: i64 = 0
32 var j: i64 = 0
33 while a[j] != (0 as u8) { if a[j] >= (48 as u8) { if a[j] <= (57 as u8) { v = v*10 + (a[j] as i64 - 48) } } j = j + 1 }
34 if v > 0 { N = v }
35 }
36 let fd: i64 = sys_openat_rd("knowledge/index/reader_rows.bin" as *u8)
37 if fd < 0 { return 1 }
38 let cap: i64 = K_MAGIC_16777216
39 let raw: *u8 = sys_mmap(cap)
40 var got: i64 = 0
41 var r: i64 = 1
42 while r > 0 { r = sys_read(fd, (raw as i64 + got) as *u8, cap - got); if r > 0 { got = got + r } }
43 sys_close(fd)
44 if got < 16 { return 1 }
45 var off: i64 = 8 // skip magic
46 var nrows: i64 = 0
47 while off + 24 < got {
48 if nrows >= N { off = got } else {
49 let hp: *i64 = (raw as i64 + off) as *i64
50 let qlen: i64 = hp[0]
51 if qlen < 0 { off = got } else { if qlen > K_MAGIC_100000 { off = got } else {
52 let qp: *u8 = (raw as i64 + off + 8) as *u8
53 let hp2: *i64 = (raw as i64 + off + 8 + qlen) as *i64
54 let clen: i64 = hp2[0]
55 let cp: *u8 = (raw as i64 + off + 16 + qlen) as *u8
56 let hp3: *i64 = (raw as i64 + off + 16 + qlen + clen) as *i64
57 let alen: i64 = hp3[0]
58 let ap: *u8 = (raw as i64 + off + 24 + qlen + clen) as *u8
59 off = off + 24 + qlen + clen + alen
60 rt_field(qp, qlen)
61 rt_putc(9)
62 rt_field(cp, clen)
63 rt_putc(9)
64 rt_field(ap, alen)
65 rt_putc(10)
66 nrows = nrows + 1
67 } }
68 }
69 }
70 return 0
71}