code wiki / _hdl_build / nx_pdf_reflow_charset_gate.nx
`nx_pdf_reflow_charset_gate.nx` source

1import "nx_gate_gn.nx"
2// nx_pdf_reflow_charset_gate.nx -- liar-kill gate for the PDF-reflow mojibake fix (operator: "Copyright (c) 2008
3// by Wiley Publishing... still there" -- a born-digital PDF whose WinAnsi/CP1252 text byte 0xA9 = © reached the
4// UTF-8 reflow page raw -> U+FFFD). Builds a minimal PDF content stream carrying a raw 0xA9, runs the REAL
5// pr_pdf_to_reflow, and asserts the emitted HTML has the © as clean UTF-8 (0xC2 0xA9) with NO lone 0xA9 and NO
6// U+FFFD.  expect_exit: 0  license_tier: ORIGINAL
7import "nx_syscalls.nx"
8import "_pdf_reflow_lib.nx"
9
10func gp(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
11func gcat(d: *u8, o: i64, s: *u8) -> i64 { var i: i64=0; while s[i]!=(0 as u8){d[o+i]=s[i];i=i+1} return o+i }
12func has3(buf: *u8, n: i64, a: i64, b: i64, c: i64) -> i64 { var i: i64=0; while i+3<=n { if (buf[i]&0xff)==a { if (buf[i+1]&0xff)==b { if (buf[i+2]&0xff)==c { return 1 } } } i=i+1 } return 0 }
13// is there a 0xA9 byte NOT immediately preceded by 0xC2 (= a raw, un-repaired WinAnsi copyright) ?
14func lone_a9(buf: *u8, n: i64) -> i64 { var i: i64=0; while i<n { if (buf[i]&0xff)==0xa9 { if i==0 { return 1 } if (buf[i-1]&0xff)!=0xc2 { return 1 } } i=i+1 } return 0 }
15
16func main() -> i64 {
17    gp("=== nx_pdf_reflow_charset_gate: WinAnsi 0xA9 (c) in a PDF stream -> clean UTF-8 reflow ===\n" as *u8)
18    let pf: *i64 = sys_mmap(16) as *i64; pf[0]=0; pf[1]=0
19
20    // minimal PDF: one uncompressed content stream with a Tj string literal carrying a raw 0xA9.
21    let pdf: *u8 = sys_mmap(256); var k: i64 = 0
22    k = gcat(pdf, k, "stream\n(Copyright " as *u8)
23    pdf[k] = 0xa9 as u8; k = k + 1                       // the raw WinAnsi copyright byte
24    k = gcat(pdf, k, " 2008 by Wiley Publishing)Tj\nendstream\n" as *u8)
25
26    let html: *u8 = sys_mmap(4194304)
27    let txt:  *u8 = sys_mmap(4194304)
28    let dec:  *u8 = sys_mmap(4194304)
29    let tco:  *i64 = sys_mmap(8) as *i64; tco[0]=0
30    let w: i64 = pr_pdf_to_reflow(pdf, k, html, txt, dec, tco)
31    gp("  reflow html bytes="); gn(w); gp(" extracted_text_chars="); gn(tco[0]); gp("\n" as *u8)
32
33    // (1) the © arrives as clean UTF-8 right after "Copyright " (space 0x20, then 0xC2 0xA9)
34    if has3(html, w, 0x20, 0xc2, 0xa9) == 1 { gp("  PASS copyright -> U+00A9 clean UTF-8\n" as *u8); pf[0]=pf[0]+1 } else { gp("  FAIL copyright UTF-8 absent\n" as *u8); pf[1]=pf[1]+1 }
35    // (2) teeth: NO lone/raw 0xA9 survived
36    if lone_a9(html, w) == 0 { gp("  PASS no raw 0xA9\n" as *u8); pf[0]=pf[0]+1 } else { gp("  FAIL raw 0xA9 survived\n" as *u8); pf[1]=pf[1]+1 }
37    // (3) NO U+FFFD replacement char produced
38    if has3(html, w, 0xef, 0xbf, 0xbd) == 0 { gp("  PASS no U+FFFD\n" as *u8); pf[0]=pf[0]+1 } else { gp("  FAIL U+FFFD produced\n" as *u8); pf[1]=pf[1]+1 }
39    // (4) surrounding prose intact (proves we didn't mangle ASCII)
40    if has3(html, w, 0x57, 0x69, 0x6c) == 1 { gp("  PASS 'Wil'ey prose intact\n" as *u8); pf[0]=pf[0]+1 } else { gp("  FAIL prose mangled\n" as *u8); pf[1]=pf[1]+1 }
41
42    gp("PDF-REFLOW-CHARSET pass="); gn(pf[0]); gp(" fail="); gn(pf[1])
43    if pf[1] == 0 { gp(" verdict=GREEN\n" as *u8); sys_exit(0); return 0 }
44    gp(" verdict=RED\n" as *u8); sys_exit(1); return 1
45}
nx_pdf_reflow_charset_gate.nx source

`nx_pdf_reflow_charset_gate.nx` source
