`nx_kf8_book.nx` source

↩ module page · 256 lines · 15225 B
1// nx_kf8_book.nx -- SOVEREIGN KF8 / AZW3 (the newer Kindle format) -> Nishi format. KF8 lives in the SAME PDB
2// container as MOBI6; a dual file has a MOBI6 part (record 0) + a KF8 part whose start record is named by EXTH 121
3// ("KF8 boundary"); a pure-KF8 file has mobiType==8 at record 0. The KF8 part has its OWN PalmDOC/MOBI header +
4// its own text records, compressed exactly like MOBI6 (PalmDOC or HUFF/CDIC) -- so this REUSES the validated
5// nx_palmdoc + nx_huffcdic codecs and only adds the boundary glue. nx_mobi_book (validated) is left untouched.
6//   LEGAL LINE: detects encryptionType and DECLINES DRM (exit 2), never circumvents.
7// Rung 1: decompress the KF8 text flows + nx_html_to_text -> readable book.json (the text is all there). Rung 2 =
8// SKEL/FRAG/FDST reassembly for exact structure, + unify the decode glue with nx_mobi_book (DRY). expect_exit: 0
9// Usage: nx_kf8_book <mobi/azw3-path> <slug>.  license_tier: ORIGINAL
10import "nx_syscalls.nx"
11import "nx_itoa_lib.nx"   // shared MSB-first emitter (zero-alloc)
12import "nx_palmdoc.nx"
13import "nx_huffcdic.nx"
14import "nx_html_to_text.nx"
15import "nx_mobi_cover.nx"
16import "nx_chapter_split.nx"
17import "nx_charset.nx"   // MOJIBAKE FIX: nx_charset_repair_utf8 -- stray Windows-1252 bytes -> clean UTF-8
18const K_MAGIC_1252: i64 = 1252
19const K_MAGIC_1024: i64 = 1024
20const K_MAGIC_8192: i64 = 8192
21const K_MAGIC_262144: i64 = 262144
22const K_MAGIC_17480: i64 = 17480
23
24func kf_slen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
25func kf_w(fd: i64, s: *u8) -> i64 { sys_write(fd, s, kf_slen(s)); return 0 }
26// MIGRATED to the shared emitter (debt 1785563586). The old body mmapped a scratch buffer
27// per call and never freed it. At PAGE granularity that is 4096B leaked PER CALL -- the
28// defect that took 28.5GB of a 36GB host in nx_ts_lumadiff (2MB input, ~3.66M calls).
29// nxi_* is MSB-first, allocates NOTHING, and emits identical bytes including the sign.
30func kf_n(fd: i64, v: i64) -> i64 { nxi_fd(fd, v); return 0 }
31func kf_p(s: *u8) -> i64 { kf_w(1, s); return 0 }
32func kf_pn(v: i64) -> i64 { kf_n(1, v); return 0 }
33func ensure_dir(path: *u8) -> i64 { __syscall(258, 0-100, path, 0x1ed, 0, 0, 0); return 0 }
34func be16(b: *u8, o: i64) -> i64 { return ((b[o] as i64)<<8)|(b[o+1] as i64) }
35func be32(b: *u8, o: i64) -> i64 { return ((b[o] as i64)<<24)|((b[o+1] as i64)<<16)|((b[o+2] as i64)<<8)|(b[o+3] as i64) }
36func find_sub(hay: *u8, hl: i64, needle: *u8) -> i64 {
  let nl: i64 = kf_slen(needle); if nl == 0 { return 0-1 }
  var i: i64 = 0
  while i + nl <= hl { var k: i64=0; var hit: i64=1; while k<nl { if hay[i+k]!=needle[k]{hit=0;k=nl}else{k=k+1} } if hit==1 {return i} i=i+1 }
  return 0-1
41}
42func er_cat(dst: *u8, off: i64, s: *u8) -> i64 { var i: i64=0; while s[i]!=(0 as u8){dst[off+i]=s[i];i=i+1} return off+i }
43// JSON string emit. MOJIBAKE FIX: repair each metadata string to clean UTF-8 first (idempotent on valid UTF-8).
44func er_wjson(fd: i64, s: *u8) -> i64 {
  var sl: i64 = 0; while s[sl] != (0 as u8) { sl = sl + 1 }
  let rcap: i64 = sl * 3 + 16
  let rep: *u8 = sys_mmap(rcap)
  let rn: i64 = nx_charset_repair_utf8(s, sl, rep, rcap)
  var i: i64 = 0
  while i < rn { let c: i64 = rep[i] as i64; if c == 0x22 { sys_write(fd, "\\\"" as *u8, 2) } else { if c == 0x5c { sys_write(fd, "\\\\" as *u8, 2) } else { if c < 0x20 { sys_write(fd, " " as *u8, 1) } else { sys_write(fd, (rep as i64 + i) as *u8, 1) } } } i = i + 1 }
  sys_munmap(rep, rcap)
  return 0
53}
54func utf8_put(out: *u8, o: i64, cp: i64) -> i64 {
  if cp < 0x80 { out[o] = cp as u8; return o + 1 }
  if cp < 0x800 { out[o] = (0xC0 | (cp >> 6)) as u8; out[o+1] = (0x80 | (cp & 0x3F)) as u8; return o + 2 }
  if cp < 0x10000 { out[o] = (0xE0 | (cp >> 12)) as u8; out[o+1] = (0x80 | ((cp >> 6) & 0x3F)) as u8; out[o+2] = (0x80 | (cp & 0x3F)) as u8; return o + 3 }
  out[o] = (0xF0 | (cp >> 18)) as u8; out[o+1] = (0x80 | ((cp >> 12) & 0x3F)) as u8; out[o+2] = (0x80 | ((cp >> 6) & 0x3F)) as u8; out[o+3] = (0x80 | (cp & 0x3F)) as u8; return o + 4
59}
60func decode_numeric_refs(txt: *u8, n: i64, out: *u8, outcap: i64) -> i64 {
  var i: i64 = 0
  var o: i64 = 0
  while i < n {
      let c: i64 = txt[i] as i64
      var done: i64 = 0
      if c == 0x26 { if i + 1 < n { if (txt[i+1] as i64) == 0x23 {
          var j: i64 = i + 2
          var hex: i64 = 0
          if j < n { let xc: i64 = txt[j] as i64; if xc == 0x78 { hex = 1; j = j + 1 } else { if xc == 0x58 { hex = 1; j = j + 1 } } }
          var cp: i64 = 0
          var ndig: i64 = 0
          var scan: i64 = 1
          while scan == 1 {
              if j >= n { scan = 0 } else {
                  let dc: i64 = txt[j] as i64
                  var dv: i64 = 0-1
                  if hex == 1 { if dc >= 48 { if dc <= 57 { dv = dc - 48 } } if dc >= 97 { if dc <= 102 { dv = dc - 87 } } if dc >= 65 { if dc <= 70 { dv = dc - 55 } } }
                  else { if dc >= 48 { if dc <= 57 { dv = dc - 48 } } }
                  if dv < 0 { scan = 0 } else { if hex == 1 { cp = cp * 16 + dv } else { cp = cp * 10 + dv } ndig = ndig + 1; j = j + 1 }
              }
          }
          if ndig > 0 { if j < n { if (txt[j] as i64) == 0x3b { if cp > 0 { if cp <= 0x10FFFF { if o + 4 < outcap { o = utf8_put(out, o, cp) } i = j + 1; done = 1 } } } } }
      } } }
      if done == 0 { if o < outcap - 1 { out[o] = txt[i]; o = o + 1 } i = i + 1 }
  }
  out[o] = 0 as u8
  return o
88}
89func exth_copy(z: *u8, p: i64, rlen: i64, out: *u8, cap: i64) -> i64 {
  var o: i64 = 0; var s: i64 = p + 8; let e: i64 = p + rlen
  while s < e { if o < cap-1 { out[o] = z[s]; o = o + 1 } s = s + 1 }
  out[o] = 0 as u8; return o
93}
94
95const KCAP: i64 = 16777216
96
97func do_kf8(path: *u8, slug: *u8) -> i64 {
  let lb: *i64 = sys_mmap(16) as *i64
  let z: *u8 = sys_read_file(path, lb)
  if (z as i64) == 0 { kf_p("KF8-FAIL read " as *u8); kf_p(path); kf_p("\n" as *u8); return 0-1 }
  let zl: i64 = lb[0]
  if zl < 80 { kf_p("KF8-FAIL too-small\n" as *u8); return 0-1 }
  let numRec: i64 = be16(z, 76)
  let rec0off: i64 = be32(z, 78)
105
  // ---- EXTH: KF8 boundary (121) + metadata (author 100, title 503) ----
  let title: *u8 = sys_mmap(K_MAGIC_1024); title[0]=0 as u8
  let author: *u8 = sys_mmap(K_MAGIC_1024); author[0]=0 as u8
  let publisher: *u8 = sys_mmap(K_MAGIC_1024); publisher[0]=0 as u8   // EXTH 101
  let isbn: *u8 = sys_mmap(256); isbn[0]=0 as u8               // EXTH 104
  let pubdate: *u8 = sys_mmap(256); pubdate[0]=0 as u8         // EXTH 106
  let descr_raw: *u8 = sys_mmap(K_MAGIC_8192); descr_raw[0]=0 as u8    // EXTH 103 description (HTML)
  var boundary: i64 = 0-1
  var coveroff: i64 = 0; var coverfound: i64 = 0; var thumboff: i64 = 0; var thumbfound: i64 = 0   // EMBEDDED cover (shared absolute image records)
  let rec1off: i64 = be32(z, 78 + 8)
  var r0ext: i64 = rec1off - rec0off
  if r0ext <= 0 { r0ext = zl - rec0off }
  let exrel: i64 = find_sub((z as i64 + rec0off) as *u8, r0ext, "EXTH" as *u8)
  if exrel >= 0 {
      let exth: i64 = rec0off + exrel
      let cnt: i64 = be32(z, exth + 8)
      var p: i64 = exth + 12; var ci: i64 = 0
      while ci < cnt {
          if p + 8 > zl { ci = cnt } else {
              let rtype: i64 = be32(z, p); let rlen: i64 = be32(z, p+4)
              if rlen < 8 { ci = cnt } else {
                  if rtype == 121 { boundary = be32(z, p+8) }
                  if rtype == 100 { exth_copy(z, p, rlen, author, K_MAGIC_1024) }
                  if rtype == 503 { exth_copy(z, p, rlen, title, K_MAGIC_1024) }
                  if rtype == 201 { if rlen >= 12 { coveroff = be32(z, p+8); coverfound = 1 } }   // EXTH 201 coveroffset
                  if rtype == 202 { if rlen >= 12 { thumboff = be32(z, p+8); thumbfound = 1 } }   // EXTH 202 thumboffset
                  if rtype == 101 { exth_copy(z, p, rlen, publisher, K_MAGIC_1024) }   // EXTH 101 publisher
                  if rtype == 104 { exth_copy(z, p, rlen, isbn, 256) }         // EXTH 104 isbn
                  if rtype == 106 { exth_copy(z, p, rlen, pubdate, 256) }      // EXTH 106 publishing date
                  if rtype == 103 { exth_copy(z, p, rlen, descr_raw, K_MAGIC_8192) }   // EXTH 103 description (HTML)
                  p = p + rlen; ci = ci + 1
              }
          }
      }
  }
  if title[0]==(0 as u8) { var k: i64=0; while k<31 { let c: i64=z[k] as i64; if c==0 {k=31} else {title[k]=z[k] as u8; k=k+1} } title[k]=0 as u8 }
142
  // ---- locate the KF8 part: dual (EXTH 121) or pure-KF8 (mobiType==8) ----
  let mobiType: i64 = be32(z, rec0off + 16 + 8)
  var kfrec0: i64 = 0
  var B: i64 = 0
  if boundary >= 0 { if boundary < numRec { B = boundary; kfrec0 = be32(z, 78 + B*8) } else { kf_p("KF8-FAIL boundary-out-of-range\n" as *u8); return 0-1 } }
  else { if mobiType == 8 { B = 0; kfrec0 = rec0off } else { kf_p("KF8-NOT-KF8 (no EXTH-121 boundary and mobiType=" as *u8); kf_pn(mobiType); kf_p("; this is a MOBI6-only file -- use nx_mobi_book)\n" as *u8); return 0-1 } }
149
  // ---- KF8 PalmDOC header ----
  let comp: i64 = be16(z, kfrec0 + 0)
  let textLen: i64 = be32(z, kfrec0 + 4)
  let recCount: i64 = be16(z, kfrec0 + 8)
  let enc: i64 = be16(z, kfrec0 + 12)
  if enc != 0 { kf_p("KF8-DRM-DECLINED encryptionType=" as *u8); kf_pn(enc); kf_p(" (decode a DRM-free copy; no circumvention)\n" as *u8); sys_exit(2); return 2 }
156
  // ---- HUFF/CDIC tables (if compressed that way) ----
  let d1cl: *i64 = sys_mmap(256*8) as *i64
  let d1tm: *i64 = sys_mmap(256*8) as *i64
  let d1mx: *i64 = sys_mmap(256*8) as *i64
  let hmin: *i64 = sys_mmap(64*8) as *i64
  let hmax: *i64 = sys_mmap(64*8) as *i64
  let dptr: *i64 = sys_mmap(K_MAGIC_262144*8) as *i64
  let dlen: *i64 = sys_mmap(K_MAGIC_262144*8) as *i64
  let dflag: *i64 = sys_mmap(K_MAGIC_262144*8) as *i64
  let dcnt: *i64 = sys_mmap(8) as *i64; dcnt[0]=0
  var is_huff: i64 = 0
  if comp == K_MAGIC_17480 { is_huff = 1 } else { if comp != 1 { if comp != 2 { kf_p("KF8-UNSUPPORTED compression=" as *u8); kf_pn(comp); kf_p("\n" as *u8); return 0-1 } } }
  if is_huff == 1 {
      let huffidx: i64 = B + be32(z, kfrec0 + 112)     // KF8 part record refs are RELATIVE to the boundary (B+40 = abs 86)
      let huffcnt: i64 = be32(z, kfrec0 + 116)
      let hs: i64 = be32(z, 78 + huffidx*8)
      var he: i64 = zl; if huffidx + 1 < numRec { he = be32(z, 78 + (huffidx+1)*8) }
      if huff_load((z as i64 + hs) as *u8, he - hs, d1cl, d1tm, d1mx, hmin, hmax) != 0 { kf_p("KF8-HUFF-FAIL bad-HUFF-record\n" as *u8); return 0-1 }
      var ci: i64 = 1
      while ci < huffcnt {
          let cidx: i64 = huffidx + ci
          let cs: i64 = be32(z, 78 + cidx*8)
          var ce: i64 = zl; if cidx + 1 < numRec { ce = be32(z, 78 + (cidx+1)*8) }
          cdic_load((z as i64 + cs) as *u8, ce - cs, dptr, dlen, dflag, dcnt)
          ci = ci + 1
      }
  }
184
  // ---- decompress KF8 text records (boundary+1 .. boundary+recCount) ----
  let tbuf: *u8 = sys_mmap(KCAP)
  var to: i64 = 0
  var ri: i64 = B + 1
  let last: i64 = B + recCount
  while ri <= last {
      if ri >= numRec { ri = last + 1 } else {
          let rs: i64 = be32(z, 78 + ri*8)
          var re: i64 = zl; if ri + 1 < numRec { re = be32(z, 78 + (ri+1)*8) }
          let rlen: i64 = re - rs
          if rlen > 0 { if rs + rlen <= zl {
              if comp == 2 { to = to + palmdoc_decompress((z as i64 + rs) as *u8, rlen, (tbuf as i64 + to) as *u8, KCAP - to) }
              else { if is_huff == 1 { to = to + hc_unpack((z as i64 + rs) as *u8, rlen, (tbuf as i64 + to) as *u8, KCAP - to, d1cl, d1tm, d1mx, hmin, hmax, dptr, dlen, dflag, dcnt[0]) }
              else { var k: i64=0; while k < rlen { if to < KCAP { tbuf[to]=z[rs+k]; to=to+1 } k=k+1 } } }
          } }
          ri = ri + 1
      }
  }
  var htmllen: i64 = to
204
  // ---- (chapter split + per-chapter html_to_text + chap<i>.txt are done by nx_chapter_split, in book.json below) ----
206
  // description (EXTH 103) is HTML -> strip tags + decode numeric refs for book.json
  let descr_clean: *u8 = sys_mmap(K_MAGIC_8192); descr_clean[0] = 0 as u8
  if descr_raw[0] != (0 as u8) {
      let dtmp: *u8 = sys_mmap(K_MAGIC_8192)
      let dtn: i64 = nx_html_to_text(descr_raw, kf_slen(descr_raw), dtmp, K_MAGIC_8192)
      decode_numeric_refs(dtmp, dtn, descr_clean, K_MAGIC_8192)
  }
214
  ensure_dir("knowledge/staging/media/reader" as *u8)
  let dir: *u8 = sys_mmap(512)
  var dl: i64 = er_cat(dir, 0, "knowledge/staging/media/reader/" as *u8); dl = er_cat(dir, dl, slug); dir[dl]=0 as u8
  ensure_dir(dir)
  // EMBEDDED cover (offline, from the file; shared with nx_mobi_book via nx_mobi_cover). KF8 image records are
  // absolute, so the record-0 First Image index + EXTH 201 reach the SAME cover the MOBI6 part references.
  let covername: *u8 = sys_mmap(64); covername[0] = 0 as u8
  mobi_extract_cover(z, zl, numRec, rec0off, coveroff, coverfound, thumboff, thumbfound, dir, covername)
  let jp: *u8 = sys_mmap(K_MAGIC_1024)
  var jl: i64 = er_cat(jp, 0, dir as *u8); jl = er_cat(jp, jl, "/book.json" as *u8); jp[jl]=0 as u8
  let jfd: i64 = sys_openat_wr(jp, 0x1a4)
  if jfd < 0 { kf_p("KF8-FAIL open-json\n" as *u8); return 0-1 }
  kf_w(jfd, "{\"title\":\"" as *u8); er_wjson(jfd, title)
  kf_w(jfd, "\",\"author\":\"" as *u8); er_wjson(jfd, author)
  kf_w(jfd, "\",\"publisher\":\"" as *u8); er_wjson(jfd, publisher)
  kf_w(jfd, "\",\"pubdate\":\"" as *u8); er_wjson(jfd, pubdate)
  kf_w(jfd, "\",\"isbn\":\"" as *u8); er_wjson(jfd, isbn)
  kf_w(jfd, "\",\"description\":\"" as *u8); er_wjson(jfd, descr_clean)
  kf_w(jfd, "\",\"format\":\"kf8\",\"rfix\":1,\"slug\":\"" as *u8); er_wjson(jfd, slug)   // rfix=reader-fix schema -> server re-extracts older caches
  kf_w(jfd, "\"," as *u8)
  let totalp: *i64 = sys_mmap(8) as *i64; totalp[0]=0
  let nseg: i64 = chapter_split(tbuf, htmllen, dir, title, jfd, totalp, "kf8\x00" as *u8)   // split -> chap<i>.txt + chapters[]/toc[]
  let total: i64 = totalp[0]
  kf_w(jfd, ",\"assets\":[],\"nassets\":0,\"cover\":\"" as *u8); er_wjson(jfd, covername)
  kf_w(jfd, "\"}" as *u8)
  sys_close(jfd)
241
  kf_p("KF8-BOOK title=\"" as *u8); kf_w(1, title); kf_p("\" author=\"" as *u8); kf_w(1, author)
  kf_p("\" boundary=" as *u8); kf_pn(B); kf_p(" comp=" as *u8); kf_pn(comp); kf_p(" kf8_text_records=" as *u8); kf_pn(recCount)
  kf_p(" html_bytes=" as *u8); kf_pn(htmllen); kf_p(" chapters=" as *u8); kf_pn(nseg); kf_p(" chars=" as *u8); kf_pn(total); kf_p(" -> " as *u8); kf_p(dir); kf_p("\n" as *u8)
  if total < 20 { return 0-1 }
  return 0
247}
248
249func main(argc: i64, argv: *i64) -> i64 {
  var path: *u8 = "knowledge/benchmarks/refs/libmobi-sample-huffdic.ref" as *u8
  var slug: *u8 = "real_kf8" as *u8
  if argc >= 3 { path = argv[1] as *u8; slug = argv[2] as *u8 }
  let r: i64 = do_kf8(path, slug)
  if r == 0 { kf_p("KF8-BOOK-OK\n" as *u8); sys_exit(0); return 0 }
  kf_p("KF8-BOOK-FAIL\n" as *u8); sys_exit(1); return 1
256}
nx_kf8_book.nx source

`nx_kf8_book.nx` source
