code wiki / _hdl_build / nx_mobi_book.nx
`nx_mobi_book.nx` source

↩ module page · 305 lines · 16854 B
1// nx_mobi_book.nx -- SOVEREIGN MOBI/AZW (Kindle) -> Nishi format. Operator goal: read your OWN purchased books on
2// your OWN device, freed from the lock-in container. Parses the PDB database + PalmDOC-decompresses the text +
3// reads EXTH metadata, then reuses the SAME pipeline as the EPUB reader (nx_html_to_text -> book.json + chap.txt)
4// so the zero-JS reader renders it unchanged.
5//   LEGAL LINE: this DECODES the FORMAT (interoperability). It DETECTS the MOBI encryptionType flag and DECLINES
6//   DRM-encrypted files (exit 2) -- it does NOT circumvent DRM. DRM-FREE purchases/side-loads/library books decode.
7// Rung 1: whole book -> one chapter (readable); HUFF/CDIC compression + KF8/AZW3 + pagebreak chapter-splitting are
8// next rungs. Usage: nx_mobi_book <mobi-path> <slug>.  expect_exit: 0  license_tier: ORIGINAL
9import "nx_syscalls.nx"
10import "nx_itoa_lib.nx"   // shared MSB-first emitter (zero-alloc)
11import "nx_palmdoc.nx"
12import "nx_huffcdic.nx"
13import "nx_html_to_text.nx"
14import "nx_mobi_cover.nx"
15import "nx_chapter_split.nx"
16import "nx_charset.nx"   // MOJIBAKE FIX: nx_charset_repair_utf8 -- stray Windows-1252 bytes -> clean UTF-8
17const K_MAGIC_1252: i64 = 1252
18const K_MAGIC_17480: i64 = 17480
19const K_MAGIC_262144: i64 = 262144
20const K_MAGIC_1024: i64 = 1024
21const K_MAGIC_8192: i64 = 8192
22
23func mb_slen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
24func mb_w(fd: i64, s: *u8) -> i64 { sys_write(fd, s, mb_slen(s)); return 0 }
25// MIGRATED to the shared emitter (debt 1785563586). The old body mmapped a scratch buffer
26// per call and never freed it. At PAGE granularity that is 4096B leaked PER CALL -- the
27// defect that took 28.5GB of a 36GB host in nx_ts_lumadiff (2MB input, ~3.66M calls).
28// nxi_* is MSB-first, allocates NOTHING, and emits identical bytes including the sign.
29func mb_n(fd: i64, v: i64) -> i64 { nxi_fd(fd, v); return 0 }
30func mb_p(s: *u8) -> i64 { mb_w(1, s); return 0 }
31func mb_pn(v: i64) -> i64 { mb_n(1, v); return 0 }
32func ensure_dir(path: *u8) -> i64 { __syscall(258, 0-100, path, 0x1ed, 0, 0, 0); return 0 }
33
34func be16(b: *u8, o: i64) -> i64 { return ((b[o] as i64) << 8) | (b[o+1] as i64) }
35func be32(b: *u8, o: i64) -> i64 { return ((b[o] as i64) << 24) | ((b[o+1] as i64) << 16) | ((b[o+2] as i64) << 8) | (b[o+3] as i64) }
36
37func find_sub(hay: *u8, hl: i64, needle: *u8) -> i64 {
  let nl: i64 = mb_slen(needle); if nl == 0 { return 0-1 }
  var i: i64 = 0
  while i + nl <= hl { var k: i64=0; var hit: i64=1; while k<nl { if hay[i+k]!=needle[k]{hit=0;k=nl}else{k=k+1} } if hit==1 {return i} i=i+1 }
  return 0-1
42}
43// extract the <h1> text (inner tags stripped) from html[0..hl) into out; ret len (0 if no <h1>). Used for chapter titles.
44func extract_h1(html: *u8, hl: i64, out: *u8, cap: i64) -> i64 {
  out[0] = 0 as u8
  let h1: i64 = find_sub(html, hl, "<h1" as *u8)
  if h1 < 0 { return 0 }
  var gt: i64 = 0-1; var p: i64 = h1
  while p < hl { if html[p] == (0x3e as u8) { gt = p; p = hl } else { p = p + 1 } }
  if gt < 0 { return 0 }
  let cs: i64 = gt + 1
  let he: i64 = find_sub((html as i64 + cs) as *u8, hl - cs, "</h1" as *u8)
  var ce: i64 = hl
  if he >= 0 { ce = cs + he }
  var o: i64 = 0; var i: i64 = cs; var intag: i64 = 0
  while i < ce {
      let c: i64 = html[i] as i64
      if c == 0x3c { intag = 1 } else { if c == 0x3e { intag = 0 } else { if intag == 0 { if o < cap-1 { out[o]=html[i] as u8; o=o+1 } } } }
      i = i + 1
  }
  out[o] = 0 as u8
  return o
63}
64func er_cat(dst: *u8, off: i64, s: *u8) -> i64 { var i: i64=0; while s[i]!=(0 as u8){dst[off+i]=s[i];i=i+1} return off+i }
65func er_catn(dst: *u8, off: i64, v: i64) -> i64 { let t: *u8=sys_mmap(28); var m: i64=v; var k: i64=0; if m==0{t[0]=48 as u8;k=1}; while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1}; var i: i64=0; while i<k{dst[off+i]=t[k-1-i];i=i+1} return off+k }
66// JSON string emit. MOJIBAKE FIX: repair each metadata string (title/author/publisher/description...) to clean
67// UTF-8 first, so a stray Windows-1252 byte renders correctly instead of U+FFFD. Idempotent on valid UTF-8.
68func er_wjson(fd: i64, s: *u8) -> i64 {
  var sl: i64 = 0; while s[sl] != (0 as u8) { sl = sl + 1 }
  let rcap: i64 = sl * 3 + 16
  let rep: *u8 = sys_mmap(rcap)
  let rn: i64 = nx_charset_repair_utf8(s, sl, rep, rcap)
  var i: i64 = 0
  while i < rn {
      let c: i64 = rep[i] as i64
      if c == 0x22 { sys_write(fd, "\\\"" as *u8, 2) }
      else { if c == 0x5c { sys_write(fd, "\\\\" as *u8, 2) }
      else { if c < 0x20 { sys_write(fd, " " as *u8, 1) }
      else { sys_write(fd, (rep as i64 + i) as *u8, 1) } } }
      i = i + 1
  }
  sys_munmap(rep, rcap)
  return 0
84}
85// numeric character-reference decode (&#DDDD; / &#xHHHH; -> UTF-8), same proven shape as nx_epub_book.
86func utf8_put(out: *u8, o: i64, cp: i64) -> i64 {
  if cp < 0x80 { out[o] = cp as u8; return o + 1 }
  if cp < 0x800 { out[o] = (0xC0 | (cp >> 6)) as u8; out[o+1] = (0x80 | (cp & 0x3F)) as u8; return o + 2 }
  if cp < 0x10000 { out[o] = (0xE0 | (cp >> 12)) as u8; out[o+1] = (0x80 | ((cp >> 6) & 0x3F)) as u8; out[o+2] = (0x80 | (cp & 0x3F)) as u8; return o + 3 }
  out[o] = (0xF0 | (cp >> 18)) as u8; out[o+1] = (0x80 | ((cp >> 12) & 0x3F)) as u8; out[o+2] = (0x80 | ((cp >> 6) & 0x3F)) as u8; out[o+3] = (0x80 | (cp & 0x3F)) as u8; return o + 4
91}
92func decode_numeric_refs(txt: *u8, n: i64, out: *u8, outcap: i64) -> i64 {
  var i: i64 = 0
  var o: i64 = 0
  while i < n {
      let c: i64 = txt[i] as i64
      var done: i64 = 0
      if c == 0x26 { if i + 1 < n { if (txt[i+1] as i64) == 0x23 {
          var j: i64 = i + 2
          var hex: i64 = 0
          if j < n { let xc: i64 = txt[j] as i64; if xc == 0x78 { hex = 1; j = j + 1 } else { if xc == 0x58 { hex = 1; j = j + 1 } } }
          var cp: i64 = 0
          var ndig: i64 = 0
          var scan: i64 = 1
          while scan == 1 {
              if j >= n { scan = 0 } else {
                  let dc: i64 = txt[j] as i64
                  var dv: i64 = 0-1
                  if hex == 1 { if dc >= 48 { if dc <= 57 { dv = dc - 48 } } if dc >= 97 { if dc <= 102 { dv = dc - 87 } } if dc >= 65 { if dc <= 70 { dv = dc - 55 } } }
                  else { if dc >= 48 { if dc <= 57 { dv = dc - 48 } } }
                  if dv < 0 { scan = 0 } else { if hex == 1 { cp = cp * 16 + dv } else { cp = cp * 10 + dv } ndig = ndig + 1; j = j + 1 }
              }
          }
          if ndig > 0 { if j < n { if (txt[j] as i64) == 0x3b { if cp > 0 { if cp <= 0x10FFFF { if o + 4 < outcap { o = utf8_put(out, o, cp) } i = j + 1; done = 1 } } } } }
      } } }
      if done == 0 { if o < outcap - 1 { out[o] = txt[i]; o = o + 1 } i = i + 1 }
  }
  out[o] = 0 as u8
  return o
120}
121// copy EXTH record value [p+8, p+rlen) of record at p into out (NUL-term).
122func exth_copy(z: *u8, p: i64, rlen: i64, out: *u8, cap: i64) -> i64 {
  var o: i64 = 0
  var s: i64 = p + 8
  let e: i64 = p + rlen
  while s < e { if o < cap-1 { out[o] = z[s]; o = o + 1 } s = s + 1 }
  out[o] = 0 as u8
  return o
129}
130
131const MCAP: i64 = 16777216
132
133func do_mobi(path: *u8, slug: *u8) -> i64 {
  let lb: *i64 = sys_mmap(16) as *i64
  let z: *u8 = sys_read_file(path, lb)
  if (z as i64) == 0 { mb_p("MOBI-FAIL read " as *u8); mb_p(path); mb_p("\n" as *u8); return 0-1 }
  let zl: i64 = lb[0]
  if zl < 80 { mb_p("MOBI-FAIL too-small\n" as *u8); return 0-1 }
139
  let numRec: i64 = be16(z, 76)
  if numRec < 2 { mb_p("MOBI-FAIL no-text-records\n" as *u8); return 0-1 }
  let rec0off: i64 = be32(z, 78)
  if rec0off + 16 > zl { mb_p("MOBI-FAIL bad-rec0\n" as *u8); return 0-1 }
144
  let comp: i64 = be16(z, rec0off + 0)
  let textLen: i64 = be32(z, rec0off + 4)
  let recCount: i64 = be16(z, rec0off + 8)
  let enc: i64 = be16(z, rec0off + 12)
149
  // ---- LEGAL LINE: DRM is detected and DECLINED, never circumvented ----
  if enc != 0 {
      mb_p("MOBI-DRM-DECLINED encryptionType=" as *u8); mb_pn(enc)
      mb_p(" (the sovereign reader decodes DRM-FREE MOBI/AZW; it does NOT circumvent DRM. Use a DRM-free copy.)\n" as *u8)
      sys_exit(2); return 2
  }
  var is_huff: i64 = 0
  if comp == K_MAGIC_17480 { is_huff = 1 } else { if comp != 1 { if comp != 2 {
      mb_p("MOBI-UNSUPPORTED compression=" as *u8); mb_pn(comp); mb_p(" (supported: none/PalmDOC/HUFF-CDIC)\n" as *u8)
      sys_exit(3); return 3
  } } }
161
  // ---- HUFF/CDIC: load the Huffman tables (HUFF record) + dictionary (CDIC records) before decode ----
  let d1cl: *i64 = sys_mmap(256*8) as *i64
  let d1tm: *i64 = sys_mmap(256*8) as *i64
  let d1mx: *i64 = sys_mmap(256*8) as *i64
  let hmin: *i64 = sys_mmap(64*8) as *i64
  let hmax: *i64 = sys_mmap(64*8) as *i64
  let dptr: *i64 = sys_mmap(K_MAGIC_262144*8) as *i64
  let dlen: *i64 = sys_mmap(K_MAGIC_262144*8) as *i64
  let dflag: *i64 = sys_mmap(K_MAGIC_262144*8) as *i64
  let dcnt: *i64 = sys_mmap(8) as *i64; dcnt[0]=0
  if is_huff == 1 {
      let huffidx: i64 = be32(z, rec0off + 112)        // Huffman Record Offset: the spec's 0x70 is RECORD-0-relative (incl the 16B PalmDOC hdr), verified on a real kindlegen file
      let huffcnt: i64 = be32(z, rec0off + 116)        // Huffman Record Count (record-0 0x74)
      let hs: i64 = be32(z, 78 + huffidx*8)
      var he: i64 = zl
      if huffidx + 1 < numRec { he = be32(z, 78 + (huffidx+1)*8) }
      if huff_load((z as i64 + hs) as *u8, he - hs, d1cl, d1tm, d1mx, hmin, hmax) != 0 {
          mb_p("MOBI-HUFF-FAIL bad-HUFF-record\n" as *u8); sys_exit(3); return 3
      }
      var ci: i64 = 1
      while ci < huffcnt {
          let cidx: i64 = huffidx + ci
          let cs: i64 = be32(z, 78 + cidx*8)
          var ce: i64 = zl
          if cidx + 1 < numRec { ce = be32(z, 78 + (cidx+1)*8) }
          cdic_load((z as i64 + cs) as *u8, ce - cs, dptr, dlen, dflag, dcnt)
          ci = ci + 1
      }
  }
191
  // ---- decompress/concat text records 1..recCount ----
  let tbuf: *u8 = sys_mmap(MCAP)
  var to: i64 = 0
  var ri: i64 = 1
  while ri <= recCount {
      if ri >= numRec { ri = recCount + 1 } else {
          let rs: i64 = be32(z, 78 + ri*8)
          var re: i64 = zl
          if ri + 1 < numRec { re = be32(z, 78 + (ri+1)*8) }
          let rlen: i64 = re - rs
          if rlen > 0 { if rs + rlen <= zl {
              if comp == 2 { to = to + palmdoc_decompress((z as i64 + rs) as *u8, rlen, (tbuf as i64 + to) as *u8, MCAP - to) }
              else { if is_huff == 1 { to = to + hc_unpack((z as i64 + rs) as *u8, rlen, (tbuf as i64 + to) as *u8, MCAP - to, d1cl, d1tm, d1mx, hmin, hmax, dptr, dlen, dflag, dcnt[0]) }
              else { var k: i64=0; while k < rlen { if to < MCAP { tbuf[to]=z[rs+k]; to=to+1 } k=k+1 } } }
          } }
          ri = ri + 1
      }
  }
  var htmllen: i64 = to
  if textLen > 0 { if textLen < htmllen { htmllen = textLen } }
212
  // ---- metadata: EXTH (author=100, title=503), fallback title = PDB database name ----
  let title: *u8 = sys_mmap(K_MAGIC_1024); title[0] = 0 as u8
  let author: *u8 = sys_mmap(K_MAGIC_1024); author[0] = 0 as u8
  // richer EXTH metadata ("all the book info"): publisher 101 / isbn 104 / pubdate 106 -- captured from the file itself
  let publisher: *u8 = sys_mmap(K_MAGIC_1024); publisher[0] = 0 as u8
  let isbn: *u8 = sys_mmap(256); isbn[0] = 0 as u8
  let pubdate: *u8 = sys_mmap(256); pubdate[0] = 0 as u8
  let descr_raw: *u8 = sys_mmap(K_MAGIC_8192); descr_raw[0] = 0 as u8   // EXTH 103 description (HTML; html_to_text'd before book.json)
  // EMBEDDED-COVER pointers (spec mobileread-mobi-spec.ref): EXTH 201 coveroffset / 202 thumboffset are ADDED to
  // the MOBI header's "First Image index" to find the cover PDB record. Captured here during the EXTH scan.
  var coveroff: i64 = 0; var coverfound: i64 = 0; var thumboff: i64 = 0; var thumbfound: i64 = 0
  let rec1off: i64 = be32(z, 78 + 8)
  var r0ext: i64 = rec1off - rec0off
  if r0ext <= 0 { r0ext = zl - rec0off }
  let exrel: i64 = find_sub((z as i64 + rec0off) as *u8, r0ext, "EXTH" as *u8)
  if exrel >= 0 {
      let exth: i64 = rec0off + exrel
      let cnt: i64 = be32(z, exth + 8)
      var p: i64 = exth + 12
      var ci: i64 = 0
      while ci < cnt {
          if p + 8 > zl { ci = cnt } else {
              let rtype: i64 = be32(z, p)
              let rlen: i64 = be32(z, p + 4)
              if rlen < 8 { ci = cnt } else {
                  if rtype == 100 { exth_copy(z, p, rlen, author, K_MAGIC_1024) }
                  if rtype == 503 { exth_copy(z, p, rlen, title, K_MAGIC_1024) }
                  if rtype == 201 { if rlen >= 12 { coveroff = be32(z, p + 8); coverfound = 1 } }   // EXTH 201 coveroffset
                  if rtype == 202 { if rlen >= 12 { thumboff = be32(z, p + 8); thumbfound = 1 } }   // EXTH 202 thumboffset
                  if rtype == 101 { exth_copy(z, p, rlen, publisher, K_MAGIC_1024) }   // EXTH 101 publisher
                  if rtype == 104 { exth_copy(z, p, rlen, isbn, 256) }         // EXTH 104 isbn
                  if rtype == 106 { exth_copy(z, p, rlen, pubdate, 256) }      // EXTH 106 publishing date
                  if rtype == 103 { exth_copy(z, p, rlen, descr_raw, K_MAGIC_8192) }   // EXTH 103 description (HTML)
                  p = p + rlen
                  ci = ci + 1
              }
          }
      }
  }
  if title[0] == (0 as u8) { var k: i64=0; while k < 31 { let c: i64 = z[k] as i64; if c==0 { k=31 } else { title[k]=z[k] as u8; k=k+1 } } title[k]=0 as u8 }
253
  // ---- description (EXTH 103) is HTML -> strip tags + decode numeric refs so book.json carries clean prose ----
  let descr_clean: *u8 = sys_mmap(K_MAGIC_8192); descr_clean[0] = 0 as u8
  if descr_raw[0] != (0 as u8) {
      let dtmp: *u8 = sys_mmap(K_MAGIC_8192)
      let dtn0: i64 = nx_html_to_text(descr_raw, mb_slen(descr_raw), dtmp, K_MAGIC_8192)
      decode_numeric_refs(dtmp, dtn0, descr_clean, K_MAGIC_8192)
  }
261
  // ---- emit reader/<slug>/ dir + cover (the chapter split + chap<i>.txt files are done by nx_chapter_split, below) ----
  ensure_dir("knowledge/staging/media/reader" as *u8)
  let dir: *u8 = sys_mmap(512)
  var dl: i64 = er_cat(dir, 0, "knowledge/staging/media/reader/" as *u8)
  dl = er_cat(dir, dl, slug); dir[dl] = 0 as u8
  ensure_dir(dir)
  let covername: *u8 = sys_mmap(64); covername[0] = 0 as u8   // EMBEDDED cover (shared w/ nx_kf8_book via nx_mobi_cover)
  mobi_extract_cover(z, zl, numRec, rec0off, coveroff, coverfound, thumboff, thumbfound, dir, covername)
270
  // ---- book.json (metadata + chapters[] + toc[]) ----
  let jp: *u8 = sys_mmap(K_MAGIC_1024)
  var jl: i64 = er_cat(jp, 0, dir as *u8); jl = er_cat(jp, jl, "/book.json" as *u8); jp[jl] = 0 as u8
  let jfd: i64 = sys_openat_wr(jp, 0x1a4)
  if jfd < 0 { mb_p("MOBI-FAIL open-json\n" as *u8); return 0-1 }
  mb_w(jfd, "{\"title\":\"" as *u8); er_wjson(jfd, title)
  mb_w(jfd, "\",\"author\":\"" as *u8); er_wjson(jfd, author)
  mb_w(jfd, "\",\"publisher\":\"" as *u8); er_wjson(jfd, publisher)
  mb_w(jfd, "\",\"pubdate\":\"" as *u8); er_wjson(jfd, pubdate)
  mb_w(jfd, "\",\"isbn\":\"" as *u8); er_wjson(jfd, isbn)
  mb_w(jfd, "\",\"description\":\"" as *u8); er_wjson(jfd, descr_clean)
  mb_w(jfd, "\",\"format\":\"mobi\",\"rfix\":1,\"slug\":\"" as *u8); er_wjson(jfd, slug)   // rfix=reader-fix schema -> server re-extracts older caches
  mb_w(jfd, "\"," as *u8)
  let totalp: *i64 = sys_mmap(8) as *i64; totalp[0]=0
  let nseg: i64 = chapter_split(tbuf, htmllen, dir, title, jfd, totalp, "mobi\x00" as *u8)   // split -> chap<i>.txt + chapters[]/toc[]
  let total: i64 = totalp[0]
  mb_w(jfd, ",\"assets\":[],\"nassets\":0,\"cover\":\"" as *u8); er_wjson(jfd, covername)
  mb_w(jfd, "\"}" as *u8)
  sys_close(jfd)
290
  mb_p("MOBI-BOOK title=\"" as *u8); mb_w(1, title); mb_p("\" author=\"" as *u8); mb_w(1, author)
  mb_p("\" comp=" as *u8); mb_pn(comp); mb_p(" enc=" as *u8); mb_pn(enc); mb_p(" text_records=" as *u8); mb_pn(recCount)
  mb_p(" html_bytes=" as *u8); mb_pn(htmllen); mb_p(" chapters=" as *u8); mb_pn(nseg); mb_p(" chars=" as *u8); mb_pn(total); mb_p(" -> " as *u8); mb_p(dir); mb_p("\n" as *u8)
  if total < 20 { return 0-1 }
  return 0
296}
297
298func main(argc: i64, argv: *i64) -> i64 {
  var path: *u8 = "knowledge/fixtures/nishi_mobi_fixture.mobi" as *u8
  var slug: *u8 = "nishi_mobi_fixture" as *u8
  if argc >= 3 { path = argv[1] as *u8; slug = argv[2] as *u8 }
  let r: i64 = do_mobi(path, slug)
  if r == 0 { mb_p("MOBI-BOOK-OK\n" as *u8); sys_exit(0); return 0 }
  mb_p("MOBI-BOOK-FAIL\n" as *u8); sys_exit(1); return 1
305}
nx_mobi_book.nx source

`nx_mobi_book.nx` source
