code wiki / _hdl_build / nx_epub_read.nx
nx_epub_read.nx source
↩ module page · 349 lines · 15245 B
1// nx_epub_read.nx -- SOVEREIGN EPUB read (Reader-arc RUNG 2).
2//
3// "The agnostic Nishi read working" on EPUB, over the operator's REAL Calibre library.
4// Composes (anti-reinvention): nx_deflate.nx (nx_deflate_inflate -- the GREEN inflate keystone),
5// nx_html_to_text.nx (XHTML -> plain text). The ONLY new logic is a ZIP central-directory walker
6// (name-based entry lookup; the authored _zip_extract does first-entry only) + the EPUB/OPF
7// structure walk (container.xml -> OPF -> metadata/manifest/spine) + first-real-chapter pick.
8//
9// SOVEREIGN: parse path is pure Nishi organs + raw syscalls. NO python/unzip/3rd-party-xml/gcc/sh,
10// NO calibre binaries. (Browser HTML/CSS RENDER is a later rung 2b; this rung is parse+extract only.)
11// NO-FAKE-GREEN: the 3 inputs are REAL files on the NAS; title/author/spine/text are MEASURED from
12// the bytes, never hardcoded. Emits the unified book model (book<N>.json + book<N>.ch1.txt), ADDITIVE.
13// license_tier: ORIGINAL
14import "nx_syscalls.nx"
15import "nx_itoa_lib.nx" // shared MSB-first emitter (zero-alloc)
16import "nx_deflate.nx"
17import "nx_html_to_text.nx"
18const CHAP_MAGIC_262144: i64 = 262144
19const CHAP_MAGIC_1024: i64 = 1024
20const CHAP_MAGIC_2097152: i64 = 2097152
21const CHAP_MAGIC_65536: i64 = 65536
22const CHAP_MAGIC_2048: i64 = 2048
23
24// ---- io + string helpers ----
25func er_slen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
26func er_w(fd: i64, s: *u8) -> i64 { sys_write(fd, s, er_slen(s)); return 0 }
27// MIGRATED to the shared emitter (debt 1785563586). The old body mmapped a scratch buffer
28// per call and never freed it. At PAGE granularity that is 4096B leaked PER CALL -- the
29// defect that took 28.5GB of a 36GB host in nx_ts_lumadiff (2MB input, ~3.66M calls).
30// nxi_* is MSB-first, allocates NOTHING, and emits identical bytes including the sign.
31func er_n(fd: i64, v: i64) -> i64 { nxi_fd(fd, v); return 0 }
32func er_p(s: *u8) -> i64 { er_w(1, s); return 0 }
33func er_pn(v: i64) -> i64 { er_n(1, v); return 0 }
34
35func le16(b: *u8, o: i64) -> i64 { return (b[o] as i64) | ((b[o+1] as i64) << 8) }
36func le32(b: *u8, o: i64) -> i64 { return (b[o] as i64) | ((b[o+1] as i64) << 8) | ((b[o+2] as i64) << 16) | ((b[o+3] as i64) << 24) }
37
38func er_cat(dst: *u8, off: i64, s: *u8) -> i64 { var i: i64=0; while s[i]!=(0 as u8){dst[off+i]=s[i];i=i+1} return off+i }
39func er_catn(dst: *u8, off: i64, v: i64) -> i64 { let t: *u8=sys_mmap(28); var m: i64=v; var k: i64=0; if m==0{t[0]=48 as u8;k=1}; while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1}; var i: i64=0; while i<k{dst[off+i]=t[k-1-i];i=i+1} return off+k }
40
41// substring search; return index in hay[0..hl] or -1
42func find_sub(hay: *u8, hl: i64, needle: *u8) -> i64 {
43 let nl: i64 = er_slen(needle)
44 if nl == 0 { return 0-1 }
45 var i: i64 = 0
46 while i + nl <= hl {
47 var k: i64 = 0
48 var hit: i64 = 1
49 while k < nl { if hay[i+k] != needle[k] { hit = 0; k = nl } else { k = k + 1 } }
50 if hit == 1 { return i }
51 i = i + 1
52 }
53 return 0-1
54}
55
56// count non-overlapping occurrences of needle in hay[0..hl]
57func count_sub(hay: *u8, hl: i64, needle: *u8) -> i64 {
58 let nl: i64 = er_slen(needle)
59 if nl == 0 { return 0 }
60 var c: i64 = 0
61 var i: i64 = 0
62 while i + nl <= hl {
63 var k: i64 = 0
64 var hit: i64 = 1
65 while k < nl { if hay[i+k] != needle[k] { hit = 0; k = nl } else { k = k + 1 } }
66 if hit == 1 { c = c + 1; i = i + nl } else { i = i + 1 }
67 }
68 return c
69}
70
71// extract attribute value: find attr (e.g. `full-path="`) then copy until the next '"'
72func attr_val(hay: *u8, hl: i64, attr: *u8, out: *u8, outcap: i64) -> i64 {
73 let at: i64 = find_sub(hay, hl, attr)
74 if at < 0 { return 0-1 }
75 var s: i64 = at + er_slen(attr)
76 var o: i64 = 0
77 while s < hl { if hay[s] == (0x22 as u8) { break } if o < outcap-1 { out[o] = hay[s]; o = o + 1 } s = s + 1 }
78 out[o] = 0 as u8
79 return o
80}
81
82// extract element text: find opentag (e.g. `<dc:title`), skip to '>', copy until next '<'
83func elem_text(hay: *u8, hl: i64, opentag: *u8, out: *u8, outcap: i64) -> i64 {
84 let at: i64 = find_sub(hay, hl, opentag)
85 if at < 0 { return 0-1 }
86 var s: i64 = at
87 while s < hl { if hay[s] == (0x3e as u8) { s = s + 1; break } s = s + 1 }
88 var o: i64 = 0
89 while s < hl { if hay[s] == (0x3c as u8) { break } if o < outcap-1 { out[o] = hay[s]; o = o + 1 } s = s + 1 }
90 out[o] = 0 as u8
91 return o
92}
93
94// ---- ZIP central-directory walker (name-based extract) ----
95// find EOCD (End Of Central Directory, sig PK\x05\x06) scanning backward
96func find_eocd(z: *u8, zl: i64) -> i64 {
97 var i: i64 = zl - 22
98 if i < 0 { return 0-1 }
99 while i >= 0 {
100 if z[i] == (0x50 as u8) { if z[i+1] == (0x4b as u8) { if z[i+2] == (0x05 as u8) { if z[i+3] == (0x06 as u8) { return i } } } }
101 i = i - 1
102 }
103 return 0-1
104}
105
106// locate named entry; fills ob[0]=method ob[1]=compsize ob[2]=uncompsize ob[3]=dataoff; ret 1/0
107func zip_find(z: *u8, zl: i64, target: *u8, ob: *i64) -> i64 {
108 let eocd: i64 = find_eocd(z, zl)
109 if eocd < 0 { return 0 }
110 let count: i64 = le16(z, eocd + 10)
111 var cd: i64 = le32(z, eocd + 16)
112 let tlen: i64 = er_slen(target)
113 var k: i64 = 0
114 while k < count {
115 if cd + 46 > zl { return 0 }
116 if z[cd] != (0x50 as u8) { return 0 }
117 let method: i64 = le16(z, cd + 10)
118 let comp: i64 = le32(z, cd + 20)
119 let uncomp: i64 = le32(z, cd + 24)
120 let fnlen: i64 = le16(z, cd + 28)
121 let extralen: i64 = le16(z, cd + 30)
122 let commentlen: i64 = le16(z, cd + 32)
123 let lho: i64 = le32(z, cd + 42)
124 var mt: i64 = 0
125 if fnlen == tlen {
126 mt = 1
127 var i: i64 = 0
128 while i < fnlen { if z[cd + 46 + i] != target[i] { mt = 0; i = fnlen } else { i = i + 1 } }
129 }
130 if mt == 1 {
131 let lfn: i64 = le16(z, lho + 26)
132 let lex: i64 = le16(z, lho + 28)
133 ob[0] = method; ob[1] = comp; ob[2] = uncomp; ob[3] = lho + 30 + lfn + lex
134 return 1
135 }
136 cd = cd + 46 + fnlen + extralen + commentlen
137 k = k + 1
138 }
139 return 0
140}
141
142// extract named entry into out (cap); return uncompressed size or <0
143func zip_read_named(z: *u8, zl: i64, target: *u8, out: *u8, outcap: i64) -> i64 {
144 let ob: *i64 = sys_mmap(64) as *i64
145 if zip_find(z, zl, target, ob) == 0 { return 0-1 }
146 let method: i64 = ob[0]
147 let comp: i64 = ob[1]
148 let dataoff: i64 = ob[3]
149 if method == 0 {
150 var i: i64 = 0
151 while i < comp { if i < outcap { out[i] = z[dataoff + i] } i = i + 1 }
152 return comp
153 }
154 if method == 8 {
155 let r: *NxDeflateResult = nx_deflate_inflate((z as i64 + dataoff) as *u8, comp, outcap)
156 if (r as i64) == 0 { return 0-1 }
157 let n: i64 = r.output_size
158 var i: i64 = 0
159 while i < n { if i < outcap { out[i] = r.output_data[i] } i = i + 1 }
160 return n
161 }
162 return 0-2
163}
164
165// href for a given manifest id: find id="<idref>", isolate the enclosing <...> tag, read href="
166func href_for_id(opf: *u8, ol: i64, idref: *u8, out: *u8, outcap: i64) -> i64 {
167 let needle: *u8 = sys_mmap(512)
168 var w: i64 = er_cat(needle, 0, "id=\"" as *u8)
169 w = er_cat(needle, w, idref)
170 needle[w] = 0x22 as u8; w = w + 1
171 needle[w] = 0 as u8
172 let at: i64 = find_sub(opf, ol, needle)
173 if at < 0 { return 0-1 }
174 var ts: i64 = at
175 while ts > 0 { if opf[ts] == (0x3c as u8) { break } ts = ts - 1 }
176 var te: i64 = at
177 while te < ol { if opf[te] == (0x3e as u8) { break } te = te + 1 }
178 return attr_val((opf as i64 + ts) as *u8, te - ts, "href=\"" as *u8, out, outcap)
179}
180
181// join OPF directory + href -> full zip path
182func join_path(opfpath: *u8, href: *u8, out: *u8) -> i64 {
183 let ol: i64 = er_slen(opfpath)
184 var lastslash: i64 = 0-1
185 var i: i64 = 0
186 while i < ol { if opfpath[i] == (0x2f as u8) { lastslash = i } i = i + 1 }
187 var o: i64 = 0
188 if lastslash >= 0 { var k: i64 = 0; while k <= lastslash { out[o] = opfpath[k]; o = o + 1; k = k + 1 } }
189 var j: i64 = 0
190 while href[j] != (0 as u8) { out[o] = href[j]; o = o + 1; j = j + 1 }
191 out[o] = 0 as u8
192 return o
193}
194
195// collect spine idref strings (in reading order) into arena (NUL-separated), offs[]=starts; ret count
196func get_spine_idrefs(opf: *u8, ol: i64, arena: *u8, offs: *i64, cap: i64) -> i64 {
197 let spos: i64 = find_sub(opf, ol, "<spine" as *u8)
198 if spos < 0 { return 0 }
199 var spe: i64 = find_sub((opf as i64 + spos) as *u8, ol - spos, "</spine" as *u8)
200 if spe < 0 { spe = ol - spos }
201 spe = spos + spe
202 var p: i64 = spos
203 var n: i64 = 0
204 var aoff: i64 = 0
205 while p < spe {
206 if n >= cap { break }
207 let rem: i64 = spe - p
208 let at: i64 = find_sub((opf as i64 + p) as *u8, rem, "idref=\"" as *u8)
209 if at < 0 { break }
210 var s: i64 = p + at + 7
211 offs[n] = aoff
212 while s < spe { if opf[s] == (0x22 as u8) { break } arena[aoff] = opf[s]; aoff = aoff + 1; s = s + 1 }
213 arena[aoff] = 0 as u8; aoff = aoff + 1
214 n = n + 1
215 p = s + 1
216 }
217 return n
218}
219
220// write the unified book model (additive): book<idx>.json + book<idx>.ch1.txt
221func er_wjson(fd: i64, s: *u8) -> i64 {
222 var i: i64 = 0
223 while s[i] != (0 as u8) {
224 let c: i64 = s[i] as i64
225 if c == 0x22 { sys_write(fd, "\\\"" as *u8, 2) }
226 else { if c == 0x5c { sys_write(fd, "\\\\" as *u8, 2) }
227 else { if c < 0x20 { sys_write(fd, " " as *u8, 1) }
228 else { sys_write(fd, (s as i64 + i) as *u8, 1) } } }
229 i = i + 1
230 }
231 return 0
232}
233
234func write_book(idx: i64, title: *u8, author: *u8, nspine: i64, chosen: i64, textn: i64, text: *u8) -> i64 {
235 let jp: *u8 = sys_mmap(256)
236 var o: i64 = er_cat(jp, 0, "knowledge/staging/media/reader/book" as *u8)
237 o = er_catn(jp, o, idx)
238 o = er_cat(jp, o, ".json" as *u8); jp[o] = 0 as u8
239 let fd: i64 = sys_openat_wr(jp, 0x1a4)
240 if fd < 0 { return 0-1 }
241 er_w(fd, "{\"title\":\"" as *u8); er_wjson(fd, title)
242 er_w(fd, "\",\"author\":\"" as *u8); er_wjson(fd, author)
243 er_w(fd, "\",\"format\":\"epub\",\"spine\":" as *u8); er_n(fd, nspine)
244 er_w(fd, ",\"chapter_index\":" as *u8); er_n(fd, chosen)
245 er_w(fd, ",\"ch1_chars\":" as *u8); er_n(fd, textn)
246 er_w(fd, ",\"ch1_file\":\"book" as *u8); er_n(fd, idx); er_w(fd, ".ch1.txt\"}" as *u8)
247 sys_close(fd)
248 let tp: *u8 = sys_mmap(256)
249 o = er_cat(tp, 0, "knowledge/staging/media/reader/book" as *u8)
250 o = er_catn(tp, o, idx)
251 o = er_cat(tp, o, ".ch1.txt" as *u8); tp[o] = 0 as u8
252 let tfd: i64 = sys_openat_wr(tp, 0x1a4)
253 if tfd < 0 { return 0-1 }
254 sys_write(tfd, text, textn)
255 sys_close(tfd)
256 return 0
257}
258
259const CHAP_CAP: i64 = 4194304
260
261func do_book(epath: *u8, idx: i64) -> i64 {
262 let lb: *i64 = sys_mmap(16) as *i64
263 let zip: *u8 = sys_read_file(epath, lb)
264 if (zip as i64) == 0 { er_p("EPUB-READ-FAIL read book" as *u8); er_pn(idx); er_p("\n" as *u8); return 0-1 }
265 let zl: i64 = lb[0]
266
267 // container.xml -> OPF path
268 let cbuf: *u8 = sys_mmap(CHAP_MAGIC_262144)
269 let cn: i64 = zip_read_named(zip, zl, "META-INF/container.xml" as *u8, cbuf, CHAP_MAGIC_262144)
270 if cn < 0 { er_p("EPUB-READ-FAIL container book" as *u8); er_pn(idx); er_p("\n" as *u8); return 0-1 }
271 let opfpath: *u8 = sys_mmap(CHAP_MAGIC_1024)
272 let opn: i64 = attr_val(cbuf, cn, "full-path=\"" as *u8, opfpath, CHAP_MAGIC_1024)
273 if opn < 0 { er_p("EPUB-READ-FAIL opfpath book" as *u8); er_pn(idx); er_p("\n" as *u8); return 0-1 }
274
275 // OPF -> metadata + spine
276 let obuf: *u8 = sys_mmap(CHAP_MAGIC_2097152)
277 let on: i64 = zip_read_named(zip, zl, opfpath, obuf, CHAP_MAGIC_2097152)
278 if on < 0 { er_p("EPUB-READ-FAIL opf book" as *u8); er_pn(idx); er_p("\n" as *u8); return 0-1 }
279 let title: *u8 = sys_mmap(CHAP_MAGIC_1024)
280 var tn: i64 = elem_text(obuf, on, "<dc:title" as *u8, title, CHAP_MAGIC_1024)
281 if tn < 0 { tn = elem_text(obuf, on, "<title" as *u8, title, CHAP_MAGIC_1024) }
282 if tn < 0 { title[0] = 0 as u8 }
283 let author: *u8 = sys_mmap(CHAP_MAGIC_1024)
284 var an: i64 = elem_text(obuf, on, "<dc:creator" as *u8, author, CHAP_MAGIC_1024)
285 if an < 0 { author[0] = 0 as u8 }
286
287 let arena: *u8 = sys_mmap(CHAP_MAGIC_65536)
288 let offs: *i64 = sys_mmap(8 * 512) as *i64
289 let nspine: i64 = get_spine_idrefs(obuf, on, arena, offs, 512)
290
291 // pick the first real chapter (running max text; stop once >= 400 chars), scan up to 16 spine items
292 let xbuf: *u8 = sys_mmap(CHAP_CAP)
293 let tbuf: *u8 = sys_mmap(CHAP_CAP)
294 let bestbuf: *u8 = sys_mmap(CHAP_CAP)
295 var bestlen: i64 = 0
296 var chosen: i64 = 0-1
297 let href: *u8 = sys_mmap(CHAP_MAGIC_1024)
298 let chpath: *u8 = sys_mmap(CHAP_MAGIC_2048)
299 var si: i64 = 0
300 while si < nspine {
301 if si >= 16 { si = nspine }
302 else {
303 let idref: *u8 = (arena as i64 + offs[si]) as *u8
304 let hn: i64 = href_for_id(obuf, on, idref, href, CHAP_MAGIC_1024)
305 if hn > 0 {
306 join_path(opfpath, href, chpath)
307 let xn: i64 = zip_read_named(zip, zl, chpath, xbuf, CHAP_CAP)
308 if xn > 0 {
309 let tn2: i64 = nx_html_to_text(xbuf, xn, tbuf, CHAP_CAP)
310 if tn2 > bestlen {
311 bestlen = tn2
312 chosen = si
313 var c: i64 = 0
314 while c < tn2 { bestbuf[c] = tbuf[c]; c = c + 1 }
315 }
316 if bestlen >= 400 { si = nspine }
317 }
318 }
319 si = si + 1
320 }
321 }
322 if bestlen <= 0 { er_p("EPUB-READ-FAIL no-chapter-text book" as *u8); er_pn(idx); er_p("\n" as *u8); return 0-1 }
323
324 write_book(idx, title, author, nspine, chosen, bestlen, bestbuf)
325
326 er_p("EPUB-READ book" as *u8); er_pn(idx)
327 er_p(" title=\"" as *u8); er_w(1, title)
328 er_p("\" author=\"" as *u8); er_w(1, author)
329 er_p("\" spine=" as *u8); er_pn(nspine)
330 er_p(" chapter=" as *u8); er_pn(chosen)
331 er_p(" ch1_chars=" as *u8); er_pn(bestlen)
332 er_p("\n" as *u8)
333 return 0
334}
335
336func main() -> i64 {
337 er_p("EPUB-READ: start (sovereign zip-central-dir + nx_deflate inflate + html->text)\n" as *u8)
338 // create the reader output dir (idempotent; -EEXIST is harmless). x86_64 mkdirat=258, AT_FDCWD=-100, 0755.
339 __syscall(258, 0-100, "knowledge/staging/media/reader" as *u8, 0x1ed, 0, 0, 0)
340
341 var ok: i64 = 0
342 if do_book("/mnt/nas_ai/pythonbook.epub" as *u8, 1) == 0 { ok = ok + 1 }
343 if do_book("/mnt/nas_homes/elderwesto/books/Richard Lee Byers/Dissolution_ War of the Spider Queen - Book I (7926)/Dissolution_ War of the Spider Queen - Boo - Richard Lee Byers.epub" as *u8, 2) == 0 { ok = ok + 1 }
344 if do_book("/mnt/nas_homes/elderwesto/books/Fox, Scott/Click Millionaires_ Work Less, Live More with an Internet Business You Love (6633)/Click Millionaires_ Work Less, Live More w - Fox, Scott.epub" as *u8, 3) == 0 { ok = ok + 1 }
345
346 er_p("EPUB-READ: parsed=" as *u8); er_pn(ok); er_p("/3\n" as *u8)
347 if ok == 3 { er_p("EPUB-READ-OK\n" as *u8); sys_exit(0); return 0 }
348 er_p("EPUB-READ-FAIL not-all-3\n" as *u8); sys_exit(1); return 1
349}