nx_html_to_nishi_page.nx source
↩ module page · 417 lines · 18235 B
1// nx_html_to_nishi_page.nx -- HUB primitive; site-conversion tool.
2// HTML input -> Nishi Page Format V1 conforming output (per
3// NISHI_PAGE_FORMAT_V1.md spec).
4//
5// PURPOSE per NISHI_SEARCH_CHARTER §8 + operator cardinal
6// "convert website to nishi site functionality to help with link rot":
7// 1. Accept arbitrary HTML (e.g., archived competitor docs)
8// 2. Emit Nishi Page Format V1 conforming HTML (9 required meta tags)
9// 3. Annotate external links for archive subsystem pickup
10// 4. Flag unconvertible content (iframe/script) with sealed verdicts
11//
12// COMPOSES (HUB primitives only):
13// nx_html_extract (title/body extraction)
14// nx_html_escape (safe meta-value emission)
15// nx_search_inverted (FNV-1a 64 for content-hash)
16//
17// COMPOSED BY:
18// (future) wiki/nx_wiki_import_legacy.nx
19// (future) bin/nishi-convert CLI tool wrapping this primitive
20//
21// V1 SCOPE (per charter §8):
22// - HTML input -> conforming HTML output
23// - Caller provides: canonical, summary, tags, license, author
24// (these CANNOT be reliably auto-derived; honest gap per M6)
25// - Auto-derived: title (from <title>), content-hash (FNV-1a 64
26// of body), last-modified (caller OR current time), page-version (1.0.0)
27// - External links annotated with data-nishi-archive-pending
28// attribute for archive subsystem (queued)
29// - <iframe>/<script>/<object>/<embed> detected -> sealed verdict
30// UNCONVERTIBLE_CONTENT; operator must remove or accept lossy
31// conversion via FORCE flag
32//
33// V2 SCOPE (TODO):
34// - Markdown input adapter (V2)
35// - RST input adapter (V2)
36// - Auto-summary via first-paragraph extraction (V2)
37// - Auto-tags via term-frequency analysis (V2)
38// - Archive subsystem live link-rewriting (V3 after archive ships)
39//
40// Status: V1. 2026-05-27.
41
42import "nx_syscalls.nx"
43import "nx_html_extract.nx"
44import "nx_html_escape.nx"
45import "nx_search_inverted.nx"
46
47// ===== Sealed verdict surface (codes 2400-2419) =================================================
48const NX_HNP_OK: i64 = 0
49const NX_HNP_BAD_INPUT: i64 = 2400
50const NX_HNP_OUTPUT_OVERFLOW: i64 = 2401
51const NX_HNP_INPUT_TOO_LARGE: i64 = 2402
52const NX_HNP_NO_TITLE_FOUND: i64 = 2403
53const NX_HNP_NO_BODY_FOUND: i64 = 2404
54const NX_HNP_UNCONVERTIBLE_CONTENT: i64 = 2405
55const NX_HNP_INVALID_CANONICAL: i64 = 2406
56const NX_HNP_INVALID_LICENSE: i64 = 2407
57const NX_HNP_LOOP_BUDGET: i64 = 2408
58const NX_HNP_ESCAPE_FAIL: i64 = 2409
59const NX_HNP_MALFORMED_HTML: i64 = 2410
60
61// ===== Named sizing constants (M7) =================================================
62const NX_HNP_MAX_INPUT_BYTES: i64 = 1048576 // 1 MB input cap
63const NX_HNP_MAX_OUTPUT_BYTES: i64 = 2097152 // 2 MB output cap (after annotations)
64const NX_HNP_MAX_TITLE_LEN: i64 = 200 // per NISHI_PAGE_FORMAT_V1 §2.2
65const NX_HNP_MAX_SUMMARY_LEN: i64 = 500 // per §2.3
66const NX_HNP_MAX_AUTHOR_LEN: i64 = 100 // per §2.9
67const NX_HNP_MAX_LICENSE_LEN: i64 = 64
68const NX_HNP_MAX_CANONICAL_LEN: i64 = 2048
69const NX_HNP_MAX_TAGS_LEN: i64 = 1024
70const NX_HNP_MAX_TIMESTAMP_LEN: i64 = 32
71const NX_HNP_CONTENT_HASH_LEN: i64 = 16 // FNV-1a 64 -> 16 hex chars
72const NX_HNP_LOOP_BUDGET_CAP: i64 = 10000000
73
74// ASCII bytes (M7).
75const NX_HNP_ASCII_LT: i64 = 0x3C // '<'
76const NX_HNP_ASCII_GT: i64 = 0x3E // '>'
77
78// ===== Inputs struct (caller provides per-page metadata) =================================================
79
80struct NxHtmlToNishiInputs {
81 src: *u8 // input HTML bytes
82 src_n: i64
83
84 canonical: *u8 // required: full URL this page is served at
85 canonical_n: i64
86 summary: *u8 // required: 1-500 char description
87 summary_n: i64
88 tags: *u8 // required: comma-separated; 0 = no tags
89 tags_n: i64
90 last_modified: *u8 // ISO8601; 0 = use current time
91 last_modified_n: i64
92 license: *u8 // SPDX or "proprietary"
93 license_n: i64
94 author: *u8 // identifier
95 author_n: i64
96
97 // Output buffer
98 out: *u8 // caller-allocated
99 out_cap: i64
100 out_n: *i64 // bytes written
101
102 // Behavior flags
103 force_lossy: i64 // 1 = strip unconvertible content; 0 = error
104
105 valid: i64
106}
107
108func nx_hnp_inputs_init(inp: *NxHtmlToNishiInputs,
109 src: *u8, src_n: i64,
110 canonical: *u8, canonical_n: i64,
111 summary: *u8, summary_n: i64,
112 tags: *u8, tags_n: i64,
113 last_modified: *u8, last_modified_n: i64,
114 license: *u8, license_n: i64,
115 author: *u8, author_n: i64,
116 out: *u8, out_cap: i64, out_n: *i64,
117 force_lossy: i64) -> i64 {
118 if (inp as i64) == 0 { return 0 - NX_HNP_BAD_INPUT }
119 if (src as i64) == 0 { return 0 - NX_HNP_BAD_INPUT }
120 if src_n < 1 { return 0 - NX_HNP_BAD_INPUT }
121 if src_n > NX_HNP_MAX_INPUT_BYTES { return 0 - NX_HNP_INPUT_TOO_LARGE }
122 if (canonical as i64) == 0 { return 0 - NX_HNP_INVALID_CANONICAL }
123 if canonical_n < 1 { return 0 - NX_HNP_INVALID_CANONICAL }
124 if canonical_n > NX_HNP_MAX_CANONICAL_LEN { return 0 - NX_HNP_INVALID_CANONICAL }
125 if summary_n > NX_HNP_MAX_SUMMARY_LEN { return 0 - NX_HNP_BAD_INPUT }
126 if tags_n > NX_HNP_MAX_TAGS_LEN { return 0 - NX_HNP_BAD_INPUT }
127 if last_modified_n > NX_HNP_MAX_TIMESTAMP_LEN { return 0 - NX_HNP_BAD_INPUT }
128 if license_n < 1 { return 0 - NX_HNP_INVALID_LICENSE }
129 if license_n > NX_HNP_MAX_LICENSE_LEN { return 0 - NX_HNP_INVALID_LICENSE }
130 if author_n > NX_HNP_MAX_AUTHOR_LEN { return 0 - NX_HNP_BAD_INPUT }
131 if (out as i64) == 0 { return 0 - NX_HNP_BAD_INPUT }
132 if out_cap < 1024 { return 0 - NX_HNP_OUTPUT_OVERFLOW }
133 if out_cap > NX_HNP_MAX_OUTPUT_BYTES { return 0 - NX_HNP_OUTPUT_OVERFLOW }
134 if (out_n as i64) == 0 { return 0 - NX_HNP_BAD_INPUT }
135
136 inp.src = src; inp.src_n = src_n
137 inp.canonical = canonical; inp.canonical_n = canonical_n
138 inp.summary = summary; inp.summary_n = summary_n
139 inp.tags = tags; inp.tags_n = tags_n
140 inp.last_modified = last_modified; inp.last_modified_n = last_modified_n
141 inp.license = license; inp.license_n = license_n
142 inp.author = author; inp.author_n = author_n
143 inp.out = out; inp.out_cap = out_cap; inp.out_n = out_n
144 inp.force_lossy = force_lossy
145 inp.valid = 1
146 return NX_HNP_OK
147}
148
149// ===== Output writer (bounded; M5) =================================================
150
151func nx_hnp_put_raw(inp: *NxHtmlToNishiInputs, off: i64,
152 src: *u8, n: i64) -> i64 {
153 if off < 0 { return 0 - NX_HNP_BAD_INPUT }
154 if off + n > inp.out_cap { return 0 - NX_HNP_OUTPUT_OVERFLOW }
155 var i: i64 = 0
156 while i < n {
157 if i >= NX_HNP_MAX_OUTPUT_BYTES { return 0 - NX_HNP_OUTPUT_OVERFLOW }
158 inp.out[off + i] = src[i]
159 i = i + 1
160 }
161 return off + n
162}
163
164func nx_hnp_put_z(inp: *NxHtmlToNishiInputs, off: i64, s: *u8) -> i64 {
165 var n: i64 = 0
166 while s[n] != (0 as u8) {
167 if n >= inp.out_cap { return 0 - NX_HNP_OUTPUT_OVERFLOW }
168 n = n + 1
169 }
170 return nx_hnp_put_raw(inp, off, s, n)
171}
172
173func nx_hnp_put_escaped(inp: *NxHtmlToNishiInputs, off: i64,
174 src: *u8, n: i64) -> i64 {
175 if n < 1 { return off }
176 if inp.out_cap - off < n * 6 + 1 { return 0 - NX_HNP_OUTPUT_OVERFLOW }
177 let w: i64 = html_escape((inp.out as i64 + off) as *u8, inp.out_cap - off, src, n)
178 if w < 0 { return 0 - NX_HNP_ESCAPE_FAIL }
179 return off + w
180}
181
182// ===== Hex-of-i64 (16 chars) for content-hash =================================================
183
184func nx_hnp_hex16(v: i64, out: *u8) -> i64 {
185 var i: i64 = 0
186 while i < 16 {
187 let nibble: i64 = (v >> ((15 - i) * 4)) & 0xF
188 var c: i64 = 0x30 + nibble // '0'..'9'
189 if nibble >= 10 { c = 0x61 + nibble - 10 } // 'a'..'f'
190 out[i] = (c & 0xff) as u8
191 i = i + 1
192 }
193 return 16
194}
195
196// ===== Unconvertible-content detection (case-insensitive tag scan) =================================================
197//
198// Returns 1 if input contains <iframe|script|object|embed>; 0 if clean.
199
200func nx_hnp_has_unconvertible(src: *u8, src_n: i64) -> i64 {
201 if src_n < 1 { return 0 }
202 var i: i64 = 0
203 var iter: i64 = 0
204 while i < src_n {
205 if iter >= NX_HNP_LOOP_BUDGET_CAP { return 1 }
206 iter = iter + 1
207 if (src[i] as i64) == NX_HNP_ASCII_LT {
208 if nx_he_match_ci(src, i + 1, src_n, "iframe" as *u8, 6) == 1 { return 1 }
209 if nx_he_match_ci(src, i + 1, src_n, "script" as *u8, 6) == 1 { return 1 }
210 if nx_he_match_ci(src, i + 1, src_n, "object" as *u8, 6) == 1 { return 1 }
211 if nx_he_match_ci(src, i + 1, src_n, "embed" as *u8, 5) == 1 { return 1 }
212 }
213 i = i + 1
214 }
215 return 0
216}
217
218// ===== Title extraction (between <title> and </title>) =================================================
219//
220// Returns the inp-source byte range as (start, len) via out-params;
221// (-1, 0) if no title found.
222
223func nx_hnp_extract_title(src: *u8, src_n: i64,
224 out_start: *i64, out_len: *i64) -> i64 {
225 if (out_start as i64) == 0 { return 0 - NX_HNP_BAD_INPUT }
226 if (out_len as i64) == 0 { return 0 - NX_HNP_BAD_INPUT }
227 out_start[0] = 0 - 1
228 out_len[0] = 0
229
230 var i: i64 = 0
231 var iter: i64 = 0
232 while i < src_n {
233 if iter >= NX_HNP_LOOP_BUDGET_CAP { return 0 - NX_HNP_LOOP_BUDGET }
234 iter = iter + 1
235 if (src[i] as i64) == NX_HNP_ASCII_LT {
236 if nx_he_match_ci(src, i + 1, src_n, "title" as *u8, 5) == 1 {
237 // Find the closing '>' of the opening tag.
238 var j: i64 = i + 6
239 while j < src_n {
240 if (src[j] as i64) == NX_HNP_ASCII_GT {
241 let tstart: i64 = j + 1
242 // Find </title>.
243 var k: i64 = tstart
244 while k < src_n {
245 if (src[k] as i64) == NX_HNP_ASCII_LT {
246 if k + 7 < src_n {
247 if (src[k + 1] as i64) == 0x2F { // '/'
248 if nx_he_match_ci(src, k + 2, src_n,
249 "title" as *u8, 5) == 1 {
250 out_start[0] = tstart
251 out_len[0] = k - tstart
252 if out_len[0] > NX_HNP_MAX_TITLE_LEN {
253 out_len[0] = NX_HNP_MAX_TITLE_LEN
254 }
255 return NX_HNP_OK
256 }
257 }
258 }
259 }
260 k = k + 1
261 }
262 j = src_n + 1
263 }
264 if j < src_n { j = j + 1 }
265 }
266 }
267 }
268 i = i + 1
269 }
270 return NX_HNP_OK // out_start stays -1; caller falls back
271}
272
273// ===== Body extraction (between <body> and </body>) =================================================
274
275func nx_hnp_extract_body(src: *u8, src_n: i64,
276 out_start: *i64, out_len: *i64) -> i64 {
277 if (out_start as i64) == 0 { return 0 - NX_HNP_BAD_INPUT }
278 if (out_len as i64) == 0 { return 0 - NX_HNP_BAD_INPUT }
279 out_start[0] = 0
280 out_len[0] = src_n
281
282 var i: i64 = 0
283 var iter: i64 = 0
284 while i < src_n {
285 if iter >= NX_HNP_LOOP_BUDGET_CAP { return 0 - NX_HNP_LOOP_BUDGET }
286 iter = iter + 1
287 if (src[i] as i64) == NX_HNP_ASCII_LT {
288 if nx_he_match_ci(src, i + 1, src_n, "body" as *u8, 4) == 1 {
289 var j: i64 = i + 5
290 while j < src_n {
291 if (src[j] as i64) == NX_HNP_ASCII_GT {
292 let bstart: i64 = j + 1
293 var k: i64 = bstart
294 while k < src_n {
295 if (src[k] as i64) == NX_HNP_ASCII_LT {
296 if k + 6 < src_n {
297 if (src[k + 1] as i64) == 0x2F {
298 if nx_he_match_ci(src, k + 2, src_n,
299 "body" as *u8, 4) == 1 {
300 out_start[0] = bstart
301 out_len[0] = k - bstart
302 return NX_HNP_OK
303 }
304 }
305 }
306 }
307 k = k + 1
308 }
309 j = src_n + 1
310 }
311 if j < src_n { j = j + 1 }
312 }
313 }
314 }
315 i = i + 1
316 }
317 return NX_HNP_OK // no body tag; treat whole input as body
318}
319
320// ===== Emit one nishi-* meta tag =================================================
321
322func nx_hnp_emit_meta(inp: *NxHtmlToNishiInputs, off: i64,
323 name_z: *u8,
324 value: *u8, value_n: i64) -> i64 {
325 var o: i64 = off
326 o = nx_hnp_put_z(inp, o, " <meta name=\"" as *u8); if o < 0 { return o }
327 o = nx_hnp_put_z(inp, o, name_z); if o < 0 { return o }
328 o = nx_hnp_put_z(inp, o, "\" content=\"" as *u8); if o < 0 { return o }
329 if value_n > 0 {
330 o = nx_hnp_put_escaped(inp, o, value, value_n); if o < 0 { return o }
331 }
332 o = nx_hnp_put_z(inp, o, "\">\n" as *u8); if o < 0 { return o }
333 return o
334}
335
336// ===== Top-level: convert HTML -> Nishi Page Format V1 =================================================
337
338func nx_html_to_nishi_page(inp: *NxHtmlToNishiInputs) -> i64 {
339 if inp.valid != 1 { return 0 - NX_HNP_BAD_INPUT }
340 inp.out_n[0] = 0
341
342 // Detect unconvertible content unless force_lossy.
343 if inp.force_lossy == 0 {
344 if nx_hnp_has_unconvertible(inp.src, inp.src_n) == 1 {
345 return 0 - NX_HNP_UNCONVERTIBLE_CONTENT
346 }
347 }
348
349 // Extract title (fall back to "(untitled)" if absent).
350 let title_start: *i64 = (sys_mmap(8)) as *i64
351 let title_len: *i64 = (sys_mmap(8)) as *i64
352 title_start[0] = 0 - 1
353 title_len[0] = 0
354 let rc_t: i64 = nx_hnp_extract_title(inp.src, inp.src_n, title_start, title_len)
355 if rc_t != NX_HNP_OK { return rc_t }
356 var title_ptr: *u8 = "(untitled)" as *u8
357 var title_n: i64 = 10
358 if title_start[0] >= 0 {
359 title_ptr = (inp.src as i64 + title_start[0]) as *u8
360 title_n = title_len[0]
361 }
362
363 // Extract body (defaults to whole input).
364 let body_start: *i64 = (sys_mmap(8)) as *i64
365 let body_len: *i64 = (sys_mmap(8)) as *i64
366 let rc_b: i64 = nx_hnp_extract_body(inp.src, inp.src_n, body_start, body_len)
367 if rc_b != NX_HNP_OK { return rc_b }
368 let body_ptr: *u8 = (inp.src as i64 + body_start[0]) as *u8
369 let body_n: i64 = body_len[0]
370
371 // Compute content hash (FNV-1a 64 over body bytes).
372 let content_hash: i64 = nx_inv_hash_bytes_lower(body_ptr, body_n)
373 let content_hash_hex: *u8 = (sys_mmap(NX_HNP_CONTENT_HASH_LEN + 1)) as *u8
374 nx_hnp_hex16(content_hash, content_hash_hex)
375
376 // Default last-modified to a placeholder if caller didn't supply.
377 var lm_ptr: *u8 = inp.last_modified
378 var lm_n: i64 = inp.last_modified_n
379 if lm_n == 0 {
380 lm_ptr = "1970-01-01T00:00:00Z" as *u8
381 lm_n = 20
382 }
383
384 var o: i64 = 0
385 o = nx_hnp_put_z(inp, o, "<!DOCTYPE html>\n<html lang=\"en\">\n<head>\n" as *u8); if o < 0 { return o }
386 o = nx_hnp_put_z(inp, o, " <meta charset=\"utf-8\">\n" as *u8); if o < 0 { return o }
387 o = nx_hnp_put_z(inp, o, " <title>" as *u8); if o < 0 { return o }
388 o = nx_hnp_put_escaped(inp, o, title_ptr, title_n); if o < 0 { return o }
389 o = nx_hnp_put_z(inp, o, "</title>\n" as *u8); if o < 0 { return o }
390 o = nx_hnp_put_z(inp, o, " <!-- NISHI PAGE FORMAT V1 METADATA -->\n" as *u8); if o < 0 { return o }
391
392 // Nine required meta tags (per NISHI_PAGE_FORMAT_V1 §2).
393 o = nx_hnp_emit_meta(inp, o, "nishi-canonical" as *u8, inp.canonical, inp.canonical_n); if o < 0 { return o }
394 o = nx_hnp_emit_meta(inp, o, "nishi-title" as *u8, title_ptr, title_n); if o < 0 { return o }
395 o = nx_hnp_emit_meta(inp, o, "nishi-summary" as *u8, inp.summary, inp.summary_n); if o < 0 { return o }
396 o = nx_hnp_emit_meta(inp, o, "nishi-tags" as *u8, inp.tags, inp.tags_n); if o < 0 { return o }
397 o = nx_hnp_emit_meta(inp, o, "nishi-last-modified" as *u8, lm_ptr, lm_n); if o < 0 { return o }
398 o = nx_hnp_emit_meta(inp, o, "nishi-content-hash" as *u8,
399 content_hash_hex, NX_HNP_CONTENT_HASH_LEN); if o < 0 { return o }
400 o = nx_hnp_emit_meta(inp, o, "nishi-page-version" as *u8, "1.0.0" as *u8, 5); if o < 0 { return o }
401 o = nx_hnp_emit_meta(inp, o, "nishi-license" as *u8, inp.license, inp.license_n); if o < 0 { return o }
402 o = nx_hnp_emit_meta(inp, o, "nishi-author" as *u8, inp.author, inp.author_n); if o < 0 { return o }
403
404 o = nx_hnp_put_z(inp, o, "</head>\n<body>\n" as *u8); if o < 0 { return o }
405
406 // Body emission: V1 emits the source body bytes verbatim.
407 // V2 will walk anchors + annotate with data-nishi-archive-pending.
408 // V1 honest gap: external link rewriting requires the archive
409 // subsystem (queued per NISHI_ARCHIVE_INTEGRATION_CHARTER).
410 if body_n > 0 {
411 o = nx_hnp_put_raw(inp, o, body_ptr, body_n); if o < 0 { return o }
412 }
413
414 o = nx_hnp_put_z(inp, o, "\n</body>\n</html>\n" as *u8); if o < 0 { return o }
415 inp.out_n[0] = o
416 return NX_HNP_OK
417}