code wiki / _hdl_build / nx_migrate_ingest.nx
nx_migrate_ingest.nx source
↩ module page · 155 lines · 7064 B
1// nx_migrate_ingest.nx -- SITE-MIGRATION ingest (R3b): transform a fetched external page (HTML bytes) into a
2// sovereign `.site` BLUEPRINT that nx_site_provision consumes -> the basis of /api/migrate (e.g. jasonewest.com).
3// Operator: "migrate sites like jasonewest.com easily -- not just hosting but migrating/updating." PURE transform
4// mi_emit_site(html,n,domain,dn,out) -> out_n (bytes in, bytes out, independently gateable); a thin main fetches
5// (adapter) + writes. Extracts <title>/<h1>/<p>/<h2> into title|header|hero|search|cards|... ; STRIPS nested tags,
6// collapses whitespace, and SANITIZES the '|' delimiter out of extracted text (else it corrupts the .site format).
7// (NB: distinct from nx_site_ingest.nx, which is research-library mirroring -- different concern.) license_tier: ORIGINAL
8import "nx_syscalls.nx"
9const K_MAGIC_4096: i64 = 4096
10const K_MAGIC_262144: i64 = 262144
11
12func mi_lower(c: i64) -> i64 { if c >= 65 { if c <= 90 { return c + 32 } } return c }
13func mi_slen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
14func mi_put(out: *u8, o: i64, s: *u8) -> i64 { var i: i64 = 0; while s[i] != (0 as u8) { out[o] = s[i]; o = o + 1; i = i + 1 } return o }
15func mi_putb(out: *u8, o: i64, b: *u8, len: i64) -> i64 { var i: i64 = 0; while i < len { out[o] = b[i]; o = o + 1; i = i + 1 } return o }
16
17// case-insensitive find of pat in html[start..n). returns index or -1.
18func mi_find_ci(html: *u8, n: i64, start: i64, pat: *u8) -> i64 {
19 let pl: i64 = mi_slen(pat)
20 if pl == 0 { return 0 - 1 }
21 var i: i64 = start
22 while i + pl <= n {
23 var k: i64 = 0
24 var hit: i64 = 1
25 while k < pl { if mi_lower(html[i + k] as i64) != mi_lower(pat[k] as i64) { hit = 0; k = pl } else { k = k + 1 } }
26 if hit == 1 { return i }
27 i = i + 1
28 }
29 return 0 - 1
30}
31
32// from the '<' of an open tag at oidx: skip to '>', copy inner text until closepat; strip nested <...>,
33// collapse whitespace, drop '|'. writes to out (cap), returns len; sets endbox to index past closepat (or n).
34func mi_inner(html: *u8, n: i64, oidx: i64, closepat: *u8, out: *u8, cap: i64, endbox: *i64) -> i64 {
35 var i: i64 = oidx
36 var f: i64 = 0
37 while f == 0 { if i >= n { f = 1 } else { if (html[i] as i64) == 62 { f = 1 } else { i = i + 1 } } }
38 i = i + 1
39 let cidx: i64 = mi_find_ci(html, n, i, closepat)
40 var stop: i64 = cidx
41 if stop < 0 { stop = n }
42 var o: i64 = 0
43 var intag: i64 = 0
44 var lastsp: i64 = 1
45 while i < stop {
46 let c: i64 = html[i] as i64
47 if c == 60 { intag = 1 }
48 if intag == 0 {
49 var ch: i64 = c
50 if ch == 124 { ch = 32 }
51 if ch == 9 { ch = 32 }
52 if ch == 10 { ch = 32 }
53 if ch == 13 { ch = 32 }
54 if ch == 32 {
55 if lastsp == 0 { if o < cap - 1 { out[o] = 32 as u8; o = o + 1 } lastsp = 1 }
56 } else {
57 if o < cap - 1 { out[o] = ch as u8; o = o + 1 } lastsp = 0
58 }
59 }
60 if c == 62 { intag = 0 }
61 i = i + 1
62 }
63 if o > 0 { if out[o - 1] == (32 as u8) { o = o - 1 } }
64 out[o] = 0 as u8
65 if cidx < 0 { endbox[0] = n } else { endbox[0] = cidx + mi_slen(closepat) }
66 return o
67}
68
69// the pure transform: HTML bytes -> a compliant .site blueprint (title|header|hero|search|cards|card|trust|sig|footer).
70func mi_emit_site(html: *u8, n: i64, domain: *u8, dn: i64, out: *u8) -> i64 {
71 let tmp: *u8 = sys_mmap(K_MAGIC_4096)
72 let pbuf: *u8 = sys_mmap(K_MAGIC_4096)
73 let eb: *i64 = sys_mmap(8) as *i64
74 var o: i64 = 0
75
76 // title (fallback = domain)
77 var tlen: i64 = 0
78 let ti: i64 = mi_find_ci(html, n, 0, "<title" as *u8)
79 if ti >= 0 { tlen = mi_inner(html, n, ti, "</title" as *u8, tmp, K_MAGIC_4096, eb) }
80 o = mi_put(out, o, "title|" as *u8)
81 if tlen > 0 { o = mi_putb(out, o, tmp, tlen) } else { o = mi_putb(out, o, domain, dn) }
82 o = mi_put(out, o, "\n" as *u8)
83
84 // header|<brand>|Contact|#contact
85 o = mi_put(out, o, "header|" as *u8)
86 if tlen > 0 { o = mi_putb(out, o, tmp, tlen) } else { o = mi_putb(out, o, domain, dn) }
87 o = mi_put(out, o, "|Contact|#contact\n" as *u8)
88
89 // hero|<h1>|<first p>|Get in touch|#contact
90 var h1len: i64 = 0
91 let hi: i64 = mi_find_ci(html, n, 0, "<h1" as *u8)
92 if hi >= 0 { h1len = mi_inner(html, n, hi, "</h1" as *u8, tmp, K_MAGIC_4096, eb) }
93 o = mi_put(out, o, "hero|" as *u8)
94 if h1len > 0 { o = mi_putb(out, o, tmp, h1len) } else { o = mi_put(out, o, "Welcome" as *u8) }
95 o = mi_put(out, o, "|" as *u8)
96 var plen: i64 = 0
97 let pi: i64 = mi_find_ci(html, n, 0, "<p" as *u8)
98 if pi >= 0 { plen = mi_inner(html, n, pi, "</p" as *u8, pbuf, K_MAGIC_4096, eb) }
99 if plen > 0 { o = mi_putb(out, o, pbuf, plen) } else { o = mi_put(out, o, "Migrated to the Nishi sovereign host." as *u8) }
100 o = mi_put(out, o, "|Get in touch|#contact\n" as *u8)
101
102 // search
103 o = mi_put(out, o, "search|/search|Search...\n" as *u8)
104
105 // cards from up to 3 <h2>
106 o = mi_put(out, o, "cards|Highlights\n" as *u8)
107 var start: i64 = 0
108 var emitted: i64 = 0
109 var done: i64 = 0
110 while done == 0 {
111 if emitted >= 3 { done = 1 } else {
112 let h2: i64 = mi_find_ci(html, n, start, "<h2" as *u8)
113 if h2 < 0 { done = 1 } else {
114 let cl: i64 = mi_inner(html, n, h2, "</h2" as *u8, tmp, K_MAGIC_4096, eb)
115 start = eb[0]
116 if cl > 0 {
117 o = mi_put(out, o, "card|" as *u8)
118 o = mi_putb(out, o, tmp, cl)
119 o = mi_put(out, o, "|Imported from the original site.\n" as *u8)
120 emitted = emitted + 1
121 }
122 }
123 }
124 }
125 if emitted == 0 { o = mi_put(out, o, "card|About|Imported from the original site.\n" as *u8) }
126
127 // trust + sig + footer
128 o = mi_put(out, o, "trust|\n" as *u8)
129 o = mi_put(out, o, "sig|Migrated to the Nishi sovereign host\n" as *u8)
130 o = mi_put(out, o, "footer|" as *u8)
131 o = mi_putb(out, o, domain, dn)
132 o = mi_put(out, o, "\n" as *u8)
133 return o
134}
135
136func main(argc: i64, argv: *i64) -> i64 {
137 if argc < 4 { sys_write(2, "usage: nx_migrate_ingest <captured_html_file> <domain> <out.site>\n" as *u8, 65); return 1 }
138 let htmlpath: *u8 = argv[1] as *u8
139 let domain: *u8 = argv[2] as *u8
140 let outpath: *u8 = argv[3] as *u8
141 let szb: *i64 = sys_mmap(16) as *i64
142 szb[0] = 0
143 let html: *u8 = sys_read_file(htmlpath, szb)
144 if (html as i64) == 0 { sys_write(2, "ingest: cannot read html\n" as *u8, 25); return 2 }
145 let out: *u8 = sys_mmap(K_MAGIC_262144)
146 let on: i64 = mi_emit_site(html, szb[0], domain, mi_slen(domain), out)
147 let fd: i64 = sys_openat_wr(outpath, 0x1a4)
148 if fd < 0 { sys_write(2, "ingest: cannot write .site\n" as *u8, 27); return 3 }
149 sys_write(fd, out, on)
150 sys_close(fd)
151 sys_write(1, "MIGRATE-INGEST -> " as *u8, 18)
152 sys_write(1, outpath, mi_slen(outpath))
153 sys_write(1, "\n" as *u8, 1)
154 return 0
155}