nx_meal_capture.nx source
↩ module page · 185 lines · 7488 B
1// nx_meal_capture.nx -- MAIN: preserve ONE origin recipe page into the sovereign WARC, correctly.
2//
3// WHY THIS IS ITS OWN ORGAN. It began as a stage inside nx_meal_emit. Pulling in nx_gzip_wrap there dragged
4// nx_runtime + nx_tier + nx_deflate + nx_crc32 in behind it, and the emitter's seg-store seeding -- code that
5// had not been touched and had run green minutes earlier -- started dying silently. Rather than fight that,
6// the two jobs are separated the way they should have been from the start (rule 9): CAPTURE owns bytes off
7// the wire and touches no store; EMIT owns the stores and the page and touches no decompressor. Each organ
8// now imports only what its own job needs.
9//
10// WHAT IT FIXES. A raw sovereign fetch returns the body exactly as it travelled: HTTP/1.1 CHUNK-FRAMED, and
11// then DEFLATE-compressed. Archiving those bytes preserves something nobody can read, while still answering
12// "is this URL archived?" with YES -- so a page would claim the link cannot rot over a blob it cannot serve.
13// That is the estate's own gzip-undecoded class, already paid for once in the crawler.
14//
15// SO IT REFUSES. If the decoded payload is not recognisably HTML, NOTHING is written to the archive, and the
16// meal page will honestly report the source as unpreserved. An empty archive that admits it is empty beats a
17// full one that lies.
18//
19// Decoders are the INCUMBENTS: nx_http_chunked (gate-proven) and nx_gzip_wrap (~10 consumers, KAT-proven).
20// nx_inflate.nx is a known second implementation whose own header says DO NOT ADD NEW CONSUMERS.
21//
22// argv[1] = raw fetch file argv[2] = WARC path argv[3] = target URI argv[4] = ISO date
23// license_tier: ORIGINAL No hw writes (Rule 26).
24import "_hdl_build/nx_web_archive.nx"
25import "_hdl_build/nx_http_chunked.nx"
26import "nx_gzip_wrap.nx"
27import "nx_syscalls.nx"
28
29const MCAP_WARC: i64 = 4194304
30const MCAP_GZ_MAGIC1: i64 = 31
31const MCAP_GZ_MAGIC2: i64 = 139
32
33func mcap_p(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
34func mcap_n(v: i64) -> i64 {
35 let b: *u8 = sys_mmap(32)
36 let t: *u8 = sys_mmap(32)
37 var m: i64 = v
38 var k: i64 = 0
39 if m < 0 { sys_write(1, "-" as *u8, 1); m = 0 - m }
40 if m == 0 { t[0] = 48 as u8; k = 1 }
41 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
42 var i: i64 = 0
43 while i < k { b[i] = t[k - 1 - i]; i = i + 1 }
44 sys_write(1, b, k)
45 return 0
46}
47func mcap_find(hay: *u8, hl: i64, needle: *u8, nl: i64, from: i64) -> i64 {
48 if nl == 0 { return from }
49 var i: i64 = from
50 let last: i64 = hl - nl
51 while i <= last {
52 var j: i64 = 0
53 var m: i64 = 1
54 while j < nl { if (hay[i+j] & 0xff) != (needle[j] & 0xff) { m = 0; j = nl } else { j = j + 1 } }
55 if m == 1 { return i }
56 i = i + 1
57 }
58 return 0 - 1
59}
60
61func main(argc: i64, argv: *i64) -> i64 {
62 var fetchf: *u8 = "/tmp/bs_origin.txt" as *u8
63 var warcpath: *u8 = "knowledge/store/meal.warc" as *u8
64 var uri: *u8 = "https://en.wikibooks.org/wiki/Cookbook:Beef_Stew" as *u8
65 var date: *u8 = "2026-08-06T18:22:06Z" as *u8
66 if argc > 1 { fetchf = argv[1] as *u8 }
67 if argc > 2 { warcpath = argv[2] as *u8 }
68 if argc > 3 { uri = argv[3] as *u8 }
69 if argc > 4 { date = argv[4] as *u8 }
70
71 let box: *i64 = sys_mmap(16) as *i64
72 box[0] = 0
73 let raw: *u8 = sys_read_file(fetchf, box)
74 let rn: i64 = box[0]
75 if rn <= 0 {
76 mcap_p("NX-MEAL-CAPTURE FAIL no fetch bytes at " as *u8); mcap_p(fetchf); mcap_p("\n" as *u8)
77 sys_exit(1)
78 return 1
79 }
80 mcap_p(" fetch file: " as *u8); mcap_n(rn); mcap_p(" bytes\n" as *u8)
81
82 // the transport chatter, then the HTTP response. The resource is the BODY.
83 let hb: i64 = mcap_find(raw, rn, "\r\n\r\n" as *u8, 4, 0)
84 var bs: i64 = 0
85 if hb >= 0 { bs = hb + 4 }
86 var payload: *u8 = ((raw as i64) + bs) as *u8
87 var plen: i64 = rn - bs
88
89 // 1) strip chunk framing
90 if hb > 0 {
91 if hc_is_chunked(raw, bs) == 1 {
92 let dech: *u8 = sys_mmap(MCAP_WARC)
93 let dn: i64 = chunked_decode(payload, plen, dech)
94 if dn > 0 {
95 payload = dech
96 plen = dn
97 mcap_p(" de-chunked -> " as *u8); mcap_n(dn); mcap_p(" bytes\n" as *u8)
98 }
99 }
100 }
101
102 // 2) inflate gzip
103 if plen > 2 {
104 if ((payload[0] as i64) & 255) == MCAP_GZ_MAGIC1 {
105 if ((payload[1] as i64) & 255) == MCAP_GZ_MAGIC2 {
106 let g: *NxGzipResult = nx_gzip_inflate(payload, plen, MCAP_WARC)
107 if g.error_code == NX_GZ_OK {
108 payload = g.output_data
109 plen = g.output_size
110 mcap_p(" gunzipped -> " as *u8); mcap_n(plen); mcap_p(" bytes\n" as *u8)
111 } else {
112 mcap_p(" gzip FAILED err=" as *u8); mcap_n(g.error_code); mcap_p("\n" as *u8)
113 plen = 0
114 }
115 }
116 }
117 }
118
119 // 3) REFUSE to archive what did not decode into readable HTML.
120 var readable: i64 = 0
121 if plen > 0 { if mcap_find(payload, plen, "<html" as *u8, 5, 0) >= 0 { readable = 1 } }
122 if readable == 0 {
123 mcap_p("NX-MEAL-CAPTURE REFUSED decoded bytes are not HTML -- archiving NOTHING so the page tells the truth\n" as *u8)
124 sys_exit(2)
125 return 2
126 }
127
128 let warc: *u8 = sys_mmap(MCAP_WARC)
129 var wl: i64 = wa_load(warcpath, warc, MCAP_WARC)
130 if wl < 0 { wl = 0 }
131
132 // IDEMPOTENT: if this exact resource is already preserved byte-for-byte, append nothing.
133 let op: *i64 = sys_mmap(16) as *i64
134 let ol: *i64 = sys_mmap(16) as *i64
135 var ulen: i64 = 0
136 while uri[ulen] != (0 as u8) { ulen = ulen + 1 }
137 var same: i64 = 0
138 if wa_lookup(warc, wl, uri, ulen, op, ol) == 1 {
139 if ol[0] == plen {
140 var i: i64 = 0
141 var m: i64 = 1
142 let base: i64 = op[0]
143 while i < plen { if (warc[base + i] & 0xff) != (payload[i] & 0xff) { m = 0; i = plen } else { i = i + 1 } }
144 same = m
145 }
146 }
147 if same == 1 {
148 mcap_p("NX-MEAL-CAPTURE OK already preserved byte-identical bytes=" as *u8); mcap_n(plen)
149 mcap_p(" warc=" as *u8); mcap_n(wl); mcap_p("\n" as *u8)
150 sys_exit(0)
151 return 0
152 }
153
154 wl = wa_write_resource(warc, wl, uri, date, "text/html" as *u8, payload, plen)
155 if wa_save(warcpath, warc, wl) < 0 {
156 mcap_p("NX-MEAL-CAPTURE FAIL cannot write " as *u8); mcap_p(warcpath); mcap_p("\n" as *u8)
157 sys_exit(1)
158 return 1
159 }
160
161 // PROVE IT ROUND-TRIPPED before claiming success: re-read the saved file and byte-compare.
162 let back: *u8 = sys_mmap(MCAP_WARC)
163 let bn: i64 = wa_load(warcpath, back, MCAP_WARC)
164 var ok: i64 = 0
165 if wa_lookup(back, bn, uri, ulen, op, ol) == 1 {
166 if ol[0] == plen {
167 var i: i64 = 0
168 var m: i64 = 1
169 let base: i64 = op[0]
170 while i < plen { if (back[base + i] & 0xff) != (payload[i] & 0xff) { m = 0; i = plen } else { i = i + 1 } }
171 ok = m
172 }
173 }
174 if ok == 0 {
175 mcap_p("NX-MEAL-CAPTURE FAIL saved archive does not read back byte-identical\n" as *u8)
176 sys_exit(3)
177 return 3
178 }
179
180 mcap_p("NX-MEAL-CAPTURE OK preserved=" as *u8); mcap_n(plen)
181 mcap_p(" warc=" as *u8); mcap_n(wl)
182 mcap_p(" verified=byte-identical-on-reload\n" as *u8)
183 sys_exit(0)
184 return 0
185}