code wiki / _hdl_build / nx_ims_batch2.nx
nx_ims_batch2.nx source
↩ module page · 124 lines · 19499 B
1// nx_ims_batch2.nx -- INGEST batch #2: the SUBSTRATE hub + 6 REAL pages, each rendered
2// through the ONE reusable generator nx_ims_page (no per-page cloned emitters). This
3// DEEPENS the wiki IA from a flat star into a real tree: a new "substrate" hub (child of
4// start) introduces the low-level sovereign stack and parents nishi_os / sovereign_email /
5// simd_backend; three more product pages (neural_mt / vision_fr / voxelworld) parent under
6// the existing products hub. Summaries are GENUINE (drawn from the live arc memory files;
7// gate counts + measured numbers are cited as the arcs' last recorded state, not invented).
8// Each page's nav relationships come from its wiki_links.tsv row so it joins the walkable
9// tree the moment that row exists. Writes web_assets/{substrate,nishi_os,sovereign_email,
10// simd_backend,neural_mt,vision_fr,voxelworld}.html. Publish is a SEPARATE guarded+versioned
11// step (nx_ims_batch2_publish via vpub). license_tier: ORIGINAL
12import "nx_ims_page.nx"
13import "nx_syscalls.nx"
14
15func b_w(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
16func b_num(v: i64) -> i64 {
17 let bb: *u8 = sys_mmap(28); var m: i64 = v; if m < 0 { sys_write(1, "-" as *u8, 1); m = 0 - m }
18 let t: *u8 = sys_mmap(28); var k: i64 = 0
19 if m == 0 { t[0] = 48 as u8; k = 1 }
20 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
21 var i: i64 = 0; while i < k { bb[i] = t[k - 1 - i]; i = i + 1 }
22 sys_write(1, bb, k); return 0
23}
24
25func b_emit(slug: *u8, doc_type: *u8, title: *u8, body: *u8) -> i64 {
26 let w: i64 = ims_page(slug, doc_type, title, body)
27 b_w(" ims_page("); b_w(slug); b_w(", "); b_w(doc_type); b_w(") -> bytes="); b_num(w); b_w("\n")
28 if w > 0 { return 0 }
29 return 0 - 1
30}
31
32func main() -> i64 {
33 b_w("=== IMS BATCH #2: substrate hub + 6 real pages via the ONE reusable generator ===\n")
34 var ok: i64 = 0
35
36 // ---------- 1) SUBSTRATE HUB (explanation) ----------
37 let substrate_body: *u8 =
38"<p>The <strong>substrate</strong> is the low-level sovereign stack everything else stands on: the layers from the hardware rung up — the operating system, the network & protocol stacks, the vector/SIMD throughput floor, and the crypto/codec primitives — all built in NishiLang organs (<code>nx_cc</code> → <code>nxasm_x86</code>, no gcc) with no borrowed third-party stack on the product path. This hub introduces those layers and links the pages that document each one.</p>\n"
39"<p>Why a substrate layer at all: a capability is only sovereign if the ground it runs on is sovereign too. The <a href=\"/wiki/genesis_genealogy.html\">genesis genealogy</a> traces this lineage end-to-end (god → ISA → toolchain → organs → OS → math → AI → apps → IO endpoint); the substrate pages are the OS-and-below rungs of that tree, where the never-brick law (Rule 26) matters most.</p>\n"
40"<h3>The substrate pages</h3>\n"
41"<ul>\n"
42"<li><a href=\"/wiki/nishi_os.html\"><strong>NishiOS</strong></a> — the sovereign installable operating system; a sovereign UEFI boot app is GREEN.</li>\n"
43"<li><a href=\"/wiki/sovereign_email.html\"><strong>Sovereign email</strong></a> — an 8-rung email stack (MX → SMTP → MIME → store → IMAP/POP3 → SPF/DKIM/DMARC → submission → MTA) riding the sovereign net stack.</li>\n"
44"<li><a href=\"/wiki/simd_backend.html\"><strong>SIMD backend</strong></a> — the sovereign SIMD/vector throughput floor; its ChaCha20 cipher was measured to BEAT OpenSSL by about 3% on Zen1, byte-exact.</li>\n"
45"</ul>\n"
46"<p>Apps consume the substrate through stable interfaces and never reach down to the mnemonics — ship a new SIMD width or a new OS port and the apps above get it for free. See the <a href=\"/wiki/products.html\">product catalog</a> for the application layer this floor unblocks.</p>\n"
47 if b_emit("substrate" as *u8, "explanation" as *u8, "Substrate — the sovereign low-level stack" as *u8, substrate_body) == 0 { ok = ok + 1 }
48
49 // ---------- 2) NISHI OS (explanation) ----------
50 let nishios_body: *u8 =
51"<p><strong>NishiOS</strong> is the project's sovereign operating system: a C microkernel (~10k LoC target, AGPL 3) with a NishiVM userspace, RISC-V first, a 9P-style unified namespace, and mesh-native networking — the same OS intended to scale from an 80 MHz MCU up to Nishi silicon. The thesis is that five device tiers (MCU / edge / workstation / data center / silicon) need five OSes today and none meets the full sovereign requirement (no Big-Tech governance, no cloud tie, 30+ year ABI, deterministic real-time, zero binary blobs), so writing one deliberately is cheaper than the status quo.</p>\n"
52"<h3>Where it is (honest)</h3>\n"
53"<p>The architecture is locked (microkernel + NishiVM <code>.nxb</code> userspace; first target RISC-V; ABI stability via message protocols, not function signatures; init = NishiVM as PID 0, no systemd; own TCP/IP + mesh, no lwIP/POSIX sockets). The first concrete sovereign rung is GREEN: <strong>NOS-R0</strong> is a sovereign UEFI boot application (<code>nx_boot_uefi</code> → an EFI app that prints and returns), the same real <code>.efi</code> artifact the never-brick capsule chain golden-recovers and boots byte-exact in the emulator.</p>\n"
54"<p>Design notes credit the closest prior art (seL4 for the formally-verified microkernel, Inferno/Plan 9 for the Dis-VM + 9P namespace) and state plainly that neither alone suffices — \"NishiOS = seL4 + Inferno + our glue.\" Most of NishiOS is still a phased plan (QEMU bring-up → Milk-V Pioneer port → MCU → workstation/server → silicon); the boot rung is what is built and gated today. It sits at the OS rung of the <a href=\"/wiki/genesis_genealogy.html\">genesis lineage</a>, just above the firmware layer governed by the never-brick law, alongside the other <a href=\"/wiki/substrate.html\">substrate</a> pages.</p>\n"
55 if b_emit("nishi_os" as *u8, "explanation" as *u8, "NishiOS — the sovereign operating system" as *u8, nishios_body) == 0 { ok = ok + 1 }
56
57 // ---------- 3) SOVEREIGN EMAIL (reference) ----------
58 let email_body: *u8 =
59"<p>The <strong>sovereign email stack</strong> is a full email capability built from the hardware rung up, riding the already-shipped sovereign net stack (<code>nx_socket</code> TCP/UDP syscalls, <code>nx_dns</code>, <code>nx_tls13</code>, <code>nx_sha256</code>/<code>nx_chacha20</code>/ed25519) — transport is composed, not rebuilt. Each rung is a pure NishiLang organ gated GREEN with a negative-control liar-kill before climbing; the posture is honest (graded on completeness + security-hardening + sovereignty + integer-determinism vs incumbents like getmxrr/c-ares, libesmtp, opendkim; raw throughput-exceed is NOT claimed).</p>\n"
60"<h3>The 8 rungs (all recorded DONE-GREEN)</h3>\n"
61"<ul>\n"
62"<li><strong>R0 MX resolution</strong> — <code>nx_email_mx</code>: MX rdata parse + lowest-preference-first sort (RFC 5321), txid spoof gate, compression-pointer-loop bounded.</li>\n"
63"<li><strong>R1 SMTP client</strong> — <code>nx_smtp_client</code>: multiline reply-code parser + EHLO/MAIL/RCPT/DATA/QUIT builders + dot-stuffing (injection defense) + a state machine.</li>\n"
64"<li><strong>R2 RFC 5322 + MIME</strong> — <code>nx_email_mime</code>: header find/unfold, quoted-printable + base64 (76-col wrap), multipart assemble (17 checks).</li>\n"
65"<li><strong>R3 mailbox store</strong> — <code>nx_email_mailbox</code> over the append-only segment store: deliver / soft-delete (tombstone) / byte-exact fetch / history.</li>\n"
66"<li><strong>R4 POP3 + IMAP retrieval</strong> — <code>nx_email_retrieve</code>: POP3 de-dot + IMAP tagged-status parsing (untagged-line defense).</li>\n"
67"<li><strong>R5 SPF + DKIM + DMARC</strong> — <code>nx_email_auth</code>: DKIM ed25519-sha256 sign/verify (RFC 8463), SPF cidr/qualifiers, DMARC policy; tamper-rejected.</li>\n"
68"<li><strong>R6 authenticated submission</strong> — <code>nx_email_submit</code>: AUTH PLAIN/LOGIN + STARTTLS detect, and a downgrade defense (refuse cleartext credentials if STARTTLS is not offered).</li>\n"
69"<li><strong>R7 receiving MTA</strong> — <code>nx_email_mta</code>: server FSM with an open-relay defense (550 for foreign recipients); the gate composes ACROSS rungs (inbound session → de-dot → store → fetch byte-exact).</li>\n"
70"</ul>\n"
71"<p>A live loopback layer was also demonstrated (a real SMTP message to <code>self@jasonewest.com</code> traveled the wire and landed in the store; a webmail UI rendered the inbox), with outbound DKIM signing using the same ed25519 key type the DNS zone generator publishes. The remaining work is the public deploy (operator-gated, outward-facing: a public host, MX records, port 25 reachable) and 3rd-party interop benchmarking. Built on the same sovereign <a href=\"/wiki/substrate.html\">substrate</a> as <a href=\"/wiki/nishi_os.html\">NishiOS</a>, and gated by the <a href=\"/wiki/access_wall.html\">access wall</a> when served.</p>\n"
72 if b_emit("sovereign_email" as *u8, "reference" as *u8, "Sovereign email — the 8-rung stack" as *u8, email_body) == 0 { ok = ok + 1 }
73
74 // ---------- 4) SIMD BACKEND (explanation) ----------
75 let simd_body: *u8 =
76"<p>The <strong>SIMD / vector backend</strong> is the throughput-floor hardware rung: the sovereign x86 vector path (SSE/AVX2 emitted by <code>nxasm_x86</code>, where the codegen previously emitted no xmm/SSE/AVX at all) plus a loosely-coupled, width-agnostic abstraction so that apps (ChaCha, FNet, GEMM, the trainer) consume a stable interface and never reach back down to the mnemonics — ship a new width (AVX-512, a new ISA) and every app gets faster untouched. All of it is built and run-proven on real silicon (no gcc/qemu on the product path); external incumbents are used as a benchmark only.</p>\n"
77"<h3>The measured exceed (ChaCha20 vs OpenSSL)</h3>\n"
78"<p>The headline result: a <strong>sovereign ChaCha20 that BEATS OpenSSL by about 3% on the room hardware (AMD Zen1 NAS), byte-exact.</strong> Full-cipher head-to-head on that silicon, interleaved in the same contention window: <strong>Nishi ~988–1020 MB/s vs OpenSSL ~958–988 MB/s</strong>. The path there was an honest design-space search across many kernels (4-block vertical 962, OpenSSL-faithful 8x SoA 936, …) that all plateaued at ~95% of OpenSSL — until ONE profile-targeted stall-kill (the <em>c-resident</em> optimization: keep the invariant ChaCha c-words resident across the round boundaries to remove a store→load-forward stall, something even OpenSSL's published allocation does not do) pushed the faithful kernel from 936 to ~1020 full-cipher. Correctness is an RFC-8439 byte-exact known-answer test; a port profiler measured the Zen1 shift port at 1.05 G/s as the true bottleneck. The exceed was not a different algorithm or ISA — it was killing overhead the profiler pointed at.</p>\n"
79"<p>Beyond ChaCha: the Poly1305 MAC has a byte-exact scalar reference (poly1305-donna, 5×26-bit limbs, RFC-8439 KAT), the SIMD primitives it needs are run-verified, and a SIMD census measures coverage of the full x86 vector ISA (recorded at 39/88 capabilities, evidence-based: present = the assembler actually dispatches the mnemonic, not asserted). Honest caveats: 512-wide is unmeasured on this silicon (no AVX-512); and because NishiLang compiles to one static ELF with no FFI, the hand-asm kernels are not yet callable from organs — the FFI bridge is a known planned rung, not a surprise. This floor is what unblocks <a href=\"/wiki/neural_mt.html\">neural MT</a> and the rest of the <a href=\"/wiki/products.html\">catalog</a>; it lives alongside the other <a href=\"/wiki/substrate.html\">substrate</a> pages.</p>\n"
80 if b_emit("simd_backend" as *u8, "explanation" as *u8, "SIMD backend — the throughput floor" as *u8, simd_body) == 0 { ok = ok + 1 }
81
82 // ---------- 5) NEURAL MT (explanation) ----------
83 let mt_body: *u8 =
84"<p><strong>Sovereign neural machine translation</strong> is a real neural MT arc (not a phrasebook) for live translation on nishifamily.com/video. It composes the existing sovereign ML substrate — scalar autograd (<code>nx_autograd</code>), the f32 tower (matmul/softmax/layernorm/activations), the sub-quadratic FNet mixer (with a working backward pass), and the production BPE tokenizer (<code>nx_bpe</code>) — rather than reinventing them. Each rung is gated GREEN with an anti-hardcode liar-kill (retrain on different data → learns the new mapping and forgets the old, proving nothing is baked in).</p>\n"
85"<h3>Where it is (honest)</h3>\n"
86"<ul>\n"
87"<li><strong>R0</strong> — the trainable atom: a net learns a bilingual word-map from zero-init via the scalar autograd (acc 6/6, loss converges, bit-exact retrain, data-driven).</li>\n"
88"<li><strong>R1</strong> — sequences: a shared position-independent word-map generalizes to held-out sentences (9/9 held-out).</li>\n"
89"<li><strong>R2</strong> — cross-token reordering (EN adjective-noun → ES noun-adjective) with a measured necessity contrast: the per-token model gets only 8/26 on the same task, so cross-token capacity is required, not decorative.</li>\n"
90"<li><strong>R3</strong> — real text via the production BPE: round-trip byte-exact, out-of-vocabulary words decompose into subwords, end-to-end text translate 3/3.</li>\n"
91"<li><strong>R5</strong> — wired LIVE: a separate sovereign translate daemon (kept apart from the video daemon so it cannot crash video) serves per-token translate-or-passthrough behind nginx, folded into the existing sovereign supervisor for crash-recovery + reboot survival.</li>\n"
92"</ul>\n"
93"<p>Honest scope: the vocabulary is still a toy, and real open-vocab sentence fluency needs real training compute (or loading trained weights, which is in tension with sovereignty) — that is R4, where the arc gets expensive. The arc is fully sovereign (no third party, no gcc, no shell) and rides the <a href=\"/wiki/simd_backend.html\">SIMD throughput floor</a>; it ships in the family-video product line alongside the <a href=\"/wiki/media_studio.html\">media studio</a>.</p>\n"
94 if b_emit("neural_mt" as *u8, "explanation" as *u8, "Neural MT — sovereign machine translation" as *u8, mt_body) == 0 { ok = ok + 1 }
95
96 // ---------- 6) VISION FR (explanation) ----------
97 let vision_body: *u8 =
98"<p><strong>Sovereign image facial-recognition</strong> groups the AI-image library by person (\"who is who\") using a weight-free <strong>LBP (Local Binary Pattern)</strong> descriptor built from first principles in NishiLang, hardware-rung-up. The method is classic and integer-only — grayscale → LBP histogram → chi-square distance → greedy clustering → person ids — needing NO trained weights and illumination-invariant by construction. Per the operator's method law, the product is built ONLY in NishiLang organs; the external incumbent <strong>insightface is allowed purely as a benchmark / measuring-stick</strong>, never as the product (an earlier Python insightface indexer was reverted).</p>\n"
99"<h3>The algorithmic core (rungs GREEN)</h3>\n"
100"<ul>\n"
101"<li><strong>R1 <code>nx_lbp</code></strong> — LBP code + histogram + chi-square, bit-exact KATs; illumination invariance proven (chi2 of two flat images = 0); same-vs-different separation.</li>\n"
102"<li><strong>R2 <code>nx_lbp_img</code></strong> — real PNG decode → luma gray → LBP histogram on actual 768×1024 images; chi2(A,A)=0 while chi2(A,B) is large and positive.</li>\n"
103"<li><strong>R3 <code>nx_lbp_grid</code></strong> — spatial-grid descriptor (g×g cells, each a 256-bin histogram) with a spatial-locality KAT: a localized change moves only the affected cell's distance, far cells stay bit-identical.</li>\n"
104"<li><strong>R4 <code>nx_lbp_cluster</code></strong> — greedy chi-square-threshold clustering: three separated synthetic groups cluster correctly, with threshold-controlled granularity.</li>\n"
105"</ul>\n"
106"<p>So the descriptor (R1–R3) and the clusterer (R4) are complete and gated GREEN, all sovereign. Honest scope: this is the classic weight-free pipeline; later rungs (face-region crop, Eigenfaces/PCA, Viola-Jones integral-image detection) and the head-to-head measure against the insightface benchmark are still ahead. It serves the <a href=\"/wiki/media_studio.html\">media studio</a>'s image library and shares the <a href=\"/wiki/products.html\">product</a> vision line.</p>\n"
107 if b_emit("vision_fr" as *u8, "explanation" as *u8, "Vision FR — sovereign face recognition (LBP)" as *u8, vision_body) == 0 { ok = ok + 1 }
108
109 // ---------- 7) VOXELWORLD (tutorial) ----------
110 let voxel_body: *u8 =
111"<p><strong>Voxelworld</strong> is the live, sovereign first-person 3D voxel world, playable now at <a href=\"https://nishifamily.com/voxelworld\">nishifamily.com/voxelworld</a>. It proves the whole sovereign-game chain end-to-end: the renderer is written in NishiLang and compiled to WebAssembly through the sovereign wat toolchain — no third-party game engine. It is the practical demonstration of the <a href=\"/wiki/substrate.html\">substrate</a> reaching all the way up to a thing you can actually play in a browser.</p>\n"
112"<h3>Try it (and how it is built)</h3>\n"
113"<ol>\n"
114"<li><strong>Open</strong> <a href=\"https://nishifamily.com/voxelworld\">https://nishifamily.com/voxelworld</a> in a browser.</li>\n"
115"<li><strong>Click the canvas</strong> to capture the pointer (pointer-lock mouse-look).</li>\n"
116"<li><strong>Move</strong> with WASD; <strong>Space / Shift</strong> for up / down. The camera renders per frame — there is no auto-spin (an earlier orbit \"diorama\" was rejected by the operator as \"not a game\" and replaced by a real navigable world).</li>\n"
117"</ol>\n"
118"<p>Under the hood: <code>nx_wasmfps.nx</code> is a first-person voxel renderer written base-relative (every buffer = base + offset) so the EXACT same code runs native (base = mmap) and in wasm (base = 0) — a 96×96 z-buffered, perspective, distance-culled world with trees and realism shading. The build pipeline is sovereign: <code>nx_compile_wat</code> (~6877 lines of wat, 0 TODO) → <code>nx_wat_compiler</code> (WATC GREEN, ~11 KB wasm) → <code>nx_wasm_html_fps</code> emits the interactive HTML shim (the JS does only pointer-lock + key state and calls <code>render()</code>; it is emitted output, not hand-authored app logic). The ~17.7 KB HTML was pushed to the live Nishi sites daemon and verified over HTTPS through the sovereign SNI router. Correctness is checked by eye via a sovereign PNG render (a native gate renders the same code to a PNG, closing the can't-see-wasm gap). It shares the game line with <a href=\"/wiki/econsim.html\">econsim</a> and the <a href=\"/wiki/products.html\">catalog</a>.</p>\n"
119 if b_emit("voxelworld" as *u8, "tutorial" as *u8, "Voxelworld — the live sovereign voxel game" as *u8, voxel_body) == 0 { ok = ok + 1 }
120
121 b_w("=== emitted "); b_num(ok); b_w("/7 pages (substrate hub + 6) via nx_ims_page ===\n")
122 if ok == 7 { sys_exit(0); return 0 }
123 sys_exit(1); return 1
124}