nx_docportal_lib.nx source

1// nx_docportal_lib.nx -- shared core for the PER-DOMAIN DOCUMENT PORTAL (operator 2026-06-28: "a document portal 2// to upload documents for their onsite search ... all our sites sharded off per domain ... public and private"). 3// The s-class substrate is the SOVEREIGN seg_store (immutable append-only segments + content-addressed values + 4// never-delete) -- NOT a new TSV database (operator: get off TSV). ONE call ingests a document into the domain's 5// PUBLIC store (also emitted to the domain's derived BM25 search source so it is findable on <domain>/search) or 6// PRIVATE store (the client vault, NEVER emitted to the public search source). 7// 8// THE PUBLIC/PRIVATE SPLIT IS BY CONSTRUCTION: public and private are DIFFERENT store prefixes => DIFFERENT 9// physical segment files => a public search query cannot read the private shard (it never opens those files). 10// THE PER-DOMAIN SHARD is the domain: prefix carries <domain>, so a query scoped to domain A cannot read domain 11// B's segments the same way. This is the records-management "balance of confidentiality, data privacy, and public 12// access" (ldm_records_mgmt) enforced mechanically, not by an ACL check that can be misconfigured. 13// 14// CID = deterministic content id (FNV-1a, folded positive): same bytes -> same CID (exceed dimension #8); used as 15// the segid so distinct content never overwrites a prior doc's segment (never-delete) and identical re-ingest is 16// idempotent. The cryptographic seal (sha256 via nx_doc_seal) is a later rung. Composes nx_seg_store + nx_fnv. 17// license_tier: ORIGINAL 18import "nx_seg_store.nx" 19import "nx_fnv.nx" 20import "nx_syscalls.nx" 21import "nx_docprose.nx" // dpr_http_body / dpr_is_html: capture front matter, and IS this really html 22import "nx_block_density.nx" // bd_fit_text: the estate's shipping Kohlschuetter boilerplate remover 23 24const DP_VIS_PUBLIC: i64 = 0 25const DP_VIS_PRIVATE: i64 = 1 26 27// ---- owner USAGE-POLICY flags (operator: owners flag how their uploads are managed; extensible -- "...or whatever") ---- 28const DP_USE_PUB_SEARCH: i64 = 1 // index this doc for the public onsite search 29const DP_USE_AI_BLOG: i64 = 2 // an AI writer MAY publish blog posts derived from this content 30const DP_USE_AI_SUMMARY: i64 = 4 // AI may summarize/derive internally (non-publishing) 31const DP_USE_SHARE_EXT: i64 = 8 // may be shared outside the firm 32 33func dp_len(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n } 34func dp_cat(out: *u8, o: i64, s: *u8) -> i64 { var i: i64 = 0; while s[i] != (0 as u8) { out[o] = s[i]; o = o + 1; i = i + 1 } return o } 35func dp_catn(out: *u8, o: i64, v: i64) -> i64 { 36 if v == 0 { out[o] = 48 as u8; return o + 1 } 37 var m: i64 = v 38 if m < 0 { out[o] = 45 as u8; o = o + 1; m = 0 - m } 39 let t: *u8 = sys_mmap(24); var k: i64 = 0 40 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 } 41 var i: i64 = 0; while i < k { out[o] = t[k - 1 - i]; o = o + 1; i = i + 1 } 42 return o 43} 44 45// deterministic content id (FNV-1a over the doc bytes, folded positive). Same bytes -> same CID. 46func dp_cid(text: *u8, n: i64) -> i64 { 47 var h: i64 = fnv1a_init() 48 h = fnv1a_update(h, text, n) 49 return h & 0x7fffffffffffffff 50} 51 52// store prefix for (domain, visibility): knowledge/store/dp-<domain>-pub- / knowledge/store/dp-<domain>-prv- 53func dp_prefix(domain: *u8, vis: i64, out: *u8) -> i64 { 54 var o: i64 = dp_cat(out, 0, "knowledge/store/dp-" as *u8) 55 o = dp_cat(out, o, domain) 56 if vis == DP_VIS_PRIVATE { o = dp_cat(out, o, "-prv-" as *u8) } else { o = dp_cat(out, o, "-pub-" as *u8) } 57 out[o] = 0 as u8 58 return o 59} 60 61// content-addressed doc key: doc:<cid> 62func dp_key(cid: i64, out: *u8) -> i64 { 63 var o: i64 = dp_cat(out, 0, "doc:" as *u8) 64 o = dp_catn(out, o, cid) 65 out[o] = 0 as u8 66 return o 67} 68 69// RETIRED (operator 2026-07-02, "no tsv -- nishi ecosystem hardware rung up"): dp_pubsrc/dp_append_pubsrc, 70// the derived knowledge/index/<domain>_src.tsv search-source emitter, are DELETED. The sovereign store IS the 71// search index (ss_write_seg builds .terms for every segment); nx_docportal_search_seg queries it directly and 72// enforces DP_USE_PUB_SEARCH consent LIVE per hit (pol:<cid>), so the emission-time filter has no job left. 73 74// INGEST one document into the domain's shard by visibility. Writes the raw bytes content-addressed into the 75// sovereign seg_store (immutable; segid=CID so distinct content never overwrites a prior doc, identical re-ingest 76// is idempotent). PUBLIC docs are ALSO emitted to the derived BM25 search source. A PRIVATE doc touches ONLY the 77// private store -- no public store write, no search-source line. Returns 0 ok; cidout[0] = content id. 78// INGEST with the DEFAULT owner policy (public: searchable, AI-blog opt-in OFF; private: nothing). Backward- 79// compatible wrapper -- existing callers get sensible, consent-respecting defaults; owners set explicit flags via 80// dp_ingest_policy (the admin handler passes the owner's choices). Returns 0 ok; cidout[0]=cid. 81func dp_ingest(domain: *u8, vis: i64, text: *u8, n: i64, cidout: *i64) -> i64 { 82 return dp_ingest_policy(domain, vis, dp_default_policy(vis), text, n, cidout) 83} 84 85// READ a doc back from the (domain, visibility) shard by cid, byte-exact. 1 = found (ptrout[0]/lenout[0] set). 86// Uses the PROVEN ss_open+ss_hget path (nx_raci_sov): an empty/absent shard -> ss_open returns 0 -> not found, 87// which is exactly the cross-shard isolation guarantee (a public/other-domain prefix with no segments answers 0). 88func dp_read(domain: *u8, vis: i64, cid: i64, ptrout: *i64, lenout: *i64) -> i64 { 89 let prefix: *u8 = sys_mmap(512); dp_prefix(domain, vis, prefix) 90 let key: *u8 = sys_mmap(64); dp_key(cid, key) 91 // ss_open_cached (seq905/962 class fix, seq1347 migration, 2026-07-30). ss_open reads every live 92 // segment into anon RAM and nothing frees it, so a per-request reader leaks the WHOLE store per 93 // request -- the fingerprint behind ~23.6 GiB across nine organs. SAFE HERE: open -> one ss_hget -> 94 // return, so the handle's shared query-scratch arena is never re-entered while held. Manifest 95 // (st_size, st_mtime) invalidation means live edits are still picked up on the next call. 96 let h: *i64 = ss_open_cached(prefix) 97 if (h as i64) == 0 { return 0 } 98 if ss_hget(h, key, ptrout, lenout) == 1 { return 1 } // normalize: ss_hget returns -1 (not 0) for not-found 99 return 0 100} 101 102// ================= OWNER USAGE POLICY (operator: lawyers/users flag how their uploads are managed) ================= 103// Per-doc flags, set by the owner at upload + amendable, stored in the doc's shard (pol:<cid>) ATOMICALLY with the 104// doc, and ENFORCED BY CONSTRUCTION: a consumer (the public indexer, the AI-blog writer) is only ever fed a doc whose 105// flag permits that use -- a non-permitted doc never enters the consumer's corpus, and consumers re-check dp_may LIVE 106// so a consent amendment takes effect at once. Conservative + defensible (the privilege research): private content is 107// never auto-published. Extensible bitset ("...or whatever etc"). 108 109func dp_default_policy(vis: i64) -> i64 { 110 if vis == DP_VIS_PRIVATE { return 0 } // private: nothing permitted by default (most restrictive) 111 return DP_USE_PUB_SEARCH // public: searchable; AI-blog is OPT-IN (owner must flag it) 112} 113func dp_polkey(cid: i64, out: *u8) -> i64 { var o: i64 = dp_cat(out, 0, "pol:" as *u8); o = dp_catn(out, o, cid); out[o] = 0 as u8; return o } 114 115// AI-blog-permitted corpus (DERIVED, policy-gated): knowledge/index/<domain>_blogsrc.tsv (cid<TAB>text). A doc enters 116// it ONLY if PUBLIC visibility AND carries DP_USE_AI_BLOG -- a private/unflagged doc is never here, so it can never 117// reach a published blog (private content is never auto-published). 118func dp_blogsrc(domain: *u8, out: *u8) -> i64 { 119 var o: i64 = dp_cat(out, 0, "knowledge/index/" as *u8); o = dp_cat(out, o, domain); o = dp_cat(out, o, "_blogsrc.tsv" as *u8); out[o] = 0 as u8; return o 120} 121func dp_append_blogsrc(domain: *u8, cid: i64, text: *u8, n: i64) -> i64 { 122 let path: *u8 = sys_mmap(512); dp_blogsrc(domain, path) 123 let fd: i64 = sys_openat_append(path, 0x1a4); if fd < 0 { return 0 - 1 } 124 let hdr: *u8 = sys_mmap(64); var ho: i64 = dp_catn(hdr, 0, cid); hdr[ho] = 9 as u8; ho = ho + 1; sys_write(fd, hdr, ho) 125 let san: *u8 = sys_mmap(n + 8); var i: i64 = 0 126 while i < n { var c: i64 = text[i] as i64; if c == 9 { c = 32 } if c == 10 { c = 32 } if c == 13 { c = 32 } san[i] = c as u8; i = i + 1 } 127 sys_write(fd, san, n); sys_write(fd, "\n" as *u8, 1); sys_close(fd); return 0 128} 129 130// INGEST with an explicit owner policy. Stores doc + policy ATOMICALLY (one commit, segid=CID); emits to the public 131// search source iff (PUBLIC & DP_USE_PUB_SEARCH); emits to the AI-blog corpus iff (PUBLIC & DP_USE_AI_BLOG). 132func dp_ingest_policy(domain: *u8, vis: i64, flags: i64, text_in: *u8, n_in: i64, cidout: *i64) -> i64 { 133 // ---- DERIVE CLEAN TEXT BEFORE THE CID IS TAKEN (2026-08-25) ------------------------------------ 134 // WAS: the raw uploaded bytes were hashed, stored AND tokenised, with no extraction step anywhere on 135 // this path. That is why nishifamily.com/search served titles like 136 // "HTTP/1.1 200 OK Date: Thu, 02 Jul 2026 17:07:07 GMT Content-Type: text/h" 137 // -- evidence-mirror captures reached the index with their wire bytes intact. 138 // It is ALSO why one page appears twice: dp_cid is FNV-1a over exactly these bytes, so two fetches of 139 // the same page differing only in a Date: header are two different documents with two different cids. 140 // Cleaning the text BEFORE dp_cid fixes BOTH: the duplicate collapses to one cid, and this function's 141 // existing identical-re-ingest idempotence then absorbs it. ORDER IS LOAD-BEARING: 142 // 1. strip the HTTP header block -- protocol-exact and anchored at byte 0. bd_fit_text does NOT do 143 // this: on unstructured input it self-degrades to nx_html_to_text, a tag stripper, which would 144 // leave the whole status line and every header sitting in the indexed text. 145 // 2. extract article text, but ONLY when the document really is html. dpr_is_html decides on a 146 // STRUCTURAL fact (a declared Content-Type, or a doctype/html/body marker), never on tag density, 147 // because this estate indexes its own source and a tag stripper would silently eat every `<u8>`. 148 let bstart: i64 = dpr_http_body(text_in, n_in) 149 let ishtml: i64 = dpr_is_html(text_in, n_in) 150 var text: *u8 = ((text_in as i64) + bstart) as *u8 151 var n: i64 = n_in - bstart 152 if ishtml == 1 { if n > 0 { 153 let fcap: i64 = n + 1 154 let fout: *u8 = sys_mmap(fcap) 155 let fm: i64 = bd_fit_text(text, n, fout, fcap) 156 // A fit that yields NOTHING is a fit that failed: keep the header-stripped original rather than 157 // index an empty document. Falling through silently here would turn a boilerplate-heavy page into 158 // a zero-byte doc that still gets a cid and still occupies a result slot. 159 if fm > 0 { text = fout; n = fm } 160 } } 161 let cid: i64 = dp_cid(text, n); cidout[0] = cid 162 let prefix: *u8 = sys_mmap(512); dp_prefix(domain, vis, prefix) 163 let dkey: *u8 = sys_mmap(64); dp_key(cid, dkey) 164 let pkey: *u8 = sys_mmap(64); dp_polkey(cid, pkey) 165 let polbuf: *u8 = sys_mmap(32); let pl: i64 = dp_catn(polbuf, 0, flags) 166 let w: *i64 = ss_begin() 167 if ss_add(w, 1, dkey, text, n) < 0 { return 0 - 2 } 168 if ss_add(w, 1, pkey, polbuf, pl) < 0 { return 0 - 2 } 169 ss_commit(prefix, w, cid) 170 // NO derived search emission: the committed segment's .terms postings make the doc searchable at once, 171 // and the stored pol:<cid> row is what the search consumer (dss_search) checks LIVE for DP_USE_PUB_SEARCH. 172 if vis == DP_VIS_PUBLIC { 173 if (flags & DP_USE_AI_BLOG) != 0 { dp_append_blogsrc(domain, cid, text, n) } 174 } 175 return 0 176} 177 178// read a doc's stored policy flags (0 if absent) 179func dp_get_policy(domain: *u8, vis: i64, cid: i64) -> i64 { 180 let prefix: *u8 = sys_mmap(512); dp_prefix(domain, vis, prefix) 181 let pkey: *u8 = sys_mmap(64); dp_polkey(cid, pkey) 182 let h: *i64 = ss_open_cached(prefix); if (h as i64) == 0 { return 0 } // seq1347: same class fix as dp_read 183 let pq: *i64 = sys_mmap(16) as *i64; let lq: *i64 = sys_mmap(16) as *i64 184 if ss_hget(h, pkey, pq, lq) != 1 { return 0 } 185 let p: *u8 = pq[0] as *u8; let l: i64 = lq[0] 186 var v: i64 = 0; var i: i64 = 0 187 while i < l { let c: i64 = p[i] as i64; if c >= 48 { if c <= 57 { v = v * 10 + (c - 48) } } i = i + 1 } 188 return v 189} 190 191// does the owner's policy permit `use_flag` for this doc? 1/0. The LIVE authoritative check every consumer makes. 192func dp_may(domain: *u8, vis: i64, cid: i64, use_flag: i64) -> i64 { 193 let f: i64 = dp_get_policy(domain, vis, cid) 194 if (f & use_flag) != 0 { return 1 } 195 return 0 196} 197 198// AMEND a doc's policy (owner changes consent). Re-commits the doc bytes (copied, unchanged) + the new flags with 199// segid=CID -- idempotent on the doc (bytes preserved, never-delete), policy updated. The LIVE policy is authoritative 200// immediately (consumers re-check dp_may), so withdrawing AI-blog consent excludes the doc at once. 0 ok, <0 absent. 201func dp_set_policy(domain: *u8, vis: i64, cid: i64, flags: i64) -> i64 { 202 let pq: *i64 = sys_mmap(16) as *i64; let lq: *i64 = sys_mmap(16) as *i64 203 if dp_read(domain, vis, cid, pq, lq) == 0 { return 0 - 1 } 204 let srclen: i64 = lq[0] 205 let copy: *u8 = sys_mmap(srclen + 8); let src: *u8 = pq[0] as *u8 206 var i: i64 = 0; while i < srclen { copy[i] = src[i]; i = i + 1 } 207 let prefix: *u8 = sys_mmap(512); dp_prefix(domain, vis, prefix) 208 let dkey: *u8 = sys_mmap(64); dp_key(cid, dkey) 209 let pkey: *u8 = sys_mmap(64); dp_polkey(cid, pkey) 210 let polbuf: *u8 = sys_mmap(32); let pl: i64 = dp_catn(polbuf, 0, flags) 211 let w: *i64 = ss_begin() 212 ss_add(w, 1, dkey, copy, srclen) 213 ss_add(w, 1, pkey, polbuf, pl) 214 ss_commit(prefix, w, cid) 215 return 0 216}