nx_docportal_lib.nx source

1// nx_docportal_lib.nx -- shared core for the PER-DOMAIN DOCUMENT PORTAL (operator 2026-06-28: "a document portal 2// to upload documents for their onsite search ... all our sites sharded off per domain ... public and private"). 3// The s-class substrate is the SOVEREIGN seg_store (immutable append-only segments + content-addressed values + 4// never-delete) -- NOT a new TSV database (operator: get off TSV). ONE call ingests a document into the domain's 5// PUBLIC store (also emitted to the domain's derived BM25 search source so it is findable on <domain>/search) or 6// PRIVATE store (the client vault, NEVER emitted to the public search source). 7// 8// THE PUBLIC/PRIVATE SPLIT IS BY CONSTRUCTION: public and private are DIFFERENT store prefixes => DIFFERENT 9// physical segment files => a public search query cannot read the private shard (it never opens those files). 10// THE PER-DOMAIN SHARD is the domain: prefix carries <domain>, so a query scoped to domain A cannot read domain 11// B's segments the same way. This is the records-management "balance of confidentiality, data privacy, and public 12// access" (ldm_records_mgmt) enforced mechanically, not by an ACL check that can be misconfigured. 13// 14// CID = deterministic content id (FNV-1a, folded positive): same bytes -> same CID (exceed dimension #8); used as 15// the segid so distinct content never overwrites a prior doc's segment (never-delete) and identical re-ingest is 16// idempotent. The cryptographic seal (sha256 via nx_doc_seal) is a later rung. Composes nx_seg_store + nx_fnv. 17// license_tier: ORIGINAL 18import "nx_seg_store.nx" 19import "nx_fnv.nx" 20import "nx_syscalls.nx" 21 22const DP_VIS_PUBLIC: i64 = 0 23const DP_VIS_PRIVATE: i64 = 1 24 25// ---- owner USAGE-POLICY flags (operator: owners flag how their uploads are managed; extensible -- "...or whatever") ---- 26const DP_USE_PUB_SEARCH: i64 = 1 // index this doc for the public onsite search 27const DP_USE_AI_BLOG: i64 = 2 // an AI writer MAY publish blog posts derived from this content 28const DP_USE_AI_SUMMARY: i64 = 4 // AI may summarize/derive internally (non-publishing) 29const DP_USE_SHARE_EXT: i64 = 8 // may be shared outside the firm 30 31func dp_len(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n } 32func dp_cat(out: *u8, o: i64, s: *u8) -> i64 { var i: i64 = 0; while s[i] != (0 as u8) { out[o] = s[i]; o = o + 1; i = i + 1 } return o } 33func dp_catn(out: *u8, o: i64, v: i64) -> i64 { 34 if v == 0 { out[o] = 48 as u8; return o + 1 } 35 var m: i64 = v 36 if m < 0 { out[o] = 45 as u8; o = o + 1; m = 0 - m } 37 let t: *u8 = sys_mmap(24); var k: i64 = 0 38 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 } 39 var i: i64 = 0; while i < k { out[o] = t[k - 1 - i]; o = o + 1; i = i + 1 } 40 return o 41} 42 43// deterministic content id (FNV-1a over the doc bytes, folded positive). Same bytes -> same CID. 44func dp_cid(text: *u8, n: i64) -> i64 { 45 var h: i64 = fnv1a_init() 46 h = fnv1a_update(h, text, n) 47 return h & 0x7fffffffffffffff 48} 49 50// store prefix for (domain, visibility): knowledge/store/dp-<domain>-pub- / knowledge/store/dp-<domain>-prv- 51func dp_prefix(domain: *u8, vis: i64, out: *u8) -> i64 { 52 var o: i64 = dp_cat(out, 0, "knowledge/store/dp-" as *u8) 53 o = dp_cat(out, o, domain) 54 if vis == DP_VIS_PRIVATE { o = dp_cat(out, o, "-prv-" as *u8) } else { o = dp_cat(out, o, "-pub-" as *u8) } 55 out[o] = 0 as u8 56 return o 57} 58 59// content-addressed doc key: doc:<cid> 60func dp_key(cid: i64, out: *u8) -> i64 { 61 var o: i64 = dp_cat(out, 0, "doc:" as *u8) 62 o = dp_catn(out, o, cid) 63 out[o] = 0 as u8 64 return o 65} 66 67// RETIRED (operator 2026-07-02, "no tsv -- nishi ecosystem hardware rung up"): dp_pubsrc/dp_append_pubsrc, 68// the derived knowledge/index/<domain>_src.tsv search-source emitter, are DELETED. The sovereign store IS the 69// search index (ss_write_seg builds .terms for every segment); nx_docportal_search_seg queries it directly and 70// enforces DP_USE_PUB_SEARCH consent LIVE per hit (pol:<cid>), so the emission-time filter has no job left. 71 72// INGEST one document into the domain's shard by visibility. Writes the raw bytes content-addressed into the 73// sovereign seg_store (immutable; segid=CID so distinct content never overwrites a prior doc, identical re-ingest 74// is idempotent). PUBLIC docs are ALSO emitted to the derived BM25 search source. A PRIVATE doc touches ONLY the 75// private store -- no public store write, no search-source line. Returns 0 ok; cidout[0] = content id. 76// INGEST with the DEFAULT owner policy (public: searchable, AI-blog opt-in OFF; private: nothing). Backward- 77// compatible wrapper -- existing callers get sensible, consent-respecting defaults; owners set explicit flags via 78// dp_ingest_policy (the admin handler passes the owner's choices). Returns 0 ok; cidout[0]=cid. 79func dp_ingest(domain: *u8, vis: i64, text: *u8, n: i64, cidout: *i64) -> i64 { 80 return dp_ingest_policy(domain, vis, dp_default_policy(vis), text, n, cidout) 81} 82 83// READ a doc back from the (domain, visibility) shard by cid, byte-exact. 1 = found (ptrout[0]/lenout[0] set). 84// Uses the PROVEN ss_open+ss_hget path (nx_raci_sov): an empty/absent shard -> ss_open returns 0 -> not found, 85// which is exactly the cross-shard isolation guarantee (a public/other-domain prefix with no segments answers 0). 86func dp_read(domain: *u8, vis: i64, cid: i64, ptrout: *i64, lenout: *i64) -> i64 { 87 let prefix: *u8 = sys_mmap(512); dp_prefix(domain, vis, prefix) 88 let key: *u8 = sys_mmap(64); dp_key(cid, key) 89 // ss_open_cached (seq905/962 class fix, seq1347 migration, 2026-07-30). ss_open reads every live 90 // segment into anon RAM and nothing frees it, so a per-request reader leaks the WHOLE store per 91 // request -- the fingerprint behind ~23.6 GiB across nine organs. SAFE HERE: open -> one ss_hget -> 92 // return, so the handle's shared query-scratch arena is never re-entered while held. Manifest 93 // (st_size, st_mtime) invalidation means live edits are still picked up on the next call. 94 let h: *i64 = ss_open_cached(prefix) 95 if (h as i64) == 0 { return 0 } 96 if ss_hget(h, key, ptrout, lenout) == 1 { return 1 } // normalize: ss_hget returns -1 (not 0) for not-found 97 return 0 98} 99 100// ================= OWNER USAGE POLICY (operator: lawyers/users flag how their uploads are managed) ================= 101// Per-doc flags, set by the owner at upload + amendable, stored in the doc's shard (pol:<cid>) ATOMICALLY with the 102// doc, and ENFORCED BY CONSTRUCTION: a consumer (the public indexer, the AI-blog writer) is only ever fed a doc whose 103// flag permits that use -- a non-permitted doc never enters the consumer's corpus, and consumers re-check dp_may LIVE 104// so a consent amendment takes effect at once. Conservative + defensible (the privilege research): private content is 105// never auto-published. Extensible bitset ("...or whatever etc"). 106 107func dp_default_policy(vis: i64) -> i64 { 108 if vis == DP_VIS_PRIVATE { return 0 } // private: nothing permitted by default (most restrictive) 109 return DP_USE_PUB_SEARCH // public: searchable; AI-blog is OPT-IN (owner must flag it) 110} 111func dp_polkey(cid: i64, out: *u8) -> i64 { var o: i64 = dp_cat(out, 0, "pol:" as *u8); o = dp_catn(out, o, cid); out[o] = 0 as u8; return o } 112 113// AI-blog-permitted corpus (DERIVED, policy-gated): knowledge/index/<domain>_blogsrc.tsv (cid<TAB>text). A doc enters 114// it ONLY if PUBLIC visibility AND carries DP_USE_AI_BLOG -- a private/unflagged doc is never here, so it can never 115// reach a published blog (private content is never auto-published). 116func dp_blogsrc(domain: *u8, out: *u8) -> i64 { 117 var o: i64 = dp_cat(out, 0, "knowledge/index/" as *u8); o = dp_cat(out, o, domain); o = dp_cat(out, o, "_blogsrc.tsv" as *u8); out[o] = 0 as u8; return o 118} 119func dp_append_blogsrc(domain: *u8, cid: i64, text: *u8, n: i64) -> i64 { 120 let path: *u8 = sys_mmap(512); dp_blogsrc(domain, path) 121 let fd: i64 = sys_openat_append(path, 0x1a4); if fd < 0 { return 0 - 1 } 122 let hdr: *u8 = sys_mmap(64); var ho: i64 = dp_catn(hdr, 0, cid); hdr[ho] = 9 as u8; ho = ho + 1; sys_write(fd, hdr, ho) 123 let san: *u8 = sys_mmap(n + 8); var i: i64 = 0 124 while i < n { var c: i64 = text[i] as i64; if c == 9 { c = 32 } if c == 10 { c = 32 } if c == 13 { c = 32 } san[i] = c as u8; i = i + 1 } 125 sys_write(fd, san, n); sys_write(fd, "\n" as *u8, 1); sys_close(fd); return 0 126} 127 128// INGEST with an explicit owner policy. Stores doc + policy ATOMICALLY (one commit, segid=CID); emits to the public 129// search source iff (PUBLIC & DP_USE_PUB_SEARCH); emits to the AI-blog corpus iff (PUBLIC & DP_USE_AI_BLOG). 130func dp_ingest_policy(domain: *u8, vis: i64, flags: i64, text: *u8, n: i64, cidout: *i64) -> i64 { 131 let cid: i64 = dp_cid(text, n); cidout[0] = cid 132 let prefix: *u8 = sys_mmap(512); dp_prefix(domain, vis, prefix) 133 let dkey: *u8 = sys_mmap(64); dp_key(cid, dkey) 134 let pkey: *u8 = sys_mmap(64); dp_polkey(cid, pkey) 135 let polbuf: *u8 = sys_mmap(32); let pl: i64 = dp_catn(polbuf, 0, flags) 136 let w: *i64 = ss_begin() 137 if ss_add(w, 1, dkey, text, n) < 0 { return 0 - 2 } 138 if ss_add(w, 1, pkey, polbuf, pl) < 0 { return 0 - 2 } 139 ss_commit(prefix, w, cid) 140 // NO derived search emission: the committed segment's .terms postings make the doc searchable at once, 141 // and the stored pol:<cid> row is what the search consumer (dss_search) checks LIVE for DP_USE_PUB_SEARCH. 142 if vis == DP_VIS_PUBLIC { 143 if (flags & DP_USE_AI_BLOG) != 0 { dp_append_blogsrc(domain, cid, text, n) } 144 } 145 return 0 146} 147 148// read a doc's stored policy flags (0 if absent) 149func dp_get_policy(domain: *u8, vis: i64, cid: i64) -> i64 { 150 let prefix: *u8 = sys_mmap(512); dp_prefix(domain, vis, prefix) 151 let pkey: *u8 = sys_mmap(64); dp_polkey(cid, pkey) 152 let h: *i64 = ss_open_cached(prefix); if (h as i64) == 0 { return 0 } // seq1347: same class fix as dp_read 153 let pq: *i64 = sys_mmap(16) as *i64; let lq: *i64 = sys_mmap(16) as *i64 154 if ss_hget(h, pkey, pq, lq) != 1 { return 0 } 155 let p: *u8 = pq[0] as *u8; let l: i64 = lq[0] 156 var v: i64 = 0; var i: i64 = 0 157 while i < l { let c: i64 = p[i] as i64; if c >= 48 { if c <= 57 { v = v * 10 + (c - 48) } } i = i + 1 } 158 return v 159} 160 161// does the owner's policy permit `use_flag` for this doc? 1/0. The LIVE authoritative check every consumer makes. 162func dp_may(domain: *u8, vis: i64, cid: i64, use_flag: i64) -> i64 { 163 let f: i64 = dp_get_policy(domain, vis, cid) 164 if (f & use_flag) != 0 { return 1 } 165 return 0 166} 167 168// AMEND a doc's policy (owner changes consent). Re-commits the doc bytes (copied, unchanged) + the new flags with 169// segid=CID -- idempotent on the doc (bytes preserved, never-delete), policy updated. The LIVE policy is authoritative 170// immediately (consumers re-check dp_may), so withdrawing AI-blog consent excludes the doc at once. 0 ok, <0 absent. 171func dp_set_policy(domain: *u8, vis: i64, cid: i64, flags: i64) -> i64 { 172 let pq: *i64 = sys_mmap(16) as *i64; let lq: *i64 = sys_mmap(16) as *i64 173 if dp_read(domain, vis, cid, pq, lq) == 0 { return 0 - 1 } 174 let srclen: i64 = lq[0] 175 let copy: *u8 = sys_mmap(srclen + 8); let src: *u8 = pq[0] as *u8 176 var i: i64 = 0; while i < srclen { copy[i] = src[i]; i = i + 1 } 177 let prefix: *u8 = sys_mmap(512); dp_prefix(domain, vis, prefix) 178 let dkey: *u8 = sys_mmap(64); dp_key(cid, dkey) 179 let pkey: *u8 = sys_mmap(64); dp_polkey(cid, pkey) 180 let polbuf: *u8 = sys_mmap(32); let pl: i64 = dp_catn(polbuf, 0, flags) 181 let w: *i64 = ss_begin() 182 ss_add(w, 1, dkey, copy, srclen) 183 ss_add(w, 1, pkey, polbuf, pl) 184 ss_commit(prefix, w, cid) 185 return 0 186}