code wiki / (root) / nx_comparestale_dependency_t138.nx
`nx_comparestale_dependency_t138.nx` source

↩ module page · 352 lines · 18189 B
1// nx_comparestale.nx -- IS EVERY PUBLISHED COMPARE PAGE STILL TRUE TO ITS SOURCE?
2//
3// WHY THIS EXISTS. /compare/doctor sat a MONTH stale, publishing "THE root blocker: emitted ELFs
4// carry no line mapping" long after that stopped being true -- with a perfectly valid
5// generated_unix. Nothing could see it, because nothing compared the page to the thing it was
6// generated from. A TIMESTAMP TELLS YOU WHEN A PAGE WAS MADE, NEVER WHETHER IT IS CURRENT.
7//
8// HOW. Both generators now stamp "source_hash" = djb2-48 over the .matrix/.sota they read. This
9// organ recomputes that hash from the source on disk and compares it to the published page.
10// Deliberately CLOCK-FREE: mtimes lie across copies (this tree has TWO knowledge/compare
11// directories and I lost an edit to exactly that) and clocks lie across machines. Content does not.
12//
13// VERDICTS, and the distinction is the point:
14//   FRESH       page hash == source hash
15//   STALE       page exists and disagrees with its source  -> the source of truth moved, publish did not
16//   UNPUBLISHED source exists, no page                     -> never generated, or gate-refused
17//   NOSTAMP     page predates the provenance stamp         -> republish once and it becomes decidable
18// Exit 1 if any STALE. UNPUBLISHED does NOT fail the gate: a domain whose matrix gate legitimately
19// refuses (legal, present>=5) must not be reported as drift -- CONFLATING "REFUSED" WITH "DRIFTED"
20// IS THE SAME DEFECT THIS ESTATE KEEPS PAYING FOR.
21//
22// license_tier: ORIGINAL  expect_exit: 0
23import "nx_syscalls.nx"
24const CS_MAGIC_5381: i64 = 5381
25const CS_MAGIC_8192: i64 = 8192
26const CS_MAGIC_1024: i64 = 1024
27const CS_MAGIC_8000: i64 = 8000
28
29const CS_CAP: i64 = 262144
30
31func w(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
32func wn(v: i64) -> i64 {
  var m: i64 = v
  if m < 0 { w("-" as *u8); m = 0 - m }
  let t: *u8 = sys_mmap(24)
  var k: i64 = 0
  if m == 0 { t[0] = 48 as u8; k = 1 }
  while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
  let o: *u8 = sys_mmap(24)
  var i: i64 = 0
  while i < k { o[i] = t[k-1-i]; i = i + 1 }
  sys_write(1, o, k)
  return 0
44}
45func cs_read(path: *u8, buf: *u8, cap: i64) -> i64 {
  let fd: i64 = sys_openat_rd(path)
  if fd < 0 { return 0 - 1 }
  var tot: i64 = 0
  while tot < cap {
      let r: i64 = sys_read(fd, (buf as i64 + tot) as *u8, cap - tot)
      if r <= 0 { break }
      tot = tot + r
  }
  sys_close(fd)
  return tot
56}
57func cs_cat(dst: *u8, off: i64, s: *u8) -> i64 {
  var i: i64 = 0
  while s[i] != (0 as u8) { dst[off+i] = s[i]; i = i + 1 }
  return off + i
61}
62// MUST match the generators byte for byte. If this ever diverges the organ reports drift that is
63// not there -- a comparator and a writer that disagree on canonical form never converge.
64func cs_hash(b: *u8, n: i64) -> i64 {
  var h: i64 = CS_MAGIC_5381
  var i: i64 = 0
  while i < n { h = h * 33 + (b[i] as i64); i = i + 1 }
  if h < 0 { h = 0 - h }
  return h & 0xFFFFFFFFFFFF
70}
71func cs_find(b: *u8, n: i64, pat: *u8) -> i64 {
  var pl: i64 = 0
  while pat[pl] != (0 as u8) { pl = pl + 1 }
  var i: i64 = 0
  while i + pl <= n {
      var j: i64 = 0
      var ok: i64 = 1
      while j < pl { if b[i+j] != pat[j] { ok = 0; j = pl } else { j = j + 1 } }
      if ok == 1 { return i }
      i = i + 1
  }
  return 0 - 1
83}
84func cs_int_at(b: *u8, n: i64, off: i64) -> i64 {
  var i: i64 = off
  var v: i64 = 0
  while i < n {
      let c: i64 = b[i] as i64
      if c < 48 { break }
      if c > 57 { break }
      v = v * 10 + (c - 48)
      i = i + 1
  }
  return v
95}
96
97func cs_legacy_main(argc: i64, argv: *i64) -> i64 {
  var srcd: *u8 = "buildroot/knowledge/compare/" as *u8
  var docd: *u8 = "sites/nishifamily/compare/" as *u8
  if argc >= 2 { srcd = argv[1] as *u8 }
  if argc >= 3 { docd = argv[2] as *u8 }
102
  let lst: *u8 = sys_mmap(CS_MAGIC_8192)
  let lp: *u8 = sys_mmap(CS_MAGIC_1024)
  var lo: i64 = cs_cat(lp, 0, srcd)
  lo = cs_cat(lp, lo, "regen.list" as *u8)
  lp[lo] = 0 as u8
  let ln: i64 = cs_read(lp, lst, CS_MAGIC_8000)
  if ln <= 0 { w("FAIL: cannot read " as *u8); w(lp); w("\n" as *u8); sys_exit(2); return 2 }
110
  let sbuf: *u8 = sys_mmap(CS_CAP)
  let pbuf: *u8 = sys_mmap(CS_CAP)
  let path: *u8 = sys_mmap(CS_MAGIC_1024)
  var fresh: i64 = 0
  var stale: i64 = 0
  var unpub: i64 = 0
  var nostamp: i64 = 0
118
  w("=== NX-COMPARESTALE -- does each published page still match the source it came from? ===\n" as *u8)
  var p: i64 = 0
  while p < ln {
      var e: i64 = p
      while e < ln { if lst[e] == (10 as u8) { break } e = e + 1 }
      lst[e] = 0 as u8
      let dom: *u8 = (lst as i64 + p) as *u8
      p = e + 1
      if dom[0] != (0 as u8) {
      if dom[0] != (35 as u8) {
          // WHICH SOURCE DID THIS PAGE COME FROM? THE PAGE SAYS. WE DO NOT INFER.
          // This organ first guessed ".matrix, else .sota". llm and lang each have BOTH, and both
          // are published from their .sota -- so the guess hashed the wrong file and reported two
          // perfectly current pages as STALE. AN ARTEFACT MUST DECLARE ITS OWN PROVENANCE; A
          // CONSUMER THAT INFERS IT IS GUESSING, AND A GUESS THAT IS RIGHT 44 TIMES IN 46 STILL
          // LIES TWICE -- and lies in the direction of a false alarm, which is the expensive one.
          var o: i64 = cs_cat(path, 0, docd); o = cs_cat(path, o, dom); o = cs_cat(path, o, "/api.json" as *u8); path[o] = 0 as u8
          let pn: i64 = cs_read(path, pbuf, CS_CAP)
          if pn <= 0 {
              unpub = unpub + 1
              w("  UNPUBLISHED  " as *u8); w(dom); w("\n" as *u8)
          } else {
              let sf: i64 = cs_find(pbuf, pn, "\"source_file\":\"" as *u8)
              let ha: i64 = cs_find(pbuf, pn, "\"source_hash\":" as *u8)
              if sf < 0 {
                  nostamp = nostamp + 1
                  w("  NOSTAMP      " as *u8); w(dom); w("  (no source_file stamp -- republish once and it becomes decidable)\n" as *u8)
              } else { if ha < 0 {
                  nostamp = nostamp + 1
                  w("  NOSTAMP      " as *u8); w(dom); w("  (no source_hash stamp -- republish once)\n" as *u8)
              } else {
                  // lift the declared basename straight out of the page
                  var bo: i64 = cs_cat(path, 0, srcd)
                  var i: i64 = sf + 15
                  while i < pn { if pbuf[i] == (34 as u8) { break } path[bo] = pbuf[i]; bo = bo + 1; i = i + 1 }
                  path[bo] = 0 as u8
                  let sn: i64 = cs_read(path, sbuf, CS_CAP)
                  if sn <= 0 {
                      // The page names a source that is not there. That is a REAL defect -- either the
                      // source was deleted out from under a live page or the stamp is wrong -- so it
                      // must fail, and it must not be silently folded into "stale".
                      stale = stale + 1
                      w("  MISSINGSRC   " as *u8); w(dom); w("  page names a source that does not exist: " as *u8); w(path); w("\n" as *u8)
                  } else {
                      let want: i64 = cs_hash(sbuf, sn)
                      let got: i64 = cs_int_at(pbuf, pn, ha + 14)
                      if got == want { fresh = fresh + 1 } else {
                          stale = stale + 1
                          w("  STALE        " as *u8); w(dom)
                          w("  page=" as *u8); wn(got)
                          w(" source=" as *u8); wn(want)
                          w("  src=" as *u8); w(path)
                          w("\n" as *u8)
                      }
                  }
              } }
          }
      } }
  }
  w("COMPARESTALE fresh=" as *u8); wn(fresh)
  w(" stale=" as *u8); wn(stale)
  w(" unpublished=" as *u8); wn(unpub)
  w(" nostamp=" as *u8); wn(nostamp)
  if stale > 0 {
      w(" verdict=STALE (a published page no longer matches its source -- rerun nishi_compare_regen)\n" as *u8)
      sys_exit(1)
      return 1
  }
  w(" verdict=GREEN (every published page matches the source it was generated from)\n" as *u8)
  return 0
189}
190// Consumer and dependency artifacts must be nonempty; matching empty digests are not usable artifacts.
191// One explicitly selected edge. Expected hashes are caller assertions, not qualification receipts.
192// Literal occurrence binds an observation to consumer bytes; it cannot prove runtime reachability.
193import "nx_fio.nx"
194import "nx_sha256.nx"
195import "json_emit.nx"
196
197// Current Nishi ABI: each declared pointer/i64 field occupies one8-byte slot.
198const CD_ABI_WORD_BYTES: i64=8
199const CD_REGION_BYTES: i64=9*CD_ABI_WORD_BYTES
200const CD_FILE_BYTES: i64=6*CD_ABI_WORD_BYTES
201const CD_WRITER_BYTES: i64=5*CD_ABI_WORD_BYTES
202const CD_HASH_CHARS: i64=64
203const CD_I64_MAX: i64=9223372036854775807
204const CD_SCHEMA_FIELDS: i64=32
205const CD_JSON_INTEGER_CHARS: i64=21
206const CD_JSON_ESCAPE_EXPANSION: i64=6
207struct CsDependencyFile {
code: i64, bytes: i64, read_bytes: i64, reference_found: i64,
hash: *u8, stage: *u8,
210}
211func cd_len(s: *u8) -> i64 { var n: i64=0;while s[n]!=(0 as u8){n=n+1};return n }
212func cd_equal(a: *u8,b: *u8) -> i64 { var i: i64=0;while a[i]==b[i]{if a[i]==(0 as u8){return 1};i=i+1};return 0 }
213func cd_hash_valid(s: *u8) -> i64 {
if cd_len(s)!=CD_HASH_CHARS {return 0};var i: i64=0
while i<CD_HASH_CHARS {let c: i64=s[i] as i64;if (c<48||c>57)&&(c<97||c>102){return 0};i=i+1};return 1
216}
217func cd_file_init(o: *CsDependencyFile,hex: *u8) -> i64 {
o.code=0;o.bytes=0-1;o.read_bytes=0;o.reference_found=0;o.hash=hex;o.hash[0]=0;o.stage="not-observed";return 0
219}
220// KMP keeps literal matching linear and preserves partial matches across transport windows.
221func cd_prefix(token: *u8,n: i64,pi: *i64) -> i64 {
if n==0{return 0};pi[0]=0;var i: i64=1;var j: i64=0
while i<n {while j>0&&token[i]!=token[j]{j=pi[j-1]};if token[i]==token[j]{j=j+1};pi[i]=j;i=i+1};return 0
224}
225func cd_scan(path: *u8,token: *u8,o: *CsDependencyFile) -> i64 {
cd_file_init(o,o.hash)
let wn: i64=sha256_workspace_bytes();let tn: i64=cd_len(token)
if tn>(CD_I64_MAX-8)/8 {o.code=0-22;o.stage="reference-length";return o.code}
let ws: *u8=sys_mmap_shared(wn);let buffer: *u8=sys_mmap_shared(wn)
let region: *NxFileReadRegion=sys_mmap_shared(CD_REGION_BYTES) as *NxFileReadRegion
let pi: *i64=sys_mmap_shared((tn+1)*8) as *i64
if (ws as i64)<=0||(buffer as i64)<=0||(region as i64)<=0||(pi as i64)<=0 {
if (ws as i64)>0{sys_munmap(ws,wn)};if (buffer as i64)>0{sys_munmap(buffer,wn)}
if (region as i64)>0{sys_munmap(region as *u8,CD_REGION_BYTES)};if (pi as i64)>0{sys_munmap(pi as *u8,(tn+1)*8)}
o.code=0-12;o.stage="allocation";return o.code
}
fio_region_init(region);var rc: i64=sha256_init_workspace(ws,wn)
if rc==0 {rc=fio_region_open(path,region)} else {o.stage="hash-workspace"}
if rc==0 {
o.bytes=region.total
if region.total>CD_I64_MAX/SHA256_BITS_PER_BYTE {rc=0-75;o.stage="hash-bit-length-overflow"}
}
cd_prefix(token,tn,pi);var matched: i64=0
while rc==0 {
let got: i64=fio_region_next(region,buffer,wn)
if got<0{rc=got;break};if got==0{break}
sha256_update(ws as *Sha256,buffer,got)
var i: i64=0
while i<got&&tn>0 {
 while matched>0&&buffer[i]!=token[matched]{matched=pi[matched-1]}
 if buffer[i]==token[matched]{matched=matched+1}
 if matched==tn{o.reference_found=1;matched=pi[matched-1]};i=i+1
}
}
if rc==0 {
// Digest output reuses the transport workspace after the last checked read.
sha256_final(ws as *Sha256,buffer);let digits: *u8="0123456789abcdef";var i: i64=0
while i<SHA256_DIGEST_BYTES {let c: i64=buffer[i] as i64;o.hash[i*2]=digits[c/16];o.hash[i*2+1]=digits[c%16];i=i+1};o.hash[CD_HASH_CHARS]=0
}
fio_region_close(region);o.read_bytes=region.read_bytes
if rc==0&&region.code!=0{rc=region.code}
if cd_equal(o.stage,"not-observed")==1{o.stage=region.stage}
var released: i64=0
if sys_munmap(pi as *u8,(tn+1)*8)!=0{released=1};if sys_munmap(region as *u8,CD_REGION_BYTES)!=0{released=1}
if sys_munmap(buffer,wn)!=0{released=1};if sys_munmap(ws,wn)!=0{released=1}
if rc==0&&released==1{rc=0-5;o.stage="workspace-release"}
o.code=rc;return rc
268}
269func cd_file_state(o: *CsDependencyFile,expected: *u8) -> *u8 {
if o.code==(0-2)&&cd_equal(o.stage,"open")==1{return "MISSING"}
if o.code!=0{return "UNOBSERVABLE"}
if o.bytes==0{return "EMPTY"}
if cd_equal(o.hash,expected)==0{return "IDENTITY_MISMATCH"}
return "IDENTITY_MATCH"
275}
276func cd_pair(j: *JsonWriter,key: *u8,value: *u8) -> i64 {
if json_emit_key(j,key,cd_len(key))<0{return 0-1};return json_emit_string(j,value,cd_len(value))
278}
279func cd_num(j: *JsonWriter,key: *u8,value: i64) -> i64 {
if json_emit_key(j,key,cd_len(key))<0{return 0-1};return json_emit_int(j,value)
281}
282func cd_file_json(j: *JsonWriter,key: *u8,path: *u8,expected: *u8,o: *CsDependencyFile) -> i64 {
var rc: i64=json_emit_key(j,key,cd_len(key));rc=rc|json_begin_object(j)
rc=rc|cd_pair(j,"path",path);rc=rc|cd_pair(j,"expected_sha256_caller_assertion",expected)
rc=rc|cd_pair(j,"observed_sha256",o.hash);rc=rc|cd_pair(j,"state",cd_file_state(o,expected))
rc=rc|cd_num(j,"bytes",o.bytes);rc=rc|cd_num(j,"read_bytes",o.read_bytes)
rc=rc|cd_pair(j,"io_stage",o.stage);rc=rc|cd_num(j,"io_code",o.code)
return rc|json_end_object(j)
289}
290func cd_report(cp: *u8,ch: *u8,dp: *u8,dh: *u8,token: *u8,c: *CsDependencyFile,d: *CsDependencyFile,buffer: *u8,capacity: i64,j: *JsonWriter,prior: *u8) -> i64 {
j.buf=buffer;j.pos=0;j.cap=capacity;j.depth=0;j.prior=prior
var i: i64=0;while i<JE_MAX_DEPTH{prior[i]=0;i=i+1}
var matched: i64=0
if cd_equal(cd_file_state(c,ch),"IDENTITY_MATCH")==1&&cd_equal(cd_file_state(d,dh),"IDENTITY_MATCH")==1&&c.reference_found==1{matched=1}
var rc: i64=json_begin_object(j)
rc=rc|cd_pair(j,"schema","estate-dependency-observation/v1")
rc=rc|cd_pair(j,"overall_state","PARTIAL_OBSERVATION")
rc=rc|cd_pair(j,"artifact_contract","consumer and dependency must be nonempty")
rc=rc|cd_pair(j,"scope","one caller-selected edge; literal presence is not runtime reachability or complete dependency coverage")
rc=rc|cd_num(j,"observed_unix",sys_now_realtime_sec())
rc=rc|cd_pair(j,"expected_identity_authority","caller assertion; qualified receipt linkage unknown")
rc=rc|cd_pair(j,"reference_token",token);rc=rc|cd_num(j,"literal_reference_present",c.reference_found)
rc=rc|cd_file_json(j,"consumer",cp,ch,c);rc=rc|cd_file_json(j,"dependency",dp,dh,d)
rc=rc|cd_num(j,"eligible_selected_edges",1);rc=rc|cd_num(j,"identity_and_literal_matches",matched)
var unknown: i64=0;if cd_equal(cd_file_state(c,ch),"UNOBSERVABLE")==1||cd_equal(cd_file_state(d,dh),"UNOBSERVABLE")==1{unknown=1}
rc=rc|cd_num(j,"unobservable_selected_edges",unknown)
rc=rc|cd_num(j,"observed_selected_edge_failures",1-matched-unknown)
rc=rc|cd_pair(j,"source_build_qualification","UNKNOWN")
rc=rc|cd_pair(j,"runtime_compatibility","UNKNOWN")
rc=rc|cd_pair(j,"configured_selection","UNKNOWN")
rc=rc|cd_pair(j,"executed_identity","UNKNOWN")
rc=rc|cd_pair(j,"served_identity","UNKNOWN")
rc=rc|cd_pair(j,"complete_dependency_coverage","UNKNOWN")
rc=rc|cd_pair(j,"freshness_policy","none applied; no timestamp-derived desired version")
rc=rc|cd_pair(j,"next_action","resolve named identity/I/O failures; join qualified receipt and observe actual consumer execution and served dependency")
rc=rc|json_end_object(j);if rc<0{return 0-1};return j.pos
317}
318func cd_command(argc: i64,argv: *i64) -> i64 {
if argc!=7{w("{\"error\":\"usage: nx_comparestale dependency-check consumer-file expected-consumer-sha256 dependency-file expected-dependency-sha256 literal-reference\"}\n");return 2}
let cp: *u8=argv[2] as *u8;let ch: *u8=argv[3] as *u8;let dp: *u8=argv[4] as *u8;let dh: *u8=argv[5] as *u8;let token: *u8=argv[6] as *u8
if cd_hash_valid(ch)==0||cd_hash_valid(dh)==0||cd_len(token)==0||cd_len(cp)==0||cd_len(dp)==0{w("{\"error\":\"invalid dependency boundary\"}\n");return 2}
let total: i64=cd_len(cp)+cd_len(dp)+cd_len(token)
if total>(CD_I64_MAX/2){return 2}
// Worst-case escaped input plus fixed schema field/key/value envelope, not a file-size ceiling.
let fixed: i64=CD_SCHEMA_FIELDS*(CD_HASH_CHARS+CD_JSON_INTEGER_CHARS)*CD_JSON_ESCAPE_EXPANSION
if total>(CD_I64_MAX-fixed)/CD_JSON_ESCAPE_EXPANSION{return 2}
let cap: i64=total*CD_JSON_ESCAPE_EXPANSION+fixed
let buffer: *u8=sys_mmap_shared(cap);let j: *JsonWriter=sys_mmap_shared(CD_WRITER_BYTES) as *JsonWriter
let prior: *u8=sys_mmap_shared(JE_MAX_DEPTH);let c: *CsDependencyFile=sys_mmap_shared(CD_FILE_BYTES) as *CsDependencyFile
let d: *CsDependencyFile=sys_mmap_shared(CD_FILE_BYTES) as *CsDependencyFile
let hashes: *u8=sys_mmap_shared((CD_HASH_CHARS+1)*2)
var exitcode: i64=4
if (buffer as i64)>0&&(j as i64)>0&&(prior as i64)>0&&(c as i64)>0&&(d as i64)>0&&(hashes as i64)>0 {
cd_file_init(c,hashes);cd_file_init(d,hashes+CD_HASH_CHARS+1)
cd_scan(cp,token,c);cd_scan(dp,"",d)
let n: i64=cd_report(cp,ch,dp,dh,token,c,d,buffer,cap,j,prior)
if n>=0 {
 var wrote: i64=0;while wrote<n{let rc: i64=sys_write(1,buffer+wrote,n-wrote);if rc<=0{break};wrote=wrote+rc}
 if wrote==n{if sys_write(1,"\n",1)==1{
  exitcode=0
  if cd_equal(cd_file_state(c,ch),"UNOBSERVABLE")==1||cd_equal(cd_file_state(d,dh),"UNOBSERVABLE")==1{exitcode=4}else{if cd_equal(cd_file_state(c,ch),"IDENTITY_MATCH")==0||cd_equal(cd_file_state(d,dh),"IDENTITY_MATCH")==0||c.reference_found==0{exitcode=1}}
 }}
}
} else {w("{\"error\":\"dependency observation allocation failed\"}\n")}
if (hashes as i64)>0{if sys_munmap(hashes,(CD_HASH_CHARS+1)*2)!=0{exitcode=4}}
if (d as i64)>0{if sys_munmap(d as *u8,CD_FILE_BYTES)!=0{exitcode=4}}
if (c as i64)>0{if sys_munmap(c as *u8,CD_FILE_BYTES)!=0{exitcode=4}}
if (prior as i64)>0{if sys_munmap(prior,JE_MAX_DEPTH)!=0{exitcode=4}}
if (j as i64)>0{if sys_munmap(j as *u8,CD_WRITER_BYTES)!=0{exitcode=4}}
if (buffer as i64)>0{if sys_munmap(buffer,cap)!=0{exitcode=4}}
return exitcode
352}
nx_comparestale_dependency_t138.nx source

`nx_comparestale_dependency_t138.nx` source
