code wiki / (root) / nx_research_synth_qwen.nx
`nx_research_synth_qwen.nx` source

↩ module page · 369 lines · 15708 B
1// nx_research_synth_qwen.nx -- S1 of the Tesla growth ladder: SOVEREIGN GROUNDED SYNTHESIS.
2// Given a question + N retrieved source passages, the pretrained sovereign Qwen (nsv_generate, in-process,
3// no HTTP, $0, integer-deterministic) generates a MULTI-SENTENCE answer that cites each claim [n] back to a
4// numbered source. The organ's RESONANT property (Tesla law #1) is not the generation -- it is the
5// DETERMINISTIC GROUNDING CHECK that runs at $0 with NO model: every content word of the answer must trace
6// to a cited source, and every citation must be in range. A hallucinated claim (word absent from all sources)
7// or a fabricated citation ([5] when 2 sources exist) is caught mechanically. This is the anti-hallucination
8// discipline FARS lacked (its own audit: 5.7% hallucinated citations, 7.4% fabricated results).
9//
10// Composes: nx_nofloat_serve_core (nsv_init/nsv_generate). Model-call pattern mirrors nx_qwen_extract (proven).
11// The grounding functions (rsq_ground_report / rsq_parse_cites / rsq_is_refusal) need NO model -> the gate
12// proves the mechanism in miniature (law #3), and the generative path is marker-gated for NAS measurement.
13// NEXT (S2): each ungrounded claim -> nx_evidence_verify_gate for external confirm/discredit. license_tier: ORIGINAL
14import "nx_syscalls.nx"
15import "nx_nofloat_serve_core.nx"
16import "nx_text_sentences.nx"
17
18const RSQ_MODEL: *u8 = "/home/elderwesto/nx_stage/nx_real_model.gguf"
19const RSQ_SRCCAP: i64 = 600       // per-source byte cap inside the prompt
20const RSQ_MAXPROMPT: i64 = 4000   // hard byte guard (<< NSV_MAXIN=16384 tok-scratch; ~1600 tok < NSV_MAXT=2048)
21const RSQ_MAXNEW: i64 = 64        // synthesis needs a 2-4 sentence answer (extraction used 16)
22const RSQ_MODE: i64 = 0           // 0 = i32 lossless (faithful default)
23const RSQ_MINWORD: i64 = 5        // a content word is len>=5 (filters most function words)
24
25static g_rsq_ready: i64           // 0 until nsv_init succeeds (BSS-zero, safe to read pre-init)
26static g_rsq_pr: *u8              // preallocated prompt buffer
27static g_rsq_out: *u8             // model output buffer
28static g_rsq_meta: *i64           // nsv meta block
29static g_rsq_gp: *i64             // nsv generate-params block
30static g_rsq_ctp: *i64            // one-time scratch for txt_parse_cite (Consolidation STEP C; lazy-init, NOT mmap-per-call)
31static g_rsq_ctp_init: i64
32
33// ---- small byte helpers (declared before use; fwd static-ref would miscompile) ----
34func rsq_len(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
35func rsq_lc(c: i64) -> i64 { if c >= 65 { if c <= 90 { return c + 32 } } return c }
36func rsq_is_alnum(c: i64) -> i64 {
  if c >= 48 { if c <= 57 { return 1 } }
  if c >= 65 { if c <= 90 { return 1 } }
  if c >= 97 { if c <= 122 { return 1 } }
  return 0
41}
42func rsq_cat(dst: *u8, off: i64, s: *u8, n: i64, cap: i64) -> i64 {
  var i: i64 = 0; var o: i64 = off
  while i < n { if o < cap { dst[o] = s[i]; o = o + 1 } i = i + 1 }
  return o
46}
47func rsq_cats(dst: *u8, off: i64, s: *u8, cap: i64) -> i64 {
  var i: i64 = 0; var o: i64 = off
  while s[i] != (0 as u8) { if o < cap { dst[o] = s[i]; o = o + 1 } i = i + 1 }
  return o
51}
52func rsq_catd(dst: *u8, off: i64, d: i64, cap: i64) -> i64 {
  var o: i64 = off
  if o < cap { dst[o] = (48 + d) as u8; o = o + 1 }
  return o
56}
57
58// case-insensitive: does the (ptr,wlen) word occur anywhere in buf[0,blen)?
59func rsq_ci_match_at(buf: *u8, pos: i64, blen: i64, word: *u8, wlen: i64) -> i64 {
  var k: i64 = 0
  while k < wlen {
      if pos + k >= blen { return 0 }
      if rsq_lc(buf[pos+k] as i64 & 0xff) != rsq_lc(word[k] as i64 & 0xff) { return 0 }
      k = k + 1
  }
  return 1
67}
68func rsq_ci_contains(buf: *u8, blen: i64, word: *u8, wlen: i64) -> i64 {
  if wlen == 0 { return 0 }
  var i: i64 = 0
  while i + wlen <= blen {
      if rsq_ci_match_at(buf, i, blen, word, wlen) == 1 { return 1 }
      i = i + 1
  }
  return 0
76}
77// end of the alnum run starting at i (i points at an alnum byte); always returns > i
78func rsq_word_end(ans: *u8, i: i64, alen: i64) -> i64 {
  var j: i64 = i
  var go: i64 = 1
  while go == 1 {
      go = 0
      if j < alen { if rsq_is_alnum(ans[j] as i64 & 0xff) == 1 { j = j + 1; go = 1 } }
  }
  return j
86}
87// case-insensitive equality of answer span [off,off+wlen) to a NUL-terminated word
88func rsq_eqword(ans: *u8, off: i64, wlen: i64, w: *u8) -> i64 {
  var k: i64 = 0
  while k < wlen {
      if w[k] == (0 as u8) { return 0 }
      if rsq_lc(ans[off+k] as i64 & 0xff) != rsq_lc(w[k] as i64 & 0xff) { return 0 }
      k = k + 1
  }
  if w[wlen] != (0 as u8) { return 0 }
  return 1
97}
98// a small len>=5 function-word list so the hallucination signal is about CONTENT, not grammar
99func rsq_is_stop(ans: *u8, off: i64, wlen: i64) -> i64 {
  if rsq_eqword(ans, off, wlen, "which" as *u8) == 1 { return 1 }
  if rsq_eqword(ans, off, wlen, "there" as *u8) == 1 { return 1 }
  if rsq_eqword(ans, off, wlen, "these" as *u8) == 1 { return 1 }
  if rsq_eqword(ans, off, wlen, "those" as *u8) == 1 { return 1 }
  if rsq_eqword(ans, off, wlen, "their" as *u8) == 1 { return 1 }
  if rsq_eqword(ans, off, wlen, "would" as *u8) == 1 { return 1 }
  if rsq_eqword(ans, off, wlen, "could" as *u8) == 1 { return 1 }
  if rsq_eqword(ans, off, wlen, "should" as *u8) == 1 { return 1 }
  if rsq_eqword(ans, off, wlen, "about" as *u8) == 1 { return 1 }
  if rsq_eqword(ans, off, wlen, "after" as *u8) == 1 { return 1 }
  if rsq_eqword(ans, off, wlen, "other" as *u8) == 1 { return 1 }
  if rsq_eqword(ans, off, wlen, "being" as *u8) == 1 { return 1 }
  if rsq_eqword(ans, off, wlen, "where" as *u8) == 1 { return 1 }
  if rsq_eqword(ans, off, wlen, "while" as *u8) == 1 { return 1 }
  if rsq_eqword(ans, off, wlen, "first" as *u8) == 1 { return 1 }
  return 0
116}
117// is a content word (ptr at ans+off, wlen) present in ANY of the nsrc sources?
118func rsq_present_in_sources(ans: *u8, off: i64, wlen: i64, srcv: *i64, nsrc: i64) -> i64 {
  var k: i64 = 0
  while k < nsrc {
      let sp: *u8 = srcv[k] as *u8
      let sl: i64 = rsq_len(sp)
      if rsq_ci_contains(sp, sl, (ans as i64 + off) as *u8, wlen) == 1 { return 1 }
      k = k + 1
  }
  return 0
127}
128// THE grounding report: out[0]=content words, out[1]=hallucinated (present in NO source). returns halluc.
129func rsq_ground_report(ans: *u8, alen: i64, srcv: *i64, nsrc: i64, out: *i64) -> i64 {
  var content: i64 = 0
  var halluc: i64 = 0
  var i: i64 = 0
  while i < alen {
      let c: i64 = ans[i] as i64 & 0xff
      if rsq_is_alnum(c) == 1 {
          let je: i64 = rsq_word_end(ans, i, alen)
          let wlen: i64 = je - i
          if wlen >= RSQ_MINWORD {
              if rsq_is_stop(ans, i, wlen) == 0 {
                  content = content + 1
                  if rsq_present_in_sources(ans, i, wlen, srcv, nsrc) == 0 { halluc = halluc + 1 }
              }
          }
          i = je
      } else {
          i = i + 1
      }
  }
  out[0] = content
  out[1] = halluc
  return halluc
152}
153// parse [n] citations: out[0]=count out[1]=in-range out[2]=out-of-range. returns count.
154func rsq_parse_cites(ans: *u8, alen: i64, cites: *i64, maxc: i64, srcrange: i64, out: *i64) -> i64 {
  var nc: i64 = 0; var inr: i64 = 0; var outr: i64 = 0
  var i: i64 = 0
  while i < alen {
      if (ans[i] as i64 & 0xff) == 91 {          // '['
          var j: i64 = i + 1
          var val: i64 = 0
          var got: i64 = 0
          var go: i64 = 1
          while go == 1 {
              go = 0
              if j < alen {
                  let d: i64 = ans[j] as i64 & 0xff
                  if d >= 48 { if d <= 57 { val = val*10 + (d-48); got = got + 1; j = j + 1; go = 1 } }
              }
          }
          if got > 0 { if j < alen { if (ans[j] as i64 & 0xff) == 93 {   // ']'
              if nc < maxc { cites[nc] = val; nc = nc + 1 }
              if val >= 1 { if val <= srcrange { inr = inr + 1 } else { outr = outr + 1 } } else { outr = outr + 1 }
              i = j
          } } }
      }
      i = i + 1
  }
  out[0] = nc; out[1] = inr; out[2] = outr
  return nc
180}
181// did the model decline (grounded refusal, the DeepResearcher honesty-refusal behavior)?
182func rsq_is_refusal(ans: *u8, alen: i64) -> i64 {
  if rsq_ci_contains(ans, alen, "not supported" as *u8, 13) == 1 { return 1 }
  return 0
185}
186// COMBINED grounded verdict (the complete S1 check). out[0]=content out[1]=halluc out[2]=ncites
187// out[3]=inrange out[4]=outrange. Returns 1 GROUNDED iff halluc<=htol AND no out-of-range citation
188// AND (has content OR is an explicit refusal). Catches BOTH hallucinated claims and fabricated citations
189// (the real 0.5B over-cites -- Tier-2 emitted a [3] with 2 sources; this rejects it). Not a hot path.
190func rsq_verdict(ans: *u8, alen: i64, srcv: *i64, nsrc: i64, htol: i64, out: *i64) -> i64 {
  let gr: *i64 = sys_mmap(64) as *i64
  let cr: *i64 = sys_mmap(64) as *i64
  let cbuf: *i64 = sys_mmap(256) as *i64
  rsq_ground_report(ans, alen, srcv, nsrc, gr)
  rsq_parse_cites(ans, alen, cbuf, 24, nsrc, cr)
  out[0] = gr[0]
  out[1] = gr[1]
  out[2] = cr[0]
  out[3] = cr[1]
  out[4] = cr[2]
  var grounded: i64 = 1
  if gr[1] > htol { grounded = 0 }
  if cr[2] > 0 { grounded = 0 }
  if gr[0] == 0 { if rsq_is_refusal(ans, alen) == 0 { grounded = 0 } }
  return grounded
206}
207
208// ---- per-claim attribution (the S1->S2 bridge: each sentence-claim independently checkable) ----
209// first IN-RANGE [c] citation in sentence [s,e) (1..nsrc), or 0 if none; outp[0]=1 if any OUT-OF-RANGE cite.
210func rsq_sentence_cite(ans: *u8, s: i64, e: i64, nsrc: i64, outp: *i64) -> i64 {
  if g_rsq_ctp_init == 0 { g_rsq_ctp = sys_mmap(32) as *i64; g_rsq_ctp_init = 1 }
  let tp: *i64 = g_rsq_ctp
  var c: i64 = 0
  var oor: i64 = 0
  var i: i64 = s
  while i < e {
      var adv: i64 = 1
      if (ans[i] as i64 & 0xff) == 91 {                 // '['
          let endp: i64 = txt_parse_cite(ans, e, i, tp)     // shared canonical parser (Consolidation STEP C)
          if tp[1] == 1 {                                   // complete [n] token
              let val: i64 = tp[0]
              if val >= 1 { if val <= nsrc { if c == 0 { c = val } } else { oor = 1 } } else { oor = 1 }
              i = endp
              adv = 0
          }
      }
      if adv == 1 { i = i + 1 }
  }
  outp[0] = oor
  return c
231}
232// does any content word of ans[s,e) appear in the SPECIFICALLY cited source srcv[c-1]? (mis-citation catcher)
233func rsq_span_overlaps_src(ans: *u8, s: i64, e: i64, srcv: *i64, c: i64) -> i64 {
  if c < 1 { return 0 }
  let sp: *u8 = srcv[c-1] as *u8
  let sl: i64 = rsq_len(sp)
  var i: i64 = s
  while i < e {
      if rsq_is_alnum(ans[i] as i64 & 0xff) == 1 {
          let je: i64 = rsq_word_end(ans, i, e)
          let wlen: i64 = je - i
          if wlen >= RSQ_MINWORD { if rsq_is_stop(ans, i, wlen) == 0 {
              if rsq_ci_contains(sp, sl, (ans as i64 + i) as *u8, wlen) == 1 { return 1 }
          } }
          i = je
      } else { i = i + 1 }
  }
  return 0
249}
250// per-claim (sentence) verdict. out[0]=nclaims out[1]=grounded out[2]=ungrounded. A claim is grounded iff it
251// has an in-range citation AND >=1 content word present in THAT cited source. returns nclaims.
252func rsq_claim_verdict(ans: *u8, alen: i64, srcv: *i64, nsrc: i64, out: *i64) -> i64 {
  var nclaims: i64 = 0
  var ng: i64 = 0
  var nu: i64 = 0
  let op: *i64 = sys_mmap(32) as *i64
  var s: i64 = 0
  while s < alen {
      let e: i64 = txt_sentence_end(ans, s, alen)
      var content: i64 = 0
      var i: i64 = s
      while i < e {
          if rsq_is_alnum(ans[i] as i64 & 0xff) == 1 {
              let je: i64 = rsq_word_end(ans, i, e)
              let wlen: i64 = je - i
              if wlen >= RSQ_MINWORD { if rsq_is_stop(ans, i, wlen) == 0 { content = content + 1 } }
              i = je
          } else { i = i + 1 }
      }
      if content > 0 {
          nclaims = nclaims + 1
          let c: i64 = rsq_sentence_cite(ans, s, e, nsrc, op)
          var grounded: i64 = 0
          if c >= 1 { if op[0] == 0 { if rsq_span_overlaps_src(ans, s, e, srcv, c) == 1 { grounded = 1 } } }
          if grounded == 1 { ng = ng + 1 } else { nu = nu + 1 }
      }
      s = e + 1
  }
  out[0] = nclaims
  out[1] = ng
  out[2] = nu
  return nclaims
283}
284// build the synthesis prompt: instruction + numbered sources + question. returns byte length.
285func rsq_build_prompt(q: *u8, srcv: *i64, nsrc: i64, pr: *u8, cap: i64) -> i64 {
  var po: i64 = 0
  po = rsq_cats(pr, po, "Answer the question using ONLY the numbered sources. After each sentence cite the source in brackets like [1]. If the sources do not contain the answer, reply exactly: NOT SUPPORTED.\n\n" as *u8, cap)
  var k: i64 = 0
  while k < nsrc {
      let sp: *u8 = srcv[k] as *u8
      po = rsq_cat(pr, po, "[" as *u8, 1, cap)
      po = rsq_catd(pr, po, k + 1, cap)
      po = rsq_cats(pr, po, "] " as *u8, cap)
      var sl: i64 = rsq_len(sp)
      if sl > RSQ_SRCCAP { sl = RSQ_SRCCAP }
      po = rsq_cat(pr, po, sp, sl, cap)
      po = rsq_cats(pr, po, "\n" as *u8, cap)
      k = k + 1
  }
  po = rsq_cats(pr, po, "\nQuestion: " as *u8, cap)
  var qc: i64 = rsq_len(q)
  if qc > 200 { qc = 200 }
  po = rsq_cat(pr, po, q, qc, cap)
  po = rsq_cats(pr, po, "\nAnswer:" as *u8, cap)
  return po
306}
307
308// ---- model boot (idempotent; path override like nx_qwen_extract) ----
309func rsq_load() -> i64 {
  if g_rsq_ready == 1 { return 0 }
  var mp: *u8 = RSQ_MODEL
  let pfd: i64 = sys_openat_rd("knowledge/index/qwen_model_path.txt" as *u8)
  if pfd >= 0 {
      let pb: *u8 = sys_mmap(512)
      let pn: i64 = sys_read(pfd, pb, 511)
      sys_close(pfd)
      var pe: i64 = pn
      var trim: i64 = 1
      while trim == 1 {
          trim = 0
          if pe > 0 {
              let c: i64 = pb[pe-1] as i64
              if c == 10 { pe = pe - 1; trim = 1 } else { if c == 13 { pe = pe - 1; trim = 1 } else { if c == 32 { pe = pe - 1; trim = 1 } } }
          }
      }
      if pe > 0 { pb[pe] = 0 as u8; mp = pb }
  }
  let rc: i64 = nsv_init(mp)
  if rc != 0 { return rc }
  g_rsq_pr = sys_mmap(8192)
  g_rsq_out = sys_mmap(4096)
  g_rsq_meta = sys_mmap(8*8) as *i64
  g_rsq_gp = sys_mmap(12*8) as *i64
  g_rsq_ready = 1
  return 0
336}
337func rsq_ready() -> i64 { return g_rsq_ready }
338
339// generate a grounded synthesis for q over nsrc sources; first-block answer into ans (NUL-term).
340// returns answer byte length, or <=0 on not-ready / too-long / empty.
341func rsq_synth(q: *u8, srcv: *i64, nsrc: i64, ans: *u8) -> i64 {
  if g_rsq_ready == 0 { return 0 }
  let pr: *u8 = g_rsq_pr
  let po: i64 = rsq_build_prompt(q, srcv, nsrc, pr, 8192)
  if po > RSQ_MAXPROMPT { return 0 }
  let out: *u8 = g_rsq_out
  let gp: *i64 = g_rsq_gp
  gp[0] = pr as i64
  gp[1] = po
  gp[2] = RSQ_MAXNEW
  gp[3] = RSQ_MODE
  gp[4] = out as i64
  gp[5] = 4096
  gp[6] = g_rsq_meta as i64
  gp[7] = 0 - 1
  gp[8] = 0
  gp[9] = 0
  gp[10] = 0
  gp[11] = 0
  let tlen: i64 = nsv_generate(gp)
  if tlen <= 0 { return 0 }
  var st: i64 = 0
  if (out[0] as i64) == 32 { st = 1 }              // trim leading space
  var w: i64 = 0
  var k: i64 = st
  while k < tlen { if w < 1000 { ans[w] = out[k]; w = w + 1 } k = k + 1 }
  ans[w] = 0 as u8
  return w
369}
nx_research_synth_qwen.nx source

`nx_research_synth_qwen.nx` source
