code wiki / _hdl_build / nx_nofloat_grammar_gate.nx
`nx_nofloat_grammar_gate.nx` source

↩ module page · 158 lines · 9686 B
1// nx_nofloat_grammar_gate.nx -- CAP-NF-GRAMMAR: the AFFORDABLE "landing" step the sovereign researcher
2// identified (knowledge/research/2026-06-23-nofloat-affordable-land-roadmap.md, R1, grounded in TinyStories
3// arXiv 2305.07759 + Textbooks arXiv 2306.11644: a tiny model is COHERENT iff the DOMAIN is constrained).
4// Instead of memorizing one fixed sentence, the model learns a small GRAMMAR and GENERATES coherent NOVEL
5// sentences. Grammar: a stream of [ARTICLE, NOUN, VERB] triples. Vocab 8: articles {the=0,a=1}, nouns
6// {cat=2,dog=3,bird=4}, verbs {ran=5,sat=6,ate=7}. EVERY training stream is fresh-random members -> the model
7// cannot memorize sentences; it must learn the CATEGORY GRAMMAR (article->noun->verb->article ...).
8//   T1 GENERATES coherent grammar: from a seed, free argmax-decode produces a stream whose every token is in
9//      the grammatically-correct category for its position (>= 90%, vs ~33% chance) = coherent NOVEL generation.
10//   T2 learned the grammar on HELD-OUT streams: teacher-forced next-token category accuracy >= 90% (>> chance).
11// Pure integer Q16, attention-only block, 3 clm_fwd sites in separate helpers (nx_cc-lean). Sovereign.
12// HONEST: a CONSTRAINED grammar domain (the affordable landing per the research), not open prose. expect_exit: 0
13import "nx_nofloat_autograd.nx"
14import "nx_syscalls.nx"
15import "nx_gate_emit_lib.nx"
16import "nx_gate_verdict.nx"
17const Q16: i64 = 65536
18
19
20func dini(a: *i64, n: i64, sd: i64) -> i64 { var i: i64=0; while i<n { a[i]=(((i*7+sd*13+1)%11)-5)*13107; i=i+1 } return 0 }
21func lcg(st: *i64) -> i64 { st[0]=(st[0]*1103515245 + 12345) & 2147483647; return (st[0] >> 15) }
22func cat_of(tok: i64) -> i64 { if tok<2 { return 0 } if tok<5 { return 1 } return 2 }
23// fresh-random grammatical stream: position cat = i%3; article 0..1, noun 2..4, verb 5..7. tgt = next token.
24func make_stream(S: *i64, tgt: *i64, P: i64, st: *i64) -> i64 {
  var i: i64=0
  while i<P { let c: i64=i%3; if c==0 { S[i]=lcg(st)%2 } if c==1 { S[i]=2+lcg(st)%3 } if c==2 { S[i]=5+lcg(st)%3 } i=i+1 }
  i=0; while i<P-1 { tgt[i]=S[i+1]; i=i+1 }
  tgt[P-1]=S[0]
  return 0
30}
31
32func clm_fwd(tape: *i64, vals: *i64, st: *i64, W: *i64, ids: *i64, tgt: *i64, T: i64, dm: i64, V: i64, scale: i64, leaves: *i64) -> i64 {
  let E: *i64=W[0] as *i64; let Wq: *i64=W[1] as *i64; let Wk: *i64=W[2] as *i64; let Wv: *i64=W[3] as *i64; let Wo: *i64=W[4] as *i64; let Wlm: *i64=W[5] as *i64
  st[0]=0; st[1]=0
  let nE: i64=nfa_leaf(tape,vals,st,V,dm,E,0)
  let nWq: i64=nfa_leaf(tape,vals,st,dm,dm,Wq,0)
  let nWk: i64=nfa_leaf(tape,vals,st,dm,dm,Wk,0)
  let nWv: i64=nfa_leaf(tape,vals,st,dm,dm,Wv,0)
  let nWo: i64=nfa_leaf(tape,vals,st,dm,dm,Wo,0)
  let nWlm: i64=nfa_leaf(tape,vals,st,dm,V,Wlm,0)
  let nX: i64=nfa_embed(tape,vals,st,nE,ids,T)
  let nXn: i64=nfa_rmsnorm_rows(tape,vals,st,nX)
  let nQ: i64=nfa_matmul(tape,vals,st,nXn,nWq)
  let nK: i64=nfa_matmul(tape,vals,st,nXn,nWk)
  let nV: i64=nfa_matmul(tape,vals,st,nXn,nWv)
  let nQr: i64=nfa_rope(tape,vals,st,nQ)
  let nKr: i64=nfa_rope(tape,vals,st,nK)
  let nS: i64=nfa_matmul_nt(tape,vals,st,nQr,nKr)
  let nSs: i64=nfa_cmul(tape,vals,st,nS,scale)
  let nA: i64=nfa_softmax_rows(tape,vals,st,nSs,1)
  let nO: i64=nfa_matmul(tape,vals,st,nA,nV)
  let nOp: i64=nfa_matmul(tape,vals,st,nO,nWo)
  let nH: i64=nfa_vadd(tape,vals,st,nX,nOp)
  let nHn: i64=nfa_rmsnorm_rows(tape,vals,st,nH)
  let nLg: i64=nfa_matmul(tape,vals,st,nHn,nWlm)
  let nLoss: i64=nfa_softce_rows(tape,vals,st,nLg,tgt)
  leaves[0]=nE; leaves[1]=nWq; leaves[2]=nWk; leaves[3]=nWv; leaves[4]=nWo; leaves[5]=nWlm; leaves[6]=nLg
  return nLoss
59}
60func step_all(tape: *i64, grads: *i64, W: *i64, WN: *i64, leaves: *i64, nW: i64, lr: i64, clip: i64, gb: *i64) -> i64 {
  var i: i64=0
  while i<nW { let ar: *i64=W[i] as *i64; let cn: i64=WN[i]; let nd: i64=leaves[i]; var c: i64=0; while c<cn { var g: i64=nfa_grad(tape,grads,nd,c); if g>clip{g=clip} if g<0-clip{g=0-clip} gb[c]=g; c=c+1 } nfa_sgd(ar,gb,cn,lr); i=i+1 }
  return 0
64}
65func amx(tape: *i64, vals: *i64, logn: i64, r: i64, V: i64) -> i64 { let o: i64=tape[7*logn+5]; var b: i64=0; var bv: i64=vals[o+r*V]; var j: i64=1; while j<V { if vals[o+r*V+j]>bv { bv=vals[o+r*V+j]; b=j } j=j+1 } return b }
66// site 1: train on fresh-random grammatical streams
67func do_train(tape: *i64, vals: *i64, grads: *i64, st: *i64, W: *i64, WN: *i64, S: *i64, tgt: *i64, P: i64, dm: i64, V: i64, scale: i64, leaves: *i64, gb: *i64, steps: i64, sdat: *i64) -> i64 {
  var ep: i64=0
  while ep < steps { make_stream(S,tgt,P,sdat); let nl: i64=clm_fwd(tape,vals,st,W,S,tgt,P-1,dm,V,scale,leaves); nfa_backward(tape,vals,grads,st[0],nl); step_all(tape,grads,W,WN,leaves,6,6554,262144,gb); ep=ep+1 }
  return 0
71}
72// site 2: teacher-forced next-CATEGORY accuracy over N held-out fresh streams (position i predicts cat (i+1)%3)
73func eval_tf(tape: *i64, vals: *i64, st: *i64, W: *i64, S: *i64, tgt: *i64, P: i64, dm: i64, V: i64, scale: i64, leaves: *i64, N: i64, sdat: *i64, totp: *i64) -> i64 {
  var ok: i64=0; var tp: i64=0; var e: i64=0
  while e<N {
      make_stream(S,tgt,P,sdat)
      let nl: i64=clm_fwd(tape,vals,st,W,S,tgt,P-1,dm,V,scale,leaves)
      var i: i64=0
      while i<P-1 { if cat_of(amx(tape,vals,leaves[6],i,V))==((i+1)%3) { ok=ok+1 } tp=tp+1; i=i+1 }
      e=e+1
  }
  totp[0]=tp
  return ok
84}
85// site 3: free generation coherence -- fixed-T decode from a seed article; count tokens in the grammatically-correct category
86func gen_coherent(tape: *i64, vals: *i64, st: *i64, W: *i64, gen: *i64, P: i64, dm: i64, V: i64, scale: i64, leaves: *i64, outtok: *i64) -> i64 {
  var z: i64=0; while z<P { gen[z]=0; z=z+1 }    // 0 = "the" (valid article seed)
  let Tf: i64 = P-1
  var step: i64=1
  while step <= Tf {
      let nl: i64=clm_fwd(tape,vals,st,W,gen,gen,Tf,dm,V,scale,leaves)
      gen[step]=amx(tape,vals,leaves[6],step-1,V)
      step=step+1
  }
  var ok: i64=0; var i: i64=1
  while i<P { if cat_of(gen[i])==(i%3) { ok=ok+1 } outtok[i]=gen[i]; i=i+1 }
  outtok[0]=gen[0]
  return ok
99}
100
101func main() -> i64 {
  g_puts("nx_nofloat_grammar gate (constrained-grammar -> coherent NOVEL generation; the affordable land)\n" as *u8)
  var pass: i64=0; var total: i64=0
  let V: i64=8; let P: i64=12; let dm: i64=24; let scale: i64=13377
  let tape:  *i64 = sys_mmap(512*7*8) as *i64
  let vals:  *i64 = sys_mmap(65536*8) as *i64
  let grads: *i64 = sys_mmap(65536*8) as *i64
  let st:    *i64 = sys_mmap(2*8) as *i64
  let nW: i64=6
  let W: *i64 = sys_mmap(nW*8) as *i64
  let WN: *i64 = sys_mmap(nW*8) as *i64
  WN[0]=V*dm; WN[1]=dm*dm; WN[2]=dm*dm; WN[3]=dm*dm; WN[4]=dm*dm; WN[5]=dm*V
  var wi: i64=0
  while wi<nW { let a: *i64=sys_mmap(WN[wi]*8) as *i64; dini(a,WN[wi],wi+1); W[wi]=a as i64; wi=wi+1 }
  let leaves: *i64 = sys_mmap(8*8) as *i64
  let gbuf: *i64 = sys_mmap(4096*8) as *i64
  let S: *i64 = sys_mmap(P*8) as *i64; let tgt: *i64 = sys_mmap(P*8) as *i64
  let totp: *i64 = sys_mmap(8) as *i64; let sdat: *i64 = sys_mmap(8) as *i64
  let outtok: *i64 = sys_mmap(P*8) as *i64
120
  sdat[0]=12345
  do_train(tape,vals,grads,st,W,WN,S,tgt,P,dm,V,scale,leaves,gbuf,30000,sdat)
123
  // T2: held-out grammar accuracy (teacher-forced next-category)
  sdat[0]=55554444
  let ok: i64 = eval_tf(tape,vals,st,W,S,tgt,P,dm,V,scale,leaves,150,sdat,totp)
  let tp: i64 = totp[0]
  // T1: free generation coherence (the actual "landing")
  let gen: *i64 = sys_mmap(P*8) as *i64
  let gok: i64 = gen_coherent(tape,vals,st,W,gen,P,dm,V,scale,leaves,outtok)
  g_puts("  [measure] held-out next-category acc=" as *u8); g_pn(ok); g_puts("/" as *u8); g_pn(tp); g_puts("   free-gen coherent tokens=" as *u8); g_pn(gok); g_puts("/" as *u8); g_pn(P-1); g_puts("  (chance~33%)\n" as *u8)
  // print the generated sentence as category letters (A=article,N=noun,V=verb) + token ids
  g_puts("  generated grammar: " as *u8)
  var pi: i64=0
  while pi<P { let c: i64=cat_of(outtok[pi]); let lab: *u8=sys_mmap(2); if c==0 { lab[0]=65 as u8 } if c==1 { lab[0]=78 as u8 } if c==2 { lab[0]=86 as u8 } sys_write(1,lab,1); pi=pi+1 }
  g_puts("  (expected A N V A N V ...)\n" as *u8)
137
  var t1: i64=0; if gok*100 >= (P-1)*90 { t1=1 }
  pass=pass+g_check("T1: GENERATES coherent grammar -- free-decoded stream is category-valid >= 90% (novel coherent gen)" as *u8, t1); total=total+1
  var t2: i64=0; if ok*100 >= tp*90 { t2=1 }
  pass=pass+g_check("T2: learned the grammar on HELD-OUT streams (teacher-forced next-category >= 90% >> 33% chance)" as *u8, t2); total=total+1
142
  var okall: i64=0; if pass==total { okall=1 }
  if okall==1 {
  let logf: i64 = sys_openat_append("knowledge/status/nofloat_grammar.log" as *u8, 420)
  if logf >= 0 { let w0: i64=sys_write(logf,"NOFLOATGRAMMAR constrained-grammar coherent generation measured\n" as *u8,63); sys_close(logf) }
  }
  g_puts("---- grammar gate: passed " as *u8); g_pn(pass); g_puts(" / " as *u8); g_pn(total); g_puts(" ----\n" as *u8)
  // MIGRATED onto nx_gate_verdict by nx_gate_dry_apply (D001, minimal form): every check
  // row above is untouched, so the PASS/FAIL vector cannot change; only the hand-rolled
  // verdict emission is replaced by the ONE shared base class. Proven by nx_gate_migrate verify.
  let ctr__dry: *i64 = gv_ctr()
  ctr__dry[0] = pass
  ctr__dry[1] = total
  let rc__dry: i64 = gv_verdict("NOFLOAT-GRAMMAR-GATE" as *u8, ctr__dry, "the affordable landing: a no-float LM learns a grammar and GENERATES coherent novel sentences)" as *u8)
  sys_exit(rc__dry)
  return rc__dry
158}
nx_nofloat_grammar_gate.nx source

`nx_nofloat_grammar_gate.nx` source
