code wiki / _hdl_build / nx_usage_artifact_t173.nx
nx_usage_artifact_t173.nx source
↩ module page · 129 lines · 6838 B
1// nx_usage_artifact_t173.nx -- Extracts and validates structured request artifacts from binary data in the Nishi sovereign ecosystem.
2import "nx_claude_harvest.nx"
3
4// Native request artifacts are a closed observation stream, not arbitrary NDJSON.
5// Projection shares the legacy request validator and preserves unknown attribution.
6func chua_key(q:*u8,s:i64,n:i64,names:*u8)->i64 {
7 var p:i64=0;var start:i64=0;var ix:i64=0
8 while 1==1{if names[p]==(124 as u8)||names[p]==(0 as u8){
9 if p-start==n{var j:i64=0;while j<n{if q[s+j]!=names[start+j]{break};j=j+1};if j==n{return ix}}
10 if names[p]==(0 as u8){return -1};ix=ix+1;start=p+1
11 };p=p+1}
12 return -1
13}
14func chua_fields(q:*u8,n:i64,names:*u8,count:i64,f:*i64,t:*NxJsonTok,r:*i64)->i64 {
15 var j:i64=0;while j<count*3{f[j]=0;j=j+1}
16 t.src=q;t.len=n;t.pos=0
17 if nx_json_next(t)!=NX_JSON_LBRACE{return -1}
18 var mask:i64=0;var k:i64=nx_json_next(t)
19 if k==NX_JSON_RBRACE{return -1}
20 while 1==1{
21 if k!=NX_JSON_STRING{return -1}
22 let ix:i64=chua_key(q,t.tok_off+1,t.tok_len-2,names)
23 if ix>=0{if (mask&(1<<ix))!=0{return -1};mask=mask|(1<<ix)}
24 if nx_json_next(t)!=NX_JSON_COLON{return -1}
25 k=nx_json_next(t);let start:i64=t.tok_off;let kind:i64=k
26 if chu_value(t,0,r,0)!=0{return -1}
27 if ix>=0{f[ix*3]=start;f[ix*3+1]=t.pos-start;f[ix*3+2]=kind}
28 k=nx_json_next(t)
29 if k==NX_JSON_RBRACE{break};if k!=NX_JSON_COMMA{return -1};k=nx_json_next(t)
30 }
31 if nx_json_next(t)!=NX_JSON_EOF{return -1};return mask
32}
33func chua_lit(q:*u8,f:*i64,i:i64,value:*u8)->i64 {
34 if f[i*3+2]!=NX_JSON_STRING{return 0}
35 return sj_lit_eq(q,f[i*3]+1,f[i*3]+f[i*3+1]-1,value)
36}
37func chua_num(q:*u8,f:*i64,i:i64)->i64 {
38 if f[i*3+2]!=NX_JSON_NUMBER{return -1};return chu_uint(q,f[i*3],f[i*3+1])
39}
40func chua_equal(a:*u8,f:*i64,i:i64,b:*u8,s:i64,n:i64)->i64 {
41 if f[i*3+1]!=n{return 0};var j:i64=0
42 while j<n{if a[f[i*3]+j]!=b[s+j]{return 0};j=j+1};return 1
43}
44func chua_copy(q:*u8,f:*i64,i:i64,out:*u8,o:i64)->i64 {
45 var j:i64=0;while j<f[i*3+1]{out[o+j]=q[f[i*3]+j];j=j+1};return o+j
46}
47const CHUA_HEADER:*u8="schema|state|code|scan_sha256|start_inclusive_epoch|end_exclusive_epoch|totals|unique_requests|snapshot_unique_requests|duplicate_records|ignored_records"
48const CHUA_REQUEST:*u8="schema|scan_sha256|sequence|timestamp|thread_id|response_id|turn_id|session_id|root_turn_id|usage|provider|requested_model|actual_model|role|task_id|attempt_id|review_id|accepted_deployed_outcome|elapsed_ms|billed_spend|attribution"
49const CHUA_END:*u8="schema|scan_sha256|emitted_requests|expected_requests|complete"
50const CHUA_COUNTERS:*u8="input_tokens|cached_input_tokens|cache_write_input_tokens|output_tokens|reasoning_output_tokens|total_tokens"
51// meta: upstream quoted hash offset/length, from/to, count, six counters.
52func chua_project(q:*u8,n:i64,out:*u8,meta:*i64,t:*NxJsonTok,r:*i64,f:*i64)->i64 {
53 let head:i64=sj_le(q,0,n);if head==n{return -1}
54 if chua_fields(q,head,CHUA_HEADER,11,f,t,r)!=2047{return -1}
55 if chua_lit(q,f,0,"nishi.context-request-window.v1")!=1||chua_lit(q,f,1,"observed")!=1||chua_num(q,f,2)!=0{return -1}
56 if f[11]!=NX_JSON_STRING||f[10]!=66{return -1}
57 var j:i64=0;while j<64{let b:i64=q[f[9]+1+j] as i64;if (b<48||b>57)&&(b<97||b>102){return -1};j=j+1}
58 meta[0]=f[9];meta[1]=f[10];meta[2]=chua_num(q,f,4);meta[3]=chua_num(q,f,5);meta[4]=chua_num(q,f,7)
59 if meta[2]<0||meta[3]<=meta[2]||meta[4]<0{return -1}
60 if chua_num(q,f,8)<meta[4]||chua_num(q,f,9)<0||chua_num(q,f,10)<0{return -1}
61 let ts:i64=f[18];let tn:i64=f[19]
62 if f[20]!=NX_JSON_LBRACE||chua_fields(q+ts,tn,CHUA_COUNTERS,6,f,t,r)!=63{return -1}
63 j=0;while j<6{meta[5+j]=chua_num(q+ts,f,j);if meta[5+j]<0{return -1};j=j+1}
64 var pos:i64=head+1;var o:i64=0;var seq:i64=0;var ended:i64=0
65 while pos<n{
66 let end:i64=sj_le(q,pos,n);if end==n||end==pos{return -1}
67 let line:*u8=q+pos;let ln:i64=end-pos
68 let mask:i64=chua_fields(line,ln,CHUA_REQUEST,21,f,t,r)
69 if mask<0{return -1}
70 if chua_lit(line,f,0,"nishi.context-request-stream-end.v1")==1{
71 if chua_fields(line,ln,CHUA_END,5,f,t,r)!=31{return -1}
72 if chua_equal(line,f,1,q,meta[0],meta[1])!=1||chua_num(line,f,2)!=seq||chua_num(line,f,3)!=seq||f[14]!=NX_JSON_TRUE{return -1}
73 if seq!=meta[4]||end+1!=n{return -1};ended=1;break
74 }
75 if mask!=2097151||chua_lit(line,f,0,"nishi.context-request-observation.v1")!=1{return -1}
76 if chua_equal(line,f,1,q,meta[0],meta[1])!=1||chua_num(line,f,2)!=seq{return -1}
77 j=10;while j<20{if f[j*3+2]!=NX_JSON_NULL{return -1};j=j+1}
78 if chua_lit(line,f,20,"recorded identities only; joins unverified")!=1{return -1}
79 // Each projected row is shorter than its source row. Check before emitting.
80 if o+ln>n{return -1}
81 o=sj_cat(out,o,"{\"type\":\"token_usage_record\",\"timestamp\":")
82 o=chua_copy(line,f,3,out,o);o=sj_cat(out,o,",\"payload\":{\"thread_id\":")
83 o=chua_copy(line,f,4,out,o);o=sj_cat(out,o,",\"response_id\":")
84 o=chua_copy(line,f,5,out,o);o=sj_cat(out,o,",\"turn_id\":")
85 o=chua_copy(line,f,6,out,o);o=sj_cat(out,o,",\"session_id\":")
86 o=chua_copy(line,f,7,out,o);o=sj_cat(out,o,",\"root_turn_id\":")
87 o=chua_copy(line,f,8,out,o);o=sj_cat(out,o,",\"usage\":")
88 o=chua_copy(line,f,9,out,o);o=sj_cat(out,o,"}}\n")
89 seq=seq+1;pos=end+1
90 }
91 if ended!=1{return -1};return o
92}
93// raw remains original until validation succeeds. Caller hashes it BEFORE this call.
94// On success table spans address the normalized bytes now in raw. Artifact identity
95// still refers to the original immutable file; upstream identifies its source scan.
96func chu_artifact_totals(raw:*u8,n:i64,from:i64,to:i64,table:*i64,t:*NxJsonTok,row:*i64,c:*i64,upstream:*u8)->i64 {
97 let f:*i64=sys_mmap_shared(21*3*8) as *i64;let meta:*i64=sys_mmap_shared(11*8) as *i64
98 let out:*u8=sys_mmap_shared(n+1)
99 var rc:i64=0;var normalized:i64=0
100 if (f as i64)<=0||(meta as i64)<=0||(out as i64)<=0{rc=3}else{
101 let head:i64=sj_le(raw,0,n)
102 if head==n{rc=-2}else{
103 let mask:i64=chua_fields(raw,head,"schema",1,f,t,row)
104 if mask<0{rc=-3}else{
105 if mask==0{rc=chu_totals(raw,n,from,to,table,t,row,c)}else{
106 if chua_lit(raw,f,0,"nishi.context-request-window.v1")!=1{rc=-3}else{
107 let m:i64=chua_project(raw,n,out,meta,t,row,f)
108 if m<0||from<meta[2]||to>meta[3]{rc=-3}else{
109 rc=chu_totals(out,m,meta[2],meta[3],table,t,row,c)
110 if rc==0{
111 if c[0]!=meta[4]||c[1]!=0||c[2]!=meta[4]||c[3]!=0{rc=-4}
112 var j:i64=0;while j<6{if c[4+j]!=meta[5+j]{rc=-4};j=j+1}
113 }
114 if rc==0{
115 var j:i64=0;while j<64{upstream[j]=raw[meta[0]+1+j];j=j+1};upstream[64]=0
116 j=0;while j<m{raw[j]=out[j];j=j+1};raw[m]=0;normalized=m
117 rc=chu_totals(raw,m,from,to,table,t,row,c)
118 }
119 }
120 }
121 }
122 }
123 }
124 }
125 if (out as i64)>0{if sys_munmap(out,n+1)!=0{rc=3}}
126 if (f as i64)>0{if sys_munmap(f as *u8,21*3*8)!=0{rc=3}}
127 if (meta as i64)>0{if sys_munmap(meta as *u8,11*8)!=0{rc=3}}
128 return rc
129}