code wiki / _hdl_build / nx_tok_debug.nx
nx_tok_debug.nx source
↩ module page · 104 lines · 4388 B
1// nx_tok_debug.nx -- GROUND-TRUTH: how does nx_html_next_token tokenize <style>...</style><cite>?
2// The render shows `<cite class="...">` as LITERAL TEXT in Wikipedia references; this dumps the token
3// stream for a minimal fixture to see whether <cite> after </style> is a START_TAG or leaks as TEXT.
4import "nx_syscalls.nx"
5import "nx_html_tokenizer.nx"
6
7func ps(s: *u8) -> i64 { var k: i64=0; while s[k]!=(0 as u8){k=k+1} sys_write(1,s,k); return 0 }
8func pn(v0: i64) -> i64 { var v: i64=v0; if v<0 { let m: *u8=sys_mmap(8); m[0]=45 as u8; sys_write(1,m,1); v=0-v } let b: *u8=sys_mmap(24); var k: i64=0; if v==0 {b[0]=48;k=1} while v>0 {b[k]=(48+(v%10)) as u8; v=v/10; k=k+1} let o: *u8=sys_mmap(24); var j: i64=0; while j<k {o[j]=b[k-1-j];j=j+1} sys_write(1,o,k); return 0 }
9func kname(k: i64) -> i64 {
10 if k==NX_HTML_TOK_START_TAG { ps("START \x00" as *u8) }
11 if k==NX_HTML_TOK_END_TAG { ps("END \x00" as *u8) }
12 if k==NX_HTML_TOK_SELF_CLOSING { ps("SELFCL \x00" as *u8) }
13 if k==NX_HTML_TOK_TEXT { ps("TEXT \x00" as *u8) }
14 if k==NX_HTML_TOK_COMMENT { ps("COMMENT\x00" as *u8) }
15 if k==NX_HTML_TOK_DOCTYPE { ps("DOCTYPE\x00" as *u8) }
16 if k==NX_HTML_TOK_EOF { ps("EOF \x00" as *u8) }
17 return 0
18}
19
20func run(html: *u8) -> i64 {
21 var hlen: i64 = 0
22 while html[hlen]!=(0 as u8) { hlen=hlen+1 }
23 ps("=== fixture: \x00" as *u8); sys_write(1, html, hlen); ps("\n\x00" as *u8)
24 let c: *HtmlCursor = sys_mmap(24) as *HtmlCursor
25 nx_html_cursor_init(c, html, hlen)
26 let tok: *HtmlToken = sys_mmap(56) as *HtmlToken
27 var n: i64 = 0
28 var keep: i64 = 1
29 while keep==1 {
30 nx_html_next_token(c, tok)
31 if tok.kind==NX_HTML_TOK_EOF { keep=0 }
32 else {
33 if n>=40 { keep=0 }
34 else {
35 ps(" \x00" as *u8); kname(tok.kind); ps(" src=[\x00" as *u8)
36 var sl: i64 = tok.src_len; if sl>50 { sl=50 }
37 sys_write(1, ((html as i64)+tok.src_off) as *u8, sl)
38 ps("]\n\x00" as *u8)
39 n=n+1
40 }
41 }
42 }
43 ps("\n\x00" as *u8)
44 return 0
45}
46
47// substring search: first index of pat in src[from..len), or -1
48func findsub(src: *u8, len: i64, from: i64, pat: *u8, patlen: i64) -> i64 {
49 var i: i64 = from
50 while i <= len - patlen {
51 var j: i64 = 0; var ok: i64 = 1
52 while j < patlen { if (src[i+j]&0xff) != (pat[j]&0xff) { ok=0; j=patlen } else { j=j+1 } }
53 if ok==1 { return i }
54 i=i+1
55 }
56 return 0-1
57}
58// does token src contain "class=" ? (signature of a tag leaking as TEXT)
59func has_classeq(src: *u8, off: i64, len: i64) -> i64 {
60 let pat: *u8 = "class=\x00" as *u8
61 if len < 6 { return 0 }
62 var i: i64 = 0
63 while i <= len-6 {
64 var j: i64 = 0; var ok: i64 = 1
65 while j<6 { if (src[off+i+j]&0xff)!=(pat[j]&0xff){ok=0;j=6}else{j=j+1} }
66 if ok==1 { return 1 }
67 i=i+1
68 }
69 return 0
70}
71
72func main() -> i64 {
73 let lp: *i64 = sys_mmap(16) as *i64
74 let html: *u8 = sys_read_file("knowledge/fetched/last_fetch.html\x00" as *u8, lp)
75 let hlen: i64 = *lp
76 if hlen<=0 { ps("no file\n\x00" as *u8); return 1 }
77 let start: i64 = findsub(html, hlen, 0, "ol class=\"references\"\x00" as *u8, 21)
78 ps("references at offset \x00" as *u8); pn(start); ps(" / hlen \x00" as *u8); pn(hlen); ps("\n\x00" as *u8)
79 if start<0 { return 1 }
80 let c: *HtmlCursor = sys_mmap(24) as *HtmlCursor
81 nx_html_cursor_init(c, html, hlen)
82 c.pos = start - 1 // start just before the <ol ...> '<'
83 let tok: *HtmlToken = sys_mmap(56) as *HtmlToken
84 var n: i64 = 0
85 var keep: i64 = 1
86 while keep==1 {
87 nx_html_next_token(c, tok)
88 if tok.kind==NX_HTML_TOK_EOF { keep=0 }
89 else {
90 if n>=70 { keep=0 }
91 else {
92 var leak: i64 = 0
93 if tok.kind==NX_HTML_TOK_TEXT { if has_classeq(html, tok.src_off, tok.src_len)==1 { leak=1 } }
94 if leak==1 { ps(">>> LEAK \x00" as *u8) } else { ps(" \x00" as *u8) }
95 kname(tok.kind); ps(" @\x00" as *u8); pn(tok.src_off); ps(" [\x00" as *u8)
96 var sl: i64 = tok.src_len; if sl>46 { sl=46 }
97 sys_write(1, ((html as i64)+tok.src_off) as *u8, sl)
98 ps("]\n\x00" as *u8)
99 n=n+1
100 }
101 }
102 }
103 return 0
104}