code wiki / _hdl_build / nx_dom_tokenize.nx
nx_dom_tokenize.nx source
↩ module page · 146 lines · 7337 B
1// nx_dom_tokenize.nx -- DOM TOKENIZER rung 1: the foundation for true layout-aware quality criteria
2// (contrast, focus order, overlap, responsive behavior all need STRUCTURE, not byte offsets). Parses
3// HTML into a flat ELEMENT TABLE, stride 5: {kind 0=open 1=close 2=self/void, name_off, pos, depth,
4// name_len}; names in a pool. Handles void elements (img/br/input/meta/link/hr), skips comments and
5// attribute text. Gates on the REAL artifact: (a) tag balance -- every open has a close, final
6// depth 0; (b) counts agree with the quality gate's independent byte-scan (nav/main/footer/h1);
7// (c) max depth sane (<32). The element table is what rung-2 criteria walk. license_tier: ORIGINAL
8import "nx_syscalls.nx"
9const K_MAGIC_4194320: i64 = 4194320
10const K_MAGIC_2097152: i64 = 2097152
11const K_MAGIC_1048576: i64 = 1048576
12const K_MAGIC_262144: i64 = 262144
13const K_MAGIC_20000: i64 = 20000
14func _p(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
15func _pn(v: i64) -> i64 { let bb: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m;sys_write(1,"-" as *u8,1)}; let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=48;k=1}; while m>0{t[k]=48+(m%10);m=m/10;k=k+1}; var i: i64=0; while i<k{bb[i]=t[k-1-i];i=i+1}; sys_write(1,bb,k); return 0 }
16func dt_isalpha(c: i64) -> i64 { if c >= 97 { if c <= 122 { return 1 } } if c >= 65 { if c <= 90 { return 1 } } return 0 }
17func dt_lower(c: i64) -> i64 { if c >= 65 { if c <= 90 { return c + 32 } } return c }
18// is the tag name at pool[off..off+len) a void element?
19func dt_isvoid(pool: *u8, off: i64, len: i64) -> i64 {
20 // table: img br input meta link hr area base col embed source track wbr
21 if len == 3 { if pool[off] == (105 as u8) { if pool[off+1] == (109 as u8) { if pool[off+2] == (103 as u8) { return 1 } } } }
22 if len == 2 { if pool[off] == (98 as u8) { if pool[off+1] == (114 as u8) { return 1 } } }
23 if len == 2 { if pool[off] == (104 as u8) { if pool[off+1] == (114 as u8) { return 1 } } }
24 if len == 5 { if pool[off] == (105 as u8) { if pool[off+1] == (110 as u8) { return 1 } } }
25 if len == 4 { if pool[off] == (109 as u8) { if pool[off+1] == (101 as u8) { return 1 } } }
26 if len == 4 { if pool[off] == (108 as u8) { if pool[off+1] == (105 as u8) { if pool[off+2] == (110 as u8) { return 1 } } } }
27 return 0
28}
29// tokenize buf[0,n) into elems (stride 5) + pool; returns element count; state[0]=final depth, state[1]=max depth
30func dt_tokenize(buf: *u8, n: i64, elems: *i64, maxel: i64, pool: *u8, state: *i64) -> i64 {
31 var ne: i64 = 0
32 var pp: i64 = 0
33 var depth: i64 = 0
34 var maxd: i64 = 0
35 var i: i64 = 0
36 while i < n {
37 if buf[i] == (60 as u8) {
38 // comment?
39 if i + 3 < n { if buf[i+1] == (33 as u8) {
40 // skip to --> (or next > for doctype)
41 var j: i64 = i + 2
42 while j < n { if buf[j] == (62 as u8) { break } j = j + 1 }
43 i = j + 1
44 } else {
45 var kind: i64 = 0
46 var j: i64 = i + 1
47 if buf[j] == (47 as u8) { kind = 1; j = j + 1 }
48 if dt_isalpha(buf[j] as i64) == 1 {
49 let ns: i64 = pp
50 var nl: i64 = 0
51 while j < n {
52 let c: i64 = buf[j] as i64
53 if dt_isalpha(c) == 1 { pool[pp] = dt_lower(c) as u8; pp = pp + 1; nl = nl + 1; j = j + 1 }
54 else { if c >= 48 { if c <= 57 { pool[pp] = c as u8; pp = pp + 1; nl = nl + 1; j = j + 1 } else { break } } else { break } }
55 }
56 // scan to tag end '>' (skip attrs; respect quotes)
57 var inq: i64 = 0
58 while j < n {
59 let c2: i64 = buf[j] as i64
60 if inq == 0 { if c2 == 34 { inq = 1 } else { if c2 == 62 { break } } }
61 else { if c2 == 34 { inq = 0 } }
62 j = j + 1
63 }
64 var selfc: i64 = 0
65 if j > 0 { if buf[j-1] == (47 as u8) { selfc = 1 } }
66 if kind == 0 { if dt_isvoid(pool, ns, nl) == 1 { selfc = 1 } }
67 var k2: i64 = kind
68 if selfc == 1 { if kind == 0 { k2 = 2 } }
69 if ne < maxel {
70 if k2 == 1 { depth = depth - 1 }
71 elems[ne*5+0] = k2
72 elems[ne*5+1] = ns
73 elems[ne*5+2] = i
74 elems[ne*5+3] = depth
75 elems[ne*5+4] = nl
76 if k2 == 0 { depth = depth + 1; if depth > maxd { maxd = depth } }
77 ne = ne + 1
78 }
79 i = j + 1
80 } else { i = i + 1 }
81 } } else { i = i + 1 }
82 } else { i = i + 1 }
83 }
84 state[0] = depth
85 state[1] = maxd
86 return ne
87}
88func dt_count_named(elems: *i64, ne: i64, pool: *u8, name: *u8, kind: i64) -> i64 {
89 let nl: i64 = 0
90 var want: i64 = 0
91 while name[want] != (0 as u8) { want = want + 1 }
92 var cnt: i64 = 0
93 var e: i64 = 0
94 while e < ne {
95 if elems[e*5+0] == kind { if elems[e*5+4] == want {
96 var k: i64 = 0
97 var hit: i64 = 1
98 while k < want { if pool[elems[e*5+1]+k] != name[k] { hit = 0; k = want } else { k = k + 1 } }
99 if hit == 1 { cnt = cnt + 1 }
100 } }
101 e = e + 1
102 }
103 return cnt
104}
105func main() -> i64 {
106 _p("=== DOM TOKENIZER rung 1: element table over the real artifact (the layout-criteria foundation) ===\n" as *u8)
107 let buf: *u8 = sys_mmap(K_MAGIC_4194320)
108 let fd: i64 = sys_openat_rd("knowledge/andelinwest.html" as *u8)
109 if fd < 0 { sys_exit(1); return 1 }
110 var n: i64 = 0
111 var go: i64 = 1
112 while go == 1 {
113 let base: i64 = buf as i64
114 let r: i64 = sys_read(fd, (base + n) as *u8, K_MAGIC_2097152 - n)
115 if r <= 0 { go = 0 } else { n = n + r }
116 }
117 sys_close(fd)
118 let elems: *i64 = sys_mmap(K_MAGIC_1048576) as *i64
119 let pool: *u8 = sys_mmap(K_MAGIC_262144)
120 let state: *i64 = sys_mmap(64) as *i64
121 let ne: i64 = dt_tokenize(buf, n, elems, K_MAGIC_20000, pool, state)
122 _p(" elements=" as *u8); _pn(ne)
123 _p(" final-depth=" as *u8); _pn(state[0])
124 _p(" max-depth=" as *u8); _pn(state[1]); _p("\n" as *u8)
125 let nnav: i64 = dt_count_named(elems, ne, pool, "nav" as *u8, 0)
126 let nmain: i64 = dt_count_named(elems, ne, pool, "main" as *u8, 0)
127 let nfoot: i64 = dt_count_named(elems, ne, pool, "footer" as *u8, 0)
128 let nh1: i64 = dt_count_named(elems, ne, pool, "h1" as *u8, 0)
129 _p(" nav=" as *u8); _pn(nnav)
130 _p(" main=" as *u8); _pn(nmain)
131 _p(" footer=" as *u8); _pn(nfoot)
132 _p(" h1=" as *u8); _pn(nh1); _p("\n" as *u8)
133 var pass: i64 = 1
134 if state[0] != 0 { pass = 0; _p(" GATE: unbalanced tags (final depth != 0)\n" as *u8) }
135 if state[1] >= 32 { pass = 0 }
136 if ne < 20 { pass = 0 }
137 // cross-check vs the quality gate's independent byte-scan facts (1 nav, 1 main, 1 footer, 1 h1)
138 if nnav != 1 { pass = 0 }
139 if nmain != 1 { pass = 0 }
140 if nfoot != 1 { pass = 0 }
141 if nh1 != 1 { pass = 0 }
142 if pass == 1 { _p(" DOM-TOKENIZE GATE: PASS (balanced, sane depth, counts cross-checked vs the byte-scan)\n" as *u8); sys_exit(0); return 0 }
143 _p(" DOM-TOKENIZE GATE: FAIL\n" as *u8)
144 sys_exit(1)
145 return 1
146}