nx_css_tokenize.nx source
↩ module page · 499 lines · 16746 B
1// nx_css_tokenize.nx -- CSS tokenizer for the Nishi browser.
2// Phase 2 (CSS subset) of docs/NISHI_BROWSER_ROADMAP.md.
3//
4// Recognises the subset andelinwest.com + ide.html + most modern
5// hand-written CSS uses:
6// tag selectors `body`, `div`, `h1`
7// class selectors `.head`, `.menu-item`
8// id selectors `#main`, `#sidebar`
9// descendant combinator (whitespace between simple selectors)
10// property:value `color: red`, `font-size: 16px`
11// hex colors `#ff0000`, `#fff`
12// number+unit `16px`, `1.5em`, `100%`
13// quoted strings `"Arial"`, `'sans-serif'`
14// block delimiters `{ ... }`
15// statement separator `;`
16// selector list `,` between selectors
17//
18// What it does NOT handle yet (named Phase 2b improvements):
19// - @-rules (@media, @keyframes, @import, @font-face)
20// - pseudo-classes (:hover, :nth-child)
21// - pseudo-elements (::before, ::after)
22// - attribute selectors ([href^="https"])
23// - sibling/child combinators (+ ~ >)
24// - calc() / var() / CSS variables
25// - URL functions
26// - escape sequences inside identifiers
27// - block comments / and / asterisk
28//
29// Subset is sufficient for Phase 2 demo: parse a sovereign-served
30// stylesheet, build a selector -> declarations map, apply to the
31// DOM tree from nx_html_tokenizer. Per cardinal
32// `feedback-honest-perf-verdict-no-aspirational-claims`: gap list is
33// EXACT and lives in this header.
34//
35// genealogy_id: w3c_css_syntax_module_level_3 + whatwg_css_tokenizer +
36// substrate_browser_phase_2_seed
37// lineage_id: nishi_browser_css_tokenize_v1
38//
39// nx_safety_envelope:
40// intended_use: "CSS tokenizer -- browser front-end style apply"
41// sil_target: SIL1 (style apply; failure -> wrong colors,
42// not security boundary violation)
43// evidence: [W3C_CSS_syntax_canonical_basis,
44// sealed_token_kind_enum,
45// bounded_state_machine,
46// no_allocation_caller_owned_memory]
47// verdict: NOT_YET_EVALUATED
48
49import "nx_syscalls.nx"
50
51// Sealed token kind enum.
52const NX_CSS_TOK_UNKNOWN: i64 = 0
53const NX_CSS_TOK_IDENT: i64 = 1 // body / div / color / margin
54const NX_CSS_TOK_DOT: i64 = 2 // .
55const NX_CSS_TOK_HASH: i64 = 3 // # (class or color prefix)
56const NX_CSS_TOK_COLON: i64 = 4 // :
57const NX_CSS_TOK_SEMI: i64 = 5 // ;
58const NX_CSS_TOK_LBRACE: i64 = 6 // {
59const NX_CSS_TOK_RBRACE: i64 = 7 // }
60const NX_CSS_TOK_COMMA: i64 = 8 // ,
61const NX_CSS_TOK_WHITESPACE: i64 = 9 // descendant combinator
62const NX_CSS_TOK_STRING: i64 = 10 // "Arial" / 'sans-serif'
63const NX_CSS_TOK_NUMBER: i64 = 11 // 16 / 1.5
64const NX_CSS_TOK_UNIT: i64 = 12 // px / em / rem / % (follows NUMBER)
65const NX_CSS_TOK_HEXCOLOR: i64 = 13 // #ff0000 / #fff (recognized as a unit)
66const NX_CSS_TOK_EOF: i64 = 14
67const NX_CSS_TOK_N: i64 = 15
68
69// One token's view into the source. No string allocation;
70// (src_off, src_len) point into the caller's buffer.
71struct CssToken {
72 kind: i64,
73 src_off: i64,
74 src_len: i64,
75 body_off: i64, // for IDENT / STRING / NUMBER / HEXCOLOR -- body offset
76 body_len: i64 // body length (without quotes / hash prefix)
77}
78
79const NX_CSS_TOKEN_BYTES: i64 = 40
80
81// Tokenizer cursor. Caller advances; reuses the same CssToken slot.
82struct CssCursor {
83 src: *u8,
84 src_len: i64,
85 pos: i64
86}
87
88const NX_CSS_CURSOR_BYTES: i64 = 24
89
90// ---- char classifiers (no forward refs; defined before use) ----
91
92func _css_is_space(c: i64) -> i64 {
93 if c == 32 { return 1 } // ' '
94 if c == 9 { return 1 } // \t
95 if c == 10 { return 1 } // \n
96 if c == 13 { return 1 } // \r
97 if c == 12 { return 1 } // \f
98 return 0
99}
100
101func _css_is_alpha(c: i64) -> i64 {
102 if c >= 65 { // 'A'
103 if c <= 90 { return 1 } // 'Z'
104 }
105 if c >= 97 { // 'a'
106 if c <= 122 { return 1 } // 'z'
107 }
108 return 0
109}
110
111func _css_is_digit(c: i64) -> i64 {
112 if c >= 48 { // '0'
113 if c <= 57 { return 1 } // '9'
114 }
115 return 0
116}
117
118func _css_is_hex(c: i64) -> i64 {
119 if _css_is_digit(c) == 1 { return 1 }
120 if c >= 65 { // 'A'
121 if c <= 70 { return 1 } // 'F'
122 }
123 if c >= 97 { // 'a'
124 if c <= 102 { return 1 } // 'f'
125 }
126 return 0
127}
128
129// Ident-start chars (per CSS Syntax §4.2). For our subset: alpha + '-' + '_'.
130func _css_is_ident_start(c: i64) -> i64 {
131 if _css_is_alpha(c) == 1 { return 1 }
132 if c == 95 { return 1 } // '_'
133 if c == 45 { return 1 } // '-' (CSS allows leading - for vendor prefixes)
134 return 0
135}
136
137// Ident-continue chars: alpha + digit + '-' + '_'.
138func _css_is_ident_continue(c: i64) -> i64 {
139 if _css_is_alpha(c) == 1 { return 1 }
140 if _css_is_digit(c) == 1 { return 1 }
141 if c == 95 { return 1 }
142 if c == 45 { return 1 }
143 return 0
144}
145
146// ---- cursor helpers ----
147
148func _css_peek(cur: *CssCursor, offset: i64) -> i64 {
149 let p: i64 = cur.pos + offset
150 if p < 0 { return -1 }
151 if p >= cur.src_len { return -1 }
152 return (cur.src[p] as i64) & 255
153}
154
155// Consume ident body. Returns end-position (exclusive); cur.pos advances.
156// Caller must have already verified the first char is ident-start.
157func _css_consume_ident(cur: *CssCursor) -> i64 {
158 let start: i64 = cur.pos
159 cur.pos = cur.pos + 1
160 var keep: i64 = 1
161 while keep == 1 {
162 let c: i64 = _css_peek(cur, 0)
163 if c < 0 { keep = 0 }
164 else {
165 if _css_is_ident_continue(c) == 1 {
166 cur.pos = cur.pos + 1
167 } else {
168 keep = 0
169 }
170 }
171 }
172 return cur.pos
173}
174
175// Consume a hex run (after we already saw '#'). Returns count of hex chars
176// consumed (3, 4, 6, or 8 are valid CSS hex color shapes; anything else
177// leaves cur.pos at the failure point).
178func _css_consume_hex_run(cur: *CssCursor) -> i64 {
179 var n: i64 = 0
180 var keep: i64 = 1
181 while keep == 1 {
182 let c: i64 = _css_peek(cur, 0)
183 if c < 0 { keep = 0 }
184 else {
185 if _css_is_hex(c) == 1 {
186 cur.pos = cur.pos + 1
187 n = n + 1
188 } else {
189 keep = 0
190 }
191 }
192 }
193 return n
194}
195
196// Consume a number body. Returns number of digits + optional '.'.
197// Caller has already verified first char is a digit.
198func _css_consume_number(cur: *CssCursor) -> i64 {
199 let start: i64 = cur.pos
200 var saw_dot: i64 = 0
201 var keep: i64 = 1
202 while keep == 1 {
203 let c: i64 = _css_peek(cur, 0)
204 if c < 0 { keep = 0 }
205 else {
206 if _css_is_digit(c) == 1 {
207 cur.pos = cur.pos + 1
208 } else {
209 if c == 46 { // '.'
210 if saw_dot == 0 {
211 let nxt: i64 = _css_peek(cur, 1)
212 if nxt >= 0 {
213 if _css_is_digit(nxt) == 1 {
214 cur.pos = cur.pos + 1
215 saw_dot = 1
216 } else {
217 keep = 0
218 }
219 } else {
220 keep = 0
221 }
222 } else {
223 keep = 0
224 }
225 } else {
226 keep = 0
227 }
228 }
229 }
230 }
231 return cur.pos - start
232}
233
234// Consume a quoted string body. Caller has already verified first char
235// is the quote (' or "). Returns 1 on success, 0 on unterminated.
236func _css_consume_string(cur: *CssCursor, quote: i64) -> i64 {
237 cur.pos = cur.pos + 1 // skip opening quote
238 var keep: i64 = 1
239 var ok: i64 = 0
240 while keep == 1 {
241 let c: i64 = _css_peek(cur, 0)
242 if c < 0 { keep = 0 }
243 else {
244 if c == quote {
245 cur.pos = cur.pos + 1
246 ok = 1
247 keep = 0
248 } else {
249 // CSS strings may not contain raw newlines; we treat
250 // \n as unterminated for safety per W3C spec §4.3.5.
251 if c == 10 { keep = 0 }
252 else { cur.pos = cur.pos + 1 }
253 }
254 }
255 }
256 return ok
257}
258
259// ---- public API ----
260
261// Init a cursor over the given source bytes.
262func nx_css_cursor_init(cur: *CssCursor, src: *u8, src_len: i64) -> i64 {
263 cur.src = src
264 cur.src_len = src_len
265 cur.pos = 0
266 return 0
267}
268
269// Read the next token into *tok. Returns 1 if a token was produced
270// (kind != EOF), 0 if EOF reached. tok.kind is set in both cases.
271func nx_css_next_token(cur: *CssCursor, tok: *CssToken) -> i64 {
272 let start: i64 = cur.pos
273 let c: i64 = _css_peek(cur, 0)
274
275 tok.body_off = 0
276 tok.body_len = 0
277
278 if c < 0 {
279 tok.kind = NX_CSS_TOK_EOF
280 tok.src_off = start
281 tok.src_len = 0
282 return 0
283 }
284
285 // Whitespace run -> single WHITESPACE token (descendant combinator).
286 if _css_is_space(c) == 1 {
287 var keep: i64 = 1
288 while keep == 1 {
289 let cw: i64 = _css_peek(cur, 0)
290 if cw < 0 { keep = 0 }
291 else {
292 if _css_is_space(cw) == 1 { cur.pos = cur.pos + 1 }
293 else { keep = 0 }
294 }
295 }
296 tok.kind = NX_CSS_TOK_WHITESPACE
297 tok.src_off = start
298 tok.src_len = cur.pos - start
299 return 1
300 }
301
302 // Punctuation singles.
303 if c == 123 { // '{'
304 cur.pos = cur.pos + 1
305 tok.kind = NX_CSS_TOK_LBRACE
306 tok.src_off = start
307 tok.src_len = 1
308 return 1
309 }
310 if c == 125 { // '}'
311 cur.pos = cur.pos + 1
312 tok.kind = NX_CSS_TOK_RBRACE
313 tok.src_off = start
314 tok.src_len = 1
315 return 1
316 }
317 if c == 58 { // ':'
318 cur.pos = cur.pos + 1
319 tok.kind = NX_CSS_TOK_COLON
320 tok.src_off = start
321 tok.src_len = 1
322 return 1
323 }
324 if c == 59 { // ';'
325 cur.pos = cur.pos + 1
326 tok.kind = NX_CSS_TOK_SEMI
327 tok.src_off = start
328 tok.src_len = 1
329 return 1
330 }
331 if c == 44 { // ','
332 cur.pos = cur.pos + 1
333 tok.kind = NX_CSS_TOK_COMMA
334 tok.src_off = start
335 tok.src_len = 1
336 return 1
337 }
338 if c == 46 { // '.'
339 cur.pos = cur.pos + 1
340 tok.kind = NX_CSS_TOK_DOT
341 tok.src_off = start
342 tok.src_len = 1
343 return 1
344 }
345
346 // '#' is either HEXCOLOR (followed by 3/4/6/8 hex chars) or HASH
347 // (followed by an ident -- id selector). Disambiguate by lookahead.
348 if c == 35 { // '#'
349 let nxt: i64 = _css_peek(cur, 1)
350 if nxt >= 0 {
351 if _css_is_hex(nxt) == 1 {
352 // Tentatively parse as hex color. Consume the run.
353 cur.pos = cur.pos + 1
354 let body_start: i64 = cur.pos
355 let n_hex: i64 = _css_consume_hex_run(cur)
356 if n_hex == 3 { tok.kind = NX_CSS_TOK_HEXCOLOR }
357 else {
358 if n_hex == 4 { tok.kind = NX_CSS_TOK_HEXCOLOR }
359 else {
360 if n_hex == 6 { tok.kind = NX_CSS_TOK_HEXCOLOR }
361 else {
362 if n_hex == 8 { tok.kind = NX_CSS_TOK_HEXCOLOR }
363 else {
364 // Not a valid hex color shape -> an ID selector whose name starts with hex
365 // digits (#f / #ab / #abcde). REWIND past the consumed run so the name
366 // re-tokenizes as a normal IDENT (consistent with the '#'+non-hex path),
367 // letting the selector parser read it via _css_advance_one. Leaving the name
368 // inside the HASH body made the parser see '{' next -> the rule was DROPPED.
369 cur.pos = body_start
370 tok.kind = NX_CSS_TOK_HASH
371 tok.src_off = start
372 tok.src_len = 1
373 tok.body_off = body_start
374 tok.body_len = 0
375 return 1
376 }
377 }
378 }
379 }
380 tok.src_off = start
381 tok.src_len = cur.pos - start
382 tok.body_off = body_start
383 tok.body_len = n_hex
384 return 1
385 }
386 if _css_is_ident_start(nxt) == 1 {
387 cur.pos = cur.pos + 1 // skip '#'
388 tok.kind = NX_CSS_TOK_HASH
389 tok.src_off = start
390 tok.src_len = 1
391 return 1
392 }
393 }
394 // Lone '#' with nothing valid after.
395 cur.pos = cur.pos + 1
396 tok.kind = NX_CSS_TOK_UNKNOWN
397 tok.src_off = start
398 tok.src_len = 1
399 return 1
400 }
401
402 // Quoted string.
403 if c == 34 { // '"'
404 let body_start: i64 = cur.pos + 1
405 let ok: i64 = _css_consume_string(cur, 34)
406 if ok == 1 {
407 tok.kind = NX_CSS_TOK_STRING
408 tok.src_off = start
409 tok.src_len = cur.pos - start
410 tok.body_off = body_start
411 tok.body_len = (cur.pos - 1) - body_start
412 } else {
413 tok.kind = NX_CSS_TOK_UNKNOWN
414 tok.src_off = start
415 tok.src_len = cur.pos - start
416 }
417 return 1
418 }
419 if c == 39 { // '\''
420 let body_start: i64 = cur.pos + 1
421 let ok: i64 = _css_consume_string(cur, 39)
422 if ok == 1 {
423 tok.kind = NX_CSS_TOK_STRING
424 tok.src_off = start
425 tok.src_len = cur.pos - start
426 tok.body_off = body_start
427 tok.body_len = (cur.pos - 1) - body_start
428 } else {
429 tok.kind = NX_CSS_TOK_UNKNOWN
430 tok.src_off = start
431 tok.src_len = cur.pos - start
432 }
433 return 1
434 }
435
436 // Number / dimension. We emit NUMBER first; the unit (if any) is
437 // emitted on the NEXT call as a NX_CSS_TOK_UNIT (an ident-shaped
438 // token whose preceding token was NUMBER). This keeps the
439 // tokenizer state machine flat -- the parser layer handles
440 // number+unit pairing.
441 // Negative number (e.g. -100px, -1.5em): a '-' IMMEDIATELY followed by a digit is a NUMBER whose body
442 // INCLUDES the '-' (nx_css_number_parse applies the sign). A '-' NOT followed by a digit falls through
443 // to the ident-start branch below (vendor prefixes -webkit-, custom props --x, a lone '-').
444 if c == 45 {
445 let nd: i64 = _css_peek(cur, 1)
446 if nd >= 0 { if _css_is_digit(nd) == 1 {
447 let nstart: i64 = cur.pos
448 cur.pos = cur.pos + 1
449 _css_consume_number(cur)
450 tok.kind = NX_CSS_TOK_NUMBER
451 tok.src_off = start
452 tok.src_len = cur.pos - start
453 tok.body_off = nstart
454 tok.body_len = cur.pos - nstart
455 return 1
456 } }
457 }
458 if _css_is_digit(c) == 1 {
459 let body_start: i64 = cur.pos
460 let n_digits: i64 = _css_consume_number(cur)
461 tok.kind = NX_CSS_TOK_NUMBER
462 tok.src_off = start
463 tok.src_len = cur.pos - start
464 tok.body_off = body_start
465 tok.body_len = n_digits
466 return 1
467 }
468
469 // Ident or unit (the parser disambiguates based on prior token).
470 if _css_is_ident_start(c) == 1 {
471 let body_start: i64 = cur.pos
472 _css_consume_ident(cur)
473 tok.kind = NX_CSS_TOK_IDENT
474 tok.src_off = start
475 tok.src_len = cur.pos - start
476 tok.body_off = body_start
477 tok.body_len = cur.pos - body_start
478 return 1
479 }
480
481 // '%' as a standalone unit (e.g., 50%) -- emitted as IDENT-shaped
482 // token; parser turns it into a UNIT if it follows a NUMBER.
483 if c == 37 { // '%'
484 cur.pos = cur.pos + 1
485 tok.kind = NX_CSS_TOK_IDENT
486 tok.src_off = start
487 tok.src_len = 1
488 tok.body_off = start
489 tok.body_len = 1
490 return 1
491 }
492
493 // Anything else: single-char UNKNOWN, advance so we don't loop.
494 cur.pos = cur.pos + 1
495 tok.kind = NX_CSS_TOK_UNKNOWN
496 tok.src_off = start
497 tok.src_len = 1
498 return 1
499}