code wiki / _hdl_build / nx_struct_extract.nx
nx_struct_extract.nx source
↩ module page · 185 lines · 10161 B
1// nx_struct_extract.nx -- STRUCTURED-DATA extractor (grow-capabilities 2026-07-05): mine the high-quality
2// metadata modern sites embed for SEO/rich-results -- OpenGraph, Twitter-card, <meta name=description>,
3// <title>, and JSON-LD -- straight from the HTML. MEASURED motivation (SPA-shell probe): BBC/NYT ship
4// og:+jsonld+twitter in the shell; x.com ships og. This yields a CLEAN title + description (snippet) + type
5// + site even when the visible <body> is JS-rendered, and IS the knowledge-card source the SERP lacked. It is
6// a PARSER, not a JS VM -- it does NOT execute client-only SPAs (reddit-class, no embedded data = still need
7// the browser-JS arc). Precedence: og > twitter > JSON-LD > <meta description>/<title>. license_tier: ORIGINAL
8import "nx_syscalls.nx"
9const K_MAGIC_1000000: i64 = 1000000
10
11func se_lc(c: i64) -> i64 { if c >= 65 { if c <= 90 { return c + 32 } } return c }
12// case-insensitive equal: does hay[off..off+blen) == b[0..blen) (b assumed lowercase)?
13func se_ci_at(hay: *u8, off: i64, hlen: i64, b: *u8, blen: i64) -> i64 {
14 if off + blen > hlen { return 0 }
15 var i: i64 = 0
16 while i < blen { if se_lc(hay[off + i] as i64) != (b[i] as i64) { return 0 } i = i + 1 }
17 return 1
18}
19// find attribute `name` within a tag spanning [ts, te); copy its (unescaped-enough) value into out.
20// Handles name="v" / name='v' / name=v; attr order-independent. Returns value length, or -1 if absent.
21func se_attr(html: *u8, ts: i64, te: i64, name: *u8, nlen: i64, out: *u8, cap: i64) -> i64 {
22 var i: i64 = ts
23 while i < te {
24 // an attribute name starts after whitespace; match `name` then '='
25 if se_ci_at(html, i, te, name, nlen) == 1 {
26 // char before must be a boundary (space or '<... ' ) and char at i+nlen must be '=' (maybe ws)
27 var bok: i64 = 0
28 if i == ts { bok = 1 } else { let pc: i64 = html[i-1] as i64; if pc==32 { bok=1 } if pc==9 { bok=1 } if pc==10 { bok=1 } if pc==13 { bok=1 } }
29 if bok == 1 {
30 var j: i64 = i + nlen
31 while j < te { let c: i64 = html[j] as i64; if c==32 { j=j+1 } else { if c==9 { j=j+1 } else { j=te+K_MAGIC_1000000 } } }
32 // j overshot -> real j = first non-space; recompute cleanly
33 var p: i64 = i + nlen
34 var st: i64 = 0
35 while st == 0 { if p >= te { st = 1 } else { let c2: i64 = html[p] as i64; if c2==32 { p=p+1 } else { if c2==9 { p=p+1 } else { st=1 } } } }
36 if p < te { if html[p] == (61 as u8) { // '='
37 p = p + 1
38 // skip ws after '='
39 var st2: i64 = 0
40 while st2 == 0 { if p >= te { st2 = 1 } else { let c3: i64 = html[p] as i64; if c3==32 { p=p+1 } else { if c3==9 { p=p+1 } else { st2=1 } } } }
41 if p < te {
42 var q: i64 = html[p] as i64
43 var vstart: i64 = 0
44 var vend: i64 = 0
45 if q == 34 { p=p+1; vstart=p; while p<te { if html[p]==(34 as u8) { vend=p; p=te } else { p=p+1 } } } // "
46 else { if q == 39 { p=p+1; vstart=p; while p<te { if html[p]==(39 as u8) { vend=p; p=te } else { p=p+1 } } } // '
47 else { vstart=p; while p<te { let c4: i64=html[p] as i64; if c4==32 { vend=p; p=te } else { if c4==62 { vend=p; p=te } else { p=p+1 } } } if vend==0 { vend=te } } }
48 if vend > vstart {
49 let vl: i64 = vend - vstart
50 var o: i64 = 0
51 while o < vl { if o < cap-1 { out[o] = html[vstart + o] } o = o + 1 }
52 var ol: i64 = vl; if ol > cap-1 { ol = cap-1 }
53 out[ol] = 0 as u8
54 return ol
55 }
56 }
57 } }
58 }
59 }
60 i = i + 1
61 }
62 return 0 - 1
63}
64// find a <meta ...> whose `property`/`name` attr equals key, return its `content` value. -1 if absent.
65func se_meta(html: *u8, hlen: i64, keyattr: *u8, kalen: i64, key: *u8, klen: i64, out: *u8, cap: i64) -> i64 {
66 var i: i64 = 0
67 let mtag: *u8 = "meta" as *u8
68 while i + 5 < hlen {
69 if html[i] == (60 as u8) { // '<'
70 if se_ci_at(html, i+1, hlen, mtag, 4) == 1 {
71 let after: i64 = i + 5
72 var boundary: i64 = 0
73 let ac: i64 = html[after] as i64
74 if ac==32 { boundary=1 } if ac==9 { boundary=1 } if ac==10 { boundary=1 } if ac==13 { boundary=1 } if ac==47 { boundary=1 }
75 if boundary == 1 {
76 var te: i64 = after
77 var found: i64 = 0
78 while found == 0 { if te >= hlen { found = 1 } else { if html[te] == (62 as u8) { found = 1 } else { te = te + 1 } } }
79 // does keyattr == key?
80 let kv: *u8 = sys_mmap(256)
81 let kvl: i64 = se_attr(html, i+5, te, keyattr, kalen, kv, 256)
82 if kvl > 0 {
83 if kvl == klen {
84 var eq: i64 = 1; var z: i64 = 0
85 while z < klen { if se_lc(kv[z] as i64) != se_lc(key[z] as i64) { eq = 0; z = klen } else { z = z + 1 } }
86 if eq == 1 {
87 let cl: i64 = se_attr(html, i+5, te, "content" as *u8, 7, out, cap)
88 if cl > 0 { return cl }
89 }
90 }
91 }
92 i = te
93 }
94 }
95 }
96 i = i + 1
97 }
98 return 0 - 1
99}
100// <title>...</title>
101func se_title(html: *u8, hlen: i64, out: *u8, cap: i64) -> i64 {
102 var i: i64 = 0
103 let t: *u8 = "title" as *u8
104 while i + 7 < hlen {
105 if html[i] == (60 as u8) { if se_ci_at(html, i+1, hlen, t, 5) == 1 { if html[i+6] == (62 as u8) {
106 let s: i64 = i + 7
107 var e: i64 = s
108 var fnd: i64 = 0
109 while fnd == 0 { if e >= hlen { fnd = 1 } else { if html[e] == (60 as u8) { fnd = 1 } else { e = e + 1 } } }
110 let vl: i64 = e - s
111 var o: i64 = 0
112 while o < vl { if o < cap-1 { out[o] = html[s+o] } o = o + 1 }
113 var ol: i64 = vl; if ol > cap-1 { ol = cap-1 }
114 out[ol] = 0 as u8
115 return ol
116 } } }
117 i = i + 1
118 }
119 return 0 - 1
120}
121// a JSON string field's value inside the first ld+json block: "field":"value" (ws-tolerant). -1 absent.
122// (string-scan, not a full JSON parse -- robust enough for headline/description extraction.)
123func se_jsonld(html: *u8, hlen: i64, field: *u8, flen: i64, out: *u8, cap: i64) -> i64 {
124 // locate an application/ld+json script block start
125 var i: i64 = 0
126 let needle: *u8 = "ld+json" as *u8
127 var blk: i64 = 0 - 1
128 while i + 7 < hlen { if se_ci_at(html, i, hlen, needle, 7) == 1 { blk = i; i = hlen } else { i = i + 1 } }
129 if blk < 0 { return 0 - 1 }
130 // scan forward from the block for "field"
131 var p: i64 = blk
132 while p + flen + 3 < hlen {
133 if html[p] == (34 as u8) {
134 var m: i64 = 1; var k: i64 = 0
135 while k < flen { if html[p+1+k] != field[k] { m = 0; k = flen } else { k = k + 1 } }
136 if m == 1 { if html[p+1+flen] == (34 as u8) { if html[p+2+flen] == (58 as u8) { // "field":
137 var q: i64 = p + 3 + flen
138 var st: i64 = 0
139 while st == 0 { if q >= hlen { st = 1 } else { let c: i64 = html[q] as i64; if c==32 { q=q+1 } else { st=1 } } }
140 if q < hlen { if html[q] == (34 as u8) {
141 q = q + 1; var s: i64 = q
142 var e: i64 = s; var fnd: i64 = 0
143 while fnd == 0 { if e >= hlen { fnd = 1 } else { if html[e]==(92 as u8) { e=e+2 } else { if html[e]==(34 as u8) { fnd = 1 } else { e = e + 1 } } } }
144 let vl: i64 = e - s
145 var o: i64 = 0
146 while o < vl { if o < cap-1 { out[o] = html[s+o] } o = o + 1 }
147 var ol: i64 = vl; if ol > cap-1 { ol = cap-1 }
148 out[ol] = 0 as u8
149 return ol
150 } }
151 } } }
152 }
153 p = p + 1
154 }
155 return 0 - 1
156}
157
158// TOP-LEVEL: best TITLE + DESCRIPTION + TYPE + SITE with precedence. Returns how many of {title,desc} filled.
159// Precedence title: og:title > twitter:title > JSON-LD headline > JSON-LD name > <title>.
160// desc: og:description > twitter:description > JSON-LD description > <meta name=description>.
161func nx_struct_extract(html: *u8, hlen: i64,
162 title: *u8, title_cap: i64, desc: *u8, desc_cap: i64,
163 stype: *u8, stype_cap: i64, ssite: *u8, ssite_cap: i64) -> i64 {
164 title[0] = 0 as u8; desc[0] = 0 as u8; stype[0] = 0 as u8; ssite[0] = 0 as u8
165 var got: i64 = 0
166 // title
167 if se_meta(html, hlen, "property" as *u8, 8, "og:title" as *u8, 8, title, title_cap) < 0 {
168 if se_meta(html, hlen, "name" as *u8, 4, "twitter:title" as *u8, 13, title, title_cap) < 0 {
169 if se_jsonld(html, hlen, "headline" as *u8, 8, title, title_cap) < 0 {
170 if se_jsonld(html, hlen, "name" as *u8, 4, title, title_cap) < 0 {
171 se_title(html, hlen, title, title_cap)
172 } } } }
173 if title[0] != (0 as u8) { got = got + 1 }
174 // description
175 if se_meta(html, hlen, "property" as *u8, 8, "og:description" as *u8, 14, desc, desc_cap) < 0 {
176 if se_meta(html, hlen, "name" as *u8, 4, "twitter:description" as *u8, 19, desc, desc_cap) < 0 {
177 if se_jsonld(html, hlen, "description" as *u8, 11, desc, desc_cap) < 0 {
178 se_meta(html, hlen, "name" as *u8, 4, "description" as *u8, 11, desc, desc_cap)
179 } } }
180 if desc[0] != (0 as u8) { got = got + 1 }
181 // type + site (card metadata)
182 se_meta(html, hlen, "property" as *u8, 8, "og:type" as *u8, 7, stype, stype_cap)
183 se_meta(html, hlen, "property" as *u8, 8, "og:site_name" as *u8, 12, ssite, ssite_cap)
184 return got
185}