nx_html_extract_data.nx source

1// nx_html_extract_data.nx -- JS-RENDER FETCHER rung 1: STRUCTURED-DATA EXTRACTION. 2// 3// Many "JS-rendered" pages embed content as JSON-LD (<script application/ld+json>) 4// + OpenGraph meta -- recoverable WITHOUT executing JS. Pulls prose fields 5// (headline/name/description/articleBody) AND og:description out of raw HTML so the 6// researcher can read JS sites' structured content. Reuses re_find/re_strlen (DRY). 7// HONEST SCOPE: embedded/SSR data only; fully client-fetched content needs rung 2 8// (API discovery) then rung 3 (a sovereign JS interpreter + DOM). license_tier: ORIGINAL. 9 10import "nx_syscalls.nx" 11import "nx_research_extract.nx" 12const K_MAGIC_65536: i64 = 65536 13 14// copy a JSON string value following keypat (e.g. "description":) into out: 15// skips optional spaces, requires an opening quote, reads to the closing quote 16// (handles \-escapes). Returns length (0 if absent / not a string value). 17func hed_field(html: *u8, n: i64, keypat: *u8, out: *u8, cap: i64) -> i64 { 18 let p: i64 = re_find(html, n, keypat) 19 if p < 0 { return 0 } 20 var i: i64 = p + re_strlen(keypat) 21 while i < n { if html[i] == (32 as u8) { i = i + 1 } else { break } } // skip optional spaces 22 if i >= n { return 0 } 23 if html[i] != (34 as u8) { return 0 } // value must be a string 24 i = i + 1 25 var k: i64 = 0 26 while i < n { 27 let c: u8 = html[i] 28 if c == (34 as u8) { i = n } 29 else { 30 if c == (92 as u8) { i = i + 1; if i < n { if k < cap - 1 { out[k] = html[i]; k = k + 1 } } i = i + 1 } 31 else { if k < cap - 1 { out[k] = c; k = k + 1 } i = i + 1 } 32 } 33 } 34 out[k] = 0 as u8 35 return k 36} 37 38// extract an HTML meta value: find marker (e.g. og:description), then the next 39// content="..." after it; reads until the matching quote. Returns length. 40func hed_meta(html: *u8, n: i64, marker: *u8, out: *u8, cap: i64) -> i64 { 41 let p: i64 = re_find(html, n, marker) 42 if p < 0 { return 0 } 43 let cp: i64 = re_find(html + p, n - p, "content=" as *u8) 44 if cp < 0 { return 0 } 45 var i: i64 = p + cp + 8 // past "content=" 46 var q: u8 = 34 as u8 // default closing quote = " 47 if i < n { if html[i] == (39 as u8) { q = 39 as u8 } } 48 if i < n { if html[i] == (34 as u8) { i = i + 1 } else { if html[i] == (39 as u8) { i = i + 1 } } } 49 var k: i64 = 0 50 while i < n { if html[i] == q { i = n } else { if k < cap - 1 { out[k] = html[i]; k = k + 1 } i = i + 1 } } 51 out[k] = 0 as u8 52 return k 53} 54 55func hed_app(out: *u8, off: *i64, s: *u8, slen: i64, cap: i64) -> i64 { 56 var i: i64 = 0 57 while i < slen { if off[0] < cap - 2 { out[off[0]] = s[i]; off[0] = off[0] + 1 } i = i + 1 } 58 if off[0] < cap - 2 { out[off[0]] = 10 as u8; off[0] = off[0] + 1 } 59 return 0 60} 61 62// extract JSON-LD prose + og:description from raw HTML -> out 63func nx_html_extract_data(html: *u8, n: i64, out: *u8, cap: i64) -> i64 { 64 let off: *i64 = (sys_mmap(8)) as *i64 65 off[0] = 0 66 let tmp: *u8 = sys_mmap(K_MAGIC_65536) 67 var L: i64 = 0 68 L = hed_field(html, n, "\"headline\":" as *u8, tmp, K_MAGIC_65536); if L > 0 { hed_app(out, off, tmp, L, cap) } 69 L = hed_field(html, n, "\"name\":" as *u8, tmp, K_MAGIC_65536); if L > 0 { hed_app(out, off, tmp, L, cap) } 70 L = hed_field(html, n, "\"description\":" as *u8, tmp, K_MAGIC_65536); if L > 0 { hed_app(out, off, tmp, L, cap) } 71 L = hed_field(html, n, "\"articleBody\":" as *u8, tmp, K_MAGIC_65536); if L > 0 { hed_app(out, off, tmp, L, cap) } 72 L = hed_meta(html, n, "og:description" as *u8, tmp, K_MAGIC_65536); if L > 0 { hed_app(out, off, tmp, L, cap) } 73 L = hed_meta(html, n, "og:title" as *u8, tmp, K_MAGIC_65536); if L > 0 { hed_app(out, off, tmp, L, cap) } 74 out[off[0]] = 0 as u8 75 return off[0] 76}