nx_crawl_doc.nx source
↩ module page · 42 lines · 2035 B
1// nx_crawl_doc.nx -- per-page processing unit for the sovereign crawler.
2//
3// module: nishi-core.search.crawl_doc
4// depends: nx_html_to_text.nx, nx_simhash.nx, nx_search_inverted.nx
5// capability: CORE_COMPUTE
6// wired_status: FULLY_WIRED
7//
8// The back half of the crawl loop (the half that is design-sensitive and
9// SEO-resistant), composing already-proven sovereign modules:
10// fetched HTML --nx_html_to_text--> visible text (tags/script/style stripped)
11// --nx_simhash--------> 64-bit content fingerprint
12// The crawler then uses the fingerprint for content near-dup / novelty BEFORE
13// indexing, so a scraped copy never pollutes the index while unique content is
14// kept -- demotion EARNED from measured content, never a domain/title prejudice
15// (operator cardinal 2026-05-29). Front half (resolve + fetch over our own
16// DNS/TLS/HTTP) runs in the operator's networked environment.
17
18import "syscalls.nx"
19import "nx_html_to_text.nx"
20import "nx_simhash.nx"
21import "nx_search_inverted.nx"
22
23// Extract visible text from a fetched HTML page into text_out and compute its
24// content SimHash into fp_out[0]. Returns the extracted text length.
25func nx_crawl_extract(html: *u8, html_len: i64, text_out: *u8, text_cap: i64,
26 fp_out: *i64) -> i64 {
27 let tlen: i64 = nx_html_to_text(html, html_len, text_out, text_cap)
28 fp_out[0] = nx_simhash_fingerprint(text_out, tlen)
29 return tlen
30}
31
32// Crawl-time dedup decision: should this page be indexed, given the
33// fingerprints of pages already kept? Returns 1 (index it -- novel content) or
34// 0 (skip -- it is a near-duplicate of something already indexed, within
35// `threshold` Hamming). This is the SEO-resistant gate: mass-produced /
36// scraped copies collapse onto the first canonical; genuinely new content
37// always passes.
38func nx_crawl_should_index(fp: i64, kept_fps: *i64, n_kept: i64,
39 threshold: i64) -> i64 {
40 if nx_simhash_min_hamming(fp, kept_fps, n_kept) <= threshold { return 0 }
41 return 1
42}