nx_crawl_doc_test.nx source
↩ module page · 76 lines · 3859 B
1// nx_crawl_doc_test.nx -- KAT for the crawl back-half on real CONTENT.
2// Native sovereign lane; exit 0 = pass, N = assertion N failed.
3//
4// Proves the dedup lesson resolved: SimHash on extracted PAGE CONTENT (not
5// titles) correctly treats a scraped copy (same body, different boilerplate)
6// as a near-duplicate while an unrelated page stays distinct, and the kept
7// pages index + retrieve.
8
9import "fx.nx"
10import "nx_str.nx"
11import "nx_crawl_doc.nx"
12
13func main() -> i64 {
14 // A: canonical page. B: a scraper's copy -- SAME article body, different
15 // nav/boilerplate + a <script> (suppressed). C: unrelated page.
16 let htmlA: *u8 = "<html><head><title>x</title><style>.a{color:red}</style></head><body><nav>Home About Contact</nav><h1>Diora Baird</h1><p>Diora Baird is an American actress and model known for her roles in Wedding Crashers, Hostel Part Two, and Cobra Kai.</p></body></html>"
17 let htmlB: *u8 = "<html><head><script>track();analytics();</script></head><body><nav>Menu Login Signup Cart</nav><h1>Diora Baird</h1><p>Diora Baird is an American actress and model known for her roles in Wedding Crashers, Hostel Part Two, and Cobra Kai.</p></body></html>"
18 let htmlC: *u8 = "<html><body><p>Quantum chromodynamics is the theory of the strong interaction between quarks mediated by gluons in particle physics.</p></body></html>"
19
20 let cap: i64 = 4096
21 let tA: *u8 = sys_mmap(cap)
22 let tC: *u8 = sys_mmap(cap)
23 let fpbox: *i64 = sys_mmap(8) as *i64
24
25 let lenA: i64 = nx_crawl_extract(htmlA, nx_str_len(htmlA), tA, cap, fpbox)
26 let fpA: i64 = fpbox[0]
27 let lenB: i64 = nx_crawl_extract(htmlB, nx_str_len(htmlB), sys_mmap(cap), cap, fpbox)
28 let fpB: i64 = fpbox[0]
29 let lenC: i64 = nx_crawl_extract(htmlC, nx_str_len(htmlC), tC, cap, fpbox)
30 let fpC: i64 = fpbox[0]
31
32 if lenA <= 0 { return 1 }
33 // tags must be stripped: no '<' (0x3C) survives in extracted text
34 var i: i64 = 0
35 while i < lenA {
36 if (tA[i] as i64) == 0x3C { return 2 }
37 i = i + 1
38 }
39
40 // content near-dup: the scraped copy B is strictly closer to A than the
41 // unrelated page C (this is the title-limitation lesson resolved -- we
42 // measure CONTENT, where B really is a copy of A's article)
43 let ham_copy: i64 = nx_simhash_hamming(fpA, fpB)
44 let ham_diff: i64 = nx_simhash_hamming(fpA, fpC)
45 if ham_copy >= ham_diff { return 3 }
46
47 // crawl-time dedup gate: with A kept, B is rejected (copy), C admitted.
48 // Pick a fair threshold midway between the copy and the unrelated spread.
49 let thr: i64 = (ham_copy + ham_diff) / 2
50 let keptfps: *i64 = sys_mmap(8) as *i64
51 keptfps[0] = fpA
52 if nx_crawl_should_index(fpB, keptfps, 1, thr) != 0 { return 4 } // B is a dup -> skip
53 if nx_crawl_should_index(fpC, keptfps, 1, thr) != 1 { return 5 } // C is novel -> index
54
55 // index the KEPT canonicals (A, C) and retrieve by content term
56 let idx: *NxInvIndex = nx_inv_new(8)
57 nx_inv_index_row(idx, tA, lenA, 0)
58 nx_inv_index_row(idx, tC, lenC, 1)
59 nx_inv_finalize_offsets(idx)
60 nx_inv_emit_row(idx, tA, lenA, 0)
61 nx_inv_emit_row(idx, tC, lenC, 1)
62
63 let rowids: *i64 = sys_mmap(16 * 8) as *i64
64 let rr: *NxInvQueryResult = sys_mmap(NX_INV_QUERY_RESULT_BYTES) as *NxInvQueryResult
65 let qd: *u8 = "crashers"
66 nx_inv_query_term(idx, qd, nx_str_len(qd), rowids, 16, rr)
67 if rr.postings_count < 1 { return 6 } // A's content ("Wedding Crashers") indexed
68 let qq: *u8 = "quarks"
69 nx_inv_query_term(idx, qq, nx_str_len(qq), rowids, 16, rr)
70 if rr.postings_count < 1 { return 7 } // C's content ("quarks") indexed
71 let qz: *u8 = "zqxjwkvb"
72 nx_inv_query_term(idx, qz, nx_str_len(qz), rowids, 16, rr)
73 if rr.postings_count != 0 { return 8 } // absent term -> no match (extraction is real)
74
75 return 0
76}