nx_urlcanon.nx source
↩ module page · 79 lines · 3052 B
1// nx_urlcanon.nx -- URL canonicalization for crawl de-duplication.
2//
3// module: nishi-core.search.urlcanon
4// depends: nx_str.nx
5// capability: CORE_COMPUTE
6// wired_status: FULLY_WIRED
7//
8// WHY: the same resource is reachable under many URL spellings (host-case,
9// #fragment, default :80/:443). Canonicalizing before the visited-set check
10// means we DON'T re-fetch the same page under a cosmetic variant -- saving
11// crawl budget and, crucially, not hammering hosts (-> not getting
12// blacklisted). RFC 3986 syntax-based normalization (the safe subset):
13// * lowercase scheme + host (case-insensitive per RFC 3986 sec 6.2.2.1)
14// * drop the fragment (never sent to the origin server)
15// * drop the default port (:80 for http, :443 for https)
16// Path + query are preserved verbatim (case- and order-significant). Tracking-
17// param stripping is a queued refinement.
18
19import "nx_str.nx"
20
21func _uc_lc(c: i64) -> i64 { if c >= 0x41 { if c <= 0x5A { return c + 0x20 } } return c }
22
23// Canonicalize url[0..n) into out (null-terminated). Returns out length.
24func nx_url_canon(url: *u8, n: i64, out: *u8, cap: i64) -> i64 {
25 var o: i64 = 0
26 // locate "://"
27 var i: i64 = 0
28 var se: i64 = 0 - 1
29 while i < n - 2 {
30 if se < 0 { if (url[i] as i64)==0x3A { if (url[i+1] as i64)==0x2F { if (url[i+2] as i64)==0x2F { se = i + 3 } } } }
31 i = i + 1
32 }
33 if se < 0 {
34 // no scheme: copy, dropping fragment
35 var p0: i64 = 0
36 while p0 < n { if (url[p0] as i64) == 0x23 { p0 = n } else { if o < cap { out[o] = url[p0]; o = o + 1 } p0 = p0 + 1 } }
37 out[o] = 0
38 return o
39 }
40 // scheme:// lowercased
41 i = 0
42 while i < se { if o < cap { out[o] = _uc_lc(url[i] as i64); o = o + 1 } i = i + 1 }
43 // host end = first of '/', ':', '?', '#'
44 var he: i64 = se
45 var hrun: i64 = 1
46 while hrun == 1 {
47 hrun = 0
48 if he < n {
49 let c: i64 = url[he] as i64
50 if c != 0x2F { if c != 0x3A { if c != 0x3F { if c != 0x23 { he = he + 1; hrun = 1 } } } }
51 }
52 }
53 // lowercase host
54 i = se
55 while i < he { if o < cap { out[o] = _uc_lc(url[i] as i64); o = o + 1 } i = i + 1 }
56 // optional :port
57 var rest: i64 = he
58 if he < n {
59 if (url[he] as i64) == 0x3A {
60 var pe: i64 = he + 1
61 var port: i64 = 0
62 var prun: i64 = 1
63 while prun == 1 {
64 prun = 0
65 if pe < n { let c: i64 = url[pe] as i64; if c >= 0x30 { if c <= 0x39 { port = port * 10 + (c - 0x30); pe = pe + 1; prun = 1 } } }
66 }
67 var deflt: i64 = 80
68 if o > 4 { if (out[4] as i64) == 0x73 { deflt = 443 } } // 's' of https
69 if port == deflt { rest = pe } // drop default port; else leave ':' so :port is copied
70 }
71 }
72 // path + query, dropping fragment
73 var p: i64 = rest
74 while p < n {
75 if (url[p] as i64) == 0x23 { p = n } else { if o < cap { out[o] = url[p]; o = o + 1 } p = p + 1 }
76 }
77 out[o] = 0
78 return o
79}