nx_fetch_rights.nx source
↩ module page · 107 lines · 3372 B
1// nx_fetch_rights.nx -- rights-status policy for acquisition, modelled on how
2// modern search engines treat content. Four statuses:
3//
4// CRAWLABLE unknown rights, public -> may index (robots/ToS-respecting),
5// like Google/Bing; NOT usable in our products.
6// BLOCKED rights NOT available (Getty, DMCA-flagged) -> never fetch/index.
7// OWNED licensed talent/partner -> full product use.
8// TO_ACQUIRE flagged: we want to license -> may index, NOT product-usable
9// yet, and pursue the agent/contact to make an offer.
10//
11// Unknown host defaults to CRAWLABLE. This is the policy layer; the fetcher
12// (nx_crawl_https) checks nx_fr_may_index AND robots.txt before fetching, and
13// only OWNED content is used in products.
14// license_tier: ORIGINAL
15
16import "nx_syscalls.nx"
17const NX_MAGIC_1469598103934665603: i64 = 1469598103934665603
18const NX_MAGIC_1099511628211: i64 = 1099511628211
19
20const NX_FR_CRAWLABLE: i64 = 0
21const NX_FR_BLOCKED: i64 = 1
22const NX_FR_OWNED: i64 = 2
23const NX_FR_TO_ACQUIRE: i64 = 3
24const NX_FR_N: i64 = 4
25
26struct NxFrEntry {
27 host_hash: i64,
28 status: i64,
29}
30
31struct NxFrPolicy {
32 entries: *NxFrEntry,
33 capacity: i64,
34 count: i64,
35}
36
37const NX_FR_ENTRY_BYTES: i64 = 16
38
39// FNV-1a 64, lowercased (hosts are case-insensitive).
40func _fr_hash(host: *u8) -> i64 {
41 var h: i64 = NX_MAGIC_1469598103934665603
42 var i: i64 = 0
43 while host[i] != (0 as u8) {
44 var c: i64 = host[i] as i64
45 if c >= 65 { if c <= 90 { c = c + 32 } }
46 h = h ^ c
47 h = h * NX_MAGIC_1099511628211
48 i = i + 1
49 }
50 return h
51}
52
53func nx_fr_new(capacity: i64) -> *NxFrPolicy {
54 let p: *NxFrPolicy = (sys_mmap(24)) as *NxFrPolicy
55 p.entries = (sys_mmap(capacity * NX_FR_ENTRY_BYTES)) as *NxFrEntry
56 p.capacity = capacity
57 p.count = 0
58 return p
59}
60
61func _fr_at(p: *NxFrPolicy, i: i64) -> *NxFrEntry {
62 return (p.entries as i64 + i * NX_FR_ENTRY_BYTES) as *NxFrEntry
63}
64
65// Set/override a host's rights status (additive; last write wins on lookup).
66func nx_fr_set(p: *NxFrPolicy, host: *u8, status: i64) -> i64 {
67 if status < 0 { return 0 - 1 }
68 if status >= NX_FR_N { return 0 - 1 }
69 if p.count >= p.capacity { return 0 - 1 }
70 let e: *NxFrEntry = _fr_at(p, p.count)
71 e.host_hash = _fr_hash(host)
72 e.status = status
73 p.count = p.count + 1
74 return 0
75}
76
77// Classify a host; unknown -> CRAWLABLE (search-engine default). Last matching
78// entry wins (a later override supersedes).
79func nx_fr_classify(p: *NxFrPolicy, host: *u8) -> i64 {
80 let hh: i64 = _fr_hash(host)
81 var found: i64 = NX_FR_CRAWLABLE
82 var i: i64 = 0
83 while i < p.count {
84 let e: *NxFrEntry = _fr_at(p, i)
85 if e.host_hash == hh { found = e.status }
86 i = i + 1
87 }
88 return found
89}
90
91// May we fetch/index this at all? BLOCKED -> no; else yes (robots still applies).
92func nx_fr_may_index(status: i64) -> i64 {
93 if status == NX_FR_BLOCKED { return 0 }
94 return 1
95}
96
97// May we USE this content in our products? OWNED (licensed) only.
98func nx_fr_may_use_in_product(status: i64) -> i64 {
99 if status == NX_FR_OWNED { return 1 }
100 return 0
101}
102
103// Should we pursue a license (find agent/contact, make an offer)?
104func nx_fr_is_to_acquire(status: i64) -> i64 {
105 if status == NX_FR_TO_ACQUIRE { return 1 }
106 return 0
107}