robots_txt.nx source
↩ module page · 174 lines · 5574 B
1// robots_txt.nx -- build robots.txt files.
2//
3// Robots Exclusion Protocol (RFC 9309 -- finally standardised 2022
4// after 30 years of informal use). Served at /robots.txt at the
5// site root; crawlers fetch it before any other URL on the site.
6//
7// Format:
8// # comments
9// User-agent: *
10// Disallow: /private
11// Disallow: /admin
12// Allow: /private/public-page
13//
14// User-agent: Googlebot
15// Disallow: /experimental
16//
17// Sitemap: https://example.com/sitemap.xml
18//
19// Groups: lines grouped under User-agent headers apply only to
20// that agent. "*" matches everything without a dedicated group.
21// Sitemap lines are global and can appear anywhere.
22//
23// Use cases: nishi-pages emits this at /robots.txt pointing at
24// sitemap.xml and blocking /admin. Every site that wants to
25// appear (or NOT appear) in search results needs it.
26//
27// Invariants:
28// RT1 Output is plain ASCII; no escaping (robots.txt doesn't
29// have a concept of encoding).
30// RT2 Path values are literals -- caller is responsible for
31// URL-encoding special characters if needed (rare).
32// RT3 Stream builder: robots_agent -> robots_disallow* /
33// robots_allow*, then robots_sitemap for global links.
34
35import "syscalls.nx"
36
37const RT_ERR_SHORT: i64 = -1
38
39func rt_put(out: *u8, cap: i64, off: i64, src: *u8, n: i64) -> i64 {
40 if off + n > cap { return RT_ERR_SHORT }
41 var i: i64 = 0
42 while i < n {
43 out[off + i] = src[i]
44 i = i + 1
45 }
46 return off + n
47}
48
49// Comment line. "# <text>\n".
50func robots_comment(out: *u8, cap: i64, off: i64,
51 text: *u8, text_len: i64) -> i64 {
52 var cur: i64 = off
53 cur = rt_put(out, cap, cur, "# ", 2)
54 if cur < 0 { return cur }
55 cur = rt_put(out, cap, cur, text, text_len)
56 if cur < 0 { return cur }
57 cur = rt_put(out, cap, cur, "\n", 1)
58 return cur
59}
60
61// Start a group: "User-agent: <name>\n". Pass "*" for wildcard.
62func robots_agent(out: *u8, cap: i64, off: i64,
63 name: *u8, name_len: i64) -> i64 {
64 var cur: i64 = off
65 cur = rt_put(out, cap, cur, "User-agent: ", 12)
66 if cur < 0 { return cur }
67 cur = rt_put(out, cap, cur, name, name_len)
68 if cur < 0 { return cur }
69 cur = rt_put(out, cap, cur, "\n", 1)
70 return cur
71}
72
73// Disallow: <path>\n.
74func robots_disallow(out: *u8, cap: i64, off: i64,
75 path: *u8, path_len: i64) -> i64 {
76 var cur: i64 = off
77 cur = rt_put(out, cap, cur, "Disallow: ", 10)
78 if cur < 0 { return cur }
79 cur = rt_put(out, cap, cur, path, path_len)
80 if cur < 0 { return cur }
81 cur = rt_put(out, cap, cur, "\n", 1)
82 return cur
83}
84
85// Allow: <path>\n. Used to open exceptions inside a disallowed
86// subtree.
87func robots_allow(out: *u8, cap: i64, off: i64,
88 path: *u8, path_len: i64) -> i64 {
89 var cur: i64 = off
90 cur = rt_put(out, cap, cur, "Allow: ", 7)
91 if cur < 0 { return cur }
92 cur = rt_put(out, cap, cur, path, path_len)
93 if cur < 0 { return cur }
94 cur = rt_put(out, cap, cur, "\n", 1)
95 return cur
96}
97
98// Sitemap: <url>\n. Global; appears outside any group.
99func robots_sitemap(out: *u8, cap: i64, off: i64,
100 url: *u8, url_len: i64) -> i64 {
101 var cur: i64 = off
102 cur = rt_put(out, cap, cur, "Sitemap: ", 9)
103 if cur < 0 { return cur }
104 cur = rt_put(out, cap, cur, url, url_len)
105 if cur < 0 { return cur }
106 cur = rt_put(out, cap, cur, "\n", 1)
107 return cur
108}
109
110// Crawl-delay: <seconds>\n. Non-standard but widely honoured;
111// tells crawlers to wait N seconds between requests.
112func robots_crawl_delay(out: *u8, cap: i64, off: i64,
113 seconds: i64) -> i64 {
114 var cur: i64 = off
115 cur = rt_put(out, cap, cur, "Crawl-delay: ", 13)
116 if cur < 0 { return cur }
117 if seconds == 0 {
118 cur = rt_put(out, cap, cur, "0", 1)
119 } else {
120 // Emit decimal.
121 let digits_raw: *u8 = sys_mmap(32)
122 var tmp: i64 = seconds
123 var n: i64 = 0
124 while tmp > 0 {
125 digits_raw[n] = 0x30 + (tmp % 10)
126 tmp = tmp / 10
127 n = n + 1
128 }
129 if cur + n > cap { return RT_ERR_SHORT }
130 var i: i64 = n - 1
131 while i >= 0 {
132 out[cur] = digits_raw[i]
133 cur = cur + 1
134 i = i - 1
135 }
136 }
137 if cur < 0 { return cur }
138 cur = rt_put(out, cap, cur, "\n", 1)
139 return cur
140}
141
142// Blank line separator between groups.
143func robots_blank(out: *u8, cap: i64, off: i64) -> i64 {
144 return rt_put(out, cap, off, "\n", 1)
145}
146
147// Compile-only smoke.
148func main() -> i64 {
149 let out: *u8 = sys_mmap(1024)
150 var off: i64 = 0
151
152 off = robots_comment(out, 1024, off, "Nishi Pages default", 19)
153 if off < 0 { return 1 }
154
155 off = robots_agent(out, 1024, off, "*", 1)
156 if off < 0 { return 2 }
157 off = robots_disallow(out, 1024, off, "/admin/", 7)
158 if off < 0 { return 3 }
159 off = robots_allow(out, 1024, off, "/admin/public", 13)
160 if off < 0 { return 4 }
161 off = robots_crawl_delay(out, 1024, off, 2)
162 if off < 0 { return 5 }
163
164 off = robots_blank(out, 1024, off)
165 if off < 0 { return 6 }
166 off = robots_sitemap(out, 1024, off,
167 "https://nishifamily.com/sitemap.xml", 35)
168 if off < 0 { return 7 }
169
170 // First two bytes should be \"# \".
171 if out[0] != 0x23 { return 8 }
172 if out[1] != 0x20 { return 9 }
173 return 0
174}