nx_robots_txt.nx source
↩ module page · 181 lines · 5758 B
1// robots_txt.nx -- build robots.txt files.
2//
3// Robots Exclusion Protocol (RFC 9309 -- finally standardised 2022
4// after 30 years of informal use). Served at /robots.txt at the
5// site root; crawlers fetch it before any other URL on the site.
6//
7// Format:
8// # comments
9// User-agent: *
10// Disallow: /private
11// Disallow: /admin
12// Allow: /private/public-page
13//
14// User-agent: Googlebot
15// Disallow: /experimental
16//
17// Sitemap: https://example.com/sitemap.xml
18//
19// Groups: lines grouped under User-agent headers apply only to
20// that agent. "*" matches everything without a dedicated group.
21// Sitemap lines are global and can appear anywhere.
22//
23// Use cases: nishi-pages emits this at /robots.txt pointing at
24// sitemap.xml and blocking /admin. Every site that wants to
25// appear (or NOT appear) in search results needs it.
26//
27// Invariants:
28// RT1 Output is plain ASCII; no escaping (robots.txt doesn't
29// have a concept of encoding).
30// RT2 Path values are literals -- caller is responsible for
31// URL-encoding special characters if needed (rare).
32// RT3 Stream builder: robots_agent -> robots_disallow* /
33// robots_allow*, then robots_sitemap for global links.
34
35// nx_safety_envelope:
36// intended_use: AUTO_APPLIED -- primitive-specific tuning queued
37// sil_target: SIL1
38// evidence: [bulk_applied_2026-05-16, see-file-comment-for-detail]
39// verdict: NOT_YET_EVALUATED
40
41import "nx_syscalls.nx"
42const RT_MAGIC_1024: i64 = 1024
43
44const RT_ERR_SHORT: i64 = -1
45
46func rt_put(out: *u8, cap: i64, off: i64, src: *u8, n: i64) -> i64 {
47 if off + n > cap { return RT_ERR_SHORT }
48 var i: i64 = 0
49 while i < n {
50 out[off + i] = src[i]
51 i = i + 1
52 }
53 return off + n
54}
55
56// Comment line. "# <text>\n".
57func robots_comment(out: *u8, cap: i64, off: i64,
58 text: *u8, text_len: i64) -> i64 {
59 var cur: i64 = off
60 cur = rt_put(out, cap, cur, "# ", 2)
61 if cur < 0 { return cur }
62 cur = rt_put(out, cap, cur, text, text_len)
63 if cur < 0 { return cur }
64 cur = rt_put(out, cap, cur, "\n", 1)
65 return cur
66}
67
68// Start a group: "User-agent: <name>\n". Pass "*" for wildcard.
69func robots_agent(out: *u8, cap: i64, off: i64,
70 name: *u8, name_len: i64) -> i64 {
71 var cur: i64 = off
72 cur = rt_put(out, cap, cur, "User-agent: ", 12)
73 if cur < 0 { return cur }
74 cur = rt_put(out, cap, cur, name, name_len)
75 if cur < 0 { return cur }
76 cur = rt_put(out, cap, cur, "\n", 1)
77 return cur
78}
79
80// Disallow: <path>\n.
81func robots_disallow(out: *u8, cap: i64, off: i64,
82 path: *u8, path_len: i64) -> i64 {
83 var cur: i64 = off
84 cur = rt_put(out, cap, cur, "Disallow: ", 10)
85 if cur < 0 { return cur }
86 cur = rt_put(out, cap, cur, path, path_len)
87 if cur < 0 { return cur }
88 cur = rt_put(out, cap, cur, "\n", 1)
89 return cur
90}
91
92// Allow: <path>\n. Used to open exceptions inside a disallowed
93// subtree.
94func robots_allow(out: *u8, cap: i64, off: i64,
95 path: *u8, path_len: i64) -> i64 {
96 var cur: i64 = off
97 cur = rt_put(out, cap, cur, "Allow: ", 7)
98 if cur < 0 { return cur }
99 cur = rt_put(out, cap, cur, path, path_len)
100 if cur < 0 { return cur }
101 cur = rt_put(out, cap, cur, "\n", 1)
102 return cur
103}
104
105// Sitemap: <url>\n. Global; appears outside any group.
106func robots_sitemap(out: *u8, cap: i64, off: i64,
107 url: *u8, url_len: i64) -> i64 {
108 var cur: i64 = off
109 cur = rt_put(out, cap, cur, "Sitemap: ", 9)
110 if cur < 0 { return cur }
111 cur = rt_put(out, cap, cur, url, url_len)
112 if cur < 0 { return cur }
113 cur = rt_put(out, cap, cur, "\n", 1)
114 return cur
115}
116
117// Crawl-delay: <seconds>\n. Non-standard but widely honoured;
118// tells crawlers to wait N seconds between requests.
119func robots_crawl_delay(out: *u8, cap: i64, off: i64,
120 seconds: i64) -> i64 {
121 var cur: i64 = off
122 cur = rt_put(out, cap, cur, "Crawl-delay: ", 13)
123 if cur < 0 { return cur }
124 if seconds == 0 {
125 cur = rt_put(out, cap, cur, "0", 1)
126 } else {
127 // Emit decimal.
128 let digits_raw: *u8 = sys_mmap(32)
129 var tmp: i64 = seconds
130 var n: i64 = 0
131 while tmp > 0 {
132 digits_raw[n] = 0x30 + (tmp % 10)
133 tmp = tmp / 10
134 n = n + 1
135 }
136 if cur + n > cap { return RT_ERR_SHORT }
137 var i: i64 = n - 1
138 while i >= 0 {
139 out[cur] = digits_raw[i]
140 cur = cur + 1
141 i = i - 1
142 }
143 }
144 if cur < 0 { return cur }
145 cur = rt_put(out, cap, cur, "\n", 1)
146 return cur
147}
148
149// Blank line separator between groups.
150func robots_blank(out: *u8, cap: i64, off: i64) -> i64 {
151 return rt_put(out, cap, off, "\n", 1)
152}
153
154// Compile-only smoke.
155func main() -> i64 {
156 let out: *u8 = sys_mmap(RT_MAGIC_1024)
157 var off: i64 = 0
158
159 off = robots_comment(out, RT_MAGIC_1024, off, "Nishi Pages default", 19)
160 if off < 0 { return 1 }
161
162 off = robots_agent(out, RT_MAGIC_1024, off, "*", 1)
163 if off < 0 { return 2 }
164 off = robots_disallow(out, RT_MAGIC_1024, off, "/admin/", 7)
165 if off < 0 { return 3 }
166 off = robots_allow(out, RT_MAGIC_1024, off, "/admin/public", 13)
167 if off < 0 { return 4 }
168 off = robots_crawl_delay(out, RT_MAGIC_1024, off, 2)
169 if off < 0 { return 5 }
170
171 off = robots_blank(out, RT_MAGIC_1024, off)
172 if off < 0 { return 6 }
173 off = robots_sitemap(out, RT_MAGIC_1024, off,
174 "https://nishifamily.com/sitemap.xml", 35)
175 if off < 0 { return 7 }
176
177 // First two bytes should be \"# \".
178 if out[0] != 0x23 { return 8 }
179 if out[1] != 0x20 { return 9 }
180 return 0
181}