From a88d2aa007686d5766d7f0de14dbca4eca429348 Mon Sep 17 00:00:00 2001
From: Douglas Eichelberger <d@eic.email>
Date: Fri, 10 Jul 2026 08:26:16 -0700
Subject: [PATCH 1/2] Generate a robots.txt (#1068)

Hugo skips robots.txt unless `enableRobotsTXT` is set, so
https://www.shift2bikes.org/robots.txt has been a 404.

Turn it on, and supply a template rather than take hugo's
permissive default. Search indexing stays open for all real
content, event images, and the calendar; crawling is turned
away from the emailed event-management links and from the two
netlify rules that proxy through to our own api server.
---
 site/config/_default/hugo.toml |  3 +++
 site/layouts/robots.txt        | 25 +++++++++++++++++++++++++
 2 files changed, 28 insertions(+)
 create mode 100644 site/layouts/robots.txt

diff --git a/site/config/_default/hugo.toml b/site/config/_default/hugo.toml
index 461c9a6b..2b40f23b 100644
--- a/site/config/_default/hugo.toml
+++ b/site/config/_default/hugo.toml
@@ -13,6 +13,9 @@ theme = "s2b_hugo_theme"
 # and  https://www.shift2bikes.org/categories/ for all categories.
 # taxonomies are autogenerated based on 'tags:' and 'categories:' specified on individual pages.
 disableKinds = ["taxonomy", "term"]
+# generate /robots.txt from site/layouts/robots.txt
+# ( without this, hugo skips the file entirely. )
+enableRobotsTXT = true
 
 # -----------------------------
 # Main menu used by the site's nav bar
diff --git a/site/layouts/robots.txt b/site/layouts/robots.txt
new file mode 100644
index 00000000..5f3c5d89
--- /dev/null
+++ b/site/layouts/robots.txt
@@ -0,0 +1,25 @@
+# https://www.shift2bikes.org/robots.txt
+# re: #1068
+#
+# hugo only generates this file when `enableRobotsTXT`
+# is set in site/config/_default/hugo.toml.
+
+User-agent: *
+
+# emailed event-management links; unlisted, but
+# they should never be indexed if one leaks.
+# ( netlify serves /addevent/* as a 200. )
+Disallow: /addevent/edit-
+
+# proxied straight through to our api server;
+# crawling these is load with no indexable value.
+Disallow: /api/
+Disallow: /socialapi/
+
+# the 404 page is a real page, and hugo
+# would otherwise offer it up for indexing.
+Disallow: /404/
+
+# note: the entries in this sitemap are relative
+# until baseURL is set per-context. see #1068.
+Sitemap: https://www.shift2bikes.org/sitemap.xml

From 791acfcfb85ccfced1d2de17918e3cf010bb5688 Mon Sep 17 00:00:00 2001
From: Douglas Eichelberger <d@eic.email>
Date: Fri, 10 Jul 2026 08:45:27 -0700
Subject: [PATCH 2/2] Point 404 disallow at /404.html for #862

---
 site/layouts/robots.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/site/layouts/robots.txt b/site/layouts/robots.txt
index 5f3c5d89..87bae141 100644
--- a/site/layouts/robots.txt
+++ b/site/layouts/robots.txt
@@ -18,7 +18,7 @@ Disallow: /socialapi/
 
 # the 404 page is a real page, and hugo
 # would otherwise offer it up for indexing.
-Disallow: /404/
+Disallow: /404.html
 
 # note: the entries in this sitemap are relative
 # until baseURL is set per-context. see #1068.