diff --git a/site/config/_default/hugo.toml b/site/config/_default/hugo.toml index 461c9a6b..2b40f23b 100644 --- a/site/config/_default/hugo.toml +++ b/site/config/_default/hugo.toml @@ -13,6 +13,9 @@ theme = "s2b_hugo_theme" # and https://www.shift2bikes.org/categories/ for all categories. # taxonomies are autogenerated based on 'tags:' and 'categories:' specified on individual pages. disableKinds = ["taxonomy", "term"] +# generate /robots.txt from site/layouts/robots.txt +# ( without this, hugo skips the file entirely. ) +enableRobotsTXT = true # ----------------------------- # Main menu used by the site's nav bar diff --git a/site/layouts/robots.txt b/site/layouts/robots.txt new file mode 100644 index 00000000..87bae141 --- /dev/null +++ b/site/layouts/robots.txt @@ -0,0 +1,25 @@ +# https://www.shift2bikes.org/robots.txt +# re: #1068 +# +# hugo only generates this file when `enableRobotsTXT` +# is set in site/config/_default/hugo.toml. + +User-agent: * + +# emailed event-management links; unlisted, but +# they should never be indexed if one leaks. +# ( netlify serves /addevent/* as a 200. ) +Disallow: /addevent/edit- + +# proxied straight through to our api server; +# crawling these is load with no indexable value. +Disallow: /api/ +Disallow: /socialapi/ + +# the 404 page is a real page, and hugo +# would otherwise offer it up for indexing. +Disallow: /404.html + +# note: the entries in this sitemap are relative +# until baseURL is set per-context. see #1068. +Sitemap: https://www.shift2bikes.org/sitemap.xml