docs: add robots.txt

saucelabs · Nov 30, 2023 · 00fe9a5 · 00fe9a5
1 parent 077bf46
commit 00fe9a5
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 0 deletions.
diff --git a/docs/hugo.toml b/docs/hugo.toml
@@ -1,6 +1,7 @@
 baseURL = 'https://opensource.saucelabs.com/forwarder/'
 languageCode = 'en-us'
 title = 'Forwarder Proxy'
+enableRobotsTXT = true
 
 [module]
 [[module.imports]]

diff --git a/docs/layouts/robots.txt b/docs/layouts/robots.txt
@@ -0,0 +1,7 @@
+User-agent: *
+
+Disallow: /categories/
+Disallow: /fonts/
+Disallow: /katex/
+
+Sitemap: {{ .Site.BaseURL }}sitemap.xml