diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go index e71e48d4124..ee0d5c56368 100644 --- a/hugolib/hugo_sites.go +++ b/hugolib/hugo_sites.go @@ -49,6 +49,7 @@ import ( "github.com/gohugoio/hugo/langs/i18n" "github.com/gohugoio/hugo/resources/page" + "github.com/gohugoio/hugo/resources/page/pagemeta" "github.com/gohugoio/hugo/tpl" "github.com/gohugoio/hugo/tpl/tplimpl" ) @@ -690,7 +691,7 @@ func (cfg *BuildCfg) shouldRender(p *pageState) bool { return false } -func (h *HugoSites) renderCrossSitesArtifacts() error { +func (h *HugoSites) renderCrossSitesSitemap() error { if !h.multilingual.enabled() || h.IsMultihost() { return nil @@ -716,6 +717,38 @@ func (h *HugoSites) renderCrossSitesArtifacts() error { s.siteCfg.sitemap.Filename, h.toSiteInfos(), templ) } +func (h *HugoSites) renderCrossSitesRobotsTXT() error { + if h.multihost { + return nil + } + if !h.Cfg.GetBool("enableRobotsTXT") { + return nil + } + + s := h.Sites[0] + + p, err := newPageStandalone(&pageMeta{ + s: s, + kind: kindRobotsTXT, + urlPaths: pagemeta.URLPath{ + URL: "robots.txt", + }, + }, + output.RobotsTxtFormat) + + if err != nil { + return err + } + + if !p.render { + return nil + } + + templ := s.lookupLayouts("robots.txt", "_default/robots.txt", "_internal/_default/robots.txt") + + return s.renderAndWritePage(&s.PathSpec.ProcessingStats.Pages, "Robots Txt", "robots.txt", p, templ) +} + func (h *HugoSites) removePageByFilename(filename string) { h.getContentMaps().withMaps(func(m *pageMap) error { m.deleteBundleMatching(func(b *contentNode) bool { diff --git a/hugolib/hugo_sites_build.go b/hugolib/hugo_sites_build.go index f39e7a7e53f..67ee10e0978 100644 --- a/hugolib/hugo_sites_build.go +++ b/hugolib/hugo_sites_build.go @@ -334,7 +334,10 @@ func (h *HugoSites) render(config *BuildCfg) error { } if !config.SkipRender { - if err := h.renderCrossSitesArtifacts(); err != nil { + if err := h.renderCrossSitesSitemap(); err != nil { + return err + } + if err := h.renderCrossSitesRobotsTXT(); err != nil { return err } } diff --git a/hugolib/hugo_sites_build_test.go b/hugolib/hugo_sites_build_test.go index 84655c1f20d..8d0872bd5e0 100644 --- a/hugolib/hugo_sites_build_test.go +++ b/hugolib/hugo_sites_build_test.go @@ -255,8 +255,10 @@ func doTestMultiSitesBuild(t *testing.T, configTemplate, configSuffix string) { b.AssertFileContent("public/fr/404.html", "404|fr|404 Page not found") // Check robots.txt - b.AssertFileContent("public/en/robots.txt", "robots|en|") - b.AssertFileContent("public/nn/robots.txt", "robots|nn|") + // the domain root is the public directory, so the robots.txt has to be created there and not in the language directories + b.AssertFileContent("public/robots.txt", "robots") + b.AssertFileDoesNotExist("public/en/robots.txt") + b.AssertFileDoesNotExist("public/nn/robots.txt") b.AssertFileContent("public/en/sect/doc1-slug/index.html", "Permalink: http://example.com/blog/en/sect/doc1-slug/") b.AssertFileContent("public/en/sect/doc2/index.html", "Permalink: http://example.com/blog/en/sect/doc2/") diff --git a/hugolib/hugo_sites_multihost_test.go b/hugolib/hugo_sites_multihost_test.go index a15e8cd43cb..4fe4960422c 100644 --- a/hugolib/hugo_sites_multihost_test.go +++ b/hugolib/hugo_sites_multihost_test.go @@ -19,6 +19,7 @@ disablePathToLower = true defaultContentLanguage = "fr" defaultContentLanguageInSubdir = false staticDir = ["s1", "s2"] +enableRobotsTXT = true [permalinks] other = "/somewhere/else/:filename" @@ -73,6 +74,12 @@ languageName = "Nynorsk" c.Assert(pageWithURLInFrontMatter.RelPermalink(), qt.Equals, "/docs/superbob/") b.AssertFileContent("public/en/superbob/index.html", "doc3|Hello|en") + // the domain root is the language directory for each language, so the robots.txt is created in the language directories + b.AssertFileContent("public/en/robots.txt", "robots|en") + b.AssertFileContent("public/fr/robots.txt", "robots|fr") + b.AssertFileContent("public/nn/robots.txt", "robots|nn") + b.AssertFileDoesNotExist("public/robots.txt") + // check alias: b.AssertFileContent("public/en/al/alias1/index.html", `content="0; url=https://example.com/docs/superbob/"`) b.AssertFileContent("public/en/al/alias2/index.html", `content="0; url=https://example.com/docs/superbob/"`) diff --git a/hugolib/site.go b/hugolib/site.go index 5507d7a787e..ac65931d0c8 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -1209,8 +1209,10 @@ func (s *Site) render(ctx *siteRenderContext) (err error) { return } - if err = s.renderRobotsTXT(); err != nil { - return + if ctx.multihost { + if err = s.renderRobotsTXT(); err != nil { + return + } } if err = s.render404(); err != nil { diff --git a/hugolib/testhelpers_test.go b/hugolib/testhelpers_test.go index c1ee27557ea..2af4691d173 100644 --- a/hugolib/testhelpers_test.go +++ b/hugolib/testhelpers_test.go @@ -726,6 +726,12 @@ func (s *sitesBuilder) AssertFileContent(filename string, matches ...string) { } } +func (s *sitesBuilder) AssertFileDoesNotExist(filename string) { + if s.CheckExists(filename) { + s.Fatalf("File %q exists but must not exist.", filename) + } +} + func (s *sitesBuilder) AssertImage(width, height int, filename string) { filename = filepath.Join(s.workingDir, filename) f, err := s.Fs.Destination.Open(filename)