From 3466884e364d0df39394742697d6fa0b37d49e64 Mon Sep 17 00:00:00 2001 From: Nico Date: Sat, 20 Jun 2020 15:00:25 +0200 Subject: [PATCH] Create robots.txt in the domain root directory Before a robots.txt is created in every Site. So in public/robots.txt if there are no languages (was correct). But if there are multiple languages in every language directory, too (was wrong). If defaultContentLanguageInSubdir is true, no language is created into the root directory, so no robots.txt is in the root directory (was wrong). If multihosts are configured for each language, that is the only case where one robots.txt must be created in each language directory (was correct). I've changed the behaviour, that only in the multihost case the robots.txt is created in the language directories. In any other case it is created in public/robots.txt. I've also added tests that files are not created in the wrong directories. Fixes #5160 See also #4193 --- hugolib/hugo_sites.go | 35 +++++++++++++++++++++++++++- hugolib/hugo_sites_build.go | 5 +++- hugolib/hugo_sites_build_test.go | 6 +++-- hugolib/hugo_sites_multihost_test.go | 7 ++++++ hugolib/site.go | 6 +++-- hugolib/testhelpers_test.go | 6 +++++ 6 files changed, 59 insertions(+), 6 deletions(-) diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go index e71e48d4124..ee0d5c56368 100644 --- a/hugolib/hugo_sites.go +++ b/hugolib/hugo_sites.go @@ -49,6 +49,7 @@ import ( "github.com/gohugoio/hugo/langs/i18n" "github.com/gohugoio/hugo/resources/page" + "github.com/gohugoio/hugo/resources/page/pagemeta" "github.com/gohugoio/hugo/tpl" "github.com/gohugoio/hugo/tpl/tplimpl" ) @@ -690,7 +691,7 @@ func (cfg *BuildCfg) shouldRender(p *pageState) bool { return false } -func (h *HugoSites) renderCrossSitesArtifacts() error { +func (h *HugoSites) renderCrossSitesSitemap() error { if !h.multilingual.enabled() || h.IsMultihost() { return nil @@ -716,6 +717,38 @@ func (h *HugoSites) renderCrossSitesArtifacts() error { s.siteCfg.sitemap.Filename, h.toSiteInfos(), templ) } +func (h *HugoSites) renderCrossSitesRobotsTXT() error { + if h.multihost { + return nil + } + if !h.Cfg.GetBool("enableRobotsTXT") { + return nil + } + + s := h.Sites[0] + + p, err := newPageStandalone(&pageMeta{ + s: s, + kind: kindRobotsTXT, + urlPaths: pagemeta.URLPath{ + URL: "robots.txt", + }, + }, + output.RobotsTxtFormat) + + if err != nil { + return err + } + + if !p.render { + return nil + } + + templ := s.lookupLayouts("robots.txt", "_default/robots.txt", "_internal/_default/robots.txt") + + return s.renderAndWritePage(&s.PathSpec.ProcessingStats.Pages, "Robots Txt", "robots.txt", p, templ) +} + func (h *HugoSites) removePageByFilename(filename string) { h.getContentMaps().withMaps(func(m *pageMap) error { m.deleteBundleMatching(func(b *contentNode) bool { diff --git a/hugolib/hugo_sites_build.go b/hugolib/hugo_sites_build.go index f39e7a7e53f..67ee10e0978 100644 --- a/hugolib/hugo_sites_build.go +++ b/hugolib/hugo_sites_build.go @@ -334,7 +334,10 @@ func (h *HugoSites) render(config *BuildCfg) error { } if !config.SkipRender { - if err := h.renderCrossSitesArtifacts(); err != nil { + if err := h.renderCrossSitesSitemap(); err != nil { + return err + } + if err := h.renderCrossSitesRobotsTXT(); err != nil { return err } } diff --git a/hugolib/hugo_sites_build_test.go b/hugolib/hugo_sites_build_test.go index 84655c1f20d..8d0872bd5e0 100644 --- a/hugolib/hugo_sites_build_test.go +++ b/hugolib/hugo_sites_build_test.go @@ -255,8 +255,10 @@ func doTestMultiSitesBuild(t *testing.T, configTemplate, configSuffix string) { b.AssertFileContent("public/fr/404.html", "404|fr|404 Page not found") // Check robots.txt - b.AssertFileContent("public/en/robots.txt", "robots|en|") - b.AssertFileContent("public/nn/robots.txt", "robots|nn|") + // the domain root is the public directory, so the robots.txt has to be created there and not in the language directories + b.AssertFileContent("public/robots.txt", "robots") + b.AssertFileDoesNotExist("public/en/robots.txt") + b.AssertFileDoesNotExist("public/nn/robots.txt") b.AssertFileContent("public/en/sect/doc1-slug/index.html", "Permalink: http://example.com/blog/en/sect/doc1-slug/") b.AssertFileContent("public/en/sect/doc2/index.html", "Permalink: http://example.com/blog/en/sect/doc2/") diff --git a/hugolib/hugo_sites_multihost_test.go b/hugolib/hugo_sites_multihost_test.go index a15e8cd43cb..4fe4960422c 100644 --- a/hugolib/hugo_sites_multihost_test.go +++ b/hugolib/hugo_sites_multihost_test.go @@ -19,6 +19,7 @@ disablePathToLower = true defaultContentLanguage = "fr" defaultContentLanguageInSubdir = false staticDir = ["s1", "s2"] +enableRobotsTXT = true [permalinks] other = "/somewhere/else/:filename" @@ -73,6 +74,12 @@ languageName = "Nynorsk" c.Assert(pageWithURLInFrontMatter.RelPermalink(), qt.Equals, "/docs/superbob/") b.AssertFileContent("public/en/superbob/index.html", "doc3|Hello|en") + // the domain root is the language directory for each language, so the robots.txt is created in the language directories + b.AssertFileContent("public/en/robots.txt", "robots|en") + b.AssertFileContent("public/fr/robots.txt", "robots|fr") + b.AssertFileContent("public/nn/robots.txt", "robots|nn") + b.AssertFileDoesNotExist("public/robots.txt") + // check alias: b.AssertFileContent("public/en/al/alias1/index.html", `content="0; url=https://example.com/docs/superbob/"`) b.AssertFileContent("public/en/al/alias2/index.html", `content="0; url=https://example.com/docs/superbob/"`) diff --git a/hugolib/site.go b/hugolib/site.go index 5507d7a787e..ac65931d0c8 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -1209,8 +1209,10 @@ func (s *Site) render(ctx *siteRenderContext) (err error) { return } - if err = s.renderRobotsTXT(); err != nil { - return + if ctx.multihost { + if err = s.renderRobotsTXT(); err != nil { + return + } } if err = s.render404(); err != nil { diff --git a/hugolib/testhelpers_test.go b/hugolib/testhelpers_test.go index c1ee27557ea..2af4691d173 100644 --- a/hugolib/testhelpers_test.go +++ b/hugolib/testhelpers_test.go @@ -726,6 +726,12 @@ func (s *sitesBuilder) AssertFileContent(filename string, matches ...string) { } } +func (s *sitesBuilder) AssertFileDoesNotExist(filename string) { + if s.CheckExists(filename) { + s.Fatalf("File %q exists but must not exist.", filename) + } +} + func (s *sitesBuilder) AssertImage(width, height int, filename string) { filename = filepath.Join(s.workingDir, filename) f, err := s.Fs.Destination.Open(filename)