diff --git a/common/hugio/readers.go b/common/hugio/readers.go new file mode 100644 index 00000000000..ba55e2d08da --- /dev/null +++ b/common/hugio/readers.go @@ -0,0 +1,53 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugio + +import ( + "io" + "strings" +) + +// ReadSeeker wraps io.Reader and io.Seeker. +type ReadSeeker interface { + io.Reader + io.Seeker +} + +// ReadSeekCloser is implemented by afero.File. We use this as the common type for +// content in Resource objects, even for strings. +type ReadSeekCloser interface { + ReadSeeker + io.Closer +} + +// ReadSeekerNoOpCloser implements ReadSeekCloser by doing nothing in Close. +type ReadSeekerNoOpCloser struct { + ReadSeeker +} + +// Close does nothing. +func (r ReadSeekerNoOpCloser) Close() error { + return nil +} + +// NewReadSeekerNoOpCloser creates a new ReadSeekerNoOpCloser with the given ReadSeeker. +func NewReadSeekerNoOpCloser(r ReadSeeker) ReadSeekerNoOpCloser { + return ReadSeekerNoOpCloser{r} +} + +// NewReadSeekerNoOpCloserFromString uses strings.NewReader to create a new ReadSeekerNoOpCloser +// from the given string. +func NewReadSeekerNoOpCloserFromString(content string) ReadSeekerNoOpCloser { + return ReadSeekerNoOpCloser{strings.NewReader(content)} +} diff --git a/common/hugio/writers.go b/common/hugio/writers.go new file mode 100644 index 00000000000..2766146e1e5 --- /dev/null +++ b/common/hugio/writers.go @@ -0,0 +1,43 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugio + +import ( + "io" +) + +type multiWriteCloser struct { + io.Writer + closers []io.WriteCloser +} + +func (m multiWriteCloser) Close() error { + var err error + for _, c := range m.closers { + if closeErr := c.Close(); err != nil { + err = closeErr + } + } + return err +} + +// NewMultiWriteCloser creates a new io.WriteCloser that duplicates its writes to all the +// provided writers. +func NewMultiWriteCloser(writeClosers ...io.WriteCloser) io.WriteCloser { + writers := make([]io.Writer, len(writeClosers)) + for i, w := range writeClosers { + writers[i] = w + } + return multiWriteCloser{Writer: io.MultiWriter(writers...), closers: writeClosers} +} diff --git a/helpers/path.go b/helpers/path.go index 134a2052799..3586c574424 100644 --- a/helpers/path.go +++ b/helpers/path.go @@ -24,6 +24,7 @@ import ( "strings" "unicode" + "github.com/gohugoio/hugo/common/hugio" "github.com/spf13/afero" "golang.org/x/text/transform" "golang.org/x/text/unicode/norm" @@ -515,6 +516,24 @@ func WriteToDisk(inpath string, r io.Reader, fs afero.Fs) (err error) { return afero.WriteReader(fs, inpath, r) } +// OpenFileForWriting opens all the given filenames for writing. +func OpenFilesForWriting(fs afero.Fs, filenames ...string) (io.WriteCloser, error) { + var writeClosers []io.WriteCloser + for _, filename := range filenames { + f, err := OpenFileForWriting(fs, filename) + if err != nil { + for _, wc := range writeClosers { + wc.Close() + } + return nil, err + } + writeClosers = append(writeClosers, f) + } + + return hugio.NewMultiWriteCloser(writeClosers...), nil + +} + // OpenFileForWriting opens or creates the given file. If the target directory // does not exist, it gets created. func OpenFileForWriting(fs afero.Fs, filename string) (afero.File, error) { diff --git a/hugolib/hugo_sites_multihost_test.go b/hugolib/hugo_sites_multihost_test.go index 4062868e6a8..83d6bfc9e9a 100644 --- a/hugolib/hugo_sites_multihost_test.go +++ b/hugolib/hugo_sites_multihost_test.go @@ -81,8 +81,10 @@ languageName = "Nynorsk" s2h := s2.getPage(KindHome) assert.Equal("https://example.fr/", s2h.Permalink()) - b.AssertFileContent("public/fr/index.html", "French Home Page") - b.AssertFileContent("public/en/index.html", "Default Home Page") + b.AssertFileContent("public/fr/index.html", "French Home Page", "String Resource: /docs/text/pipes.txt") + b.AssertFileContent("public/fr/text/pipes.txt", "Hugo Pipes") + b.AssertFileContent("public/en/index.html", "Default Home Page", "String Resource: /docs/text/pipes.txt") + b.AssertFileContent("public/en/text/pipes.txt", "Hugo Pipes") // Check paginators b.AssertFileContent("public/en/page/1/index.html", `refresh" content="0; url=https://example.com/docs/"`) diff --git a/hugolib/page_bundler_handlers.go b/hugolib/page_bundler_handlers.go index e0eac3ac49d..9050052ac85 100644 --- a/hugolib/page_bundler_handlers.go +++ b/hugolib/page_bundler_handlers.go @@ -332,7 +332,7 @@ func (c *contentHandlers) createResource() contentHandler { SourceFile: ctx.source, RelTargetFilename: ctx.target, URLBase: c.s.GetURLLanguageBasePath(), - TargetPathBase: c.s.GetTargetLanguageBasePath(), + TargetBasePaths: []string{c.s.GetTargetLanguageBasePath()}, }) return handlerResult{err: err, handled: true, resource: resource} diff --git a/hugolib/paths/paths.go b/hugolib/paths/paths.go index 3be034fef3e..10acf0cb38b 100644 --- a/hugolib/paths/paths.go +++ b/hugolib/paths/paths.go @@ -53,6 +53,10 @@ type Paths struct { PublishDir string + // When in multihost mode, this returns a list of base paths below PublishDir + // for each language. + MultihostTargetBasePaths []string + DisablePathToLower bool RemovePathAccents bool UglyURLs bool @@ -135,6 +139,15 @@ func New(fs *hugofs.Fs, cfg config.Provider) (*Paths, error) { absResourcesDir = FilePathSeparator } + multilingual := cfg.GetBool("multilingual") + + var multihostTargetBasePaths []string + if multilingual { + for _, l := range languages { + multihostTargetBasePaths = append(multihostTargetBasePaths, l.Lang) + } + } + p := &Paths{ Fs: fs, Cfg: cfg, @@ -154,12 +167,13 @@ func New(fs *hugofs.Fs, cfg config.Provider) (*Paths, error) { themes: config.GetStringSlicePreserveString(cfg, "theme"), - multilingual: cfg.GetBool("multilingual"), + multilingual: multilingual, defaultContentLanguageInSubdir: cfg.GetBool("defaultContentLanguageInSubdir"), DefaultContentLanguage: defaultContentLanguage, - Language: language, - Languages: languages, + Language: language, + Languages: languages, + MultihostTargetBasePaths: multihostTargetBasePaths, PaginatePath: cfg.GetString("paginatePath"), } diff --git a/hugolib/testhelpers_test.go b/hugolib/testhelpers_test.go index 4ba95144987..ec84449ec30 100644 --- a/hugolib/testhelpers_test.go +++ b/hugolib/testhelpers_test.go @@ -403,8 +403,8 @@ date: "2018-02-28" defaultTemplates = []string{ "_default/single.html", "Single: {{ .Title }}|{{ i18n \"hello\" }}|{{.Lang}}|{{ .Content }}", "_default/list.html", "{{ $p := .Paginator }}List Page {{ $p.PageNumber }}: {{ .Title }}|{{ i18n \"hello\" }}|{{ .Permalink }}|Pager: {{ template \"_internal/pagination.html\" . }}", - "index.html", "{{ $p := .Paginator }}Default Home Page {{ $p.PageNumber }}: {{ .Title }}|{{ .IsHome }}|{{ i18n \"hello\" }}|{{ .Permalink }}|{{ .Site.Data.hugo.slogan }}", - "index.fr.html", "{{ $p := .Paginator }}French Home Page {{ $p.PageNumber }}: {{ .Title }}|{{ .IsHome }}|{{ i18n \"hello\" }}|{{ .Permalink }}|{{ .Site.Data.hugo.slogan }}", + "index.html", "{{ $p := .Paginator }}Default Home Page {{ $p.PageNumber }}: {{ .Title }}|{{ .IsHome }}|{{ i18n \"hello\" }}|{{ .Permalink }}|{{ .Site.Data.hugo.slogan }}|String Resource: {{ ( \"Hugo Pipes\" | resources.FromString \"text/pipes.txt\").RelPermalink }}", + "index.fr.html", "{{ $p := .Paginator }}French Home Page {{ $p.PageNumber }}: {{ .Title }}|{{ .IsHome }}|{{ i18n \"hello\" }}|{{ .Permalink }}|{{ .Site.Data.hugo.slogan }}|String Resource: {{ ( \"Hugo Pipes\" | resources.FromString \"text/pipes.txt\").RelPermalink }}", // Shortcodes "shortcodes/shortcode.html", "Shortcode: {{ i18n \"hello\" }}", diff --git a/resource/image.go b/resource/image.go index 57da4f93daf..fd8aea37658 100644 --- a/resource/image.go +++ b/resource/image.go @@ -268,7 +268,7 @@ func (i *Image) doWithImageConfig(action, spec string, f func(src image.Image, c ci.config = image.Config{Width: b.Max.X, Height: b.Max.Y} ci.configLoaded = true - return ci, i.encodeToDestinations(converted, conf, resourceCacheFilename, ci.targetFilename()) + return ci, i.encodeToDestinations(converted, conf, resourceCacheFilename, ci.targetFilenames()...) }) } @@ -447,13 +447,21 @@ func (i *Image) decodeSource() (image.Image, error) { func (i *Image) copyToDestination(src string) error { var res error i.copyToDestinationInit.Do(func() { - target := i.targetFilename() + targetFilenames := i.targetFilenames() + var changedFilenames []string // Fast path: // This is a processed version of the original. // If it exists on destination with the same filename and file size, it is // the same file, so no need to transfer it again. - if fi, err := i.spec.BaseFs.PublishFs.Stat(target); err == nil && fi.Size() == i.osFileInfo.Size() { + for _, targetFilename := range targetFilenames { + if fi, err := i.spec.BaseFs.PublishFs.Stat(targetFilename); err == nil && fi.Size() == i.osFileInfo.Size() { + continue + } + changedFilenames = append(changedFilenames, targetFilename) + } + + if len(changedFilenames) == 0 { return } @@ -464,7 +472,7 @@ func (i *Image) copyToDestination(src string) error { } defer in.Close() - out, err := helpers.OpenFileForWriting(i.spec.BaseFs.PublishFs, target) + out, err := helpers.OpenFilesForWriting(i.spec.BaseFs.PublishFs, changedFilenames...) if err != nil { res = err @@ -485,9 +493,9 @@ func (i *Image) copyToDestination(src string) error { return nil } -func (i *Image) encodeToDestinations(img image.Image, conf imageConfig, resourceCacheFilename, targetFilename string) error { +func (i *Image) encodeToDestinations(img image.Image, conf imageConfig, resourceCacheFilename string, targetFilenames ...string) error { - file1, err := helpers.OpenFileForWriting(i.spec.BaseFs.PublishFs, targetFilename) + file1, err := helpers.OpenFilesForWriting(i.spec.BaseFs.PublishFs, targetFilenames...) if err != nil { return err } diff --git a/resource/image_cache.go b/resource/image_cache.go index fb2996c9deb..470c24c9983 100644 --- a/resource/image_cache.go +++ b/resource/image_cache.go @@ -69,7 +69,7 @@ func (c *imageCache) getOrCreate( parent *Image, conf imageConfig, create func(resourceCacheFilename string) (*Image, error)) (*Image, error) { relTarget := parent.relTargetPathFromConfig(conf) - key := parent.relTargetPathForRel(relTarget.path(), false) + key := parent.relTargetPathForRel(relTarget.path(), false, false) // First check the in-memory store, then the disk. c.mu.RLock() diff --git a/resource/resource.go b/resource/resource.go index a1e29c52f57..01e66078b7e 100644 --- a/resource/resource.go +++ b/resource/resource.go @@ -16,6 +16,7 @@ package resource import ( "errors" "fmt" + "io" "io/ioutil" "mime" "os" @@ -62,8 +63,8 @@ type Source interface { type permalinker interface { relPermalinkFor(target string) string permalinkFor(target string) string - relTargetPathFor(target string) string - relTargetPath() string + relTargetPathsFor(target string) []string + relTargetPaths() []string targetPath() string } @@ -332,10 +333,12 @@ type ResourceSourceDescriptor struct { // Typically the language code if this resource should be published to its sub-folder. URLBase string - // Any base path prepended to the target path. This will also typically be the + // Any base paths prepended to the target path. This will also typically be the // language code, but setting it here means that it should not have any effect on // the permalink. - TargetPathBase string + // This may be several values. In multihost mode we may publish the same resources to + // multiple targets. + TargetBasePaths []string // Delay publishing until either Permalink or RelPermalink is called. Maybe never. LazyPublish bool @@ -373,6 +376,11 @@ func (r *Spec) newResourceForFs(sourceFs afero.Fs, fd ResourceSourceDescriptor) fd.RelTargetFilename = fd.Filename() } + if len(fd.TargetBasePaths) == 0 { + // If not set, we publish the same resource to all hosts. + fd.TargetBasePaths = r.MultihostTargetBasePaths + } + return r.newResource(sourceFs, fd) } @@ -418,7 +426,7 @@ func (r *Spec) newResource(sourceFs afero.Fs, fd ResourceSourceDescriptor) (Reso fd.LazyPublish, fd.OpenReadSeekCloser, fd.URLBase, - fd.TargetPathBase, + fd.TargetBasePaths, fd.TargetPathBuilder, fi, sourceFilename, @@ -505,8 +513,8 @@ type resourcePathDescriptor struct { baseURLDir string // This will normally be the same as above, but this will only apply to publishing - // of resources. - baseTargetPathDir string + // of resources. It may be mulltiple values when in multihost mode. + baseTargetPathDirs []string // baseOffset is set when the output format's path has a offset, e.g. for AMP. baseOffset string @@ -688,12 +696,12 @@ func (l *genericResource) permalinkFor(target string) string { return l.spec.PermalinkForBaseURL(l.relPermalinkForRel(target), l.spec.BaseURL.HostURL()) } -func (l *genericResource) relTargetPathFor(target string) string { - return l.relTargetPathForRel(target, false) +func (l *genericResource) relTargetPathsFor(target string) []string { + return l.relTargetPathsForRel(target) } -func (l *genericResource) relTargetPath() string { - return l.relTargetPathForRel(l.targetPath(), false) +func (l *genericResource) relTargetPaths() []string { + return l.relTargetPathsForRel(l.targetPath()) } func (l *genericResource) Name() string { @@ -731,11 +739,34 @@ func (l *genericResource) updateParams(params map[string]interface{}) { } func (l *genericResource) relPermalinkForRel(rel string) string { - return l.spec.PathSpec.URLizeFilename(l.relTargetPathForRel(rel, true)) + return l.spec.PathSpec.URLizeFilename(l.relTargetPathForRel(rel, false, true)) +} + +func (l *genericResource) relTargetPathsForRel(rel string) []string { + if len(l.baseTargetPathDirs) == 0 { + return []string{l.relTargetPathForRelAndBasePath(rel, "", false)} + } + + var targetPaths = make([]string, len(l.baseTargetPathDirs)) + for i, dir := range l.baseTargetPathDirs { + targetPaths[i] = l.relTargetPathForRelAndBasePath(rel, dir, false) + } + return targetPaths } -func (l *genericResource) relTargetPathForRel(rel string, isURL bool) string { +func (l *genericResource) relTargetPathForRel(rel string, addBaseTargetPath, isURL bool) string { + if addBaseTargetPath && len(l.baseTargetPathDirs) > 1 { + panic("multiple baseTargetPathDirs") + } + var basePath string + if addBaseTargetPath && len(l.baseTargetPathDirs) > 0 { + basePath = l.baseTargetPathDirs[0] + } + return l.relTargetPathForRelAndBasePath(rel, basePath, isURL) +} + +func (l *genericResource) relTargetPathForRelAndBasePath(rel, basePath string, isURL bool) string { if l.targetPathBuilder != nil { rel = l.targetPathBuilder(rel) } @@ -744,8 +775,8 @@ func (l *genericResource) relTargetPathForRel(rel string, isURL bool) string { rel = path.Join(l.baseURLDir, rel) } - if !isURL && l.baseTargetPathDir != "" { - rel = path.Join(l.baseTargetPathDir, rel) + if basePath != "" { + rel = path.Join(basePath, rel) } if l.baseOffset != "" { @@ -772,12 +803,19 @@ func (l *genericResource) String() string { } func (l *genericResource) Publish() error { - f, err := l.ReadSeekCloser() + fr, err := l.ReadSeekCloser() if err != nil { return err } - defer f.Close() - return helpers.WriteToDisk(l.targetFilename(), f, l.spec.BaseFs.PublishFs) + defer fr.Close() + fw, err := helpers.OpenFilesForWriting(l.spec.BaseFs.PublishFs, l.targetFilenames()...) + if err != nil { + return err + } + defer fw.Close() + + _, err = io.Copy(fw, fr) + return err } // Path is stored with Unix style slashes. @@ -785,8 +823,12 @@ func (l *genericResource) targetPath() string { return l.relTargetDirFile.path() } -func (l *genericResource) targetFilename() string { - return filepath.Clean(l.relTargetPath()) +func (l *genericResource) targetFilenames() []string { + paths := l.relTargetPaths() + for i, p := range paths { + paths[i] = filepath.Clean(p) + } + return paths } // TODO(bep) clean up below @@ -801,7 +843,7 @@ func (r *Spec) newGenericResource(sourceFs afero.Fs, false, nil, "", - "", + nil, targetPathBuilder, osFileInfo, sourceFilename, @@ -816,7 +858,7 @@ func (r *Spec) newGenericResourceWithBase( lazyPublish bool, openReadSeekerCloser OpenReadSeekCloser, urlBaseDir string, - targetPathBaseDir string, + targetPathBaseDirs []string, targetPathBuilder func(base string) string, osFileInfo os.FileInfo, sourceFilename, @@ -836,10 +878,10 @@ func (r *Spec) newGenericResourceWithBase( } pathDescriptor := resourcePathDescriptor{ - baseURLDir: urlBaseDir, - baseTargetPathDir: targetPathBaseDir, - targetPathBuilder: targetPathBuilder, - relTargetDirFile: dirFile{dir: fpath, file: fname}, + baseURLDir: urlBaseDir, + baseTargetPathDirs: targetPathBaseDirs, + targetPathBuilder: targetPathBuilder, + relTargetDirFile: dirFile{dir: fpath, file: fname}, } var po *publishOnce diff --git a/resource/resource_test.go b/resource/resource_test.go index e699e6f3fcf..b76f0a604c8 100644 --- a/resource/resource_test.go +++ b/resource/resource_test.go @@ -93,7 +93,7 @@ func TestNewResourceFromFilenameSubPathInBaseURL(t *testing.T) { assert.Equal("/docs/a/b/logo.png", r.RelPermalink()) assert.Equal("https://example.com/docs/a/b/logo.png", r.Permalink()) img := r.(*Image) - assert.Equal(filepath.FromSlash("/a/b/logo.png"), img.targetFilename()) + assert.Equal(filepath.FromSlash("/a/b/logo.png"), img.targetFilenames()[0]) } diff --git a/resource/transform.go b/resource/transform.go index c61a9771e35..dd9cd143bdb 100644 --- a/resource/transform.go +++ b/resource/transform.go @@ -267,7 +267,7 @@ func (r *transformedResource) initContent() error { func (r *transformedResource) transform(setContent bool) (err error) { openPublishFileForWriting := func(relTargetPath string) (io.WriteCloser, error) { - return helpers.OpenFileForWriting(r.cache.rs.PublishFs, r.linker.relTargetPathFor(relTargetPath)) + return helpers.OpenFilesForWriting(r.cache.rs.PublishFs, r.linker.relTargetPathsFor(relTargetPath)...) } // This can be the last resource in a chain. @@ -299,7 +299,7 @@ func (r *transformedResource) transform(setContent bool) (err error) { key = key + "_" + v.transformation.Key().key() case permalinker: r.linker = v - p := v.relTargetPath() + p := v.targetPath() if p == "" { panic("target path needed for key creation") }