From 40072a4966e49b23c518f9f5c66c16de542ee0ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Fri, 3 May 2019 09:16:58 +0200 Subject: [PATCH] Add Hugo Modules Fixes #5973 Fixes #5996 Fixes #6010 Fixes #5911 Fixes #5940 Fixes #6074 Fixes #6082 Fixes #6092 --- benchbep.sh | 3 +- cache/filecache/filecache.go | 2 +- cache/filecache/filecache_config_test.go | 37 +- cache/filecache/filecache_pruner_test.go | 3 +- cache/filecache/filecache_test.go | 7 +- commands/commandeer.go | 31 +- commands/commands.go | 20 +- commands/convert.go | 5 +- commands/hugo.go | 178 +--- commands/import_jekyll.go | 71 +- commands/mod.go | 188 +++++ commands/new.go | 2 - commands/server.go | 2 +- common/herrors/errors.go | 11 + common/hugio/copy.go | 96 +++ common/hugo/version.go | 14 + common/hugo/version_test.go | 6 + common/maps/params.go | 96 +++ common/maps/params_test.go | 45 + config/configProvider.go | 8 +- config/env.go | 24 + config/env_test.go | 32 + create/content.go | 44 +- create/content_template_handler.go | 5 +- create/content_test.go | 45 +- deps/deps.go | 6 +- go.mod | 5 + go.sum | 8 + helpers/docshelper.go | 3 +- helpers/general.go | 57 +- helpers/general_test.go | 70 ++ helpers/path.go | 78 +- helpers/path_test.go | 13 +- helpers/pathspec_test.go | 9 +- helpers/testhelpers_test.go | 9 + htesting/test_helpers.go | 39 + htesting/test_structs.go | 100 --- hugofs/basepath_real_filename_fs.go | 91 -- hugofs/decorators.go | 202 +++++ hugofs/fileinfo.go | 283 +++++++ hugofs/files/classifier.go | 123 +++ hugofs/files/classifier_test.go | 49 ++ hugofs/filter_fs.go | 341 ++++++++ hugofs/fs.go | 6 +- hugofs/language_composite_fs.go | 40 +- hugofs/language_composite_fs_test.go | 107 --- hugofs/language_fs.go | 346 -------- hugofs/language_fs_test.go | 100 --- hugofs/nosymlink_fs.go | 85 ++ hugofs/nosymlink_test.go | 97 +++ hugofs/rootmapping_fs.go | 457 +++++++--- hugofs/rootmapping_fs_test.go | 202 ++++- hugofs/slice_fs.go | 293 +++++++ hugofs/{nolstat_fs.go => slice_fs_test.go} | 25 - hugofs/walk.go | 317 +++++++ hugofs/walk_test.go | 225 +++++ hugolib/config.go | 288 +++---- hugolib/config_test.go | 154 +++- hugolib/data/hugo.toml | 1 + hugolib/disableKinds_test.go | 89 +- hugolib/fileInfo.go | 77 +- hugolib/fileInfo_test.go | 1 + hugolib/filesystems/basefs.go | 698 +++++++--------- hugolib/filesystems/basefs_test.go | 183 +++-- hugolib/hugo_modules_test.go | 382 +++++++++ hugolib/hugo_sites.go | 198 +++-- hugolib/hugo_sites_build.go | 8 +- hugolib/hugo_sites_build_errors_test.go | 3 +- hugolib/hugo_sites_build_test.go | 55 +- hugolib/hugo_themes_test.go | 268 ------ hugolib/language_content_dir_test.go | 11 +- hugolib/menu_test.go | 44 +- hugolib/multilingual.go | 56 -- hugolib/page.go | 18 +- hugolib/page__meta.go | 13 +- hugolib/page_permalink_test.go | 3 +- hugolib/page_test.go | 52 +- hugolib/pagebundler.go | 206 ----- hugolib/pagebundler_capture.go | 773 ----------------- hugolib/pagebundler_capture_test.go | 272 ------ hugolib/pagebundler_handlers.go | 305 ------- hugolib/pagebundler_test.go | 384 +++++---- hugolib/pagecollections.go | 1 - hugolib/pages_capture.go | 777 ++++++++++++++++++ hugolib/pages_capture_test.go | 87 ++ hugolib/paths/paths.go | 62 +- hugolib/paths/paths_test.go | 9 +- hugolib/paths/themes.go | 154 ---- hugolib/resource_chain_test.go | 66 +- hugolib/shortcode_test.go | 70 +- hugolib/site.go | 104 +-- hugolib/site_output_test.go | 47 +- hugolib/site_sections_test.go | 3 +- hugolib/site_stats_test.go | 23 +- hugolib/site_test.go | 5 +- hugolib/taxonomy_test.go | 54 +- hugolib/template_engines_test.go | 3 +- hugolib/template_test.go | 23 +- hugolib/testhelpers_test.go | 267 ++++-- langs/config.go | 217 +++++ langs/i18n/i18n_test.go | 13 +- langs/i18n/translationProvider.go | 56 +- langs/language.go | 25 +- magefile.go | 21 +- modules/client.go | 537 ++++++++++++ modules/client_test.go | 103 +++ modules/collect.go | 496 +++++++++++ modules/collect_test.go | 38 + modules/config.go | 279 +++++++ modules/config_test.go | 95 +++ modules/module.go | 166 ++++ output/layout.go | 2 +- parser/metadecoders/decoder.go | 24 + parser/metadecoders/decoder_test.go | 32 + resources/image_cache.go | 2 +- resources/page/page_nop.go | 5 +- resources/page/page_wrappers.autogen.go | 4 +- resources/page/pagemeta/page_frontmatter.go | 2 +- resources/page/permalinks.go | 5 + resources/page/site.go | 71 ++ resources/page/testhelpers_test.go | 11 +- resources/page/zero_file.autogen.go | 4 +- resources/resource.go | 49 +- resources/resource/params.go | 63 +- .../resource_factories/bundler/bundler.go | 4 +- resources/resource_factories/create/create.go | 12 +- resources/resource_test.go | 12 +- .../resource_transformers/postcss/postcss.go | 4 +- .../tocss/scss/client.go | 3 +- .../resource_transformers/tocss/scss/tocss.go | 11 +- resources/sunset.jpg | Bin 0 -> 90587 bytes resources/testhelpers_test.go | 58 +- resources/transform.go | 3 + source/fileInfo.go | 82 +- source/fileInfo_test.go | 57 +- source/filesystem.go | 79 +- source/filesystem_test.go | 48 +- source/sourceSpec.go | 13 + tpl/cast/docshelper.go | 10 +- tpl/data/init_test.go | 2 + tpl/data/resources_test.go | 15 +- tpl/hugo/init_test.go | 8 +- tpl/os/os.go | 26 +- tpl/site/init_test.go | 8 +- tpl/template.go | 7 +- tpl/tplimpl/embedded/templates.autogen.go | 3 +- tpl/tplimpl/template.go | 6 +- tpl/tplimpl/template_funcs_test.go | 14 +- 148 files changed, 8834 insertions(+), 5029 deletions(-) create mode 100644 commands/mod.go create mode 100644 common/hugio/copy.go create mode 100644 common/maps/params.go create mode 100644 common/maps/params_test.go create mode 100644 config/env_test.go create mode 100644 htesting/test_helpers.go delete mode 100644 htesting/test_structs.go delete mode 100644 hugofs/basepath_real_filename_fs.go create mode 100644 hugofs/decorators.go create mode 100644 hugofs/fileinfo.go create mode 100644 hugofs/files/classifier.go create mode 100644 hugofs/files/classifier_test.go create mode 100644 hugofs/filter_fs.go delete mode 100644 hugofs/language_composite_fs_test.go delete mode 100644 hugofs/language_fs.go delete mode 100644 hugofs/language_fs_test.go create mode 100644 hugofs/nosymlink_fs.go create mode 100644 hugofs/nosymlink_test.go create mode 100644 hugofs/slice_fs.go rename hugofs/{nolstat_fs.go => slice_fs_test.go} (58%) create mode 100644 hugofs/walk.go create mode 100644 hugofs/walk_test.go create mode 100755 hugolib/data/hugo.toml create mode 100644 hugolib/hugo_modules_test.go delete mode 100644 hugolib/hugo_themes_test.go delete mode 100644 hugolib/pagebundler.go delete mode 100644 hugolib/pagebundler_capture.go delete mode 100644 hugolib/pagebundler_capture_test.go delete mode 100644 hugolib/pagebundler_handlers.go create mode 100644 hugolib/pages_capture.go create mode 100644 hugolib/pages_capture_test.go delete mode 100644 hugolib/paths/themes.go create mode 100644 langs/config.go create mode 100644 modules/client.go create mode 100644 modules/client_test.go create mode 100644 modules/collect.go create mode 100644 modules/collect_test.go create mode 100644 modules/config.go create mode 100644 modules/config_test.go create mode 100644 modules/module.go create mode 100644 resources/sunset.jpg diff --git a/benchbep.sh b/benchbep.sh index fabd30c189b..efd616c8859 100755 --- a/benchbep.sh +++ b/benchbep.sh @@ -1,2 +1 @@ -gobench -package=./hugolib -bench="BenchmarkSiteBuilding/YAML,num_langs=3,num_pages=5000,tags_per_page=5,shortcodes,render" -count=3 > 1.bench -benchcmp -best 0.bench 1.bench \ No newline at end of file +gobench -package=./hugolib -bench="BenchmarkSiteNew/Deep_content_tree" \ No newline at end of file diff --git a/cache/filecache/filecache.go b/cache/filecache/filecache.go index 6ad417117e0..86c93e3eb85 100644 --- a/cache/filecache/filecache.go +++ b/cache/filecache/filecache.go @@ -319,7 +319,7 @@ func NewCaches(p *helpers.PathSpec) (Caches, error) { var cfs afero.Fs if v.isResourceDir { - cfs = p.BaseFs.Resources.Fs + cfs = p.BaseFs.ResourcesCache } else { cfs = fs } diff --git a/cache/filecache/filecache_config_test.go b/cache/filecache/filecache_config_test.go index b0f5d2dc0b3..4da2417f276 100644 --- a/cache/filecache/filecache_config_test.go +++ b/cache/filecache/filecache_config_test.go @@ -20,6 +20,10 @@ import ( "testing" "time" + "github.com/gohugoio/hugo/modules" + + "github.com/gohugoio/hugo/langs" + "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/config" @@ -55,8 +59,9 @@ dir = "/path/to/c3" ` - cfg, err := config.FromConfigString(configStr, "toml") + cfg, err := configFromString(configStr) assert.NoError(err) + fs := hugofs.NewMem(cfg) p, err := helpers.NewPathSpec(fs, cfg) assert.NoError(err) @@ -76,6 +81,23 @@ dir = "/path/to/c3" } +func configFromString(configStr string) (config.Provider, error) { + cfg, err := config.FromConfigString(configStr, "toml") + if err != nil { + return nil, err + } + if _, err := langs.LoadLanguageSettings(cfg, nil); err != nil { + return nil, err + } + mod, err := modules.CreateProjectModule(cfg) + if err != nil { + return nil, err + } + cfg.Set("allModules", modules.Modules{mod}) + + return cfg, nil +} + func TestDecodeConfigIgnoreCache(t *testing.T) { t.Parallel() @@ -103,8 +125,7 @@ dir = "/path/to/c3" ` - cfg, err := config.FromConfigString(configStr, "toml") - assert.NoError(err) + cfg, err := configFromString(configStr) fs := hugofs.NewMem(cfg) p, err := helpers.NewPathSpec(fs, cfg) assert.NoError(err) @@ -181,8 +202,7 @@ dir = "/" configStr = strings.Replace(configStr, "/", "c:\\\\", 1) } - cfg, err := config.FromConfigString(configStr, "toml") - assert.NoError(err) + cfg, err := configFromString(configStr) fs := hugofs.NewMem(cfg) p, err := helpers.NewPathSpec(fs, cfg) assert.NoError(err) @@ -203,5 +223,12 @@ func newTestConfig() *viper.Viper { cfg.Set("archetypeDir", "archetypes") cfg.Set("assetDir", "assets") + langs.LoadLanguageSettings(cfg, nil) + mod, err := modules.CreateProjectModule(cfg) + if err != nil { + panic(err) + } + cfg.Set("allModules", modules.Modules{mod}) + return cfg } diff --git a/cache/filecache/filecache_pruner_test.go b/cache/filecache/filecache_pruner_test.go index e62a6315a74..047bc5d0aca 100644 --- a/cache/filecache/filecache_pruner_test.go +++ b/cache/filecache/filecache_pruner_test.go @@ -18,7 +18,6 @@ import ( "testing" "time" - "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/hugofs" @@ -54,7 +53,7 @@ maxAge = "200ms" dir = ":resourceDir/_gen" ` - cfg, err := config.FromConfigString(configStr, "toml") + cfg, err := configFromString(configStr) assert.NoError(err) for _, name := range []string{cacheKeyGetCSV, cacheKeyGetJSON, cacheKeyAssets, cacheKeyImages} { diff --git a/cache/filecache/filecache_test.go b/cache/filecache/filecache_test.go index 5ac2e9beb81..c97ba5ae614 100644 --- a/cache/filecache/filecache_test.go +++ b/cache/filecache/filecache_test.go @@ -26,7 +26,6 @@ import ( "time" "github.com/gohugoio/hugo/common/hugio" - "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/hugofs" @@ -83,8 +82,7 @@ dir = ":cacheDir/c" configStr = replacer.Replace(configStr) configStr = strings.Replace(configStr, "\\", winPathSep, -1) - cfg, err := config.FromConfigString(configStr, "toml") - assert.NoError(err) + cfg, err := configFromString(configStr) fs := hugofs.NewFrom(osfs, cfg) p, err := helpers.NewPathSpec(fs, cfg) @@ -207,8 +205,7 @@ dir = "/cache/c" ` - cfg, err := config.FromConfigString(configStr, "toml") - assert.NoError(err) + cfg, err := configFromString(configStr) fs := hugofs.NewMem(cfg) p, err := helpers.NewPathSpec(fs, cfg) assert.NoError(err) diff --git a/commands/commandeer.go b/commands/commandeer.go index 8c9da53b92b..a3d42e8b269 100644 --- a/commands/commandeer.go +++ b/commands/commandeer.go @@ -17,6 +17,8 @@ import ( "bytes" "errors" + "github.com/gohugoio/hugo/modules" + "io/ioutil" "github.com/gohugoio/hugo/common/herrors" @@ -27,7 +29,6 @@ import ( "os" "path/filepath" "regexp" - "strings" "sync" "time" @@ -290,7 +291,7 @@ func (c *commandeer) loadConfig(mustHaveConfigFile, running bool) error { if mustHaveConfigFile { return err } - if err != hugolib.ErrNoConfigFile { + if err != hugolib.ErrNoConfigFile && !modules.IsNotExist(err) { return err } @@ -388,21 +389,23 @@ func (c *commandeer) loadConfig(mustHaveConfigFile, running bool) error { cfg.Logger.INFO.Println("Using config file:", config.ConfigFileUsed()) - themeDir := c.hugo.PathSpec.GetFirstThemeDir() - if themeDir != "" { - if _, err := sourceFs.Stat(themeDir); os.IsNotExist(err) { - return newSystemError("Unable to find theme Directory:", themeDir) + // TODO(bep) mod + /* + themeDir := c.hugo.PathSpec.GetFirstThemeDir() + if themeDir != "" { + if _, err := sourceFs.Stat(themeDir); os.IsNotExist(err) { + return newSystemError("Unable to find theme Directory:", themeDir) + } } - } - dir, themeVersionMismatch, minVersion := c.isThemeVsHugoVersionMismatch(sourceFs) - - if themeVersionMismatch { - name := filepath.Base(dir) - cfg.Logger.ERROR.Printf("%s theme does not support Hugo version %s. Minimum version required is %s\n", - strings.ToUpper(name), hugo.CurrentVersion.ReleaseVersion(), minVersion) - } + dir, themeVersionMismatch, minVersion := c.isThemeVsHugoVersionMismatch(sourceFs) + if themeVersionMismatch { + name := filepath.Base(dir) + cfg.Logger.ERROR.Printf("%s theme does not support Hugo version %s. Minimum version required is %s\n", + strings.ToUpper(name), hugo.CurrentVersion.ReleaseVersion(), minVersion) + } + */ return nil } diff --git a/commands/commands.go b/commands/commands.go index 51bfb47635f..f1c9dc824dd 100644 --- a/commands/commands.go +++ b/commands/commands.go @@ -54,6 +54,7 @@ func (b *commandsBuilder) addAll() *commandsBuilder { newImportCmd(), newGenCmd(), createReleaser(), + b.newModCmd(), ) return b @@ -189,9 +190,10 @@ Complete documentation is available at http://gohugo.io/.`, } type hugoBuilderCommon struct { - source string - baseURL string - environment string + source string + baseURL string + environment string + ignoreVendor bool buildWatch bool @@ -243,20 +245,26 @@ func (cc *hugoBuilderCommon) getEnvironment(isServer bool) string { return hugo.EnvironmentProduction } +func (cc *hugoBuilderCommon) handleCommonBuilderFlags(cmd *cobra.Command) { + cmd.PersistentFlags().StringVarP(&cc.source, "source", "s", "", "filesystem path to read files relative from") + cmd.PersistentFlags().SetAnnotation("source", cobra.BashCompSubdirsInDir, []string{}) + cmd.PersistentFlags().StringVarP(&cc.environment, "environment", "e", "", "build environment") + cmd.PersistentFlags().StringP("themesDir", "", "", "filesystem path to themes directory") + cmd.PersistentFlags().BoolP("ignoreVendor", "", false, "ignores any _vendor directory") +} + func (cc *hugoBuilderCommon) handleFlags(cmd *cobra.Command) { + cc.handleCommonBuilderFlags(cmd) cmd.Flags().Bool("cleanDestinationDir", false, "remove files from destination not found in static directories") cmd.Flags().BoolP("buildDrafts", "D", false, "include content marked as draft") cmd.Flags().BoolP("buildFuture", "F", false, "include content with publishdate in the future") cmd.Flags().BoolP("buildExpired", "E", false, "include expired content") - cmd.Flags().StringVarP(&cc.source, "source", "s", "", "filesystem path to read files relative from") - cmd.Flags().StringVarP(&cc.environment, "environment", "e", "", "build environment") cmd.Flags().StringP("contentDir", "c", "", "filesystem path to content directory") cmd.Flags().StringP("layoutDir", "l", "", "filesystem path to layout directory") cmd.Flags().StringP("cacheDir", "", "", "filesystem path to cache directory. Defaults: $TMPDIR/hugo_cache/") cmd.Flags().BoolP("ignoreCache", "", false, "ignores the cache directory") cmd.Flags().StringP("destination", "d", "", "filesystem path to write files to") cmd.Flags().StringSliceP("theme", "t", []string{}, "themes to use (located in /themes/THEMENAME/)") - cmd.Flags().StringP("themesDir", "", "", "filesystem path to themes directory") cmd.Flags().StringVarP(&cc.baseURL, "baseURL", "b", "", "hostname (and path) to the root, e.g. http://spf13.com/") cmd.Flags().Bool("enableGitInfo", false, "add Git revision, date and author info to the pages") cmd.Flags().BoolVar(&cc.gc, "gc", false, "enable to run some cleanup tasks (remove unused cache files) after the build") diff --git a/commands/convert.go b/commands/convert.go index d0a46a6417b..e4ff1ac61c5 100644 --- a/commands/convert.go +++ b/commands/convert.go @@ -30,7 +30,6 @@ import ( "github.com/gohugoio/hugo/parser/metadecoders" "github.com/gohugoio/hugo/parser/pageparser" - src "github.com/gohugoio/hugo/source" "github.com/pkg/errors" "github.com/gohugoio/hugo/hugolib" @@ -152,8 +151,8 @@ func (cc *convertCmd) convertAndSavePage(p page.Page, site *hugolib.Site, target site.Log.INFO.Println("Attempting to convert", p.File().Filename()) - f, _ := p.File().(src.ReadableFile) - file, err := f.Open() + f := p.File() + file, err := f.FileInfo().Meta().Open() if err != nil { site.Log.ERROR.Println(errMsg) file.Close() diff --git a/commands/hugo.go b/commands/hugo.go index 07f2b95a2bb..6836ec934ad 100644 --- a/commands/hugo.go +++ b/commands/hugo.go @@ -21,14 +21,12 @@ import ( "os/signal" "runtime/pprof" "runtime/trace" - "sort" "sync/atomic" "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/resources/page" - "github.com/gohugoio/hugo/common/hugo" "github.com/pkg/errors" "github.com/gohugoio/hugo/common/herrors" @@ -49,7 +47,6 @@ import ( "github.com/gohugoio/hugo/config" - "github.com/gohugoio/hugo/parser/metadecoders" flag "github.com/spf13/pflag" "github.com/fsnotify/fsnotify" @@ -196,6 +193,7 @@ func initializeFlags(cmd *cobra.Command, cfg config.Provider) { "forceSyncStatic", "noTimes", "noChmod", + "ignoreVendor", "templateMetrics", "templateMetricsHints", @@ -309,13 +307,9 @@ func (c *commandeer) fullBuild() error { cnt, err := c.copyStatic() if err != nil { - if !os.IsNotExist(err) { - return errors.Wrap(err, "Error copying static files") - } - c.logger.INFO.Println("No Static directory found") + return errors.Wrap(err, "Error copying static files") } langCount = cnt - langCount = cnt return nil } buildSitesFunc := func() error { @@ -547,7 +541,11 @@ func (c *commandeer) serverBuild() error { } func (c *commandeer) copyStatic() (map[string]uint64, error) { - return c.doWithPublishDirs(c.copyStaticTo) + m, err := c.doWithPublishDirs(c.copyStaticTo) + if err == nil || os.IsNotExist(err) { + return m, nil + } + return m, err } func (c *commandeer) doWithPublishDirs(f func(sourceFs *filesystems.SourceFilesystem) (uint64, error)) (map[string]uint64, error) { @@ -566,6 +564,7 @@ func (c *commandeer) doWithPublishDirs(f func(sourceFs *filesystems.SourceFilesy if err != nil { return langCount, err } + if lang == "" { // Not multihost for _, l := range c.languages { @@ -609,7 +608,8 @@ func (c *commandeer) copyStaticTo(sourceFs *filesystems.SourceFilesystem) (uint6 syncer := fsync.NewSyncer() syncer.NoTimes = c.Cfg.GetBool("noTimes") - syncer.NoChmod = c.Cfg.GetBool("noChmod") + // TODO(bep) mod Go module cache has 0555 directories. + syncer.NoChmod = true // c.Cfg.GetBool("noChmod") syncer.SrcFs = fs syncer.DestFs = c.Fs.Destination // Now that we are using a unionFs for the static directories @@ -652,120 +652,39 @@ func (c *commandeer) timeTrack(start time.Time, name string) { // getDirList provides NewWatcher() with a list of directories to watch for changes. func (c *commandeer) getDirList() ([]string, error) { - var a []string - - // To handle nested symlinked content dirs - var seen = make(map[string]bool) - var nested []string - - newWalker := func(allowSymbolicDirs bool) func(path string, fi os.FileInfo, err error) error { - return func(path string, fi os.FileInfo, err error) error { - if err != nil { - if os.IsNotExist(err) { - return nil - } - - c.logger.ERROR.Println("Walker: ", err) - return nil - } - - // Skip .git directories. - // Related to https://github.com/gohugoio/hugo/issues/3468. - if fi.Name() == ".git" { - return nil - } - - if fi.Mode()&os.ModeSymlink == os.ModeSymlink { - link, err := filepath.EvalSymlinks(path) - if err != nil { - c.logger.ERROR.Printf("Cannot read symbolic link '%s', error was: %s", path, err) - return nil - } - linkfi, err := helpers.LstatIfPossible(c.Fs.Source, link) - if err != nil { - c.logger.ERROR.Printf("Cannot stat %q: %s", link, err) - return nil - } - if !allowSymbolicDirs && !linkfi.Mode().IsRegular() { - c.logger.ERROR.Printf("Symbolic links for directories not supported, skipping %q", path) - return nil - } + var dirnames []string - if allowSymbolicDirs && linkfi.IsDir() { - // afero.Walk will not walk symbolic links, so wee need to do it. - if !seen[path] { - seen[path] = true - nested = append(nested, path) - } - return nil - } - - fi = linkfi - } - - if fi.IsDir() { - if fi.Name() == ".git" || - fi.Name() == "node_modules" || fi.Name() == "bower_components" { - return filepath.SkipDir - } - a = append(a, path) - } + walkFn := func(path string, fi hugofs.FileMetaInfo, err error) error { + if err != nil { + c.logger.ERROR.Println("walker: ", err) return nil } - } - - symLinkWalker := newWalker(true) - regularWalker := newWalker(false) - - // SymbolicWalk will log anny ERRORs - // Also note that the Dirnames fetched below will contain any relevant theme - // directories. - for _, contentDir := range c.hugo.PathSpec.BaseFs.Content.Dirnames { - _ = helpers.SymbolicWalk(c.Fs.Source, contentDir, symLinkWalker) - } - - for _, staticDir := range c.hugo.PathSpec.BaseFs.Data.Dirnames { - _ = helpers.SymbolicWalk(c.Fs.Source, staticDir, regularWalker) - } - - for _, staticDir := range c.hugo.PathSpec.BaseFs.I18n.Dirnames { - _ = helpers.SymbolicWalk(c.Fs.Source, staticDir, regularWalker) - } - for _, staticDir := range c.hugo.PathSpec.BaseFs.Layouts.Dirnames { - _ = helpers.SymbolicWalk(c.Fs.Source, staticDir, regularWalker) - } + if fi.IsDir() { + if fi.Name() == ".git" || + fi.Name() == "node_modules" || fi.Name() == "bower_components" { + return filepath.SkipDir + } - for _, staticFilesystem := range c.hugo.PathSpec.BaseFs.Static { - for _, staticDir := range staticFilesystem.Dirnames { - _ = helpers.SymbolicWalk(c.Fs.Source, staticDir, regularWalker) + dirnames = append(dirnames, fi.Meta().Filename()) } - } - - for _, assetDir := range c.hugo.PathSpec.BaseFs.Assets.Dirnames { - _ = helpers.SymbolicWalk(c.Fs.Source, assetDir, regularWalker) - } - if len(nested) > 0 { - for { + return nil - toWalk := nested - nested = nested[:0] + } - for _, d := range toWalk { - _ = helpers.SymbolicWalk(c.Fs.Source, d, symLinkWalker) - } + watchDirs := c.hugo.PathSpec.BaseFs.WatchDirs() + for _, watchDir := range watchDirs { - if len(nested) == 0 { - break - } + w := hugofs.NewWalkway(hugofs.WalkwayConfig{Logger: c.logger, Info: watchDir, WalkFn: walkFn}) + if err := w.Walk(); err != nil { + c.logger.ERROR.Println("walker: ", err) } } - a = helpers.UniqueStrings(a) - sort.Strings(a) + dirnames = helpers.UniqueStringsSorted(dirnames) - return a, nil + return dirnames, nil } func (c *commandeer) buildSites() (err error) { @@ -825,7 +744,13 @@ func (c *commandeer) fullRebuild() { } if !c.paused { - err := c.buildSites() + _, err := c.copyStatic() + if err != nil { + c.logger.ERROR.Println(err) + return + } + + err = c.buildSites() if err != nil { c.logger.ERROR.Println(err) } else if !c.h.buildWatch && !c.Cfg.GetBool("disableLiveReload") { @@ -1015,7 +940,7 @@ func (c *commandeer) handleEvents(watcher *watcher.Batcher, continue } - walkAdder := func(path string, f os.FileInfo, err error) error { + walkAdder := func(path string, f hugofs.FileMetaInfo, err error) error { if f.IsDir() { c.logger.FEEDBACK.Println("adding created directory to watchlist", path) if err := watcher.Add(path); err != nil { @@ -1170,38 +1095,11 @@ func pickOneWriteOrCreatePath(events []fsnotify.Event) string { // isThemeVsHugoVersionMismatch returns whether the current Hugo version is // less than any of the themes' min_version. +// TODO(bep) mod func (c *commandeer) isThemeVsHugoVersionMismatch(fs afero.Fs) (dir string, mismatch bool, requiredMinVersion string) { if !c.hugo.PathSpec.ThemeSet() { return } - for _, absThemeDir := range c.hugo.BaseFs.AbsThemeDirs { - - path := filepath.Join(absThemeDir, "theme.toml") - - exists, err := helpers.Exists(path, fs) - - if err != nil || !exists { - continue - } - - b, err := afero.ReadFile(fs, path) - if err != nil { - continue - } - - tomlMeta, err := metadecoders.Default.UnmarshalToMap(b, metadecoders.TOML) - if err != nil { - continue - } - - if minVersion, ok := tomlMeta["min_version"]; ok { - if hugo.CompareVersion(minVersion) > 0 { - return absThemeDir, true, fmt.Sprint(minVersion) - } - } - - } - return } diff --git a/commands/import_jekyll.go b/commands/import_jekyll.go index 1d37cfd9d3a..e5c39dc3418 100644 --- a/commands/import_jekyll.go +++ b/commands/import_jekyll.go @@ -17,7 +17,6 @@ import ( "bytes" "errors" "fmt" - "io" "io/ioutil" "os" "path/filepath" @@ -27,6 +26,8 @@ import ( "time" "unicode" + "github.com/gohugoio/hugo/common/hugio" + "github.com/gohugoio/hugo/parser/metadecoders" "github.com/gohugoio/hugo/helpers" @@ -113,7 +114,7 @@ func (i *importCmd) importFromJekyll(cmd *cobra.Command, args []string) error { jww.FEEDBACK.Println("Importing...") fileCount := 0 - callback := func(path string, fi os.FileInfo, err error) error { + callback := func(path string, fi hugofs.FileMetaInfo, err error) error { if err != nil { return err } @@ -302,66 +303,10 @@ func (i *importCmd) createConfigFromJekyll(fs afero.Fs, inpath string, kind meta return helpers.WriteToDisk(filepath.Join(inpath, "config."+string(kind)), &buf, fs) } -func copyFile(source string, dest string) error { - sf, err := os.Open(source) - if err != nil { - return err - } - defer sf.Close() - df, err := os.Create(dest) - if err != nil { - return err - } - defer df.Close() - _, err = io.Copy(df, sf) - if err == nil { - si, err := os.Stat(source) - if err != nil { - err = os.Chmod(dest, si.Mode()) - - if err != nil { - return err - } - } - - } - return nil -} - -func copyDir(source string, dest string) error { - fi, err := os.Stat(source) - if err != nil { - return err - } - if !fi.IsDir() { - return errors.New(source + " is not a directory") - } - err = os.MkdirAll(dest, fi.Mode()) - if err != nil { - return err - } - entries, _ := ioutil.ReadDir(source) - for _, entry := range entries { - sfp := filepath.Join(source, entry.Name()) - dfp := filepath.Join(dest, entry.Name()) - if entry.IsDir() { - err = copyDir(sfp, dfp) - if err != nil { - jww.ERROR.Println(err) - } - } else { - err = copyFile(sfp, dfp) - if err != nil { - jww.ERROR.Println(err) - } - } - - } - return nil -} - func (i *importCmd) copyJekyllFilesAndFolders(jekyllRoot, dest string, jekyllPostDirs map[string]bool) (err error) { - fi, err := os.Stat(jekyllRoot) + fs := hugofs.Os + + fi, err := fs.Stat(jekyllRoot) if err != nil { return err } @@ -383,7 +328,7 @@ func (i *importCmd) copyJekyllFilesAndFolders(jekyllRoot, dest string, jekyllPos if entry.IsDir() { if entry.Name()[0] != '_' && entry.Name()[0] != '.' { if _, ok := jekyllPostDirs[entry.Name()]; !ok { - err = copyDir(sfp, dfp) + err = hugio.CopyDir(fs, sfp, dfp, nil) if err != nil { jww.ERROR.Println(err) } @@ -402,7 +347,7 @@ func (i *importCmd) copyJekyllFilesAndFolders(jekyllRoot, dest string, jekyllPos } if !isExcept && entry.Name()[0] != '.' && entry.Name()[0] != '_' { - err = copyFile(sfp, dfp) + err = hugio.CopyFile(fs, sfp, dfp) if err != nil { jww.ERROR.Println(err) } diff --git a/commands/mod.go b/commands/mod.go new file mode 100644 index 00000000000..aecd5f15d9f --- /dev/null +++ b/commands/mod.go @@ -0,0 +1,188 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package commands + +import ( + "os" + "path/filepath" + "strings" + + "github.com/gohugoio/hugo/config" + + "github.com/gohugoio/hugo/hugofs" + + "github.com/gohugoio/hugo/modules" + "github.com/spf13/cobra" +) + +var _ cmder = (*modCmd)(nil) + +type modCmd struct { + *baseBuilderCmd +} + +func (b *commandsBuilder) newModCmd() *modCmd { + c := &modCmd{} + + const commonUsage = ` +Note that Hugo will always start out by resolving the components defined in the site +configuration, provided by a _vendor directory (if no --ignoreVendor flag provided), +Go Modules, or a folder inside the themes directory, in that order. + +` + + cmd := &cobra.Command{ + Use: "mod", + Short: "Various Hugo Modules helpers.", + Long: "LONG V", + + RunE: nil, + } + + cmd.AddCommand( + &cobra.Command{ + // go get [-d] [-m] [-u] [-v] [-insecure] [build flags] [packages] + Use: "get", + DisableFlagParsing: true, + Short: "Resolves dependencies in your current Hugo Project.", + Long: ` +Resolves dependencies in your current Hugo Project. + +You 'go get github.com/gohugoio/testshortcodes@v0.3.0' + +Run "go help get" for more information. +` + commonUsage, + RunE: func(cmd *cobra.Command, args []string) error { + return c.withModsClient(false, func(c *modules.Client) error { + // We currently just pass on the flags we get to Go and + // need to do the flag handling manually. + if len(args) == 1 && strings.Contains(args[0], "-h") { + return cmd.Help() + } + return c.Get(args...) + }) + }, + }, + &cobra.Command{ + Use: "graph", + Short: "TODO(bep)", + RunE: func(cmd *cobra.Command, args []string) error { + return c.withModsClient(true, func(c *modules.Client) error { + return c.Graph(os.Stdout) + }) + }, + }, + &cobra.Command{ + Use: "init", + Short: "TODO(bep) ", + RunE: func(cmd *cobra.Command, args []string) error { + var path string + if len(args) >= 1 { + path = args[0] + } + return c.withModsClient(false, func(c *modules.Client) error { + return c.Init(path) + }) + }, + }, + &cobra.Command{ + Use: "vendor", + Short: "TODO(bep)", + RunE: func(cmd *cobra.Command, args []string) error { + return c.withModsClient(true, func(c *modules.Client) error { + return c.Vendor() + }) + }, + }, + &cobra.Command{ + Use: "tidy", + Short: "TODO(bep)", + RunE: func(cmd *cobra.Command, args []string) error { + return c.withModsClient(true, func(c *modules.Client) error { + return c.Tidy() + }) + }, + }, + ) + + c.baseBuilderCmd = b.newBuilderCmd(cmd) + + return c + +} + +func (c *modCmd) withModsClient(failOnMissingConfig bool, f func(*modules.Client) error) error { + com, err := c.initConfig(failOnMissingConfig) + if err != nil { + return err + } + client, err := c.newModsClient(com.Cfg) + if err != nil { + return err + } + return f(client) +} + +func (c *modCmd) initConfig(failOnNoConfig bool) (*commandeer, error) { + com, err := initializeConfig(failOnNoConfig, false, &c.hugoBuilderCommon, c, nil) + if err != nil { + return nil, err + } + return com, nil +} + +func (c *modCmd) newModsClient(cfg config.Provider) (*modules.Client, error) { + var ( + workingDir string + themesDir string + modProxy string + modConfig modules.Config + ignoreVendor bool + ) + + if c.source != "" { + workingDir = c.source + } else { + var err error + workingDir, err = os.Getwd() + if err != nil { + return nil, err + } + } + + if cfg != nil { + themesDir = cfg.GetString("themesDir") + if themesDir != "" && !filepath.IsAbs(themesDir) { + themesDir = filepath.Join(workingDir, themesDir) + } + var err error + modConfig, err = modules.DecodeConfig(cfg) + // TODO(bep) mod + if err != nil { + return nil, err + } + ignoreVendor = cfg.GetBool("ignoreVendor") + modProxy = cfg.GetString("modProxy") + } + + return modules.NewClient(modules.ClientConfig{ + Fs: hugofs.Os, + WorkingDir: workingDir, + ThemesDir: themesDir, + ModuleConfig: modConfig, + IgnoreVendor: ignoreVendor, + ModProxy: modProxy, + }), nil + +} diff --git a/commands/new.go b/commands/new.go index f10369837da..61617434a57 100644 --- a/commands/new.go +++ b/commands/new.go @@ -53,8 +53,6 @@ Ensure you run this within the root directory of your site.`, cc := &newCmd{baseBuilderCmd: b.newBuilderCmd(cmd)} cmd.Flags().StringVarP(&cc.contentType, "kind", "k", "", "content type to create") - cmd.PersistentFlags().StringVarP(&cc.source, "source", "s", "", "filesystem path to read files relative from") - cmd.PersistentFlags().SetAnnotation("source", cobra.BashCompSubdirsInDir, []string{}) cmd.Flags().StringVar(&cc.contentEditor, "editor", "", "edit new content with this editor, if provided") cmd.AddCommand(newNewSiteCmd().getCommand()) diff --git a/commands/server.go b/commands/server.go index 5d50ebe2cf0..de0ab2cb08d 100644 --- a/commands/server.go +++ b/commands/server.go @@ -262,7 +262,7 @@ func (sc *serverCmd) server(cmd *cobra.Command, args []string) error { relWatchDirs[i], _ = helpers.GetRelativePath(dir, baseWatchDir) } - rootWatchDirs := strings.Join(helpers.UniqueStrings(helpers.ExtractRootPaths(relWatchDirs)), ",") + rootWatchDirs := strings.Join(helpers.UniqueStringsSorted(helpers.ExtractRootPaths(relWatchDirs)), ",") jww.FEEDBACK.Printf("Watching for changes in %s%s{%s}\n", baseWatchDir, helpers.FilePathSeparator, rootWatchDirs) watcher, err := c.newWatcher(watchDirs...) diff --git a/common/herrors/errors.go b/common/herrors/errors.go index be98ceb39b6..1a61070501a 100644 --- a/common/herrors/errors.go +++ b/common/herrors/errors.go @@ -19,6 +19,7 @@ import ( "fmt" "io" "os" + "runtime/debug" _errors "github.com/pkg/errors" ) @@ -46,6 +47,16 @@ func FprintStackTrace(w io.Writer, err error) { } } +// Recover is a helper function that can be used to capture panics. +// Put this at the top of a method/function that crashes in a template: +// defer herrors.Recover() +func Recover() { + if r := recover(); r != nil { + fmt.Println("stacktrace from panic: \n" + string(debug.Stack())) + } + +} + // ErrFeatureNotAvailable denotes that a feature is unavailable. // // We will, at least to begin with, make some Hugo features (SCSS with libsass) optional, diff --git a/common/hugio/copy.go b/common/hugio/copy.go new file mode 100644 index 00000000000..e7ed8577ab3 --- /dev/null +++ b/common/hugio/copy.go @@ -0,0 +1,96 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugio + +import ( + "io" + "io/ioutil" + "os" + "path/filepath" + "strings" + + "github.com/pkg/errors" + + "github.com/spf13/afero" +) + +// CopyFile copies a file. +func CopyFile(fs afero.Fs, from, to string) error { + sf, err := os.Open(from) + if err != nil { + return err + } + defer sf.Close() + df, err := os.Create(to) + if err != nil { + return err + } + defer df.Close() + _, err = io.Copy(df, sf) + // TODDO(bep) mod + if false && err == nil { + si, err := os.Stat(from) + if err != nil { + err = os.Chmod(to, si.Mode()) + + if err != nil { + return err + } + } + + } + return nil +} + +// CopyDir copies a directory. +func CopyDir(fs afero.Fs, from, to string, shouldCopy func(filename string) bool) error { + fi, err := os.Stat(from) + if err != nil { + return err + } + + if !fi.IsDir() { + return errors.Errorf("%q is not a directory", from) + } + + err = fs.MkdirAll(to, 0777) // before umask + if err != nil { + return err + } + + entries, _ := ioutil.ReadDir(from) + for _, entry := range entries { + fromFilename := filepath.Join(from, entry.Name()) + toFilename := filepath.Join(to, entry.Name()) + if entry.IsDir() { + if shouldCopy != nil && !shouldCopy(fromFilename) { + continue + } + if err := CopyDir(fs, fromFilename, toFilename, shouldCopy); err != nil { + return err + } + } else { + // TODO(bep) mod + if strings.Contains(toFilename, "go.") { + continue + } + if err := CopyFile(fs, fromFilename, toFilename); err != nil { + return err + } + } + + } + + return nil +} diff --git a/common/hugo/version.go b/common/hugo/version.go index 47641f10c09..727552cb31e 100644 --- a/common/hugo/version.go +++ b/common/hugo/version.go @@ -15,6 +15,7 @@ package hugo import ( "fmt" + "strconv" "runtime" "strings" @@ -235,3 +236,16 @@ func compareFloatVersions(version float32, v float32) int { } return 1 } + +func GoMinorVersion() int { + return goMinorVersion(runtime.Version()) +} + +func goMinorVersion(version string) int { + if strings.HasPrefix(version, "devel") { + return 9999 // magic + } + i, _ := strconv.Atoi(strings.Split(version, ".")[1]) + return i + +} diff --git a/common/hugo/version_test.go b/common/hugo/version_test.go index 08059189e09..e2aeeabbf88 100644 --- a/common/hugo/version_test.go +++ b/common/hugo/version_test.go @@ -77,3 +77,9 @@ func TestParseHugoVersion(t *testing.T) { require.Equal(t, "0.25-DEV", MustParseVersion("0.25-DEV").String()) } + +func TestGoMinorVersion(t *testing.T) { + assert := require.New(t) + assert.Equal(12, goMinorVersion("go1.12.5")) + assert.True(GoMinorVersion() >= 11) +} diff --git a/common/maps/params.go b/common/maps/params.go new file mode 100644 index 00000000000..0b81057b148 --- /dev/null +++ b/common/maps/params.go @@ -0,0 +1,96 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package maps + +import ( + "strings" + + "github.com/pkg/errors" + + "github.com/spf13/cast" +) + +// GetNestedParam gets the first match of the keyStr in the candidates given. +// It will first try the exact match and then try to find it as a nested map value, +// using the given separator, e.g. "mymap.name". +// It assumes that all the maps given have lower cased keys. +func GetNestedParam(keyStr, separator string, candidates ...map[string]interface{}) (interface{}, error) { + keyStr = strings.ToLower(keyStr) + + lookupFn := func(key string) interface{} { + for _, m := range candidates { + if v, ok := m[key]; ok { + return v + } + } + + return nil + } + + v, _, _, err := GetNestedParamFn(keyStr, separator, lookupFn) + return v, err +} + +func GetNestedParamFn(keyStr, separator string, lookupFn func(key string) interface{}) (interface{}, string, map[string]interface{}, error) { + result, _ := traverseDirectParams(keyStr, lookupFn) + if result != nil { + return result, keyStr, nil, nil + } + + keySegments := strings.Split(keyStr, separator) + if len(keySegments) == 1 { + return nil, keyStr, nil, nil + } + + return traverseNestedParams(keySegments, lookupFn) +} + +func traverseDirectParams(keyStr string, lookupFn func(key string) interface{}) (interface{}, error) { + return lookupFn(keyStr), nil +} + +func traverseNestedParams(keySegments []string, lookupFn func(key string) interface{}) (interface{}, string, map[string]interface{}, error) { + firstKey, rest := keySegments[0], keySegments[1:] + result := lookupFn(firstKey) + if result == nil || len(rest) == 0 { + return result, firstKey, nil, nil + } + + switch m := result.(type) { + case map[string]interface{}: + v, key, owner := traverseParams(rest, m) + return v, key, owner, nil + default: + return nil, "", nil, errors.Errorf("unsupported Params type: %T", result) + } +} + +func traverseParams(keys []string, m map[string]interface{}) (interface{}, string, map[string]interface{}) { + // Shift first element off. + firstKey, rest := keys[0], keys[1:] + result := m[firstKey] + + // No point in continuing here. + if result == nil { + return result, "", nil + } + + if len(rest) == 0 { + // That was the last key. + return result, firstKey, m + } + + // That was not the last key. + return traverseParams(rest, cast.ToStringMap(result)) +} diff --git a/common/maps/params_test.go b/common/maps/params_test.go new file mode 100644 index 00000000000..89b149617c5 --- /dev/null +++ b/common/maps/params_test.go @@ -0,0 +1,45 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package maps + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestGetNestedParam(t *testing.T) { + + m := map[string]interface{}{ + "first": 1, + "with_underscore": 2, + "nested": map[string]interface{}{ + "color": "blue", + }, + } + + assert := require.New(t) + + must := func(keyStr, separator string, candidates ...map[string]interface{}) interface{} { + v, err := GetNestedParam(keyStr, separator, candidates...) + assert.NoError(err) + return v + } + + assert.Equal(1, must("first", "_", m)) + assert.Equal(1, must("First", "_", m)) + assert.Equal(2, must("with_underscore", "_", m)) + assert.Equal("blue", must("nested_color", "_", m)) + +} diff --git a/config/configProvider.go b/config/configProvider.go index 31914c38bc5..187fb7b1064 100644 --- a/config/configProvider.go +++ b/config/configProvider.go @@ -35,10 +35,14 @@ type Provider interface { // we do not attempt to split it into fields. func GetStringSlicePreserveString(cfg Provider, key string) []string { sd := cfg.Get(key) - if sds, ok := sd.(string); ok { + return toStringSlicePreserveString(sd) +} + +func toStringSlicePreserveString(v interface{}) []string { + if sds, ok := v.(string); ok { return []string{sds} } - return cast.ToStringSlice(sd) + return cast.ToStringSlice(v) } // SetBaseTestDefaults provides some common config defaults used in tests. diff --git a/config/env.go b/config/env.go index adf6f9b6811..f482cd24733 100644 --- a/config/env.go +++ b/config/env.go @@ -17,6 +17,7 @@ import ( "os" "runtime" "strconv" + "strings" ) // GetNumWorkerMultiplier returns the base value used to calculate the number @@ -31,3 +32,26 @@ func GetNumWorkerMultiplier() int { } return runtime.NumCPU() } + +// SetEnvVars sets vars on the form key=value in the oldVars slice. +func SetEnvVars(oldVars *[]string, keyValues ...string) { + for i := 0; i < len(keyValues); i += 2 { + setEnvVar(oldVars, keyValues[i], keyValues[i+1]) + } +} + +func SplitEnvVar(v string) (string, string) { + parts := strings.Split(v, "=") + return parts[0], parts[1] +} + +func setEnvVar(vars *[]string, key, value string) { + for i := range *vars { + if strings.HasPrefix((*vars)[i], key+"=") { + (*vars)[i] = key + "=" + value + return + } + } + // New var. + *vars = append(*vars, key+"="+value) +} diff --git a/config/env_test.go b/config/env_test.go new file mode 100644 index 00000000000..594c3e871b3 --- /dev/null +++ b/config/env_test.go @@ -0,0 +1,32 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestSetEnvVars(t *testing.T) { + t.Parallel() + assert := require.New(t) + vars := []string{"FOO=bar", "HUGO=cool", "BAR=foo"} + SetEnvVars(&vars, "HUGO", "rocking!", "NEW", "bar") + assert.Equal([]string{"FOO=bar", "HUGO=rocking!", "BAR=foo", "NEW=bar"}, vars) + + key, val := SplitEnvVar("HUGO=rocks") + assert.Equal("HUGO", key) + assert.Equal("rocks", val) +} diff --git a/create/content.go b/create/content.go index e48dfc078bb..f817d317f32 100644 --- a/create/content.go +++ b/create/content.go @@ -25,6 +25,8 @@ import ( "path/filepath" "strings" + "github.com/gohugoio/hugo/hugofs/files" + "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/helpers" @@ -50,7 +52,10 @@ func NewContent( if isDir { - langFs := hugofs.NewLanguageFs(s.Language().Lang, sites.LanguageSet(), archetypeFs) + langFs, err := hugofs.NewLanguageFs(sites.LanguageSet(), archetypeFs) + if err != nil { + return err + } cm, err := mapArcheTypeDir(ps, langFs, archetypeFilename) if err != nil { @@ -64,7 +69,7 @@ func NewContent( } name := filepath.Base(targetPath) - return newContentFromDir(archetypeFilename, sites, archetypeFs, sourceFs, cm, name, contentPath) + return newContentFromDir(archetypeFilename, sites, sourceFs, cm, name, contentPath) } // Building the sites can be expensive, so only do it if really needed. @@ -111,9 +116,9 @@ func NewContent( return nil } -func targetSite(sites *hugolib.HugoSites, fi *hugofs.LanguageFileInfo) *hugolib.Site { +func targetSite(sites *hugolib.HugoSites, fi hugofs.FileMetaInfo) *hugolib.Site { for _, s := range sites.Sites { - if fi.Lang() == s.Language().Lang { + if fi.Meta().Lang() == s.Language().Lang { return s } } @@ -123,13 +128,14 @@ func targetSite(sites *hugolib.HugoSites, fi *hugofs.LanguageFileInfo) *hugolib. func newContentFromDir( archetypeDir string, sites *hugolib.HugoSites, - sourceFs, targetFs afero.Fs, + targetFs afero.Fs, cm archetypeMap, name, targetPath string) error { for _, f := range cm.otherFiles { - filename := f.Filename() + meta := f.Meta() + filename := meta.Path() // Just copy the file to destination. - in, err := sourceFs.Open(filename) + in, err := meta.Open() if err != nil { return errors.Wrap(err, "failed to open non-content file") } @@ -156,7 +162,7 @@ func newContentFromDir( } for _, f := range cm.contentFiles { - filename := f.Filename() + filename := f.Meta().Path() s := targetSite(sites, f) targetFilename := filepath.Join(targetPath, strings.TrimPrefix(filename, archetypeDir)) @@ -177,9 +183,9 @@ func newContentFromDir( type archetypeMap struct { // These needs to be parsed and executed as Go templates. - contentFiles []*hugofs.LanguageFileInfo + contentFiles []hugofs.FileMetaInfo // These are just copied to destination. - otherFiles []*hugofs.LanguageFileInfo + otherFiles []hugofs.FileMetaInfo // If the templates needs a fully built site. This can potentially be // expensive, so only do when needed. siteUsed bool @@ -192,7 +198,7 @@ func mapArcheTypeDir( var m archetypeMap - walkFn := func(filename string, fi os.FileInfo, err error) error { + walkFn := func(path string, fi hugofs.FileMetaInfo, err error) error { if err != nil { return err @@ -202,12 +208,12 @@ func mapArcheTypeDir( return nil } - fil := fi.(*hugofs.LanguageFileInfo) + fil := fi.(hugofs.FileMetaInfo) - if hugolib.IsContentFile(filename) { + if files.IsContentFile(path) { m.contentFiles = append(m.contentFiles, fil) if !m.siteUsed { - m.siteUsed, err = usesSiteVar(fs, filename) + m.siteUsed, err = usesSiteVar(fs, path) if err != nil { return err } @@ -220,7 +226,15 @@ func mapArcheTypeDir( return nil } - if err := helpers.SymbolicWalk(fs, archetypeDir, walkFn); err != nil { + walkCfg := hugofs.WalkwayConfig{ + WalkFn: walkFn, + Fs: fs, + Root: archetypeDir, + } + + w := hugofs.NewWalkway(walkCfg) + + if err := w.Walk(); err != nil { return m, errors.Wrapf(err, "failed to walk archetype dir %q", archetypeDir) } diff --git a/create/content_template_handler.go b/create/content_template_handler.go index 5a8b4f63cbf..1576fabdb4e 100644 --- a/create/content_template_handler.go +++ b/create/content_template_handler.go @@ -90,7 +90,10 @@ func executeArcheTypeAsTemplate(s *hugolib.Site, name, kind, targetPath, archety err error ) - f := s.SourceSpec.NewFileInfo("", targetPath, false, nil) + f, err := s.SourceSpec.NewFileInfoFrom(targetPath, targetPath) + if err != nil { + return nil, err + } if name == "" { name = f.TranslationBaseName() diff --git a/create/content_test.go b/create/content_test.go index e321900bcf0..b5ae40f9f69 100644 --- a/create/content_test.go +++ b/create/content_test.go @@ -64,8 +64,9 @@ func TestNewContent(t *testing.T) { } for i, c := range cases { - cfg, fs := newTestCfg(assert) - assert.NoError(initFs(fs)) + mm := afero.NewMemMapFs() + assert.NoError(initFs(mm)) + cfg, fs := newTestCfg(assert, mm) h, err := hugolib.NewHugoSites(deps.DepsCfg{Cfg: cfg, Fs: fs}) assert.NoError(err) @@ -86,15 +87,14 @@ func TestNewContent(t *testing.T) { } func TestNewContentFromDir(t *testing.T) { + mm := afero.NewMemMapFs() assert := require.New(t) - cfg, fs := newTestCfg(assert) - assert.NoError(initFs(fs)) archetypeDir := filepath.Join("archetypes", "my-bundle") - assert.NoError(fs.Source.Mkdir(archetypeDir, 0755)) + assert.NoError(mm.MkdirAll(archetypeDir, 0755)) archetypeThemeDir := filepath.Join("themes", "mytheme", "archetypes", "my-theme-bundle") - assert.NoError(fs.Source.Mkdir(archetypeThemeDir, 0755)) + assert.NoError(mm.MkdirAll(archetypeThemeDir, 0755)) contentFile := ` File: %s @@ -103,15 +103,18 @@ Name: {{ replace .Name "-" " " | title }} i18n: {{ T "hugo" }} ` - assert.NoError(afero.WriteFile(fs.Source, filepath.Join(archetypeDir, "index.md"), []byte(fmt.Sprintf(contentFile, "index.md")), 0755)) - assert.NoError(afero.WriteFile(fs.Source, filepath.Join(archetypeDir, "index.nn.md"), []byte(fmt.Sprintf(contentFile, "index.nn.md")), 0755)) + assert.NoError(afero.WriteFile(mm, filepath.Join(archetypeDir, "index.md"), []byte(fmt.Sprintf(contentFile, "index.md")), 0755)) + assert.NoError(afero.WriteFile(mm, filepath.Join(archetypeDir, "index.nn.md"), []byte(fmt.Sprintf(contentFile, "index.nn.md")), 0755)) + + assert.NoError(afero.WriteFile(mm, filepath.Join(archetypeDir, "pages", "bio.md"), []byte(fmt.Sprintf(contentFile, "bio.md")), 0755)) + assert.NoError(afero.WriteFile(mm, filepath.Join(archetypeDir, "resources", "hugo1.json"), []byte(`hugo1: {{ printf "no template handling in here" }}`), 0755)) + assert.NoError(afero.WriteFile(mm, filepath.Join(archetypeDir, "resources", "hugo2.xml"), []byte(`hugo2: {{ printf "no template handling in here" }}`), 0755)) - assert.NoError(afero.WriteFile(fs.Source, filepath.Join(archetypeDir, "pages", "bio.md"), []byte(fmt.Sprintf(contentFile, "bio.md")), 0755)) - assert.NoError(afero.WriteFile(fs.Source, filepath.Join(archetypeDir, "resources", "hugo1.json"), []byte(`hugo1: {{ printf "no template handling in here" }}`), 0755)) - assert.NoError(afero.WriteFile(fs.Source, filepath.Join(archetypeDir, "resources", "hugo2.xml"), []byte(`hugo2: {{ printf "no template handling in here" }}`), 0755)) + assert.NoError(afero.WriteFile(mm, filepath.Join(archetypeThemeDir, "index.md"), []byte(fmt.Sprintf(contentFile, "index.md")), 0755)) + assert.NoError(afero.WriteFile(mm, filepath.Join(archetypeThemeDir, "resources", "hugo1.json"), []byte(`hugo1: {{ printf "no template handling in here" }}`), 0755)) - assert.NoError(afero.WriteFile(fs.Source, filepath.Join(archetypeThemeDir, "index.md"), []byte(fmt.Sprintf(contentFile, "index.md")), 0755)) - assert.NoError(afero.WriteFile(fs.Source, filepath.Join(archetypeThemeDir, "resources", "hugo1.json"), []byte(`hugo1: {{ printf "no template handling in here" }}`), 0755)) + assert.NoError(initFs(mm)) + cfg, fs := newTestCfg(assert, mm) h, err := hugolib.NewHugoSites(deps.DepsCfg{Cfg: cfg, Fs: fs}) assert.NoError(err) @@ -135,7 +138,7 @@ i18n: {{ T "hugo" }} } -func initFs(fs *hugofs.Fs) error { +func initFs(fs afero.Fs) error { perm := os.FileMode(0755) var err error @@ -146,8 +149,8 @@ func initFs(fs *hugofs.Fs) error { filepath.Join("themes", "sample", "archetypes"), } for _, dir := range dirs { - err = fs.Source.Mkdir(dir, perm) - if err != nil { + err = fs.Mkdir(dir, perm) + if err != nil && !os.IsExist(err) { return err } } @@ -198,7 +201,7 @@ Some text. `, }, } { - f, err := fs.Source.Create(v.path) + f, err := fs.Create(v.path) if err != nil { return err } @@ -221,6 +224,7 @@ func assertContains(assert *require.Assertions, v interface{}, matches ...string // TODO(bep) extract common testing package with this and some others func readFileFromFs(t *testing.T, fs afero.Fs, filename string) string { + t.Helper() filename = filepath.FromSlash(filename) b, err := afero.ReadFile(fs, filename) if err != nil { @@ -238,7 +242,7 @@ func readFileFromFs(t *testing.T, fs afero.Fs, filename string) string { return string(b) } -func newTestCfg(assert *require.Assertions) (*viper.Viper, *hugofs.Fs) { +func newTestCfg(assert *require.Assertions, mm afero.Fs) (*viper.Viper, *hugofs.Fs) { cfg := ` @@ -254,8 +258,11 @@ languageName = "Nynorsk" contentDir = "content_nn" ` + if mm == nil { + mm = afero.NewMemMapFs() + } - mm := afero.NewMemMapFs() + mm.MkdirAll(filepath.FromSlash("themes/mytheme"), 0777) assert.NoError(afero.WriteFile(mm, filepath.Join("i18n", "en.toml"), []byte(`[hugo] other = "Hugo Rocks!"`), 0755)) diff --git a/deps/deps.go b/deps/deps.go index fa62fe5ae54..8ef015ac95b 100644 --- a/deps/deps.go +++ b/deps/deps.go @@ -159,11 +159,11 @@ func (d *Deps) TemplateHandler() tpl.TemplateHandler { func (d *Deps) LoadResources() error { // Note that the translations need to be loaded before the templates. if err := d.translationProvider.Update(d); err != nil { - return err + return errors.Wrap(err, "loading translations") } if err := d.templateProvider.Update(d); err != nil { - return err + return errors.Wrap(err, "loading templates") } return nil @@ -210,7 +210,7 @@ func New(cfg DepsCfg) (*Deps, error) { ps, err := helpers.NewPathSpec(fs, cfg.Language) if err != nil { - return nil, err + return nil, errors.Wrap(err, "create PathSpec") } fileCaches, err := filecache.NewCaches(ps) diff --git a/go.mod b/go.mod index 74051d659d6..3a1c0dbbcdc 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,9 @@ require ( github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 github.com/fortytw2/leaktest v1.3.0 github.com/fsnotify/fsnotify v1.4.7 + github.com/go-errors/errors v1.0.1 github.com/gobwas/glob v0.2.3 + github.com/gohugoio/testmodBuilder/mods v0.0.0-20190520184928-c56af20f2e95 github.com/google/go-cmp v0.3.0 github.com/gorilla/websocket v1.4.0 github.com/hashicorp/go-immutable-radix v1.0.0 @@ -43,6 +45,7 @@ require ( github.com/olekukonko/tablewriter v0.0.0-20180506121414-d4647c9c7a84 github.com/pelletier/go-toml v1.4.0 // indirect github.com/pkg/errors v0.8.1 + github.com/rogpeppe/go-internal v1.3.0 github.com/russross/blackfriday v1.5.3-0.20190124082335-a477dd164691 github.com/sanity-io/litter v1.1.0 github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect @@ -71,3 +74,5 @@ require ( ) replace github.com/markbates/inflect => github.com/markbates/inflect v0.0.0-20171215194931-a12c3aec81a6 + +go 1.13 diff --git a/go.sum b/go.sum index cebd1059348..be92e89cc47 100644 --- a/go.sum +++ b/go.sum @@ -107,6 +107,7 @@ github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHqu github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= @@ -118,6 +119,8 @@ github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJA github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gohugoio/testmodBuilder/mods v0.0.0-20190520184928-c56af20f2e95 h1:sgew0XCnZwnzpWxTt3V8LLiCO7OQi3C6dycaE67wfkU= +github.com/gohugoio/testmodBuilder/mods v0.0.0-20190520184928-c56af20f2e95/go.mod h1:bOlVlCa1/RajcHpXkrUXPSHB/Re1UnlXxD1Qp8SKOd8= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -207,6 +210,7 @@ github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.7 h1:UvyT9uN+3r7yLEYSlJsbQGdsaB/a0DlgWP3pql6iwOc= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-runewidth v0.0.3 h1:a+kO+98RDGEfo6asOGMmpodZq4FNtnGP54yps8BzLR4= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= @@ -261,6 +265,8 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7z github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/go-internal v1.3.0 h1:RR9dF3JtopPvtkroDZuVD7qquD0bnHlKSqaQhgwt8yk= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday v1.5.3-0.20190124082335-a477dd164691 h1:auJkuUc4uOuZNoH9jGLvqVaDLiuCOh/LY+Qw5NBFo4I= @@ -398,6 +404,7 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190530182044-ad28b68e88f1 h1:R4dVlxdmKenVdMRS/tTspEpSTRWINYrHD8ySIU9yCIU= golang.org/x/sys v0.0.0-20190530182044-ad28b68e88f1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7 h1:LepdCS8Gf/MVejFIt8lsiexZATdoGVyp5bcyS+rYoUI= golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -450,6 +457,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= diff --git a/helpers/docshelper.go b/helpers/docshelper.go index 8ad817d12ea..66cbfa7d3bf 100644 --- a/helpers/docshelper.go +++ b/helpers/docshelper.go @@ -36,8 +36,7 @@ func init() { } } - sort.Strings(aliases) - aliases = UniqueStrings(aliases) + aliases = UniqueStringsSorted(aliases) lexerEntry := struct { Name string diff --git a/helpers/general.go b/helpers/general.go index 3cf7ba8af67..5eabda3c605 100644 --- a/helpers/general.go +++ b/helpers/general.go @@ -22,15 +22,16 @@ import ( "net" "os" "path/filepath" + "sort" "strings" "sync" "unicode" "unicode/utf8" - "github.com/gohugoio/hugo/common/hugo" - "github.com/gohugoio/hugo/hugofs" + "github.com/gohugoio/hugo/common/hugo" + "github.com/spf13/afero" "github.com/jdkato/prose/transform" @@ -106,7 +107,7 @@ func FirstUpper(s string) string { // UniqueStrings returns a new slice with any duplicates removed. func UniqueStrings(s []string) []string { - var unique []string + unique := make([]string, 0, len(s)) set := map[string]interface{}{} for _, val := range s { if _, ok := set[val]; !ok { @@ -117,6 +118,40 @@ func UniqueStrings(s []string) []string { return unique } +// UniqueStringsReuse returns a slice with any duplicates removed. +// It will modify the input slice. +func UniqueStringsReuse(s []string) []string { + set := map[string]interface{}{} + result := s[:0] + for _, val := range s { + if _, ok := set[val]; !ok { + result = append(result, val) + set[val] = val + } + } + return result +} + +// UniqueStringsReuse returns a sorted slice with any duplicates removed. +// It will modify the input slice. +func UniqueStringsSorted(s []string) []string { + if len(s) == 0 { + return nil + } + ss := sort.StringSlice(s) + ss.Sort() + i := 0 + for j := 1; j < len(s); j++ { + if !ss.Less(i, j) { + continue + } + i++ + s[i] = s[j] + } + + return s[:i+1] +} + // ReaderToBytes takes an io.Reader argument, reads from it // and returns bytes. func ReaderToBytes(lines io.Reader) []byte { @@ -459,17 +494,15 @@ func PrintFs(fs afero.Fs, path string, w io.Writer) { if fs == nil { return } + afero.Walk(fs, path, func(path string, info os.FileInfo, err error) error { - if info != nil && !info.IsDir() { - s := path - if lang, ok := info.(hugofs.LanguageAnnouncer); ok { - s = s + "\tLANG: " + lang.Lang() - } - if fp, ok := info.(hugofs.FilePather); ok { - s = s + "\tRF: " + fp.Filename() + "\tBP: " + fp.BaseDir() - } - fmt.Fprintln(w, " ", s) + var filename string + var meta interface{} + if fim, ok := info.(hugofs.FileMetaInfo); ok { + filename = fim.Meta().Filename() + meta = fim.Meta() } + fmt.Fprintf(w, " %q %q\t\t%v\n", path, filename, meta) return nil }) } diff --git a/helpers/general_test.go b/helpers/general_test.go index ed4c3d2c27b..dd61d89485b 100644 --- a/helpers/general_test.go +++ b/helpers/general_test.go @@ -234,6 +234,24 @@ func TestUniqueStrings(t *testing.T) { } } +func TestUniqueStringsReuse(t *testing.T) { + in := []string{"a", "b", "a", "b", "c", "", "a", "", "d"} + output := UniqueStringsReuse(in) + expected := []string{"a", "b", "c", "", "d"} + if !reflect.DeepEqual(output, expected) { + t.Errorf("Expected %#v, got %#v\n", expected, output) + } +} + +func TestUniqueStringsSorted(t *testing.T) { + assert := require.New(t) + in := []string{"a", "a", "b", "c", "b", "", "a", "", "d"} + output := UniqueStringsSorted(in) + expected := []string{"", "a", "b", "c", "d"} + assert.Equal(expected, output) + assert.Nil(UniqueStringsSorted(nil)) +} + func TestFindAvailablePort(t *testing.T) { addr, err := FindAvailablePort() assert.Nil(t, err) @@ -328,3 +346,55 @@ func BenchmarkMD5FromFileFast(b *testing.B) { } } + +func BenchmarkUniqueStrings(b *testing.B) { + input := []string{"a", "b", "d", "e", "d", "h", "a", "i"} + + b.Run("Safe", func(b *testing.B) { + for i := 0; i < b.N; i++ { + result := UniqueStrings(input) + if len(result) != 6 { + b.Fatal(fmt.Sprintf("invalid count: %d", len(result))) + } + } + }) + + b.Run("Reuse slice", func(b *testing.B) { + b.StopTimer() + inputs := make([][]string, b.N) + for i := 0; i < b.N; i++ { + inputc := make([]string, len(input)) + copy(inputc, input) + inputs[i] = inputc + } + b.StartTimer() + for i := 0; i < b.N; i++ { + inputc := inputs[i] + + result := UniqueStringsReuse(inputc) + if len(result) != 6 { + b.Fatal(fmt.Sprintf("invalid count: %d", len(result))) + } + } + }) + + b.Run("Reuse slice sorted", func(b *testing.B) { + b.StopTimer() + inputs := make([][]string, b.N) + for i := 0; i < b.N; i++ { + inputc := make([]string, len(input)) + copy(inputc, input) + inputs[i] = inputc + } + b.StartTimer() + for i := 0; i < b.N; i++ { + inputc := inputs[i] + + result := UniqueStringsSorted(inputc) + if len(result) != 6 { + b.Fatal(fmt.Sprintf("invalid count: %d", len(result))) + } + } + }) + +} diff --git a/helpers/path.go b/helpers/path.go index 36bd3269bf8..5cca1d9e39f 100644 --- a/helpers/path.go +++ b/helpers/path.go @@ -26,6 +26,8 @@ import ( "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/hugofs" + "github.com/gohugoio/hugo/common/hugio" _errors "github.com/pkg/errors" "github.com/spf13/afero" @@ -172,32 +174,6 @@ func ReplaceExtension(path string, newExt string) string { return f + "." + newExt } -// GetFirstThemeDir gets the root directory of the first theme, if there is one. -// If there is no theme, returns the empty string. -func (p *PathSpec) GetFirstThemeDir() string { - if p.ThemeSet() { - return p.AbsPathify(filepath.Join(p.ThemesDir, p.Themes()[0])) - } - return "" -} - -// GetThemesDir gets the absolute root theme dir path. -func (p *PathSpec) GetThemesDir() string { - if p.ThemeSet() { - return p.AbsPathify(p.ThemesDir) - } - return "" -} - -// GetRelativeThemeDir gets the relative root directory of the current theme, if there is one. -// If there is no theme, returns the empty string. -func (p *PathSpec) GetRelativeThemeDir() string { - if p.ThemeSet() { - return strings.TrimPrefix(filepath.Join(p.ThemesDir, p.Themes()[0]), FilePathSeparator) - } - return "" -} - func makePathRelative(inPath string, possibleDirectories ...string) (string, error) { for _, currentPath := range possibleDirectories { @@ -425,47 +401,20 @@ func FindCWD() (string, error) { return path, nil } -// SymbolicWalk is like filepath.Walk, but it supports the root being a -// symbolic link. It will still not follow symbolic links deeper down in -// the file structure. -func SymbolicWalk(fs afero.Fs, root string, walker filepath.WalkFunc) error { - - // Sanity check - if root != "" && len(root) < 4 { - return errors.New("path is too short") +// SymbolicWalk is like filepath.Walk, but it follows symbolic links. +func SymbolicWalk(fs afero.Fs, root string, walker hugofs.WalkFunc) error { + if _, isOs := fs.(*afero.OsFs); isOs { + // Mainly to track symlinks. + fs = hugofs.NewBaseFileDecorator(fs) } - // Handle the root first - fileInfo, realPath, err := getRealFileInfo(fs, root) - - if err != nil { - return walker(root, nil, err) - } - - if !fileInfo.IsDir() { - return fmt.Errorf("cannot walk regular file %s", root) - } - - if err := walker(realPath, fileInfo, err); err != nil && err != filepath.SkipDir { - return err - } - - // Some of Hugo's filesystems represents an ordered root folder, i.e. project first, then theme folders. - // Make sure that order is preserved. afero.Walk will sort the directories down in the file tree, - // but we don't care about that. - rootContent, err := readDir(fs, root, false) - - if err != nil { - return walker(root, nil, err) - } - - for _, fi := range rootContent { - if err := afero.Walk(fs, filepath.Join(root, fi.Name()), walker); err != nil { - return err - } - } + w := hugofs.NewWalkway(hugofs.WalkwayConfig{ + Fs: fs, + Root: root, + WalkFn: walker, + }) - return nil + return w.Walk() } @@ -485,6 +434,7 @@ func readDir(fs afero.Fs, dirname string, doSort bool) ([]os.FileInfo, error) { return list, nil } +// TODO(bep) mod add theme fs with no lstat and a check on Open func getRealFileInfo(fs afero.Fs, path string) (os.FileInfo, string, error) { fileInfo, err := LstatIfPossible(fs, path) realPath := path diff --git a/helpers/path_test.go b/helpers/path_test.go index 98291936c1a..2b6e01be23f 100644 --- a/helpers/path_test.go +++ b/helpers/path_test.go @@ -73,18 +73,9 @@ func TestMakePath(t *testing.T) { } func TestMakePathSanitized(t *testing.T) { - v := viper.New() - v.Set("contentDir", "content") - v.Set("dataDir", "data") - v.Set("i18nDir", "i18n") - v.Set("layoutDir", "layouts") - v.Set("assetDir", "assets") - v.Set("resourceDir", "resources") - v.Set("publishDir", "public") - v.Set("archetypeDir", "archetypes") + v := newTestCfg() - l := langs.NewDefaultLanguage(v) - p, _ := NewPathSpec(hugofs.NewMem(v), l) + p, _ := NewPathSpec(hugofs.NewMem(v), v) tests := []struct { input string diff --git a/helpers/pathspec_test.go b/helpers/pathspec_test.go index 00dd9cd7b1a..1c27f7e1159 100644 --- a/helpers/pathspec_test.go +++ b/helpers/pathspec_test.go @@ -14,6 +14,7 @@ package helpers import ( + "path/filepath" "testing" "github.com/gohugoio/hugo/hugofs" @@ -36,8 +37,12 @@ func TestNewPathSpecFromConfig(t *testing.T) { v.Set("workingDir", "thework") v.Set("staticDir", "thestatic") v.Set("theme", "thetheme") + langs.LoadLanguageSettings(v, nil) - p, err := NewPathSpec(hugofs.NewMem(v), l) + fs := hugofs.NewMem(v) + fs.Source.MkdirAll(filepath.FromSlash("thework/thethemes/thetheme"), 0777) + + p, err := NewPathSpec(fs, l) require.NoError(t, err) require.True(t, p.CanonifyURLs) @@ -50,5 +55,5 @@ func TestNewPathSpecFromConfig(t *testing.T) { require.Equal(t, "http://base.com", p.BaseURL.String()) require.Equal(t, "thethemes", p.ThemesDir) require.Equal(t, "thework", p.WorkingDir) - require.Equal(t, []string{"thetheme"}, p.Themes()) + } diff --git a/helpers/testhelpers_test.go b/helpers/testhelpers_test.go index c9da4f12919..b74dccfc46d 100644 --- a/helpers/testhelpers_test.go +++ b/helpers/testhelpers_test.go @@ -5,6 +5,7 @@ import ( "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/langs" + "github.com/gohugoio/hugo/modules" ) func newTestPathSpec(fs *hugofs.Fs, v *viper.Viper) *PathSpec { @@ -42,6 +43,14 @@ func newTestCfg() *viper.Viper { v.Set("resourceDir", "resources") v.Set("publishDir", "public") v.Set("archetypeDir", "archetypes") + langs.LoadLanguageSettings(v, nil) + langs.LoadLanguageSettings(v, nil) + mod, err := modules.CreateProjectModule(v) + if err != nil { + panic(err) + } + v.Set("allModules", modules.Modules{mod}) + return v } diff --git a/htesting/test_helpers.go b/htesting/test_helpers.go new file mode 100644 index 00000000000..dc303b2e50c --- /dev/null +++ b/htesting/test_helpers.go @@ -0,0 +1,39 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package htesting + +import ( + "runtime" + "strings" + + "github.com/spf13/afero" +) + +// CreateTempDir creates a temp dir in the given filesystem and +// returns the dirnam and a func that removes it when done. +func CreateTempDir(fs afero.Fs, prefix string) (string, func(), error) { + tempDir, err := afero.TempDir(fs, "", prefix) + if err != nil { + return "", nil, err + } + + _, isOsFs := fs.(*afero.OsFs) + + if isOsFs && runtime.GOOS == "darwin" && !strings.HasPrefix(tempDir, "/private") { + // To get the entry folder in line with the rest. This its a little bit + // mysterious, but so be it. + tempDir = "/private" + tempDir + } + return tempDir, func() { fs.RemoveAll(tempDir) }, nil +} diff --git a/htesting/test_structs.go b/htesting/test_structs.go deleted file mode 100644 index 72dc7f3fc63..00000000000 --- a/htesting/test_structs.go +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright 2019 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package htesting - -import ( - "html/template" - "time" - - "github.com/gohugoio/hugo/common/hugo" - "github.com/gohugoio/hugo/langs" - "github.com/gohugoio/hugo/navigation" - "github.com/gohugoio/hugo/resources/page" - "github.com/spf13/viper" -) - -type testSite struct { - h hugo.Info - l *langs.Language -} - -func (t testSite) Hugo() hugo.Info { - return t.h -} - -func (t testSite) ServerPort() int { - return 1313 -} - -func (testSite) LastChange() (t time.Time) { - return -} - -func (t testSite) Title() string { - return "foo" -} - -func (t testSite) Sites() page.Sites { - return nil -} - -func (t testSite) IsServer() bool { - return false -} - -func (t testSite) Language() *langs.Language { - return t.l -} - -func (t testSite) Pages() page.Pages { - return nil -} - -func (t testSite) RegularPages() page.Pages { - return nil -} - -func (t testSite) Menus() navigation.Menus { - return nil -} - -func (t testSite) Taxonomies() interface{} { - return nil -} - -func (t testSite) BaseURL() template.URL { - return "" -} - -func (t testSite) Params() map[string]interface{} { - return nil -} - -func (t testSite) Data() map[string]interface{} { - return nil -} - -// NewTestHugoSite creates a new minimal test site. -func NewTestHugoSite() page.Site { - return testSite{ - h: hugo.NewInfo(hugo.EnvironmentProduction), - l: langs.NewLanguage("en", newTestConfig()), - } -} - -func newTestConfig() *viper.Viper { - v := viper.New() - v.Set("contentDir", "content") - return v -} diff --git a/hugofs/basepath_real_filename_fs.go b/hugofs/basepath_real_filename_fs.go deleted file mode 100644 index 1024c4d3034..00000000000 --- a/hugofs/basepath_real_filename_fs.go +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugofs - -import ( - "os" - - "github.com/spf13/afero" -) - -// RealFilenameInfo is a thin wrapper around os.FileInfo adding the real filename. -type RealFilenameInfo interface { - os.FileInfo - - // This is the real filename to the file in the underlying filesystem. - RealFilename() string -} - -type realFilenameInfo struct { - os.FileInfo - realFilename string -} - -func (f *realFilenameInfo) RealFilename() string { - return f.realFilename -} - -// NewBasePathRealFilenameFs returns a new BasePathRealFilenameFs instance -// using base. -func NewBasePathRealFilenameFs(base *afero.BasePathFs) *BasePathRealFilenameFs { - return &BasePathRealFilenameFs{BasePathFs: base} -} - -// BasePathRealFilenameFs is a thin wrapper around afero.BasePathFs that -// provides the real filename in Stat and LstatIfPossible. -type BasePathRealFilenameFs struct { - *afero.BasePathFs -} - -// Stat returns the os.FileInfo structure describing a given file. If there is -// an error, it will be of type *os.PathError. -func (b *BasePathRealFilenameFs) Stat(name string) (os.FileInfo, error) { - fi, err := b.BasePathFs.Stat(name) - if err != nil { - return nil, err - } - - if _, ok := fi.(RealFilenameInfo); ok { - return fi, nil - } - - filename, err := b.RealPath(name) - if err != nil { - return nil, &os.PathError{Op: "stat", Path: name, Err: err} - } - - return &realFilenameInfo{FileInfo: fi, realFilename: filename}, nil -} - -// LstatIfPossible returns the os.FileInfo structure describing a given file. -// It attempts to use Lstat if supported or defers to the os. In addition to -// the FileInfo, a boolean is returned telling whether Lstat was called. -func (b *BasePathRealFilenameFs) LstatIfPossible(name string) (os.FileInfo, bool, error) { - - fi, ok, err := b.BasePathFs.LstatIfPossible(name) - if err != nil { - return nil, false, err - } - - if _, ok := fi.(RealFilenameInfo); ok { - return fi, ok, nil - } - - filename, err := b.RealPath(name) - if err != nil { - return nil, false, &os.PathError{Op: "lstat", Path: name, Err: err} - } - - return &realFilenameInfo{FileInfo: fi, realFilename: filename}, ok, nil -} diff --git a/hugofs/decorators.go b/hugofs/decorators.go new file mode 100644 index 00000000000..81778688d7b --- /dev/null +++ b/hugofs/decorators.go @@ -0,0 +1,202 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugofs + +import ( + "os" + "path/filepath" + "strings" + + "github.com/pkg/errors" + + "github.com/spf13/afero" +) + +func decorateDirs(fs afero.Fs, meta FileMeta) afero.Fs { + ffs := &baseFileDecoratorFs{Fs: fs} + + decorator := func(fi os.FileInfo, name string) (os.FileInfo, error) { + if !fi.IsDir() { + // Leave regular files as they are. + return fi, nil + } + + return decorateFileInfo(fi, fs, nil, "", "", meta), nil + } + + ffs.decorate = decorator + + return ffs + +} + +func decoratePath(fs afero.Fs, createPath func(name string) string) afero.Fs { + + ffs := &baseFileDecoratorFs{Fs: fs} + + decorator := func(fi os.FileInfo, name string) (os.FileInfo, error) { + path := createPath(name) + + return decorateFileInfo(fi, fs, nil, "", path, nil), nil + } + + ffs.decorate = decorator + + return ffs + +} + +// DecorateBasePathFs adds Path info to files and directories in the +// provided BasePathFs, using the base as base. +func DecorateBasePathFs(base *afero.BasePathFs) afero.Fs { + basePath, _ := base.RealPath("") + if !strings.HasSuffix(basePath, filepathSeparator) { + basePath += filepathSeparator + } + + ffs := &baseFileDecoratorFs{Fs: base} + + decorator := func(fi os.FileInfo, name string) (os.FileInfo, error) { + path := strings.TrimPrefix(name, basePath) + + return decorateFileInfo(fi, base, nil, "", path, nil), nil + } + + ffs.decorate = decorator + + return ffs +} + +// NewBaseFileDecorator decorates the given Fs to provide the real filename +// and an Opener func. If +func NewBaseFileDecorator(fs afero.Fs) afero.Fs { + + ffs := &baseFileDecoratorFs{Fs: fs} + + decorator := func(fi os.FileInfo, filename string) (os.FileInfo, error) { + // Store away the original in case it's a symlink. + meta := FileMeta{metaKeyName: fi.Name()} + isSymlink := isSymlink(fi) + if isSymlink { + meta[metaKeyOriginalFilename] = filename + link, err := filepath.EvalSymlinks(filename) + if err != nil { + return nil, err + } + + fi, err = fs.Stat(link) + if err != nil { + return nil, err + } + + filename = link + meta[metaKeyIsSymlink] = true + + } + + opener := func() (afero.File, error) { + return ffs.open(filename) + + } + + return decorateFileInfo(fi, ffs, opener, filename, "", meta), nil + } + + ffs.decorate = decorator + return ffs +} + +type baseFileDecoratorFs struct { + afero.Fs + decorate func(fi os.FileInfo, filename string) (os.FileInfo, error) +} + +func (fs *baseFileDecoratorFs) Stat(name string) (os.FileInfo, error) { + fi, err := fs.Fs.Stat(name) + if err != nil { + return nil, err + } + + return fs.decorate(fi, name) + +} + +func (fs *baseFileDecoratorFs) LstatIfPossible(name string) (os.FileInfo, bool, error) { + var ( + fi os.FileInfo + err error + ok bool + ) + + if lstater, isLstater := fs.Fs.(afero.Lstater); isLstater { + fi, ok, err = lstater.LstatIfPossible(name) + } else { + fi, err = fs.Fs.Stat(name) + } + + if err != nil { + return nil, false, err + } + + fi, err = fs.decorate(fi, name) + + return fi, ok, err +} + +func (fs *baseFileDecoratorFs) Open(name string) (afero.File, error) { + return fs.open(name) +} + +func (fs *baseFileDecoratorFs) open(name string) (afero.File, error) { + f, err := fs.Fs.Open(name) + if err != nil { + return nil, err + } + return &baseFileDecoratorFile{File: f, fs: fs}, nil +} + +type baseFileDecoratorFile struct { + afero.File + fs *baseFileDecoratorFs +} + +func (l *baseFileDecoratorFile) Readdir(c int) (ofi []os.FileInfo, err error) { + dirnames, err := l.File.Readdirnames(c) + if err != nil { + return nil, err + } + + fisp := make([]os.FileInfo, len(dirnames)) + + for i, dirname := range dirnames { + filename := dirname + + if l.Name() != "" && l.Name() != filepathSeparator { + filename = filepath.Join(l.Name(), dirname) + } + + // We need to resolve any symlink info. + fi, err := lstatIfPossible(l.fs.Fs, filename) + if err != nil { + return nil, err + } + fi, err = l.fs.decorate(fi, filename) + if err != nil { + return nil, errors.Wrap(err, "decorate") + } + fisp[i] = fi + } + + return fisp, err +} diff --git a/hugofs/fileinfo.go b/hugofs/fileinfo.go new file mode 100644 index 00000000000..62602a3b9ba --- /dev/null +++ b/hugofs/fileinfo.go @@ -0,0 +1,283 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package hugofs provides the file systems used by Hugo. +package hugofs + +import ( + "os" + "path/filepath" + "strings" + "time" + + "github.com/gohugoio/hugo/hugofs/files" + + "github.com/pkg/errors" + + "github.com/spf13/cast" + + "github.com/gohugoio/hugo/common/hreflect" + + "github.com/spf13/afero" +) + +const ( + metaKeyFilename = "filename" + metaKeyOriginalFilename = "originalFilename" + metaKeyName = "name" + metaKeyPath = "path" + metaKeyPathWalk = "pathWalk" + metaKeyLang = "lang" + metaKeyWeight = "weight" + metaKeyOrdinal = "ordinal" + metaKeyFs = "fs" + metaKeyOpener = "opener" + metaKeyIsOrdered = "isOrdered" + metaKeyIsSymlink = "isSymlink" + metaKeySkipDir = "skipDir" + metaKeyClassifier = "classifier" + metaKeyTranslationBaseName = "translationBaseName" + metaKeyTranslationBaseNameWithExt = "translationBaseNameWithExt" + metaKeyTranslations = "translations" + metaKeyDecoraterPath = "decoratorPath" +) + +type FileMeta map[string]interface{} + +func (f FileMeta) GetInt(key string) int { + return cast.ToInt(f[key]) +} + +func (f FileMeta) GetString(key string) string { + return cast.ToString(f[key]) +} + +func (f FileMeta) GetBool(key string) bool { + return cast.ToBool(f[key]) +} + +func (f FileMeta) Filename() string { + return f.stringV(metaKeyFilename) +} + +func (f FileMeta) OriginalFilename() string { + return f.stringV(metaKeyOriginalFilename) +} + +func (f FileMeta) SkipDir() bool { + return f.GetBool(metaKeySkipDir) +} +func (f FileMeta) TranslationBaseName() string { + return f.stringV(metaKeyTranslationBaseName) +} + +func (f FileMeta) TranslationBaseNameWithExt() string { + return f.stringV(metaKeyTranslationBaseNameWithExt) +} + +func (f FileMeta) Translations() []string { + return cast.ToStringSlice(f[metaKeyTranslations]) +} + +func (f FileMeta) Name() string { + return f.stringV(metaKeyName) +} + +func (f FileMeta) Classifier() string { + c := f.stringV(metaKeyClassifier) + if c != "" { + return c + } + + return files.ContentClassFile // For sorting +} + +func (f FileMeta) Lang() string { + return f.stringV(metaKeyLang) +} + +func (f FileMeta) Path() string { + return f.stringV(metaKeyPath) +} + +func (f FileMeta) Weight() int { + return f.GetInt(metaKeyWeight) +} + +func (f FileMeta) Ordinal() int { + return f.GetInt(metaKeyOrdinal) +} + +func (f FileMeta) IsOrdered() bool { + return f.GetBool(metaKeyIsOrdered) +} + +// IsSymlink returns whether this comes from a symlinked file or directory. +func (f FileMeta) IsSymlink() bool { + return f.GetBool(metaKeyIsSymlink) +} + +func (f FileMeta) Watch() bool { + if v, found := f["watch"]; found { + return v.(bool) + } + return false +} + +func (f FileMeta) Fs() afero.Fs { + if v, found := f[metaKeyFs]; found { + return v.(afero.Fs) + } + return nil +} + +func (f FileMeta) GetOpener() func() (afero.File, error) { + o, found := f[metaKeyOpener] + if !found { + return nil + } + return o.(func() (afero.File, error)) +} + +func (f FileMeta) Open() (afero.File, error) { + v, found := f[metaKeyOpener] + if !found { + return nil, errors.New("file opener not found") + } + return v.(func() (afero.File, error))() +} + +func (f FileMeta) stringV(key string) string { + if v, found := f[key]; found { + return v.(string) + } + return "" +} + +func (f FileMeta) setIfNotZero(key string, val interface{}) { + if !hreflect.IsTruthful(val) { + return + } + f[key] = val +} + +type FileMetaInfo interface { + os.FileInfo + Meta() FileMeta +} + +type fileInfoMeta struct { + os.FileInfo + m FileMeta +} + +func (fi *fileInfoMeta) Meta() FileMeta { + return fi.m +} + +func NewFileMetaInfo(fi os.FileInfo, m FileMeta) FileMetaInfo { + + if fim, ok := fi.(FileMetaInfo); ok { + mergeFileMeta(fim.Meta(), m) + } + return &fileInfoMeta{FileInfo: fi, m: m} +} + +// Merge metadata, last entry wins. +func mergeFileMeta(from, to FileMeta) { + if from == nil { + return + } + for k, v := range from { + if _, found := to[k]; !found { + to[k] = v + } + } +} + +type dirNameOnlyFileInfo struct { + name string +} + +func (fi *dirNameOnlyFileInfo) Name() string { + return fi.name +} + +func (fi *dirNameOnlyFileInfo) Size() int64 { + panic("not implemented") +} + +func (fi *dirNameOnlyFileInfo) Mode() os.FileMode { + return os.ModeDir +} + +func (fi *dirNameOnlyFileInfo) ModTime() time.Time { + return time.Time{} +} + +func (fi *dirNameOnlyFileInfo) IsDir() bool { + return true +} + +func (fi *dirNameOnlyFileInfo) Sys() interface{} { + return nil +} + +func newDirNameOnlyFileInfo(name string, isOrdered bool, fileOpener func() (afero.File, error)) FileMetaInfo { + _, base := filepath.Split(name) + return NewFileMetaInfo(&dirNameOnlyFileInfo{name: base}, FileMeta{ + metaKeyFilename: name, + metaKeyIsOrdered: isOrdered, + metaKeyOpener: fileOpener}) +} + +func decorateFileInfo( + fi os.FileInfo, + fs afero.Fs, opener func() (afero.File, error), + filename, filepath string, inMeta FileMeta) FileMetaInfo { + + var meta FileMeta + var fim FileMetaInfo + + filepath = strings.TrimPrefix(filepath, filepathSeparator) + + var ok bool + if fim, ok = fi.(FileMetaInfo); ok { + meta = fim.Meta() + } else { + meta = make(FileMeta) + fim = NewFileMetaInfo(fi, meta) + } + + meta.setIfNotZero(metaKeyOpener, opener) + meta.setIfNotZero(metaKeyFs, fs) + meta.setIfNotZero(metaKeyPath, filepath) + meta.setIfNotZero(metaKeyFilename, filename) + + mergeFileMeta(inMeta, meta) + + return fim + +} + +func isSymlink(fi os.FileInfo) bool { + return fi != nil && fi.Mode()&os.ModeSymlink == os.ModeSymlink +} + +func fileInfosToFileMetaInfos(fis []os.FileInfo) []FileMetaInfo { + fims := make([]FileMetaInfo, len(fis)) + for i, v := range fis { + fims[i] = v.(FileMetaInfo) + } + return fims +} diff --git a/hugofs/files/classifier.go b/hugofs/files/classifier.go new file mode 100644 index 00000000000..392afd01d78 --- /dev/null +++ b/hugofs/files/classifier.go @@ -0,0 +1,123 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package files + +import ( + "os" + "path/filepath" + "sort" + "strings" +) + +var ( + // This should be the only list of valid extensions for content files. + contentFileExtensions = []string{ + "html", "htm", + "mdown", "markdown", "md", + "asciidoc", "adoc", "ad", + "rest", "rst", + "mmark", + "org", + "pandoc", "pdc"} + + contentFileExtensionsSet map[string]bool +) + +func init() { + contentFileExtensionsSet = make(map[string]bool) + for _, ext := range contentFileExtensions { + contentFileExtensionsSet[ext] = true + } +} + +func IsContentFile(filename string) bool { + return contentFileExtensionsSet[strings.TrimPrefix(filepath.Ext(filename), ".")] +} + +func IsContentExt(ext string) bool { + return contentFileExtensionsSet[ext] +} + +const ( + ContentClassLeaf = "leaf" + ContentClassBranch = "branch" + ContentClassFile = "zfile" // Sort below + ContentClassContent = "zcontent" +) + +func ClassifyContentFile(filename string) string { + if !IsContentFile(filename) { + return ContentClassFile + } + if strings.HasPrefix(filename, "_index.") { + return ContentClassBranch + } + + if strings.HasPrefix(filename, "index.") { + return ContentClassLeaf + } + + return ContentClassContent +} + +const ( + ComponentFolderArchetypes = "archetypes" + ComponentFolderStatic = "static" + ComponentFolderLayouts = "layouts" + ComponentFolderContent = "content" + ComponentFolderData = "data" + ComponentFolderAssets = "assets" + ComponentFolderI18n = "i18n" + + // TODO(bep) mod I don't think this qualifies as a component folder. + ComponentFolderResources = "resources" +) + +var ( + ComponentFolders = []string{ + ComponentFolderArchetypes, + ComponentFolderStatic, + ComponentFolderLayouts, + ComponentFolderContent, + ComponentFolderData, + ComponentFolderAssets, + ComponentFolderI18n, + ComponentFolderResources, + } + + componentFoldersSet = make(map[string]bool) +) + +func init() { + sort.Strings(ComponentFolders) + for _, f := range ComponentFolders { + componentFoldersSet[f] = true + } +} + +// ResolveComponentFolder returns "content" from "content/blog/foo.md" etc. +func ResolveComponentFolder(filename string) string { + filename = strings.TrimPrefix(filename, string(os.PathSeparator)) + for _, cf := range ComponentFolders { + if strings.HasPrefix(filename, cf) { + return cf + } + } + + return "" +} + +func IsComponentFolder(name string) bool { + return componentFoldersSet[name] +} diff --git a/hugofs/files/classifier_test.go b/hugofs/files/classifier_test.go new file mode 100644 index 00000000000..c17d337f98f --- /dev/null +++ b/hugofs/files/classifier_test.go @@ -0,0 +1,49 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package files + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestIsContentFile(t *testing.T) { + assert := require.New(t) + + assert.True(IsContentFile(filepath.FromSlash("my/file.md"))) + assert.True(IsContentFile(filepath.FromSlash("my/file.ad"))) + assert.False(IsContentFile(filepath.FromSlash("textfile.txt"))) + assert.True(IsContentExt("md")) + assert.False(IsContentExt("json")) +} + +func TestComponentFolders(t *testing.T) { + assert := require.New(t) + + // It's important that these are absolutely right and not changed. + assert.Equal(len(ComponentFolders), len(componentFoldersSet)) + assert.True(IsComponentFolder("archetypes")) + assert.True(IsComponentFolder("layouts")) + assert.True(IsComponentFolder("data")) + assert.True(IsComponentFolder("i18n")) + assert.True(IsComponentFolder("assets")) + assert.True(IsComponentFolder("resources")) + assert.True(IsComponentFolder("static")) + assert.True(IsComponentFolder("content")) + assert.False(IsComponentFolder("foo")) + assert.False(IsComponentFolder("")) + +} diff --git a/hugofs/filter_fs.go b/hugofs/filter_fs.go new file mode 100644 index 00000000000..d616aec1d47 --- /dev/null +++ b/hugofs/filter_fs.go @@ -0,0 +1,341 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugofs + +import ( + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + "syscall" + "time" + + "github.com/gohugoio/hugo/hugofs/files" + + "github.com/spf13/afero" +) + +var ( + _ afero.Fs = (*FilterFs)(nil) + _ afero.Lstater = (*FilterFs)(nil) + _ afero.File = (*filterDir)(nil) +) + +func NewLanguageFs(langs map[string]int, fs afero.Fs) (afero.Fs, error) { + + applyMeta := func(fs *FilterFs, name string, fis []os.FileInfo) { + + for i, fi := range fis { + if fi.IsDir() { + filename := filepath.Join(name, fi.Name()) + fis[i] = decorateFileInfo(fi, fs, fs.getOpener(filename), "", "", nil) + continue + } + + meta := fi.(FileMetaInfo).Meta() + lang := meta.Lang() + + fileLang, translationBaseName, translationBaseNameWithExt := langInfoFrom(langs, fi.Name()) + weight := 0 + + if fileLang != "" { + weight = 1 + if fileLang == lang { + // Give priority to myfile.sv.txt inside the sv filesystem. + weight++ + } + lang = fileLang + } + + fim := NewFileMetaInfo(fi, FileMeta{ + metaKeyLang: lang, + metaKeyWeight: weight, + metaKeyOrdinal: langs[lang], + metaKeyTranslationBaseName: translationBaseName, + metaKeyTranslationBaseNameWithExt: translationBaseNameWithExt, + metaKeyClassifier: files.ClassifyContentFile(fi.Name()), + }) + + fis[i] = fim + } + } + + all := func(fis []os.FileInfo) { + // Maps translation base name to a list of language codes. + translations := make(map[string][]string) + trackTranslation := func(meta FileMeta) { + name := meta.TranslationBaseNameWithExt() + translations[name] = append(translations[name], meta.Lang()) + } + for _, fi := range fis { + if fi.IsDir() { + continue + } + meta := fi.(FileMetaInfo).Meta() + + trackTranslation(meta) + + } + + for _, fi := range fis { + fim := fi.(FileMetaInfo) + langs := translations[fim.Meta().TranslationBaseNameWithExt()] + if len(langs) > 0 { + fim.Meta()["translations"] = sortAndremoveStringDuplicates(langs) + } + } + } + + return &FilterFs{ + fs: fs, + applyPerSource: applyMeta, + applyAll: all, + }, nil + +} + +func NewFilterFs(fs afero.Fs) (afero.Fs, error) { + + applyMeta := func(fs *FilterFs, name string, fis []os.FileInfo) { + for i, fi := range fis { + if fi.IsDir() { + fis[i] = decorateFileInfo(fi, fs, fs.getOpener(fi.(FileMetaInfo).Meta().Filename()), "", "", nil) + } + } + } + + ffs := &FilterFs{ + fs: fs, + applyPerSource: applyMeta, + } + + return ffs, nil + +} + +// FilterFs is an ordered composite filesystem. +type FilterFs struct { + fs afero.Fs + + applyPerSource func(fs *FilterFs, name string, fis []os.FileInfo) + applyAll func(fis []os.FileInfo) +} + +func (fs *FilterFs) Chmod(n string, m os.FileMode) error { + return syscall.EPERM +} + +func (fs *FilterFs) Chtimes(n string, a, m time.Time) error { + return syscall.EPERM +} + +func (fs *FilterFs) LstatIfPossible(name string) (os.FileInfo, bool, error) { + fi, err := lstatIfPossible(fs.fs, name) + + if err != nil { + return nil, false, err + } + + if fi.IsDir() { + return decorateFileInfo(fi, fs, fs.getOpener(name), "", "", nil), false, nil + } + + fs.applyFilters(name, -1, fi) + + return fi, false, nil + +} + +func (fs *FilterFs) Mkdir(n string, p os.FileMode) error { + return syscall.EPERM +} + +func (fs *FilterFs) MkdirAll(n string, p os.FileMode) error { + return syscall.EPERM +} + +func (fs *FilterFs) Name() string { + return "WeightedFileSystem" +} + +func (fs *FilterFs) Open(name string) (afero.File, error) { + f, err := fs.fs.Open(name) + if err != nil { + return nil, err + } + + return &filterDir{ + File: f, + ffs: fs, + }, nil + +} + +func (fs *FilterFs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) { + panic("not implemented") +} + +func (fs *FilterFs) ReadDir(name string) ([]os.FileInfo, error) { + panic("not implemented") +} + +func (fs *FilterFs) Remove(n string) error { + return syscall.EPERM +} + +func (fs *FilterFs) RemoveAll(p string) error { + return syscall.EPERM +} + +func (fs *FilterFs) Rename(o, n string) error { + return syscall.EPERM +} + +func (fs *FilterFs) Stat(name string) (os.FileInfo, error) { + fi, _, err := fs.LstatIfPossible(name) + return fi, err +} + +func (fs *FilterFs) Create(n string) (afero.File, error) { + return nil, syscall.EPERM +} + +func (fs *FilterFs) getOpener(name string) func() (afero.File, error) { + return func() (afero.File, error) { + return fs.Open(name) + } +} + +func (fs *FilterFs) applyFilters(name string, count int, fis ...os.FileInfo) ([]os.FileInfo, error) { + if fs.applyPerSource != nil { + fs.applyPerSource(fs, name, fis) + } + + seen := make(map[string]bool) + var duplicates []int + for i, dir := range fis { + if !dir.IsDir() { + continue + } + if seen[dir.Name()] { + duplicates = append(duplicates, i) + } else { + seen[dir.Name()] = true + } + } + + // Remove duplicate directories, keep first. + if len(duplicates) > 0 { + for i := len(duplicates) - 1; i >= 0; i-- { + idx := duplicates[i] + fis = append(fis[:idx], fis[idx+1:]...) + } + } + + if fs.applyAll != nil { + fs.applyAll(fis) + } + + if count > 0 && len(fis) >= count { + return fis[:count], nil + } + + return fis, nil + +} + +type filterDir struct { + afero.File + ffs *FilterFs +} + +func (f *filterDir) Readdir(count int) ([]os.FileInfo, error) { + fis, err := f.File.Readdir(-1) + if err != nil { + return nil, err + } + return f.ffs.applyFilters(f.Name(), count, fis...) +} + +func (f *filterDir) Readdirnames(count int) ([]string, error) { + dirsi, err := f.Readdir(count) + if err != nil { + return nil, err + } + + dirs := make([]string, len(dirsi)) + for i, d := range dirsi { + dirs[i] = d.Name() + } + return dirs, nil +} + +// Try to extract the language from the given filename. +// Any valid language identificator in the name will win over the +// language set on the file system, e.g. "mypost.en.md". +func langInfoFrom(languages map[string]int, name string) (string, string, string) { + var lang string + + baseName := filepath.Base(name) + ext := filepath.Ext(baseName) + translationBaseName := baseName + + if ext != "" { + translationBaseName = strings.TrimSuffix(translationBaseName, ext) + } + + fileLangExt := filepath.Ext(translationBaseName) + fileLang := strings.TrimPrefix(fileLangExt, ".") + + if _, found := languages[fileLang]; found { + lang = fileLang + translationBaseName = strings.TrimSuffix(translationBaseName, fileLangExt) + } + + translationBaseNameWithExt := translationBaseName + + if ext != "" { + translationBaseNameWithExt += ext + } + + return lang, translationBaseName, translationBaseNameWithExt + +} + +func printFs(fs afero.Fs, path string, w io.Writer) { + if fs == nil { + return + } + afero.Walk(fs, path, func(path string, info os.FileInfo, err error) error { + fmt.Println("p:::", path) + return nil + }) +} + +func sortAndremoveStringDuplicates(s []string) []string { + ss := sort.StringSlice(s) + ss.Sort() + i := 0 + for j := 1; j < len(s); j++ { + if !ss.Less(i, j) { + continue + } + i++ + s[i] = s[j] + } + + return s[:i+1] +} diff --git a/hugofs/fs.go b/hugofs/fs.go index 38590a64e0a..163807704a8 100644 --- a/hugofs/fs.go +++ b/hugofs/fs.go @@ -21,8 +21,10 @@ import ( "github.com/spf13/afero" ) -// Os points to an Os Afero file system. -var Os = &afero.OsFs{} +var ( + // Os points to the (real) Os filesystem. + Os = &afero.OsFs{} +) // Fs abstracts the file system to separate source and destination file systems // and allows both to be mocked for testing. diff --git a/hugofs/language_composite_fs.go b/hugofs/language_composite_fs.go index 2889f8a00fc..5dbd252c0be 100644 --- a/hugofs/language_composite_fs.go +++ b/hugofs/language_composite_fs.go @@ -14,6 +14,9 @@ package hugofs import ( + "os" + "path" + "github.com/spf13/afero" ) @@ -30,8 +33,8 @@ type languageCompositeFs struct { // This is a hybrid filesystem. To get a specific file in Open, Stat etc., use the full filename // to the target filesystem. This information is available in Readdir, Stat etc. via the // special LanguageFileInfo FileInfo implementation. -func NewLanguageCompositeFs(base afero.Fs, overlay *LanguageFs) afero.Fs { - return afero.NewReadOnlyFs(&languageCompositeFs{afero.NewCopyOnWriteFs(base, overlay).(*afero.CopyOnWriteFs)}) +func NewLanguageCompositeFs(base, overlay afero.Fs) afero.Fs { + return &languageCompositeFs{afero.NewCopyOnWriteFs(base, overlay).(*afero.CopyOnWriteFs)} } // Open takes the full path to the file in the target filesystem. If it is a directory, it gets merged @@ -49,3 +52,36 @@ func (fs *languageCompositeFs) Open(name string) (afero.File, error) { } return f, nil } + +// LanguageDirsMerger implements the afero.DirsMerger interface, which is used +// to merge two directories. +var LanguageDirsMerger = func(lofi, bofi []os.FileInfo) ([]os.FileInfo, error) { + m := make(map[string]FileMetaInfo) + + getKey := func(fim FileMetaInfo) string { + return path.Join(fim.Meta().Lang(), fim.Name()) + } + + for _, fi := range lofi { + fim := fi.(FileMetaInfo) + m[getKey(fim)] = fim + } + + for _, fi := range bofi { + fim := fi.(FileMetaInfo) + key := getKey(fim) + _, found := m[key] + if !found { + m[key] = fim + } + } + + merged := make([]os.FileInfo, len(m)) + i := 0 + for _, v := range m { + merged[i] = v + i++ + } + + return merged, nil +} diff --git a/hugofs/language_composite_fs_test.go b/hugofs/language_composite_fs_test.go deleted file mode 100644 index ab4e25fc0ac..00000000000 --- a/hugofs/language_composite_fs_test.go +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugofs - -import ( - "path/filepath" - - "strings" - - "testing" - - "github.com/spf13/afero" - "github.com/stretchr/testify/require" -) - -func TestCompositeLanguagFsTest(t *testing.T) { - assert := require.New(t) - - languages := map[string]bool{ - "sv": true, - "en": true, - "nn": true, - } - msv := afero.NewMemMapFs() - baseSv := "/content/sv" - lfssv := NewLanguageFs("sv", languages, afero.NewBasePathFs(msv, baseSv)) - mnn := afero.NewMemMapFs() - baseNn := "/content/nn" - lfsnn := NewLanguageFs("nn", languages, afero.NewBasePathFs(mnn, baseNn)) - men := afero.NewMemMapFs() - baseEn := "/content/en" - lfsen := NewLanguageFs("en", languages, afero.NewBasePathFs(men, baseEn)) - - // The order will be sv, en, nn - composite := NewLanguageCompositeFs(lfsnn, lfsen) - composite = NewLanguageCompositeFs(composite, lfssv) - - afero.WriteFile(msv, filepath.Join(baseSv, "f1.txt"), []byte("some sv"), 0755) - afero.WriteFile(mnn, filepath.Join(baseNn, "f1.txt"), []byte("some nn"), 0755) - afero.WriteFile(men, filepath.Join(baseEn, "f1.txt"), []byte("some en"), 0755) - - // Swedish is the top layer. - assertLangFile(t, composite, "f1.txt", "sv") - - afero.WriteFile(msv, filepath.Join(baseSv, "f2.en.txt"), []byte("some sv"), 0755) - afero.WriteFile(mnn, filepath.Join(baseNn, "f2.en.txt"), []byte("some nn"), 0755) - afero.WriteFile(men, filepath.Join(baseEn, "f2.en.txt"), []byte("some en"), 0755) - - // English is in the middle, but the most specific language match wins. - //assertLangFile(t, composite, "f2.en.txt", "en") - - // Fetch some specific language versions - assertLangFile(t, composite, filepath.Join(baseNn, "f2.en.txt"), "nn") - assertLangFile(t, composite, filepath.Join(baseEn, "f2.en.txt"), "en") - assertLangFile(t, composite, filepath.Join(baseSv, "f2.en.txt"), "sv") - - // Read the root - f, err := composite.Open("/") - assert.NoError(err) - defer f.Close() - files, err := f.Readdir(-1) - assert.NoError(err) - assert.Equal(4, len(files)) - expected := map[string]bool{ - filepath.FromSlash("/content/en/f1.txt"): true, - filepath.FromSlash("/content/nn/f1.txt"): true, - filepath.FromSlash("/content/sv/f1.txt"): true, - filepath.FromSlash("/content/en/f2.en.txt"): true, - } - got := make(map[string]bool) - - for _, fi := range files { - fil, ok := fi.(*LanguageFileInfo) - assert.True(ok) - got[fil.Filename()] = true - } - assert.Equal(expected, got) -} - -func assertLangFile(t testing.TB, fs afero.Fs, filename, match string) { - f, err := fs.Open(filename) - if err != nil { - t.Fatal(err) - } - defer f.Close() - b, err := afero.ReadAll(f) - if err != nil { - t.Fatal(err) - } - - s := string(b) - if !strings.Contains(s, match) { - t.Fatalf("got %q expected it to contain %q", s, match) - - } -} diff --git a/hugofs/language_fs.go b/hugofs/language_fs.go deleted file mode 100644 index db77c1fabd3..00000000000 --- a/hugofs/language_fs.go +++ /dev/null @@ -1,346 +0,0 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugofs - -import ( - "fmt" - "os" - "path/filepath" - "strings" - - "github.com/spf13/afero" -) - -const hugoFsMarker = "__hugofs" - -var ( - _ LanguageAnnouncer = (*LanguageFileInfo)(nil) - _ FilePather = (*LanguageFileInfo)(nil) - _ afero.Lstater = (*LanguageFs)(nil) -) - -// LanguageAnnouncer is aware of its language. -type LanguageAnnouncer interface { - Lang() string - TranslationBaseName() string -} - -// FilePather is aware of its file's location. -type FilePather interface { - // Filename gets the full path and filename to the file. - Filename() string - - // Path gets the content relative path including file name and extension. - // The directory is relative to the content root where "content" is a broad term. - Path() string - - // RealName is FileInfo.Name in its original form. - RealName() string - - BaseDir() string -} - -// LanguageDirsMerger implements the afero.DirsMerger interface, which is used -// to merge two directories. -var LanguageDirsMerger = func(lofi, bofi []os.FileInfo) ([]os.FileInfo, error) { - m := make(map[string]*LanguageFileInfo) - - for _, fi := range lofi { - fil, ok := fi.(*LanguageFileInfo) - if !ok { - return nil, fmt.Errorf("received %T, expected *LanguageFileInfo", fi) - } - m[fil.virtualName] = fil - } - - for _, fi := range bofi { - fil, ok := fi.(*LanguageFileInfo) - if !ok { - return nil, fmt.Errorf("received %T, expected *LanguageFileInfo", fi) - } - existing, found := m[fil.virtualName] - - if !found || existing.weight < fil.weight { - m[fil.virtualName] = fil - } - } - - merged := make([]os.FileInfo, len(m)) - i := 0 - for _, v := range m { - merged[i] = v - i++ - } - - return merged, nil -} - -// LanguageFileInfo is a super-set of os.FileInfo with additional information -// about the file in relation to its Hugo language. -type LanguageFileInfo struct { - os.FileInfo - lang string - baseDir string - realFilename string - relFilename string - name string - realName string - virtualName string - translationBaseName string - - // We add some weight to the files in their own language's content directory. - weight int -} - -// Filename returns a file's real filename including the base (ie. -// "/my/base/sect/page.md"). -func (fi *LanguageFileInfo) Filename() string { - return fi.realFilename -} - -// Path returns a file's filename relative to the base (ie. "sect/page.md"). -func (fi *LanguageFileInfo) Path() string { - return fi.relFilename -} - -// RealName returns a file's real base name (ie. "page.md"). -func (fi *LanguageFileInfo) RealName() string { - return fi.realName -} - -// BaseDir returns a file's base directory (ie. "/my/base"). -func (fi *LanguageFileInfo) BaseDir() string { - return fi.baseDir -} - -// Lang returns a file's language (ie. "sv"). -func (fi *LanguageFileInfo) Lang() string { - return fi.lang -} - -// TranslationBaseName returns the base filename without any extension or language -// identifiers (ie. "page"). -func (fi *LanguageFileInfo) TranslationBaseName() string { - return fi.translationBaseName -} - -// Name is the name of the file within this filesystem without any path info. -// It will be marked with language information so we can identify it as ours -// (ie. "__hugofs_sv_page.md"). -func (fi *LanguageFileInfo) Name() string { - return fi.name -} - -type languageFile struct { - afero.File - fs *LanguageFs -} - -// Readdir creates FileInfo entries by calling Lstat if possible. -func (l *languageFile) Readdir(c int) (ofi []os.FileInfo, err error) { - names, err := l.File.Readdirnames(c) - if err != nil { - return nil, err - } - - fis := make([]os.FileInfo, len(names)) - - for i, name := range names { - fi, _, err := l.fs.LstatIfPossible(filepath.Join(l.Name(), name)) - - if err != nil { - return nil, err - } - fis[i] = fi - } - - return fis, err -} - -// LanguageFs represents a language filesystem. -type LanguageFs struct { - // This Fs is usually created with a BasePathFs - basePath string - lang string - nameMarker string - languages map[string]bool - afero.Fs -} - -// NewLanguageFs creates a new language filesystem. -func NewLanguageFs(lang string, languages map[string]bool, fs afero.Fs) *LanguageFs { - if lang == "" { - panic("no lang set for the language fs") - } - var basePath string - - if bfs, ok := fs.(*afero.BasePathFs); ok { - basePath, _ = bfs.RealPath("") - } - - marker := hugoFsMarker + "_" + lang + "_" - - return &LanguageFs{lang: lang, languages: languages, basePath: basePath, Fs: fs, nameMarker: marker} -} - -// Lang returns a language filesystem's language (ie. "sv"). -func (fs *LanguageFs) Lang() string { - return fs.lang -} - -// Stat returns the os.FileInfo of a given file. -func (fs *LanguageFs) Stat(name string) (os.FileInfo, error) { - name, err := fs.realName(name) - if err != nil { - return nil, err - } - - fi, err := fs.Fs.Stat(name) - if err != nil { - return nil, err - } - - return fs.newLanguageFileInfo(name, fi) -} - -// Open opens the named file for reading. -func (fs *LanguageFs) Open(name string) (afero.File, error) { - name, err := fs.realName(name) - if err != nil { - return nil, err - } - f, err := fs.Fs.Open(name) - - if err != nil { - return nil, err - } - return &languageFile{File: f, fs: fs}, nil -} - -// LstatIfPossible returns the os.FileInfo structure describing a given file. -// It attempts to use Lstat if supported or defers to the os. In addition to -// the FileInfo, a boolean is returned telling whether Lstat was called. -func (fs *LanguageFs) LstatIfPossible(name string) (os.FileInfo, bool, error) { - name, err := fs.realName(name) - if err != nil { - return nil, false, err - } - - var fi os.FileInfo - var b bool - - if lif, ok := fs.Fs.(afero.Lstater); ok { - fi, b, err = lif.LstatIfPossible(name) - } else { - fi, err = fs.Fs.Stat(name) - } - - if err != nil { - return nil, b, err - } - - lfi, err := fs.newLanguageFileInfo(name, fi) - - return lfi, b, err -} - -func (fs *LanguageFs) realPath(name string) (string, error) { - if baseFs, ok := fs.Fs.(*afero.BasePathFs); ok { - return baseFs.RealPath(name) - } - return name, nil -} - -func (fs *LanguageFs) realName(name string) (string, error) { - if strings.Contains(name, hugoFsMarker) { - if !strings.Contains(name, fs.nameMarker) { - return "", os.ErrNotExist - } - return strings.Replace(name, fs.nameMarker, "", 1), nil - } - - if fs.basePath == "" { - return name, nil - } - - return strings.TrimPrefix(name, fs.basePath), nil -} - -func (fs *LanguageFs) newLanguageFileInfo(filename string, fi os.FileInfo) (*LanguageFileInfo, error) { - filename = filepath.Clean(filename) - _, name := filepath.Split(filename) - - realName := name - virtualName := name - - realPath, err := fs.realPath(filename) - if err != nil { - return nil, err - } - - lang := fs.Lang() - - baseNameNoExt := "" - - if !fi.IsDir() { - - // Try to extract the language from the file name. - // Any valid language identificator in the name will win over the - // language set on the file system, e.g. "mypost.en.md". - baseName := filepath.Base(name) - ext := filepath.Ext(baseName) - baseNameNoExt = baseName - - if ext != "" { - baseNameNoExt = strings.TrimSuffix(baseNameNoExt, ext) - } - - fileLangExt := filepath.Ext(baseNameNoExt) - fileLang := strings.TrimPrefix(fileLangExt, ".") - - if fs.languages[fileLang] { - lang = fileLang - baseNameNoExt = strings.TrimSuffix(baseNameNoExt, fileLangExt) - } - - // This connects the filename to the filesystem, not the language. - virtualName = baseNameNoExt + "." + lang + ext - - name = fs.nameMarker + name - } - - weight := 1 - // If this file's language belongs in this directory, add some weight to it - // to make it more important. - if lang == fs.Lang() { - weight = 2 - } - - if fi.IsDir() { - // For directories we always want to start from the union view. - realPath = strings.TrimPrefix(realPath, fs.basePath) - } - - return &LanguageFileInfo{ - lang: lang, - weight: weight, - realFilename: realPath, - realName: realName, - relFilename: strings.TrimPrefix(strings.TrimPrefix(realPath, fs.basePath), string(os.PathSeparator)), - name: name, - virtualName: virtualName, - translationBaseName: baseNameNoExt, - baseDir: fs.basePath, - FileInfo: fi}, nil -} diff --git a/hugofs/language_fs_test.go b/hugofs/language_fs_test.go deleted file mode 100644 index 52e8dc9de74..00000000000 --- a/hugofs/language_fs_test.go +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugofs - -import ( - "path/filepath" - "testing" - - "github.com/spf13/afero" - "github.com/stretchr/testify/require" -) - -func TestLanguagFs(t *testing.T) { - languages := map[string]bool{ - "sv": true, - } - base := filepath.FromSlash("/my/base") - assert := require.New(t) - m := afero.NewMemMapFs() - bfs := afero.NewBasePathFs(m, base) - lfs := NewLanguageFs("sv", languages, bfs) - assert.NotNil(lfs) - assert.Equal("sv", lfs.Lang()) - err := afero.WriteFile(lfs, filepath.FromSlash("sect/page.md"), []byte("abc"), 0777) - assert.NoError(err) - fi, err := lfs.Stat(filepath.FromSlash("sect/page.md")) - assert.NoError(err) - assert.Equal("__hugofs_sv_page.md", fi.Name()) - - languager, ok := fi.(LanguageAnnouncer) - assert.True(ok) - - assert.Equal("sv", languager.Lang()) - - lfi, ok := fi.(*LanguageFileInfo) - assert.True(ok) - assert.Equal(filepath.FromSlash("/my/base/sect/page.md"), lfi.Filename()) - assert.Equal(filepath.FromSlash("sect/page.md"), lfi.Path()) - assert.Equal("page.sv.md", lfi.virtualName) - assert.Equal("__hugofs_sv_page.md", lfi.Name()) - assert.Equal("page.md", lfi.RealName()) - assert.Equal(filepath.FromSlash("/my/base"), lfi.BaseDir()) - assert.Equal("sv", lfi.Lang()) - assert.Equal("page", lfi.TranslationBaseName()) -} - -// Issue 4559 -func TestFilenamesHandling(t *testing.T) { - languages := map[string]bool{ - "sv": true, - } - base := filepath.FromSlash("/my/base") - assert := require.New(t) - m := afero.NewMemMapFs() - bfs := afero.NewBasePathFs(m, base) - lfs := NewLanguageFs("sv", languages, bfs) - assert.NotNil(lfs) - assert.Equal("sv", lfs.Lang()) - - for _, test := range []struct { - filename string - check func(fi *LanguageFileInfo) - }{ - {"tc-lib-color/class-Com.Tecnick.Color.Css", func(fi *LanguageFileInfo) { - assert.Equal("class-Com.Tecnick.Color", fi.TranslationBaseName()) - assert.Equal(filepath.FromSlash("/my/base"), fi.BaseDir()) - assert.Equal(filepath.FromSlash("tc-lib-color/class-Com.Tecnick.Color.Css"), fi.Path()) - assert.Equal("class-Com.Tecnick.Color.Css", fi.RealName()) - assert.Equal(filepath.FromSlash("/my/base/tc-lib-color/class-Com.Tecnick.Color.Css"), fi.Filename()) - }}, - {"tc-lib-color/class-Com.Tecnick.Color.sv.Css", func(fi *LanguageFileInfo) { - assert.Equal("class-Com.Tecnick.Color", fi.TranslationBaseName()) - assert.Equal("class-Com.Tecnick.Color.sv.Css", fi.RealName()) - assert.Equal(filepath.FromSlash("/my/base/tc-lib-color/class-Com.Tecnick.Color.sv.Css"), fi.Filename()) - }}, - } { - err := afero.WriteFile(lfs, filepath.FromSlash(test.filename), []byte("abc"), 0777) - assert.NoError(err) - fi, err := lfs.Stat(filepath.FromSlash(test.filename)) - assert.NoError(err) - - lfi, ok := fi.(*LanguageFileInfo) - assert.True(ok) - assert.Equal("sv", lfi.Lang()) - test.check(lfi) - - } - -} diff --git a/hugofs/nosymlink_fs.go b/hugofs/nosymlink_fs.go new file mode 100644 index 00000000000..42ab94b5ce7 --- /dev/null +++ b/hugofs/nosymlink_fs.go @@ -0,0 +1,85 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugofs + +import ( + "errors" + "os" + + "github.com/spf13/afero" +) + +var ( + ErrPermissionSymlink = errors.New("symlinks not allowed in this filesystem") +) + +func NewNoSymlinkFs(fs afero.Fs) afero.Fs { + return &noSymlinkFs{Fs: fs} +} + +// noSymlinkFs is a filesystem that prevents symlinking. +type noSymlinkFs struct { + afero.Fs +} + +func (fs *noSymlinkFs) LstatIfPossible(name string) (os.FileInfo, bool, error) { + return fs.stat(name) +} + +func (fs *noSymlinkFs) Stat(name string) (os.FileInfo, error) { + fi, _, err := fs.stat(name) + return fi, err +} + +func (fs *noSymlinkFs) stat(name string) (os.FileInfo, bool, error) { + + var ( + fi os.FileInfo + wasLstat bool + err error + ) + + if lstater, ok := fs.Fs.(afero.Lstater); ok { + fi, wasLstat, err = lstater.LstatIfPossible(name) + } else { + + fi, err = fs.Fs.Stat(name) + } + + var metaIsSymlink bool + + if fim, ok := fi.(FileMetaInfo); ok { + metaIsSymlink = fim.Meta().IsSymlink() + } + + if metaIsSymlink || isSymlink(fi) { + return nil, wasLstat, ErrPermissionSymlink + } + + return fi, wasLstat, err +} + +func (fs *noSymlinkFs) Open(name string) (afero.File, error) { + if _, _, err := fs.stat(name); err != nil { + return nil, err + } + return fs.Fs.Open(name) +} + +func (fs *noSymlinkFs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) { + if _, _, err := fs.stat(name); err != nil { + return nil, err + } + return fs.Fs.OpenFile(name, flag, perm) +} diff --git a/hugofs/nosymlink_test.go b/hugofs/nosymlink_test.go new file mode 100644 index 00000000000..6d0b99dccbb --- /dev/null +++ b/hugofs/nosymlink_test.go @@ -0,0 +1,97 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugofs + +import ( + "os" + "path/filepath" + "testing" + + "github.com/gohugoio/hugo/htesting" + + "github.com/spf13/afero" + + "github.com/stretchr/testify/require" +) + +func TestNoSymlinkFs(t *testing.T) { + if skipSymlink() { + t.Skip("Skip; os.Symlink needs administrator rights on Windows") + } + assert := require.New(t) + workDir, clean, err := htesting.CreateTempDir(Os, "hugo-nosymlink") + assert.NoError(err) + defer clean() + wd, _ := os.Getwd() + defer func() { + os.Chdir(wd) + }() + + blogDir := filepath.Join(workDir, "blog") + blogFile := filepath.Join(blogDir, "a.txt") + assert.NoError(os.MkdirAll(blogDir, 0777)) + afero.WriteFile(Os, filepath.Join(blogFile), []byte("content"), 0777) + os.Chdir(workDir) + assert.NoError(os.Symlink("blog", "symlinkdedir")) + os.Chdir(blogDir) + assert.NoError(os.Symlink("a.txt", "symlinkdedfile.txt")) + + fs := NewNoSymlinkFs(Os) + ls := fs.(afero.Lstater) + symlinkedDir := filepath.Join(workDir, "symlinkdedir") + symlinkedFile := filepath.Join(blogDir, "symlinkdedfile.txt") + + // Check Stat and Lstat + for _, stat := range []func(name string) (os.FileInfo, error){ + func(name string) (os.FileInfo, error) { + return fs.Stat(name) + }, + func(name string) (os.FileInfo, error) { + fi, _, err := ls.LstatIfPossible(name) + return fi, err + }, + } { + _, err = stat(symlinkedDir) + assert.Equal(ErrPermissionSymlink, err) + _, err = stat(symlinkedFile) + assert.Equal(ErrPermissionSymlink, err) + + fi, err := stat(filepath.Join(workDir, "blog")) + assert.NoError(err) + assert.NotNil(fi) + + fi, err = stat(blogFile) + assert.NoError(err) + assert.NotNil(fi) + } + + // Check Open + _, err = fs.Open(symlinkedDir) + assert.Equal(ErrPermissionSymlink, err) + _, err = fs.OpenFile(symlinkedDir, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0666) + assert.Equal(ErrPermissionSymlink, err) + _, err = fs.OpenFile(symlinkedFile, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0666) + assert.Equal(ErrPermissionSymlink, err) + _, err = fs.Open(symlinkedFile) + assert.Equal(ErrPermissionSymlink, err) + f, err := fs.Open(blogDir) + assert.NoError(err) + f.Close() + f, err = fs.Open(blogFile) + assert.NoError(err) + f.Close() + + // os.OpenFile(logFile, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0666) + +} diff --git a/hugofs/rootmapping_fs.go b/hugofs/rootmapping_fs.go index 2b8b8d2c012..a1214a02c7f 100644 --- a/hugofs/rootmapping_fs.go +++ b/hugofs/rootmapping_fs.go @@ -1,4 +1,4 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. +// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,10 +14,14 @@ package hugofs import ( + "fmt" "os" "path/filepath" "strings" - "time" + + "github.com/gohugoio/hugo/hugofs/files" + + "github.com/pkg/errors" radix "github.com/hashicorp/go-immutable-radix" "github.com/spf13/afero" @@ -25,151 +29,429 @@ import ( var filepathSeparator = string(filepath.Separator) -// A RootMappingFs maps several roots into one. Note that the root of this filesystem -// is directories only, and they will be returned in Readdir and Readdirnames -// in the order given. -type RootMappingFs struct { - afero.Fs - rootMapToReal *radix.Node - virtualRoots []string +// NewRootMappingFs creates a new RootMappingFs on top of the provided with +// of root mappings with some optional metadata about the root. +// Note that From represents a virtual root that maps to the actual filename in To. +func NewRootMappingFs(fs afero.Fs, rms ...RootMapping) (*RootMappingFs, error) { + rootMapToReal := radix.New().Txn() + + for _, rm := range rms { + (&rm).clean() + + fromBase := files.ResolveComponentFolder(rm.From) + if fromBase == "" { + panic("unrecognised component folder in" + rm.From) + } + + if len(rm.To) < 2 { + panic(fmt.Sprintf("invalid root mapping; from/to: %s/%s", rm.From, rm.To)) + } + + _, err := fs.Stat(rm.To) + if err != nil { + if os.IsNotExist(err) { + continue + } + return nil, err + } + + // Extract "blog" from "content/blog" + rm.path = strings.TrimPrefix(strings.TrimPrefix(rm.From, fromBase), filepathSeparator) + + key := []byte(rm.rootKey()) + var mappings []RootMapping + v, found := rootMapToReal.Get(key) + if found { + // There may be more than one language pointing to the same root. + mappings = v.([]RootMapping) + } + mappings = append(mappings, rm) + rootMapToReal.Insert(key, mappings) + } + + rfs := &RootMappingFs{Fs: fs, + virtualRoots: rms, + rootMapToReal: rootMapToReal.Commit().Root()} + + return rfs, nil } -type rootMappingFile struct { - afero.File - fs *RootMappingFs - name string +// NewRootMappingFsFromFromTo is a convenicence variant of NewRootMappingFs taking +// From and To as string pairs. +func NewRootMappingFsFromFromTo(fs afero.Fs, fromTo ...string) (*RootMappingFs, error) { + rms := make([]RootMapping, len(fromTo)/2) + for i, j := 0, 0; j < len(fromTo); i, j = i+1, j+2 { + rms[i] = RootMapping{ + From: fromTo[j], + To: fromTo[j+1], + } + } + + return NewRootMappingFs(fs, rms...) } -type rootMappingFileInfo struct { - name string +type RootMapping struct { + From string + To string + + path string // The virtual mount point, e.g. "blog". + Meta FileMeta // File metadata (lang etc.) } -func (fi *rootMappingFileInfo) Name() string { - return fi.name +func (rm *RootMapping) clean() { + rm.From = filepath.Clean(rm.From) + rm.To = filepath.Clean(rm.To) } -func (fi *rootMappingFileInfo) Size() int64 { - panic("not implemented") +func (r RootMapping) filename(name string) string { + return filepath.Join(r.To, strings.TrimPrefix(name, r.From)) } -func (fi *rootMappingFileInfo) Mode() os.FileMode { - return os.ModeDir +func (r RootMapping) rootKey() string { + return r.From } -func (fi *rootMappingFileInfo) ModTime() time.Time { - panic("not implemented") +// A RootMappingFs maps several roots into one. Note that the root of this filesystem +// is directories only, and they will be returned in Readdir and Readdirnames +// in the order given. +type RootMappingFs struct { + afero.Fs + rootMapToReal *radix.Node + virtualRoots []RootMapping + filter func(r RootMapping) bool } -func (fi *rootMappingFileInfo) IsDir() bool { - return true +func (fs *RootMappingFs) Dirs(base string) ([]FileMetaInfo, error) { + roots := fs.getRootsWithPrefix(base) + + if roots == nil { + return nil, nil + } + + fss := make([]FileMetaInfo, len(roots)) + for i, r := range roots { + bfs := afero.NewBasePathFs(fs.Fs, r.To) + bfs = decoratePath(bfs, func(name string) string { + p := strings.TrimPrefix(name, r.To) + if r.path != "" { + // Make sure it's mounted to a any sub path, e.g. blog + p = filepath.Join(r.path, p) + } + p = strings.TrimLeft(p, filepathSeparator) + return p + }) + fs := decorateDirs(bfs, r.Meta) + fi, err := fs.Stat("") + if err != nil { + return nil, errors.Wrap(err, "RootMappingFs.Dirs") + } + fss[i] = fi.(FileMetaInfo) + } + + return fss, nil } -func (fi *rootMappingFileInfo) Sys() interface{} { - return nil +// LstatIfPossible returns the os.FileInfo structure describing a given file. +func (fs *RootMappingFs) LstatIfPossible(name string) (os.FileInfo, bool, error) { + fis, b, err := fs.doLstat(name, false) + if err != nil { + return nil, b, err + } + return fis[0], b, nil + } -func newRootMappingDirFileInfo(name string) *rootMappingFileInfo { - return &rootMappingFileInfo{name: name} +func (fs *RootMappingFs) virtualDirOpener(name string, isRoot bool) func() (afero.File, error) { + return func() (afero.File, error) { return &rootMappingFile{name: name, isRoot: isRoot, fs: fs}, nil } } -// NewRootMappingFs creates a new RootMappingFs on top of the provided with -// a list of from, to string pairs of root mappings. -// Note that 'from' represents a virtual root that maps to the actual filename in 'to'. -func NewRootMappingFs(fs afero.Fs, fromTo ...string) (*RootMappingFs, error) { - rootMapToReal := radix.New().Txn() - var virtualRoots []string +func (fs *RootMappingFs) doLstat(name string, allowMultiple bool) ([]FileMetaInfo, bool, error) { + + if fs.isRoot(name) { + return []FileMetaInfo{newDirNameOnlyFileInfo(name, true, fs.virtualDirOpener(name, true))}, false, nil + } + + roots := fs.getRoots(name) - for i := 0; i < len(fromTo); i += 2 { - vr := filepath.Clean(fromTo[i]) - rr := filepath.Clean(fromTo[i+1]) + if len(roots) == 0 { + roots := fs.getRootsWithPrefix(name) + if len(roots) != 0 { + // We have root mappings below name, let's make it look like + // a directory. + return []FileMetaInfo{newDirNameOnlyFileInfo(name, true, fs.virtualDirOpener(name, false))}, false, nil + } + + return nil, false, os.ErrNotExist + } - // We need to preserve the original order for Readdir - virtualRoots = append(virtualRoots, vr) + var ( + fis []FileMetaInfo + b bool + fi os.FileInfo + root RootMapping + err error + ) + + for _, root = range roots { + fi, b, err = fs.statRoot(root, name) + if err != nil { + if os.IsNotExist(err) { + continue + } + return nil, false, err + } + fim := fi.(FileMetaInfo) + fis = append(fis, fim) + } - rootMapToReal.Insert([]byte(vr), rr) + if len(fis) == 0 { + return nil, false, os.ErrNotExist } - return &RootMappingFs{Fs: fs, - virtualRoots: virtualRoots, - rootMapToReal: rootMapToReal.Commit().Root()}, nil + if allowMultiple || len(fis) == 1 { + return fis, b, nil + } + + // Open it in this composite filesystem. + opener := func() (afero.File, error) { + return fs.Open(name) + } + + return []FileMetaInfo{decorateFileInfo(fi, fs, opener, "", "", root.Meta)}, b, nil + } -// Stat returns the os.FileInfo structure describing a given file. If there is -// an error, it will be of type *os.PathError. -func (fs *RootMappingFs) Stat(name string) (os.FileInfo, error) { +// Open opens the namedrootMappingFile file for reading. +func (fs *RootMappingFs) Open(name string) (afero.File, error) { if fs.isRoot(name) { - return newRootMappingDirFileInfo(name), nil + return &rootMappingFile{name: name, fs: fs, isRoot: true}, nil } - realName := fs.realName(name) - fi, err := fs.Fs.Stat(realName) - if rfi, ok := fi.(RealFilenameInfo); ok { - return rfi, err + fis, _, err := fs.doLstat(name, true) + if err != nil { + return nil, err } - return &realFilenameInfo{FileInfo: fi, realFilename: realName}, err + if len(fis) == 1 { + fi := fis[0] + meta := fi.(FileMetaInfo).Meta() + f, err := meta.Open() + if err != nil { + return nil, err + } + return &rootMappingFile{File: f, fs: fs, name: name, meta: meta}, nil + } + + return fs.newUnionFile(fis...) } +// Stat returns the os.FileInfo structure describing a given file. If there is +// an error, it will be of type *os.PathError. +func (fs *RootMappingFs) Stat(name string) (os.FileInfo, error) { + fi, _, err := fs.LstatIfPossible(name) + return fi, err + +} + +// Filter creates a copy of this filesystem with the applied filter. +func (fs RootMappingFs) Filter(f func(m RootMapping) bool) *RootMappingFs { + fs.filter = f + return &fs +} + func (fs *RootMappingFs) isRoot(name string) bool { return name == "" || name == filepathSeparator } -// Open opens the named file for reading. -func (fs *RootMappingFs) Open(name string) (afero.File, error) { - if fs.isRoot(name) { - return &rootMappingFile{name: name, fs: fs}, nil +func (fs *RootMappingFs) getRoots(name string) []RootMapping { + nameb := []byte(filepath.Clean(name)) + _, v, found := fs.rootMapToReal.LongestPrefix(nameb) + if !found { + return nil } - realName := fs.realName(name) - f, err := fs.Fs.Open(realName) + + rm := v.([]RootMapping) + + if fs.filter != nil { + var filtered []RootMapping + for _, m := range rm { + if fs.filter(m) { + filtered = append(filtered, m) + } + } + return filtered + } + + return rm +} + +func (fs *RootMappingFs) getRootsWithPrefix(prefix string) []RootMapping { + if fs.isRoot(prefix) { + return fs.virtualRoots + } + prefixb := []byte(filepath.Clean(prefix)) + var roots []RootMapping + + fs.rootMapToReal.WalkPrefix(prefixb, func(b []byte, v interface{}) bool { + roots = append(roots, v.([]RootMapping)...) + return false + }) + + return roots +} + +func (fs *RootMappingFs) newUnionFile(fis ...FileMetaInfo) (afero.File, error) { + meta := fis[0].Meta() + f, err := meta.Open() if err != nil { return nil, err } - return &rootMappingFile{File: f, name: name, fs: fs}, nil -} + rf := &rootMappingFile{File: f, fs: fs, name: meta.Name(), meta: meta} + if len(fis) == 1 { + return rf, err + } -// LstatIfPossible returns the os.FileInfo structure describing a given file. -// It attempts to use Lstat if supported or defers to the os. In addition to -// the FileInfo, a boolean is returned telling whether Lstat was called. -func (fs *RootMappingFs) LstatIfPossible(name string) (os.FileInfo, bool, error) { + next, err := fs.newUnionFile(fis[1:]...) + if err != nil { + return nil, err + } - if fs.isRoot(name) { - return newRootMappingDirFileInfo(name), false, nil + uf := &afero.UnionFile{Base: rf, Layer: next} + + uf.Merger = func(lofi, bofi []os.FileInfo) ([]os.FileInfo, error) { + // Ignore duplicate directory entries + seen := make(map[string]bool) + var result []os.FileInfo + + for _, fis := range [][]os.FileInfo{bofi, lofi} { + for _, fi := range fis { + + if fi.IsDir() && seen[fi.Name()] { + continue + } + + if fi.IsDir() { + seen[fi.Name()] = true + } + + result = append(result, fi) + } + } + + return result, nil } - name = fs.realName(name) + + return uf, nil + +} + +func (fs *RootMappingFs) statRoot(root RootMapping, name string) (os.FileInfo, bool, error) { + filename := root.filename(name) + + var b bool + var fi os.FileInfo + var err error if ls, ok := fs.Fs.(afero.Lstater); ok { - fi, b, err := ls.LstatIfPossible(name) - return &realFilenameInfo{FileInfo: fi, realFilename: name}, b, err + fi, b, err = ls.LstatIfPossible(filename) + if err != nil { + return nil, b, err + } + + } else { + fi, err = fs.Fs.Stat(filename) + if err != nil { + return nil, b, err + } + } + + // Opens the real directory/file. + opener := func() (afero.File, error) { + return fs.Fs.Open(filename) + } + + if fi.IsDir() { + _, name = filepath.Split(name) + fi = newDirNameOnlyFileInfo(name, false, opener) } - fi, err := fs.Stat(name) - return fi, false, err + + return decorateFileInfo(fi, fs.Fs, opener, "", "", root.Meta), b, nil + } -func (fs *RootMappingFs) realName(name string) string { - key, val, found := fs.rootMapToReal.LongestPrefix([]byte(filepath.Clean(name))) - if !found { - return name +type rootMappingFile struct { + afero.File + fs *RootMappingFs + name string + meta FileMeta + isRoot bool +} + +func (f *rootMappingFile) Close() error { + if f.File == nil { + return nil } - keystr := string(key) + return f.File.Close() +} - return filepath.Join(val.(string), strings.TrimPrefix(name, keystr)) +func (f *rootMappingFile) Name() string { + return f.name } func (f *rootMappingFile) Readdir(count int) ([]os.FileInfo, error) { if f.File == nil { dirsn := make([]os.FileInfo, 0) - for i := 0; i < len(f.fs.virtualRoots); i++ { - if count != -1 && i >= count { + roots := f.fs.getRootsWithPrefix(f.name) + seen := make(map[string]bool) + + j := 0 + for _, rm := range roots { + if count != -1 && j >= count { break } - dirsn = append(dirsn, newRootMappingDirFileInfo(f.fs.virtualRoots[i])) + + opener := func() (afero.File, error) { + return f.fs.Open(rm.From) + } + + name := rm.From + if !f.isRoot { + _, name = filepath.Split(rm.From) + } + + if seen[name] { + continue + } + seen[name] = true + + j++ + + fi := newDirNameOnlyFileInfo(name, false, opener) + if rm.Meta != nil { + mergeFileMeta(rm.Meta, fi.Meta()) + } + + dirsn = append(dirsn, fi) } return dirsn, nil } - return f.File.Readdir(count) + if f.File == nil { + panic(fmt.Sprintf("no File for %q", f.name)) + } + + fis, err := f.File.Readdir(count) + if err != nil { + return nil, err + } + + for i, fi := range fis { + fis[i] = decorateFileInfo(fi, f.fs, nil, "", "", f.meta) + } + + return fis, nil } func (f *rootMappingFile) Readdirnames(count int) ([]string, error) { @@ -183,14 +465,3 @@ func (f *rootMappingFile) Readdirnames(count int) ([]string, error) { } return dirss, nil } - -func (f *rootMappingFile) Name() string { - return f.name -} - -func (f *rootMappingFile) Close() error { - if f.File == nil { - return nil - } - return f.File.Close() -} diff --git a/hugofs/rootmapping_fs_test.go b/hugofs/rootmapping_fs_test.go index e6a0301c941..2fbd4337461 100644 --- a/hugofs/rootmapping_fs_test.go +++ b/hugofs/rootmapping_fs_test.go @@ -1,4 +1,4 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. +// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -19,24 +19,118 @@ import ( "path/filepath" "testing" + "github.com/spf13/viper" + + "github.com/gohugoio/hugo/htesting" "github.com/spf13/afero" "github.com/stretchr/testify/require" ) -func TestRootMappingFsRealName(t *testing.T) { +// TODO(bep) mod +// tc-lib-color/class-Com.Tecnick.Color.Css and class-Com.Tecnick.Color.sv.Css + +func TestLanguageRootMapping(t *testing.T) { assert := require.New(t) - fs := afero.NewMemMapFs() + v := viper.New() + v.Set("contentDir", "content") + + fs := NewBaseFileDecorator(afero.NewMemMapFs()) + + assert.NoError(afero.WriteFile(fs, filepath.Join("content/sv/svdir", "main.txt"), []byte("main sv"), 0755)) + assert.NoError(afero.WriteFile(fs, filepath.Join("themes/a/mysvblogcontent", "sv-f.txt"), []byte("some sv blog content"), 0755)) + assert.NoError(afero.WriteFile(fs, filepath.Join("themes/a/myenblogcontent", "en-f.txt"), []byte("some en blog content in a"), 0755)) + assert.NoError(afero.WriteFile(fs, filepath.Join("themes/a/myotherenblogcontent", "en-f2.txt"), []byte("some en content"), 0755)) + assert.NoError(afero.WriteFile(fs, filepath.Join("themes/a/mysvdocs", "sv-docs.txt"), []byte("some sv docs content"), 0755)) + assert.NoError(afero.WriteFile(fs, filepath.Join("themes/b/myenblogcontent", "en-b-f.txt"), []byte("some en content"), 0755)) + + rfs, err := NewRootMappingFs(fs, + RootMapping{ + From: "content/blog", // Virtual path, first element is one of content, static, layouts etc. + To: "themes/a/mysvblogcontent", // Real path + Meta: FileMeta{"lang": "sv"}, + }, + RootMapping{ + From: "content/blog", + To: "themes/a/myenblogcontent", + Meta: FileMeta{"lang": "en"}, + }, + RootMapping{ + From: "content/blog", + To: "content/sv", + Meta: FileMeta{"lang": "sv"}, + }, + RootMapping{ + From: "content/blog", + To: "themes/a/myotherenblogcontent", + Meta: FileMeta{"lang": "en"}, + }, + RootMapping{ + From: "content/docs", + To: "themes/a/mysvdocs", + Meta: FileMeta{"lang": "sv"}, + }, + ) + + assert.NoError(err) - rfs, err := NewRootMappingFs(fs, "f1", "f1t", "f2", "f2t") + collected, err := collectFilenames(rfs, "content", "content") assert.NoError(err) + assert.Equal([]string{"blog/en-f.txt", "blog/en-f2.txt", "blog/sv-f.txt", "blog/svdir/main.txt", "docs/sv-docs.txt"}, collected) + + bfs := afero.NewBasePathFs(rfs, "content") + collected, err = collectFilenames(bfs, "", "") + assert.NoError(err) + assert.Equal([]string{"blog/en-f.txt", "blog/en-f2.txt", "blog/sv-f.txt", "blog/svdir/main.txt", "docs/sv-docs.txt"}, collected) + + dirs, err := rfs.Dirs(filepath.FromSlash("content/blog")) + assert.NoError(err) + + assert.Equal(4, len(dirs)) + + getDirnames := func(name string, rfs *RootMappingFs) []string { + filename := filepath.FromSlash(name) + f, err := rfs.Open(filename) + assert.NoError(err) + names, err := f.Readdirnames(-1) + + f.Close() + assert.NoError(err) - assert.Equal(filepath.FromSlash("f1t/foo/file.txt"), rfs.realName(filepath.Join("f1", "foo", "file.txt"))) + info, err := rfs.Stat(filename) + assert.NoError(err) + f2, err := info.(FileMetaInfo).Meta().Open() + assert.NoError(err) + names2, err := f2.Readdirnames(-1) + assert.NoError(err) + assert.Equal(names, names2) + f2.Close() + + return names + } + + rfsEn := rfs.Filter(func(rm RootMapping) bool { + return rm.Meta.Lang() == "en" + }) + + assert.Equal([]string{"en-f.txt", "en-f2.txt"}, getDirnames("content/blog", rfsEn)) + + rfsSv := rfs.Filter(func(rm RootMapping) bool { + return rm.Meta.Lang() == "sv" + }) + + assert.Equal([]string{"sv-f.txt", "svdir"}, getDirnames("content/blog", rfsSv)) + + // Make sure we have not messed with the original + assert.Equal([]string{"sv-f.txt", "en-f.txt", "svdir", "en-f2.txt"}, getDirnames("content/blog", rfs)) + + assert.Equal([]string{"blog", "docs"}, getDirnames("content", rfsSv)) + assert.Equal([]string{"blog", "docs"}, getDirnames("content", rfs)) } func TestRootMappingFsDirnames(t *testing.T) { assert := require.New(t) - fs := afero.NewMemMapFs() + fs := NewBaseFileDecorator(afero.NewMemMapFs()) testfile := "myfile.txt" assert.NoError(fs.Mkdir("f1t", 0755)) @@ -44,13 +138,14 @@ func TestRootMappingFsDirnames(t *testing.T) { assert.NoError(fs.Mkdir("f3t", 0755)) assert.NoError(afero.WriteFile(fs, filepath.Join("f2t", testfile), []byte("some content"), 0755)) - rfs, err := NewRootMappingFs(fs, "bf1", "f1t", "cf2", "f2t", "af3", "f3t") + rfs, err := NewRootMappingFsFromFromTo(fs, "static/bf1", "f1t", "static/cf2", "f2t", "static/af3", "f3t") assert.NoError(err) - fif, err := rfs.Stat(filepath.Join("cf2", testfile)) + fif, err := rfs.Stat(filepath.Join("static/cf2", testfile)) assert.NoError(err) assert.Equal("myfile.txt", fif.Name()) - assert.Equal(filepath.FromSlash("f2t/myfile.txt"), fif.(RealFilenameInfo).RealFilename()) + fifm := fif.(FileMetaInfo).Meta() + assert.Equal(filepath.FromSlash("f2t/myfile.txt"), fifm.Filename()) root, err := rfs.Open(filepathSeparator) assert.NoError(err) @@ -61,6 +156,91 @@ func TestRootMappingFsDirnames(t *testing.T) { } +func TestRootMappingFsFilename(t *testing.T) { + assert := require.New(t) + workDir, clean, err := htesting.CreateTempDir(Os, "hugo-root-filename") + assert.NoError(err) + defer clean() + fs := NewBaseFileDecorator(Os) + + testfilename := filepath.Join(workDir, "f1t/foo/file.txt") + + assert.NoError(fs.MkdirAll(filepath.Join(workDir, "f1t/foo"), 0777)) + assert.NoError(afero.WriteFile(fs, testfilename, []byte("content"), 0666)) + + rfs, err := NewRootMappingFsFromFromTo(fs, "static/f1", filepath.Join(workDir, "f1t"), "static/f2", filepath.Join(workDir, "f2t")) + assert.NoError(err) + + fi, err := rfs.Stat(filepath.FromSlash("static/f1/foo/file.txt")) + assert.NoError(err) + fim := fi.(FileMetaInfo) + assert.Equal(testfilename, fim.Meta().Filename()) + _, err = rfs.Stat(filepath.FromSlash("static/f1")) + assert.NoError(err) +} + +func TestRootMappingFsMount(t *testing.T) { + assert := require.New(t) + fs := NewBaseFileDecorator(afero.NewMemMapFs()) + + testfile := "test.txt" + + assert.NoError(afero.WriteFile(fs, filepath.Join("themes/a/mynoblogcontent", testfile), []byte("some no content"), 0755)) + assert.NoError(afero.WriteFile(fs, filepath.Join("themes/a/myenblogcontent", testfile), []byte("some en content"), 0755)) + assert.NoError(afero.WriteFile(fs, filepath.Join("themes/a/mysvblogcontent", testfile), []byte("some sv content"), 0755)) + assert.NoError(afero.WriteFile(fs, filepath.Join("themes/a/mysvblogcontent", "other.txt"), []byte("some sv content"), 0755)) + + bfs := afero.NewBasePathFs(fs, "themes/a").(*afero.BasePathFs) + rm := []RootMapping{ + RootMapping{From: "content/blog", + To: "mynoblogcontent", + Meta: FileMeta{"lang": "no"}, + }, + RootMapping{From: "content/blog", + To: "myenblogcontent", + Meta: FileMeta{"lang": "en"}, + }, + RootMapping{From: "content/blog", + To: "mysvblogcontent", + Meta: FileMeta{"lang": "sv"}, + }, + } + + rfs, err := NewRootMappingFs(bfs, rm...) + assert.NoError(err) + + blog, err := rfs.Stat(filepath.FromSlash("content/blog")) + assert.NoError(err) + blogm := blog.(FileMetaInfo).Meta() + assert.Equal("sv", blogm.Lang()) // Last match + + f, err := blogm.Open() + assert.NoError(err) + defer f.Close() + dirs1, err := f.Readdirnames(-1) + assert.NoError(err) + // Union with duplicate dir names filtered. + assert.Equal([]string{"test.txt", "test.txt", "other.txt", "test.txt"}, dirs1) + + files, err := afero.ReadDir(rfs, filepath.FromSlash("content/blog")) + assert.NoError(err) + assert.Equal(4, len(files)) + + testfilefi := files[1] + assert.Equal(testfile, testfilefi.Name()) + + testfilem := testfilefi.(FileMetaInfo).Meta() + assert.Equal(filepath.FromSlash("themes/a/mynoblogcontent/test.txt"), testfilem.Filename()) + + tf, err := testfilem.Open() + assert.NoError(err) + defer tf.Close() + c, err := ioutil.ReadAll(tf) + assert.NoError(err) + assert.Equal("some no content", string(c)) + +} + func TestRootMappingFsOs(t *testing.T) { assert := require.New(t) fs := afero.NewOsFs() @@ -77,10 +257,10 @@ func TestRootMappingFsOs(t *testing.T) { assert.NoError(fs.Mkdir(filepath.Join(d, "f3t"), 0755)) assert.NoError(afero.WriteFile(fs, filepath.Join(d, "f2t", testfile), []byte("some content"), 0755)) - rfs, err := NewRootMappingFs(fs, "bf1", filepath.Join(d, "f1t"), "cf2", filepath.Join(d, "f2t"), "af3", filepath.Join(d, "f3t")) + rfs, err := NewRootMappingFsFromFromTo(fs, "static/bf1", filepath.Join(d, "f1t"), "static/cf2", filepath.Join(d, "f2t"), "static/af3", filepath.Join(d, "f3t")) assert.NoError(err) - fif, err := rfs.Stat(filepath.Join("cf2", testfile)) + fif, err := rfs.Stat(filepath.Join("static/cf2", testfile)) assert.NoError(err) assert.Equal("myfile.txt", fif.Name()) diff --git a/hugofs/slice_fs.go b/hugofs/slice_fs.go new file mode 100644 index 00000000000..7a61401ec52 --- /dev/null +++ b/hugofs/slice_fs.go @@ -0,0 +1,293 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugofs + +import ( + "os" + "syscall" + "time" + + "github.com/pkg/errors" + + "github.com/spf13/afero" +) + +var ( + _ afero.Fs = (*SliceFs)(nil) + _ afero.Lstater = (*SliceFs)(nil) + _ afero.File = (*sliceDir)(nil) +) + +func NewSliceFs(dirs ...FileMetaInfo) (afero.Fs, error) { + if len(dirs) == 0 { + return NoOpFs, nil + } + + for _, dir := range dirs { + if !dir.IsDir() { + return nil, errors.New("this fs supports directories only") + } + } + + fs := &SliceFs{ + dirs: dirs, + } + + return fs, nil + +} + +// SliceFs is an ordered composite filesystem. +type SliceFs struct { + dirs []FileMetaInfo +} + +func (fs *SliceFs) Chmod(n string, m os.FileMode) error { + return syscall.EPERM +} + +func (fs *SliceFs) Chtimes(n string, a, m time.Time) error { + return syscall.EPERM +} + +func (fs *SliceFs) LstatIfPossible(name string) (os.FileInfo, bool, error) { + fi, _, err := fs.pickFirst(name) + + if err != nil { + return nil, false, err + } + + if fi.IsDir() { + return decorateFileInfo(fi, fs, fs.getOpener(name), "", "", nil), false, nil + } + + return nil, false, errors.Errorf("lstat: files not supported: %q", name) + +} + +func (fs *SliceFs) Mkdir(n string, p os.FileMode) error { + return syscall.EPERM +} + +func (fs *SliceFs) MkdirAll(n string, p os.FileMode) error { + return syscall.EPERM +} + +func (fs *SliceFs) Name() string { + return "SliceFs" +} + +func (fs *SliceFs) Open(name string) (afero.File, error) { + fi, idx, err := fs.pickFirst(name) + if err != nil { + return nil, err + } + + if !fi.IsDir() { + panic("currently only dirs in here") + } + + return &sliceDir{ + lfs: fs, + idx: idx, + dirname: name, + }, nil + +} + +func (fs *SliceFs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) { + panic("not implemented") +} + +func (fs *SliceFs) ReadDir(name string) ([]os.FileInfo, error) { + panic("not implemented") +} + +func (fs *SliceFs) Remove(n string) error { + return syscall.EPERM +} + +func (fs *SliceFs) RemoveAll(p string) error { + return syscall.EPERM +} + +func (fs *SliceFs) Rename(o, n string) error { + return syscall.EPERM +} + +func (fs *SliceFs) Stat(name string) (os.FileInfo, error) { + fi, _, err := fs.LstatIfPossible(name) + return fi, err +} + +func (fs *SliceFs) Create(n string) (afero.File, error) { + return nil, syscall.EPERM +} + +func (fs *SliceFs) getOpener(name string) func() (afero.File, error) { + return func() (afero.File, error) { + return fs.Open(name) + } +} + +func (fs *SliceFs) pickFirst(name string) (os.FileInfo, int, error) { + for i, mfs := range fs.dirs { + meta := mfs.Meta() + fs := meta.Fs() + fi, err := lstatIfPossible(fs, name) + if err == nil { + // Gotta match! + return fi, i, nil + } + + if !os.IsNotExist(err) { + // Real error + return nil, -1, err + } + } + + // Not found + return nil, -1, os.ErrNotExist +} + +func (fs *SliceFs) readDirs(name string, startIdx, count int) ([]os.FileInfo, error) { + collect := func(lfs FileMeta) ([]os.FileInfo, error) { + d, err := lfs.Fs().Open(name) + if err != nil { + if !os.IsNotExist(err) { + return nil, err + } + return nil, nil + } else { + defer d.Close() + dirs, err := d.Readdir(-1) + if err != nil { + return nil, err + } + return dirs, nil + } + } + + var dirs []os.FileInfo + + for i := startIdx; i < len(fs.dirs); i++ { + mfs := fs.dirs[i] + + fis, err := collect(mfs.Meta()) + if err != nil { + return nil, err + } + + dirs = append(dirs, fis...) + + } + + seen := make(map[string]bool) + var duplicates []int + for i, fi := range dirs { + if !fi.IsDir() { + continue + } + + if seen[fi.Name()] { + duplicates = append(duplicates, i) + } else { + // Make sure it's opened by this filesystem. + dirs[i] = decorateFileInfo(fi, fs, fs.getOpener(fi.(FileMetaInfo).Meta().Filename()), "", "", nil) + seen[fi.Name()] = true + } + } + + // Remove duplicate directories, keep first. + if len(duplicates) > 0 { + for i := len(duplicates) - 1; i >= 0; i-- { + idx := duplicates[i] + dirs = append(dirs[:idx], dirs[idx+1:]...) + } + } + + if count > 0 && len(dirs) >= count { + return dirs[:count], nil + } + + return dirs, nil + +} + +type sliceDir struct { + lfs *SliceFs + idx int + dirname string +} + +func (f *sliceDir) Close() error { + return nil +} + +func (f *sliceDir) Name() string { + return f.dirname +} + +func (f *sliceDir) Read(p []byte) (n int, err error) { + panic("not implemented") +} + +func (f *sliceDir) ReadAt(p []byte, off int64) (n int, err error) { + panic("not implemented") +} + +func (f *sliceDir) Readdir(count int) ([]os.FileInfo, error) { + return f.lfs.readDirs(f.dirname, f.idx, count) +} + +func (f *sliceDir) Readdirnames(count int) ([]string, error) { + dirsi, err := f.Readdir(count) + if err != nil { + return nil, err + } + + dirs := make([]string, len(dirsi)) + for i, d := range dirsi { + dirs[i] = d.Name() + } + return dirs, nil +} + +func (f *sliceDir) Seek(offset int64, whence int) (int64, error) { + panic("not implemented") +} + +func (f *sliceDir) Stat() (os.FileInfo, error) { + panic("not implemented") +} + +func (f *sliceDir) Sync() error { + panic("not implemented") +} + +func (f *sliceDir) Truncate(size int64) error { + panic("not implemented") +} + +func (f *sliceDir) Write(p []byte) (n int, err error) { + panic("not implemented") +} + +func (f *sliceDir) WriteAt(p []byte, off int64) (n int, err error) { + panic("not implemented") +} + +func (f *sliceDir) WriteString(s string) (ret int, err error) { + panic("not implemented") +} diff --git a/hugofs/nolstat_fs.go b/hugofs/slice_fs_test.go similarity index 58% rename from hugofs/nolstat_fs.go rename to hugofs/slice_fs_test.go index 6b27e8e1f64..4646c2de916 100644 --- a/hugofs/nolstat_fs.go +++ b/hugofs/slice_fs_test.go @@ -12,28 +12,3 @@ // limitations under the License. package hugofs - -import ( - "os" - - "github.com/spf13/afero" -) - -var ( - _ afero.Fs = (*noLstatFs)(nil) -) - -type noLstatFs struct { - afero.Fs -} - -// NewNoLstatFs creates a new filesystem with no Lstat support. -func NewNoLstatFs(fs afero.Fs) afero.Fs { - return &noLstatFs{Fs: fs} -} - -// LstatIfPossible always delegates to Stat. -func (fs *noLstatFs) LstatIfPossible(name string) (os.FileInfo, bool, error) { - fi, err := fs.Stat(name) - return fi, false, err -} diff --git a/hugofs/walk.go b/hugofs/walk.go new file mode 100644 index 00000000000..3edddf07870 --- /dev/null +++ b/hugofs/walk.go @@ -0,0 +1,317 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugofs + +import ( + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/gohugoio/hugo/common/loggers" + + "github.com/pkg/errors" + + "github.com/spf13/afero" +) + +type ( + WalkFunc func(path string, info FileMetaInfo, err error) error + WalkHook func(dir FileMetaInfo, path string, readdir []FileMetaInfo) ([]FileMetaInfo, error) +) + +type Walkway struct { + fs afero.Fs + root string + basePath string + + logger *loggers.Logger + + // May be pre-set + fi FileMetaInfo + dirEntries []FileMetaInfo + + walkFn WalkFunc + walked bool + + // We may traverse symbolic links and bite ourself. + seen map[string]bool + + // Optional hooks + hookPre WalkHook + hookPost WalkHook +} + +type WalkwayConfig struct { + Fs afero.Fs + Root string + BasePath string + + Logger *loggers.Logger + + // One or both of these may be pre-set. + Info FileMetaInfo + DirEntries []FileMetaInfo + + WalkFn WalkFunc + HookPre WalkHook + HookPost WalkHook +} + +func NewWalkway(cfg WalkwayConfig) *Walkway { + var fs afero.Fs + if cfg.Info != nil { + fs = cfg.Info.Meta().Fs() + } else { + fs = cfg.Fs + } + + basePath := cfg.BasePath + if basePath != "" && !strings.HasSuffix(basePath, filepathSeparator) { + basePath += filepathSeparator + } + + logger := cfg.Logger + if logger == nil { + logger = loggers.NewWarningLogger() + } + + return &Walkway{ + fs: fs, + root: cfg.Root, + basePath: basePath, + fi: cfg.Info, + dirEntries: cfg.DirEntries, + walkFn: cfg.WalkFn, + hookPre: cfg.HookPre, + hookPost: cfg.HookPost, + logger: logger, + seen: make(map[string]bool)} +} + +// TODO(bep) make content use this +func (w *Walkway) Walk() error { + if w.walked { + panic("this walkway is already walked") + } + w.walked = true + + if w.fs == NoOpFs { + return nil + } + + var fi FileMetaInfo + if w.fi != nil { + fi = w.fi + } else { + info, err := lstatIfPossible(w.fs, w.root) + if err != nil { + if os.IsNotExist(err) { + return nil + } + + if err == ErrPermissionSymlink { + w.logger.WARN.Printf("Unsupported symlink found in %q, skipping.", w.root) + return nil + } + + return w.walkFn(w.root, nil, errors.Wrapf(err, "walk: %q", w.root)) + } + fi = info.(FileMetaInfo) + } + + if !fi.IsDir() { + return w.walkFn(w.root, nil, errors.New("file to walk must be a directory")) + } + + return w.walk(w.root, fi, w.dirEntries, w.walkFn) + +} + +// if the filesystem supports it, use Lstat, else use fs.Stat +func lstatIfPossible(fs afero.Fs, path string) (os.FileInfo, error) { + if lfs, ok := fs.(afero.Lstater); ok { + fi, _, err := lfs.LstatIfPossible(path) + return fi, err + } + return fs.Stat(path) +} + +// walk recursively descends path, calling walkFn. +// It follow symlinks if supported by the filesystem, but only the same path once. +func (w *Walkway) walk(path string, info FileMetaInfo, dirEntries []FileMetaInfo, walkFn WalkFunc) error { + err := walkFn(path, info, nil) + if err != nil { + if info.IsDir() && err == filepath.SkipDir { + return nil + } + return err + } + if !info.IsDir() { + return nil + } + + // TODO(bep) mod make sure that slice_fs etc. do the stat even for symlinks + + meta := info.Meta() + filename := meta.Filename() + + // Prevent infinite recursion. + // TODO(bep) mod with the mount support this may be a bit crude. Check + // if we can somehow check if we're going up. + w.isSeen(filename) + + if dirEntries == nil { + f, err := w.fs.Open(path) + + if err != nil { + return walkFn(path, info, errors.Wrapf(err, "walk: open %q (%q)", path, w.root)) + } + + fis, err := f.Readdir(-1) + f.Close() + if err != nil { + if err == ErrPermissionSymlink { + w.logger.WARN.Printf("Unsupported symlink found in %q, skipping.", filename) + return nil + } + return walkFn(path, info, errors.Wrap(err, "walk: Readdir")) + } + + dirEntries = fileInfosToFileMetaInfos(fis) + + if !meta.IsOrdered() { + sort.Slice(dirEntries, func(i, j int) bool { + fii := dirEntries[i] + fij := dirEntries[j] + + fim, fjm := fii.Meta(), fij.Meta() + + // Pull bundle headers to the top. + ficlass, fjclass := fim.Classifier(), fjm.Classifier() + if ficlass != fjclass { + return ficlass < fjclass + } + + // With multiple content dirs with different languages, + // there can be duplicate files, and a weight will be added + // to the closest one. + fiw, fjw := fim.Weight(), fjm.Weight() + if fiw != fjw { + return fiw > fjw + } + + // Explicit order set. + fio, fjo := fim.Ordinal(), fjm.Ordinal() + if fio != fjo { + return fio < fjo + } + + // When we walk into a symlink, we keep the reference to + // the original name. + fin, fjn := fim.Name(), fjm.Name() + if fin != "" && fjn != "" { + return fin < fjn + } + + return fii.Name() < fij.Name() + }) + } + } + + // First add some metadata to the dir entries + for _, fi := range dirEntries { + fim := fi.(FileMetaInfo) + + meta := fim.Meta() + + // Note that we use the original Name even if it's a symlink. + name := meta.Name() + if name == "" { + name = fim.Name() + } + + if name == "" { + panic(fmt.Sprintf("[%s] no name set in %v", path, meta)) + } + pathn := filepath.Join(path, name) + + pathMeta := pathn + if w.basePath != "" { + pathMeta = strings.TrimPrefix(pathn, w.basePath) + } + + meta[metaKeyPath] = pathMeta + meta[metaKeyPathWalk] = pathn + + if fim.IsDir() && w.isSeen(meta.Filename()) { + + // Prevent infinite recursion + // Possible cyclic reference + // TODO(bep) mod check if we log some warning about this in the + // existing content walker. + meta[metaKeySkipDir] = true + } + } + + if w.hookPre != nil { + dirEntries, err = w.hookPre(info, path, dirEntries) + if err != nil { + if err == filepath.SkipDir { + return nil + } + return err + } + } + + for _, fi := range dirEntries { + fim := fi.(FileMetaInfo) + meta := fim.Meta() + + if meta.SkipDir() { + continue + } + + err := w.walk(meta.GetString(metaKeyPathWalk), fim, nil, walkFn) + if err != nil { + if !fi.IsDir() || err != filepath.SkipDir { + return err + } + } + } + + if w.hookPost != nil { + dirEntries, err = w.hookPost(info, path, dirEntries) + if err != nil { + if err == filepath.SkipDir { + return nil + } + return err + } + } + return nil +} + +func (w *Walkway) isSeen(filename string) bool { + if filename == "" { + return false + } + if w.seen[filename] { + return true + } + + w.seen[filename] = true + return false +} diff --git a/hugofs/walk_test.go b/hugofs/walk_test.go new file mode 100644 index 00000000000..d492041ae04 --- /dev/null +++ b/hugofs/walk_test.go @@ -0,0 +1,225 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugofs + +import ( + "fmt" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/gohugoio/hugo/common/hugo" + + "github.com/pkg/errors" + + "github.com/gohugoio/hugo/htesting" + + "github.com/spf13/afero" + + "github.com/stretchr/testify/require" +) + +func TestWalk(t *testing.T) { + assert := require.New(t) + + fs := NewBaseFileDecorator(afero.NewMemMapFs()) + + afero.WriteFile(fs, "b.txt", []byte("content"), 0777) + afero.WriteFile(fs, "c.txt", []byte("content"), 0777) + afero.WriteFile(fs, "a.txt", []byte("content"), 0777) + + names, err := collectFilenames(fs, "", "") + + assert.NoError(err) + assert.Equal([]string{"a.txt", "b.txt", "c.txt"}, names) +} + +func TestWalkRootMappingFs(t *testing.T) { + assert := require.New(t) + fs := NewBaseFileDecorator(afero.NewMemMapFs()) + + testfile := "test.txt" + + assert.NoError(afero.WriteFile(fs, filepath.Join("a/b", testfile), []byte("some content"), 0755)) + assert.NoError(afero.WriteFile(fs, filepath.Join("c/d", testfile), []byte("some content"), 0755)) + assert.NoError(afero.WriteFile(fs, filepath.Join("e/f", testfile), []byte("some content"), 0755)) + + rm := []RootMapping{ + RootMapping{ + From: "static/b", + To: "e/f", + }, + RootMapping{ + From: "static/a", + To: "c/d", + }, + + RootMapping{ + From: "static/c", + To: "a/b", + }, + } + + rfs, err := NewRootMappingFs(fs, rm...) + assert.NoError(err) + bfs := afero.NewBasePathFs(rfs, "static") + + names, err := collectFilenames(bfs, "", "") + + assert.NoError(err) + assert.Equal([]string{"a/test.txt", "b/test.txt", "c/test.txt"}, names) + +} + +func skipSymlink() bool { + return runtime.GOOS == "windows" && os.Getenv("CI") == "" +} + +func TestWalkSymbolicLink(t *testing.T) { + if skipSymlink() { + t.Skip("Skip; os.Symlink needs administrator rights on Windows") + } + assert := require.New(t) + workDir, clean, err := htesting.CreateTempDir(Os, "hugo-walk-sym") + assert.NoError(err) + defer clean() + wd, _ := os.Getwd() + defer func() { + os.Chdir(wd) + }() + + fs := NewBaseFileDecorator(Os) + + blogDir := filepath.Join(workDir, "blog") + docsDir := filepath.Join(workDir, "docs") + blogReal := filepath.Join(blogDir, "real") + blogRealSub := filepath.Join(blogReal, "sub") + assert.NoError(os.MkdirAll(blogRealSub, 0777)) + assert.NoError(os.MkdirAll(docsDir, 0777)) + afero.WriteFile(fs, filepath.Join(blogRealSub, "a.txt"), []byte("content"), 0777) + afero.WriteFile(fs, filepath.Join(docsDir, "b.txt"), []byte("content"), 0777) + + os.Chdir(blogDir) + assert.NoError(os.Symlink("real", "symlinked")) + os.Chdir(blogReal) + assert.NoError(os.Symlink("../real", "cyclic")) + os.Chdir(docsDir) + assert.NoError(os.Symlink("../blog/real/cyclic", "docsreal")) + + t.Run("OS Fs", func(t *testing.T) { + assert := require.New(t) + + names, err := collectFilenames(fs, workDir, workDir) + assert.NoError(err) + + assert.Equal([]string{"blog/real/sub/a.txt", "docs/b.txt"}, names) + }) + + t.Run("BasePath Fs", func(t *testing.T) { + if hugo.GoMinorVersion() < 12 { + // https://github.com/golang/go/issues/30520 + // This is fixed in Go 1.13 and in the latest Go 1.12 + t.Skip("skip this for Go <= 1.11 due to a bug in Go's stdlib") + + } + assert := require.New(t) + + docsFs := afero.NewBasePathFs(fs, docsDir) + + names, err := collectFilenames(docsFs, "", "") + assert.NoError(err) + + // Note: the docsreal folder is considered cyclic when walking from the root, but this works. + assert.Equal([]string{"b.txt", "docsreal/sub/a.txt"}, names) + }) + +} + +func collectFilenames(fs afero.Fs, base, root string) ([]string, error) { + var names []string + + walkFn := func(path string, info FileMetaInfo, err error) error { + if err != nil { + return err + } + + if info.IsDir() { + return nil + } + + filename := info.Meta().Path() + filename = filepath.ToSlash(filename) + + names = append(names, filename) + + return nil + } + + w := NewWalkway(WalkwayConfig{Fs: fs, BasePath: base, Root: root, WalkFn: walkFn}) + + err := w.Walk() + + return names, err + +} + +func BenchmarkWalk(b *testing.B) { + assert := require.New(b) + fs := NewBaseFileDecorator(afero.NewMemMapFs()) + + writeFiles := func(dir string, numfiles int) { + for i := 0; i < numfiles; i++ { + filename := filepath.Join(dir, fmt.Sprintf("file%d.txt", i)) + assert.NoError(afero.WriteFile(fs, filename, []byte("content"), 0777)) + } + } + + const numFilesPerDir = 20 + + writeFiles("root", numFilesPerDir) + writeFiles("root/l1_1", numFilesPerDir) + writeFiles("root/l1_1/l2_1", numFilesPerDir) + writeFiles("root/l1_1/l2_2", numFilesPerDir) + writeFiles("root/l1_2", numFilesPerDir) + writeFiles("root/l1_2/l2_1", numFilesPerDir) + writeFiles("root/l1_3", numFilesPerDir) + + walkFn := func(path string, info FileMetaInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + + filename := info.Meta().Filename() + if !strings.HasPrefix(filename, "root") { + return errors.New(filename) + } + + return nil + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + w := NewWalkway(WalkwayConfig{Fs: fs, Root: "root", WalkFn: walkFn}) + + if err := w.Walk(); err != nil { + b.Fatal(err) + } + } + +} diff --git a/hugolib/config.go b/hugolib/config.go index 50e4ca6ec29..036d080e328 100644 --- a/hugolib/config.go +++ b/hugolib/config.go @@ -14,21 +14,20 @@ package hugolib import ( - "fmt" - "os" "path/filepath" "strings" + "github.com/gohugoio/hugo/common/maps" + "github.com/gohugoio/hugo/parser/metadecoders" "github.com/gohugoio/hugo/common/herrors" "github.com/gohugoio/hugo/common/hugo" "github.com/gohugoio/hugo/hugolib/paths" - "github.com/pkg/errors" - _errors "github.com/pkg/errors" - "github.com/gohugoio/hugo/langs" + "github.com/gohugoio/hugo/modules" + "github.com/pkg/errors" "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/config/privacy" @@ -84,6 +83,9 @@ type ConfigSourceDescriptor struct { // production, development Environment string + + // Defaults to os.Environ if not set. + Environ []string } func (d ConfigSourceDescriptor) configFilenames() []string { @@ -110,52 +112,46 @@ var ErrNoConfigFile = errors.New("Unable to locate config file or config directo // LoadConfig loads Hugo configuration into a new Viper and then adds // a set of defaults. +var counter int + func LoadConfig(d ConfigSourceDescriptor, doWithConfig ...func(cfg config.Provider) error) (*viper.Viper, []string, error) { + if d.Environment == "" { d.Environment = hugo.EnvironmentProduction } + if len(d.Environ) == 0 { + d.Environ = os.Environ() + } + var configFiles []string v := viper.New() l := configLoader{ConfigSourceDescriptor: d} - v.AutomaticEnv() - v.SetEnvPrefix("hugo") - - var cerr error - for _, name := range d.configFilenames() { var filename string - if filename, cerr = l.loadConfig(name, v); cerr != nil && cerr != ErrNoConfigFile { - return nil, nil, cerr + filename, err := l.loadConfig(name, v) + if err == nil { + configFiles = append(configFiles, filename) + } else if err != ErrNoConfigFile { + return nil, nil, err } - configFiles = append(configFiles, filename) } if d.AbsConfigDir != "" { dirnames, err := l.loadConfigFromConfigDir(v) if err == nil { configFiles = append(configFiles, dirnames...) + } else if err != ErrNoConfigFile { + return nil, nil, err } - cerr = err } if err := loadDefaultSettingsFor(v); err != nil { return v, configFiles, err } - if cerr == nil { - themeConfigFiles, err := l.loadThemeConfig(v) - if err != nil { - return v, configFiles, err - } - - if len(themeConfigFiles) > 0 { - configFiles = append(configFiles, themeConfigFiles...) - } - } - // We create languages based on the settings, so we need to make sure that // all configuration is loaded/set before doing that. for _, d := range doWithConfig { @@ -164,12 +160,74 @@ func LoadConfig(d ConfigSourceDescriptor, doWithConfig ...func(cfg config.Provid } } + // Apply environment overrides + if len(d.Environ) > 0 { + // Extract all that start with the HUGO_ prefix + const hugoEnvPrefix = "HUGO_" + var hugoEnv []string + for _, v := range d.Environ { + key, val := config.SplitEnvVar(v) + if strings.HasPrefix(key, hugoEnvPrefix) { + hugoEnv = append(hugoEnv, strings.ToLower(strings.TrimPrefix(key, hugoEnvPrefix)), val) + } + } + + if len(hugoEnv) > 0 { + for i := 0; i < len(hugoEnv); i += 2 { + key, valStr := strings.ToLower(hugoEnv[i]), hugoEnv[i+1] + + existing, nestedKey, owner, err := maps.GetNestedParamFn(key, "_", v.Get) + if err != nil { + return v, configFiles, err + } + + if existing != nil { + val, err := metadecoders.Default.UnmarshalStringTo(valStr, existing) + if err != nil { + continue + } + + if owner != nil { + owner[nestedKey] = val + } else { + v.Set(key, val) + } + + } + } + } + } + + modulesConfig, err := l.loadModulesConfig(v) + if err != nil { + return v, configFiles, err + } + + mods, modulesConfigFiles, err := l.collectModules(modulesConfig, v) + if err != nil { + return v, configFiles, err + } + if err := loadLanguageSettings(v, nil); err != nil { return v, configFiles, err } - return v, configFiles, cerr + // Apply default project mounts. + if err := modules.ApplyProjectConfigDefaults(v, mods[len(mods)-1]); err != nil { + return v, configFiles, err + } + if len(modulesConfigFiles) > 0 { + configFiles = append(configFiles, modulesConfigFiles...) + } + + return v, configFiles, nil + +} + +func loadLanguageSettings(cfg config.Provider, oldLangs langs.Languages) error { + _, err := langs.LoadLanguageSettings(cfg, oldLangs) + return err } type configLoader struct { @@ -334,145 +392,70 @@ func (l configLoader) loadConfigFromConfigDir(v *viper.Viper) ([]string, error) return dirnames, nil } -func loadLanguageSettings(cfg config.Provider, oldLangs langs.Languages) error { - - defaultLang := cfg.GetString("defaultContentLanguage") - - var languages map[string]interface{} - - languagesFromConfig := cfg.GetStringMap("languages") - disableLanguages := cfg.GetStringSlice("disableLanguages") - - if len(disableLanguages) == 0 { - languages = languagesFromConfig - } else { - languages = make(map[string]interface{}) - for k, v := range languagesFromConfig { - for _, disabled := range disableLanguages { - if disabled == defaultLang { - return fmt.Errorf("cannot disable default language %q", defaultLang) - } - - if strings.EqualFold(k, disabled) { - v.(map[string]interface{})["disabled"] = true - break - } - } - languages[k] = v - } - } - - var ( - languages2 langs.Languages - err error - ) +func (l configLoader) loadModulesConfig(v1 *viper.Viper) (modules.Config, error) { - if len(languages) == 0 { - languages2 = append(languages2, langs.NewDefaultLanguage(cfg)) - } else { - languages2, err = toSortedLanguages(cfg, languages) - if err != nil { - return _errors.Wrap(err, "Failed to parse multilingual config") - } - } - - if oldLangs != nil { - // When in multihost mode, the languages are mapped to a server, so - // some structural language changes will need a restart of the dev server. - // The validation below isn't complete, but should cover the most - // important cases. - var invalid bool - if languages2.IsMultihost() != oldLangs.IsMultihost() { - invalid = true - } else { - if languages2.IsMultihost() && len(languages2) != len(oldLangs) { - invalid = true - } - } - - if invalid { - return errors.New("language change needing a server restart detected") - } - - if languages2.IsMultihost() { - // We need to transfer any server baseURL to the new language - for i, ol := range oldLangs { - nl := languages2[i] - nl.Set("baseURL", ol.GetString("baseURL")) - } - } - } - - // The defaultContentLanguage is something the user has to decide, but it needs - // to match a language in the language definition list. - langExists := false - for _, lang := range languages2 { - if lang.Lang == defaultLang { - langExists = true - break - } - } - - if !langExists { - return fmt.Errorf("site config value %q for defaultContentLanguage does not match any language definition", defaultLang) + modConfig, err := modules.DecodeConfig(v1) + if err != nil { + return modules.Config{}, err } - cfg.Set("languagesSorted", languages2) - cfg.Set("multilingual", len(languages2) > 1) - - multihost := languages2.IsMultihost() + return modConfig, nil +} - if multihost { - cfg.Set("defaultContentLanguageInSubdir", true) - cfg.Set("multihost", true) +func (l configLoader) collectModules(modConfig modules.Config, v1 *viper.Viper) (modules.Modules, []string, error) { + workingDir := l.WorkingDir + if workingDir == "" { + workingDir = v1.GetString("workingDir") } - if multihost { - // The baseURL may be provided at the language level. If that is true, - // then every language must have a baseURL. In this case we always render - // to a language sub folder, which is then stripped from all the Permalink URLs etc. - for _, l := range languages2 { - burl := l.GetLocal("baseURL") - if burl == nil { - return errors.New("baseURL must be set on all or none of the languages") - } - } - - } + themesDir := paths.AbsPathify(l.WorkingDir, v1.GetString("themesDir")) - return nil -} + ignoreVendor := v1.GetBool("ignoreVendor") + modProxy := v1.GetString("modProxy") -func (l configLoader) loadThemeConfig(v1 *viper.Viper) ([]string, error) { - themesDir := paths.AbsPathify(l.WorkingDir, v1.GetString("themesDir")) - themes := config.GetStringSlicePreserveString(v1, "theme") + modulesClient := modules.NewClient(modules.ClientConfig{ + Fs: l.Fs, + WorkingDir: workingDir, + ThemesDir: themesDir, + ModuleConfig: modConfig, + IgnoreVendor: ignoreVendor, + ModProxy: modProxy, + }) - themeConfigs, err := paths.CollectThemes(l.Fs, themesDir, themes) + moduleConfig, err := modulesClient.Collect() if err != nil { - return nil, err + return nil, nil, err } - if len(themeConfigs) == 0 { - return nil, nil - } + // Avoid recreating these later. + v1.Set("allModules", moduleConfig.Modules) + v1.Set("modulesClient", modulesClient) - v1.Set("allThemes", themeConfigs) + if len(moduleConfig.Modules) == 0 { + return nil, nil, nil + } var configFilenames []string - for _, tc := range themeConfigs { - if tc.ConfigFilename != "" { - configFilenames = append(configFilenames, tc.ConfigFilename) + for _, tc := range moduleConfig.Modules { + if tc.ConfigFilename() != "" { + configFilenames = append(configFilenames, tc.ConfigFilename()) if err := l.applyThemeConfig(v1, tc); err != nil { - return nil, err + return nil, nil, err } } } - return configFilenames, nil + if moduleConfig.GoModulesFilename != "" { + // We want to watch this for changes and trigger rebuild on version + // changes etc. + configFilenames = append(configFilenames, moduleConfig.GoModulesFilename) + } + + return moduleConfig.Modules, configFilenames, nil } -func (l configLoader) applyThemeConfig(v1 *viper.Viper, theme paths.ThemeConfig) error { +func (l configLoader) applyThemeConfig(v1 *viper.Viper, theme modules.Module) error { const ( paramsKey = "params" @@ -480,22 +463,12 @@ func (l configLoader) applyThemeConfig(v1 *viper.Viper, theme paths.ThemeConfig) menuKey = "menus" ) - v2 := theme.Cfg + v2 := theme.Cfg() for _, key := range []string{paramsKey, "outputformats", "mediatypes"} { l.mergeStringMapKeepLeft("", key, v1, v2) } - themeLower := strings.ToLower(theme.Name) - themeParamsNamespace := paramsKey + "." + themeLower - - // Set namespaced params - if v2.IsSet(paramsKey) && !v1.IsSet(themeParamsNamespace) { - // Set it in the default store to make sure it gets in the same or - // behind the others. - v1.SetDefault(themeParamsNamespace, v2.Get(paramsKey)) - } - // Only add params and new menu entries, we do not add language definitions. if v1.IsSet(languagesKey) && v2.IsSet(languagesKey) { v1Langs := v1.GetStringMap(languagesKey) @@ -508,12 +481,6 @@ func (l configLoader) applyThemeConfig(v1 *viper.Viper, theme paths.ThemeConfig) if k == "" { continue } - langParamsKey := languagesKey + "." + k + "." + paramsKey - langParamsThemeNamespace := langParamsKey + "." + themeLower - // Set namespaced params - if v2.IsSet(langParamsKey) && !v1.IsSet(langParamsThemeNamespace) { - v1.SetDefault(langParamsThemeNamespace, v2.Get(langParamsKey)) - } langMenuKey := languagesKey + "." + k + "." + menuKey if v2.IsSet(langMenuKey) { @@ -635,5 +602,8 @@ func loadDefaultSettingsFor(v *viper.Viper) error { v.SetDefault("disableFastRender", false) v.SetDefault("timeout", 10000) // 10 seconds v.SetDefault("enableInlineShortcodes", false) + + // Translates to GOPROXY when doing "go get" etc. + v.SetDefault("modProxy", "direct") return nil } diff --git a/hugolib/config_test.go b/hugolib/config_test.go index 885a07ee951..87a77209cf3 100644 --- a/hugolib/config_test.go +++ b/hugolib/config_test.go @@ -14,6 +14,9 @@ package hugolib import ( + "bytes" + "fmt" + "path/filepath" "testing" "github.com/spf13/afero" @@ -42,8 +45,7 @@ func TestLoadConfig(t *testing.T) { assert.Equal("side", cfg.GetString("paginatePath")) // default assert.Equal("layouts", cfg.GetString("layoutDir")) - // no themes - assert.False(cfg.IsSet("allThemes")) + } func TestLoadMultiConfig(t *testing.T) { @@ -188,11 +190,6 @@ map[string]interface {}{ "p1": "p1 main", "p2": "p2 main", "p3": "p3 theme", - "test-theme": map[string]interface {}{ - "p1": "p1 theme", - "p2": "p2 theme", - "p3": "p3 theme", - }, "top": "top", }`, got["params"]) @@ -257,10 +254,6 @@ map[string]interface {}{ "params": map[string]interface {}{ "pl1": "p1-en-main", "pl2": "p2-en-theme", - "test-theme": map[string]interface {}{ - "pl1": "p1-en-theme", - "pl2": "p2-en-theme", - }, }, }, "nb": map[string]interface {}{ @@ -275,11 +268,6 @@ map[string]interface {}{ "params": map[string]interface {}{ "pl1": "p1-nb-main", "pl2": "p2-nb-theme", - "test-theme": map[string]interface {}{ - "pl1": "p1-nb-theme", - "pl2": "p2-nb-theme", - "top": "top-nb-theme", - }, }, }, } @@ -397,3 +385,137 @@ privacyEnhanced = true assert.True(b.H.Sites[0].Info.Config().Privacy.YouTube.PrivacyEnhanced) } + +func TestLoadConfigModules(t *testing.T) { + t.Parallel() + + assert := require.New(t) + + // https://github.com/gohugoio/hugoThemes#themetoml + + const ( + // Before Hugo 0.56 each theme/component could have its own theme.toml + // with some settings, mostly used on the Hugo themes site. + // To preserve combability we read these files into the new "modules" + // section in config.toml. + o1t = ` +name = "Component o1" +license = "MIT" +min_version = 0.38 +` + // This is the component's config.toml, using the old theme syntax. + o1c = ` +theme = ["n2"] +` + + n1 = ` +title = "Component n1" + +[module] +description = "Component n1 description" +[module.hugoVersion] +min = "0.40.0" +max = "0.50.0" +extended = true +[[module.imports]] +path="o1" +[[module.imports]] +path="n3" + + +` + + n2 = ` +title = "Component n2" +` + + n3 = ` +title = "Component n3" +` + + n4 = ` +title = "Component n4" +` + ) + + b := newTestSitesBuilder(t) + + writeThemeFiles := func(name, configTOML, themeTOML string) { + b.WithSourceFile(filepath.Join("themes", name, "data", "module.toml"), fmt.Sprintf("name=%q", name)) + if configTOML != "" { + b.WithSourceFile(filepath.Join("themes", name, "config.toml"), configTOML) + } + if themeTOML != "" { + b.WithSourceFile(filepath.Join("themes", name, "theme.toml"), themeTOML) + } + } + + writeThemeFiles("n1", n1, "") + writeThemeFiles("n2", n2, "") + writeThemeFiles("n3", n3, "") + writeThemeFiles("n4", n4, "") + writeThemeFiles("o1", o1c, o1t) + + b.WithConfigFile("toml", ` +[module] +[[module.imports]] +path="n1" +[[module.imports]] +path="n4" + +`) + + b.Build(BuildCfg{}) + + modulesClient := b.H.Paths.ModulesClient + var graphb bytes.Buffer + modulesClient.Graph(&graphb) + + assert.Equal(`project n1 +n1 o1 +o1 n2 +n1 n3 +project n4 +`, graphb.String()) + +} + +func TestLoadConfigWithOsEnvOverrides(t *testing.T) { + + assert := require.New(t) + + baseConfig := ` + +environment = "production" +intSlice = [5,7,9] +floatSlice = [3.14, 5.19] +stringSlice = ["a", "b"] + +[imaging] +anchor = "smart" +quality = 75 +resamplefilter = "CatmullRom" +` + + b := newTestSitesBuilder(t).WithConfigFile("toml", baseConfig) + + b.WithEnviron( + "HUGO_ENVIRONMENT", "test", + "HUGO_IMAGING_ANCHOR", "top", + "HUGO_STRINGSLICE", `["c", "d"]`, + "HUGO_INTSLICE", `[5, 8, 9]`, + "HUGO_FLOATSLICE", `[5.32]`, + ) + + b.Build(BuildCfg{}) + + cfg := b.H.Cfg + + assert.Equal("test", cfg.Get("environment")) + assert.Equal("top", cfg.Get("imaging.anchor")) + assert.Equal(int64(75), cfg.Get("imaging.quality")) + assert.Equal([]interface{}{"c", "d"}, cfg.Get("stringSlice")) + assert.Equal([]interface{}{5.32}, cfg.Get("floatSlice")) + assert.Equal([]interface{}{5, 8, 9}, cfg.Get("intSlice")) + +} diff --git a/hugolib/data/hugo.toml b/hugolib/data/hugo.toml new file mode 100755 index 00000000000..eb1dbc42cb1 --- /dev/null +++ b/hugolib/data/hugo.toml @@ -0,0 +1 @@ +slogan = "Hugo Rocks!" \ No newline at end of file diff --git a/hugolib/disableKinds_test.go b/hugolib/disableKinds_test.go index f5c093646c2..c191dfef1ef 100644 --- a/hugolib/disableKinds_test.go +++ b/hugolib/disableKinds_test.go @@ -20,11 +20,7 @@ import ( "github.com/gohugoio/hugo/resources/page" - "github.com/gohugoio/hugo/deps" - "github.com/spf13/afero" - "github.com/gohugoio/hugo/helpers" - "github.com/gohugoio/hugo/hugofs" "github.com/stretchr/testify/require" ) @@ -80,8 +76,6 @@ categories: # Doc ` - mf := afero.NewMemMapFs() - disabledStr := "[]" if len(disabled) > 0 { @@ -90,47 +84,41 @@ categories: } siteConfig := fmt.Sprintf(siteConfigTemplate, disabledStr) - writeToFs(t, mf, "config.toml", siteConfig) - - cfg, err := LoadConfigDefault(mf) - require.NoError(t, err) - fs := hugofs.NewFrom(mf, cfg) - th := testHelper{cfg, fs, t} + b := newTestSitesBuilder(t).WithConfigFile("toml", siteConfig) - writeSource(t, fs, "layouts/index.html", "Home|{{ .Title }}|{{ .Content }}") - writeSource(t, fs, "layouts/_default/single.html", "Single|{{ .Title }}|{{ .Content }}") - writeSource(t, fs, "layouts/_default/list.html", "List|{{ .Title }}|{{ .Content }}") - writeSource(t, fs, "layouts/_default/terms.html", "Terms List|{{ .Title }}|{{ .Content }}") - writeSource(t, fs, "layouts/404.html", "Page Not Found") + b.WithTemplates( + "index.html", "Home|{{ .Title }}|{{ .Content }}", + "_default/single.html", "Single|{{ .Title }}|{{ .Content }}", + "_default/list.html", "List|{{ .Title }}|{{ .Content }}", + "_default/terms.html", "Terms List|{{ .Title }}|{{ .Content }}", + "layouts/404.html", "Page Not Found", + ) - writeSource(t, fs, "content/sect/p1.md", fmt.Sprintf(pageTemplate, "P1", "- tag1")) + b.WithContent( + "sect/p1.md", fmt.Sprintf(pageTemplate, "P1", "- tag1"), + "categories/_index.md", newTestPage("Category Terms", "2017-01-01", 10), + "tags/tag1/_index.md", newTestPage("Tag1 List", "2017-01-01", 10), + ) - writeNewContentFile(t, fs.Source, "Category Terms", "2017-01-01", "content/categories/_index.md", 10) - writeNewContentFile(t, fs.Source, "Tag1 List", "2017-01-01", "content/tags/tag1/_index.md", 10) + b.Build(BuildCfg{}) + h := b.H - h, err := NewHugoSites(deps.DepsCfg{Fs: fs, Cfg: cfg}) - - require.NoError(t, err) require.Len(t, h.Sites, 1) - err = h.Build(BuildCfg{}) - - require.NoError(t, err) - - assertDisabledKinds(th, h.Sites[0], disabled...) + assertDisabledKinds(b, h.Sites[0], disabled...) } -func assertDisabledKinds(th testHelper, s *Site, disabled ...string) { - assertDisabledKind(th, +func assertDisabledKinds(b *sitesBuilder, s *Site, disabled ...string) { + assertDisabledKind(b, func(isDisabled bool) bool { if isDisabled { return len(s.RegularPages()) == 0 } return len(s.RegularPages()) > 0 }, disabled, page.KindPage, "public/sect/p1/index.html", "Single|P1") - assertDisabledKind(th, + assertDisabledKind(b, func(isDisabled bool) bool { p := s.getPage(page.KindHome) if isDisabled { @@ -138,7 +126,7 @@ func assertDisabledKinds(th testHelper, s *Site, disabled ...string) { } return p != nil }, disabled, page.KindHome, "public/index.html", "Home") - assertDisabledKind(th, + assertDisabledKind(b, func(isDisabled bool) bool { p := s.getPage(page.KindSection, "sect") if isDisabled { @@ -146,7 +134,7 @@ func assertDisabledKinds(th testHelper, s *Site, disabled ...string) { } return p != nil }, disabled, page.KindSection, "public/sect/index.html", "Sects") - assertDisabledKind(th, + assertDisabledKind(b, func(isDisabled bool) bool { p := s.getPage(page.KindTaxonomy, "tags", "tag1") @@ -156,7 +144,7 @@ func assertDisabledKinds(th testHelper, s *Site, disabled ...string) { return p != nil }, disabled, page.KindTaxonomy, "public/tags/tag1/index.html", "Tag1") - assertDisabledKind(th, + assertDisabledKind(b, func(isDisabled bool) bool { p := s.getPage(page.KindTaxonomyTerm, "tags") if isDisabled { @@ -165,7 +153,7 @@ func assertDisabledKinds(th testHelper, s *Site, disabled ...string) { return p != nil }, disabled, page.KindTaxonomyTerm, "public/tags/index.html", "Tags") - assertDisabledKind(th, + assertDisabledKind(b, func(isDisabled bool) bool { p := s.getPage(page.KindTaxonomyTerm, "categories") @@ -175,7 +163,7 @@ func assertDisabledKinds(th testHelper, s *Site, disabled ...string) { return p != nil }, disabled, page.KindTaxonomyTerm, "public/categories/index.html", "Category Terms") - assertDisabledKind(th, + assertDisabledKind(b, func(isDisabled bool) bool { p := s.getPage(page.KindTaxonomy, "categories", "hugo") if isDisabled { @@ -185,15 +173,15 @@ func assertDisabledKinds(th testHelper, s *Site, disabled ...string) { }, disabled, page.KindTaxonomy, "public/categories/hugo/index.html", "Hugo") // The below have no page in any collection. - assertDisabledKind(th, func(isDisabled bool) bool { return true }, disabled, kindRSS, "public/index.xml", "") - assertDisabledKind(th, func(isDisabled bool) bool { return true }, disabled, kindSitemap, "public/sitemap.xml", "sitemap") - assertDisabledKind(th, func(isDisabled bool) bool { return true }, disabled, kindRobotsTXT, "public/robots.txt", "User-agent") - assertDisabledKind(th, func(isDisabled bool) bool { return true }, disabled, kind404, "public/404.html", "Page Not Found") + assertDisabledKind(b, func(isDisabled bool) bool { return true }, disabled, kindRSS, "public/index.xml", "") + assertDisabledKind(b, func(isDisabled bool) bool { return true }, disabled, kindSitemap, "public/sitemap.xml", "sitemap") + assertDisabledKind(b, func(isDisabled bool) bool { return true }, disabled, kindRobotsTXT, "public/robots.txt", "User-agent") + assertDisabledKind(b, func(isDisabled bool) bool { return true }, disabled, kind404, "public/404.html", "Page Not Found") } -func assertDisabledKind(th testHelper, kindAssert func(bool) bool, disabled []string, kind, path, matcher string) { +func assertDisabledKind(b *sitesBuilder, kindAssert func(bool) bool, disabled []string, kind, path, matcher string) { isDisabled := stringSliceContains(kind, disabled...) - require.True(th.T, kindAssert(isDisabled), fmt.Sprintf("%s: %t", kind, isDisabled)) + require.True(b.T, kindAssert(isDisabled), fmt.Sprintf("%s: %t", kind, isDisabled)) if kind == kindRSS && !isDisabled { // If the home page is also disabled, there is not RSS to look for. @@ -204,20 +192,11 @@ func assertDisabledKind(th testHelper, kindAssert func(bool) bool, disabled []st if isDisabled { // Path should not exist - fileExists, err := helpers.Exists(path, th.Fs.Destination) - require.False(th.T, fileExists) - require.NoError(th.T, err) + fileExists, err := helpers.Exists(path, b.Fs.Destination) + require.False(b.T, fileExists) + require.NoError(b.T, err) } else { - th.assertFileContent(path, matcher) - } -} - -func stringSliceContains(k string, values ...string) bool { - for _, v := range values { - if k == v { - return true - } + b.AssertFileContent(path, matcher) } - return false } diff --git a/hugolib/fileInfo.go b/hugolib/fileInfo.go index ea3b15ef328..a418c16414f 100644 --- a/hugolib/fileInfo.go +++ b/hugolib/fileInfo.go @@ -16,82 +16,67 @@ package hugolib import ( "strings" - "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/hugofs/files" + + "github.com/pkg/errors" + + "github.com/gohugoio/hugo/hugofs" + + "github.com/spf13/afero" + "github.com/gohugoio/hugo/source" ) // fileInfo implements the File and ReadableFile interface. var ( - _ source.File = (*fileInfo)(nil) - _ source.ReadableFile = (*fileInfo)(nil) - _ pathLangFile = (*fileInfo)(nil) + _ source.File = (*fileInfo)(nil) ) -// A partial interface to prevent ambigous compiler error. -type basePather interface { - Filename() string - RealName() string - BaseDir() string -} - type fileInfo struct { - bundleTp bundleDirType - - source.ReadableFile - basePather + source.File overriddenLang string +} + +func (fi *fileInfo) Open() (afero.File, error) { + f, err := fi.FileInfo().Meta().Open() + if err != nil { + err = errors.Wrap(err, "fileInfo") + } - // Set if the content language for this file is disabled. - disabled bool + return f, err } func (fi *fileInfo) Lang() string { if fi.overriddenLang != "" { return fi.overriddenLang } - return fi.ReadableFile.Lang() -} - -func (fi *fileInfo) Filename() string { - if fi == nil || fi.basePather == nil { - return "" - } - return fi.basePather.Filename() + return fi.File.Lang() } func (fi *fileInfo) String() string { - if fi == nil || fi.ReadableFile == nil { + if fi == nil || fi.File == nil { return "" } return fi.Path() } -func (fi *fileInfo) isOwner() bool { - return fi.bundleTp > bundleNot -} - -func IsContentFile(filename string) bool { - return contentFileExtensionsSet[strings.TrimPrefix(helpers.Ext(filename), ".")] -} +// TODO(bep) rename +func newFileInfo2(sp *source.SourceSpec, fi hugofs.FileMetaInfo) (*fileInfo, error) { -func (fi *fileInfo) isContentFile() bool { - return contentFileExtensionsSet[fi.Ext()] -} - -func newFileInfo(sp *source.SourceSpec, baseDir, filename string, fi pathLangFileFi, tp bundleDirType) *fileInfo { + baseFi, err := sp.NewFileInfo(fi) + if err != nil { + return nil, err + } - baseFi := sp.NewFileInfo(baseDir, filename, tp == bundleLeaf, fi) f := &fileInfo{ - bundleTp: tp, - ReadableFile: baseFi, - basePather: fi, + // TODO(bep) mod bundleTp: tp, + File: baseFi, } - lang := f.Lang() - f.disabled = lang != "" && sp.DisabledLanguages[lang] + // TODO(bep) mod remove disabled f.disabled = lang != "" && sp.DisabledLanguages[lang] - return f + return f, nil } @@ -108,7 +93,7 @@ const ( // Returns the given file's name's bundle type and whether it is a content // file or not. func classifyBundledFile(name string) (bundleDirType, bool) { - if !IsContentFile(name) { + if !files.IsContentFile(name) { return bundleNot, false } if strings.HasPrefix(name, "_index.") { diff --git a/hugolib/fileInfo_test.go b/hugolib/fileInfo_test.go index 10f5f051774..c51ff5033d7 100644 --- a/hugolib/fileInfo_test.go +++ b/hugolib/fileInfo_test.go @@ -22,6 +22,7 @@ import ( func TestFileInfo(t *testing.T) { t.Run("String", func(t *testing.T) { + t.Parallel() assert := require.New(t) fi := &fileInfo{} _, err := cast.ToStringE(fi) diff --git a/hugolib/filesystems/basefs.go b/hugolib/filesystems/basefs.go index d88141efd7d..152d200b63a 100644 --- a/hugolib/filesystems/basefs.go +++ b/hugolib/filesystems/basefs.go @@ -16,27 +16,27 @@ package filesystems import ( - "errors" + "io" "os" + "path" "path/filepath" "strings" + "sync" - "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/hugofs/files" + + "github.com/pkg/errors" + + "github.com/gohugoio/hugo/modules" "github.com/gohugoio/hugo/hugofs" "fmt" "github.com/gohugoio/hugo/hugolib/paths" - "github.com/gohugoio/hugo/langs" "github.com/spf13/afero" ) -// When we create a virtual filesystem with data and i18n bundles for the project and the themes, -// this is the name of the project's virtual root. It got it's funky name to make sure -// (or very unlikely) that it collides with a theme name. -const projectVirtualFolder = "__h__project" - var filePathSeparator = string(filepath.Separator) // BaseFs contains the core base filesystems used by Hugo. The name "base" is used @@ -51,16 +51,43 @@ type BaseFs struct { // This usually maps to /my-project/public. PublishFs afero.Fs - themeFs afero.Fs + theBigFs *filesystemsCollector +} + +func (fs *BaseFs) WatchDirs() []hugofs.FileMetaInfo { + var dirs []hugofs.FileMetaInfo + for _, dir := range fs.AllDirs() { + if dir.Meta().Watch() { + dirs = append(dirs, dir) + } + } + + return dirs +} + +func (fs *BaseFs) AllDirs() []hugofs.FileMetaInfo { + var dirs []hugofs.FileMetaInfo + for _, dirSet := range [][]hugofs.FileMetaInfo{ + fs.Archetypes.Dirs, + fs.I18n.Dirs, + fs.Data.Dirs, + fs.Content.Dirs, + fs.Assets.Dirs, + fs.Layouts.Dirs, + //fs.Resources.Dirs, + fs.StaticDirs, + } { + dirs = append(dirs, dirSet...) + } - // TODO(bep) improve the "theme interaction" - AbsThemeDirs []string + return dirs } // RelContentDir tries to create a path relative to the content root from // the given filename. The return value is the path and language code. func (b *BaseFs) RelContentDir(filename string) string { - for _, dirname := range b.SourceFilesystems.Content.Dirnames { + for _, dir := range b.SourceFilesystems.Content.Dirs { + dirname := dir.Meta().Filename() if strings.HasPrefix(filename, dirname) { rel := strings.TrimPrefix(filename, dirname) return strings.TrimPrefix(rel, filePathSeparator) @@ -82,14 +109,20 @@ type SourceFilesystems struct { Assets *SourceFilesystem Resources *SourceFilesystem - // This is a unified read-only view of the project's and themes' workdir. - Work *SourceFilesystem + // Writable filesystem to the project's resources directory. + ResourcesCache afero.Fs + + // The project folder. + Work afero.Fs // When in multihost we have one static filesystem per language. The sync // static files is currently done outside of the Hugo build (where there is // a concept of a site per language). // When in non-multihost mode there will be one entry in this map with a blank key. Static map[string]*SourceFilesystem + + // All the /static dirs (including themes/modules). + StaticDirs []hugofs.FileMetaInfo } // A SourceFilesystem holds the filesystem for a given source type in Hugo (data, @@ -99,12 +132,9 @@ type SourceFilesystem struct { // This is a virtual composite filesystem. It expects path relative to a context. Fs afero.Fs - // This is the base source filesystem. In real Hugo, this will be the OS filesystem. - // Use this if you need to resolve items in Dirnames below. - SourceFs afero.Fs - - // Dirnames is absolute filenames to the directories in this filesystem. - Dirnames []string + // This filesystem as separate root directories, starting from project and down + // to the themes/modules. + Dirs []hugofs.FileMetaInfo // When syncing a source folder to the target (e.g. /public), this may // be set to publish into a subfolder. This is used for static syncing @@ -207,7 +237,8 @@ func (s SourceFilesystems) MakeStaticPathRelative(filename string) string { // MakePathRelative creates a relative path from the given filename. // It will return an empty string if the filename is not a member of this filesystem. func (d *SourceFilesystem) MakePathRelative(filename string) string { - for _, currentPath := range d.Dirnames { + for _, dir := range d.Dirs { + currentPath := dir.(hugofs.FileMetaInfo).Meta().Filename() if strings.HasPrefix(filename, currentPath) { return strings.TrimPrefix(filename, currentPath) } @@ -220,8 +251,8 @@ func (d *SourceFilesystem) RealFilename(rel string) string { if err != nil { return rel } - if realfi, ok := fi.(hugofs.RealFilenameInfo); ok { - return realfi.RealFilename() + if realfi, ok := fi.(hugofs.FileMetaInfo); ok { + return realfi.Meta().Filename() } return rel @@ -229,8 +260,8 @@ func (d *SourceFilesystem) RealFilename(rel string) string { // Contains returns whether the given filename is a member of the current filesystem. func (d *SourceFilesystem) Contains(filename string) bool { - for _, dir := range d.Dirnames { - if strings.HasPrefix(filename, dir) { + for _, dir := range d.Dirs { + if strings.HasPrefix(filename, dir.Meta().Filename()) { return true } } @@ -241,9 +272,12 @@ func (d *SourceFilesystem) Contains(filename string) bool { // path. func (d *SourceFilesystem) RealDirs(from string) []string { var dirnames []string - for _, dir := range d.Dirnames { - dirname := filepath.Join(dir, from) - if _, err := d.SourceFs.Stat(dirname); err == nil { + for _, dir := range d.Dirs { + meta := dir.Meta() + dirname := filepath.Join(meta.Filename(), from) + _, err := meta.Fs().Stat(from) + + if err == nil { dirnames = append(dirnames, dirname) } } @@ -254,40 +288,18 @@ func (d *SourceFilesystem) RealDirs(from string) []string { // the same across sites/languages. func WithBaseFs(b *BaseFs) func(*BaseFs) error { return func(bb *BaseFs) error { - bb.themeFs = b.themeFs - bb.AbsThemeDirs = b.AbsThemeDirs + bb.theBigFs = b.theBigFs + bb.SourceFilesystems = b.SourceFilesystems return nil } } -func newRealBase(base afero.Fs) afero.Fs { - return hugofs.NewBasePathRealFilenameFs(base.(*afero.BasePathFs)) - -} - // NewBase builds the filesystems used by Hugo given the paths and options provided.NewBase func NewBase(p *paths.Paths, options ...func(*BaseFs) error) (*BaseFs, error) { fs := p.Fs publishFs := afero.NewBasePathFs(fs.Destination, p.AbsPublishDir) - contentFs, absContentDirs, err := createContentFs(fs.Source, p.WorkingDir, p.DefaultContentLanguage, p.Languages) - if err != nil { - return nil, err - } - - // Make sure we don't have any overlapping content dirs. That will never work. - for i, d1 := range absContentDirs { - for j, d2 := range absContentDirs { - if i == j { - continue - } - if strings.HasPrefix(d1, d2) || strings.HasPrefix(d2, d1) { - return nil, fmt.Errorf("found overlapping content dirs (%q and %q)", d1, d2) - } - } - } - b := &BaseFs{ PublishFs: publishFs, } @@ -298,463 +310,385 @@ func NewBase(p *paths.Paths, options ...func(*BaseFs) error) (*BaseFs, error) { } } + if b.theBigFs != nil && b.SourceFilesystems != nil { + return b, nil + } + builder := newSourceFilesystemsBuilder(p, b) sourceFilesystems, err := builder.Build() if err != nil { - return nil, err - } - - sourceFilesystems.Content = &SourceFilesystem{ - SourceFs: fs.Source, - Fs: contentFs, - Dirnames: absContentDirs, + return nil, errors.Wrap(err, "build filesystems") } b.SourceFilesystems = sourceFilesystems - b.themeFs = builder.themeFs - b.AbsThemeDirs = builder.absThemeDirs + b.theBigFs = builder.theBigFs return b, nil } type sourceFilesystemsBuilder struct { - p *paths.Paths - result *SourceFilesystems - themeFs afero.Fs - hasTheme bool - absThemeDirs []string + p *paths.Paths + sourceFs afero.Fs + result *SourceFilesystems + theBigFs *filesystemsCollector } func newSourceFilesystemsBuilder(p *paths.Paths, b *BaseFs) *sourceFilesystemsBuilder { - return &sourceFilesystemsBuilder{p: p, themeFs: b.themeFs, absThemeDirs: b.AbsThemeDirs, result: &SourceFilesystems{}} + sourceFs := hugofs.NewBaseFileDecorator(p.Fs.Source) + return &sourceFilesystemsBuilder{p: p, sourceFs: sourceFs, theBigFs: b.theBigFs, result: &SourceFilesystems{}} } +func (b *sourceFilesystemsBuilder) newSourceFilesystem(fs afero.Fs, dirs []hugofs.FileMetaInfo) *SourceFilesystem { + return &SourceFilesystem{ + Fs: fs, + Dirs: dirs, + } +} func (b *sourceFilesystemsBuilder) Build() (*SourceFilesystems, error) { - if b.themeFs == nil && b.p.ThemeSet() { - themeFs, absThemeDirs, err := createThemesOverlayFs(b.p) + + if b.theBigFs == nil { + + theBigFs, err := b.createMainOverlayFs(b.p) if err != nil { - return nil, err - } - if themeFs == nil { - panic("createThemesFs returned nil") + return nil, errors.Wrap(err, "create main fs") } - b.themeFs = themeFs - b.absThemeDirs = absThemeDirs + b.theBigFs = theBigFs } - b.hasTheme = len(b.absThemeDirs) > 0 + createView := func(componentID string) *SourceFilesystem { + if b.theBigFs == nil || b.theBigFs.overlayMounts == nil { + return b.newSourceFilesystem(hugofs.NoOpFs, nil) + } - sfs, err := b.createRootMappingFs("dataDir", "data") - if err != nil { - return nil, err - } - b.result.Data = sfs + dirs := b.theBigFs.overlayDirs[componentID] - sfs, err = b.createRootMappingFs("i18nDir", "i18n") - if err != nil { - return nil, err - } - b.result.I18n = sfs + return b.newSourceFilesystem(afero.NewBasePathFs(b.theBigFs.overlayMounts, componentID), dirs) - sfs, err = b.createFs(false, true, "layoutDir", "layouts") - if err != nil { - return nil, err } - b.result.Layouts = sfs - sfs, err = b.createFs(false, true, "archetypeDir", "archetypes") - if err != nil { - return nil, err - } - b.result.Archetypes = sfs + b.theBigFs.finalizeDirs() - sfs, err = b.createFs(false, true, "assetDir", "assets") - if err != nil { - return nil, err - } - b.result.Assets = sfs + b.result.Archetypes = createView(files.ComponentFolderArchetypes) + b.result.Layouts = createView(files.ComponentFolderLayouts) + b.result.Assets = createView(files.ComponentFolderAssets) + b.result.Resources = createView(files.ComponentFolderResources) + b.result.ResourcesCache = afero.NewBasePathFs(b.sourceFs, b.p.AbsResourcesDir) - sfs, err = b.createFs(true, false, "resourceDir", "resources") + // Data, i18n and content cannot use the overlay fs + dataDirs := b.theBigFs.overlayDirs[files.ComponentFolderData] + dataFs, err := hugofs.NewSliceFs(dataDirs...) if err != nil { return nil, err } - b.result.Resources = sfs + b.result.Data = b.newSourceFilesystem(dataFs, dataDirs) - sfs, err = b.createFs(false, true, "", "") + i18nDirs := b.theBigFs.overlayDirs[files.ComponentFolderI18n] + i18nFs, err := hugofs.NewSliceFs(i18nDirs...) if err != nil { return nil, err } - b.result.Work = sfs + b.result.I18n = b.newSourceFilesystem(i18nFs, i18nDirs) + + contentDirs := b.theBigFs.overlayDirs[files.ComponentFolderContent] + contentBfs := afero.NewBasePathFs(b.theBigFs.overlayMountsContent, files.ComponentFolderContent) - err = b.createStaticFs() + contentFs, err := hugofs.NewLanguageFs(b.p.LanguagesDefaultFirst.AsOrdinalSet(), contentBfs) if err != nil { - return nil, err + return nil, errors.Wrap(err, "create content filesystem") } - return b.result, nil -} - -func (b *sourceFilesystemsBuilder) createFs( - mkdir bool, - readOnly bool, - dirKey, themeFolder string) (*SourceFilesystem, error) { - s := &SourceFilesystem{ - SourceFs: b.p.Fs.Source, - } + b.result.Content = b.newSourceFilesystem(contentFs, contentDirs) - if themeFolder == "" { - themeFolder = filePathSeparator - } + b.result.Work = afero.NewReadOnlyFs(b.theBigFs.overlayFull) - var dir string - if dirKey != "" { - dir = b.p.Cfg.GetString(dirKey) - if dir == "" { - return s, fmt.Errorf("config %q not set", dirKey) + // Create static filesystem(s) + ms := make(map[string]*SourceFilesystem) + b.result.Static = ms + b.result.StaticDirs = b.theBigFs.overlayDirs[files.ComponentFolderStatic] + + if b.theBigFs.staticPerLanguage != nil { + // Multihost mode + for k, v := range b.theBigFs.staticPerLanguage { + sfs := b.newSourceFilesystem(v, b.result.StaticDirs) + sfs.PublishFolder = k + ms[k] = sfs } + } else { + bfs := afero.NewBasePathFs(b.theBigFs.overlayMounts, files.ComponentFolderStatic) + ms[""] = b.newSourceFilesystem(bfs, b.result.StaticDirs) } - var fs afero.Fs + return b.result, nil - absDir := b.p.AbsPathify(dir) - existsInSource := b.existsInSource(absDir) - if !existsInSource && mkdir { - // We really need this directory. Make it. - if err := b.p.Fs.Source.MkdirAll(absDir, 0777); err == nil { - existsInSource = true - } - } - if existsInSource { - fs = newRealBase(afero.NewBasePathFs(b.p.Fs.Source, absDir)) - s.Dirnames = []string{absDir} - } +} - if b.hasTheme { - if !strings.HasPrefix(themeFolder, filePathSeparator) { - themeFolder = filePathSeparator + themeFolder - } - themeFolderFs := newRealBase(afero.NewBasePathFs(b.themeFs, themeFolder)) - if fs == nil { - fs = themeFolderFs - } else { - fs = afero.NewCopyOnWriteFs(themeFolderFs, fs) - } +func (b *sourceFilesystemsBuilder) createMainOverlayFs(p *paths.Paths) (*filesystemsCollector, error) { - for _, absThemeDir := range b.absThemeDirs { - absThemeFolderDir := filepath.Join(absThemeDir, themeFolder) - if b.existsInSource(absThemeFolderDir) { - s.Dirnames = append(s.Dirnames, absThemeFolderDir) - } - } + var staticFsMap map[string]afero.Fs + if b.p.Cfg.GetBool("multihost") { + staticFsMap = make(map[string]afero.Fs) } - if fs == nil { - s.Fs = hugofs.NoOpFs - } else if readOnly { - s.Fs = afero.NewReadOnlyFs(fs) - } else { - s.Fs = fs + collector := &filesystemsCollector{ + sourceProject: b.sourceFs, + sourceModules: hugofs.NewNoSymlinkFs(b.sourceFs), + overlayDirs: make(map[string][]hugofs.FileMetaInfo), + staticPerLanguage: staticFsMap, } - return s, nil -} + mods := p.AllModules -// Used for data, i18n -- we cannot use overlay filsesystems for those, but we need -// to keep a strict order. -func (b *sourceFilesystemsBuilder) createRootMappingFs(dirKey, themeFolder string) (*SourceFilesystem, error) { - s := &SourceFilesystem{ - SourceFs: b.p.Fs.Source, + if len(mods) == 0 { + return collector, nil } - projectDir := b.p.Cfg.GetString(dirKey) - if projectDir == "" { - return nil, fmt.Errorf("config %q not set", dirKey) - } + modsReversed := make([]mountsDescriptor, len(mods)) - var fromTo []string - to := b.p.AbsPathify(projectDir) + // The theme components are ordered from left to right. + // We need to revert it to get the + // overlay logic below working as expected, with the project on top (last). - if b.existsInSource(to) { - s.Dirnames = []string{to} - fromTo = []string{projectVirtualFolder, to} - } + for i, mod := range mods { + dir := mod.Dir() - for _, theme := range b.p.AllThemes { - to := b.p.AbsPathify(filepath.Join(b.p.ThemesDir, theme.Name, themeFolder)) - if b.existsInSource(to) { - s.Dirnames = append(s.Dirnames, to) - from := theme - fromTo = append(fromTo, from.Name, to) + if i < len(mods)-1 { + i = len(mods) - 2 - i } - } - if len(fromTo) == 0 { - s.Fs = hugofs.NoOpFs - return s, nil + isMainProject := mod.Owner() == nil + modsReversed[i] = mountsDescriptor{ + mounts: mod.Mounts(), + dir: dir, + watch: isMainProject || !mod.IsGoMod(), // TODO(bep) mod consider Replace + isMainProject: isMainProject, + } } - fs, err := hugofs.NewRootMappingFs(b.p.Fs.Source, fromTo...) - if err != nil { - return nil, err - } + err := b.createOverlayFs(collector, modsReversed) - s.Fs = afero.NewReadOnlyFs(fs) + return collector, err - return s, nil } -func (b *sourceFilesystemsBuilder) existsInSource(abspath string) bool { - exists, _ := afero.Exists(b.p.Fs.Source, abspath) - return exists +func (b *sourceFilesystemsBuilder) isContentMount(mnt modules.Mount) bool { + return strings.HasPrefix(mnt.Target, files.ComponentFolderContent) } -func (b *sourceFilesystemsBuilder) createStaticFs() error { - isMultihost := b.p.Cfg.GetBool("multihost") - ms := make(map[string]*SourceFilesystem) - b.result.Static = ms - - if isMultihost { - for _, l := range b.p.Languages { - s := &SourceFilesystem{ - SourceFs: b.p.Fs.Source, - PublishFolder: l.Lang} - staticDirs := removeDuplicatesKeepRight(getStaticDirs(l)) - if len(staticDirs) == 0 { - continue - } +func (b *sourceFilesystemsBuilder) createModFs( + collector *filesystemsCollector, + md mountsDescriptor) error { - for _, dir := range staticDirs { - absDir := b.p.AbsPathify(dir) - if !b.existsInSource(absDir) { - continue - } + var ( + fromTo []hugofs.RootMapping + fromToContent []hugofs.RootMapping + ) - s.Dirnames = append(s.Dirnames, absDir) - } + absPathify := func(path string) string { + return paths.AbsPathify(md.dir, path) + } - fs, err := createOverlayFs(b.p.Fs.Source, s.Dirnames) - if err != nil { - return err - } + seen := make(map[string]bool) - if b.hasTheme { - themeFolder := "static" - fs = afero.NewCopyOnWriteFs(newRealBase(afero.NewBasePathFs(b.themeFs, themeFolder)), fs) - for _, absThemeDir := range b.absThemeDirs { - s.Dirnames = append(s.Dirnames, filepath.Join(absThemeDir, themeFolder)) - } - } + var mounts []modules.Mount - s.Fs = fs - ms[l.Lang] = s +OUTER: + for i, mount := range md.mounts { + key := path.Join(mount.Lang, mount.Source, mount.Target) + if seen[key] { + continue + } + seen[key] = true + // Prevent overlapping mounts + for j, mount2 := range md.mounts { + if j == i || mount2.Target != mount.Target { + continue + } + source := mount.Source + if !strings.HasSuffix(source, filePathSeparator) { + source += filePathSeparator + } + if strings.HasPrefix(mount2.Source, source) { + continue OUTER + } } - return nil + mounts = append(mounts, mount) } - s := &SourceFilesystem{ - SourceFs: b.p.Fs.Source, - } + for _, mount := range mounts { - var staticDirs []string + mountWeight := 1 + if md.isMainProject { + mountWeight++ + } - for _, l := range b.p.Languages { - staticDirs = append(staticDirs, getStaticDirs(l)...) - } + rm := hugofs.RootMapping{ + From: mount.Target, + To: absPathify(mount.Source), + Meta: hugofs.FileMeta{ + "watch": md.watch, + "mountWeight": mountWeight, + }, + } - staticDirs = removeDuplicatesKeepRight(staticDirs) - if len(staticDirs) == 0 { - return nil - } + isContentMount := b.isContentMount(mount) - for _, dir := range staticDirs { - absDir := b.p.AbsPathify(dir) - if !b.existsInSource(absDir) { - continue + lang := mount.Lang + if lang == "" && isContentMount { + lang = b.p.DefaultContentLanguage } - s.Dirnames = append(s.Dirnames, absDir) - } - fs, err := createOverlayFs(b.p.Fs.Source, s.Dirnames) - if err != nil { - return err - } + rm.Meta["lang"] = lang - if b.hasTheme { - themeFolder := "static" - fs = afero.NewCopyOnWriteFs(newRealBase(afero.NewBasePathFs(b.themeFs, themeFolder)), fs) - for _, absThemeDir := range b.absThemeDirs { - s.Dirnames = append(s.Dirnames, filepath.Join(absThemeDir, themeFolder)) + if isContentMount { + fromToContent = append(fromToContent, rm) + } else { + fromTo = append(fromTo, rm) } } - s.Fs = fs - ms[""] = s - - return nil -} - -func getStaticDirs(cfg config.Provider) []string { - var staticDirs []string - for i := -1; i <= 10; i++ { - staticDirs = append(staticDirs, getStringOrStringSlice(cfg, "staticDir", i)...) + modBase := collector.sourceProject + if !md.isMainProject { + modBase = collector.sourceModules } - return staticDirs -} -func getStringOrStringSlice(cfg config.Provider, key string, id int) []string { - - if id >= 0 { - key = fmt.Sprintf("%s%d", key, id) + rmfs, err := hugofs.NewRootMappingFs(modBase, fromTo...) + if err != nil { + return err } - - return config.GetStringSlicePreserveString(cfg, key) - -} - -func createContentFs(fs afero.Fs, - workingDir, - defaultContentLanguage string, - languages langs.Languages) (afero.Fs, []string, error) { - - var contentLanguages langs.Languages - var contentDirSeen = make(map[string]bool) - languageSet := make(map[string]bool) - - // The default content language needs to be first. - for _, language := range languages { - if language.Lang == defaultContentLanguage { - contentLanguages = append(contentLanguages, language) - contentDirSeen[language.ContentDir] = true - } - languageSet[language.Lang] = true + rmfsContent, err := hugofs.NewRootMappingFs(modBase, fromToContent...) + if err != nil { + return err } - for _, language := range languages { - if contentDirSeen[language.ContentDir] { - continue - } - if language.ContentDir == "" { - language.ContentDir = defaultContentLanguage - } - contentDirSeen[language.ContentDir] = true - contentLanguages = append(contentLanguages, language) + // We need to keep the ordered list of directories for watching and + // some special merge operations (data, i18n). + collector.addDirs(rmfs) + collector.addDirs(rmfsContent) - } + if collector.staticPerLanguage != nil { + for _, l := range b.p.Languages { + lang := l.Lang - var absContentDirs []string + lfs := rmfs.Filter(func(rm hugofs.RootMapping) bool { + rlang := rm.Meta.Lang() + return rlang == "" || rlang == lang + }) - fs, err := createContentOverlayFs(fs, workingDir, contentLanguages, languageSet, &absContentDirs) - return fs, absContentDirs, err + bfs := afero.NewBasePathFs(lfs, files.ComponentFolderStatic) -} + sfs, found := collector.staticPerLanguage[lang] + if found { + collector.staticPerLanguage[lang] = afero.NewCopyOnWriteFs(sfs, bfs) -func createContentOverlayFs(source afero.Fs, - workingDir string, - languages langs.Languages, - languageSet map[string]bool, - absContentDirs *[]string) (afero.Fs, error) { - if len(languages) == 0 { - return source, nil + } else { + collector.staticPerLanguage[lang] = bfs + } + } } - language := languages[0] - - contentDir := language.ContentDir - if contentDir == "" { - panic("missing contentDir") + if collector.overlayMounts == nil { + collector.overlayMounts = rmfs + collector.overlayMountsContent = rmfsContent + collector.overlayFull = afero.NewBasePathFs(modBase, md.dir) + } else { + collector.overlayMounts = afero.NewCopyOnWriteFs(collector.overlayMounts, rmfs) + collector.overlayMountsContent = hugofs.NewLanguageCompositeFs(collector.overlayMountsContent, rmfsContent) + collector.overlayFull = afero.NewCopyOnWriteFs(collector.overlayFull, afero.NewBasePathFs(modBase, md.dir)) } - absContentDir := paths.AbsPathify(workingDir, language.ContentDir) - if !strings.HasSuffix(absContentDir, paths.FilePathSeparator) { - absContentDir += paths.FilePathSeparator - } + return nil - // If root, remove the second '/' - if absContentDir == "//" { - absContentDir = paths.FilePathSeparator - } +} - if len(absContentDir) < 6 { - return nil, fmt.Errorf("invalid content dir %q: Path is too short", absContentDir) +func printFs(fs afero.Fs, path string, w io.Writer) { + if fs == nil { + return } + afero.Walk(fs, path, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + var filename string + if fim, ok := info.(hugofs.FileMetaInfo); ok { + filename = fim.Meta().Filename() + } + fmt.Fprintf(w, " %q %q\n", path, filename) + return nil + }) +} - *absContentDirs = append(*absContentDirs, absContentDir) +type filesystemsCollector struct { + sourceProject afero.Fs // Source for project folders + sourceModules afero.Fs // Source for modules/themes - overlay := hugofs.NewLanguageFs(language.Lang, languageSet, afero.NewBasePathFs(source, absContentDir)) - if len(languages) == 1 { - return overlay, nil - } + overlayMounts afero.Fs + overlayMountsContent afero.Fs + overlayFull afero.Fs - base, err := createContentOverlayFs(source, workingDir, languages[1:], languageSet, absContentDirs) - if err != nil { - return nil, err - } + // Maps component type (layouts, static, content etc.) an ordered list of + // directories representing the overlay filesystems above. + overlayDirs map[string][]hugofs.FileMetaInfo - return hugofs.NewLanguageCompositeFs(base, overlay), nil + // Set if in multihost mode + staticPerLanguage map[string]afero.Fs + finalizerInit sync.Once } -func createThemesOverlayFs(p *paths.Paths) (afero.Fs, []string, error) { - - themes := p.AllThemes +func (c *filesystemsCollector) addDirs(rfs *hugofs.RootMappingFs) { + for _, componentFolder := range files.ComponentFolders { + dirs, err := rfs.Dirs(componentFolder) - if len(themes) == 0 { - panic("AllThemes not set") + if err == nil { + c.overlayDirs[componentFolder] = append(c.overlayDirs[componentFolder], dirs...) + } } +} - themesDir := p.AbsPathify(p.ThemesDir) - if themesDir == "" { - return nil, nil, errors.New("no themes dir set") - } +func (c *filesystemsCollector) finalizeDirs() { + c.finalizerInit.Do(func() { + // Order the directories from top to bottom (project, theme a, theme ...). + for _, dirs := range c.overlayDirs { + c.reverseFis(dirs) + } + }) - absPaths := make([]string, len(themes)) +} - // The themes are ordered from left to right. We need to revert it to get the - // overlay logic below working as expected. - for i := 0; i < len(themes); i++ { - absPaths[i] = filepath.Join(themesDir, themes[len(themes)-1-i].Name) +func (c *filesystemsCollector) reverseFis(fis []hugofs.FileMetaInfo) { + for i := len(fis)/2 - 1; i >= 0; i-- { + opp := len(fis) - 1 - i + fis[i], fis[opp] = fis[opp], fis[i] } - - fs, err := createOverlayFs(p.Fs.Source, absPaths) - fs = hugofs.NewNoLstatFs(fs) - - return fs, absPaths, err - } -func createOverlayFs(source afero.Fs, absPaths []string) (afero.Fs, error) { - if len(absPaths) == 0 { - return hugofs.NoOpFs, nil - } +type mountsDescriptor struct { + mounts []modules.Mount + dir string + watch bool // whether this is a candidate for watching in server mode. + isMainProject bool +} - if len(absPaths) == 1 { - return afero.NewReadOnlyFs(newRealBase(afero.NewBasePathFs(source, absPaths[0]))), nil +func (b *sourceFilesystemsBuilder) createOverlayFs(collector *filesystemsCollector, mounts []mountsDescriptor) error { + if len(mounts) == 0 { + return nil } - base := afero.NewReadOnlyFs(newRealBase(afero.NewBasePathFs(source, absPaths[0]))) - overlay, err := createOverlayFs(source, absPaths[1:]) + err := b.createModFs(collector, mounts[0]) if err != nil { - return nil, err + return err } - return afero.NewCopyOnWriteFs(base, overlay), nil -} - -func removeDuplicatesKeepRight(in []string) []string { - seen := make(map[string]bool) - var out []string - for i := len(in) - 1; i >= 0; i-- { - v := in[i] - if seen[v] { - continue - } - out = append([]string{v}, out...) - seen[v] = true + if len(mounts) == 1 { + return nil } - return out + return b.createOverlayFs(collector, mounts[1:]) } diff --git a/hugolib/filesystems/basefs_test.go b/hugolib/filesystems/basefs_test.go index ec6ccb30c41..c6298858864 100644 --- a/hugolib/filesystems/basefs_test.go +++ b/hugolib/filesystems/basefs_test.go @@ -1,4 +1,4 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. +// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,18 +18,59 @@ import ( "fmt" "os" "path/filepath" + "strings" "testing" + "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/langs" "github.com/spf13/afero" "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/hugolib/paths" + "github.com/gohugoio/hugo/modules" "github.com/spf13/viper" "github.com/stretchr/testify/require" ) +func initConfig(fs afero.Fs, cfg config.Provider) error { + if _, err := langs.LoadLanguageSettings(cfg, nil); err != nil { + return err + } + + modConfig, err := modules.DecodeConfig(cfg) + if err != nil { + return err + } + + workingDir := cfg.GetString("workingDir") + themesDir := cfg.GetString("themesDir") + if !filepath.IsAbs(themesDir) { + themesDir = filepath.Join(workingDir, themesDir) + } + modulesClient := modules.NewClient(modules.ClientConfig{ + Fs: fs, + WorkingDir: workingDir, + ThemesDir: themesDir, + ModuleConfig: modConfig, + IgnoreVendor: true, + }) + + moduleConfig, err := modulesClient.Collect() + if err != nil { + return err + } + + if err := modules.ApplyProjectConfigDefaults(cfg, moduleConfig.Modules[len(moduleConfig.Modules)-1]); err != nil { + return err + } + + cfg.Set("allModules", moduleConfig.Modules) + + return nil +} + func TestNewBaseFs(t *testing.T) { assert := require.New(t) v := viper.New() @@ -40,16 +81,21 @@ func TestNewBaseFs(t *testing.T) { workingDir := filepath.FromSlash("/my/work") v.Set("workingDir", workingDir) + v.Set("contentDir", "content") v.Set("themesDir", "themes") + v.Set("defaultContentLanguage", "en") v.Set("theme", themes[:1]) // Write some data to the themes for _, theme := range themes { for _, dir := range []string{"i18n", "data", "archetypes", "layouts"} { base := filepath.Join(workingDir, "themes", theme, dir) - filename := filepath.Join(base, fmt.Sprintf("theme-file-%s.txt", theme)) + filenameTheme := filepath.Join(base, fmt.Sprintf("theme-file-%s.txt", theme)) + filenameOverlap := filepath.Join(base, "f3.txt") fs.Source.Mkdir(base, 0755) - afero.WriteFile(fs.Source, filename, []byte(fmt.Sprintf("content:%s:%s", theme, dir)), 0755) + content := []byte(fmt.Sprintf("content:%s:%s", theme, dir)) + afero.WriteFile(fs.Source, filenameTheme, content, 0755) + afero.WriteFile(fs.Source, filenameOverlap, content, 0755) } // Write some files to the root of the theme base := filepath.Join(workingDir, "themes", theme) @@ -73,6 +119,7 @@ theme = ["atheme"] setConfigAndWriteSomeFilesTo(fs.Source, v, "resourceDir", "myrsesource", 10) v.Set("publishDir", "public") + assert.NoError(initConfig(fs.Source, v)) p, err := paths.New(fs, v) assert.NoError(err) @@ -85,33 +132,27 @@ theme = ["atheme"] assert.NoError(err) dirnames, err := root.Readdirnames(-1) assert.NoError(err) - assert.Equal([]string{projectVirtualFolder, "btheme", "atheme"}, dirnames) - ff, err := bfs.I18n.Fs.Open("myi18n") - assert.NoError(err) - _, err = ff.Readdirnames(-1) - assert.NoError(err) + assert.Equal([]string{"f1.txt", "f2.txt", "f3.txt", "f4.txt", "f3.txt", "theme-file-btheme.txt", "f3.txt", "theme-file-atheme.txt"}, dirnames) root, err = bfs.Data.Fs.Open("") assert.NoError(err) dirnames, err = root.Readdirnames(-1) assert.NoError(err) - assert.Equal([]string{projectVirtualFolder, "btheme", "atheme"}, dirnames) - ff, err = bfs.I18n.Fs.Open("mydata") - assert.NoError(err) - _, err = ff.Readdirnames(-1) - assert.NoError(err) + assert.Equal([]string{"f1.txt", "f2.txt", "f3.txt", "f4.txt", "f5.txt", "f6.txt", "f7.txt", "f3.txt", "theme-file-btheme.txt", "f3.txt", "theme-file-atheme.txt"}, dirnames) + + //printFs(bfs.Work, "", os.Stdout) - checkFileCount(bfs.Content.Fs, "", assert, 3) - checkFileCount(bfs.I18n.Fs, "", assert, 6) // 4 + 2 themes checkFileCount(bfs.Layouts.Fs, "", assert, 7) + + checkFileCount(bfs.Content.Fs, "", assert, 3) + checkFileCount(bfs.I18n.Fs, "", assert, 8) // 4 + 4 themes + checkFileCount(bfs.Static[""].Fs, "", assert, 6) - checkFileCount(bfs.Data.Fs, "", assert, 9) // 7 + 2 themes + checkFileCount(bfs.Data.Fs, "", assert, 11) // 7 + 4 themes checkFileCount(bfs.Archetypes.Fs, "", assert, 10) // 8 + 2 themes checkFileCount(bfs.Assets.Fs, "", assert, 9) checkFileCount(bfs.Resources.Fs, "", assert, 10) - checkFileCount(bfs.Work.Fs, "", assert, 78) - - assert.Equal([]string{filepath.FromSlash("/my/work/mydata"), filepath.FromSlash("/my/work/themes/btheme/data"), filepath.FromSlash("/my/work/themes/atheme/data")}, bfs.Data.Dirnames) + checkFileCount(bfs.Work, "", assert, 82) assert.True(bfs.IsData(filepath.Join(workingDir, "mydata", "file1.txt"))) assert.True(bfs.IsI18n(filepath.Join(workingDir, "myi18n", "file1.txt"))) @@ -125,13 +166,13 @@ theme = ["atheme"] assert.Equal("file1.txt", rel) // Check Work fs vs theme - checkFileContent(bfs.Work.Fs, "file-root.txt", assert, "content-project") - checkFileContent(bfs.Work.Fs, "theme-root-atheme.txt", assert, "content:atheme") + checkFileContent(bfs.Work, "file-root.txt", assert, "content-project") + checkFileContent(bfs.Work, "theme-root-atheme.txt", assert, "content:atheme") // https://github.com/gohugoio/hugo/issues/5318 // Check both project and theme. for _, fs := range []afero.Fs{bfs.Archetypes.Fs, bfs.Layouts.Fs} { - for _, filename := range []string{"/file1.txt", "/theme-file-atheme.txt"} { + for _, filename := range []string{"/f1.txt", "/theme-file-atheme.txt"} { filename = filepath.FromSlash(filename) f, err := fs.Open(filename) assert.NoError(err) @@ -153,6 +194,7 @@ func createConfig() *viper.Viper { v.Set("assetDir", "myassets") v.Set("resourceDir", "resources") v.Set("publishDir", "public") + v.Set("defaultContentLanguage", "en") return v } @@ -161,17 +203,18 @@ func TestNewBaseFsEmpty(t *testing.T) { assert := require.New(t) v := createConfig() fs := hugofs.NewMem(v) + assert.NoError(initConfig(fs.Source, v)) + p, err := paths.New(fs, v) assert.NoError(err) bfs, err := NewBase(p) assert.NoError(err) assert.NotNil(bfs) - assert.Equal(hugofs.NoOpFs, bfs.Archetypes.Fs) - assert.Equal(hugofs.NoOpFs, bfs.Layouts.Fs) - assert.Equal(hugofs.NoOpFs, bfs.Data.Fs) - assert.Equal(hugofs.NoOpFs, bfs.Assets.Fs) - assert.Equal(hugofs.NoOpFs, bfs.I18n.Fs) - assert.NotNil(bfs.Work.Fs) + assert.NotNil(bfs.Archetypes.Fs) + assert.NotNil(bfs.Layouts.Fs) + assert.NotNil(bfs.Data.Fs) + assert.NotNil(bfs.I18n.Fs) + assert.NotNil(bfs.Work) assert.NotNil(bfs.Content.Fs) assert.NotNil(bfs.Static) } @@ -217,11 +260,14 @@ func TestRealDirs(t *testing.T) { afero.WriteFile(sfs, filepath.Join(filepath.Join(root, "myassets", "js", "f2", "a1.js")), []byte("content"), 0755) afero.WriteFile(sfs, filepath.Join(filepath.Join(root, "myassets", "js", "a2.js")), []byte("content"), 0755) + assert.NoError(initConfig(fs.Source, v)) + p, err := paths.New(fs, v) assert.NoError(err) bfs, err := NewBase(p) assert.NoError(err) assert.NotNil(bfs) + checkFileCount(bfs.Assets.Fs, "", assert, 6) realDirs := bfs.Assets.RealDirs("scss") @@ -231,10 +277,9 @@ func TestRealDirs(t *testing.T) { checkFileCount(bfs.Resources.Fs, "", assert, 3) - assert.NotNil(bfs.themeFs) - fi, b, err := bfs.themeFs.(afero.Lstater).LstatIfPossible(filepath.Join("resources", "t1.txt")) + assert.NotNil(bfs.theBigFs) + fi, _, err := bfs.theBigFs.overlayMounts.(afero.Lstater).LstatIfPossible(filepath.Join("resources", "t1.txt")) assert.NoError(err) - assert.False(b) assert.Equal("t1.txt", fi.Name()) } @@ -245,20 +290,25 @@ func TestStaticFs(t *testing.T) { workDir := "mywork" v.Set("workingDir", workDir) v.Set("themesDir", "themes") - v.Set("theme", "t1") + v.Set("theme", []string{"t1", "t2"}) fs := hugofs.NewMem(v) themeStaticDir := filepath.Join(workDir, "themes", "t1", "static") + themeStaticDir2 := filepath.Join(workDir, "themes", "t2", "static") afero.WriteFile(fs.Source, filepath.Join(workDir, "mystatic", "f1.txt"), []byte("Hugo Rocks!"), 0755) afero.WriteFile(fs.Source, filepath.Join(themeStaticDir, "f1.txt"), []byte("Hugo Themes Rocks!"), 0755) afero.WriteFile(fs.Source, filepath.Join(themeStaticDir, "f2.txt"), []byte("Hugo Themes Still Rocks!"), 0755) + afero.WriteFile(fs.Source, filepath.Join(themeStaticDir2, "f2.txt"), []byte("Hugo Themes Rocks in t2!"), 0755) + + assert.NoError(initConfig(fs.Source, v)) p, err := paths.New(fs, v) assert.NoError(err) bfs, err := NewBase(p) assert.NoError(err) + sfs := bfs.StaticFs("en") checkFileContent(sfs, "f1.txt", assert, "Hugo Rocks!") checkFileContent(sfs, "f2.txt", assert, "Hugo Themes Still Rocks!") @@ -272,21 +322,19 @@ func TestStaticFsMultiHost(t *testing.T) { v.Set("workingDir", workDir) v.Set("themesDir", "themes") v.Set("theme", "t1") - v.Set("multihost", true) - - vn := viper.New() - vn.Set("staticDir", "nn_static") - - en := langs.NewLanguage("en", v) - no := langs.NewLanguage("no", v) - no.Set("staticDir", "static_no") - - languages := langs.Languages{ - en, - no, + v.Set("defaultContentLanguage", "en") + + langConfig := map[string]interface{}{ + "no": map[string]interface{}{ + "staticDir": "static_no", + "baseURL": "https://example.org/no/", + }, + "en": map[string]interface{}{ + "baseURL": "https://example.org/en/", + }, } - v.Set("languagesSorted", languages) + v.Set("languages", langConfig) fs := hugofs.NewMem(v) @@ -298,6 +346,8 @@ func TestStaticFsMultiHost(t *testing.T) { afero.WriteFile(fs.Source, filepath.Join(themeStaticDir, "f1.txt"), []byte("Hugo Themes Rocks!"), 0755) afero.WriteFile(fs.Source, filepath.Join(themeStaticDir, "f2.txt"), []byte("Hugo Themes Still Rocks!"), 0755) + assert.NoError(initConfig(fs.Source, v)) + p, err := paths.New(fs, v) assert.NoError(err) bfs, err := NewBase(p) @@ -312,9 +362,9 @@ func TestStaticFsMultiHost(t *testing.T) { } func checkFileCount(fs afero.Fs, dirname string, assert *require.Assertions, expected int) { - count, _, err := countFileaAndGetDirs(fs, dirname) - assert.NoError(err) - assert.Equal(expected, count) + count, fnames, err := countFileaAndGetFilenames(fs, dirname) + assert.NoError(err, fnames) + assert.Equal(expected, count, fnames) } func checkFileContent(fs afero.Fs, filename string, assert *require.Assertions, expected ...string) { @@ -329,27 +379,38 @@ func checkFileContent(fs afero.Fs, filename string, assert *require.Assertions, } } -func countFileaAndGetDirs(fs afero.Fs, dirname string) (int, []string, error) { +func countFileaAndGetFilenames(fs afero.Fs, dirname string) (int, []string, error) { if fs == nil { return 0, nil, errors.New("no fs") } counter := 0 - var dirs []string - - afero.Walk(fs, dirname, func(path string, info os.FileInfo, err error) error { - if info != nil { - if !info.IsDir() { - counter++ - } else if info.Name() != "." { - dirs = append(dirs, filepath.Join(path, info.Name())) - } + var filenames []string + + wf := func(path string, info hugofs.FileMetaInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + counter++ + } + + if info.Name() != "." { + name := info.Name() + name = strings.Replace(name, filepath.FromSlash("/my/work"), "WORK_DIR", 1) + filenames = append(filenames, name) } return nil - }) + } + + w := hugofs.NewWalkway(hugofs.WalkwayConfig{Fs: fs, Root: dirname, WalkFn: wf}) + + if err := w.Walk(); err != nil { + return -1, nil, err + } - return counter, dirs, nil + return counter, filenames, nil } func setConfigAndWriteSomeFilesTo(fs afero.Fs, v *viper.Viper, key, val string, num int) { @@ -357,7 +418,7 @@ func setConfigAndWriteSomeFilesTo(fs afero.Fs, v *viper.Viper, key, val string, v.Set(key, val) fs.Mkdir(val, 0755) for i := 0; i < num; i++ { - filename := filepath.Join(workingDir, val, fmt.Sprintf("file%d.txt", i+1)) + filename := filepath.Join(workingDir, val, fmt.Sprintf("f%d.txt", i+1)) afero.WriteFile(fs, filename, []byte(fmt.Sprintf("content:%s:%d", key, i+1)), 0755) } } diff --git a/hugolib/hugo_modules_test.go b/hugolib/hugo_modules_test.go new file mode 100644 index 00000000000..7a171c089f1 --- /dev/null +++ b/hugolib/hugo_modules_test.go @@ -0,0 +1,382 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "fmt" + "math/rand" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/spf13/afero" + + "github.com/gohugoio/hugo/hugofs/files" + + "github.com/gohugoio/hugo/common/hugo" + + "github.com/gohugoio/hugo/htesting" + "github.com/gohugoio/hugo/hugofs" + + "github.com/gohugoio/testmodBuilder/mods" + "github.com/spf13/viper" + "github.com/stretchr/testify/require" +) + +// TODO(bep) mod this fails when testmodBuilder is also building ... +func TestHugoModules(t *testing.T) { + t.Parallel() + + if hugo.GoMinorVersion() < 12 { + // https://github.com/golang/go/issues/26794 + // There were some concurrent issues with Go modules in < Go 12. + t.Skip("skip this for Go <= 1.11 due to a bug in Go's stdlib") + } + + if testing.Short() { + t.Skip() + } + + rnd := rand.New(rand.NewSource(time.Now().UnixNano())) + gooss := []string{"linux", "darwin", "windows"} + goos := gooss[rnd.Intn(len(gooss))] + ignoreVendor := rnd.Intn(2) == 0 + testmods := mods.CreateModules(goos).Collect() + rnd.Shuffle(len(testmods), func(i, j int) { testmods[i], testmods[j] = testmods[j], testmods[i] }) + + for _, m := range testmods[:2] { + assert := require.New(t) + + v := viper.New() + + workingDir, clean, err := htesting.CreateTempDir(hugofs.Os, "hugo-modules-test") + assert.NoError(err) + defer clean() + + configTemplate := ` +baseURL = "https://example.com" +title = "My Modular Site" +workingDir = %q +theme = %q +ignoreVendor = %t + +` + + config := fmt.Sprintf(configTemplate, workingDir, m.Path(), ignoreVendor) + + b := newTestSitesBuilder(t) + + // Need to use OS fs for this. + b.Fs = hugofs.NewDefault(v) + + b.WithWorkingDir(workingDir).WithConfigFile("toml", config) + b.WithContent("page.md", ` +--- +title: "Foo" +--- +`) + b.WithTemplates("home.html", ` + +{{ $mod := .Site.Data.modinfo.module }} +Mod Name: {{ $mod.name }} +Mod Version: {{ $mod.version }} +---- +{{ range $k, $v := .Site.Data.modinfo }} +- {{ $k }}: {{ range $kk, $vv := $v }}{{ $kk }}: {{ $vv }}|{{ end -}} +{{ end }} + + +`) + b.WithSourceFile("go.mod", ` +module github.com/gohugoio/tests/testHugoModules + + +`) + + b.Build(BuildCfg{}) + + // Verify that go.mod is autopopulated with all the modules in config.toml. + b.AssertFileContent("go.mod", m.Path()) + + b.AssertFileContent("public/index.html", + "Mod Name: "+m.Name(), + "Mod Version: v1.4.0") + + b.AssertFileContent("public/index.html", createChildModMatchers(m, ignoreVendor, m.Vendor)...) + + } +} + +func createChildModMatchers(m *mods.Md, ignoreVendor, vendored bool) []string { + // Child depdendencies are one behind. + expectMinorVersion := 3 + + if !ignoreVendor && vendored { + // Vendored modules are stuck at v1.1.0. + expectMinorVersion = 1 + } + + expectVersion := fmt.Sprintf("v1.%d.0", expectMinorVersion) + + var matchers []string + + for _, mm := range m.Children { + matchers = append( + matchers, + fmt.Sprintf("%s: name: %s|version: %s", mm.Name(), mm.Name(), expectVersion)) + matchers = append(matchers, createChildModMatchers(mm, ignoreVendor, vendored || mm.Vendor)...) + } + return matchers +} + +func TestThemeWithContent(t *testing.T) { + t.Parallel() + + b := newTestSitesBuilder(t).WithWorkingDir("/site").WithConfigFile("toml", ` +baseURL="https://example.org" + +workingDir="/site" + +defaultContentLanguage = "en" + +[module] +[[module.imports]] +path="a" +[[module.imports.mounts]] +source="myacontent" +target="content/blog" +lang="en" +[[module.imports]] +path="b" +[[module.imports.mounts]] +source="mybcontent" +target="content/blog" +lang="nn" +[[module.imports]] +path="c" +[[module.imports]] +path="d" + +[languages] + +[languages.en] +title = "Title in English" +languageName = "English" +weight = 1 +[languages.nn] +languageName = "Nynorsk" +weight = 2 +title = "Tittel på nynorsk" +[languages.nb] +languageName = "Bokmål" +weight = 3 +title = "Tittel på bokmål" +[languages.fr] +languageName = "French" +weight = 4 +title = "French Title" + + +`) + + b.WithTemplatesAdded("index.html", ` +{{ range .Site.RegularPages }} +|{{ .Title }}|{{ .RelPermalink }}|{{ .Plain }} +{{ end }} +{{ $data := .Site.Data }} +Data Common: {{ $data.common.value }} +Data C: {{ $data.c.value }} +Data D: {{ $data.d.value }} +All Data: {{ $data }} + +i18n hello: {{ i18n "hello" . }} +i18n theme: {{ i18n "theme" . }} +i18n theme2: {{ i18n "theme2" . }} +`) + + content := func(id string) string { + return fmt.Sprintf(`--- +title: Title %s +--- +Content %s + +`, id, id) + } + + i18nContent := func(id, value string) string { + return fmt.Sprintf(` +[%s] +other = %q +`, id, value) + } + + // Content files + b.WithSourceFile("themes/a/myacontent/page.md", content("theme-a-en")) + b.WithSourceFile("themes/b/mybcontent/page.md", content("theme-b-nn")) + b.WithSourceFile("themes/c/content/blog/c.md", content("theme-c-nn")) + + // Data files + b.WithSourceFile("data/common.toml", `value="Project"`) + b.WithSourceFile("themes/c/data/common.toml", `value="Theme C"`) + b.WithSourceFile("themes/c/data/c.toml", `value="Hugo Rocks!"`) + b.WithSourceFile("themes/d/data/c.toml", `value="Hugo Rodcks!"`) + b.WithSourceFile("themes/d/data/d.toml", `value="Hugo Rodks!"`) + + // i18n files + b.WithSourceFile("i18n/en.toml", i18nContent("hello", "Project")) + b.WithSourceFile("themes/c/en.toml", i18nContent("hello", "Theme C")) + b.WithSourceFile("themes/c/i18n/en.toml", i18nContent("theme", "Theme C")) + b.WithSourceFile("themes/d/i18n/en.toml", i18nContent("theme", "Theme D")) + b.WithSourceFile("themes/d/i18n/en.toml", i18nContent("theme2", "Theme2 D")) + + // Static files + b.WithSourceFile("themes/c/static/hello.txt", `Hugo Rocks!"`) + + b.Build(BuildCfg{}) + + b.AssertFileContent("public/index.html", "|Title theme-a-en|/blog/page/|Content theme-a-en") + b.AssertFileContent("public/nn/index.html", "|Title theme-b-nn|/nn/blog/page/|Content theme-b-nn") + + // Data + b.AssertFileContent("public/index.html", + "Data Common: Project", + "Data C: Hugo Rocks!", + "Data D: Hugo Rodks!", + ) + + // i18n + b.AssertFileContent("public/index.html", + "i18n hello: Project", + "i18n theme: Theme C", + "i18n theme2: Theme2 D", + ) + +} + +func TestHugoModulesSymlinks(t *testing.T) { + skipSymlink(t) + + wd, _ := os.Getwd() + defer func() { + os.Chdir(wd) + }() + + assert := require.New(t) + // We need to use the OS fs for this. + cfg := viper.New() + fs := hugofs.NewFrom(hugofs.Os, cfg) + + workDir, clean, err := htesting.CreateTempDir(hugofs.Os, "hugo-mod-sym") + assert.NoError(err) + + defer clean() + + const homeTemplate = ` +Data: {{ .Site.Data }} +` + + createDirsAndFiles := func(baseDir string) { + for _, dir := range files.ComponentFolders { + realDir := filepath.Join(baseDir, dir, "real") + assert.NoError(os.MkdirAll(realDir, 0777)) + assert.NoError(afero.WriteFile(fs.Source, filepath.Join(realDir, "data.toml"), []byte("[hello]\nother = \"hello\""), 0777)) + } + + assert.NoError(afero.WriteFile(fs.Source, filepath.Join(baseDir, "layouts", "index.html"), []byte(homeTemplate), 0777)) + } + + // Create project dirs and files. + createDirsAndFiles(workDir) + // Create one module inside the default themes folder. + themeDir := filepath.Join(workDir, "themes", "mymod") + createDirsAndFiles(themeDir) + + createSymlinks := func(baseDir, id string) { + for _, dir := range files.ComponentFolders { + assert.NoError(os.Chdir(filepath.Join(baseDir, dir))) + assert.NoError(os.Symlink("real", fmt.Sprintf("realsym%s", id))) + assert.NoError(os.Chdir(filepath.Join(baseDir, dir, "real"))) + assert.NoError(os.Symlink("data.toml", fmt.Sprintf(filepath.FromSlash("datasym%s.toml"), id))) + } + } + + createSymlinks(workDir, "project") + createSymlinks(themeDir, "mod") + + config := ` +baseURL = "https://example.com" +theme="mymod" +defaultContentLanguage="nn" +defaultContentLanguageInSubDir=true + +[languages] +[languages.nn] +weight = 1 +[languages.en] +weight = 2 + + +` + + b := newTestSitesBuilder(t).WithNothingAdded().WithWorkingDir(workDir) + b.Fs = fs + + b.WithConfigFile("toml", config) + assert.NoError(os.Chdir(workDir)) + + b.Build(BuildCfg{}) + + b.AssertFileContentFn(filepath.Join("public", "en", "index.html"), func(s string) bool { + // Symbolic links only followed in project. There should be WARNING logs. + return !strings.Contains(s, "symmod") && strings.Contains(s, "symproject") + }) + + bfs := b.H.BaseFs + + for _, componentFs := range []afero.Fs{ + bfs.Archetypes.Fs, + bfs.Content.Fs, + bfs.Data.Fs, + bfs.Assets.Fs, + bfs.Static[""].Fs, + bfs.I18n.Fs, + bfs.Resources.Fs} { + + for i, id := range []string{"mod", "project"} { + + statCheck := func(fs afero.Fs, filename string) { + shouldFail := i == 0 + _, err := fs.Stat(filepath.FromSlash(filename)) + if err != nil { + if strings.HasSuffix(filename, "toml") && strings.Contains(err.Error(), "files not supported") { + // OK + return + } + } + if shouldFail { + assert.Error(err) + assert.Equal(hugofs.ErrPermissionSymlink, err) + } else { + assert.NoError(err) + } + } + + statCheck(componentFs, fmt.Sprintf("realsym%s", id)) + statCheck(componentFs, fmt.Sprintf("real/datasym%s.toml", id)) + + } + } +} diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go index e852e7f1dd4..edaa9f40308 100644 --- a/hugolib/hugo_sites.go +++ b/hugolib/hugo_sites.go @@ -22,6 +22,8 @@ import ( "strings" "sync" + radix "github.com/hashicorp/go-immutable-radix" + "github.com/gohugoio/hugo/output" "github.com/gohugoio/hugo/parser/metadecoders" @@ -33,7 +35,6 @@ import ( "github.com/bep/gitmap" "github.com/gohugoio/hugo/config" - "github.com/spf13/afero" "github.com/gohugoio/hugo/publisher" @@ -199,10 +200,11 @@ func (h *HugoSites) IsMultihost() bool { return h != nil && h.multihost } -func (h *HugoSites) LanguageSet() map[string]bool { - set := make(map[string]bool) - for _, s := range h.Sites { - set[s.language.Lang] = true +// TODO(bep) mod default content first +func (h *HugoSites) LanguageSet() map[string]int { + set := make(map[string]int) + for i, s := range h.Sites { + set[s.language.Lang] = i } return set } @@ -222,14 +224,6 @@ func (h *HugoSites) PrintProcessingStats(w io.Writer) { helpers.ProcessingStatsTable(w, stats...) } -func (h *HugoSites) langSite() map[string]*Site { - m := make(map[string]*Site) - for _, s := range h.Sites { - m[s.language.Lang] = s - } - return m -} - // GetContentPage finds a Page with content given the absolute filename. // Returns nil if none found. func (h *HugoSites) GetContentPage(filename string) page.Page { @@ -265,7 +259,7 @@ func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) { langConfig, err := newMultiLingualFromSites(cfg.Cfg, sites...) if err != nil { - return nil, err + return nil, errors.Wrap(err, "failed to create language config") } var contentChangeTracker *contentChangeMap @@ -288,8 +282,11 @@ func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) { } h.init.data.Add(func() (interface{}, error) { - err := h.loadData(h.PathSpec.BaseFs.Data.Fs) - return err, nil + err := h.loadData(h.PathSpec.BaseFs.Data.Dirs) + if err != nil { + return nil, errors.Wrap(err, "failed to load data") + } + return nil, nil }) h.init.translations.Add(func() (interface{}, error) { @@ -303,7 +300,10 @@ func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) { h.init.gitInfo.Add(func() (interface{}, error) { err := h.loadGitInfo() - return nil, err + if err != nil { + return nil, errors.Wrap(err, "failed to load Git info") + } + return nil, nil }) for _, s := range sites { @@ -311,7 +311,7 @@ func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) { } if err := applyDeps(cfg, sites...); err != nil { - return nil, err + return nil, errors.Wrap(err, "add site dependencies") } h.Deps = sites[0].Deps @@ -319,7 +319,12 @@ func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) { // Only needed in server mode. // TODO(bep) clean up the running vs watching terms if cfg.Running { - contentChangeTracker = &contentChangeMap{pathSpec: h.PathSpec, symContent: make(map[string]map[string]bool)} + contentChangeTracker = &contentChangeMap{ + pathSpec: h.PathSpec, + symContent: make(map[string]map[string]bool), + leafBundles: radix.New(), + branchBundles: make(map[string]bool), + } h.ContentChanges = contentChangeTracker } @@ -371,7 +376,7 @@ func applyDeps(cfg deps.DepsCfg, sites ...*Site) error { siteConfig, err := loadSiteConfig(s.language) if err != nil { - return err + return errors.Wrap(err, "load site config") } s.siteConfigConfig = siteConfig s.siteRefLinker, err = newSiteRefLinker(s.language, s) @@ -388,17 +393,17 @@ func applyDeps(cfg deps.DepsCfg, sites ...*Site) error { var err error d, err = deps.New(cfg) if err != nil { - return err + return errors.Wrap(err, "create deps") } d.OutputFormatsConfig = s.outputFormatsConfig if err := onCreated(d); err != nil { - return err + return errors.Wrap(err, "on created") } if err = d.LoadResources(); err != nil { - return err + return errors.Wrap(err, "load resources") } } else { @@ -418,7 +423,7 @@ func applyDeps(cfg deps.DepsCfg, sites ...*Site) error { func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) { sites, err := createSitesFromConfig(cfg) if err != nil { - return nil, err + return nil, errors.Wrap(err, "from config") } return newHugoSites(cfg, sites...) } @@ -800,41 +805,45 @@ func (h *HugoSites) Pages() page.Pages { return h.Sites[0].AllPages() } -func (h *HugoSites) loadData(fs afero.Fs) (err error) { - spec := source.NewSourceSpec(h.PathSpec, fs) - fileSystem := spec.NewFilesystem("") +func (h *HugoSites) loadData(fis []hugofs.FileMetaInfo) (err error) { + spec := source.NewSourceSpec(h.PathSpec, nil) + h.data = make(map[string]interface{}) - for _, r := range fileSystem.Files() { - if err := h.handleDataFile(r); err != nil { + for _, fi := range fis { + fileSystem := spec.NewFilesystemFromFileMetaInfo(fi) + files, err := fileSystem.Files() + if err != nil { return err } + for _, r := range files { + if err := h.handleDataFile(r); err != nil { + return err + } + } } return } -func (h *HugoSites) handleDataFile(r source.ReadableFile) error { +func (h *HugoSites) handleDataFile(r source.File) error { var current map[string]interface{} - f, err := r.Open() + f, err := r.FileInfo().Meta().Open() if err != nil { - return errors.Wrapf(err, "Failed to open data file %q:", r.LogicalName()) + return errors.Wrapf(err, "data: failed to open %q:", r.LogicalName()) } defer f.Close() // Crawl in data tree to insert data current = h.data keyParts := strings.Split(r.Dir(), helpers.FilePathSeparator) - // The first path element is the virtual folder (typically theme name), which is - // not part of the key. - if len(keyParts) > 1 { - for _, key := range keyParts[1:] { - if key != "" { - if _, ok := current[key]; !ok { - current[key] = make(map[string]interface{}) - } - current = current[key].(map[string]interface{}) + + for _, key := range keyParts { + if key != "" { + if _, ok := current[key]; !ok { + current[key] = make(map[string]interface{}) } + current = current[key].(map[string]interface{}) } } @@ -848,15 +857,10 @@ func (h *HugoSites) handleDataFile(r source.ReadableFile) error { } // filepath.Walk walks the files in lexical order, '/' comes before '.' - // this warning could happen if - // 1. A theme uses the same key; the main data folder wins - // 2. A sub folder uses the same key: the sub folder wins higherPrecedentData := current[r.BaseFileName()] switch data.(type) { case nil: - // hear the crickets? - case map[string]interface{}: switch higherPrecedentData.(type) { @@ -868,7 +872,11 @@ func (h *HugoSites) handleDataFile(r source.ReadableFile) error { higherPrecedentMap := higherPrecedentData.(map[string]interface{}) for key, value := range data.(map[string]interface{}) { if _, exists := higherPrecedentMap[key]; exists { - h.Log.WARN.Printf("Data for key '%s' in path '%s' is overridden by higher precedence data already in the data tree", key, r.Path()) + // this warning could happen if + // 1. A theme uses the same key; the main data folder wins + // 2. A sub folder uses the same key: the sub folder wins + // TODO(bep) figure out a way to detect 2) above and make that a WARN + h.Log.INFO.Printf("Data for key '%s' in path '%s' is overridden by higher precedence data already in the data tree", key, r.Path()) } else { higherPrecedentMap[key] = value } @@ -896,12 +904,12 @@ func (h *HugoSites) handleDataFile(r source.ReadableFile) error { } func (h *HugoSites) errWithFileContext(err error, f source.File) error { - rfi, ok := f.FileInfo().(hugofs.RealFilenameInfo) + fim, ok := f.FileInfo().(hugofs.FileMetaInfo) if !ok { return err } - realFilename := rfi.RealFilename() + realFilename := fim.Meta().Filename() err, _ = herrors.WithFileContextForFile( err, @@ -913,8 +921,8 @@ func (h *HugoSites) errWithFileContext(err error, f source.File) error { return err } -func (h *HugoSites) readData(f source.ReadableFile) (interface{}, error) { - file, err := f.Open() +func (h *HugoSites) readData(f source.File) (interface{}, error) { + file, err := f.FileInfo().Meta().Open() if err != nil { return nil, errors.Wrap(err, "readData: failed to open data file") } @@ -939,9 +947,14 @@ func (h *HugoSites) findPagesByShortcode(shortcode string) page.Pages { // Used in partial reloading to determine if the change is in a bundle. type contentChangeMap struct { - mu sync.RWMutex - branches []string - leafs []string + mu sync.RWMutex + + // Holds directories with leaf bundles. + leafBundles *radix.Tree + leafBundlesTxn *radix.Txn + + // Holds directories with branch bundles. + branchBundles map[string]bool pathSpec *helpers.PathSpec @@ -950,9 +963,22 @@ type contentChangeMap struct { // locations in /content -- which is really cool, but also means we have to // go an extra mile to handle changes. // This map is only used in watch mode. - // It maps either file to files or the real dir to a set of content directories where it is in use. - symContent map[string]map[string]bool + // It maps either file to files or the real dir to a set of content directories + // where it is in use. symContentMu sync.Mutex + symContent map[string]map[string]bool +} + +func (m *contentChangeMap) start() { + m.mu.Lock() + m.leafBundlesTxn = m.leafBundles.Txn() + m.mu.Unlock() +} + +func (m *contentChangeMap) stop() { + m.mu.Lock() + m.leafBundles = m.leafBundlesTxn.Commit() + m.mu.Unlock() } func (m *contentChangeMap) add(filename string, tp bundleDirType) { @@ -961,68 +987,63 @@ func (m *contentChangeMap) add(filename string, tp bundleDirType) { dir = strings.TrimPrefix(dir, ".") switch tp { case bundleBranch: - m.branches = append(m.branches, dir) + m.branchBundles[dir] = true case bundleLeaf: - m.leafs = append(m.leafs, dir) + m.leafBundlesTxn.Insert([]byte(dir), true) default: panic("invalid bundle type") } m.mu.Unlock() } -// Track the addition of bundle dirs. -func (m *contentChangeMap) handleBundles(b *bundleDirs) { - for _, bd := range b.bundles { - m.add(bd.fi.Path(), bd.tp) - } -} - -// resolveAndRemove resolves the given filename to the root folder of a bundle, if relevant. -// It also removes the entry from the map. It will be re-added again by the partial -// build if it still is a bundle. func (m *contentChangeMap) resolveAndRemove(filename string) (string, string, bundleDirType) { m.mu.RLock() defer m.mu.RUnlock() // Bundles share resources, so we need to start from the virtual root. - relPath := m.pathSpec.RelContentDir(filename) - dir, name := filepath.Split(relPath) + relFilename := m.pathSpec.RelContentDir(filename) + dir, name := filepath.Split(relFilename) if !strings.HasSuffix(dir, helpers.FilePathSeparator) { dir += helpers.FilePathSeparator } - fileTp, isContent := classifyBundledFile(name) - - // This may be a member of a bundle. Start with branch bundles, the most specific. - if fileTp == bundleBranch || (fileTp == bundleNot && !isContent) { - for i, b := range m.branches { - if b == dir { - m.branches = append(m.branches[:i], m.branches[i+1:]...) - return dir, b, bundleBranch - } - } + if _, found := m.branchBundles[dir]; found { + delete(m.branchBundles, dir) + return dir, dir, bundleBranch } - // And finally the leaf bundles, which can contain anything. - for i, l := range m.leafs { - if strings.HasPrefix(dir, l) { - m.leafs = append(m.leafs[:i], m.leafs[i+1:]...) - return dir, l, bundleLeaf - } + if key, _, found := m.leafBundles.Root().LongestPrefix([]byte(dir)); found { + m.leafBundlesTxn.Delete(key) + dir = string(key) + return dir, dir, bundleLeaf } + fileTp, isContent := classifyBundledFile(name) if isContent && fileTp != bundleNot { // A new bundle. return dir, dir, fileTp } - // Not part of any bundle return dir, filename, bundleNot + } -func (m *contentChangeMap) addSymbolicLinkMapping(from, to string) { +func (m *contentChangeMap) addSymbolicLinkMapping(fim hugofs.FileMetaInfo) { + meta := fim.Meta() + if !meta.IsSymlink() { + return + } m.symContentMu.Lock() + + from, to := meta.Filename(), meta.OriginalFilename() + if fim.IsDir() { + if !strings.HasSuffix(from, helpers.FilePathSeparator) { + from += helpers.FilePathSeparator + } + } + mm, found := m.symContent[from] + if !found { mm = make(map[string]bool) m.symContent[from] = mm @@ -1044,5 +1065,6 @@ func (m *contentChangeMap) GetSymbolicLinkMappings(dir string) []string { } sort.Strings(dirs) + return dirs } diff --git a/hugolib/hugo_sites_build.go b/hugolib/hugo_sites_build.go index 7f725def2f9..d20932599c3 100644 --- a/hugolib/hugo_sites_build.go +++ b/hugolib/hugo_sites_build.go @@ -22,7 +22,7 @@ import ( "github.com/gohugoio/hugo/output" - "errors" + "github.com/pkg/errors" "github.com/fsnotify/fsnotify" "github.com/gohugoio/hugo/helpers" @@ -82,11 +82,11 @@ func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error { if len(events) > 0 { // Rebuild if err := h.initRebuild(conf); err != nil { - return err + return errors.Wrap(err, "initRebuild") } } else { if err := h.initSites(conf); err != nil { - return err + return errors.Wrap(err, "initSites") } } @@ -97,7 +97,7 @@ func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error { } trace.WithRegion(ctx, "process", f) if err != nil { - return err + return errors.Wrap(err, "process") } f = func() { diff --git a/hugolib/hugo_sites_build_errors_test.go b/hugolib/hugo_sites_build_errors_test.go index 6fe4901a1c8..993b4e613ab 100644 --- a/hugolib/hugo_sites_build_errors_test.go +++ b/hugolib/hugo_sites_build_errors_test.go @@ -40,7 +40,6 @@ func (t testSiteBuildErrorAsserter) assertErrorMessage(e1, e2 string) { } func TestSiteBuildErrors(t *testing.T) { - t.Parallel() const ( yamlcontent = "yamlcontent" @@ -226,7 +225,9 @@ func TestSiteBuildErrors(t *testing.T) { } for _, test := range tests { + test := test t.Run(test.name, func(t *testing.T) { + t.Parallel() assert := require.New(t) errorAsserter := testSiteBuildErrorAsserter{ assert: assert, diff --git a/hugolib/hugo_sites_build_test.go b/hugolib/hugo_sites_build_test.go index 236fd11a64c..876f21cfa6d 100644 --- a/hugolib/hugo_sites_build_test.go +++ b/hugolib/hugo_sites_build_test.go @@ -5,7 +5,6 @@ import ( "strings" "testing" - "os" "path/filepath" "time" @@ -246,6 +245,8 @@ func doTestMultiSitesBuild(t *testing.T, configTemplate, configSuffix string) { require.Equal(t, "en", enSite.language.Lang) + //dumpPages(enSite.RegularPages()...) + assert.Equal(5, len(enSite.RegularPages())) assert.Equal(32, len(enSite.AllPages())) @@ -447,7 +448,7 @@ func TestMultiSitesRebuild(t *testing.T) { require.NotNil(t, homeEn) assert.Len(homeEn.Translations(), 3) - contentFs := b.H.BaseFs.Content.Fs + contentFs := b.H.Fs.Source for i, this := range []struct { preFunc func(t *testing.T) @@ -480,9 +481,9 @@ func TestMultiSitesRebuild(t *testing.T) { }, { func(t *testing.T) { - writeNewContentFile(t, contentFs, "new_en_1", "2016-07-31", "new1.en.md", -5) - writeNewContentFile(t, contentFs, "new_en_2", "1989-07-30", "new2.en.md", -10) - writeNewContentFile(t, contentFs, "new_fr_1", "2016-07-30", "new1.fr.md", 10) + writeNewContentFile(t, contentFs, "new_en_1", "2016-07-31", "content/new1.en.md", -5) + writeNewContentFile(t, contentFs, "new_en_2", "1989-07-30", "content/new2.en.md", -10) + writeNewContentFile(t, contentFs, "new_fr_1", "2016-07-30", "content/new1.fr.md", 10) }, []fsnotify.Event{ {Name: filepath.FromSlash("content/new1.en.md"), Op: fsnotify.Create}, @@ -503,7 +504,7 @@ func TestMultiSitesRebuild(t *testing.T) { }, { func(t *testing.T) { - p := "sect/doc1.en.md" + p := "content/sect/doc1.en.md" doc1 := readFileFromFs(t, contentFs, p) doc1 += "CHANGED" writeToFs(t, contentFs, p, doc1) @@ -519,7 +520,7 @@ func TestMultiSitesRebuild(t *testing.T) { // Rename a file { func(t *testing.T) { - if err := contentFs.Rename("new1.en.md", "new1renamed.en.md"); err != nil { + if err := contentFs.Rename("content/new1.en.md", "content/new1renamed.en.md"); err != nil { t.Fatalf("Rename failed: %s", err) } }, @@ -672,38 +673,6 @@ title = "Svenska" } -func TestChangeDefaultLanguage(t *testing.T) { - t.Parallel() - - assert := require.New(t) - - b := newMultiSiteTestBuilder(t, "", "", map[string]interface{}{ - "DefaultContentLanguage": "fr", - "DefaultContentLanguageInSubdir": false, - }) - b.CreateSites().Build(BuildCfg{}) - - b.AssertFileContent("public/sect/doc1/index.html", "Single", "Bonjour") - b.AssertFileContent("public/en/sect/doc2/index.html", "Single", "Hello") - - // Switch language - b.WithNewConfigData(map[string]interface{}{ - "DefaultContentLanguage": "en", - "DefaultContentLanguageInSubdir": false, - }) - - assert.NoError(b.LoadConfig()) - err := b.H.Build(BuildCfg{NewConfig: b.Cfg}) - - if err != nil { - t.Fatalf("Failed to rebuild sites: %s", err) - } - - // Default language is now en, so that should now be the "root" language - b.AssertFileContent("public/fr/sect/doc1/index.html", "Single", "Bonjour") - b.AssertFileContent("public/sect/doc2/index.html", "Single", "Hello") -} - // https://github.com/gohugoio/hugo/issues/4706 func TestContentStressTest(t *testing.T) { b := newTestSitesBuilder(t) @@ -1261,16 +1230,19 @@ var multiSiteJSONConfigTemplate = ` ` func writeSource(t testing.TB, fs *hugofs.Fs, filename, content string) { + t.Helper() writeToFs(t, fs.Source, filename, content) } func writeToFs(t testing.TB, fs afero.Fs, filename, content string) { + t.Helper() if err := afero.WriteFile(fs, filepath.FromSlash(filename), []byte(content), 0755); err != nil { t.Fatalf("Failed to write file: %s", err) } } func readDestination(t testing.TB, fs *hugofs.Fs, filename string) string { + t.Helper() return readFileFromFs(t, fs.Destination, filename) } @@ -1287,6 +1259,7 @@ func readSource(t *testing.T, fs *hugofs.Fs, filename string) string { } func readFileFromFs(t testing.TB, fs afero.Fs, filename string) string { + t.Helper() filename = filepath.Clean(filename) b, err := afero.ReadFile(fs, filename) if err != nil { @@ -1308,8 +1281,8 @@ func readFileFromFs(t testing.TB, fs afero.Fs, filename string) string { root = helpers.FilePathSeparator + root } - helpers.PrintFs(fs, root, os.Stdout) - Fatalf(t, "Failed to read file: %s", err) + //helpers.PrintFs(fs, root, os.Stdout) + t.Fatalf("Failed to read file: %s", err) } return string(b) } diff --git a/hugolib/hugo_themes_test.go b/hugolib/hugo_themes_test.go deleted file mode 100644 index 05bfaa692bc..00000000000 --- a/hugolib/hugo_themes_test.go +++ /dev/null @@ -1,268 +0,0 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugolib - -import ( - "fmt" - "os" - "path/filepath" - "testing" - - "github.com/gohugoio/hugo/common/loggers" -) - -func TestThemesGraph(t *testing.T) { - t.Parallel() - - const ( - themeStandalone = ` -title = "Theme Standalone" -[params] -v1 = "v1s" -v2 = "v2s" -` - themeCyclic = ` -title = "Theme Cyclic" -theme = "theme3" -[params] -v1 = "v1c" -v2 = "v2c" -` - theme1 = ` -title = "Theme #1" -theme = "themeStandalone" -[params] -v2 = "v21" -` - - theme2 = ` -title = "Theme #2" -theme = "theme1" -[params] -v1 = "v12" -` - - theme3 = ` -title = "Theme #3" -theme = ["theme2", "themeStandalone", "themeCyclic"] -[params] -v1 = "v13" -v2 = "v24" -` - - theme4 = ` -title = "Theme #4" -theme = "theme3" -[params] -v1 = "v14" -v2 = "v24" -` - - site1 = ` - theme = "theme4" - - [params] - v1 = "site" -` - site2 = ` - theme = ["theme2", "themeStandalone"] -` - ) - - var ( - testConfigs = []struct { - siteConfig string - - // The name of theme somewhere in the middle to write custom key/files. - offset string - - check func(b *sitesBuilder) - }{ - {site1, "theme3", func(b *sitesBuilder) { - - // site1: theme4 theme3 theme2 theme1 themeStandalone themeCyclic - - // Check data - // theme3 should win the offset competition - b.AssertFileContent("public/index.html", "theme1o::[offset][v]theme3", "theme4o::[offset][v]theme3", "themeStandaloneo::[offset][v]theme3") - b.AssertFileContent("public/index.html", "nproject::[inner][other]project|[project][other]project|[theme][other]theme4|[theme1][other]theme1") - b.AssertFileContent("public/index.html", "ntheme::[inner][other]theme4|[theme][other]theme4|[theme1][other]theme1|[theme2][other]theme2|[theme3][other]theme3") - b.AssertFileContent("public/index.html", "theme1::[inner][other]project|[project][other]project|[theme][other]theme1|[theme1][other]theme1|") - b.AssertFileContent("public/index.html", "theme4::[inner][other]project|[project][other]project|[theme][other]theme4|[theme4][other]theme4|") - - // Check layouts - b.AssertFileContent("public/index.html", "partial ntheme: theme4", "partial theme2o: theme3") - - // Check i18n - b.AssertFileContent("public/index.html", "i18n: project theme4") - - // Check static files - // TODO(bep) static files not currently part of the build b.AssertFileContent("public/nproject.txt", "TODO") - - // Check site params - b.AssertFileContent("public/index.html", "v1::site", "v2::v24") - }}, - {site2, "", func(b *sitesBuilder) { - - // site2: theme2 theme1 themeStandalone - b.AssertFileContent("public/index.html", "nproject::[inner][other]project|[project][other]project|[theme][other]theme2|[theme1][other]theme1|[theme2][other]theme2|[themeStandalone][other]themeStandalone|") - b.AssertFileContent("public/index.html", "ntheme::[inner][other]theme2|[theme][other]theme2|[theme1][other]theme1|[theme2][other]theme2|[themeStandalone][other]themeStandalone|") - b.AssertFileContent("public/index.html", "i18n: project theme2") - b.AssertFileContent("public/index.html", "partial ntheme: theme2") - - // Params only set in themes - b.AssertFileContent("public/index.html", "v1::v12", "v2::v21") - - }}, - } - - themeConfigs = []struct { - name string - config string - }{ - {"themeStandalone", themeStandalone}, - {"themeCyclic", themeCyclic}, - {"theme1", theme1}, - {"theme2", theme2}, - {"theme3", theme3}, - {"theme4", theme4}, - } - ) - - for i, testConfig := range testConfigs { - t.Log(fmt.Sprintf("Test %d", i)) - b := newTestSitesBuilder(t).WithLogger(loggers.NewErrorLogger()) - b.WithConfigFile("toml", testConfig.siteConfig) - - for _, tc := range themeConfigs { - var variationsNameBase = []string{"nproject", "ntheme", tc.name} - - themeRoot := filepath.Join("themes", tc.name) - b.WithSourceFile(filepath.Join(themeRoot, "config.toml"), tc.config) - - b.WithSourceFile(filepath.Join("layouts", "partials", "m.html"), `{{- range $k, $v := . }}{{ $k }}::{{ template "printv" $v }} -{{ end }} -{{ define "printv" }} -{{- $tp := printf "%T" . -}} -{{- if (strings.HasSuffix $tp "map[string]interface {}") -}} -{{- range $k, $v := . }}[{{ $k }}]{{ template "printv" $v }}{{ end -}} -{{- else -}} -{{- . }}| -{{- end -}} -{{ end }} -`) - - for _, nameVariaton := range variationsNameBase { - roots := []string{"", themeRoot} - - for _, root := range roots { - name := tc.name - if root == "" { - name = "project" - } - - if nameVariaton == "ntheme" && name == "project" { - continue - } - - // static - b.WithSourceFile(filepath.Join(root, "static", nameVariaton+".txt"), name) - - // layouts - if i == 1 { - b.WithSourceFile(filepath.Join(root, "layouts", "partials", "theme2o.html"), "Not Set") - } - b.WithSourceFile(filepath.Join(root, "layouts", "partials", nameVariaton+".html"), name) - if root != "" && testConfig.offset == tc.name { - for _, tc2 := range themeConfigs { - b.WithSourceFile(filepath.Join(root, "layouts", "partials", tc2.name+"o.html"), name) - } - } - - // i18n + data - - var dataContent string - if root == "" { - dataContent = fmt.Sprintf(` -[%s] -other = %q - -[inner] -other = %q - -`, name, name, name) - } else { - dataContent = fmt.Sprintf(` -[%s] -other = %q - -[inner] -other = %q - -[theme] -other = %q - -`, name, name, name, name) - } - - b.WithSourceFile(filepath.Join(root, "data", nameVariaton+".toml"), dataContent) - b.WithSourceFile(filepath.Join(root, "i18n", "en.toml"), dataContent) - - // If an offset is set, duplicate a data key with a winner in the middle. - if root != "" && testConfig.offset == tc.name { - for _, tc2 := range themeConfigs { - dataContent := fmt.Sprintf(` -[offset] -v = %q -`, tc.name) - b.WithSourceFile(filepath.Join(root, "data", tc2.name+"o.toml"), dataContent) - } - } - } - - } - - } - - for _, themeConfig := range themeConfigs { - b.WithSourceFile(filepath.Join("themes", "config.toml"), themeConfig.config) - } - - b.WithContent(filepath.Join("content", "page.md"), `--- -title: "Page" ---- - -`) - - homeTpl := ` -data: {{ partial "m" .Site.Data }} -i18n: {{ i18n "inner" }} {{ i18n "theme" }} -partial ntheme: {{ partial "ntheme" . }} -partial theme2o: {{ partial "theme2o" . }} -params: {{ partial "m" .Site.Params }} - -` - - b.WithTemplates(filepath.Join("layouts", "home.html"), homeTpl) - - b.Build(BuildCfg{}) - - var _ = os.Stdout - - // printFs(b.H.Deps.BaseFs.LayoutsFs, "", os.Stdout) - testConfig.check(b) - - } - -} diff --git a/hugolib/language_content_dir_test.go b/hugolib/language_content_dir_test.go index ad1e1fb5345..2fea91fe1ab 100644 --- a/hugolib/language_content_dir_test.go +++ b/hugolib/language_content_dir_test.go @@ -211,7 +211,11 @@ Content. _ = os.Stdout - b.Build(BuildCfg{}) + err := b.BuildE(BuildCfg{}) + + //dumpPages(b.H.Sites[1].RegularPages()...) + + assert.NoError(err) assert.Equal(3, len(b.H.Sites)) @@ -222,7 +226,8 @@ Content. b.AssertFileContent("/my/project/public/en/mystatic/file1.yaml", "en") b.AssertFileContent("/my/project/public/nn/mystatic/file1.yaml", "nn") - //dumpPages(nnSite.RegularPages...) + //dumpPages(nnSite.RegularPages()...) + assert.Equal(12, len(nnSite.RegularPages())) assert.Equal(13, len(enSite.RegularPages())) @@ -281,9 +286,9 @@ Content. assert.Equal("/en/sect/mybundle/", bundleEn.RelPermalink()) assert.Equal("/sv/sect/mybundle/", bundleSv.RelPermalink()) - assert.Equal(4, len(bundleEn.Resources())) assert.Equal(4, len(bundleNn.Resources())) assert.Equal(4, len(bundleSv.Resources())) + assert.Equal(4, len(bundleEn.Resources())) b.AssertFileContent("/my/project/public/en/sect/mybundle/index.html", "image/png: /en/sect/mybundle/logo.png") b.AssertFileContent("/my/project/public/nn/sect/mybundle/index.html", "image/png: /nn/sect/mybundle/logo.png") diff --git a/hugolib/menu_test.go b/hugolib/menu_test.go index 4a2b176039d..3708173d931 100644 --- a/hugolib/menu_test.go +++ b/hugolib/menu_test.go @@ -18,8 +18,6 @@ import ( "fmt" - "github.com/spf13/afero" - "github.com/stretchr/testify/require" ) @@ -45,11 +43,10 @@ title = "Section Menu" sectionPagesMenu = "sect" ` - th, h := newTestSitesFromConfig( - t, - afero.NewMemMapFs(), - siteConfig, - "layouts/partials/menu.html", + b := newTestSitesBuilder(t).WithConfigFile("toml", siteConfig) + + b.WithTemplates( + "partials/menu.html", `{{- $p := .page -}} {{- $m := .menu -}} {{ range (index $p.Site.Menus $m) -}} @@ -58,28 +55,25 @@ sectionPagesMenu = "sect" {{- if $p.HasMenuCurrent $m . }}HasMenuCurrent{{ else }}-{{ end -}}| {{- end -}} `, - "layouts/_default/single.html", + "_default/single.html", `Single|{{ .Title }} Menu Sect: {{ partial "menu.html" (dict "page" . "menu" "sect") }} Menu Main: {{ partial "menu.html" (dict "page" . "menu" "main") }}`, - "layouts/_default/list.html", "List|{{ .Title }}|{{ .Content }}", + "_default/list.html", "List|{{ .Title }}|{{ .Content }}", ) - require.Len(t, h.Sites, 1) - - fs := th.Fs - - writeSource(t, fs, "content/sect1/p1.md", fmt.Sprintf(menuPageTemplate, "p1", 1, "main", "atitle1", 40)) - writeSource(t, fs, "content/sect1/p2.md", fmt.Sprintf(menuPageTemplate, "p2", 2, "main", "atitle2", 30)) - writeSource(t, fs, "content/sect2/p3.md", fmt.Sprintf(menuPageTemplate, "p3", 3, "main", "atitle3", 20)) - writeSource(t, fs, "content/sect2/p4.md", fmt.Sprintf(menuPageTemplate, "p4", 4, "main", "atitle4", 10)) - writeSource(t, fs, "content/sect3/p5.md", fmt.Sprintf(menuPageTemplate, "p5", 5, "main", "atitle5", 5)) - writeNewContentFile(t, fs.Source, "Section One", "2017-01-01", "content/sect1/_index.md", 100) - writeNewContentFile(t, fs.Source, "Section Five", "2017-01-01", "content/sect5/_index.md", 10) - - err := h.Build(BuildCfg{}) + b.WithContent( + "sect1/p1.md", fmt.Sprintf(menuPageTemplate, "p1", 1, "main", "atitle1", 40), + "sect1/p2.md", fmt.Sprintf(menuPageTemplate, "p2", 2, "main", "atitle2", 30), + "sect2/p3.md", fmt.Sprintf(menuPageTemplate, "p3", 3, "main", "atitle3", 20), + "sect2/p4.md", fmt.Sprintf(menuPageTemplate, "p4", 4, "main", "atitle4", 10), + "sect3/p5.md", fmt.Sprintf(menuPageTemplate, "p5", 5, "main", "atitle5", 5), + "sect1/_index.md", newTestPage("Section One", "2017-01-01", 100), + "sect5/_index.md", newTestPage("Section Five", "2017-01-01", 10), + ) - require.NoError(t, err) + b.Build(BuildCfg{}) + h := b.H s := h.Sites[0] @@ -90,7 +84,7 @@ Menu Main: {{ partial "menu.html" (dict "page" . "menu" "main") }}`, // There is only one menu in the page, but it is "member of" 2 require.Len(t, p1, 1) - th.assertFileContent("public/sect1/p1/index.html", "Single", + b.AssertFileContent("public/sect1/p1/index.html", "Single", "Menu Sect: "+ "/sect5/|Section Five|Section Five|10|-|-|"+ "/sect1/|Section One|Section One|100|-|HasMenuCurrent|"+ @@ -104,7 +98,7 @@ Menu Main: {{ partial "menu.html" (dict "page" . "menu" "main") }}`, "/sect1/p1/|p1|atitle1|40|IsMenuCurrent|-|", ) - th.assertFileContent("public/sect2/p3/index.html", "Single", + b.AssertFileContent("public/sect2/p3/index.html", "Single", "Menu Sect: "+ "/sect5/|Section Five|Section Five|10|-|-|"+ "/sect1/|Section One|Section One|100|-|-|"+ diff --git a/hugolib/multilingual.go b/hugolib/multilingual.go index 6f744f3a532..9b34c75e6a3 100644 --- a/hugolib/multilingual.go +++ b/hugolib/multilingual.go @@ -16,17 +16,11 @@ package hugolib import ( "sync" - "github.com/gohugoio/hugo/common/maps" - - "sort" - "errors" - "fmt" "github.com/gohugoio/hugo/langs" "github.com/gohugoio/hugo/config" - "github.com/spf13/cast" ) // Multilingual manages the all languages used in a multilingual site. @@ -88,53 +82,3 @@ func (s *Site) multilingualEnabled() bool { } return s.h.multilingual != nil && s.h.multilingual.enabled() } - -func toSortedLanguages(cfg config.Provider, l map[string]interface{}) (langs.Languages, error) { - languages := make(langs.Languages, len(l)) - i := 0 - - for lang, langConf := range l { - langsMap, err := cast.ToStringMapE(langConf) - - if err != nil { - return nil, fmt.Errorf("Language config is not a map: %T", langConf) - } - - language := langs.NewLanguage(lang, cfg) - - for loki, v := range langsMap { - switch loki { - case "title": - language.Title = cast.ToString(v) - case "languagename": - language.LanguageName = cast.ToString(v) - case "weight": - language.Weight = cast.ToInt(v) - case "contentdir": - language.ContentDir = cast.ToString(v) - case "disabled": - language.Disabled = cast.ToBool(v) - case "params": - m := cast.ToStringMap(v) - // Needed for case insensitive fetching of params values - maps.ToLower(m) - for k, vv := range m { - language.SetParam(k, vv) - } - } - - // Put all into the Params map - language.SetParam(loki, v) - - // Also set it in the configuration map (for baseURL etc.) - language.Set(loki, v) - } - - languages[i] = language - i++ - } - - sort.Sort(languages) - - return languages, nil -} diff --git a/hugolib/page.go b/hugolib/page.go index 537482fb269..676cba762ac 100644 --- a/hugolib/page.go +++ b/hugolib/page.go @@ -23,6 +23,8 @@ import ( "sort" "strings" + "github.com/gohugoio/hugo/hugofs/files" + "github.com/bep/gitmap" "github.com/gohugoio/hugo/helpers" @@ -290,7 +292,9 @@ func (p *pageState) getLayoutDescriptor() output.LayoutDescriptor { switch p.Kind() { case page.KindSection: - section = sections[0] + if len(sections) > 0 { + section = sections[0] + } case page.KindTaxonomyTerm: section = p.getTaxonomyNodeInfo().singular case page.KindTaxonomy: @@ -365,6 +369,7 @@ func (p *pageState) renderResources() (err error) { var toBeDeleted []int for i, r := range p.Resources() { + if _, ok := r.(page.Page); ok { // Pages gets rendered with the owning page but we count them here. p.s.PathSpec.ProcessingStats.Incr(&p.s.PathSpec.ProcessingStats.Pages) @@ -491,14 +496,6 @@ func (p *pageState) addSectionToParent() { p.parent.subSections = append(p.parent.subSections, p) } -func (p *pageState) contentMarkupType() string { - if p.m.markup != "" { - return p.m.markup - - } - return p.File().Ext() -} - func (p *pageState) mapContent(meta *pageMeta) error { s := p.shortcodeState @@ -843,6 +840,7 @@ func (ps pageStatePages) findPagePosByFilnamePrefix(prefix string) int { func (s *Site) sectionsFromFile(fi source.File) []string { dirname := fi.Dir() + dirname = strings.Trim(dirname, helpers.FilePathSeparator) if dirname == "" { return nil @@ -850,7 +848,7 @@ func (s *Site) sectionsFromFile(fi source.File) []string { parts := strings.Split(dirname, helpers.FilePathSeparator) if fii, ok := fi.(*fileInfo); ok { - if fii.bundleTp == bundleLeaf && len(parts) > 0 { + if len(parts) > 0 && fii.FileInfo().Meta().Classifier() == files.ContentClassLeaf { // my-section/mybundle/index.md => my-section return parts[:len(parts)-1] } diff --git a/hugolib/page__meta.go b/hugolib/page__meta.go index d14b9d724f5..551f4797744 100644 --- a/hugolib/page__meta.go +++ b/hugolib/page__meta.go @@ -21,6 +21,8 @@ import ( "strings" "time" + "github.com/gohugoio/hugo/hugofs/files" + "github.com/gohugoio/hugo/common/hugo" "github.com/gohugoio/hugo/related" @@ -591,15 +593,14 @@ func (p *pageMeta) applyDefaultValues() error { } if p.IsNode() { - p.bundleType = "branch" + p.bundleType = files.ContentClassBranch } else { source := p.File() if fi, ok := source.(*fileInfo); ok { - switch fi.bundleTp { - case bundleBranch: - p.bundleType = "branch" - case bundleLeaf: - p.bundleType = "leaf" + class := fi.FileInfo().Meta().Classifier() + switch class { + case files.ContentClassBranch, files.ContentClassLeaf: + p.bundleType = class } } } diff --git a/hugolib/page_permalink_test.go b/hugolib/page_permalink_test.go index 526f9578b5b..ffbb50cd525 100644 --- a/hugolib/page_permalink_test.go +++ b/hugolib/page_permalink_test.go @@ -63,8 +63,9 @@ func TestPermalink(t *testing.T) { } for i, test := range tests { + test := test t.Run(fmt.Sprintf("%s-%d", test.file, i), func(t *testing.T) { - + t.Parallel() cfg, fs := newTestCfg() cfg.Set("uglyURLs", test.uglyURLs) diff --git a/hugolib/page_test.go b/hugolib/page_test.go index 79a8a267f8c..e754a5e4f91 100644 --- a/hugolib/page_test.go +++ b/hugolib/page_test.go @@ -18,6 +18,8 @@ import ( "html/template" "os" + "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/common/loggers" "path/filepath" @@ -29,7 +31,6 @@ import ( "github.com/gohugoio/hugo/resources/page" "github.com/gohugoio/hugo/resources/resource" - "github.com/spf13/afero" "github.com/spf13/viper" @@ -301,6 +302,7 @@ func checkPageTitle(t *testing.T, page page.Page, title string) { } func checkPageContent(t *testing.T, page page.Page, expected string, msg ...interface{}) { + t.Helper() a := normalizeContent(expected) b := normalizeContent(content(page)) if a != b { @@ -387,11 +389,13 @@ func testAllMarkdownEnginesForPages(t *testing.T, continue } - cfg, fs := newTestCfg() + cfg, fs := newTestCfg(func(cfg config.Provider) error { + for k, v := range settings { + cfg.Set(k, v) + } + return nil - for k, v := range settings { - cfg.Set(k, v) - } + }) contentDir := "content" @@ -413,7 +417,10 @@ func testAllMarkdownEnginesForPages(t *testing.T, homePath := fmt.Sprintf("_index.%s", e.ext) writeSource(t, fs, filepath.Join(contentDir, homePath), homePage) - s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{SkipRender: true}) + b := newTestSitesBuilderFromDepsCfg(t, deps.DepsCfg{Fs: fs, Cfg: cfg}).WithNothingAdded() + b.Build(BuildCfg{SkipRender: true}) + + s := b.H.Sites[0] require.Len(t, s.RegularPages(), len(pageSources)) @@ -770,6 +777,9 @@ func TestPageWithLastmodFromGitInfo(t *testing.T) { fs := hugofs.NewFrom(hugofs.Os, cfg) fs.Destination = &afero.MemMapFs{} + wd, err := os.Getwd() + assrt.NoError(err) + cfg.Set("frontmatter", map[string]interface{}{ "lastmod": []string{":git", "lastmod"}, }) @@ -791,19 +801,14 @@ func TestPageWithLastmodFromGitInfo(t *testing.T) { cfg.Set("languages", langConfig) cfg.Set("enableGitInfo", true) - assrt.NoError(loadDefaultSettingsFor(cfg)) - assrt.NoError(loadLanguageSettings(cfg, nil)) - - wd, err := os.Getwd() - assrt.NoError(err) cfg.Set("workingDir", filepath.Join(wd, "testsite")) - h, err := NewHugoSites(deps.DepsCfg{Fs: fs, Cfg: cfg}) + b := newTestSitesBuilderFromDepsCfg(t, deps.DepsCfg{Fs: fs, Cfg: cfg}).WithNothingAdded() - assrt.NoError(err) - assrt.Len(h.Sites, 2) + b.Build(BuildCfg{SkipRender: true}) + h := b.H - require.NoError(t, h.Build(BuildCfg{SkipRender: true})) + assrt.Len(h.Sites, 2) enSite := h.Sites[0] assrt.Len(enSite.RegularPages(), 1) @@ -820,10 +825,10 @@ func TestPageWithLastmodFromGitInfo(t *testing.T) { } func TestPageWithFrontMatterConfig(t *testing.T) { - t.Parallel() - for _, dateHandler := range []string{":filename", ":fileModTime"} { + dateHandler := dateHandler t.Run(fmt.Sprintf("dateHandler=%q", dateHandler), func(t *testing.T) { + t.Parallel() assrt := require.New(t) cfg, fs := newTestCfg() @@ -852,8 +857,10 @@ Content c2fi, err := fs.Source.Stat(c2) assrt.NoError(err) - s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{SkipRender: true}) + b := newTestSitesBuilderFromDepsCfg(t, deps.DepsCfg{Fs: fs, Cfg: cfg}).WithNothingAdded() + b.Build(BuildCfg{SkipRender: true}) + s := b.H.Sites[0] assrt.Len(s.RegularPages(), 2) noSlug := s.RegularPages()[0] @@ -1051,10 +1058,8 @@ func TestPageWithEmoji(t *testing.T) { for _, enableEmoji := range []bool{true, false} { v := viper.New() v.Set("enableEmoji", enableEmoji) - b := newTestSitesBuilder(t) - b.WithViper(v) - b.WithSimpleConfigFile() + b := newTestSitesBuilder(t).WithViper(v) b.WithContent("page-emoji.md", `--- title: "Hugo Smile" @@ -1329,11 +1334,12 @@ func TestShouldBuild(t *testing.T) { // "dot" in path: #1885 and #2110 // disablePathToLower regression: #3374 func TestPathIssues(t *testing.T) { - t.Parallel() for _, disablePathToLower := range []bool{false, true} { for _, uglyURLs := range []bool{false, true} { + disablePathToLower := disablePathToLower + uglyURLs := uglyURLs t.Run(fmt.Sprintf("disablePathToLower=%t,uglyURLs=%t", disablePathToLower, uglyURLs), func(t *testing.T) { - + t.Parallel() cfg, fs := newTestCfg() th := testHelper{cfg, fs, t} diff --git a/hugolib/pagebundler.go b/hugolib/pagebundler.go deleted file mode 100644 index 5149968bcfb..00000000000 --- a/hugolib/pagebundler.go +++ /dev/null @@ -1,206 +0,0 @@ -// Copyright 2019 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugolib - -import ( - "context" - "fmt" - "math" - "path/filepath" - - "github.com/gohugoio/hugo/config" - - _errors "github.com/pkg/errors" - - "golang.org/x/sync/errgroup" -) - -type siteContentProcessor struct { - site *Site - - handleContent contentHandler - - ctx context.Context - - // The input file bundles. - fileBundlesChan chan *bundleDir - - // The input file singles. - fileSinglesChan chan *fileInfo - - // These assets should be just copied to destination. - fileAssetsChan chan pathLangFile - - numWorkers int - - // The output Pages - pagesChan chan *pageState - - // Used for partial rebuilds (aka. live reload) - // Will signal replacement of pages in the site collection. - partialBuild bool -} - -func (s *siteContentProcessor) processBundle(b *bundleDir) { - select { - case s.fileBundlesChan <- b: - case <-s.ctx.Done(): - } -} - -func (s *siteContentProcessor) processSingle(fi *fileInfo) { - select { - case s.fileSinglesChan <- fi: - case <-s.ctx.Done(): - } -} - -func (s *siteContentProcessor) processAsset(asset pathLangFile) { - select { - case s.fileAssetsChan <- asset: - case <-s.ctx.Done(): - } -} - -func newSiteContentProcessor(ctx context.Context, partialBuild bool, s *Site) *siteContentProcessor { - numWorkers := config.GetNumWorkerMultiplier() * 3 - - numWorkers = int(math.Ceil(float64(numWorkers) / float64(len(s.h.Sites)))) - - return &siteContentProcessor{ - ctx: ctx, - partialBuild: partialBuild, - site: s, - handleContent: newHandlerChain(s), - fileBundlesChan: make(chan *bundleDir, numWorkers), - fileSinglesChan: make(chan *fileInfo, numWorkers), - fileAssetsChan: make(chan pathLangFile, numWorkers), - numWorkers: numWorkers, - pagesChan: make(chan *pageState, numWorkers), - } -} - -func (s *siteContentProcessor) closeInput() { - close(s.fileSinglesChan) - close(s.fileBundlesChan) - close(s.fileAssetsChan) -} - -func (s *siteContentProcessor) process(ctx context.Context) error { - g1, ctx := errgroup.WithContext(ctx) - g2, ctx := errgroup.WithContext(ctx) - - // There can be only one of these per site. - g1.Go(func() error { - for p := range s.pagesChan { - if p.s != s.site { - panic(fmt.Sprintf("invalid page site: %v vs %v", p.s, s)) - } - - p.forceRender = s.partialBuild - - if p.forceRender { - s.site.replacePage(p) - } else { - s.site.addPage(p) - } - } - return nil - }) - - for i := 0; i < s.numWorkers; i++ { - g2.Go(func() error { - for { - select { - case f, ok := <-s.fileSinglesChan: - if !ok { - return nil - } - - err := s.readAndConvertContentFile(f) - if err != nil { - return err - } - case <-ctx.Done(): - return ctx.Err() - } - } - }) - - g2.Go(func() error { - for { - select { - case file, ok := <-s.fileAssetsChan: - if !ok { - return nil - } - f, err := s.site.BaseFs.Content.Fs.Open(file.Filename()) - if err != nil { - return _errors.Wrap(err, "failed to open assets file") - } - filename := filepath.Join(s.site.GetTargetLanguageBasePath(), file.Path()) - err = s.site.publish(&s.site.PathSpec.ProcessingStats.Files, filename, f) - f.Close() - if err != nil { - return err - } - case <-ctx.Done(): - return ctx.Err() - } - } - }) - - g2.Go(func() error { - for { - select { - case bundle, ok := <-s.fileBundlesChan: - if !ok { - return nil - } - err := s.readAndConvertContentBundle(bundle) - if err != nil { - return err - } - case <-ctx.Done(): - return ctx.Err() - } - } - }) - } - - err := g2.Wait() - - close(s.pagesChan) - - if err != nil { - return err - } - - if err := g1.Wait(); err != nil { - return err - } - - return nil - -} - -func (s *siteContentProcessor) readAndConvertContentFile(file *fileInfo) error { - ctx := &handlerContext{source: file, pages: s.pagesChan} - return s.handleContent(ctx).err -} - -func (s *siteContentProcessor) readAndConvertContentBundle(bundle *bundleDir) error { - ctx := &handlerContext{bundle: bundle, pages: s.pagesChan} - return s.handleContent(ctx).err -} diff --git a/hugolib/pagebundler_capture.go b/hugolib/pagebundler_capture.go deleted file mode 100644 index 7c01a751dbf..00000000000 --- a/hugolib/pagebundler_capture.go +++ /dev/null @@ -1,773 +0,0 @@ -// Copyright 2017-present The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugolib - -import ( - "errors" - "fmt" - "os" - "path" - "path/filepath" - - "github.com/gohugoio/hugo/config" - - "github.com/gohugoio/hugo/common/loggers" - _errors "github.com/pkg/errors" - - "sort" - "strings" - "sync" - - "github.com/spf13/afero" - - "github.com/gohugoio/hugo/hugofs" - - "github.com/gohugoio/hugo/helpers" - - "golang.org/x/sync/errgroup" - - "github.com/gohugoio/hugo/source" -) - -var errSkipCyclicDir = errors.New("skip potential cyclic dir") - -type capturer struct { - // To prevent symbolic link cycles: Visit same folder only once. - seen map[string]bool - seenMu sync.Mutex - - handler captureResultHandler - - sourceSpec *source.SourceSpec - fs afero.Fs - logger *loggers.Logger - - // Filenames limits the content to process to a list of filenames/directories. - // This is used for partial building in server mode. - filenames []string - - // Used to determine how to handle content changes in server mode. - contentChanges *contentChangeMap - - // Semaphore used to throttle the concurrent sub directory handling. - sem chan bool -} - -func newCapturer( - logger *loggers.Logger, - sourceSpec *source.SourceSpec, - handler captureResultHandler, - contentChanges *contentChangeMap, - filenames ...string) *capturer { - - numWorkers := config.GetNumWorkerMultiplier() - - // TODO(bep) the "index" vs "_index" check/strings should be moved in one place. - isBundleHeader := func(filename string) bool { - base := filepath.Base(filename) - name := helpers.Filename(base) - return IsContentFile(base) && (name == "index" || name == "_index") - } - - // Make sure that any bundle header files are processed before the others. This makes - // sure that any bundle head is processed before its resources. - sort.Slice(filenames, func(i, j int) bool { - a, b := filenames[i], filenames[j] - ac, bc := isBundleHeader(a), isBundleHeader(b) - - if ac { - return true - } - - if bc { - return false - } - - return a < b - }) - - c := &capturer{ - sem: make(chan bool, numWorkers), - handler: handler, - sourceSpec: sourceSpec, - fs: sourceSpec.SourceFs, - logger: logger, - contentChanges: contentChanges, - seen: make(map[string]bool), - filenames: filenames} - - return c -} - -// Captured files and bundles ready to be processed will be passed on to -// these channels. -type captureResultHandler interface { - handleSingles(fis ...*fileInfo) - handleCopyFile(fi pathLangFile) - captureBundlesHandler -} - -type captureBundlesHandler interface { - handleBundles(b *bundleDirs) -} - -type captureResultHandlerChain struct { - handlers []captureBundlesHandler -} - -func (c *captureResultHandlerChain) handleSingles(fis ...*fileInfo) { - for _, h := range c.handlers { - if hh, ok := h.(captureResultHandler); ok { - hh.handleSingles(fis...) - } - } -} -func (c *captureResultHandlerChain) handleBundles(b *bundleDirs) { - for _, h := range c.handlers { - h.handleBundles(b) - } -} - -func (c *captureResultHandlerChain) handleCopyFile(file pathLangFile) { - for _, h := range c.handlers { - if hh, ok := h.(captureResultHandler); ok { - hh.handleCopyFile(file) - } - } -} - -func (c *capturer) capturePartial(filenames ...string) error { - handled := make(map[string]bool) - - for _, filename := range filenames { - dir, resolvedFilename, tp := c.contentChanges.resolveAndRemove(filename) - if handled[resolvedFilename] { - continue - } - - handled[resolvedFilename] = true - - switch tp { - case bundleLeaf: - if err := c.handleDir(resolvedFilename); err != nil { - // Directory may have been deleted. - if !os.IsNotExist(err) { - return err - } - } - case bundleBranch: - if err := c.handleBranchDir(resolvedFilename); err != nil { - // Directory may have been deleted. - if !os.IsNotExist(err) { - return err - } - } - default: - fi, err := c.resolveRealPath(resolvedFilename) - if os.IsNotExist(err) { - // File has been deleted. - continue - } - - // Just in case the owning dir is a new symlink -- this will - // create the proper mapping for it. - c.resolveRealPath(dir) - - f, active := c.newFileInfo(fi, tp) - if active { - c.copyOrHandleSingle(f) - } - } - } - - return nil -} - -func (c *capturer) capture() error { - if len(c.filenames) > 0 { - return c.capturePartial(c.filenames...) - } - - err := c.handleDir(helpers.FilePathSeparator) - if err != nil { - return err - } - - return nil -} - -func (c *capturer) handleNestedDir(dirname string) error { - select { - case c.sem <- true: - var g errgroup.Group - - g.Go(func() error { - defer func() { - <-c.sem - }() - return c.handleDir(dirname) - }) - return g.Wait() - default: - // For deeply nested file trees, waiting for a semaphore wil deadlock. - return c.handleDir(dirname) - } -} - -// This handles a bundle branch and its resources only. This is used -// in server mode on changes. If this dir does not (anymore) represent a bundle -// branch, the handling is upgraded to the full handleDir method. -func (c *capturer) handleBranchDir(dirname string) error { - files, err := c.readDir(dirname) - if err != nil { - - return err - } - - var ( - dirType bundleDirType - ) - - for _, fi := range files { - if !fi.IsDir() { - tp, _ := classifyBundledFile(fi.RealName()) - if dirType == bundleNot { - dirType = tp - } - - if dirType == bundleLeaf { - return c.handleDir(dirname) - } - } - } - - if dirType != bundleBranch { - return c.handleDir(dirname) - } - - dirs := newBundleDirs(bundleBranch, c) - - var secondPass []*fileInfo - - // Handle potential bundle headers first. - for _, fi := range files { - if fi.IsDir() { - continue - } - - tp, isContent := classifyBundledFile(fi.RealName()) - - f, active := c.newFileInfo(fi, tp) - - if !active { - continue - } - - if !f.isOwner() { - if !isContent { - // This is a partial update -- we only care about the files that - // is in this bundle. - secondPass = append(secondPass, f) - } - continue - } - dirs.addBundleHeader(f) - } - - for _, f := range secondPass { - dirs.addBundleFiles(f) - } - - c.handler.handleBundles(dirs) - - return nil - -} - -func (c *capturer) handleDir(dirname string) error { - - files, err := c.readDir(dirname) - if err != nil { - return err - } - - type dirState int - - const ( - dirStateDefault dirState = iota - - dirStateAssetsOnly - dirStateSinglesOnly - ) - - var ( - fileBundleTypes = make([]bundleDirType, len(files)) - - // Start with the assumption that this dir contains only non-content assets (images etc.) - // If that is still true after we had a first look at the list of files, we - // can just copy the files to destination. We will still have to look at the - // sub-folders for potential bundles. - state = dirStateAssetsOnly - - // Start with the assumption that this dir is not a bundle. - // A directory is a bundle if it contains a index content file, - // e.g. index.md (a leaf bundle) or a _index.md (a branch bundle). - bundleType = bundleNot - ) - - /* First check for any content files. - - If there are none, then this is a assets folder only (images etc.) - and we can just plainly copy them to - destination. - - If this is a section with no image etc. or similar, we can just handle it - as it was a single content file. - */ - var hasNonContent, isBranch bool - - for i, fi := range files { - if !fi.IsDir() { - tp, isContent := classifyBundledFile(fi.RealName()) - - fileBundleTypes[i] = tp - if !isBranch { - isBranch = tp == bundleBranch - } - - if isContent { - // This is not a assets-only folder. - state = dirStateDefault - } else { - hasNonContent = true - } - } - } - - if isBranch && !hasNonContent { - // This is a section or similar with no need for any bundle handling. - state = dirStateSinglesOnly - } - - if state > dirStateDefault { - return c.handleNonBundle(dirname, files, state == dirStateSinglesOnly) - } - - var fileInfos = make([]*fileInfo, 0, len(files)) - - for i, fi := range files { - - currentType := bundleNot - - if !fi.IsDir() { - currentType = fileBundleTypes[i] - if bundleType == bundleNot && currentType != bundleNot { - bundleType = currentType - } - } - - if bundleType == bundleNot && currentType != bundleNot { - bundleType = currentType - } - - f, active := c.newFileInfo(fi, currentType) - - if !active { - continue - } - - fileInfos = append(fileInfos, f) - } - - var todo []*fileInfo - - if bundleType != bundleLeaf { - for _, fi := range fileInfos { - if fi.FileInfo().IsDir() { - // Handle potential nested bundles. - if err := c.handleNestedDir(fi.Path()); err != nil { - return err - } - } else if bundleType == bundleNot || (!fi.isOwner() && fi.isContentFile()) { - // Not in a bundle. - c.copyOrHandleSingle(fi) - } else { - // This is a section folder or similar with non-content files in it. - todo = append(todo, fi) - } - } - } else { - todo = fileInfos - } - - if len(todo) == 0 { - return nil - } - - dirs, err := c.createBundleDirs(todo, bundleType) - if err != nil { - return err - } - - // Send the bundle to the next step in the processor chain. - c.handler.handleBundles(dirs) - - return nil -} - -func (c *capturer) handleNonBundle( - dirname string, - fileInfos pathLangFileFis, - singlesOnly bool) error { - - for _, fi := range fileInfos { - if fi.IsDir() { - if err := c.handleNestedDir(fi.Filename()); err != nil { - return err - } - } else { - if singlesOnly { - f, active := c.newFileInfo(fi, bundleNot) - if !active { - continue - } - c.handler.handleSingles(f) - } else { - c.handler.handleCopyFile(fi) - } - } - } - - return nil -} - -func (c *capturer) copyOrHandleSingle(fi *fileInfo) { - if fi.isContentFile() { - c.handler.handleSingles(fi) - } else { - // These do not currently need any further processing. - c.handler.handleCopyFile(fi) - } -} - -func (c *capturer) createBundleDirs(fileInfos []*fileInfo, bundleType bundleDirType) (*bundleDirs, error) { - dirs := newBundleDirs(bundleType, c) - - for _, fi := range fileInfos { - if fi.FileInfo().IsDir() { - var collector func(fis ...*fileInfo) - - if bundleType == bundleBranch { - // All files in the current directory are part of this bundle. - // Trying to include sub folders in these bundles are filled with ambiguity. - collector = func(fis ...*fileInfo) { - for _, fi := range fis { - c.copyOrHandleSingle(fi) - } - } - } else { - // All nested files and directories are part of this bundle. - collector = func(fis ...*fileInfo) { - fileInfos = append(fileInfos, fis...) - } - } - err := c.collectFiles(fi.Path(), collector) - if err != nil { - return nil, err - } - - } else if fi.isOwner() { - // There can be more than one language, so: - // 1. Content files must be attached to its language's bundle. - // 2. Other files must be attached to all languages. - // 3. Every content file needs a bundle header. - dirs.addBundleHeader(fi) - } - } - - for _, fi := range fileInfos { - if fi.FileInfo().IsDir() || fi.isOwner() { - continue - } - - if fi.isContentFile() { - if bundleType != bundleBranch { - dirs.addBundleContentFile(fi) - } - } else { - dirs.addBundleFiles(fi) - } - } - - return dirs, nil -} - -func (c *capturer) collectFiles(dirname string, handleFiles func(fis ...*fileInfo)) error { - - filesInDir, err := c.readDir(dirname) - if err != nil { - return err - } - - for _, fi := range filesInDir { - if fi.IsDir() { - err := c.collectFiles(fi.Filename(), handleFiles) - if err != nil { - return err - } - } else { - f, active := c.newFileInfo(fi, bundleNot) - if active { - handleFiles(f) - } - } - } - - return nil -} - -func (c *capturer) readDir(dirname string) (pathLangFileFis, error) { - if c.sourceSpec.IgnoreFile(dirname) { - return nil, nil - } - - dir, err := c.fs.Open(dirname) - if err != nil { - return nil, err - } - defer dir.Close() - fis, err := dir.Readdir(-1) - if err != nil { - return nil, err - } - - pfis := make(pathLangFileFis, 0, len(fis)) - - for _, fi := range fis { - fip := fi.(pathLangFileFi) - - if !c.sourceSpec.IgnoreFile(fip.Filename()) { - - err := c.resolveRealPathIn(fip) - - if err != nil { - // It may have been deleted in the meantime. - if err == errSkipCyclicDir || os.IsNotExist(err) { - continue - } - return nil, err - } - - pfis = append(pfis, fip) - } - } - - return pfis, nil -} - -func (c *capturer) newFileInfo(fi pathLangFileFi, tp bundleDirType) (*fileInfo, bool) { - f := newFileInfo(c.sourceSpec, "", "", fi, tp) - return f, !f.disabled -} - -type pathLangFile interface { - hugofs.LanguageAnnouncer - hugofs.FilePather -} - -type pathLangFileFi interface { - os.FileInfo - pathLangFile -} - -type pathLangFileFis []pathLangFileFi - -type bundleDirs struct { - tp bundleDirType - // Maps languages to bundles. - bundles map[string]*bundleDir - - // Keeps track of language overrides for non-content files, e.g. logo.en.png. - langOverrides map[string]bool - - c *capturer -} - -func newBundleDirs(tp bundleDirType, c *capturer) *bundleDirs { - return &bundleDirs{tp: tp, bundles: make(map[string]*bundleDir), langOverrides: make(map[string]bool), c: c} -} - -type bundleDir struct { - tp bundleDirType - fi *fileInfo - - resources map[string]*fileInfo -} - -func (b bundleDir) clone() *bundleDir { - b.resources = make(map[string]*fileInfo) - fic := *b.fi - b.fi = &fic - return &b -} - -func newBundleDir(fi *fileInfo, bundleType bundleDirType) *bundleDir { - return &bundleDir{fi: fi, tp: bundleType, resources: make(map[string]*fileInfo)} -} - -func (b *bundleDirs) addBundleContentFile(fi *fileInfo) { - dir, found := b.bundles[fi.Lang()] - if !found { - // Every bundled content file needs a bundle header. - // If one does not exist in its language, we pick the default - // language version, or a random one if that doesn't exist, either. - tl := b.c.sourceSpec.DefaultContentLanguage - ldir, found := b.bundles[tl] - if !found { - // Just pick one. - for _, v := range b.bundles { - ldir = v - break - } - } - - if ldir == nil { - panic(fmt.Sprintf("bundle not found for file %q", fi.Filename())) - } - - dir = ldir.clone() - dir.fi.overriddenLang = fi.Lang() - b.bundles[fi.Lang()] = dir - } - - dir.resources[fi.Path()] = fi -} - -func (b *bundleDirs) addBundleFiles(fi *fileInfo) { - dir := filepath.ToSlash(fi.Dir()) - p := dir + fi.TranslationBaseName() + "." + fi.Ext() - for lang, bdir := range b.bundles { - key := path.Join(lang, p) - - // Given mypage.de.md (German translation) and mypage.md we pick the most - // specific for that language. - if fi.Lang() == lang || !b.langOverrides[key] { - bdir.resources[key] = fi - } - b.langOverrides[key] = true - } -} - -func (b *bundleDirs) addBundleHeader(fi *fileInfo) { - b.bundles[fi.Lang()] = newBundleDir(fi, b.tp) -} - -func (c *capturer) isSeen(dirname string) bool { - c.seenMu.Lock() - defer c.seenMu.Unlock() - seen := c.seen[dirname] - c.seen[dirname] = true - if seen { - c.logger.INFO.Printf("Content dir %q already processed; skipped to avoid infinite recursion.", dirname) - return true - - } - return false -} - -func (c *capturer) resolveRealPath(path string) (pathLangFileFi, error) { - fileInfo, err := c.lstatIfPossible(path) - if err != nil { - return nil, err - } - return fileInfo, c.resolveRealPathIn(fileInfo) -} - -func (c *capturer) resolveRealPathIn(fileInfo pathLangFileFi) error { - - basePath := fileInfo.BaseDir() - path := fileInfo.Filename() - - realPath := path - - if fileInfo.Mode()&os.ModeSymlink == os.ModeSymlink { - link, err := filepath.EvalSymlinks(path) - if err != nil { - return _errors.Wrapf(err, "Cannot read symbolic link %q, error was:", path) - } - - // This is a file on the outside of any base fs, so we have to use the os package. - sfi, err := os.Stat(link) - if err != nil { - return _errors.Wrapf(err, "Cannot stat %q, error was:", link) - } - - // TODO(bep) improve all of this. - if a, ok := fileInfo.(*hugofs.LanguageFileInfo); ok { - a.FileInfo = sfi - } - - realPath = link - - if realPath != path && sfi.IsDir() && c.isSeen(realPath) { - // Avoid cyclic symlinks. - // Note that this may prevent some uses that isn't cyclic and also - // potential useful, but this implementation is both robust and simple: - // We stop at the first directory that we have seen before, e.g. - // /content/blog will only be processed once. - return errSkipCyclicDir - } - - if c.contentChanges != nil { - // Keep track of symbolic links in watch mode. - var from, to string - if sfi.IsDir() { - from = realPath - to = path - - if !strings.HasSuffix(to, helpers.FilePathSeparator) { - to = to + helpers.FilePathSeparator - } - if !strings.HasSuffix(from, helpers.FilePathSeparator) { - from = from + helpers.FilePathSeparator - } - - if !strings.HasSuffix(basePath, helpers.FilePathSeparator) { - basePath = basePath + helpers.FilePathSeparator - } - - if strings.HasPrefix(from, basePath) { - // With symbolic links inside /content we need to keep - // a reference to both. This may be confusing with --navigateToChanged - // but the user has chosen this him or herself. - c.contentChanges.addSymbolicLinkMapping(from, from) - } - - } else { - from = realPath - to = path - } - - c.contentChanges.addSymbolicLinkMapping(from, to) - } - } - - return nil -} - -func (c *capturer) lstatIfPossible(path string) (pathLangFileFi, error) { - fi, err := helpers.LstatIfPossible(c.fs, path) - if err != nil { - return nil, err - } - return fi.(pathLangFileFi), nil -} diff --git a/hugolib/pagebundler_capture_test.go b/hugolib/pagebundler_capture_test.go deleted file mode 100644 index b6d9822af86..00000000000 --- a/hugolib/pagebundler_capture_test.go +++ /dev/null @@ -1,272 +0,0 @@ -// Copyright 2017-present The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugolib - -import ( - "fmt" - "os" - "path" - "path/filepath" - "sort" - - "github.com/gohugoio/hugo/common/loggers" - - "runtime" - "strings" - "sync" - "testing" - - "github.com/gohugoio/hugo/helpers" - "github.com/gohugoio/hugo/source" - "github.com/stretchr/testify/require" -) - -type storeFilenames struct { - sync.Mutex - filenames []string - copyNames []string - dirKeys []string -} - -func (s *storeFilenames) handleSingles(fis ...*fileInfo) { - s.Lock() - defer s.Unlock() - for _, fi := range fis { - s.filenames = append(s.filenames, filepath.ToSlash(fi.Filename())) - } -} - -func (s *storeFilenames) handleBundles(d *bundleDirs) { - s.Lock() - defer s.Unlock() - var keys []string - for _, b := range d.bundles { - res := make([]string, len(b.resources)) - i := 0 - for _, r := range b.resources { - res[i] = path.Join(r.Lang(), filepath.ToSlash(r.Filename())) - i++ - } - sort.Strings(res) - keys = append(keys, path.Join("__bundle", b.fi.Lang(), filepath.ToSlash(b.fi.Filename()), "resources", strings.Join(res, "|"))) - } - s.dirKeys = append(s.dirKeys, keys...) -} - -func (s *storeFilenames) handleCopyFile(file pathLangFile) { - s.Lock() - defer s.Unlock() - s.copyNames = append(s.copyNames, filepath.ToSlash(file.Filename())) -} - -func (s *storeFilenames) sortedStr() string { - s.Lock() - defer s.Unlock() - sort.Strings(s.filenames) - sort.Strings(s.dirKeys) - sort.Strings(s.copyNames) - return "\nF:\n" + strings.Join(s.filenames, "\n") + "\nD:\n" + strings.Join(s.dirKeys, "\n") + - "\nC:\n" + strings.Join(s.copyNames, "\n") + "\n" -} - -func TestPageBundlerCaptureSymlinks(t *testing.T) { - if runtime.GOOS == "windows" && os.Getenv("CI") == "" { - t.Skip("Skip TestPageBundlerCaptureSymlinks as os.Symlink needs administrator rights on Windows") - } - - assert := require.New(t) - ps, clean, workDir := newTestBundleSymbolicSources(t) - sourceSpec := source.NewSourceSpec(ps, ps.BaseFs.Content.Fs) - defer clean() - - fileStore := &storeFilenames{} - logger := loggers.NewErrorLogger() - c := newCapturer(logger, sourceSpec, fileStore, nil) - - assert.NoError(c.capture()) - - expected := ` -F: -/base/a/page_s.md -/base/a/regular.md -/base/symbolic1/s1.md -/base/symbolic1/s2.md -/base/symbolic3/circus/a/page_s.md -/base/symbolic3/circus/a/regular.md -D: -__bundle/en/base/symbolic2/a1/index.md/resources/en/base/symbolic2/a1/logo.png|en/base/symbolic2/a1/page.md -C: -/base/symbolic3/s1.png -/base/symbolic3/s2.png -` - - got := strings.Replace(fileStore.sortedStr(), filepath.ToSlash(workDir), "", -1) - got = strings.Replace(got, "//", "/", -1) - - if expected != got { - diff := helpers.DiffStringSlices(strings.Fields(expected), strings.Fields(got)) - t.Log(got) - t.Fatalf("Failed:\n%s", diff) - } -} - -func TestPageBundlerCaptureBasic(t *testing.T) { - t.Parallel() - - assert := require.New(t) - fs, cfg := newTestBundleSources(t) - assert.NoError(loadDefaultSettingsFor(cfg)) - assert.NoError(loadLanguageSettings(cfg, nil)) - ps, err := helpers.NewPathSpec(fs, cfg) - assert.NoError(err) - - sourceSpec := source.NewSourceSpec(ps, ps.BaseFs.Content.Fs) - - fileStore := &storeFilenames{} - - c := newCapturer(loggers.NewErrorLogger(), sourceSpec, fileStore, nil) - - assert.NoError(c.capture()) - - expected := ` -F: -/work/base/_1.md -/work/base/a/1.md -/work/base/a/2.md -/work/base/assets/pages/mypage.md -D: -__bundle/en/work/base/_index.md/resources/en/work/base/_1.png -__bundle/en/work/base/a/b/index.md/resources/en/work/base/a/b/ab1.md -__bundle/en/work/base/b/my-bundle/index.md/resources/en/work/base/b/my-bundle/1.md|en/work/base/b/my-bundle/2.md|en/work/base/b/my-bundle/c/logo.png|en/work/base/b/my-bundle/custom-mime.bep|en/work/base/b/my-bundle/sunset1.jpg|en/work/base/b/my-bundle/sunset2.jpg -__bundle/en/work/base/c/bundle/index.md/resources/en/work/base/c/bundle/logo-은행.png -__bundle/en/work/base/root/index.md/resources/en/work/base/root/1.md|en/work/base/root/c/logo.png -C: -/work/base/assets/pic1.png -/work/base/assets/pic2.png -/work/base/images/hugo-logo.png -` - - got := fileStore.sortedStr() - - if expected != got { - diff := helpers.DiffStringSlices(strings.Fields(expected), strings.Fields(got)) - t.Log(got) - t.Fatalf("Failed:\n%s", diff) - } -} - -func TestPageBundlerCaptureMultilingual(t *testing.T) { - t.Parallel() - - assert := require.New(t) - fs, cfg := newTestBundleSourcesMultilingual(t) - assert.NoError(loadDefaultSettingsFor(cfg)) - assert.NoError(loadLanguageSettings(cfg, nil)) - - ps, err := helpers.NewPathSpec(fs, cfg) - assert.NoError(err) - - sourceSpec := source.NewSourceSpec(ps, ps.BaseFs.Content.Fs) - fileStore := &storeFilenames{} - c := newCapturer(loggers.NewErrorLogger(), sourceSpec, fileStore, nil) - - assert.NoError(c.capture()) - - expected := ` -F: -/work/base/1s/mypage.md -/work/base/1s/mypage.nn.md -/work/base/bb/_1.md -/work/base/bb/_1.nn.md -/work/base/bb/en.md -/work/base/bc/page.md -/work/base/bc/page.nn.md -/work/base/be/_index.md -/work/base/be/page.md -/work/base/be/page.nn.md -D: -__bundle/en/work/base/bb/_index.md/resources/en/work/base/bb/a.png|en/work/base/bb/b.png|nn/work/base/bb/c.nn.png -__bundle/en/work/base/bc/_index.md/resources/en/work/base/bc/logo-bc.png -__bundle/en/work/base/bd/index.md/resources/en/work/base/bd/page.md -__bundle/en/work/base/bf/my-bf-bundle/index.md/resources/en/work/base/bf/my-bf-bundle/page.md -__bundle/en/work/base/lb/index.md/resources/en/work/base/lb/1.md|en/work/base/lb/2.md|en/work/base/lb/c/d/deep.png|en/work/base/lb/c/logo.png|en/work/base/lb/c/one.png|en/work/base/lb/c/page.md -__bundle/nn/work/base/bb/_index.nn.md/resources/en/work/base/bb/a.png|nn/work/base/bb/b.nn.png|nn/work/base/bb/c.nn.png -__bundle/nn/work/base/bd/index.md/resources/nn/work/base/bd/page.nn.md -__bundle/nn/work/base/bf/my-bf-bundle/index.nn.md/resources -__bundle/nn/work/base/lb/index.nn.md/resources/en/work/base/lb/c/d/deep.png|en/work/base/lb/c/one.png|nn/work/base/lb/2.nn.md|nn/work/base/lb/c/logo.nn.png -C: -/work/base/1s/mylogo.png -/work/base/bb/b/d.nn.png -` - - got := fileStore.sortedStr() - - if expected != got { - diff := helpers.DiffStringSlices(strings.Fields(expected), strings.Fields(got)) - t.Log(got) - t.Fatalf("Failed:\n%s", strings.Join(diff, "\n")) - } - -} - -type noOpFileStore int - -func (noOpFileStore) handleSingles(fis ...*fileInfo) {} -func (noOpFileStore) handleBundles(b *bundleDirs) {} -func (noOpFileStore) handleCopyFile(file pathLangFile) {} - -func BenchmarkPageBundlerCapture(b *testing.B) { - capturers := make([]*capturer, b.N) - - for i := 0; i < b.N; i++ { - cfg, fs := newTestCfg() - ps, _ := helpers.NewPathSpec(fs, cfg) - sourceSpec := source.NewSourceSpec(ps, fs.Source) - - base := fmt.Sprintf("base%d", i) - for j := 1; j <= 5; j++ { - js := fmt.Sprintf("j%d", j) - writeSource(b, fs, filepath.Join(base, js, "index.md"), "content") - writeSource(b, fs, filepath.Join(base, js, "logo1.png"), "content") - writeSource(b, fs, filepath.Join(base, js, "sub", "logo2.png"), "content") - writeSource(b, fs, filepath.Join(base, js, "section", "_index.md"), "content") - writeSource(b, fs, filepath.Join(base, js, "section", "logo.png"), "content") - writeSource(b, fs, filepath.Join(base, js, "section", "sub", "logo.png"), "content") - - for k := 1; k <= 5; k++ { - ks := fmt.Sprintf("k%d", k) - writeSource(b, fs, filepath.Join(base, js, ks, "logo1.png"), "content") - writeSource(b, fs, filepath.Join(base, js, "section", ks, "logo.png"), "content") - } - } - - for i := 1; i <= 5; i++ { - writeSource(b, fs, filepath.Join(base, "assetsonly", fmt.Sprintf("image%d.png", i)), "image") - } - - for i := 1; i <= 5; i++ { - writeSource(b, fs, filepath.Join(base, "contentonly", fmt.Sprintf("c%d.md", i)), "content") - } - - capturers[i] = newCapturer(loggers.NewErrorLogger(), sourceSpec, new(noOpFileStore), nil, base) - } - - b.ResetTimer() - for i := 0; i < b.N; i++ { - err := capturers[i].capture() - if err != nil { - b.Fatal(err) - } - } -} diff --git a/hugolib/pagebundler_handlers.go b/hugolib/pagebundler_handlers.go deleted file mode 100644 index e745a04f2ac..00000000000 --- a/hugolib/pagebundler_handlers.go +++ /dev/null @@ -1,305 +0,0 @@ -// Copyright 2019 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugolib - -import ( - "errors" - "fmt" - "path/filepath" - - "github.com/gohugoio/hugo/common/hugio" - - "strings" - - "github.com/gohugoio/hugo/resources" - "github.com/gohugoio/hugo/resources/resource" -) - -var ( - // This should be the only list of valid extensions for content files. - contentFileExtensions = []string{ - "html", "htm", - "mdown", "markdown", "md", - "asciidoc", "adoc", "ad", - "rest", "rst", - "mmark", - "org", - "pandoc", "pdc"} - - contentFileExtensionsSet map[string]bool -) - -func init() { - contentFileExtensionsSet = make(map[string]bool) - for _, ext := range contentFileExtensions { - contentFileExtensionsSet[ext] = true - } -} - -func newHandlerChain(s *Site) contentHandler { - c := &contentHandlers{s: s} - - contentFlow := c.parsePage( - c.handlePageContent(), - ) - - c.rootHandler = c.processFirstMatch( - contentFlow, - - // Creates a file resource (image, CSS etc.) if there is a parent - // page set on the current context. - c.createResource(), - - // Everything that isn't handled above, will just be copied - // to destination. - c.copyFile(), - ) - - return c.rootHandler - -} - -type contentHandlers struct { - s *Site - rootHandler contentHandler -} - -func (c *contentHandlers) processFirstMatch(handlers ...contentHandler) func(ctx *handlerContext) handlerResult { - return func(ctx *handlerContext) handlerResult { - for _, h := range handlers { - res := h(ctx) - if res.handled || res.err != nil { - return res - } - } - return handlerResult{err: errors.New("no matching handler found")} - } -} - -type handlerContext struct { - // These are the pages stored in Site. - pages chan<- *pageState - - doNotAddToSiteCollections bool - - currentPage *pageState - parentPage *pageState - - bundle *bundleDir - - source *fileInfo - - // Relative path to the target. - target string -} - -func (c *handlerContext) ext() string { - if c.currentPage != nil { - return c.currentPage.contentMarkupType() - } - - if c.bundle != nil { - return c.bundle.fi.Ext() - } else { - return c.source.Ext() - } -} - -func (c *handlerContext) targetPath() string { - if c.target != "" { - return c.target - } - - return c.source.Filename() -} - -func (c *handlerContext) file() *fileInfo { - if c.bundle != nil { - return c.bundle.fi - } - - return c.source -} - -// Create a copy with the current context as its parent. -func (c handlerContext) childCtx(fi *fileInfo) *handlerContext { - if c.currentPage == nil { - panic("Need a Page to create a child context") - } - - c.target = strings.TrimPrefix(fi.Path(), c.bundle.fi.Dir()) - c.source = fi - - c.doNotAddToSiteCollections = c.bundle != nil && c.bundle.tp != bundleBranch - - c.bundle = nil - - c.parentPage = c.currentPage - c.currentPage = nil - - return &c -} - -func (c *handlerContext) supports(exts ...string) bool { - ext := c.ext() - for _, s := range exts { - if s == ext { - return true - } - } - - return false -} - -func (c *handlerContext) isContentFile() bool { - return contentFileExtensionsSet[c.ext()] -} - -type ( - handlerResult struct { - err error - handled bool - result interface{} - } - - contentHandler func(ctx *handlerContext) handlerResult -) - -var ( - notHandled handlerResult -) - -func (c *contentHandlers) parsePage(h contentHandler) contentHandler { - return func(ctx *handlerContext) handlerResult { - if !ctx.isContentFile() { - return notHandled - } - - result := handlerResult{handled: true} - fi := ctx.file() - - content := func() (hugio.ReadSeekCloser, error) { - f, err := fi.Open() - if err != nil { - return nil, fmt.Errorf("failed to open content file %q: %s", fi.Filename(), err) - } - return f, nil - } - - ps, err := newPageWithContent(fi, c.s, ctx.parentPage != nil, content) - if err != nil { - return handlerResult{err: err} - } - - if !c.s.shouldBuild(ps) { - if !ctx.doNotAddToSiteCollections { - ctx.pages <- ps - } - return result - } - - ctx.currentPage = ps - - if ctx.bundle != nil { - // Add the bundled files - for _, fi := range ctx.bundle.resources { - childCtx := ctx.childCtx(fi) - res := c.rootHandler(childCtx) - if res.err != nil { - return res - } - if res.result != nil { - switch resv := res.result.(type) { - case *pageState: - resv.m.resourcePath = filepath.ToSlash(childCtx.target) - resv.parent = ps - ps.addResources(resv) - case resource.Resource: - ps.addResources(resv) - - default: - panic("Unknown type") - } - } - } - } - - return h(ctx) - } -} - -func (c *contentHandlers) handlePageContent() contentHandler { - return func(ctx *handlerContext) handlerResult { - p := ctx.currentPage - - if !ctx.doNotAddToSiteCollections { - ctx.pages <- p - } - - return handlerResult{handled: true, result: p} - } -} - -func (c *contentHandlers) createResource() contentHandler { - return func(ctx *handlerContext) handlerResult { - if ctx.parentPage == nil { - return notHandled - } - - // TODO(bep) consolidate with multihost logic + clean up - outputFormats := ctx.parentPage.m.outputFormats() - seen := make(map[string]bool) - var targetBasePaths []string - // Make sure bundled resources are published to all of the ouptput formats' - // sub paths. - for _, f := range outputFormats { - p := f.Path - if seen[p] { - continue - } - seen[p] = true - targetBasePaths = append(targetBasePaths, p) - - } - - resource, err := c.s.ResourceSpec.New( - resources.ResourceSourceDescriptor{ - TargetPaths: ctx.parentPage.getTargetPaths, - SourceFile: ctx.source, - RelTargetFilename: ctx.target, - TargetBasePaths: targetBasePaths, - }) - - return handlerResult{err: err, handled: true, result: resource} - } -} - -func (c *contentHandlers) copyFile() contentHandler { - return func(ctx *handlerContext) handlerResult { - f, err := c.s.BaseFs.Content.Fs.Open(ctx.source.Filename()) - if err != nil { - err := fmt.Errorf("failed to open file in copyFile: %s", err) - return handlerResult{err: err} - } - - target := ctx.targetPath() - - defer f.Close() - if err := c.s.publish(&c.s.PathSpec.ProcessingStats.Files, target, f); err != nil { - return handlerResult{err: err} - } - - return handlerResult{handled: true} - } -} diff --git a/hugolib/pagebundler_test.go b/hugolib/pagebundler_test.go index 79e0a107d60..f5a6d9a84e3 100644 --- a/hugolib/pagebundler_test.go +++ b/hugolib/pagebundler_test.go @@ -16,18 +16,18 @@ package hugolib import ( "os" "path" - "runtime" + "regexp" "strings" "testing" + "github.com/gohugoio/hugo/hugofs" + "github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/resources/page" - "github.com/gohugoio/hugo/helpers" - "io" - "github.com/spf13/afero" + "github.com/gohugoio/hugo/htesting" "github.com/gohugoio/hugo/media" @@ -36,7 +36,6 @@ import ( "fmt" "github.com/gohugoio/hugo/deps" - "github.com/gohugoio/hugo/hugofs" "github.com/spf13/viper" "github.com/stretchr/testify/require" @@ -54,8 +53,11 @@ func TestPageBundlerSiteRegular(t *testing.T) { if baseURLPathId == "" { baseURLPathId = "NONE" } + ugly := ugly + canonify := canonify t.Run(fmt.Sprintf("ugly=%t,canonify=%t,path=%s", ugly, canonify, baseURLPathId), func(t *testing.T) { + t.Parallel() baseURL := baseBaseURL + baseURLPath relURLBase := baseURLPath if canonify { @@ -65,14 +67,12 @@ func TestPageBundlerSiteRegular(t *testing.T) { fs, cfg := newTestBundleSources(t) cfg.Set("baseURL", baseURL) cfg.Set("canonifyURLs", canonify) - assert.NoError(loadDefaultSettingsFor(cfg)) - assert.NoError(loadLanguageSettings(cfg, nil)) cfg.Set("permalinks", map[string]string{ "a": ":sections/:filename", "b": ":year/:slug/", "c": ":sections/:slug", - "": ":filename/", + "/": ":filename/", }) cfg.Set("outputFormats", map[string]interface{}{ @@ -92,9 +92,11 @@ func TestPageBundlerSiteRegular(t *testing.T) { cfg.Set("uglyURLs", ugly) - s := buildSingleSite(t, deps.DepsCfg{Logger: loggers.NewErrorLogger(), Fs: fs, Cfg: cfg}, BuildCfg{}) + b := newTestSitesBuilderFromDepsCfg(t, deps.DepsCfg{Logger: loggers.NewErrorLogger(), Fs: fs, Cfg: cfg}).WithNothingAdded() + + b.Build(BuildCfg{}) - th := testHelper{s.Cfg, s.Fs, t} + s := b.H.Sites[0] assert.Len(s.RegularPages(), 8) @@ -127,22 +129,22 @@ func TestPageBundlerSiteRegular(t *testing.T) { // Check both output formats rel, filename := relFilename("/a/1/", "index.html") - th.assertFileContent(filepath.Join("/work/public", filename), + b.AssertFileContent(filepath.Join("/work/public", filename), "TheContent", "Single RelPermalink: "+rel, ) rel, filename = relFilename("/cpath/a/1/", "cindex.html") - th.assertFileContent(filepath.Join("/work/public", filename), + b.AssertFileContent(filepath.Join("/work/public", filename), "TheContent", "Single RelPermalink: "+rel, ) - th.assertFileContent(filepath.FromSlash("/work/public/images/hugo-logo.png"), "content") + b.AssertFileContent(filepath.FromSlash("/work/public/images/hugo-logo.png"), "content") // This should be just copied to destination. - th.assertFileContent(filepath.FromSlash("/work/public/assets/pic1.png"), "content") + b.AssertFileContent(filepath.FromSlash("/work/public/assets/pic1.png"), "content") leafBundle1 := s.getPage(page.KindPage, "b/my-bundle/index.md") assert.NotNil(leafBundle1) @@ -159,8 +161,8 @@ func TestPageBundlerSiteRegular(t *testing.T) { assert.NotNil(rootBundle) assert.True(rootBundle.Parent().IsHome()) if !ugly { - th.assertFileContent(filepath.FromSlash("/work/public/root/index.html"), "Single RelPermalink: "+relURLBase+"/root/") - th.assertFileContent(filepath.FromSlash("/work/public/cpath/root/cindex.html"), "Single RelPermalink: "+relURLBase+"/cpath/root/") + b.AssertFileContent(filepath.FromSlash("/work/public/root/index.html"), "Single RelPermalink: "+relURLBase+"/root/") + b.AssertFileContent(filepath.FromSlash("/work/public/cpath/root/cindex.html"), "Single RelPermalink: "+relURLBase+"/cpath/root/") } leafBundle2 := s.getPage(page.KindPage, "a/b/index.md") @@ -172,6 +174,7 @@ func TestPageBundlerSiteRegular(t *testing.T) { assert.Len(pageResources, 2) firstPage := pageResources[0].(page.Page) secondPage := pageResources[1].(page.Page) + assert.Equal(filepath.FromSlash("/work/base/b/my-bundle/1.md"), firstPage.File().Filename(), secondPage.File().Filename()) assert.Contains(content(firstPage), "TheContent") assert.Equal(6, len(leafBundle1.Resources())) @@ -201,17 +204,17 @@ func TestPageBundlerSiteRegular(t *testing.T) { } if ugly { - th.assertFileContent("/work/public/2017/pageslug.html", + b.AssertFileContent("/work/public/2017/pageslug.html", relPermalinker("Single RelPermalink: %s/2017/pageslug.html"), permalinker("Single Permalink: %s/2017/pageslug.html"), relPermalinker("Sunset RelPermalink: %s/2017/pageslug/sunset1.jpg"), permalinker("Sunset Permalink: %s/2017/pageslug/sunset1.jpg")) } else { - th.assertFileContent("/work/public/2017/pageslug/index.html", + b.AssertFileContent("/work/public/2017/pageslug/index.html", relPermalinker("Sunset RelPermalink: %s/2017/pageslug/sunset1.jpg"), permalinker("Sunset Permalink: %s/2017/pageslug/sunset1.jpg")) - th.assertFileContent("/work/public/cpath/2017/pageslug/cindex.html", + b.AssertFileContent("/work/public/cpath/2017/pageslug/cindex.html", relPermalinker("Single RelPermalink: %s/cpath/2017/pageslug/"), relPermalinker("Short Sunset RelPermalink: %s/cpath/2017/pageslug/sunset2.jpg"), relPermalinker("Sunset RelPermalink: %s/cpath/2017/pageslug/sunset1.jpg"), @@ -219,15 +222,15 @@ func TestPageBundlerSiteRegular(t *testing.T) { ) } - th.assertFileContent(filepath.FromSlash("/work/public/2017/pageslug/c/logo.png"), "content") - th.assertFileContent(filepath.FromSlash("/work/public/cpath/2017/pageslug/c/logo.png"), "content") - th.assertFileNotExist("/work/public/cpath/cpath/2017/pageslug/c/logo.png") + b.AssertFileContent(filepath.FromSlash("/work/public/2017/pageslug/c/logo.png"), "content") + b.AssertFileContent(filepath.FromSlash("/work/public/cpath/2017/pageslug/c/logo.png"), "content") + assert.False(b.CheckExists("/work/public/cpath/cpath/2017/pageslug/c/logo.png")) // Custom media type defined in site config. assert.Len(leafBundle1.Resources().ByType("bepsays"), 1) if ugly { - th.assertFileContent(filepath.FromSlash("/work/public/2017/pageslug.html"), + b.AssertFileContent(filepath.FromSlash("/work/public/2017/pageslug.html"), "TheContent", relPermalinker("Sunset RelPermalink: %s/2017/pageslug/sunset1.jpg"), permalinker("Sunset Permalink: %s/2017/pageslug/sunset1.jpg"), @@ -245,19 +248,19 @@ func TestPageBundlerSiteRegular(t *testing.T) { ) // https://github.com/gohugoio/hugo/issues/5882 - th.assertFileContent( + b.AssertFileContent( filepath.FromSlash("/work/public/2017/pageslug.html"), "0: Page RelPermalink: |") - th.assertFileContent(filepath.FromSlash("/work/public/cpath/2017/pageslug.html"), "TheContent") + b.AssertFileContent(filepath.FromSlash("/work/public/cpath/2017/pageslug.html"), "TheContent") // 은행 - th.assertFileContent(filepath.FromSlash("/work/public/c/은행/logo-은행.png"), "은행 PNG") + b.AssertFileContent(filepath.FromSlash("/work/public/c/은행/logo-은행.png"), "은행 PNG") } else { - th.assertFileContent(filepath.FromSlash("/work/public/2017/pageslug/index.html"), "TheContent") - th.assertFileContent(filepath.FromSlash("/work/public/cpath/2017/pageslug/cindex.html"), "TheContent") - th.assertFileContent(filepath.FromSlash("/work/public/2017/pageslug/index.html"), "Single Title") - th.assertFileContent(filepath.FromSlash("/work/public/root/index.html"), "Single Title") + b.AssertFileContent(filepath.FromSlash("/work/public/2017/pageslug/index.html"), "TheContent") + b.AssertFileContent(filepath.FromSlash("/work/public/cpath/2017/pageslug/cindex.html"), "TheContent") + b.AssertFileContent(filepath.FromSlash("/work/public/2017/pageslug/index.html"), "Single Title") + b.AssertFileContent(filepath.FromSlash("/work/public/root/index.html"), "Single Title") } @@ -272,25 +275,26 @@ func TestPageBundlerSiteMultilingual(t *testing.T) { t.Parallel() for _, ugly := range []bool{false, true} { + ugly := ugly t.Run(fmt.Sprintf("ugly=%t", ugly), func(t *testing.T) { - + t.Parallel() assert := require.New(t) fs, cfg := newTestBundleSourcesMultilingual(t) cfg.Set("uglyURLs", ugly) - assert.NoError(loadDefaultSettingsFor(cfg)) - assert.NoError(loadLanguageSettings(cfg, nil)) - sites, err := NewHugoSites(deps.DepsCfg{Fs: fs, Cfg: cfg}) - assert.NoError(err) - assert.Equal(2, len(sites.Sites)) + b := newTestSitesBuilderFromDepsCfg(t, deps.DepsCfg{Fs: fs, Cfg: cfg}).WithNothingAdded() + b.Build(BuildCfg{SkipRender: true}) - assert.NoError(sites.Build(BuildCfg{})) + sites := b.H + + assert.Equal(2, len(sites.Sites)) s := sites.Sites[0] assert.Equal(8, len(s.RegularPages())) assert.Equal(16, len(s.Pages())) + //dumpPages(s.AllPages()...) assert.Equal(31, len(s.AllPages())) bundleWithSubPath := s.getPage(page.KindPage, "lb/index") @@ -358,15 +362,12 @@ func TestMultilingualDisableLanguage(t *testing.T) { fs, cfg := newTestBundleSourcesMultilingual(t) cfg.Set("disableLanguages", []string{"nn"}) - assert.NoError(loadDefaultSettingsFor(cfg)) - assert.NoError(loadLanguageSettings(cfg, nil)) + b := newTestSitesBuilderFromDepsCfg(t, deps.DepsCfg{Fs: fs, Cfg: cfg}).WithNothingAdded() + b.Build(BuildCfg{}) + sites := b.H - sites, err := NewHugoSites(deps.DepsCfg{Fs: fs, Cfg: cfg}) - assert.NoError(err) assert.Equal(1, len(sites.Sites)) - assert.NoError(sites.Build(BuildCfg{})) - s := sites.Sites[0] assert.Equal(8, len(s.RegularPages())) @@ -383,20 +384,98 @@ func TestMultilingualDisableLanguage(t *testing.T) { } func TestPageBundlerSiteWitSymbolicLinksInContent(t *testing.T) { - if runtime.GOOS == "windows" && os.Getenv("CI") == "" { - t.Skip("Skip TestPageBundlerSiteWitSymbolicLinksInContent as os.Symlink needs administrator rights on Windows") - } + skipSymlink(t) + + wd, _ := os.Getwd() + defer func() { + os.Chdir(wd) + }() assert := require.New(t) - ps, clean, workDir := newTestBundleSymbolicSources(t) + // We need to use the OS fs for this. + cfg := viper.New() + fs := hugofs.NewFrom(hugofs.Os, cfg) + + workDir, clean, err := htesting.CreateTempDir(hugofs.Os, "hugosym") + assert.NoError(err) + + contentDirName := "content" + + contentDir := filepath.Join(workDir, contentDirName) + assert.NoError(os.MkdirAll(filepath.Join(contentDir, "a"), 0777)) + + for i := 1; i <= 3; i++ { + assert.NoError(os.MkdirAll(filepath.Join(workDir, fmt.Sprintf("symcontent%d", i)), 0777)) + } + + assert.NoError(os.MkdirAll(filepath.Join(workDir, "symcontent2", "a1"), 0777)) + + // Symlinked sections inside content. + os.Chdir(contentDir) + for i := 1; i <= 3; i++ { + assert.NoError(os.Symlink(filepath.FromSlash(fmt.Sprintf(("../symcontent%d"), i)), fmt.Sprintf("symbolic%d", i))) + } + + assert.NoError(os.Chdir(filepath.Join(contentDir, "a"))) + + // Create a symlink to one single content file + assert.NoError(os.Symlink(filepath.FromSlash("../../symcontent2/a1/page.md"), "page_s.md")) + + assert.NoError(os.Chdir(filepath.FromSlash("../../symcontent3"))) + + // Create a circular symlink. Will print some warnings. + assert.NoError(os.Symlink(filepath.Join("..", contentDirName), filepath.FromSlash("circus"))) + + assert.NoError(os.Chdir(workDir)) + defer clean() - cfg := ps.Cfg - fs := ps.Fs + cfg.Set("workingDir", workDir) + cfg.Set("contentDir", contentDirName) + cfg.Set("baseURL", "https://example.com") + + layout := `{{ .Title }}|{{ .Content }}` + pageContent := `--- +slug: %s +date: 2017-10-09 +--- + +TheContent. +` + + b := newTestSitesBuilderFromDepsCfg(t, deps.DepsCfg{ + Fs: fs, + Cfg: cfg, + }) - s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg, Logger: loggers.NewErrorLogger()}, BuildCfg{}) + b.WithTemplates( + "_default/single.html", layout, + "_default/list.html", layout, + ) - th := testHelper{s.Cfg, s.Fs, t} + b.WithContent( + "a/regular.md", fmt.Sprintf(pageContent, "a1"), + ) + + b.WithSourceFile( + "symcontent1/s1.md", fmt.Sprintf(pageContent, "s1"), + "symcontent1/s2.md", fmt.Sprintf(pageContent, "s2"), + // Regular files inside symlinked folder. + "symcontent1/s1.md", fmt.Sprintf(pageContent, "s1"), + "symcontent1/s2.md", fmt.Sprintf(pageContent, "s2"), + + // A bundle + "symcontent2/a1/index.md", fmt.Sprintf(pageContent, ""), + "symcontent2/a1/page.md", fmt.Sprintf(pageContent, "page"), + "symcontent2/a1/logo.png", "image", + + // Assets + "symcontent3/s1.png", "image", + "symcontent3/s2.png", "image", + ) + + b.Build(BuildCfg{}) + s := b.H.Sites[0] assert.Equal(7, len(s.RegularPages())) a1Bundle := s.getPage(page.KindPage, "symbolic2/a1/index.md") @@ -404,9 +483,9 @@ func TestPageBundlerSiteWitSymbolicLinksInContent(t *testing.T) { assert.Equal(2, len(a1Bundle.Resources())) assert.Equal(1, len(a1Bundle.Resources().ByType(pageResourceType))) - th.assertFileContent(filepath.FromSlash(workDir+"/public/a/page/index.html"), "TheContent") - th.assertFileContent(filepath.FromSlash(workDir+"/public/symbolic1/s1/index.html"), "TheContent") - th.assertFileContent(filepath.FromSlash(workDir+"/public/symbolic2/a1/index.html"), "TheContent") + b.AssertFileContent(filepath.FromSlash(workDir+"/public/a/page/index.html"), "TheContent") + b.AssertFileContent(filepath.FromSlash(workDir+"/public/symbolic1/s1/index.html"), "TheContent") + b.AssertFileContent(filepath.FromSlash(workDir+"/public/symbolic2/a1/index.html"), "TheContent") } @@ -576,7 +655,7 @@ Single content. } func newTestBundleSources(t *testing.T) (*hugofs.Fs, *viper.Viper) { - cfg, fs := newTestCfg() + cfg, fs := newTestCfgBasic() assert := require.New(t) workDir := "/work" @@ -743,7 +822,7 @@ Content for 은행. } func newTestBundleSourcesMultilingual(t *testing.T) (*hugofs.Fs, *viper.Viper) { - cfg, fs := newTestCfg() + cfg, fs := newTestCfgBasic() workDir := "/work" cfg.Set("workingDir", workDir) @@ -825,90 +904,6 @@ TheContent. return fs, cfg } -func newTestBundleSymbolicSources(t *testing.T) (*helpers.PathSpec, func(), string) { - assert := require.New(t) - // We need to use the OS fs for this. - cfg := viper.New() - fs := hugofs.NewFrom(hugofs.Os, cfg) - fs.Destination = &afero.MemMapFs{} - loadDefaultSettingsFor(cfg) - - workDir, clean, err := createTempDir("hugosym") - assert.NoError(err) - - contentDir := "base" - cfg.Set("workingDir", workDir) - cfg.Set("contentDir", contentDir) - cfg.Set("baseURL", "https://example.com") - - if err := loadLanguageSettings(cfg, nil); err != nil { - t.Fatal(err) - } - - layout := `{{ .Title }}|{{ .Content }}` - pageContent := `--- -slug: %s -date: 2017-10-09 ---- - -TheContent. -` - - fs.Source.MkdirAll(filepath.Join(workDir, "layouts", "_default"), 0777) - fs.Source.MkdirAll(filepath.Join(workDir, contentDir), 0777) - fs.Source.MkdirAll(filepath.Join(workDir, contentDir, "a"), 0777) - for i := 1; i <= 3; i++ { - fs.Source.MkdirAll(filepath.Join(workDir, fmt.Sprintf("symcontent%d", i)), 0777) - - } - fs.Source.MkdirAll(filepath.Join(workDir, "symcontent2", "a1"), 0777) - - writeSource(t, fs, filepath.Join(workDir, "layouts", "_default", "single.html"), layout) - writeSource(t, fs, filepath.Join(workDir, "layouts", "_default", "list.html"), layout) - - writeSource(t, fs, filepath.Join(workDir, contentDir, "a", "regular.md"), fmt.Sprintf(pageContent, "a1")) - - // Regular files inside symlinked folder. - writeSource(t, fs, filepath.Join(workDir, "symcontent1", "s1.md"), fmt.Sprintf(pageContent, "s1")) - writeSource(t, fs, filepath.Join(workDir, "symcontent1", "s2.md"), fmt.Sprintf(pageContent, "s2")) - - // A bundle - writeSource(t, fs, filepath.Join(workDir, "symcontent2", "a1", "index.md"), fmt.Sprintf(pageContent, "")) - writeSource(t, fs, filepath.Join(workDir, "symcontent2", "a1", "page.md"), fmt.Sprintf(pageContent, "page")) - writeSource(t, fs, filepath.Join(workDir, "symcontent2", "a1", "logo.png"), "image") - - // Assets - writeSource(t, fs, filepath.Join(workDir, "symcontent3", "s1.png"), "image") - writeSource(t, fs, filepath.Join(workDir, "symcontent3", "s2.png"), "image") - - wd, _ := os.Getwd() - defer func() { - os.Chdir(wd) - }() - // Symlinked sections inside content. - os.Chdir(filepath.Join(workDir, contentDir)) - for i := 1; i <= 3; i++ { - assert.NoError(os.Symlink(filepath.FromSlash(fmt.Sprintf(("../symcontent%d"), i)), fmt.Sprintf("symbolic%d", i))) - } - - os.Chdir(filepath.Join(workDir, contentDir, "a")) - - // Create a symlink to one single content file - assert.NoError(os.Symlink(filepath.FromSlash("../../symcontent2/a1/page.md"), "page_s.md")) - - os.Chdir(filepath.FromSlash("../../symcontent3")) - - // Create a circular symlink. Will print some warnings. - assert.NoError(os.Symlink(filepath.Join("..", contentDir), filepath.FromSlash("circus"))) - - os.Chdir(workDir) - assert.NoError(err) - - ps, _ := helpers.NewPathSpec(fs, cfg) - - return ps, clean, workDir -} - // https://github.com/gohugoio/hugo/issues/5858 func TestBundledResourcesWhenMultipleOutputFormats(t *testing.T) { t.Parallel() @@ -971,3 +966,112 @@ slug: %s assert.True(b.CheckExists("public/about/services2/this-is-another-slug/index.html")) } + +func TestBundleMisc(t *testing.T) { + config := ` +baseURL = "https://example.com" +defaultContentLanguage = "en" +defaultContentLanguageInSubdir = true +ignoreFiles = ["README\\.md", "content/en/ignore"] + +[Languages] +[Languages.en] +weight = 99999 +contentDir = "content/en" +[Languages.nn] +weight = 20 +contentDir = "content/nn" +[Languages.sv] +weight = 30 +contentDir = "content/sv" +[Languages.nb] +weight = 40 +contentDir = "content/nb" + +` + + const pageContent = `--- +title: %q +--- +` + createPage := func(s string) string { + return fmt.Sprintf(pageContent, s) + } + + b := newTestSitesBuilder(t).WithConfigFile("toml", config) + b.WithLogger(loggers.NewWarningLogger()) + + b.WithTemplates("_default/list.html", `{{ range .Site.Pages }} +{{ .Kind }}|{{ .Path }}|{{ with .CurrentSection }}CurrentSection: {{ .Path }}{{ end }}|{{ .RelPermalink }}{{ end }} +`) + + b.WithTemplates("_default/single.html", `Single: {{ .Title }}`) + + b.WithContent("en/sect1/sect2/_index.md", createPage("en: Sect 2")) + b.WithContent("en/sect1/sect2/page.md", createPage("en: Page")) + b.WithContent("en/sect1/sect2/data-branch.json", "mydata") + b.WithContent("nn/sect1/sect2/page.md", createPage("nn: Page")) + b.WithContent("nn/sect1/sect2/data-branch.json", "my nn data") + + // En only + b.WithContent("en/enonly/myen.md", createPage("en: Page")) + b.WithContent("en/enonly/myendata.json", "mydata") + + // Leaf + + b.WithContent("nn/b1/index.md", createPage("nn: leaf")) + b.WithContent("en/b1/index.md", createPage("en: leaf")) + b.WithContent("sv/b1/index.md", createPage("sv: leaf")) + b.WithContent("nb/b1/index.md", createPage("nb: leaf")) + + // Should be ignored + b.WithContent("en/ignore/page.md", createPage("en: ignore")) + b.WithContent("en/README.md", createPage("en: ignore")) + + // Both leaf and branch bundle in same dir + b.WithContent("en/b2/index.md", `--- +slug: leaf +--- +`) + b.WithContent("en/b2/_index.md", createPage("en: branch")) + + b.WithContent("en/b1/data1.json", "en: data") + b.WithContent("sv/b1/data1.json", "sv: data") + b.WithContent("sv/b1/data2.json", "sv: data2") + b.WithContent("nb/b1/data2.json", "nb: data2") + + b.Build(BuildCfg{}) + + b.AssertFileContent("public/en/index.html", + filepath.FromSlash("section|sect1/sect2/_index.md|CurrentSection: sect1/sect2/_index.md"), + "myen.md|CurrentSection: enonly") + + b.AssertFileContentFn("public/en/index.html", func(s string) bool { + // Check ignored files + return !regexp.MustCompile("README|ignore").MatchString(s) + + }) + + b.AssertFileContent("public/nn/index.html", filepath.FromSlash("page|sect1/sect2/page.md|CurrentSection: sect1")) + b.AssertFileContentFn("public/nn/index.html", func(s string) bool { + return !strings.Contains(s, "enonly") + + }) + + // Check order of inherited data file + b.AssertFileContent("public/nb/b1/data1.json", "en: data") // Default content + b.AssertFileContent("public/nn/b1/data2.json", "sv: data") // First match + + b.AssertFileContent("public/en/enonly/myen/index.html", "Single: en: Page") + b.AssertFileContent("public/en/enonly/myendata.json", "mydata") + + assert := require.New(t) + assert.False(b.CheckExists("public/sv/enonly/myen/index.html")) + + // Both leaf and branch bundle in same dir + // We log a warning about it, but we keep both. + b.AssertFileContent("public/en/b2/index.html", + "/en/b2/leaf/", + filepath.FromSlash("section|sect1/sect2/_index.md|CurrentSection: sect1/sect2/_index.md")) + +} diff --git a/hugolib/pagecollections.go b/hugolib/pagecollections.go index 24318232034..aedcf40901e 100644 --- a/hugolib/pagecollections.go +++ b/hugolib/pagecollections.go @@ -148,7 +148,6 @@ func newPageCollectionsFromPages(pages pageStatePages) *PageCollections { for _, p := range pageCollection { if p.IsPage() { sourceRef := p.sourceRef() - if sourceRef != "" { // index the canonical ref // e.g. /section/article.md diff --git a/hugolib/pages_capture.go b/hugolib/pages_capture.go new file mode 100644 index 00000000000..b8132cbceed --- /dev/null +++ b/hugolib/pages_capture.go @@ -0,0 +1,777 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "context" + "fmt" + "os" + pth "path" + "path/filepath" + "strings" + + "github.com/gohugoio/hugo/config" + + "github.com/gohugoio/hugo/hugofs/files" + + "github.com/gohugoio/hugo/resources" + + "github.com/pkg/errors" + "golang.org/x/sync/errgroup" + + "github.com/gohugoio/hugo/common/hugio" + + "github.com/gohugoio/hugo/resources/resource" + + "github.com/gohugoio/hugo/source" + + "github.com/gohugoio/hugo/hugofs" + + "github.com/gohugoio/hugo/common/loggers" + "github.com/spf13/afero" +) + +func newPagesCollector( + sp *source.SourceSpec, + logger *loggers.Logger, + contentTracker *contentChangeMap, + proc pagesCollectorProcessorProvider, filenames ...string) *pagesCollector { + + return &pagesCollector{ + fs: sp.SourceFs, + proc: proc, + sp: sp, + logger: logger, + filenames: filenames, + tracker: contentTracker, + } +} + +func newPagesProcessor(h *HugoSites, sp *source.SourceSpec, partialBuild bool) *pagesProcessor { + + return &pagesProcessor{ + h: h, + sp: sp, + partialBuild: partialBuild, + numWorkers: config.GetNumWorkerMultiplier() * 3, + } +} + +type fileinfoBundle struct { + header hugofs.FileMetaInfo + resources []hugofs.FileMetaInfo +} + +func (b *fileinfoBundle) containsResource(name string) bool { + for _, r := range b.resources { + if r.Name() == name { + return true + } + } + + return false + +} + +type pageBundles map[string]*fileinfoBundle + +type pagesCollector struct { + sp *source.SourceSpec + fs afero.Fs + logger *loggers.Logger + + // Ordered list (bundle headers first) used in partial builds. + filenames []string + + // Content files tracker used in partial builds. + tracker *contentChangeMap + + proc pagesCollectorProcessorProvider +} + +type contentDirKey struct { + dirname string + filename string + tp bundleDirType +} + +// Collect. +func (c *pagesCollector) Collect() error { + c.proc.Start(context.Background()) + if c.tracker != nil { + c.tracker.start() + defer c.tracker.stop() + } + + var collectErr error + if len(c.filenames) == 0 { + // Collect everything. + collectErr = c.collectDir("", false, nil) + } else { + dirs := make(map[contentDirKey]bool) + for _, filename := range c.filenames { + dir, filename, btype := c.tracker.resolveAndRemove(filename) + dirs[contentDirKey{dir, filename, btype}] = true + } + + for dir, _ := range dirs { + switch dir.tp { + case bundleLeaf, bundleBranch: + collectErr = c.collectDir(dir.dirname, true, nil) + default: + // We always start from a directory. + collectErr = c.collectDir(dir.dirname, true, func(fim hugofs.FileMetaInfo) bool { + return strings.HasSuffix(dir.filename, fim.Meta().Path()) + }) + } + + if collectErr != nil { + break + } + } + + } + + err := c.proc.Wait() + + if collectErr != nil { + return collectErr + } + + return err +} + +func (c *pagesCollector) collectDir(dirname string, partial bool, inFilter func(fim hugofs.FileMetaInfo) bool) error { + fi, err := c.fs.Stat(dirname) + if err != nil { + if os.IsNotExist(err) { + // May have been deleted. + return nil + } + return err + } + + handleDir := func( + btype bundleDirType, + dir hugofs.FileMetaInfo, + path string, + readdir []hugofs.FileMetaInfo) error { + + if btype > bundleNot && c.tracker != nil { + c.tracker.add(path, btype) + } + + if btype == bundleBranch { + if err := c.handleBundleBranch(readdir); err != nil { + return err + } + // A branch bundle is only this directory level, so keep walking. + return nil + } else if btype == bundleLeaf { + if err := c.handleBundleLeaf(dir, path, readdir); err != nil { + return err + } + + return nil + } + + if err := c.handleFiles(readdir...); err != nil { + return err + } + + return nil + + } + + filter := func(fim hugofs.FileMetaInfo) bool { + if fim.Meta().SkipDir() { + return false + } + + if c.sp.IgnoreFile(fim.Meta().Filename()) { + return false + } + + if inFilter != nil { + return inFilter(fim) + } + return true + } + + preHook := func(dir hugofs.FileMetaInfo, path string, readdir []hugofs.FileMetaInfo) ([]hugofs.FileMetaInfo, error) { + var btype bundleDirType + + filtered := readdir[:0] + for _, fi := range readdir { + if filter(fi) { + filtered = append(filtered, fi) + if c.tracker != nil { + // Track symlinks. + c.tracker.addSymbolicLinkMapping(fi) + } + } + } + readdir = filtered + + // We merge language directories, so there can be duplicates, but they + // will be ordered, most important first. + var duplicates []int + seen := make(map[string]bool) + + for i, fi := range readdir { + + if fi.IsDir() { + continue + } + + meta := fi.Meta() + class := meta.Classifier() + translationBase := meta.TranslationBaseNameWithExt() + key := pth.Join(meta.Lang(), translationBase) + + if seen[key] { + duplicates = append(duplicates, i) + continue + } + seen[key] = true + + var thisBtype bundleDirType + + switch class { + case files.ContentClassLeaf: + thisBtype = bundleLeaf + case files.ContentClassBranch: + thisBtype = bundleBranch + } + + // Folders with both index.md and _index.md type of files have + // undefined behaviour and can never work. + // The branch variant will win because of sort order, but log + // a warning about it. + if thisBtype > bundleNot && btype > bundleNot && thisBtype != btype { + c.logger.WARN.Printf("Content directory %q have both index.* and _index.* files, pick one.", dir.Meta().Filename()) + // Reclassify it so it will be handled as a content file inside the + // section, which is in line with the <= 0.55 behaviour. + meta["classifier"] = files.ContentClassContent + } else if thisBtype > bundleNot { + btype = thisBtype + } + + } + + if len(duplicates) > 0 { + for i := len(duplicates) - 1; i >= 0; i-- { + idx := duplicates[i] + readdir = append(readdir[:idx], readdir[idx+1:]...) + } + } + + err := handleDir(btype, dir, path, readdir) + if err != nil { + return nil, err + } + + if btype == bundleLeaf || partial { + return nil, filepath.SkipDir + } + + // Keep walking. + return readdir, nil + + } + + var postHook hugofs.WalkHook + if c.tracker != nil { + postHook = func(dir hugofs.FileMetaInfo, path string, readdir []hugofs.FileMetaInfo) ([]hugofs.FileMetaInfo, error) { + if c.tracker == nil { + // Nothing to do. + return readdir, nil + } + + return readdir, nil + } + } + + wfn := func(path string, info hugofs.FileMetaInfo, err error) error { + if err != nil { + return err + } + + return nil + } + + w := hugofs.NewWalkway(hugofs.WalkwayConfig{ + Fs: c.fs, + Logger: c.logger, + Root: dirname, + Info: fi.(hugofs.FileMetaInfo), + HookPre: preHook, + HookPost: postHook, + WalkFn: wfn}) + + return w.Walk() + +} + +func (c *pagesCollector) isBundleHeader(fi hugofs.FileMetaInfo) bool { + class := fi.Meta().Classifier() + return class == files.ContentClassLeaf || class == files.ContentClassBranch +} + +func (c *pagesCollector) getLang(fi hugofs.FileMetaInfo) string { + lang := fi.Meta().Lang() + if lang != "" { + return lang + } + + return c.sp.DefaultContentLanguage +} + +func (c *pagesCollector) addToBundle(info hugofs.FileMetaInfo, btyp bundleDirType, bundles pageBundles) error { + getBundle := func(lang string) *fileinfoBundle { + return bundles[lang] + } + + // TODO(bep) mod check headless kub template + + cloneBundle := func(lang string) *fileinfoBundle { + // Every bundled file needs a content file header. + // Use the default content language if found, else just + // pick one. + var ( + source *fileinfoBundle + found bool + ) + + source, found = bundles[c.sp.DefaultContentLanguage] + if !found { + for _, b := range bundles { + source = b + break + } + } + + if source == nil { + panic(fmt.Sprintf("no source found, %d", len(bundles))) + } + + clone := c.cloneFileInfo(source.header) + clone.Meta()["lang"] = lang + + return &fileinfoBundle{ + header: clone, + } + } + + lang := c.getLang(info) + bundle := getBundle(lang) + isBundleHeader := c.isBundleHeader(info) + classifier := info.Meta().Classifier() + if bundle == nil { + if isBundleHeader { + bundle = &fileinfoBundle{header: info} + bundles[lang] = bundle + } else { + if btyp == bundleBranch { + // No special logic for branch bundles. + // Every language needs its own _index.md file. + return c.handleFiles(info) + } + + bundle = cloneBundle(lang) + bundles[lang] = bundle + } + } + + if !isBundleHeader { + bundle.resources = append(bundle.resources, info) + } + + if classifier == files.ContentClassFile { + translations := info.Meta().Translations() + if len(translations) < len(bundles) { + for lang, b := range bundles { + if !stringSliceContains(lang, translations...) && !b.containsResource(info.Name()) { + // Clone and add it to the bundle. + clone := c.cloneFileInfo(info) + clone.Meta()["lang"] = lang + b.resources = append(b.resources, clone) + } + } + } + } + + return nil +} + +func (c *pagesCollector) cloneFileInfo(fi hugofs.FileMetaInfo) hugofs.FileMetaInfo { + cm := hugofs.FileMeta{} + meta := fi.Meta() + if meta == nil { + panic(fmt.Sprintf("not meta: %v", fi.Name())) + } + for k, v := range meta { + cm[k] = v + } + + return hugofs.NewFileMetaInfo(fi, cm) +} + +func (c *pagesCollector) handleBundleBranch(readdir []hugofs.FileMetaInfo) error { + + // Maps bundles to its language. + bundles := pageBundles{} + + for _, fim := range readdir { + + if fim.IsDir() { + continue + } + + meta := fim.Meta() + + switch meta.Classifier() { + case files.ContentClassContent: + if err := c.handleFiles(fim); err != nil { + return err + } + default: + if err := c.addToBundle(fim, bundleBranch, bundles); err != nil { + return err + } + } + + } + + return c.proc.Process(bundles) + +} + +func (c *pagesCollector) handleBundleLeaf(dir hugofs.FileMetaInfo, path string, readdir []hugofs.FileMetaInfo) error { + // Maps bundles to its language. + bundles := pageBundles{} + + walk := func(path string, info hugofs.FileMetaInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + + return c.addToBundle(info, bundleLeaf, bundles) + + } + + // Start a new walker from the given path. + w := hugofs.NewWalkway(hugofs.WalkwayConfig{ + Root: path, + Fs: c.fs, + Logger: c.logger, + Info: dir, + DirEntries: readdir, + WalkFn: walk}) + + if err := w.Walk(); err != nil { + return err + } + + return c.proc.Process(bundles) + +} + +func (c *pagesCollector) handleFiles(fis ...hugofs.FileMetaInfo) error { + for _, fi := range fis { + if fi.IsDir() { + continue + } + + if err := c.proc.Process(fi); err != nil { + return err + } + } + return nil +} + +type pagesCollectorProcessorProvider interface { + Process(item interface{}) error + Start(ctx context.Context) context.Context + Wait() error +} + +type pagesProcessor struct { + h *HugoSites + sp *source.SourceSpec + + itemChan chan interface{} + itemGroup *errgroup.Group + + // The output Pages + pagesChan chan *pageState + pagesGroup *errgroup.Group + + numWorkers int + + partialBuild bool +} + +func (proc *pagesProcessor) Process(item interface{}) error { + proc.itemChan <- item + return nil +} + +func (proc *pagesProcessor) Start(ctx context.Context) context.Context { + proc.pagesChan = make(chan *pageState, proc.numWorkers) + proc.pagesGroup, ctx = errgroup.WithContext(ctx) + proc.itemChan = make(chan interface{}, proc.numWorkers) + proc.itemGroup, ctx = errgroup.WithContext(ctx) + + proc.pagesGroup.Go(func() error { + for p := range proc.pagesChan { + s := p.s + p.forceRender = proc.partialBuild + + if p.forceRender { + s.replacePage(p) + } else { + s.addPage(p) + } + } + return nil + }) + + for i := 0; i < proc.numWorkers; i++ { + proc.itemGroup.Go(func() error { + for item := range proc.itemChan { + if err := proc.process(item); err != nil { + // TODO(bep) mod handle global cancellation + proc.h.SendError(err) + } + } + + return nil + }) + } + + return ctx +} + +func (proc *pagesProcessor) Wait() error { + close(proc.itemChan) + + err := proc.itemGroup.Wait() + + close(proc.pagesChan) + + if err != nil { + return err + } + + return proc.pagesGroup.Wait() +} + +func (proc *pagesProcessor) newPageFromBundle(b *fileinfoBundle) (*pageState, error) { + p, err := proc.newPageFromFi(b.header, nil) + if err != nil { + return nil, err + } + + if len(b.resources) > 0 { + + resources := make(resource.Resources, len(b.resources)) + + for i, rfi := range b.resources { + meta := rfi.Meta() + classifier := meta.Classifier() + var r resource.Resource + switch classifier { + case files.ContentClassContent: + rp, err := proc.newPageFromFi(rfi, p) + if err != nil { + return nil, err + } + rp.m.resourcePath = filepath.ToSlash(strings.TrimPrefix(rp.Path(), p.File().Dir())) + + r = rp + + case files.ContentClassFile: + r, err = proc.newResource(rfi, p) + if err != nil { + return nil, err + } + default: + panic(fmt.Sprintf("invalid classifier: %q", classifier)) + } + + resources[i] = r + + } + + p.addResources(resources...) + } + + return p, nil +} + +func (proc *pagesProcessor) newPageFromFi(fim hugofs.FileMetaInfo, owner *pageState) (*pageState, error) { + fi, err := newFileInfo2(proc.sp, fim) + if err != nil { + return nil, err + } + + var s *Site + meta := fim.Meta() + + if owner != nil { + s = owner.s + } else { + lang := meta.Lang() + s = proc.getSite(lang) + } + + r := func() (hugio.ReadSeekCloser, error) { + return meta.Open() + } + + p, err := newPageWithContent(fi, s, owner != nil, r) + if err != nil { + return nil, err + } + p.parent = owner + return p, nil +} + +func (proc *pagesProcessor) newResource(fim hugofs.FileMetaInfo, owner *pageState) (resource.Resource, error) { + + // TODO(bep) consolidate with multihost logic + clean up + outputFormats := owner.m.outputFormats() + seen := make(map[string]bool) + var targetBasePaths []string + // Make sure bundled resources are published to all of the ouptput formats' + // sub paths. + for _, f := range outputFormats { + p := f.Path + if seen[p] { + continue + } + seen[p] = true + targetBasePaths = append(targetBasePaths, p) + + } + + meta := fim.Meta() + r := func() (hugio.ReadSeekCloser, error) { + return meta.Open() + } + + target := strings.TrimPrefix(meta.Path(), owner.File().Dir()) + + return owner.s.ResourceSpec.New( + resources.ResourceSourceDescriptor{ + TargetPaths: owner.getTargetPaths, + OpenReadSeekCloser: r, + FileInfo: fim, + RelTargetFilename: target, + TargetBasePaths: targetBasePaths, + }) +} + +func (proc *pagesProcessor) getSite(lang string) *Site { + if lang == "" { + return proc.h.Sites[0] + } + + for _, s := range proc.h.Sites { + if lang == s.Lang() { + return s + } + } + return proc.h.Sites[0] +} + +func (proc *pagesProcessor) copyFile(fim hugofs.FileMetaInfo) error { + meta := fim.Meta() + s := proc.getSite(meta.Lang()) + f, err := meta.Open() + if err != nil { + return errors.Wrap(err, "copyFile: failed to open") + } + + target := filepath.Join(s.PathSpec.GetTargetLanguageBasePath(), meta.Path()) + + defer f.Close() + + return s.publish(&s.PathSpec.ProcessingStats.Files, target, f) + +} + +func (proc *pagesProcessor) process(item interface{}) error { + send := func(p *pageState, err error) { + if err != nil { + proc.sendError(err) + } else { + proc.pagesChan <- p + } + } + + switch v := item.(type) { + // Page bundles mapped to their language. + case pageBundles: + for _, bundle := range v { + if proc.shouldSkip(bundle.header) { + continue + } + send(proc.newPageFromBundle(bundle)) + } + case hugofs.FileMetaInfo: + if proc.shouldSkip(v) { + return nil + } + meta := v.Meta() + + classifier := meta.Classifier() + switch classifier { + case files.ContentClassContent: + send(proc.newPageFromFi(v, nil)) + case files.ContentClassFile: + proc.sendError(proc.copyFile(v)) + default: + panic(fmt.Sprintf("invalid classifier: %q", classifier)) + } + default: + panic(fmt.Sprintf("unrecognized item type in Process: %T", item)) + } + + return nil +} + +func (proc *pagesProcessor) sendError(err error) { + if err == nil { + return + } + proc.h.SendError(err) +} + +func (proc *pagesProcessor) shouldSkip(fim hugofs.FileMetaInfo) bool { + return proc.sp.DisabledLanguages[fim.Meta().Lang()] +} + +func stringSliceContains(k string, values ...string) bool { + for _, v := range values { + if k == v { + return true + } + } + return false +} diff --git a/hugolib/pages_capture_test.go b/hugolib/pages_capture_test.go new file mode 100644 index 00000000000..6095ec8f236 --- /dev/null +++ b/hugolib/pages_capture_test.go @@ -0,0 +1,87 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "context" + "fmt" + "path/filepath" + "testing" + + "github.com/pkg/errors" + + "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/source" + + "github.com/gohugoio/hugo/common/loggers" + + "github.com/gohugoio/hugo/hugofs" + "github.com/spf13/afero" + "github.com/stretchr/testify/require" +) + +func TestPagesCapture(t *testing.T) { + + cfg, hfs := newTestCfg() + fs := hfs.Source + assert := require.New(t) + + var writeFile = func(filename string) { + assert.NoError(afero.WriteFile(fs, filepath.FromSlash(filename), []byte(fmt.Sprintf("content-%s", filename)), 0755)) + } + + writeFile("_index.md") + writeFile("logo.png") + writeFile("root.md") + writeFile("blog/index.md") + writeFile("blog/hello.md") + writeFile("blog/images/sunset.png") + writeFile("pages/page1.md") + writeFile("pages/page2.md") + writeFile("pages/page.png") + + ps, err := helpers.NewPathSpec(hugofs.NewFrom(fs, cfg), cfg) + assert.NoError(err) + sourceSpec := source.NewSourceSpec(ps, fs) + + t.Run("Collect", func(t *testing.T) { + assert := require.New(t) + proc := &testPagesCollectorProcessor{} + c := newPagesCollector(sourceSpec, loggers.NewErrorLogger(), nil, proc) + assert.NoError(c.Collect()) + assert.Equal(4, len(proc.items)) + }) + + t.Run("error in Wait", func(t *testing.T) { + assert := require.New(t) + c := newPagesCollector(sourceSpec, loggers.NewErrorLogger(), nil, + &testPagesCollectorProcessor{waitErr: errors.New("failed")}) + assert.Error(c.Collect()) + }) +} + +type testPagesCollectorProcessor struct { + items []interface{} + waitErr error +} + +func (proc *testPagesCollectorProcessor) Process(item interface{}) error { + proc.items = append(proc.items, item) + return nil +} +func (proc *testPagesCollectorProcessor) Start(ctx context.Context) context.Context { + return ctx +} + +func (proc *testPagesCollectorProcessor) Wait() error { return proc.waitErr } diff --git a/hugolib/paths/paths.go b/hugolib/paths/paths.go index df66e2a46e2..9baf183d333 100644 --- a/hugolib/paths/paths.go +++ b/hugolib/paths/paths.go @@ -20,6 +20,7 @@ import ( "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/langs" + "github.com/gohugoio/hugo/modules" "github.com/pkg/errors" "github.com/gohugoio/hugo/hugofs" @@ -62,8 +63,9 @@ type Paths struct { UglyURLs bool CanonifyURLs bool - Language *langs.Language - Languages langs.Languages + Language *langs.Language + Languages langs.Languages + LanguagesDefaultFirst langs.Languages // The PathSpec looks up its config settings in both the current language // and then in the global Viper config. @@ -74,8 +76,8 @@ type Paths struct { DefaultContentLanguage string multilingual bool - themes []string - AllThemes []ThemeConfig + AllModules modules.Modules + ModulesClient *modules.Client } func New(fs *hugofs.Fs, cfg config.Provider) (*Paths, error) { @@ -91,12 +93,6 @@ func New(fs *hugofs.Fs, cfg config.Provider) (*Paths, error) { resourceDir := filepath.Clean(cfg.GetString("resourceDir")) publishDir := filepath.Clean(cfg.GetString("publishDir")) - if contentDir == "" { - return nil, fmt.Errorf("contentDir not set") - } - if resourceDir == "" { - return nil, fmt.Errorf("resourceDir not set") - } if publishDir == "" { return nil, fmt.Errorf("publishDir not set") } @@ -104,8 +100,9 @@ func New(fs *hugofs.Fs, cfg config.Provider) (*Paths, error) { defaultContentLanguage := cfg.GetString("defaultContentLanguage") var ( - language *langs.Language - languages langs.Languages + language *langs.Language + languages langs.Languages + languagesDefaultFirst langs.Languages ) if l, ok := cfg.(*langs.Language); ok { @@ -117,6 +114,12 @@ func New(fs *hugofs.Fs, cfg config.Provider) (*Paths, error) { languages = l } + if l, ok := cfg.Get("languagesSortedDefaultFirst").(langs.Languages); ok { + languagesDefaultFirst = l + } + + // + if len(languages) == 0 { // We have some old tests that does not test the entire chain, hence // they have no languages. So create one so we get the proper filesystem. @@ -163,26 +166,24 @@ func New(fs *hugofs.Fs, cfg config.Provider) (*Paths, error) { AbsResourcesDir: absResourcesDir, AbsPublishDir: absPublishDir, - themes: config.GetStringSlicePreserveString(cfg, "theme"), - multilingual: cfg.GetBool("multilingual"), defaultContentLanguageInSubdir: cfg.GetBool("defaultContentLanguageInSubdir"), DefaultContentLanguage: defaultContentLanguage, Language: language, Languages: languages, + LanguagesDefaultFirst: languagesDefaultFirst, MultihostTargetBasePaths: multihostTargetBasePaths, PaginatePath: cfg.GetString("paginatePath"), } - if !cfg.IsSet("theme") && cfg.IsSet("allThemes") { - p.AllThemes = cfg.Get("allThemes").([]ThemeConfig) - } else { - p.AllThemes, err = collectThemeNames(p) - if err != nil { - return nil, err - } + if cfg.IsSet("allModules") { + p.AllModules = cfg.Get("allModules").(modules.Modules) + } + + if cfg.IsSet("modulesClient") { + p.ModulesClient = cfg.Get("modulesClient").(*modules.Client) } // TODO(bep) remove this, eventually @@ -208,12 +209,9 @@ func (p *Paths) Lang() string { } // ThemeSet checks whether a theme is in use or not. +// TODO(bep) mod remove this func (p *Paths) ThemeSet() bool { - return len(p.themes) > 0 -} - -func (p *Paths) Themes() []string { - return p.themes + return len(p.AllModules) > 0 } func (p *Paths) GetTargetLanguageBasePath() string { @@ -269,6 +267,18 @@ func (p *Paths) AbsPathify(inPath string) string { return AbsPathify(p.WorkingDir, inPath) } +// RelPathify trims any WorkingDir prefix from the given filename. If +// the filename is not considered to be absolute, the path is just cleaned. +func (p *Paths) RelPathify(filename string) string { + filename = filepath.Clean(filename) + if !filepath.IsAbs(filename) { + return filename + } + + return strings.TrimPrefix(strings.TrimPrefix(filename, p.WorkingDir), FilePathSeparator) + +} + // AbsPathify creates an absolute path if given a working dir and arelative path. // If already absolute, the path is just cleaned. func AbsPathify(workingDir, inPath string) string { diff --git a/hugolib/paths/paths_test.go b/hugolib/paths/paths_test.go index 3bd445b8bc6..95a8be782ef 100644 --- a/hugolib/paths/paths_test.go +++ b/hugolib/paths/paths_test.go @@ -16,6 +16,8 @@ package paths import ( "testing" + "github.com/gohugoio/hugo/langs" + "github.com/gohugoio/hugo/hugofs" "github.com/spf13/viper" "github.com/stretchr/testify/require" @@ -27,14 +29,19 @@ func TestNewPaths(t *testing.T) { v := viper.New() fs := hugofs.NewMem(v) + v.Set("languages", map[string]interface{}{ + "no": map[string]interface{}{}, + "en": map[string]interface{}{}, + }) v.Set("defaultContentLanguageInSubdir", true) v.Set("defaultContentLanguage", "no") - v.Set("multilingual", true) v.Set("contentDir", "content") v.Set("workingDir", "work") v.Set("resourceDir", "resources") v.Set("publishDir", "public") + langs.LoadLanguageSettings(v, nil) + p, err := New(fs, v) assert.NoError(err) diff --git a/hugolib/paths/themes.go b/hugolib/paths/themes.go deleted file mode 100644 index a526953f16e..00000000000 --- a/hugolib/paths/themes.go +++ /dev/null @@ -1,154 +0,0 @@ -// Copyright 2019 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package paths - -import ( - "path/filepath" - "strings" - - "github.com/gohugoio/hugo/config" - "github.com/spf13/afero" - "github.com/spf13/cast" -) - -type ThemeConfig struct { - // The theme name as provided by the folder name below /themes. - Name string - - // Optional configuration filename (e.g. "/themes/mytheme/config.json"). - ConfigFilename string - - // Optional config read from the ConfigFile above. - Cfg config.Provider -} - -// Create file system, an ordered theme list from left to right, no duplicates. -type themesCollector struct { - themesDir string - fs afero.Fs - seen map[string]bool - themes []ThemeConfig -} - -func (c *themesCollector) isSeen(theme string) bool { - loki := strings.ToLower(theme) - if c.seen[loki] { - return true - } - c.seen[loki] = true - return false -} - -func (c *themesCollector) addAndRecurse(themes ...string) error { - for i := 0; i < len(themes); i++ { - theme := themes[i] - configFilename := c.getConfigFileIfProvided(theme) - if !c.isSeen(theme) { - tc, err := c.add(theme, configFilename) - if err != nil { - return err - } - if err := c.addThemeNamesFromTheme(tc); err != nil { - return err - } - } - } - return nil -} - -func (c *themesCollector) add(name, configFilename string) (ThemeConfig, error) { - var cfg config.Provider - var tc ThemeConfig - - if configFilename != "" { - var err error - cfg, err = config.FromFile(c.fs, configFilename) - if err != nil { - return tc, err - } - } - - tc = ThemeConfig{Name: name, ConfigFilename: configFilename, Cfg: cfg} - c.themes = append(c.themes, tc) - return tc, nil - -} - -func collectThemeNames(p *Paths) ([]ThemeConfig, error) { - return CollectThemes(p.Fs.Source, p.AbsPathify(p.ThemesDir), p.Themes()) - -} - -func CollectThemes(fs afero.Fs, themesDir string, themes []string) ([]ThemeConfig, error) { - if len(themes) == 0 { - return nil, nil - } - - c := &themesCollector{ - fs: fs, - themesDir: themesDir, - seen: make(map[string]bool)} - - for i := 0; i < len(themes); i++ { - theme := themes[i] - if err := c.addAndRecurse(theme); err != nil { - return nil, err - } - } - - return c.themes, nil - -} - -func (c *themesCollector) getConfigFileIfProvided(theme string) string { - configDir := filepath.Join(c.themesDir, theme) - - var ( - configFilename string - exists bool - ) - - // Viper supports more, but this is the sub-set supported by Hugo. - for _, configFormats := range config.ValidConfigFileExtensions { - configFilename = filepath.Join(configDir, "config."+configFormats) - exists, _ = afero.Exists(c.fs, configFilename) - if exists { - break - } - } - - if !exists { - // No theme config set. - return "" - } - - return configFilename - -} - -func (c *themesCollector) addThemeNamesFromTheme(theme ThemeConfig) error { - if theme.Cfg != nil && theme.Cfg.IsSet("theme") { - v := theme.Cfg.Get("theme") - switch vv := v.(type) { - case []string: - return c.addAndRecurse(vv...) - case []interface{}: - return c.addAndRecurse(cast.ToStringSlice(vv)...) - default: - return c.addAndRecurse(cast.ToString(vv)) - } - } - - return nil -} diff --git a/hugolib/resource_chain_test.go b/hugolib/resource_chain_test.go index 8f55e112e68..ea95a3f09a6 100644 --- a/hugolib/resource_chain_test.go +++ b/hugolib/resource_chain_test.go @@ -18,6 +18,8 @@ import ( "path/filepath" "testing" + "github.com/gohugoio/hugo/htesting" + "github.com/spf13/viper" "github.com/stretchr/testify/require" @@ -33,17 +35,17 @@ func TestSCSSWithIncludePaths(t *testing.T) { t.Skip("Skip SCSS") } assert := require.New(t) - workDir, clean, err := createTempDir("hugo-scss-include") + workDir, clean, err := htesting.CreateTempDir(hugofs.Os, "hugo-scss-include") assert.NoError(err) defer clean() v := viper.New() v.Set("workingDir", workDir) b := newTestSitesBuilder(t).WithLogger(loggers.NewErrorLogger()) - b.WithViper(v) - b.WithWorkingDir(workDir) // Need to use OS fs for this. b.Fs = hugofs.NewDefault(v) + b.WithWorkingDir(workDir) + b.WithViper(v) fooDir := filepath.Join(workDir, "node_modules", "foo") scssDir := filepath.Join(workDir, "assets", "scss") @@ -84,7 +86,7 @@ func TestSCSSWithThemeOverrides(t *testing.T) { t.Skip("Skip SCSS") } assert := require.New(t) - workDir, clean, err := createTempDir("hugo-scss-include") + workDir, clean, err := htesting.CreateTempDir(hugofs.Os, "hugo-scss-include") assert.NoError(err) defer clean() @@ -95,10 +97,10 @@ func TestSCSSWithThemeOverrides(t *testing.T) { v.Set("workingDir", workDir) v.Set("theme", theme) b := newTestSitesBuilder(t).WithLogger(loggers.NewErrorLogger()) - b.WithViper(v) - b.WithWorkingDir(workDir) // Need to use OS fs for this. b.Fs = hugofs.NewDefault(v) + b.WithWorkingDir(workDir) + b.WithViper(v) fooDir := filepath.Join(workDir, "node_modules", "foo") scssDir := filepath.Join(workDir, "assets", "scss") @@ -385,14 +387,15 @@ CSV2: {{ $csv2 }} } for _, test := range tests { - if !test.shouldRun() { - t.Log("Skip", test.name) - continue - } - - b := newTestSitesBuilder(t).WithLogger(loggers.NewErrorLogger()) - b.WithSimpleConfigFile() - b.WithContent("_index.md", ` + test := test + t.Run(test.name, func(t *testing.T) { + if !test.shouldRun() { + t.Skip() + } + t.Parallel() + + b := newTestSitesBuilder(t).WithLogger(loggers.NewErrorLogger()) + b.WithContent("_index.md", ` --- title: Home --- @@ -400,37 +403,37 @@ title: Home Home. `, - "page1.md", ` + "page1.md", ` --- title: Hello1 --- Hello1 `, - "page2.md", ` + "page2.md", ` --- title: Hello2 --- Hello2 `, - "t1.txt", "t1t|", - "t2.txt", "t2t|", - ) + "t1.txt", "t1t|", + "t2.txt", "t2t|", + ) - b.WithSourceFile(filepath.Join("assets", "css", "styles1.css"), ` + b.WithSourceFile(filepath.Join("assets", "css", "styles1.css"), ` h1 { font-style: bold; } `) - b.WithSourceFile(filepath.Join("assets", "js", "script1.js"), ` + b.WithSourceFile(filepath.Join("assets", "js", "script1.js"), ` var x; x = 5; document.getElementById("demo").innerHTML = x * 10; `) - b.WithSourceFile(filepath.Join("assets", "mydata", "json1.json"), ` + b.WithSourceFile(filepath.Join("assets", "mydata", "json1.json"), ` { "employees":[ {"firstName":"John", "lastName":"Doe"}, @@ -440,19 +443,19 @@ document.getElementById("demo").innerHTML = x * 10; } `) - b.WithSourceFile(filepath.Join("assets", "mydata", "svg1.svg"), ` + b.WithSourceFile(filepath.Join("assets", "mydata", "svg1.svg"), ` `) - b.WithSourceFile(filepath.Join("assets", "mydata", "xml1.xml"), ` + b.WithSourceFile(filepath.Join("assets", "mydata", "xml1.xml"), ` Hugo Rocks! `) - b.WithSourceFile(filepath.Join("assets", "mydata", "html1.html"), ` + b.WithSourceFile(filepath.Join("assets", "mydata", "html1.html"), ` Cool @@ -460,7 +463,7 @@ Cool `) - b.WithSourceFile(filepath.Join("assets", "scss", "styles2.scss"), ` + b.WithSourceFile(filepath.Join("assets", "scss", "styles2.scss"), ` $color: #333; body { @@ -468,7 +471,7 @@ body { } `) - b.WithSourceFile(filepath.Join("assets", "sass", "styles3.sass"), ` + b.WithSourceFile(filepath.Join("assets", "sass", "styles3.sass"), ` $color: #333; .content-navigation @@ -476,10 +479,11 @@ $color: #333; `) - t.Log("Test", test.name) - test.prepare(b) - b.Build(BuildCfg{}) - test.verify(b) + test.prepare(b) + b.Build(BuildCfg{}) + test.verify(b) + + }) } } diff --git a/hugolib/shortcode_test.go b/hugolib/shortcode_test.go index 42eef61ae0a..24ef77503cb 100644 --- a/hugolib/shortcode_test.go +++ b/hugolib/shortcode_test.go @@ -26,10 +26,6 @@ import ( "strings" "testing" - "github.com/spf13/viper" - - "github.com/spf13/afero" - "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/tpl" @@ -54,12 +50,8 @@ title: "Title" writeSource(t, fs, "content/simple.md", contentFile) - h, err := NewHugoSites(deps.DepsCfg{Fs: fs, Cfg: cfg, WithTemplate: withTemplate}) - - require.NoError(t, err) - require.Len(t, h.Sites, 1) - - err = h.Build(BuildCfg{}) + b := newTestSitesBuilderFromDepsCfg(t, deps.DepsCfg{Fs: fs, Cfg: cfg, WithTemplate: withTemplate}).WithNothingAdded() + err := b.BuildE(BuildCfg{}) if err != nil && !expectError { t.Fatalf("Shortcode rendered error %s.", err) @@ -69,6 +61,9 @@ title: "Title" t.Fatalf("No error from shortcode") } + h := b.H + require.Len(t, h.Sites, 1) + require.Len(t, h.Sites[0].RegularPages(), 1) output := strings.TrimSpace(content(h.Sites[0].RegularPages()[0])) @@ -78,7 +73,7 @@ title: "Title" expected = strings.TrimSpace(expected) if output != expected { - Fatalf(t, "Shortcode render didn't match. got \n%q but expected \n%q", output, expected) + t.Fatalf("Shortcode render didn't match. got \n%q but expected \n%q", output, expected) } } @@ -341,7 +336,6 @@ func TestShortcodeWrappedInPIssue(t *testing.T) { } func TestExtractShortcodes(t *testing.T) { - t.Parallel() b := newTestSitesBuilder(t).WithSimpleConfigFile() b.WithTemplates( @@ -413,7 +407,10 @@ title: "Shortcodes Galore!" {"inline", `{{< my.inline >}}Hi{{< /my.inline >}}`, regexpCheck("my.inline;inline:true;closing:true;inner:{Hi};")}, } { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() assert := require.New(t) counter := 0 @@ -437,7 +434,6 @@ title: "Shortcodes Galore!" } func TestShortcodesInSite(t *testing.T) { - t.Parallel() baseURL := "http://foo/bar" tests := []struct { @@ -577,7 +573,9 @@ title: "Foo" s := buildSingleSite(t, deps.DepsCfg{WithTemplate: addTemplates, Fs: fs, Cfg: cfg}, BuildCfg{}) for i, test := range tests { + test := test t.Run(fmt.Sprintf("test=%d;contentPath=%s", i, test.contentPath), func(t *testing.T) { + t.Parallel() if strings.HasSuffix(test.contentPath, ".ad") && !helpers.HasAsciidoc() { t.Skip("Skip Asciidoc test case as no Asciidoc present.") } else if strings.HasSuffix(test.contentPath, ".rst") && !helpers.HasRst() { @@ -632,9 +630,8 @@ outputs: ["CSV"] CSV: {{< myShort >}} ` - mf := afero.NewMemMapFs() - - th, h := newTestSitesFromConfig(t, mf, siteConfig, + b := newTestSitesBuilder(t).WithConfigFile("toml", siteConfig) + b.WithTemplates( "layouts/_default/single.html", `Single HTML: {{ .Title }}|{{ .Content }}`, "layouts/_default/single.json", `Single JSON: {{ .Title }}|{{ .Content }}`, "layouts/_default/single.csv", `Single CSV: {{ .Title }}|{{ .Content }}`, @@ -651,14 +648,13 @@ CSV: {{< myShort >}} "layouts/shortcodes/myInner.html", `myInner:--{{- .Inner -}}--`, ) - fs := th.Fs - - writeSource(t, fs, "content/_index.md", fmt.Sprintf(pageTemplate, "Home")) - writeSource(t, fs, "content/sect/mypage.md", fmt.Sprintf(pageTemplate, "Single")) - writeSource(t, fs, "content/sect/mycsvpage.md", fmt.Sprintf(pageTemplateCSVOnly, "Single CSV")) + b.WithContent("_index.md", fmt.Sprintf(pageTemplate, "Home"), + "sect/mypage.md", fmt.Sprintf(pageTemplate, "Single"), + "sect/mycsvpage.md", fmt.Sprintf(pageTemplateCSVOnly, "Single CSV"), + ) - err := h.Build(BuildCfg{}) - require.NoError(t, err) + b.Build(BuildCfg{}) + h := b.H require.Len(t, h.Sites, 1) s := h.Sites[0] @@ -666,7 +662,7 @@ CSV: {{< myShort >}} require.NotNil(t, home) require.Len(t, home.OutputFormats(), 3) - th.assertFileContent("public/index.html", + b.AssertFileContent("public/index.html", "Home HTML", "ShortHTML", "ShortNoExt", @@ -674,7 +670,7 @@ CSV: {{< myShort >}} "myInner:--ShortHTML--", ) - th.assertFileContent("public/amp/index.html", + b.AssertFileContent("public/amp/index.html", "Home AMP", "ShortAMP", "ShortNoExt", @@ -682,7 +678,7 @@ CSV: {{< myShort >}} "myInner:--ShortAMP--", ) - th.assertFileContent("public/index.ics", + b.AssertFileContent("public/index.ics", "Home Calendar", "ShortCalendar", "ShortNoExt", @@ -690,7 +686,7 @@ CSV: {{< myShort >}} "myInner:--ShortCalendar--", ) - th.assertFileContent("public/sect/mypage/index.html", + b.AssertFileContent("public/sect/mypage/index.html", "Single HTML", "ShortHTML", "ShortNoExt", @@ -698,7 +694,7 @@ CSV: {{< myShort >}} "myInner:--ShortHTML--", ) - th.assertFileContent("public/sect/mypage/index.json", + b.AssertFileContent("public/sect/mypage/index.json", "Single JSON", "ShortJSON", "ShortNoExt", @@ -706,7 +702,7 @@ CSV: {{< myShort >}} "myInner:--ShortJSON--", ) - th.assertFileContent("public/amp/sect/mypage/index.html", + b.AssertFileContent("public/amp/sect/mypage/index.html", // No special AMP template "Single HTML", "ShortAMP", @@ -715,7 +711,7 @@ CSV: {{< myShort >}} "myInner:--ShortAMP--", ) - th.assertFileContent("public/sect/mycsvpage/index.csv", + b.AssertFileContent("public/sect/mycsvpage/index.csv", "Single CSV", "ShortCSV", ) @@ -864,10 +860,6 @@ weight: %d --- C-%s` - v := viper.New() - - v.Set("timeout", 500) - templates = append(templates, []string{"shortcodes/c.html", contentShortcode}...) templates = append(templates, []string{"_default/single.html", "Single Content: {{ .Content }}"}...) templates = append(templates, []string{"_default/list.html", "List Content: {{ .Content }}"}...) @@ -884,21 +876,21 @@ C-%s` builder := newTestSitesBuilder(t).WithDefaultMultiSiteConfig() - builder.WithViper(v).WithContent(content...).WithTemplates(templates...).CreateSites().Build(BuildCfg{}) + builder.WithContent(content...).WithTemplates(templates...).CreateSites().Build(BuildCfg{}) s := builder.H.Sites[0] assert.Equal(3, len(s.RegularPages())) - builder.AssertFileContent("public/section1/index.html", + builder.AssertFileContent("public/en/section1/index.html", "List Content:

Logo:P1:|P2:logo.png/PNG logo|:P1: P1:|P2:docs1p1/

C-s1p1

\n|", "BP1:P1:|P2:docbp1/

C-bp1

", ) - builder.AssertFileContent("public/b1/index.html", + builder.AssertFileContent("public/en/b1/index.html", "Single Content:

Logo:P1:|P2:logo.png/PNG logo|:P1: P1:|P2:docs1p1/

C-s1p1

\n|", "P2:docbp1/

C-bp1

", ) - builder.AssertFileContent("public/section2/s2p1/index.html", + builder.AssertFileContent("public/en/section2/s2p1/index.html", "Single Content:

Logo:P1:|P2:logo.png/PNG logo|:P1: P1:|P2:docs1p1/

C-s1p1

\n|", "P2:docbp1/

C-bp1

", ) @@ -1062,8 +1054,10 @@ String: {{ . | safeHTML }} func TestInlineShortcodes(t *testing.T) { for _, enableInlineShortcodes := range []bool{true, false} { + enableInlineShortcodes := enableInlineShortcodes t.Run(fmt.Sprintf("enableInlineShortcodes=%t", enableInlineShortcodes), func(t *testing.T) { + t.Parallel() conf := fmt.Sprintf(` baseURL = "https://example.com" enableInlineShortcodes = %t diff --git a/hugolib/site.go b/hugolib/site.go index b1441ca8a13..882874db947 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -14,7 +14,6 @@ package hugolib import ( - "context" "fmt" "html/template" "io" @@ -29,6 +28,8 @@ import ( "strings" "time" + "github.com/gohugoio/hugo/hugofs/files" + "github.com/gohugoio/hugo/common/maps" "github.com/pkg/errors" @@ -45,7 +46,6 @@ import ( "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/lazy" - "golang.org/x/sync/errgroup" "github.com/gohugoio/hugo/media" @@ -1028,7 +1028,8 @@ func (s *Site) processPartial(events []fsnotify.Event) (whatChanged, error) { removed = true } } - if removed && IsContentFile(ev.Name) { + + if removed && files.IsContentFile(ev.Name) { h.removePageByFilename(ev.Name) } @@ -1058,7 +1059,7 @@ func (s *Site) processPartial(events []fsnotify.Event) (whatChanged, error) { filenamesChanged = append(filenamesChanged, contentFilesChanged...) } - filenamesChanged = helpers.UniqueStrings(filenamesChanged) + filenamesChanged = helpers.UniqueStringsReuse(filenamesChanged) if err := s.readAndProcessContent(filenamesChanged...); err != nil { return whatChanged{}, err @@ -1078,10 +1079,12 @@ func (s *Site) processPartial(events []fsnotify.Event) (whatChanged, error) { func (s *Site) process(config BuildCfg) (err error) { if err = s.initialize(); err != nil { + err = errors.Wrap(err, "initialize") return } - if err := s.readAndProcessContent(); err != nil { - return err + if err = s.readAndProcessContent(); err != nil { + err = errors.Wrap(err, "readAndProcessContent") + return } return err @@ -1304,93 +1307,14 @@ func (s *Site) isContentDirEvent(e fsnotify.Event) bool { return s.BaseFs.IsContent(e.Name) } -type contentCaptureResultHandler struct { - defaultContentProcessor *siteContentProcessor - contentProcessors map[string]*siteContentProcessor -} - -func (c *contentCaptureResultHandler) getContentProcessor(lang string) *siteContentProcessor { - proc, found := c.contentProcessors[lang] - if found { - return proc - } - return c.defaultContentProcessor -} - -func (c *contentCaptureResultHandler) handleSingles(fis ...*fileInfo) { - for _, fi := range fis { - proc := c.getContentProcessor(fi.Lang()) - proc.processSingle(fi) - } -} -func (c *contentCaptureResultHandler) handleBundles(d *bundleDirs) { - for _, b := range d.bundles { - proc := c.getContentProcessor(b.fi.Lang()) - proc.processBundle(b) - } -} - -func (c *contentCaptureResultHandler) handleCopyFile(f pathLangFile) { - proc := c.getContentProcessor(f.Lang()) - proc.processAsset(f) -} - func (s *Site) readAndProcessContent(filenames ...string) error { - - ctx := context.Background() - g, ctx := errgroup.WithContext(ctx) - - defaultContentLanguage := s.SourceSpec.DefaultContentLanguage - - contentProcessors := make(map[string]*siteContentProcessor) - var defaultContentProcessor *siteContentProcessor - sites := s.h.langSite() - for k, v := range sites { - if v.language.Disabled { - continue - } - proc := newSiteContentProcessor(ctx, len(filenames) > 0, v) - contentProcessors[k] = proc - if k == defaultContentLanguage { - defaultContentProcessor = proc - } - g.Go(func() error { - return proc.process(ctx) - }) - } - - var ( - handler captureResultHandler - bundleMap *contentChangeMap - ) - - mainHandler := &contentCaptureResultHandler{contentProcessors: contentProcessors, defaultContentProcessor: defaultContentProcessor} - sourceSpec := source.NewSourceSpec(s.PathSpec, s.BaseFs.Content.Fs) - if s.running() { - // Need to track changes. - bundleMap = s.h.ContentChanges - handler = &captureResultHandlerChain{handlers: []captureBundlesHandler{mainHandler, bundleMap}} - - } else { - handler = mainHandler - } - - c := newCapturer(s.Log, sourceSpec, handler, bundleMap, filenames...) + proc := newPagesProcessor(s.h, sourceSpec, len(filenames) > 0) - err1 := c.capture() + c := newPagesCollector(sourceSpec, s.Log, s.h.ContentChanges, proc, filenames...) - for _, proc := range contentProcessors { - proc.closeInput() - } - - err2 := g.Wait() - - if err1 != nil { - return err1 - } - return err2 + return c.Collect() } func (s *Site) getMenusFromConfig() navigation.Menus { @@ -1831,8 +1755,8 @@ func (s *Site) kindFromFileInfoOrSections(fi *fileInfo, sections []string) strin } func (s *Site) kindFromSections(sections []string) string { - if len(sections) == 0 || len(s.siteCfg.taxonomiesConfig) == 0 { - return page.KindSection + if len(sections) == 0 { + return page.KindHome } sectionPath := path.Join(sections...) diff --git a/hugolib/site_output_test.go b/hugolib/site_output_test.go index 71b87b63692..1536817b091 100644 --- a/hugolib/site_output_test.go +++ b/hugolib/site_output_test.go @@ -32,14 +32,15 @@ import ( func TestSiteWithPageOutputs(t *testing.T) { for _, outputs := range [][]string{{"html", "json", "calendar"}, {"json"}} { + outputs := outputs t.Run(fmt.Sprintf("%v", outputs), func(t *testing.T) { + t.Parallel() doTestSiteWithPageOutputs(t, outputs) }) } } func doTestSiteWithPageOutputs(t *testing.T, outputs []string) { - t.Parallel() outputsStr := strings.Replace(fmt.Sprintf("%q", outputs), " ", ", ", -1) @@ -84,19 +85,16 @@ outputs: %s ` - mf := afero.NewMemMapFs() - - writeToFs(t, mf, "i18n/en.toml", ` + b := newTestSitesBuilder(t).WithConfigFile("toml", siteConfig) + b.WithI18n("en.toml", ` [elbow] other = "Elbow" -`) - writeToFs(t, mf, "i18n/nn.toml", ` +`, "nn.toml", ` [elbow] other = "Olboge" `) - th, h := newTestSitesFromConfig(t, mf, siteConfig, - + b.WithTemplates( // Case issue partials #3333 "layouts/partials/GoHugo.html", `Go Hugo Partial`, "layouts/_default/baseof.json", `START JSON:{{block "main" .}}default content{{ end }}:END JSON`, @@ -133,23 +131,17 @@ Len Pages: {{ .Kind }} {{ len .Site.RegularPages }} Page Number: {{ .Paginator.P `, "layouts/_default/single.html", `{{ define "main" }}{{ .Content }}{{ end }}`, ) - require.Len(t, h.Sites, 2) - - fs := th.Fs - writeSource(t, fs, "content/_index.md", fmt.Sprintf(pageTemplate, "JSON Home", outputsStr)) - writeSource(t, fs, "content/_index.nn.md", fmt.Sprintf(pageTemplate, "JSON Nynorsk Heim", outputsStr)) + b.WithContent("_index.md", fmt.Sprintf(pageTemplate, "JSON Home", outputsStr)) + b.WithContent("_index.nn.md", fmt.Sprintf(pageTemplate, "JSON Nynorsk Heim", outputsStr)) for i := 1; i <= 10; i++ { - writeSource(t, fs, fmt.Sprintf("content/p%d.md", i), fmt.Sprintf(pageTemplate, fmt.Sprintf("Page %d", i), outputsStr)) - + b.WithContent(fmt.Sprintf("p%d.md", i), fmt.Sprintf(pageTemplate, fmt.Sprintf("Page %d", i), outputsStr)) } - err := h.Build(BuildCfg{}) - - require.NoError(t, err) + b.Build(BuildCfg{}) - s := h.Sites[0] + s := b.H.Sites[0] require.Equal(t, "en", s.language.Lang) home := s.getPage(page.KindHome) @@ -163,13 +155,13 @@ Len Pages: {{ .Kind }} {{ len .Site.RegularPages }} Page Number: {{ .Paginator.P // There is currently always a JSON output to make it simpler ... altFormats := lenOut - 1 hasHTML := helpers.InStringArray(outputs, "html") - th.assertFileContent("public/index.json", + b.AssertFileContent("public/index.json", "List JSON", fmt.Sprintf("Alt formats: %d", altFormats), ) if hasHTML { - th.assertFileContent("public/index.json", + b.AssertFileContent("public/index.json", "Alt Output: HTML", "Output/Rel: JSON/alternate|", "Output/Rel: HTML/canonical|", @@ -178,7 +170,7 @@ Len Pages: {{ .Kind }} {{ len .Site.RegularPages }} Page Number: {{ .Paginator.P "OtherShort:

Hi!

", ) - th.assertFileContent("public/index.html", + b.AssertFileContent("public/index.html", // The HTML entity is a deliberate part of this test: The HTML templates are // parsed with html/template. `List HTML|JSON Home|`, @@ -187,21 +179,22 @@ Len Pages: {{ .Kind }} {{ len .Site.RegularPages }} Page Number: {{ .Paginator.P "OtherShort:

Hi!

", "Len Pages: home 10", ) - th.assertFileContent("public/page/2/index.html", "Page Number: 2") - th.assertFileNotExist("public/page/2/index.json") + assert := require.New(t) + b.AssertFileContent("public/page/2/index.html", "Page Number: 2") + assert.False(b.CheckExists("public/page/2/index.json")) - th.assertFileContent("public/nn/index.html", + b.AssertFileContent("public/nn/index.html", "List HTML|JSON Nynorsk Heim|", "nn: Olboge") } else { - th.assertFileContent("public/index.json", + b.AssertFileContent("public/index.json", "Output/Rel: JSON/canonical|", // JSON is plain text, so no need to safeHTML this and that ``, "ShortJSON", "OtherShort:

Hi!

", ) - th.assertFileContent("public/nn/index.json", + b.AssertFileContent("public/nn/index.json", "List JSON|JSON Nynorsk Heim|", "nn: Olboge", "ShortJSON", diff --git a/hugolib/site_sections_test.go b/hugolib/site_sections_test.go index 199947c3131..d4aa9d354b9 100644 --- a/hugolib/site_sections_test.go +++ b/hugolib/site_sections_test.go @@ -25,7 +25,6 @@ import ( ) func TestNestedSections(t *testing.T) { - t.Parallel() var ( assert = require.New(t) @@ -294,7 +293,9 @@ PAG|{{ .Title }}|{{ $sect.InSection . }} home := s.getPage(page.KindHome) for _, test := range tests { + test := test t.Run(fmt.Sprintf("sections %s", test.sections), func(t *testing.T) { + t.Parallel() assert := require.New(t) sections := strings.Split(test.sections, ",") p := s.getPage(page.KindSection, sections...) diff --git a/hugolib/site_stats_test.go b/hugolib/site_stats_test.go index c722037b4eb..bbefc95774f 100644 --- a/hugolib/site_stats_test.go +++ b/hugolib/site_stats_test.go @@ -20,7 +20,6 @@ import ( "testing" "github.com/gohugoio/hugo/helpers" - "github.com/spf13/afero" "github.com/stretchr/testify/require" ) @@ -60,30 +59,28 @@ aliases: [/Ali%d] # Doc ` - th, h := newTestSitesFromConfig(t, afero.NewMemMapFs(), siteConfig, - "layouts/_default/single.html", "Single|{{ .Title }}|{{ .Content }}", - "layouts/_default/list.html", `List|{{ .Title }}|Pages: {{ .Paginator.TotalPages }}|{{ .Content }}`, - "layouts/_default/terms.html", "Terms List|{{ .Title }}|{{ .Content }}", - ) - require.Len(t, h.Sites, 2) + b := newTestSitesBuilder(t).WithConfigFile("toml", siteConfig) - fs := th.Fs + b.WithTemplates( + "_default/single.html", "Single|{{ .Title }}|{{ .Content }}", + "_default/list.html", `List|{{ .Title }}|Pages: {{ .Paginator.TotalPages }}|{{ .Content }}`, + "_default/terms.html", "Terms List|{{ .Title }}|{{ .Content }}", + ) for i := 0; i < 2; i++ { for j := 0; j < 2; j++ { pageID := i + j + 1 - writeSource(t, fs, fmt.Sprintf("content/sect/p%d.md", pageID), + b.WithContent(fmt.Sprintf("content/sect/p%d.md", pageID), fmt.Sprintf(pageTemplate, pageID, fmt.Sprintf("- tag%d", j), fmt.Sprintf("- category%d", j), pageID)) } } for i := 0; i < 5; i++ { - writeSource(t, fs, fmt.Sprintf("content/assets/image%d.png", i+1), "image") + b.WithContent(fmt.Sprintf("assets/image%d.png", i+1), "image") } - err := h.Build(BuildCfg{}) - - assert.NoError(err) + b.Build(BuildCfg{}) + h := b.H stats := []*helpers.ProcessingStats{ h.Sites[0].PathSpec.ProcessingStats, diff --git a/hugolib/site_test.go b/hugolib/site_test.go index 5912abbc9c0..bbf101fc406 100644 --- a/hugolib/site_test.go +++ b/hugolib/site_test.go @@ -362,11 +362,14 @@ func TestShouldNotWriteZeroLengthFilesToDestination(t *testing.T) { // Issue #1176 func TestSectionNaming(t *testing.T) { - t.Parallel() for _, canonify := range []bool{true, false} { for _, uglify := range []bool{true, false} { for _, pluralize := range []bool{true, false} { + canonify := canonify + uglify := uglify + pluralize := pluralize t.Run(fmt.Sprintf("canonify=%t,uglify=%t,pluralize=%t", canonify, uglify, pluralize), func(t *testing.T) { + t.Parallel() doTestSectionNaming(t, canonify, uglify, pluralize) }) } diff --git a/hugolib/taxonomy_test.go b/hugolib/taxonomy_test.go index f4902ae8d65..2edc36d63e1 100644 --- a/hugolib/taxonomy_test.go +++ b/hugolib/taxonomy_test.go @@ -64,14 +64,15 @@ YAML frontmatter with tags and categories taxonomy.` // func TestTaxonomiesWithAndWithoutContentFile(t *testing.T) { for _, uglyURLs := range []bool{false, true} { + uglyURLs := uglyURLs t.Run(fmt.Sprintf("uglyURLs=%t", uglyURLs), func(t *testing.T) { + t.Parallel() doTestTaxonomiesWithAndWithoutContentFile(t, uglyURLs) }) } } func doTestTaxonomiesWithAndWithoutContentFile(t *testing.T, uglyURLs bool) { - t.Parallel() siteConfig := ` baseURL = "http://example.com/blog" @@ -104,25 +105,20 @@ permalinkeds: siteConfig = fmt.Sprintf(siteConfig, uglyURLs) - th, h := newTestSitesFromConfigWithDefaultTemplates(t, siteConfig) - require.Len(t, h.Sites, 1) - - fs := th.Fs - - writeSource(t, fs, "content/p1.md", fmt.Sprintf(pageTemplate, "t1/c1", "- Tag1", "- cAt1", "- o1", "- Pl1")) - writeSource(t, fs, "content/p2.md", fmt.Sprintf(pageTemplate, "t2/c1", "- tag2", "- cAt1", "- o1", "- Pl1")) - writeSource(t, fs, "content/p3.md", fmt.Sprintf(pageTemplate, "t2/c12", "- tag2", "- cat2", "- o1", "- Pl1")) - writeSource(t, fs, "content/p4.md", fmt.Sprintf(pageTemplate, "Hello World", "", "", "- \"Hello Hugo world\"", "- Pl1")) + b := newTestSitesBuilder(t).WithConfigFile("toml", siteConfig) - writeNewContentFile(t, fs.Source, "Category Terms", "2017-01-01", "content/categories/_index.md", 10) - writeNewContentFile(t, fs.Source, "Tag1 List", "2017-01-01", "content/tags/Tag1/_index.md", 10) + b.WithContent( + "p1.md", fmt.Sprintf(pageTemplate, "t1/c1", "- Tag1", "- cAt1", "- o1", "- Pl1"), + "p2.md", fmt.Sprintf(pageTemplate, "t2/c1", "- tag2", "- cAt1", "- o1", "- Pl1"), + "p3.md", fmt.Sprintf(pageTemplate, "t2/c12", "- tag2", "- cat2", "- o1", "- Pl1"), + "p4.md", fmt.Sprintf(pageTemplate, "Hello World", "", "", "- \"Hello Hugo world\"", "- Pl1"), + "categories/_index.md", newTestPage("Category Terms", "2017-01-01", 10), + "tags/Tag1/_index.md", newTestPage("Tag1 List", "2017-01-01", 10), + // https://github.com/gohugoio/hugo/issues/5847 + "/tags/not-used/_index.md", newTestPage("Unused Tag List", "2018-01-01", 10), + ) - // https://github.com/gohugoio/hugo/issues/5847 - writeNewContentFile(t, fs.Source, "Unused Tag List", "2018-01-01", "content/tags/not-used/_index.md", 10) - - err := h.Build(BuildCfg{}) - - require.NoError(t, err) + b.Build(BuildCfg{}) // So what we have now is: // 1. categories with terms content page, but no content page for the only c1 category @@ -138,26 +134,26 @@ permalinkeds: } // 1. - th.assertFileContent(pathFunc("public/categories/cat1/index.html"), "List", "cAt1") - th.assertFileContent(pathFunc("public/categories/index.html"), "Terms List", "Category Terms") + b.AssertFileContent(pathFunc("public/categories/cat1/index.html"), "List", "cAt1") + b.AssertFileContent(pathFunc("public/categories/index.html"), "Taxonomy Term Page", "Category Terms") // 2. - th.assertFileContent(pathFunc("public/tags/tag2/index.html"), "List", "tag2") - th.assertFileContent(pathFunc("public/tags/tag1/index.html"), "List", "Tag1") - th.assertFileContent(pathFunc("public/tags/index.html"), "Terms List", "Tags") + b.AssertFileContent(pathFunc("public/tags/tag2/index.html"), "List", "tag2") + b.AssertFileContent(pathFunc("public/tags/tag1/index.html"), "List", "Tag1") + b.AssertFileContent(pathFunc("public/tags/index.html"), "Taxonomy Term Page", "Tags") // 3. - th.assertFileContent(pathFunc("public/others/o1/index.html"), "List", "o1") - th.assertFileContent(pathFunc("public/others/index.html"), "Terms List", "Others") + b.AssertFileContent(pathFunc("public/others/o1/index.html"), "List", "o1") + b.AssertFileContent(pathFunc("public/others/index.html"), "Taxonomy Term Page", "Others") // 4. - th.assertFileContent(pathFunc("public/perma/pl1/index.html"), "List", "Pl1") + b.AssertFileContent(pathFunc("public/perma/pl1/index.html"), "List", "Pl1") // This looks kind of funky, but the taxonomy terms do not have a permalinks definition, // for good reasons. - th.assertFileContent(pathFunc("public/permalinkeds/index.html"), "Terms List", "Permalinkeds") + b.AssertFileContent(pathFunc("public/permalinkeds/index.html"), "Taxonomy Term Page", "Permalinkeds") - s := h.Sites[0] + s := b.H.Sites[0] // Make sure that each page.KindTaxonomyTerm page has an appropriate number // of page.KindTaxonomy pages in its Pages slice. @@ -204,7 +200,7 @@ permalinkeds: require.Equal(t, "Hello Hugo world", helloWorld.Title()) // Issue #2977 - th.assertFileContent(pathFunc("public/empties/index.html"), "Terms List", "Empties") + b.AssertFileContent(pathFunc("public/empties/index.html"), "Taxonomy Term Page", "Empties") } diff --git a/hugolib/template_engines_test.go b/hugolib/template_engines_test.go index 6a046c9f59a..ec229a29941 100644 --- a/hugolib/template_engines_test.go +++ b/hugolib/template_engines_test.go @@ -24,7 +24,6 @@ import ( ) func TestAllTemplateEngines(t *testing.T) { - t.Parallel() noOp := func(s string) string { return s } @@ -48,8 +47,10 @@ func TestAllTemplateEngines(t *testing.T) { {"html", noOp}, {"ace", noOp}, } { + config := config t.Run(config.suffix, func(t *testing.T) { + t.Parallel() doTestTemplateEngine(t, config.suffix, config.templateFixer) }) } diff --git a/hugolib/template_test.go b/hugolib/template_test.go index 3ec81323b4b..6ed9643c7fd 100644 --- a/hugolib/template_test.go +++ b/hugolib/template_test.go @@ -25,7 +25,6 @@ import ( ) func TestTemplateLookupOrder(t *testing.T) { - t.Parallel() var ( fs *hugofs.Fs cfg *viper.Viper @@ -193,22 +192,26 @@ func TestTemplateLookupOrder(t *testing.T) { }, } { - cfg, fs = newTestCfg() - th = testHelper{cfg, fs, t} + this := this + t.Run(this.name, func(t *testing.T) { + // TODO(bep) there are some function vars need to pull down here to enable => t.Parallel() + cfg, fs = newTestCfg() + th = testHelper{cfg, fs, t} - for i := 1; i <= 3; i++ { - writeSource(t, fs, filepath.Join("content", fmt.Sprintf("sect%d", i), fmt.Sprintf("page%d.md", i)), `--- + for i := 1; i <= 3; i++ { + writeSource(t, fs, filepath.Join("content", fmt.Sprintf("sect%d", i), fmt.Sprintf("page%d.md", i)), `--- title: Template test --- Some content `) - } + } - this.setup(t) + this.setup(t) - buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{}) - t.Log(this.name) - this.assert(t) + buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{}) + //helpers.PrintFs(s.BaseFs.Layouts.Fs, "", os.Stdout) + this.assert(t) + }) } } diff --git a/hugolib/testhelpers_test.go b/hugolib/testhelpers_test.go index 8c72e10d0c1..37215e8a027 100644 --- a/hugolib/testhelpers_test.go +++ b/hugolib/testhelpers_test.go @@ -2,13 +2,17 @@ package hugolib import ( "io" - "io/ioutil" "path/filepath" "runtime" "strconv" "testing" "unicode/utf8" + "github.com/gohugoio/hugo/parser/metadecoders" + + "github.com/gohugoio/hugo/parser" + "github.com/pkg/errors" + "bytes" "fmt" "regexp" @@ -39,9 +43,12 @@ import ( ) type sitesBuilder struct { - Cfg config.Provider - Fs *hugofs.Fs - T testing.TB + Cfg config.Provider + environ []string + + Fs *hugofs.Fs + T testing.TB + depsCfg deps.DepsCfg *require.Assertions @@ -60,13 +67,16 @@ type sitesBuilder struct { theme string // Default toml - configFormat string + configFormat string + configFileSet bool + viperSet bool // Default is empty. // TODO(bep) revisit this and consider always setting it to something. // Consider this in relation to using the BaseFs.PublishFs to all publishing. workingDir string + addNothing bool // Base data/content contentFilePairs []string templateFilePairs []string @@ -94,18 +104,22 @@ func newTestSitesBuilder(t testing.TB) *sitesBuilder { return &sitesBuilder{T: t, Assertions: require.New(t), Fs: fs, configFormat: "toml", dumper: litterOptions} } -func createTempDir(prefix string) (string, func(), error) { - workDir, err := ioutil.TempDir("", prefix) - if err != nil { - return "", nil, err - } +func newTestSitesBuilderFromDepsCfg(t testing.TB, d deps.DepsCfg) *sitesBuilder { + assert := require.New(t) - if runtime.GOOS == "darwin" && !strings.HasPrefix(workDir, "/private") { - // To get the entry folder in line with the rest. This its a little bit - // mysterious, but so be it. - workDir = "/private" + workDir + litterOptions := litter.Options{ + HidePrivateFields: true, + StripPackageNames: true, + Separator: " ", } - return workDir, func() { os.RemoveAll(workDir) }, nil + + b := &sitesBuilder{T: t, Assertions: assert, depsCfg: d, Fs: d.Fs, dumper: litterOptions} + workingDir := d.Cfg.GetString("workingDir") + + b.WithWorkingDir(workingDir) + + return b.WithViper(d.Cfg.(*viper.Viper)) + } func (s *sitesBuilder) Running() *sitesBuilder { @@ -113,17 +127,31 @@ func (s *sitesBuilder) Running() *sitesBuilder { return s } +func (s *sitesBuilder) WithNothingAdded() *sitesBuilder { + s.addNothing = true + return s +} + func (s *sitesBuilder) WithLogger(logger *loggers.Logger) *sitesBuilder { s.logger = logger return s } func (s *sitesBuilder) WithWorkingDir(dir string) *sitesBuilder { - s.workingDir = dir + s.workingDir = filepath.FromSlash(dir) + return s +} + +func (s *sitesBuilder) WithEnviron(env ...string) *sitesBuilder { + for i := 0; i < len(env); i += 2 { + s.environ = append(s.environ, fmt.Sprintf("%s=%s", env[i], env[i+1])) + } return s } func (s *sitesBuilder) WithConfigTemplate(data interface{}, format, configTemplate string) *sitesBuilder { + s.T.Helper() + if format == "" { format = "toml" } @@ -138,32 +166,59 @@ func (s *sitesBuilder) WithConfigTemplate(data interface{}, format, configTempla } func (s *sitesBuilder) WithViper(v *viper.Viper) *sitesBuilder { - loadDefaultSettingsFor(v) - s.Cfg = v + s.T.Helper() + if s.configFileSet { + s.T.Fatal("WithViper: use Viper or config.toml, not both") + } + defer func() { + s.viperSet = true + }() - return s + // Write to a config file to make sure the tests follow the same code path. + var buff bytes.Buffer + m := v.AllSettings() + s.Assertions.NoError(parser.InterfaceToConfig(m, metadecoders.TOML, &buff)) + return s.WithConfigFile("toml", buff.String()) } func (s *sitesBuilder) WithConfigFile(format, conf string) *sitesBuilder { - writeSource(s.T, s.Fs, "config."+format, conf) + s.T.Helper() + if s.viperSet { + s.T.Fatal("WithConfigFile: use Viper or config.toml, not both") + } + s.configFileSet = true + filename := s.absFilename("config." + format) + writeSource(s.T, s.Fs, filename, conf) s.configFormat = format return s } func (s *sitesBuilder) WithThemeConfigFile(format, conf string) *sitesBuilder { + s.T.Helper() if s.theme == "" { s.theme = "test-theme" } filename := filepath.Join("themes", s.theme, "config."+format) - writeSource(s.T, s.Fs, filename, conf) + writeSource(s.T, s.Fs, s.absFilename(filename), conf) return s } -func (s *sitesBuilder) WithSourceFile(filename, content string) *sitesBuilder { - writeSource(s.T, s.Fs, filepath.FromSlash(filename), content) +func (s *sitesBuilder) WithSourceFile(filenameContent ...string) *sitesBuilder { + s.T.Helper() + for i := 0; i < len(filenameContent); i += 2 { + writeSource(s.T, s.Fs, s.absFilename(filenameContent[i]), filenameContent[i+1]) + } return s } +func (s *sitesBuilder) absFilename(filename string) string { + filename = filepath.FromSlash(filename) + if s.workingDir != "" && !strings.HasPrefix(filename, s.workingDir) { + filename = filepath.Join(s.workingDir, filename) + } + return filename +} + const commonConfigSections = ` [services] @@ -191,10 +246,12 @@ privacyEnhanced = true ` func (s *sitesBuilder) WithSimpleConfigFile() *sitesBuilder { + s.T.Helper() return s.WithSimpleConfigFileAndBaseURL("http://example.com/") } func (s *sitesBuilder) WithSimpleConfigFileAndBaseURL(baseURL string) *sitesBuilder { + s.T.Helper() config := fmt.Sprintf("baseURL = %q", baseURL) config = config + commonConfigSections @@ -323,7 +380,7 @@ func (s *sitesBuilder) EditFiles(filenameContent ...string) *sitesBuilder { for i := 0; i < len(filenameContent); i += 2 { filename, content := filepath.FromSlash(filenameContent[i]), filenameContent[i+1] changedFiles = append(changedFiles, filename) - writeSource(s.T, s.Fs, filename, content) + writeSource(s.T, s.Fs, s.absFilename(filename), content) } s.changedFiles = changedFiles @@ -354,6 +411,7 @@ func (s *sitesBuilder) writeFilePairs(folder string, filenameContent []string) * func (s *sitesBuilder) CreateSites() *sitesBuilder { if err := s.CreateSitesE(); err != nil { + herrors.PrintStackTrace(err) s.Fatalf("Failed to create sites: %s", err) } @@ -361,34 +419,71 @@ func (s *sitesBuilder) CreateSites() *sitesBuilder { } func (s *sitesBuilder) LoadConfig() error { - cfg, _, err := LoadConfig(ConfigSourceDescriptor{Fs: s.Fs.Source, Filename: "config." + s.configFormat}) + if !s.configFileSet { + s.WithSimpleConfigFile() + } + + cfg, _, err := LoadConfig(ConfigSourceDescriptor{ + WorkingDir: s.workingDir, + Fs: s.Fs.Source, + Environ: s.environ, + Filename: "config." + s.configFormat}, func(cfg config.Provider) error { + + return nil + }) + if err != nil { return err } + s.Cfg = cfg + return nil } func (s *sitesBuilder) CreateSitesE() error { - s.addDefaults() - s.writeFilePairs("content", s.contentFilePairs) - s.writeFilePairs("content", s.contentFilePairsAdded) - s.writeFilePairs("layouts", s.templateFilePairs) - s.writeFilePairs("layouts", s.templateFilePairsAdded) - s.writeFilePairs("data", s.dataFilePairs) - s.writeFilePairs("data", s.dataFilePairsAdded) - s.writeFilePairs("i18n", s.i18nFilePairs) - s.writeFilePairs("i18n", s.i18nFilePairsAdded) - - if s.Cfg == nil { - if err := s.LoadConfig(); err != nil { - return err + if !s.addNothing { + if _, ok := s.Fs.Source.(*afero.OsFs); ok { + for _, dir := range []string{ + "content/sect", + "layouts/_default", + "layouts/partials", + "layouts/shortcodes", + "data", + "i18n", + } { + if err := os.MkdirAll(filepath.Join(s.workingDir, dir), 0777); err != nil { + return errors.Wrapf(err, "failed to create %q", dir) + } + } } + + s.addDefaults() + s.writeFilePairs("content", s.contentFilePairsAdded) + s.writeFilePairs("layouts", s.templateFilePairsAdded) + s.writeFilePairs("data", s.dataFilePairsAdded) + s.writeFilePairs("i18n", s.i18nFilePairsAdded) + + s.writeFilePairs("i18n", s.i18nFilePairs) + s.writeFilePairs("data", s.dataFilePairs) + s.writeFilePairs("content", s.contentFilePairs) + s.writeFilePairs("layouts", s.templateFilePairs) + } - sites, err := NewHugoSites(deps.DepsCfg{Fs: s.Fs, Cfg: s.Cfg, Logger: s.logger, Running: s.running}) + if err := s.LoadConfig(); err != nil { + return errors.Wrap(err, "failed to load config") + } + + depsCfg := s.depsCfg + depsCfg.Fs = s.Fs + depsCfg.Cfg = s.Cfg + depsCfg.Logger = s.logger + depsCfg.Running = s.running + + sites, err := NewHugoSites(depsCfg) if err != nil { - return err + return errors.Wrap(err, "failed to create sites") } s.H = sites @@ -404,10 +499,12 @@ func (s *sitesBuilder) BuildE(cfg BuildCfg) error { } func (s *sitesBuilder) Build(cfg BuildCfg) *sitesBuilder { + s.T.Helper() return s.build(cfg, false) } func (s *sitesBuilder) BuildFail(cfg BuildCfg) *sitesBuilder { + s.T.Helper() return s.build(cfg, true) } @@ -528,14 +625,8 @@ hello: } func (s *sitesBuilder) Fatalf(format string, args ...interface{}) { - Fatalf(s.T, format, args...) -} - -func Fatalf(t testing.TB, format string, args ...interface{}) { - trace := stackTrace() - format = format + "\n%s" - args = append(args, trace) - t.Fatalf(format, args...) + s.T.Helper() + s.T.Fatalf(format, args...) } func stackTrace() string { @@ -543,9 +634,10 @@ func stackTrace() string { } func (s *sitesBuilder) AssertFileContentFn(filename string, f func(s string) bool) { + s.T.Helper() content := s.FileContent(filename) if !f(content) { - s.Fatalf("Assert failed for %q", filename) + s.Fatalf("Assert failed for %q in content\n%s", filename, content) } } @@ -554,6 +646,11 @@ func (s *sitesBuilder) AssertHome(matches ...string) { } func (s *sitesBuilder) AssertFileContent(filename string, matches ...string) { + s.T.Helper() + filename = filepath.FromSlash(filename) + if !strings.HasPrefix(filename, s.workingDir) { + filename = filepath.Join(s.workingDir, filename) + } content := s.FileContent(filename) for _, match := range matches { if !strings.Contains(content, match) { @@ -563,10 +660,12 @@ func (s *sitesBuilder) AssertFileContent(filename string, matches ...string) { } func (s *sitesBuilder) FileContent(filename string) string { + s.T.Helper() return readDestination(s.T, s.Fs, filename) } func (s *sitesBuilder) AssertObject(expected string, object interface{}) { + s.T.Helper() got := s.dumper.Sdump(object) expected = strings.TrimSpace(expected) @@ -633,17 +732,41 @@ func (th testHelper) replaceDefaultContentLanguageValue(value string) string { return value } -func newTestCfg() (*viper.Viper, *hugofs.Fs) { +func loadTestConfig(fs afero.Fs, withConfig ...func(cfg config.Provider) error) (*viper.Viper, error) { + v, _, err := LoadConfig(ConfigSourceDescriptor{Fs: fs}, withConfig...) + return v, err +} +func newTestCfgBasic() (*viper.Viper, *hugofs.Fs) { + mm := afero.NewMemMapFs() v := viper.New() - fs := hugofs.NewMem(v) + v.Set("defaultContentLanguageInSubdir", true) - v.SetFs(fs.Source) + fs := hugofs.NewFrom(hugofs.NewBaseFileDecorator(mm), v) - loadDefaultSettingsFor(v) + return v, fs - // Default is false, but true is easier to use as default in tests - v.Set("defaultContentLanguageInSubdir", true) +} + +func newTestCfg(withConfig ...func(cfg config.Provider) error) (*viper.Viper, *hugofs.Fs) { + mm := afero.NewMemMapFs() + + v, err := loadTestConfig(mm, func(cfg config.Provider) error { + // Default is false, but true is easier to use as default in tests + cfg.Set("defaultContentLanguageInSubdir", true) + + for _, w := range withConfig { + w(cfg) + } + + return nil + }) + + if err != nil && err != ErrNoConfigFile { + panic(err) + } + + fs := hugofs.NewFrom(hugofs.NewBaseFileDecorator(mm), v) return v, fs @@ -651,9 +774,10 @@ func newTestCfg() (*viper.Viper, *hugofs.Fs) { func newTestSitesFromConfig(t testing.TB, afs afero.Fs, tomlConfig string, layoutPathContentPairs ...string) (testHelper, *HugoSites) { if len(layoutPathContentPairs)%2 != 0 { - Fatalf(t, "Layouts must be provided in pairs") + t.Fatalf("Layouts must be provided in pairs") } + writeToFs(t, afs, filepath.Join("content", ".gitkeep"), "") writeToFs(t, afs, "config.toml", tomlConfig) cfg, err := LoadConfigDefault(afs) @@ -673,14 +797,6 @@ func newTestSitesFromConfig(t testing.TB, afs afero.Fs, tomlConfig string, layou return th, h } -func newTestSitesFromConfigWithDefaultTemplates(t testing.TB, tomlConfig string) (testHelper, *HugoSites) { - return newTestSitesFromConfig(t, afero.NewMemMapFs(), tomlConfig, - "layouts/_default/single.html", "Single|{{ .Title }}|{{ .Content }}", - "layouts/_default/list.html", "List|{{ .Title }}|{{ .Content }}", - "layouts/_default/terms.html", "Terms List|{{ .Title }}|{{ .Content }}", - ) -} - func createWithTemplateFromNameValues(additionalTemplates ...string) func(templ tpl.TemplateHandler) error { return func(templ tpl.TemplateHandler) error { @@ -694,12 +810,16 @@ func createWithTemplateFromNameValues(additionalTemplates ...string) func(templ } } +// TODO(bep) replace these with the builder func buildSingleSite(t testing.TB, depsCfg deps.DepsCfg, buildCfg BuildCfg) *Site { return buildSingleSiteExpected(t, false, false, depsCfg, buildCfg) } func buildSingleSiteExpected(t testing.TB, expectSiteInitEror, expectBuildError bool, depsCfg deps.DepsCfg, buildCfg BuildCfg) *Site { - h, err := NewHugoSites(depsCfg) + t.Helper() + b := newTestSitesBuilderFromDepsCfg(t, depsCfg).WithNothingAdded() + + err := b.CreateSitesE() if expectSiteInitEror { require.Error(t, err) @@ -708,6 +828,8 @@ func buildSingleSiteExpected(t testing.TB, expectSiteInitEror, expectBuildError require.NoError(t, err) } + h := b.H + require.Len(t, h.Sites, 1) if expectBuildError { @@ -751,9 +873,13 @@ func content(c resource.ContentProvider) string { func dumpPages(pages ...page.Page) { fmt.Println("---------") for i, p := range pages { - fmt.Printf("%d: Kind: %s Title: %-10s RelPermalink: %-10s Path: %-10s sections: %s\n", + var meta interface{} + if p.File() != nil && p.File().FileInfo() != nil { + meta = p.File().FileInfo().Meta() + } + fmt.Printf("%d: Kind: %s Title: %-10s RelPermalink: %-10s Path: %-10s sections: %s Lang: %s Meta: %v\n", i+1, - p.Kind(), p.Title(), p.RelPermalink(), p.Path(), p.SectionsPath()) + p.Kind(), p.Title(), p.RelPermalink(), p.Path(), p.SectionsPath(), p.Lang(), meta) } } @@ -802,3 +928,10 @@ func parallel(t *testing.T) { t.Parallel() } } + +func skipSymlink(t *testing.T) { + if runtime.GOOS == "windows" && os.Getenv("CI") == "" { + t.Skip("skip symlink test on local Windows (needs admin)") + } + +} diff --git a/langs/config.go b/langs/config.go new file mode 100644 index 00000000000..927f3558fa4 --- /dev/null +++ b/langs/config.go @@ -0,0 +1,217 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package langs + +import ( + "fmt" + "path/filepath" + "sort" + "strings" + + "github.com/gohugoio/hugo/common/maps" + + "github.com/spf13/cast" + + "github.com/pkg/errors" + + "github.com/gohugoio/hugo/config" +) + +type LanguagesConfig struct { + Languages Languages + Multihost bool + DefaultContentLanguageInSubdir bool +} + +func LoadLanguageSettings(cfg config.Provider, oldLangs Languages) (c LanguagesConfig, err error) { + + defaultLang := cfg.GetString("defaultContentLanguage") + if defaultLang == "" { + defaultLang = "en" + cfg.Set("defaultContentLanguage", defaultLang) + } + + var languages map[string]interface{} + + languagesFromConfig := cfg.GetStringMap("languages") + disableLanguages := cfg.GetStringSlice("disableLanguages") + + if len(disableLanguages) == 0 { + languages = languagesFromConfig + } else { + languages = make(map[string]interface{}) + for k, v := range languagesFromConfig { + for _, disabled := range disableLanguages { + if disabled == defaultLang { + return c, fmt.Errorf("cannot disable default language %q", defaultLang) + } + + if strings.EqualFold(k, disabled) { + v.(map[string]interface{})["disabled"] = true + break + } + } + languages[k] = v + } + } + + var languages2 Languages + + if len(languages) == 0 { + languages2 = append(languages2, NewDefaultLanguage(cfg)) + } else { + languages2, err = toSortedLanguages(cfg, languages) + if err != nil { + return c, errors.Wrap(err, "Failed to parse multilingual config") + } + } + + if oldLangs != nil { + // When in multihost mode, the languages are mapped to a server, so + // some structural language changes will need a restart of the dev server. + // The validation below isn't complete, but should cover the most + // important cases. + var invalid bool + if languages2.IsMultihost() != oldLangs.IsMultihost() { + invalid = true + } else { + if languages2.IsMultihost() && len(languages2) != len(oldLangs) { + invalid = true + } + } + + if invalid { + return c, errors.New("language change needing a server restart detected") + } + + if languages2.IsMultihost() { + // We need to transfer any server baseURL to the new language + for i, ol := range oldLangs { + nl := languages2[i] + nl.Set("baseURL", ol.GetString("baseURL")) + } + } + } + + // The defaultContentLanguage is something the user has to decide, but it needs + // to match a language in the language definition list. + langExists := false + for _, lang := range languages2 { + if lang.Lang == defaultLang { + langExists = true + break + } + } + + if !langExists { + return c, fmt.Errorf("site config value %q for defaultContentLanguage does not match any language definition", defaultLang) + } + + c.Languages = languages2 + c.Multihost = languages2.IsMultihost() + c.DefaultContentLanguageInSubdir = c.Multihost + + sortedDefaultFirst := make(Languages, len(c.Languages)) + for i, v := range c.Languages { + sortedDefaultFirst[i] = v + } + sort.Slice(sortedDefaultFirst, func(i, j int) bool { + li, lj := sortedDefaultFirst[i], sortedDefaultFirst[j] + if li.Lang == defaultLang { + return true + } + + if lj.Lang == defaultLang { + return false + } + + return i < j + }) + + cfg.Set("languagesSorted", c.Languages) + cfg.Set("languagesSortedDefaultFirst", sortedDefaultFirst) + cfg.Set("multilingual", len(languages2) > 1) + + multihost := c.Multihost + + if multihost { + cfg.Set("defaultContentLanguageInSubdir", true) + cfg.Set("multihost", true) + } + + if multihost { + // The baseURL may be provided at the language level. If that is true, + // then every language must have a baseURL. In this case we always render + // to a language sub folder, which is then stripped from all the Permalink URLs etc. + for _, l := range languages2 { + burl := l.GetLocal("baseURL") + if burl == nil { + return c, errors.New("baseURL must be set on all or none of the languages") + } + } + + } + + return c, nil +} + +func toSortedLanguages(cfg config.Provider, l map[string]interface{}) (Languages, error) { + languages := make(Languages, len(l)) + i := 0 + + for lang, langConf := range l { + langsMap, err := cast.ToStringMapE(langConf) + + if err != nil { + return nil, fmt.Errorf("Language config is not a map: %T", langConf) + } + + language := NewLanguage(lang, cfg) + + for loki, v := range langsMap { + switch loki { + case "title": + language.Title = cast.ToString(v) + case "languagename": + language.LanguageName = cast.ToString(v) + case "weight": + language.Weight = cast.ToInt(v) + case "contentdir": + language.ContentDir = filepath.Clean(cast.ToString(v)) + case "disabled": + language.Disabled = cast.ToBool(v) + case "params": + m := cast.ToStringMap(v) + // Needed for case insensitive fetching of params values + maps.ToLower(m) + for k, vv := range m { + language.SetParam(k, vv) + } + } + + // Put all into the Params map + language.SetParam(loki, v) + + // Also set it in the configuration map (for baseURL etc.) + language.Set(loki, v) + } + + languages[i] = language + i++ + } + + sort.Sort(languages) + + return languages, nil +} diff --git a/langs/i18n/i18n_test.go b/langs/i18n/i18n_test.go index b67cabc5524..e08210848fe 100644 --- a/langs/i18n/i18n_test.go +++ b/langs/i18n/i18n_test.go @@ -17,11 +17,13 @@ import ( "path/filepath" "testing" + "github.com/gohugoio/hugo/modules" + "github.com/gohugoio/hugo/tpl/tplimpl" "github.com/gohugoio/hugo/common/loggers" - "github.com/gohugoio/hugo/htesting" "github.com/gohugoio/hugo/langs" + "github.com/gohugoio/hugo/resources/page" "github.com/spf13/afero" "github.com/spf13/viper" @@ -199,7 +201,7 @@ func newDepsConfig(tp *TranslationProvider, cfg config.Provider, fs *hugofs.Fs) l.Set("i18nDir", "i18n") return deps.DepsCfg{ Language: l, - Site: htesting.NewTestHugoSite(), + Site: page.NewDummyHugoSite(cfg), Cfg: cfg, Fs: fs, Logger: logger, @@ -219,6 +221,13 @@ func getConfig() *viper.Viper { v.Set("assetDir", "assets") v.Set("resourceDir", "resources") v.Set("publishDir", "public") + langs.LoadLanguageSettings(v, nil) + mod, err := modules.CreateProjectModule(v) + if err != nil { + panic(err) + } + v.Set("allModules", modules.Modules{mod}) + return v } diff --git a/langs/i18n/translationProvider.go b/langs/i18n/translationProvider.go index 74e144007ef..c7b4839eef1 100644 --- a/langs/i18n/translationProvider.go +++ b/langs/i18n/translationProvider.go @@ -40,8 +40,7 @@ func NewTranslationProvider() *TranslationProvider { // Update updates the i18n func in the provided Deps. func (tp *TranslationProvider) Update(d *deps.Deps) error { - sp := source.NewSourceSpec(d.PathSpec, d.BaseFs.SourceFilesystems.I18n.Fs) - src := sp.NewFilesystem("") + spec := source.NewSourceSpec(d.PathSpec, nil) i18nBundle := bundle.New() @@ -51,25 +50,33 @@ func (tp *TranslationProvider) Update(d *deps.Deps) error { } var newLangs []string - for _, r := range src.Files() { - currentSpec := language.GetPluralSpec(r.BaseFileName()) - if currentSpec == nil { - // This may is a language code not supported by go-i18n, it may be - // Klingon or ... not even a fake language. Make sure it works. - newLangs = append(newLangs, r.BaseFileName()) + for _, dir := range d.BaseFs.I18n.Dirs { + src := spec.NewFilesystemFromFileMetaInfo(dir) + + files, err := src.Files() + if err != nil { + return err } - } - if len(newLangs) > 0 { - language.RegisterPluralSpec(newLangs, en) - } + for _, r := range files { + currentSpec := language.GetPluralSpec(r.BaseFileName()) + if currentSpec == nil { + // This may is a language code not supported by go-i18n, it may be + // Klingon or ... not even a fake language. Make sure it works. + newLangs = append(newLangs, r.BaseFileName()) + } + } - // The source files are ordered so the most important comes first. Since this is a - // last key win situation, we have to reverse the iteration order. - files := src.Files() - for i := len(files) - 1; i >= 0; i-- { - if err := addTranslationFile(i18nBundle, files[i]); err != nil { - return err + if len(newLangs) > 0 { + language.RegisterPluralSpec(newLangs, en) + } + + // The source files are ordered so the most important comes first. Since this is a + // last key win situation, we have to reverse the iteration order. + for i := len(files) - 1; i >= 0; i-- { + if err := addTranslationFile(i18nBundle, files[i]); err != nil { + return err + } } } @@ -81,8 +88,8 @@ func (tp *TranslationProvider) Update(d *deps.Deps) error { } -func addTranslationFile(bundle *bundle.Bundle, r source.ReadableFile) error { - f, err := r.Open() +func addTranslationFile(bundle *bundle.Bundle, r source.File) error { + f, err := r.FileInfo().Meta().Open() if err != nil { return _errors.Wrapf(err, "failed to open translations file %q:", r.LogicalName()) } @@ -101,14 +108,15 @@ func (tp *TranslationProvider) Clone(d *deps.Deps) error { return nil } -func errWithFileContext(inerr error, r source.ReadableFile) error { - rfi, ok := r.FileInfo().(hugofs.RealFilenameInfo) +func errWithFileContext(inerr error, r source.File) error { + fim, ok := r.FileInfo().(hugofs.FileMetaInfo) if !ok { return inerr } - realFilename := rfi.RealFilename() - f, err := r.Open() + meta := fim.Meta() + realFilename := meta.Filename() + f, err := meta.Open() if err != nil { return inerr } diff --git a/langs/language.go b/langs/language.go index 14e3263aeb9..f71b0255b38 100644 --- a/langs/language.go +++ b/langs/language.go @@ -78,12 +78,7 @@ func NewLanguage(lang string, cfg config.Provider) *Language { } maps.ToLower(params) - defaultContentDir := cfg.GetString("contentDir") - if defaultContentDir == "" { - panic("contentDir not set") - } - - l := &Language{Lang: lang, ContentDir: defaultContentDir, Cfg: cfg, params: params, settings: make(map[string]interface{})} + l := &Language{Lang: lang, ContentDir: cfg.GetString("contentDir"), Cfg: cfg, params: params, settings: make(map[string]interface{})} return l } @@ -132,6 +127,24 @@ func (l *Language) Params() map[string]interface{} { return l.params } +func (l Languages) AsSet() map[string]bool { + m := make(map[string]bool) + for _, lang := range l { + m[lang.Lang] = true + } + + return m +} + +func (l Languages) AsOrdinalSet() map[string]int { + m := make(map[string]int) + for i, lang := range l { + m[lang.Lang] = i + } + + return m +} + // IsMultihost returns whether there are more than one language and at least one of // the languages has baseURL specificed on the language level. func (l Languages) IsMultihost() bool { diff --git a/magefile.go b/magefile.go index 3b74a7e940e..d0b7c8d98a6 100644 --- a/magefile.go +++ b/magefile.go @@ -143,20 +143,31 @@ func Check() { mg.Deps(TestRace) } +func testGoFlags() string { + if isCI() { + return "" + } + + return "-test.short" +} + // Run tests in 32-bit mode // Note that we don't run with the extended tag. Currently not supported in 32 bit. func Test386() error { - return sh.RunWith(map[string]string{"GOARCH": "386"}, goexe, "test", "./...") + env := map[string]string{"GOARCH": "386", "GOFLAGS": testGoFlags()} + return sh.RunWith(env, goexe, "test", "./...") } // Run tests func Test() error { - return sh.Run(goexe, "test", "./...", "-tags", buildTags()) + env := map[string]string{"GOFLAGS": testGoFlags()} + return sh.RunWith(env, goexe, "test", "./...", "-tags", buildTags()) } // Run tests with race detector func TestRace() error { - return sh.Run(goexe, "test", "-race", "./...", "-tags", buildTags()) + env := map[string]string{"GOFLAGS": testGoFlags()} + return sh.RunWith(env, goexe, "test", "-race", "./...", "-tags", buildTags()) } // Run gofmt linter @@ -296,6 +307,10 @@ func isGoLatest() bool { return strings.Contains(runtime.Version(), "1.11") } +func isCI() bool { + return os.Getenv("CI") != "" +} + func buildTags() string { // To build the extended Hugo SCSS/SASS enabled version, build with // HUGO_BUILD_TAGS=extended mage install etc. diff --git a/modules/client.go b/modules/client.go new file mode 100644 index 00000000000..1e4f5437a1f --- /dev/null +++ b/modules/client.go @@ -0,0 +1,537 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package modules + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/gohugoio/hugo/config" + + "github.com/rogpeppe/go-internal/module" + + "github.com/gohugoio/hugo/common/hugio" + + "github.com/pkg/errors" + "github.com/spf13/afero" +) + +var ( + fileSeparator = string(os.PathSeparator) +) + +const ( + goBinaryStatusOK goBinaryStatus = iota + goBinaryStatusNotFound + goBinaryStatusTooOld +) + +// The "vendor" dir is reserved for Go Modules. +const vendord = "_vendor" + +const ( + goModFilename = "go.mod" + goSumFilename = "go.sum" +) + +type ClientConfig struct { + Fs afero.Fs + IgnoreVendor bool + WorkingDir string + ThemesDir string // Absolute directory path + ModProxy string + ModuleConfig Config +} + +// NewClient creates a new Client that can be used to manage the Hugo Components +// in a given workingDir. +// The Client will resolve the dependencies recursively, but needs the top +// level imports to start out. +func NewClient(cfg ClientConfig) *Client { + fs := cfg.Fs + + n := filepath.Join(cfg.WorkingDir, goModFilename) + goModEnabled, _ := afero.Exists(fs, n) + var goModFilename string + if goModEnabled { + goModFilename = n + } + + env := os.Environ() + mcfg := cfg.ModuleConfig + + config.SetEnvVars(&env, + "PWD", cfg.WorkingDir, + "GOPROXY", mcfg.Proxy, + "GOPRIVATE", mcfg.Private, + "GONOPROXY", mcfg.NoProxy) + + return &Client{ + fs: fs, + ignoreVendor: cfg.IgnoreVendor, + workingDir: cfg.WorkingDir, + themesDir: cfg.ThemesDir, + moduleConfig: mcfg, + environ: env, + GoModulesFilename: goModFilename} +} + +// Client contains most of the API provided by this package. +type Client struct { + fs afero.Fs + + // Ignore any _vendor directory. + ignoreVendor bool + + // Absolute path to the project dir. + workingDir string + + // Absolute path to the project's themes dir. + themesDir string + + // The top level module config + moduleConfig Config + + // Environment variables used in "go get" etc. + environ []string + + // Set when Go modules are initialized in the current repo, that is: + // a go.mod file exists. + GoModulesFilename string + + // Set if we get a exec.ErrNotFound when running Go, which is most likely + // due to being run on a system without Go installed. We record it here + // so we can give an instructional error at the end if module/theme + // resolution fails. + goBinaryStatus goBinaryStatus +} + +// TODO(bep) merge with _vendor + /theme +func (m *Client) Graph(w io.Writer) error { + mc, err := m.Collect() + if err != nil { + return err + } + for _, module := range mc.Modules { + if module.Owner() == nil { + continue + } + dep := pathVersion(module.Owner()) + " " + pathVersion(module) + if replace := module.Replace(); replace != nil { + dep += " => " + replace.Dir() + } + fmt.Fprintln(w, dep) + + } + + return nil +} + +// Tidy can be used to remove unused dependencies from go.mod and go.sum. +func (m *Client) Tidy() error { + tc, err := m.Collect() + if err != nil { + return err + } + + isGoMod := make(map[string]bool) + for _, m := range tc.Modules { + if m.Owner() == nil { + continue + } + if m.IsGoMod() { + // Matching the format in go.mod + isGoMod[m.Path()+" "+m.Version()] = true + } + } + + if err := m.rewriteGoMod(goModFilename, isGoMod); err != nil { + return err + } + + // Now go.mod contains only in-use modules. The go.sum file will + // contain the entire dependency graph, so we need to check against that. + // TODO(bep) check if needed + /*graph, err := m.graphStr() + if err != nil { + return err + } + + isGoMod = make(map[string]bool) + graphItems := strings.Split(graph, "\n") + for _, item := range graphItems { + item = strings.TrimSpace(item) + if item == "" { + continue + } + modver := strings.Replace(strings.Fields(item)[1], "@", " ", 1) + isGoMod[modver] = true + }*/ + + if err := m.rewriteGoMod(goSumFilename, isGoMod); err != nil { + return err + } + + return nil +} + +func (m *Client) Get(args ...string) error { + if err := m.runGo(context.Background(), os.Stdout, append([]string{"get"}, args...)...); err != nil { + errors.Wrapf(err, "failed to get %q", args) + } + return nil +} + +func (m *Client) Init(path string) error { + + err := m.runGo(context.Background(), os.Stdout, "mod", "init", path) + if err != nil { + return errors.Wrap(err, "failed to init modules") + } + + m.GoModulesFilename = filepath.Join(m.workingDir, goModFilename) + + return nil +} + +func (m *Client) IsProbablyModule(path string) bool { + return module.CheckPath(path) == nil +} + +// Like Go, Hugo supports writing the dependencies to a /_vendor folder. +// Unlike Go, we support it for any level. +// We, by defaults, use the /_vendor folder first, if found. To disable, +// run with +// hugo --ignoreVendor +// +// Given a module tree, Hugo will pick the first module for a given path, +// meaning that if the top-level module is vendored, that will be the full +// set of dependencies. +func (c *Client) Vendor() error { + vendorDir := filepath.Join(c.workingDir, vendord) + if err := c.rmVendorDir(vendorDir); err != nil { + return err + } + + // Write the modules list to modules.txt. + // + // On the form: + // + // # github.com/alecthomas/chroma v0.6.3 + // + // This is how "go mod vendor" does it. Go also lists + // the packages below it, but that is currently not applicable to us. + // + var modulesContent bytes.Buffer + + tc, err := c.Collect() + if err != nil { + return err + } + + for _, t := range tc.Modules { + if t.Owner() == nil { + // This is the project. + continue + } + // We respect the --ignoreVendor flag even for the vendor command. + if !t.IsGoMod() && !t.Vendor() { + // We currently do not vendor components living in the + // theme directory, see https://github.com/gohugoio/hugo/issues/5993 + continue + } + + fmt.Fprintln(&modulesContent, "# "+t.Path()+" "+t.Version()) + + dir := t.Dir() + + shouldCopy := func(filename string) bool { + return true + } + + for _, mount := range t.Mounts() { + if err := hugio.CopyDir(c.fs, filepath.Join(dir, mount.Source), filepath.Join(vendorDir, t.Path(), mount.Source), shouldCopy); err != nil { + return errors.Wrap(err, "failed to copy module to vendor dir") + } + + // Also include any theme.toml or config.* files in the root. + configFiles, _ := afero.Glob(c.fs, filepath.Join(dir, "config.*")) + configFiles = append(configFiles, filepath.Join(dir, "theme.toml")) + for _, configFile := range configFiles { + if err := hugio.CopyFile(c.fs, configFile, filepath.Join(vendorDir, t.Path(), filepath.Base(configFile))); err != nil { + if !os.IsNotExist(err) { + return err + } + } + } + } + } + + if modulesContent.Len() > 0 { + if err := afero.WriteFile(c.fs, filepath.Join(vendorDir, vendorModulesFilename), modulesContent.Bytes(), 0666); err != nil { + return err + } + } + + return nil +} + +func (m *Client) listGoMods() (goModules, error) { + if m.GoModulesFilename == "" { + return nil, nil + } + /// + // TODO(bep) mod check permissions + // 0555 directories + + // GOCACHE + + out := ioutil.Discard + err := m.runGo(context.Background(), out, "mod", "download") + if err != nil { + return nil, errors.Wrap(err, "failed to download modules") + } + + b := &bytes.Buffer{} + err = m.runGo(context.Background(), b, "list", "-m", "-json", "all") + if err != nil { + return nil, errors.Wrap(err, "failed to list modules") + } + + var modules goModules + + dec := json.NewDecoder(b) + for { + m := &goModule{} + if err := dec.Decode(m); err != nil { + if err == io.EOF { + break + } + return nil, errors.Wrap(err, "failed to decode modules list") + } + + modules = append(modules, m) + } + + return modules, err + +} + +func (m *Client) rewriteGoMod(name string, isGoMod map[string]bool) error { + data, err := m.rewriteGoModRewrite(name, isGoMod) + if err != nil { + return err + } + if data != nil { + // Rewrite the file. + if err := afero.WriteFile(m.fs, filepath.Join(m.workingDir, name), data, 0666); err != nil { + return err + } + } + + return nil +} + +func (m *Client) rewriteGoModRewrite(name string, isGoMod map[string]bool) ([]byte, error) { + if name == goModFilename && m.GoModulesFilename == "" { + // Already checked. + return nil, nil + } + + isModLine := func(s string) bool { + return true + } + + if name == goModFilename { + isModLine = func(s string) bool { + return strings.HasPrefix(s, "mod require") || strings.HasPrefix(s, "\t") + } + } + + b := &bytes.Buffer{} + f, err := m.fs.Open(filepath.Join(m.workingDir, name)) + if err != nil { + if os.IsNotExist(err) { + // It's been deleted. + return nil, nil + } + return nil, err + } + defer f.Close() + + scanner := bufio.NewScanner(f) + var dirty bool + + for scanner.Scan() { + line := scanner.Text() + var doWrite bool + + if isModLine(line) { + modname := strings.TrimSpace(line) + if modname == "" { + doWrite = true + } else { + // TODO(bep) mod: mod require + parts := strings.Fields(modname) + if len(parts) >= 2 { + // [module path] [version]/go.mod + modname, modver := parts[0], parts[1] + modver = strings.TrimSuffix(modver, "/"+goModFilename) + doWrite = isGoMod[modname+" "+modver] + } + } + } else { + doWrite = true + } + + if doWrite { + fmt.Fprintln(b, line) + } else { + dirty = true + } + } + + if !dirty { + // Nothing changed + return nil, nil + } + + return b.Bytes(), nil + +} + +func (c *Client) rmVendorDir(vendorDir string) error { + modulestxt := filepath.Join(vendorDir, vendorModulesFilename) + + if _, err := c.fs.Stat(vendorDir); err != nil { + return nil + } + + _, err := c.fs.Stat(modulestxt) + if err != nil { + // If we have a _vendor dir without modules.txt it sounds like + // a _vendor dir created by others. + return errors.New("found _vendor dir without modules.txt, skip delete") + } + + return c.fs.RemoveAll(vendorDir) +} + +func (m *Client) runGo( + ctx context.Context, + stdout io.Writer, + args ...string) error { + + if m.goBinaryStatus != 0 { + return nil + } + + stderr := new(bytes.Buffer) + cmd := exec.CommandContext(ctx, "go", args...) + + cmd.Env = m.environ + cmd.Dir = m.workingDir + cmd.Stdout = stdout + cmd.Stderr = io.MultiWriter(stderr, os.Stderr) + + if err := cmd.Run(); err != nil { + if ee, ok := err.(*exec.Error); ok && ee.Err == exec.ErrNotFound { + m.goBinaryStatus = goBinaryStatusNotFound + return nil + } + + _, ok := err.(*exec.ExitError) + if !ok { + return errors.Errorf("failed to execute 'go %v': %s %T", args, err, err) + } + + // Too old Go version + if strings.Contains(stderr.String(), "flag provided but not defined") { + m.goBinaryStatus = goBinaryStatusTooOld + return nil + } + + return errors.Errorf("go command failed: %s", stderr) + + } + + return nil +} + +type ModuleError struct { + Err string // the error itself +} + +type goBinaryStatus int + +type goModule struct { + Path string // module path + Version string // module version + Versions []string // available module versions (with -versions) + Replace *goModule // replaced by this module + Time *time.Time // time version was created + Update *goModule // available update, if any (with -u) + Main bool // is this the main module? + Indirect bool // is this module only an indirect dependency of main module? + Dir string // directory holding files for this module, if any + GoMod string // path to go.mod file for this module, if any + Error *ModuleError // error loading module +} + +type goModules []*goModule + +func (modules goModules) GetByPath(p string) *goModule { + if modules == nil { + return nil + } + + for _, m := range modules { + if strings.EqualFold(p, m.Path) { + return m + } + } + + return nil +} + +func (modules goModules) GetMain() *goModule { + for _, m := range modules { + if m.Main { + return m + } + } + + return nil +} + +func pathVersion(m Module) string { + versionStr := m.Version() + if m.Vendor() { + versionStr += "+vendor" + } + if versionStr == "" { + return m.Path() + } + return fmt.Sprintf("%s@%s", m.Path(), versionStr) +} diff --git a/modules/client_test.go b/modules/client_test.go new file mode 100644 index 00000000000..072e7664004 --- /dev/null +++ b/modules/client_test.go @@ -0,0 +1,103 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package modules + +import ( + "bytes" + "testing" + + "github.com/gohugoio/hugo/common/hugo" + + "github.com/gohugoio/hugo/htesting" + + "github.com/gohugoio/hugo/hugofs" + + "github.com/stretchr/testify/require" +) + +func TestClient(t *testing.T) { + if hugo.GoMinorVersion() < 12 { + // https://github.com/golang/go/issues/26794 + // There were some concurrent issues with Go modules in < Go 12. + t.Skip("skip this for Go <= 1.11 due to a bug in Go's stdlib") + } + + t.Parallel() + + modName := "hugo-modules-basic-test" + modPath := "github.com/gohugoio/tests/" + modName + modConfig := defaultModuleConfig + modConfig.Imports = []Import{Import{Path: "github.com/gohugoio/hugoTestModules1_darwin/modh2_2"}} + + assert := require.New(t) + + workingDir, clean, err := htesting.CreateTempDir(hugofs.Os, modName) + assert.NoError(err) + defer clean() + + client := NewClient(ClientConfig{ + Fs: hugofs.Os, + WorkingDir: workingDir, + ModuleConfig: modConfig, + }) + + // Test Init + assert.NoError(client.Init(modPath)) + + // Test Collect + mc, err := client.Collect() + assert.NoError(err) + assert.Equal(4, len(mc.Modules)) + for _, m := range mc.Modules { + assert.NotNil(m) + } + + // Test Graph + var graphb bytes.Buffer + assert.NoError(client.Graph(&graphb)) + + expect := `github.com/gohugoio/tests/hugo-modules-basic-test github.com/gohugoio/hugoTestModules1_darwin/modh2_2@v1.4.0 +github.com/gohugoio/hugoTestModules1_darwin/modh2_2@v1.4.0 github.com/gohugoio/hugoTestModules1_darwin/modh2_2_1v@v1.3.0 +github.com/gohugoio/hugoTestModules1_darwin/modh2_2@v1.4.0 github.com/gohugoio/hugoTestModules1_darwin/modh2_2_2@v1.3.0 +` + + assert.Equal(expect, graphb.String()) + + // Test Vendor + assert.NoError(client.Vendor()) + graphb.Reset() + assert.NoError(client.Graph(&graphb)) + expectVendored := `github.com/gohugoio/tests/hugo-modules-basic-test github.com/gohugoio/hugoTestModules1_darwin/modh2_2@v1.4.0+vendor +github.com/gohugoio/tests/hugo-modules-basic-test github.com/gohugoio/hugoTestModules1_darwin/modh2_2_1v@v1.3.0+vendor +github.com/gohugoio/tests/hugo-modules-basic-test github.com/gohugoio/hugoTestModules1_darwin/modh2_2_2@v1.3.0+vendor +` + assert.Equal(expectVendored, graphb.String()) + + // Test the ignoreVendor setting + clientIgnoreVendor := NewClient(ClientConfig{ + Fs: hugofs.Os, + WorkingDir: workingDir, + ModuleConfig: modConfig, + IgnoreVendor: true, + }) + + graphb.Reset() + assert.NoError(clientIgnoreVendor.Graph(&graphb)) + assert.Equal(expect, graphb.String()) + + // Test Tidy + // TODO(bep) improve + assert.NoError(client.Tidy()) + +} diff --git a/modules/collect.go b/modules/collect.go new file mode 100644 index 00000000000..0795a84b5b4 --- /dev/null +++ b/modules/collect.go @@ -0,0 +1,496 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package modules + +import ( + "bufio" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/gohugoio/hugo/hugofs/files" + + "github.com/rogpeppe/go-internal/module" + + "github.com/pkg/errors" + + "github.com/gohugoio/hugo/config" + "github.com/spf13/afero" +) + +var ErrNotExist = errors.New("module does not exist") + +// IsNotExist returns whether an error means that a module could not be found. +func IsNotExist(err error) bool { + return errors.Cause(err) == ErrNotExist +} + +const vendorModulesFilename = "modules.txt" + +func (h *Client) hasImports() bool { + return len(h.moduleConfig.Imports) > 0 +} + +func (h *Client) Collect() (ModulesConfig, error) { + c := &collector{ + Client: h, + } + + if err := c.collect(); err != nil { + return ModulesConfig{}, err + } + + return ModulesConfig{ + Modules: c.modules, + GoModulesFilename: c.GoModulesFilename, + }, c.err + +} + +type ModulesConfig struct { + Modules Modules + + // Set if this is a Go modules enabled project. + GoModulesFilename string +} + +type collected struct { + // Pick the first and prevent circular loops. + seen map[string]bool + + // Maps module path to a _vendor dir. These values are fetched from + // _vendor/modules.txt, and the first (top-most) will win. + vendored map[string]vendoredModule + + // Set if a Go modules enabled project. + gomods goModules + + // Ordered list of collected modules, including Go Modules and theme + // components stored below /themes. + modules Modules +} + +// Collects and creates a module tree. +type collector struct { + *Client + + // Store away any non-fatal error and return at the end. + err error + + *collected +} + +type vendoredModule struct { + Owner Module + Dir string + Version string +} + +func (c *collector) initModules() error { + c.collected = &collected{ + seen: make(map[string]bool), + vendored: make(map[string]vendoredModule), + } + + // We may fail later if we don't find the mods. + return c.loadModules() +} + +// TODO(bep) mod: +// - no-vendor +func (c *collector) isSeen(path string) bool { + key := pathKey(path) + if c.seen[key] { + return true + } + c.seen[key] = true + return false +} + +func (c *collector) getVendoredDir(path string) (vendoredModule, bool) { + v, found := c.vendored[path] + return v, found +} + +// TODO(bep) mod +const zeroVersion = "" + +func (c *collector) add(owner *moduleAdapter, moduleImport Import) (*moduleAdapter, error) { + var ( + mod *goModule + moduleDir string + version string + vendored bool + ) + + modulePath := moduleImport.Path + var realOwner Module = owner + + if !c.ignoreVendor { + if err := c.collectModulesTXT(owner); err != nil { + return nil, err + } + + // Try _vendor first. + var vm vendoredModule + vm, vendored = c.getVendoredDir(modulePath) + if vendored { + moduleDir = vm.Dir + realOwner = vm.Owner + version = vm.Version + + } + } + + if moduleDir == "" { + mod = c.gomods.GetByPath(modulePath) + if mod != nil { + moduleDir = mod.Dir + } + + if moduleDir == "" { + + if c.GoModulesFilename != "" && c.IsProbablyModule(modulePath) { + // Try to "go get" it and reload the module configuration. + if err := c.Get(modulePath); err != nil { + return nil, err + } + if err := c.loadModules(); err != nil { + return nil, err + } + + mod = c.gomods.GetByPath(modulePath) + if mod != nil { + moduleDir = mod.Dir + } + } + + // Fall back to /themes/ + if moduleDir == "" { + moduleDir = filepath.Join(c.themesDir, modulePath) + + if found, _ := afero.Exists(c.fs, moduleDir); !found { + c.err = c.wrapModuleNotFound(errors.Errorf(`module %q not found; either add it as a Hugo Module or store it in %q. +TODO(bep) mod add note about hugo mod init +`, modulePath, c.themesDir)) + return nil, nil + } + } + } + } + + if found, _ := afero.Exists(c.fs, moduleDir); !found { + c.err = c.wrapModuleNotFound(errors.Errorf("%q not found", moduleDir)) + return nil, nil + } + + if !strings.HasSuffix(moduleDir, fileSeparator) { + moduleDir += fileSeparator + } + + ma := &moduleAdapter{ + dir: moduleDir, + vendor: vendored, + gomod: mod, + version: version, + // This may be the owner of the _vendor dir + owner: realOwner, + } + if mod == nil { + ma.path = modulePath + } + + if err := ma.validateAndApplyDefaults(c.fs); err != nil { + return nil, err + } + + if err := c.applyThemeConfig(ma); err != nil { + return nil, err + } + + if err := c.applyMounts(moduleImport, ma); err != nil { + return nil, err + } + + c.modules = append(c.modules, ma) + return ma, nil + +} + +func (c *collector) addAndRecurse(owner *moduleAdapter) error { + moduleConfig := owner.Config() + if owner.projectMod { + if err := c.applyMounts(Import{}, owner); err != nil { + return err + } + } + + for _, moduleImport := range moduleConfig.Imports { + if !c.isSeen(moduleImport.Path) { + tc, err := c.add(owner, moduleImport) + if err != nil { + return err + } + if tc == nil { + continue + } + if err := c.addAndRecurse(tc); err != nil { + return err + } + } + } + return nil +} + +func (c *collector) applyMounts(moduleImport Import, mod *moduleAdapter) error { + mounts := moduleImport.Mounts + + if !mod.projectMod && len(mounts) == 0 { + modConfig := mod.Config() + mounts = modConfig.Mounts + if len(mounts) == 0 { + // Create default mount points for every component folder that + // exists in the module. + for _, componentFolder := range files.ComponentFolders { + sourceDir := filepath.Join(mod.Dir(), componentFolder) + _, err := c.fs.Stat(sourceDir) + if err == nil { + mounts = append(mounts, Mount{ + Source: componentFolder, + Target: componentFolder, + }) + } + } + } + } + + var err error + mounts, err = c.normalizeMounts(mod, mounts) + if err != nil { + return err + } + + mod.mounts = mounts + return nil +} + +func (c *collector) normalizeMounts(owner Module, mounts []Mount) ([]Mount, error) { + var out []Mount + dir := owner.Dir() + + for _, mnt := range mounts { + errMsg := fmt.Sprintf("invalid module config for %q", owner.Path()) + + if mnt.Source == "" || mnt.Target == "" { + return nil, errors.New(errMsg + ": both source and target must be set") + } + + mnt.Source = filepath.Clean(mnt.Source) + mnt.Target = filepath.Clean(mnt.Target) + + // Verify that Source exists + sourceDir := filepath.Join(dir, mnt.Source) + _, err := c.fs.Stat(sourceDir) + if err != nil { + //fmt.Println(">>> ::::: NOT FOUND", sourceDir) + + continue + //return nil, errors.Errorf("%s: module mount source not found: %q", errMsg, sourceDir) + } + + // Verify that target points to one of the predefined component dirs + targetBase := mnt.Target + idxPathSep := strings.Index(mnt.Target, string(os.PathSeparator)) + if idxPathSep != -1 { + targetBase = mnt.Target[0:idxPathSep] + } + if !files.IsComponentFolder(targetBase) { + return nil, errors.Errorf("%s: mount target must be one of: %v", errMsg, files.ComponentFolders) + } + + out = append(out, mnt) + } + + return out, nil +} + +func (c *collector) applyThemeConfig(tc *moduleAdapter) error { + + var ( + configFilename string + cfg config.Provider + exists bool + ) + + // Viper supports more, but this is the sub-set supported by Hugo. + for _, configFormats := range config.ValidConfigFileExtensions { + configFilename = filepath.Join(tc.Dir(), "config."+configFormats) + exists, _ = afero.Exists(c.fs, configFilename) + if exists { + break + } + } + + if !exists { + // No theme config set. + return nil + } + + if configFilename != "" { + var err error + cfg, err = config.FromFile(c.fs, configFilename) + if err != nil { + return err + } + } + + tc.configFilename = configFilename + tc.cfg = cfg + + config, err := DecodeConfig(cfg) + if err != nil { + return err + } + tc.config = config + + return nil + +} + +// CreateProjectModule creates modules from the given config. +// This is used in tests only. +func CreateProjectModule(cfg config.Provider) (Module, error) { + workingDir := cfg.GetString("workingDir") + var modConfig Config + + mod := createProjectModule(nil, workingDir, modConfig) + if err := ApplyProjectConfigDefaults(cfg, mod); err != nil { + return nil, err + } + + return mod, nil +} + +func createProjectModule(gomod *goModule, workingDir string, conf Config) *moduleAdapter { + // Create a pseudo module for the main project. + var path string + if gomod == nil { + path = "project" + } + + return &moduleAdapter{ + path: path, + dir: workingDir, + gomod: gomod, + projectMod: true, + config: conf, + } + +} + +func (c *collector) collect() error { + if err := c.initModules(); err != nil { + return err + } + + projectMod := createProjectModule(c.gomods.GetMain(), c.workingDir, c.moduleConfig) + + if err := c.addAndRecurse(projectMod); err != nil { + return err + } + + // Append the project module at the tail. + c.modules = append(c.modules, projectMod) + + return nil +} + +func (c *collector) collectModulesTXT(owner Module) error { + vendorDir := filepath.Join(owner.Dir(), vendord) + filename := filepath.Join(vendorDir, vendorModulesFilename) + + f, err := c.fs.Open(filename) + + if err != nil { + if os.IsNotExist(err) { + return nil + } + + return err + } + + defer f.Close() + + scanner := bufio.NewScanner(f) + + for scanner.Scan() { + // # github.com/alecthomas/chroma v0.6.3 + line := scanner.Text() + line = strings.Trim(line, "# ") + line = strings.TrimSpace(line) + parts := strings.Fields(line) + if len(parts) != 2 { + return errors.Errorf("invalid modules list: %q", filename) + } + path := parts[0] + if _, found := c.vendored[path]; !found { + c.vendored[path] = vendoredModule{ + Owner: owner, + Dir: filepath.Join(vendorDir, path), + Version: parts[1], + } + } + + } + return nil +} + +func (c *collector) loadModules() error { + modules, err := c.listGoMods() + if err != nil { + return err + } + c.gomods = modules + return nil +} + +func (c *collector) wrapModuleNotFound(err error) error { + err = errors.Wrap(ErrNotExist, err.Error()) + if c.GoModulesFilename == "" { + return err + } + + baseMsg := "we found a go.mod file in your project, but" + + switch c.goBinaryStatus { + case goBinaryStatusNotFound: + return errors.Wrap(err, baseMsg+" you need to install Go to use it. See https://golang.org/dl/.") + case goBinaryStatusTooOld: + return errors.Wrap(err, baseMsg+" you need to a newer version of Go to use it. See https://golang.org/dl/.") + } + + return err + +} + +// In the first iteration of Hugo Modules, we do not support multiple +// major versions running at the same time, so we pick the first (upper most). +// We will investigate namespaces in future versions. +// TODO(bep) mod add a warning when the above happens. +func pathKey(p string) string { + prefix, _, _ := module.SplitPathVersion(p) + + return strings.ToLower(prefix) +} diff --git a/modules/collect_test.go b/modules/collect_test.go new file mode 100644 index 00000000000..d76c0b2bbed --- /dev/null +++ b/modules/collect_test.go @@ -0,0 +1,38 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package modules + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestPathKey(t *testing.T) { + assert := require.New(t) + + for _, test := range []struct { + in string + expect string + }{ + {"github.com/foo", "github.com/foo"}, + {"github.com/foo/v2", "github.com/foo"}, + {"github.com/foo/v12", "github.com/foo"}, + {"github.com/foo/v3d", "github.com/foo/v3d"}, + {"MyTheme", "mytheme"}, + } { + assert.Equal(test.expect, pathKey(test.in)) + } + +} diff --git a/modules/config.go b/modules/config.go new file mode 100644 index 00000000000..7468439e2d9 --- /dev/null +++ b/modules/config.go @@ -0,0 +1,279 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package modules + +import ( + "fmt" + "path" + "path/filepath" + "strings" + + "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/hugofs/files" + "github.com/gohugoio/hugo/langs" + "github.com/mitchellh/mapstructure" +) + +var defaultModuleConfig = Config{ + + // Default to direct, which means "git clone" and similar. We + // will investigate proxy settings in more depth later. + // See https://github.com/golang/go/issues/26334 + Proxy: "direct", + + // Comma separated glob list matching paths that should not use the + // proxy configured above. + NoProxy: "none", + + // Comma separated glob list matching paths that should be + // treated as private. + Private: "*.*", +} + +// Config holds a module config. +type Config struct { + Mounts []Mount + Imports []Import + + // Configures GOPROXY. + Proxy string + // Configures GONOPROXY. + NoProxy string + // Configures GOPRIVATE. + Private string +} + +type Import struct { + Path string // Module path + IgnoreConfig bool // Ignore any config.toml found. TODO(bep) implement + Mounts []Mount +} + +type Mount struct { + Source string // relative path in source repo, e.g. "scss" + Target string // relative target path, e.g. "assets/bootstrap/scss" + + // TODO(bep) mod + Lang string +} + +func (m Mount) Component() string { + return strings.Split(m.Target, fileSeparator)[0] +} + +// ApplyProjectConfigDefaults applies default/missing module configuration for +// the main project. +func ApplyProjectConfigDefaults(cfg config.Provider, mod Module) error { + moda := mod.(*moduleAdapter) + + // TODO(bep) mod we need a way to check if contentDir etc. is really set. + // ... so remove the default settings for these. + + // Map legacy directory config into the new module. + languages := cfg.Get("languagesSortedDefaultFirst").(langs.Languages) + isMultiHost := languages.IsMultihost() + + // To bridge between old and new configuration format we need + // a way to make sure all of the core components are configured on + // the basic level. + componentsConfigured := make(map[string]bool) + for _, mnt := range moda.mounts { + componentsConfigured[mnt.Component()] = true + } + + type dirKeyComponent struct { + key string + component string + multilingual bool + } + + dirKeys := []dirKeyComponent{ + {"contentDir", files.ComponentFolderContent, true}, + {"dataDir", files.ComponentFolderData, false}, + {"layoutDir", files.ComponentFolderLayouts, false}, + {"i18nDir", files.ComponentFolderI18n, false}, + {"archetypeDir", files.ComponentFolderArchetypes, false}, + {"assetDir", files.ComponentFolderAssets, false}, + {"resourceDir", files.ComponentFolderResources, false}, + {"", files.ComponentFolderStatic, isMultiHost}, + } + + createMountsFor := func(d dirKeyComponent, cfg config.Provider) []Mount { + var lang string + if language, ok := cfg.(*langs.Language); ok { + lang = language.Lang + } + + // Static mounts are a little special. + if d.component == files.ComponentFolderStatic { + var mounts []Mount + staticDirs := getStaticDirs(cfg) + if len(staticDirs) > 0 { + componentsConfigured[d.component] = true + } + + for _, dir := range staticDirs { + mounts = append(mounts, Mount{Lang: lang, Source: dir, Target: d.component}) + } + + return mounts + + } + + if cfg.IsSet(d.key) { + source := cfg.GetString(d.key) + componentsConfigured[d.component] = true + + return []Mount{Mount{ + // No lang set for layouts etc. + Source: source, + Target: d.component}} + } + + return nil + } + + createMounts := func(d dirKeyComponent) []Mount { + var mounts []Mount + if d.multilingual { + if d.component == files.ComponentFolderContent { + seen := make(map[string]bool) + for _, language := range languages { + contentDir := language.ContentDir + if seen[contentDir] { + continue + } + seen[contentDir] = true + mounts = append(mounts, Mount{Lang: language.Lang, Source: contentDir, Target: d.component}) + } + + componentsConfigured[d.component] = len(seen) > 0 + + } else { + for _, language := range languages { + mounts = append(mounts, createMountsFor(d, language)...) + } + } + } else { + mounts = append(mounts, createMountsFor(d, cfg)...) + } + + return mounts + } + + var mounts []Mount + for _, dirKey := range dirKeys { + if componentsConfigured[dirKey.component] { + + continue + } + + mounts = append(mounts, createMounts(dirKey)...) + + } + + // Add default configuration + for _, dirKey := range dirKeys { + if componentsConfigured[dirKey.component] { + continue + } + mounts = append(mounts, Mount{Source: dirKey.component, Target: dirKey.component}) + } + + // Remove duplicates + seen := make(map[string]bool) + tmp := mounts[:0] + for _, m := range mounts { + key := path.Join(m.Lang, m.Source, m.Target) + if !seen[key] { + tmp = append(tmp, m) + } + seen[key] = true + } + + moda.mounts = mounts + + return nil +} + +// DecodeConfig creates a modules Config from a given Hugo configuration. +func DecodeConfig(cfg config.Provider) (Config, error) { + c := defaultModuleConfig + + if cfg == nil { + return c, nil + } + + themeSet := cfg.IsSet("theme") + moduleSet := cfg.IsSet("module") + + if moduleSet { + m := cfg.GetStringMap("module") + if err := mapstructure.WeakDecode(m, &c); err != nil { + return c, err + } + + for i, mnt := range c.Mounts { + mnt.Source = filepath.Clean(mnt.Source) + mnt.Target = filepath.Clean(mnt.Target) + c.Mounts[i] = mnt + } + + } + + if themeSet { + imports := config.GetStringSlicePreserveString(cfg, "theme") + for _, imp := range imports { + c.Imports = append(c.Imports, Import{ + Path: imp, + }) + } + + } + + return c, nil +} + +func removeDuplicatesKeepRight(in []string) []string { + seen := make(map[string]bool) + var out []string + for i := len(in) - 1; i >= 0; i-- { + v := in[i] + if seen[v] { + continue + } + out = append([]string{v}, out...) + seen[v] = true + } + + return out +} + +func getStaticDirs(cfg config.Provider) []string { + var staticDirs []string + for i := -1; i <= 10; i++ { + staticDirs = append(staticDirs, getStringOrStringSlice(cfg, "staticDir", i)...) + } + return staticDirs +} + +func getStringOrStringSlice(cfg config.Provider, key string, id int) []string { + + if id >= 0 { + key = fmt.Sprintf("%s%d", key, id) + } + + return config.GetStringSlicePreserveString(cfg, key) + +} diff --git a/modules/config_test.go b/modules/config_test.go new file mode 100644 index 00000000000..a372c39b1fc --- /dev/null +++ b/modules/config_test.go @@ -0,0 +1,95 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package modules + +import ( + "testing" + + "github.com/gohugoio/hugo/config" + + "github.com/stretchr/testify/require" +) + +func TestDecodeConfig(t *testing.T) { + assert := require.New(t) + tomlConfig := ` +[module] +[[module.mounts]] +source="src/project/blog" +target="content/blog" +lang="en" +[[module.imports]] +path="github.com/bep/mycomponent" +[[module.imports.mounts]] +source="scss" +target="assets/bootstrap/scss" +[[module.imports.mounts]] +source="src/markdown/blog" +target="content/blog" +lang="en" +` + cfg, err := config.FromConfigString(tomlConfig, "toml") + assert.NoError(err) + + mcfg, err := DecodeConfig(cfg) + assert.NoError(err) + + assert.Len(mcfg.Mounts, 1) + assert.Len(mcfg.Imports, 1) + imp := mcfg.Imports[0] + imp.Path = "github.com/bep/mycomponent" + assert.Equal("src/markdown/blog", imp.Mounts[1].Source) + assert.Equal("content/blog", imp.Mounts[1].Target) + assert.Equal("en", imp.Mounts[1].Lang) + +} + +// Test old style theme import. +func TestDecodeConfigTheme(t *testing.T) { + assert := require.New(t) + tomlConfig := ` + +theme = ["a", "b"] +` + cfg, err := config.FromConfigString(tomlConfig, "toml") + assert.NoError(err) + + mcfg, err := DecodeConfig(cfg) + assert.NoError(err) + + assert.Len(mcfg.Imports, 2) + assert.Equal("a", mcfg.Imports[0].Path) + assert.Equal("b", mcfg.Imports[1].Path) +} + +func TestDecodeConfigBothOldAndNewProvided(t *testing.T) { + assert := require.New(t) + tomlConfig := ` + +theme = ["b", "c"] + +[module] +[[module.imports]] +path="a" + +` + cfg, err := config.FromConfigString(tomlConfig, "toml") + assert.NoError(err) + + modCfg, err := DecodeConfig(cfg) + assert.NoError(err) + assert.Len(modCfg.Imports, 3) + assert.Equal("a", modCfg.Imports[0].Path) + +} diff --git a/modules/module.go b/modules/module.go new file mode 100644 index 00000000000..c9202aa56df --- /dev/null +++ b/modules/module.go @@ -0,0 +1,166 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package modules provides a client that can be used to manage Hugo Components, +// what's refered to as Hugo Modules. Hugo Modules is built on top of Go Modules, +// but also supports vendoring and components stored directly in the themes dir. +package modules + +import ( + "github.com/gohugoio/hugo/config" + "github.com/spf13/afero" +) + +var _ Module = (*moduleAdapter)(nil) + +type Module interface { + + // Optional configuration filename (e.g. "/themes/mytheme/config.json"). + // This will be added to the special configuration watch list when in + // server mode. + ConfigFilename() string + + // Optional config read from the configFilename above. + Cfg() config.Provider + + // The decoded module config and mounts. + Config() Config + + // Directory holding files for this module. + Dir() string + + // Returns whether this is a Go Module. + IsGoMod() bool + + // In the dependency tree, this is the first module that defines this module + // as a dependency. + Owner() Module + + // Replaced by this module. + Replace() Module + + // Any directory remappings. + Mounts() []Mount + + // Returns the path to this module. + // This will either be the module path, e.g. "github.com/gohugoio/myshortcodes", + // or the path below your /theme folder, e.g. "mytheme". + Path() string + + // Returns whether Dir points below the _vendor dir. + Vendor() bool + + // The module version. + Version() string +} + +type Modules []Module + +type moduleAdapter struct { + path string + dir string + version string + vendor bool + projectMod bool + owner Module + + mounts []Mount + + configFilename string + cfg config.Provider + config Config + + // Set if a Go module. + gomod *goModule +} + +func (m *moduleAdapter) Cfg() config.Provider { + return m.cfg +} + +func (m *moduleAdapter) Config() Config { + return m.config +} + +func (m *moduleAdapter) ConfigFilename() string { + return m.configFilename +} + +func (m *moduleAdapter) Dir() string { + // This may point to the _vendor dir. + if !m.IsGoMod() || m.dir != "" { + return m.dir + } + return m.gomod.Dir +} + +func (m *moduleAdapter) IsGoMod() bool { + return m.gomod != nil +} + +func (m *moduleAdapter) Owner() Module { + return m.owner +} + +func (m *moduleAdapter) Replace() Module { + if m.IsGoMod() && !m.Vendor() && m.gomod.Replace != nil { + return &moduleAdapter{ + gomod: m.gomod.Replace, + owner: m.owner, + } + } + return nil +} + +func (m *moduleAdapter) Mounts() []Mount { + return m.mounts +} + +func (m *moduleAdapter) Path() string { + if !m.IsGoMod() || m.path != "" { + return m.path + } + return m.gomod.Path +} + +func (m *moduleAdapter) Vendor() bool { + return m.vendor +} + +func (m *moduleAdapter) Version() string { + if !m.IsGoMod() || m.version != "" { + return m.version + } + return m.gomod.Version +} + +func (m *moduleAdapter) validateAndApplyDefaults(fs afero.Fs) error { + + /*if len(m.modImport.Mounts) == 0 { + // Create default mount points for every component folder that + // exists in the module. + for _, componentFolder := range files.ComponentFolders { + sourceDir := filepath.Join(dir, componentFolder) + _, err := fs.Stat(sourceDir) + if err == nil { + m.modImport.Mounts = append(m.modImport.Mounts, Mount{ + Source: componentFolder, + Target: componentFolder, + }) + } + } + }*/ + + return nil + +} diff --git a/output/layout.go b/output/layout.go index 5d72938af58..055d742b15f 100644 --- a/output/layout.go +++ b/output/layout.go @@ -71,7 +71,7 @@ func (l *LayoutHandler) For(d LayoutDescriptor, f Format) ([]string, error) { layouts := resolvePageTemplate(d, f) layouts = prependTextPrefixIfNeeded(f, layouts...) - layouts = helpers.UniqueStrings(layouts) + layouts = helpers.UniqueStringsReuse(layouts) l.mu.Lock() l.cache[key] = layouts diff --git a/parser/metadecoders/decoder.go b/parser/metadecoders/decoder.go index e7b0e3c97b8..8af1b49e0ce 100644 --- a/parser/metadecoders/decoder.go +++ b/parser/metadecoders/decoder.go @@ -82,6 +82,30 @@ func (d Decoder) UnmarshalFileToMap(fs afero.Fs, filename string) (map[string]in return d.UnmarshalToMap(data, format) } +// UnmarshalStringTo tries to unmarshal data to a new instance of type typ. +func (d Decoder) UnmarshalStringTo(data string, typ interface{}) (interface{}, error) { + data = strings.TrimSpace(data) + // We only check for the possible types in YAML, JSON and TOML. + switch typ.(type) { + case string: + return data, nil + case map[string]interface{}: + format := d.FormatFromContentString(data) + return d.UnmarshalToMap([]byte(data), format) + case []interface{}: + // A standalone slice. Let YAML handle it. + return d.Unmarshal([]byte(data), YAML) + case int: + return cast.ToIntE(data) + case int64: + return cast.ToInt64E(data) + case float64: + return cast.ToFloat64E(data) + default: + return nil, errors.Errorf("unmarshal: %T not supportedd", typ) + } +} + // Unmarshal will unmarshall data in format f into an interface{}. // This is what's needed for Hugo's /data handling. func (d Decoder) Unmarshal(data []byte, f Format) (interface{}, error) { diff --git a/parser/metadecoders/decoder_test.go b/parser/metadecoders/decoder_test.go index 146df506900..7cb66d7360f 100644 --- a/parser/metadecoders/decoder_test.go +++ b/parser/metadecoders/decoder_test.go @@ -90,6 +90,38 @@ func TestUnmarshalToInterface(t *testing.T) { } +func TestUnmarshalStringTo(t *testing.T) { + assert := require.New(t) + + d := Default + + expectMap := map[string]interface{}{"a": "b"} + + for i, test := range []struct { + data string + to interface{} + expect interface{} + }{ + {"a string", "string", "a string"}, + {`{ "a": "b" }`, make(map[string]interface{}), expectMap}, + {"32", int64(1234), int64(32)}, + {"32", int(1234), int(32)}, + {"3.14159", float64(1), float64(3.14159)}, + {"[3,7,9]", []interface{}{}, []interface{}{3, 7, 9}}, + {"[3.1,7.2,9.3]", []interface{}{}, []interface{}{3.1, 7.2, 9.3}}, + } { + msg := fmt.Sprintf("%d: %T", i, test.to) + m, err := d.UnmarshalStringTo(test.data, test.to) + if b, ok := test.expect.(bool); ok && !b { + assert.Error(err, msg) + } else { + assert.NoError(err, msg) + assert.Equal(test.expect, m, msg) + } + + } +} + func TestStringifyYAMLMapKeys(t *testing.T) { cases := []struct { input interface{} diff --git a/resources/image_cache.go b/resources/image_cache.go index cf1e999badc..4072851e29d 100644 --- a/resources/image_cache.go +++ b/resources/image_cache.go @@ -145,7 +145,7 @@ func (c *imageCache) getOrCreate( } // The file is now stored in this cache. - img.overriddenSourceFs = c.fileCache.Fs + img.sourceFs = c.fileCache.Fs c.mu.Lock() if img2, found := c.store[key]; found { diff --git a/resources/page/page_nop.go b/resources/page/page_nop.go index 229bcb077e4..c3a4819f1f4 100644 --- a/resources/page/page_nop.go +++ b/resources/page/page_nop.go @@ -17,9 +17,10 @@ package page import ( "html/template" - "os" "time" + "github.com/gohugoio/hugo/hugofs" + "github.com/bep/gitmap" "github.com/gohugoio/hugo/navigation" @@ -147,7 +148,7 @@ func (p *nopPage) File() source.File { return nilFile } -func (p *nopPage) FileInfo() os.FileInfo { +func (p *nopPage) FileInfo() hugofs.FileMetaInfo { return nil } diff --git a/resources/page/page_wrappers.autogen.go b/resources/page/page_wrappers.autogen.go index d7fcb520109..d2d14dee6f1 100644 --- a/resources/page/page_wrappers.autogen.go +++ b/resources/page/page_wrappers.autogen.go @@ -18,8 +18,8 @@ package page import ( "github.com/gohugoio/hugo/common/hugo" "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/hugofs" "html/template" - "os" ) // NewDeprecatedWarningPage adds deprecation warnings to the given implementation. @@ -91,7 +91,7 @@ func (p *pageDeprecated) UniqueID() string { helpers.Deprecated("Page", ".UniqueID", "Use .File.UniqueID", false) return p.p.UniqueID() } -func (p *pageDeprecated) FileInfo() os.FileInfo { +func (p *pageDeprecated) FileInfo() hugofs.FileMetaInfo { helpers.Deprecated("Page", ".FileInfo", "Use .File.FileInfo", false) return p.p.FileInfo() } diff --git a/resources/page/pagemeta/page_frontmatter.go b/resources/page/pagemeta/page_frontmatter.go index 1ce3fbee4fc..7b9f13e622e 100644 --- a/resources/page/pagemeta/page_frontmatter.go +++ b/resources/page/pagemeta/page_frontmatter.go @@ -236,7 +236,7 @@ func addDateFieldAliases(values []string) []string { complete = append(complete, aliases...) } } - return helpers.UniqueStrings(complete) + return helpers.UniqueStringsReuse(complete) } func expandDefaultValues(values []string, defaults []string) []string { diff --git a/resources/page/permalinks.go b/resources/page/permalinks.go index 98489231b2c..59f2da91671 100644 --- a/resources/page/permalinks.go +++ b/resources/page/permalinks.go @@ -15,6 +15,7 @@ package page import ( "fmt" + "os" "path/filepath" "regexp" "strconv" @@ -90,7 +91,11 @@ func (l PermalinkExpander) parse(patterns map[string]string) (map[string]func(Pa expanders := make(map[string]func(Page) (string, error)) + // Allow " " and / to represent the root section. + const sectionCutSet = " /" + string(os.PathSeparator) + for k, pattern := range patterns { + k = strings.Trim(k, sectionCutSet) if !l.validate(pattern) { return nil, &permalinkExpandError{pattern: pattern, err: errPermalinkIllFormed} } diff --git a/resources/page/site.go b/resources/page/site.go index 25df063f1b1..9153c855614 100644 --- a/resources/page/site.go +++ b/resources/page/site.go @@ -17,6 +17,8 @@ import ( "html/template" "time" + "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/common/hugo" "github.com/gohugoio/hugo/langs" "github.com/gohugoio/hugo/navigation" @@ -51,3 +53,72 @@ func (s Sites) First() Site { } return s[0] } + +type testSite struct { + h hugo.Info + l *langs.Language +} + +func (t testSite) Hugo() hugo.Info { + return t.h +} + +func (t testSite) ServerPort() int { + return 1313 +} + +func (testSite) LastChange() (t time.Time) { + return +} + +func (t testSite) Title() string { + return "foo" +} + +func (t testSite) Sites() Sites { + return nil +} + +func (t testSite) IsServer() bool { + return false +} + +func (t testSite) Language() *langs.Language { + return t.l +} + +func (t testSite) Pages() Pages { + return nil +} + +func (t testSite) RegularPages() Pages { + return nil +} + +func (t testSite) Menus() navigation.Menus { + return nil +} + +func (t testSite) Taxonomies() interface{} { + return nil +} + +func (t testSite) BaseURL() template.URL { + return "" +} + +func (t testSite) Params() map[string]interface{} { + return nil +} + +func (t testSite) Data() map[string]interface{} { + return nil +} + +// NewDummyHugoSite creates a new minimal test site. +func NewDummyHugoSite(cfg config.Provider) Site { + return testSite{ + h: hugo.NewInfo(hugo.EnvironmentProduction), + l: langs.NewLanguage("en", cfg), + } +} diff --git a/resources/page/testhelpers_test.go b/resources/page/testhelpers_test.go index 60a6c0816fb..fa5f8e9c8a0 100644 --- a/resources/page/testhelpers_test.go +++ b/resources/page/testhelpers_test.go @@ -16,10 +16,11 @@ package page import ( "fmt" "html/template" - "os" "path/filepath" "time" + "github.com/gohugoio/hugo/modules" + "github.com/bep/gitmap" "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/resources/resource" @@ -65,6 +66,12 @@ func newTestPathSpec() *helpers.PathSpec { func newTestPathSpecFor(cfg config.Provider) *helpers.PathSpec { config.SetBaseTestDefaults(cfg) + langs.LoadLanguageSettings(cfg, nil) + mod, err := modules.CreateProjectModule(cfg) + if err != nil { + panic(err) + } + cfg.Set("allModules", modules.Modules{mod}) fs := hugofs.NewMem(cfg) s, err := helpers.NewPathSpec(fs, cfg) if err != nil { @@ -189,7 +196,7 @@ func (p *testPage) File() source.File { return p.file } -func (p *testPage) FileInfo() os.FileInfo { +func (p *testPage) FileInfo() hugofs.FileMetaInfo { panic("not implemented") } diff --git a/resources/page/zero_file.autogen.go b/resources/page/zero_file.autogen.go index eec1dd66dc0..23e36b76490 100644 --- a/resources/page/zero_file.autogen.go +++ b/resources/page/zero_file.autogen.go @@ -17,8 +17,8 @@ package page import ( "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/source" - "os" ) // ZeroFile represents a zero value of source.File with warnings if invoked. @@ -82,7 +82,7 @@ func (z zeroFile) UniqueID() (o0 string) { z.log.Println(".File.UniqueID on zero object. Wrap it in if or with: {{ with .File }}{{ .UniqueID }}{{ end }}") return } -func (z zeroFile) FileInfo() (o0 os.FileInfo) { +func (z zeroFile) FileInfo() (o0 hugofs.FileMetaInfo) { z.log.Println(".File.FileInfo on zero object. Wrap it in if or with: {{ with .File }}{{ .FileInfo }}{{ end }}") return } diff --git a/resources/resource.go b/resources/resource.go index c120a8dd090..66f4220f35c 100644 --- a/resources/resource.go +++ b/resources/resource.go @@ -24,6 +24,8 @@ import ( "strings" "sync" + "github.com/gohugoio/hugo/hugofs" + "github.com/gohugoio/hugo/media" "github.com/gohugoio/hugo/output" @@ -133,9 +135,13 @@ type ResourceSourceDescriptor struct { SourceFile source.File OpenReadSeekCloser resource.OpenReadSeekCloser + FileInfo os.FileInfo + // If OpenReadSeekerCloser is not set, we use this to open the file. SourceFilename string + Fs afero.Fs + // The relative target filename without any language code. RelTargetFilename string @@ -157,19 +163,11 @@ func (r ResourceSourceDescriptor) Filename() string { return r.SourceFilename } -func (r *Spec) sourceFs() afero.Fs { - return r.PathSpec.BaseFs.Content.Fs -} - func (r *Spec) New(fd ResourceSourceDescriptor) (resource.Resource, error) { - return r.newResourceForFs(r.sourceFs(), fd) -} - -func (r *Spec) NewForFs(sourceFs afero.Fs, fd ResourceSourceDescriptor) (resource.Resource, error) { - return r.newResourceForFs(sourceFs, fd) + return r.newResourceFor(fd) } -func (r *Spec) newResourceForFs(sourceFs afero.Fs, fd ResourceSourceDescriptor) (resource.Resource, error) { +func (r *Spec) newResourceFor(fd ResourceSourceDescriptor) (resource.Resource, error) { if fd.OpenReadSeekCloser == nil { if fd.SourceFile != nil && fd.SourceFilename != "" { return nil, errors.New("both SourceFile and AbsSourceFilename provided") @@ -187,15 +185,14 @@ func (r *Spec) newResourceForFs(sourceFs afero.Fs, fd ResourceSourceDescriptor) fd.TargetBasePaths = r.MultihostTargetBasePaths } - return r.newResource(sourceFs, fd) + return r.newResource(fd.Fs, fd) } func (r *Spec) newResource(sourceFs afero.Fs, fd ResourceSourceDescriptor) (resource.Resource, error) { - var fi os.FileInfo + fi := fd.FileInfo var sourceFilename string if fd.OpenReadSeekCloser != nil { - } else if fd.SourceFilename != "" { var err error fi, err = sourceFs.Stat(fd.SourceFilename) @@ -207,7 +204,6 @@ func (r *Spec) newResource(sourceFs afero.Fs, fd ResourceSourceDescriptor) (reso } sourceFilename = fd.SourceFilename } else { - fi = fd.SourceFile.FileInfo() sourceFilename = fd.SourceFile.Filename() } @@ -245,8 +241,6 @@ func (r *Spec) newResource(sourceFs afero.Fs, fd ResourceSourceDescriptor) (reso mimeType) if mimeType.MainType == "image" { - ext := strings.ToLower(helpers.Ext(sourceFilename)) - imgFormat, ok := imageFormats[ext] if !ok { // This allows SVG etc. to be used as resources. They will not have the methods of the Image, but @@ -376,7 +370,7 @@ type genericResource struct { // This may be set to tell us to look in another filesystem for this resource. // We, by default, use the sourceFs filesystem in the spec below. - overriddenSourceFs afero.Fs + sourceFs afero.Fs spec *Spec @@ -411,7 +405,11 @@ func (l *genericResource) ReadSeekCloser() (hugio.ReadSeekCloser, error) { if l.openReadSeekerCloser != nil { return l.openReadSeekerCloser() } - f, err := l.sourceFs().Open(l.sourceFilename) + if fim, ok := l.osFileInfo.(hugofs.FileMetaInfo); ok { + return fim.Meta().Open() + } + // TODO(bep) mod + f, err := l.getSourceFs().Open(l.sourceFilename) if err != nil { return nil, err } @@ -497,11 +495,8 @@ func (l *genericResource) initContent() error { return err } -func (l *genericResource) sourceFs() afero.Fs { - if l.overriddenSourceFs != nil { - return l.overriddenSourceFs - } - return l.spec.sourceFs() +func (l *genericResource) getSourceFs() afero.Fs { + return l.sourceFs } func (l *genericResource) publishIfNeeded() { @@ -711,6 +706,10 @@ func (r *Spec) newGenericResourceWithBase( baseFilename string, mediaType media.Type) *genericResource { + if osFileInfo != nil && osFileInfo.IsDir() { + panic(fmt.Sprintf("dirs nto supported resource types: %v", osFileInfo)) + } + // This value is used both to construct URLs and file paths, but start // with a Unix-styled path. baseFilename = helpers.ToSlashTrimLeading(baseFilename) @@ -724,7 +723,7 @@ func (r *Spec) newGenericResourceWithBase( } pathDescriptor := resourcePathDescriptor{ - baseTargetPathDirs: helpers.UniqueStrings(targetPathBaseDirs), + baseTargetPathDirs: helpers.UniqueStringsReuse(targetPathBaseDirs), targetPathBuilder: targetPathBuilder, relTargetDirFile: dirFile{dir: fpath, file: fname}, } @@ -738,7 +737,7 @@ func (r *Spec) newGenericResourceWithBase( openReadSeekerCloser: openReadSeekerCloser, publishOnce: po, resourcePathDescriptor: pathDescriptor, - overriddenSourceFs: sourceFs, + sourceFs: sourceFs, osFileInfo: osFileInfo, sourceFilename: sourceFilename, mediaType: mediaType, diff --git a/resources/resource/params.go b/resources/resource/params.go index f6ecea35ad1..4cb41715da1 100644 --- a/resources/resource/params.go +++ b/resources/resource/params.go @@ -14,7 +14,7 @@ package resource import ( - "strings" + "github.com/gohugoio/hugo/common/maps" "github.com/spf13/cast" ) @@ -25,65 +25,6 @@ func Param(r ResourceParamsProvider, fallback map[string]interface{}, key interf return nil, err } - keyStr = strings.ToLower(keyStr) - result, _ := traverseDirectParams(r, fallback, keyStr) - if result != nil { - return result, nil - } - - keySegments := strings.Split(keyStr, ".") - if len(keySegments) == 1 { - return nil, nil - } - - return traverseNestedParams(r, fallback, keySegments) -} - -func traverseDirectParams(r ResourceParamsProvider, fallback map[string]interface{}, key string) (interface{}, error) { - keyStr := strings.ToLower(key) - if val, ok := r.Params()[keyStr]; ok { - return val, nil - } - - if fallback == nil { - return nil, nil - } - - return fallback[keyStr], nil -} - -func traverseNestedParams(r ResourceParamsProvider, fallback map[string]interface{}, keySegments []string) (interface{}, error) { - result := traverseParams(keySegments, r.Params()) - if result != nil { - return result, nil - } - - if fallback != nil { - result = traverseParams(keySegments, fallback) - if result != nil { - return result, nil - } - } - - // Didn't find anything, but also no problems. - return nil, nil -} - -func traverseParams(keys []string, m map[string]interface{}) interface{} { - // Shift first element off. - firstKey, rest := keys[0], keys[1:] - result := m[firstKey] - - // No point in continuing here. - if result == nil { - return result - } - - if len(rest) == 0 { - // That was the last key. - return result - } + return maps.GetNestedParam(keyStr, ".", r.Params(), fallback) - // That was not the last key. - return traverseParams(rest, cast.ToStringMap(result)) } diff --git a/resources/resource_factories/bundler/bundler.go b/resources/resource_factories/bundler/bundler.go index 59810e34741..6655ee5c330 100644 --- a/resources/resource_factories/bundler/bundler.go +++ b/resources/resource_factories/bundler/bundler.go @@ -124,9 +124,9 @@ func (c *Client) Concat(targetPath string, r resource.Resources) (resource.Resou return &multiReadSeekCloser{mr: mr, sources: rcsources}, nil } - composite, err := c.rs.NewForFs( - c.rs.FileCaches.AssetsCache().Fs, + composite, err := c.rs.New( resources.ResourceSourceDescriptor{ + Fs: c.rs.FileCaches.AssetsCache().Fs, LazyPublish: true, OpenReadSeekCloser: concatr, RelTargetFilename: filepath.Clean(targetPath)}) diff --git a/resources/resource_factories/create/create.go b/resources/resource_factories/create/create.go index dc565056d8e..36a29e733fb 100644 --- a/resources/resource_factories/create/create.go +++ b/resources/resource_factories/create/create.go @@ -40,10 +40,10 @@ func New(rs *resources.Spec) *Client { func (c *Client) Get(fs afero.Fs, filename string) (resource.Resource, error) { filename = filepath.Clean(filename) return c.rs.ResourceCache.GetOrCreate(resources.ResourceKeyPartition(filename), filename, func() (resource.Resource, error) { - return c.rs.NewForFs(fs, - resources.ResourceSourceDescriptor{ - LazyPublish: true, - SourceFilename: filename}) + return c.rs.New(resources.ResourceSourceDescriptor{ + Fs: fs, + LazyPublish: true, + SourceFilename: filename}) }) } @@ -51,9 +51,9 @@ func (c *Client) Get(fs afero.Fs, filename string) (resource.Resource, error) { // FromString creates a new Resource from a string with the given relative target path. func (c *Client) FromString(targetPath, content string) (resource.Resource, error) { return c.rs.ResourceCache.GetOrCreate(resources.CACHE_OTHER, targetPath, func() (resource.Resource, error) { - return c.rs.NewForFs( - c.rs.FileCaches.AssetsCache().Fs, + return c.rs.New( resources.ResourceSourceDescriptor{ + Fs: c.rs.FileCaches.AssetsCache().Fs, LazyPublish: true, OpenReadSeekCloser: func() (hugio.ReadSeekCloser, error) { return hugio.NewReadSeekerNoOpCloserFromString(content), nil diff --git a/resources/resource_test.go b/resources/resource_test.go index af7867eb1c9..565ae06c433 100644 --- a/resources/resource_test.go +++ b/resources/resource_test.go @@ -21,6 +21,8 @@ import ( "testing" "time" + "github.com/spf13/afero" + "github.com/gohugoio/hugo/resources/resource" "github.com/gohugoio/hugo/media" @@ -61,7 +63,9 @@ func TestNewResourceFromFilename(t *testing.T) { writeSource(t, spec.Fs, "content/a/b/logo.png", "image") writeSource(t, spec.Fs, "content/a/b/data.json", "json") - r, err := spec.New(ResourceSourceDescriptor{SourceFilename: "a/b/logo.png"}) + bfs := afero.NewBasePathFs(spec.Fs.Source, "content") + + r, err := spec.New(ResourceSourceDescriptor{Fs: bfs, SourceFilename: "a/b/logo.png"}) assert.NoError(err) assert.NotNil(r) @@ -69,7 +73,7 @@ func TestNewResourceFromFilename(t *testing.T) { assert.Equal("/a/b/logo.png", r.RelPermalink()) assert.Equal("https://example.com/a/b/logo.png", r.Permalink()) - r, err = spec.New(ResourceSourceDescriptor{SourceFilename: "a/b/data.json"}) + r, err = spec.New(ResourceSourceDescriptor{Fs: bfs, SourceFilename: "a/b/data.json"}) assert.NoError(err) assert.NotNil(r) @@ -85,8 +89,10 @@ func TestNewResourceFromFilenameSubPathInBaseURL(t *testing.T) { spec := newTestResourceSpecForBaseURL(assert, "https://example.com/docs") writeSource(t, spec.Fs, "content/a/b/logo.png", "image") + bfs := afero.NewBasePathFs(spec.Fs.Source, "content") - r, err := spec.New(ResourceSourceDescriptor{SourceFilename: filepath.FromSlash("a/b/logo.png")}) + fmt.Println() + r, err := spec.New(ResourceSourceDescriptor{Fs: bfs, SourceFilename: filepath.FromSlash("a/b/logo.png")}) assert.NoError(err) assert.NotNil(r) diff --git a/resources/resource_transformers/postcss/postcss.go b/resources/resource_transformers/postcss/postcss.go index d26ffad5460..125989a10dd 100644 --- a/resources/resource_transformers/postcss/postcss.go +++ b/resources/resource_transformers/postcss/postcss.go @@ -130,7 +130,7 @@ func (t *postcssTransformation) Transform(ctx *resources.ResourceTransformationC if !filepath.IsAbs(configFile) { // We resolve this against the virtual Work filesystem, to allow // this config file to live in one of the themes if needed. - fi, err := t.rs.BaseFs.Work.Fs.Stat(configFile) + fi, err := t.rs.BaseFs.Work.Stat(configFile) if err != nil { if t.options.Config != "" { // Only fail if the user specificed config file is not found. @@ -138,7 +138,7 @@ func (t *postcssTransformation) Transform(ctx *resources.ResourceTransformationC } configFile = "" } else { - configFile = fi.(hugofs.RealFilenameInfo).RealFilename() + configFile = fi.(hugofs.FileMetaInfo).Meta().Filename() } } diff --git a/resources/resource_transformers/tocss/scss/client.go b/resources/resource_transformers/tocss/scss/client.go index 41ff6743355..e69af2f748f 100644 --- a/resources/resource_transformers/tocss/scss/client.go +++ b/resources/resource_transformers/tocss/scss/client.go @@ -19,6 +19,7 @@ import ( "github.com/gohugoio/hugo/hugolib/filesystems" "github.com/gohugoio/hugo/resources" "github.com/gohugoio/hugo/resources/resource" + "github.com/spf13/afero" "github.com/mitchellh/mapstructure" ) @@ -26,7 +27,7 @@ import ( type Client struct { rs *resources.Spec sfs *filesystems.SourceFilesystem - workFs *filesystems.SourceFilesystem + workFs afero.Fs } func New(fs *filesystems.SourceFilesystem, rs *resources.Spec) (*Client, error) { diff --git a/resources/resource_transformers/tocss/scss/tocss.go b/resources/resource_transformers/tocss/scss/tocss.go index 17c32ea8ece..ad581d681c6 100644 --- a/resources/resource_transformers/tocss/scss/tocss.go +++ b/resources/resource_transformers/tocss/scss/tocss.go @@ -55,7 +55,11 @@ func (t *toCSSTransformation) Transform(ctx *resources.ResourceTransformationCtx // Append any workDir relative include paths for _, ip := range options.from.IncludePaths { - options.to.IncludePaths = append(options.to.IncludePaths, t.c.workFs.RealDirs(filepath.Clean(ip))...) + info, err := t.c.workFs.Stat(filepath.Clean(ip)) + if err == nil { + filename := info.(hugofs.FileMetaInfo).Meta().Filename() + options.to.IncludePaths = append(options.to.IncludePaths, filename) + } } // To allow for overrides of SCSS files anywhere in the project/theme hierarchy, we need @@ -74,6 +78,7 @@ func (t *toCSSTransformation) Transform(ctx *resources.ResourceTransformationCtx prevDir = baseDir } else { prevDir = t.c.sfs.MakePathRelative(filepath.Dir(prev)) + if prevDir == "" { // Not a member of this filesystem. Let LibSASS handle it. return "", "", false @@ -100,8 +105,8 @@ func (t *toCSSTransformation) Transform(ctx *resources.ResourceTransformationCtx filenameToCheck := filepath.Join(basePath, fmt.Sprintf(namePattern, name)) fi, err := t.c.sfs.Fs.Stat(filenameToCheck) if err == nil { - if fir, ok := fi.(hugofs.RealFilenameInfo); ok { - return fir.RealFilename(), "", true + if fim, ok := fi.(hugofs.FileMetaInfo); ok { + return fim.Meta().Filename(), "", true } } } diff --git a/resources/sunset.jpg b/resources/sunset.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7d7307bed36efb65bf0443b62ebec6be6d7d3420 GIT binary patch literal 90587 zcmeFa2V9d^^f&%wK}Z+@LO=w4Ku}Od*kLG17=p3{R8Uj`1c@?27@}4S2)L-CMcgu~ z;x16MXdM`F)Vgqs3)}-OTD59X{^veni+$Vo)3=}BuYZF{Ztgti-0vCpJWov9)wU+R zO0Nt_EQa~|4#nsghA}V=5*agqQXV{6P*#WcGI;8fXc!6JE8y9`y<7ujGZKZUCqvl= z`VWKmT6o$*nGNsF@N|Xp0C-0oCG)97{{c`oU7#qNQ7~)?6~lskJpz4jUw^?!-++<0 zN1%U@ARycV_Y3v3!h*a4LIq*?XiGcX*B8rIOxF1>h)IeP<3VvrsYxkuNmATTGBqw$ zmXwr$J6hS;;%5G$Xh|Y!GY87JHcl=!HZHbyIM><5!O_LR9*dfmBulgs%OumSq~aK{ zEJd8ipPC? KutC4fBEWGlE&1vb~Wm%oCt@d5?+zyLO+y&Qu$El|dQA&hHl$E6e~ zB#{ICsqN)eP$n&B5R%vdW%6>x{MD#Wd;NYWD=$~fPlLHs+RH2`Gur#tp?OLaebDZx zmnr5s1@#*3<%>{em37gn+g^rhjJ#1%4@Hd9txRh#4}^N9ZuRtTWtDDaMz^wRdl|{Z zb|a0D!wp!>exm{%t!QC*AbL;0lLt=*JQe%Gb+L8AxjYv;o{NKRpLOTTQZc?z1B<;j zns|qy#CUrEFp~X`B^V9M?0PS0Z+`^~qID3uyT7L-MB@r{S$YbF;x>+!4%~zU+}qZQ zhkJSA0zZ7BulFQ8#M2+cv|%iX;D`+DkuWm+VORh>$*?HmSPQnD(q2CvK2zK4heAEt z8^kqwC=s79ObGSL?R`c;nGR#oJX{cqO8Yzl_{?Z8+d)~iy*wJqYVGqx!C3Y7`cSCX zXs@3J^{j67n(g&6sMmseW#~H#%7##8LOC1C=wSj+M5`c!hOvgA8pUVSjOwe)sXgjF zK@c%euhiZ?xcwdf1;ZWzLx?BT&Re8-N9abM$BzuLXR(x^%_nv#kVJehPQB|1}; z!C*2Nsw`FF(f+3R(EF97tgNi2s;0+c>9N(-)!FDpovlf9(^R~vw!ifrs`%V?6TpGD zJ;&4u>xISCB0}40F-|6tMxxS`lj;Wn} zc3D-6CiEl&zR4uTKp3h?)}m-r*<)w|8>2xYTVa%JSz8093Y!W|nwUG*dQpF0MM)i7 zZ$YVkSzVlECGK6fWsf}jI>U%MbP=nhlEGlbW#{PQrDU}hR#17p($NtMh8=k{Z|*?P zeofVa4gHC?@BGVD>>G?XUWhCC*1wEy{~rG~5Nh6=nbcfzfN?Uzvv;I%cJ9qHEi7K6_Z!}S#7C$OC-ba&Q2(80l zY8qRhCEx`uL1wbxg~7saQ#|`O#?9mD<7YL>)b!`Dh(%JBp3GtO-}-8d_an5>K124v zw)3s;FI%07iI3B%+?E>?*KgU-ewFyRx~z!|YW&dB{tQ~YnPo+ZfWcJA?NN3e-GUD< zX8i127Y6&09g#U~@e`puC>6uh?^Z+l9Bo^nP+) zQMoM-tx=Nhxh2*!YpoV7euA-1MTxh3+loqj&yg}}yxHRTp;Q)L39Q6rQT6dXhs(<9 z^zq8TGFZ&fyjn21zo&Obg64#lBUQ%!jtjm^vs$3?YM&YH)joVkNL=Rn09U;2;@jzi zCapVjqF(jH3iT5OskW9ZJ7fOo+@RCBRJasf17CpoSDNbVfZmJikZ>lGaSYg3@JVy9FfkoLGIQ|aM@79Wq7)0)@oqTr(Y$Nes#>KtD!0k@@+PgCxsOnr*HhZ7y8t zSyf~{Kc;GJOM_|QS+%kW7ra-ck2Y6Yl^%Rjd+YI-s_gg4*DOj#6npM2cYXXS_~h4Z zSKisjPuOFA$v`4@iT=i-++)ya^9%67aGB{&mwv3#W~UaI7N&c$tMaePC)7_Iuu!a1 z`KV}2fBm8{YxTpurfZ$F8x(xf&Um!h>4g?$Rji^h6V(@r`&9sW@=ZG}ghjzQ=3=MA z{)?v49L?PfwWF3lN(~X09;99^IjF|ITJG9d?`VGJt#(v#+qrkCV|H8%sJZLEDD}90 ze8^_rw^8eDj+U)uKZ^0Ks=uZcA5vPd`Dnk2t(D#}E`xvZj@LJ@+zQK8J@J;W9lR-i zX#AM9jQD-43*t6E*c{JD+_$0v3NgNWw@%bryJmGI`^Lqc-#9shEp&1S%5|DF_OSoy z@Wb`~r^7`4S3*~X-nf_uwZgZdH!htuG2gp=eaqIp+uQ2j-LrpK_44_%@2ZAIUvsig z8#yYp**^8S-|Qh#kDe?kd=pYpUvFsCTpso4(UO8U0Y^r9m}`h#bv>$9dN$kJNm4^9 zQ|-)CgJ*KMut@<&pN4H7U%8%sa*lSzxt-3b;)Mww|i%;iVE2~D>ypjz%1`a7k_o( zzZo(sb?>d(^~Xatc<0Uz-{91C{#A{>?$7IPdyGsSvwLb~YPy#m5R^8u|7`QC`(Ven zrCTjobyd5f{@jU%@~GGQ^p0)4nznk!v~+zVzRVQPOns-D_o_Rm*sa+Un>WVFQ z@u;)YUN74jaAC(caUr{Bow%I4`oZL?YMSfHtqG*5duJ7q*Pl+a2oHF@g!2-ZnHX|) za^B1EV$&^+(@j^uoVbMg6XR^v0j}<~K=CYtm#hsxKi#`gR(a3PP2D#_*YB(O7BvIg zF1+FmU0XE0Y4lT%eM65e*}m@jMvq-XAIu0&kX+M=pL6-^Z5Ot=T>WFmjokr7<3`>b zy`tP>nDO&XKX1JjuA63=#@)Ge-%o=#i6iD*hz@bwW_M|7?Cyk1Er$)h-Jf|i?6F}> z^yot|`McuQO|yJ)R$TRLR>JGNY3I(Ur8=mYaBlZM&X!%;{$%F5Rc8&Z{4vG-=)&9Y zKi*Chk9PfjvGc*rc?lt|JDM-&==eR~{Iu-)*nw%QPtQ6rH2h}lO#P?BruDlSl`@TY zVkml_zB8(H{lyL5`!1u8Gxe94GV*3bJX}$8XX?VJx|iQibQ?Wx>ZtAP&DVO5e_$3UK#87=GlVpXV#Ux zT2^xX{4dTKHGz+Je3f5%?c?r-x_r06S*JvUKW<8S)sl4W=PUk)T)k>)@}r+8{xWIL zg`JmWEtMaf283=H{;uK2-;FQa`eW+|&cOT8Cx1SF^KC=!9?d$#xBG8zx;)2X?k|%s z2P~LzTXKJi+07QKfqTP?nFDu@{MOwtIq5)v(IVA-MTgwCL_9pabm#GJKMwFVY{`}1 zKO<^xdb98D%7ull(P#7bg&&X>Kgm7wRs2Dhkyh@Lhs^r%_=loBYjii5df7GnOgp;n zp+nLAD_XzaeVbD4_i@va6E6paFL?Xvn}(eY7bhmgjypSk34L>93Ui<9N!gFTTW>mh z_paMa&ykOR+dtPg|3&f3_bs>5=e1#GucvHywDR^X&dn!NA83qS-B|RU*{~Tqi&qUF zHza#Pe#tktYT7XFtQ~>WecnN_zeY@1HGI5t@b8a>-{Y`mG_y*NAi||_~4nd!@ zO8#D&KV8#!pb_+gTR$iExL+gO*AU&W=w<&n3wOViv?Zc^-jJ`7>rUM=GyE{_eeTH& z@nv4rB+X|vH(T#sO4;D-`(lG|rNsNC+ue&+AvybRth3XIEJ-n;oEk1{`bC@dZht-Z zH|uM`8G)KBH4eyI{8C>3KK;Y$U%vnPT^n}cLm+jg&A~%I?tJESROk7drK=|2a*Gwc zdK)EvGI8YeI{DWNZU-IEYdLkWc%bTjwaG6-s@`sjdzbs53izki(g>X}o~e&_k~ zA6UjmD#wLBUAp+1`^5f_t1GTm@~50Wq9pwAL&*0!&A&XldwbFk$0}oZ{8Q57H(e?x z-#EAd)V!jmzR7< zSj$|n>dZ!OdBSF|$@y~)-_22VeX-lT<#_*^@A^sa*gGHB75e->SH+FjI;hxs*Kf}# z@khBECdBasH5sIX!M`4zedo}JUj`hsoZat3s$D}`r25ex9ybf*7K!dJLoOZte!BJc zsk>c@9G_onsaZSW3T5()kc&s>?EkneX}ZHbrz6de2I=Z8{yBXTzaq(@I8f=pt-z{B z1Fk&5B>ji4+*{|lWcR*_rVCr7X-nO|ZEnM!?pxdN@tW4|7r`S9hP<2naPRH@+3v3= z&ka1*#C_B9akT4f?9H>?{XE}08%y>?Dj(9=bmh!1FKm*&nt5Z!Ztt_AOXP{p%oIb!WnOgbl$yz_ z3U{}n&}C;G&W!)jqT>EG<@jII$I*_sPIPUR@#W)pug<&U_I%ddlWKw0k4r?$r&c@6 zSPTuZ&tXm1gL)pwJZz)3vHbU9^OyO%epHv;&P(p+^00F3O1nI=fHNaa zx$R2plT-Y=9BcAD&5uFDHm~DEPWK)0Sa;vNxkGo@-6XeRr;ES$EYX>mzL_IZi(V(A zW{-SQIPpl}c(-4tZ%Q}5etp=4ciZeuL{9kA7|!wICsZAuIF}rATP@vPXaBod`Fv*B z@Afxp{x~t%W&TRefsN}~2^G6{rtJ9Y#{3xlOox1{fiGQGKPzLXE%AGyJA_B?m1>h5M;3xtwHo5qGMr zzjk%re0!Z4KM&wqBtk9AC5mViwBuPz4 z!|)3#R;Ism?vYiK^Y5Lbz1*j^PF1{KcK6YD!uu_rX1Z~eJY@&{qGW>$%PRJa`eB;( z!7C4%q5e?j)Y_B?FEz^`~3#ryqgnbpB6xMD6!AaD;{h1!`<1yUwc3wa? zUw)t@Q|2aD)%Kyy+ba_YH%FLg&n_!p^vCT7*)$qCJgqnLBz@M;aq zpcwBnCYe6JCvBK! z!SQ_{H5Me_WKTJAaJ-ALmYeA=$?thl_kF{(=LhR}FP%^nO z3$Fh5`1`Ug*6MdJ6#Qa5BD~D`!RAwIm0n2?9Fm*$_lx(>>p%NS&3k-a|74NUhnbn| z{!7-MP0O^rxcFn%m?fK54NShRbzb?Bd3**Vi)Npjbz0Xj;dkMA$-P5H(I;+%(-!$s zGHV83F>YN^cG)~;@aeeAAATx1;(a|Uf68i-S>?g?Z+D!tJT+yF3U`!qu9@$`#)$la zxQPDObhiI~n&xm?ZTAn)=Z!z~PEfgE@#2_yG5fBVd+na8`SmtcTJ45?TXx8wPcep1a9RG&|zB=)m{Izn+#Tz0kv18B-mL?q#lz;<6?cM+)9OSa9xz z!Mv~LuUKeMFlPEXUjsMK*bB3-|7fDrAWO*l)yX7J!+L9l*R)a7N0AQ8WJeb^yf$-Q ztD5_kuaV*Qu7a{_i_2rlRpl2H|0<0G&*rc~3%u=2mew^NKe9nUX83p{P1l{hbcWkc zlW2REoZ{(6+$yc5yb2A}aIE6_)||4Bm{Q(`g$rmKrti*sQ_0!;imK!N&7c^)TLI*> zLuXw-uskH|Y*@AW8WATRuV195KQ8QQ#BH|)L&<5HDZX(Wx8;keg<&c8TTYwDTW6hq zn*3BzP_s$sq! z9#evX1HFCyywDARn5vH-kc$k0-gYtKlxQgDKskO!sub0? zK$#si9hKoOOeo8MKq%{>@>E6Hg6N|t^HDh_Au$H}AeyBy2{EW#2jvCnX<{f-{Ggnd zE)maw@;$nW1#YrMnNpKfadaG%xlmS>g$8>-*$r-}Rh``tY(N;*Hosa)xht8g# zFLYba3a!^EJKR$oD@uz`#gR~>lj4)oWO#~H6fMRrdxQA@dSExQy3t_~ERGe+#EG!f zI6#>saVl(gVvHnJl9Y%`5}SesT@? zenvm|UdABI470>+F(=Fw<6~afC@c^Q#lo@4STq)gC15ga2KE(}1>Xo+j1^!jur*jQ zwh`Nk?ZT?CAF=(|VeA<83w92>gk8rPvHRFltOa|6{QPsg$&pR7u)PI!yYRR8P7_Y9c)*y(GOS zQ^=}hHrbFom~2hvkp<+@Fs%c@{Z`yo9`pTt?nO-a|f2{)K#r+(>>(ehqg@ zGbtR35yg_iqj*pPDHABM6d7eMWf7%_QbyTD*+)4}xj<>8G*jMDm8jZOoN7sRrTS3E zQlqKK)Vb7r>Kf`6>KO^{M(p^~dPX(BGhcRKK}DyT5Dy=>CiPSM_f& zAR7!d2sW5)@SVXagV%-w3`ZJD4Obf;F>D^d8NeSfZ9w6Gg99E9)E+oupk!d-z(WI{ z;5xVfkH=Txb$E-Bfsv0adk4m~@JJj`*JWZ3#)^%gV>XNyFOGK(vgs+I!Fbj$6QO;#K$ zKdWr3AFZBSn^;e6prGs-60W}nSV+o876wrg!K+Ns%j+kI_U zWB0;-n7!D3gZ)(pZ3lmc`3`c24~|ZbGRIwxkDW}MBAwPbUE#5LLA-q431^zK(D`fU z{m$=P99>dfs$70`wRD~Cy3O^G+YmRg+eWv$!;OcFhL;X+9D$FB98of&(cQ>hdH6YLN?7jlI%;cnsY9xfhpJZe2Do+CXMd7kmo@*3;4#_N{1vG-K( z?cOaT9Y)R`S?fdd8SS&o=ZdePZ?x}L-$^7>Dll@2e zf9roEz%(E+;Kx8L&^Pegz?(rsf|7%N8lyBOXw2F%4}xuizY0Dcq8&0NWLwCaP+{oO z(CcH(#-@#xkJA_zF>dR)-^Y87Up~Gu%sOmN*e~Jw;gay03Ca`3P1ro)^+d0UD<M>FPu8CtKl#8EjVYojKSa_Z$4739Y!wBH%0#cDe52MywM2VFuZ(^g z!;dM9c`P0-UM_wZJ3MxI?4zl}r+z#2ah!WxQCzdcL$XHlVw%si;%UE651d{;y)Axx z{Pzj;gvf-wiCT%%6YG)&BxNMkONUAGq>afg$t#juWTR!9Q%EV3QfgA!5KNs)GfT@& zYf2xHzBc{sjBzu5$k5D4%s4f3=*)#PAI|cgwP`kWcJyrdS4LlDebqEaFsF1bX>Qb9 z`PYNK&iVR&=E%$~S&XddS*NqP*~_zE%^N>&Uyfl;X3pK*k-6LR)bnI{SLVCUFJ3@h zAX#u`q20pO3qLN3S#)wScX83;*8J%Fli%2Uv+A3+C9zA+EOlJEp+Ko1vEcGD_hnm_ zYc8L){O-5@-|i{I3l|l>S}|qC&qek{#VZ*r(^fXF@>^B2+GKUX>ee-JYc8!7uKj+U z!McU(eqSH6{`?01hF!&m#fyvImq<#kmX0jlQ#QD)=sTtF(!YDKaoon^n|PbHZr0zt zc=Lzy#PY^1!CUIKI&Iy$&0yQo?d0ug+aK?U*l}*B=gyj4mb*&7=X}4o0;@=?cv2Zz zdAVwImHY?Z54);OtJm$;-o0oKY0u0(EkDNn*jN)@Q@?lAUim)PeY=0M`l)=s(f+ju zbPg;%$T*mDu)&o`QtB+zw9~XaBBByhts>yIG)*a*6D1`Ip=fx>fP!Oo_9Zg_=4w!V;6ld z{&FecQvKzzm#c_j4YoJt%rO@Zr`+wvYBd_I`Zs$;2o3pGu$p(VX{8^Vz!R zX3wjC<^OuBC9LKC3)zddm-)Z-`>p(y{VVzFpx3wFB)s|K_eF2}z1{MT_wM-n@$Vo0 zF{4$vb=3#+5Bolj{&=G;p{=d0X`;A2&PzJHGAkf7de-!4=NM!65e&Bsmy85y zf;d%#XC%ZYrnqDbHzp=`fikMLHpYoAsndrWE55K9<{t#HmQ0L0SUFlo+t~7OCub`g z2WJ~wdkY+Hz;LwY+F9H1EN$#vxHc{h4tV>+cmxA?lf}fk1bce7F9x568+VIdXV_UK z$);LEH1BN9wY9djwS*p)DKisOMH!ZfDTB}yia9(Xq?bvgM7W3M6h$Sar4BbXhIYJt za#0%a(wE4@@aUiSv)~YOtiH)UYsCKOih6?Ha+@--rL2f`};{(TSX*!+Zq9(1LoLUjD+BT_NM zQoYhantchcW67Qqc%_4YdQ5?Y>=o8IN3=vLvYIBf?v#D^<$R^7(e}V5LRIGo*xYDh z2U1dFth@Ih1Vn(f2h-8Al+Lj|HX~Y;AeMU^OP%B{oS091b@IF-{d*Pm_wL5);9$ zx_E$qQpKL|9Bzzm(6Y4QTG~2=c6?vBXTKy_Xi`#qhpp+lYwYu14(}ZACyElo$Z&Cd zeA0~0T7NKNF``sa_w{WoZS6ukzDGU`tW^6RBWnGVVkEIM|7!1G*v!PKfD~fAP9=6h z;}v`ZBtFMdfKB-S0K)HD5?7A-=Ag&w90cB&9)e3FaVfMC)MQ-`~X(5RI%C z!UC{o6(fmpiFM*dJJ>}zTiSDZ(U$gNF3(bAV`F0(BeJ)%caDmUa*9##0|Q_R-$aEz zbW9=n`%?r$szw4g(sifC+rysg>A~YVI0$UH4&Dw9PD1#zc6rVNV1pcnG+G&Q?KMY7)_-bF7UW zYypqgjz=`RZ(=mu;|Mz_iVqbhNWq+>rXe$_uu8pqz<-#~y(t8BChSd*+BlI+ng}F! zuk?ciT_PegBq1p&HI6`a*BTy>+>r$&Nm9U9#H101ulwK*TOmk5IPBgM5(nYLjL=?m z!6ZIuJ(z%+zKI`WJ@Fjx>XEiEM#$sDNa z&Pomh%${r+=L3EfZ1Fu5ZsU8-wtI<;=oBo+5A};w%TPJCsX;-XCNCkof zhfcT9wXnY=CN(Kh+}_2ZFTy()Y47CJH4n@$lZq3AMKO{zAYm9py`5{cw|PQzBmr;P zZT0pRub8P~e^JWxUXCCrDJ9iImL&bW#UCkyTj4)-1jsdmr;L@vi@g#>Q9W%G;kQMJ zX%HcFhS;A@5d`j6ilXjxF*r;Jr$s<{7faomeUfC7SxLxeex^fkk|Lnub`wdr@gXoZ z7}1XCg7ETPiwg)UJ~Q)JDREiqGp%qUf+hQ;IlNaB3JyBX4}$K~d1XkGQXmF^u+p~I z7`O(KDifuowl{U|b|*+udbcPr_(?Spm3&ewjE4-vC$%2&A{hi@y=R7NLoxu5SYb^* z-OZlupx}OREV1hn%_r?XlHtZqpSBZr@$=IY&F!<>e`Yi^qsbk+H^+%)^>CwL*FYbn zA*pcHX=-X*_YDE5rvn~ClK}cNQxwqg7R9G1&VdL#Xs`!d%@GHHxfyN@7lg$kS?BKb zXjO=@R}T;34BM_g5RxWDiDP2KF`YAXQ?3wkY8s>^Pz^{BU7btm?ADIWu4vQ2#*W8> zV7qq@4+s*(G89R7=d*pdqA>std<2@i?9C_b1f)M{heNg~{xfqEZ9qFn*v=s#0RZvI z@F1BOU4R>I+*8Eui-$r~mVyqWL<|pm+M6|R$asW^rxF>R_C8(g`-oCNengI-+esen zAhaN}*L&4IPyZ-Og}cF_7$SEOBn*hpio8{bI9c&Y@iAU9;!nSVV3iCmP6q>m${$bsi1cIL%eI@-m=SlaXK>@A&bcyK6nigFS=Iy%_FJF$7V zVvd+TPO^V@4sZ*9{~T&f$giD0AHqM)>?Yuj{bFk`66q`CjBohnJN#t)QdOD|x zjzh;UIG$N|@ZMWJ6oBl$VQ$^h1R)SfNMa#|K;rwe?Hr*y2=DsWyGTR4!5Id zBCa~M=d=} zgsAp*PENLVJw^t`#=_MgMSyDO%xgaxbc_vzOQ}SDnQLX^f*#QboO$v zced*_HbjiP5)h*hAh={C4Rn-l7bz@4$gL}I=&0$89oktKABQffprO`Y@$nLA`-uWk zso)Vcjf1bFgOK5HPdqxgRP-SDx3h7wcUA<_U1K|Vx8=2mS)H?nNM?x@2UR;K8++$o zGon_-S+%cLLA*2$r5F3AMsalbbBf^@E`gkkSk~UHbE=LI2gFB&#ogrRX8nnb6@9v4 zvwb^ydOF2UbOhM#8`Evj|4=~NI@&te+H`IVV!kJ!ot=BZ{l5liXFG>34i!-E0CacW z+f6|yplzM(ofJpHE=A}9bYHDs0B!s)SZG@~47OiVLmSf@(Dt@mZZEiZ0ou;a#<7<& z{4cc74o)_`?XUvSJ$e5EXp|B8Utpmjp<&l$X?g?N0q&RTa>zvMe+|%(TI?Ipc060# zF3EfW^#2Y(+dFb?Z4{Z(J}tBj&rxyHNN4MR4bZj@HoalnZlP`LY`VNtZ$R5SIPrRq z>k2vgYW)&&{2L($&)J^)c|b#aZEx2VVg3g#w6hb>p4S_;1fYR;=RP5;I277CeQMde z0Nq#X7eN0jfaW>#oO;>dt}F+#Uff=-|1Asc>;(7Dbp=%lJKOGfI8Jv;rmK}u25w)iUo7;$ zVWAxzoa}9S1r=Qu+Qu0|j$RS=e+$qK&W>E?E)Kg{X!!n+gI!;l=Jq@^fVQvJc0l*h z`UTMc2B4jI4i4O2Hn=z{l6m*ZDZ@~)Gbo%ZJ{Bl)qXqk ze>4vb-ra z*Gh2@jRmf#3n%K}$O3oISl}|^8#Y80m$)eg?$YhL#R=*^y@KA+LovDH2BnU=&dK4n zB&ch@j|!$iH&R*PNs2=uu^hTP=`(k<^duhLFYyWSaHm91;)xp|I>;dI&-x3*D{e;V z7}ZHUG5J43Ji0dCi+JLy_&-5BapOTR;@cGMdn__ z6PK2u?$2QJmx)JN<6gu=2E)dtC-KBZtPV2%Dr^)N6gx(>!^R$6M*L@pS6tudNxb69 zMo;323l<$@{8i!=7mqqdDTr5GGWw^8N4e}zh=(lpKS4Zkp`bVM=<>inMLfz|_9h-> zYkLw;BqBS=_^XmvB!xRhbx2+@`9FbqMUJW`@rvwJFXGXGr-O{YO1vTg*fB~$ydv!f zb^n0ED)JFMiRTiTh<|{1MY5+S@ro4BKSR9Y?Aeoe#i{cjAYO5>?@7EZah(5Wh*z95 zdJ?ZVVf+KcD-LlziB}xm{wd;7Oxc@w6ixmU#1jE~PvUKfF#T^6|0fY%M=Yo4((V2* z2=Ekj@ZV0s|955G`5#_&+@6ij3IFdN9hcVr->m>Me_7;jiT8`RzKH8@N#JjF{(`PA z;`&<>_*8S5i_|Qlc~HD)68)7^>)@cqNM6UPx3L zjjl}BU@$b;>gwuj^rFtzBzkBn-c;M)x{c~qY`Y1+_>tW998)L$%WW*C7JmMN)K-gW z(#f`z915n1ku*tUO;TGO{GJE=Xouo~ek24QWD*6V(&$Rc9rdKnUmGEjDO8%0GM$Aw zk!d6)3XP^rB~d9PGDgwVq8g#!r_*EiXP$d;G2yr6SD zs!Vk4qf_MP?`V=R3PvJRsT5@-Em)bPsYRw3QMGNx(AWZz&azTra8_k~Q;Ts_vJz*x ztw-5mawx@?BCiIX5dA{)8vZ!i2|^qe--^!?*<=RA9w*)zj6} zrR&lSv}khpV^S(uRxDS#E|cGkN8mvl;3b{qs-dW)8|E-`)S;iUh*Bd({j}$knRaoeZx z!~I~r!9bSb(gg$C&b1DgJO;iAp2-c$6M)Zrro;=lWz)Tm(a5x8+Ay68Oh}arxGO?bltuJr(Ui9ORsq5K zo-986C78i`&P3tPQe8bm*lCtl4u=N(laJAuW&&V}73Ot}OmJU_R{+!M=gh*`7F6g2 zfM~4Xu*||>qxIMp%zRq@po-j}g2nkmbAUNv8oWqz&^B8Ue>@Xr4nqq_M;i-GS;E?8 zTvPp$F1O;+eS6c zQoy|oK#$-~u4ZsT(Uy;8()oNrBwn?M1KdZqV_kR>z@SIBFoFt3sfcaC;E<0gQ-PP{ zOhP@xY!ko{@uz+aKfHuxbOC9M!MPWJ= zbQnTZ=SpX7R5_9>EKu>K%E#Q`u{LJ2Ge_}xz^$+V?X)%^pDld>>=G!b&9%gV>)PDN zw34seF0_tTY)uz5UL7X57wdwD>fNxX-xJBHK*qq@xPB_?Z_5oXafdV zJLuy`=472L?qeJ_J`yRppONSHw*cy^PPn&SYz>q=mg1&tIx=CFB|&(QDV@%yqbFMh z@EcakhX-&WMhqjE=LA*d5QX*~1qf7_fyT$t0pNgoCK(9>EFQlUbUp&iVZ;oO4T$1H zN&4_cFaoR`Egj&owg#~RhJZD>08|9*AfP!pR~lycO(U`nhV$b<+wyIE#s<$>)2$Dg6^woO(WB}gS9C6>T=RB zkcLf`4cEQxQtMdB6KN2RHamfwiGvt4_5&Yu&|bj38G<+3ZqD!$yn-{rh;5-nEu{gB zk&9NqA&#)$}FpD8v=nEo% z0^c-JVG*ciU%%Bxj!Ko@{_uzjXRo*Od>A-Dh(K>bZv8_DCS=43Vt zB;o#h9P;u})Ym)~(tOZ+I$cPnRhN?5AF$tG%{`dtBX~N*z5*HT(qOeqgM61R5cv@7 z)1h0{Vr2f1bb_jw`K3rYr7^lHLc-dU(8rqE=CAmrU;l)8<09KKi2Gnf zX$;$+;GfYXmBK)}*k#9?K^qZWc5Ld#itNbgB>5!!r( z36K+tkcWcZQrL7*m18U<7&owWKtEv+fXyGx=oV#n30ELI0WNaMITQ}E%?aMK!SR81 zvBJSDTs>jM<#2fxwD}9Yj_l9QEyR~Iw0D%!ASh`z0o16RK{3Gjb4W5vIwFFKtfR=n z_vd@z`~oW!5hzicYI9v`A^w|B7TrOiPK$UI(kS}}m&)+jd@uPUYOr*haJ2*O^HAs~c+2o&fUiJFx@(YDQVv8t1GS4>1vKzK?Ey!W=GA zoElUp6_4RSfV^b`1b-3u42?QPaP$})X3bm(6;+NrN3Zf{WPKDP+PCrtuOmxp2d~Rq zGh4o}99tt@AdzR4-y+#JY6p-EYMYJjBr!<3_SyMytqRHdyxummkp5Jeby)YongP{c$?XB6U2iNmVR%q)!KP|&{1!Mz|Uv9jf>kf+T>;t%bJY41noG_#@&JH^l%Bb)*LIAX-hCGwl-FfGbphh*mR zyukE){Sv}OVcJtLy%CO?8PX-_kcFZxdhj3oyyA~wNGV62Q0bDl;EdgP)bcR+9aG5( zl5eQLvr_CjN{)KFH^#}!LG5VqGx-pF(W2lJ`0&0p-MFvAiMy{l$Ku_ z4L`kU0ic>mhImvoz8^ZbgV7CCG3xlNGI&=m4Tc!Vi)KHhAqixM1c2L;LJme* z+KNok06f`;6@tPwNAT1~STh}1PLLT@W98?I-MRQy76!HK`iPgr(5%NJAb;;cf z{1yA_jfC^sh}^YYUf?5S-eJpY!xk_H?g!-ny{Hw;f@;t`zyw*Ya9^mw) z3z~B>YrrzWzYOHn`K7LWk^65X(H%{pikBShJUyhqDB$+}pQem02X|x1ziIB z;=@#L;S3;hpDbURe2h>J#^~>**gB)n{0DYYP-Wv%{li9HIJYZ53G1rbyf1>FI= z!!Ll;&;`)E!RyqI)Il0h&Aplo#+u72AuJE69JeT-ggnQ|CAEW#7m$`KdEpTBXa}(1 z7+Fr|@ger9D4VrTAMB7E7*+9Jq<{}-UMPrzfD*`u!mqH0WAGnfsp8lW1NeatV-X(C z48p(+`runJs@54OCekveJ)?`nW10A<19f1ZN=0-Na|A0e-G|~7Apk2o4KN~z1QQM! zMv&b(mJiz+(oiJGvgY8$altAiDi}GiJ%KF!O(l3!wZ~+yX zG^@6BMyC7?$VKEy)y-gALs|&TyQ zgw@teMtLe9R}HPj^UY~0iZ~mRPIO4JgIgzyu)-gGsGAz$FtsPw&+FX;emJfK#|tc2 zDZYTTS=DUOoESKi(yUQ%l!S>6S#Kl`5%mQr8hl*kQxcWS z5-`FbkOqkiKDUu21Cv-1R2=7_f|t71X4IDA_+}c|I!a9xO-^{nrfOym^9V}fAmJHJ z$^nGcPHq{Z3MmKTJYJrGZKQ`xuhmNcD@|uF+m9FKY6oEE_~2DsJNwoIIH^)>3rK;i zEs{AKE7W+@jbW#kW3?zus@pzXT@A}BTj&iUfNioTD=n(}4r$sg3iJ0&6bpzV9mY_+ zrfB8q{)(TYgCs#uKWwC1k9e7&E=Wbg?=Aj!PSJb6KBWs{f{)i3-#t&GfY&6;DJq8+uod<#H?!i86GUc!`T{R?8>Qz1GBr`s z;)C*9f-R@0+&~8&_hyhk04xvWD}|&D2aYFT1Fg6KSszGn&0k;$i66M$0GDefSgY$C zD_(%ETU0V;Kxn|b0O@+(jK}z?3z?%ba%a#<*lu?gshp!yjt^cp;RpLxuT05;3UG}o z<6-BAkG=mf|89ZCR&Q&_=HN)|EZG)^AqQwbNd=9_M>kzIXm zc(3vXuhNUBJD*dnv~CbYX2Q~P&JcB0hIE(xJ?%UVAL<3evcvX`lz`D=?C%`5zf(I+ zYG>bAYTv{!$y&{aPzpj#2&xKH%I{cw4H4C4>hn9(09~3TT6t7(g7xDlW079ck%DH* zpPAK=3qXfv6(G6Yz? z!^)Ny_*y^!?*y0)6S4x&=wwJ){(JfLk_LXZoHb?R(7T|4@{COR(fPS|j!u|sd_+E) zKEy{gWHAI)taNw{i{MoXCs=&P{{2ANMY*p?#Ie#8yV9$T0(X{OJj$!K8tAjTjQz15 zB!=VJ7D3CRS{jW4EZ{H<1HD|m7U!rp{VcXtxA0R9D+NVGX*RyU^Qd%9@ep;38S0~{ z7Z!&$y#Zed)*tLSia5!^Ua=@FXJ@T7A>wl#2(V}%K@EtnZj_}b$CW8%j?!OJVLz5- zPj1nJ7tN9|5F8x)vPveE#BDbd%|4+wX%hroR5c8k7kqu0Kw9>PCO-?K+-2}Wdx4}? zMnP$0q|p|$d^T!bAC<%M73JjO#h}Iv2+0Q2i<;hK%2!onCvfz9Dj<%MCvnWeEdXlV z4G=Rmd*!gyi@|>wvwS^{@#XR&O%Y8~l)G4U$n70r0_B88e3^(=5L7W>gt44?hjM%e zGxIS8z%24JF%m)Wm+e&;ma%xO3cb|$Vz$tV%vnq~znYPiT!5~@p$m`boD9jvnhcb3 zgPdx!33X$c2)v)Lf&|AtGe|yzbWn8K4zT4|NF4^*H;S7ZA7ZV>vNTy-ndTM{QV>Xo zrN2O*gj7{IPR&zICo&G{Ef<2qEQ%M)a6g}jB=|}1N9j#z_0ig)M9P$3eFh3w5*B>S z0xw=y1c8x-l6FRg96U&vASQ%qxk&d}gN2*r%K~nth<4~aS%qUZ>0U+luN+8dhE=?H zOp2NXIVvA(s|}FK3=>4+?&mEO$t^YVufNO$eI}hJlRLEB51q2v*(bgs%L@WqQP7Lh z6!t1`WWo#JETM1!xkO6029Dn7yp0mp5cIn;g+KM99)rs*(U8Zq0h5MAW(xih+n9OR z%+zGJ)=Dt@%OVRGdy9=PGWuDNmqk1JRIU!IU>waH2U>Zt^)mj$frlZlI9Il+B}fh3S*xJjVWUbh{VJp2^XhvOjFvfrM%`@`sHT5P3~5$|?zG za4V;KZ@p!~-T%{3a1o?!AMRDu9FS6G=1zu3 zWdUwVSAWS+UO?B_PsK8izUN2sQHqC5sma`li)QlqP$5bpTyH7(XIMQ8%x0}<7RU+G zIfSo48K0n8NdVW~fW5mMxCwyCVTD7Y5H5cmU7Q~VX+Y!S(XeXeRj62U%pwA=RBeSz zG^43IRPy(@_OnP1t4Pjb99@`Sq~f!iyeyKvn7T3CC!{Xy)Q%#GMHnrrb{NNQm??-X zb{xnslD+(-YSeLv#GaH%PFEPt60S15@4X10_FWssa$xRPBcHGz`^-!fbu4REZT%VymjN=oNg zhK*eK!ImUa1Be42e8_qU3qn2yEnb+5j{rBTA2pJvON1LmU>U(|Qk79x0IdmR*>exQ z0wb444U0^u4jZ=^pd0A}VwXST=!I2e!-|!nT+OK5b?%3$iVVuD>k^s5o zr2H_`s5<#*bDPCxqr7$+P5fEHA7>etyx6Rt@yY*>it~VKqWJ=S#11NofJhgmH>pZT zB=krLH8iOvM5G8(qzI^VP5{iO=bP{?C5ZW93{_ni=&e`nD&fJ~d zy}x_!lwF{BZ+A*vAd{CKE(&sL<*#xQDm!x!I<(;kiWp;t=4>!rTMIN~44~}2z=#Vd z2#cSsK{<_B`^Uq2>hv1kSYi zEsX$snbjFl;|E&C4^E!`>UmuFCEoB0!oASf^qSO6uCw$)C-ID)rWl+~iutFjs=MVT zDmNUb+Sxf8DFoO#+V6r?lc%9M&PwYO1C5C?{Yk55ML=-m2?T}K?RU%Ut{Rso-W14f z5mXZ}fkwN~>rJ0*x@W_AUBX$AWiH{oN^N#B3{_YgdKrfD^aztjEdn54nWSzfbMt8q z(Csg1ToN=i0?`{E&P%CochR`~brG=80S)!Xbm2Ud&{uju$((VWIVOrA#}CvUV|?lT zW4ehr6KycsBV{Q+%{o^AI^YmS+*O9Fksu^y(PdtZgsjpRQ#40XY#jW(%h6IGtJo`i z^jJczEK|2zu>q8a0HsAj!l_u_g9l>dYtDaj%?ffU;Up#ii`uO1aJ@G#=`bsN~{ty8Z6?#oQ zDtg@j`nU(yL;sEivM5fr?0*}3nH|08$r-s+^py{++s*#}?S+V)(7)|Jm1{;I`fcdp zWijn$wpBU*^si?FOx*64an*+wfgFC$I8N(!nlihKCMOEki34hHj_WKkQS9K1J8Ru; z-$q|~_D=b$A``9l=D7R2;k;nXcvL8Y{B0O6XFOGl5H|=n46Ds*DJZj1 z{OZK{mR>;OnvH>Fw*az`9SqyoI#?8K=^;TM&S@fU%zP*1a|IE%$E zo;>ehrZG@!y@^0}%jkvj#0%vi!p2qE7dhjsUI1MK^*VAEUo~xxdms$#thyVhvCZmr z_$40tO?rAJXxs;mcJ?xq@{J}F%?M!6A7(k@9>{O}R=I%%SljHbg5geUQ-bc9e{J>N zNvH&d%pL=j*@srSUW7QQ2!s{`$`JwyC)$q?Fj1z3g53Yi9nesw2Q)kE7)-#O)9bzg zHV*E_dMJ?r-0BE$27#>qlniQs1GvNtrAHv(W#I^b384801=0T|W+;}S65z;bU8b*G zvTj!@H&HSa$Y~HnYW&X)OcafpK$+$V4#0t;%nk(Ye_#v+^6YF+h&+q{UeI?8Ic<>U zXv)_dHz0t(8<+y~-i>KT(hKB;)p4|hmDvH!!WHYX+L8J6aHwT%0vu{wmX(9RL)FSt zPBd{-ATMbvoM+Il&yK2+VPdkJJ4g!5(sJ654(ivxyv44I{`>@&(S zQK8Y~Sp&*it(Sj~Rt5)v%>$tP^r`l{F^Im`qn_mvK)qRMKoYplKSlj&&OPxTpu+2` zfZjhJ(R~dRcvB$j!b_lbb%AVftI$XNpn;g7h&w4ffxg+AD%WdMc$Heh?tZ%sy{guy zOd}2=3{PN77<)M~i;9saP$lecSyqH_$iJ#Hk+ctNpjXw!0owjJ0Yu+fP{0YUnxOZO z2v6BRVxODizy=im4+|OuCO|HyMZ9w9PI)3j)rngqb$}=Z$^=o(*6murNB$xD0DJ_5 zCXF_`Q^qF@6#paCSrP({65yg`C*{fUCw)fTasM&oqCp1QEw3HpS)NjGm{Xntp7+GW z`Jlz(RGe{rdVtdVe}te9=-~zrwqk}N;jKxPc?GF{?*R%>rUo==0;BMqsRDY8>lFf~ zN&^7dmPhJLAOXR3Kx!Xa%mC!rW7;Ji{R3X9@fNg}J&IA(L zeU*gp;3)vdvmgH1-V}r|4FMqg)|96-=3f}PUI{pa^MKHyQ*Htc9uY(~`0lHtVRYxa zpJ>9A)N|V?{qFS71ghs|T^tv;4OEGEr!H=%1aQ31z5^(oxt4v0UR-Yy*%y0}QS^ar zUn~_!Y&4bWfpAO2JL7)avPdB{p2(Z*What$N-YryBH8>A$o4mpH_dERj2QajE0OKf zjG{|+eUDmTv`aRc@h#$Zzgogxh@xDVY@jj>{j(yGv`e;qaV_F{HjHL@BH0{*o7l5Spc1*iNK7C~`ca*N3 zIKuB9a6H~a;A*@r2@&woIAk{MOdss5{CZZzty&WpJ!l2}NFXmT4OHx_XGOsGcOnJF zX{VuA^MuL3#NUIWRj!1HJD~m;!qcb13mo%}l(=q~OK$eXlZq6C$Rn8G)jZq&KA>0) z-j-PYMj8UIr=ci8wDx}ABid!bT(&DP+7F!I?qk~76AARXXJR#>0kP1lCgOvPMcs*l z6ad}pWW0&&3B9(g5MeSv*aF-t@}|IT6}3Alq7P0IWZPMQ2P9`Axzz5UT&?c(#Q>0> zH341N|CpVC#2NR+w1@+ifSG0tVEnly;wwN0NV)vuXdZ#=IH|SkOSItvRsFNSP9#9m zmU~j6RE})7VHAC1qXKAwMDsm|xUE{`O`1D>?>HorDQ;`N;}B1#C`WdP-d0fz>9KFDT$slk;PysDEUz4`UCo;=1k-~_#9kE3ZL{J^ z{72?}E{15U?^lQQjbG1Yjtj20DCN*=pRE z5QagC!=Y*bWzsS!RuguXr!OH~YwHAi8xS-_t0bgQTCW3i)-XYFgiBT=yb_rgu9=g< zn~9w|VSwy-@CbGjiBlN*&(#pVO+Y{kc&HwmTK~+h$KDB1aoaZvz(AGKkXe%_FhPg| zp4A9cmOuc$2~Y0FE5@LZ{{kW-!tAzs%y)$OO)Wu24KurbJMj9GAgY1!{4W}w_!7W% z9R&!)N}DCz_|y+P%-(HCO8^*ziFl2fathC@NI~GVdefkjYl#4bz(WOjjQgMw$XqI} zgb)GH|8Yem{d2z>o&V^u|HoNh00|(PeG~-=*D&9ie^JyBpaXI9b4w@Acx2X*Nvk*; zorsXM8;ZQ?{=Zx-D1d!5M@bIR^i^D`OhsQrNm^ZqOwv@Z`u?8jE~oA0rHj|*7$|Oc z-aVs!lG;N;MjitJdz=5DCO(}^rdht*l{MudQEALx#NUTLjBL0z`FK=u?@Lc*VH zs9-jfheBNMmr6{`Xh%-lsS$D82GDhx09S0wZF>`|8A|~ZQWNM-X?r6X3$s&d%jTko z2^sdE1UwPXP?$ z&hL28ykTDj-27ChQWAyT zh*j)On^dtMu{S@@HLcmVJd-Muy?OoKC+=6*g_1gshmRrry6t|}6RxSkp@OLL=i$5y z2FX=_2 zMg;qvKIJng}+}MZVTtj`t=ar z>?80$uvsa%NFl9L;6Bczh$?~z9Ke;*b_&E>P;>{}Jh&H4WZxrjdmz*{A=@Cpw<}Su zz#kC+YOE%Jev=)9>{pSr;OzDQa2AlDfPK_gKj9gdYKDtdQBxV_=K@=2k)vxC4B#Cy zXA&)UbY1uz4*);y$hJ3>{Oslx9<0?P(a<(0!OS8yClRa;ep)@cMbb|?9qvmv0M75{ z7?kd8-k@;gPWpVBJL&O-=)2!<$~HHOv06XAUMd!1tzhj^Nz^ z_Cn5LNq8-obAUn6nS{zg;>_chXZy~1TY1fREbCe zy)uZ~fiOXaNKhLQ1I7UlKwty|#@i~0)1V8@hJxT$Yoc)h%yciAsNAW_x@LDZW+>qp zf)dCDm#hxgFl{crPt|6p(CWX8sJUKmd-@MX;&LK{jK#90RGR;IY=pvHWc z%nQIfiAL16JPSU`XFl$zXVX*miPUrphkQx9;Q)$QtRd8<`$&I`Va(X6j72q+tZmeq z@7H=5xUatGH}LY(_MyM=$T38>7wv#LmsgyU|Dge^m-K?E?B?CCFYy}_R|)HcZ3R5t zb?UE^Qrhw?hZ95!pDwu?B*5MT;yDen?V|XQ)D6)0MvfPD+P02w?xRn?h7T1P`=^V~f<+a;Zh6>EsyX617t9BV$LZK0XB6-H(T%w=)*bV_IQDok$w%lwQMBiz#+}v{~?jgt{YNF}c z7js4>0>=TPtp<_eCg>TIQ2?}xzWRyP%sz(WrO})qz6gToWC0U0cVQG& z3>B}@vF(emP5Eam&w#w`zfBo*DcvI-IU}>SBeQrJGzdRJwNE=rPLfpK-|AI4?Xb9| zycc?QTrXtMbkX+GL zmG^XGFFfICgQdvbu9Roe5k0VQ*+1?K8JCi=YDB4?b6T4@&!1;p3h!D55eCeY*;O&o ze(?*cN%6rS*y1{?FAtKE918}L5l^dZ|NW3vejBud4h^3WS=*#uKFeAY6-G_O-3EbdLmC8LaT+Xk z-fht2X$+v(4Q82Yx8N4qH0E%iGr zY9Dj9;r`Zr5QR3zx#PFJ{5y7gTFHTVr;Uu^N4mPjr26C~oO6wT%`t=^Geo3*dgy_d zaPMH#+3H8~mr2Pd6x6+{mK7D7Ja`g(3@PA}k0E0&&mPr;lIwL19<%x>O?<%egH-^O zAhJ;%t<0c~R$;XtijxIvFxl&ZA(JC}v)_znBfq-WT3Sa-3FLNzO%uTm6i`lsLI#0Y z#4BF;;B^Y`LzRTE4HW7DFtoc#88F+Ri|G=vXVmj>0oSE@A6(1LZHL`7ZOd&RhY8Zn z^d+9=O62qcVXc!;bP|i4#6XR;mqdJRcgl%~^pU+AynK@@Osf`5+Lgc*J{u|om)6h% zR}!T`Vr{uz=>5@JX7Q5bN$%&Nag3=Tod-Vk8e zSo3K^^oh4JD>B^c+l$3=b zvi@ovr^YQaXq4Q;&C+xuQTdGm+22~7Lfc04?k?1dlX4#y(%FxhJC5NgxD#X z{kf~n@nprO#f|L3?C?^7O;*Y^8RP2oSnfFs5q03V9=#B&yM6^T=98hl=v|>0sOTIuu@LAga`B+IxBEP`axXd6>}%xQ6BT%by_?;i zlVylV@C%=V-F(Cwt76`q+;8U@gKS^ta*O*g=hk%WVAYq-UbJbaUKL{89)+MhvW{zh&PhM$ zeon(M9G9HyAR%v?`zbtswl?n5pu5OJ=)fK*{8Ff{X)j?u19ByzdZW<6)17S(B8`%A zkrEcRxwi^7InPO0SjV-LbjC3Z?u~2cG1kj8pOf$db%iU|Zx~CxM@}=A+M)LtcRc(G;_9Ic8 zyBM+Sg01KMxSu^26I4BWpxo5q=`qFS=)5A^mGlVwJlHzHj#X1jCYm$pmW91tS_y+k z4E_#ZCqYyRo18q;xsI&jatsYJs_iPf`&%oasT#J|*;%9MPVqeB5nM3vV(84gsFh?= z##7TsDa_qZj#}!UvW%r+sD92lTo(FKnXsUy=Cjg`0S?m_!3o)yXRD1tGGgi;;+26* z;OxI&yJ`%($822XBnE4x7qocB7$tzHPsTaiFLP8)#=R}?s$#Xzx*?FNBh%F+k-)8Y zYF=7dBoVw$_JV^Evl;GlSiZ(Fq$%XIZ`XXFEcYY%XrJ9E*Y@Db@fwv(oJ2JIi+)A^ zUXVmbquZPujr74Zac6)uF_i9t{gLBcp4)S%9NI2>LwkiF&z}lYsJ5t5JKELzSeNXq z;Lh>Lf(k32NX5#Z=DqVWmK0oYe!LgYKb3u&{0RQ{_85Mi`!^FQn0P~0j^(OCFqf;} zr>ZG=w$^d&wAtk=Y?a#ZrhuFe7?qu6Qmr3b&?JmQc8Qcq@JK$K4|m$@mOWzIFJEkJX&qw2_mVJui*P!)x$5- zW(z&5wmHcaRotQ+Yu?Ik;&&L0{^=m&GaI#}D!%^YBbSXj;GvRI_H@2`a? zTWXNP9}u_hlNoFHQspEW1|H6MB^LJ1;!|eNYZtbrnaRHzjGA)pD6VA8)V|-IO^(Wl z9}1B|6%sr^Af4;iwR_0nFEck%*Vy$NsQa(ij-_}@?Q2&=2b%wxDg!FiF} zG;9WB!4nRg_u{!lyj>fYC#QI&TrX++)VKyhgh&PUx+N?#xmgF!tDF)V5{P9|Zy*qeyC~8AOb@#`(9ZIFt=JOf^>I^7aHgfYvqQJ(ZK+36!gQ$LU615M zwW<2>&AWe`g`2&S6V_Inz25s&`*C-+dw~wf-tKOF$R>xC|Ft>0yAO5e?9#55%qcfQ zOXjRX|MsE53Ebgsg>!c8o}V;gqmb{OlTB%+2eZ_in%5=yDeg&X z;ytN&@h+^eM^*Nemh9_M-cI# ztMy!V(D@&LvkEM0s+i73HI8XBeL81a6nxYnOSzNufyZs^4F9_M_CbUqNNpnRi|0=>*s2W$h<-Oh>g5k15t- zYxL`mW_p-Cg)n2py^sD+g5ir9Wku}6L;B^Z1=){J?FnN2Sa+8?%d&-`SVAem^bNx%%Um=)oP4&wE+}g5TZ< zn{21}pT{|z-8hycmH#E;?@{;GJ?(*5|F2xR*v4Vo>&FhK6=le0g z-J-7#L>G=~k7}FiQAseqviOUy0<-Lk*CTeX#O)tFdp|Axe#w6yKeQy7?fgvp(u$0a zo*!HPeo42xd>4K++JIqJDadrH)cMgdWZh9j;mYKyU~A<3GWCME{Gj)<;#1!BZ;OZG zZJa7xIeb>IK}2VaW^~19W7FE~HnuO9=*Ab`UlfH!7v}%jn0b=o4I($^LX@t2%xG(v7M+ z2WEGtrL=>uzeF6&_b%*Gi)DA@7jzxUWw4e+Y>>pytg-D=9$ zNcYv`3)Eg^(S-*4Tp}x+nbhq`JW$=+`{vQ&Cz4_uw1aZ)~_+0)|)I5x_)4(U7Ci!jRzQO>{P+nIa?vs_h-RihW5b;oVNw^^W zNtyh+72_&Kr#z|Whg~WLEK))oey3ZvDyF1-_v_vN?k#?3@ef)$67F2LThuMu{q7vn7=cP%K)p3~x`*4gBukMIsoPLoiUQ}l?vtV^xKGUr_R&_bqj_H*D#V3l%U zZ>itMi~acT@V;9J=t#t5SvW4BqrF1^60Y{5Ydf{is|7Nyik(yzflAY;yAO@#v@%RC zKS@r*COkBl)2f(#H|UkzA%+*p-xN#W4ze5YO3Dy={^N@gkT;tT!fR9~hdIwSFXig6 zmcx76S&0JFCdpHquED+8sLLbI&5P5`oBTxCuGaI|=<_{=`SYNfWjd%CZZPK_m0INN z#piI)U50myq81|_bic)yfV^e19%hbnX$ZCxr`zKni)w7&bBPl1{5pEf@jryl|9G^Tf_lt#{7{nUqmBJ;pTm&pOWs~m<# zGE%AK-9k{Q6+>uvU)`x*6ykF_1Qy>AvhP+=q!+XVd9KKAxbT8*ay%6)pO z?r&?Q%(8mO26WC(du6ywwZecDpfFZ}!jm zQ&4l>uJf8%kk>qV?$3+2ne(cT*^Y$y8hv7<+-BRsxslw^H2u!9S3pVt>$L!!ne zxJw3RJDOZDB~Laj#?afW*~{k8eBHv!qI&pzoA(T_nyB_{q<1aSoC&0`YPOpL8NMbp zU%Y(fOZ9csZ`^r4TZ2)J&YpX|?b|NIeA*@cXU*+*EUHpb79YnH4ux;v43*fp4D=|c zZ9e^3-HcgE!jjpRrU#Yw?-{`i1{kHWe8N^wlxHq~3EguF-Lm&C6Jb*&9^Dv>=CXd$ zZwcm_C_J}A_|ElNuj@?e}fCTH?{TzTki`m+a95k7pgwrBDNhgJZVeVp*k$q-U}W-xxmB0*gY^l=Pdp~ zZTPUOb^J*i+Btg=bqu+`-!{{}a>2V;VV=!PHu+zvzrs9sJpX6mWrTmX{Dzm_*I;Tx z?7{j>JMn^d)vW`YRk4{4#2eOP&ArzdS|2lm3-L!TJINb%pmC$zgXvZ7Sixr(yz5R{ z#BiBLIok~4!Nu*pMF(7iSj)8k%M+OYwia=3hGMwPZ>Nb{8s{ENS$m5Tm@|imc>VA1 zclM~5-;O&eKU(u`YTB~vmwITh?XA!)zj?hc(RlXqjlqQzF>Mudn>a~J!w>%x8#)ow zIDZUz=R3!pD^#ZU+gBy@7&61=l}u=#V*L0Gvl$vK$ek-_WUdBQu97!qyS?E{V-l6e zknxM5lS0q?F;DU`$}#&tq(nX={mB%%w=al#E}v|26Nr@tn|8 zK_G04zt{nf=UMBst6f1vZPT6{T6=0X_XKkfd_aU&*UE2p^$cvH@FLKoWXh%ibykYS9T{-{^2G)VZ5}@d^>i= z!JX0NW%=R!0@+tcXUBQ6ZG9Wj#a{od-fJ<6W@nAC;R7@LswTYOyx(2cV6nm?WYH5^ zuzgvQgyWiiUH3=Y48x>LWb~iEUgS2rINhJvQ9=~Iw*y8}yxzC5!Wg50g=2{LDhuz6 zsI*$TI>T3Uo7Jol#me5_e4R}3CF|qfp1Xc}^Bska;qykM;77u%+S+Uv+h*pcb{pr0 zNKw(x<;;ATYh@3f+~_G@W)XX70Zx$e+}r)->9DVet=(E*bK(|eqUQG|?LS(dWHnhY z1)o+vmb>_Kq_kQOT5r-l>b%8u4DqQqE)C8cT9}?k!jHZe39g2-y$cs?>`gm{Y?Nqs zQwc9|GcO8$9Yy~{L_ctS%KxES;i{?jVZ9ONrejYWVx**EBv2ApW`{kR7A7L%vz*P* zv^O#PQiZbVPCZTsKhVdJ>MDm*FAOd{6+U`<1LjYfD?)Trn+KDBn z4SO+zzSoTvtwPVgT;;OzX(J)Og%`-YeK=zwc0N_UoMhP{fVV0nk;fRi zZMn5naY|0pHwv>~eX-Z&#pbpbU9r-?t*zlpxj1U75---B8ryvN;B$D4AN+bhy2qz% z*)O1QjnIeN@*iKK?xFE78h*bcZ?i=9jz(-1OE4O|%|$b2Eo!_I)S+HI{nW2QUg@Cq z_txQE?n0e|M{l#_?lxaBQeuEO+H5Az_LmrDm&o_b$$cOdAKo(@PbV~_BL+ytxNk_N zJ;PBN=`QC5<(&j6&t&q$AB6Ey@~eUyrt18C^|d;W|9XCEaY_uHmJe-hb|u$W^?GA+ z$!}UxEfu9>E=w2x@U4}@UR;1zdSWWr^hnWoIc}X4hwhPJTe?&I0p^N#!fCJ>{jI_E z1wMnmY&{NgIlcFa`?gPO>hRopSa&QHc7BXD>{z zRa-n~PQOo zW#O)#a*Xz%#ScP^m&vN<#_d0QHKaqMeSUp zA48aX+Shjyj{Fm=iH&Ql#VvOR@N4IG%`2+DZY0in8V$<(r;{JP*gA&j=$$33{(M_c z5ZkEr9XWUiPMGao4E?14WA^FH-Xr<>DdxI*14e2c_G_VroDV6~K1at*uhfUQk`DD3 z&AeODyC9KJFl=nZ^YalqjRq72aeBpuPGG_lFqlT)Q3JNMR%2M)$80jrI&H&!S3hk zgQfm*#Mv`)$y+s_^-<2Mi#>gHj4fzuaW`54sY{U0eGJoh>{_?F{c8i<=*zaA@bzQJ3h$}z{q)dk&pmlI_XF+x<#XTS3vi0q z!mOt8M&}pgwv9CT#a#@1txDSiba#ot5@~zAlfg^edV7^%xeH?;gNn zhlrn3+QZC|I$A$1jI51M`Nft{2d~c(p7u1?tZ#>S*JW&97Ot!$-ov6Hg=ztYZxB#( z*Iz72ffq;qc5bdQdy+@icdO+s-m?`hMXPO7sL{cNelcNAdv(n0`i>sJ3zymQk0G|` zAUf$cZO}2Oq_zBGhCMK$FKPDv)<>>njV`DHednVl%fr-g>U$hE;IxtGux;P$?dt^T zUVRp&z-i7p$r~)qoj!@$EA`>-phkQv-Sl1Mc3kK)j@f+aQ7c2A43cY1nJ;b7RQZW_s*8>&x*;P$L<`Ag4%Br zamNt2d}N_{J!1`GH%v?0e=Zr^=ENp?J*s-&pZBQ1ms`J{{!0>z{|oi=nWZ+}@~YTn z{@r2I!+LP40c&#pugli!)N2y5YI|cOa_2#?_Ue)Ux#?+Nf6@{B5J9auc<%K2cN+O? zYUu%0Kh8`5V`mXY9gH44FFBfM_b=2Xc#oFnmCru&|9xuh7!vkm?I^MG$g#DxR(0QB zcxBL6W)21qCGv+J^?p-nzYp1}S4qOW+4|Q8Iks7KIQmFotklww z3Nx)D8|VDsPE@KSp6!jB9Cp+F7*Zo^wC}%uw5*54MK}?~<4Wu6U@z+Zy{-MO_mMDN zIBzD&mlP({ewkc)MrzaHOPY@mg54eonL3 z(E{~A_w%)9;4#r`Eb|Iq_xwG+VT#DVcUh|mOZgt=uqQkYwt!$gS zb(M9w+l+=C1LtKZcqoKNNNLzfX8HFDsX3lUY2+rqlEOR^b|B#PzWOP+&3+EYk)v$w z(aje5m@OBgn-sZqQ5Xo+51y(fMasS6+S0##@LtyEi0#^z8|Lz44Ow+jxCZni-YNR0 zw$Pz-r3W}DH1`SKF zwKHZ1^Yfp5ZfDC`m|M7f#7brO)MyVU@(v4)zuWh_K52Hp$;_{0+TCKnyYbJ*+Ox+H zG9h5Eh+HdsgYC(m>AzoOeLauLcM%iFL(5n+-<277j^HswWpe_*>wlAcUG557#lZxd z?jma*QJa)H;aG|tPWZuH5#1>5Z=q0Bdr#sf!=EnR)xTZV~7-CdMzV*rFH&r+|TkRu{8&* zG(|6L7!~T^o@~p(WMV`dkXOd)`efF3Ej4#;Zzi%@GB37b6^>;6v%BtvnqpXhG?!ia;HO^4F4$B<~Yg>}g^eIIQcQH|TR@TY%=+_kl%2)`!T zKi_1g=8qw+{yay~l30BHZAt?Pa+z*q;vF;_S%G&VI*{Nu1K5jA`PHc(LllC#jQYg` zPC44=b}&($yFsQ)-`#zLS$WveF_ppLzr%fSM9oiD%}ce^Ent5~#e1%4=gest)i;y- zrJEbOi3tB^CPO6u{s8#ymtFZEU9ZlUalcQE3T7J%=3O7NoJh>TR%E1nj&2%4r(SnC z2yZ_blUQ^l*Q9 z>O$ur5KI5sN)t{49t!+Dil$OXMr}l#DPH}Z^AB4iIhVSU-d`-soILfDgcQs-Q`~uU zRnJdjvvzS|t=D!qQFe=rv(ep6ey+MUF*7`wbZp6djEyC=9^?8~nK*EyJI_A3>a!-+ zWqLPUeNe1Vds=`NL~yiPQDgBvI=YZE8>LKaX_X0s4m#eWLl3pCcuf4HzvHVF_sPeN zj+XJ~uZ56EiZR zQdrE*rRBnl?{Sc3z4Alis?!!t;fP%CD#njlg=Q7ZsY13f9&yidcxq%e<8K-a(whm~ zU3qF-<<+Hf$Bm^&A@iL#>RU!+T1-z*>8oQ%CFQ%ts+xmb*63C_=y?Id;ZhNY^|zOX zb)>fE*+-IFK5)lf_Gg2778*yKV#vSotry`ZT`0J<)lgg{Y~`QVYkKbE5L=kNz09=h z`Tn8WdF{^8-MYAd50gozh!I}N-&1y&4;QR z?kvD$wPNv84#vXE+5?Q)6WSUz_sSZnuDr5t`0Ev+MEJ{A>F>Z=$Y-dS@QYI2y=J6k zcu(b)yD**q2z3=M=dvlC|E#{w`S4Q=L9W}nhuQnOT9>woe=oT@xP{8Awl0afQWODT6!?8jMd_)1`3+ z)t`hI+@j2G$LV8P-L@SRP<{grR5IlFr#FrvQ7O8b(J`4xQNavkX|H|~()SZ7Slbwl zLZc}MAzblxd0Qlk?|43a-!s~p5zV?R>b}s!{FnMGbjOf$)05u``4Gu!_A~P{F8fE0 zUgc{JzEz}How-8tD|jhIJf-+@G{1G0dsHHWy>~1}#<=(3?da)x@60L2bZ*D)d3Tw; zP=d?td;y1jm&K<=3RMocXD{sF-3P_2mTY!^Y6X|M3%t`GKo1|)A>yj@9$ux5$_&L# zvUc`JVQ`Buf4#!iZ8=S&!FQ)VwmKIGWbRZBxP)xk+uM~?_~DI63IiCEJglmb$$Nrx z6;)GKyQ7VhhcmQg6CAy$N`>xii>l&5BGG|3TrkoeVV*Q_uCvugp4w3D)V z_7e+!-F|~a?wd>Q+vIWMVQ^sI~cmh+;%N4Rea zrYibLYaV>CZ|j5U0N;H?M$$|_;>GxZopr8{&2y=Pdj7&A!M$2^ANb^C+h1L$`t)y_ zo69kgXwii8<5uo7UI@$h+5L$uul4=20FMd4)BK~`Na}r)Q z?@2>XKO(C3C3U{vP#s@5`&N%?p|RZY21KPgqO$ZF|e{lhPObvf_zow{2gswPux!H@7qU5R?y>+E zEytSTvMmSP2)nvL=d;Wiht9xV6(3XEJfBlrJ1FeSCpMGoQ-+U5>SDJ>9`T@32lxx7 zDtUS}#5Ldk4jQ1lTs%#XKEU$si)=Dm{UsTZx;|_>>E>fe!T8FfRBVY$-Z0_4(yg-c zA)!dC!|mC@mloL9Nzq%K?au4EV&lArP__|l>2?+Np{dSgjYumTOe804xG6|KSV?=6 zuCWtXP8zha(ld&>jA${qe~Tb(V`M^<=KB%i4lPkBHa%7PDOs()>xsOakQuh>sQ&QF z%I%?g?IR3V;_|~^uQRJ0$Zs=pRh8}w#540eJBEB=+L@P-XG$ku8=l_AY;O3na93yi z7{E*BMdu?4S!22;WxLfCehYG~?f68YV7;+j_s_Bx*nK0?`97o_Va3+Xn%Qt} zo-bZ0vK4xHEA5+UVf#x`aA9-^VJ-Y?VvFUWqPN$;nWF|=icUj!6;D{EdwGhj~Znbqv#5a3Sp&)x7NkS1ROh z_j-!pC8@0urh6FO!}6CBk76dXiU!#Z3sTfqcE_=Gd)nB(HRigBdrh9df?M+U%}JuU z?hciFCaW(T?341w)(s1Leqi9ifkfpOn9VJpN7%?ov2lf-K_Yhf7*c9LKQXfPt~{_M z4_QM23vs_kJ|&|XOx&0C`Esb>(Xt&qvQs*a(F}s}dFR@!GJ4;oiQlOa`s*h9ddqpu z*97w<8WT=v@>gvGK8*SGrAjZLe=$GzBb#pHceWWDH8E4a7pQSrZurv1#=5VlVg_MSZ$OUI_I&2Kma!w=dg1!QgtqNWgQM|#6QH6pB;pWn+n)<@7+s-s?M z*3#}aCcNx<@g$$LG{4Qxn^IMo7nMZq(pd0HTOc_Z!m`4Z9Xu9!m_eEG0gQvz7osRL zA+J8?6C%GJbP|}knCIhULBM~ID}WBbo%L;^Ru{SBekl{v?QmbKLNHF-@3);o(<}*<8I}?gH^4vd zNre5ZBVNrepqu@J*rZH@-b!|;8{Vxr6?p)?V*c2%$c&|Y$0WL^NY`#R80&iaEd3|0 z`He8j5**yYAsm+v4H-*g%2b+Ap180cnKcaL-m;2j4RXc`+37o4i($BaOzr6NM3wKe@cghaON_OA={2Tf1 zwQ-r9E<{WRT53zex_0#yqSs_|KgrMa4Wx` z@A3+iL&s;oP19*o4om7Pq#TbW*ixREHr3Cjzw=!G#{Fp7x#=*i`ru;4((d;HWBTbw zo5*$hh0#CPIvH5U`1Re}M?GE<-_5v~+RdD~Kp*o-(eg-e(jxaq;$zy37c|*jmXX@e ztudYV-aQUC{P6Q>edj23PLLe}W$jBkvguf+#D_)4eEYkx&L{6{cq=TZboTVf>pDJU z(5}iMa5WyIPf-ae0jW+?*ZA;?)K~IU(`z*3&Bdi7bvq`Rs{0>ojarTL^LH4Jhn{{h zm+2uIuC0laY>TJ1_HU1u+Dg874-ICmJ55bpxT{r1UD&kx=6N%zn0cwP9Ziql2 z_T319gXPWkJ>N`gvOYq?Ft4q`rZZp)YLKh(<@_JqFPq2O1FNUqCpp3#KDcc9>HHr6 z=|C30HL0lqO6}8s5-ZU~u^k$lZZ`VB`MTi*^XhA>rw9w+NcWE~AdTT4I+=1X zOO`x<-d0~rNvEv%ic5{bXUpB+>nK8&r%hEVuAhj!$WjS`?H;YAWw>LXRbjp6DZ6~# zX1h?;)zpO`Sqk==rY2aOMxv|a)eoBy`Px#%))d$_ol8{Y^$z#$1e8RzT{Tux)k^}~ z{Coh5nqf(}r9t}#0I|4@k z07)gNnr_FwUG38>mp8g#N%IbKl^?Z$IstwB0 zPX7Q2p57fd%ec~NKUWvO`A)IH=H=8A ztN}tbAmp8xTsP%499-v;HAJTNsX-Y}!*X`}T3Ci=TaR4SQlt1}sOdxf-8z*OH~#Kc zn#3oWESpj*wcB%UZb?5&nW|@2$UQ*U2&^x^Iazx&ey^3mzqpzu2 znwHaX8>vTHkXNwrx*AVD;*R zW%z?x2jqWAE>6u8`#dOWFGxFeVh#N6rcUeG?4sIYp=!3C-bea@`Aqj-$*6rbGNnMO zpnIVie=UIg<%=ecic=JcsZHn+xMgG@{{V{6JgcnHdT8LGvQK>6#UF!k|{Gv#syXE?ytm!=?PBj|cP-+C) zk|oPA8`&|$pEGe5%iyQ&582kc>a^O0s)ouHQHu_S`2s$q%bEHY4-d4y-_X_iaMOA+ z)TLKath$$~0BUK=6tT6~e-~)u!`zvFpDWI+y%gA(P1w9QQ zR|8c405hJyAqeu-rKVQmDs9LqHy7^$)2=U;+iWcXzNg*T+`!D#TyK5($gQ-r@lxmF zb94P5X{dZq_b@#4Un;T{gM3`^Y^ay~amw&^Oa4);GUnO%1m;B=&;eUYfNlGLRK)IDN-S zq2i55kXw7g#CMQ*D${#}rr!~;Q_fU-snh&23S)QXSobnY#570AZF83+Vf|#H z*?t2yQq>OTTfi|b3pLLQcg!@U!~@N)fC-Q+0k5V?Ynp>=fn&^>ET@+H{{Tr* z_jFo1bOM$atiyIZ#@E_nWzr$^)xbb`8|~8Ys?`yJR{^dpLvr2+^ID}EcQnZlOx&Rc z`(MP<64r(ZNL0YH9$kPEg;P|BH@Q7}Kp%*wQLIIWesBd!%ER#JE?C%fglq2ISNKz9 zUY|Htl>k)8osPhc@UI;qQoVi;RD)9T^+FPyjnv+{08@1Ye1pw=QrvxC7w7 z%*deim~tQwD zQNBAJ{K+ufKF*t0PO#>2`y0-44*YpFYh}ZtZ*g+XF;4NxD7_j^38uDs1 zYSSTey}N#~9g~-KG-$3lG2g7~4kULL9Hj+7=X)5WJ|RqJCI9dX(=s^ zY)?1{Rbko(aai&nXhyi)0!7c#EouMM{HaS*T&QEEcguWP6-Yh74>;G|yGq*1%;jn6 zK}NO7w@%|DEI~5Xnk9P*)R;Pr$Gody+Q!m)4i$80NT#Jq^$x7V^D*5!IBI=8>DbmQ za(%!H8~lW|CBUY%Dd12_kA7F8<#zA{P~C?RE7n>{0l&f+3w+2T6;8U6wiRfr+jYpe zQ+}4^8B2Obr$opawJK^yz=77+KjKp*O7>w)>8rJX6>0eoeTx|T*eS?aT`zDn_N0oT*hrAr)|rXs=B*$|WTa}hR^ z6vs`as^^Gw;HVb3sP5nOT54v9g48Ndte=JzX}zp!EkR9wK$0Yhzf;kcr`K0OjZJ2| z05-+-3d~P{w8@OZ+xX92_WuCZby}P~D^XKNpi2a?4N=JcCSp4D0@G2#3FR;5{Y%$9 zOV#UBtLl1&)YYcEG-d!i0%IXiAHs@<~cyllCwBf=3slr zxVW#>z%0t3-2BW2Dp3kDOpl&d`@*rURUJy;tJ=cb!aGGwkj!}#Z?q4hqi7yYS^dj+ zFg|U}4QMrq`ADy2F`I=SGvGk>a$G>N_LO-wROyv06f1yD*K6Cjf#6nlIH0-gBuAo_ zovs?QSw{HtFt4YlD0+OQ8rP#nqFDQyODd`7z05})NU!YecOjO~!Ba5jv7y1(*)A87YQw9`<v7w2!Jwv_sbub>@;Yb8)|vDoh; zR`eqZjr+S>zc~TaCkc0b`HR>Om8Dn9tmySh8erI8zs$HS+?XC}Lw05x z9j_og!)NrKoZ; zB#_5r+Gd-MKwKkaN6!BM_9r=qNaW2cij_z&`@UsQD5TZNmP<9c4UClOprZUY+}Izi za^rBEIaIgQV^uB*Hu_9(?Je~t;o-qE1ReUB-Q;^twvwTzOaO+%5(*LkWKr>?F}Ww4(=5xe>j;3bl?<@tdG>)WK(N^N zft#+^>()R~8lweKb0X7c0+7LZ^S^jd;??i@2~ywx)%>0ssJgi(nO%<({{UUdB64Zw z#q#~#dakR?2V-OY8%-n0sC1^K6{`mvp6&vZ<%HBD(WTLv)OeLOxf_4-y+X+S z%e1tX^--(pE`V2}_;byw<$sneNSUZ!?JBmgD92DKD*M`dW~sH;mlI-VMV zcBpv1@$+j4xEa%?D=EE7wVu~+cB=vhP;Nh2QFQ*I)XAhY)kDOLHpn8Z2@DDkrUewg;*s8S#MFQ?iGMjg}zSA`g zTR}%dQ1L~5SyRmws#Lk~K~Zm{ozBkFR@2k6GSm#j;huF{ZbXAoM6(h^yIJdc4@5z! z)#@p!B=G3)SXFCw0gds#pONKwR=`st#X%n_P|}?s4hR>yfs=(^QY*(%@;*CAHe+ju7=)=+ zv8JVuj-NS4)2B^ns-^i2%nwQ$WL%4nGK_UJwDl>Gx|7(z`4wmc87pMHdJ!Pa4U7-1 zq)aJfemC6s9idv2X$Ol?>Mjp1R{6pfg#~vizU4Let<1_kmj;53)o}7RFg!Yn!h?fM zdxvk70@ck?mG*^st2rf9n~R7?)6+M)0OtzQy)1x!Ru*I8RD9&((ry4kR1!fPtWD$% zi>VjbR^DktkZ~!^D2RcXB-cFXu~Hs>=i)6>T*dRlXGEp{W9)+ucbHcRB$vFmNOo2D!t z_V-Hwg23Bvw@F!NSt>ObV^9V~$yjqSD*2T7u6AMvR=Js1yc|?5GqCubj^v*+EeNW_ zVx$9pcG?E(Ug`lLzeyG0*ReIgBak06d8lSWma4rGYE=r3X3&=UDEhs93yqk6@;AC6 zUYZ(ubb(vtRQyDDmdaN|rUhUQOUm}&=N3>dtTo===;|hsjcsE#!rSyG^$M1amh~guF~`Im0hGd+l$$D@ty()Q{%sCRW+hY>cqXjz(!8|JMAAO!|I!@HkEnA`73% zjK_8CuUXXJPfWgt*Ezz2SAF(dc1j?VqRj0wBP&Zbll<28HRWaCg zDq5;W^;)+Hc_NU@bKb^Vgznysk##i-Vnc(`Kqu;4Ni$1Ls%dERZ9BVIs5Ht8AD&Hr zw3blP(C`q{I)|l9?}bJ~f4B$omda%Mm!n$LYP!`nECniVE%UnCRD0D-SSEyvz#MLvqv(Iv4<*!#k&Sl_riOm|+&bz$}^ zwT>64%WtV5MHbADPQ*43Ec%Jdu2VU}xD%Dh5 z5A`sQD5qCYNgF3t=s(Uuw*r!k6*F$%yf2omPu|bh6n5vF~QRAxR zTNc_uIK2VbKuu6K*O6XIyQy0e#`d%#>9kc{!&O4G#^(E8QS?9~bF?cutzjVfmDsU_ zX{4ys)$ummLcJ!vM%cH|b%1Jn^vbthP&|uSlilau0cwTVA25%xr0ae|yra`nrYT@| zJAEY?>_$hn!0!VzKvu_d2=QwHr0V0X!TypfwDld^6x_%+thMQ<5rx;Sx3sFyND{d( zQ~j7f2wDBo*! zi630oM5UfPD6*ip=0wAz=W2E4sk>YO$^xX-bu{s2#)lz(XXP~zmf2T^wx#fcxaDr$ z=9y|5sT;MATXMf*Z38u~QJEiL+a0$CRC#rEUC>caeTdwSgUo}@sZb^*tfz7ZBPA`V zrlgC5WA&1YXcof&EC(Wa##c-+tc0>0i0Zu}lG;G|{m{2YvCeH~GjWx}q9juWihz`E=^==>pAicVK+-nnon6 zo1w(^JBS%ne7W--U~i<-Hlz2BaiR^0>c_B|SbHjHbl3QeCL>^27+QS#x^|~~FlFt? z{iS4R(lt)xEiufUzJdp>LN38cw;-b5oTEj2K_mnNbG6JX&#hqF+se0F9)@NXUO8DX zETEWG7Pb3rXdD{X>y#GOoh7C+ag&})+C_uQk;j(y9E6RtdgX!N#8OOxWY;67c%mg% zjaOno`IstNVCl%;xRIhtyJO#^#KW&1Pitw-8#%x-+k1jd;w0r1*e;@?=g)4Z^b)YZ zlEj9&w&qlsc-s z=6fCAEYqb_l4V6-E28w9Le{Z7h}OK8DV+dM$i9uUV)hnxm?}Zfrxav)e zvlbxNY0|Y+O-WK0;%ZG%ul?34KUkvMQ1wF8fS}c?z4t~1LHVRQbk#96PK@5cZsOaHqi|)v zxlf-|qT+RQ>C&b;o)}z@A-VpuY`#~odWBCx2d3BIxjwDkp_LN}4?X(Y2dZc4We(iP^9Q>J4pi^LKi?{Oh zfumR35nwH_9H3^p!Hv0Fqz&lm!mixUITV{z%r?K)2a{AwrdRib479Y(dYH2ipj_LW z52;U0ra|!K<&EHL#6{UaQR8R<>T})%UB$`qXjeToVasSAA5}@IR5$ofBM4ew;~oL~ zNUee1@GV9b+nf&@i?Ot*(@m#+ML?^vkBPfL`*Y8z?<;LUh<>r>GbJBm^QI8gIcYmSInltYC5jM_Yw`6g}K1VH1+E~bd-u` zW^O_U(W1*p1Z4|pL#e7!wd5Xl)%LkPAxJGld)oHp*XuJfu;EspCQ?swY(FV2Y36c6 zo4u2o@4VbGEP<%U8f1mM2w>^)UN2k-CpAw=Q3tHFS{O>a@ac`{E=^mL% zl#OP!l>8xl<<3}wW%8GXgfUaMB226`YSmp;1ACuAB`vXFR9|ZjkXh5>)nKT`*7;Zo zN1}ylx?G97Q+o`pwd7kD^$LN3rg5e@sN~1Y7W3AwUZ`Qp-s+A*@Qrw?h{E8vBj!<^ zDrL{SMX&If9`FIg4cH6cp!-UUYSV18Pk8p_BBe1=4qSoe4>$ooa^#wrt?m#VzOoKy z1m;S!4o9efO;$H0z0-ZATXhPRD6wS&6VK~2OKodvjbyVOj@IoITMN?10HBVSz2!!3 zs-oZ!d~P6VrqRBn_7=UlePmbKR1}mT03wTAc=<@%!>!cOU`d{45;-zn^d!GPd*|RMgY8Itf*oxsyRGD%FD6{IL?{ zCP3@;OjVMRsK;dus5ZA@VRJ1Ov~@blY+UFmIT!frIkE5WZ4)1Hy84c*B6t)jx#ep1 zQQu)OZ`>i+=Uy2Vh< zO|8`lcE9K?G3CPQTz#n5RHm%7Y8&pvBoC13C^dC@s`RPQty$_(qj5p3LdV#l{313|)3V2UMDH7tjeBOV|mUw+an zM4%79P+PKi^C~f~MxD+e3-Yvz!kbuQ)EFCvkabFRarg|QUrp0haMdk}6Y(D%r5{D; zQboQ;F8*MA>Xphi4mTT#05$SqzS0j>I*H$NDBLi&h#2Zw zJ%kUhjWkhBM#KL1lz262stYOJ(hoMB8nfcO&oj^nA6}Bv)2Kza`JV7HuJtTfHp8cx zMk!3zP{ohS%!8jzer&_-6b(HU5cjC%!2TOrS{}sV<^_VN)|BoVhk%bX51 zdMLMJ+5^(IHA{agnHA^^4cHqm^35Zv>5V`wVdC4uFl022ENnj`^@V9on1Gl9Z+wg+ z*+!Bj>eSdCWLo6*5TV6dwQwMsd+a)P8_2IVt50E)bLPYP!aUUOO~voh&@F}SZ+Zr_ zUFi#{11|jK8m6?;1Z?HGvtMtNSf;tEtxzF0*yG3;7#&Lka&k5}_5}En)t+41{{S)b zkZ6=@)l8KMzwt=4*>qLx32-<0#gx?Ps-Uv4cKKRe6!6trsN}fwev+bbd#~CD4C=Nh z3m%|#f%Nn%Su@LEP0JIt*PxPC9jw+GN-0w}C*nO}W?wTh%u5@}!W6oijNkB%rbTN@ z7EQ>=ckLw0P{o*%P4?+4EokYQsmaa>-1C-)wJFl3JEn{FWz%sN1+ z&tqdEvtR@TSeVZVN2WwtdGAB0=wG{lt!0dB+M{UTBt5W8IM$VjZ7qYDK$ zw%743B+8?)-2JARRn(LuD;ssb;zd8j9$`iL5wyy=tFf^xZ`K75kQp)Y;J`~%-*QiT z%8KqW_dv&=Iiz^25XUjGJj97203uX>|Iz$mMy+s+e-0HkA*vO&-Z2w2G+j_&&0d<+ zAm7}pOl{i0n4-z+(OO~GLsNwJwV6TNZ|0Z3xWb3woG?uT79Ygre?=1aPR$=rOrtqM zjON>|fbC;p`p7($bgtX`*)iq`bvQ(zV_H2Oz zF4kMgYfP%^sp-|IK94d5wp)*w-UrU9tkbGFI#x#_$QZUaJ$Z>0^2ug7F(jU3?qN~~ zKakB2sun;#_tSNv_DhA`6fR}3qV{zUF z70^~s?~jDQ`Q);j9f;Znl8>_{%(6L&DI z&0f7>9z9#+g!1Y1kz467L{z8oH!~>Nz~yn;_uc^{HNXIp4gPQq8t4WjmA69!)0&hP zvEBe8++C%n$r|}JsQOW@Dh1SYW5@}sN3hUjvl1`R2`y=bbXGvFcLVLYd;TeLvB{E{UCX?)M(YMe|rVq+ixVSKAlb> zw)Rjq24$qcvW%MDw~8H1Vt{~ddYGY=N~d~2S$~TBx8(!o3M|!)B^&-*O)&LeqM%0a~(b5bCC zR<=qPac*GBIoy$8KGK36S8I~cF=k@c=42AUFb%sNLKVG zH@^KL9#^c@923ZL5Q=XKp7$sbf%v%*16nnO?0@XY)Jt(tHsWGihDd^=pX41m0%oC^ zHgQZ?SQF54w5+kYX6<=UU2)v+=cFMmL2Cg6I)P5>14?v;A&XodLI$C!D7GT~P2oh; z<_5#WQd=%7@+aNt1C?BZep1p{I)&M}!VQObAOF()ToqlhZ926;HpuPsA&8Jq-jxiR zV1$1bbkRk;?-o;Z)>9Nz>gv%_dJ{(9F^!CrJZ-AbxI)l&-eeYrV&0Hh-{{T{EI=4o@ z?)~3;V}7R4Jdmz|&^iY|=p6%~G^pwDn_dCP<#soLuN6oO$aOFyR2H)GkYr8e9RuE# zg3|*1B_3w1O;E$9DF?qxSX3Cc{tOM-K{$_u?edg(vna(Inwvl^r{f&K#9qYnxsXAm zrVW{x{fGI=Jv6lR7T)do^EZTUsZDQ)0Qa;FRaBdEC#l{Q)hVw_0o#;V@>C?TMIPuJ z?E}cChgA1$6DkIb0+3O-h_6nxSgqvWOFh!^ea)MxUaap#DNV>ATMm3@+Jlq z>Z>qZ1^!zQU|?uhEK0B|&~LYnQrc2f%>q)Wr5BSh-u^Ewr8G)uPM~!b18hdbf;mct ziU1&Qi6+CblyW$vOM&j4s{Fbiqztthm0E;`y8&wt9Iqhr)T%WW$XV@c^t=g6NnWY4 zt%r8lNo!AaA52hv_AG6^wi7nhJujrv)4IJdTEf@9(puBR;AvP?rUp4*<@B0E{8F$j zVoz2aq@vR*fMmBG2qtQh)`p}O!TF8A?;_egWG-7Qp6v0cOn7$z?jqoVoMIt9Ys>1iNxEL;Ca~?HNm<3 z;9f7gx5Rmv1zZLUM<^dlO{diA6C^=Sn{uefbMg{fQ0l&sPz=pao%jB-TGPu@Ls3nT z+zw>zDpSl=klBTdAzYXpa{?2#<^b!o4^0$%g4-|jnV4=R>Z@9i0a0uROT;xVKgGKc z1k;<8C_C;tK+dUK`=-iy_Jn3g>9Owvlor_Xfw2mRHoT)e#kUsm<_Bg@!(rkOl|q}^ z=b5~yj7`q_9z&E2$=(6Hga6a~J5K0BEhMZ7++bAi-sV(lR+Um1(=;j#?3j?>^YStc z>zxzPMy*PqW;R?=RPEPe9mc7nrAP{kLCV~GIE#bM`57%<(gh-FQd)E}BX?S5zC!l` zWn0p#QBF&!MXI8}Y%W=wa!*@c!d6()s$cg+ z(%f8ys(`m{VE(d?9bHzPY6@hMQ2W@{77g@U8KihMI#sPzs)F}#<_yIBaT25FwwF>^ zQxsSe$PsJ(;aBiA#?FOL$i)_G>GiHiT!&N054=)a%~KHAt+v?Plqg6#2SU0BKMQ0fD5D~7nkzBPN zp|TGb=_vJ6q={x>oAkdS2&`8A{{Ua%=>x}VnSSEsPWJ5s)lKS^v#qVx1*p%Bg}2xf z0}5J8K2;7@JiNg6h8q$rqrIR^z_}X}))kFyNVmFg+Ck=8L5_6|@4dNNYLK{Oukxte z8BwO2P_`)60L-fw)HSS|2pNi{wsJ`8Zg37)_f%PX4ziJfT9ww1^lofb&4T|lziKbbNQ&`x0gZPNC+FM36nV%}GcjPbm#0@mzOBQ2p z(u-Z$K&G1)INz;|gZc_3X{t~!Y*bsg1EkAIl9nSayzeN=vohSK%BJGdE982WXQ)># zx7re3C8^(g^_7!aiVoY3$CL^xt@j(kn$}_tKf}zMZ3E9yfj7O+QvwDfmpBN8Wi82v z=k$T=(rM`;iu#2l-`qpU%(W2uACFK5Bv^ADj5N(PfZf@yJaW8+x#}*Wjz-u18-H0* z^V|b)R_pn~F=|bfse-`nB|#|aY(=0kMb75JEJIDH*}0Ce60~ zvF6I5K|e$gDP2V&`>jn%^xK|HjDIpM5ku6vEgh*jsamZYT!bLFKQ+9UD^Sgw(%zZd zRLmOKGgKA>x#zrhJ2Ixelphw+aY?<p++#L}}cI*Kr5?P)R*`eFowpM~_?S?D{%_n^~+kOXI%wB!Ovce>VP~s-|sK3Lbmd z?evN*Q$)ZQaxon_U+XO;n`zW?-?fI{Qz*fxoBRit@b4hV({a%Jq*Sr(3XnPx(5{4Z zE1+}^fzUb!KjO>^uks zt5j}mOaiqT4b<8O<6BfNHUxQqe-~EP7W+UaQCRbDlmOKo{w62qBBjg*Dzd0m0C|?t z)aqd2nQ|RLl@7Ibs@M2OYuv!dTy7vvXMVCO+QWWwgenyHO^;I`jaA#56Mk?G=ada8 zeP4fdgVU9d9t*yyc#!P@~P@O0?Aw4t{)@ zN1X-U+=Q&XoMtB1_jb6p{G<&k1q7Ru)3l1ss@aJxdybr;M@dvtu2qeZw?8{X!4Gk& zLAW*@CR$-vP!VoT*z#Fo8JT(Mv$OFkFt=m=lXT6MZ9=nxS8y%heI*8xn(vJ?FI4Yg z*h-H(yOWC^TU(ThOxV*}W2MVm%m+#F_gI{J2^5m0^;C<02^)N&A5-zv)~MoNt@e># zqphQWD8aAsbT;TFsVqMc#JNWSK;GM#QSkMQ#-)p7FK|SP^N#}N1;vN(m>tWV;2~xh zfxHOb>9OA1KtCu7n+0_(d_lRigWPov)c0(!$V|5KwK}5gM$)4&5ajfwhl zfvVc8n{C`ccT`&e&#Vl~->vWB2L`#^kR%1`C9P>hqf}D_fw4bWA@5qLQ9uh}<7tV9 zSeKTwI-Uso&DW`c$Up`FBp62% z3WgCNTNoez)%-xhv#H_W2;{Wd_WETADWlbtH6IqIe|<%X-@Ys&%4XUsOIW3PTqD_}ir+|&VIow~s z1_!o_QL9GL|&8RBvIStoAoj z)hrFUbTKhL>k|)Vny#$WdfH!2R0Ly^)N!|xKoc__^0(~|T|ZIvs!3{rLW_{<1zQfs z6*6awT9u=$^$LxW{`5BaB^5#UY)Sh|Q4HL!t551WhF&!UQ~mG?{Ox_BCJBjVx*bZL zl@&$*0ND`0kC5dqp(tx;wGCL);u`^Qs#o*E-<+{9j4RP}P8Drk5luzym7nE(jHP=#7`iD_-VepeE!4Y>2P?9-YAJE7J zO*sbimu9)Wvit1{E2g;$%5&t`_ec4dqSmDfib_2{clnLS+jvGMfCA*=tWG73zod%p z5F_2hk=03n8wCUf-<63U(gsIuk2w{_2SDf)5n?g9J$lL^kaP}(bPj>gItM_uKWH6j z5(hx&9Rr|r4uQ}*1p~wjzsd(?ZLA)VDadLO%77fXP&=R{ZBeayU zFJiHD1ap%sZe;g&AojepmhKHdi(x7-6+%=t<;%R17ZY+=coNZIY{vfpX=t!>b0V76 zszy0j0c*)iV9rU~VairU{$O>-&Ih;CYYKIm`xZhJ-rL`3SbYlo4ig%x8xebpbM8FC zz8>sNxgJs@YG`)=-MdWNO@#|xh`Wn^v1?3gJLT^SOAN>g&VtHz>L#t0g1S`r*3ILT zwhJL*1@`Ikky;KI9z!+1qC zD^(#!A;=C;8qra5=Iz_0qb3HMTbMkcd#KbiLKn_5#83|y%C zd*7TmVhpP+iRLz`+U zyY_O{x75j~h0MJl*vm%ks;t!9{6|jbFR#6C$~At4G8%72SG9VTtCDsPOLP7u-dp6I zo|R1nH@z(iaFBc17gs+ljil~&d3B9mxCc)`Xa+V?_Obeb`pXFU--oV%rF@;&l~fsT z+oakuO+K4XQ;6$O)NVk9wZ6BsE8C7LDV-yNcVBfbeB70Uc(s(&?MZaTH6qB)&t=~#s1I*uX_WugMi<);3A3!&I#>f z{X_%vf!$zm*!h8R?jT%7+x<^S2a)-9fjahqV0S&>b)a_qpmZyscQ_qm{a|-bFgSRD zaz8($B0{)h*6=G7C*2s1+~81(Me_r-_JJ@ru=kH)yaXs!JvKavAWK7Z&{ko4sPTK4 zq_z)2suCF(TdlT{S-|KW0)j!hn-~pER@f76y`XhYDtoSekP46pJz*Cso3`?ckiE|G zT88Y~3zO0ZT=Ib+bOZqA#=-}92WTCU#@Qd2NE)lF0Ijea*m9Vb;P}f(g0^n`B8z8J zP|n4<1JY`ez?_B2HvX|mV`4Xz6$nFeFACF$m+NU1)z!HkT{%EG3*<+Ra5XHn_Zi zJxg=mQJvcRn-4Ql4A|nQaBcFIlHG#+JVh4eVyE5RWK&v-df54Z9k`4D9H3mWZ+D+5 z7TX0QdW~V*tjkOqru%+#OpvTCW)h-g7oQ(0AXXdC=yXsS&ful5vZYWjjTLC z4g$55-V`JqDnI|%{798FVH9dKtjs@&sw0p7%N@|sbQKt~(4YtMlB4c&le)CWOKYx{ zi36Zuc}H;(sod}qkno9eKIBS(b(OTuegGuwG8B}4rb@Bsr249%>eHsE2;CXLZ~nH9 z%BxlGDZZRLfq#0iPyYbaK;0UtsMLz0F4-n(O^<8=kzRzN7+_dnIi;=u`eJ0T>r>AV zLX|;qMk;picG^f79dM!!RH-wOVUF^t4nRC)@v&Jx-n5I5qnUpMfrDD0L;us z)na{_ooB35)EYH(?OdJB>8if{$Z`5jR}$o^>9xABRB80|RFHk5BIl?!xs*yCpR1}km5njI z&6nK(b|mxp!aVi#)~s5FsnJ2cb{_A5L6ip9RTiYWokc`DSw$+Weq$pgttBd&+O8Ee zl}%>b-fU@`4!Lauv#m{usimZ>J=R>Uk^b=p5VW*CMza^wP^C>q$Y`x5VdCsWy}+va zz#Tn63Pl@sH0hJua&9D7in&%QX(6b?U;?H#@B6VJFX`3US=Oy8gnU%P5XwA;0?|uv zFr*QPty6}1jD7zA(7@29omof^QCMF(fS|bTVs8U;POU%x03kGNsH@F^Dm?Efz%^R+ zR+AK@u5K;`HTf2|>mUtAp?6woejd$%%}3O20UZ{hMnyIok{M6+iY!urbp#Xzz7SXg z_JF?!sZ;QajnBNheD~TYuzD3KY|CaFld%>&w*_^%MK<%J#Cif%64%?R^I>4mt0N9ba?*QMW_JEfLKfELWHn#Wsm=JBg z#?UDtiDE6_3kVTWn1G{lYy=KuC`z`>4TymVx7Gz2CNFM(2_oRz=L$;YcLw*|^Bc;I z(b3!YjZVXs#drD$8t}!4)2|eP{24gE&eEe5XQh(VY*>E_0!Wcbbq$r3fFI7+w6-P! zuI$;6ek5JLNdVw3R>%u;xwq;B48j1|x!bz$BE1Wv4xCDCSX%pU_m+~`ag8B}0Nbg7 zi6oLvF^2fzX5&;^f(RK%0EvNE)vJX@b{kr>7+Y&sbJugUDJD%B1pv z+{hODK%(~pvBW|$~g-j@Fnfa2Nt;J12f+6Bnbom(ELzq>GR5!AxW*sQ^5X`QEdjUlCD-_ z-HQ-U=3MJ-0HP?4upRy#dfV2`FUOhISF)v&ug8bJR5 z7Yvba7b4M6S%;}w;OkJ;AKeHK*q_opK8m#|`hi`k(W|!Xjc5Ckv2b^Jl8stqn(PtJ zEUKNpKpAcsYWhr&I@3ejh{DJ6H;Y;YT`Ge0hL?&z_>$$mxpI!^^-|QRTds>v*7#Y2 zw%_0<^pawI*8au#jdrBB;i(Y3ic>+4-mr>@^(`q-){4AU();6`+fJkMQ~F78t+@Ki zI;C=G>8g{?nWQcAx-5}Yt?TP4*9-W)eJg|S6;FTv0HusXeUjc<^(!@Ow8+(I6SLm{ zN6Aq@nutqds?;b$Mzv}*$lX?(7i_$dd8$i%_SETAsr*e+{{RhuTMxLJq=89LIzM7LCIhgJt|2-?g*-D zW{rM(pdt%-MvlIoyERQs5p!c$GgP;LM>rTT3~*Ys>$@8kWo&@&c9BM<14$UEs%^2l z5SIDe?E}%K(bQqYU3E0t!xy`6(;bYdWRw!M!_{3=Q#-R7qQkbpz%}X62*GJ-&evQj zRL}mda6WQ7(;?Gtm*+Cm(7H`#vn@p$F6sPr4S>1xu$Ge8y$wYmOVecoV%9eM@{k60 zatkqHHWxRvE9z0uY0)TF)Q}dg_iCu2f8q+RN7ibR&r+V1MyiHURa{?;l}mrp5o5A< zd3c&SD0n3+0^bcvWIiXe5=PPw8CREUi5R;oa0_(fumf(_ zBb-7@TDomd3H4QU^idYT0LZKK0j^A=!YMn3b;iS=yrBI7H!~{R@Qf*san_~oO+XM^ z#m@4jv#qADK&YyPOK!Ke4K>ev-jnD`ThgX zU+>Sy{eHb&Z>ceLnH7xK<7l*kv5!yYBthaEP_l%oaq35gQ~Jz-gYjo*tRsK*u}rK^ z$zO)DJV_NCI8z9fNdRk#s%XwdejnX`gRi5!mwKMXxj12b>JV?Y29oZS^9kuocEiV*HJgS2lzFA6s*kwO9C(NKMI(}aT5!@%z$}ie6nijC~y8=$Ivi2U;-)dVvI`DIoju(r>+RC zeXdn@_eN1>;T3MB`Gu_FalWZS{1MXUz-Kw)6R7~JkSD*QU&aNBL36LTT-jg_`qv7>Wb(m}&PmYi`ccD5~XxDgL<6vJJ z(HGx_TnNCkCvK+y?8<-fh<7I*U-!<3{dIZK$Zeo#dBcs>?rkfEy+)7IM31=u%%*WrNPm5yVVPPhT6 zvT*67k_yq&OLaW7}zfTM#sF^>9> zm;ciQko?FuV9?_7uHU0ipCcrWIo;9C-+kxUaRL(y$^BG+-pz(M;(}VyYvjb{7ej&f zz6nK!b`Rj`pZxk4qv z)&3gsy5f!ytsi})#=W+YDZ+KME)^?us8fl=%J_*>VBKjlGS;#o_WvZ#t|Vo>%9T!{ zIxW8yQfhll575YF3>5_=seWd^e!;B}^5g5vSS@~P9dp%24eAR@6zJgVGHYQ(i6NVG zr4rqz#?_27B%{}LBOFU`AXY2L5P;M3YLYQ~5X#y~yd4yD!Z}z1nJfCUsX^$^ITA_o z9h7MC*Uk)1?AV|+pB8OsaD%Z7Bq%YjNh1}#Px%Z|wa4ckmCOv6mfEEllv`Y5nM8+d zZSGmWT*ZWxv;}oDIsYrS=^0?lADY5&DeWuQJ`+@JsOCzYCFSKjvkw~bU%%^ZwCS2=~AGQB9om!Hk2pA!?&m)#1J+Sb1PBMXq?t6xSKs?5rhD=p`3cbIXzIq?%Cq)p+zgBBlt z*rxmJ{BmN;L*CWU_o$?smX{H_WJkppqsrtK4GM}XfvUxapAkZ4;Gu4I4t<);z_*xUeT@cFuDN% zN<_VFMEIwdmi0vZ+fZs{u!c)Ro>)NL=I02(^2`sNs8dj6K^!huP_S}K1Y{BZR`$iY z?PNKObfapqhc$#ac`CPlgVu#@zXJ3OGC-qMtdZz19gut1!{07{v=xOiK?>OLSm`Qa z|0h4aiLu^*o<#H9HG%n4mV^Yu8FK$*u#1Gifbh<(XB;gs-=C*;Q{Ss2bB{{HN~@LO zB_-K~2j4AYBX-Xp&n!j4jlg*@iD2%JD2f0udlGkhd$0E%^FXf+SA@>$WTiT7_U<1uqd%gBELS&|6cOuP?=$L z6=V?n_UTX){NdB*4KMj5fdMX&_pGaJPTpI*;+5xksdhuDnUjfaIL1@#p|B#!5gSbi zxTj_DWza4?5=SU_E?*qoeP&Y|xTl{&kgmK{BiUZy*j;Q`s?|FqFNHGwyyo4GT@;a; zuI1V9_%6Pskn*qp>Y~d=Q;xKV(UZUzy~Pb8WC&uW?I0`4)fBTlS7u9+_4e^d{Vbf6 z6o>1|igl#4bnx^=YiIYxF7t%h@(mW7jyHA$>3+H(l`D@wcg-UBu9#xr@$XREi%dj_?5lSa82$1thw`7r z(S6AwZ^iDfa}N*iKB_MFH6Dl;+s^I}%xX2r*nJc{6AIhqT&ff+=I67sZXIqX2Shx4 z&b8kW(XY8shdvWO+6nzkMSk6fK3sKw6*T}p_z+D302pi}gJcd%>rIGa|h9O=j`c)9A#lw5$96Durhcey4H&-y0B-1c7s z6?5c>1}MAFvu_FBk@q04wI35!cbINYXxR0ST|axj<{itOXWD?-m0&mhg+J3ruG!Cq zhZQP3(O>Zk zD?1XG!?Z06K=kD)=V;$QXbEhr|$0b3`tJ-zY6198aID!|1aYcNhYW5^m z+3jEk9^&j!4ePG7u=x}A-BJkhw67`dag+q16_+B(Gq1JB!#)C3M-!8()Xn3iU@uHi zNRs3pApIH<{B#g3W+Gk9+R30@LHNMs7A0oDcKQ^;1<8V>q*0|0l<&ukM51|6K-98a zS@o3k50@RyvDw8Nt`1RUg$SmKb;8aC)t*1@;H7`PS+ywLZDYcI(n+S+=QsMT&;*Fd z+9n83c8jO9<5~4i^hrn>M(~ z<6(^n82V>fUBSbO_-2uIS66?`zS} z23QZH6K)9~n#*f9QYK%=WoOP5vznzf5{aNN@L(>4-C6J-*LobGCSt0|qxYo7NOqq}Sy_KKfs*?M#y85#>3k$zhG#)P*5zD{ z(V(8x(x^IcxsuiF<$(8spbY8y<}he*$S3cP10NDCai<`cM10h2h8_Hm>#bV%xwfBU+6X9kz*I@G6U{{H&BmC&T)roc$(Vd$ zd4_D7&g*{m#kp#KJS~L{*F+h@9(<1}4v+kMcKzvwAdelX%+kpKC*JU3wTJ=qqeb5r zznZneV8jzs0XwG)AH5n9P)r-LEvFNQR%2#L5Tw{ZXm&?NU%aC>P!eS_Tv6>lVxf#h zb`IWK(&QD{krrg0rQSu4=?9=zVFNsUx!T$l4YCLZL*2oC+=(mRdAG{^fCu?Ajn6&j zS#_EE1HgP*r=?yKZ;nLiihl_Zj!gK_tv)56|J<=20}|=ygMa6ey0vctv@Sl~$eVgQ z<=bsF;Gs<6+>!2%1v?Wz;W0@*F51*-xjQDvuEI+Z_!F$zCnA5r8cIjM1C{Wx|NmlN z_q;l>wF(0+O+h~2R?;+E2Pd`&*_R1Vak&*6DWEwfU&Awj1|Wb=de^OwEgfG`4CPZz ztM)@Oh%yIGb5f}^w(cvYucAWc!4&%lU#;75uZrWES0wqte(s^)Z=QI2E~x0lho6_J z-%fhlLpZd6vXCGL%rbAwptRXn!~T9n-nr|C%vW)u>d4fpfE@1j{>7#&m9US3;^Od?e!tB(<4L2&qdqzEW|JL$}^fYR;1;`N1U-O4%I+{bzgS zF7p1O6}nawL5>5T_B^{mJQX4ka#+`GZQEsltc*}Op604n@D!O;x~=f(0vFwx`O1+< zOt6`ie%Vo2H4n#xF=^jQ;)w4q5<6xWC&f;2y?b57h_mM}m>S2uSQz{m-71ao!_UU! zWna%%3?}Ty*q-Y_rdPV~7hx&XxE{lGL-~j7?09k*Agl0a!{ys49h^5}!E6?3FZ^Up z@EK&WLEMK~Hzag(1X2VCtaQO%(+1Rg2Cl&CbaB(9cd&-a!Qcw z`M%K7o3|GKb{_BYvZ<6f(TK}G8R#m@YJLB+EXseClI%_@Q-YWy8X5Y-+*f9LWmFt;*vN{4WF%AjElJ&YLhE z{l~?gQKR7+3!xw8{xVOi^-g==Th#Lwj-4zWL8An!)>6El6VO*pgLfkcROV5e-8AxZTRE~TJey}Lx#eUhmsB$R&nfd?AHfU$G14tg zwYO){2^-;NWyL3r`Grjd{>e8O4HZCe_aA!qWc|iO(zeRq8?+*E7m)I}(b6;R1?Pq_pzaz)bQnd2n ze5^s9``aqOR^8UZ9bPm4N~z;K(wJ;#{hykRI>SNyuSV7QN)qt|9~=04a$W2|P{4t< zcG$Y~5`74B3s#}hcWvpMW9t0qZrZ!{e(+OFdz^%+D#7&g9>G^$$BH%z#*8^2365;E zh*Fav07|z08{*TD6cbq^B5Q;WY{0ImKZHT)(j*tT+ebbuhPH^>87a`wgGT0=1Pgr) zR~Gf${k~WCZn3lEyQC3<noxjDvH9d^z@NXCDoba@ zYwMB=hEf$`Z`H1QBg{T@@`>j)D%>BJOTU({-&xqkZ&R~(42rdb+`jByP}<GQc%bUs0z-uPa#Pe7hk*!<kp zz6Ub!*dzOhPS{1f2KLA*B`bmM|8v(r&AH|Na>t5DViVAjxF!3p^AOrT_+Yec;QbD_ zBSSX}mGMbaq>Yw6hkH#dcs&ZBt|kdZ_Urvc`u*ub==_e9BjyCTVxzz7s2N{UnP6Vs zq|k?(tQjlvg`DcHavn`Ep)Io1%VTzW$NIKhv>^AsoyF_eo*Cz4 z8_%hH@{GRRSh!`fb8T~xh_oQI>cke2vK~o&Cs1OpjmhU_H#D@Kx3<#o;+I*nZTgRk z-0e>ccXlW>ZrKt3aub>p|JBw7MKOm=gjZI|g3ZQbo_fLsD3?@;&qvlp;?klX$@HgM zZ`x*emXlYgT=Kq0IAg7S^t|vL``D*eunQtB&wSwMeGgfHYN-PW6Kt1oW@0*U@MSio z83+HNK2{B-$*hO$9oE$$UU6FUf@PEgzG~8VmhK7eZT`WSqMb=>D`kF>vBJ&@%fH~P z(dZ$BJH-i;z2wVT-Hs9koUOy7iCNN=)}W#9YF>9R+kY;#?zNHW)D@|BB7S$#Pax)&0;oxLKz4d zxlXl4yHeufPpWj*^)~ffg+2F7pXB?h*@f?2#28x?Siqj>fv{-sYgaEjle}SeF73qX z^%`2hJSMcI5sHckFz=xpS3$RTj)S6qjj3u9_fzhWR$2OdD6L5;;wJ3h6`4q=?==Wx zKSk$J!|IstltCQ?$`*?He7%0Sbac(t1x_}LX@A#Mfp$`Pl4m9VIPkH$V5qpaE{+A$vhB03kg0h*v9@Agnju@Ib}t~saQ5%>x~Ygc>&9{fgfzMz zNhZ^ks|<)u?K;4llAL+uhJSce7nyt?PR3NyvN;d#$}1F1YVi@mB?sHTuj~NQ3x}%k z0B~|*iDdkO2GDueVzW8nxj{+ZJ~lwi|tsrQzR@;@N{}vbph=_0+?1Z$jWj89tD?!&oUk z!YUI%yqU!&a}x8z*BwgSfU+venxzS75kraKM$Hcb@1U%SK_iv&G7nj#+Oryr3teHO zHPOjfO-_FV1d^UGK#C_UJ7RSO@oYybPega@5s)+vXcYidCHo@ibnB4d$+s7(3l4Kv z&1rKL4GEVS<#sug%hZYm!_#CfY5&$YJ33SVRkhO6Lj)k9x!N1`NFpGz7F=Im8X;k7 z;b8+c&u{Gn7@3%?i$^v_uNg=&9VOJp!bY-=GnLNB;Il%Mh|%%qt23RkB*n1)vwGUtnn{j2Y5vFG|dQP0KQIOAypf# zaVF+do3#z1clgVJ23XI0PJFYsMxme2oFnuEFmDivW2V^$anE#w8!PxA8<4Qm+2fDo zNBP9NPNnEU99q4B32XiQAR%?Zq!hcilHTBVzfV-l#Ds$9=91?28*V7oehENw?Hv_ADcQ?9vUDrbkGpUCxlCu(PsQl5=pp6f z&;K8H%E5>IvdE5fLm1e+Y1%~KS6=lyTYytVLooHUJnvxlb-lerWO=;+X~6KLEz7nA zM}Ty8m@N7<&0FpT@ks9{E!BeDt9?EU)vVq~KJjq#=)RB>sifP`>-$Z4;Pls2TZe%e zaV?Ya&PF`N{+pRGfo&k}wLq-ghWI_!yhcA|ODg-3_G4q;UTk&aTAB!k>gsm>)|VXB z2MPCHclbJ!?sL;twe{6S!;Or%&E_+CX2le>b0Aqyio5?z_(#awJfs96lG8S24r|iP zU}XpRH3A0H}vqV-H{}sFA}-yB=!DU)-?xx z;%RDkZS-ZY0g^0>xLV`Yq00}@R+t;bw4WRMCZC^)yp=K<5G7L(;tlIuZ&jDcOUmhN zCdYYiUBnw5$|c6LI9loIPGNYr#>!ZdR~RU#>Q!wg9K)9w=laa&n3490x?eu!o*i7; zBPh9wHy3KpW$}?qEm;#}Vhh-}e*Cu4+^CfT;2oB?mSco}kWH|^d?wfJZHCwAD8bTy z?E+X>eM&}XW@^bYif0v>Z&v%EJnfa3GL_GqdWVh4VfZyYnR=xoU-XL86}S~;byH>A z2rk;oF-(iHl&PYn#l@!V70=9zS3g1=u8^z*oAa-$cwD3`+m}h;UA}RxNdC_7g3;CK zAr`h-pf?lW&XCxC5nks!e}JrRTog}~DvNqp7!1>?IgGb4i=SiM&77`1S;~oPx)T%n z#LSX^adBS)Uf*wS#)zwV#wp_?W~0pxGiyeYs!%mEW80vke(=~-Bu+y*{9h;7;%(s@9K`TuGWnZpU5=nxQs^21)1Moec9SEA!8O z_0`C}gjPQ1RL?MbS6=HKF*o#4j;y^RNktlrT5B=r+HuFAUuyEtL<~kqU$-$Y6M6K~ zXu0O?^1wiU`@qh(^Aw=?u7CYka!OXr!=|MRbo33@fsIRp=rM4`scD`N$r!H?8%Khj ziXN$FGGaXQ@gC0VF@Q;muwqFCpFq^PFMMh+`Tf)XXV%Zr<-%swH?(BtSMt6a?sl=0 zyLGNuyA3Crwf|30q7^&DS?610zrv}`kk5fG!S=rjoj)RVp{QmU;_d(s7dQ1?$8LS| zzQi#AGL`i7Fty3iGmA<)5xUvXYX?~IEU$7As*hQE zN00lw6Y>qt)aXG>ty)W}2+33qzWtryZXzLO{%Ml<)9C!n-Li1KrmKw&M@&3Sy4^zc zCjHMi^Juq}a%#-{ubKHdNjX}2b5C@kBt+ly7|;LxbW|X#IPG_oD<6jfGQCiBLit>= V6Yqu8|FQfEE;s&{==tyK{{g?L5tsk~ literal 0 HcmV?d00001 diff --git a/resources/testhelpers_test.go b/resources/testhelpers_test.go index d064fa57052..61a9a6e1e72 100644 --- a/resources/testhelpers_test.go +++ b/resources/testhelpers_test.go @@ -4,7 +4,6 @@ import ( "path/filepath" "testing" - "fmt" "image" "io" "io/ioutil" @@ -12,6 +11,9 @@ import ( "runtime" "strings" + "github.com/gohugoio/hugo/langs" + "github.com/gohugoio/hugo/modules" + "github.com/gohugoio/hugo/cache/filecache" "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/hugofs" @@ -28,9 +30,8 @@ func newTestResourceSpec(assert *require.Assertions) *Spec { return newTestResourceSpecForBaseURL(assert, "https://example.com/") } -func newTestResourceSpecForBaseURL(assert *require.Assertions, baseURL string) *Spec { +func createTestCfg() *viper.Viper { cfg := viper.New() - cfg.Set("baseURL", baseURL) cfg.Set("resourceDir", "resources") cfg.Set("contentDir", "content") cfg.Set("dataDir", "data") @@ -40,6 +41,21 @@ func newTestResourceSpecForBaseURL(assert *require.Assertions, baseURL string) * cfg.Set("archetypeDir", "archetypes") cfg.Set("publishDir", "public") + langs.LoadLanguageSettings(cfg, nil) + mod, err := modules.CreateProjectModule(cfg) + if err != nil { + panic(err) + } + cfg.Set("allModules", modules.Modules{mod}) + + return cfg + +} + +func newTestResourceSpecForBaseURL(assert *require.Assertions, baseURL string) *Spec { + cfg := createTestCfg() + cfg.Set("baseURL", baseURL) + imagingCfg := map[string]interface{}{ "resampleFilter": "linear", "quality": 68, @@ -71,7 +87,7 @@ func newTargetPaths(link string) func() page.TargetPaths { } func newTestResourceOsFs(assert *require.Assertions) *Spec { - cfg := viper.New() + cfg := createTestCfg() cfg.Set("baseURL", "https://example.com") workDir, _ := ioutil.TempDir("", "hugores") @@ -83,14 +99,6 @@ func newTestResourceOsFs(assert *require.Assertions) *Spec { } cfg.Set("workingDir", workDir) - cfg.Set("resourceDir", "resources") - cfg.Set("contentDir", "content") - cfg.Set("dataDir", "data") - cfg.Set("i18nDir", "i18n") - cfg.Set("layoutDir", "layouts") - cfg.Set("assetDir", "assets") - cfg.Set("archetypeDir", "archetypes") - cfg.Set("publishDir", "public") fs := hugofs.NewFrom(hugofs.Os, cfg) fs.Destination = &afero.MemMapFs{} @@ -126,7 +134,7 @@ func fetchResourceForSpec(spec *Spec, assert *require.Assertions, name string) r src, err := os.Open(filepath.FromSlash("testdata/" + name)) assert.NoError(err) - out, err := helpers.OpenFileForWriting(spec.BaseFs.Content.Fs, name) + out, err := helpers.OpenFileForWriting(spec.Fs.Source, name) assert.NoError(err) _, err = io.Copy(out, src) out.Close() @@ -135,7 +143,7 @@ func fetchResourceForSpec(spec *Spec, assert *require.Assertions, name string) r factory := newTargetPaths("/a") - r, err := spec.New(ResourceSourceDescriptor{TargetPaths: factory, LazyPublish: true, SourceFilename: name}) + r, err := spec.New(ResourceSourceDescriptor{Fs: spec.Fs.Source, TargetPaths: factory, LazyPublish: true, SourceFilename: name}) assert.NoError(err) return r.(resource.ContentResource) @@ -144,9 +152,6 @@ func fetchResourceForSpec(spec *Spec, assert *require.Assertions, name string) r func assertImageFile(assert *require.Assertions, fs afero.Fs, filename string, width, height int) { filename = filepath.Clean(filename) f, err := fs.Open(filename) - if err != nil { - printFs(fs, "", os.Stdout) - } assert.NoError(err) defer f.Close() @@ -170,22 +175,3 @@ func writeToFs(t testing.TB, fs afero.Fs, filename, content string) { t.Fatalf("Failed to write file: %s", err) } } - -func printFs(fs afero.Fs, path string, w io.Writer) { - if fs == nil { - return - } - afero.Walk(fs, path, func(path string, info os.FileInfo, err error) error { - if info != nil && !info.IsDir() { - s := path - if lang, ok := info.(hugofs.LanguageAnnouncer); ok { - s = s + "\t" + lang.Lang() - } - if fp, ok := info.(hugofs.FilePather); ok { - s += "\tFilename: " + fp.Filename() + "\tBase: " + fp.BaseDir() - } - fmt.Fprintln(w, " ", s) - } - return nil - }) -} diff --git a/resources/transform.go b/resources/transform.go index 934c713277b..b848556c61c 100644 --- a/resources/transform.go +++ b/resources/transform.go @@ -515,6 +515,9 @@ func (r *transformedResource) initTransform(setContent, publish bool) error { // Copy the file from cache to /public _, src, err := r.cache.fileCache.Get(r.sourceFilename) + if src == nil { + panic(fmt.Sprintf("[BUG] resource cache file not found: %q", r.sourceFilename)) + } if err == nil { defer src.Close() diff --git a/source/fileInfo.go b/source/fileInfo.go index 072b55b6cbb..a4cbf6fe621 100644 --- a/source/fileInfo.go +++ b/source/fileInfo.go @@ -14,11 +14,14 @@ package source import ( - "os" "path/filepath" "strings" "sync" + "github.com/gohugoio/hugo/hugofs/files" + + "github.com/pkg/errors" + "github.com/gohugoio/hugo/common/hugio" "github.com/gohugoio/hugo/hugofs" @@ -28,8 +31,7 @@ import ( // fileInfo implements the File interface. var ( - _ File = (*FileInfo)(nil) - _ ReadableFile = (*FileInfo)(nil) + _ File = (*FileInfo)(nil) ) // File represents a source file. @@ -90,13 +92,7 @@ type FileWithoutOverlap interface { // Hugo content files being one of them, considered to be unique. UniqueID() string - FileInfo() os.FileInfo -} - -// A ReadableFile is a File that is readable. -type ReadableFile interface { - File - Open() (hugio.ReadSeekCloser, error) + FileInfo() hugofs.FileMetaInfo } // FileInfo describes a source file. @@ -107,7 +103,7 @@ type FileInfo struct { sp *SourceSpec - fi os.FileInfo + fi hugofs.FileMetaInfo // Derived from filename ext string // Extension without any "." @@ -179,13 +175,14 @@ func (fi *FileInfo) UniqueID() string { } // FileInfo returns a file's underlying os.FileInfo. -func (fi *FileInfo) FileInfo() os.FileInfo { return fi.fi } +func (fi *FileInfo) FileInfo() hugofs.FileMetaInfo { return fi.fi } func (fi *FileInfo) String() string { return fi.BaseFileName() } // Open implements ReadableFile. func (fi *FileInfo) Open() (hugio.ReadSeekCloser, error) { - f, err := fi.sp.SourceFs.Open(fi.Filename()) + f, err := fi.fi.Meta().Open() + return f, err } @@ -225,38 +222,45 @@ func NewTestFile(filename string) *FileInfo { } } -// NewFileInfo returns a new FileInfo structure. -func (sp *SourceSpec) NewFileInfo(baseDir, filename string, isLeafBundle bool, fi os.FileInfo) *FileInfo { +func (sp *SourceSpec) NewFileInfoFrom(path, filename string) (*FileInfo, error) { + meta := hugofs.FileMeta{ + "filename": filename, + "path": path, + } - var lang, translationBaseName, relPath string + return sp.NewFileInfo(hugofs.NewFileMetaInfo(nil, meta)) +} - if fp, ok := fi.(hugofs.FilePather); ok { - filename = fp.Filename() - baseDir = fp.BaseDir() - relPath = fp.Path() - } +func (sp *SourceSpec) NewFileInfo(fi hugofs.FileMetaInfo) (*FileInfo, error) { - if fl, ok := fi.(hugofs.LanguageAnnouncer); ok { - lang = fl.Lang() - translationBaseName = fl.TranslationBaseName() - } + m := fi.Meta() - dir, name := filepath.Split(filename) - if !strings.HasSuffix(dir, helpers.FilePathSeparator) { - dir = dir + helpers.FilePathSeparator + filename := m.Filename() + relPath := m.Path() + isLeafBundle := m.Classifier() == files.ContentClassLeaf + + if relPath == "" || strings.Contains(relPath, "TODO") { + return nil, errors.Errorf("no Path provided by %v (%T)", m, m.Fs()) } - baseDir = strings.TrimSuffix(baseDir, helpers.FilePathSeparator) + if filename == "" || strings.Contains(filename, "TODO") { + return nil, errors.Errorf("no Filename provided by %v (%T)", m, m.Fs()) + } - relDir := "" - if dir != baseDir { - relDir = strings.TrimPrefix(dir, baseDir) + relDir := filepath.Dir(relPath) + if relDir == "." { + relDir = "" + } + if !strings.HasSuffix(relDir, helpers.FilePathSeparator) { + relDir = relDir + helpers.FilePathSeparator } - relDir = strings.TrimPrefix(relDir, helpers.FilePathSeparator) + lang := m.Lang() + translationBaseName := m.GetString("translationBaseName") - if relPath == "" { - relPath = filepath.Join(relDir, name) + dir, name := filepath.Split(relPath) + if !strings.HasSuffix(dir, helpers.FilePathSeparator) { + dir = dir + helpers.FilePathSeparator } ext := strings.ToLower(strings.TrimPrefix(filepath.Ext(name), ".")) @@ -277,14 +281,14 @@ func (sp *SourceSpec) NewFileInfo(baseDir, filename string, isLeafBundle bool, f lang: lang, ext: ext, dir: dir, - relDir: relDir, - relPath: relPath, + relDir: relDir, // Dir() + relPath: relPath, // Path() name: name, - baseName: baseName, + baseName: baseName, // BaseFileName() translationBaseName: translationBaseName, isLeafBundle: isLeafBundle, } - return f + return f, nil } diff --git a/source/fileInfo_test.go b/source/fileInfo_test.go index 9390c624706..0c024de1882 100644 --- a/source/fileInfo_test.go +++ b/source/fileInfo_test.go @@ -15,12 +15,9 @@ package source import ( "path/filepath" + "strings" "testing" - "github.com/gohugoio/hugo/helpers" - - "github.com/gohugoio/hugo/hugofs" - "github.com/spf13/afero" "github.com/stretchr/testify/require" ) @@ -55,56 +52,10 @@ func TestFileInfo(t *testing.T) { }}, } { - f := s.NewFileInfo(this.base, this.filename, false, nil) + path := strings.TrimPrefix(this.filename, this.base) + f, err := s.NewFileInfoFrom(path, this.filename) + assert.NoError(err) this.assert(f) } } - -func TestFileInfoLanguage(t *testing.T) { - assert := require.New(t) - langs := map[string]bool{ - "sv": true, - "en": true, - } - - m := afero.NewMemMapFs() - lfs := hugofs.NewLanguageFs("sv", langs, m) - v := newTestConfig() - - fs := hugofs.NewFrom(m, v) - - ps, err := helpers.NewPathSpec(fs, v) - assert.NoError(err) - s := SourceSpec{SourceFs: lfs, PathSpec: ps} - s.Languages = map[string]interface{}{ - "en": true, - } - - err = afero.WriteFile(lfs, "page.md", []byte("abc"), 0777) - assert.NoError(err) - err = afero.WriteFile(lfs, "page.en.md", []byte("abc"), 0777) - assert.NoError(err) - - sv, _ := lfs.Stat("page.md") - en, _ := lfs.Stat("page.en.md") - - fiSv := s.NewFileInfo("", "page.md", false, sv) - fiEn := s.NewFileInfo("", "page.en.md", false, en) - - assert.Equal("sv", fiSv.Lang()) - assert.Equal("en", fiEn.Lang()) - - // test contentBaseName implementation - fi := s.NewFileInfo("", "2018-10-01-contentbasename.md", false, nil) - assert.Equal("2018-10-01-contentbasename", fi.ContentBaseName()) - - fi = s.NewFileInfo("", "2018-10-01-contentbasename.en.md", false, nil) - assert.Equal("2018-10-01-contentbasename", fi.ContentBaseName()) - - fi = s.NewFileInfo("", filepath.Join("2018-10-01-contentbasename", "index.en.md"), true, nil) - assert.Equal("2018-10-01-contentbasename", fi.ContentBaseName()) - - fi = s.NewFileInfo("", filepath.Join("2018-10-01-contentbasename", "_index.en.md"), false, nil) - assert.Equal("_index", fi.ContentBaseName()) -} diff --git a/source/filesystem.go b/source/filesystem.go index 0c1a6ac7b9e..f3357750878 100644 --- a/source/filesystem.go +++ b/source/filesystem.go @@ -19,24 +19,24 @@ import ( "runtime" "sync" - "github.com/gohugoio/hugo/helpers" + "github.com/pkg/errors" + + "github.com/gohugoio/hugo/hugofs" + jww "github.com/spf13/jwalterweatherman" - "golang.org/x/text/unicode/norm" ) // Filesystem represents a source filesystem. type Filesystem struct { - files []ReadableFile - filesInit sync.Once + files []File + filesInit sync.Once + filesInitErr error Base string - SourceSpec -} + fi hugofs.FileMetaInfo -// Input describes a source input. -type Input interface { - Files() []ReadableFile + SourceSpec } // NewFilesystem returns a new filesytem for a given source spec. @@ -44,63 +44,84 @@ func (sp SourceSpec) NewFilesystem(base string) *Filesystem { return &Filesystem{SourceSpec: sp, Base: base} } +func (sp SourceSpec) NewFilesystemFromFileMetaInfo(fi hugofs.FileMetaInfo) *Filesystem { + return &Filesystem{SourceSpec: sp, fi: fi} +} + // Files returns a slice of readable files. -func (f *Filesystem) Files() []ReadableFile { +func (f *Filesystem) Files() ([]File, error) { f.filesInit.Do(func() { - f.captureFiles() + err := f.captureFiles() + if err != nil { + f.filesInitErr = errors.Wrap(err, "capture files") + } }) - return f.files + return f.files, f.filesInitErr } // add populates a file in the Filesystem.files -func (f *Filesystem) add(name string, fi os.FileInfo) (err error) { - var file ReadableFile +func (f *Filesystem) add(name string, fi hugofs.FileMetaInfo) (err error) { + var file File if runtime.GOOS == "darwin" { // When a file system is HFS+, its filepath is in NFD form. - name = norm.NFC.String(name) + // TODO(bep) mod move this to hugofs name = norm.NFC.String(name) + } + + file, err = f.SourceSpec.NewFileInfo(fi) + if err != nil { + return err } - file = f.SourceSpec.NewFileInfo(f.Base, name, false, fi) f.files = append(f.files, file) return err } -func (f *Filesystem) captureFiles() { - walker := func(filePath string, fi os.FileInfo, err error) error { +func (f *Filesystem) captureFiles() error { + walker := func(path string, fi hugofs.FileMetaInfo, err error) error { if err != nil { + return err + } + + if fi.IsDir() { return nil } - b, err := f.shouldRead(filePath, fi) + meta := fi.Meta() + filename := meta.Filename() + + b, err := f.shouldRead(filename, fi) if err != nil { return err } + if b { - f.add(filePath, fi) + err = f.add(filename, fi) } + return err } - if f.SourceFs == nil { - panic("Must have a fs") - } - err := helpers.SymbolicWalk(f.SourceFs, f.Base, walker) + w := hugofs.NewWalkway(hugofs.WalkwayConfig{ + Fs: f.SourceFs, + Info: f.fi, + Root: f.Base, + WalkFn: walker, + }) - if err != nil { - jww.ERROR.Println(err) - } + return w.Walk() } -func (f *Filesystem) shouldRead(filename string, fi os.FileInfo) (bool, error) { +func (f *Filesystem) shouldRead(filename string, fi hugofs.FileMetaInfo) (bool, error) { if fi.Mode()&os.ModeSymlink == os.ModeSymlink { link, err := filepath.EvalSymlinks(filename) if err != nil { jww.ERROR.Printf("Cannot read symbolic link '%s', error was: %s", filename, err) return false, nil } + // TODO(bep) mod this vs root mapping linkfi, err := f.SourceFs.Stat(link) if err != nil { jww.ERROR.Printf("Cannot stat '%s', error was: %s", link, err) @@ -113,7 +134,7 @@ func (f *Filesystem) shouldRead(filename string, fi os.FileInfo) (bool, error) { return false, nil } - ignore := f.SourceSpec.IgnoreFile(filename) + ignore := f.SourceSpec.IgnoreFile(fi.Meta().Filename()) if fi.IsDir() { if ignore { diff --git a/source/filesystem_test.go b/source/filesystem_test.go index 8c8e30413f2..416799c9ddc 100644 --- a/source/filesystem_test.go +++ b/source/filesystem_test.go @@ -14,30 +14,44 @@ package source import ( - "os" + "fmt" + "path/filepath" "runtime" "testing" + "github.com/gohugoio/hugo/modules" + + "github.com/gohugoio/hugo/langs" + + "github.com/spf13/afero" + "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/hugofs" + "github.com/stretchr/testify/require" "github.com/spf13/viper" ) func TestEmptySourceFilesystem(t *testing.T) { + assert := require.New(t) ss := newTestSourceSpec() - src := ss.NewFilesystem("Empty") - if len(src.Files()) != 0 { + src := ss.NewFilesystem("") + files, err := src.Files() + assert.NoError(err) + if len(files) != 0 { t.Errorf("new filesystem should contain 0 files.") } } -func TestUnicodeNorm(t *testing.T) { +// TODO(bep) mod fix me; think a little where to apply this. +func _TestUnicodeNorm(t *testing.T) { if runtime.GOOS != "darwin" { // Normalization code is only for Mac OS, since it is not necessary for other OSes. return } + assert := require.New(t) + paths := []struct { NFC string NFD string @@ -47,14 +61,18 @@ func TestUnicodeNorm(t *testing.T) { } ss := newTestSourceSpec() - var fi os.FileInfo + fi := hugofs.NewFileMetaInfo(nil, hugofs.FileMeta{}) - for _, path := range paths { - src := ss.NewFilesystem("base") + for i, path := range paths { + base := fmt.Sprintf("base%d", i) + assert.NoError(afero.WriteFile(ss.Fs.Source, filepath.Join(base, path.NFD), []byte("some data"), 0777)) + src := ss.NewFilesystem(base) _ = src.add(path.NFD, fi) - f := src.Files()[0] + files, err := src.Files() + assert.NoError(err) + f := files[0] if f.BaseFileName() != path.NFC { - t.Fatalf("file name in NFD form should be normalized (%s)", path.NFC) + t.Fatalf("file %q name in NFD form should be normalized (%s)", f.BaseFileName(), path.NFC) } } @@ -70,12 +88,22 @@ func newTestConfig() *viper.Viper { v.Set("resourceDir", "resources") v.Set("publishDir", "public") v.Set("assetDir", "assets") + _, err := langs.LoadLanguageSettings(v, nil) + if err != nil { + panic(err) + } + mod, err := modules.CreateProjectModule(v) + if err != nil { + panic(err) + } + v.Set("allModules", modules.Modules{mod}) + return v } func newTestSourceSpec() *SourceSpec { v := newTestConfig() - fs := hugofs.NewMem(v) + fs := hugofs.NewFrom(hugofs.NewBaseFileDecorator(afero.NewMemMapFs()), v) ps, err := helpers.NewPathSpec(fs, v) if err != nil { panic(err) diff --git a/source/sourceSpec.go b/source/sourceSpec.go index 9731a8d8d4a..504a3a22da3 100644 --- a/source/sourceSpec.go +++ b/source/sourceSpec.go @@ -17,6 +17,7 @@ import ( "os" "path/filepath" "regexp" + "runtime" "github.com/gohugoio/hugo/langs" "github.com/spf13/afero" @@ -107,6 +108,18 @@ func (s *SourceSpec) IgnoreFile(filename string) bool { } } + if runtime.GOOS == "windows" { + // Also check the forward slash variant if different. + unixFilename := filepath.ToSlash(filename) + if unixFilename != filename { + for _, re := range s.ignoreFilesRe { + if re.MatchString(unixFilename) { + return true + } + } + } + } + return false } diff --git a/tpl/cast/docshelper.go b/tpl/cast/docshelper.go index 6fc35f3c72d..1ee614b10c5 100644 --- a/tpl/cast/docshelper.go +++ b/tpl/cast/docshelper.go @@ -17,7 +17,7 @@ import ( "github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/docshelper" - "github.com/gohugoio/hugo/htesting" + "github.com/gohugoio/hugo/resources/page" "github.com/gohugoio/hugo/tpl/internal" "github.com/spf13/viper" ) @@ -30,7 +30,7 @@ func init() { Cfg: viper.New(), Log: loggers.NewErrorLogger(), BuildStartListeners: &deps.Listeners{}, - Site: htesting.NewTestHugoSite(), + Site: page.NewDummyHugoSite(newTestConfig()), } var namespaces internal.TemplateFuncsNamespaces @@ -47,3 +47,9 @@ func init() { docshelper.AddDocProvider("tpl", docsProvider) } + +func newTestConfig() *viper.Viper { + v := viper.New() + v.Set("contentDir", "content") + return v +} diff --git a/tpl/data/init_test.go b/tpl/data/init_test.go index c4751e8925e..94c8408ea6c 100644 --- a/tpl/data/init_test.go +++ b/tpl/data/init_test.go @@ -16,6 +16,7 @@ package data import ( "testing" + "github.com/gohugoio/hugo/langs" "github.com/gohugoio/hugo/tpl/internal" "github.com/spf13/viper" "github.com/stretchr/testify/require" @@ -27,6 +28,7 @@ func TestInit(t *testing.T) { v := viper.New() v.Set("contentDir", "content") + langs.LoadLanguageSettings(v, nil) for _, nsf := range internal.TemplateFuncsNamespaceRegistry { ns = nsf(newDeps(v)) diff --git a/tpl/data/resources_test.go b/tpl/data/resources_test.go index a42232f940b..dc436dce6ae 100644 --- a/tpl/data/resources_test.go +++ b/tpl/data/resources_test.go @@ -23,6 +23,8 @@ import ( "testing" "time" + "github.com/gohugoio/hugo/modules" + "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/cache/filecache" @@ -186,14 +188,19 @@ func newDeps(cfg config.Provider) *deps.Deps { cfg.Set("layoutDir", "layouts") cfg.Set("archetypeDir", "archetypes") - l := langs.NewLanguage("en", cfg) - l.Set("i18nDir", "i18n") - cs, err := helpers.NewContentSpec(l) + langs.LoadLanguageSettings(cfg, nil) + mod, err := modules.CreateProjectModule(cfg) + if err != nil { + panic(err) + } + cfg.Set("allModules", modules.Modules{mod}) + + cs, err := helpers.NewContentSpec(cfg) if err != nil { panic(err) } - fs := hugofs.NewMem(l) + fs := hugofs.NewMem(cfg) logger := loggers.NewErrorLogger() p, err := helpers.NewPathSpec(fs, cfg) diff --git a/tpl/hugo/init_test.go b/tpl/hugo/init_test.go index 128f6fc19f0..f4e31f622cb 100644 --- a/tpl/hugo/init_test.go +++ b/tpl/hugo/init_test.go @@ -16,17 +16,19 @@ package hugo import ( "testing" - "github.com/gohugoio/hugo/htesting" - "github.com/gohugoio/hugo/deps" + "github.com/gohugoio/hugo/resources/page" "github.com/gohugoio/hugo/tpl/internal" + "github.com/spf13/viper" "github.com/stretchr/testify/require" ) func TestInit(t *testing.T) { var found bool var ns *internal.TemplateFuncsNamespace - s := htesting.NewTestHugoSite() + v := viper.New() + v.Set("contentDir", "content") + s := page.NewDummyHugoSite(v) for _, nsf := range internal.TemplateFuncsNamespaceRegistry { ns = nsf(&deps.Deps{Site: s}) diff --git a/tpl/os/os.go b/tpl/os/os.go index 2dab5c4906a..28731e53318 100644 --- a/tpl/os/os.go +++ b/tpl/os/os.go @@ -18,6 +18,7 @@ package os import ( "errors" "fmt" + "os" _os "os" "github.com/gohugoio/hugo/deps" @@ -26,23 +27,21 @@ import ( ) // New returns a new instance of the os-namespaced template functions. -func New(deps *deps.Deps) *Namespace { +func New(d *deps.Deps) *Namespace { - // Since Hugo 0.38 we can have multiple content dirs. This can make it hard to - // reason about where the file is placed relative to the project root. - // To make the {{ readFile .Filename }} variant just work, we create a composite - // filesystem that first checks the work dir fs and then the content fs. var rfs afero.Fs - if deps.Fs != nil { - rfs = deps.Fs.WorkingDir - if deps.PathSpec != nil && deps.PathSpec.BaseFs != nil { - rfs = afero.NewReadOnlyFs(afero.NewCopyOnWriteFs(deps.PathSpec.BaseFs.Content.Fs, deps.Fs.WorkingDir)) + if d.Fs != nil { + // TODO(bep) mod consider afero.NewCopyOnWriteFs(BaseFs.Work.Fs. ...) + rfs = d.Fs.WorkingDir + if d.PathSpec != nil && d.PathSpec.BaseFs != nil { + rfs = afero.NewReadOnlyFs(afero.NewCopyOnWriteFs(d.PathSpec.BaseFs.Content.Fs, d.Fs.WorkingDir)) } + } return &Namespace{ readFileFs: rfs, - deps: deps, + deps: d, } } @@ -76,6 +75,9 @@ func readFile(fs afero.Fs, filename string) (string, error) { return "", fmt.Errorf("file %q is too big", filename) } } else { + if os.IsNotExist(err) { + return "", fmt.Errorf("file %q does not exist", filename) + } return "", err } b, err := afero.ReadFile(fs, filename) @@ -96,6 +98,10 @@ func (ns *Namespace) ReadFile(i interface{}) (string, error) { return "", err } + if ns.deps.PathSpec != nil { + s = ns.deps.PathSpec.RelPathify(s) + } + return readFile(ns.readFileFs, s) } diff --git a/tpl/site/init_test.go b/tpl/site/init_test.go index 00704d94355..5ef8856770e 100644 --- a/tpl/site/init_test.go +++ b/tpl/site/init_test.go @@ -16,8 +16,10 @@ package site import ( "testing" + "github.com/spf13/viper" + "github.com/gohugoio/hugo/deps" - "github.com/gohugoio/hugo/htesting" + "github.com/gohugoio/hugo/resources/page" "github.com/gohugoio/hugo/tpl/internal" "github.com/stretchr/testify/require" ) @@ -25,7 +27,9 @@ import ( func TestInit(t *testing.T) { var found bool var ns *internal.TemplateFuncsNamespace - s := htesting.NewTestHugoSite() + v := viper.New() + v.Set("contentDir", "content") + s := page.NewDummyHugoSite(v) for _, nsf := range internal.TemplateFuncsNamespaceRegistry { ns = nsf(&deps.Deps{Site: s}) diff --git a/tpl/template.go b/tpl/template.go index 93577136407..cd00d8061bc 100644 --- a/tpl/template.go +++ b/tpl/template.go @@ -252,12 +252,15 @@ func (t *TemplateAdapter) fileAndFilename(name string) (afero.File, string, erro if err != nil { return nil, "", err } - f, err := fs.Open(filename) + fim := fi.(hugofs.FileMetaInfo) + meta := fim.Meta() + + f, err := meta.Open() if err != nil { return nil, "", errors.Wrapf(err, "failed to open template file %q:", filename) } - return f, fi.(hugofs.RealFilenameInfo).RealFilename(), nil + return f, meta.Filename(), nil } // ExecuteToString executes the current template and returns the result as a diff --git a/tpl/tplimpl/embedded/templates.autogen.go b/tpl/tplimpl/embedded/templates.autogen.go index bdbc842223a..b2c97de6af5 100644 --- a/tpl/tplimpl/embedded/templates.autogen.go +++ b/tpl/tplimpl/embedded/templates.autogen.go @@ -268,8 +268,7 @@ if (!doNotTrack) { {{ end }} -{{ end }} -`}, +{{ end }}`}, {`schema.html`, ` diff --git a/tpl/tplimpl/template.go b/tpl/tplimpl/template.go index f0d3066e2ec..cce867ac2ba 100644 --- a/tpl/tplimpl/template.go +++ b/tpl/tplimpl/template.go @@ -711,7 +711,7 @@ func (t *templateHandler) RebuildClone() { func (t *templateHandler) loadTemplates(prefix string) error { - walker := func(path string, fi os.FileInfo, err error) error { + walker := func(path string, fi hugofs.FileMetaInfo, err error) error { if err != nil || fi.IsDir() { return err } @@ -928,8 +928,8 @@ func (t *templateHandler) addTemplateFile(name, baseTemplatePath, path string) e realFilename := filename if fi, err := fs.Stat(filename); err == nil { - if fir, ok := fi.(hugofs.RealFilenameInfo); ok { - realFilename = fir.RealFilename() + if fim, ok := fi.(hugofs.FileMetaInfo); ok { + realFilename = fim.Meta().Filename() } } diff --git a/tpl/tplimpl/template_funcs_test.go b/tpl/tplimpl/template_funcs_test.go index 449d20fd4c5..faf5b01fe45 100644 --- a/tpl/tplimpl/template_funcs_test.go +++ b/tpl/tplimpl/template_funcs_test.go @@ -21,7 +21,9 @@ import ( "testing" "time" - "github.com/gohugoio/hugo/htesting" + "github.com/gohugoio/hugo/modules" + + "github.com/gohugoio/hugo/resources/page" "github.com/gohugoio/hugo/common/hugo" "github.com/gohugoio/hugo/common/loggers" @@ -52,6 +54,14 @@ func newTestConfig() config.Provider { v.Set("assetDir", "assets") v.Set("resourceDir", "resources") v.Set("publishDir", "public") + + langs.LoadLanguageSettings(v, nil) + mod, err := modules.CreateProjectModule(v) + if err != nil { + panic(err) + } + v.Set("allModules", modules.Modules{mod}) + return v } @@ -59,7 +69,7 @@ func newDepsConfig(cfg config.Provider) deps.DepsCfg { l := langs.NewLanguage("en", cfg) return deps.DepsCfg{ Language: l, - Site: htesting.NewTestHugoSite(), + Site: page.NewDummyHugoSite(cfg), Cfg: cfg, Fs: hugofs.NewMem(l), Logger: logger,