diff --git a/benchbep.sh b/benchbep.sh index fabd30c189b..efd616c8859 100755 --- a/benchbep.sh +++ b/benchbep.sh @@ -1,2 +1 @@ -gobench -package=./hugolib -bench="BenchmarkSiteBuilding/YAML,num_langs=3,num_pages=5000,tags_per_page=5,shortcodes,render" -count=3 > 1.bench -benchcmp -best 0.bench 1.bench \ No newline at end of file +gobench -package=./hugolib -bench="BenchmarkSiteNew/Deep_content_tree" \ No newline at end of file diff --git a/cache/filecache/filecache.go b/cache/filecache/filecache.go index 6ad417117e0..bf004c8f7b6 100644 --- a/cache/filecache/filecache.go +++ b/cache/filecache/filecache.go @@ -44,6 +44,9 @@ type Cache struct { // 0 is effectively turning this cache off. maxAge time.Duration + // When set, we just remove this entire root directory on expiration. + pruneAllRootDir string + nlocker *lockTracker } @@ -77,11 +80,12 @@ type ItemInfo struct { } // NewCache creates a new file cache with the given filesystem and max age. -func NewCache(fs afero.Fs, maxAge time.Duration) *Cache { +func NewCache(fs afero.Fs, maxAge time.Duration, pruneAllRootDir string) *Cache { return &Cache{ - Fs: fs, - nlocker: &lockTracker{Locker: locker.NewLocker(), seen: make(map[string]struct{})}, - maxAge: maxAge, + Fs: fs, + nlocker: &lockTracker{Locker: locker.NewLocker(), seen: make(map[string]struct{})}, + maxAge: maxAge, + pruneAllRootDir: pruneAllRootDir, } } @@ -307,9 +311,15 @@ func (f Caches) Get(name string) *Cache { // NewCaches creates a new set of file caches from the given // configuration. func NewCaches(p *helpers.PathSpec) (Caches, error) { - dcfg, err := decodeConfig(p) - if err != nil { - return nil, err + var dcfg Configs + if c, ok := p.Cfg.Get("filecacheConfigs").(Configs); ok { + dcfg = c + } else { + var err error + dcfg, err = DecodeConfig(p.Fs.Source, p.Cfg) + if err != nil { + return nil, err + } } fs := p.Fs.Source @@ -319,30 +329,25 @@ func NewCaches(p *helpers.PathSpec) (Caches, error) { var cfs afero.Fs if v.isResourceDir { - cfs = p.BaseFs.Resources.Fs + cfs = p.BaseFs.ResourcesCache } else { cfs = fs } - var baseDir string - if !strings.HasPrefix(v.Dir, "_gen") { - // We do cache eviction (file removes) and since the user can set - // his/hers own cache directory, we really want to make sure - // we do not delete any files that do not belong to this cache. - // We do add the cache name as the root, but this is an extra safe - // guard. We skip the files inside /resources/_gen/ because - // that would be breaking. - baseDir = filepath.Join(v.Dir, filecacheRootDirname, k) - } else { - baseDir = filepath.Join(v.Dir, k) - } - if err = cfs.MkdirAll(baseDir, 0777); err != nil && !os.IsExist(err) { + baseDir := v.Dir + + if err := cfs.MkdirAll(baseDir, 0777); err != nil && !os.IsExist(err) { return nil, err } bfs := afero.NewBasePathFs(cfs, baseDir) - m[k] = NewCache(bfs, v.MaxAge) + var pruneAllRootDir string + if k == cacheKeyModules { + pruneAllRootDir = "pkg" + } + + m[k] = NewCache(bfs, v.MaxAge, pruneAllRootDir) } return m, nil diff --git a/cache/filecache/filecache_config.go b/cache/filecache/filecache_config.go index a6a0252b24b..0c6b569c1e3 100644 --- a/cache/filecache/filecache_config.go +++ b/cache/filecache/filecache_config.go @@ -19,6 +19,8 @@ import ( "strings" "time" + "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/helpers" "github.com/mitchellh/mapstructure" @@ -32,7 +34,7 @@ const ( resourcesGenDir = ":resourceDir/_gen" ) -var defaultCacheConfig = cacheConfig{ +var defaultCacheConfig = Config{ MaxAge: -1, // Never expire Dir: ":cacheDir/:project", } @@ -42,9 +44,20 @@ const ( cacheKeyGetCSV = "getcsv" cacheKeyImages = "images" cacheKeyAssets = "assets" + cacheKeyModules = "modules" ) -var defaultCacheConfigs = map[string]cacheConfig{ +type Configs map[string]Config + +func (c Configs) CacheDirModules() string { + return c[cacheKeyModules].Dir +} + +var defaultCacheConfigs = Configs{ + cacheKeyModules: { + MaxAge: -1, + Dir: ":cacheDir/modules", + }, cacheKeyGetJSON: defaultCacheConfig, cacheKeyGetCSV: defaultCacheConfig, cacheKeyImages: { @@ -57,9 +70,7 @@ var defaultCacheConfigs = map[string]cacheConfig{ }, } -type cachesConfig map[string]cacheConfig - -type cacheConfig struct { +type Config struct { // Max age of cache entries in this cache. Any items older than this will // be removed and not returned from the cache. // a negative value means forever, 0 means cache is disabled. @@ -88,13 +99,18 @@ func (f Caches) ImageCache() *Cache { return f[cacheKeyImages] } +// ModulesCache gets the file cache for Hugo Modules. +func (f Caches) ModulesCache() *Cache { + return f[cacheKeyModules] +} + // AssetsCache gets the file cache for assets (processed resources, SCSS etc.). func (f Caches) AssetsCache() *Cache { return f[cacheKeyAssets] } -func decodeConfig(p *helpers.PathSpec) (cachesConfig, error) { - c := make(cachesConfig) +func DecodeConfig(fs afero.Fs, cfg config.Provider) (Configs, error) { + c := make(Configs) valid := make(map[string]bool) // Add defaults for k, v := range defaultCacheConfigs { @@ -102,11 +118,9 @@ func decodeConfig(p *helpers.PathSpec) (cachesConfig, error) { valid[k] = true } - cfg := p.Cfg - m := cfg.GetStringMap(cachesConfigKey) - _, isOsFs := p.Fs.Source.(*afero.OsFs) + _, isOsFs := fs.(*afero.OsFs) for k, v := range m { cc := defaultCacheConfig @@ -148,7 +162,7 @@ func decodeConfig(p *helpers.PathSpec) (cachesConfig, error) { for i, part := range parts { if strings.HasPrefix(part, ":") { - resolved, isResource, err := resolveDirPlaceholder(p, part) + resolved, isResource, err := resolveDirPlaceholder(fs, cfg, part) if err != nil { return c, err } @@ -176,6 +190,18 @@ func decodeConfig(p *helpers.PathSpec) (cachesConfig, error) { } } + if !strings.HasPrefix(v.Dir, "_gen") { + // We do cache eviction (file removes) and since the user can set + // his/hers own cache directory, we really want to make sure + // we do not delete any files that do not belong to this cache. + // We do add the cache name as the root, but this is an extra safe + // guard. We skip the files inside /resources/_gen/ because + // that would be breaking. + v.Dir = filepath.Join(v.Dir, filecacheRootDirname, k) + } else { + v.Dir = filepath.Join(v.Dir, k) + } + if disabled { v.MaxAge = 0 } @@ -187,15 +213,17 @@ func decodeConfig(p *helpers.PathSpec) (cachesConfig, error) { } // Resolves :resourceDir => /myproject/resources etc., :cacheDir => ... -func resolveDirPlaceholder(p *helpers.PathSpec, placeholder string) (cacheDir string, isResource bool, err error) { +func resolveDirPlaceholder(fs afero.Fs, cfg config.Provider, placeholder string) (cacheDir string, isResource bool, err error) { + workingDir := cfg.GetString("workingDir") + switch strings.ToLower(placeholder) { case ":resourcedir": return "", true, nil case ":cachedir": - d, err := helpers.GetCacheDir(p.Fs.Source, p.Cfg) + d, err := helpers.GetCacheDir(fs, cfg) return d, false, err case ":project": - return filepath.Base(p.WorkingDir), false, nil + return filepath.Base(workingDir), false, nil } return "", false, errors.Errorf("%q is not a valid placeholder (valid values are :cacheDir or :resourceDir)", placeholder) diff --git a/cache/filecache/filecache_config_test.go b/cache/filecache/filecache_config_test.go index b0f5d2dc0b3..f2f75344b60 100644 --- a/cache/filecache/filecache_config_test.go +++ b/cache/filecache/filecache_config_test.go @@ -20,10 +20,9 @@ import ( "testing" "time" - "github.com/gohugoio/hugo/helpers" + "github.com/spf13/afero" "github.com/gohugoio/hugo/config" - "github.com/gohugoio/hugo/hugofs" "github.com/spf13/viper" "github.com/stretchr/testify/require" @@ -57,22 +56,19 @@ dir = "/path/to/c3" cfg, err := config.FromConfigString(configStr, "toml") assert.NoError(err) - fs := hugofs.NewMem(cfg) - p, err := helpers.NewPathSpec(fs, cfg) + fs := afero.NewMemMapFs() + decoded, err := DecodeConfig(fs, cfg) assert.NoError(err) - decoded, err := decodeConfig(p) - assert.NoError(err) - - assert.Equal(4, len(decoded)) + assert.Equal(5, len(decoded)) c2 := decoded["getcsv"] assert.Equal("11h0m0s", c2.MaxAge.String()) - assert.Equal(filepath.FromSlash("/path/to/c2"), c2.Dir) + assert.Equal(filepath.FromSlash("/path/to/c2/filecache/getcsv"), c2.Dir) c3 := decoded["images"] assert.Equal(time.Duration(-1), c3.MaxAge) - assert.Equal(filepath.FromSlash("/path/to/c3"), c3.Dir) + assert.Equal(filepath.FromSlash("/path/to/c3/filecache/images"), c3.Dir) } @@ -105,14 +101,11 @@ dir = "/path/to/c3" cfg, err := config.FromConfigString(configStr, "toml") assert.NoError(err) - fs := hugofs.NewMem(cfg) - p, err := helpers.NewPathSpec(fs, cfg) - assert.NoError(err) - - decoded, err := decodeConfig(p) + fs := afero.NewMemMapFs() + decoded, err := DecodeConfig(fs, cfg) assert.NoError(err) - assert.Equal(4, len(decoded)) + assert.Equal(5, len(decoded)) for _, v := range decoded { assert.Equal(time.Duration(0), v.MaxAge) @@ -133,24 +126,22 @@ func TestDecodeConfigDefault(t *testing.T) { cfg.Set("cacheDir", "/cache/thecache") } - fs := hugofs.NewMem(cfg) - p, err := helpers.NewPathSpec(fs, cfg) - assert.NoError(err) + fs := afero.NewMemMapFs() - decoded, err := decodeConfig(p) + decoded, err := DecodeConfig(fs, cfg) assert.NoError(err) - assert.Equal(4, len(decoded)) + assert.Equal(5, len(decoded)) imgConfig := decoded[cacheKeyImages] jsonConfig := decoded[cacheKeyGetJSON] if runtime.GOOS == "windows" { - assert.Equal("_gen", imgConfig.Dir) + assert.Equal(filepath.FromSlash("_gen/images"), imgConfig.Dir) } else { - assert.Equal("_gen", imgConfig.Dir) - assert.Equal("/cache/thecache/hugoproject", jsonConfig.Dir) + assert.Equal("_gen/images", imgConfig.Dir) + assert.Equal("/cache/thecache/hugoproject/filecache/getjson", jsonConfig.Dir) } assert.True(imgConfig.isResourceDir) @@ -183,11 +174,9 @@ dir = "/" cfg, err := config.FromConfigString(configStr, "toml") assert.NoError(err) - fs := hugofs.NewMem(cfg) - p, err := helpers.NewPathSpec(fs, cfg) - assert.NoError(err) + fs := afero.NewMemMapFs() - _, err = decodeConfig(p) + _, err = DecodeConfig(fs, cfg) assert.Error(err) } diff --git a/cache/filecache/filecache_pruner.go b/cache/filecache/filecache_pruner.go index 322eabf9256..c6fd4497e4c 100644 --- a/cache/filecache/filecache_pruner.go +++ b/cache/filecache/filecache_pruner.go @@ -28,53 +28,100 @@ import ( func (c Caches) Prune() (int, error) { counter := 0 for k, cache := range c { - err := afero.Walk(cache.Fs, "", func(name string, info os.FileInfo, err error) error { - if info == nil { - return nil - } - name = cleanID(name) - - if info.IsDir() { - f, err := cache.Fs.Open(name) - if err != nil { - // This cache dir may not exist. - return nil - } - defer f.Close() - _, err = f.Readdirnames(1) - if err == io.EOF { - // Empty dir. - return cache.Fs.Remove(name) - } + count, err := cache.Prune(false) + + if err != nil { + return counter, errors.Wrapf(err, "failed to prune cache %q", k) + } + + counter += count + + } + + return counter, nil +} + +// Prune removes expired and unused items from this cache. +// If force is set, everything will be removed not considering expiry time. +func (c *Cache) Prune(force bool) (int, error) { + if c.pruneAllRootDir != "" { + return c.pruneRootDir(force) + } + + counter := 0 + + err := afero.Walk(c.Fs, "", func(name string, info os.FileInfo, err error) error { + if info == nil { + return nil + } + + name = cleanID(name) + if info.IsDir() { + f, err := c.Fs.Open(name) + if err != nil { + // This cache dir may not exist. return nil } + defer f.Close() + _, err = f.Readdirnames(1) + if err == io.EOF { + // Empty dir. + return c.Fs.Remove(name) + } + + return nil + } - shouldRemove := cache.isExpired(info.ModTime()) + shouldRemove := force || c.isExpired(info.ModTime()) - if !shouldRemove && len(cache.nlocker.seen) > 0 { - // Remove it if it's not been touched/used in the last build. - _, seen := cache.nlocker.seen[name] - shouldRemove = !seen - } + if !shouldRemove && len(c.nlocker.seen) > 0 { + // Remove it if it's not been touched/used in the last build. + _, seen := c.nlocker.seen[name] + shouldRemove = !seen + } - if shouldRemove { - err := cache.Fs.Remove(name) - if err == nil { - counter++ - } - return err + if shouldRemove { + err := c.Fs.Remove(name) + if err == nil { + counter++ } + return err + } - return nil - }) + return nil + }) - if err != nil { - return counter, errors.Wrapf(err, "failed to prune cache %q", k) + return counter, err +} + +func (c *Cache) pruneRootDir(force bool) (int, error) { + + info, err := c.Fs.Stat(c.pruneAllRootDir) + if err != nil { + if os.IsNotExist(err) { + return 0, nil } + return 0, err + } + if !force && !c.isExpired(info.ModTime()) { + return 0, nil } - return counter, nil + counter := 0 + // Module cache has 0555 directories; make them writable in order to remove content. + afero.Walk(c.Fs, c.pruneAllRootDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return nil + } + if info.IsDir() { + counter++ + c.Fs.Chmod(path, 0777) + } + return nil + }) + return 1, c.Fs.RemoveAll(c.pruneAllRootDir) + } diff --git a/cache/filecache/filecache_pruner_test.go b/cache/filecache/filecache_pruner_test.go index e62a6315a74..72c6781ac2b 100644 --- a/cache/filecache/filecache_pruner_test.go +++ b/cache/filecache/filecache_pruner_test.go @@ -18,9 +18,7 @@ import ( "testing" "time" - "github.com/gohugoio/hugo/config" - "github.com/gohugoio/hugo/helpers" - "github.com/gohugoio/hugo/hugofs" + "github.com/spf13/afero" "github.com/stretchr/testify/require" ) @@ -54,14 +52,9 @@ maxAge = "200ms" dir = ":resourceDir/_gen" ` - cfg, err := config.FromConfigString(configStr, "toml") - assert.NoError(err) - for _, name := range []string{cacheKeyGetCSV, cacheKeyGetJSON, cacheKeyAssets, cacheKeyImages} { msg := fmt.Sprintf("cache: %s", name) - fs := hugofs.NewMem(cfg) - p, err := helpers.NewPathSpec(fs, cfg) - assert.NoError(err) + p := newPathsSpec(t, afero.NewMemMapFs(), configStr) caches, err := NewCaches(p) assert.NoError(err) cache := caches[name] diff --git a/cache/filecache/filecache_test.go b/cache/filecache/filecache_test.go index 5ac2e9beb81..a03c3116a61 100644 --- a/cache/filecache/filecache_test.go +++ b/cache/filecache/filecache_test.go @@ -25,6 +25,9 @@ import ( "testing" "time" + "github.com/gohugoio/hugo/langs" + "github.com/gohugoio/hugo/modules" + "github.com/gohugoio/hugo/common/hugio" "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/helpers" @@ -83,12 +86,7 @@ dir = ":cacheDir/c" configStr = replacer.Replace(configStr) configStr = strings.Replace(configStr, "\\", winPathSep, -1) - cfg, err := config.FromConfigString(configStr, "toml") - assert.NoError(err) - - fs := hugofs.NewFrom(osfs, cfg) - p, err := helpers.NewPathSpec(fs, cfg) - assert.NoError(err) + p := newPathsSpec(t, osfs, configStr) caches, err := NewCaches(p) assert.NoError(err) @@ -207,11 +205,7 @@ dir = "/cache/c" ` - cfg, err := config.FromConfigString(configStr, "toml") - assert.NoError(err) - fs := hugofs.NewMem(cfg) - p, err := helpers.NewPathSpec(fs, cfg) - assert.NoError(err) + p := newPathsSpec(t, afero.NewMemMapFs(), configStr) caches, err := NewCaches(p) assert.NoError(err) @@ -255,3 +249,51 @@ func TestCleanID(t *testing.T) { assert.Equal(filepath.FromSlash("a/b/c.txt"), cleanID(filepath.FromSlash("/a/b//c.txt"))) assert.Equal(filepath.FromSlash("a/b/c.txt"), cleanID(filepath.FromSlash("a/b//c.txt"))) } + +func initConfig(fs afero.Fs, cfg config.Provider) error { + if _, err := langs.LoadLanguageSettings(cfg, nil); err != nil { + return err + } + + modConfig, err := modules.DecodeConfig(cfg) + if err != nil { + return err + } + + workingDir := cfg.GetString("workingDir") + themesDir := cfg.GetString("themesDir") + if !filepath.IsAbs(themesDir) { + themesDir = filepath.Join(workingDir, themesDir) + } + modulesClient := modules.NewClient(modules.ClientConfig{ + Fs: fs, + WorkingDir: workingDir, + ThemesDir: themesDir, + ModuleConfig: modConfig, + IgnoreVendor: true, + }) + + moduleConfig, err := modulesClient.Collect() + if err != nil { + return err + } + + if err := modules.ApplyProjectConfigDefaults(cfg, moduleConfig.ActiveModules[len(moduleConfig.ActiveModules)-1]); err != nil { + return err + } + + cfg.Set("allModules", moduleConfig.ActiveModules) + + return nil +} + +func newPathsSpec(t *testing.T, fs afero.Fs, configStr string) *helpers.PathSpec { + assert := require.New(t) + cfg, err := config.FromConfigString(configStr, "toml") + assert.NoError(err) + initConfig(fs, cfg) + p, err := helpers.NewPathSpec(hugofs.NewFrom(fs, cfg), cfg) + assert.NoError(err) + return p + +} diff --git a/commands/commandeer.go b/commands/commandeer.go index 8c9da53b92b..cf60d12f476 100644 --- a/commands/commandeer.go +++ b/commands/commandeer.go @@ -16,6 +16,11 @@ package commands import ( "bytes" "errors" + "sync" + + "golang.org/x/sync/semaphore" + + "github.com/gohugoio/hugo/modules" "io/ioutil" @@ -27,8 +32,6 @@ import ( "os" "path/filepath" "regexp" - "strings" - "sync" "time" "github.com/gohugoio/hugo/common/loggers" @@ -88,6 +91,8 @@ type commandeer struct { configured bool paused bool + fullRebuildSem *semaphore.Weighted + // Any error from the last build. buildErr error } @@ -153,6 +158,7 @@ func newCommandeer(mustHaveConfigFile, running bool, h *hugoBuilderCommon, f fla doWithCommandeer: doWithCommandeer, visitedURLs: types.NewEvictingStringQueue(10), debounce: rebuildDebouncer, + fullRebuildSem: semaphore.NewWeighted(1), // This will be replaced later, but we need something to log to before the configuration is read. logger: loggers.NewLogger(jww.LevelError, jww.LevelError, os.Stdout, ioutil.Discard, running), } @@ -290,7 +296,7 @@ func (c *commandeer) loadConfig(mustHaveConfigFile, running bool) error { if mustHaveConfigFile { return err } - if err != hugolib.ErrNoConfigFile { + if err != hugolib.ErrNoConfigFile && !modules.IsNotExist(err) { return err } @@ -388,21 +394,6 @@ func (c *commandeer) loadConfig(mustHaveConfigFile, running bool) error { cfg.Logger.INFO.Println("Using config file:", config.ConfigFileUsed()) - themeDir := c.hugo.PathSpec.GetFirstThemeDir() - if themeDir != "" { - if _, err := sourceFs.Stat(themeDir); os.IsNotExist(err) { - return newSystemError("Unable to find theme Directory:", themeDir) - } - } - - dir, themeVersionMismatch, minVersion := c.isThemeVsHugoVersionMismatch(sourceFs) - - if themeVersionMismatch { - name := filepath.Base(dir) - cfg.Logger.ERROR.Printf("%s theme does not support Hugo version %s. Minimum version required is %s\n", - strings.ToUpper(name), hugo.CurrentVersion.ReleaseVersion(), minVersion) - } - return nil } diff --git a/commands/commands.go b/commands/commands.go index 51bfb47635f..3096e1fe023 100644 --- a/commands/commands.go +++ b/commands/commands.go @@ -54,6 +54,7 @@ func (b *commandsBuilder) addAll() *commandsBuilder { newImportCmd(), newGenCmd(), createReleaser(), + b.newModCmd(), ) return b @@ -243,20 +244,26 @@ func (cc *hugoBuilderCommon) getEnvironment(isServer bool) string { return hugo.EnvironmentProduction } +func (cc *hugoBuilderCommon) handleCommonBuilderFlags(cmd *cobra.Command) { + cmd.PersistentFlags().StringVarP(&cc.source, "source", "s", "", "filesystem path to read files relative from") + cmd.PersistentFlags().SetAnnotation("source", cobra.BashCompSubdirsInDir, []string{}) + cmd.PersistentFlags().StringVarP(&cc.environment, "environment", "e", "", "build environment") + cmd.PersistentFlags().StringP("themesDir", "", "", "filesystem path to themes directory") + cmd.PersistentFlags().BoolP("ignoreVendor", "", false, "ignores any _vendor directory") +} + func (cc *hugoBuilderCommon) handleFlags(cmd *cobra.Command) { + cc.handleCommonBuilderFlags(cmd) cmd.Flags().Bool("cleanDestinationDir", false, "remove files from destination not found in static directories") cmd.Flags().BoolP("buildDrafts", "D", false, "include content marked as draft") cmd.Flags().BoolP("buildFuture", "F", false, "include content with publishdate in the future") cmd.Flags().BoolP("buildExpired", "E", false, "include expired content") - cmd.Flags().StringVarP(&cc.source, "source", "s", "", "filesystem path to read files relative from") - cmd.Flags().StringVarP(&cc.environment, "environment", "e", "", "build environment") cmd.Flags().StringP("contentDir", "c", "", "filesystem path to content directory") cmd.Flags().StringP("layoutDir", "l", "", "filesystem path to layout directory") cmd.Flags().StringP("cacheDir", "", "", "filesystem path to cache directory. Defaults: $TMPDIR/hugo_cache/") cmd.Flags().BoolP("ignoreCache", "", false, "ignores the cache directory") cmd.Flags().StringP("destination", "d", "", "filesystem path to write files to") cmd.Flags().StringSliceP("theme", "t", []string{}, "themes to use (located in /themes/THEMENAME/)") - cmd.Flags().StringP("themesDir", "", "", "filesystem path to themes directory") cmd.Flags().StringVarP(&cc.baseURL, "baseURL", "b", "", "hostname (and path) to the root, e.g. http://spf13.com/") cmd.Flags().Bool("enableGitInfo", false, "add Git revision, date and author info to the pages") cmd.Flags().BoolVar(&cc.gc, "gc", false, "enable to run some cleanup tasks (remove unused cache files) after the build") diff --git a/commands/convert.go b/commands/convert.go index d0a46a6417b..e4ff1ac61c5 100644 --- a/commands/convert.go +++ b/commands/convert.go @@ -30,7 +30,6 @@ import ( "github.com/gohugoio/hugo/parser/metadecoders" "github.com/gohugoio/hugo/parser/pageparser" - src "github.com/gohugoio/hugo/source" "github.com/pkg/errors" "github.com/gohugoio/hugo/hugolib" @@ -152,8 +151,8 @@ func (cc *convertCmd) convertAndSavePage(p page.Page, site *hugolib.Site, target site.Log.INFO.Println("Attempting to convert", p.File().Filename()) - f, _ := p.File().(src.ReadableFile) - file, err := f.Open() + f := p.File() + file, err := f.FileInfo().Meta().Open() if err != nil { site.Log.ERROR.Println(errMsg) file.Close() diff --git a/commands/hugo.go b/commands/hugo.go index 07f2b95a2bb..36d3812ebc2 100644 --- a/commands/hugo.go +++ b/commands/hugo.go @@ -16,19 +16,18 @@ package commands import ( + "context" "fmt" "io/ioutil" "os/signal" "runtime/pprof" "runtime/trace" - "sort" "sync/atomic" "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/resources/page" - "github.com/gohugoio/hugo/common/hugo" "github.com/pkg/errors" "github.com/gohugoio/hugo/common/herrors" @@ -49,7 +48,6 @@ import ( "github.com/gohugoio/hugo/config" - "github.com/gohugoio/hugo/parser/metadecoders" flag "github.com/spf13/pflag" "github.com/fsnotify/fsnotify" @@ -196,6 +194,7 @@ func initializeFlags(cmd *cobra.Command, cfg config.Provider) { "forceSyncStatic", "noTimes", "noChmod", + "ignoreVendor", "templateMetrics", "templateMetricsHints", @@ -291,6 +290,7 @@ func ifTerminal(s string) string { } func (c *commandeer) fullBuild() error { + var ( g errgroup.Group langCount map[string]uint64 @@ -309,13 +309,9 @@ func (c *commandeer) fullBuild() error { cnt, err := c.copyStatic() if err != nil { - if !os.IsNotExist(err) { - return errors.Wrap(err, "Error copying static files") - } - c.logger.INFO.Println("No Static directory found") + return errors.Wrap(err, "Error copying static files") } langCount = cnt - langCount = cnt return nil } buildSitesFunc := func() error { @@ -503,7 +499,11 @@ func (c *commandeer) build() error { if err != nil { return err } - c.logger.FEEDBACK.Println("Watching for changes in", c.hugo.PathSpec.AbsPathify(c.Cfg.GetString("contentDir"))) + + baseWatchDir := c.Cfg.GetString("workingDir") + rootWatchDirs := getRootWatchDirsStr(baseWatchDir, watchDirs) + + c.logger.FEEDBACK.Printf("Watching for changes in %s%s{%s}\n", baseWatchDir, helpers.FilePathSeparator, rootWatchDirs) c.logger.FEEDBACK.Println("Press Ctrl+C to stop") watcher, err := c.newWatcher(watchDirs...) checkErr(c.Logger, err) @@ -547,7 +547,11 @@ func (c *commandeer) serverBuild() error { } func (c *commandeer) copyStatic() (map[string]uint64, error) { - return c.doWithPublishDirs(c.copyStaticTo) + m, err := c.doWithPublishDirs(c.copyStaticTo) + if err == nil || os.IsNotExist(err) { + return m, nil + } + return m, err } func (c *commandeer) doWithPublishDirs(f func(sourceFs *filesystems.SourceFilesystem) (uint64, error)) (map[string]uint64, error) { @@ -566,6 +570,7 @@ func (c *commandeer) doWithPublishDirs(f func(sourceFs *filesystems.SourceFilesy if err != nil { return langCount, err } + if lang == "" { // Not multihost for _, l := range c.languages { @@ -594,6 +599,16 @@ func (fs *countingStatFs) Stat(name string) (os.FileInfo, error) { return f, err } +func chmodFilter(dst, src os.FileInfo) bool { + // Hugo publishes data from multiple sources, potentially + // with overlapping directory structures. We cannot sync permissions + // for directories as that would mean that we might end up with write-protected + // directories inside /public. + // One example of this would be syncing from the Go Module cache, + // which have 0555 directories. + return src.IsDir() +} + func (c *commandeer) copyStaticTo(sourceFs *filesystems.SourceFilesystem) (uint64, error) { publishDir := c.hugo.PathSpec.PublishDir // If root, remove the second '/' @@ -610,6 +625,7 @@ func (c *commandeer) copyStaticTo(sourceFs *filesystems.SourceFilesystem) (uint6 syncer := fsync.NewSyncer() syncer.NoTimes = c.Cfg.GetBool("noTimes") syncer.NoChmod = c.Cfg.GetBool("noChmod") + syncer.ChmodFilter = chmodFilter syncer.SrcFs = fs syncer.DestFs = c.Fs.Destination // Now that we are using a unionFs for the static directories @@ -652,120 +668,39 @@ func (c *commandeer) timeTrack(start time.Time, name string) { // getDirList provides NewWatcher() with a list of directories to watch for changes. func (c *commandeer) getDirList() ([]string, error) { - var a []string - - // To handle nested symlinked content dirs - var seen = make(map[string]bool) - var nested []string - - newWalker := func(allowSymbolicDirs bool) func(path string, fi os.FileInfo, err error) error { - return func(path string, fi os.FileInfo, err error) error { - if err != nil { - if os.IsNotExist(err) { - return nil - } - - c.logger.ERROR.Println("Walker: ", err) - return nil - } + var dirnames []string - // Skip .git directories. - // Related to https://github.com/gohugoio/hugo/issues/3468. - if fi.Name() == ".git" { - return nil - } - - if fi.Mode()&os.ModeSymlink == os.ModeSymlink { - link, err := filepath.EvalSymlinks(path) - if err != nil { - c.logger.ERROR.Printf("Cannot read symbolic link '%s', error was: %s", path, err) - return nil - } - linkfi, err := helpers.LstatIfPossible(c.Fs.Source, link) - if err != nil { - c.logger.ERROR.Printf("Cannot stat %q: %s", link, err) - return nil - } - if !allowSymbolicDirs && !linkfi.Mode().IsRegular() { - c.logger.ERROR.Printf("Symbolic links for directories not supported, skipping %q", path) - return nil - } - - if allowSymbolicDirs && linkfi.IsDir() { - // afero.Walk will not walk symbolic links, so wee need to do it. - if !seen[path] { - seen[path] = true - nested = append(nested, path) - } - return nil - } - - fi = linkfi - } - - if fi.IsDir() { - if fi.Name() == ".git" || - fi.Name() == "node_modules" || fi.Name() == "bower_components" { - return filepath.SkipDir - } - a = append(a, path) - } + walkFn := func(path string, fi hugofs.FileMetaInfo, err error) error { + if err != nil { + c.logger.ERROR.Println("walker: ", err) return nil } - } - - symLinkWalker := newWalker(true) - regularWalker := newWalker(false) - - // SymbolicWalk will log anny ERRORs - // Also note that the Dirnames fetched below will contain any relevant theme - // directories. - for _, contentDir := range c.hugo.PathSpec.BaseFs.Content.Dirnames { - _ = helpers.SymbolicWalk(c.Fs.Source, contentDir, symLinkWalker) - } - - for _, staticDir := range c.hugo.PathSpec.BaseFs.Data.Dirnames { - _ = helpers.SymbolicWalk(c.Fs.Source, staticDir, regularWalker) - } - for _, staticDir := range c.hugo.PathSpec.BaseFs.I18n.Dirnames { - _ = helpers.SymbolicWalk(c.Fs.Source, staticDir, regularWalker) - } - - for _, staticDir := range c.hugo.PathSpec.BaseFs.Layouts.Dirnames { - _ = helpers.SymbolicWalk(c.Fs.Source, staticDir, regularWalker) - } + if fi.IsDir() { + if fi.Name() == ".git" || + fi.Name() == "node_modules" || fi.Name() == "bower_components" { + return filepath.SkipDir + } - for _, staticFilesystem := range c.hugo.PathSpec.BaseFs.Static { - for _, staticDir := range staticFilesystem.Dirnames { - _ = helpers.SymbolicWalk(c.Fs.Source, staticDir, regularWalker) + dirnames = append(dirnames, fi.Meta().Filename()) } - } - for _, assetDir := range c.hugo.PathSpec.BaseFs.Assets.Dirnames { - _ = helpers.SymbolicWalk(c.Fs.Source, assetDir, regularWalker) - } - - if len(nested) > 0 { - for { + return nil - toWalk := nested - nested = nested[:0] + } - for _, d := range toWalk { - _ = helpers.SymbolicWalk(c.Fs.Source, d, symLinkWalker) - } + watchDirs := c.hugo.PathSpec.BaseFs.WatchDirs() + for _, watchDir := range watchDirs { - if len(nested) == 0 { - break - } + w := hugofs.NewWalkway(hugofs.WalkwayConfig{Logger: c.logger, Info: watchDir, WalkFn: walkFn}) + if err := w.Walk(); err != nil { + c.logger.ERROR.Println("walker: ", err) } } - a = helpers.UniqueStrings(a) - sort.Strings(a) + dirnames = helpers.UniqueStringsSorted(dirnames) - return a, nil + return dirnames, nil } func (c *commandeer) buildSites() (err error) { @@ -812,26 +747,60 @@ func (c *commandeer) partialReRender(urls ...string) error { return c.hugo.Build(hugolib.BuildCfg{RecentlyVisited: visited, PartialReRender: true}) } -func (c *commandeer) fullRebuild() { - c.commandeerHugoState = &commandeerHugoState{} - err := c.loadConfig(true, true) - if err != nil { - // Set the processing on pause until the state is recovered. - c.paused = true - c.handleBuildErr(err, "Failed to reload config") - - } else { - c.paused = false +func (c *commandeer) fullRebuild(changeType string) { + if changeType == configChangeGoMod { + // go.mod may be changed during the build itself, and + // we really want to prevent superfluous builds. + if !c.fullRebuildSem.TryAcquire(1) { + return + } + c.fullRebuildSem.Release(1) } - if !c.paused { - err := c.buildSites() + c.fullRebuildSem.Acquire(context.Background(), 1) + + go func() { + + defer c.fullRebuildSem.Release(1) + + c.printChangeDetected(changeType) + + defer func() { + + // Allow any file system events to arrive back. + // This will block any rebuild on config changes for the + // duration of the sleep. + time.Sleep(2 * time.Second) + }() + + defer c.timeTrack(time.Now(), "Total") + + c.commandeerHugoState = &commandeerHugoState{} + err := c.loadConfig(true, true) if err != nil { - c.logger.ERROR.Println(err) - } else if !c.h.buildWatch && !c.Cfg.GetBool("disableLiveReload") { - livereload.ForceRefresh() + // Set the processing on pause until the state is recovered. + c.paused = true + c.handleBuildErr(err, "Failed to reload config") + + } else { + c.paused = false } - } + + if !c.paused { + _, err := c.copyStatic() + if err != nil { + c.logger.ERROR.Println(err) + return + } + + err = c.buildSites() + if err != nil { + c.logger.ERROR.Println(err) + } else if !c.h.buildWatch && !c.Cfg.GetBool("disableLiveReload") { + livereload.ForceRefresh() + } + } + }() } // newWatcher creates a new watcher to watch filesystem events. @@ -886,26 +855,53 @@ func (c *commandeer) newWatcher(dirList ...string) (*watcher.Batcher, error) { return watcher, nil } +func (c *commandeer) printChangeDetected(typ string) { + msg := "\nChange" + if typ != "" { + msg += " of " + typ + } + msg += " detected, rebuilding site." + + c.logger.FEEDBACK.Println(msg) + const layout = "2006-01-02 15:04:05.000 -0700" + c.logger.FEEDBACK.Println(time.Now().Format(layout)) +} + +const ( + configChangeConfig = "config file" + configChangeGoMod = "go.mod file" +) + func (c *commandeer) handleEvents(watcher *watcher.Batcher, staticSyncer *staticSyncer, evs []fsnotify.Event, configSet map[string]bool) { + var isHandled bool + for _, ev := range evs { isConfig := configSet[ev.Name] + configChangeType := configChangeConfig + if isConfig { + if strings.Contains(ev.Name, "go.mod") { + configChangeType = configChangeGoMod + } + } if !isConfig { // It may be one of the /config folders dirname := filepath.Dir(ev.Name) if dirname != "." && configSet[dirname] { isConfig = true } - } if isConfig { + isHandled = true + if ev.Op&fsnotify.Chmod == fsnotify.Chmod { continue } + if ev.Op&fsnotify.Remove == fsnotify.Remove || ev.Op&fsnotify.Rename == fsnotify.Rename { for _, configFile := range c.configFiles { counter := 0 @@ -917,13 +913,20 @@ func (c *commandeer) handleEvents(watcher *watcher.Batcher, time.Sleep(100 * time.Millisecond) } } + + // A write event will follow. + continue } + // Config file(s) changed. Need full rebuild. - c.fullRebuild() - break + c.fullRebuild(configChangeType) } } + if isHandled { + return + } + if c.paused { // Wait for the server to get into a consistent state before // we continue with processing. @@ -933,7 +936,9 @@ func (c *commandeer) handleEvents(watcher *watcher.Batcher, if len(evs) > 50 { // This is probably a mass edit of the content dir. // Schedule a full rebuild for when it slows down. - c.debounce(c.fullRebuild) + c.debounce(func() { + c.fullRebuild("") + }) return } @@ -1015,7 +1020,7 @@ func (c *commandeer) handleEvents(watcher *watcher.Batcher, continue } - walkAdder := func(path string, f os.FileInfo, err error) error { + walkAdder := func(path string, f hugofs.FileMetaInfo, err error) error { if f.IsDir() { c.logger.FEEDBACK.Println("adding created directory to watchlist", path) if err := watcher.Add(path); err != nil { @@ -1046,9 +1051,7 @@ func (c *commandeer) handleEvents(watcher *watcher.Batcher, } if len(staticEvents) > 0 { - c.logger.FEEDBACK.Println("\nStatic file changes detected") - const layout = "2006-01-02 15:04:05.000 -0700" - c.logger.FEEDBACK.Println(time.Now().Format(layout)) + c.printChangeDetected("Static files") if c.Cfg.GetBool("forceSyncStatic") { c.logger.FEEDBACK.Printf("Syncing all static files\n") @@ -1087,10 +1090,7 @@ func (c *commandeer) handleEvents(watcher *watcher.Batcher, doLiveReload := !c.h.buildWatch && !c.Cfg.GetBool("disableLiveReload") onePageName := pickOneWriteOrCreatePath(partitionedEvents.ContentEvents) - c.logger.FEEDBACK.Println("\nChange detected, rebuilding site") - const layout = "2006-01-02 15:04:05.000 -0700" - c.logger.FEEDBACK.Println(time.Now().Format(layout)) - + c.printChangeDetected("") c.changeDetector.PrepareNew() if err := c.rebuildSites(dynamicEvents); err != nil { c.handleBuildErr(err, "Rebuild failed") @@ -1167,41 +1167,3 @@ func pickOneWriteOrCreatePath(events []fsnotify.Event) string { return name } - -// isThemeVsHugoVersionMismatch returns whether the current Hugo version is -// less than any of the themes' min_version. -func (c *commandeer) isThemeVsHugoVersionMismatch(fs afero.Fs) (dir string, mismatch bool, requiredMinVersion string) { - if !c.hugo.PathSpec.ThemeSet() { - return - } - - for _, absThemeDir := range c.hugo.BaseFs.AbsThemeDirs { - - path := filepath.Join(absThemeDir, "theme.toml") - - exists, err := helpers.Exists(path, fs) - - if err != nil || !exists { - continue - } - - b, err := afero.ReadFile(fs, path) - if err != nil { - continue - } - - tomlMeta, err := metadecoders.Default.UnmarshalToMap(b, metadecoders.TOML) - if err != nil { - continue - } - - if minVersion, ok := tomlMeta["min_version"]; ok { - if hugo.CompareVersion(minVersion) > 0 { - return absThemeDir, true, fmt.Sprint(minVersion) - } - } - - } - - return -} diff --git a/commands/import_jekyll.go b/commands/import_jekyll.go index 1d37cfd9d3a..e5c39dc3418 100644 --- a/commands/import_jekyll.go +++ b/commands/import_jekyll.go @@ -17,7 +17,6 @@ import ( "bytes" "errors" "fmt" - "io" "io/ioutil" "os" "path/filepath" @@ -27,6 +26,8 @@ import ( "time" "unicode" + "github.com/gohugoio/hugo/common/hugio" + "github.com/gohugoio/hugo/parser/metadecoders" "github.com/gohugoio/hugo/helpers" @@ -113,7 +114,7 @@ func (i *importCmd) importFromJekyll(cmd *cobra.Command, args []string) error { jww.FEEDBACK.Println("Importing...") fileCount := 0 - callback := func(path string, fi os.FileInfo, err error) error { + callback := func(path string, fi hugofs.FileMetaInfo, err error) error { if err != nil { return err } @@ -302,66 +303,10 @@ func (i *importCmd) createConfigFromJekyll(fs afero.Fs, inpath string, kind meta return helpers.WriteToDisk(filepath.Join(inpath, "config."+string(kind)), &buf, fs) } -func copyFile(source string, dest string) error { - sf, err := os.Open(source) - if err != nil { - return err - } - defer sf.Close() - df, err := os.Create(dest) - if err != nil { - return err - } - defer df.Close() - _, err = io.Copy(df, sf) - if err == nil { - si, err := os.Stat(source) - if err != nil { - err = os.Chmod(dest, si.Mode()) - - if err != nil { - return err - } - } - - } - return nil -} - -func copyDir(source string, dest string) error { - fi, err := os.Stat(source) - if err != nil { - return err - } - if !fi.IsDir() { - return errors.New(source + " is not a directory") - } - err = os.MkdirAll(dest, fi.Mode()) - if err != nil { - return err - } - entries, _ := ioutil.ReadDir(source) - for _, entry := range entries { - sfp := filepath.Join(source, entry.Name()) - dfp := filepath.Join(dest, entry.Name()) - if entry.IsDir() { - err = copyDir(sfp, dfp) - if err != nil { - jww.ERROR.Println(err) - } - } else { - err = copyFile(sfp, dfp) - if err != nil { - jww.ERROR.Println(err) - } - } - - } - return nil -} - func (i *importCmd) copyJekyllFilesAndFolders(jekyllRoot, dest string, jekyllPostDirs map[string]bool) (err error) { - fi, err := os.Stat(jekyllRoot) + fs := hugofs.Os + + fi, err := fs.Stat(jekyllRoot) if err != nil { return err } @@ -383,7 +328,7 @@ func (i *importCmd) copyJekyllFilesAndFolders(jekyllRoot, dest string, jekyllPos if entry.IsDir() { if entry.Name()[0] != '_' && entry.Name()[0] != '.' { if _, ok := jekyllPostDirs[entry.Name()]; !ok { - err = copyDir(sfp, dfp) + err = hugio.CopyDir(fs, sfp, dfp, nil) if err != nil { jww.ERROR.Println(err) } @@ -402,7 +347,7 @@ func (i *importCmd) copyJekyllFilesAndFolders(jekyllRoot, dest string, jekyllPos } if !isExcept && entry.Name()[0] != '.' && entry.Name()[0] != '_' { - err = copyFile(sfp, dfp) + err = hugio.CopyFile(fs, sfp, dfp) if err != nil { jww.ERROR.Println(err) } diff --git a/commands/list_test.go b/commands/list_test.go index f2ce7001091..55a0674444b 100644 --- a/commands/list_test.go +++ b/commands/list_test.go @@ -62,6 +62,7 @@ func TestListAll(t *testing.T) { }, header) record, err := r.Read() + assert.NoError(err) assert.Equal([]string{ filepath.Join("content", "p1.md"), "", "P1", "0001-01-01T00:00:00Z", "0001-01-01T00:00:00Z", "0001-01-01T00:00:00Z", diff --git a/commands/mod.go b/commands/mod.go new file mode 100644 index 00000000000..861704e35b7 --- /dev/null +++ b/commands/mod.go @@ -0,0 +1,189 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package commands + +import ( + "os" + "strings" + + "github.com/gohugoio/hugo/modules" + "github.com/spf13/cobra" +) + +var _ cmder = (*modCmd)(nil) + +type modCmd struct { + *baseBuilderCmd +} + +func (b *commandsBuilder) newModCmd() *modCmd { + c := &modCmd{} + + const commonUsage = ` +Note that Hugo will always start out by resolving the components defined in the site +configuration, provided by a _vendor directory (if no --ignoreVendor flag provided), +Go Modules, or a folder inside the themes directory, in that order. + +See https://gohugo.io/hugo-modules/ for more information. + +` + + cmd := &cobra.Command{ + Use: "mod", + Short: "Various Hugo Modules helpers.", + Long: `Various helpers to help manage the modules in your project's dependency graph. + +Most operations here requires a Go version installed on your system (>= Go 1.12) and the relevant VCS client (typically Git). +This is not needed if you only operate on modules inside /themes or if you have vendored them via "hugo mod vendor". + +` + commonUsage, + + RunE: nil, + } + + cmd.AddCommand( + &cobra.Command{ + Use: "get", + DisableFlagParsing: true, + Short: "Resolves dependencies in your current Hugo Project.", + Long: ` +Resolves dependencies in your current Hugo Project. + +Some examples: + +Install the latest version possible for a given module: + + hugo mod get github.com/gohugoio/testshortcodes + +Install a specific version: + + hugo mod get github.com/gohugoio/testshortcodes@v0.3.0 + +Install the latest versions of all module dependencies: + + hugo mod get -u + +Run "go help get" for more information. All flags available for "go get" is also relevant here. +` + commonUsage, + RunE: func(cmd *cobra.Command, args []string) error { + return c.withModsClient(false, func(c *modules.Client) error { + // We currently just pass on the flags we get to Go and + // need to do the flag handling manually. + if len(args) == 1 && strings.Contains(args[0], "-h") { + return cmd.Help() + } + return c.Get(args...) + }) + }, + }, + &cobra.Command{ + Use: "graph", + Short: "Print a module dependency graph.", + Long: `Print a module dependency graph with information about module status (disabled, vendored). +Note that for vendored modules, that is the version listed and not the one from go.mod. +`, + RunE: func(cmd *cobra.Command, args []string) error { + return c.withModsClient(true, func(c *modules.Client) error { + return c.Graph(os.Stdout) + }) + }, + }, + &cobra.Command{ + Use: "init", + Short: "Initialize this project as a Hugo Module.", + Long: `Initialize this project as a Hugo Module. +It will try to guess the module path, but you may help by passing it as an argument, e.g: + + hugo mod init github.com/gohugoio/testshortcodes + +Note that Hugo Modules supports multi-module projects, so you can initialize a Hugo Module +inside a subfolder on GitHub, as one example. +`, + RunE: func(cmd *cobra.Command, args []string) error { + var path string + if len(args) >= 1 { + path = args[0] + } + return c.withModsClient(false, func(c *modules.Client) error { + return c.Init(path) + }) + }, + }, + &cobra.Command{ + Use: "vendor", + Short: "Vendor all module dependencies into the _vendor directory.", + Long: `Vendor all module dependencies into the _vendor directory. + +If a module is vendored, that is where Hugo will look for it's dependencies. +`, + RunE: func(cmd *cobra.Command, args []string) error { + return c.withModsClient(true, func(c *modules.Client) error { + return c.Vendor() + }) + }, + }, + &cobra.Command{ + Use: "tidy", + Short: "Remove unused entries in go.mod and go.sum.", + RunE: func(cmd *cobra.Command, args []string) error { + return c.withModsClient(true, func(c *modules.Client) error { + return c.Tidy() + }) + }, + }, + &cobra.Command{ + Use: "clean", + Short: "Delete the entire Hugo Module cache.", + Long: `Delete the entire Hugo Module cache. + +Note that after you run this command, all of your dependencies will be re-downloaded next time you run "hugo". + +Also note that if you configure a positive maxAge for the "modules" file cache, it will also be cleaned as part of "hugo --gc". + +`, + RunE: func(cmd *cobra.Command, args []string) error { + com, err := c.initConfig(true) + if err != nil { + return err + } + + _, err = com.hugo.FileCaches.ModulesCache().Prune(true) + return err + + }, + }, + ) + + c.baseBuilderCmd = b.newBuilderCmd(cmd) + + return c + +} + +func (c *modCmd) withModsClient(failOnMissingConfig bool, f func(*modules.Client) error) error { + com, err := c.initConfig(failOnMissingConfig) + if err != nil { + return err + } + + return f(com.hugo.ModulesClient) +} + +func (c *modCmd) initConfig(failOnNoConfig bool) (*commandeer, error) { + com, err := initializeConfig(failOnNoConfig, false, &c.hugoBuilderCommon, c, nil) + if err != nil { + return nil, err + } + return com, nil +} diff --git a/commands/new.go b/commands/new.go index f10369837da..649be521e3f 100644 --- a/commands/new.go +++ b/commands/new.go @@ -53,8 +53,6 @@ Ensure you run this within the root directory of your site.`, cc := &newCmd{baseBuilderCmd: b.newBuilderCmd(cmd)} cmd.Flags().StringVarP(&cc.contentType, "kind", "k", "", "content type to create") - cmd.PersistentFlags().StringVarP(&cc.source, "source", "s", "", "filesystem path to read files relative from") - cmd.PersistentFlags().SetAnnotation("source", cobra.BashCompSubdirsInDir, []string{}) cmd.Flags().StringVar(&cc.contentEditor, "editor", "", "edit new content with this editor, if provided") cmd.AddCommand(newNewSiteCmd().getCommand()) @@ -120,8 +118,8 @@ func newContentPathSection(h *hugolib.HugoSites, path string) (string, string) { createpath := filepath.FromSlash(path) if h != nil { - for _, s := range h.Sites { - createpath = strings.TrimPrefix(createpath, s.PathSpec.ContentDir) + for _, dir := range h.BaseFs.Content.Dirs { + createpath = strings.TrimPrefix(createpath, dir.Meta().Filename()) } } diff --git a/commands/server.go b/commands/server.go index 5d50ebe2cf0..ade8da5e89c 100644 --- a/commands/server.go +++ b/commands/server.go @@ -256,15 +256,11 @@ func (sc *serverCmd) server(cmd *cobra.Command, args []string) error { return err } - baseWatchDir := c.Cfg.GetString("workingDir") - relWatchDirs := make([]string, len(watchDirs)) - for i, dir := range watchDirs { - relWatchDirs[i], _ = helpers.GetRelativePath(dir, baseWatchDir) - } - - rootWatchDirs := strings.Join(helpers.UniqueStrings(helpers.ExtractRootPaths(relWatchDirs)), ",") + watchGroups := helpers.ExtractAndGroupRootPaths(watchDirs) - jww.FEEDBACK.Printf("Watching for changes in %s%s{%s}\n", baseWatchDir, helpers.FilePathSeparator, rootWatchDirs) + for _, group := range watchGroups { + jww.FEEDBACK.Printf("Watching for changes in %s\n", group) + } watcher, err := c.newWatcher(watchDirs...) if err != nil { @@ -279,6 +275,15 @@ func (sc *serverCmd) server(cmd *cobra.Command, args []string) error { } +func getRootWatchDirsStr(baseDir string, watchDirs []string) string { + relWatchDirs := make([]string, len(watchDirs)) + for i, dir := range watchDirs { + relWatchDirs[i], _ = helpers.GetRelativePath(dir, baseDir) + } + + return strings.Join(helpers.UniqueStringsSorted(helpers.ExtractRootPaths(relWatchDirs)), ",") +} + type fileServer struct { baseURLs []string roots []string diff --git a/commands/static_syncer.go b/commands/static_syncer.go index 2374538683f..6103fe71c65 100644 --- a/commands/static_syncer.go +++ b/commands/static_syncer.go @@ -53,6 +53,7 @@ func (s *staticSyncer) syncsStaticEvents(staticEvents []fsnotify.Event) error { syncer := fsync.NewSyncer() syncer.NoTimes = c.Cfg.GetBool("noTimes") syncer.NoChmod = c.Cfg.GetBool("noChmod") + syncer.ChmodFilter = chmodFilter syncer.SrcFs = sourceFs.Fs syncer.DestFs = c.Fs.Destination diff --git a/common/herrors/errors.go b/common/herrors/errors.go index be98ceb39b6..1a61070501a 100644 --- a/common/herrors/errors.go +++ b/common/herrors/errors.go @@ -19,6 +19,7 @@ import ( "fmt" "io" "os" + "runtime/debug" _errors "github.com/pkg/errors" ) @@ -46,6 +47,16 @@ func FprintStackTrace(w io.Writer, err error) { } } +// Recover is a helper function that can be used to capture panics. +// Put this at the top of a method/function that crashes in a template: +// defer herrors.Recover() +func Recover() { + if r := recover(); r != nil { + fmt.Println("stacktrace from panic: \n" + string(debug.Stack())) + } + +} + // ErrFeatureNotAvailable denotes that a feature is unavailable. // // We will, at least to begin with, make some Hugo features (SCSS with libsass) optional, diff --git a/common/hugio/copy.go b/common/hugio/copy.go new file mode 100644 index 00000000000..2b756cb4428 --- /dev/null +++ b/common/hugio/copy.go @@ -0,0 +1,90 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugio + +import ( + "io" + "io/ioutil" + "os" + "path/filepath" + + "github.com/pkg/errors" + + "github.com/spf13/afero" +) + +// CopyFile copies a file. +func CopyFile(fs afero.Fs, from, to string) error { + sf, err := os.Open(from) + if err != nil { + return err + } + defer sf.Close() + df, err := os.Create(to) + if err != nil { + return err + } + defer df.Close() + _, err = io.Copy(df, sf) + if err == nil { + si, err := os.Stat(from) + if err != nil { + err = os.Chmod(to, si.Mode()) + + if err != nil { + return err + } + } + + } + return nil +} + +// CopyDir copies a directory. +func CopyDir(fs afero.Fs, from, to string, shouldCopy func(filename string) bool) error { + fi, err := os.Stat(from) + if err != nil { + return err + } + + if !fi.IsDir() { + return errors.Errorf("%q is not a directory", from) + } + + err = fs.MkdirAll(to, 0777) // before umask + if err != nil { + return err + } + + entries, _ := ioutil.ReadDir(from) + for _, entry := range entries { + fromFilename := filepath.Join(from, entry.Name()) + toFilename := filepath.Join(to, entry.Name()) + if entry.IsDir() { + if shouldCopy != nil && !shouldCopy(fromFilename) { + continue + } + if err := CopyDir(fs, fromFilename, toFilename, shouldCopy); err != nil { + return err + } + } else { + if err := CopyFile(fs, fromFilename, toFilename); err != nil { + return err + } + } + + } + + return nil +} diff --git a/common/hugo/vars_extended.go b/common/hugo/vars_extended.go index 20683b8042c..bb96bade6c5 100644 --- a/common/hugo/vars_extended.go +++ b/common/hugo/vars_extended.go @@ -15,4 +15,4 @@ package hugo -var isExtended = true +var IsExtended = true diff --git a/common/hugo/vars_regular.go b/common/hugo/vars_regular.go index e1ece83fbb9..fae18df14d3 100644 --- a/common/hugo/vars_regular.go +++ b/common/hugo/vars_regular.go @@ -15,4 +15,4 @@ package hugo -var isExtended = false +var IsExtended = false diff --git a/common/hugo/version.go b/common/hugo/version.go index 47641f10c09..848393f970d 100644 --- a/common/hugo/version.go +++ b/common/hugo/version.go @@ -15,6 +15,7 @@ package hugo import ( "fmt" + "strconv" "runtime" "strings" @@ -133,7 +134,7 @@ func BuildVersionString() string { if commitHash != "" { version += "-" + strings.ToUpper(commitHash) } - if isExtended { + if IsExtended { version += "/extended" } @@ -235,3 +236,16 @@ func compareFloatVersions(version float32, v float32) int { } return 1 } + +func GoMinorVersion() int { + return goMinorVersion(runtime.Version()) +} + +func goMinorVersion(version string) int { + if strings.HasPrefix(version, "devel") { + return 9999 // magic + } + i, _ := strconv.Atoi(strings.Split(version, ".")[1]) + return i + +} diff --git a/common/hugo/version_test.go b/common/hugo/version_test.go index 08059189e09..e2aeeabbf88 100644 --- a/common/hugo/version_test.go +++ b/common/hugo/version_test.go @@ -77,3 +77,9 @@ func TestParseHugoVersion(t *testing.T) { require.Equal(t, "0.25-DEV", MustParseVersion("0.25-DEV").String()) } + +func TestGoMinorVersion(t *testing.T) { + assert := require.New(t) + assert.Equal(12, goMinorVersion("go1.12.5")) + assert.True(GoMinorVersion() >= 11) +} diff --git a/common/loggers/loggers.go b/common/loggers/loggers.go index e711de46898..082fd148789 100644 --- a/common/loggers/loggers.go +++ b/common/loggers/loggers.go @@ -40,6 +40,7 @@ func init() { type Logger struct { *jww.Notepad ErrorCounter *jww.Counter + WarnCounter *jww.Counter // This is only set in server mode. errors *bytes.Buffer @@ -143,9 +144,10 @@ func getLogWriters(outHandle, logHandle io.Writer) (io.Writer, io.Writer) { func newLogger(stdoutThreshold, logThreshold jww.Threshold, outHandle, logHandle io.Writer, saveErrors bool) *Logger { errorCounter := &jww.Counter{} + warnCounter := &jww.Counter{} outHandle, logHandle = getLogWriters(outHandle, logHandle) - listeners := []jww.LogListener{jww.LogCounter(errorCounter, jww.LevelError)} + listeners := []jww.LogListener{jww.LogCounter(errorCounter, jww.LevelError), jww.LogCounter(warnCounter, jww.LevelWarn)} var errorBuff *bytes.Buffer if saveErrors { errorBuff = new(bytes.Buffer) @@ -164,6 +166,7 @@ func newLogger(stdoutThreshold, logThreshold jww.Threshold, outHandle, logHandle return &Logger{ Notepad: jww.NewNotepad(stdoutThreshold, logThreshold, outHandle, logHandle, "", log.Ldate|log.Ltime, listeners...), ErrorCounter: errorCounter, + WarnCounter: warnCounter, errors: errorBuff, } } diff --git a/common/maps/params.go b/common/maps/params.go new file mode 100644 index 00000000000..0b81057b148 --- /dev/null +++ b/common/maps/params.go @@ -0,0 +1,96 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package maps + +import ( + "strings" + + "github.com/pkg/errors" + + "github.com/spf13/cast" +) + +// GetNestedParam gets the first match of the keyStr in the candidates given. +// It will first try the exact match and then try to find it as a nested map value, +// using the given separator, e.g. "mymap.name". +// It assumes that all the maps given have lower cased keys. +func GetNestedParam(keyStr, separator string, candidates ...map[string]interface{}) (interface{}, error) { + keyStr = strings.ToLower(keyStr) + + lookupFn := func(key string) interface{} { + for _, m := range candidates { + if v, ok := m[key]; ok { + return v + } + } + + return nil + } + + v, _, _, err := GetNestedParamFn(keyStr, separator, lookupFn) + return v, err +} + +func GetNestedParamFn(keyStr, separator string, lookupFn func(key string) interface{}) (interface{}, string, map[string]interface{}, error) { + result, _ := traverseDirectParams(keyStr, lookupFn) + if result != nil { + return result, keyStr, nil, nil + } + + keySegments := strings.Split(keyStr, separator) + if len(keySegments) == 1 { + return nil, keyStr, nil, nil + } + + return traverseNestedParams(keySegments, lookupFn) +} + +func traverseDirectParams(keyStr string, lookupFn func(key string) interface{}) (interface{}, error) { + return lookupFn(keyStr), nil +} + +func traverseNestedParams(keySegments []string, lookupFn func(key string) interface{}) (interface{}, string, map[string]interface{}, error) { + firstKey, rest := keySegments[0], keySegments[1:] + result := lookupFn(firstKey) + if result == nil || len(rest) == 0 { + return result, firstKey, nil, nil + } + + switch m := result.(type) { + case map[string]interface{}: + v, key, owner := traverseParams(rest, m) + return v, key, owner, nil + default: + return nil, "", nil, errors.Errorf("unsupported Params type: %T", result) + } +} + +func traverseParams(keys []string, m map[string]interface{}) (interface{}, string, map[string]interface{}) { + // Shift first element off. + firstKey, rest := keys[0], keys[1:] + result := m[firstKey] + + // No point in continuing here. + if result == nil { + return result, "", nil + } + + if len(rest) == 0 { + // That was the last key. + return result, firstKey, m + } + + // That was not the last key. + return traverseParams(rest, cast.ToStringMap(result)) +} diff --git a/common/maps/params_test.go b/common/maps/params_test.go new file mode 100644 index 00000000000..89b149617c5 --- /dev/null +++ b/common/maps/params_test.go @@ -0,0 +1,45 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package maps + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestGetNestedParam(t *testing.T) { + + m := map[string]interface{}{ + "first": 1, + "with_underscore": 2, + "nested": map[string]interface{}{ + "color": "blue", + }, + } + + assert := require.New(t) + + must := func(keyStr, separator string, candidates ...map[string]interface{}) interface{} { + v, err := GetNestedParam(keyStr, separator, candidates...) + assert.NoError(err) + return v + } + + assert.Equal(1, must("first", "_", m)) + assert.Equal(1, must("First", "_", m)) + assert.Equal(2, must("with_underscore", "_", m)) + assert.Equal("blue", must("nested_color", "_", m)) + +} diff --git a/config/configLoader.go b/config/configLoader.go index b8aa3fda38b..2e37a5b3559 100644 --- a/config/configLoader.go +++ b/config/configLoader.go @@ -120,8 +120,6 @@ func RenameKeys(m map[string]interface{}) { func newViper() *viper.Viper { v := viper.New() - v.AutomaticEnv() - v.SetEnvPrefix("hugo") return v } diff --git a/config/configProvider.go b/config/configProvider.go index 31914c38bc5..187fb7b1064 100644 --- a/config/configProvider.go +++ b/config/configProvider.go @@ -35,10 +35,14 @@ type Provider interface { // we do not attempt to split it into fields. func GetStringSlicePreserveString(cfg Provider, key string) []string { sd := cfg.Get(key) - if sds, ok := sd.(string); ok { + return toStringSlicePreserveString(sd) +} + +func toStringSlicePreserveString(v interface{}) []string { + if sds, ok := v.(string); ok { return []string{sds} } - return cast.ToStringSlice(sd) + return cast.ToStringSlice(v) } // SetBaseTestDefaults provides some common config defaults used in tests. diff --git a/config/env.go b/config/env.go index adf6f9b6811..f482cd24733 100644 --- a/config/env.go +++ b/config/env.go @@ -17,6 +17,7 @@ import ( "os" "runtime" "strconv" + "strings" ) // GetNumWorkerMultiplier returns the base value used to calculate the number @@ -31,3 +32,26 @@ func GetNumWorkerMultiplier() int { } return runtime.NumCPU() } + +// SetEnvVars sets vars on the form key=value in the oldVars slice. +func SetEnvVars(oldVars *[]string, keyValues ...string) { + for i := 0; i < len(keyValues); i += 2 { + setEnvVar(oldVars, keyValues[i], keyValues[i+1]) + } +} + +func SplitEnvVar(v string) (string, string) { + parts := strings.Split(v, "=") + return parts[0], parts[1] +} + +func setEnvVar(vars *[]string, key, value string) { + for i := range *vars { + if strings.HasPrefix((*vars)[i], key+"=") { + (*vars)[i] = key + "=" + value + return + } + } + // New var. + *vars = append(*vars, key+"="+value) +} diff --git a/hugofs/nolstat_fs.go b/config/env_test.go similarity index 53% rename from hugofs/nolstat_fs.go rename to config/env_test.go index 6b27e8e1f64..594c3e871b3 100644 --- a/hugofs/nolstat_fs.go +++ b/config/env_test.go @@ -1,4 +1,4 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. +// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,29 +11,22 @@ // See the License for the specific language governing permissions and // limitations under the License. -package hugofs +package config import ( - "os" + "testing" - "github.com/spf13/afero" + "github.com/stretchr/testify/require" ) -var ( - _ afero.Fs = (*noLstatFs)(nil) -) - -type noLstatFs struct { - afero.Fs -} - -// NewNoLstatFs creates a new filesystem with no Lstat support. -func NewNoLstatFs(fs afero.Fs) afero.Fs { - return &noLstatFs{Fs: fs} -} +func TestSetEnvVars(t *testing.T) { + t.Parallel() + assert := require.New(t) + vars := []string{"FOO=bar", "HUGO=cool", "BAR=foo"} + SetEnvVars(&vars, "HUGO", "rocking!", "NEW", "bar") + assert.Equal([]string{"FOO=bar", "HUGO=rocking!", "BAR=foo", "NEW=bar"}, vars) -// LstatIfPossible always delegates to Stat. -func (fs *noLstatFs) LstatIfPossible(name string) (os.FileInfo, bool, error) { - fi, err := fs.Stat(name) - return fi, false, err + key, val := SplitEnvVar("HUGO=rocks") + assert.Equal("HUGO", key) + assert.Equal("rocks", val) } diff --git a/create/content.go b/create/content.go index e48dfc078bb..0e05adf93de 100644 --- a/create/content.go +++ b/create/content.go @@ -25,6 +25,8 @@ import ( "path/filepath" "strings" + "github.com/gohugoio/hugo/hugofs/files" + "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/helpers" @@ -50,7 +52,10 @@ func NewContent( if isDir { - langFs := hugofs.NewLanguageFs(s.Language().Lang, sites.LanguageSet(), archetypeFs) + langFs, err := hugofs.NewLanguageFs(sites.LanguageSet(), archetypeFs) + if err != nil { + return err + } cm, err := mapArcheTypeDir(ps, langFs, archetypeFilename) if err != nil { @@ -64,7 +69,7 @@ func NewContent( } name := filepath.Base(targetPath) - return newContentFromDir(archetypeFilename, sites, archetypeFs, sourceFs, cm, name, contentPath) + return newContentFromDir(archetypeFilename, sites, sourceFs, cm, name, contentPath) } // Building the sites can be expensive, so only do it if really needed. @@ -111,9 +116,9 @@ func NewContent( return nil } -func targetSite(sites *hugolib.HugoSites, fi *hugofs.LanguageFileInfo) *hugolib.Site { +func targetSite(sites *hugolib.HugoSites, fi hugofs.FileMetaInfo) *hugolib.Site { for _, s := range sites.Sites { - if fi.Lang() == s.Language().Lang { + if fi.Meta().Lang() == s.Language().Lang { return s } } @@ -123,13 +128,14 @@ func targetSite(sites *hugolib.HugoSites, fi *hugofs.LanguageFileInfo) *hugolib. func newContentFromDir( archetypeDir string, sites *hugolib.HugoSites, - sourceFs, targetFs afero.Fs, + targetFs afero.Fs, cm archetypeMap, name, targetPath string) error { for _, f := range cm.otherFiles { - filename := f.Filename() + meta := f.Meta() + filename := meta.Path() // Just copy the file to destination. - in, err := sourceFs.Open(filename) + in, err := meta.Open() if err != nil { return errors.Wrap(err, "failed to open non-content file") } @@ -156,7 +162,7 @@ func newContentFromDir( } for _, f := range cm.contentFiles { - filename := f.Filename() + filename := f.Meta().Path() s := targetSite(sites, f) targetFilename := filepath.Join(targetPath, strings.TrimPrefix(filename, archetypeDir)) @@ -177,9 +183,9 @@ func newContentFromDir( type archetypeMap struct { // These needs to be parsed and executed as Go templates. - contentFiles []*hugofs.LanguageFileInfo + contentFiles []hugofs.FileMetaInfo // These are just copied to destination. - otherFiles []*hugofs.LanguageFileInfo + otherFiles []hugofs.FileMetaInfo // If the templates needs a fully built site. This can potentially be // expensive, so only do when needed. siteUsed bool @@ -192,7 +198,7 @@ func mapArcheTypeDir( var m archetypeMap - walkFn := func(filename string, fi os.FileInfo, err error) error { + walkFn := func(path string, fi hugofs.FileMetaInfo, err error) error { if err != nil { return err @@ -202,12 +208,12 @@ func mapArcheTypeDir( return nil } - fil := fi.(*hugofs.LanguageFileInfo) + fil := fi.(hugofs.FileMetaInfo) - if hugolib.IsContentFile(filename) { + if files.IsContentFile(path) { m.contentFiles = append(m.contentFiles, fil) if !m.siteUsed { - m.siteUsed, err = usesSiteVar(fs, filename) + m.siteUsed, err = usesSiteVar(fs, path) if err != nil { return err } @@ -220,7 +226,15 @@ func mapArcheTypeDir( return nil } - if err := helpers.SymbolicWalk(fs, archetypeDir, walkFn); err != nil { + walkCfg := hugofs.WalkwayConfig{ + WalkFn: walkFn, + Fs: fs, + Root: archetypeDir, + } + + w := hugofs.NewWalkway(walkCfg) + + if err := w.Walk(); err != nil { return m, errors.Wrapf(err, "failed to walk archetype dir %q", archetypeDir) } @@ -254,17 +268,29 @@ func resolveContentPath(sites *hugolib.HugoSites, fs afero.Fs, targetPath string } } - for _, ss := range sites.Sites { - contentDir := ss.PathSpec.ContentDir + var dirLang string + + for _, dir := range sites.BaseFs.Content.Dirs { + meta := dir.Meta() + contentDir := meta.Filename() + if !strings.HasSuffix(contentDir, helpers.FilePathSeparator) { contentDir += helpers.FilePathSeparator } + if strings.HasPrefix(targetPath, contentDir) { - siteContentDir = ss.PathSpec.ContentDir - if s == nil { + siteContentDir = contentDir + dirLang = meta.Lang() + break + } + } + + if s == nil && dirLang != "" { + for _, ss := range sites.Sites { + if ss.Lang() == dirLang { s = ss + break } - break } } @@ -280,12 +306,22 @@ func resolveContentPath(sites *hugolib.HugoSites, fs afero.Fs, targetPath string } } + if siteContentDir == "" { + + } + if siteContentDir != "" { pp := filepath.Join(siteContentDir, strings.TrimPrefix(targetPath, siteContentDir)) return s.PathSpec.AbsPathify(pp), s - } else { - return s.PathSpec.AbsPathify(filepath.Join(first.PathSpec.ContentDir, targetPath)), s + var contentDir string + for _, dir := range sites.BaseFs.Content.Dirs { + contentDir = dir.Meta().Filename() + if dir.Meta().Lang() == s.Lang() { + break + } + } + return s.PathSpec.AbsPathify(filepath.Join(contentDir, targetPath)), s } } diff --git a/create/content_template_handler.go b/create/content_template_handler.go index 5a8b4f63cbf..1576fabdb4e 100644 --- a/create/content_template_handler.go +++ b/create/content_template_handler.go @@ -90,7 +90,10 @@ func executeArcheTypeAsTemplate(s *hugolib.Site, name, kind, targetPath, archety err error ) - f := s.SourceSpec.NewFileInfo("", targetPath, false, nil) + f, err := s.SourceSpec.NewFileInfoFrom(targetPath, targetPath) + if err != nil { + return nil, err + } if name == "" { name = f.TranslationBaseName() diff --git a/create/content_test.go b/create/content_test.go index e321900bcf0..6a3852d291b 100644 --- a/create/content_test.go +++ b/create/content_test.go @@ -35,7 +35,6 @@ import ( ) func TestNewContent(t *testing.T) { - assert := require.New(t) cases := []struct { kind string @@ -50,12 +49,12 @@ func TestNewContent(t *testing.T) { {"product", "product/sample-4.md", []string{`title = "SAMPLE-4"`}}, // empty archetype front matter {"lang", "post/lang-1.md", []string{`Site Lang: en|Name: Lang 1|i18n: Hugo Rocks!`}}, {"lang", "post/lang-2.en.md", []string{`Site Lang: en|Name: Lang 2|i18n: Hugo Rocks!`}}, - {"lang", "post/lang-3.nn.md", []string{`Site Lang: nn|Name: Lang 3|i18n: Hugo Rokkar!`}}, + {"lang", "content/post/lang-3.nn.md", []string{`Site Lang: nn|Name: Lang 3|i18n: Hugo Rokkar!`}}, {"lang", "content_nn/post/lang-4.md", []string{`Site Lang: nn|Name: Lang 4|i18n: Hugo Rokkar!`}}, {"lang", "content_nn/post/lang-5.en.md", []string{`Site Lang: en|Name: Lang 5|i18n: Hugo Rocks!`}}, {"lang", "post/my-bundle/index.md", []string{`Site Lang: en|Name: My Bundle|i18n: Hugo Rocks!`}}, {"lang", "post/my-bundle/index.en.md", []string{`Site Lang: en|Name: My Bundle|i18n: Hugo Rocks!`}}, - {"lang", "post/my-bundle/index.nn.md", []string{`Site Lang: nn|Name: My Bundle|i18n: Hugo Rokkar!`}}, + {"lang", "content/post/my-bundle/index.nn.md", []string{`Site Lang: nn|Name: My Bundle|i18n: Hugo Rokkar!`}}, {"shortcodes", "shortcodes/go.md", []string{ `title = "GO"`, "{{< myshortcode >}}", @@ -64,37 +63,43 @@ func TestNewContent(t *testing.T) { } for i, c := range cases { - cfg, fs := newTestCfg(assert) - assert.NoError(initFs(fs)) - h, err := hugolib.NewHugoSites(deps.DepsCfg{Cfg: cfg, Fs: fs}) - assert.NoError(err) - - assert.NoError(create.NewContent(h, c.kind, c.path)) - - fname := filepath.FromSlash(c.path) - if !strings.HasPrefix(fname, "content") { - fname = filepath.Join("content", fname) - } - content := readFileFromFs(t, fs.Source, fname) - for _, v := range c.expected { - found := strings.Contains(content, v) - if !found { - t.Fatalf("[%d] %q missing from output:\n%q", i, v, content) + c := c + t.Run(fmt.Sprintf("%s-%d", c.kind, i), func(t *testing.T) { + t.Parallel() + assert := require.New(t) + mm := afero.NewMemMapFs() + assert.NoError(initFs(mm)) + cfg, fs := newTestCfg(assert, mm) + h, err := hugolib.NewHugoSites(deps.DepsCfg{Cfg: cfg, Fs: fs}) + assert.NoError(err) + + assert.NoError(create.NewContent(h, c.kind, c.path)) + + fname := filepath.FromSlash(c.path) + if !strings.HasPrefix(fname, "content") { + fname = filepath.Join("content", fname) } - } + content := readFileFromFs(t, fs.Source, fname) + for _, v := range c.expected { + found := strings.Contains(content, v) + if !found { + t.Fatalf("[%d] %q missing from output:\n%q", i, v, content) + } + } + }) + } } func TestNewContentFromDir(t *testing.T) { + mm := afero.NewMemMapFs() assert := require.New(t) - cfg, fs := newTestCfg(assert) - assert.NoError(initFs(fs)) archetypeDir := filepath.Join("archetypes", "my-bundle") - assert.NoError(fs.Source.Mkdir(archetypeDir, 0755)) + assert.NoError(mm.MkdirAll(archetypeDir, 0755)) archetypeThemeDir := filepath.Join("themes", "mytheme", "archetypes", "my-theme-bundle") - assert.NoError(fs.Source.Mkdir(archetypeThemeDir, 0755)) + assert.NoError(mm.MkdirAll(archetypeThemeDir, 0755)) contentFile := ` File: %s @@ -103,15 +108,18 @@ Name: {{ replace .Name "-" " " | title }} i18n: {{ T "hugo" }} ` - assert.NoError(afero.WriteFile(fs.Source, filepath.Join(archetypeDir, "index.md"), []byte(fmt.Sprintf(contentFile, "index.md")), 0755)) - assert.NoError(afero.WriteFile(fs.Source, filepath.Join(archetypeDir, "index.nn.md"), []byte(fmt.Sprintf(contentFile, "index.nn.md")), 0755)) + assert.NoError(afero.WriteFile(mm, filepath.Join(archetypeDir, "index.md"), []byte(fmt.Sprintf(contentFile, "index.md")), 0755)) + assert.NoError(afero.WriteFile(mm, filepath.Join(archetypeDir, "index.nn.md"), []byte(fmt.Sprintf(contentFile, "index.nn.md")), 0755)) - assert.NoError(afero.WriteFile(fs.Source, filepath.Join(archetypeDir, "pages", "bio.md"), []byte(fmt.Sprintf(contentFile, "bio.md")), 0755)) - assert.NoError(afero.WriteFile(fs.Source, filepath.Join(archetypeDir, "resources", "hugo1.json"), []byte(`hugo1: {{ printf "no template handling in here" }}`), 0755)) - assert.NoError(afero.WriteFile(fs.Source, filepath.Join(archetypeDir, "resources", "hugo2.xml"), []byte(`hugo2: {{ printf "no template handling in here" }}`), 0755)) + assert.NoError(afero.WriteFile(mm, filepath.Join(archetypeDir, "pages", "bio.md"), []byte(fmt.Sprintf(contentFile, "bio.md")), 0755)) + assert.NoError(afero.WriteFile(mm, filepath.Join(archetypeDir, "resources", "hugo1.json"), []byte(`hugo1: {{ printf "no template handling in here" }}`), 0755)) + assert.NoError(afero.WriteFile(mm, filepath.Join(archetypeDir, "resources", "hugo2.xml"), []byte(`hugo2: {{ printf "no template handling in here" }}`), 0755)) - assert.NoError(afero.WriteFile(fs.Source, filepath.Join(archetypeThemeDir, "index.md"), []byte(fmt.Sprintf(contentFile, "index.md")), 0755)) - assert.NoError(afero.WriteFile(fs.Source, filepath.Join(archetypeThemeDir, "resources", "hugo1.json"), []byte(`hugo1: {{ printf "no template handling in here" }}`), 0755)) + assert.NoError(afero.WriteFile(mm, filepath.Join(archetypeThemeDir, "index.md"), []byte(fmt.Sprintf(contentFile, "index.md")), 0755)) + assert.NoError(afero.WriteFile(mm, filepath.Join(archetypeThemeDir, "resources", "hugo1.json"), []byte(`hugo1: {{ printf "no template handling in here" }}`), 0755)) + + assert.NoError(initFs(mm)) + cfg, fs := newTestCfg(assert, mm) h, err := hugolib.NewHugoSites(deps.DepsCfg{Cfg: cfg, Fs: fs}) assert.NoError(err) @@ -135,7 +143,7 @@ i18n: {{ T "hugo" }} } -func initFs(fs *hugofs.Fs) error { +func initFs(fs afero.Fs) error { perm := os.FileMode(0755) var err error @@ -146,8 +154,8 @@ func initFs(fs *hugofs.Fs) error { filepath.Join("themes", "sample", "archetypes"), } for _, dir := range dirs { - err = fs.Source.Mkdir(dir, perm) - if err != nil { + err = fs.Mkdir(dir, perm) + if err != nil && !os.IsExist(err) { return err } } @@ -198,7 +206,7 @@ Some text. `, }, } { - f, err := fs.Source.Create(v.path) + f, err := fs.Create(v.path) if err != nil { return err } @@ -221,6 +229,7 @@ func assertContains(assert *require.Assertions, v interface{}, matches ...string // TODO(bep) extract common testing package with this and some others func readFileFromFs(t *testing.T, fs afero.Fs, filename string) string { + t.Helper() filename = filepath.FromSlash(filename) b, err := afero.ReadFile(fs, filename) if err != nil { @@ -238,12 +247,11 @@ func readFileFromFs(t *testing.T, fs afero.Fs, filename string) string { return string(b) } -func newTestCfg(assert *require.Assertions) (*viper.Viper, *hugofs.Fs) { +func newTestCfg(assert *require.Assertions, mm afero.Fs) (*viper.Viper, *hugofs.Fs) { cfg := ` theme = "mytheme" - [languages] [languages.en] weight = 1 @@ -254,8 +262,13 @@ languageName = "Nynorsk" contentDir = "content_nn" ` + if mm == nil { + mm = afero.NewMemMapFs() + } - mm := afero.NewMemMapFs() + mm.MkdirAll(filepath.FromSlash("content_nn"), 0777) + + mm.MkdirAll(filepath.FromSlash("themes/mytheme"), 0777) assert.NoError(afero.WriteFile(mm, filepath.Join("i18n", "en.toml"), []byte(`[hugo] other = "Hugo Rocks!"`), 0755)) diff --git a/deps/deps.go b/deps/deps.go index fa62fe5ae54..8ef015ac95b 100644 --- a/deps/deps.go +++ b/deps/deps.go @@ -159,11 +159,11 @@ func (d *Deps) TemplateHandler() tpl.TemplateHandler { func (d *Deps) LoadResources() error { // Note that the translations need to be loaded before the templates. if err := d.translationProvider.Update(d); err != nil { - return err + return errors.Wrap(err, "loading translations") } if err := d.templateProvider.Update(d); err != nil { - return err + return errors.Wrap(err, "loading templates") } return nil @@ -210,7 +210,7 @@ func New(cfg DepsCfg) (*Deps, error) { ps, err := helpers.NewPathSpec(fs, cfg.Language) if err != nil { - return nil, err + return nil, errors.Wrap(err, "create PathSpec") } fileCaches, err := filecache.NewCaches(ps) diff --git a/go.mod b/go.mod index 249f74015ae..e834dc1d40d 100644 --- a/go.mod +++ b/go.mod @@ -12,41 +12,43 @@ require ( github.com/bep/debounce v1.2.0 github.com/bep/gitmap v1.1.0 github.com/bep/go-tocss v0.6.0 - github.com/cpuguy83/go-md2man v1.0.8 // indirect github.com/disintegration/imaging v1.6.0 github.com/dustin/go-humanize v1.0.0 github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 github.com/fortytw2/leaktest v1.3.0 github.com/fsnotify/fsnotify v1.4.7 + github.com/go-errors/errors v1.0.1 github.com/gobwas/glob v0.2.3 + github.com/gohugoio/testmodBuilder/mods v0.0.0-20190520184928-c56af20f2e95 github.com/google/go-cmp v0.3.0 github.com/gorilla/websocket v1.4.0 github.com/hashicorp/go-immutable-radix v1.0.0 github.com/hashicorp/go-uuid v1.0.1 // indirect - github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jdkato/prose v1.1.0 github.com/kyokomi/emoji v1.5.1 github.com/magefile/mage v1.4.0 github.com/magiconair/properties v1.8.1 // indirect github.com/markbates/inflect v1.0.0 github.com/mattn/go-isatty v0.0.8 - github.com/mattn/go-runewidth v0.0.3 // indirect github.com/miekg/mmark v1.3.6 github.com/mitchellh/hashstructure v1.0.0 github.com/mitchellh/mapstructure v1.1.2 github.com/muesli/smartcrop v0.0.0-20180228075044-f6ebaa786a12 + github.com/ncw/rclone v1.48.0 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 // indirect github.com/nicksnyder/go-i18n v1.10.0 github.com/niklasfasching/go-org v0.1.1 github.com/olekukonko/tablewriter v0.0.0-20180506121414-d4647c9c7a84 + github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 github.com/pelletier/go-toml v1.4.0 // indirect github.com/pkg/errors v0.8.1 + github.com/rogpeppe/go-internal v1.3.0 github.com/russross/blackfriday v1.5.3-0.20190124082335-a477dd164691 github.com/sanity-io/litter v1.1.0 github.com/spf13/afero v1.2.2 github.com/spf13/cast v1.3.0 - github.com/spf13/cobra v0.0.3 - github.com/spf13/fsync v0.0.0-20170320142552-12a01e648f05 + github.com/spf13/cobra v0.0.4-0.20190321000552-67fc4837d267 + github.com/spf13/fsync v0.9.0 github.com/spf13/jwalterweatherman v1.1.0 github.com/spf13/pflag v1.0.3 github.com/spf13/viper v1.4.0 @@ -67,3 +69,5 @@ require ( ) replace github.com/markbates/inflect => github.com/markbates/inflect v0.0.0-20171215194931-a12c3aec81a6 + +go 1.13 diff --git a/go.sum b/go.sum index 5c862510209..5a1786cde3d 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,4 @@ +bazil.org/fuse v0.0.0-20180421153158-65cc252bf669/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.37.4/go.mod h1:NHPJ89PdicEuT9hdPXMROBD91xc5uRDxsMtSB16k7hw= @@ -19,6 +20,7 @@ github.com/Azure/azure-sdk-for-go v27.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9mo github.com/Azure/azure-service-bus-go v0.4.1/go.mod h1:d9ho9e/06euiTwGpKxmlbpPhFUsfCsq6a4tZ68r51qI= github.com/Azure/azure-storage-blob-go v0.6.0 h1:SEATKb3LIHcaSIX+E6/K4kJpwfuozFEsmt5rS56N6CE= github.com/Azure/azure-storage-blob-go v0.6.0/go.mod h1:oGfmITT1V6x//CswqY2gtAHND+xIP64/qL7a5QJix0Y= +github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-autorest v11.0.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest v11.1.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest v11.1.2+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= @@ -35,6 +37,11 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/Unknwon/goconfig v0.0.0-20181105214110-56bd8ab18619 h1:6X8iB881g299aNEv6KXrcjL31iLOH7yA6NXoQX+MbDg= +github.com/Unknwon/goconfig v0.0.0-20181105214110-56bd8ab18619/go.mod h1:wngxua9XCNjvHjDiTiV26DaKDT+0c63QR6H5hjVUUxw= +github.com/a8m/tree v0.0.0-20181222104329-6a0b80129de4/go.mod h1:FSdwKX97koS5efgm8WevNf7XS3PqtyFkKDDXrz778cg= +github.com/abbot/go-http-auth v0.4.0 h1:QjmvZ5gSC7jm3Zg54DqWE/T5m1t2AfDu6QlXJT0EVT0= +github.com/abbot/go-http-auth v0.4.0/go.mod h1:Cz6ARTIzApMJDzh5bRMSUou6UMSp0IEXg9km/ci7TJM= github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38 h1:smF2tmSOzy2Mm+0dGI2AIUHY+w0BUc+4tn40djz7+6U= github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38/go.mod h1:r7bzyVFMNntcxPZXK3/+KdruV1H5KSlyVY0gc+NgInI= github.com/alecthomas/chroma v0.6.4 h1:Gn37/7W4a1qkmKLzfUpDy2rt3jt4X8CWycb4Gm7L360= @@ -48,10 +55,12 @@ github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1 h1:GDQdwm/gAcJcLAK github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/anacrolix/dms v0.0.0-20180117034613-8af4925bffb5/go.mod h1:DGqLjaZ3ziKKNRt+U5Q9PLWJ52Q/4rxfaaH/b3QYKaE= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= github.com/aws/aws-sdk-go v1.18.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.19.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.19.16/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.19.40 h1:omRrS4bCM/IbzU6UEb8Ojg1PvlElZzYZkOh8vWWgFMc= github.com/aws/aws-sdk-go v1.19.40/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= @@ -63,19 +72,24 @@ github.com/bep/gitmap v1.1.0 h1:vOMIdVB+2gd1VbfJPNJdLqAmn0af6NK98t4fK/GoCdA= github.com/bep/gitmap v1.1.0/go.mod h1:g9VRETxFUXNWzMiuxOwcudo6DfZkW9jOsOW0Ft4kYaY= github.com/bep/go-tocss v0.6.0 h1:lJf+nIjsQDpifUr+NgHi9QMBnrr9cFvMvEBT+uV9Q9E= github.com/bep/go-tocss v0.6.0/go.mod h1:d9d3crzlTl+PUZLFzBUjfFCpp68K+ku10mzTlnqU/+A= +github.com/billziss-gh/cgofuse v1.1.0/go.mod h1:LJjoaUojlVjgo5GQoEJTcJNqZJeRU0nCR84CyxKt2YM= github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= +github.com/coreos/bbolt v1.3.2 h1:wZwiHHUieZCquLkDL0B8UhzreNWsPHooDAG3q34zk0s= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.8 h1:DwoNytLphI8hzS2Af4D0dfaEaiSq2bN05mEm4R6vf8M= github.com/cpuguy83/go-md2man v1.0.8/go.mod h1:N6JayAiVKtlHSnuTCeuLSQVs75hb8q+dYQLjr7cDsKY= +github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 h1:y5HC9v93H5EPKqaS1UYVg1uYah5Xf51mBfIoWehClUQ= github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9hchkHSWYkEqJwUGisez3G1QY8Ryz0sdWrLPMGjLk= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -86,8 +100,10 @@ github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8 github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= github.com/disintegration/imaging v1.6.0 h1:nVPXRUUQ36Z7MNf0O77UzgnOb1mkMMor7lmJMJXc/mA= github.com/disintegration/imaging v1.6.0/go.mod h1:xuIt+sRxDFrHS0drzXUlCJthkJ8k7lkkUojDSR247MQ= +github.com/djherbis/times v1.2.0/go.mod h1:CGMZlo255K5r4Yw0b9RRfFQpM2y7uOmxg4jm9HsaVf8= github.com/dlclark/regexp2 v1.1.6 h1:CqB4MjHw0MFCDj+PHHjiESmHX+N7t0tJzKvC6M97BRg= github.com/dlclark/regexp2 v1.1.6/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= +github.com/dropbox/dropbox-sdk-go-unofficial v5.4.0+incompatible/go.mod h1:lr+LhMM3F6Y3lW1T9j2U5l7QeuWm87N9+PPXo3yH4qY= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= @@ -95,6 +111,7 @@ github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 h1:clC1lXBpe2kTj2VHdaIu9ajZQe4kcEY9j0NsnDDBZ3o= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= +github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= github.com/fortytw2/leaktest v1.2.0 h1:cj6GCiwJDH7l3tMHLjZDo0QqPtrXJiWSI9JgpeQKw+Q= github.com/fortytw2/leaktest v1.2.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= @@ -102,6 +119,7 @@ github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHqu github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= @@ -110,9 +128,14 @@ github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/goftp/file-driver v0.0.0-20180502053751-5d604a0fc0c9/go.mod h1:GpOj6zuVBG3Inr9qjEnuVTgBlk2lZ1S9DcoFiXWyKss= +github.com/goftp/server v0.0.0-20190304020633-eabccc535b5a/go.mod h1:k/SS6VWkxY7dHPhoMQ8IdRu8L4lQtmGbhyXGg+vCnXE= +github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gohugoio/testmodBuilder/mods v0.0.0-20190520184928-c56af20f2e95 h1:sgew0XCnZwnzpWxTt3V8LLiCO7OQi3C6dycaE67wfkU= +github.com/gohugoio/testmodBuilder/mods v0.0.0-20190520184928-c56af20f2e95/go.mod h1:bOlVlCa1/RajcHpXkrUXPSHB/Re1UnlXxD1Qp8SKOd8= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -130,6 +153,7 @@ github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian v2.1.1-0.20190517191504-25dcb96d9e51+incompatible h1:xmapqc1AyLoB+ddYT6r04bD9lIjlOqGaREovi0SzFaE= @@ -143,6 +167,8 @@ github.com/googleapis/gax-go v2.0.2+incompatible h1:silFMLAnr330+NRuag/VjIGF7TLp github.com/googleapis/gax-go v2.0.2+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= github.com/googleapis/gax-go/v2 v2.0.4 h1:hU4mGcQI4DaAYW+IbTun+2qEZVFxK0ySjQLTbS0VQKc= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20190411002643-bd77b112433e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= @@ -168,16 +194,22 @@ github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NH github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jdkato/prose v1.1.0 h1:LpvmDGwbKGTgdCH3a8VJL56sr7p/wOFPw/R4lM4PfFg= github.com/jdkato/prose v1.1.0/go.mod h1:jkF0lkxaX5PFSlk9l4Gh9Y+T57TqUZziWT7uZbW5ADg= +github.com/jlaffaye/ftp v0.0.0-20190519203911-8f5b34ce006f/go.mod h1:lli8NYPQOFy3O++YmYbqVgOcQ1JPCwdOy+5zSjKJ9qY= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/koofr/go-httpclient v0.0.0-20180104120329-03786175608a/go.mod h1:3xszwh+rNrYk1r9SStc4iJ326gne1OaBcrdB1ACsbzI= +github.com/koofr/go-koofrclient v0.0.0-20190131164641-7f327592caff/go.mod h1:MRAz4Gsxd+OzrZ0owwrUHc0zLESL+1Y5syqK/sJxK2A= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -187,6 +219,7 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kyokomi/emoji v1.5.1 h1:qp9dub1mW7C4MlvoRENH6EAENb9skEFOvIEbp1Waj38= github.com/kyokomi/emoji v1.5.1/go.mod h1:mZ6aGCD7yk8j6QY6KICwnZ2pxoszVseX1DNoGtU2tBA= github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magefile/mage v1.4.0 h1:RI7B1CgnPAuu2O9lWszwya61RLmfL0KCdo+QyyI/Bhk= github.com/magefile/mage v1.4.0/go.mod h1:IUDi13rsHje59lecXokTfGX0QIzO45uVPlXnJYsXepA= github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= @@ -199,13 +232,18 @@ github.com/matryer/try v0.0.0-20161228173917-9ac251b645a2/go.mod h1:0KeJpeMD6o+O github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.7 h1:UvyT9uN+3r7yLEYSlJsbQGdsaB/a0DlgWP3pql6iwOc= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-runewidth v0.0.3 h1:a+kO+98RDGEfo6asOGMmpodZq4FNtnGP54yps8BzLR4= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y= +github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/mmark v1.3.6 h1:t47x5vThdwgLJzofNsbsAl7gmIiJ7kbDQN5BxwBmwvY= github.com/miekg/mmark v1.3.6/go.mod h1:w7r9mkTvpS55jlfyn22qJ618itLryxXBhA7Jp3FIlkw= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/hashstructure v1.0.0 h1:ZkRJX1CyOoTkar7p/mLS5TZU4nJ1Rn/F8u9dGS02Q3Y= github.com/mitchellh/hashstructure v1.0.0/go.mod h1:QjSHrPWS+BGUVBYkbTZWEnOh3G1DutKwClXU/ABz6AQ= @@ -214,29 +252,43 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh github.com/muesli/smartcrop v0.0.0-20180228075044-f6ebaa786a12 h1:l0X/8IDy2UoK+oXcQFMRSIOcyuYb5iEPytPGplnM41Y= github.com/muesli/smartcrop v0.0.0-20180228075044-f6ebaa786a12/go.mod h1:i2fCI/UorTfgEpPPLWiFBv4pye+YAG78RwcQLUkocpI= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/ncw/go-acd v0.0.0-20171120105400-887eb06ab6a2/go.mod h1:MLIrzg7gp/kzVBxRE1olT7CWYMCklcUWU+ekoxOD9x0= +github.com/ncw/rclone v1.48.0 h1:Rc7A4YEQDeMPgnc1IzA6PsJ4YikyP+zS68rgGMYKJ7o= +github.com/ncw/rclone v1.48.0/go.mod h1:CXDUKN1OQ3Y2ya1Ma6jTZ7m9ZarGzF3ZTHsdPLHWWzY= +github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= github.com/nicksnyder/go-i18n v1.10.0 h1:5AzlPKvXBH4qBzmZ09Ua9Gipyruv6uApMcrNZdo96+Q= github.com/nicksnyder/go-i18n v1.10.0/go.mod h1:HrK7VCrbOvQoUAQ7Vpy7i87N7JZZZ7R2xBGjv0j365Q= github.com/niklasfasching/go-org v0.1.1 h1:yoU+dQLuc8P1Y0nxY3DV9SoBQTXSy9CogEvhOXSq9/Q= github.com/niklasfasching/go-org v0.1.1/go.mod h1:AsLD6X7djzRIz4/RFZu8vwRL0VGjUvGZCCH1Nz0VdrU= +github.com/nsf/termbox-go v0.0.0-20190325093121-288510b9734e/go.mod h1:IuKpRQcYE1Tfu+oAQqaLisqDeXgjyyltCfsaoYN18NQ= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/okzk/sdnotify v0.0.0-20180710141335-d9becc38acbd/go.mod h1:4soZNh0zW0LtYGdQ416i0jO0EIqMGcbtaspRS4BDvRQ= github.com/olekukonko/tablewriter v0.0.0-20180506121414-d4647c9c7a84 h1:fiKJgB4JDUd43CApkmCeTSQlWjtTtABrU2qsgbuP0BI= github.com/olekukonko/tablewriter v0.0.0-20180506121414-d4647c9c7a84/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88= +github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= +github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= +github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.4.0 h1:u3Z1r+oOXJIkxqw34zVhyPgjBsm6X2wn21NWs/HfSeg= github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= +github.com/pengsrc/go-shared v0.2.0/go.mod h1:jVblp62SafmidSkvWrXyxAme3gaTfEtWwRPGz5cpvHg= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.1-0.20190523025818-e98a7bef6829/go.mod h1:NxmoDg/QLVWluQDUYG7XBZTLUpKeFa8e3aMf1BfjyHk= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -253,14 +305,27 @@ github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rfjakob/eme v0.0.0-20171028163933-2222dbd4ba46 h1:w2CpS5muK+jyydnmlkqpAhzKmHmMBzBkfYUDjQNS1Dk= +github.com/rfjakob/eme v0.0.0-20171028163933-2222dbd4ba46/go.mod h1:U2bmx0hDj8EyDdcxmD5t3XHDnBFnyNNc22n1R4008eM= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/go-internal v1.3.0 h1:RR9dF3JtopPvtkroDZuVD7qquD0bnHlKSqaQhgwt8yk= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday v1.5.3-0.20190124082335-a477dd164691 h1:auJkuUc4uOuZNoH9jGLvqVaDLiuCOh/LY+Qw5NBFo4I= github.com/russross/blackfriday v1.5.3-0.20190124082335-a477dd164691/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/sanity-io/litter v1.1.0 h1:BllcKWa3VbZmOZbDCoszYLk7zCsKHz5Beossi8SUcTc= github.com/sanity-io/litter v1.1.0/go.mod h1:CJ0VCw2q4qKU7LaQr3n7UOSHzgEMgcGco7N/SkZQPjw= github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/sevlyar/go-daemon v0.1.4/go.mod h1:6dJpPatBT9eUwM5VCw9Bt6CdX9Tk6UWvhW3MebLDRKE= +github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/skratchdot/open-golang v0.0.0-20190402232053-79abb63cd66e/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/assertions v0.0.0-20190401211740-f487f9de1cd3/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= @@ -271,14 +336,19 @@ github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v0.0.4-0.20190321000552-67fc4837d267 h1:I9j1PLS64+NgCtkgbomGInboj1NFH1KF1tkVKlt3yF4= +github.com/spf13/cobra v0.0.4-0.20190321000552-67fc4837d267/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/fsync v0.0.0-20170320142552-12a01e648f05 h1:pQHm7pxjSgC54M1rtLSLmju25phy6RgYf3p4O6XanYE= github.com/spf13/fsync v0.0.0-20170320142552-12a01e648f05/go.mod h1:jdsEoy1w+v0NpuwXZEaRAH6ADTDmzfRnE2eVwshwFrM= +github.com/spf13/fsync v0.9.0 h1:f9CEt3DOB2mnHxZaftmEOFWjABEvKM/xpf3cUwJrGOY= +github.com/spf13/fsync v0.9.0/go.mod h1:fNtJEfG3HiltN3y4cPOz6MLjos9+2pIEqLIgszqhp/0= github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= @@ -288,6 +358,7 @@ github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/t3rm1n4l/go-mega v0.0.0-20190430100803-72151b53bb44/go.mod h1:XWL4vDyd3JKmJx+hZWUVgCNmmhZ2dTBcaNDcxH465s0= github.com/tdewolff/minify/v2 v2.3.7 h1:nhk7MKYRdTDwTxqEQZKLDkLe04tDHht8mBI+VJrsYvk= github.com/tdewolff/minify/v2 v2.3.7/go.mod h1:DD1stRlSx6JsHfl1+E/HVMQeXiec9rD1UQ0epklIZLc= github.com/tdewolff/parse/v2 v2.3.5 h1:/uS8JfhwVJsNkEh769GM5ENv6L9LOh2Z9uW3tCdlhs0= @@ -300,14 +371,17 @@ github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex github.com/uber/jaeger-client-go v2.15.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v1.5.0/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/wellington/go-libsass v0.9.3-0.20181113175235-c63644206701 h1:9vG9vvVNVupO4Y7uwFkRgIMNe9rdaJMCINDe8vhAhLo= github.com/wellington/go-libsass v0.9.3-0.20181113175235-c63644206701/go.mod h1:mxgxgam0N0E+NAUMHLcu20Ccfc3mVpDkyrLDayqfiTs= +github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yosssi/ace v0.0.5 h1:tUkIP/BLdKqrlrPwcmH0shwEEhTRHoGnc1wFIWmaBUA= github.com/yosssi/ace v0.0.5/go.mod h1:ALfIzm2vT7t5ZE7uoIZqF3TQ7SAOyupFZnkrF5id+K0= +github.com/yunify/qingstor-sdk-go v2.2.15+incompatible/go.mod h1:w6wqLDQ5bBTzxGJ55581UrSwLrsTAsdo9N6yX/8d9RY= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.mongodb.org/mongo-driver v1.0.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.opencensus.io v0.15.0/go.mod h1:UffZAU+4sDEINUGP/B7UfBBkq4fqLu9zXAX7ke6CHW0= @@ -325,8 +399,13 @@ gocloud.dev v0.15.0 h1:Tl8dkOHWVZiYBYPxG2ouhpfmluoQGt3mY323DaAHaC8= gocloud.dev v0.15.0/go.mod h1:ShXCyJaGrJu9y/7a6+DSCyBb9MFGZ1P5wwPa0Wu6w34= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181001203147-e3636079e1a4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190422183909-d864b10871cd/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5 h1:58fnuSXlxZmFdJyvtTFVmVhcMLU6v5fEb/ok4wyqtNU= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81 h1:00VmoueYNlNz/aHIilyyQz/MHSqGoWJzpFv/HW8xpzI= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= @@ -355,6 +434,8 @@ golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092 h1:4QSRKanuywn15aTZvI/mIDEgPQpswuFndXpOj3rKEco= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190606173856-1492cefac77f h1:IWHgpgFqnL5AhBUBZSgBdjl2vkQUEzcY+JNKWfcgAU0= +golang.org/x/net v0.0.0-20190606173856-1492cefac77f/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -379,11 +460,16 @@ golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181128092732-4ed8d59d0b35/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190530182044-ad28b68e88f1 h1:R4dVlxdmKenVdMRS/tTspEpSTRWINYrHD8ySIU9yCIU= +golang.org/x/sys v0.0.0-20190530182044-ad28b68e88f1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7 h1:LepdCS8Gf/MVejFIt8lsiexZATdoGVyp5bcyS+rYoUI= golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= @@ -394,6 +480,7 @@ golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5f golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -402,7 +489,9 @@ golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190422233926-fe54fb35175b/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190606174628-0139d5756a7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -413,6 +502,7 @@ google.golang.org/api v0.5.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEt google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.0 h1:Tfd7cKwKbFRsI8RMAD3oqqw7JPFRrvFlOsfbgVkjOOw= google.golang.org/appengine v1.6.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= @@ -427,6 +517,7 @@ google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3 google.golang.org/grpc v1.19.0 h1:cfg4PD8YEdSFnm7qLV4++93WcmhH2nIUhMjhdCvl3j8= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.19.1/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.0 h1:G+97AoqBnmZIT91cLG/EkCoK9NSelj64P8bOHHNmGn0= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -435,6 +526,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= diff --git a/helpers/content.go b/helpers/content.go index 6d9f1ca0878..f6576c04f75 100644 --- a/helpers/content.go +++ b/helpers/content.go @@ -511,12 +511,6 @@ func TotalWords(s string) int { return n } -// Old implementation only kept for benchmark comparison. -// TODO(bep) remove -func totalWordsOld(s string) int { - return len(strings.Fields(s)) -} - // TruncateWordsByRune truncates words by runes. func (c *ContentSpec) TruncateWordsByRune(in []string) (string, bool) { words := make([]string, len(in)) diff --git a/helpers/content_test.go b/helpers/content_test.go index 709c811420c..04e9466d0f5 100644 --- a/helpers/content_test.go +++ b/helpers/content_test.go @@ -506,13 +506,3 @@ func BenchmarkTotalWords(b *testing.B) { } } } - -func BenchmarkTotalWordsOld(b *testing.B) { - b.ResetTimer() - for i := 0; i < b.N; i++ { - wordCount := totalWordsOld(totalWordsBenchmarkString) - if wordCount != 400 { - b.Fatal("Wordcount error") - } - } -} diff --git a/helpers/docshelper.go b/helpers/docshelper.go index 8ad817d12ea..66cbfa7d3bf 100644 --- a/helpers/docshelper.go +++ b/helpers/docshelper.go @@ -36,8 +36,7 @@ func init() { } } - sort.Strings(aliases) - aliases = UniqueStrings(aliases) + aliases = UniqueStringsSorted(aliases) lexerEntry := struct { Name string diff --git a/helpers/general.go b/helpers/general.go index 3cf7ba8af67..5eabda3c605 100644 --- a/helpers/general.go +++ b/helpers/general.go @@ -22,15 +22,16 @@ import ( "net" "os" "path/filepath" + "sort" "strings" "sync" "unicode" "unicode/utf8" - "github.com/gohugoio/hugo/common/hugo" - "github.com/gohugoio/hugo/hugofs" + "github.com/gohugoio/hugo/common/hugo" + "github.com/spf13/afero" "github.com/jdkato/prose/transform" @@ -106,7 +107,7 @@ func FirstUpper(s string) string { // UniqueStrings returns a new slice with any duplicates removed. func UniqueStrings(s []string) []string { - var unique []string + unique := make([]string, 0, len(s)) set := map[string]interface{}{} for _, val := range s { if _, ok := set[val]; !ok { @@ -117,6 +118,40 @@ func UniqueStrings(s []string) []string { return unique } +// UniqueStringsReuse returns a slice with any duplicates removed. +// It will modify the input slice. +func UniqueStringsReuse(s []string) []string { + set := map[string]interface{}{} + result := s[:0] + for _, val := range s { + if _, ok := set[val]; !ok { + result = append(result, val) + set[val] = val + } + } + return result +} + +// UniqueStringsReuse returns a sorted slice with any duplicates removed. +// It will modify the input slice. +func UniqueStringsSorted(s []string) []string { + if len(s) == 0 { + return nil + } + ss := sort.StringSlice(s) + ss.Sort() + i := 0 + for j := 1; j < len(s); j++ { + if !ss.Less(i, j) { + continue + } + i++ + s[i] = s[j] + } + + return s[:i+1] +} + // ReaderToBytes takes an io.Reader argument, reads from it // and returns bytes. func ReaderToBytes(lines io.Reader) []byte { @@ -459,17 +494,15 @@ func PrintFs(fs afero.Fs, path string, w io.Writer) { if fs == nil { return } + afero.Walk(fs, path, func(path string, info os.FileInfo, err error) error { - if info != nil && !info.IsDir() { - s := path - if lang, ok := info.(hugofs.LanguageAnnouncer); ok { - s = s + "\tLANG: " + lang.Lang() - } - if fp, ok := info.(hugofs.FilePather); ok { - s = s + "\tRF: " + fp.Filename() + "\tBP: " + fp.BaseDir() - } - fmt.Fprintln(w, " ", s) + var filename string + var meta interface{} + if fim, ok := info.(hugofs.FileMetaInfo); ok { + filename = fim.Meta().Filename() + meta = fim.Meta() } + fmt.Fprintf(w, " %q %q\t\t%v\n", path, filename, meta) return nil }) } diff --git a/helpers/general_test.go b/helpers/general_test.go index ed4c3d2c27b..dd61d89485b 100644 --- a/helpers/general_test.go +++ b/helpers/general_test.go @@ -234,6 +234,24 @@ func TestUniqueStrings(t *testing.T) { } } +func TestUniqueStringsReuse(t *testing.T) { + in := []string{"a", "b", "a", "b", "c", "", "a", "", "d"} + output := UniqueStringsReuse(in) + expected := []string{"a", "b", "c", "", "d"} + if !reflect.DeepEqual(output, expected) { + t.Errorf("Expected %#v, got %#v\n", expected, output) + } +} + +func TestUniqueStringsSorted(t *testing.T) { + assert := require.New(t) + in := []string{"a", "a", "b", "c", "b", "", "a", "", "d"} + output := UniqueStringsSorted(in) + expected := []string{"", "a", "b", "c", "d"} + assert.Equal(expected, output) + assert.Nil(UniqueStringsSorted(nil)) +} + func TestFindAvailablePort(t *testing.T) { addr, err := FindAvailablePort() assert.Nil(t, err) @@ -328,3 +346,55 @@ func BenchmarkMD5FromFileFast(b *testing.B) { } } + +func BenchmarkUniqueStrings(b *testing.B) { + input := []string{"a", "b", "d", "e", "d", "h", "a", "i"} + + b.Run("Safe", func(b *testing.B) { + for i := 0; i < b.N; i++ { + result := UniqueStrings(input) + if len(result) != 6 { + b.Fatal(fmt.Sprintf("invalid count: %d", len(result))) + } + } + }) + + b.Run("Reuse slice", func(b *testing.B) { + b.StopTimer() + inputs := make([][]string, b.N) + for i := 0; i < b.N; i++ { + inputc := make([]string, len(input)) + copy(inputc, input) + inputs[i] = inputc + } + b.StartTimer() + for i := 0; i < b.N; i++ { + inputc := inputs[i] + + result := UniqueStringsReuse(inputc) + if len(result) != 6 { + b.Fatal(fmt.Sprintf("invalid count: %d", len(result))) + } + } + }) + + b.Run("Reuse slice sorted", func(b *testing.B) { + b.StopTimer() + inputs := make([][]string, b.N) + for i := 0; i < b.N; i++ { + inputc := make([]string, len(input)) + copy(inputc, input) + inputs[i] = inputc + } + b.StartTimer() + for i := 0; i < b.N; i++ { + inputc := inputs[i] + + result := UniqueStringsSorted(inputc) + if len(result) != 6 { + b.Fatal(fmt.Sprintf("invalid count: %d", len(result))) + } + } + }) + +} diff --git a/helpers/path.go b/helpers/path.go index 36bd3269bf8..12ddfeb56f5 100644 --- a/helpers/path.go +++ b/helpers/path.go @@ -26,6 +26,8 @@ import ( "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/hugofs" + "github.com/gohugoio/hugo/common/hugio" _errors "github.com/pkg/errors" "github.com/spf13/afero" @@ -172,32 +174,6 @@ func ReplaceExtension(path string, newExt string) string { return f + "." + newExt } -// GetFirstThemeDir gets the root directory of the first theme, if there is one. -// If there is no theme, returns the empty string. -func (p *PathSpec) GetFirstThemeDir() string { - if p.ThemeSet() { - return p.AbsPathify(filepath.Join(p.ThemesDir, p.Themes()[0])) - } - return "" -} - -// GetThemesDir gets the absolute root theme dir path. -func (p *PathSpec) GetThemesDir() string { - if p.ThemeSet() { - return p.AbsPathify(p.ThemesDir) - } - return "" -} - -// GetRelativeThemeDir gets the relative root directory of the current theme, if there is one. -// If there is no theme, returns the empty string. -func (p *PathSpec) GetRelativeThemeDir() string { - if p.ThemeSet() { - return strings.TrimPrefix(filepath.Join(p.ThemesDir, p.Themes()[0]), FilePathSeparator) - } - return "" -} - func makePathRelative(inPath string, possibleDirectories ...string) (string, error) { for _, currentPath := range possibleDirectories { @@ -379,6 +355,107 @@ func prettifyPath(in string, b filepathPathBridge) string { return b.Join(b.Dir(in), name, "index"+ext) } +type NamedSlice struct { + Name string + Slice []string +} + +func (n NamedSlice) String() string { + if len(n.Slice) == 0 { + return n.Name + } + return fmt.Sprintf("%s%s{%s}", n.Name, FilePathSeparator, strings.Join(n.Slice, ",")) +} + +func ExtractAndGroupRootPaths(paths []string) []NamedSlice { + if len(paths) == 0 { + return nil + } + + pathsCopy := make([]string, len(paths)) + hadSlashPrefix := strings.HasPrefix(paths[0], FilePathSeparator) + + for i, p := range paths { + pathsCopy[i] = strings.Trim(filepath.ToSlash(p), "/") + } + + sort.Strings(pathsCopy) + + pathsParts := make([][]string, len(pathsCopy)) + + for i, p := range pathsCopy { + pathsParts[i] = strings.Split(p, "/") + } + + var groups [][]string + + for i, p1 := range pathsParts { + c1 := -1 + + for j, p2 := range pathsParts { + if i == j { + continue + } + + c2 := -1 + + for i, v := range p1 { + if i >= len(p2) { + break + } + if v != p2[i] { + break + } + + c2 = i + } + + if c1 == -1 || (c2 != -1 && c2 < c1) { + c1 = c2 + } + } + + if c1 != -1 { + groups = append(groups, p1[:c1+1]) + } else { + groups = append(groups, p1) + } + } + + groupsStr := make([]string, len(groups)) + for i, g := range groups { + groupsStr[i] = strings.Join(g, "/") + } + + groupsStr = UniqueStringsSorted(groupsStr) + + var result []NamedSlice + + for _, g := range groupsStr { + name := filepath.FromSlash(g) + if hadSlashPrefix { + name = FilePathSeparator + name + } + ns := NamedSlice{Name: name} + for _, p := range pathsCopy { + if !strings.HasPrefix(p, g) { + continue + } + + p = strings.TrimPrefix(p, g) + if p != "" { + ns.Slice = append(ns.Slice, p) + } + } + + ns.Slice = UniqueStrings(ExtractRootPaths(ns.Slice)) + + result = append(result, ns) + } + + return result +} + // ExtractRootPaths extracts the root paths from the supplied list of paths. // The resulting root path will not contain any file separators, but there // may be duplicates. @@ -425,98 +502,21 @@ func FindCWD() (string, error) { return path, nil } -// SymbolicWalk is like filepath.Walk, but it supports the root being a -// symbolic link. It will still not follow symbolic links deeper down in -// the file structure. -func SymbolicWalk(fs afero.Fs, root string, walker filepath.WalkFunc) error { - - // Sanity check - if root != "" && len(root) < 4 { - return errors.New("path is too short") - } - - // Handle the root first - fileInfo, realPath, err := getRealFileInfo(fs, root) - - if err != nil { - return walker(root, nil, err) - } - - if !fileInfo.IsDir() { - return fmt.Errorf("cannot walk regular file %s", root) - } - - if err := walker(realPath, fileInfo, err); err != nil && err != filepath.SkipDir { - return err - } - - // Some of Hugo's filesystems represents an ordered root folder, i.e. project first, then theme folders. - // Make sure that order is preserved. afero.Walk will sort the directories down in the file tree, - // but we don't care about that. - rootContent, err := readDir(fs, root, false) - - if err != nil { - return walker(root, nil, err) - } - - for _, fi := range rootContent { - if err := afero.Walk(fs, filepath.Join(root, fi.Name()), walker); err != nil { - return err - } +// SymbolicWalk is like filepath.Walk, but it follows symbolic links. +func SymbolicWalk(fs afero.Fs, root string, walker hugofs.WalkFunc) error { + if _, isOs := fs.(*afero.OsFs); isOs { + // Mainly to track symlinks. + fs = hugofs.NewBaseFileDecorator(fs) } - return nil + w := hugofs.NewWalkway(hugofs.WalkwayConfig{ + Fs: fs, + Root: root, + WalkFn: walker, + }) -} - -func readDir(fs afero.Fs, dirname string, doSort bool) ([]os.FileInfo, error) { - f, err := fs.Open(dirname) - if err != nil { - return nil, err - } - list, err := f.Readdir(-1) - f.Close() - if err != nil { - return nil, err - } - if doSort { - sort.Slice(list, func(i, j int) bool { return list[i].Name() < list[j].Name() }) - } - return list, nil -} - -func getRealFileInfo(fs afero.Fs, path string) (os.FileInfo, string, error) { - fileInfo, err := LstatIfPossible(fs, path) - realPath := path - - if err != nil { - return nil, "", err - } - - if fileInfo.Mode()&os.ModeSymlink == os.ModeSymlink { - link, err := filepath.EvalSymlinks(path) - if err != nil { - return nil, "", _errors.Wrapf(err, "Cannot read symbolic link %q", path) - } - fileInfo, err = LstatIfPossible(fs, link) - if err != nil { - return nil, "", _errors.Wrapf(err, "Cannot stat %q", link) - } - realPath = link - } - return fileInfo, realPath, nil -} - -// GetRealPath returns the real file path for the given path, whether it is a -// symlink or not. -func GetRealPath(fs afero.Fs, path string) (string, error) { - _, realPath, err := getRealFileInfo(fs, path) - - if err != nil { - return "", err - } + return w.Walk() - return realPath, nil } // LstatIfPossible can be used to call Lstat if possible, else Stat. diff --git a/helpers/path_test.go b/helpers/path_test.go index 98291936c1a..e58a045c19e 100644 --- a/helpers/path_test.go +++ b/helpers/path_test.go @@ -29,8 +29,6 @@ import ( "github.com/stretchr/testify/require" - "github.com/stretchr/testify/assert" - "github.com/gohugoio/hugo/hugofs" "github.com/spf13/afero" "github.com/spf13/viper" @@ -73,18 +71,9 @@ func TestMakePath(t *testing.T) { } func TestMakePathSanitized(t *testing.T) { - v := viper.New() - v.Set("contentDir", "content") - v.Set("dataDir", "data") - v.Set("i18nDir", "i18n") - v.Set("layoutDir", "layouts") - v.Set("assetDir", "assets") - v.Set("resourceDir", "resources") - v.Set("publishDir", "public") - v.Set("archetypeDir", "archetypes") + v := newTestCfg() - l := langs.NewDefaultLanguage(v) - p, _ := NewPathSpec(hugofs.NewMem(v), l) + p, _ := NewPathSpec(hugofs.NewMem(v), v) tests := []struct { input string @@ -166,33 +155,6 @@ func TestGetRelativePath(t *testing.T) { } } -func TestGetRealPath(t *testing.T) { - if runtime.GOOS == "windows" && os.Getenv("CI") == "" { - t.Skip("Skip TestGetRealPath as os.Symlink needs administrator rights on Windows") - } - - d1, _ := ioutil.TempDir("", "d1") - defer os.Remove(d1) - fs := afero.NewOsFs() - - rp1, err := GetRealPath(fs, d1) - require.NoError(t, err) - assert.Equal(t, d1, rp1) - - sym := filepath.Join(os.TempDir(), "d1sym") - err = os.Symlink(d1, sym) - require.NoError(t, err) - defer os.Remove(sym) - - rp2, err := GetRealPath(fs, sym) - require.NoError(t, err) - - // On OS X, the temp folder is itself a symbolic link (to /private...) - // This has to do for now. - assert.True(t, strings.HasSuffix(rp2, d1)) - -} - func TestMakePathRelative(t *testing.T) { type test struct { inPath, path1, path2, output string @@ -659,6 +621,29 @@ func TestPrettifyPath(t *testing.T) { } +func TestExtractAndGroupRootPaths(t *testing.T) { + in := []string{ + filepath.FromSlash("/a/b/c/d"), + filepath.FromSlash("/a/b/c/e"), + filepath.FromSlash("/a/b/e/f"), + filepath.FromSlash("/a/b"), + filepath.FromSlash("/a/b/c/b/g"), + filepath.FromSlash("/c/d/e"), + } + + inCopy := make([]string, len(in)) + copy(inCopy, in) + + result := ExtractAndGroupRootPaths(in) + + assert := require.New(t) + assert.Equal(filepath.FromSlash("[/a/b/{c,e} /c/d/e]"), fmt.Sprint(result)) + + // Make sure the original is preserved + assert.Equal(inCopy, in) + +} + func TestExtractRootPaths(t *testing.T) { tests := []struct { input []string diff --git a/helpers/pathspec_test.go b/helpers/pathspec_test.go index 00dd9cd7b1a..1c27f7e1159 100644 --- a/helpers/pathspec_test.go +++ b/helpers/pathspec_test.go @@ -14,6 +14,7 @@ package helpers import ( + "path/filepath" "testing" "github.com/gohugoio/hugo/hugofs" @@ -36,8 +37,12 @@ func TestNewPathSpecFromConfig(t *testing.T) { v.Set("workingDir", "thework") v.Set("staticDir", "thestatic") v.Set("theme", "thetheme") + langs.LoadLanguageSettings(v, nil) - p, err := NewPathSpec(hugofs.NewMem(v), l) + fs := hugofs.NewMem(v) + fs.Source.MkdirAll(filepath.FromSlash("thework/thethemes/thetheme"), 0777) + + p, err := NewPathSpec(fs, l) require.NoError(t, err) require.True(t, p.CanonifyURLs) @@ -50,5 +55,5 @@ func TestNewPathSpecFromConfig(t *testing.T) { require.Equal(t, "http://base.com", p.BaseURL.String()) require.Equal(t, "thethemes", p.ThemesDir) require.Equal(t, "thework", p.WorkingDir) - require.Equal(t, []string{"thetheme"}, p.Themes()) + } diff --git a/helpers/testhelpers_test.go b/helpers/testhelpers_test.go index c9da4f12919..b74dccfc46d 100644 --- a/helpers/testhelpers_test.go +++ b/helpers/testhelpers_test.go @@ -5,6 +5,7 @@ import ( "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/langs" + "github.com/gohugoio/hugo/modules" ) func newTestPathSpec(fs *hugofs.Fs, v *viper.Viper) *PathSpec { @@ -42,6 +43,14 @@ func newTestCfg() *viper.Viper { v.Set("resourceDir", "resources") v.Set("publishDir", "public") v.Set("archetypeDir", "archetypes") + langs.LoadLanguageSettings(v, nil) + langs.LoadLanguageSettings(v, nil) + mod, err := modules.CreateProjectModule(v) + if err != nil { + panic(err) + } + v.Set("allModules", modules.Modules{mod}) + return v } diff --git a/htesting/test_helpers.go b/htesting/test_helpers.go new file mode 100644 index 00000000000..dc303b2e50c --- /dev/null +++ b/htesting/test_helpers.go @@ -0,0 +1,39 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package htesting + +import ( + "runtime" + "strings" + + "github.com/spf13/afero" +) + +// CreateTempDir creates a temp dir in the given filesystem and +// returns the dirnam and a func that removes it when done. +func CreateTempDir(fs afero.Fs, prefix string) (string, func(), error) { + tempDir, err := afero.TempDir(fs, "", prefix) + if err != nil { + return "", nil, err + } + + _, isOsFs := fs.(*afero.OsFs) + + if isOsFs && runtime.GOOS == "darwin" && !strings.HasPrefix(tempDir, "/private") { + // To get the entry folder in line with the rest. This its a little bit + // mysterious, but so be it. + tempDir = "/private" + tempDir + } + return tempDir, func() { fs.RemoveAll(tempDir) }, nil +} diff --git a/htesting/test_structs.go b/htesting/test_structs.go deleted file mode 100644 index 72dc7f3fc63..00000000000 --- a/htesting/test_structs.go +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright 2019 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package htesting - -import ( - "html/template" - "time" - - "github.com/gohugoio/hugo/common/hugo" - "github.com/gohugoio/hugo/langs" - "github.com/gohugoio/hugo/navigation" - "github.com/gohugoio/hugo/resources/page" - "github.com/spf13/viper" -) - -type testSite struct { - h hugo.Info - l *langs.Language -} - -func (t testSite) Hugo() hugo.Info { - return t.h -} - -func (t testSite) ServerPort() int { - return 1313 -} - -func (testSite) LastChange() (t time.Time) { - return -} - -func (t testSite) Title() string { - return "foo" -} - -func (t testSite) Sites() page.Sites { - return nil -} - -func (t testSite) IsServer() bool { - return false -} - -func (t testSite) Language() *langs.Language { - return t.l -} - -func (t testSite) Pages() page.Pages { - return nil -} - -func (t testSite) RegularPages() page.Pages { - return nil -} - -func (t testSite) Menus() navigation.Menus { - return nil -} - -func (t testSite) Taxonomies() interface{} { - return nil -} - -func (t testSite) BaseURL() template.URL { - return "" -} - -func (t testSite) Params() map[string]interface{} { - return nil -} - -func (t testSite) Data() map[string]interface{} { - return nil -} - -// NewTestHugoSite creates a new minimal test site. -func NewTestHugoSite() page.Site { - return testSite{ - h: hugo.NewInfo(hugo.EnvironmentProduction), - l: langs.NewLanguage("en", newTestConfig()), - } -} - -func newTestConfig() *viper.Viper { - v := viper.New() - v.Set("contentDir", "content") - return v -} diff --git a/hugofs/basepath_real_filename_fs.go b/hugofs/basepath_real_filename_fs.go deleted file mode 100644 index 1024c4d3034..00000000000 --- a/hugofs/basepath_real_filename_fs.go +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugofs - -import ( - "os" - - "github.com/spf13/afero" -) - -// RealFilenameInfo is a thin wrapper around os.FileInfo adding the real filename. -type RealFilenameInfo interface { - os.FileInfo - - // This is the real filename to the file in the underlying filesystem. - RealFilename() string -} - -type realFilenameInfo struct { - os.FileInfo - realFilename string -} - -func (f *realFilenameInfo) RealFilename() string { - return f.realFilename -} - -// NewBasePathRealFilenameFs returns a new BasePathRealFilenameFs instance -// using base. -func NewBasePathRealFilenameFs(base *afero.BasePathFs) *BasePathRealFilenameFs { - return &BasePathRealFilenameFs{BasePathFs: base} -} - -// BasePathRealFilenameFs is a thin wrapper around afero.BasePathFs that -// provides the real filename in Stat and LstatIfPossible. -type BasePathRealFilenameFs struct { - *afero.BasePathFs -} - -// Stat returns the os.FileInfo structure describing a given file. If there is -// an error, it will be of type *os.PathError. -func (b *BasePathRealFilenameFs) Stat(name string) (os.FileInfo, error) { - fi, err := b.BasePathFs.Stat(name) - if err != nil { - return nil, err - } - - if _, ok := fi.(RealFilenameInfo); ok { - return fi, nil - } - - filename, err := b.RealPath(name) - if err != nil { - return nil, &os.PathError{Op: "stat", Path: name, Err: err} - } - - return &realFilenameInfo{FileInfo: fi, realFilename: filename}, nil -} - -// LstatIfPossible returns the os.FileInfo structure describing a given file. -// It attempts to use Lstat if supported or defers to the os. In addition to -// the FileInfo, a boolean is returned telling whether Lstat was called. -func (b *BasePathRealFilenameFs) LstatIfPossible(name string) (os.FileInfo, bool, error) { - - fi, ok, err := b.BasePathFs.LstatIfPossible(name) - if err != nil { - return nil, false, err - } - - if _, ok := fi.(RealFilenameInfo); ok { - return fi, ok, nil - } - - filename, err := b.RealPath(name) - if err != nil { - return nil, false, &os.PathError{Op: "lstat", Path: name, Err: err} - } - - return &realFilenameInfo{FileInfo: fi, realFilename: filename}, ok, nil -} diff --git a/hugofs/decorators.go b/hugofs/decorators.go new file mode 100644 index 00000000000..0a2b3971270 --- /dev/null +++ b/hugofs/decorators.go @@ -0,0 +1,205 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugofs + +import ( + "os" + "path/filepath" + "strings" + + "github.com/pkg/errors" + + "github.com/spf13/afero" +) + +func decorateDirs(fs afero.Fs, meta FileMeta) afero.Fs { + ffs := &baseFileDecoratorFs{Fs: fs} + + decorator := func(fi os.FileInfo, name string) (os.FileInfo, error) { + if !fi.IsDir() { + // Leave regular files as they are. + return fi, nil + } + + return decorateFileInfo(fi, fs, nil, "", "", meta), nil + } + + ffs.decorate = decorator + + return ffs + +} + +func decoratePath(fs afero.Fs, createPath func(name string) string) afero.Fs { + + ffs := &baseFileDecoratorFs{Fs: fs} + + decorator := func(fi os.FileInfo, name string) (os.FileInfo, error) { + path := createPath(name) + + return decorateFileInfo(fi, fs, nil, "", path, nil), nil + } + + ffs.decorate = decorator + + return ffs + +} + +// DecorateBasePathFs adds Path info to files and directories in the +// provided BasePathFs, using the base as base. +func DecorateBasePathFs(base *afero.BasePathFs) afero.Fs { + basePath, _ := base.RealPath("") + if !strings.HasSuffix(basePath, filepathSeparator) { + basePath += filepathSeparator + } + + ffs := &baseFileDecoratorFs{Fs: base} + + decorator := func(fi os.FileInfo, name string) (os.FileInfo, error) { + path := strings.TrimPrefix(name, basePath) + + return decorateFileInfo(fi, base, nil, "", path, nil), nil + } + + ffs.decorate = decorator + + return ffs +} + +// NewBaseFileDecorator decorates the given Fs to provide the real filename +// and an Opener func. If +func NewBaseFileDecorator(fs afero.Fs) afero.Fs { + + ffs := &baseFileDecoratorFs{Fs: fs} + + decorator := func(fi os.FileInfo, filename string) (os.FileInfo, error) { + // Store away the original in case it's a symlink. + meta := FileMeta{metaKeyName: fi.Name()} + isSymlink := isSymlink(fi) + if isSymlink { + meta[metaKeyOriginalFilename] = filename + link, err := filepath.EvalSymlinks(filename) + if err != nil { + return nil, err + } + + fi, err = fs.Stat(link) + if err != nil { + return nil, err + } + + filename = link + meta[metaKeyIsSymlink] = true + + } + + opener := func() (afero.File, error) { + return ffs.open(filename) + + } + + return decorateFileInfo(fi, ffs, opener, filename, "", meta), nil + } + + ffs.decorate = decorator + return ffs +} + +type baseFileDecoratorFs struct { + afero.Fs + decorate func(fi os.FileInfo, filename string) (os.FileInfo, error) +} + +func (fs *baseFileDecoratorFs) Stat(name string) (os.FileInfo, error) { + fi, err := fs.Fs.Stat(name) + if err != nil { + return nil, err + } + + return fs.decorate(fi, name) + +} + +func (fs *baseFileDecoratorFs) LstatIfPossible(name string) (os.FileInfo, bool, error) { + var ( + fi os.FileInfo + err error + ok bool + ) + + if lstater, isLstater := fs.Fs.(afero.Lstater); isLstater { + fi, ok, err = lstater.LstatIfPossible(name) + } else { + fi, err = fs.Fs.Stat(name) + } + + if err != nil { + return nil, false, err + } + + fi, err = fs.decorate(fi, name) + + return fi, ok, err +} + +func (fs *baseFileDecoratorFs) Open(name string) (afero.File, error) { + return fs.open(name) +} + +func (fs *baseFileDecoratorFs) open(name string) (afero.File, error) { + f, err := fs.Fs.Open(name) + if err != nil { + return nil, err + } + return &baseFileDecoratorFile{File: f, fs: fs}, nil +} + +type baseFileDecoratorFile struct { + afero.File + fs *baseFileDecoratorFs +} + +func (l *baseFileDecoratorFile) Readdir(c int) (ofi []os.FileInfo, err error) { + dirnames, err := l.File.Readdirnames(c) + if err != nil { + return nil, err + } + + fisp := make([]os.FileInfo, 0, len(dirnames)) + + for _, dirname := range dirnames { + filename := dirname + + if l.Name() != "" && l.Name() != filepathSeparator { + filename = filepath.Join(l.Name(), dirname) + } + + // We need to resolve any symlink info. + fi, _, err := lstatIfPossible(l.fs.Fs, filename) + if err != nil { + if os.IsNotExist(err) { + continue + } + return nil, err + } + fi, err = l.fs.decorate(fi, filename) + if err != nil { + return nil, errors.Wrap(err, "decorate") + } + fisp = append(fisp, fi) + } + + return fisp, err +} diff --git a/hugofs/fileinfo.go b/hugofs/fileinfo.go new file mode 100644 index 00000000000..a2f12c429dd --- /dev/null +++ b/hugofs/fileinfo.go @@ -0,0 +1,297 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package hugofs provides the file systems used by Hugo. +package hugofs + +import ( + "os" + "path/filepath" + "runtime" + "strings" + "time" + + "github.com/gohugoio/hugo/hugofs/files" + "golang.org/x/text/unicode/norm" + + "github.com/pkg/errors" + + "github.com/spf13/cast" + + "github.com/gohugoio/hugo/common/hreflect" + + "github.com/spf13/afero" +) + +const ( + metaKeyFilename = "filename" + metaKeyOriginalFilename = "originalFilename" + metaKeyName = "name" + metaKeyPath = "path" + metaKeyPathWalk = "pathWalk" + metaKeyLang = "lang" + metaKeyWeight = "weight" + metaKeyOrdinal = "ordinal" + metaKeyFs = "fs" + metaKeyOpener = "opener" + metaKeyIsOrdered = "isOrdered" + metaKeyIsSymlink = "isSymlink" + metaKeySkipDir = "skipDir" + metaKeyClassifier = "classifier" + metaKeyTranslationBaseName = "translationBaseName" + metaKeyTranslationBaseNameWithExt = "translationBaseNameWithExt" + metaKeyTranslations = "translations" + metaKeyDecoraterPath = "decoratorPath" +) + +type FileMeta map[string]interface{} + +func (f FileMeta) GetInt(key string) int { + return cast.ToInt(f[key]) +} + +func (f FileMeta) GetString(key string) string { + return cast.ToString(f[key]) +} + +func (f FileMeta) GetBool(key string) bool { + return cast.ToBool(f[key]) +} + +func (f FileMeta) Filename() string { + return f.stringV(metaKeyFilename) +} + +func (f FileMeta) OriginalFilename() string { + return f.stringV(metaKeyOriginalFilename) +} + +func (f FileMeta) SkipDir() bool { + return f.GetBool(metaKeySkipDir) +} +func (f FileMeta) TranslationBaseName() string { + return f.stringV(metaKeyTranslationBaseName) +} + +func (f FileMeta) TranslationBaseNameWithExt() string { + return f.stringV(metaKeyTranslationBaseNameWithExt) +} + +func (f FileMeta) Translations() []string { + return cast.ToStringSlice(f[metaKeyTranslations]) +} + +func (f FileMeta) Name() string { + return f.stringV(metaKeyName) +} + +func (f FileMeta) Classifier() string { + c := f.stringV(metaKeyClassifier) + if c != "" { + return c + } + + return files.ContentClassFile // For sorting +} + +func (f FileMeta) Lang() string { + return f.stringV(metaKeyLang) +} + +func (f FileMeta) Path() string { + return f.stringV(metaKeyPath) +} + +func (f FileMeta) Weight() int { + return f.GetInt(metaKeyWeight) +} + +func (f FileMeta) Ordinal() int { + return f.GetInt(metaKeyOrdinal) +} + +func (f FileMeta) IsOrdered() bool { + return f.GetBool(metaKeyIsOrdered) +} + +// IsSymlink returns whether this comes from a symlinked file or directory. +func (f FileMeta) IsSymlink() bool { + return f.GetBool(metaKeyIsSymlink) +} + +func (f FileMeta) Watch() bool { + if v, found := f["watch"]; found { + return v.(bool) + } + return false +} + +func (f FileMeta) Fs() afero.Fs { + if v, found := f[metaKeyFs]; found { + return v.(afero.Fs) + } + return nil +} + +func (f FileMeta) GetOpener() func() (afero.File, error) { + o, found := f[metaKeyOpener] + if !found { + return nil + } + return o.(func() (afero.File, error)) +} + +func (f FileMeta) Open() (afero.File, error) { + v, found := f[metaKeyOpener] + if !found { + return nil, errors.New("file opener not found") + } + return v.(func() (afero.File, error))() +} + +func (f FileMeta) stringV(key string) string { + if v, found := f[key]; found { + return v.(string) + } + return "" +} + +func (f FileMeta) setIfNotZero(key string, val interface{}) { + if !hreflect.IsTruthful(val) { + return + } + f[key] = val +} + +type FileMetaInfo interface { + os.FileInfo + Meta() FileMeta +} + +type fileInfoMeta struct { + os.FileInfo + m FileMeta +} + +func (fi *fileInfoMeta) Meta() FileMeta { + return fi.m +} + +func NewFileMetaInfo(fi os.FileInfo, m FileMeta) FileMetaInfo { + + if fim, ok := fi.(FileMetaInfo); ok { + mergeFileMeta(fim.Meta(), m) + } + return &fileInfoMeta{FileInfo: fi, m: m} +} + +// Merge metadata, last entry wins. +func mergeFileMeta(from, to FileMeta) { + if from == nil { + return + } + for k, v := range from { + if _, found := to[k]; !found { + to[k] = v + } + } +} + +type dirNameOnlyFileInfo struct { + name string +} + +func (fi *dirNameOnlyFileInfo) Name() string { + return fi.name +} + +func (fi *dirNameOnlyFileInfo) Size() int64 { + panic("not implemented") +} + +func (fi *dirNameOnlyFileInfo) Mode() os.FileMode { + return os.ModeDir +} + +func (fi *dirNameOnlyFileInfo) ModTime() time.Time { + return time.Time{} +} + +func (fi *dirNameOnlyFileInfo) IsDir() bool { + return true +} + +func (fi *dirNameOnlyFileInfo) Sys() interface{} { + return nil +} + +func newDirNameOnlyFileInfo(name string, isOrdered bool, fileOpener func() (afero.File, error)) FileMetaInfo { + name = normalizeFilename(name) + _, base := filepath.Split(name) + return NewFileMetaInfo(&dirNameOnlyFileInfo{name: base}, FileMeta{ + metaKeyFilename: name, + metaKeyIsOrdered: isOrdered, + metaKeyOpener: fileOpener}) +} + +func decorateFileInfo( + fi os.FileInfo, + fs afero.Fs, opener func() (afero.File, error), + filename, filepath string, inMeta FileMeta) FileMetaInfo { + + var meta FileMeta + var fim FileMetaInfo + + filepath = strings.TrimPrefix(filepath, filepathSeparator) + + var ok bool + if fim, ok = fi.(FileMetaInfo); ok { + meta = fim.Meta() + } else { + meta = make(FileMeta) + fim = NewFileMetaInfo(fi, meta) + } + + meta.setIfNotZero(metaKeyOpener, opener) + meta.setIfNotZero(metaKeyFs, fs) + meta.setIfNotZero(metaKeyPath, normalizeFilename(filepath)) + meta.setIfNotZero(metaKeyFilename, normalizeFilename(filename)) + + mergeFileMeta(inMeta, meta) + + return fim + +} + +func isSymlink(fi os.FileInfo) bool { + return fi != nil && fi.Mode()&os.ModeSymlink == os.ModeSymlink +} + +func fileInfosToFileMetaInfos(fis []os.FileInfo) []FileMetaInfo { + fims := make([]FileMetaInfo, len(fis)) + for i, v := range fis { + fims[i] = v.(FileMetaInfo) + } + return fims +} + +func normalizeFilename(filename string) string { + if filename == "" { + return "" + } + if runtime.GOOS == "darwin" { + // When a file system is HFS+, its filepath is in NFD form. + return norm.NFC.String(filename) + } + return filename +} diff --git a/hugofs/files/classifier.go b/hugofs/files/classifier.go new file mode 100644 index 00000000000..9aa2476b7bf --- /dev/null +++ b/hugofs/files/classifier.go @@ -0,0 +1,121 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package files + +import ( + "os" + "path/filepath" + "sort" + "strings" +) + +var ( + // This should be the only list of valid extensions for content files. + contentFileExtensions = []string{ + "html", "htm", + "mdown", "markdown", "md", + "asciidoc", "adoc", "ad", + "rest", "rst", + "mmark", + "org", + "pandoc", "pdc"} + + contentFileExtensionsSet map[string]bool +) + +func init() { + contentFileExtensionsSet = make(map[string]bool) + for _, ext := range contentFileExtensions { + contentFileExtensionsSet[ext] = true + } +} + +func IsContentFile(filename string) bool { + return contentFileExtensionsSet[strings.TrimPrefix(filepath.Ext(filename), ".")] +} + +func IsContentExt(ext string) bool { + return contentFileExtensionsSet[ext] +} + +const ( + ContentClassLeaf = "leaf" + ContentClassBranch = "branch" + ContentClassFile = "zfile" // Sort below + ContentClassContent = "zcontent" +) + +func ClassifyContentFile(filename string) string { + if !IsContentFile(filename) { + return ContentClassFile + } + if strings.HasPrefix(filename, "_index.") { + return ContentClassBranch + } + + if strings.HasPrefix(filename, "index.") { + return ContentClassLeaf + } + + return ContentClassContent +} + +const ( + ComponentFolderArchetypes = "archetypes" + ComponentFolderStatic = "static" + ComponentFolderLayouts = "layouts" + ComponentFolderContent = "content" + ComponentFolderData = "data" + ComponentFolderAssets = "assets" + ComponentFolderI18n = "i18n" + + FolderResources = "resources" +) + +var ( + ComponentFolders = []string{ + ComponentFolderArchetypes, + ComponentFolderStatic, + ComponentFolderLayouts, + ComponentFolderContent, + ComponentFolderData, + ComponentFolderAssets, + ComponentFolderI18n, + } + + componentFoldersSet = make(map[string]bool) +) + +func init() { + sort.Strings(ComponentFolders) + for _, f := range ComponentFolders { + componentFoldersSet[f] = true + } +} + +// ResolveComponentFolder returns "content" from "content/blog/foo.md" etc. +func ResolveComponentFolder(filename string) string { + filename = strings.TrimPrefix(filename, string(os.PathSeparator)) + for _, cf := range ComponentFolders { + if strings.HasPrefix(filename, cf) { + return cf + } + } + + return "" +} + +func IsComponentFolder(name string) bool { + return componentFoldersSet[name] +} diff --git a/hugofs/files/classifier_test.go b/hugofs/files/classifier_test.go new file mode 100644 index 00000000000..d576b4e58a7 --- /dev/null +++ b/hugofs/files/classifier_test.go @@ -0,0 +1,49 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package files + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestIsContentFile(t *testing.T) { + assert := require.New(t) + + assert.True(IsContentFile(filepath.FromSlash("my/file.md"))) + assert.True(IsContentFile(filepath.FromSlash("my/file.ad"))) + assert.False(IsContentFile(filepath.FromSlash("textfile.txt"))) + assert.True(IsContentExt("md")) + assert.False(IsContentExt("json")) +} + +func TestComponentFolders(t *testing.T) { + assert := require.New(t) + + // It's important that these are absolutely right and not changed. + assert.Equal(len(ComponentFolders), len(componentFoldersSet)) + assert.True(IsComponentFolder("archetypes")) + assert.True(IsComponentFolder("layouts")) + assert.True(IsComponentFolder("data")) + assert.True(IsComponentFolder("i18n")) + assert.True(IsComponentFolder("assets")) + assert.False(IsComponentFolder("resources")) + assert.True(IsComponentFolder("static")) + assert.True(IsComponentFolder("content")) + assert.False(IsComponentFolder("foo")) + assert.False(IsComponentFolder("")) + +} diff --git a/hugofs/filter_fs.go b/hugofs/filter_fs.go new file mode 100644 index 00000000000..952b276cff3 --- /dev/null +++ b/hugofs/filter_fs.go @@ -0,0 +1,341 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugofs + +import ( + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + "syscall" + "time" + + "github.com/gohugoio/hugo/hugofs/files" + + "github.com/spf13/afero" +) + +var ( + _ afero.Fs = (*FilterFs)(nil) + _ afero.Lstater = (*FilterFs)(nil) + _ afero.File = (*filterDir)(nil) +) + +func NewLanguageFs(langs map[string]int, fs afero.Fs) (afero.Fs, error) { + + applyMeta := func(fs *FilterFs, name string, fis []os.FileInfo) { + + for i, fi := range fis { + if fi.IsDir() { + filename := filepath.Join(name, fi.Name()) + fis[i] = decorateFileInfo(fi, fs, fs.getOpener(filename), "", "", nil) + continue + } + + meta := fi.(FileMetaInfo).Meta() + lang := meta.Lang() + + fileLang, translationBaseName, translationBaseNameWithExt := langInfoFrom(langs, fi.Name()) + weight := 0 + + if fileLang != "" { + weight = 1 + if fileLang == lang { + // Give priority to myfile.sv.txt inside the sv filesystem. + weight++ + } + lang = fileLang + } + + fim := NewFileMetaInfo(fi, FileMeta{ + metaKeyLang: lang, + metaKeyWeight: weight, + metaKeyOrdinal: langs[lang], + metaKeyTranslationBaseName: translationBaseName, + metaKeyTranslationBaseNameWithExt: translationBaseNameWithExt, + metaKeyClassifier: files.ClassifyContentFile(fi.Name()), + }) + + fis[i] = fim + } + } + + all := func(fis []os.FileInfo) { + // Maps translation base name to a list of language codes. + translations := make(map[string][]string) + trackTranslation := func(meta FileMeta) { + name := meta.TranslationBaseNameWithExt() + translations[name] = append(translations[name], meta.Lang()) + } + for _, fi := range fis { + if fi.IsDir() { + continue + } + meta := fi.(FileMetaInfo).Meta() + + trackTranslation(meta) + + } + + for _, fi := range fis { + fim := fi.(FileMetaInfo) + langs := translations[fim.Meta().TranslationBaseNameWithExt()] + if len(langs) > 0 { + fim.Meta()["translations"] = sortAndremoveStringDuplicates(langs) + } + } + } + + return &FilterFs{ + fs: fs, + applyPerSource: applyMeta, + applyAll: all, + }, nil + +} + +func NewFilterFs(fs afero.Fs) (afero.Fs, error) { + + applyMeta := func(fs *FilterFs, name string, fis []os.FileInfo) { + for i, fi := range fis { + if fi.IsDir() { + fis[i] = decorateFileInfo(fi, fs, fs.getOpener(fi.(FileMetaInfo).Meta().Filename()), "", "", nil) + } + } + } + + ffs := &FilterFs{ + fs: fs, + applyPerSource: applyMeta, + } + + return ffs, nil + +} + +// FilterFs is an ordered composite filesystem. +type FilterFs struct { + fs afero.Fs + + applyPerSource func(fs *FilterFs, name string, fis []os.FileInfo) + applyAll func(fis []os.FileInfo) +} + +func (fs *FilterFs) Chmod(n string, m os.FileMode) error { + return syscall.EPERM +} + +func (fs *FilterFs) Chtimes(n string, a, m time.Time) error { + return syscall.EPERM +} + +func (fs *FilterFs) LstatIfPossible(name string) (os.FileInfo, bool, error) { + fi, b, err := lstatIfPossible(fs.fs, name) + + if err != nil { + return nil, false, err + } + + if fi.IsDir() { + return decorateFileInfo(fi, fs, fs.getOpener(name), "", "", nil), false, nil + } + + fs.applyFilters(name, -1, fi) + + return fi, b, nil + +} + +func (fs *FilterFs) Mkdir(n string, p os.FileMode) error { + return syscall.EPERM +} + +func (fs *FilterFs) MkdirAll(n string, p os.FileMode) error { + return syscall.EPERM +} + +func (fs *FilterFs) Name() string { + return "WeightedFileSystem" +} + +func (fs *FilterFs) Open(name string) (afero.File, error) { + f, err := fs.fs.Open(name) + if err != nil { + return nil, err + } + + return &filterDir{ + File: f, + ffs: fs, + }, nil + +} + +func (fs *FilterFs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) { + panic("not implemented") +} + +func (fs *FilterFs) ReadDir(name string) ([]os.FileInfo, error) { + panic("not implemented") +} + +func (fs *FilterFs) Remove(n string) error { + return syscall.EPERM +} + +func (fs *FilterFs) RemoveAll(p string) error { + return syscall.EPERM +} + +func (fs *FilterFs) Rename(o, n string) error { + return syscall.EPERM +} + +func (fs *FilterFs) Stat(name string) (os.FileInfo, error) { + fi, _, err := fs.LstatIfPossible(name) + return fi, err +} + +func (fs *FilterFs) Create(n string) (afero.File, error) { + return nil, syscall.EPERM +} + +func (fs *FilterFs) getOpener(name string) func() (afero.File, error) { + return func() (afero.File, error) { + return fs.Open(name) + } +} + +func (fs *FilterFs) applyFilters(name string, count int, fis ...os.FileInfo) ([]os.FileInfo, error) { + if fs.applyPerSource != nil { + fs.applyPerSource(fs, name, fis) + } + + seen := make(map[string]bool) + var duplicates []int + for i, dir := range fis { + if !dir.IsDir() { + continue + } + if seen[dir.Name()] { + duplicates = append(duplicates, i) + } else { + seen[dir.Name()] = true + } + } + + // Remove duplicate directories, keep first. + if len(duplicates) > 0 { + for i := len(duplicates) - 1; i >= 0; i-- { + idx := duplicates[i] + fis = append(fis[:idx], fis[idx+1:]...) + } + } + + if fs.applyAll != nil { + fs.applyAll(fis) + } + + if count > 0 && len(fis) >= count { + return fis[:count], nil + } + + return fis, nil + +} + +type filterDir struct { + afero.File + ffs *FilterFs +} + +func (f *filterDir) Readdir(count int) ([]os.FileInfo, error) { + fis, err := f.File.Readdir(-1) + if err != nil { + return nil, err + } + return f.ffs.applyFilters(f.Name(), count, fis...) +} + +func (f *filterDir) Readdirnames(count int) ([]string, error) { + dirsi, err := f.Readdir(count) + if err != nil { + return nil, err + } + + dirs := make([]string, len(dirsi)) + for i, d := range dirsi { + dirs[i] = d.Name() + } + return dirs, nil +} + +// Try to extract the language from the given filename. +// Any valid language identificator in the name will win over the +// language set on the file system, e.g. "mypost.en.md". +func langInfoFrom(languages map[string]int, name string) (string, string, string) { + var lang string + + baseName := filepath.Base(name) + ext := filepath.Ext(baseName) + translationBaseName := baseName + + if ext != "" { + translationBaseName = strings.TrimSuffix(translationBaseName, ext) + } + + fileLangExt := filepath.Ext(translationBaseName) + fileLang := strings.TrimPrefix(fileLangExt, ".") + + if _, found := languages[fileLang]; found { + lang = fileLang + translationBaseName = strings.TrimSuffix(translationBaseName, fileLangExt) + } + + translationBaseNameWithExt := translationBaseName + + if ext != "" { + translationBaseNameWithExt += ext + } + + return lang, translationBaseName, translationBaseNameWithExt + +} + +func printFs(fs afero.Fs, path string, w io.Writer) { + if fs == nil { + return + } + afero.Walk(fs, path, func(path string, info os.FileInfo, err error) error { + fmt.Println("p:::", path) + return nil + }) +} + +func sortAndremoveStringDuplicates(s []string) []string { + ss := sort.StringSlice(s) + ss.Sort() + i := 0 + for j := 1; j < len(s); j++ { + if !ss.Less(i, j) { + continue + } + i++ + s[i] = s[j] + } + + return s[:i+1] +} diff --git a/hugofs/filter_fs_test.go b/hugofs/filter_fs_test.go new file mode 100644 index 00000000000..cc9a5e0443a --- /dev/null +++ b/hugofs/filter_fs_test.go @@ -0,0 +1,48 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugofs + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestLangInfoFrom(t *testing.T) { + + langs := map[string]int{ + "sv": 10, + "en": 20, + } + + assert := require.New(t) + + tests := []struct { + input string + expected []string + }{ + {"page.sv.md", []string{"sv", "page", "page.md"}}, + {"page.en.md", []string{"en", "page", "page.md"}}, + {"page.no.md", []string{"", "page.no", "page.no.md"}}, + {filepath.FromSlash("tc-lib-color/class-Com.Tecnick.Color.Css"), []string{"", "class-Com.Tecnick.Color", "class-Com.Tecnick.Color.Css"}}, + {filepath.FromSlash("class-Com.Tecnick.Color.sv.Css"), []string{"sv", "class-Com.Tecnick.Color", "class-Com.Tecnick.Color.Css"}}, + } + + for _, test := range tests { + v1, v2, v3 := langInfoFrom(langs, test.input) + assert.Equal(test.expected, []string{v1, v2, v3}) + } + +} diff --git a/hugofs/fs.go b/hugofs/fs.go index 38590a64e0a..163807704a8 100644 --- a/hugofs/fs.go +++ b/hugofs/fs.go @@ -21,8 +21,10 @@ import ( "github.com/spf13/afero" ) -// Os points to an Os Afero file system. -var Os = &afero.OsFs{} +var ( + // Os points to the (real) Os filesystem. + Os = &afero.OsFs{} +) // Fs abstracts the file system to separate source and destination file systems // and allows both to be mocked for testing. diff --git a/hugofs/language_composite_fs.go b/hugofs/language_composite_fs.go index 2889f8a00fc..5dbd252c0be 100644 --- a/hugofs/language_composite_fs.go +++ b/hugofs/language_composite_fs.go @@ -14,6 +14,9 @@ package hugofs import ( + "os" + "path" + "github.com/spf13/afero" ) @@ -30,8 +33,8 @@ type languageCompositeFs struct { // This is a hybrid filesystem. To get a specific file in Open, Stat etc., use the full filename // to the target filesystem. This information is available in Readdir, Stat etc. via the // special LanguageFileInfo FileInfo implementation. -func NewLanguageCompositeFs(base afero.Fs, overlay *LanguageFs) afero.Fs { - return afero.NewReadOnlyFs(&languageCompositeFs{afero.NewCopyOnWriteFs(base, overlay).(*afero.CopyOnWriteFs)}) +func NewLanguageCompositeFs(base, overlay afero.Fs) afero.Fs { + return &languageCompositeFs{afero.NewCopyOnWriteFs(base, overlay).(*afero.CopyOnWriteFs)} } // Open takes the full path to the file in the target filesystem. If it is a directory, it gets merged @@ -49,3 +52,36 @@ func (fs *languageCompositeFs) Open(name string) (afero.File, error) { } return f, nil } + +// LanguageDirsMerger implements the afero.DirsMerger interface, which is used +// to merge two directories. +var LanguageDirsMerger = func(lofi, bofi []os.FileInfo) ([]os.FileInfo, error) { + m := make(map[string]FileMetaInfo) + + getKey := func(fim FileMetaInfo) string { + return path.Join(fim.Meta().Lang(), fim.Name()) + } + + for _, fi := range lofi { + fim := fi.(FileMetaInfo) + m[getKey(fim)] = fim + } + + for _, fi := range bofi { + fim := fi.(FileMetaInfo) + key := getKey(fim) + _, found := m[key] + if !found { + m[key] = fim + } + } + + merged := make([]os.FileInfo, len(m)) + i := 0 + for _, v := range m { + merged[i] = v + i++ + } + + return merged, nil +} diff --git a/hugofs/language_composite_fs_test.go b/hugofs/language_composite_fs_test.go deleted file mode 100644 index ab4e25fc0ac..00000000000 --- a/hugofs/language_composite_fs_test.go +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugofs - -import ( - "path/filepath" - - "strings" - - "testing" - - "github.com/spf13/afero" - "github.com/stretchr/testify/require" -) - -func TestCompositeLanguagFsTest(t *testing.T) { - assert := require.New(t) - - languages := map[string]bool{ - "sv": true, - "en": true, - "nn": true, - } - msv := afero.NewMemMapFs() - baseSv := "/content/sv" - lfssv := NewLanguageFs("sv", languages, afero.NewBasePathFs(msv, baseSv)) - mnn := afero.NewMemMapFs() - baseNn := "/content/nn" - lfsnn := NewLanguageFs("nn", languages, afero.NewBasePathFs(mnn, baseNn)) - men := afero.NewMemMapFs() - baseEn := "/content/en" - lfsen := NewLanguageFs("en", languages, afero.NewBasePathFs(men, baseEn)) - - // The order will be sv, en, nn - composite := NewLanguageCompositeFs(lfsnn, lfsen) - composite = NewLanguageCompositeFs(composite, lfssv) - - afero.WriteFile(msv, filepath.Join(baseSv, "f1.txt"), []byte("some sv"), 0755) - afero.WriteFile(mnn, filepath.Join(baseNn, "f1.txt"), []byte("some nn"), 0755) - afero.WriteFile(men, filepath.Join(baseEn, "f1.txt"), []byte("some en"), 0755) - - // Swedish is the top layer. - assertLangFile(t, composite, "f1.txt", "sv") - - afero.WriteFile(msv, filepath.Join(baseSv, "f2.en.txt"), []byte("some sv"), 0755) - afero.WriteFile(mnn, filepath.Join(baseNn, "f2.en.txt"), []byte("some nn"), 0755) - afero.WriteFile(men, filepath.Join(baseEn, "f2.en.txt"), []byte("some en"), 0755) - - // English is in the middle, but the most specific language match wins. - //assertLangFile(t, composite, "f2.en.txt", "en") - - // Fetch some specific language versions - assertLangFile(t, composite, filepath.Join(baseNn, "f2.en.txt"), "nn") - assertLangFile(t, composite, filepath.Join(baseEn, "f2.en.txt"), "en") - assertLangFile(t, composite, filepath.Join(baseSv, "f2.en.txt"), "sv") - - // Read the root - f, err := composite.Open("/") - assert.NoError(err) - defer f.Close() - files, err := f.Readdir(-1) - assert.NoError(err) - assert.Equal(4, len(files)) - expected := map[string]bool{ - filepath.FromSlash("/content/en/f1.txt"): true, - filepath.FromSlash("/content/nn/f1.txt"): true, - filepath.FromSlash("/content/sv/f1.txt"): true, - filepath.FromSlash("/content/en/f2.en.txt"): true, - } - got := make(map[string]bool) - - for _, fi := range files { - fil, ok := fi.(*LanguageFileInfo) - assert.True(ok) - got[fil.Filename()] = true - } - assert.Equal(expected, got) -} - -func assertLangFile(t testing.TB, fs afero.Fs, filename, match string) { - f, err := fs.Open(filename) - if err != nil { - t.Fatal(err) - } - defer f.Close() - b, err := afero.ReadAll(f) - if err != nil { - t.Fatal(err) - } - - s := string(b) - if !strings.Contains(s, match) { - t.Fatalf("got %q expected it to contain %q", s, match) - - } -} diff --git a/hugofs/language_fs.go b/hugofs/language_fs.go deleted file mode 100644 index db77c1fabd3..00000000000 --- a/hugofs/language_fs.go +++ /dev/null @@ -1,346 +0,0 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugofs - -import ( - "fmt" - "os" - "path/filepath" - "strings" - - "github.com/spf13/afero" -) - -const hugoFsMarker = "__hugofs" - -var ( - _ LanguageAnnouncer = (*LanguageFileInfo)(nil) - _ FilePather = (*LanguageFileInfo)(nil) - _ afero.Lstater = (*LanguageFs)(nil) -) - -// LanguageAnnouncer is aware of its language. -type LanguageAnnouncer interface { - Lang() string - TranslationBaseName() string -} - -// FilePather is aware of its file's location. -type FilePather interface { - // Filename gets the full path and filename to the file. - Filename() string - - // Path gets the content relative path including file name and extension. - // The directory is relative to the content root where "content" is a broad term. - Path() string - - // RealName is FileInfo.Name in its original form. - RealName() string - - BaseDir() string -} - -// LanguageDirsMerger implements the afero.DirsMerger interface, which is used -// to merge two directories. -var LanguageDirsMerger = func(lofi, bofi []os.FileInfo) ([]os.FileInfo, error) { - m := make(map[string]*LanguageFileInfo) - - for _, fi := range lofi { - fil, ok := fi.(*LanguageFileInfo) - if !ok { - return nil, fmt.Errorf("received %T, expected *LanguageFileInfo", fi) - } - m[fil.virtualName] = fil - } - - for _, fi := range bofi { - fil, ok := fi.(*LanguageFileInfo) - if !ok { - return nil, fmt.Errorf("received %T, expected *LanguageFileInfo", fi) - } - existing, found := m[fil.virtualName] - - if !found || existing.weight < fil.weight { - m[fil.virtualName] = fil - } - } - - merged := make([]os.FileInfo, len(m)) - i := 0 - for _, v := range m { - merged[i] = v - i++ - } - - return merged, nil -} - -// LanguageFileInfo is a super-set of os.FileInfo with additional information -// about the file in relation to its Hugo language. -type LanguageFileInfo struct { - os.FileInfo - lang string - baseDir string - realFilename string - relFilename string - name string - realName string - virtualName string - translationBaseName string - - // We add some weight to the files in their own language's content directory. - weight int -} - -// Filename returns a file's real filename including the base (ie. -// "/my/base/sect/page.md"). -func (fi *LanguageFileInfo) Filename() string { - return fi.realFilename -} - -// Path returns a file's filename relative to the base (ie. "sect/page.md"). -func (fi *LanguageFileInfo) Path() string { - return fi.relFilename -} - -// RealName returns a file's real base name (ie. "page.md"). -func (fi *LanguageFileInfo) RealName() string { - return fi.realName -} - -// BaseDir returns a file's base directory (ie. "/my/base"). -func (fi *LanguageFileInfo) BaseDir() string { - return fi.baseDir -} - -// Lang returns a file's language (ie. "sv"). -func (fi *LanguageFileInfo) Lang() string { - return fi.lang -} - -// TranslationBaseName returns the base filename without any extension or language -// identifiers (ie. "page"). -func (fi *LanguageFileInfo) TranslationBaseName() string { - return fi.translationBaseName -} - -// Name is the name of the file within this filesystem without any path info. -// It will be marked with language information so we can identify it as ours -// (ie. "__hugofs_sv_page.md"). -func (fi *LanguageFileInfo) Name() string { - return fi.name -} - -type languageFile struct { - afero.File - fs *LanguageFs -} - -// Readdir creates FileInfo entries by calling Lstat if possible. -func (l *languageFile) Readdir(c int) (ofi []os.FileInfo, err error) { - names, err := l.File.Readdirnames(c) - if err != nil { - return nil, err - } - - fis := make([]os.FileInfo, len(names)) - - for i, name := range names { - fi, _, err := l.fs.LstatIfPossible(filepath.Join(l.Name(), name)) - - if err != nil { - return nil, err - } - fis[i] = fi - } - - return fis, err -} - -// LanguageFs represents a language filesystem. -type LanguageFs struct { - // This Fs is usually created with a BasePathFs - basePath string - lang string - nameMarker string - languages map[string]bool - afero.Fs -} - -// NewLanguageFs creates a new language filesystem. -func NewLanguageFs(lang string, languages map[string]bool, fs afero.Fs) *LanguageFs { - if lang == "" { - panic("no lang set for the language fs") - } - var basePath string - - if bfs, ok := fs.(*afero.BasePathFs); ok { - basePath, _ = bfs.RealPath("") - } - - marker := hugoFsMarker + "_" + lang + "_" - - return &LanguageFs{lang: lang, languages: languages, basePath: basePath, Fs: fs, nameMarker: marker} -} - -// Lang returns a language filesystem's language (ie. "sv"). -func (fs *LanguageFs) Lang() string { - return fs.lang -} - -// Stat returns the os.FileInfo of a given file. -func (fs *LanguageFs) Stat(name string) (os.FileInfo, error) { - name, err := fs.realName(name) - if err != nil { - return nil, err - } - - fi, err := fs.Fs.Stat(name) - if err != nil { - return nil, err - } - - return fs.newLanguageFileInfo(name, fi) -} - -// Open opens the named file for reading. -func (fs *LanguageFs) Open(name string) (afero.File, error) { - name, err := fs.realName(name) - if err != nil { - return nil, err - } - f, err := fs.Fs.Open(name) - - if err != nil { - return nil, err - } - return &languageFile{File: f, fs: fs}, nil -} - -// LstatIfPossible returns the os.FileInfo structure describing a given file. -// It attempts to use Lstat if supported or defers to the os. In addition to -// the FileInfo, a boolean is returned telling whether Lstat was called. -func (fs *LanguageFs) LstatIfPossible(name string) (os.FileInfo, bool, error) { - name, err := fs.realName(name) - if err != nil { - return nil, false, err - } - - var fi os.FileInfo - var b bool - - if lif, ok := fs.Fs.(afero.Lstater); ok { - fi, b, err = lif.LstatIfPossible(name) - } else { - fi, err = fs.Fs.Stat(name) - } - - if err != nil { - return nil, b, err - } - - lfi, err := fs.newLanguageFileInfo(name, fi) - - return lfi, b, err -} - -func (fs *LanguageFs) realPath(name string) (string, error) { - if baseFs, ok := fs.Fs.(*afero.BasePathFs); ok { - return baseFs.RealPath(name) - } - return name, nil -} - -func (fs *LanguageFs) realName(name string) (string, error) { - if strings.Contains(name, hugoFsMarker) { - if !strings.Contains(name, fs.nameMarker) { - return "", os.ErrNotExist - } - return strings.Replace(name, fs.nameMarker, "", 1), nil - } - - if fs.basePath == "" { - return name, nil - } - - return strings.TrimPrefix(name, fs.basePath), nil -} - -func (fs *LanguageFs) newLanguageFileInfo(filename string, fi os.FileInfo) (*LanguageFileInfo, error) { - filename = filepath.Clean(filename) - _, name := filepath.Split(filename) - - realName := name - virtualName := name - - realPath, err := fs.realPath(filename) - if err != nil { - return nil, err - } - - lang := fs.Lang() - - baseNameNoExt := "" - - if !fi.IsDir() { - - // Try to extract the language from the file name. - // Any valid language identificator in the name will win over the - // language set on the file system, e.g. "mypost.en.md". - baseName := filepath.Base(name) - ext := filepath.Ext(baseName) - baseNameNoExt = baseName - - if ext != "" { - baseNameNoExt = strings.TrimSuffix(baseNameNoExt, ext) - } - - fileLangExt := filepath.Ext(baseNameNoExt) - fileLang := strings.TrimPrefix(fileLangExt, ".") - - if fs.languages[fileLang] { - lang = fileLang - baseNameNoExt = strings.TrimSuffix(baseNameNoExt, fileLangExt) - } - - // This connects the filename to the filesystem, not the language. - virtualName = baseNameNoExt + "." + lang + ext - - name = fs.nameMarker + name - } - - weight := 1 - // If this file's language belongs in this directory, add some weight to it - // to make it more important. - if lang == fs.Lang() { - weight = 2 - } - - if fi.IsDir() { - // For directories we always want to start from the union view. - realPath = strings.TrimPrefix(realPath, fs.basePath) - } - - return &LanguageFileInfo{ - lang: lang, - weight: weight, - realFilename: realPath, - realName: realName, - relFilename: strings.TrimPrefix(strings.TrimPrefix(realPath, fs.basePath), string(os.PathSeparator)), - name: name, - virtualName: virtualName, - translationBaseName: baseNameNoExt, - baseDir: fs.basePath, - FileInfo: fi}, nil -} diff --git a/hugofs/language_fs_test.go b/hugofs/language_fs_test.go deleted file mode 100644 index 52e8dc9de74..00000000000 --- a/hugofs/language_fs_test.go +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugofs - -import ( - "path/filepath" - "testing" - - "github.com/spf13/afero" - "github.com/stretchr/testify/require" -) - -func TestLanguagFs(t *testing.T) { - languages := map[string]bool{ - "sv": true, - } - base := filepath.FromSlash("/my/base") - assert := require.New(t) - m := afero.NewMemMapFs() - bfs := afero.NewBasePathFs(m, base) - lfs := NewLanguageFs("sv", languages, bfs) - assert.NotNil(lfs) - assert.Equal("sv", lfs.Lang()) - err := afero.WriteFile(lfs, filepath.FromSlash("sect/page.md"), []byte("abc"), 0777) - assert.NoError(err) - fi, err := lfs.Stat(filepath.FromSlash("sect/page.md")) - assert.NoError(err) - assert.Equal("__hugofs_sv_page.md", fi.Name()) - - languager, ok := fi.(LanguageAnnouncer) - assert.True(ok) - - assert.Equal("sv", languager.Lang()) - - lfi, ok := fi.(*LanguageFileInfo) - assert.True(ok) - assert.Equal(filepath.FromSlash("/my/base/sect/page.md"), lfi.Filename()) - assert.Equal(filepath.FromSlash("sect/page.md"), lfi.Path()) - assert.Equal("page.sv.md", lfi.virtualName) - assert.Equal("__hugofs_sv_page.md", lfi.Name()) - assert.Equal("page.md", lfi.RealName()) - assert.Equal(filepath.FromSlash("/my/base"), lfi.BaseDir()) - assert.Equal("sv", lfi.Lang()) - assert.Equal("page", lfi.TranslationBaseName()) -} - -// Issue 4559 -func TestFilenamesHandling(t *testing.T) { - languages := map[string]bool{ - "sv": true, - } - base := filepath.FromSlash("/my/base") - assert := require.New(t) - m := afero.NewMemMapFs() - bfs := afero.NewBasePathFs(m, base) - lfs := NewLanguageFs("sv", languages, bfs) - assert.NotNil(lfs) - assert.Equal("sv", lfs.Lang()) - - for _, test := range []struct { - filename string - check func(fi *LanguageFileInfo) - }{ - {"tc-lib-color/class-Com.Tecnick.Color.Css", func(fi *LanguageFileInfo) { - assert.Equal("class-Com.Tecnick.Color", fi.TranslationBaseName()) - assert.Equal(filepath.FromSlash("/my/base"), fi.BaseDir()) - assert.Equal(filepath.FromSlash("tc-lib-color/class-Com.Tecnick.Color.Css"), fi.Path()) - assert.Equal("class-Com.Tecnick.Color.Css", fi.RealName()) - assert.Equal(filepath.FromSlash("/my/base/tc-lib-color/class-Com.Tecnick.Color.Css"), fi.Filename()) - }}, - {"tc-lib-color/class-Com.Tecnick.Color.sv.Css", func(fi *LanguageFileInfo) { - assert.Equal("class-Com.Tecnick.Color", fi.TranslationBaseName()) - assert.Equal("class-Com.Tecnick.Color.sv.Css", fi.RealName()) - assert.Equal(filepath.FromSlash("/my/base/tc-lib-color/class-Com.Tecnick.Color.sv.Css"), fi.Filename()) - }}, - } { - err := afero.WriteFile(lfs, filepath.FromSlash(test.filename), []byte("abc"), 0777) - assert.NoError(err) - fi, err := lfs.Stat(filepath.FromSlash(test.filename)) - assert.NoError(err) - - lfi, ok := fi.(*LanguageFileInfo) - assert.True(ok) - assert.Equal("sv", lfi.Lang()) - test.check(lfi) - - } - -} diff --git a/hugofs/nosymlink_fs.go b/hugofs/nosymlink_fs.go new file mode 100644 index 00000000000..42ab94b5ce7 --- /dev/null +++ b/hugofs/nosymlink_fs.go @@ -0,0 +1,85 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugofs + +import ( + "errors" + "os" + + "github.com/spf13/afero" +) + +var ( + ErrPermissionSymlink = errors.New("symlinks not allowed in this filesystem") +) + +func NewNoSymlinkFs(fs afero.Fs) afero.Fs { + return &noSymlinkFs{Fs: fs} +} + +// noSymlinkFs is a filesystem that prevents symlinking. +type noSymlinkFs struct { + afero.Fs +} + +func (fs *noSymlinkFs) LstatIfPossible(name string) (os.FileInfo, bool, error) { + return fs.stat(name) +} + +func (fs *noSymlinkFs) Stat(name string) (os.FileInfo, error) { + fi, _, err := fs.stat(name) + return fi, err +} + +func (fs *noSymlinkFs) stat(name string) (os.FileInfo, bool, error) { + + var ( + fi os.FileInfo + wasLstat bool + err error + ) + + if lstater, ok := fs.Fs.(afero.Lstater); ok { + fi, wasLstat, err = lstater.LstatIfPossible(name) + } else { + + fi, err = fs.Fs.Stat(name) + } + + var metaIsSymlink bool + + if fim, ok := fi.(FileMetaInfo); ok { + metaIsSymlink = fim.Meta().IsSymlink() + } + + if metaIsSymlink || isSymlink(fi) { + return nil, wasLstat, ErrPermissionSymlink + } + + return fi, wasLstat, err +} + +func (fs *noSymlinkFs) Open(name string) (afero.File, error) { + if _, _, err := fs.stat(name); err != nil { + return nil, err + } + return fs.Fs.Open(name) +} + +func (fs *noSymlinkFs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) { + if _, _, err := fs.stat(name); err != nil { + return nil, err + } + return fs.Fs.OpenFile(name, flag, perm) +} diff --git a/hugofs/nosymlink_test.go b/hugofs/nosymlink_test.go new file mode 100644 index 00000000000..6d0b99dccbb --- /dev/null +++ b/hugofs/nosymlink_test.go @@ -0,0 +1,97 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugofs + +import ( + "os" + "path/filepath" + "testing" + + "github.com/gohugoio/hugo/htesting" + + "github.com/spf13/afero" + + "github.com/stretchr/testify/require" +) + +func TestNoSymlinkFs(t *testing.T) { + if skipSymlink() { + t.Skip("Skip; os.Symlink needs administrator rights on Windows") + } + assert := require.New(t) + workDir, clean, err := htesting.CreateTempDir(Os, "hugo-nosymlink") + assert.NoError(err) + defer clean() + wd, _ := os.Getwd() + defer func() { + os.Chdir(wd) + }() + + blogDir := filepath.Join(workDir, "blog") + blogFile := filepath.Join(blogDir, "a.txt") + assert.NoError(os.MkdirAll(blogDir, 0777)) + afero.WriteFile(Os, filepath.Join(blogFile), []byte("content"), 0777) + os.Chdir(workDir) + assert.NoError(os.Symlink("blog", "symlinkdedir")) + os.Chdir(blogDir) + assert.NoError(os.Symlink("a.txt", "symlinkdedfile.txt")) + + fs := NewNoSymlinkFs(Os) + ls := fs.(afero.Lstater) + symlinkedDir := filepath.Join(workDir, "symlinkdedir") + symlinkedFile := filepath.Join(blogDir, "symlinkdedfile.txt") + + // Check Stat and Lstat + for _, stat := range []func(name string) (os.FileInfo, error){ + func(name string) (os.FileInfo, error) { + return fs.Stat(name) + }, + func(name string) (os.FileInfo, error) { + fi, _, err := ls.LstatIfPossible(name) + return fi, err + }, + } { + _, err = stat(symlinkedDir) + assert.Equal(ErrPermissionSymlink, err) + _, err = stat(symlinkedFile) + assert.Equal(ErrPermissionSymlink, err) + + fi, err := stat(filepath.Join(workDir, "blog")) + assert.NoError(err) + assert.NotNil(fi) + + fi, err = stat(blogFile) + assert.NoError(err) + assert.NotNil(fi) + } + + // Check Open + _, err = fs.Open(symlinkedDir) + assert.Equal(ErrPermissionSymlink, err) + _, err = fs.OpenFile(symlinkedDir, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0666) + assert.Equal(ErrPermissionSymlink, err) + _, err = fs.OpenFile(symlinkedFile, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0666) + assert.Equal(ErrPermissionSymlink, err) + _, err = fs.Open(symlinkedFile) + assert.Equal(ErrPermissionSymlink, err) + f, err := fs.Open(blogDir) + assert.NoError(err) + f.Close() + f, err = fs.Open(blogFile) + assert.NoError(err) + f.Close() + + // os.OpenFile(logFile, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0666) + +} diff --git a/hugofs/rootmapping_fs.go b/hugofs/rootmapping_fs.go index 2b8b8d2c012..a1214a02c7f 100644 --- a/hugofs/rootmapping_fs.go +++ b/hugofs/rootmapping_fs.go @@ -1,4 +1,4 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. +// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,10 +14,14 @@ package hugofs import ( + "fmt" "os" "path/filepath" "strings" - "time" + + "github.com/gohugoio/hugo/hugofs/files" + + "github.com/pkg/errors" radix "github.com/hashicorp/go-immutable-radix" "github.com/spf13/afero" @@ -25,151 +29,429 @@ import ( var filepathSeparator = string(filepath.Separator) -// A RootMappingFs maps several roots into one. Note that the root of this filesystem -// is directories only, and they will be returned in Readdir and Readdirnames -// in the order given. -type RootMappingFs struct { - afero.Fs - rootMapToReal *radix.Node - virtualRoots []string +// NewRootMappingFs creates a new RootMappingFs on top of the provided with +// of root mappings with some optional metadata about the root. +// Note that From represents a virtual root that maps to the actual filename in To. +func NewRootMappingFs(fs afero.Fs, rms ...RootMapping) (*RootMappingFs, error) { + rootMapToReal := radix.New().Txn() + + for _, rm := range rms { + (&rm).clean() + + fromBase := files.ResolveComponentFolder(rm.From) + if fromBase == "" { + panic("unrecognised component folder in" + rm.From) + } + + if len(rm.To) < 2 { + panic(fmt.Sprintf("invalid root mapping; from/to: %s/%s", rm.From, rm.To)) + } + + _, err := fs.Stat(rm.To) + if err != nil { + if os.IsNotExist(err) { + continue + } + return nil, err + } + + // Extract "blog" from "content/blog" + rm.path = strings.TrimPrefix(strings.TrimPrefix(rm.From, fromBase), filepathSeparator) + + key := []byte(rm.rootKey()) + var mappings []RootMapping + v, found := rootMapToReal.Get(key) + if found { + // There may be more than one language pointing to the same root. + mappings = v.([]RootMapping) + } + mappings = append(mappings, rm) + rootMapToReal.Insert(key, mappings) + } + + rfs := &RootMappingFs{Fs: fs, + virtualRoots: rms, + rootMapToReal: rootMapToReal.Commit().Root()} + + return rfs, nil } -type rootMappingFile struct { - afero.File - fs *RootMappingFs - name string +// NewRootMappingFsFromFromTo is a convenicence variant of NewRootMappingFs taking +// From and To as string pairs. +func NewRootMappingFsFromFromTo(fs afero.Fs, fromTo ...string) (*RootMappingFs, error) { + rms := make([]RootMapping, len(fromTo)/2) + for i, j := 0, 0; j < len(fromTo); i, j = i+1, j+2 { + rms[i] = RootMapping{ + From: fromTo[j], + To: fromTo[j+1], + } + } + + return NewRootMappingFs(fs, rms...) } -type rootMappingFileInfo struct { - name string +type RootMapping struct { + From string + To string + + path string // The virtual mount point, e.g. "blog". + Meta FileMeta // File metadata (lang etc.) } -func (fi *rootMappingFileInfo) Name() string { - return fi.name +func (rm *RootMapping) clean() { + rm.From = filepath.Clean(rm.From) + rm.To = filepath.Clean(rm.To) } -func (fi *rootMappingFileInfo) Size() int64 { - panic("not implemented") +func (r RootMapping) filename(name string) string { + return filepath.Join(r.To, strings.TrimPrefix(name, r.From)) } -func (fi *rootMappingFileInfo) Mode() os.FileMode { - return os.ModeDir +func (r RootMapping) rootKey() string { + return r.From } -func (fi *rootMappingFileInfo) ModTime() time.Time { - panic("not implemented") +// A RootMappingFs maps several roots into one. Note that the root of this filesystem +// is directories only, and they will be returned in Readdir and Readdirnames +// in the order given. +type RootMappingFs struct { + afero.Fs + rootMapToReal *radix.Node + virtualRoots []RootMapping + filter func(r RootMapping) bool } -func (fi *rootMappingFileInfo) IsDir() bool { - return true +func (fs *RootMappingFs) Dirs(base string) ([]FileMetaInfo, error) { + roots := fs.getRootsWithPrefix(base) + + if roots == nil { + return nil, nil + } + + fss := make([]FileMetaInfo, len(roots)) + for i, r := range roots { + bfs := afero.NewBasePathFs(fs.Fs, r.To) + bfs = decoratePath(bfs, func(name string) string { + p := strings.TrimPrefix(name, r.To) + if r.path != "" { + // Make sure it's mounted to a any sub path, e.g. blog + p = filepath.Join(r.path, p) + } + p = strings.TrimLeft(p, filepathSeparator) + return p + }) + fs := decorateDirs(bfs, r.Meta) + fi, err := fs.Stat("") + if err != nil { + return nil, errors.Wrap(err, "RootMappingFs.Dirs") + } + fss[i] = fi.(FileMetaInfo) + } + + return fss, nil } -func (fi *rootMappingFileInfo) Sys() interface{} { - return nil +// LstatIfPossible returns the os.FileInfo structure describing a given file. +func (fs *RootMappingFs) LstatIfPossible(name string) (os.FileInfo, bool, error) { + fis, b, err := fs.doLstat(name, false) + if err != nil { + return nil, b, err + } + return fis[0], b, nil + } -func newRootMappingDirFileInfo(name string) *rootMappingFileInfo { - return &rootMappingFileInfo{name: name} +func (fs *RootMappingFs) virtualDirOpener(name string, isRoot bool) func() (afero.File, error) { + return func() (afero.File, error) { return &rootMappingFile{name: name, isRoot: isRoot, fs: fs}, nil } } -// NewRootMappingFs creates a new RootMappingFs on top of the provided with -// a list of from, to string pairs of root mappings. -// Note that 'from' represents a virtual root that maps to the actual filename in 'to'. -func NewRootMappingFs(fs afero.Fs, fromTo ...string) (*RootMappingFs, error) { - rootMapToReal := radix.New().Txn() - var virtualRoots []string +func (fs *RootMappingFs) doLstat(name string, allowMultiple bool) ([]FileMetaInfo, bool, error) { + + if fs.isRoot(name) { + return []FileMetaInfo{newDirNameOnlyFileInfo(name, true, fs.virtualDirOpener(name, true))}, false, nil + } + + roots := fs.getRoots(name) - for i := 0; i < len(fromTo); i += 2 { - vr := filepath.Clean(fromTo[i]) - rr := filepath.Clean(fromTo[i+1]) + if len(roots) == 0 { + roots := fs.getRootsWithPrefix(name) + if len(roots) != 0 { + // We have root mappings below name, let's make it look like + // a directory. + return []FileMetaInfo{newDirNameOnlyFileInfo(name, true, fs.virtualDirOpener(name, false))}, false, nil + } + + return nil, false, os.ErrNotExist + } - // We need to preserve the original order for Readdir - virtualRoots = append(virtualRoots, vr) + var ( + fis []FileMetaInfo + b bool + fi os.FileInfo + root RootMapping + err error + ) + + for _, root = range roots { + fi, b, err = fs.statRoot(root, name) + if err != nil { + if os.IsNotExist(err) { + continue + } + return nil, false, err + } + fim := fi.(FileMetaInfo) + fis = append(fis, fim) + } - rootMapToReal.Insert([]byte(vr), rr) + if len(fis) == 0 { + return nil, false, os.ErrNotExist } - return &RootMappingFs{Fs: fs, - virtualRoots: virtualRoots, - rootMapToReal: rootMapToReal.Commit().Root()}, nil + if allowMultiple || len(fis) == 1 { + return fis, b, nil + } + + // Open it in this composite filesystem. + opener := func() (afero.File, error) { + return fs.Open(name) + } + + return []FileMetaInfo{decorateFileInfo(fi, fs, opener, "", "", root.Meta)}, b, nil + } -// Stat returns the os.FileInfo structure describing a given file. If there is -// an error, it will be of type *os.PathError. -func (fs *RootMappingFs) Stat(name string) (os.FileInfo, error) { +// Open opens the namedrootMappingFile file for reading. +func (fs *RootMappingFs) Open(name string) (afero.File, error) { if fs.isRoot(name) { - return newRootMappingDirFileInfo(name), nil + return &rootMappingFile{name: name, fs: fs, isRoot: true}, nil } - realName := fs.realName(name) - fi, err := fs.Fs.Stat(realName) - if rfi, ok := fi.(RealFilenameInfo); ok { - return rfi, err + fis, _, err := fs.doLstat(name, true) + if err != nil { + return nil, err } - return &realFilenameInfo{FileInfo: fi, realFilename: realName}, err + if len(fis) == 1 { + fi := fis[0] + meta := fi.(FileMetaInfo).Meta() + f, err := meta.Open() + if err != nil { + return nil, err + } + return &rootMappingFile{File: f, fs: fs, name: name, meta: meta}, nil + } + + return fs.newUnionFile(fis...) } +// Stat returns the os.FileInfo structure describing a given file. If there is +// an error, it will be of type *os.PathError. +func (fs *RootMappingFs) Stat(name string) (os.FileInfo, error) { + fi, _, err := fs.LstatIfPossible(name) + return fi, err + +} + +// Filter creates a copy of this filesystem with the applied filter. +func (fs RootMappingFs) Filter(f func(m RootMapping) bool) *RootMappingFs { + fs.filter = f + return &fs +} + func (fs *RootMappingFs) isRoot(name string) bool { return name == "" || name == filepathSeparator } -// Open opens the named file for reading. -func (fs *RootMappingFs) Open(name string) (afero.File, error) { - if fs.isRoot(name) { - return &rootMappingFile{name: name, fs: fs}, nil +func (fs *RootMappingFs) getRoots(name string) []RootMapping { + nameb := []byte(filepath.Clean(name)) + _, v, found := fs.rootMapToReal.LongestPrefix(nameb) + if !found { + return nil } - realName := fs.realName(name) - f, err := fs.Fs.Open(realName) + + rm := v.([]RootMapping) + + if fs.filter != nil { + var filtered []RootMapping + for _, m := range rm { + if fs.filter(m) { + filtered = append(filtered, m) + } + } + return filtered + } + + return rm +} + +func (fs *RootMappingFs) getRootsWithPrefix(prefix string) []RootMapping { + if fs.isRoot(prefix) { + return fs.virtualRoots + } + prefixb := []byte(filepath.Clean(prefix)) + var roots []RootMapping + + fs.rootMapToReal.WalkPrefix(prefixb, func(b []byte, v interface{}) bool { + roots = append(roots, v.([]RootMapping)...) + return false + }) + + return roots +} + +func (fs *RootMappingFs) newUnionFile(fis ...FileMetaInfo) (afero.File, error) { + meta := fis[0].Meta() + f, err := meta.Open() if err != nil { return nil, err } - return &rootMappingFile{File: f, name: name, fs: fs}, nil -} + rf := &rootMappingFile{File: f, fs: fs, name: meta.Name(), meta: meta} + if len(fis) == 1 { + return rf, err + } -// LstatIfPossible returns the os.FileInfo structure describing a given file. -// It attempts to use Lstat if supported or defers to the os. In addition to -// the FileInfo, a boolean is returned telling whether Lstat was called. -func (fs *RootMappingFs) LstatIfPossible(name string) (os.FileInfo, bool, error) { + next, err := fs.newUnionFile(fis[1:]...) + if err != nil { + return nil, err + } - if fs.isRoot(name) { - return newRootMappingDirFileInfo(name), false, nil + uf := &afero.UnionFile{Base: rf, Layer: next} + + uf.Merger = func(lofi, bofi []os.FileInfo) ([]os.FileInfo, error) { + // Ignore duplicate directory entries + seen := make(map[string]bool) + var result []os.FileInfo + + for _, fis := range [][]os.FileInfo{bofi, lofi} { + for _, fi := range fis { + + if fi.IsDir() && seen[fi.Name()] { + continue + } + + if fi.IsDir() { + seen[fi.Name()] = true + } + + result = append(result, fi) + } + } + + return result, nil } - name = fs.realName(name) + + return uf, nil + +} + +func (fs *RootMappingFs) statRoot(root RootMapping, name string) (os.FileInfo, bool, error) { + filename := root.filename(name) + + var b bool + var fi os.FileInfo + var err error if ls, ok := fs.Fs.(afero.Lstater); ok { - fi, b, err := ls.LstatIfPossible(name) - return &realFilenameInfo{FileInfo: fi, realFilename: name}, b, err + fi, b, err = ls.LstatIfPossible(filename) + if err != nil { + return nil, b, err + } + + } else { + fi, err = fs.Fs.Stat(filename) + if err != nil { + return nil, b, err + } + } + + // Opens the real directory/file. + opener := func() (afero.File, error) { + return fs.Fs.Open(filename) + } + + if fi.IsDir() { + _, name = filepath.Split(name) + fi = newDirNameOnlyFileInfo(name, false, opener) } - fi, err := fs.Stat(name) - return fi, false, err + + return decorateFileInfo(fi, fs.Fs, opener, "", "", root.Meta), b, nil + } -func (fs *RootMappingFs) realName(name string) string { - key, val, found := fs.rootMapToReal.LongestPrefix([]byte(filepath.Clean(name))) - if !found { - return name +type rootMappingFile struct { + afero.File + fs *RootMappingFs + name string + meta FileMeta + isRoot bool +} + +func (f *rootMappingFile) Close() error { + if f.File == nil { + return nil } - keystr := string(key) + return f.File.Close() +} - return filepath.Join(val.(string), strings.TrimPrefix(name, keystr)) +func (f *rootMappingFile) Name() string { + return f.name } func (f *rootMappingFile) Readdir(count int) ([]os.FileInfo, error) { if f.File == nil { dirsn := make([]os.FileInfo, 0) - for i := 0; i < len(f.fs.virtualRoots); i++ { - if count != -1 && i >= count { + roots := f.fs.getRootsWithPrefix(f.name) + seen := make(map[string]bool) + + j := 0 + for _, rm := range roots { + if count != -1 && j >= count { break } - dirsn = append(dirsn, newRootMappingDirFileInfo(f.fs.virtualRoots[i])) + + opener := func() (afero.File, error) { + return f.fs.Open(rm.From) + } + + name := rm.From + if !f.isRoot { + _, name = filepath.Split(rm.From) + } + + if seen[name] { + continue + } + seen[name] = true + + j++ + + fi := newDirNameOnlyFileInfo(name, false, opener) + if rm.Meta != nil { + mergeFileMeta(rm.Meta, fi.Meta()) + } + + dirsn = append(dirsn, fi) } return dirsn, nil } - return f.File.Readdir(count) + if f.File == nil { + panic(fmt.Sprintf("no File for %q", f.name)) + } + + fis, err := f.File.Readdir(count) + if err != nil { + return nil, err + } + + for i, fi := range fis { + fis[i] = decorateFileInfo(fi, f.fs, nil, "", "", f.meta) + } + + return fis, nil } func (f *rootMappingFile) Readdirnames(count int) ([]string, error) { @@ -183,14 +465,3 @@ func (f *rootMappingFile) Readdirnames(count int) ([]string, error) { } return dirss, nil } - -func (f *rootMappingFile) Name() string { - return f.name -} - -func (f *rootMappingFile) Close() error { - if f.File == nil { - return nil - } - return f.File.Close() -} diff --git a/hugofs/rootmapping_fs_test.go b/hugofs/rootmapping_fs_test.go index e6a0301c941..3845ad5e623 100644 --- a/hugofs/rootmapping_fs_test.go +++ b/hugofs/rootmapping_fs_test.go @@ -1,4 +1,4 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. +// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -19,24 +19,115 @@ import ( "path/filepath" "testing" + "github.com/spf13/viper" + + "github.com/gohugoio/hugo/htesting" "github.com/spf13/afero" "github.com/stretchr/testify/require" ) -func TestRootMappingFsRealName(t *testing.T) { +func TestLanguageRootMapping(t *testing.T) { assert := require.New(t) - fs := afero.NewMemMapFs() + v := viper.New() + v.Set("contentDir", "content") + + fs := NewBaseFileDecorator(afero.NewMemMapFs()) + + assert.NoError(afero.WriteFile(fs, filepath.Join("content/sv/svdir", "main.txt"), []byte("main sv"), 0755)) + assert.NoError(afero.WriteFile(fs, filepath.Join("themes/a/mysvblogcontent", "sv-f.txt"), []byte("some sv blog content"), 0755)) + assert.NoError(afero.WriteFile(fs, filepath.Join("themes/a/myenblogcontent", "en-f.txt"), []byte("some en blog content in a"), 0755)) + assert.NoError(afero.WriteFile(fs, filepath.Join("themes/a/myotherenblogcontent", "en-f2.txt"), []byte("some en content"), 0755)) + assert.NoError(afero.WriteFile(fs, filepath.Join("themes/a/mysvdocs", "sv-docs.txt"), []byte("some sv docs content"), 0755)) + assert.NoError(afero.WriteFile(fs, filepath.Join("themes/b/myenblogcontent", "en-b-f.txt"), []byte("some en content"), 0755)) + + rfs, err := NewRootMappingFs(fs, + RootMapping{ + From: "content/blog", // Virtual path, first element is one of content, static, layouts etc. + To: "themes/a/mysvblogcontent", // Real path + Meta: FileMeta{"lang": "sv"}, + }, + RootMapping{ + From: "content/blog", + To: "themes/a/myenblogcontent", + Meta: FileMeta{"lang": "en"}, + }, + RootMapping{ + From: "content/blog", + To: "content/sv", + Meta: FileMeta{"lang": "sv"}, + }, + RootMapping{ + From: "content/blog", + To: "themes/a/myotherenblogcontent", + Meta: FileMeta{"lang": "en"}, + }, + RootMapping{ + From: "content/docs", + To: "themes/a/mysvdocs", + Meta: FileMeta{"lang": "sv"}, + }, + ) - rfs, err := NewRootMappingFs(fs, "f1", "f1t", "f2", "f2t") assert.NoError(err) - assert.Equal(filepath.FromSlash("f1t/foo/file.txt"), rfs.realName(filepath.Join("f1", "foo", "file.txt"))) + collected, err := collectFilenames(rfs, "content", "content") + assert.NoError(err) + assert.Equal([]string{"blog/en-f.txt", "blog/en-f2.txt", "blog/sv-f.txt", "blog/svdir/main.txt", "docs/sv-docs.txt"}, collected) + + bfs := afero.NewBasePathFs(rfs, "content") + collected, err = collectFilenames(bfs, "", "") + assert.NoError(err) + assert.Equal([]string{"blog/en-f.txt", "blog/en-f2.txt", "blog/sv-f.txt", "blog/svdir/main.txt", "docs/sv-docs.txt"}, collected) + + dirs, err := rfs.Dirs(filepath.FromSlash("content/blog")) + assert.NoError(err) + + assert.Equal(4, len(dirs)) + + getDirnames := func(name string, rfs *RootMappingFs) []string { + filename := filepath.FromSlash(name) + f, err := rfs.Open(filename) + assert.NoError(err) + names, err := f.Readdirnames(-1) + + f.Close() + assert.NoError(err) + + info, err := rfs.Stat(filename) + assert.NoError(err) + f2, err := info.(FileMetaInfo).Meta().Open() + assert.NoError(err) + names2, err := f2.Readdirnames(-1) + assert.NoError(err) + assert.Equal(names, names2) + f2.Close() + + return names + } + + rfsEn := rfs.Filter(func(rm RootMapping) bool { + return rm.Meta.Lang() == "en" + }) + + assert.Equal([]string{"en-f.txt", "en-f2.txt"}, getDirnames("content/blog", rfsEn)) + + rfsSv := rfs.Filter(func(rm RootMapping) bool { + return rm.Meta.Lang() == "sv" + }) + + assert.Equal([]string{"sv-f.txt", "svdir"}, getDirnames("content/blog", rfsSv)) + + // Make sure we have not messed with the original + assert.Equal([]string{"sv-f.txt", "en-f.txt", "svdir", "en-f2.txt"}, getDirnames("content/blog", rfs)) + + assert.Equal([]string{"blog", "docs"}, getDirnames("content", rfsSv)) + assert.Equal([]string{"blog", "docs"}, getDirnames("content", rfs)) } func TestRootMappingFsDirnames(t *testing.T) { assert := require.New(t) - fs := afero.NewMemMapFs() + fs := NewBaseFileDecorator(afero.NewMemMapFs()) testfile := "myfile.txt" assert.NoError(fs.Mkdir("f1t", 0755)) @@ -44,13 +135,14 @@ func TestRootMappingFsDirnames(t *testing.T) { assert.NoError(fs.Mkdir("f3t", 0755)) assert.NoError(afero.WriteFile(fs, filepath.Join("f2t", testfile), []byte("some content"), 0755)) - rfs, err := NewRootMappingFs(fs, "bf1", "f1t", "cf2", "f2t", "af3", "f3t") + rfs, err := NewRootMappingFsFromFromTo(fs, "static/bf1", "f1t", "static/cf2", "f2t", "static/af3", "f3t") assert.NoError(err) - fif, err := rfs.Stat(filepath.Join("cf2", testfile)) + fif, err := rfs.Stat(filepath.Join("static/cf2", testfile)) assert.NoError(err) assert.Equal("myfile.txt", fif.Name()) - assert.Equal(filepath.FromSlash("f2t/myfile.txt"), fif.(RealFilenameInfo).RealFilename()) + fifm := fif.(FileMetaInfo).Meta() + assert.Equal(filepath.FromSlash("f2t/myfile.txt"), fifm.Filename()) root, err := rfs.Open(filepathSeparator) assert.NoError(err) @@ -61,6 +153,91 @@ func TestRootMappingFsDirnames(t *testing.T) { } +func TestRootMappingFsFilename(t *testing.T) { + assert := require.New(t) + workDir, clean, err := htesting.CreateTempDir(Os, "hugo-root-filename") + assert.NoError(err) + defer clean() + fs := NewBaseFileDecorator(Os) + + testfilename := filepath.Join(workDir, "f1t/foo/file.txt") + + assert.NoError(fs.MkdirAll(filepath.Join(workDir, "f1t/foo"), 0777)) + assert.NoError(afero.WriteFile(fs, testfilename, []byte("content"), 0666)) + + rfs, err := NewRootMappingFsFromFromTo(fs, "static/f1", filepath.Join(workDir, "f1t"), "static/f2", filepath.Join(workDir, "f2t")) + assert.NoError(err) + + fi, err := rfs.Stat(filepath.FromSlash("static/f1/foo/file.txt")) + assert.NoError(err) + fim := fi.(FileMetaInfo) + assert.Equal(testfilename, fim.Meta().Filename()) + _, err = rfs.Stat(filepath.FromSlash("static/f1")) + assert.NoError(err) +} + +func TestRootMappingFsMount(t *testing.T) { + assert := require.New(t) + fs := NewBaseFileDecorator(afero.NewMemMapFs()) + + testfile := "test.txt" + + assert.NoError(afero.WriteFile(fs, filepath.Join("themes/a/mynoblogcontent", testfile), []byte("some no content"), 0755)) + assert.NoError(afero.WriteFile(fs, filepath.Join("themes/a/myenblogcontent", testfile), []byte("some en content"), 0755)) + assert.NoError(afero.WriteFile(fs, filepath.Join("themes/a/mysvblogcontent", testfile), []byte("some sv content"), 0755)) + assert.NoError(afero.WriteFile(fs, filepath.Join("themes/a/mysvblogcontent", "other.txt"), []byte("some sv content"), 0755)) + + bfs := afero.NewBasePathFs(fs, "themes/a").(*afero.BasePathFs) + rm := []RootMapping{ + RootMapping{From: "content/blog", + To: "mynoblogcontent", + Meta: FileMeta{"lang": "no"}, + }, + RootMapping{From: "content/blog", + To: "myenblogcontent", + Meta: FileMeta{"lang": "en"}, + }, + RootMapping{From: "content/blog", + To: "mysvblogcontent", + Meta: FileMeta{"lang": "sv"}, + }, + } + + rfs, err := NewRootMappingFs(bfs, rm...) + assert.NoError(err) + + blog, err := rfs.Stat(filepath.FromSlash("content/blog")) + assert.NoError(err) + blogm := blog.(FileMetaInfo).Meta() + assert.Equal("sv", blogm.Lang()) // Last match + + f, err := blogm.Open() + assert.NoError(err) + defer f.Close() + dirs1, err := f.Readdirnames(-1) + assert.NoError(err) + // Union with duplicate dir names filtered. + assert.Equal([]string{"test.txt", "test.txt", "other.txt", "test.txt"}, dirs1) + + files, err := afero.ReadDir(rfs, filepath.FromSlash("content/blog")) + assert.NoError(err) + assert.Equal(4, len(files)) + + testfilefi := files[1] + assert.Equal(testfile, testfilefi.Name()) + + testfilem := testfilefi.(FileMetaInfo).Meta() + assert.Equal(filepath.FromSlash("themes/a/mynoblogcontent/test.txt"), testfilem.Filename()) + + tf, err := testfilem.Open() + assert.NoError(err) + defer tf.Close() + c, err := ioutil.ReadAll(tf) + assert.NoError(err) + assert.Equal("some no content", string(c)) + +} + func TestRootMappingFsOs(t *testing.T) { assert := require.New(t) fs := afero.NewOsFs() @@ -77,10 +254,10 @@ func TestRootMappingFsOs(t *testing.T) { assert.NoError(fs.Mkdir(filepath.Join(d, "f3t"), 0755)) assert.NoError(afero.WriteFile(fs, filepath.Join(d, "f2t", testfile), []byte("some content"), 0755)) - rfs, err := NewRootMappingFs(fs, "bf1", filepath.Join(d, "f1t"), "cf2", filepath.Join(d, "f2t"), "af3", filepath.Join(d, "f3t")) + rfs, err := NewRootMappingFsFromFromTo(fs, "static/bf1", filepath.Join(d, "f1t"), "static/cf2", filepath.Join(d, "f2t"), "static/af3", filepath.Join(d, "f3t")) assert.NoError(err) - fif, err := rfs.Stat(filepath.Join("cf2", testfile)) + fif, err := rfs.Stat(filepath.Join("static/cf2", testfile)) assert.NoError(err) assert.Equal("myfile.txt", fif.Name()) diff --git a/hugofs/slice_fs.go b/hugofs/slice_fs.go new file mode 100644 index 00000000000..4fb026ab288 --- /dev/null +++ b/hugofs/slice_fs.go @@ -0,0 +1,293 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugofs + +import ( + "os" + "syscall" + "time" + + "github.com/pkg/errors" + + "github.com/spf13/afero" +) + +var ( + _ afero.Fs = (*SliceFs)(nil) + _ afero.Lstater = (*SliceFs)(nil) + _ afero.File = (*sliceDir)(nil) +) + +func NewSliceFs(dirs ...FileMetaInfo) (afero.Fs, error) { + if len(dirs) == 0 { + return NoOpFs, nil + } + + for _, dir := range dirs { + if !dir.IsDir() { + return nil, errors.New("this fs supports directories only") + } + } + + fs := &SliceFs{ + dirs: dirs, + } + + return fs, nil + +} + +// SliceFs is an ordered composite filesystem. +type SliceFs struct { + dirs []FileMetaInfo +} + +func (fs *SliceFs) Chmod(n string, m os.FileMode) error { + return syscall.EPERM +} + +func (fs *SliceFs) Chtimes(n string, a, m time.Time) error { + return syscall.EPERM +} + +func (fs *SliceFs) LstatIfPossible(name string) (os.FileInfo, bool, error) { + fi, _, err := fs.pickFirst(name) + + if err != nil { + return nil, false, err + } + + if fi.IsDir() { + return decorateFileInfo(fi, fs, fs.getOpener(name), "", "", nil), false, nil + } + + return nil, false, errors.Errorf("lstat: files not supported: %q", name) + +} + +func (fs *SliceFs) Mkdir(n string, p os.FileMode) error { + return syscall.EPERM +} + +func (fs *SliceFs) MkdirAll(n string, p os.FileMode) error { + return syscall.EPERM +} + +func (fs *SliceFs) Name() string { + return "SliceFs" +} + +func (fs *SliceFs) Open(name string) (afero.File, error) { + fi, idx, err := fs.pickFirst(name) + if err != nil { + return nil, err + } + + if !fi.IsDir() { + panic("currently only dirs in here") + } + + return &sliceDir{ + lfs: fs, + idx: idx, + dirname: name, + }, nil + +} + +func (fs *SliceFs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) { + panic("not implemented") +} + +func (fs *SliceFs) ReadDir(name string) ([]os.FileInfo, error) { + panic("not implemented") +} + +func (fs *SliceFs) Remove(n string) error { + return syscall.EPERM +} + +func (fs *SliceFs) RemoveAll(p string) error { + return syscall.EPERM +} + +func (fs *SliceFs) Rename(o, n string) error { + return syscall.EPERM +} + +func (fs *SliceFs) Stat(name string) (os.FileInfo, error) { + fi, _, err := fs.LstatIfPossible(name) + return fi, err +} + +func (fs *SliceFs) Create(n string) (afero.File, error) { + return nil, syscall.EPERM +} + +func (fs *SliceFs) getOpener(name string) func() (afero.File, error) { + return func() (afero.File, error) { + return fs.Open(name) + } +} + +func (fs *SliceFs) pickFirst(name string) (os.FileInfo, int, error) { + for i, mfs := range fs.dirs { + meta := mfs.Meta() + fs := meta.Fs() + fi, _, err := lstatIfPossible(fs, name) + if err == nil { + // Gotta match! + return fi, i, nil + } + + if !os.IsNotExist(err) { + // Real error + return nil, -1, err + } + } + + // Not found + return nil, -1, os.ErrNotExist +} + +func (fs *SliceFs) readDirs(name string, startIdx, count int) ([]os.FileInfo, error) { + collect := func(lfs FileMeta) ([]os.FileInfo, error) { + d, err := lfs.Fs().Open(name) + if err != nil { + if !os.IsNotExist(err) { + return nil, err + } + return nil, nil + } else { + defer d.Close() + dirs, err := d.Readdir(-1) + if err != nil { + return nil, err + } + return dirs, nil + } + } + + var dirs []os.FileInfo + + for i := startIdx; i < len(fs.dirs); i++ { + mfs := fs.dirs[i] + + fis, err := collect(mfs.Meta()) + if err != nil { + return nil, err + } + + dirs = append(dirs, fis...) + + } + + seen := make(map[string]bool) + var duplicates []int + for i, fi := range dirs { + if !fi.IsDir() { + continue + } + + if seen[fi.Name()] { + duplicates = append(duplicates, i) + } else { + // Make sure it's opened by this filesystem. + dirs[i] = decorateFileInfo(fi, fs, fs.getOpener(fi.(FileMetaInfo).Meta().Filename()), "", "", nil) + seen[fi.Name()] = true + } + } + + // Remove duplicate directories, keep first. + if len(duplicates) > 0 { + for i := len(duplicates) - 1; i >= 0; i-- { + idx := duplicates[i] + dirs = append(dirs[:idx], dirs[idx+1:]...) + } + } + + if count > 0 && len(dirs) >= count { + return dirs[:count], nil + } + + return dirs, nil + +} + +type sliceDir struct { + lfs *SliceFs + idx int + dirname string +} + +func (f *sliceDir) Close() error { + return nil +} + +func (f *sliceDir) Name() string { + return f.dirname +} + +func (f *sliceDir) Read(p []byte) (n int, err error) { + panic("not implemented") +} + +func (f *sliceDir) ReadAt(p []byte, off int64) (n int, err error) { + panic("not implemented") +} + +func (f *sliceDir) Readdir(count int) ([]os.FileInfo, error) { + return f.lfs.readDirs(f.dirname, f.idx, count) +} + +func (f *sliceDir) Readdirnames(count int) ([]string, error) { + dirsi, err := f.Readdir(count) + if err != nil { + return nil, err + } + + dirs := make([]string, len(dirsi)) + for i, d := range dirsi { + dirs[i] = d.Name() + } + return dirs, nil +} + +func (f *sliceDir) Seek(offset int64, whence int) (int64, error) { + panic("not implemented") +} + +func (f *sliceDir) Stat() (os.FileInfo, error) { + panic("not implemented") +} + +func (f *sliceDir) Sync() error { + panic("not implemented") +} + +func (f *sliceDir) Truncate(size int64) error { + panic("not implemented") +} + +func (f *sliceDir) Write(p []byte) (n int, err error) { + panic("not implemented") +} + +func (f *sliceDir) WriteAt(p []byte, off int64) (n int, err error) { + panic("not implemented") +} + +func (f *sliceDir) WriteString(s string) (ret int, err error) { + panic("not implemented") +} diff --git a/hugofs/walk.go b/hugofs/walk.go new file mode 100644 index 00000000000..eca74673746 --- /dev/null +++ b/hugofs/walk.go @@ -0,0 +1,308 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugofs + +import ( + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/gohugoio/hugo/common/loggers" + + "github.com/pkg/errors" + + "github.com/spf13/afero" +) + +type ( + WalkFunc func(path string, info FileMetaInfo, err error) error + WalkHook func(dir FileMetaInfo, path string, readdir []FileMetaInfo) ([]FileMetaInfo, error) +) + +type Walkway struct { + fs afero.Fs + root string + basePath string + + logger *loggers.Logger + + // May be pre-set + fi FileMetaInfo + dirEntries []FileMetaInfo + + walkFn WalkFunc + walked bool + + // We may traverse symbolic links and bite ourself. + seen map[string]bool + + // Optional hooks + hookPre WalkHook + hookPost WalkHook +} + +type WalkwayConfig struct { + Fs afero.Fs + Root string + BasePath string + + Logger *loggers.Logger + + // One or both of these may be pre-set. + Info FileMetaInfo + DirEntries []FileMetaInfo + + WalkFn WalkFunc + HookPre WalkHook + HookPost WalkHook +} + +func NewWalkway(cfg WalkwayConfig) *Walkway { + var fs afero.Fs + if cfg.Info != nil { + fs = cfg.Info.Meta().Fs() + } else { + fs = cfg.Fs + } + + basePath := cfg.BasePath + if basePath != "" && !strings.HasSuffix(basePath, filepathSeparator) { + basePath += filepathSeparator + } + + logger := cfg.Logger + if logger == nil { + logger = loggers.NewWarningLogger() + } + + return &Walkway{ + fs: fs, + root: cfg.Root, + basePath: basePath, + fi: cfg.Info, + dirEntries: cfg.DirEntries, + walkFn: cfg.WalkFn, + hookPre: cfg.HookPre, + hookPost: cfg.HookPost, + logger: logger, + seen: make(map[string]bool)} +} + +func (w *Walkway) Walk() error { + if w.walked { + panic("this walkway is already walked") + } + w.walked = true + + if w.fs == NoOpFs { + return nil + } + + var fi FileMetaInfo + if w.fi != nil { + fi = w.fi + } else { + info, _, err := lstatIfPossible(w.fs, w.root) + if err != nil { + if os.IsNotExist(err) { + return nil + } + + if err == ErrPermissionSymlink { + w.logger.WARN.Printf("Unsupported symlink found in %q, skipping.", w.root) + return nil + } + + return w.walkFn(w.root, nil, errors.Wrapf(err, "walk: %q", w.root)) + } + fi = info.(FileMetaInfo) + } + + if !fi.IsDir() { + return w.walkFn(w.root, nil, errors.New("file to walk must be a directory")) + } + + return w.walk(w.root, fi, w.dirEntries, w.walkFn) + +} + +// if the filesystem supports it, use Lstat, else use fs.Stat +func lstatIfPossible(fs afero.Fs, path string) (os.FileInfo, bool, error) { + if lfs, ok := fs.(afero.Lstater); ok { + fi, b, err := lfs.LstatIfPossible(path) + return fi, b, err + } + fi, err := fs.Stat(path) + return fi, false, err +} + +// walk recursively descends path, calling walkFn. +// It follow symlinks if supported by the filesystem, but only the same path once. +func (w *Walkway) walk(path string, info FileMetaInfo, dirEntries []FileMetaInfo, walkFn WalkFunc) error { + err := walkFn(path, info, nil) + if err != nil { + if info.IsDir() && err == filepath.SkipDir { + return nil + } + return err + } + if !info.IsDir() { + return nil + } + + meta := info.Meta() + filename := meta.Filename() + + if dirEntries == nil { + f, err := w.fs.Open(path) + + if err != nil { + return walkFn(path, info, errors.Wrapf(err, "walk: open %q (%q)", path, w.root)) + } + + fis, err := f.Readdir(-1) + f.Close() + if err != nil { + if err == ErrPermissionSymlink { + w.logger.WARN.Printf("Unsupported symlink found in %q, skipping.", filename) + return nil + } + return walkFn(path, info, errors.Wrap(err, "walk: Readdir")) + } + + dirEntries = fileInfosToFileMetaInfos(fis) + + if !meta.IsOrdered() { + sort.Slice(dirEntries, func(i, j int) bool { + fii := dirEntries[i] + fij := dirEntries[j] + + fim, fjm := fii.Meta(), fij.Meta() + + // Pull bundle headers to the top. + ficlass, fjclass := fim.Classifier(), fjm.Classifier() + if ficlass != fjclass { + return ficlass < fjclass + } + + // With multiple content dirs with different languages, + // there can be duplicate files, and a weight will be added + // to the closest one. + fiw, fjw := fim.Weight(), fjm.Weight() + if fiw != fjw { + return fiw > fjw + } + + // Explicit order set. + fio, fjo := fim.Ordinal(), fjm.Ordinal() + if fio != fjo { + return fio < fjo + } + + // When we walk into a symlink, we keep the reference to + // the original name. + fin, fjn := fim.Name(), fjm.Name() + if fin != "" && fjn != "" { + return fin < fjn + } + + return fii.Name() < fij.Name() + }) + } + } + + // First add some metadata to the dir entries + for _, fi := range dirEntries { + fim := fi.(FileMetaInfo) + + meta := fim.Meta() + + // Note that we use the original Name even if it's a symlink. + name := meta.Name() + if name == "" { + name = fim.Name() + } + + if name == "" { + panic(fmt.Sprintf("[%s] no name set in %v", path, meta)) + } + pathn := filepath.Join(path, name) + + pathMeta := pathn + if w.basePath != "" { + pathMeta = strings.TrimPrefix(pathn, w.basePath) + } + + meta[metaKeyPath] = normalizeFilename(pathMeta) + meta[metaKeyPathWalk] = pathn + + if fim.IsDir() && w.isSeen(meta.Filename()) { + // Prevent infinite recursion + // Possible cyclic reference + meta[metaKeySkipDir] = true + } + } + + if w.hookPre != nil { + dirEntries, err = w.hookPre(info, path, dirEntries) + if err != nil { + if err == filepath.SkipDir { + return nil + } + return err + } + } + + for _, fi := range dirEntries { + fim := fi.(FileMetaInfo) + meta := fim.Meta() + + if meta.SkipDir() { + continue + } + + err := w.walk(meta.GetString(metaKeyPathWalk), fim, nil, walkFn) + if err != nil { + if !fi.IsDir() || err != filepath.SkipDir { + return err + } + } + } + + if w.hookPost != nil { + dirEntries, err = w.hookPost(info, path, dirEntries) + if err != nil { + if err == filepath.SkipDir { + return nil + } + return err + } + } + return nil +} + +func (w *Walkway) isSeen(filename string) bool { + if filename == "" { + return false + } + + if w.seen[filename] { + return true + } + + w.seen[filename] = true + return false +} diff --git a/hugofs/walk_test.go b/hugofs/walk_test.go new file mode 100644 index 00000000000..d492041ae04 --- /dev/null +++ b/hugofs/walk_test.go @@ -0,0 +1,225 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugofs + +import ( + "fmt" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/gohugoio/hugo/common/hugo" + + "github.com/pkg/errors" + + "github.com/gohugoio/hugo/htesting" + + "github.com/spf13/afero" + + "github.com/stretchr/testify/require" +) + +func TestWalk(t *testing.T) { + assert := require.New(t) + + fs := NewBaseFileDecorator(afero.NewMemMapFs()) + + afero.WriteFile(fs, "b.txt", []byte("content"), 0777) + afero.WriteFile(fs, "c.txt", []byte("content"), 0777) + afero.WriteFile(fs, "a.txt", []byte("content"), 0777) + + names, err := collectFilenames(fs, "", "") + + assert.NoError(err) + assert.Equal([]string{"a.txt", "b.txt", "c.txt"}, names) +} + +func TestWalkRootMappingFs(t *testing.T) { + assert := require.New(t) + fs := NewBaseFileDecorator(afero.NewMemMapFs()) + + testfile := "test.txt" + + assert.NoError(afero.WriteFile(fs, filepath.Join("a/b", testfile), []byte("some content"), 0755)) + assert.NoError(afero.WriteFile(fs, filepath.Join("c/d", testfile), []byte("some content"), 0755)) + assert.NoError(afero.WriteFile(fs, filepath.Join("e/f", testfile), []byte("some content"), 0755)) + + rm := []RootMapping{ + RootMapping{ + From: "static/b", + To: "e/f", + }, + RootMapping{ + From: "static/a", + To: "c/d", + }, + + RootMapping{ + From: "static/c", + To: "a/b", + }, + } + + rfs, err := NewRootMappingFs(fs, rm...) + assert.NoError(err) + bfs := afero.NewBasePathFs(rfs, "static") + + names, err := collectFilenames(bfs, "", "") + + assert.NoError(err) + assert.Equal([]string{"a/test.txt", "b/test.txt", "c/test.txt"}, names) + +} + +func skipSymlink() bool { + return runtime.GOOS == "windows" && os.Getenv("CI") == "" +} + +func TestWalkSymbolicLink(t *testing.T) { + if skipSymlink() { + t.Skip("Skip; os.Symlink needs administrator rights on Windows") + } + assert := require.New(t) + workDir, clean, err := htesting.CreateTempDir(Os, "hugo-walk-sym") + assert.NoError(err) + defer clean() + wd, _ := os.Getwd() + defer func() { + os.Chdir(wd) + }() + + fs := NewBaseFileDecorator(Os) + + blogDir := filepath.Join(workDir, "blog") + docsDir := filepath.Join(workDir, "docs") + blogReal := filepath.Join(blogDir, "real") + blogRealSub := filepath.Join(blogReal, "sub") + assert.NoError(os.MkdirAll(blogRealSub, 0777)) + assert.NoError(os.MkdirAll(docsDir, 0777)) + afero.WriteFile(fs, filepath.Join(blogRealSub, "a.txt"), []byte("content"), 0777) + afero.WriteFile(fs, filepath.Join(docsDir, "b.txt"), []byte("content"), 0777) + + os.Chdir(blogDir) + assert.NoError(os.Symlink("real", "symlinked")) + os.Chdir(blogReal) + assert.NoError(os.Symlink("../real", "cyclic")) + os.Chdir(docsDir) + assert.NoError(os.Symlink("../blog/real/cyclic", "docsreal")) + + t.Run("OS Fs", func(t *testing.T) { + assert := require.New(t) + + names, err := collectFilenames(fs, workDir, workDir) + assert.NoError(err) + + assert.Equal([]string{"blog/real/sub/a.txt", "docs/b.txt"}, names) + }) + + t.Run("BasePath Fs", func(t *testing.T) { + if hugo.GoMinorVersion() < 12 { + // https://github.com/golang/go/issues/30520 + // This is fixed in Go 1.13 and in the latest Go 1.12 + t.Skip("skip this for Go <= 1.11 due to a bug in Go's stdlib") + + } + assert := require.New(t) + + docsFs := afero.NewBasePathFs(fs, docsDir) + + names, err := collectFilenames(docsFs, "", "") + assert.NoError(err) + + // Note: the docsreal folder is considered cyclic when walking from the root, but this works. + assert.Equal([]string{"b.txt", "docsreal/sub/a.txt"}, names) + }) + +} + +func collectFilenames(fs afero.Fs, base, root string) ([]string, error) { + var names []string + + walkFn := func(path string, info FileMetaInfo, err error) error { + if err != nil { + return err + } + + if info.IsDir() { + return nil + } + + filename := info.Meta().Path() + filename = filepath.ToSlash(filename) + + names = append(names, filename) + + return nil + } + + w := NewWalkway(WalkwayConfig{Fs: fs, BasePath: base, Root: root, WalkFn: walkFn}) + + err := w.Walk() + + return names, err + +} + +func BenchmarkWalk(b *testing.B) { + assert := require.New(b) + fs := NewBaseFileDecorator(afero.NewMemMapFs()) + + writeFiles := func(dir string, numfiles int) { + for i := 0; i < numfiles; i++ { + filename := filepath.Join(dir, fmt.Sprintf("file%d.txt", i)) + assert.NoError(afero.WriteFile(fs, filename, []byte("content"), 0777)) + } + } + + const numFilesPerDir = 20 + + writeFiles("root", numFilesPerDir) + writeFiles("root/l1_1", numFilesPerDir) + writeFiles("root/l1_1/l2_1", numFilesPerDir) + writeFiles("root/l1_1/l2_2", numFilesPerDir) + writeFiles("root/l1_2", numFilesPerDir) + writeFiles("root/l1_2/l2_1", numFilesPerDir) + writeFiles("root/l1_3", numFilesPerDir) + + walkFn := func(path string, info FileMetaInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + + filename := info.Meta().Filename() + if !strings.HasPrefix(filename, "root") { + return errors.New(filename) + } + + return nil + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + w := NewWalkway(WalkwayConfig{Fs: fs, Root: "root", WalkFn: walkFn}) + + if err := w.Walk(); err != nil { + b.Fatal(err) + } + } + +} diff --git a/hugolib/config.go b/hugolib/config.go index 50e4ca6ec29..ccaf5dbaba3 100644 --- a/hugolib/config.go +++ b/hugolib/config.go @@ -14,21 +14,24 @@ package hugolib import ( - "fmt" - "os" "path/filepath" "strings" + "github.com/gohugoio/hugo/common/loggers" + + "github.com/gohugoio/hugo/cache/filecache" + + "github.com/gohugoio/hugo/common/maps" + "github.com/gohugoio/hugo/parser/metadecoders" "github.com/gohugoio/hugo/common/herrors" "github.com/gohugoio/hugo/common/hugo" "github.com/gohugoio/hugo/hugolib/paths" - "github.com/pkg/errors" - _errors "github.com/pkg/errors" - "github.com/gohugoio/hugo/langs" + "github.com/gohugoio/hugo/modules" + "github.com/pkg/errors" "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/config/privacy" @@ -67,7 +70,8 @@ func loadSiteConfig(cfg config.Provider) (scfg SiteConfig, err error) { // ConfigSourceDescriptor describes where to find the config (e.g. config.toml etc.). type ConfigSourceDescriptor struct { - Fs afero.Fs + Fs afero.Fs + Logger *loggers.Logger // Path to the config file to use, e.g. /my/project/config.toml Filename string @@ -84,6 +88,9 @@ type ConfigSourceDescriptor struct { // production, development Environment string + + // Defaults to os.Environ if not set. + Environ []string } func (d ConfigSourceDescriptor) configFilenames() []string { @@ -111,51 +118,43 @@ var ErrNoConfigFile = errors.New("Unable to locate config file or config directo // LoadConfig loads Hugo configuration into a new Viper and then adds // a set of defaults. func LoadConfig(d ConfigSourceDescriptor, doWithConfig ...func(cfg config.Provider) error) (*viper.Viper, []string, error) { + if d.Environment == "" { d.Environment = hugo.EnvironmentProduction } + if len(d.Environ) == 0 { + d.Environ = os.Environ() + } + var configFiles []string v := viper.New() l := configLoader{ConfigSourceDescriptor: d} - v.AutomaticEnv() - v.SetEnvPrefix("hugo") - - var cerr error - for _, name := range d.configFilenames() { var filename string - if filename, cerr = l.loadConfig(name, v); cerr != nil && cerr != ErrNoConfigFile { - return nil, nil, cerr + filename, err := l.loadConfig(name, v) + if err == nil { + configFiles = append(configFiles, filename) + } else if err != ErrNoConfigFile { + return nil, nil, err } - configFiles = append(configFiles, filename) } if d.AbsConfigDir != "" { dirnames, err := l.loadConfigFromConfigDir(v) if err == nil { configFiles = append(configFiles, dirnames...) + } else if err != ErrNoConfigFile { + return nil, nil, err } - cerr = err } if err := loadDefaultSettingsFor(v); err != nil { return v, configFiles, err } - if cerr == nil { - themeConfigFiles, err := l.loadThemeConfig(v) - if err != nil { - return v, configFiles, err - } - - if len(themeConfigFiles) > 0 { - configFiles = append(configFiles, themeConfigFiles...) - } - } - // We create languages based on the settings, so we need to make sure that // all configuration is loaded/set before doing that. for _, d := range doWithConfig { @@ -164,12 +163,74 @@ func LoadConfig(d ConfigSourceDescriptor, doWithConfig ...func(cfg config.Provid } } + // Apply environment overrides + if len(d.Environ) > 0 { + // Extract all that start with the HUGO_ prefix + const hugoEnvPrefix = "HUGO_" + var hugoEnv []string + for _, v := range d.Environ { + key, val := config.SplitEnvVar(v) + if strings.HasPrefix(key, hugoEnvPrefix) { + hugoEnv = append(hugoEnv, strings.ToLower(strings.TrimPrefix(key, hugoEnvPrefix)), val) + } + } + + if len(hugoEnv) > 0 { + for i := 0; i < len(hugoEnv); i += 2 { + key, valStr := strings.ToLower(hugoEnv[i]), hugoEnv[i+1] + + existing, nestedKey, owner, err := maps.GetNestedParamFn(key, "_", v.Get) + if err != nil { + return v, configFiles, err + } + + if existing != nil { + val, err := metadecoders.Default.UnmarshalStringTo(valStr, existing) + if err != nil { + continue + } + + if owner != nil { + owner[nestedKey] = val + } else { + v.Set(key, val) + } + + } + } + } + } + + modulesConfig, err := l.loadModulesConfig(v) + if err != nil { + return v, configFiles, err + } + + mods, modulesConfigFiles, err := l.collectModules(modulesConfig, v) + if err != nil { + return v, configFiles, err + } + if err := loadLanguageSettings(v, nil); err != nil { return v, configFiles, err } - return v, configFiles, cerr + // Apply default project mounts. + if err := modules.ApplyProjectConfigDefaults(v, mods[len(mods)-1]); err != nil { + return v, configFiles, err + } + if len(modulesConfigFiles) > 0 { + configFiles = append(configFiles, modulesConfigFiles...) + } + + return v, configFiles, nil + +} + +func loadLanguageSettings(cfg config.Provider, oldLangs langs.Languages) error { + _, err := langs.LoadLanguageSettings(cfg, oldLangs) + return err } type configLoader struct { @@ -334,145 +395,79 @@ func (l configLoader) loadConfigFromConfigDir(v *viper.Viper) ([]string, error) return dirnames, nil } -func loadLanguageSettings(cfg config.Provider, oldLangs langs.Languages) error { - - defaultLang := cfg.GetString("defaultContentLanguage") - - var languages map[string]interface{} - - languagesFromConfig := cfg.GetStringMap("languages") - disableLanguages := cfg.GetStringSlice("disableLanguages") - - if len(disableLanguages) == 0 { - languages = languagesFromConfig - } else { - languages = make(map[string]interface{}) - for k, v := range languagesFromConfig { - for _, disabled := range disableLanguages { - if disabled == defaultLang { - return fmt.Errorf("cannot disable default language %q", defaultLang) - } - - if strings.EqualFold(k, disabled) { - v.(map[string]interface{})["disabled"] = true - break - } - } - languages[k] = v - } - } - - var ( - languages2 langs.Languages - err error - ) +func (l configLoader) loadModulesConfig(v1 *viper.Viper) (modules.Config, error) { - if len(languages) == 0 { - languages2 = append(languages2, langs.NewDefaultLanguage(cfg)) - } else { - languages2, err = toSortedLanguages(cfg, languages) - if err != nil { - return _errors.Wrap(err, "Failed to parse multilingual config") - } - } - - if oldLangs != nil { - // When in multihost mode, the languages are mapped to a server, so - // some structural language changes will need a restart of the dev server. - // The validation below isn't complete, but should cover the most - // important cases. - var invalid bool - if languages2.IsMultihost() != oldLangs.IsMultihost() { - invalid = true - } else { - if languages2.IsMultihost() && len(languages2) != len(oldLangs) { - invalid = true - } - } - - if invalid { - return errors.New("language change needing a server restart detected") - } - - if languages2.IsMultihost() { - // We need to transfer any server baseURL to the new language - for i, ol := range oldLangs { - nl := languages2[i] - nl.Set("baseURL", ol.GetString("baseURL")) - } - } + modConfig, err := modules.DecodeConfig(v1) + if err != nil { + return modules.Config{}, err } - // The defaultContentLanguage is something the user has to decide, but it needs - // to match a language in the language definition list. - langExists := false - for _, lang := range languages2 { - if lang.Lang == defaultLang { - langExists = true - break - } - } + return modConfig, nil +} - if !langExists { - return fmt.Errorf("site config value %q for defaultContentLanguage does not match any language definition", defaultLang) +func (l configLoader) collectModules(modConfig modules.Config, v1 *viper.Viper) (modules.Modules, []string, error) { + workingDir := l.WorkingDir + if workingDir == "" { + workingDir = v1.GetString("workingDir") } - cfg.Set("languagesSorted", languages2) - cfg.Set("multilingual", len(languages2) > 1) - - multihost := languages2.IsMultihost() - - if multihost { - cfg.Set("defaultContentLanguageInSubdir", true) - cfg.Set("multihost", true) - } + themesDir := paths.AbsPathify(l.WorkingDir, v1.GetString("themesDir")) - if multihost { - // The baseURL may be provided at the language level. If that is true, - // then every language must have a baseURL. In this case we always render - // to a language sub folder, which is then stripped from all the Permalink URLs etc. - for _, l := range languages2 { - burl := l.GetLocal("baseURL") - if burl == nil { - return errors.New("baseURL must be set on all or none of the languages") - } - } + ignoreVendor := v1.GetBool("ignoreVendor") + filecacheConfigs, err := filecache.DecodeConfig(l.Fs, v1) + if err != nil { + return nil, nil, err } + v1.Set("filecacheConfigs", filecacheConfigs) - return nil -} + modulesClient := modules.NewClient(modules.ClientConfig{ + Fs: l.Fs, + Logger: l.Logger, + WorkingDir: workingDir, + ThemesDir: themesDir, + CacheDir: filecacheConfigs.CacheDirModules(), + ModuleConfig: modConfig, + IgnoreVendor: ignoreVendor, + }) -func (l configLoader) loadThemeConfig(v1 *viper.Viper) ([]string, error) { - themesDir := paths.AbsPathify(l.WorkingDir, v1.GetString("themesDir")) - themes := config.GetStringSlicePreserveString(v1, "theme") + v1.Set("modulesClient", modulesClient) - themeConfigs, err := paths.CollectThemes(l.Fs, themesDir, themes) + moduleConfig, err := modulesClient.Collect() if err != nil { - return nil, err + return nil, nil, err } - if len(themeConfigs) == 0 { - return nil, nil - } + // Avoid recreating these later. + v1.Set("allModules", moduleConfig.ActiveModules) - v1.Set("allThemes", themeConfigs) + if len(moduleConfig.ActiveModules) == 0 { + return nil, nil, nil + } var configFilenames []string - for _, tc := range themeConfigs { - if tc.ConfigFilename != "" { - configFilenames = append(configFilenames, tc.ConfigFilename) + for _, tc := range moduleConfig.ActiveModules { + if tc.ConfigFilename() != "" { + if tc.Watch() { + configFilenames = append(configFilenames, tc.ConfigFilename()) + } if err := l.applyThemeConfig(v1, tc); err != nil { - return nil, err + return nil, nil, err } } } - return configFilenames, nil + if moduleConfig.GoModulesFilename != "" { + // We want to watch this for changes and trigger rebuild on version + // changes etc. + configFilenames = append(configFilenames, moduleConfig.GoModulesFilename) + } + + return moduleConfig.ActiveModules, configFilenames, nil } -func (l configLoader) applyThemeConfig(v1 *viper.Viper, theme paths.ThemeConfig) error { +func (l configLoader) applyThemeConfig(v1 *viper.Viper, theme modules.Module) error { const ( paramsKey = "params" @@ -480,22 +475,12 @@ func (l configLoader) applyThemeConfig(v1 *viper.Viper, theme paths.ThemeConfig) menuKey = "menus" ) - v2 := theme.Cfg + v2 := theme.Cfg() for _, key := range []string{paramsKey, "outputformats", "mediatypes"} { l.mergeStringMapKeepLeft("", key, v1, v2) } - themeLower := strings.ToLower(theme.Name) - themeParamsNamespace := paramsKey + "." + themeLower - - // Set namespaced params - if v2.IsSet(paramsKey) && !v1.IsSet(themeParamsNamespace) { - // Set it in the default store to make sure it gets in the same or - // behind the others. - v1.SetDefault(themeParamsNamespace, v2.Get(paramsKey)) - } - // Only add params and new menu entries, we do not add language definitions. if v1.IsSet(languagesKey) && v2.IsSet(languagesKey) { v1Langs := v1.GetStringMap(languagesKey) @@ -508,12 +493,6 @@ func (l configLoader) applyThemeConfig(v1 *viper.Viper, theme paths.ThemeConfig) if k == "" { continue } - langParamsKey := languagesKey + "." + k + "." + paramsKey - langParamsThemeNamespace := langParamsKey + "." + themeLower - // Set namespaced params - if v2.IsSet(langParamsKey) && !v1.IsSet(langParamsThemeNamespace) { - v1.SetDefault(langParamsThemeNamespace, v2.Get(langParamsKey)) - } langMenuKey := languagesKey + "." + k + "." + menuKey if v2.IsSet(langMenuKey) { @@ -577,18 +556,23 @@ func loadDefaultSettingsFor(v *viper.Viper) error { v.RegisterAlias("indexes", "taxonomies") + /* + + TODO(bep) from 0.56 these are configured as module mounts. + v.SetDefault("contentDir", "content") + v.SetDefault("layoutDir", "layouts") + v.SetDefault("assetDir", "assets") + v.SetDefault("staticDir", "static") + v.SetDefault("dataDir", "data") + v.SetDefault("i18nDir", "i18n") + v.SetDefault("archetypeDir", "archetypes") + */ + v.SetDefault("cleanDestinationDir", false) v.SetDefault("watch", false) v.SetDefault("metaDataFormat", "toml") - v.SetDefault("contentDir", "content") - v.SetDefault("layoutDir", "layouts") - v.SetDefault("assetDir", "assets") - v.SetDefault("staticDir", "static") v.SetDefault("resourceDir", "resources") - v.SetDefault("archetypeDir", "archetypes") v.SetDefault("publishDir", "public") - v.SetDefault("dataDir", "data") - v.SetDefault("i18nDir", "i18n") v.SetDefault("themesDir", "themes") v.SetDefault("buildDrafts", false) v.SetDefault("buildFuture", false) @@ -635,5 +619,6 @@ func loadDefaultSettingsFor(v *viper.Viper) error { v.SetDefault("disableFastRender", false) v.SetDefault("timeout", 10000) // 10 seconds v.SetDefault("enableInlineShortcodes", false) + return nil } diff --git a/hugolib/config_test.go b/hugolib/config_test.go index 885a07ee951..d7e21ea88ae 100644 --- a/hugolib/config_test.go +++ b/hugolib/config_test.go @@ -14,6 +14,9 @@ package hugolib import ( + "bytes" + "fmt" + "path/filepath" "testing" "github.com/spf13/afero" @@ -40,10 +43,7 @@ func TestLoadConfig(t *testing.T) { require.NoError(t, err) assert.Equal("side", cfg.GetString("paginatePath")) - // default - assert.Equal("layouts", cfg.GetString("layoutDir")) - // no themes - assert.False(cfg.IsSet("allThemes")) + } func TestLoadMultiConfig(t *testing.T) { @@ -188,11 +188,6 @@ map[string]interface {}{ "p1": "p1 main", "p2": "p2 main", "p3": "p3 theme", - "test-theme": map[string]interface {}{ - "p1": "p1 theme", - "p2": "p2 theme", - "p3": "p3 theme", - }, "top": "top", }`, got["params"]) @@ -257,10 +252,6 @@ map[string]interface {}{ "params": map[string]interface {}{ "pl1": "p1-en-main", "pl2": "p2-en-theme", - "test-theme": map[string]interface {}{ - "pl1": "p1-en-theme", - "pl2": "p2-en-theme", - }, }, }, "nb": map[string]interface {}{ @@ -275,11 +266,6 @@ map[string]interface {}{ "params": map[string]interface {}{ "pl1": "p1-nb-main", "pl2": "p2-nb-theme", - "test-theme": map[string]interface {}{ - "pl1": "p1-nb-theme", - "pl2": "p2-nb-theme", - "top": "top-nb-theme", - }, }, }, } @@ -397,3 +383,137 @@ privacyEnhanced = true assert.True(b.H.Sites[0].Info.Config().Privacy.YouTube.PrivacyEnhanced) } + +func TestLoadConfigModules(t *testing.T) { + t.Parallel() + + assert := require.New(t) + + // https://github.com/gohugoio/hugoThemes#themetoml + + const ( + // Before Hugo 0.56 each theme/component could have its own theme.toml + // with some settings, mostly used on the Hugo themes site. + // To preserve combability we read these files into the new "modules" + // section in config.toml. + o1t = ` +name = "Component o1" +license = "MIT" +min_version = 0.38 +` + // This is the component's config.toml, using the old theme syntax. + o1c = ` +theme = ["n2"] +` + + n1 = ` +title = "Component n1" + +[module] +description = "Component n1 description" +[module.hugoVersion] +min = "0.40.0" +max = "0.50.0" +extended = true +[[module.imports]] +path="o1" +[[module.imports]] +path="n3" + + +` + + n2 = ` +title = "Component n2" +` + + n3 = ` +title = "Component n3" +` + + n4 = ` +title = "Component n4" +` + ) + + b := newTestSitesBuilder(t) + + writeThemeFiles := func(name, configTOML, themeTOML string) { + b.WithSourceFile(filepath.Join("themes", name, "data", "module.toml"), fmt.Sprintf("name=%q", name)) + if configTOML != "" { + b.WithSourceFile(filepath.Join("themes", name, "config.toml"), configTOML) + } + if themeTOML != "" { + b.WithSourceFile(filepath.Join("themes", name, "theme.toml"), themeTOML) + } + } + + writeThemeFiles("n1", n1, "") + writeThemeFiles("n2", n2, "") + writeThemeFiles("n3", n3, "") + writeThemeFiles("n4", n4, "") + writeThemeFiles("o1", o1c, o1t) + + b.WithConfigFile("toml", ` +[module] +[[module.imports]] +path="n1" +[[module.imports]] +path="n4" + +`) + + b.Build(BuildCfg{}) + + modulesClient := b.H.Paths.ModulesClient + var graphb bytes.Buffer + modulesClient.Graph(&graphb) + + assert.Equal(`project n1 +n1 o1 +o1 n2 +n1 n3 +project n4 +`, graphb.String()) + +} + +func TestLoadConfigWithOsEnvOverrides(t *testing.T) { + + assert := require.New(t) + + baseConfig := ` + +environment = "production" +intSlice = [5,7,9] +floatSlice = [3.14, 5.19] +stringSlice = ["a", "b"] + +[imaging] +anchor = "smart" +quality = 75 +resamplefilter = "CatmullRom" +` + + b := newTestSitesBuilder(t).WithConfigFile("toml", baseConfig) + + b.WithEnviron( + "HUGO_ENVIRONMENT", "test", + "HUGO_IMAGING_ANCHOR", "top", + "HUGO_STRINGSLICE", `["c", "d"]`, + "HUGO_INTSLICE", `[5, 8, 9]`, + "HUGO_FLOATSLICE", `[5.32]`, + ) + + b.Build(BuildCfg{}) + + cfg := b.H.Cfg + + assert.Equal("test", cfg.Get("environment")) + assert.Equal("top", cfg.Get("imaging.anchor")) + assert.Equal(int64(75), cfg.Get("imaging.quality")) + assert.Equal([]interface{}{"c", "d"}, cfg.Get("stringSlice")) + assert.Equal([]interface{}{5.32}, cfg.Get("floatSlice")) + assert.Equal([]interface{}{5, 8, 9}, cfg.Get("intSlice")) + +} diff --git a/hugolib/data/hugo.toml b/hugolib/data/hugo.toml new file mode 100755 index 00000000000..eb1dbc42cb1 --- /dev/null +++ b/hugolib/data/hugo.toml @@ -0,0 +1 @@ +slogan = "Hugo Rocks!" \ No newline at end of file diff --git a/hugolib/disableKinds_test.go b/hugolib/disableKinds_test.go index f5c093646c2..c191dfef1ef 100644 --- a/hugolib/disableKinds_test.go +++ b/hugolib/disableKinds_test.go @@ -20,11 +20,7 @@ import ( "github.com/gohugoio/hugo/resources/page" - "github.com/gohugoio/hugo/deps" - "github.com/spf13/afero" - "github.com/gohugoio/hugo/helpers" - "github.com/gohugoio/hugo/hugofs" "github.com/stretchr/testify/require" ) @@ -80,8 +76,6 @@ categories: # Doc ` - mf := afero.NewMemMapFs() - disabledStr := "[]" if len(disabled) > 0 { @@ -90,47 +84,41 @@ categories: } siteConfig := fmt.Sprintf(siteConfigTemplate, disabledStr) - writeToFs(t, mf, "config.toml", siteConfig) - - cfg, err := LoadConfigDefault(mf) - require.NoError(t, err) - fs := hugofs.NewFrom(mf, cfg) - th := testHelper{cfg, fs, t} + b := newTestSitesBuilder(t).WithConfigFile("toml", siteConfig) - writeSource(t, fs, "layouts/index.html", "Home|{{ .Title }}|{{ .Content }}") - writeSource(t, fs, "layouts/_default/single.html", "Single|{{ .Title }}|{{ .Content }}") - writeSource(t, fs, "layouts/_default/list.html", "List|{{ .Title }}|{{ .Content }}") - writeSource(t, fs, "layouts/_default/terms.html", "Terms List|{{ .Title }}|{{ .Content }}") - writeSource(t, fs, "layouts/404.html", "Page Not Found") + b.WithTemplates( + "index.html", "Home|{{ .Title }}|{{ .Content }}", + "_default/single.html", "Single|{{ .Title }}|{{ .Content }}", + "_default/list.html", "List|{{ .Title }}|{{ .Content }}", + "_default/terms.html", "Terms List|{{ .Title }}|{{ .Content }}", + "layouts/404.html", "Page Not Found", + ) - writeSource(t, fs, "content/sect/p1.md", fmt.Sprintf(pageTemplate, "P1", "- tag1")) + b.WithContent( + "sect/p1.md", fmt.Sprintf(pageTemplate, "P1", "- tag1"), + "categories/_index.md", newTestPage("Category Terms", "2017-01-01", 10), + "tags/tag1/_index.md", newTestPage("Tag1 List", "2017-01-01", 10), + ) - writeNewContentFile(t, fs.Source, "Category Terms", "2017-01-01", "content/categories/_index.md", 10) - writeNewContentFile(t, fs.Source, "Tag1 List", "2017-01-01", "content/tags/tag1/_index.md", 10) + b.Build(BuildCfg{}) + h := b.H - h, err := NewHugoSites(deps.DepsCfg{Fs: fs, Cfg: cfg}) - - require.NoError(t, err) require.Len(t, h.Sites, 1) - err = h.Build(BuildCfg{}) - - require.NoError(t, err) - - assertDisabledKinds(th, h.Sites[0], disabled...) + assertDisabledKinds(b, h.Sites[0], disabled...) } -func assertDisabledKinds(th testHelper, s *Site, disabled ...string) { - assertDisabledKind(th, +func assertDisabledKinds(b *sitesBuilder, s *Site, disabled ...string) { + assertDisabledKind(b, func(isDisabled bool) bool { if isDisabled { return len(s.RegularPages()) == 0 } return len(s.RegularPages()) > 0 }, disabled, page.KindPage, "public/sect/p1/index.html", "Single|P1") - assertDisabledKind(th, + assertDisabledKind(b, func(isDisabled bool) bool { p := s.getPage(page.KindHome) if isDisabled { @@ -138,7 +126,7 @@ func assertDisabledKinds(th testHelper, s *Site, disabled ...string) { } return p != nil }, disabled, page.KindHome, "public/index.html", "Home") - assertDisabledKind(th, + assertDisabledKind(b, func(isDisabled bool) bool { p := s.getPage(page.KindSection, "sect") if isDisabled { @@ -146,7 +134,7 @@ func assertDisabledKinds(th testHelper, s *Site, disabled ...string) { } return p != nil }, disabled, page.KindSection, "public/sect/index.html", "Sects") - assertDisabledKind(th, + assertDisabledKind(b, func(isDisabled bool) bool { p := s.getPage(page.KindTaxonomy, "tags", "tag1") @@ -156,7 +144,7 @@ func assertDisabledKinds(th testHelper, s *Site, disabled ...string) { return p != nil }, disabled, page.KindTaxonomy, "public/tags/tag1/index.html", "Tag1") - assertDisabledKind(th, + assertDisabledKind(b, func(isDisabled bool) bool { p := s.getPage(page.KindTaxonomyTerm, "tags") if isDisabled { @@ -165,7 +153,7 @@ func assertDisabledKinds(th testHelper, s *Site, disabled ...string) { return p != nil }, disabled, page.KindTaxonomyTerm, "public/tags/index.html", "Tags") - assertDisabledKind(th, + assertDisabledKind(b, func(isDisabled bool) bool { p := s.getPage(page.KindTaxonomyTerm, "categories") @@ -175,7 +163,7 @@ func assertDisabledKinds(th testHelper, s *Site, disabled ...string) { return p != nil }, disabled, page.KindTaxonomyTerm, "public/categories/index.html", "Category Terms") - assertDisabledKind(th, + assertDisabledKind(b, func(isDisabled bool) bool { p := s.getPage(page.KindTaxonomy, "categories", "hugo") if isDisabled { @@ -185,15 +173,15 @@ func assertDisabledKinds(th testHelper, s *Site, disabled ...string) { }, disabled, page.KindTaxonomy, "public/categories/hugo/index.html", "Hugo") // The below have no page in any collection. - assertDisabledKind(th, func(isDisabled bool) bool { return true }, disabled, kindRSS, "public/index.xml", "") - assertDisabledKind(th, func(isDisabled bool) bool { return true }, disabled, kindSitemap, "public/sitemap.xml", "sitemap") - assertDisabledKind(th, func(isDisabled bool) bool { return true }, disabled, kindRobotsTXT, "public/robots.txt", "User-agent") - assertDisabledKind(th, func(isDisabled bool) bool { return true }, disabled, kind404, "public/404.html", "Page Not Found") + assertDisabledKind(b, func(isDisabled bool) bool { return true }, disabled, kindRSS, "public/index.xml", "") + assertDisabledKind(b, func(isDisabled bool) bool { return true }, disabled, kindSitemap, "public/sitemap.xml", "sitemap") + assertDisabledKind(b, func(isDisabled bool) bool { return true }, disabled, kindRobotsTXT, "public/robots.txt", "User-agent") + assertDisabledKind(b, func(isDisabled bool) bool { return true }, disabled, kind404, "public/404.html", "Page Not Found") } -func assertDisabledKind(th testHelper, kindAssert func(bool) bool, disabled []string, kind, path, matcher string) { +func assertDisabledKind(b *sitesBuilder, kindAssert func(bool) bool, disabled []string, kind, path, matcher string) { isDisabled := stringSliceContains(kind, disabled...) - require.True(th.T, kindAssert(isDisabled), fmt.Sprintf("%s: %t", kind, isDisabled)) + require.True(b.T, kindAssert(isDisabled), fmt.Sprintf("%s: %t", kind, isDisabled)) if kind == kindRSS && !isDisabled { // If the home page is also disabled, there is not RSS to look for. @@ -204,20 +192,11 @@ func assertDisabledKind(th testHelper, kindAssert func(bool) bool, disabled []st if isDisabled { // Path should not exist - fileExists, err := helpers.Exists(path, th.Fs.Destination) - require.False(th.T, fileExists) - require.NoError(th.T, err) + fileExists, err := helpers.Exists(path, b.Fs.Destination) + require.False(b.T, fileExists) + require.NoError(b.T, err) } else { - th.assertFileContent(path, matcher) - } -} - -func stringSliceContains(k string, values ...string) bool { - for _, v := range values { - if k == v { - return true - } + b.AssertFileContent(path, matcher) } - return false } diff --git a/hugolib/fileInfo.go b/hugolib/fileInfo.go index ea3b15ef328..4997142a16a 100644 --- a/hugolib/fileInfo.go +++ b/hugolib/fileInfo.go @@ -16,82 +16,64 @@ package hugolib import ( "strings" - "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/hugofs/files" + + "github.com/pkg/errors" + + "github.com/gohugoio/hugo/hugofs" + + "github.com/spf13/afero" + "github.com/gohugoio/hugo/source" ) // fileInfo implements the File and ReadableFile interface. var ( - _ source.File = (*fileInfo)(nil) - _ source.ReadableFile = (*fileInfo)(nil) - _ pathLangFile = (*fileInfo)(nil) + _ source.File = (*fileInfo)(nil) ) -// A partial interface to prevent ambigous compiler error. -type basePather interface { - Filename() string - RealName() string - BaseDir() string -} - type fileInfo struct { - bundleTp bundleDirType - - source.ReadableFile - basePather + source.File overriddenLang string +} - // Set if the content language for this file is disabled. - disabled bool +func (fi *fileInfo) Open() (afero.File, error) { + f, err := fi.FileInfo().Meta().Open() + if err != nil { + err = errors.Wrap(err, "fileInfo") + } + + return f, err } func (fi *fileInfo) Lang() string { if fi.overriddenLang != "" { return fi.overriddenLang } - return fi.ReadableFile.Lang() -} - -func (fi *fileInfo) Filename() string { - if fi == nil || fi.basePather == nil { - return "" - } - return fi.basePather.Filename() + return fi.File.Lang() } func (fi *fileInfo) String() string { - if fi == nil || fi.ReadableFile == nil { + if fi == nil || fi.File == nil { return "" } return fi.Path() } -func (fi *fileInfo) isOwner() bool { - return fi.bundleTp > bundleNot -} - -func IsContentFile(filename string) bool { - return contentFileExtensionsSet[strings.TrimPrefix(helpers.Ext(filename), ".")] -} - -func (fi *fileInfo) isContentFile() bool { - return contentFileExtensionsSet[fi.Ext()] -} +// TODO(bep) rename +func newFileInfo(sp *source.SourceSpec, fi hugofs.FileMetaInfo) (*fileInfo, error) { -func newFileInfo(sp *source.SourceSpec, baseDir, filename string, fi pathLangFileFi, tp bundleDirType) *fileInfo { + baseFi, err := sp.NewFileInfo(fi) + if err != nil { + return nil, err + } - baseFi := sp.NewFileInfo(baseDir, filename, tp == bundleLeaf, fi) f := &fileInfo{ - bundleTp: tp, - ReadableFile: baseFi, - basePather: fi, + File: baseFi, } - lang := f.Lang() - f.disabled = lang != "" && sp.DisabledLanguages[lang] - - return f + return f, nil } @@ -108,7 +90,7 @@ const ( // Returns the given file's name's bundle type and whether it is a content // file or not. func classifyBundledFile(name string) (bundleDirType, bool) { - if !IsContentFile(name) { + if !files.IsContentFile(name) { return bundleNot, false } if strings.HasPrefix(name, "_index.") { diff --git a/hugolib/fileInfo_test.go b/hugolib/fileInfo_test.go index 10f5f051774..c51ff5033d7 100644 --- a/hugolib/fileInfo_test.go +++ b/hugolib/fileInfo_test.go @@ -22,6 +22,7 @@ import ( func TestFileInfo(t *testing.T) { t.Run("String", func(t *testing.T) { + t.Parallel() assert := require.New(t) fi := &fileInfo{} _, err := cast.ToStringE(fi) diff --git a/hugolib/filesystems/basefs.go b/hugolib/filesystems/basefs.go index d88141efd7d..47ae5d0e030 100644 --- a/hugolib/filesystems/basefs.go +++ b/hugolib/filesystems/basefs.go @@ -16,27 +16,27 @@ package filesystems import ( - "errors" + "io" "os" + "path" "path/filepath" "strings" + "sync" - "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/hugofs/files" + + "github.com/pkg/errors" + + "github.com/gohugoio/hugo/modules" "github.com/gohugoio/hugo/hugofs" "fmt" "github.com/gohugoio/hugo/hugolib/paths" - "github.com/gohugoio/hugo/langs" "github.com/spf13/afero" ) -// When we create a virtual filesystem with data and i18n bundles for the project and the themes, -// this is the name of the project's virtual root. It got it's funky name to make sure -// (or very unlikely) that it collides with a theme name. -const projectVirtualFolder = "__h__project" - var filePathSeparator = string(filepath.Separator) // BaseFs contains the core base filesystems used by Hugo. The name "base" is used @@ -51,16 +51,43 @@ type BaseFs struct { // This usually maps to /my-project/public. PublishFs afero.Fs - themeFs afero.Fs + theBigFs *filesystemsCollector +} + +func (fs *BaseFs) WatchDirs() []hugofs.FileMetaInfo { + var dirs []hugofs.FileMetaInfo + for _, dir := range fs.AllDirs() { + if dir.Meta().Watch() { + dirs = append(dirs, dir) + } + } + + return dirs +} + +func (fs *BaseFs) AllDirs() []hugofs.FileMetaInfo { + var dirs []hugofs.FileMetaInfo + for _, dirSet := range [][]hugofs.FileMetaInfo{ + fs.Archetypes.Dirs, + fs.I18n.Dirs, + fs.Data.Dirs, + fs.Content.Dirs, + fs.Assets.Dirs, + fs.Layouts.Dirs, + //fs.Resources.Dirs, + fs.StaticDirs, + } { + dirs = append(dirs, dirSet...) + } - // TODO(bep) improve the "theme interaction" - AbsThemeDirs []string + return dirs } // RelContentDir tries to create a path relative to the content root from // the given filename. The return value is the path and language code. func (b *BaseFs) RelContentDir(filename string) string { - for _, dirname := range b.SourceFilesystems.Content.Dirnames { + for _, dir := range b.SourceFilesystems.Content.Dirs { + dirname := dir.Meta().Filename() if strings.HasPrefix(filename, dirname) { rel := strings.TrimPrefix(filename, dirname) return strings.TrimPrefix(rel, filePathSeparator) @@ -80,16 +107,22 @@ type SourceFilesystems struct { Layouts *SourceFilesystem Archetypes *SourceFilesystem Assets *SourceFilesystem - Resources *SourceFilesystem - // This is a unified read-only view of the project's and themes' workdir. - Work *SourceFilesystem + // Writable filesystem on top the project's resources directory, + // with any sub module's resource fs layered below. + ResourcesCache afero.Fs + + // The project folder. + Work afero.Fs // When in multihost we have one static filesystem per language. The sync // static files is currently done outside of the Hugo build (where there is // a concept of a site per language). // When in non-multihost mode there will be one entry in this map with a blank key. Static map[string]*SourceFilesystem + + // All the /static dirs (including themes/modules). + StaticDirs []hugofs.FileMetaInfo } // A SourceFilesystem holds the filesystem for a given source type in Hugo (data, @@ -99,12 +132,9 @@ type SourceFilesystem struct { // This is a virtual composite filesystem. It expects path relative to a context. Fs afero.Fs - // This is the base source filesystem. In real Hugo, this will be the OS filesystem. - // Use this if you need to resolve items in Dirnames below. - SourceFs afero.Fs - - // Dirnames is absolute filenames to the directories in this filesystem. - Dirnames []string + // This filesystem as separate root directories, starting from project and down + // to the themes/modules. + Dirs []hugofs.FileMetaInfo // When syncing a source folder to the target (e.g. /public), this may // be set to publish into a subfolder. This is used for static syncing @@ -207,7 +237,8 @@ func (s SourceFilesystems) MakeStaticPathRelative(filename string) string { // MakePathRelative creates a relative path from the given filename. // It will return an empty string if the filename is not a member of this filesystem. func (d *SourceFilesystem) MakePathRelative(filename string) string { - for _, currentPath := range d.Dirnames { + for _, dir := range d.Dirs { + currentPath := dir.(hugofs.FileMetaInfo).Meta().Filename() if strings.HasPrefix(filename, currentPath) { return strings.TrimPrefix(filename, currentPath) } @@ -220,8 +251,8 @@ func (d *SourceFilesystem) RealFilename(rel string) string { if err != nil { return rel } - if realfi, ok := fi.(hugofs.RealFilenameInfo); ok { - return realfi.RealFilename() + if realfi, ok := fi.(hugofs.FileMetaInfo); ok { + return realfi.Meta().Filename() } return rel @@ -229,8 +260,8 @@ func (d *SourceFilesystem) RealFilename(rel string) string { // Contains returns whether the given filename is a member of the current filesystem. func (d *SourceFilesystem) Contains(filename string) bool { - for _, dir := range d.Dirnames { - if strings.HasPrefix(filename, dir) { + for _, dir := range d.Dirs { + if strings.HasPrefix(filename, dir.Meta().Filename()) { return true } } @@ -241,9 +272,12 @@ func (d *SourceFilesystem) Contains(filename string) bool { // path. func (d *SourceFilesystem) RealDirs(from string) []string { var dirnames []string - for _, dir := range d.Dirnames { - dirname := filepath.Join(dir, from) - if _, err := d.SourceFs.Stat(dirname); err == nil { + for _, dir := range d.Dirs { + meta := dir.Meta() + dirname := filepath.Join(meta.Filename(), from) + _, err := meta.Fs().Stat(from) + + if err == nil { dirnames = append(dirnames, dirname) } } @@ -254,40 +288,18 @@ func (d *SourceFilesystem) RealDirs(from string) []string { // the same across sites/languages. func WithBaseFs(b *BaseFs) func(*BaseFs) error { return func(bb *BaseFs) error { - bb.themeFs = b.themeFs - bb.AbsThemeDirs = b.AbsThemeDirs + bb.theBigFs = b.theBigFs + bb.SourceFilesystems = b.SourceFilesystems return nil } } -func newRealBase(base afero.Fs) afero.Fs { - return hugofs.NewBasePathRealFilenameFs(base.(*afero.BasePathFs)) - -} - // NewBase builds the filesystems used by Hugo given the paths and options provided.NewBase func NewBase(p *paths.Paths, options ...func(*BaseFs) error) (*BaseFs, error) { fs := p.Fs publishFs := afero.NewBasePathFs(fs.Destination, p.AbsPublishDir) - contentFs, absContentDirs, err := createContentFs(fs.Source, p.WorkingDir, p.DefaultContentLanguage, p.Languages) - if err != nil { - return nil, err - } - - // Make sure we don't have any overlapping content dirs. That will never work. - for i, d1 := range absContentDirs { - for j, d2 := range absContentDirs { - if i == j { - continue - } - if strings.HasPrefix(d1, d2) || strings.HasPrefix(d2, d1) { - return nil, fmt.Errorf("found overlapping content dirs (%q and %q)", d1, d2) - } - } - } - b := &BaseFs{ PublishFs: publishFs, } @@ -298,463 +310,395 @@ func NewBase(p *paths.Paths, options ...func(*BaseFs) error) (*BaseFs, error) { } } + if b.theBigFs != nil && b.SourceFilesystems != nil { + return b, nil + } + builder := newSourceFilesystemsBuilder(p, b) sourceFilesystems, err := builder.Build() if err != nil { - return nil, err - } - - sourceFilesystems.Content = &SourceFilesystem{ - SourceFs: fs.Source, - Fs: contentFs, - Dirnames: absContentDirs, + return nil, errors.Wrap(err, "build filesystems") } b.SourceFilesystems = sourceFilesystems - b.themeFs = builder.themeFs - b.AbsThemeDirs = builder.absThemeDirs + b.theBigFs = builder.theBigFs return b, nil } type sourceFilesystemsBuilder struct { - p *paths.Paths - result *SourceFilesystems - themeFs afero.Fs - hasTheme bool - absThemeDirs []string + p *paths.Paths + sourceFs afero.Fs + result *SourceFilesystems + theBigFs *filesystemsCollector } func newSourceFilesystemsBuilder(p *paths.Paths, b *BaseFs) *sourceFilesystemsBuilder { - return &sourceFilesystemsBuilder{p: p, themeFs: b.themeFs, absThemeDirs: b.AbsThemeDirs, result: &SourceFilesystems{}} + sourceFs := hugofs.NewBaseFileDecorator(p.Fs.Source) + return &sourceFilesystemsBuilder{p: p, sourceFs: sourceFs, theBigFs: b.theBigFs, result: &SourceFilesystems{}} } +func (b *sourceFilesystemsBuilder) newSourceFilesystem(fs afero.Fs, dirs []hugofs.FileMetaInfo) *SourceFilesystem { + return &SourceFilesystem{ + Fs: fs, + Dirs: dirs, + } +} func (b *sourceFilesystemsBuilder) Build() (*SourceFilesystems, error) { - if b.themeFs == nil && b.p.ThemeSet() { - themeFs, absThemeDirs, err := createThemesOverlayFs(b.p) + + if b.theBigFs == nil { + + theBigFs, err := b.createMainOverlayFs(b.p) if err != nil { - return nil, err - } - if themeFs == nil { - panic("createThemesFs returned nil") + return nil, errors.Wrap(err, "create main fs") } - b.themeFs = themeFs - b.absThemeDirs = absThemeDirs + b.theBigFs = theBigFs } - b.hasTheme = len(b.absThemeDirs) > 0 + createView := func(componentID string) *SourceFilesystem { + if b.theBigFs == nil || b.theBigFs.overlayMounts == nil { + return b.newSourceFilesystem(hugofs.NoOpFs, nil) + } - sfs, err := b.createRootMappingFs("dataDir", "data") - if err != nil { - return nil, err - } - b.result.Data = sfs + dirs := b.theBigFs.overlayDirs[componentID] - sfs, err = b.createRootMappingFs("i18nDir", "i18n") - if err != nil { - return nil, err - } - b.result.I18n = sfs + return b.newSourceFilesystem(afero.NewBasePathFs(b.theBigFs.overlayMounts, componentID), dirs) - sfs, err = b.createFs(false, true, "layoutDir", "layouts") - if err != nil { - return nil, err } - b.result.Layouts = sfs - sfs, err = b.createFs(false, true, "archetypeDir", "archetypes") - if err != nil { - return nil, err - } - b.result.Archetypes = sfs + b.theBigFs.finalizeDirs() - sfs, err = b.createFs(false, true, "assetDir", "assets") - if err != nil { - return nil, err - } - b.result.Assets = sfs + b.result.Archetypes = createView(files.ComponentFolderArchetypes) + b.result.Layouts = createView(files.ComponentFolderLayouts) + b.result.Assets = createView(files.ComponentFolderAssets) + b.result.ResourcesCache = b.theBigFs.overlayResources - sfs, err = b.createFs(true, false, "resourceDir", "resources") + // Data, i18n and content cannot use the overlay fs + dataDirs := b.theBigFs.overlayDirs[files.ComponentFolderData] + dataFs, err := hugofs.NewSliceFs(dataDirs...) if err != nil { return nil, err } - b.result.Resources = sfs + b.result.Data = b.newSourceFilesystem(dataFs, dataDirs) - sfs, err = b.createFs(false, true, "", "") + i18nDirs := b.theBigFs.overlayDirs[files.ComponentFolderI18n] + i18nFs, err := hugofs.NewSliceFs(i18nDirs...) if err != nil { return nil, err } - b.result.Work = sfs + b.result.I18n = b.newSourceFilesystem(i18nFs, i18nDirs) + + contentDirs := b.theBigFs.overlayDirs[files.ComponentFolderContent] + contentBfs := afero.NewBasePathFs(b.theBigFs.overlayMountsContent, files.ComponentFolderContent) - err = b.createStaticFs() + contentFs, err := hugofs.NewLanguageFs(b.p.LanguagesDefaultFirst.AsOrdinalSet(), contentBfs) if err != nil { - return nil, err + return nil, errors.Wrap(err, "create content filesystem") } - return b.result, nil -} - -func (b *sourceFilesystemsBuilder) createFs( - mkdir bool, - readOnly bool, - dirKey, themeFolder string) (*SourceFilesystem, error) { - s := &SourceFilesystem{ - SourceFs: b.p.Fs.Source, - } + b.result.Content = b.newSourceFilesystem(contentFs, contentDirs) - if themeFolder == "" { - themeFolder = filePathSeparator - } + b.result.Work = afero.NewReadOnlyFs(b.theBigFs.overlayFull) - var dir string - if dirKey != "" { - dir = b.p.Cfg.GetString(dirKey) - if dir == "" { - return s, fmt.Errorf("config %q not set", dirKey) + // Create static filesystem(s) + ms := make(map[string]*SourceFilesystem) + b.result.Static = ms + b.result.StaticDirs = b.theBigFs.overlayDirs[files.ComponentFolderStatic] + + if b.theBigFs.staticPerLanguage != nil { + // Multihost mode + for k, v := range b.theBigFs.staticPerLanguage { + sfs := b.newSourceFilesystem(v, b.result.StaticDirs) + sfs.PublishFolder = k + ms[k] = sfs } + } else { + bfs := afero.NewBasePathFs(b.theBigFs.overlayMounts, files.ComponentFolderStatic) + ms[""] = b.newSourceFilesystem(bfs, b.result.StaticDirs) } - var fs afero.Fs + return b.result, nil - absDir := b.p.AbsPathify(dir) - existsInSource := b.existsInSource(absDir) - if !existsInSource && mkdir { - // We really need this directory. Make it. - if err := b.p.Fs.Source.MkdirAll(absDir, 0777); err == nil { - existsInSource = true - } - } - if existsInSource { - fs = newRealBase(afero.NewBasePathFs(b.p.Fs.Source, absDir)) - s.Dirnames = []string{absDir} - } +} - if b.hasTheme { - if !strings.HasPrefix(themeFolder, filePathSeparator) { - themeFolder = filePathSeparator + themeFolder - } - themeFolderFs := newRealBase(afero.NewBasePathFs(b.themeFs, themeFolder)) - if fs == nil { - fs = themeFolderFs - } else { - fs = afero.NewCopyOnWriteFs(themeFolderFs, fs) - } +func (b *sourceFilesystemsBuilder) createMainOverlayFs(p *paths.Paths) (*filesystemsCollector, error) { - for _, absThemeDir := range b.absThemeDirs { - absThemeFolderDir := filepath.Join(absThemeDir, themeFolder) - if b.existsInSource(absThemeFolderDir) { - s.Dirnames = append(s.Dirnames, absThemeFolderDir) - } - } + var staticFsMap map[string]afero.Fs + if b.p.Cfg.GetBool("multihost") { + staticFsMap = make(map[string]afero.Fs) } - if fs == nil { - s.Fs = hugofs.NoOpFs - } else if readOnly { - s.Fs = afero.NewReadOnlyFs(fs) - } else { - s.Fs = fs + collector := &filesystemsCollector{ + sourceProject: b.sourceFs, + sourceModules: hugofs.NewNoSymlinkFs(b.sourceFs), + overlayDirs: make(map[string][]hugofs.FileMetaInfo), + staticPerLanguage: staticFsMap, } - return s, nil -} + mods := p.AllModules -// Used for data, i18n -- we cannot use overlay filsesystems for those, but we need -// to keep a strict order. -func (b *sourceFilesystemsBuilder) createRootMappingFs(dirKey, themeFolder string) (*SourceFilesystem, error) { - s := &SourceFilesystem{ - SourceFs: b.p.Fs.Source, + if len(mods) == 0 { + return collector, nil } - projectDir := b.p.Cfg.GetString(dirKey) - if projectDir == "" { - return nil, fmt.Errorf("config %q not set", dirKey) - } + modsReversed := make([]mountsDescriptor, len(mods)) - var fromTo []string - to := b.p.AbsPathify(projectDir) + // The theme components are ordered from left to right. + // We need to revert it to get the + // overlay logic below working as expected, with the project on top (last). - if b.existsInSource(to) { - s.Dirnames = []string{to} - fromTo = []string{projectVirtualFolder, to} - } + for i, mod := range mods { + dir := mod.Dir() - for _, theme := range b.p.AllThemes { - to := b.p.AbsPathify(filepath.Join(b.p.ThemesDir, theme.Name, themeFolder)) - if b.existsInSource(to) { - s.Dirnames = append(s.Dirnames, to) - from := theme - fromTo = append(fromTo, from.Name, to) + if i < len(mods)-1 { + i = len(mods) - 2 - i } - } - if len(fromTo) == 0 { - s.Fs = hugofs.NoOpFs - return s, nil + isMainProject := mod.Owner() == nil + modsReversed[i] = mountsDescriptor{ + mounts: mod.Mounts(), + dir: dir, + watch: mod.Watch(), + isMainProject: isMainProject, + } } - fs, err := hugofs.NewRootMappingFs(b.p.Fs.Source, fromTo...) - if err != nil { - return nil, err - } + err := b.createOverlayFs(collector, modsReversed) - s.Fs = afero.NewReadOnlyFs(fs) + return collector, err - return s, nil } -func (b *sourceFilesystemsBuilder) existsInSource(abspath string) bool { - exists, _ := afero.Exists(b.p.Fs.Source, abspath) - return exists +func (b *sourceFilesystemsBuilder) isContentMount(mnt modules.Mount) bool { + return strings.HasPrefix(mnt.Target, files.ComponentFolderContent) } -func (b *sourceFilesystemsBuilder) createStaticFs() error { - isMultihost := b.p.Cfg.GetBool("multihost") - ms := make(map[string]*SourceFilesystem) - b.result.Static = ms - - if isMultihost { - for _, l := range b.p.Languages { - s := &SourceFilesystem{ - SourceFs: b.p.Fs.Source, - PublishFolder: l.Lang} - staticDirs := removeDuplicatesKeepRight(getStaticDirs(l)) - if len(staticDirs) == 0 { - continue - } +func (b *sourceFilesystemsBuilder) createModFs( + collector *filesystemsCollector, + md mountsDescriptor) error { - for _, dir := range staticDirs { - absDir := b.p.AbsPathify(dir) - if !b.existsInSource(absDir) { - continue - } + var ( + fromTo []hugofs.RootMapping + fromToContent []hugofs.RootMapping + ) - s.Dirnames = append(s.Dirnames, absDir) - } + absPathify := func(path string) string { + return paths.AbsPathify(md.dir, path) + } - fs, err := createOverlayFs(b.p.Fs.Source, s.Dirnames) - if err != nil { - return err - } + seen := make(map[string]bool) - if b.hasTheme { - themeFolder := "static" - fs = afero.NewCopyOnWriteFs(newRealBase(afero.NewBasePathFs(b.themeFs, themeFolder)), fs) - for _, absThemeDir := range b.absThemeDirs { - s.Dirnames = append(s.Dirnames, filepath.Join(absThemeDir, themeFolder)) - } - } + var mounts []modules.Mount - s.Fs = fs - ms[l.Lang] = s +OUTER: + for i, mount := range md.mounts { + key := path.Join(mount.Lang, mount.Source, mount.Target) + if seen[key] { + continue + } + seen[key] = true + // Prevent overlapping mounts + for j, mount2 := range md.mounts { + if j == i || mount2.Target != mount.Target { + continue + } + source := mount.Source + if !strings.HasSuffix(source, filePathSeparator) { + source += filePathSeparator + } + if strings.HasPrefix(mount2.Source, source) { + continue OUTER + } } - return nil + mounts = append(mounts, mount) } - s := &SourceFilesystem{ - SourceFs: b.p.Fs.Source, - } + for _, mount := range mounts { - var staticDirs []string + mountWeight := 1 + if md.isMainProject { + mountWeight++ + } - for _, l := range b.p.Languages { - staticDirs = append(staticDirs, getStaticDirs(l)...) - } + rm := hugofs.RootMapping{ + From: mount.Target, + To: absPathify(mount.Source), + Meta: hugofs.FileMeta{ + "watch": md.watch, + "mountWeight": mountWeight, + }, + } - staticDirs = removeDuplicatesKeepRight(staticDirs) - if len(staticDirs) == 0 { - return nil - } + isContentMount := b.isContentMount(mount) - for _, dir := range staticDirs { - absDir := b.p.AbsPathify(dir) - if !b.existsInSource(absDir) { - continue + lang := mount.Lang + if lang == "" && isContentMount { + lang = b.p.DefaultContentLanguage } - s.Dirnames = append(s.Dirnames, absDir) - } - fs, err := createOverlayFs(b.p.Fs.Source, s.Dirnames) - if err != nil { - return err - } + rm.Meta["lang"] = lang - if b.hasTheme { - themeFolder := "static" - fs = afero.NewCopyOnWriteFs(newRealBase(afero.NewBasePathFs(b.themeFs, themeFolder)), fs) - for _, absThemeDir := range b.absThemeDirs { - s.Dirnames = append(s.Dirnames, filepath.Join(absThemeDir, themeFolder)) + if isContentMount { + fromToContent = append(fromToContent, rm) + } else { + fromTo = append(fromTo, rm) } } - s.Fs = fs - ms[""] = s - - return nil -} - -func getStaticDirs(cfg config.Provider) []string { - var staticDirs []string - for i := -1; i <= 10; i++ { - staticDirs = append(staticDirs, getStringOrStringSlice(cfg, "staticDir", i)...) + modBase := collector.sourceProject + if !md.isMainProject { + modBase = collector.sourceModules } - return staticDirs -} - -func getStringOrStringSlice(cfg config.Provider, key string, id int) []string { - if id >= 0 { - key = fmt.Sprintf("%s%d", key, id) + rmfs, err := hugofs.NewRootMappingFs(modBase, fromTo...) + if err != nil { + return err + } + rmfsContent, err := hugofs.NewRootMappingFs(modBase, fromToContent...) + if err != nil { + return err } - return config.GetStringSlicePreserveString(cfg, key) + // We need to keep the ordered list of directories for watching and + // some special merge operations (data, i18n). + collector.addDirs(rmfs) + collector.addDirs(rmfsContent) -} + if collector.staticPerLanguage != nil { + for _, l := range b.p.Languages { + lang := l.Lang -func createContentFs(fs afero.Fs, - workingDir, - defaultContentLanguage string, - languages langs.Languages) (afero.Fs, []string, error) { + lfs := rmfs.Filter(func(rm hugofs.RootMapping) bool { + rlang := rm.Meta.Lang() + return rlang == "" || rlang == lang + }) - var contentLanguages langs.Languages - var contentDirSeen = make(map[string]bool) - languageSet := make(map[string]bool) + bfs := afero.NewBasePathFs(lfs, files.ComponentFolderStatic) - // The default content language needs to be first. - for _, language := range languages { - if language.Lang == defaultContentLanguage { - contentLanguages = append(contentLanguages, language) - contentDirSeen[language.ContentDir] = true - } - languageSet[language.Lang] = true - } + sfs, found := collector.staticPerLanguage[lang] + if found { + collector.staticPerLanguage[lang] = afero.NewCopyOnWriteFs(sfs, bfs) - for _, language := range languages { - if contentDirSeen[language.ContentDir] { - continue - } - if language.ContentDir == "" { - language.ContentDir = defaultContentLanguage + } else { + collector.staticPerLanguage[lang] = bfs + } } - contentDirSeen[language.ContentDir] = true - contentLanguages = append(contentLanguages, language) - } - var absContentDirs []string - - fs, err := createContentOverlayFs(fs, workingDir, contentLanguages, languageSet, &absContentDirs) - return fs, absContentDirs, err - -} - -func createContentOverlayFs(source afero.Fs, - workingDir string, - languages langs.Languages, - languageSet map[string]bool, - absContentDirs *[]string) (afero.Fs, error) { - if len(languages) == 0 { - return source, nil + getResourcesDir := func() string { + if md.isMainProject { + return b.p.AbsResourcesDir + } + return absPathify(files.FolderResources) } - language := languages[0] + if collector.overlayMounts == nil { + collector.overlayMounts = rmfs + collector.overlayMountsContent = rmfsContent + collector.overlayFull = afero.NewBasePathFs(modBase, md.dir) + collector.overlayResources = afero.NewBasePathFs(modBase, getResourcesDir()) + } else { - contentDir := language.ContentDir - if contentDir == "" { - panic("missing contentDir") + collector.overlayMounts = afero.NewCopyOnWriteFs(collector.overlayMounts, rmfs) + collector.overlayMountsContent = hugofs.NewLanguageCompositeFs(collector.overlayMountsContent, rmfsContent) + collector.overlayFull = afero.NewCopyOnWriteFs(collector.overlayFull, afero.NewBasePathFs(modBase, md.dir)) + collector.overlayResources = afero.NewCopyOnWriteFs(collector.overlayResources, afero.NewBasePathFs(modBase, getResourcesDir())) } - absContentDir := paths.AbsPathify(workingDir, language.ContentDir) - if !strings.HasSuffix(absContentDir, paths.FilePathSeparator) { - absContentDir += paths.FilePathSeparator - } + return nil - // If root, remove the second '/' - if absContentDir == "//" { - absContentDir = paths.FilePathSeparator - } +} - if len(absContentDir) < 6 { - return nil, fmt.Errorf("invalid content dir %q: Path is too short", absContentDir) +func printFs(fs afero.Fs, path string, w io.Writer) { + if fs == nil { + return } + afero.Walk(fs, path, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + var filename string + if fim, ok := info.(hugofs.FileMetaInfo); ok { + filename = fim.Meta().Filename() + } + fmt.Fprintf(w, " %q %q\n", path, filename) + return nil + }) +} - *absContentDirs = append(*absContentDirs, absContentDir) +type filesystemsCollector struct { + sourceProject afero.Fs // Source for project folders + sourceModules afero.Fs // Source for modules/themes - overlay := hugofs.NewLanguageFs(language.Lang, languageSet, afero.NewBasePathFs(source, absContentDir)) - if len(languages) == 1 { - return overlay, nil - } + overlayMounts afero.Fs + overlayMountsContent afero.Fs + overlayFull afero.Fs + overlayResources afero.Fs - base, err := createContentOverlayFs(source, workingDir, languages[1:], languageSet, absContentDirs) - if err != nil { - return nil, err - } + // Maps component type (layouts, static, content etc.) an ordered list of + // directories representing the overlay filesystems above. + overlayDirs map[string][]hugofs.FileMetaInfo - return hugofs.NewLanguageCompositeFs(base, overlay), nil + // Set if in multihost mode + staticPerLanguage map[string]afero.Fs + finalizerInit sync.Once } -func createThemesOverlayFs(p *paths.Paths) (afero.Fs, []string, error) { - - themes := p.AllThemes +func (c *filesystemsCollector) addDirs(rfs *hugofs.RootMappingFs) { + for _, componentFolder := range files.ComponentFolders { + dirs, err := rfs.Dirs(componentFolder) - if len(themes) == 0 { - panic("AllThemes not set") + if err == nil { + c.overlayDirs[componentFolder] = append(c.overlayDirs[componentFolder], dirs...) + } } +} - themesDir := p.AbsPathify(p.ThemesDir) - if themesDir == "" { - return nil, nil, errors.New("no themes dir set") - } +func (c *filesystemsCollector) finalizeDirs() { + c.finalizerInit.Do(func() { + // Order the directories from top to bottom (project, theme a, theme ...). + for _, dirs := range c.overlayDirs { + c.reverseFis(dirs) + } + }) - absPaths := make([]string, len(themes)) +} - // The themes are ordered from left to right. We need to revert it to get the - // overlay logic below working as expected. - for i := 0; i < len(themes); i++ { - absPaths[i] = filepath.Join(themesDir, themes[len(themes)-1-i].Name) +func (c *filesystemsCollector) reverseFis(fis []hugofs.FileMetaInfo) { + for i := len(fis)/2 - 1; i >= 0; i-- { + opp := len(fis) - 1 - i + fis[i], fis[opp] = fis[opp], fis[i] } - - fs, err := createOverlayFs(p.Fs.Source, absPaths) - fs = hugofs.NewNoLstatFs(fs) - - return fs, absPaths, err - } -func createOverlayFs(source afero.Fs, absPaths []string) (afero.Fs, error) { - if len(absPaths) == 0 { - return hugofs.NoOpFs, nil - } +type mountsDescriptor struct { + mounts []modules.Mount + dir string + watch bool // whether this is a candidate for watching in server mode. + isMainProject bool +} - if len(absPaths) == 1 { - return afero.NewReadOnlyFs(newRealBase(afero.NewBasePathFs(source, absPaths[0]))), nil +func (b *sourceFilesystemsBuilder) createOverlayFs(collector *filesystemsCollector, mounts []mountsDescriptor) error { + if len(mounts) == 0 { + return nil } - base := afero.NewReadOnlyFs(newRealBase(afero.NewBasePathFs(source, absPaths[0]))) - overlay, err := createOverlayFs(source, absPaths[1:]) + err := b.createModFs(collector, mounts[0]) if err != nil { - return nil, err + return err } - return afero.NewCopyOnWriteFs(base, overlay), nil -} - -func removeDuplicatesKeepRight(in []string) []string { - seen := make(map[string]bool) - var out []string - for i := len(in) - 1; i >= 0; i-- { - v := in[i] - if seen[v] { - continue - } - out = append([]string{v}, out...) - seen[v] = true + if len(mounts) == 1 { + return nil } - return out + return b.createOverlayFs(collector, mounts[1:]) } diff --git a/hugolib/filesystems/basefs_test.go b/hugolib/filesystems/basefs_test.go index ec6ccb30c41..eccbe00f222 100644 --- a/hugolib/filesystems/basefs_test.go +++ b/hugolib/filesystems/basefs_test.go @@ -1,4 +1,4 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. +// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,18 +18,59 @@ import ( "fmt" "os" "path/filepath" + "strings" "testing" + "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/langs" "github.com/spf13/afero" "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/hugolib/paths" + "github.com/gohugoio/hugo/modules" "github.com/spf13/viper" "github.com/stretchr/testify/require" ) +func initConfig(fs afero.Fs, cfg config.Provider) error { + if _, err := langs.LoadLanguageSettings(cfg, nil); err != nil { + return err + } + + modConfig, err := modules.DecodeConfig(cfg) + if err != nil { + return err + } + + workingDir := cfg.GetString("workingDir") + themesDir := cfg.GetString("themesDir") + if !filepath.IsAbs(themesDir) { + themesDir = filepath.Join(workingDir, themesDir) + } + modulesClient := modules.NewClient(modules.ClientConfig{ + Fs: fs, + WorkingDir: workingDir, + ThemesDir: themesDir, + ModuleConfig: modConfig, + IgnoreVendor: true, + }) + + moduleConfig, err := modulesClient.Collect() + if err != nil { + return err + } + + if err := modules.ApplyProjectConfigDefaults(cfg, moduleConfig.ActiveModules[len(moduleConfig.ActiveModules)-1]); err != nil { + return err + } + + cfg.Set("allModules", moduleConfig.ActiveModules) + + return nil +} + func TestNewBaseFs(t *testing.T) { assert := require.New(t) v := viper.New() @@ -40,16 +81,21 @@ func TestNewBaseFs(t *testing.T) { workingDir := filepath.FromSlash("/my/work") v.Set("workingDir", workingDir) + v.Set("contentDir", "content") v.Set("themesDir", "themes") + v.Set("defaultContentLanguage", "en") v.Set("theme", themes[:1]) // Write some data to the themes for _, theme := range themes { for _, dir := range []string{"i18n", "data", "archetypes", "layouts"} { base := filepath.Join(workingDir, "themes", theme, dir) - filename := filepath.Join(base, fmt.Sprintf("theme-file-%s.txt", theme)) + filenameTheme := filepath.Join(base, fmt.Sprintf("theme-file-%s.txt", theme)) + filenameOverlap := filepath.Join(base, "f3.txt") fs.Source.Mkdir(base, 0755) - afero.WriteFile(fs.Source, filename, []byte(fmt.Sprintf("content:%s:%s", theme, dir)), 0755) + content := []byte(fmt.Sprintf("content:%s:%s", theme, dir)) + afero.WriteFile(fs.Source, filenameTheme, content, 0755) + afero.WriteFile(fs.Source, filenameOverlap, content, 0755) } // Write some files to the root of the theme base := filepath.Join(workingDir, "themes", theme) @@ -73,6 +119,7 @@ theme = ["atheme"] setConfigAndWriteSomeFilesTo(fs.Source, v, "resourceDir", "myrsesource", 10) v.Set("publishDir", "public") + assert.NoError(initConfig(fs.Source, v)) p, err := paths.New(fs, v) assert.NoError(err) @@ -85,33 +132,26 @@ theme = ["atheme"] assert.NoError(err) dirnames, err := root.Readdirnames(-1) assert.NoError(err) - assert.Equal([]string{projectVirtualFolder, "btheme", "atheme"}, dirnames) - ff, err := bfs.I18n.Fs.Open("myi18n") - assert.NoError(err) - _, err = ff.Readdirnames(-1) - assert.NoError(err) + assert.Equal([]string{"f1.txt", "f2.txt", "f3.txt", "f4.txt", "f3.txt", "theme-file-btheme.txt", "f3.txt", "theme-file-atheme.txt"}, dirnames) root, err = bfs.Data.Fs.Open("") assert.NoError(err) dirnames, err = root.Readdirnames(-1) assert.NoError(err) - assert.Equal([]string{projectVirtualFolder, "btheme", "atheme"}, dirnames) - ff, err = bfs.I18n.Fs.Open("mydata") - assert.NoError(err) - _, err = ff.Readdirnames(-1) - assert.NoError(err) + assert.Equal([]string{"f1.txt", "f2.txt", "f3.txt", "f4.txt", "f5.txt", "f6.txt", "f7.txt", "f3.txt", "theme-file-btheme.txt", "f3.txt", "theme-file-atheme.txt"}, dirnames) + + //printFs(bfs.Work, "", os.Stdout) - checkFileCount(bfs.Content.Fs, "", assert, 3) - checkFileCount(bfs.I18n.Fs, "", assert, 6) // 4 + 2 themes checkFileCount(bfs.Layouts.Fs, "", assert, 7) + + checkFileCount(bfs.Content.Fs, "", assert, 3) + checkFileCount(bfs.I18n.Fs, "", assert, 8) // 4 + 4 themes + checkFileCount(bfs.Static[""].Fs, "", assert, 6) - checkFileCount(bfs.Data.Fs, "", assert, 9) // 7 + 2 themes + checkFileCount(bfs.Data.Fs, "", assert, 11) // 7 + 4 themes checkFileCount(bfs.Archetypes.Fs, "", assert, 10) // 8 + 2 themes checkFileCount(bfs.Assets.Fs, "", assert, 9) - checkFileCount(bfs.Resources.Fs, "", assert, 10) - checkFileCount(bfs.Work.Fs, "", assert, 78) - - assert.Equal([]string{filepath.FromSlash("/my/work/mydata"), filepath.FromSlash("/my/work/themes/btheme/data"), filepath.FromSlash("/my/work/themes/atheme/data")}, bfs.Data.Dirnames) + checkFileCount(bfs.Work, "", assert, 82) assert.True(bfs.IsData(filepath.Join(workingDir, "mydata", "file1.txt"))) assert.True(bfs.IsI18n(filepath.Join(workingDir, "myi18n", "file1.txt"))) @@ -125,13 +165,13 @@ theme = ["atheme"] assert.Equal("file1.txt", rel) // Check Work fs vs theme - checkFileContent(bfs.Work.Fs, "file-root.txt", assert, "content-project") - checkFileContent(bfs.Work.Fs, "theme-root-atheme.txt", assert, "content:atheme") + checkFileContent(bfs.Work, "file-root.txt", assert, "content-project") + checkFileContent(bfs.Work, "theme-root-atheme.txt", assert, "content:atheme") // https://github.com/gohugoio/hugo/issues/5318 // Check both project and theme. for _, fs := range []afero.Fs{bfs.Archetypes.Fs, bfs.Layouts.Fs} { - for _, filename := range []string{"/file1.txt", "/theme-file-atheme.txt"} { + for _, filename := range []string{"/f1.txt", "/theme-file-atheme.txt"} { filename = filepath.FromSlash(filename) f, err := fs.Open(filename) assert.NoError(err) @@ -153,6 +193,7 @@ func createConfig() *viper.Viper { v.Set("assetDir", "myassets") v.Set("resourceDir", "resources") v.Set("publishDir", "public") + v.Set("defaultContentLanguage", "en") return v } @@ -161,17 +202,18 @@ func TestNewBaseFsEmpty(t *testing.T) { assert := require.New(t) v := createConfig() fs := hugofs.NewMem(v) + assert.NoError(initConfig(fs.Source, v)) + p, err := paths.New(fs, v) assert.NoError(err) bfs, err := NewBase(p) assert.NoError(err) assert.NotNil(bfs) - assert.Equal(hugofs.NoOpFs, bfs.Archetypes.Fs) - assert.Equal(hugofs.NoOpFs, bfs.Layouts.Fs) - assert.Equal(hugofs.NoOpFs, bfs.Data.Fs) - assert.Equal(hugofs.NoOpFs, bfs.Assets.Fs) - assert.Equal(hugofs.NoOpFs, bfs.I18n.Fs) - assert.NotNil(bfs.Work.Fs) + assert.NotNil(bfs.Archetypes.Fs) + assert.NotNil(bfs.Layouts.Fs) + assert.NotNil(bfs.Data.Fs) + assert.NotNil(bfs.I18n.Fs) + assert.NotNil(bfs.Work) assert.NotNil(bfs.Content.Fs) assert.NotNil(bfs.Static) } @@ -217,11 +259,14 @@ func TestRealDirs(t *testing.T) { afero.WriteFile(sfs, filepath.Join(filepath.Join(root, "myassets", "js", "f2", "a1.js")), []byte("content"), 0755) afero.WriteFile(sfs, filepath.Join(filepath.Join(root, "myassets", "js", "a2.js")), []byte("content"), 0755) + assert.NoError(initConfig(fs.Source, v)) + p, err := paths.New(fs, v) assert.NoError(err) bfs, err := NewBase(p) assert.NoError(err) assert.NotNil(bfs) + checkFileCount(bfs.Assets.Fs, "", assert, 6) realDirs := bfs.Assets.RealDirs("scss") @@ -229,13 +274,7 @@ func TestRealDirs(t *testing.T) { assert.Equal(filepath.Join(root, "myassets/scss"), realDirs[0]) assert.Equal(filepath.Join(themesDir, "mytheme/assets/scss"), realDirs[len(realDirs)-1]) - checkFileCount(bfs.Resources.Fs, "", assert, 3) - - assert.NotNil(bfs.themeFs) - fi, b, err := bfs.themeFs.(afero.Lstater).LstatIfPossible(filepath.Join("resources", "t1.txt")) - assert.NoError(err) - assert.False(b) - assert.Equal("t1.txt", fi.Name()) + assert.NotNil(bfs.theBigFs) } @@ -245,20 +284,25 @@ func TestStaticFs(t *testing.T) { workDir := "mywork" v.Set("workingDir", workDir) v.Set("themesDir", "themes") - v.Set("theme", "t1") + v.Set("theme", []string{"t1", "t2"}) fs := hugofs.NewMem(v) themeStaticDir := filepath.Join(workDir, "themes", "t1", "static") + themeStaticDir2 := filepath.Join(workDir, "themes", "t2", "static") afero.WriteFile(fs.Source, filepath.Join(workDir, "mystatic", "f1.txt"), []byte("Hugo Rocks!"), 0755) afero.WriteFile(fs.Source, filepath.Join(themeStaticDir, "f1.txt"), []byte("Hugo Themes Rocks!"), 0755) afero.WriteFile(fs.Source, filepath.Join(themeStaticDir, "f2.txt"), []byte("Hugo Themes Still Rocks!"), 0755) + afero.WriteFile(fs.Source, filepath.Join(themeStaticDir2, "f2.txt"), []byte("Hugo Themes Rocks in t2!"), 0755) + + assert.NoError(initConfig(fs.Source, v)) p, err := paths.New(fs, v) assert.NoError(err) bfs, err := NewBase(p) assert.NoError(err) + sfs := bfs.StaticFs("en") checkFileContent(sfs, "f1.txt", assert, "Hugo Rocks!") checkFileContent(sfs, "f2.txt", assert, "Hugo Themes Still Rocks!") @@ -272,21 +316,19 @@ func TestStaticFsMultiHost(t *testing.T) { v.Set("workingDir", workDir) v.Set("themesDir", "themes") v.Set("theme", "t1") - v.Set("multihost", true) - - vn := viper.New() - vn.Set("staticDir", "nn_static") - - en := langs.NewLanguage("en", v) - no := langs.NewLanguage("no", v) - no.Set("staticDir", "static_no") - - languages := langs.Languages{ - en, - no, + v.Set("defaultContentLanguage", "en") + + langConfig := map[string]interface{}{ + "no": map[string]interface{}{ + "staticDir": "static_no", + "baseURL": "https://example.org/no/", + }, + "en": map[string]interface{}{ + "baseURL": "https://example.org/en/", + }, } - v.Set("languagesSorted", languages) + v.Set("languages", langConfig) fs := hugofs.NewMem(v) @@ -298,6 +340,8 @@ func TestStaticFsMultiHost(t *testing.T) { afero.WriteFile(fs.Source, filepath.Join(themeStaticDir, "f1.txt"), []byte("Hugo Themes Rocks!"), 0755) afero.WriteFile(fs.Source, filepath.Join(themeStaticDir, "f2.txt"), []byte("Hugo Themes Still Rocks!"), 0755) + assert.NoError(initConfig(fs.Source, v)) + p, err := paths.New(fs, v) assert.NoError(err) bfs, err := NewBase(p) @@ -312,9 +356,9 @@ func TestStaticFsMultiHost(t *testing.T) { } func checkFileCount(fs afero.Fs, dirname string, assert *require.Assertions, expected int) { - count, _, err := countFileaAndGetDirs(fs, dirname) - assert.NoError(err) - assert.Equal(expected, count) + count, fnames, err := countFileaAndGetFilenames(fs, dirname) + assert.NoError(err, fnames) + assert.Equal(expected, count, fnames) } func checkFileContent(fs afero.Fs, filename string, assert *require.Assertions, expected ...string) { @@ -329,27 +373,38 @@ func checkFileContent(fs afero.Fs, filename string, assert *require.Assertions, } } -func countFileaAndGetDirs(fs afero.Fs, dirname string) (int, []string, error) { +func countFileaAndGetFilenames(fs afero.Fs, dirname string) (int, []string, error) { if fs == nil { return 0, nil, errors.New("no fs") } counter := 0 - var dirs []string - - afero.Walk(fs, dirname, func(path string, info os.FileInfo, err error) error { - if info != nil { - if !info.IsDir() { - counter++ - } else if info.Name() != "." { - dirs = append(dirs, filepath.Join(path, info.Name())) - } + var filenames []string + + wf := func(path string, info hugofs.FileMetaInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + counter++ + } + + if info.Name() != "." { + name := info.Name() + name = strings.Replace(name, filepath.FromSlash("/my/work"), "WORK_DIR", 1) + filenames = append(filenames, name) } return nil - }) + } + + w := hugofs.NewWalkway(hugofs.WalkwayConfig{Fs: fs, Root: dirname, WalkFn: wf}) + + if err := w.Walk(); err != nil { + return -1, nil, err + } - return counter, dirs, nil + return counter, filenames, nil } func setConfigAndWriteSomeFilesTo(fs afero.Fs, v *viper.Viper, key, val string, num int) { @@ -357,7 +412,7 @@ func setConfigAndWriteSomeFilesTo(fs afero.Fs, v *viper.Viper, key, val string, v.Set(key, val) fs.Mkdir(val, 0755) for i := 0; i < num; i++ { - filename := filepath.Join(workingDir, val, fmt.Sprintf("file%d.txt", i+1)) + filename := filepath.Join(workingDir, val, fmt.Sprintf("f%d.txt", i+1)) afero.WriteFile(fs, filename, []byte(fmt.Sprintf("content:%s:%d", key, i+1)), 0755) } } diff --git a/hugolib/hugo_modules_test.go b/hugolib/hugo_modules_test.go new file mode 100644 index 00000000000..c4bbafe7703 --- /dev/null +++ b/hugolib/hugo_modules_test.go @@ -0,0 +1,492 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "fmt" + "math/rand" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/gohugoio/hugo/common/loggers" + + "github.com/spf13/afero" + + "github.com/gohugoio/hugo/hugofs/files" + + "github.com/gohugoio/hugo/common/hugo" + + "github.com/gohugoio/hugo/htesting" + "github.com/gohugoio/hugo/hugofs" + + "github.com/gohugoio/testmodBuilder/mods" + "github.com/spf13/viper" + "github.com/stretchr/testify/require" +) + +// TODO(bep) this fails when testmodBuilder is also building ... +func TestHugoModules(t *testing.T) { + t.Parallel() + + if hugo.GoMinorVersion() < 12 { + // https://github.com/golang/go/issues/26794 + // There were some concurrent issues with Go modules in < Go 12. + t.Skip("skip this for Go <= 1.11 due to a bug in Go's stdlib") + } + + if testing.Short() { + t.Skip() + } + + rnd := rand.New(rand.NewSource(time.Now().UnixNano())) + gooss := []string{"linux", "darwin", "windows"} + goos := gooss[rnd.Intn(len(gooss))] + ignoreVendor := rnd.Intn(2) == 0 + testmods := mods.CreateModules(goos).Collect() + rnd.Shuffle(len(testmods), func(i, j int) { testmods[i], testmods[j] = testmods[j], testmods[i] }) + + for _, m := range testmods[:2] { + assert := require.New(t) + + v := viper.New() + + workingDir, clean, err := htesting.CreateTempDir(hugofs.Os, "hugo-modules-test") + assert.NoError(err) + defer clean() + + configTemplate := ` +baseURL = "https://example.com" +title = "My Modular Site" +workingDir = %q +theme = %q +ignoreVendor = %t + +` + + config := fmt.Sprintf(configTemplate, workingDir, m.Path(), ignoreVendor) + + b := newTestSitesBuilder(t) + + // Need to use OS fs for this. + b.Fs = hugofs.NewDefault(v) + + b.WithWorkingDir(workingDir).WithConfigFile("toml", config) + b.WithContent("page.md", ` +--- +title: "Foo" +--- +`) + b.WithTemplates("home.html", ` + +{{ $mod := .Site.Data.modinfo.module }} +Mod Name: {{ $mod.name }} +Mod Version: {{ $mod.version }} +---- +{{ range $k, $v := .Site.Data.modinfo }} +- {{ $k }}: {{ range $kk, $vv := $v }}{{ $kk }}: {{ $vv }}|{{ end -}} +{{ end }} + + +`) + b.WithSourceFile("go.mod", ` +module github.com/gohugoio/tests/testHugoModules + + +`) + + b.Build(BuildCfg{}) + + // Verify that go.mod is autopopulated with all the modules in config.toml. + b.AssertFileContent("go.mod", m.Path()) + + b.AssertFileContent("public/index.html", + "Mod Name: "+m.Name(), + "Mod Version: v1.4.0") + + b.AssertFileContent("public/index.html", createChildModMatchers(m, ignoreVendor, m.Vendor)...) + + } +} + +func createChildModMatchers(m *mods.Md, ignoreVendor, vendored bool) []string { + // Child depdendencies are one behind. + expectMinorVersion := 3 + + if !ignoreVendor && vendored { + // Vendored modules are stuck at v1.1.0. + expectMinorVersion = 1 + } + + expectVersion := fmt.Sprintf("v1.%d.0", expectMinorVersion) + + var matchers []string + + for _, mm := range m.Children { + matchers = append( + matchers, + fmt.Sprintf("%s: name: %s|version: %s", mm.Name(), mm.Name(), expectVersion)) + matchers = append(matchers, createChildModMatchers(mm, ignoreVendor, vendored || mm.Vendor)...) + } + return matchers +} + +func TestModulesWithContent(t *testing.T) { + t.Parallel() + + b := newTestSitesBuilder(t).WithWorkingDir("/site").WithConfigFile("toml", ` +baseURL="https://example.org" + +workingDir="/site" + +defaultContentLanguage = "en" + +[module] +[[module.imports]] +path="a" +[[module.imports.mounts]] +source="myacontent" +target="content/blog" +lang="en" +[[module.imports]] +path="b" +[[module.imports.mounts]] +source="mybcontent" +target="content/blog" +lang="nn" +[[module.imports]] +path="c" +[[module.imports]] +path="d" + +[languages] + +[languages.en] +title = "Title in English" +languageName = "English" +weight = 1 +[languages.nn] +languageName = "Nynorsk" +weight = 2 +title = "Tittel på nynorsk" +[languages.nb] +languageName = "Bokmål" +weight = 3 +title = "Tittel på bokmål" +[languages.fr] +languageName = "French" +weight = 4 +title = "French Title" + + +`) + + b.WithTemplatesAdded("index.html", ` +{{ range .Site.RegularPages }} +|{{ .Title }}|{{ .RelPermalink }}|{{ .Plain }} +{{ end }} +{{ $data := .Site.Data }} +Data Common: {{ $data.common.value }} +Data C: {{ $data.c.value }} +Data D: {{ $data.d.value }} +All Data: {{ $data }} + +i18n hello: {{ i18n "hello" . }} +i18n theme: {{ i18n "theme" . }} +i18n theme2: {{ i18n "theme2" . }} +`) + + content := func(id string) string { + return fmt.Sprintf(`--- +title: Title %s +--- +Content %s + +`, id, id) + } + + i18nContent := func(id, value string) string { + return fmt.Sprintf(` +[%s] +other = %q +`, id, value) + } + + // Content files + b.WithSourceFile("themes/a/myacontent/page.md", content("theme-a-en")) + b.WithSourceFile("themes/b/mybcontent/page.md", content("theme-b-nn")) + b.WithSourceFile("themes/c/content/blog/c.md", content("theme-c-nn")) + + // Data files + b.WithSourceFile("data/common.toml", `value="Project"`) + b.WithSourceFile("themes/c/data/common.toml", `value="Theme C"`) + b.WithSourceFile("themes/c/data/c.toml", `value="Hugo Rocks!"`) + b.WithSourceFile("themes/d/data/c.toml", `value="Hugo Rodcks!"`) + b.WithSourceFile("themes/d/data/d.toml", `value="Hugo Rodks!"`) + + // i18n files + b.WithSourceFile("i18n/en.toml", i18nContent("hello", "Project")) + b.WithSourceFile("themes/c/en.toml", i18nContent("hello", "Theme C")) + b.WithSourceFile("themes/c/i18n/en.toml", i18nContent("theme", "Theme C")) + b.WithSourceFile("themes/d/i18n/en.toml", i18nContent("theme", "Theme D")) + b.WithSourceFile("themes/d/i18n/en.toml", i18nContent("theme2", "Theme2 D")) + + // Static files + b.WithSourceFile("themes/c/static/hello.txt", `Hugo Rocks!"`) + + b.Build(BuildCfg{}) + + b.AssertFileContent("public/index.html", "|Title theme-a-en|/blog/page/|Content theme-a-en") + b.AssertFileContent("public/nn/index.html", "|Title theme-b-nn|/nn/blog/page/|Content theme-b-nn") + + // Data + b.AssertFileContent("public/index.html", + "Data Common: Project", + "Data C: Hugo Rocks!", + "Data D: Hugo Rodks!", + ) + + // i18n + b.AssertFileContent("public/index.html", + "i18n hello: Project", + "i18n theme: Theme C", + "i18n theme2: Theme2 D", + ) + +} + +func TestModulesIgnoreConfig(t *testing.T) { + b := newTestSitesBuilder(t).WithWorkingDir("/site").WithConfigFile("toml", ` +baseURL="https://example.org" + +workingDir="/site" + +[module] +[[module.imports]] +path="a" +ignoreConfig=true + +`) + + b.WithSourceFile("themes/a/config.toml", ` +[params] +a = "Should Be Ignored!" +`) + + b.WithTemplatesAdded("index.html", `Params: {{ .Site.Params }}`) + + b.Build(BuildCfg{}) + + b.AssertFileContentFn("public/index.html", func(s string) bool { + return !strings.Contains(s, "Ignored") + }) + +} + +func TestModulesDisabled(t *testing.T) { + b := newTestSitesBuilder(t).WithWorkingDir("/site").WithConfigFile("toml", ` +baseURL="https://example.org" + +workingDir="/site" + +[module] +[[module.imports]] +path="a" +[[module.imports]] +path="b" +disabled=true + + +`) + + b.WithSourceFile("themes/a/config.toml", ` +[params] +a = "A param" +`) + + b.WithSourceFile("themes/b/config.toml", ` +[params] +b = "B param" +`) + + b.WithTemplatesAdded("index.html", `Params: {{ .Site.Params }}`) + + b.Build(BuildCfg{}) + + b.AssertFileContentFn("public/index.html", func(s string) bool { + return strings.Contains(s, "A param") && !strings.Contains(s, "B param") + }) + +} + +func TestModulesIncompatible(t *testing.T) { + b := newTestSitesBuilder(t).WithWorkingDir("/site").WithConfigFile("toml", ` +baseURL="https://example.org" + +workingDir="/site" + +[module] +[[module.imports]] +path="ok" +[[module.imports]] +path="incompat1" +[[module.imports]] +path="incompat2" + + +`) + + b.WithSourceFile("themes/ok/data/ok.toml", `title = "OK"`) + + b.WithSourceFile("themes/incompat1/config.toml", ` + +[module] +[module.hugoVersion] +min = "0.33.2" +max = "0.45.0" + +`) + + // Old setup. + b.WithSourceFile("themes/incompat2/theme.toml", ` +min_version = "5.0.0" + +`) + + logger := loggers.NewWarningLogger() + b.WithLogger(logger) + + b.Build(BuildCfg{}) + + assert := require.New(t) + + assert.Equal(uint64(2), logger.WarnCounter.Count()) + +} + +func TestModulesSymlinks(t *testing.T) { + skipSymlink(t) + + wd, _ := os.Getwd() + defer func() { + os.Chdir(wd) + }() + + assert := require.New(t) + // We need to use the OS fs for this. + cfg := viper.New() + fs := hugofs.NewFrom(hugofs.Os, cfg) + + workDir, clean, err := htesting.CreateTempDir(hugofs.Os, "hugo-mod-sym") + assert.NoError(err) + + defer clean() + + const homeTemplate = ` +Data: {{ .Site.Data }} +` + + createDirsAndFiles := func(baseDir string) { + for _, dir := range files.ComponentFolders { + realDir := filepath.Join(baseDir, dir, "real") + assert.NoError(os.MkdirAll(realDir, 0777)) + assert.NoError(afero.WriteFile(fs.Source, filepath.Join(realDir, "data.toml"), []byte("[hello]\nother = \"hello\""), 0777)) + } + + assert.NoError(afero.WriteFile(fs.Source, filepath.Join(baseDir, "layouts", "index.html"), []byte(homeTemplate), 0777)) + } + + // Create project dirs and files. + createDirsAndFiles(workDir) + // Create one module inside the default themes folder. + themeDir := filepath.Join(workDir, "themes", "mymod") + createDirsAndFiles(themeDir) + + createSymlinks := func(baseDir, id string) { + for _, dir := range files.ComponentFolders { + assert.NoError(os.Chdir(filepath.Join(baseDir, dir))) + assert.NoError(os.Symlink("real", fmt.Sprintf("realsym%s", id))) + assert.NoError(os.Chdir(filepath.Join(baseDir, dir, "real"))) + assert.NoError(os.Symlink("data.toml", fmt.Sprintf(filepath.FromSlash("datasym%s.toml"), id))) + } + } + + createSymlinks(workDir, "project") + createSymlinks(themeDir, "mod") + + config := ` +baseURL = "https://example.com" +theme="mymod" +defaultContentLanguage="nn" +defaultContentLanguageInSubDir=true + +[languages] +[languages.nn] +weight = 1 +[languages.en] +weight = 2 + + +` + + b := newTestSitesBuilder(t).WithNothingAdded().WithWorkingDir(workDir) + b.Fs = fs + + b.WithConfigFile("toml", config) + assert.NoError(os.Chdir(workDir)) + + b.Build(BuildCfg{}) + + b.AssertFileContentFn(filepath.Join("public", "en", "index.html"), func(s string) bool { + // Symbolic links only followed in project. There should be WARNING logs. + return !strings.Contains(s, "symmod") && strings.Contains(s, "symproject") + }) + + bfs := b.H.BaseFs + + for _, componentFs := range []afero.Fs{ + bfs.Archetypes.Fs, + bfs.Content.Fs, + bfs.Data.Fs, + bfs.Assets.Fs, + bfs.Static[""].Fs, + bfs.I18n.Fs} { + + for i, id := range []string{"mod", "project"} { + + statCheck := func(fs afero.Fs, filename string) { + shouldFail := i == 0 + _, err := fs.Stat(filepath.FromSlash(filename)) + if err != nil { + if strings.HasSuffix(filename, "toml") && strings.Contains(err.Error(), "files not supported") { + // OK + return + } + } + if shouldFail { + assert.Error(err) + assert.Equal(hugofs.ErrPermissionSymlink, err) + } else { + assert.NoError(err) + } + } + + statCheck(componentFs, fmt.Sprintf("realsym%s", id)) + statCheck(componentFs, fmt.Sprintf("real/datasym%s.toml", id)) + + } + } +} diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go index e852e7f1dd4..6ad8715645b 100644 --- a/hugolib/hugo_sites.go +++ b/hugolib/hugo_sites.go @@ -22,6 +22,8 @@ import ( "strings" "sync" + radix "github.com/hashicorp/go-immutable-radix" + "github.com/gohugoio/hugo/output" "github.com/gohugoio/hugo/parser/metadecoders" @@ -33,7 +35,6 @@ import ( "github.com/bep/gitmap" "github.com/gohugoio/hugo/config" - "github.com/spf13/afero" "github.com/gohugoio/hugo/publisher" @@ -199,10 +200,11 @@ func (h *HugoSites) IsMultihost() bool { return h != nil && h.multihost } -func (h *HugoSites) LanguageSet() map[string]bool { - set := make(map[string]bool) - for _, s := range h.Sites { - set[s.language.Lang] = true +// TODO(bep) consolidate +func (h *HugoSites) LanguageSet() map[string]int { + set := make(map[string]int) + for i, s := range h.Sites { + set[s.language.Lang] = i } return set } @@ -222,14 +224,6 @@ func (h *HugoSites) PrintProcessingStats(w io.Writer) { helpers.ProcessingStatsTable(w, stats...) } -func (h *HugoSites) langSite() map[string]*Site { - m := make(map[string]*Site) - for _, s := range h.Sites { - m[s.language.Lang] = s - } - return m -} - // GetContentPage finds a Page with content given the absolute filename. // Returns nil if none found. func (h *HugoSites) GetContentPage(filename string) page.Page { @@ -265,7 +259,7 @@ func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) { langConfig, err := newMultiLingualFromSites(cfg.Cfg, sites...) if err != nil { - return nil, err + return nil, errors.Wrap(err, "failed to create language config") } var contentChangeTracker *contentChangeMap @@ -288,8 +282,11 @@ func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) { } h.init.data.Add(func() (interface{}, error) { - err := h.loadData(h.PathSpec.BaseFs.Data.Fs) - return err, nil + err := h.loadData(h.PathSpec.BaseFs.Data.Dirs) + if err != nil { + return nil, errors.Wrap(err, "failed to load data") + } + return nil, nil }) h.init.translations.Add(func() (interface{}, error) { @@ -303,7 +300,10 @@ func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) { h.init.gitInfo.Add(func() (interface{}, error) { err := h.loadGitInfo() - return nil, err + if err != nil { + return nil, errors.Wrap(err, "failed to load Git info") + } + return nil, nil }) for _, s := range sites { @@ -311,7 +311,7 @@ func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) { } if err := applyDeps(cfg, sites...); err != nil { - return nil, err + return nil, errors.Wrap(err, "add site dependencies") } h.Deps = sites[0].Deps @@ -319,7 +319,12 @@ func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) { // Only needed in server mode. // TODO(bep) clean up the running vs watching terms if cfg.Running { - contentChangeTracker = &contentChangeMap{pathSpec: h.PathSpec, symContent: make(map[string]map[string]bool)} + contentChangeTracker = &contentChangeMap{ + pathSpec: h.PathSpec, + symContent: make(map[string]map[string]bool), + leafBundles: radix.New(), + branchBundles: make(map[string]bool), + } h.ContentChanges = contentChangeTracker } @@ -371,7 +376,7 @@ func applyDeps(cfg deps.DepsCfg, sites ...*Site) error { siteConfig, err := loadSiteConfig(s.language) if err != nil { - return err + return errors.Wrap(err, "load site config") } s.siteConfigConfig = siteConfig s.siteRefLinker, err = newSiteRefLinker(s.language, s) @@ -388,17 +393,17 @@ func applyDeps(cfg deps.DepsCfg, sites ...*Site) error { var err error d, err = deps.New(cfg) if err != nil { - return err + return errors.Wrap(err, "create deps") } d.OutputFormatsConfig = s.outputFormatsConfig if err := onCreated(d); err != nil { - return err + return errors.Wrap(err, "on created") } if err = d.LoadResources(); err != nil { - return err + return errors.Wrap(err, "load resources") } } else { @@ -418,7 +423,7 @@ func applyDeps(cfg deps.DepsCfg, sites ...*Site) error { func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) { sites, err := createSitesFromConfig(cfg) if err != nil { - return nil, err + return nil, errors.Wrap(err, "from config") } return newHugoSites(cfg, sites...) } @@ -800,41 +805,45 @@ func (h *HugoSites) Pages() page.Pages { return h.Sites[0].AllPages() } -func (h *HugoSites) loadData(fs afero.Fs) (err error) { - spec := source.NewSourceSpec(h.PathSpec, fs) - fileSystem := spec.NewFilesystem("") +func (h *HugoSites) loadData(fis []hugofs.FileMetaInfo) (err error) { + spec := source.NewSourceSpec(h.PathSpec, nil) + h.data = make(map[string]interface{}) - for _, r := range fileSystem.Files() { - if err := h.handleDataFile(r); err != nil { + for _, fi := range fis { + fileSystem := spec.NewFilesystemFromFileMetaInfo(fi) + files, err := fileSystem.Files() + if err != nil { return err } + for _, r := range files { + if err := h.handleDataFile(r); err != nil { + return err + } + } } return } -func (h *HugoSites) handleDataFile(r source.ReadableFile) error { +func (h *HugoSites) handleDataFile(r source.File) error { var current map[string]interface{} - f, err := r.Open() + f, err := r.FileInfo().Meta().Open() if err != nil { - return errors.Wrapf(err, "Failed to open data file %q:", r.LogicalName()) + return errors.Wrapf(err, "data: failed to open %q:", r.LogicalName()) } defer f.Close() // Crawl in data tree to insert data current = h.data keyParts := strings.Split(r.Dir(), helpers.FilePathSeparator) - // The first path element is the virtual folder (typically theme name), which is - // not part of the key. - if len(keyParts) > 1 { - for _, key := range keyParts[1:] { - if key != "" { - if _, ok := current[key]; !ok { - current[key] = make(map[string]interface{}) - } - current = current[key].(map[string]interface{}) + + for _, key := range keyParts { + if key != "" { + if _, ok := current[key]; !ok { + current[key] = make(map[string]interface{}) } + current = current[key].(map[string]interface{}) } } @@ -848,15 +857,10 @@ func (h *HugoSites) handleDataFile(r source.ReadableFile) error { } // filepath.Walk walks the files in lexical order, '/' comes before '.' - // this warning could happen if - // 1. A theme uses the same key; the main data folder wins - // 2. A sub folder uses the same key: the sub folder wins higherPrecedentData := current[r.BaseFileName()] switch data.(type) { case nil: - // hear the crickets? - case map[string]interface{}: switch higherPrecedentData.(type) { @@ -868,7 +872,11 @@ func (h *HugoSites) handleDataFile(r source.ReadableFile) error { higherPrecedentMap := higherPrecedentData.(map[string]interface{}) for key, value := range data.(map[string]interface{}) { if _, exists := higherPrecedentMap[key]; exists { - h.Log.WARN.Printf("Data for key '%s' in path '%s' is overridden by higher precedence data already in the data tree", key, r.Path()) + // this warning could happen if + // 1. A theme uses the same key; the main data folder wins + // 2. A sub folder uses the same key: the sub folder wins + // TODO(bep) figure out a way to detect 2) above and make that a WARN + h.Log.INFO.Printf("Data for key '%s' in path '%s' is overridden by higher precedence data already in the data tree", key, r.Path()) } else { higherPrecedentMap[key] = value } @@ -896,12 +904,12 @@ func (h *HugoSites) handleDataFile(r source.ReadableFile) error { } func (h *HugoSites) errWithFileContext(err error, f source.File) error { - rfi, ok := f.FileInfo().(hugofs.RealFilenameInfo) + fim, ok := f.FileInfo().(hugofs.FileMetaInfo) if !ok { return err } - realFilename := rfi.RealFilename() + realFilename := fim.Meta().Filename() err, _ = herrors.WithFileContextForFile( err, @@ -913,8 +921,8 @@ func (h *HugoSites) errWithFileContext(err error, f source.File) error { return err } -func (h *HugoSites) readData(f source.ReadableFile) (interface{}, error) { - file, err := f.Open() +func (h *HugoSites) readData(f source.File) (interface{}, error) { + file, err := f.FileInfo().Meta().Open() if err != nil { return nil, errors.Wrap(err, "readData: failed to open data file") } @@ -939,9 +947,14 @@ func (h *HugoSites) findPagesByShortcode(shortcode string) page.Pages { // Used in partial reloading to determine if the change is in a bundle. type contentChangeMap struct { - mu sync.RWMutex - branches []string - leafs []string + mu sync.RWMutex + + // Holds directories with leaf bundles. + leafBundles *radix.Tree + leafBundlesTxn *radix.Txn + + // Holds directories with branch bundles. + branchBundles map[string]bool pathSpec *helpers.PathSpec @@ -950,9 +963,22 @@ type contentChangeMap struct { // locations in /content -- which is really cool, but also means we have to // go an extra mile to handle changes. // This map is only used in watch mode. - // It maps either file to files or the real dir to a set of content directories where it is in use. - symContent map[string]map[string]bool + // It maps either file to files or the real dir to a set of content directories + // where it is in use. symContentMu sync.Mutex + symContent map[string]map[string]bool +} + +func (m *contentChangeMap) start() { + m.mu.Lock() + m.leafBundlesTxn = m.leafBundles.Txn() + m.mu.Unlock() +} + +func (m *contentChangeMap) stop() { + m.mu.Lock() + m.leafBundles = m.leafBundlesTxn.Commit() + m.mu.Unlock() } func (m *contentChangeMap) add(filename string, tp bundleDirType) { @@ -961,68 +987,63 @@ func (m *contentChangeMap) add(filename string, tp bundleDirType) { dir = strings.TrimPrefix(dir, ".") switch tp { case bundleBranch: - m.branches = append(m.branches, dir) + m.branchBundles[dir] = true case bundleLeaf: - m.leafs = append(m.leafs, dir) + m.leafBundlesTxn.Insert([]byte(dir), true) default: panic("invalid bundle type") } m.mu.Unlock() } -// Track the addition of bundle dirs. -func (m *contentChangeMap) handleBundles(b *bundleDirs) { - for _, bd := range b.bundles { - m.add(bd.fi.Path(), bd.tp) - } -} - -// resolveAndRemove resolves the given filename to the root folder of a bundle, if relevant. -// It also removes the entry from the map. It will be re-added again by the partial -// build if it still is a bundle. func (m *contentChangeMap) resolveAndRemove(filename string) (string, string, bundleDirType) { m.mu.RLock() defer m.mu.RUnlock() // Bundles share resources, so we need to start from the virtual root. - relPath := m.pathSpec.RelContentDir(filename) - dir, name := filepath.Split(relPath) + relFilename := m.pathSpec.RelContentDir(filename) + dir, name := filepath.Split(relFilename) if !strings.HasSuffix(dir, helpers.FilePathSeparator) { dir += helpers.FilePathSeparator } - fileTp, isContent := classifyBundledFile(name) - - // This may be a member of a bundle. Start with branch bundles, the most specific. - if fileTp == bundleBranch || (fileTp == bundleNot && !isContent) { - for i, b := range m.branches { - if b == dir { - m.branches = append(m.branches[:i], m.branches[i+1:]...) - return dir, b, bundleBranch - } - } + if _, found := m.branchBundles[dir]; found { + delete(m.branchBundles, dir) + return dir, dir, bundleBranch } - // And finally the leaf bundles, which can contain anything. - for i, l := range m.leafs { - if strings.HasPrefix(dir, l) { - m.leafs = append(m.leafs[:i], m.leafs[i+1:]...) - return dir, l, bundleLeaf - } + if key, _, found := m.leafBundles.Root().LongestPrefix([]byte(dir)); found { + m.leafBundlesTxn.Delete(key) + dir = string(key) + return dir, dir, bundleLeaf } + fileTp, isContent := classifyBundledFile(name) if isContent && fileTp != bundleNot { // A new bundle. return dir, dir, fileTp } - // Not part of any bundle return dir, filename, bundleNot + } -func (m *contentChangeMap) addSymbolicLinkMapping(from, to string) { +func (m *contentChangeMap) addSymbolicLinkMapping(fim hugofs.FileMetaInfo) { + meta := fim.Meta() + if !meta.IsSymlink() { + return + } m.symContentMu.Lock() + + from, to := meta.Filename(), meta.OriginalFilename() + if fim.IsDir() { + if !strings.HasSuffix(from, helpers.FilePathSeparator) { + from += helpers.FilePathSeparator + } + } + mm, found := m.symContent[from] + if !found { mm = make(map[string]bool) m.symContent[from] = mm @@ -1044,5 +1065,6 @@ func (m *contentChangeMap) GetSymbolicLinkMappings(dir string) []string { } sort.Strings(dirs) + return dirs } diff --git a/hugolib/hugo_sites_build.go b/hugolib/hugo_sites_build.go index 7f725def2f9..d20932599c3 100644 --- a/hugolib/hugo_sites_build.go +++ b/hugolib/hugo_sites_build.go @@ -22,7 +22,7 @@ import ( "github.com/gohugoio/hugo/output" - "errors" + "github.com/pkg/errors" "github.com/fsnotify/fsnotify" "github.com/gohugoio/hugo/helpers" @@ -82,11 +82,11 @@ func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error { if len(events) > 0 { // Rebuild if err := h.initRebuild(conf); err != nil { - return err + return errors.Wrap(err, "initRebuild") } } else { if err := h.initSites(conf); err != nil { - return err + return errors.Wrap(err, "initSites") } } @@ -97,7 +97,7 @@ func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error { } trace.WithRegion(ctx, "process", f) if err != nil { - return err + return errors.Wrap(err, "process") } f = func() { diff --git a/hugolib/hugo_sites_build_errors_test.go b/hugolib/hugo_sites_build_errors_test.go index 6fe4901a1c8..993b4e613ab 100644 --- a/hugolib/hugo_sites_build_errors_test.go +++ b/hugolib/hugo_sites_build_errors_test.go @@ -40,7 +40,6 @@ func (t testSiteBuildErrorAsserter) assertErrorMessage(e1, e2 string) { } func TestSiteBuildErrors(t *testing.T) { - t.Parallel() const ( yamlcontent = "yamlcontent" @@ -226,7 +225,9 @@ func TestSiteBuildErrors(t *testing.T) { } for _, test := range tests { + test := test t.Run(test.name, func(t *testing.T) { + t.Parallel() assert := require.New(t) errorAsserter := testSiteBuildErrorAsserter{ assert: assert, diff --git a/hugolib/hugo_sites_build_test.go b/hugolib/hugo_sites_build_test.go index 236fd11a64c..6ff8ae4d2d5 100644 --- a/hugolib/hugo_sites_build_test.go +++ b/hugolib/hugo_sites_build_test.go @@ -2,10 +2,10 @@ package hugolib import ( "fmt" + "os" "strings" "testing" - "os" "path/filepath" "time" @@ -246,6 +246,8 @@ func doTestMultiSitesBuild(t *testing.T, configTemplate, configSuffix string) { require.Equal(t, "en", enSite.language.Lang) + //dumpPages(enSite.RegularPages()...) + assert.Equal(5, len(enSite.RegularPages())) assert.Equal(32, len(enSite.AllPages())) @@ -447,7 +449,7 @@ func TestMultiSitesRebuild(t *testing.T) { require.NotNil(t, homeEn) assert.Len(homeEn.Translations(), 3) - contentFs := b.H.BaseFs.Content.Fs + contentFs := b.H.Fs.Source for i, this := range []struct { preFunc func(t *testing.T) @@ -480,9 +482,9 @@ func TestMultiSitesRebuild(t *testing.T) { }, { func(t *testing.T) { - writeNewContentFile(t, contentFs, "new_en_1", "2016-07-31", "new1.en.md", -5) - writeNewContentFile(t, contentFs, "new_en_2", "1989-07-30", "new2.en.md", -10) - writeNewContentFile(t, contentFs, "new_fr_1", "2016-07-30", "new1.fr.md", 10) + writeNewContentFile(t, contentFs, "new_en_1", "2016-07-31", "content/new1.en.md", -5) + writeNewContentFile(t, contentFs, "new_en_2", "1989-07-30", "content/new2.en.md", -10) + writeNewContentFile(t, contentFs, "new_fr_1", "2016-07-30", "content/new1.fr.md", 10) }, []fsnotify.Event{ {Name: filepath.FromSlash("content/new1.en.md"), Op: fsnotify.Create}, @@ -503,7 +505,7 @@ func TestMultiSitesRebuild(t *testing.T) { }, { func(t *testing.T) { - p := "sect/doc1.en.md" + p := "content/sect/doc1.en.md" doc1 := readFileFromFs(t, contentFs, p) doc1 += "CHANGED" writeToFs(t, contentFs, p, doc1) @@ -519,7 +521,7 @@ func TestMultiSitesRebuild(t *testing.T) { // Rename a file { func(t *testing.T) { - if err := contentFs.Rename("new1.en.md", "new1renamed.en.md"); err != nil { + if err := contentFs.Rename("content/new1.en.md", "content/new1renamed.en.md"); err != nil { t.Fatalf("Rename failed: %s", err) } }, @@ -672,38 +674,6 @@ title = "Svenska" } -func TestChangeDefaultLanguage(t *testing.T) { - t.Parallel() - - assert := require.New(t) - - b := newMultiSiteTestBuilder(t, "", "", map[string]interface{}{ - "DefaultContentLanguage": "fr", - "DefaultContentLanguageInSubdir": false, - }) - b.CreateSites().Build(BuildCfg{}) - - b.AssertFileContent("public/sect/doc1/index.html", "Single", "Bonjour") - b.AssertFileContent("public/en/sect/doc2/index.html", "Single", "Hello") - - // Switch language - b.WithNewConfigData(map[string]interface{}{ - "DefaultContentLanguage": "en", - "DefaultContentLanguageInSubdir": false, - }) - - assert.NoError(b.LoadConfig()) - err := b.H.Build(BuildCfg{NewConfig: b.Cfg}) - - if err != nil { - t.Fatalf("Failed to rebuild sites: %s", err) - } - - // Default language is now en, so that should now be the "root" language - b.AssertFileContent("public/fr/sect/doc1/index.html", "Single", "Bonjour") - b.AssertFileContent("public/sect/doc2/index.html", "Single", "Hello") -} - // https://github.com/gohugoio/hugo/issues/4706 func TestContentStressTest(t *testing.T) { b := newTestSitesBuilder(t) @@ -1261,16 +1231,19 @@ var multiSiteJSONConfigTemplate = ` ` func writeSource(t testing.TB, fs *hugofs.Fs, filename, content string) { + t.Helper() writeToFs(t, fs.Source, filename, content) } func writeToFs(t testing.TB, fs afero.Fs, filename, content string) { + t.Helper() if err := afero.WriteFile(fs, filepath.FromSlash(filename), []byte(content), 0755); err != nil { t.Fatalf("Failed to write file: %s", err) } } func readDestination(t testing.TB, fs *hugofs.Fs, filename string) string { + t.Helper() return readFileFromFs(t, fs.Destination, filename) } @@ -1287,6 +1260,7 @@ func readSource(t *testing.T, fs *hugofs.Fs, filename string) string { } func readFileFromFs(t testing.TB, fs afero.Fs, filename string) string { + t.Helper() filename = filepath.Clean(filename) b, err := afero.ReadFile(fs, filename) if err != nil { @@ -1309,7 +1283,7 @@ func readFileFromFs(t testing.TB, fs afero.Fs, filename string) string { } helpers.PrintFs(fs, root, os.Stdout) - Fatalf(t, "Failed to read file: %s", err) + t.Fatalf("Failed to read file: %s", err) } return string(b) } diff --git a/hugolib/hugo_themes_test.go b/hugolib/hugo_themes_test.go deleted file mode 100644 index 05bfaa692bc..00000000000 --- a/hugolib/hugo_themes_test.go +++ /dev/null @@ -1,268 +0,0 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugolib - -import ( - "fmt" - "os" - "path/filepath" - "testing" - - "github.com/gohugoio/hugo/common/loggers" -) - -func TestThemesGraph(t *testing.T) { - t.Parallel() - - const ( - themeStandalone = ` -title = "Theme Standalone" -[params] -v1 = "v1s" -v2 = "v2s" -` - themeCyclic = ` -title = "Theme Cyclic" -theme = "theme3" -[params] -v1 = "v1c" -v2 = "v2c" -` - theme1 = ` -title = "Theme #1" -theme = "themeStandalone" -[params] -v2 = "v21" -` - - theme2 = ` -title = "Theme #2" -theme = "theme1" -[params] -v1 = "v12" -` - - theme3 = ` -title = "Theme #3" -theme = ["theme2", "themeStandalone", "themeCyclic"] -[params] -v1 = "v13" -v2 = "v24" -` - - theme4 = ` -title = "Theme #4" -theme = "theme3" -[params] -v1 = "v14" -v2 = "v24" -` - - site1 = ` - theme = "theme4" - - [params] - v1 = "site" -` - site2 = ` - theme = ["theme2", "themeStandalone"] -` - ) - - var ( - testConfigs = []struct { - siteConfig string - - // The name of theme somewhere in the middle to write custom key/files. - offset string - - check func(b *sitesBuilder) - }{ - {site1, "theme3", func(b *sitesBuilder) { - - // site1: theme4 theme3 theme2 theme1 themeStandalone themeCyclic - - // Check data - // theme3 should win the offset competition - b.AssertFileContent("public/index.html", "theme1o::[offset][v]theme3", "theme4o::[offset][v]theme3", "themeStandaloneo::[offset][v]theme3") - b.AssertFileContent("public/index.html", "nproject::[inner][other]project|[project][other]project|[theme][other]theme4|[theme1][other]theme1") - b.AssertFileContent("public/index.html", "ntheme::[inner][other]theme4|[theme][other]theme4|[theme1][other]theme1|[theme2][other]theme2|[theme3][other]theme3") - b.AssertFileContent("public/index.html", "theme1::[inner][other]project|[project][other]project|[theme][other]theme1|[theme1][other]theme1|") - b.AssertFileContent("public/index.html", "theme4::[inner][other]project|[project][other]project|[theme][other]theme4|[theme4][other]theme4|") - - // Check layouts - b.AssertFileContent("public/index.html", "partial ntheme: theme4", "partial theme2o: theme3") - - // Check i18n - b.AssertFileContent("public/index.html", "i18n: project theme4") - - // Check static files - // TODO(bep) static files not currently part of the build b.AssertFileContent("public/nproject.txt", "TODO") - - // Check site params - b.AssertFileContent("public/index.html", "v1::site", "v2::v24") - }}, - {site2, "", func(b *sitesBuilder) { - - // site2: theme2 theme1 themeStandalone - b.AssertFileContent("public/index.html", "nproject::[inner][other]project|[project][other]project|[theme][other]theme2|[theme1][other]theme1|[theme2][other]theme2|[themeStandalone][other]themeStandalone|") - b.AssertFileContent("public/index.html", "ntheme::[inner][other]theme2|[theme][other]theme2|[theme1][other]theme1|[theme2][other]theme2|[themeStandalone][other]themeStandalone|") - b.AssertFileContent("public/index.html", "i18n: project theme2") - b.AssertFileContent("public/index.html", "partial ntheme: theme2") - - // Params only set in themes - b.AssertFileContent("public/index.html", "v1::v12", "v2::v21") - - }}, - } - - themeConfigs = []struct { - name string - config string - }{ - {"themeStandalone", themeStandalone}, - {"themeCyclic", themeCyclic}, - {"theme1", theme1}, - {"theme2", theme2}, - {"theme3", theme3}, - {"theme4", theme4}, - } - ) - - for i, testConfig := range testConfigs { - t.Log(fmt.Sprintf("Test %d", i)) - b := newTestSitesBuilder(t).WithLogger(loggers.NewErrorLogger()) - b.WithConfigFile("toml", testConfig.siteConfig) - - for _, tc := range themeConfigs { - var variationsNameBase = []string{"nproject", "ntheme", tc.name} - - themeRoot := filepath.Join("themes", tc.name) - b.WithSourceFile(filepath.Join(themeRoot, "config.toml"), tc.config) - - b.WithSourceFile(filepath.Join("layouts", "partials", "m.html"), `{{- range $k, $v := . }}{{ $k }}::{{ template "printv" $v }} -{{ end }} -{{ define "printv" }} -{{- $tp := printf "%T" . -}} -{{- if (strings.HasSuffix $tp "map[string]interface {}") -}} -{{- range $k, $v := . }}[{{ $k }}]{{ template "printv" $v }}{{ end -}} -{{- else -}} -{{- . }}| -{{- end -}} -{{ end }} -`) - - for _, nameVariaton := range variationsNameBase { - roots := []string{"", themeRoot} - - for _, root := range roots { - name := tc.name - if root == "" { - name = "project" - } - - if nameVariaton == "ntheme" && name == "project" { - continue - } - - // static - b.WithSourceFile(filepath.Join(root, "static", nameVariaton+".txt"), name) - - // layouts - if i == 1 { - b.WithSourceFile(filepath.Join(root, "layouts", "partials", "theme2o.html"), "Not Set") - } - b.WithSourceFile(filepath.Join(root, "layouts", "partials", nameVariaton+".html"), name) - if root != "" && testConfig.offset == tc.name { - for _, tc2 := range themeConfigs { - b.WithSourceFile(filepath.Join(root, "layouts", "partials", tc2.name+"o.html"), name) - } - } - - // i18n + data - - var dataContent string - if root == "" { - dataContent = fmt.Sprintf(` -[%s] -other = %q - -[inner] -other = %q - -`, name, name, name) - } else { - dataContent = fmt.Sprintf(` -[%s] -other = %q - -[inner] -other = %q - -[theme] -other = %q - -`, name, name, name, name) - } - - b.WithSourceFile(filepath.Join(root, "data", nameVariaton+".toml"), dataContent) - b.WithSourceFile(filepath.Join(root, "i18n", "en.toml"), dataContent) - - // If an offset is set, duplicate a data key with a winner in the middle. - if root != "" && testConfig.offset == tc.name { - for _, tc2 := range themeConfigs { - dataContent := fmt.Sprintf(` -[offset] -v = %q -`, tc.name) - b.WithSourceFile(filepath.Join(root, "data", tc2.name+"o.toml"), dataContent) - } - } - } - - } - - } - - for _, themeConfig := range themeConfigs { - b.WithSourceFile(filepath.Join("themes", "config.toml"), themeConfig.config) - } - - b.WithContent(filepath.Join("content", "page.md"), `--- -title: "Page" ---- - -`) - - homeTpl := ` -data: {{ partial "m" .Site.Data }} -i18n: {{ i18n "inner" }} {{ i18n "theme" }} -partial ntheme: {{ partial "ntheme" . }} -partial theme2o: {{ partial "theme2o" . }} -params: {{ partial "m" .Site.Params }} - -` - - b.WithTemplates(filepath.Join("layouts", "home.html"), homeTpl) - - b.Build(BuildCfg{}) - - var _ = os.Stdout - - // printFs(b.H.Deps.BaseFs.LayoutsFs, "", os.Stdout) - testConfig.check(b) - - } - -} diff --git a/hugolib/language_content_dir_test.go b/hugolib/language_content_dir_test.go index ad1e1fb5345..2fea91fe1ab 100644 --- a/hugolib/language_content_dir_test.go +++ b/hugolib/language_content_dir_test.go @@ -211,7 +211,11 @@ Content. _ = os.Stdout - b.Build(BuildCfg{}) + err := b.BuildE(BuildCfg{}) + + //dumpPages(b.H.Sites[1].RegularPages()...) + + assert.NoError(err) assert.Equal(3, len(b.H.Sites)) @@ -222,7 +226,8 @@ Content. b.AssertFileContent("/my/project/public/en/mystatic/file1.yaml", "en") b.AssertFileContent("/my/project/public/nn/mystatic/file1.yaml", "nn") - //dumpPages(nnSite.RegularPages...) + //dumpPages(nnSite.RegularPages()...) + assert.Equal(12, len(nnSite.RegularPages())) assert.Equal(13, len(enSite.RegularPages())) @@ -281,9 +286,9 @@ Content. assert.Equal("/en/sect/mybundle/", bundleEn.RelPermalink()) assert.Equal("/sv/sect/mybundle/", bundleSv.RelPermalink()) - assert.Equal(4, len(bundleEn.Resources())) assert.Equal(4, len(bundleNn.Resources())) assert.Equal(4, len(bundleSv.Resources())) + assert.Equal(4, len(bundleEn.Resources())) b.AssertFileContent("/my/project/public/en/sect/mybundle/index.html", "image/png: /en/sect/mybundle/logo.png") b.AssertFileContent("/my/project/public/nn/sect/mybundle/index.html", "image/png: /nn/sect/mybundle/logo.png") diff --git a/hugolib/menu_test.go b/hugolib/menu_test.go index 4a2b176039d..3708173d931 100644 --- a/hugolib/menu_test.go +++ b/hugolib/menu_test.go @@ -18,8 +18,6 @@ import ( "fmt" - "github.com/spf13/afero" - "github.com/stretchr/testify/require" ) @@ -45,11 +43,10 @@ title = "Section Menu" sectionPagesMenu = "sect" ` - th, h := newTestSitesFromConfig( - t, - afero.NewMemMapFs(), - siteConfig, - "layouts/partials/menu.html", + b := newTestSitesBuilder(t).WithConfigFile("toml", siteConfig) + + b.WithTemplates( + "partials/menu.html", `{{- $p := .page -}} {{- $m := .menu -}} {{ range (index $p.Site.Menus $m) -}} @@ -58,28 +55,25 @@ sectionPagesMenu = "sect" {{- if $p.HasMenuCurrent $m . }}HasMenuCurrent{{ else }}-{{ end -}}| {{- end -}} `, - "layouts/_default/single.html", + "_default/single.html", `Single|{{ .Title }} Menu Sect: {{ partial "menu.html" (dict "page" . "menu" "sect") }} Menu Main: {{ partial "menu.html" (dict "page" . "menu" "main") }}`, - "layouts/_default/list.html", "List|{{ .Title }}|{{ .Content }}", + "_default/list.html", "List|{{ .Title }}|{{ .Content }}", ) - require.Len(t, h.Sites, 1) - - fs := th.Fs - - writeSource(t, fs, "content/sect1/p1.md", fmt.Sprintf(menuPageTemplate, "p1", 1, "main", "atitle1", 40)) - writeSource(t, fs, "content/sect1/p2.md", fmt.Sprintf(menuPageTemplate, "p2", 2, "main", "atitle2", 30)) - writeSource(t, fs, "content/sect2/p3.md", fmt.Sprintf(menuPageTemplate, "p3", 3, "main", "atitle3", 20)) - writeSource(t, fs, "content/sect2/p4.md", fmt.Sprintf(menuPageTemplate, "p4", 4, "main", "atitle4", 10)) - writeSource(t, fs, "content/sect3/p5.md", fmt.Sprintf(menuPageTemplate, "p5", 5, "main", "atitle5", 5)) - writeNewContentFile(t, fs.Source, "Section One", "2017-01-01", "content/sect1/_index.md", 100) - writeNewContentFile(t, fs.Source, "Section Five", "2017-01-01", "content/sect5/_index.md", 10) - - err := h.Build(BuildCfg{}) + b.WithContent( + "sect1/p1.md", fmt.Sprintf(menuPageTemplate, "p1", 1, "main", "atitle1", 40), + "sect1/p2.md", fmt.Sprintf(menuPageTemplate, "p2", 2, "main", "atitle2", 30), + "sect2/p3.md", fmt.Sprintf(menuPageTemplate, "p3", 3, "main", "atitle3", 20), + "sect2/p4.md", fmt.Sprintf(menuPageTemplate, "p4", 4, "main", "atitle4", 10), + "sect3/p5.md", fmt.Sprintf(menuPageTemplate, "p5", 5, "main", "atitle5", 5), + "sect1/_index.md", newTestPage("Section One", "2017-01-01", 100), + "sect5/_index.md", newTestPage("Section Five", "2017-01-01", 10), + ) - require.NoError(t, err) + b.Build(BuildCfg{}) + h := b.H s := h.Sites[0] @@ -90,7 +84,7 @@ Menu Main: {{ partial "menu.html" (dict "page" . "menu" "main") }}`, // There is only one menu in the page, but it is "member of" 2 require.Len(t, p1, 1) - th.assertFileContent("public/sect1/p1/index.html", "Single", + b.AssertFileContent("public/sect1/p1/index.html", "Single", "Menu Sect: "+ "/sect5/|Section Five|Section Five|10|-|-|"+ "/sect1/|Section One|Section One|100|-|HasMenuCurrent|"+ @@ -104,7 +98,7 @@ Menu Main: {{ partial "menu.html" (dict "page" . "menu" "main") }}`, "/sect1/p1/|p1|atitle1|40|IsMenuCurrent|-|", ) - th.assertFileContent("public/sect2/p3/index.html", "Single", + b.AssertFileContent("public/sect2/p3/index.html", "Single", "Menu Sect: "+ "/sect5/|Section Five|Section Five|10|-|-|"+ "/sect1/|Section One|Section One|100|-|-|"+ diff --git a/hugolib/multilingual.go b/hugolib/multilingual.go index 6f744f3a532..9b34c75e6a3 100644 --- a/hugolib/multilingual.go +++ b/hugolib/multilingual.go @@ -16,17 +16,11 @@ package hugolib import ( "sync" - "github.com/gohugoio/hugo/common/maps" - - "sort" - "errors" - "fmt" "github.com/gohugoio/hugo/langs" "github.com/gohugoio/hugo/config" - "github.com/spf13/cast" ) // Multilingual manages the all languages used in a multilingual site. @@ -88,53 +82,3 @@ func (s *Site) multilingualEnabled() bool { } return s.h.multilingual != nil && s.h.multilingual.enabled() } - -func toSortedLanguages(cfg config.Provider, l map[string]interface{}) (langs.Languages, error) { - languages := make(langs.Languages, len(l)) - i := 0 - - for lang, langConf := range l { - langsMap, err := cast.ToStringMapE(langConf) - - if err != nil { - return nil, fmt.Errorf("Language config is not a map: %T", langConf) - } - - language := langs.NewLanguage(lang, cfg) - - for loki, v := range langsMap { - switch loki { - case "title": - language.Title = cast.ToString(v) - case "languagename": - language.LanguageName = cast.ToString(v) - case "weight": - language.Weight = cast.ToInt(v) - case "contentdir": - language.ContentDir = cast.ToString(v) - case "disabled": - language.Disabled = cast.ToBool(v) - case "params": - m := cast.ToStringMap(v) - // Needed for case insensitive fetching of params values - maps.ToLower(m) - for k, vv := range m { - language.SetParam(k, vv) - } - } - - // Put all into the Params map - language.SetParam(loki, v) - - // Also set it in the configuration map (for baseURL etc.) - language.Set(loki, v) - } - - languages[i] = language - i++ - } - - sort.Sort(languages) - - return languages, nil -} diff --git a/hugolib/page.go b/hugolib/page.go index 537482fb269..676cba762ac 100644 --- a/hugolib/page.go +++ b/hugolib/page.go @@ -23,6 +23,8 @@ import ( "sort" "strings" + "github.com/gohugoio/hugo/hugofs/files" + "github.com/bep/gitmap" "github.com/gohugoio/hugo/helpers" @@ -290,7 +292,9 @@ func (p *pageState) getLayoutDescriptor() output.LayoutDescriptor { switch p.Kind() { case page.KindSection: - section = sections[0] + if len(sections) > 0 { + section = sections[0] + } case page.KindTaxonomyTerm: section = p.getTaxonomyNodeInfo().singular case page.KindTaxonomy: @@ -365,6 +369,7 @@ func (p *pageState) renderResources() (err error) { var toBeDeleted []int for i, r := range p.Resources() { + if _, ok := r.(page.Page); ok { // Pages gets rendered with the owning page but we count them here. p.s.PathSpec.ProcessingStats.Incr(&p.s.PathSpec.ProcessingStats.Pages) @@ -491,14 +496,6 @@ func (p *pageState) addSectionToParent() { p.parent.subSections = append(p.parent.subSections, p) } -func (p *pageState) contentMarkupType() string { - if p.m.markup != "" { - return p.m.markup - - } - return p.File().Ext() -} - func (p *pageState) mapContent(meta *pageMeta) error { s := p.shortcodeState @@ -843,6 +840,7 @@ func (ps pageStatePages) findPagePosByFilnamePrefix(prefix string) int { func (s *Site) sectionsFromFile(fi source.File) []string { dirname := fi.Dir() + dirname = strings.Trim(dirname, helpers.FilePathSeparator) if dirname == "" { return nil @@ -850,7 +848,7 @@ func (s *Site) sectionsFromFile(fi source.File) []string { parts := strings.Split(dirname, helpers.FilePathSeparator) if fii, ok := fi.(*fileInfo); ok { - if fii.bundleTp == bundleLeaf && len(parts) > 0 { + if len(parts) > 0 && fii.FileInfo().Meta().Classifier() == files.ContentClassLeaf { // my-section/mybundle/index.md => my-section return parts[:len(parts)-1] } diff --git a/hugolib/page__meta.go b/hugolib/page__meta.go index d14b9d724f5..551f4797744 100644 --- a/hugolib/page__meta.go +++ b/hugolib/page__meta.go @@ -21,6 +21,8 @@ import ( "strings" "time" + "github.com/gohugoio/hugo/hugofs/files" + "github.com/gohugoio/hugo/common/hugo" "github.com/gohugoio/hugo/related" @@ -591,15 +593,14 @@ func (p *pageMeta) applyDefaultValues() error { } if p.IsNode() { - p.bundleType = "branch" + p.bundleType = files.ContentClassBranch } else { source := p.File() if fi, ok := source.(*fileInfo); ok { - switch fi.bundleTp { - case bundleBranch: - p.bundleType = "branch" - case bundleLeaf: - p.bundleType = "leaf" + class := fi.FileInfo().Meta().Classifier() + switch class { + case files.ContentClassBranch, files.ContentClassLeaf: + p.bundleType = class } } } diff --git a/hugolib/page_permalink_test.go b/hugolib/page_permalink_test.go index 526f9578b5b..ffbb50cd525 100644 --- a/hugolib/page_permalink_test.go +++ b/hugolib/page_permalink_test.go @@ -63,8 +63,9 @@ func TestPermalink(t *testing.T) { } for i, test := range tests { + test := test t.Run(fmt.Sprintf("%s-%d", test.file, i), func(t *testing.T) { - + t.Parallel() cfg, fs := newTestCfg() cfg.Set("uglyURLs", test.uglyURLs) diff --git a/hugolib/page_test.go b/hugolib/page_test.go index 79a8a267f8c..e754a5e4f91 100644 --- a/hugolib/page_test.go +++ b/hugolib/page_test.go @@ -18,6 +18,8 @@ import ( "html/template" "os" + "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/common/loggers" "path/filepath" @@ -29,7 +31,6 @@ import ( "github.com/gohugoio/hugo/resources/page" "github.com/gohugoio/hugo/resources/resource" - "github.com/spf13/afero" "github.com/spf13/viper" @@ -301,6 +302,7 @@ func checkPageTitle(t *testing.T, page page.Page, title string) { } func checkPageContent(t *testing.T, page page.Page, expected string, msg ...interface{}) { + t.Helper() a := normalizeContent(expected) b := normalizeContent(content(page)) if a != b { @@ -387,11 +389,13 @@ func testAllMarkdownEnginesForPages(t *testing.T, continue } - cfg, fs := newTestCfg() + cfg, fs := newTestCfg(func(cfg config.Provider) error { + for k, v := range settings { + cfg.Set(k, v) + } + return nil - for k, v := range settings { - cfg.Set(k, v) - } + }) contentDir := "content" @@ -413,7 +417,10 @@ func testAllMarkdownEnginesForPages(t *testing.T, homePath := fmt.Sprintf("_index.%s", e.ext) writeSource(t, fs, filepath.Join(contentDir, homePath), homePage) - s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{SkipRender: true}) + b := newTestSitesBuilderFromDepsCfg(t, deps.DepsCfg{Fs: fs, Cfg: cfg}).WithNothingAdded() + b.Build(BuildCfg{SkipRender: true}) + + s := b.H.Sites[0] require.Len(t, s.RegularPages(), len(pageSources)) @@ -770,6 +777,9 @@ func TestPageWithLastmodFromGitInfo(t *testing.T) { fs := hugofs.NewFrom(hugofs.Os, cfg) fs.Destination = &afero.MemMapFs{} + wd, err := os.Getwd() + assrt.NoError(err) + cfg.Set("frontmatter", map[string]interface{}{ "lastmod": []string{":git", "lastmod"}, }) @@ -791,19 +801,14 @@ func TestPageWithLastmodFromGitInfo(t *testing.T) { cfg.Set("languages", langConfig) cfg.Set("enableGitInfo", true) - assrt.NoError(loadDefaultSettingsFor(cfg)) - assrt.NoError(loadLanguageSettings(cfg, nil)) - - wd, err := os.Getwd() - assrt.NoError(err) cfg.Set("workingDir", filepath.Join(wd, "testsite")) - h, err := NewHugoSites(deps.DepsCfg{Fs: fs, Cfg: cfg}) + b := newTestSitesBuilderFromDepsCfg(t, deps.DepsCfg{Fs: fs, Cfg: cfg}).WithNothingAdded() - assrt.NoError(err) - assrt.Len(h.Sites, 2) + b.Build(BuildCfg{SkipRender: true}) + h := b.H - require.NoError(t, h.Build(BuildCfg{SkipRender: true})) + assrt.Len(h.Sites, 2) enSite := h.Sites[0] assrt.Len(enSite.RegularPages(), 1) @@ -820,10 +825,10 @@ func TestPageWithLastmodFromGitInfo(t *testing.T) { } func TestPageWithFrontMatterConfig(t *testing.T) { - t.Parallel() - for _, dateHandler := range []string{":filename", ":fileModTime"} { + dateHandler := dateHandler t.Run(fmt.Sprintf("dateHandler=%q", dateHandler), func(t *testing.T) { + t.Parallel() assrt := require.New(t) cfg, fs := newTestCfg() @@ -852,8 +857,10 @@ Content c2fi, err := fs.Source.Stat(c2) assrt.NoError(err) - s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{SkipRender: true}) + b := newTestSitesBuilderFromDepsCfg(t, deps.DepsCfg{Fs: fs, Cfg: cfg}).WithNothingAdded() + b.Build(BuildCfg{SkipRender: true}) + s := b.H.Sites[0] assrt.Len(s.RegularPages(), 2) noSlug := s.RegularPages()[0] @@ -1051,10 +1058,8 @@ func TestPageWithEmoji(t *testing.T) { for _, enableEmoji := range []bool{true, false} { v := viper.New() v.Set("enableEmoji", enableEmoji) - b := newTestSitesBuilder(t) - b.WithViper(v) - b.WithSimpleConfigFile() + b := newTestSitesBuilder(t).WithViper(v) b.WithContent("page-emoji.md", `--- title: "Hugo Smile" @@ -1329,11 +1334,12 @@ func TestShouldBuild(t *testing.T) { // "dot" in path: #1885 and #2110 // disablePathToLower regression: #3374 func TestPathIssues(t *testing.T) { - t.Parallel() for _, disablePathToLower := range []bool{false, true} { for _, uglyURLs := range []bool{false, true} { + disablePathToLower := disablePathToLower + uglyURLs := uglyURLs t.Run(fmt.Sprintf("disablePathToLower=%t,uglyURLs=%t", disablePathToLower, uglyURLs), func(t *testing.T) { - + t.Parallel() cfg, fs := newTestCfg() th := testHelper{cfg, fs, t} diff --git a/hugolib/pagebundler.go b/hugolib/pagebundler.go deleted file mode 100644 index 5149968bcfb..00000000000 --- a/hugolib/pagebundler.go +++ /dev/null @@ -1,206 +0,0 @@ -// Copyright 2019 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugolib - -import ( - "context" - "fmt" - "math" - "path/filepath" - - "github.com/gohugoio/hugo/config" - - _errors "github.com/pkg/errors" - - "golang.org/x/sync/errgroup" -) - -type siteContentProcessor struct { - site *Site - - handleContent contentHandler - - ctx context.Context - - // The input file bundles. - fileBundlesChan chan *bundleDir - - // The input file singles. - fileSinglesChan chan *fileInfo - - // These assets should be just copied to destination. - fileAssetsChan chan pathLangFile - - numWorkers int - - // The output Pages - pagesChan chan *pageState - - // Used for partial rebuilds (aka. live reload) - // Will signal replacement of pages in the site collection. - partialBuild bool -} - -func (s *siteContentProcessor) processBundle(b *bundleDir) { - select { - case s.fileBundlesChan <- b: - case <-s.ctx.Done(): - } -} - -func (s *siteContentProcessor) processSingle(fi *fileInfo) { - select { - case s.fileSinglesChan <- fi: - case <-s.ctx.Done(): - } -} - -func (s *siteContentProcessor) processAsset(asset pathLangFile) { - select { - case s.fileAssetsChan <- asset: - case <-s.ctx.Done(): - } -} - -func newSiteContentProcessor(ctx context.Context, partialBuild bool, s *Site) *siteContentProcessor { - numWorkers := config.GetNumWorkerMultiplier() * 3 - - numWorkers = int(math.Ceil(float64(numWorkers) / float64(len(s.h.Sites)))) - - return &siteContentProcessor{ - ctx: ctx, - partialBuild: partialBuild, - site: s, - handleContent: newHandlerChain(s), - fileBundlesChan: make(chan *bundleDir, numWorkers), - fileSinglesChan: make(chan *fileInfo, numWorkers), - fileAssetsChan: make(chan pathLangFile, numWorkers), - numWorkers: numWorkers, - pagesChan: make(chan *pageState, numWorkers), - } -} - -func (s *siteContentProcessor) closeInput() { - close(s.fileSinglesChan) - close(s.fileBundlesChan) - close(s.fileAssetsChan) -} - -func (s *siteContentProcessor) process(ctx context.Context) error { - g1, ctx := errgroup.WithContext(ctx) - g2, ctx := errgroup.WithContext(ctx) - - // There can be only one of these per site. - g1.Go(func() error { - for p := range s.pagesChan { - if p.s != s.site { - panic(fmt.Sprintf("invalid page site: %v vs %v", p.s, s)) - } - - p.forceRender = s.partialBuild - - if p.forceRender { - s.site.replacePage(p) - } else { - s.site.addPage(p) - } - } - return nil - }) - - for i := 0; i < s.numWorkers; i++ { - g2.Go(func() error { - for { - select { - case f, ok := <-s.fileSinglesChan: - if !ok { - return nil - } - - err := s.readAndConvertContentFile(f) - if err != nil { - return err - } - case <-ctx.Done(): - return ctx.Err() - } - } - }) - - g2.Go(func() error { - for { - select { - case file, ok := <-s.fileAssetsChan: - if !ok { - return nil - } - f, err := s.site.BaseFs.Content.Fs.Open(file.Filename()) - if err != nil { - return _errors.Wrap(err, "failed to open assets file") - } - filename := filepath.Join(s.site.GetTargetLanguageBasePath(), file.Path()) - err = s.site.publish(&s.site.PathSpec.ProcessingStats.Files, filename, f) - f.Close() - if err != nil { - return err - } - case <-ctx.Done(): - return ctx.Err() - } - } - }) - - g2.Go(func() error { - for { - select { - case bundle, ok := <-s.fileBundlesChan: - if !ok { - return nil - } - err := s.readAndConvertContentBundle(bundle) - if err != nil { - return err - } - case <-ctx.Done(): - return ctx.Err() - } - } - }) - } - - err := g2.Wait() - - close(s.pagesChan) - - if err != nil { - return err - } - - if err := g1.Wait(); err != nil { - return err - } - - return nil - -} - -func (s *siteContentProcessor) readAndConvertContentFile(file *fileInfo) error { - ctx := &handlerContext{source: file, pages: s.pagesChan} - return s.handleContent(ctx).err -} - -func (s *siteContentProcessor) readAndConvertContentBundle(bundle *bundleDir) error { - ctx := &handlerContext{bundle: bundle, pages: s.pagesChan} - return s.handleContent(ctx).err -} diff --git a/hugolib/pagebundler_capture.go b/hugolib/pagebundler_capture.go deleted file mode 100644 index 7c01a751dbf..00000000000 --- a/hugolib/pagebundler_capture.go +++ /dev/null @@ -1,773 +0,0 @@ -// Copyright 2017-present The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugolib - -import ( - "errors" - "fmt" - "os" - "path" - "path/filepath" - - "github.com/gohugoio/hugo/config" - - "github.com/gohugoio/hugo/common/loggers" - _errors "github.com/pkg/errors" - - "sort" - "strings" - "sync" - - "github.com/spf13/afero" - - "github.com/gohugoio/hugo/hugofs" - - "github.com/gohugoio/hugo/helpers" - - "golang.org/x/sync/errgroup" - - "github.com/gohugoio/hugo/source" -) - -var errSkipCyclicDir = errors.New("skip potential cyclic dir") - -type capturer struct { - // To prevent symbolic link cycles: Visit same folder only once. - seen map[string]bool - seenMu sync.Mutex - - handler captureResultHandler - - sourceSpec *source.SourceSpec - fs afero.Fs - logger *loggers.Logger - - // Filenames limits the content to process to a list of filenames/directories. - // This is used for partial building in server mode. - filenames []string - - // Used to determine how to handle content changes in server mode. - contentChanges *contentChangeMap - - // Semaphore used to throttle the concurrent sub directory handling. - sem chan bool -} - -func newCapturer( - logger *loggers.Logger, - sourceSpec *source.SourceSpec, - handler captureResultHandler, - contentChanges *contentChangeMap, - filenames ...string) *capturer { - - numWorkers := config.GetNumWorkerMultiplier() - - // TODO(bep) the "index" vs "_index" check/strings should be moved in one place. - isBundleHeader := func(filename string) bool { - base := filepath.Base(filename) - name := helpers.Filename(base) - return IsContentFile(base) && (name == "index" || name == "_index") - } - - // Make sure that any bundle header files are processed before the others. This makes - // sure that any bundle head is processed before its resources. - sort.Slice(filenames, func(i, j int) bool { - a, b := filenames[i], filenames[j] - ac, bc := isBundleHeader(a), isBundleHeader(b) - - if ac { - return true - } - - if bc { - return false - } - - return a < b - }) - - c := &capturer{ - sem: make(chan bool, numWorkers), - handler: handler, - sourceSpec: sourceSpec, - fs: sourceSpec.SourceFs, - logger: logger, - contentChanges: contentChanges, - seen: make(map[string]bool), - filenames: filenames} - - return c -} - -// Captured files and bundles ready to be processed will be passed on to -// these channels. -type captureResultHandler interface { - handleSingles(fis ...*fileInfo) - handleCopyFile(fi pathLangFile) - captureBundlesHandler -} - -type captureBundlesHandler interface { - handleBundles(b *bundleDirs) -} - -type captureResultHandlerChain struct { - handlers []captureBundlesHandler -} - -func (c *captureResultHandlerChain) handleSingles(fis ...*fileInfo) { - for _, h := range c.handlers { - if hh, ok := h.(captureResultHandler); ok { - hh.handleSingles(fis...) - } - } -} -func (c *captureResultHandlerChain) handleBundles(b *bundleDirs) { - for _, h := range c.handlers { - h.handleBundles(b) - } -} - -func (c *captureResultHandlerChain) handleCopyFile(file pathLangFile) { - for _, h := range c.handlers { - if hh, ok := h.(captureResultHandler); ok { - hh.handleCopyFile(file) - } - } -} - -func (c *capturer) capturePartial(filenames ...string) error { - handled := make(map[string]bool) - - for _, filename := range filenames { - dir, resolvedFilename, tp := c.contentChanges.resolveAndRemove(filename) - if handled[resolvedFilename] { - continue - } - - handled[resolvedFilename] = true - - switch tp { - case bundleLeaf: - if err := c.handleDir(resolvedFilename); err != nil { - // Directory may have been deleted. - if !os.IsNotExist(err) { - return err - } - } - case bundleBranch: - if err := c.handleBranchDir(resolvedFilename); err != nil { - // Directory may have been deleted. - if !os.IsNotExist(err) { - return err - } - } - default: - fi, err := c.resolveRealPath(resolvedFilename) - if os.IsNotExist(err) { - // File has been deleted. - continue - } - - // Just in case the owning dir is a new symlink -- this will - // create the proper mapping for it. - c.resolveRealPath(dir) - - f, active := c.newFileInfo(fi, tp) - if active { - c.copyOrHandleSingle(f) - } - } - } - - return nil -} - -func (c *capturer) capture() error { - if len(c.filenames) > 0 { - return c.capturePartial(c.filenames...) - } - - err := c.handleDir(helpers.FilePathSeparator) - if err != nil { - return err - } - - return nil -} - -func (c *capturer) handleNestedDir(dirname string) error { - select { - case c.sem <- true: - var g errgroup.Group - - g.Go(func() error { - defer func() { - <-c.sem - }() - return c.handleDir(dirname) - }) - return g.Wait() - default: - // For deeply nested file trees, waiting for a semaphore wil deadlock. - return c.handleDir(dirname) - } -} - -// This handles a bundle branch and its resources only. This is used -// in server mode on changes. If this dir does not (anymore) represent a bundle -// branch, the handling is upgraded to the full handleDir method. -func (c *capturer) handleBranchDir(dirname string) error { - files, err := c.readDir(dirname) - if err != nil { - - return err - } - - var ( - dirType bundleDirType - ) - - for _, fi := range files { - if !fi.IsDir() { - tp, _ := classifyBundledFile(fi.RealName()) - if dirType == bundleNot { - dirType = tp - } - - if dirType == bundleLeaf { - return c.handleDir(dirname) - } - } - } - - if dirType != bundleBranch { - return c.handleDir(dirname) - } - - dirs := newBundleDirs(bundleBranch, c) - - var secondPass []*fileInfo - - // Handle potential bundle headers first. - for _, fi := range files { - if fi.IsDir() { - continue - } - - tp, isContent := classifyBundledFile(fi.RealName()) - - f, active := c.newFileInfo(fi, tp) - - if !active { - continue - } - - if !f.isOwner() { - if !isContent { - // This is a partial update -- we only care about the files that - // is in this bundle. - secondPass = append(secondPass, f) - } - continue - } - dirs.addBundleHeader(f) - } - - for _, f := range secondPass { - dirs.addBundleFiles(f) - } - - c.handler.handleBundles(dirs) - - return nil - -} - -func (c *capturer) handleDir(dirname string) error { - - files, err := c.readDir(dirname) - if err != nil { - return err - } - - type dirState int - - const ( - dirStateDefault dirState = iota - - dirStateAssetsOnly - dirStateSinglesOnly - ) - - var ( - fileBundleTypes = make([]bundleDirType, len(files)) - - // Start with the assumption that this dir contains only non-content assets (images etc.) - // If that is still true after we had a first look at the list of files, we - // can just copy the files to destination. We will still have to look at the - // sub-folders for potential bundles. - state = dirStateAssetsOnly - - // Start with the assumption that this dir is not a bundle. - // A directory is a bundle if it contains a index content file, - // e.g. index.md (a leaf bundle) or a _index.md (a branch bundle). - bundleType = bundleNot - ) - - /* First check for any content files. - - If there are none, then this is a assets folder only (images etc.) - and we can just plainly copy them to - destination. - - If this is a section with no image etc. or similar, we can just handle it - as it was a single content file. - */ - var hasNonContent, isBranch bool - - for i, fi := range files { - if !fi.IsDir() { - tp, isContent := classifyBundledFile(fi.RealName()) - - fileBundleTypes[i] = tp - if !isBranch { - isBranch = tp == bundleBranch - } - - if isContent { - // This is not a assets-only folder. - state = dirStateDefault - } else { - hasNonContent = true - } - } - } - - if isBranch && !hasNonContent { - // This is a section or similar with no need for any bundle handling. - state = dirStateSinglesOnly - } - - if state > dirStateDefault { - return c.handleNonBundle(dirname, files, state == dirStateSinglesOnly) - } - - var fileInfos = make([]*fileInfo, 0, len(files)) - - for i, fi := range files { - - currentType := bundleNot - - if !fi.IsDir() { - currentType = fileBundleTypes[i] - if bundleType == bundleNot && currentType != bundleNot { - bundleType = currentType - } - } - - if bundleType == bundleNot && currentType != bundleNot { - bundleType = currentType - } - - f, active := c.newFileInfo(fi, currentType) - - if !active { - continue - } - - fileInfos = append(fileInfos, f) - } - - var todo []*fileInfo - - if bundleType != bundleLeaf { - for _, fi := range fileInfos { - if fi.FileInfo().IsDir() { - // Handle potential nested bundles. - if err := c.handleNestedDir(fi.Path()); err != nil { - return err - } - } else if bundleType == bundleNot || (!fi.isOwner() && fi.isContentFile()) { - // Not in a bundle. - c.copyOrHandleSingle(fi) - } else { - // This is a section folder or similar with non-content files in it. - todo = append(todo, fi) - } - } - } else { - todo = fileInfos - } - - if len(todo) == 0 { - return nil - } - - dirs, err := c.createBundleDirs(todo, bundleType) - if err != nil { - return err - } - - // Send the bundle to the next step in the processor chain. - c.handler.handleBundles(dirs) - - return nil -} - -func (c *capturer) handleNonBundle( - dirname string, - fileInfos pathLangFileFis, - singlesOnly bool) error { - - for _, fi := range fileInfos { - if fi.IsDir() { - if err := c.handleNestedDir(fi.Filename()); err != nil { - return err - } - } else { - if singlesOnly { - f, active := c.newFileInfo(fi, bundleNot) - if !active { - continue - } - c.handler.handleSingles(f) - } else { - c.handler.handleCopyFile(fi) - } - } - } - - return nil -} - -func (c *capturer) copyOrHandleSingle(fi *fileInfo) { - if fi.isContentFile() { - c.handler.handleSingles(fi) - } else { - // These do not currently need any further processing. - c.handler.handleCopyFile(fi) - } -} - -func (c *capturer) createBundleDirs(fileInfos []*fileInfo, bundleType bundleDirType) (*bundleDirs, error) { - dirs := newBundleDirs(bundleType, c) - - for _, fi := range fileInfos { - if fi.FileInfo().IsDir() { - var collector func(fis ...*fileInfo) - - if bundleType == bundleBranch { - // All files in the current directory are part of this bundle. - // Trying to include sub folders in these bundles are filled with ambiguity. - collector = func(fis ...*fileInfo) { - for _, fi := range fis { - c.copyOrHandleSingle(fi) - } - } - } else { - // All nested files and directories are part of this bundle. - collector = func(fis ...*fileInfo) { - fileInfos = append(fileInfos, fis...) - } - } - err := c.collectFiles(fi.Path(), collector) - if err != nil { - return nil, err - } - - } else if fi.isOwner() { - // There can be more than one language, so: - // 1. Content files must be attached to its language's bundle. - // 2. Other files must be attached to all languages. - // 3. Every content file needs a bundle header. - dirs.addBundleHeader(fi) - } - } - - for _, fi := range fileInfos { - if fi.FileInfo().IsDir() || fi.isOwner() { - continue - } - - if fi.isContentFile() { - if bundleType != bundleBranch { - dirs.addBundleContentFile(fi) - } - } else { - dirs.addBundleFiles(fi) - } - } - - return dirs, nil -} - -func (c *capturer) collectFiles(dirname string, handleFiles func(fis ...*fileInfo)) error { - - filesInDir, err := c.readDir(dirname) - if err != nil { - return err - } - - for _, fi := range filesInDir { - if fi.IsDir() { - err := c.collectFiles(fi.Filename(), handleFiles) - if err != nil { - return err - } - } else { - f, active := c.newFileInfo(fi, bundleNot) - if active { - handleFiles(f) - } - } - } - - return nil -} - -func (c *capturer) readDir(dirname string) (pathLangFileFis, error) { - if c.sourceSpec.IgnoreFile(dirname) { - return nil, nil - } - - dir, err := c.fs.Open(dirname) - if err != nil { - return nil, err - } - defer dir.Close() - fis, err := dir.Readdir(-1) - if err != nil { - return nil, err - } - - pfis := make(pathLangFileFis, 0, len(fis)) - - for _, fi := range fis { - fip := fi.(pathLangFileFi) - - if !c.sourceSpec.IgnoreFile(fip.Filename()) { - - err := c.resolveRealPathIn(fip) - - if err != nil { - // It may have been deleted in the meantime. - if err == errSkipCyclicDir || os.IsNotExist(err) { - continue - } - return nil, err - } - - pfis = append(pfis, fip) - } - } - - return pfis, nil -} - -func (c *capturer) newFileInfo(fi pathLangFileFi, tp bundleDirType) (*fileInfo, bool) { - f := newFileInfo(c.sourceSpec, "", "", fi, tp) - return f, !f.disabled -} - -type pathLangFile interface { - hugofs.LanguageAnnouncer - hugofs.FilePather -} - -type pathLangFileFi interface { - os.FileInfo - pathLangFile -} - -type pathLangFileFis []pathLangFileFi - -type bundleDirs struct { - tp bundleDirType - // Maps languages to bundles. - bundles map[string]*bundleDir - - // Keeps track of language overrides for non-content files, e.g. logo.en.png. - langOverrides map[string]bool - - c *capturer -} - -func newBundleDirs(tp bundleDirType, c *capturer) *bundleDirs { - return &bundleDirs{tp: tp, bundles: make(map[string]*bundleDir), langOverrides: make(map[string]bool), c: c} -} - -type bundleDir struct { - tp bundleDirType - fi *fileInfo - - resources map[string]*fileInfo -} - -func (b bundleDir) clone() *bundleDir { - b.resources = make(map[string]*fileInfo) - fic := *b.fi - b.fi = &fic - return &b -} - -func newBundleDir(fi *fileInfo, bundleType bundleDirType) *bundleDir { - return &bundleDir{fi: fi, tp: bundleType, resources: make(map[string]*fileInfo)} -} - -func (b *bundleDirs) addBundleContentFile(fi *fileInfo) { - dir, found := b.bundles[fi.Lang()] - if !found { - // Every bundled content file needs a bundle header. - // If one does not exist in its language, we pick the default - // language version, or a random one if that doesn't exist, either. - tl := b.c.sourceSpec.DefaultContentLanguage - ldir, found := b.bundles[tl] - if !found { - // Just pick one. - for _, v := range b.bundles { - ldir = v - break - } - } - - if ldir == nil { - panic(fmt.Sprintf("bundle not found for file %q", fi.Filename())) - } - - dir = ldir.clone() - dir.fi.overriddenLang = fi.Lang() - b.bundles[fi.Lang()] = dir - } - - dir.resources[fi.Path()] = fi -} - -func (b *bundleDirs) addBundleFiles(fi *fileInfo) { - dir := filepath.ToSlash(fi.Dir()) - p := dir + fi.TranslationBaseName() + "." + fi.Ext() - for lang, bdir := range b.bundles { - key := path.Join(lang, p) - - // Given mypage.de.md (German translation) and mypage.md we pick the most - // specific for that language. - if fi.Lang() == lang || !b.langOverrides[key] { - bdir.resources[key] = fi - } - b.langOverrides[key] = true - } -} - -func (b *bundleDirs) addBundleHeader(fi *fileInfo) { - b.bundles[fi.Lang()] = newBundleDir(fi, b.tp) -} - -func (c *capturer) isSeen(dirname string) bool { - c.seenMu.Lock() - defer c.seenMu.Unlock() - seen := c.seen[dirname] - c.seen[dirname] = true - if seen { - c.logger.INFO.Printf("Content dir %q already processed; skipped to avoid infinite recursion.", dirname) - return true - - } - return false -} - -func (c *capturer) resolveRealPath(path string) (pathLangFileFi, error) { - fileInfo, err := c.lstatIfPossible(path) - if err != nil { - return nil, err - } - return fileInfo, c.resolveRealPathIn(fileInfo) -} - -func (c *capturer) resolveRealPathIn(fileInfo pathLangFileFi) error { - - basePath := fileInfo.BaseDir() - path := fileInfo.Filename() - - realPath := path - - if fileInfo.Mode()&os.ModeSymlink == os.ModeSymlink { - link, err := filepath.EvalSymlinks(path) - if err != nil { - return _errors.Wrapf(err, "Cannot read symbolic link %q, error was:", path) - } - - // This is a file on the outside of any base fs, so we have to use the os package. - sfi, err := os.Stat(link) - if err != nil { - return _errors.Wrapf(err, "Cannot stat %q, error was:", link) - } - - // TODO(bep) improve all of this. - if a, ok := fileInfo.(*hugofs.LanguageFileInfo); ok { - a.FileInfo = sfi - } - - realPath = link - - if realPath != path && sfi.IsDir() && c.isSeen(realPath) { - // Avoid cyclic symlinks. - // Note that this may prevent some uses that isn't cyclic and also - // potential useful, but this implementation is both robust and simple: - // We stop at the first directory that we have seen before, e.g. - // /content/blog will only be processed once. - return errSkipCyclicDir - } - - if c.contentChanges != nil { - // Keep track of symbolic links in watch mode. - var from, to string - if sfi.IsDir() { - from = realPath - to = path - - if !strings.HasSuffix(to, helpers.FilePathSeparator) { - to = to + helpers.FilePathSeparator - } - if !strings.HasSuffix(from, helpers.FilePathSeparator) { - from = from + helpers.FilePathSeparator - } - - if !strings.HasSuffix(basePath, helpers.FilePathSeparator) { - basePath = basePath + helpers.FilePathSeparator - } - - if strings.HasPrefix(from, basePath) { - // With symbolic links inside /content we need to keep - // a reference to both. This may be confusing with --navigateToChanged - // but the user has chosen this him or herself. - c.contentChanges.addSymbolicLinkMapping(from, from) - } - - } else { - from = realPath - to = path - } - - c.contentChanges.addSymbolicLinkMapping(from, to) - } - } - - return nil -} - -func (c *capturer) lstatIfPossible(path string) (pathLangFileFi, error) { - fi, err := helpers.LstatIfPossible(c.fs, path) - if err != nil { - return nil, err - } - return fi.(pathLangFileFi), nil -} diff --git a/hugolib/pagebundler_capture_test.go b/hugolib/pagebundler_capture_test.go deleted file mode 100644 index b6d9822af86..00000000000 --- a/hugolib/pagebundler_capture_test.go +++ /dev/null @@ -1,272 +0,0 @@ -// Copyright 2017-present The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugolib - -import ( - "fmt" - "os" - "path" - "path/filepath" - "sort" - - "github.com/gohugoio/hugo/common/loggers" - - "runtime" - "strings" - "sync" - "testing" - - "github.com/gohugoio/hugo/helpers" - "github.com/gohugoio/hugo/source" - "github.com/stretchr/testify/require" -) - -type storeFilenames struct { - sync.Mutex - filenames []string - copyNames []string - dirKeys []string -} - -func (s *storeFilenames) handleSingles(fis ...*fileInfo) { - s.Lock() - defer s.Unlock() - for _, fi := range fis { - s.filenames = append(s.filenames, filepath.ToSlash(fi.Filename())) - } -} - -func (s *storeFilenames) handleBundles(d *bundleDirs) { - s.Lock() - defer s.Unlock() - var keys []string - for _, b := range d.bundles { - res := make([]string, len(b.resources)) - i := 0 - for _, r := range b.resources { - res[i] = path.Join(r.Lang(), filepath.ToSlash(r.Filename())) - i++ - } - sort.Strings(res) - keys = append(keys, path.Join("__bundle", b.fi.Lang(), filepath.ToSlash(b.fi.Filename()), "resources", strings.Join(res, "|"))) - } - s.dirKeys = append(s.dirKeys, keys...) -} - -func (s *storeFilenames) handleCopyFile(file pathLangFile) { - s.Lock() - defer s.Unlock() - s.copyNames = append(s.copyNames, filepath.ToSlash(file.Filename())) -} - -func (s *storeFilenames) sortedStr() string { - s.Lock() - defer s.Unlock() - sort.Strings(s.filenames) - sort.Strings(s.dirKeys) - sort.Strings(s.copyNames) - return "\nF:\n" + strings.Join(s.filenames, "\n") + "\nD:\n" + strings.Join(s.dirKeys, "\n") + - "\nC:\n" + strings.Join(s.copyNames, "\n") + "\n" -} - -func TestPageBundlerCaptureSymlinks(t *testing.T) { - if runtime.GOOS == "windows" && os.Getenv("CI") == "" { - t.Skip("Skip TestPageBundlerCaptureSymlinks as os.Symlink needs administrator rights on Windows") - } - - assert := require.New(t) - ps, clean, workDir := newTestBundleSymbolicSources(t) - sourceSpec := source.NewSourceSpec(ps, ps.BaseFs.Content.Fs) - defer clean() - - fileStore := &storeFilenames{} - logger := loggers.NewErrorLogger() - c := newCapturer(logger, sourceSpec, fileStore, nil) - - assert.NoError(c.capture()) - - expected := ` -F: -/base/a/page_s.md -/base/a/regular.md -/base/symbolic1/s1.md -/base/symbolic1/s2.md -/base/symbolic3/circus/a/page_s.md -/base/symbolic3/circus/a/regular.md -D: -__bundle/en/base/symbolic2/a1/index.md/resources/en/base/symbolic2/a1/logo.png|en/base/symbolic2/a1/page.md -C: -/base/symbolic3/s1.png -/base/symbolic3/s2.png -` - - got := strings.Replace(fileStore.sortedStr(), filepath.ToSlash(workDir), "", -1) - got = strings.Replace(got, "//", "/", -1) - - if expected != got { - diff := helpers.DiffStringSlices(strings.Fields(expected), strings.Fields(got)) - t.Log(got) - t.Fatalf("Failed:\n%s", diff) - } -} - -func TestPageBundlerCaptureBasic(t *testing.T) { - t.Parallel() - - assert := require.New(t) - fs, cfg := newTestBundleSources(t) - assert.NoError(loadDefaultSettingsFor(cfg)) - assert.NoError(loadLanguageSettings(cfg, nil)) - ps, err := helpers.NewPathSpec(fs, cfg) - assert.NoError(err) - - sourceSpec := source.NewSourceSpec(ps, ps.BaseFs.Content.Fs) - - fileStore := &storeFilenames{} - - c := newCapturer(loggers.NewErrorLogger(), sourceSpec, fileStore, nil) - - assert.NoError(c.capture()) - - expected := ` -F: -/work/base/_1.md -/work/base/a/1.md -/work/base/a/2.md -/work/base/assets/pages/mypage.md -D: -__bundle/en/work/base/_index.md/resources/en/work/base/_1.png -__bundle/en/work/base/a/b/index.md/resources/en/work/base/a/b/ab1.md -__bundle/en/work/base/b/my-bundle/index.md/resources/en/work/base/b/my-bundle/1.md|en/work/base/b/my-bundle/2.md|en/work/base/b/my-bundle/c/logo.png|en/work/base/b/my-bundle/custom-mime.bep|en/work/base/b/my-bundle/sunset1.jpg|en/work/base/b/my-bundle/sunset2.jpg -__bundle/en/work/base/c/bundle/index.md/resources/en/work/base/c/bundle/logo-은행.png -__bundle/en/work/base/root/index.md/resources/en/work/base/root/1.md|en/work/base/root/c/logo.png -C: -/work/base/assets/pic1.png -/work/base/assets/pic2.png -/work/base/images/hugo-logo.png -` - - got := fileStore.sortedStr() - - if expected != got { - diff := helpers.DiffStringSlices(strings.Fields(expected), strings.Fields(got)) - t.Log(got) - t.Fatalf("Failed:\n%s", diff) - } -} - -func TestPageBundlerCaptureMultilingual(t *testing.T) { - t.Parallel() - - assert := require.New(t) - fs, cfg := newTestBundleSourcesMultilingual(t) - assert.NoError(loadDefaultSettingsFor(cfg)) - assert.NoError(loadLanguageSettings(cfg, nil)) - - ps, err := helpers.NewPathSpec(fs, cfg) - assert.NoError(err) - - sourceSpec := source.NewSourceSpec(ps, ps.BaseFs.Content.Fs) - fileStore := &storeFilenames{} - c := newCapturer(loggers.NewErrorLogger(), sourceSpec, fileStore, nil) - - assert.NoError(c.capture()) - - expected := ` -F: -/work/base/1s/mypage.md -/work/base/1s/mypage.nn.md -/work/base/bb/_1.md -/work/base/bb/_1.nn.md -/work/base/bb/en.md -/work/base/bc/page.md -/work/base/bc/page.nn.md -/work/base/be/_index.md -/work/base/be/page.md -/work/base/be/page.nn.md -D: -__bundle/en/work/base/bb/_index.md/resources/en/work/base/bb/a.png|en/work/base/bb/b.png|nn/work/base/bb/c.nn.png -__bundle/en/work/base/bc/_index.md/resources/en/work/base/bc/logo-bc.png -__bundle/en/work/base/bd/index.md/resources/en/work/base/bd/page.md -__bundle/en/work/base/bf/my-bf-bundle/index.md/resources/en/work/base/bf/my-bf-bundle/page.md -__bundle/en/work/base/lb/index.md/resources/en/work/base/lb/1.md|en/work/base/lb/2.md|en/work/base/lb/c/d/deep.png|en/work/base/lb/c/logo.png|en/work/base/lb/c/one.png|en/work/base/lb/c/page.md -__bundle/nn/work/base/bb/_index.nn.md/resources/en/work/base/bb/a.png|nn/work/base/bb/b.nn.png|nn/work/base/bb/c.nn.png -__bundle/nn/work/base/bd/index.md/resources/nn/work/base/bd/page.nn.md -__bundle/nn/work/base/bf/my-bf-bundle/index.nn.md/resources -__bundle/nn/work/base/lb/index.nn.md/resources/en/work/base/lb/c/d/deep.png|en/work/base/lb/c/one.png|nn/work/base/lb/2.nn.md|nn/work/base/lb/c/logo.nn.png -C: -/work/base/1s/mylogo.png -/work/base/bb/b/d.nn.png -` - - got := fileStore.sortedStr() - - if expected != got { - diff := helpers.DiffStringSlices(strings.Fields(expected), strings.Fields(got)) - t.Log(got) - t.Fatalf("Failed:\n%s", strings.Join(diff, "\n")) - } - -} - -type noOpFileStore int - -func (noOpFileStore) handleSingles(fis ...*fileInfo) {} -func (noOpFileStore) handleBundles(b *bundleDirs) {} -func (noOpFileStore) handleCopyFile(file pathLangFile) {} - -func BenchmarkPageBundlerCapture(b *testing.B) { - capturers := make([]*capturer, b.N) - - for i := 0; i < b.N; i++ { - cfg, fs := newTestCfg() - ps, _ := helpers.NewPathSpec(fs, cfg) - sourceSpec := source.NewSourceSpec(ps, fs.Source) - - base := fmt.Sprintf("base%d", i) - for j := 1; j <= 5; j++ { - js := fmt.Sprintf("j%d", j) - writeSource(b, fs, filepath.Join(base, js, "index.md"), "content") - writeSource(b, fs, filepath.Join(base, js, "logo1.png"), "content") - writeSource(b, fs, filepath.Join(base, js, "sub", "logo2.png"), "content") - writeSource(b, fs, filepath.Join(base, js, "section", "_index.md"), "content") - writeSource(b, fs, filepath.Join(base, js, "section", "logo.png"), "content") - writeSource(b, fs, filepath.Join(base, js, "section", "sub", "logo.png"), "content") - - for k := 1; k <= 5; k++ { - ks := fmt.Sprintf("k%d", k) - writeSource(b, fs, filepath.Join(base, js, ks, "logo1.png"), "content") - writeSource(b, fs, filepath.Join(base, js, "section", ks, "logo.png"), "content") - } - } - - for i := 1; i <= 5; i++ { - writeSource(b, fs, filepath.Join(base, "assetsonly", fmt.Sprintf("image%d.png", i)), "image") - } - - for i := 1; i <= 5; i++ { - writeSource(b, fs, filepath.Join(base, "contentonly", fmt.Sprintf("c%d.md", i)), "content") - } - - capturers[i] = newCapturer(loggers.NewErrorLogger(), sourceSpec, new(noOpFileStore), nil, base) - } - - b.ResetTimer() - for i := 0; i < b.N; i++ { - err := capturers[i].capture() - if err != nil { - b.Fatal(err) - } - } -} diff --git a/hugolib/pagebundler_handlers.go b/hugolib/pagebundler_handlers.go deleted file mode 100644 index e745a04f2ac..00000000000 --- a/hugolib/pagebundler_handlers.go +++ /dev/null @@ -1,305 +0,0 @@ -// Copyright 2019 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugolib - -import ( - "errors" - "fmt" - "path/filepath" - - "github.com/gohugoio/hugo/common/hugio" - - "strings" - - "github.com/gohugoio/hugo/resources" - "github.com/gohugoio/hugo/resources/resource" -) - -var ( - // This should be the only list of valid extensions for content files. - contentFileExtensions = []string{ - "html", "htm", - "mdown", "markdown", "md", - "asciidoc", "adoc", "ad", - "rest", "rst", - "mmark", - "org", - "pandoc", "pdc"} - - contentFileExtensionsSet map[string]bool -) - -func init() { - contentFileExtensionsSet = make(map[string]bool) - for _, ext := range contentFileExtensions { - contentFileExtensionsSet[ext] = true - } -} - -func newHandlerChain(s *Site) contentHandler { - c := &contentHandlers{s: s} - - contentFlow := c.parsePage( - c.handlePageContent(), - ) - - c.rootHandler = c.processFirstMatch( - contentFlow, - - // Creates a file resource (image, CSS etc.) if there is a parent - // page set on the current context. - c.createResource(), - - // Everything that isn't handled above, will just be copied - // to destination. - c.copyFile(), - ) - - return c.rootHandler - -} - -type contentHandlers struct { - s *Site - rootHandler contentHandler -} - -func (c *contentHandlers) processFirstMatch(handlers ...contentHandler) func(ctx *handlerContext) handlerResult { - return func(ctx *handlerContext) handlerResult { - for _, h := range handlers { - res := h(ctx) - if res.handled || res.err != nil { - return res - } - } - return handlerResult{err: errors.New("no matching handler found")} - } -} - -type handlerContext struct { - // These are the pages stored in Site. - pages chan<- *pageState - - doNotAddToSiteCollections bool - - currentPage *pageState - parentPage *pageState - - bundle *bundleDir - - source *fileInfo - - // Relative path to the target. - target string -} - -func (c *handlerContext) ext() string { - if c.currentPage != nil { - return c.currentPage.contentMarkupType() - } - - if c.bundle != nil { - return c.bundle.fi.Ext() - } else { - return c.source.Ext() - } -} - -func (c *handlerContext) targetPath() string { - if c.target != "" { - return c.target - } - - return c.source.Filename() -} - -func (c *handlerContext) file() *fileInfo { - if c.bundle != nil { - return c.bundle.fi - } - - return c.source -} - -// Create a copy with the current context as its parent. -func (c handlerContext) childCtx(fi *fileInfo) *handlerContext { - if c.currentPage == nil { - panic("Need a Page to create a child context") - } - - c.target = strings.TrimPrefix(fi.Path(), c.bundle.fi.Dir()) - c.source = fi - - c.doNotAddToSiteCollections = c.bundle != nil && c.bundle.tp != bundleBranch - - c.bundle = nil - - c.parentPage = c.currentPage - c.currentPage = nil - - return &c -} - -func (c *handlerContext) supports(exts ...string) bool { - ext := c.ext() - for _, s := range exts { - if s == ext { - return true - } - } - - return false -} - -func (c *handlerContext) isContentFile() bool { - return contentFileExtensionsSet[c.ext()] -} - -type ( - handlerResult struct { - err error - handled bool - result interface{} - } - - contentHandler func(ctx *handlerContext) handlerResult -) - -var ( - notHandled handlerResult -) - -func (c *contentHandlers) parsePage(h contentHandler) contentHandler { - return func(ctx *handlerContext) handlerResult { - if !ctx.isContentFile() { - return notHandled - } - - result := handlerResult{handled: true} - fi := ctx.file() - - content := func() (hugio.ReadSeekCloser, error) { - f, err := fi.Open() - if err != nil { - return nil, fmt.Errorf("failed to open content file %q: %s", fi.Filename(), err) - } - return f, nil - } - - ps, err := newPageWithContent(fi, c.s, ctx.parentPage != nil, content) - if err != nil { - return handlerResult{err: err} - } - - if !c.s.shouldBuild(ps) { - if !ctx.doNotAddToSiteCollections { - ctx.pages <- ps - } - return result - } - - ctx.currentPage = ps - - if ctx.bundle != nil { - // Add the bundled files - for _, fi := range ctx.bundle.resources { - childCtx := ctx.childCtx(fi) - res := c.rootHandler(childCtx) - if res.err != nil { - return res - } - if res.result != nil { - switch resv := res.result.(type) { - case *pageState: - resv.m.resourcePath = filepath.ToSlash(childCtx.target) - resv.parent = ps - ps.addResources(resv) - case resource.Resource: - ps.addResources(resv) - - default: - panic("Unknown type") - } - } - } - } - - return h(ctx) - } -} - -func (c *contentHandlers) handlePageContent() contentHandler { - return func(ctx *handlerContext) handlerResult { - p := ctx.currentPage - - if !ctx.doNotAddToSiteCollections { - ctx.pages <- p - } - - return handlerResult{handled: true, result: p} - } -} - -func (c *contentHandlers) createResource() contentHandler { - return func(ctx *handlerContext) handlerResult { - if ctx.parentPage == nil { - return notHandled - } - - // TODO(bep) consolidate with multihost logic + clean up - outputFormats := ctx.parentPage.m.outputFormats() - seen := make(map[string]bool) - var targetBasePaths []string - // Make sure bundled resources are published to all of the ouptput formats' - // sub paths. - for _, f := range outputFormats { - p := f.Path - if seen[p] { - continue - } - seen[p] = true - targetBasePaths = append(targetBasePaths, p) - - } - - resource, err := c.s.ResourceSpec.New( - resources.ResourceSourceDescriptor{ - TargetPaths: ctx.parentPage.getTargetPaths, - SourceFile: ctx.source, - RelTargetFilename: ctx.target, - TargetBasePaths: targetBasePaths, - }) - - return handlerResult{err: err, handled: true, result: resource} - } -} - -func (c *contentHandlers) copyFile() contentHandler { - return func(ctx *handlerContext) handlerResult { - f, err := c.s.BaseFs.Content.Fs.Open(ctx.source.Filename()) - if err != nil { - err := fmt.Errorf("failed to open file in copyFile: %s", err) - return handlerResult{err: err} - } - - target := ctx.targetPath() - - defer f.Close() - if err := c.s.publish(&c.s.PathSpec.ProcessingStats.Files, target, f); err != nil { - return handlerResult{err: err} - } - - return handlerResult{handled: true} - } -} diff --git a/hugolib/pagebundler_test.go b/hugolib/pagebundler_test.go index 79e0a107d60..f5a6d9a84e3 100644 --- a/hugolib/pagebundler_test.go +++ b/hugolib/pagebundler_test.go @@ -16,18 +16,18 @@ package hugolib import ( "os" "path" - "runtime" + "regexp" "strings" "testing" + "github.com/gohugoio/hugo/hugofs" + "github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/resources/page" - "github.com/gohugoio/hugo/helpers" - "io" - "github.com/spf13/afero" + "github.com/gohugoio/hugo/htesting" "github.com/gohugoio/hugo/media" @@ -36,7 +36,6 @@ import ( "fmt" "github.com/gohugoio/hugo/deps" - "github.com/gohugoio/hugo/hugofs" "github.com/spf13/viper" "github.com/stretchr/testify/require" @@ -54,8 +53,11 @@ func TestPageBundlerSiteRegular(t *testing.T) { if baseURLPathId == "" { baseURLPathId = "NONE" } + ugly := ugly + canonify := canonify t.Run(fmt.Sprintf("ugly=%t,canonify=%t,path=%s", ugly, canonify, baseURLPathId), func(t *testing.T) { + t.Parallel() baseURL := baseBaseURL + baseURLPath relURLBase := baseURLPath if canonify { @@ -65,14 +67,12 @@ func TestPageBundlerSiteRegular(t *testing.T) { fs, cfg := newTestBundleSources(t) cfg.Set("baseURL", baseURL) cfg.Set("canonifyURLs", canonify) - assert.NoError(loadDefaultSettingsFor(cfg)) - assert.NoError(loadLanguageSettings(cfg, nil)) cfg.Set("permalinks", map[string]string{ "a": ":sections/:filename", "b": ":year/:slug/", "c": ":sections/:slug", - "": ":filename/", + "/": ":filename/", }) cfg.Set("outputFormats", map[string]interface{}{ @@ -92,9 +92,11 @@ func TestPageBundlerSiteRegular(t *testing.T) { cfg.Set("uglyURLs", ugly) - s := buildSingleSite(t, deps.DepsCfg{Logger: loggers.NewErrorLogger(), Fs: fs, Cfg: cfg}, BuildCfg{}) + b := newTestSitesBuilderFromDepsCfg(t, deps.DepsCfg{Logger: loggers.NewErrorLogger(), Fs: fs, Cfg: cfg}).WithNothingAdded() + + b.Build(BuildCfg{}) - th := testHelper{s.Cfg, s.Fs, t} + s := b.H.Sites[0] assert.Len(s.RegularPages(), 8) @@ -127,22 +129,22 @@ func TestPageBundlerSiteRegular(t *testing.T) { // Check both output formats rel, filename := relFilename("/a/1/", "index.html") - th.assertFileContent(filepath.Join("/work/public", filename), + b.AssertFileContent(filepath.Join("/work/public", filename), "TheContent", "Single RelPermalink: "+rel, ) rel, filename = relFilename("/cpath/a/1/", "cindex.html") - th.assertFileContent(filepath.Join("/work/public", filename), + b.AssertFileContent(filepath.Join("/work/public", filename), "TheContent", "Single RelPermalink: "+rel, ) - th.assertFileContent(filepath.FromSlash("/work/public/images/hugo-logo.png"), "content") + b.AssertFileContent(filepath.FromSlash("/work/public/images/hugo-logo.png"), "content") // This should be just copied to destination. - th.assertFileContent(filepath.FromSlash("/work/public/assets/pic1.png"), "content") + b.AssertFileContent(filepath.FromSlash("/work/public/assets/pic1.png"), "content") leafBundle1 := s.getPage(page.KindPage, "b/my-bundle/index.md") assert.NotNil(leafBundle1) @@ -159,8 +161,8 @@ func TestPageBundlerSiteRegular(t *testing.T) { assert.NotNil(rootBundle) assert.True(rootBundle.Parent().IsHome()) if !ugly { - th.assertFileContent(filepath.FromSlash("/work/public/root/index.html"), "Single RelPermalink: "+relURLBase+"/root/") - th.assertFileContent(filepath.FromSlash("/work/public/cpath/root/cindex.html"), "Single RelPermalink: "+relURLBase+"/cpath/root/") + b.AssertFileContent(filepath.FromSlash("/work/public/root/index.html"), "Single RelPermalink: "+relURLBase+"/root/") + b.AssertFileContent(filepath.FromSlash("/work/public/cpath/root/cindex.html"), "Single RelPermalink: "+relURLBase+"/cpath/root/") } leafBundle2 := s.getPage(page.KindPage, "a/b/index.md") @@ -172,6 +174,7 @@ func TestPageBundlerSiteRegular(t *testing.T) { assert.Len(pageResources, 2) firstPage := pageResources[0].(page.Page) secondPage := pageResources[1].(page.Page) + assert.Equal(filepath.FromSlash("/work/base/b/my-bundle/1.md"), firstPage.File().Filename(), secondPage.File().Filename()) assert.Contains(content(firstPage), "TheContent") assert.Equal(6, len(leafBundle1.Resources())) @@ -201,17 +204,17 @@ func TestPageBundlerSiteRegular(t *testing.T) { } if ugly { - th.assertFileContent("/work/public/2017/pageslug.html", + b.AssertFileContent("/work/public/2017/pageslug.html", relPermalinker("Single RelPermalink: %s/2017/pageslug.html"), permalinker("Single Permalink: %s/2017/pageslug.html"), relPermalinker("Sunset RelPermalink: %s/2017/pageslug/sunset1.jpg"), permalinker("Sunset Permalink: %s/2017/pageslug/sunset1.jpg")) } else { - th.assertFileContent("/work/public/2017/pageslug/index.html", + b.AssertFileContent("/work/public/2017/pageslug/index.html", relPermalinker("Sunset RelPermalink: %s/2017/pageslug/sunset1.jpg"), permalinker("Sunset Permalink: %s/2017/pageslug/sunset1.jpg")) - th.assertFileContent("/work/public/cpath/2017/pageslug/cindex.html", + b.AssertFileContent("/work/public/cpath/2017/pageslug/cindex.html", relPermalinker("Single RelPermalink: %s/cpath/2017/pageslug/"), relPermalinker("Short Sunset RelPermalink: %s/cpath/2017/pageslug/sunset2.jpg"), relPermalinker("Sunset RelPermalink: %s/cpath/2017/pageslug/sunset1.jpg"), @@ -219,15 +222,15 @@ func TestPageBundlerSiteRegular(t *testing.T) { ) } - th.assertFileContent(filepath.FromSlash("/work/public/2017/pageslug/c/logo.png"), "content") - th.assertFileContent(filepath.FromSlash("/work/public/cpath/2017/pageslug/c/logo.png"), "content") - th.assertFileNotExist("/work/public/cpath/cpath/2017/pageslug/c/logo.png") + b.AssertFileContent(filepath.FromSlash("/work/public/2017/pageslug/c/logo.png"), "content") + b.AssertFileContent(filepath.FromSlash("/work/public/cpath/2017/pageslug/c/logo.png"), "content") + assert.False(b.CheckExists("/work/public/cpath/cpath/2017/pageslug/c/logo.png")) // Custom media type defined in site config. assert.Len(leafBundle1.Resources().ByType("bepsays"), 1) if ugly { - th.assertFileContent(filepath.FromSlash("/work/public/2017/pageslug.html"), + b.AssertFileContent(filepath.FromSlash("/work/public/2017/pageslug.html"), "TheContent", relPermalinker("Sunset RelPermalink: %s/2017/pageslug/sunset1.jpg"), permalinker("Sunset Permalink: %s/2017/pageslug/sunset1.jpg"), @@ -245,19 +248,19 @@ func TestPageBundlerSiteRegular(t *testing.T) { ) // https://github.com/gohugoio/hugo/issues/5882 - th.assertFileContent( + b.AssertFileContent( filepath.FromSlash("/work/public/2017/pageslug.html"), "0: Page RelPermalink: |") - th.assertFileContent(filepath.FromSlash("/work/public/cpath/2017/pageslug.html"), "TheContent") + b.AssertFileContent(filepath.FromSlash("/work/public/cpath/2017/pageslug.html"), "TheContent") // 은행 - th.assertFileContent(filepath.FromSlash("/work/public/c/은행/logo-은행.png"), "은행 PNG") + b.AssertFileContent(filepath.FromSlash("/work/public/c/은행/logo-은행.png"), "은행 PNG") } else { - th.assertFileContent(filepath.FromSlash("/work/public/2017/pageslug/index.html"), "TheContent") - th.assertFileContent(filepath.FromSlash("/work/public/cpath/2017/pageslug/cindex.html"), "TheContent") - th.assertFileContent(filepath.FromSlash("/work/public/2017/pageslug/index.html"), "Single Title") - th.assertFileContent(filepath.FromSlash("/work/public/root/index.html"), "Single Title") + b.AssertFileContent(filepath.FromSlash("/work/public/2017/pageslug/index.html"), "TheContent") + b.AssertFileContent(filepath.FromSlash("/work/public/cpath/2017/pageslug/cindex.html"), "TheContent") + b.AssertFileContent(filepath.FromSlash("/work/public/2017/pageslug/index.html"), "Single Title") + b.AssertFileContent(filepath.FromSlash("/work/public/root/index.html"), "Single Title") } @@ -272,25 +275,26 @@ func TestPageBundlerSiteMultilingual(t *testing.T) { t.Parallel() for _, ugly := range []bool{false, true} { + ugly := ugly t.Run(fmt.Sprintf("ugly=%t", ugly), func(t *testing.T) { - + t.Parallel() assert := require.New(t) fs, cfg := newTestBundleSourcesMultilingual(t) cfg.Set("uglyURLs", ugly) - assert.NoError(loadDefaultSettingsFor(cfg)) - assert.NoError(loadLanguageSettings(cfg, nil)) - sites, err := NewHugoSites(deps.DepsCfg{Fs: fs, Cfg: cfg}) - assert.NoError(err) - assert.Equal(2, len(sites.Sites)) + b := newTestSitesBuilderFromDepsCfg(t, deps.DepsCfg{Fs: fs, Cfg: cfg}).WithNothingAdded() + b.Build(BuildCfg{SkipRender: true}) - assert.NoError(sites.Build(BuildCfg{})) + sites := b.H + + assert.Equal(2, len(sites.Sites)) s := sites.Sites[0] assert.Equal(8, len(s.RegularPages())) assert.Equal(16, len(s.Pages())) + //dumpPages(s.AllPages()...) assert.Equal(31, len(s.AllPages())) bundleWithSubPath := s.getPage(page.KindPage, "lb/index") @@ -358,15 +362,12 @@ func TestMultilingualDisableLanguage(t *testing.T) { fs, cfg := newTestBundleSourcesMultilingual(t) cfg.Set("disableLanguages", []string{"nn"}) - assert.NoError(loadDefaultSettingsFor(cfg)) - assert.NoError(loadLanguageSettings(cfg, nil)) + b := newTestSitesBuilderFromDepsCfg(t, deps.DepsCfg{Fs: fs, Cfg: cfg}).WithNothingAdded() + b.Build(BuildCfg{}) + sites := b.H - sites, err := NewHugoSites(deps.DepsCfg{Fs: fs, Cfg: cfg}) - assert.NoError(err) assert.Equal(1, len(sites.Sites)) - assert.NoError(sites.Build(BuildCfg{})) - s := sites.Sites[0] assert.Equal(8, len(s.RegularPages())) @@ -383,20 +384,98 @@ func TestMultilingualDisableLanguage(t *testing.T) { } func TestPageBundlerSiteWitSymbolicLinksInContent(t *testing.T) { - if runtime.GOOS == "windows" && os.Getenv("CI") == "" { - t.Skip("Skip TestPageBundlerSiteWitSymbolicLinksInContent as os.Symlink needs administrator rights on Windows") - } + skipSymlink(t) + + wd, _ := os.Getwd() + defer func() { + os.Chdir(wd) + }() assert := require.New(t) - ps, clean, workDir := newTestBundleSymbolicSources(t) + // We need to use the OS fs for this. + cfg := viper.New() + fs := hugofs.NewFrom(hugofs.Os, cfg) + + workDir, clean, err := htesting.CreateTempDir(hugofs.Os, "hugosym") + assert.NoError(err) + + contentDirName := "content" + + contentDir := filepath.Join(workDir, contentDirName) + assert.NoError(os.MkdirAll(filepath.Join(contentDir, "a"), 0777)) + + for i := 1; i <= 3; i++ { + assert.NoError(os.MkdirAll(filepath.Join(workDir, fmt.Sprintf("symcontent%d", i)), 0777)) + } + + assert.NoError(os.MkdirAll(filepath.Join(workDir, "symcontent2", "a1"), 0777)) + + // Symlinked sections inside content. + os.Chdir(contentDir) + for i := 1; i <= 3; i++ { + assert.NoError(os.Symlink(filepath.FromSlash(fmt.Sprintf(("../symcontent%d"), i)), fmt.Sprintf("symbolic%d", i))) + } + + assert.NoError(os.Chdir(filepath.Join(contentDir, "a"))) + + // Create a symlink to one single content file + assert.NoError(os.Symlink(filepath.FromSlash("../../symcontent2/a1/page.md"), "page_s.md")) + + assert.NoError(os.Chdir(filepath.FromSlash("../../symcontent3"))) + + // Create a circular symlink. Will print some warnings. + assert.NoError(os.Symlink(filepath.Join("..", contentDirName), filepath.FromSlash("circus"))) + + assert.NoError(os.Chdir(workDir)) + defer clean() - cfg := ps.Cfg - fs := ps.Fs + cfg.Set("workingDir", workDir) + cfg.Set("contentDir", contentDirName) + cfg.Set("baseURL", "https://example.com") + + layout := `{{ .Title }}|{{ .Content }}` + pageContent := `--- +slug: %s +date: 2017-10-09 +--- + +TheContent. +` + + b := newTestSitesBuilderFromDepsCfg(t, deps.DepsCfg{ + Fs: fs, + Cfg: cfg, + }) - s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg, Logger: loggers.NewErrorLogger()}, BuildCfg{}) + b.WithTemplates( + "_default/single.html", layout, + "_default/list.html", layout, + ) - th := testHelper{s.Cfg, s.Fs, t} + b.WithContent( + "a/regular.md", fmt.Sprintf(pageContent, "a1"), + ) + + b.WithSourceFile( + "symcontent1/s1.md", fmt.Sprintf(pageContent, "s1"), + "symcontent1/s2.md", fmt.Sprintf(pageContent, "s2"), + // Regular files inside symlinked folder. + "symcontent1/s1.md", fmt.Sprintf(pageContent, "s1"), + "symcontent1/s2.md", fmt.Sprintf(pageContent, "s2"), + + // A bundle + "symcontent2/a1/index.md", fmt.Sprintf(pageContent, ""), + "symcontent2/a1/page.md", fmt.Sprintf(pageContent, "page"), + "symcontent2/a1/logo.png", "image", + + // Assets + "symcontent3/s1.png", "image", + "symcontent3/s2.png", "image", + ) + + b.Build(BuildCfg{}) + s := b.H.Sites[0] assert.Equal(7, len(s.RegularPages())) a1Bundle := s.getPage(page.KindPage, "symbolic2/a1/index.md") @@ -404,9 +483,9 @@ func TestPageBundlerSiteWitSymbolicLinksInContent(t *testing.T) { assert.Equal(2, len(a1Bundle.Resources())) assert.Equal(1, len(a1Bundle.Resources().ByType(pageResourceType))) - th.assertFileContent(filepath.FromSlash(workDir+"/public/a/page/index.html"), "TheContent") - th.assertFileContent(filepath.FromSlash(workDir+"/public/symbolic1/s1/index.html"), "TheContent") - th.assertFileContent(filepath.FromSlash(workDir+"/public/symbolic2/a1/index.html"), "TheContent") + b.AssertFileContent(filepath.FromSlash(workDir+"/public/a/page/index.html"), "TheContent") + b.AssertFileContent(filepath.FromSlash(workDir+"/public/symbolic1/s1/index.html"), "TheContent") + b.AssertFileContent(filepath.FromSlash(workDir+"/public/symbolic2/a1/index.html"), "TheContent") } @@ -576,7 +655,7 @@ Single content. } func newTestBundleSources(t *testing.T) (*hugofs.Fs, *viper.Viper) { - cfg, fs := newTestCfg() + cfg, fs := newTestCfgBasic() assert := require.New(t) workDir := "/work" @@ -743,7 +822,7 @@ Content for 은행. } func newTestBundleSourcesMultilingual(t *testing.T) (*hugofs.Fs, *viper.Viper) { - cfg, fs := newTestCfg() + cfg, fs := newTestCfgBasic() workDir := "/work" cfg.Set("workingDir", workDir) @@ -825,90 +904,6 @@ TheContent. return fs, cfg } -func newTestBundleSymbolicSources(t *testing.T) (*helpers.PathSpec, func(), string) { - assert := require.New(t) - // We need to use the OS fs for this. - cfg := viper.New() - fs := hugofs.NewFrom(hugofs.Os, cfg) - fs.Destination = &afero.MemMapFs{} - loadDefaultSettingsFor(cfg) - - workDir, clean, err := createTempDir("hugosym") - assert.NoError(err) - - contentDir := "base" - cfg.Set("workingDir", workDir) - cfg.Set("contentDir", contentDir) - cfg.Set("baseURL", "https://example.com") - - if err := loadLanguageSettings(cfg, nil); err != nil { - t.Fatal(err) - } - - layout := `{{ .Title }}|{{ .Content }}` - pageContent := `--- -slug: %s -date: 2017-10-09 ---- - -TheContent. -` - - fs.Source.MkdirAll(filepath.Join(workDir, "layouts", "_default"), 0777) - fs.Source.MkdirAll(filepath.Join(workDir, contentDir), 0777) - fs.Source.MkdirAll(filepath.Join(workDir, contentDir, "a"), 0777) - for i := 1; i <= 3; i++ { - fs.Source.MkdirAll(filepath.Join(workDir, fmt.Sprintf("symcontent%d", i)), 0777) - - } - fs.Source.MkdirAll(filepath.Join(workDir, "symcontent2", "a1"), 0777) - - writeSource(t, fs, filepath.Join(workDir, "layouts", "_default", "single.html"), layout) - writeSource(t, fs, filepath.Join(workDir, "layouts", "_default", "list.html"), layout) - - writeSource(t, fs, filepath.Join(workDir, contentDir, "a", "regular.md"), fmt.Sprintf(pageContent, "a1")) - - // Regular files inside symlinked folder. - writeSource(t, fs, filepath.Join(workDir, "symcontent1", "s1.md"), fmt.Sprintf(pageContent, "s1")) - writeSource(t, fs, filepath.Join(workDir, "symcontent1", "s2.md"), fmt.Sprintf(pageContent, "s2")) - - // A bundle - writeSource(t, fs, filepath.Join(workDir, "symcontent2", "a1", "index.md"), fmt.Sprintf(pageContent, "")) - writeSource(t, fs, filepath.Join(workDir, "symcontent2", "a1", "page.md"), fmt.Sprintf(pageContent, "page")) - writeSource(t, fs, filepath.Join(workDir, "symcontent2", "a1", "logo.png"), "image") - - // Assets - writeSource(t, fs, filepath.Join(workDir, "symcontent3", "s1.png"), "image") - writeSource(t, fs, filepath.Join(workDir, "symcontent3", "s2.png"), "image") - - wd, _ := os.Getwd() - defer func() { - os.Chdir(wd) - }() - // Symlinked sections inside content. - os.Chdir(filepath.Join(workDir, contentDir)) - for i := 1; i <= 3; i++ { - assert.NoError(os.Symlink(filepath.FromSlash(fmt.Sprintf(("../symcontent%d"), i)), fmt.Sprintf("symbolic%d", i))) - } - - os.Chdir(filepath.Join(workDir, contentDir, "a")) - - // Create a symlink to one single content file - assert.NoError(os.Symlink(filepath.FromSlash("../../symcontent2/a1/page.md"), "page_s.md")) - - os.Chdir(filepath.FromSlash("../../symcontent3")) - - // Create a circular symlink. Will print some warnings. - assert.NoError(os.Symlink(filepath.Join("..", contentDir), filepath.FromSlash("circus"))) - - os.Chdir(workDir) - assert.NoError(err) - - ps, _ := helpers.NewPathSpec(fs, cfg) - - return ps, clean, workDir -} - // https://github.com/gohugoio/hugo/issues/5858 func TestBundledResourcesWhenMultipleOutputFormats(t *testing.T) { t.Parallel() @@ -971,3 +966,112 @@ slug: %s assert.True(b.CheckExists("public/about/services2/this-is-another-slug/index.html")) } + +func TestBundleMisc(t *testing.T) { + config := ` +baseURL = "https://example.com" +defaultContentLanguage = "en" +defaultContentLanguageInSubdir = true +ignoreFiles = ["README\\.md", "content/en/ignore"] + +[Languages] +[Languages.en] +weight = 99999 +contentDir = "content/en" +[Languages.nn] +weight = 20 +contentDir = "content/nn" +[Languages.sv] +weight = 30 +contentDir = "content/sv" +[Languages.nb] +weight = 40 +contentDir = "content/nb" + +` + + const pageContent = `--- +title: %q +--- +` + createPage := func(s string) string { + return fmt.Sprintf(pageContent, s) + } + + b := newTestSitesBuilder(t).WithConfigFile("toml", config) + b.WithLogger(loggers.NewWarningLogger()) + + b.WithTemplates("_default/list.html", `{{ range .Site.Pages }} +{{ .Kind }}|{{ .Path }}|{{ with .CurrentSection }}CurrentSection: {{ .Path }}{{ end }}|{{ .RelPermalink }}{{ end }} +`) + + b.WithTemplates("_default/single.html", `Single: {{ .Title }}`) + + b.WithContent("en/sect1/sect2/_index.md", createPage("en: Sect 2")) + b.WithContent("en/sect1/sect2/page.md", createPage("en: Page")) + b.WithContent("en/sect1/sect2/data-branch.json", "mydata") + b.WithContent("nn/sect1/sect2/page.md", createPage("nn: Page")) + b.WithContent("nn/sect1/sect2/data-branch.json", "my nn data") + + // En only + b.WithContent("en/enonly/myen.md", createPage("en: Page")) + b.WithContent("en/enonly/myendata.json", "mydata") + + // Leaf + + b.WithContent("nn/b1/index.md", createPage("nn: leaf")) + b.WithContent("en/b1/index.md", createPage("en: leaf")) + b.WithContent("sv/b1/index.md", createPage("sv: leaf")) + b.WithContent("nb/b1/index.md", createPage("nb: leaf")) + + // Should be ignored + b.WithContent("en/ignore/page.md", createPage("en: ignore")) + b.WithContent("en/README.md", createPage("en: ignore")) + + // Both leaf and branch bundle in same dir + b.WithContent("en/b2/index.md", `--- +slug: leaf +--- +`) + b.WithContent("en/b2/_index.md", createPage("en: branch")) + + b.WithContent("en/b1/data1.json", "en: data") + b.WithContent("sv/b1/data1.json", "sv: data") + b.WithContent("sv/b1/data2.json", "sv: data2") + b.WithContent("nb/b1/data2.json", "nb: data2") + + b.Build(BuildCfg{}) + + b.AssertFileContent("public/en/index.html", + filepath.FromSlash("section|sect1/sect2/_index.md|CurrentSection: sect1/sect2/_index.md"), + "myen.md|CurrentSection: enonly") + + b.AssertFileContentFn("public/en/index.html", func(s string) bool { + // Check ignored files + return !regexp.MustCompile("README|ignore").MatchString(s) + + }) + + b.AssertFileContent("public/nn/index.html", filepath.FromSlash("page|sect1/sect2/page.md|CurrentSection: sect1")) + b.AssertFileContentFn("public/nn/index.html", func(s string) bool { + return !strings.Contains(s, "enonly") + + }) + + // Check order of inherited data file + b.AssertFileContent("public/nb/b1/data1.json", "en: data") // Default content + b.AssertFileContent("public/nn/b1/data2.json", "sv: data") // First match + + b.AssertFileContent("public/en/enonly/myen/index.html", "Single: en: Page") + b.AssertFileContent("public/en/enonly/myendata.json", "mydata") + + assert := require.New(t) + assert.False(b.CheckExists("public/sv/enonly/myen/index.html")) + + // Both leaf and branch bundle in same dir + // We log a warning about it, but we keep both. + b.AssertFileContent("public/en/b2/index.html", + "/en/b2/leaf/", + filepath.FromSlash("section|sect1/sect2/_index.md|CurrentSection: sect1/sect2/_index.md")) + +} diff --git a/hugolib/pagecollections.go b/hugolib/pagecollections.go index 24318232034..aedcf40901e 100644 --- a/hugolib/pagecollections.go +++ b/hugolib/pagecollections.go @@ -148,7 +148,6 @@ func newPageCollectionsFromPages(pages pageStatePages) *PageCollections { for _, p := range pageCollection { if p.IsPage() { sourceRef := p.sourceRef() - if sourceRef != "" { // index the canonical ref // e.g. /section/article.md diff --git a/hugolib/pages_capture.go b/hugolib/pages_capture.go new file mode 100644 index 00000000000..09493379325 --- /dev/null +++ b/hugolib/pages_capture.go @@ -0,0 +1,779 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "context" + "fmt" + "os" + pth "path" + "path/filepath" + "strings" + + "github.com/gohugoio/hugo/config" + + "github.com/gohugoio/hugo/hugofs/files" + + "github.com/gohugoio/hugo/resources" + + "github.com/pkg/errors" + "golang.org/x/sync/errgroup" + + "github.com/gohugoio/hugo/common/hugio" + + "github.com/gohugoio/hugo/resources/resource" + + "github.com/gohugoio/hugo/source" + + "github.com/gohugoio/hugo/hugofs" + + "github.com/gohugoio/hugo/common/loggers" + "github.com/spf13/afero" +) + +func newPagesCollector( + sp *source.SourceSpec, + logger *loggers.Logger, + contentTracker *contentChangeMap, + proc pagesCollectorProcessorProvider, filenames ...string) *pagesCollector { + + return &pagesCollector{ + fs: sp.SourceFs, + proc: proc, + sp: sp, + logger: logger, + filenames: filenames, + tracker: contentTracker, + } +} + +func newPagesProcessor(h *HugoSites, sp *source.SourceSpec, partialBuild bool) *pagesProcessor { + + return &pagesProcessor{ + h: h, + sp: sp, + partialBuild: partialBuild, + numWorkers: config.GetNumWorkerMultiplier() * 3, + } +} + +type fileinfoBundle struct { + header hugofs.FileMetaInfo + resources []hugofs.FileMetaInfo +} + +func (b *fileinfoBundle) containsResource(name string) bool { + for _, r := range b.resources { + if r.Name() == name { + return true + } + } + + return false + +} + +type pageBundles map[string]*fileinfoBundle + +type pagesCollector struct { + sp *source.SourceSpec + fs afero.Fs + logger *loggers.Logger + + // Ordered list (bundle headers first) used in partial builds. + filenames []string + + // Content files tracker used in partial builds. + tracker *contentChangeMap + + proc pagesCollectorProcessorProvider +} + +type contentDirKey struct { + dirname string + filename string + tp bundleDirType +} + +// Collect. +func (c *pagesCollector) Collect() error { + c.proc.Start(context.Background()) + if c.tracker != nil { + c.tracker.start() + defer c.tracker.stop() + } + + var collectErr error + if len(c.filenames) == 0 { + // Collect everything. + collectErr = c.collectDir("", false, nil) + } else { + dirs := make(map[contentDirKey]bool) + for _, filename := range c.filenames { + dir, filename, btype := c.tracker.resolveAndRemove(filename) + dirs[contentDirKey{dir, filename, btype}] = true + } + + for dir, _ := range dirs { + switch dir.tp { + case bundleLeaf, bundleBranch: + collectErr = c.collectDir(dir.dirname, true, nil) + default: + // We always start from a directory. + collectErr = c.collectDir(dir.dirname, true, func(fim hugofs.FileMetaInfo) bool { + return strings.HasSuffix(dir.filename, fim.Meta().Path()) + }) + } + + if collectErr != nil { + break + } + } + + } + + err := c.proc.Wait() + + if collectErr != nil { + return collectErr + } + + return err +} + +func (c *pagesCollector) collectDir(dirname string, partial bool, inFilter func(fim hugofs.FileMetaInfo) bool) error { + fi, err := c.fs.Stat(dirname) + if err != nil { + if os.IsNotExist(err) { + // May have been deleted. + return nil + } + return err + } + + handleDir := func( + btype bundleDirType, + dir hugofs.FileMetaInfo, + path string, + readdir []hugofs.FileMetaInfo) error { + + if btype > bundleNot && c.tracker != nil { + c.tracker.add(path, btype) + } + + if btype == bundleBranch { + if err := c.handleBundleBranch(readdir); err != nil { + return err + } + // A branch bundle is only this directory level, so keep walking. + return nil + } else if btype == bundleLeaf { + if err := c.handleBundleLeaf(dir, path, readdir); err != nil { + return err + } + + return nil + } + + if err := c.handleFiles(readdir...); err != nil { + return err + } + + return nil + + } + + filter := func(fim hugofs.FileMetaInfo) bool { + if fim.Meta().SkipDir() { + return false + } + + if c.sp.IgnoreFile(fim.Meta().Filename()) { + return false + } + + if inFilter != nil { + return inFilter(fim) + } + return true + } + + preHook := func(dir hugofs.FileMetaInfo, path string, readdir []hugofs.FileMetaInfo) ([]hugofs.FileMetaInfo, error) { + var btype bundleDirType + + filtered := readdir[:0] + for _, fi := range readdir { + if filter(fi) { + filtered = append(filtered, fi) + if c.tracker != nil { + // Track symlinks. + c.tracker.addSymbolicLinkMapping(fi) + } + } + } + readdir = filtered + + // We merge language directories, so there can be duplicates, but they + // will be ordered, most important first. + var duplicates []int + seen := make(map[string]bool) + + for i, fi := range readdir { + + if fi.IsDir() { + continue + } + + meta := fi.Meta() + class := meta.Classifier() + translationBase := meta.TranslationBaseNameWithExt() + key := pth.Join(meta.Lang(), translationBase) + + if seen[key] { + duplicates = append(duplicates, i) + continue + } + seen[key] = true + + var thisBtype bundleDirType + + switch class { + case files.ContentClassLeaf: + thisBtype = bundleLeaf + case files.ContentClassBranch: + thisBtype = bundleBranch + } + + // Folders with both index.md and _index.md type of files have + // undefined behaviour and can never work. + // The branch variant will win because of sort order, but log + // a warning about it. + if thisBtype > bundleNot && btype > bundleNot && thisBtype != btype { + c.logger.WARN.Printf("Content directory %q have both index.* and _index.* files, pick one.", dir.Meta().Filename()) + // Reclassify it so it will be handled as a content file inside the + // section, which is in line with the <= 0.55 behaviour. + meta["classifier"] = files.ContentClassContent + } else if thisBtype > bundleNot { + btype = thisBtype + } + + } + + if len(duplicates) > 0 { + for i := len(duplicates) - 1; i >= 0; i-- { + idx := duplicates[i] + readdir = append(readdir[:idx], readdir[idx+1:]...) + } + } + + err := handleDir(btype, dir, path, readdir) + if err != nil { + return nil, err + } + + if btype == bundleLeaf || partial { + return nil, filepath.SkipDir + } + + // Keep walking. + return readdir, nil + + } + + var postHook hugofs.WalkHook + if c.tracker != nil { + postHook = func(dir hugofs.FileMetaInfo, path string, readdir []hugofs.FileMetaInfo) ([]hugofs.FileMetaInfo, error) { + if c.tracker == nil { + // Nothing to do. + return readdir, nil + } + + return readdir, nil + } + } + + wfn := func(path string, info hugofs.FileMetaInfo, err error) error { + if err != nil { + return err + } + + return nil + } + + w := hugofs.NewWalkway(hugofs.WalkwayConfig{ + Fs: c.fs, + Logger: c.logger, + Root: dirname, + Info: fi.(hugofs.FileMetaInfo), + HookPre: preHook, + HookPost: postHook, + WalkFn: wfn}) + + return w.Walk() + +} + +func (c *pagesCollector) isBundleHeader(fi hugofs.FileMetaInfo) bool { + class := fi.Meta().Classifier() + return class == files.ContentClassLeaf || class == files.ContentClassBranch +} + +func (c *pagesCollector) getLang(fi hugofs.FileMetaInfo) string { + lang := fi.Meta().Lang() + if lang != "" { + return lang + } + + return c.sp.DefaultContentLanguage +} + +func (c *pagesCollector) addToBundle(info hugofs.FileMetaInfo, btyp bundleDirType, bundles pageBundles) error { + getBundle := func(lang string) *fileinfoBundle { + return bundles[lang] + } + + cloneBundle := func(lang string) *fileinfoBundle { + // Every bundled file needs a content file header. + // Use the default content language if found, else just + // pick one. + var ( + source *fileinfoBundle + found bool + ) + + source, found = bundles[c.sp.DefaultContentLanguage] + if !found { + for _, b := range bundles { + source = b + break + } + } + + if source == nil { + panic(fmt.Sprintf("no source found, %d", len(bundles))) + } + + clone := c.cloneFileInfo(source.header) + clone.Meta()["lang"] = lang + + return &fileinfoBundle{ + header: clone, + } + } + + lang := c.getLang(info) + bundle := getBundle(lang) + isBundleHeader := c.isBundleHeader(info) + classifier := info.Meta().Classifier() + if bundle == nil { + if isBundleHeader { + bundle = &fileinfoBundle{header: info} + bundles[lang] = bundle + } else { + if btyp == bundleBranch { + // No special logic for branch bundles. + // Every language needs its own _index.md file. + return c.handleFiles(info) + } + + bundle = cloneBundle(lang) + bundles[lang] = bundle + } + } + + if !isBundleHeader { + bundle.resources = append(bundle.resources, info) + } + + if classifier == files.ContentClassFile { + translations := info.Meta().Translations() + if len(translations) < len(bundles) { + for lang, b := range bundles { + if !stringSliceContains(lang, translations...) && !b.containsResource(info.Name()) { + // Clone and add it to the bundle. + clone := c.cloneFileInfo(info) + clone.Meta()["lang"] = lang + b.resources = append(b.resources, clone) + } + } + } + } + + return nil +} + +func (c *pagesCollector) cloneFileInfo(fi hugofs.FileMetaInfo) hugofs.FileMetaInfo { + cm := hugofs.FileMeta{} + meta := fi.Meta() + if meta == nil { + panic(fmt.Sprintf("not meta: %v", fi.Name())) + } + for k, v := range meta { + cm[k] = v + } + + return hugofs.NewFileMetaInfo(fi, cm) +} + +func (c *pagesCollector) handleBundleBranch(readdir []hugofs.FileMetaInfo) error { + + // Maps bundles to its language. + bundles := pageBundles{} + + for _, fim := range readdir { + + if fim.IsDir() { + continue + } + + meta := fim.Meta() + + switch meta.Classifier() { + case files.ContentClassContent: + if err := c.handleFiles(fim); err != nil { + return err + } + default: + if err := c.addToBundle(fim, bundleBranch, bundles); err != nil { + return err + } + } + + } + + return c.proc.Process(bundles) + +} + +func (c *pagesCollector) handleBundleLeaf(dir hugofs.FileMetaInfo, path string, readdir []hugofs.FileMetaInfo) error { + // Maps bundles to its language. + bundles := pageBundles{} + + walk := func(path string, info hugofs.FileMetaInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + + return c.addToBundle(info, bundleLeaf, bundles) + + } + + // Start a new walker from the given path. + w := hugofs.NewWalkway(hugofs.WalkwayConfig{ + Root: path, + Fs: c.fs, + Logger: c.logger, + Info: dir, + DirEntries: readdir, + WalkFn: walk}) + + if err := w.Walk(); err != nil { + return err + } + + return c.proc.Process(bundles) + +} + +func (c *pagesCollector) handleFiles(fis ...hugofs.FileMetaInfo) error { + for _, fi := range fis { + if fi.IsDir() { + continue + } + + if err := c.proc.Process(fi); err != nil { + return err + } + } + return nil +} + +type pagesCollectorProcessorProvider interface { + Process(item interface{}) error + Start(ctx context.Context) context.Context + Wait() error +} + +type pagesProcessor struct { + h *HugoSites + sp *source.SourceSpec + + itemChan chan interface{} + itemGroup *errgroup.Group + + // The output Pages + pagesChan chan *pageState + pagesGroup *errgroup.Group + + numWorkers int + + partialBuild bool +} + +func (proc *pagesProcessor) Process(item interface{}) error { + proc.itemChan <- item + return nil +} + +func (proc *pagesProcessor) Start(ctx context.Context) context.Context { + proc.pagesChan = make(chan *pageState, proc.numWorkers) + proc.pagesGroup, ctx = errgroup.WithContext(ctx) + proc.itemChan = make(chan interface{}, proc.numWorkers) + proc.itemGroup, ctx = errgroup.WithContext(ctx) + + proc.pagesGroup.Go(func() error { + for p := range proc.pagesChan { + s := p.s + p.forceRender = proc.partialBuild + + if p.forceRender { + s.replacePage(p) + } else { + s.addPage(p) + } + } + return nil + }) + + for i := 0; i < proc.numWorkers; i++ { + proc.itemGroup.Go(func() error { + for item := range proc.itemChan { + select { + case <-proc.h.Done(): + return nil + default: + if err := proc.process(item); err != nil { + proc.h.SendError(err) + } + } + } + + return nil + }) + } + + return ctx +} + +func (proc *pagesProcessor) Wait() error { + close(proc.itemChan) + + err := proc.itemGroup.Wait() + + close(proc.pagesChan) + + if err != nil { + return err + } + + return proc.pagesGroup.Wait() +} + +func (proc *pagesProcessor) newPageFromBundle(b *fileinfoBundle) (*pageState, error) { + p, err := proc.newPageFromFi(b.header, nil) + if err != nil { + return nil, err + } + + if len(b.resources) > 0 { + + resources := make(resource.Resources, len(b.resources)) + + for i, rfi := range b.resources { + meta := rfi.Meta() + classifier := meta.Classifier() + var r resource.Resource + switch classifier { + case files.ContentClassContent: + rp, err := proc.newPageFromFi(rfi, p) + if err != nil { + return nil, err + } + rp.m.resourcePath = filepath.ToSlash(strings.TrimPrefix(rp.Path(), p.File().Dir())) + + r = rp + + case files.ContentClassFile: + r, err = proc.newResource(rfi, p) + if err != nil { + return nil, err + } + default: + panic(fmt.Sprintf("invalid classifier: %q", classifier)) + } + + resources[i] = r + + } + + p.addResources(resources...) + } + + return p, nil +} + +func (proc *pagesProcessor) newPageFromFi(fim hugofs.FileMetaInfo, owner *pageState) (*pageState, error) { + fi, err := newFileInfo(proc.sp, fim) + if err != nil { + return nil, err + } + + var s *Site + meta := fim.Meta() + + if owner != nil { + s = owner.s + } else { + lang := meta.Lang() + s = proc.getSite(lang) + } + + r := func() (hugio.ReadSeekCloser, error) { + return meta.Open() + } + + p, err := newPageWithContent(fi, s, owner != nil, r) + if err != nil { + return nil, err + } + p.parent = owner + return p, nil +} + +func (proc *pagesProcessor) newResource(fim hugofs.FileMetaInfo, owner *pageState) (resource.Resource, error) { + + // TODO(bep) consolidate with multihost logic + clean up + outputFormats := owner.m.outputFormats() + seen := make(map[string]bool) + var targetBasePaths []string + // Make sure bundled resources are published to all of the ouptput formats' + // sub paths. + for _, f := range outputFormats { + p := f.Path + if seen[p] { + continue + } + seen[p] = true + targetBasePaths = append(targetBasePaths, p) + + } + + meta := fim.Meta() + r := func() (hugio.ReadSeekCloser, error) { + return meta.Open() + } + + target := strings.TrimPrefix(meta.Path(), owner.File().Dir()) + + return owner.s.ResourceSpec.New( + resources.ResourceSourceDescriptor{ + TargetPaths: owner.getTargetPaths, + OpenReadSeekCloser: r, + FileInfo: fim, + RelTargetFilename: target, + TargetBasePaths: targetBasePaths, + }) +} + +func (proc *pagesProcessor) getSite(lang string) *Site { + if lang == "" { + return proc.h.Sites[0] + } + + for _, s := range proc.h.Sites { + if lang == s.Lang() { + return s + } + } + return proc.h.Sites[0] +} + +func (proc *pagesProcessor) copyFile(fim hugofs.FileMetaInfo) error { + meta := fim.Meta() + s := proc.getSite(meta.Lang()) + f, err := meta.Open() + if err != nil { + return errors.Wrap(err, "copyFile: failed to open") + } + + target := filepath.Join(s.PathSpec.GetTargetLanguageBasePath(), meta.Path()) + + defer f.Close() + + return s.publish(&s.PathSpec.ProcessingStats.Files, target, f) + +} + +func (proc *pagesProcessor) process(item interface{}) error { + send := func(p *pageState, err error) { + if err != nil { + proc.sendError(err) + } else { + proc.pagesChan <- p + } + } + + switch v := item.(type) { + // Page bundles mapped to their language. + case pageBundles: + for _, bundle := range v { + if proc.shouldSkip(bundle.header) { + continue + } + send(proc.newPageFromBundle(bundle)) + } + case hugofs.FileMetaInfo: + if proc.shouldSkip(v) { + return nil + } + meta := v.Meta() + + classifier := meta.Classifier() + switch classifier { + case files.ContentClassContent: + send(proc.newPageFromFi(v, nil)) + case files.ContentClassFile: + proc.sendError(proc.copyFile(v)) + default: + panic(fmt.Sprintf("invalid classifier: %q", classifier)) + } + default: + panic(fmt.Sprintf("unrecognized item type in Process: %T", item)) + } + + return nil +} + +func (proc *pagesProcessor) sendError(err error) { + if err == nil { + return + } + proc.h.SendError(err) +} + +func (proc *pagesProcessor) shouldSkip(fim hugofs.FileMetaInfo) bool { + return proc.sp.DisabledLanguages[fim.Meta().Lang()] +} + +func stringSliceContains(k string, values ...string) bool { + for _, v := range values { + if k == v { + return true + } + } + return false +} diff --git a/hugolib/pages_capture_test.go b/hugolib/pages_capture_test.go new file mode 100644 index 00000000000..f8a7833bea3 --- /dev/null +++ b/hugolib/pages_capture_test.go @@ -0,0 +1,88 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "context" + "fmt" + "path/filepath" + "testing" + + "github.com/pkg/errors" + + "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/source" + + "github.com/gohugoio/hugo/common/loggers" + + "github.com/gohugoio/hugo/hugofs" + "github.com/spf13/afero" + "github.com/stretchr/testify/require" +) + +func TestPagesCapture(t *testing.T) { + + cfg, hfs := newTestCfg() + fs := hfs.Source + + assert := require.New(t) + + var writeFile = func(filename string) { + assert.NoError(afero.WriteFile(fs, filepath.FromSlash(filename), []byte(fmt.Sprintf("content-%s", filename)), 0755)) + } + + writeFile("_index.md") + writeFile("logo.png") + writeFile("root.md") + writeFile("blog/index.md") + writeFile("blog/hello.md") + writeFile("blog/images/sunset.png") + writeFile("pages/page1.md") + writeFile("pages/page2.md") + writeFile("pages/page.png") + + ps, err := helpers.NewPathSpec(hugofs.NewFrom(fs, cfg), cfg) + assert.NoError(err) + sourceSpec := source.NewSourceSpec(ps, fs) + + t.Run("Collect", func(t *testing.T) { + assert := require.New(t) + proc := &testPagesCollectorProcessor{} + c := newPagesCollector(sourceSpec, loggers.NewErrorLogger(), nil, proc) + assert.NoError(c.Collect()) + assert.Equal(4, len(proc.items)) + }) + + t.Run("error in Wait", func(t *testing.T) { + assert := require.New(t) + c := newPagesCollector(sourceSpec, loggers.NewErrorLogger(), nil, + &testPagesCollectorProcessor{waitErr: errors.New("failed")}) + assert.Error(c.Collect()) + }) +} + +type testPagesCollectorProcessor struct { + items []interface{} + waitErr error +} + +func (proc *testPagesCollectorProcessor) Process(item interface{}) error { + proc.items = append(proc.items, item) + return nil +} +func (proc *testPagesCollectorProcessor) Start(ctx context.Context) context.Context { + return ctx +} + +func (proc *testPagesCollectorProcessor) Wait() error { return proc.waitErr } diff --git a/hugolib/paths/paths.go b/hugolib/paths/paths.go index df66e2a46e2..97d4f17bab7 100644 --- a/hugolib/paths/paths.go +++ b/hugolib/paths/paths.go @@ -20,6 +20,7 @@ import ( "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/langs" + "github.com/gohugoio/hugo/modules" "github.com/pkg/errors" "github.com/gohugoio/hugo/hugofs" @@ -39,7 +40,6 @@ type Paths struct { // Directories // TODO(bep) when we have trimmed down mos of the dirs usage outside of this package, make // these into an interface. - ContentDir string ThemesDir string WorkingDir string @@ -62,8 +62,9 @@ type Paths struct { UglyURLs bool CanonifyURLs bool - Language *langs.Language - Languages langs.Languages + Language *langs.Language + Languages langs.Languages + LanguagesDefaultFirst langs.Languages // The PathSpec looks up its config settings in both the current language // and then in the global Viper config. @@ -74,8 +75,8 @@ type Paths struct { DefaultContentLanguage string multilingual bool - themes []string - AllThemes []ThemeConfig + AllModules modules.Modules + ModulesClient *modules.Client } func New(fs *hugofs.Fs, cfg config.Provider) (*Paths, error) { @@ -91,12 +92,6 @@ func New(fs *hugofs.Fs, cfg config.Provider) (*Paths, error) { resourceDir := filepath.Clean(cfg.GetString("resourceDir")) publishDir := filepath.Clean(cfg.GetString("publishDir")) - if contentDir == "" { - return nil, fmt.Errorf("contentDir not set") - } - if resourceDir == "" { - return nil, fmt.Errorf("resourceDir not set") - } if publishDir == "" { return nil, fmt.Errorf("publishDir not set") } @@ -104,8 +99,9 @@ func New(fs *hugofs.Fs, cfg config.Provider) (*Paths, error) { defaultContentLanguage := cfg.GetString("defaultContentLanguage") var ( - language *langs.Language - languages langs.Languages + language *langs.Language + languages langs.Languages + languagesDefaultFirst langs.Languages ) if l, ok := cfg.(*langs.Language); ok { @@ -117,6 +113,12 @@ func New(fs *hugofs.Fs, cfg config.Provider) (*Paths, error) { languages = l } + if l, ok := cfg.Get("languagesSortedDefaultFirst").(langs.Languages); ok { + languagesDefaultFirst = l + } + + // + if len(languages) == 0 { // We have some old tests that does not test the entire chain, hence // they have no languages. So create one so we get the proper filesystem. @@ -156,33 +158,30 @@ func New(fs *hugofs.Fs, cfg config.Provider) (*Paths, error) { UglyURLs: cfg.GetBool("uglyURLs"), CanonifyURLs: cfg.GetBool("canonifyURLs"), - ContentDir: contentDir, ThemesDir: cfg.GetString("themesDir"), WorkingDir: workingDir, AbsResourcesDir: absResourcesDir, AbsPublishDir: absPublishDir, - themes: config.GetStringSlicePreserveString(cfg, "theme"), - multilingual: cfg.GetBool("multilingual"), defaultContentLanguageInSubdir: cfg.GetBool("defaultContentLanguageInSubdir"), DefaultContentLanguage: defaultContentLanguage, Language: language, Languages: languages, + LanguagesDefaultFirst: languagesDefaultFirst, MultihostTargetBasePaths: multihostTargetBasePaths, PaginatePath: cfg.GetString("paginatePath"), } - if !cfg.IsSet("theme") && cfg.IsSet("allThemes") { - p.AllThemes = cfg.Get("allThemes").([]ThemeConfig) - } else { - p.AllThemes, err = collectThemeNames(p) - if err != nil { - return nil, err - } + if cfg.IsSet("allModules") { + p.AllModules = cfg.Get("allModules").(modules.Modules) + } + + if cfg.IsSet("modulesClient") { + p.ModulesClient = cfg.Get("modulesClient").(*modules.Client) } // TODO(bep) remove this, eventually @@ -207,15 +206,6 @@ func (p *Paths) Lang() string { return p.Language.Lang } -// ThemeSet checks whether a theme is in use or not. -func (p *Paths) ThemeSet() bool { - return len(p.themes) > 0 -} - -func (p *Paths) Themes() []string { - return p.themes -} - func (p *Paths) GetTargetLanguageBasePath() string { if p.Languages.IsMultihost() { // In a multihost configuration all assets will be published below the language code. @@ -269,6 +259,18 @@ func (p *Paths) AbsPathify(inPath string) string { return AbsPathify(p.WorkingDir, inPath) } +// RelPathify trims any WorkingDir prefix from the given filename. If +// the filename is not considered to be absolute, the path is just cleaned. +func (p *Paths) RelPathify(filename string) string { + filename = filepath.Clean(filename) + if !filepath.IsAbs(filename) { + return filename + } + + return strings.TrimPrefix(strings.TrimPrefix(filename, p.WorkingDir), FilePathSeparator) + +} + // AbsPathify creates an absolute path if given a working dir and arelative path. // If already absolute, the path is just cleaned. func AbsPathify(workingDir, inPath string) string { diff --git a/hugolib/paths/paths_test.go b/hugolib/paths/paths_test.go index 3bd445b8bc6..95a8be782ef 100644 --- a/hugolib/paths/paths_test.go +++ b/hugolib/paths/paths_test.go @@ -16,6 +16,8 @@ package paths import ( "testing" + "github.com/gohugoio/hugo/langs" + "github.com/gohugoio/hugo/hugofs" "github.com/spf13/viper" "github.com/stretchr/testify/require" @@ -27,14 +29,19 @@ func TestNewPaths(t *testing.T) { v := viper.New() fs := hugofs.NewMem(v) + v.Set("languages", map[string]interface{}{ + "no": map[string]interface{}{}, + "en": map[string]interface{}{}, + }) v.Set("defaultContentLanguageInSubdir", true) v.Set("defaultContentLanguage", "no") - v.Set("multilingual", true) v.Set("contentDir", "content") v.Set("workingDir", "work") v.Set("resourceDir", "resources") v.Set("publishDir", "public") + langs.LoadLanguageSettings(v, nil) + p, err := New(fs, v) assert.NoError(err) diff --git a/hugolib/paths/themes.go b/hugolib/paths/themes.go deleted file mode 100644 index a526953f16e..00000000000 --- a/hugolib/paths/themes.go +++ /dev/null @@ -1,154 +0,0 @@ -// Copyright 2019 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package paths - -import ( - "path/filepath" - "strings" - - "github.com/gohugoio/hugo/config" - "github.com/spf13/afero" - "github.com/spf13/cast" -) - -type ThemeConfig struct { - // The theme name as provided by the folder name below /themes. - Name string - - // Optional configuration filename (e.g. "/themes/mytheme/config.json"). - ConfigFilename string - - // Optional config read from the ConfigFile above. - Cfg config.Provider -} - -// Create file system, an ordered theme list from left to right, no duplicates. -type themesCollector struct { - themesDir string - fs afero.Fs - seen map[string]bool - themes []ThemeConfig -} - -func (c *themesCollector) isSeen(theme string) bool { - loki := strings.ToLower(theme) - if c.seen[loki] { - return true - } - c.seen[loki] = true - return false -} - -func (c *themesCollector) addAndRecurse(themes ...string) error { - for i := 0; i < len(themes); i++ { - theme := themes[i] - configFilename := c.getConfigFileIfProvided(theme) - if !c.isSeen(theme) { - tc, err := c.add(theme, configFilename) - if err != nil { - return err - } - if err := c.addThemeNamesFromTheme(tc); err != nil { - return err - } - } - } - return nil -} - -func (c *themesCollector) add(name, configFilename string) (ThemeConfig, error) { - var cfg config.Provider - var tc ThemeConfig - - if configFilename != "" { - var err error - cfg, err = config.FromFile(c.fs, configFilename) - if err != nil { - return tc, err - } - } - - tc = ThemeConfig{Name: name, ConfigFilename: configFilename, Cfg: cfg} - c.themes = append(c.themes, tc) - return tc, nil - -} - -func collectThemeNames(p *Paths) ([]ThemeConfig, error) { - return CollectThemes(p.Fs.Source, p.AbsPathify(p.ThemesDir), p.Themes()) - -} - -func CollectThemes(fs afero.Fs, themesDir string, themes []string) ([]ThemeConfig, error) { - if len(themes) == 0 { - return nil, nil - } - - c := &themesCollector{ - fs: fs, - themesDir: themesDir, - seen: make(map[string]bool)} - - for i := 0; i < len(themes); i++ { - theme := themes[i] - if err := c.addAndRecurse(theme); err != nil { - return nil, err - } - } - - return c.themes, nil - -} - -func (c *themesCollector) getConfigFileIfProvided(theme string) string { - configDir := filepath.Join(c.themesDir, theme) - - var ( - configFilename string - exists bool - ) - - // Viper supports more, but this is the sub-set supported by Hugo. - for _, configFormats := range config.ValidConfigFileExtensions { - configFilename = filepath.Join(configDir, "config."+configFormats) - exists, _ = afero.Exists(c.fs, configFilename) - if exists { - break - } - } - - if !exists { - // No theme config set. - return "" - } - - return configFilename - -} - -func (c *themesCollector) addThemeNamesFromTheme(theme ThemeConfig) error { - if theme.Cfg != nil && theme.Cfg.IsSet("theme") { - v := theme.Cfg.Get("theme") - switch vv := v.(type) { - case []string: - return c.addAndRecurse(vv...) - case []interface{}: - return c.addAndRecurse(cast.ToStringSlice(vv)...) - default: - return c.addAndRecurse(cast.ToString(vv)) - } - } - - return nil -} diff --git a/hugolib/resource_chain_test.go b/hugolib/resource_chain_test.go index 8f55e112e68..ea95a3f09a6 100644 --- a/hugolib/resource_chain_test.go +++ b/hugolib/resource_chain_test.go @@ -18,6 +18,8 @@ import ( "path/filepath" "testing" + "github.com/gohugoio/hugo/htesting" + "github.com/spf13/viper" "github.com/stretchr/testify/require" @@ -33,17 +35,17 @@ func TestSCSSWithIncludePaths(t *testing.T) { t.Skip("Skip SCSS") } assert := require.New(t) - workDir, clean, err := createTempDir("hugo-scss-include") + workDir, clean, err := htesting.CreateTempDir(hugofs.Os, "hugo-scss-include") assert.NoError(err) defer clean() v := viper.New() v.Set("workingDir", workDir) b := newTestSitesBuilder(t).WithLogger(loggers.NewErrorLogger()) - b.WithViper(v) - b.WithWorkingDir(workDir) // Need to use OS fs for this. b.Fs = hugofs.NewDefault(v) + b.WithWorkingDir(workDir) + b.WithViper(v) fooDir := filepath.Join(workDir, "node_modules", "foo") scssDir := filepath.Join(workDir, "assets", "scss") @@ -84,7 +86,7 @@ func TestSCSSWithThemeOverrides(t *testing.T) { t.Skip("Skip SCSS") } assert := require.New(t) - workDir, clean, err := createTempDir("hugo-scss-include") + workDir, clean, err := htesting.CreateTempDir(hugofs.Os, "hugo-scss-include") assert.NoError(err) defer clean() @@ -95,10 +97,10 @@ func TestSCSSWithThemeOverrides(t *testing.T) { v.Set("workingDir", workDir) v.Set("theme", theme) b := newTestSitesBuilder(t).WithLogger(loggers.NewErrorLogger()) - b.WithViper(v) - b.WithWorkingDir(workDir) // Need to use OS fs for this. b.Fs = hugofs.NewDefault(v) + b.WithWorkingDir(workDir) + b.WithViper(v) fooDir := filepath.Join(workDir, "node_modules", "foo") scssDir := filepath.Join(workDir, "assets", "scss") @@ -385,14 +387,15 @@ CSV2: {{ $csv2 }} } for _, test := range tests { - if !test.shouldRun() { - t.Log("Skip", test.name) - continue - } - - b := newTestSitesBuilder(t).WithLogger(loggers.NewErrorLogger()) - b.WithSimpleConfigFile() - b.WithContent("_index.md", ` + test := test + t.Run(test.name, func(t *testing.T) { + if !test.shouldRun() { + t.Skip() + } + t.Parallel() + + b := newTestSitesBuilder(t).WithLogger(loggers.NewErrorLogger()) + b.WithContent("_index.md", ` --- title: Home --- @@ -400,37 +403,37 @@ title: Home Home. `, - "page1.md", ` + "page1.md", ` --- title: Hello1 --- Hello1 `, - "page2.md", ` + "page2.md", ` --- title: Hello2 --- Hello2 `, - "t1.txt", "t1t|", - "t2.txt", "t2t|", - ) + "t1.txt", "t1t|", + "t2.txt", "t2t|", + ) - b.WithSourceFile(filepath.Join("assets", "css", "styles1.css"), ` + b.WithSourceFile(filepath.Join("assets", "css", "styles1.css"), ` h1 { font-style: bold; } `) - b.WithSourceFile(filepath.Join("assets", "js", "script1.js"), ` + b.WithSourceFile(filepath.Join("assets", "js", "script1.js"), ` var x; x = 5; document.getElementById("demo").innerHTML = x * 10; `) - b.WithSourceFile(filepath.Join("assets", "mydata", "json1.json"), ` + b.WithSourceFile(filepath.Join("assets", "mydata", "json1.json"), ` { "employees":[ {"firstName":"John", "lastName":"Doe"}, @@ -440,19 +443,19 @@ document.getElementById("demo").innerHTML = x * 10; } `) - b.WithSourceFile(filepath.Join("assets", "mydata", "svg1.svg"), ` + b.WithSourceFile(filepath.Join("assets", "mydata", "svg1.svg"), ` `) - b.WithSourceFile(filepath.Join("assets", "mydata", "xml1.xml"), ` + b.WithSourceFile(filepath.Join("assets", "mydata", "xml1.xml"), ` Hugo Rocks! `) - b.WithSourceFile(filepath.Join("assets", "mydata", "html1.html"), ` + b.WithSourceFile(filepath.Join("assets", "mydata", "html1.html"), ` Cool @@ -460,7 +463,7 @@ Cool `) - b.WithSourceFile(filepath.Join("assets", "scss", "styles2.scss"), ` + b.WithSourceFile(filepath.Join("assets", "scss", "styles2.scss"), ` $color: #333; body { @@ -468,7 +471,7 @@ body { } `) - b.WithSourceFile(filepath.Join("assets", "sass", "styles3.sass"), ` + b.WithSourceFile(filepath.Join("assets", "sass", "styles3.sass"), ` $color: #333; .content-navigation @@ -476,10 +479,11 @@ $color: #333; `) - t.Log("Test", test.name) - test.prepare(b) - b.Build(BuildCfg{}) - test.verify(b) + test.prepare(b) + b.Build(BuildCfg{}) + test.verify(b) + + }) } } diff --git a/hugolib/shortcode_test.go b/hugolib/shortcode_test.go index 42eef61ae0a..24ef77503cb 100644 --- a/hugolib/shortcode_test.go +++ b/hugolib/shortcode_test.go @@ -26,10 +26,6 @@ import ( "strings" "testing" - "github.com/spf13/viper" - - "github.com/spf13/afero" - "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/tpl" @@ -54,12 +50,8 @@ title: "Title" writeSource(t, fs, "content/simple.md", contentFile) - h, err := NewHugoSites(deps.DepsCfg{Fs: fs, Cfg: cfg, WithTemplate: withTemplate}) - - require.NoError(t, err) - require.Len(t, h.Sites, 1) - - err = h.Build(BuildCfg{}) + b := newTestSitesBuilderFromDepsCfg(t, deps.DepsCfg{Fs: fs, Cfg: cfg, WithTemplate: withTemplate}).WithNothingAdded() + err := b.BuildE(BuildCfg{}) if err != nil && !expectError { t.Fatalf("Shortcode rendered error %s.", err) @@ -69,6 +61,9 @@ title: "Title" t.Fatalf("No error from shortcode") } + h := b.H + require.Len(t, h.Sites, 1) + require.Len(t, h.Sites[0].RegularPages(), 1) output := strings.TrimSpace(content(h.Sites[0].RegularPages()[0])) @@ -78,7 +73,7 @@ title: "Title" expected = strings.TrimSpace(expected) if output != expected { - Fatalf(t, "Shortcode render didn't match. got \n%q but expected \n%q", output, expected) + t.Fatalf("Shortcode render didn't match. got \n%q but expected \n%q", output, expected) } } @@ -341,7 +336,6 @@ func TestShortcodeWrappedInPIssue(t *testing.T) { } func TestExtractShortcodes(t *testing.T) { - t.Parallel() b := newTestSitesBuilder(t).WithSimpleConfigFile() b.WithTemplates( @@ -413,7 +407,10 @@ title: "Shortcodes Galore!" {"inline", `{{< my.inline >}}Hi{{< /my.inline >}}`, regexpCheck("my.inline;inline:true;closing:true;inner:{Hi};")}, } { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() assert := require.New(t) counter := 0 @@ -437,7 +434,6 @@ title: "Shortcodes Galore!" } func TestShortcodesInSite(t *testing.T) { - t.Parallel() baseURL := "http://foo/bar" tests := []struct { @@ -577,7 +573,9 @@ title: "Foo" s := buildSingleSite(t, deps.DepsCfg{WithTemplate: addTemplates, Fs: fs, Cfg: cfg}, BuildCfg{}) for i, test := range tests { + test := test t.Run(fmt.Sprintf("test=%d;contentPath=%s", i, test.contentPath), func(t *testing.T) { + t.Parallel() if strings.HasSuffix(test.contentPath, ".ad") && !helpers.HasAsciidoc() { t.Skip("Skip Asciidoc test case as no Asciidoc present.") } else if strings.HasSuffix(test.contentPath, ".rst") && !helpers.HasRst() { @@ -632,9 +630,8 @@ outputs: ["CSV"] CSV: {{< myShort >}} ` - mf := afero.NewMemMapFs() - - th, h := newTestSitesFromConfig(t, mf, siteConfig, + b := newTestSitesBuilder(t).WithConfigFile("toml", siteConfig) + b.WithTemplates( "layouts/_default/single.html", `Single HTML: {{ .Title }}|{{ .Content }}`, "layouts/_default/single.json", `Single JSON: {{ .Title }}|{{ .Content }}`, "layouts/_default/single.csv", `Single CSV: {{ .Title }}|{{ .Content }}`, @@ -651,14 +648,13 @@ CSV: {{< myShort >}} "layouts/shortcodes/myInner.html", `myInner:--{{- .Inner -}}--`, ) - fs := th.Fs - - writeSource(t, fs, "content/_index.md", fmt.Sprintf(pageTemplate, "Home")) - writeSource(t, fs, "content/sect/mypage.md", fmt.Sprintf(pageTemplate, "Single")) - writeSource(t, fs, "content/sect/mycsvpage.md", fmt.Sprintf(pageTemplateCSVOnly, "Single CSV")) + b.WithContent("_index.md", fmt.Sprintf(pageTemplate, "Home"), + "sect/mypage.md", fmt.Sprintf(pageTemplate, "Single"), + "sect/mycsvpage.md", fmt.Sprintf(pageTemplateCSVOnly, "Single CSV"), + ) - err := h.Build(BuildCfg{}) - require.NoError(t, err) + b.Build(BuildCfg{}) + h := b.H require.Len(t, h.Sites, 1) s := h.Sites[0] @@ -666,7 +662,7 @@ CSV: {{< myShort >}} require.NotNil(t, home) require.Len(t, home.OutputFormats(), 3) - th.assertFileContent("public/index.html", + b.AssertFileContent("public/index.html", "Home HTML", "ShortHTML", "ShortNoExt", @@ -674,7 +670,7 @@ CSV: {{< myShort >}} "myInner:--ShortHTML--", ) - th.assertFileContent("public/amp/index.html", + b.AssertFileContent("public/amp/index.html", "Home AMP", "ShortAMP", "ShortNoExt", @@ -682,7 +678,7 @@ CSV: {{< myShort >}} "myInner:--ShortAMP--", ) - th.assertFileContent("public/index.ics", + b.AssertFileContent("public/index.ics", "Home Calendar", "ShortCalendar", "ShortNoExt", @@ -690,7 +686,7 @@ CSV: {{< myShort >}} "myInner:--ShortCalendar--", ) - th.assertFileContent("public/sect/mypage/index.html", + b.AssertFileContent("public/sect/mypage/index.html", "Single HTML", "ShortHTML", "ShortNoExt", @@ -698,7 +694,7 @@ CSV: {{< myShort >}} "myInner:--ShortHTML--", ) - th.assertFileContent("public/sect/mypage/index.json", + b.AssertFileContent("public/sect/mypage/index.json", "Single JSON", "ShortJSON", "ShortNoExt", @@ -706,7 +702,7 @@ CSV: {{< myShort >}} "myInner:--ShortJSON--", ) - th.assertFileContent("public/amp/sect/mypage/index.html", + b.AssertFileContent("public/amp/sect/mypage/index.html", // No special AMP template "Single HTML", "ShortAMP", @@ -715,7 +711,7 @@ CSV: {{< myShort >}} "myInner:--ShortAMP--", ) - th.assertFileContent("public/sect/mycsvpage/index.csv", + b.AssertFileContent("public/sect/mycsvpage/index.csv", "Single CSV", "ShortCSV", ) @@ -864,10 +860,6 @@ weight: %d --- C-%s` - v := viper.New() - - v.Set("timeout", 500) - templates = append(templates, []string{"shortcodes/c.html", contentShortcode}...) templates = append(templates, []string{"_default/single.html", "Single Content: {{ .Content }}"}...) templates = append(templates, []string{"_default/list.html", "List Content: {{ .Content }}"}...) @@ -884,21 +876,21 @@ C-%s` builder := newTestSitesBuilder(t).WithDefaultMultiSiteConfig() - builder.WithViper(v).WithContent(content...).WithTemplates(templates...).CreateSites().Build(BuildCfg{}) + builder.WithContent(content...).WithTemplates(templates...).CreateSites().Build(BuildCfg{}) s := builder.H.Sites[0] assert.Equal(3, len(s.RegularPages())) - builder.AssertFileContent("public/section1/index.html", + builder.AssertFileContent("public/en/section1/index.html", "List Content:

Logo:P1:|P2:logo.png/PNG logo|:P1: P1:|P2:docs1p1/

C-s1p1

\n|", "BP1:P1:|P2:docbp1/

C-bp1

", ) - builder.AssertFileContent("public/b1/index.html", + builder.AssertFileContent("public/en/b1/index.html", "Single Content:

Logo:P1:|P2:logo.png/PNG logo|:P1: P1:|P2:docs1p1/

C-s1p1

\n|", "P2:docbp1/

C-bp1

", ) - builder.AssertFileContent("public/section2/s2p1/index.html", + builder.AssertFileContent("public/en/section2/s2p1/index.html", "Single Content:

Logo:P1:|P2:logo.png/PNG logo|:P1: P1:|P2:docs1p1/

C-s1p1

\n|", "P2:docbp1/

C-bp1

", ) @@ -1062,8 +1054,10 @@ String: {{ . | safeHTML }} func TestInlineShortcodes(t *testing.T) { for _, enableInlineShortcodes := range []bool{true, false} { + enableInlineShortcodes := enableInlineShortcodes t.Run(fmt.Sprintf("enableInlineShortcodes=%t", enableInlineShortcodes), func(t *testing.T) { + t.Parallel() conf := fmt.Sprintf(` baseURL = "https://example.com" enableInlineShortcodes = %t diff --git a/hugolib/site.go b/hugolib/site.go index b1441ca8a13..882874db947 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -14,7 +14,6 @@ package hugolib import ( - "context" "fmt" "html/template" "io" @@ -29,6 +28,8 @@ import ( "strings" "time" + "github.com/gohugoio/hugo/hugofs/files" + "github.com/gohugoio/hugo/common/maps" "github.com/pkg/errors" @@ -45,7 +46,6 @@ import ( "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/lazy" - "golang.org/x/sync/errgroup" "github.com/gohugoio/hugo/media" @@ -1028,7 +1028,8 @@ func (s *Site) processPartial(events []fsnotify.Event) (whatChanged, error) { removed = true } } - if removed && IsContentFile(ev.Name) { + + if removed && files.IsContentFile(ev.Name) { h.removePageByFilename(ev.Name) } @@ -1058,7 +1059,7 @@ func (s *Site) processPartial(events []fsnotify.Event) (whatChanged, error) { filenamesChanged = append(filenamesChanged, contentFilesChanged...) } - filenamesChanged = helpers.UniqueStrings(filenamesChanged) + filenamesChanged = helpers.UniqueStringsReuse(filenamesChanged) if err := s.readAndProcessContent(filenamesChanged...); err != nil { return whatChanged{}, err @@ -1078,10 +1079,12 @@ func (s *Site) processPartial(events []fsnotify.Event) (whatChanged, error) { func (s *Site) process(config BuildCfg) (err error) { if err = s.initialize(); err != nil { + err = errors.Wrap(err, "initialize") return } - if err := s.readAndProcessContent(); err != nil { - return err + if err = s.readAndProcessContent(); err != nil { + err = errors.Wrap(err, "readAndProcessContent") + return } return err @@ -1304,93 +1307,14 @@ func (s *Site) isContentDirEvent(e fsnotify.Event) bool { return s.BaseFs.IsContent(e.Name) } -type contentCaptureResultHandler struct { - defaultContentProcessor *siteContentProcessor - contentProcessors map[string]*siteContentProcessor -} - -func (c *contentCaptureResultHandler) getContentProcessor(lang string) *siteContentProcessor { - proc, found := c.contentProcessors[lang] - if found { - return proc - } - return c.defaultContentProcessor -} - -func (c *contentCaptureResultHandler) handleSingles(fis ...*fileInfo) { - for _, fi := range fis { - proc := c.getContentProcessor(fi.Lang()) - proc.processSingle(fi) - } -} -func (c *contentCaptureResultHandler) handleBundles(d *bundleDirs) { - for _, b := range d.bundles { - proc := c.getContentProcessor(b.fi.Lang()) - proc.processBundle(b) - } -} - -func (c *contentCaptureResultHandler) handleCopyFile(f pathLangFile) { - proc := c.getContentProcessor(f.Lang()) - proc.processAsset(f) -} - func (s *Site) readAndProcessContent(filenames ...string) error { - - ctx := context.Background() - g, ctx := errgroup.WithContext(ctx) - - defaultContentLanguage := s.SourceSpec.DefaultContentLanguage - - contentProcessors := make(map[string]*siteContentProcessor) - var defaultContentProcessor *siteContentProcessor - sites := s.h.langSite() - for k, v := range sites { - if v.language.Disabled { - continue - } - proc := newSiteContentProcessor(ctx, len(filenames) > 0, v) - contentProcessors[k] = proc - if k == defaultContentLanguage { - defaultContentProcessor = proc - } - g.Go(func() error { - return proc.process(ctx) - }) - } - - var ( - handler captureResultHandler - bundleMap *contentChangeMap - ) - - mainHandler := &contentCaptureResultHandler{contentProcessors: contentProcessors, defaultContentProcessor: defaultContentProcessor} - sourceSpec := source.NewSourceSpec(s.PathSpec, s.BaseFs.Content.Fs) - if s.running() { - // Need to track changes. - bundleMap = s.h.ContentChanges - handler = &captureResultHandlerChain{handlers: []captureBundlesHandler{mainHandler, bundleMap}} - - } else { - handler = mainHandler - } - - c := newCapturer(s.Log, sourceSpec, handler, bundleMap, filenames...) + proc := newPagesProcessor(s.h, sourceSpec, len(filenames) > 0) - err1 := c.capture() + c := newPagesCollector(sourceSpec, s.Log, s.h.ContentChanges, proc, filenames...) - for _, proc := range contentProcessors { - proc.closeInput() - } - - err2 := g.Wait() - - if err1 != nil { - return err1 - } - return err2 + return c.Collect() } func (s *Site) getMenusFromConfig() navigation.Menus { @@ -1831,8 +1755,8 @@ func (s *Site) kindFromFileInfoOrSections(fi *fileInfo, sections []string) strin } func (s *Site) kindFromSections(sections []string) string { - if len(sections) == 0 || len(s.siteCfg.taxonomiesConfig) == 0 { - return page.KindSection + if len(sections) == 0 { + return page.KindHome } sectionPath := path.Join(sections...) diff --git a/hugolib/site_output_test.go b/hugolib/site_output_test.go index 71b87b63692..1536817b091 100644 --- a/hugolib/site_output_test.go +++ b/hugolib/site_output_test.go @@ -32,14 +32,15 @@ import ( func TestSiteWithPageOutputs(t *testing.T) { for _, outputs := range [][]string{{"html", "json", "calendar"}, {"json"}} { + outputs := outputs t.Run(fmt.Sprintf("%v", outputs), func(t *testing.T) { + t.Parallel() doTestSiteWithPageOutputs(t, outputs) }) } } func doTestSiteWithPageOutputs(t *testing.T, outputs []string) { - t.Parallel() outputsStr := strings.Replace(fmt.Sprintf("%q", outputs), " ", ", ", -1) @@ -84,19 +85,16 @@ outputs: %s ` - mf := afero.NewMemMapFs() - - writeToFs(t, mf, "i18n/en.toml", ` + b := newTestSitesBuilder(t).WithConfigFile("toml", siteConfig) + b.WithI18n("en.toml", ` [elbow] other = "Elbow" -`) - writeToFs(t, mf, "i18n/nn.toml", ` +`, "nn.toml", ` [elbow] other = "Olboge" `) - th, h := newTestSitesFromConfig(t, mf, siteConfig, - + b.WithTemplates( // Case issue partials #3333 "layouts/partials/GoHugo.html", `Go Hugo Partial`, "layouts/_default/baseof.json", `START JSON:{{block "main" .}}default content{{ end }}:END JSON`, @@ -133,23 +131,17 @@ Len Pages: {{ .Kind }} {{ len .Site.RegularPages }} Page Number: {{ .Paginator.P `, "layouts/_default/single.html", `{{ define "main" }}{{ .Content }}{{ end }}`, ) - require.Len(t, h.Sites, 2) - - fs := th.Fs - writeSource(t, fs, "content/_index.md", fmt.Sprintf(pageTemplate, "JSON Home", outputsStr)) - writeSource(t, fs, "content/_index.nn.md", fmt.Sprintf(pageTemplate, "JSON Nynorsk Heim", outputsStr)) + b.WithContent("_index.md", fmt.Sprintf(pageTemplate, "JSON Home", outputsStr)) + b.WithContent("_index.nn.md", fmt.Sprintf(pageTemplate, "JSON Nynorsk Heim", outputsStr)) for i := 1; i <= 10; i++ { - writeSource(t, fs, fmt.Sprintf("content/p%d.md", i), fmt.Sprintf(pageTemplate, fmt.Sprintf("Page %d", i), outputsStr)) - + b.WithContent(fmt.Sprintf("p%d.md", i), fmt.Sprintf(pageTemplate, fmt.Sprintf("Page %d", i), outputsStr)) } - err := h.Build(BuildCfg{}) - - require.NoError(t, err) + b.Build(BuildCfg{}) - s := h.Sites[0] + s := b.H.Sites[0] require.Equal(t, "en", s.language.Lang) home := s.getPage(page.KindHome) @@ -163,13 +155,13 @@ Len Pages: {{ .Kind }} {{ len .Site.RegularPages }} Page Number: {{ .Paginator.P // There is currently always a JSON output to make it simpler ... altFormats := lenOut - 1 hasHTML := helpers.InStringArray(outputs, "html") - th.assertFileContent("public/index.json", + b.AssertFileContent("public/index.json", "List JSON", fmt.Sprintf("Alt formats: %d", altFormats), ) if hasHTML { - th.assertFileContent("public/index.json", + b.AssertFileContent("public/index.json", "Alt Output: HTML", "Output/Rel: JSON/alternate|", "Output/Rel: HTML/canonical|", @@ -178,7 +170,7 @@ Len Pages: {{ .Kind }} {{ len .Site.RegularPages }} Page Number: {{ .Paginator.P "OtherShort:

Hi!

", ) - th.assertFileContent("public/index.html", + b.AssertFileContent("public/index.html", // The HTML entity is a deliberate part of this test: The HTML templates are // parsed with html/template. `List HTML|JSON Home|`, @@ -187,21 +179,22 @@ Len Pages: {{ .Kind }} {{ len .Site.RegularPages }} Page Number: {{ .Paginator.P "OtherShort:

Hi!

", "Len Pages: home 10", ) - th.assertFileContent("public/page/2/index.html", "Page Number: 2") - th.assertFileNotExist("public/page/2/index.json") + assert := require.New(t) + b.AssertFileContent("public/page/2/index.html", "Page Number: 2") + assert.False(b.CheckExists("public/page/2/index.json")) - th.assertFileContent("public/nn/index.html", + b.AssertFileContent("public/nn/index.html", "List HTML|JSON Nynorsk Heim|", "nn: Olboge") } else { - th.assertFileContent("public/index.json", + b.AssertFileContent("public/index.json", "Output/Rel: JSON/canonical|", // JSON is plain text, so no need to safeHTML this and that ``, "ShortJSON", "OtherShort:

Hi!

", ) - th.assertFileContent("public/nn/index.json", + b.AssertFileContent("public/nn/index.json", "List JSON|JSON Nynorsk Heim|", "nn: Olboge", "ShortJSON", diff --git a/hugolib/site_sections_test.go b/hugolib/site_sections_test.go index 199947c3131..d4aa9d354b9 100644 --- a/hugolib/site_sections_test.go +++ b/hugolib/site_sections_test.go @@ -25,7 +25,6 @@ import ( ) func TestNestedSections(t *testing.T) { - t.Parallel() var ( assert = require.New(t) @@ -294,7 +293,9 @@ PAG|{{ .Title }}|{{ $sect.InSection . }} home := s.getPage(page.KindHome) for _, test := range tests { + test := test t.Run(fmt.Sprintf("sections %s", test.sections), func(t *testing.T) { + t.Parallel() assert := require.New(t) sections := strings.Split(test.sections, ",") p := s.getPage(page.KindSection, sections...) diff --git a/hugolib/site_stats_test.go b/hugolib/site_stats_test.go index c722037b4eb..bbefc95774f 100644 --- a/hugolib/site_stats_test.go +++ b/hugolib/site_stats_test.go @@ -20,7 +20,6 @@ import ( "testing" "github.com/gohugoio/hugo/helpers" - "github.com/spf13/afero" "github.com/stretchr/testify/require" ) @@ -60,30 +59,28 @@ aliases: [/Ali%d] # Doc ` - th, h := newTestSitesFromConfig(t, afero.NewMemMapFs(), siteConfig, - "layouts/_default/single.html", "Single|{{ .Title }}|{{ .Content }}", - "layouts/_default/list.html", `List|{{ .Title }}|Pages: {{ .Paginator.TotalPages }}|{{ .Content }}`, - "layouts/_default/terms.html", "Terms List|{{ .Title }}|{{ .Content }}", - ) - require.Len(t, h.Sites, 2) + b := newTestSitesBuilder(t).WithConfigFile("toml", siteConfig) - fs := th.Fs + b.WithTemplates( + "_default/single.html", "Single|{{ .Title }}|{{ .Content }}", + "_default/list.html", `List|{{ .Title }}|Pages: {{ .Paginator.TotalPages }}|{{ .Content }}`, + "_default/terms.html", "Terms List|{{ .Title }}|{{ .Content }}", + ) for i := 0; i < 2; i++ { for j := 0; j < 2; j++ { pageID := i + j + 1 - writeSource(t, fs, fmt.Sprintf("content/sect/p%d.md", pageID), + b.WithContent(fmt.Sprintf("content/sect/p%d.md", pageID), fmt.Sprintf(pageTemplate, pageID, fmt.Sprintf("- tag%d", j), fmt.Sprintf("- category%d", j), pageID)) } } for i := 0; i < 5; i++ { - writeSource(t, fs, fmt.Sprintf("content/assets/image%d.png", i+1), "image") + b.WithContent(fmt.Sprintf("assets/image%d.png", i+1), "image") } - err := h.Build(BuildCfg{}) - - assert.NoError(err) + b.Build(BuildCfg{}) + h := b.H stats := []*helpers.ProcessingStats{ h.Sites[0].PathSpec.ProcessingStats, diff --git a/hugolib/site_test.go b/hugolib/site_test.go index 5912abbc9c0..bbf101fc406 100644 --- a/hugolib/site_test.go +++ b/hugolib/site_test.go @@ -362,11 +362,14 @@ func TestShouldNotWriteZeroLengthFilesToDestination(t *testing.T) { // Issue #1176 func TestSectionNaming(t *testing.T) { - t.Parallel() for _, canonify := range []bool{true, false} { for _, uglify := range []bool{true, false} { for _, pluralize := range []bool{true, false} { + canonify := canonify + uglify := uglify + pluralize := pluralize t.Run(fmt.Sprintf("canonify=%t,uglify=%t,pluralize=%t", canonify, uglify, pluralize), func(t *testing.T) { + t.Parallel() doTestSectionNaming(t, canonify, uglify, pluralize) }) } diff --git a/hugolib/taxonomy_test.go b/hugolib/taxonomy_test.go index f4902ae8d65..2edc36d63e1 100644 --- a/hugolib/taxonomy_test.go +++ b/hugolib/taxonomy_test.go @@ -64,14 +64,15 @@ YAML frontmatter with tags and categories taxonomy.` // func TestTaxonomiesWithAndWithoutContentFile(t *testing.T) { for _, uglyURLs := range []bool{false, true} { + uglyURLs := uglyURLs t.Run(fmt.Sprintf("uglyURLs=%t", uglyURLs), func(t *testing.T) { + t.Parallel() doTestTaxonomiesWithAndWithoutContentFile(t, uglyURLs) }) } } func doTestTaxonomiesWithAndWithoutContentFile(t *testing.T, uglyURLs bool) { - t.Parallel() siteConfig := ` baseURL = "http://example.com/blog" @@ -104,25 +105,20 @@ permalinkeds: siteConfig = fmt.Sprintf(siteConfig, uglyURLs) - th, h := newTestSitesFromConfigWithDefaultTemplates(t, siteConfig) - require.Len(t, h.Sites, 1) - - fs := th.Fs - - writeSource(t, fs, "content/p1.md", fmt.Sprintf(pageTemplate, "t1/c1", "- Tag1", "- cAt1", "- o1", "- Pl1")) - writeSource(t, fs, "content/p2.md", fmt.Sprintf(pageTemplate, "t2/c1", "- tag2", "- cAt1", "- o1", "- Pl1")) - writeSource(t, fs, "content/p3.md", fmt.Sprintf(pageTemplate, "t2/c12", "- tag2", "- cat2", "- o1", "- Pl1")) - writeSource(t, fs, "content/p4.md", fmt.Sprintf(pageTemplate, "Hello World", "", "", "- \"Hello Hugo world\"", "- Pl1")) + b := newTestSitesBuilder(t).WithConfigFile("toml", siteConfig) - writeNewContentFile(t, fs.Source, "Category Terms", "2017-01-01", "content/categories/_index.md", 10) - writeNewContentFile(t, fs.Source, "Tag1 List", "2017-01-01", "content/tags/Tag1/_index.md", 10) + b.WithContent( + "p1.md", fmt.Sprintf(pageTemplate, "t1/c1", "- Tag1", "- cAt1", "- o1", "- Pl1"), + "p2.md", fmt.Sprintf(pageTemplate, "t2/c1", "- tag2", "- cAt1", "- o1", "- Pl1"), + "p3.md", fmt.Sprintf(pageTemplate, "t2/c12", "- tag2", "- cat2", "- o1", "- Pl1"), + "p4.md", fmt.Sprintf(pageTemplate, "Hello World", "", "", "- \"Hello Hugo world\"", "- Pl1"), + "categories/_index.md", newTestPage("Category Terms", "2017-01-01", 10), + "tags/Tag1/_index.md", newTestPage("Tag1 List", "2017-01-01", 10), + // https://github.com/gohugoio/hugo/issues/5847 + "/tags/not-used/_index.md", newTestPage("Unused Tag List", "2018-01-01", 10), + ) - // https://github.com/gohugoio/hugo/issues/5847 - writeNewContentFile(t, fs.Source, "Unused Tag List", "2018-01-01", "content/tags/not-used/_index.md", 10) - - err := h.Build(BuildCfg{}) - - require.NoError(t, err) + b.Build(BuildCfg{}) // So what we have now is: // 1. categories with terms content page, but no content page for the only c1 category @@ -138,26 +134,26 @@ permalinkeds: } // 1. - th.assertFileContent(pathFunc("public/categories/cat1/index.html"), "List", "cAt1") - th.assertFileContent(pathFunc("public/categories/index.html"), "Terms List", "Category Terms") + b.AssertFileContent(pathFunc("public/categories/cat1/index.html"), "List", "cAt1") + b.AssertFileContent(pathFunc("public/categories/index.html"), "Taxonomy Term Page", "Category Terms") // 2. - th.assertFileContent(pathFunc("public/tags/tag2/index.html"), "List", "tag2") - th.assertFileContent(pathFunc("public/tags/tag1/index.html"), "List", "Tag1") - th.assertFileContent(pathFunc("public/tags/index.html"), "Terms List", "Tags") + b.AssertFileContent(pathFunc("public/tags/tag2/index.html"), "List", "tag2") + b.AssertFileContent(pathFunc("public/tags/tag1/index.html"), "List", "Tag1") + b.AssertFileContent(pathFunc("public/tags/index.html"), "Taxonomy Term Page", "Tags") // 3. - th.assertFileContent(pathFunc("public/others/o1/index.html"), "List", "o1") - th.assertFileContent(pathFunc("public/others/index.html"), "Terms List", "Others") + b.AssertFileContent(pathFunc("public/others/o1/index.html"), "List", "o1") + b.AssertFileContent(pathFunc("public/others/index.html"), "Taxonomy Term Page", "Others") // 4. - th.assertFileContent(pathFunc("public/perma/pl1/index.html"), "List", "Pl1") + b.AssertFileContent(pathFunc("public/perma/pl1/index.html"), "List", "Pl1") // This looks kind of funky, but the taxonomy terms do not have a permalinks definition, // for good reasons. - th.assertFileContent(pathFunc("public/permalinkeds/index.html"), "Terms List", "Permalinkeds") + b.AssertFileContent(pathFunc("public/permalinkeds/index.html"), "Taxonomy Term Page", "Permalinkeds") - s := h.Sites[0] + s := b.H.Sites[0] // Make sure that each page.KindTaxonomyTerm page has an appropriate number // of page.KindTaxonomy pages in its Pages slice. @@ -204,7 +200,7 @@ permalinkeds: require.Equal(t, "Hello Hugo world", helloWorld.Title()) // Issue #2977 - th.assertFileContent(pathFunc("public/empties/index.html"), "Terms List", "Empties") + b.AssertFileContent(pathFunc("public/empties/index.html"), "Taxonomy Term Page", "Empties") } diff --git a/hugolib/template_engines_test.go b/hugolib/template_engines_test.go index 6a046c9f59a..ec229a29941 100644 --- a/hugolib/template_engines_test.go +++ b/hugolib/template_engines_test.go @@ -24,7 +24,6 @@ import ( ) func TestAllTemplateEngines(t *testing.T) { - t.Parallel() noOp := func(s string) string { return s } @@ -48,8 +47,10 @@ func TestAllTemplateEngines(t *testing.T) { {"html", noOp}, {"ace", noOp}, } { + config := config t.Run(config.suffix, func(t *testing.T) { + t.Parallel() doTestTemplateEngine(t, config.suffix, config.templateFixer) }) } diff --git a/hugolib/template_test.go b/hugolib/template_test.go index 3ec81323b4b..6ed9643c7fd 100644 --- a/hugolib/template_test.go +++ b/hugolib/template_test.go @@ -25,7 +25,6 @@ import ( ) func TestTemplateLookupOrder(t *testing.T) { - t.Parallel() var ( fs *hugofs.Fs cfg *viper.Viper @@ -193,22 +192,26 @@ func TestTemplateLookupOrder(t *testing.T) { }, } { - cfg, fs = newTestCfg() - th = testHelper{cfg, fs, t} + this := this + t.Run(this.name, func(t *testing.T) { + // TODO(bep) there are some function vars need to pull down here to enable => t.Parallel() + cfg, fs = newTestCfg() + th = testHelper{cfg, fs, t} - for i := 1; i <= 3; i++ { - writeSource(t, fs, filepath.Join("content", fmt.Sprintf("sect%d", i), fmt.Sprintf("page%d.md", i)), `--- + for i := 1; i <= 3; i++ { + writeSource(t, fs, filepath.Join("content", fmt.Sprintf("sect%d", i), fmt.Sprintf("page%d.md", i)), `--- title: Template test --- Some content `) - } + } - this.setup(t) + this.setup(t) - buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{}) - t.Log(this.name) - this.assert(t) + buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{}) + //helpers.PrintFs(s.BaseFs.Layouts.Fs, "", os.Stdout) + this.assert(t) + }) } } diff --git a/hugolib/testhelpers_test.go b/hugolib/testhelpers_test.go index 8c72e10d0c1..ac511367d6e 100644 --- a/hugolib/testhelpers_test.go +++ b/hugolib/testhelpers_test.go @@ -2,13 +2,17 @@ package hugolib import ( "io" - "io/ioutil" "path/filepath" "runtime" "strconv" "testing" "unicode/utf8" + "github.com/gohugoio/hugo/parser/metadecoders" + + "github.com/gohugoio/hugo/parser" + "github.com/pkg/errors" + "bytes" "fmt" "regexp" @@ -39,9 +43,12 @@ import ( ) type sitesBuilder struct { - Cfg config.Provider - Fs *hugofs.Fs - T testing.TB + Cfg config.Provider + environ []string + + Fs *hugofs.Fs + T testing.TB + depsCfg deps.DepsCfg *require.Assertions @@ -60,13 +67,16 @@ type sitesBuilder struct { theme string // Default toml - configFormat string + configFormat string + configFileSet bool + viperSet bool // Default is empty. // TODO(bep) revisit this and consider always setting it to something. // Consider this in relation to using the BaseFs.PublishFs to all publishing. workingDir string + addNothing bool // Base data/content contentFilePairs []string templateFilePairs []string @@ -94,18 +104,22 @@ func newTestSitesBuilder(t testing.TB) *sitesBuilder { return &sitesBuilder{T: t, Assertions: require.New(t), Fs: fs, configFormat: "toml", dumper: litterOptions} } -func createTempDir(prefix string) (string, func(), error) { - workDir, err := ioutil.TempDir("", prefix) - if err != nil { - return "", nil, err - } +func newTestSitesBuilderFromDepsCfg(t testing.TB, d deps.DepsCfg) *sitesBuilder { + assert := require.New(t) - if runtime.GOOS == "darwin" && !strings.HasPrefix(workDir, "/private") { - // To get the entry folder in line with the rest. This its a little bit - // mysterious, but so be it. - workDir = "/private" + workDir + litterOptions := litter.Options{ + HidePrivateFields: true, + StripPackageNames: true, + Separator: " ", } - return workDir, func() { os.RemoveAll(workDir) }, nil + + b := &sitesBuilder{T: t, Assertions: assert, depsCfg: d, Fs: d.Fs, dumper: litterOptions} + workingDir := d.Cfg.GetString("workingDir") + + b.WithWorkingDir(workingDir) + + return b.WithViper(d.Cfg.(*viper.Viper)) + } func (s *sitesBuilder) Running() *sitesBuilder { @@ -113,17 +127,31 @@ func (s *sitesBuilder) Running() *sitesBuilder { return s } +func (s *sitesBuilder) WithNothingAdded() *sitesBuilder { + s.addNothing = true + return s +} + func (s *sitesBuilder) WithLogger(logger *loggers.Logger) *sitesBuilder { s.logger = logger return s } func (s *sitesBuilder) WithWorkingDir(dir string) *sitesBuilder { - s.workingDir = dir + s.workingDir = filepath.FromSlash(dir) + return s +} + +func (s *sitesBuilder) WithEnviron(env ...string) *sitesBuilder { + for i := 0; i < len(env); i += 2 { + s.environ = append(s.environ, fmt.Sprintf("%s=%s", env[i], env[i+1])) + } return s } func (s *sitesBuilder) WithConfigTemplate(data interface{}, format, configTemplate string) *sitesBuilder { + s.T.Helper() + if format == "" { format = "toml" } @@ -138,32 +166,59 @@ func (s *sitesBuilder) WithConfigTemplate(data interface{}, format, configTempla } func (s *sitesBuilder) WithViper(v *viper.Viper) *sitesBuilder { - loadDefaultSettingsFor(v) - s.Cfg = v + s.T.Helper() + if s.configFileSet { + s.T.Fatal("WithViper: use Viper or config.toml, not both") + } + defer func() { + s.viperSet = true + }() - return s + // Write to a config file to make sure the tests follow the same code path. + var buff bytes.Buffer + m := v.AllSettings() + s.Assertions.NoError(parser.InterfaceToConfig(m, metadecoders.TOML, &buff)) + return s.WithConfigFile("toml", buff.String()) } func (s *sitesBuilder) WithConfigFile(format, conf string) *sitesBuilder { - writeSource(s.T, s.Fs, "config."+format, conf) + s.T.Helper() + if s.viperSet { + s.T.Fatal("WithConfigFile: use Viper or config.toml, not both") + } + s.configFileSet = true + filename := s.absFilename("config." + format) + writeSource(s.T, s.Fs, filename, conf) s.configFormat = format return s } func (s *sitesBuilder) WithThemeConfigFile(format, conf string) *sitesBuilder { + s.T.Helper() if s.theme == "" { s.theme = "test-theme" } filename := filepath.Join("themes", s.theme, "config."+format) - writeSource(s.T, s.Fs, filename, conf) + writeSource(s.T, s.Fs, s.absFilename(filename), conf) return s } -func (s *sitesBuilder) WithSourceFile(filename, content string) *sitesBuilder { - writeSource(s.T, s.Fs, filepath.FromSlash(filename), content) +func (s *sitesBuilder) WithSourceFile(filenameContent ...string) *sitesBuilder { + s.T.Helper() + for i := 0; i < len(filenameContent); i += 2 { + writeSource(s.T, s.Fs, s.absFilename(filenameContent[i]), filenameContent[i+1]) + } return s } +func (s *sitesBuilder) absFilename(filename string) string { + filename = filepath.FromSlash(filename) + if s.workingDir != "" && !strings.HasPrefix(filename, s.workingDir) { + filename = filepath.Join(s.workingDir, filename) + } + return filename +} + const commonConfigSections = ` [services] @@ -191,10 +246,12 @@ privacyEnhanced = true ` func (s *sitesBuilder) WithSimpleConfigFile() *sitesBuilder { + s.T.Helper() return s.WithSimpleConfigFileAndBaseURL("http://example.com/") } func (s *sitesBuilder) WithSimpleConfigFileAndBaseURL(baseURL string) *sitesBuilder { + s.T.Helper() config := fmt.Sprintf("baseURL = %q", baseURL) config = config + commonConfigSections @@ -323,7 +380,7 @@ func (s *sitesBuilder) EditFiles(filenameContent ...string) *sitesBuilder { for i := 0; i < len(filenameContent); i += 2 { filename, content := filepath.FromSlash(filenameContent[i]), filenameContent[i+1] changedFiles = append(changedFiles, filename) - writeSource(s.T, s.Fs, filename, content) + writeSource(s.T, s.Fs, s.absFilename(filename), content) } s.changedFiles = changedFiles @@ -354,6 +411,7 @@ func (s *sitesBuilder) writeFilePairs(folder string, filenameContent []string) * func (s *sitesBuilder) CreateSites() *sitesBuilder { if err := s.CreateSitesE(); err != nil { + herrors.PrintStackTrace(err) s.Fatalf("Failed to create sites: %s", err) } @@ -361,34 +419,72 @@ func (s *sitesBuilder) CreateSites() *sitesBuilder { } func (s *sitesBuilder) LoadConfig() error { - cfg, _, err := LoadConfig(ConfigSourceDescriptor{Fs: s.Fs.Source, Filename: "config." + s.configFormat}) + if !s.configFileSet { + s.WithSimpleConfigFile() + } + + cfg, _, err := LoadConfig(ConfigSourceDescriptor{ + WorkingDir: s.workingDir, + Fs: s.Fs.Source, + Logger: s.logger, + Environ: s.environ, + Filename: "config." + s.configFormat}, func(cfg config.Provider) error { + + return nil + }) + if err != nil { return err } + s.Cfg = cfg + return nil } func (s *sitesBuilder) CreateSitesE() error { - s.addDefaults() - s.writeFilePairs("content", s.contentFilePairs) - s.writeFilePairs("content", s.contentFilePairsAdded) - s.writeFilePairs("layouts", s.templateFilePairs) - s.writeFilePairs("layouts", s.templateFilePairsAdded) - s.writeFilePairs("data", s.dataFilePairs) - s.writeFilePairs("data", s.dataFilePairsAdded) - s.writeFilePairs("i18n", s.i18nFilePairs) - s.writeFilePairs("i18n", s.i18nFilePairsAdded) - - if s.Cfg == nil { - if err := s.LoadConfig(); err != nil { - return err + if !s.addNothing { + if _, ok := s.Fs.Source.(*afero.OsFs); ok { + for _, dir := range []string{ + "content/sect", + "layouts/_default", + "layouts/partials", + "layouts/shortcodes", + "data", + "i18n", + } { + if err := os.MkdirAll(filepath.Join(s.workingDir, dir), 0777); err != nil { + return errors.Wrapf(err, "failed to create %q", dir) + } + } } + + s.addDefaults() + s.writeFilePairs("content", s.contentFilePairsAdded) + s.writeFilePairs("layouts", s.templateFilePairsAdded) + s.writeFilePairs("data", s.dataFilePairsAdded) + s.writeFilePairs("i18n", s.i18nFilePairsAdded) + + s.writeFilePairs("i18n", s.i18nFilePairs) + s.writeFilePairs("data", s.dataFilePairs) + s.writeFilePairs("content", s.contentFilePairs) + s.writeFilePairs("layouts", s.templateFilePairs) + } - sites, err := NewHugoSites(deps.DepsCfg{Fs: s.Fs, Cfg: s.Cfg, Logger: s.logger, Running: s.running}) + if err := s.LoadConfig(); err != nil { + return errors.Wrap(err, "failed to load config") + } + + depsCfg := s.depsCfg + depsCfg.Fs = s.Fs + depsCfg.Cfg = s.Cfg + depsCfg.Logger = s.logger + depsCfg.Running = s.running + + sites, err := NewHugoSites(depsCfg) if err != nil { - return err + return errors.Wrap(err, "failed to create sites") } s.H = sites @@ -404,10 +500,12 @@ func (s *sitesBuilder) BuildE(cfg BuildCfg) error { } func (s *sitesBuilder) Build(cfg BuildCfg) *sitesBuilder { + s.T.Helper() return s.build(cfg, false) } func (s *sitesBuilder) BuildFail(cfg BuildCfg) *sitesBuilder { + s.T.Helper() return s.build(cfg, true) } @@ -528,14 +626,8 @@ hello: } func (s *sitesBuilder) Fatalf(format string, args ...interface{}) { - Fatalf(s.T, format, args...) -} - -func Fatalf(t testing.TB, format string, args ...interface{}) { - trace := stackTrace() - format = format + "\n%s" - args = append(args, trace) - t.Fatalf(format, args...) + s.T.Helper() + s.T.Fatalf(format, args...) } func stackTrace() string { @@ -543,9 +635,10 @@ func stackTrace() string { } func (s *sitesBuilder) AssertFileContentFn(filename string, f func(s string) bool) { + s.T.Helper() content := s.FileContent(filename) if !f(content) { - s.Fatalf("Assert failed for %q", filename) + s.Fatalf("Assert failed for %q in content\n%s", filename, content) } } @@ -554,6 +647,7 @@ func (s *sitesBuilder) AssertHome(matches ...string) { } func (s *sitesBuilder) AssertFileContent(filename string, matches ...string) { + s.T.Helper() content := s.FileContent(filename) for _, match := range matches { if !strings.Contains(content, match) { @@ -563,10 +657,16 @@ func (s *sitesBuilder) AssertFileContent(filename string, matches ...string) { } func (s *sitesBuilder) FileContent(filename string) string { + s.T.Helper() + filename = filepath.FromSlash(filename) + if !strings.HasPrefix(filename, s.workingDir) { + filename = filepath.Join(s.workingDir, filename) + } return readDestination(s.T, s.Fs, filename) } func (s *sitesBuilder) AssertObject(expected string, object interface{}) { + s.T.Helper() got := s.dumper.Sdump(object) expected = strings.TrimSpace(expected) @@ -633,17 +733,41 @@ func (th testHelper) replaceDefaultContentLanguageValue(value string) string { return value } -func newTestCfg() (*viper.Viper, *hugofs.Fs) { +func loadTestConfig(fs afero.Fs, withConfig ...func(cfg config.Provider) error) (*viper.Viper, error) { + v, _, err := LoadConfig(ConfigSourceDescriptor{Fs: fs}, withConfig...) + return v, err +} +func newTestCfgBasic() (*viper.Viper, *hugofs.Fs) { + mm := afero.NewMemMapFs() v := viper.New() - fs := hugofs.NewMem(v) + v.Set("defaultContentLanguageInSubdir", true) - v.SetFs(fs.Source) + fs := hugofs.NewFrom(hugofs.NewBaseFileDecorator(mm), v) - loadDefaultSettingsFor(v) + return v, fs - // Default is false, but true is easier to use as default in tests - v.Set("defaultContentLanguageInSubdir", true) +} + +func newTestCfg(withConfig ...func(cfg config.Provider) error) (*viper.Viper, *hugofs.Fs) { + mm := afero.NewMemMapFs() + + v, err := loadTestConfig(mm, func(cfg config.Provider) error { + // Default is false, but true is easier to use as default in tests + cfg.Set("defaultContentLanguageInSubdir", true) + + for _, w := range withConfig { + w(cfg) + } + + return nil + }) + + if err != nil && err != ErrNoConfigFile { + panic(err) + } + + fs := hugofs.NewFrom(hugofs.NewBaseFileDecorator(mm), v) return v, fs @@ -651,9 +775,10 @@ func newTestCfg() (*viper.Viper, *hugofs.Fs) { func newTestSitesFromConfig(t testing.TB, afs afero.Fs, tomlConfig string, layoutPathContentPairs ...string) (testHelper, *HugoSites) { if len(layoutPathContentPairs)%2 != 0 { - Fatalf(t, "Layouts must be provided in pairs") + t.Fatalf("Layouts must be provided in pairs") } + writeToFs(t, afs, filepath.Join("content", ".gitkeep"), "") writeToFs(t, afs, "config.toml", tomlConfig) cfg, err := LoadConfigDefault(afs) @@ -673,14 +798,6 @@ func newTestSitesFromConfig(t testing.TB, afs afero.Fs, tomlConfig string, layou return th, h } -func newTestSitesFromConfigWithDefaultTemplates(t testing.TB, tomlConfig string) (testHelper, *HugoSites) { - return newTestSitesFromConfig(t, afero.NewMemMapFs(), tomlConfig, - "layouts/_default/single.html", "Single|{{ .Title }}|{{ .Content }}", - "layouts/_default/list.html", "List|{{ .Title }}|{{ .Content }}", - "layouts/_default/terms.html", "Terms List|{{ .Title }}|{{ .Content }}", - ) -} - func createWithTemplateFromNameValues(additionalTemplates ...string) func(templ tpl.TemplateHandler) error { return func(templ tpl.TemplateHandler) error { @@ -694,12 +811,16 @@ func createWithTemplateFromNameValues(additionalTemplates ...string) func(templ } } +// TODO(bep) replace these with the builder func buildSingleSite(t testing.TB, depsCfg deps.DepsCfg, buildCfg BuildCfg) *Site { return buildSingleSiteExpected(t, false, false, depsCfg, buildCfg) } func buildSingleSiteExpected(t testing.TB, expectSiteInitEror, expectBuildError bool, depsCfg deps.DepsCfg, buildCfg BuildCfg) *Site { - h, err := NewHugoSites(depsCfg) + t.Helper() + b := newTestSitesBuilderFromDepsCfg(t, depsCfg).WithNothingAdded() + + err := b.CreateSitesE() if expectSiteInitEror { require.Error(t, err) @@ -708,6 +829,8 @@ func buildSingleSiteExpected(t testing.TB, expectSiteInitEror, expectBuildError require.NoError(t, err) } + h := b.H + require.Len(t, h.Sites, 1) if expectBuildError { @@ -751,9 +874,13 @@ func content(c resource.ContentProvider) string { func dumpPages(pages ...page.Page) { fmt.Println("---------") for i, p := range pages { - fmt.Printf("%d: Kind: %s Title: %-10s RelPermalink: %-10s Path: %-10s sections: %s\n", + var meta interface{} + if p.File() != nil && p.File().FileInfo() != nil { + meta = p.File().FileInfo().Meta() + } + fmt.Printf("%d: Kind: %s Title: %-10s RelPermalink: %-10s Path: %-10s sections: %s Lang: %s Meta: %v\n", i+1, - p.Kind(), p.Title(), p.RelPermalink(), p.Path(), p.SectionsPath()) + p.Kind(), p.Title(), p.RelPermalink(), p.Path(), p.SectionsPath(), p.Lang(), meta) } } @@ -802,3 +929,10 @@ func parallel(t *testing.T) { t.Parallel() } } + +func skipSymlink(t *testing.T) { + if runtime.GOOS == "windows" && os.Getenv("CI") == "" { + t.Skip("skip symlink test on local Windows (needs admin)") + } + +} diff --git a/langs/config.go b/langs/config.go new file mode 100644 index 00000000000..927f3558fa4 --- /dev/null +++ b/langs/config.go @@ -0,0 +1,217 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package langs + +import ( + "fmt" + "path/filepath" + "sort" + "strings" + + "github.com/gohugoio/hugo/common/maps" + + "github.com/spf13/cast" + + "github.com/pkg/errors" + + "github.com/gohugoio/hugo/config" +) + +type LanguagesConfig struct { + Languages Languages + Multihost bool + DefaultContentLanguageInSubdir bool +} + +func LoadLanguageSettings(cfg config.Provider, oldLangs Languages) (c LanguagesConfig, err error) { + + defaultLang := cfg.GetString("defaultContentLanguage") + if defaultLang == "" { + defaultLang = "en" + cfg.Set("defaultContentLanguage", defaultLang) + } + + var languages map[string]interface{} + + languagesFromConfig := cfg.GetStringMap("languages") + disableLanguages := cfg.GetStringSlice("disableLanguages") + + if len(disableLanguages) == 0 { + languages = languagesFromConfig + } else { + languages = make(map[string]interface{}) + for k, v := range languagesFromConfig { + for _, disabled := range disableLanguages { + if disabled == defaultLang { + return c, fmt.Errorf("cannot disable default language %q", defaultLang) + } + + if strings.EqualFold(k, disabled) { + v.(map[string]interface{})["disabled"] = true + break + } + } + languages[k] = v + } + } + + var languages2 Languages + + if len(languages) == 0 { + languages2 = append(languages2, NewDefaultLanguage(cfg)) + } else { + languages2, err = toSortedLanguages(cfg, languages) + if err != nil { + return c, errors.Wrap(err, "Failed to parse multilingual config") + } + } + + if oldLangs != nil { + // When in multihost mode, the languages are mapped to a server, so + // some structural language changes will need a restart of the dev server. + // The validation below isn't complete, but should cover the most + // important cases. + var invalid bool + if languages2.IsMultihost() != oldLangs.IsMultihost() { + invalid = true + } else { + if languages2.IsMultihost() && len(languages2) != len(oldLangs) { + invalid = true + } + } + + if invalid { + return c, errors.New("language change needing a server restart detected") + } + + if languages2.IsMultihost() { + // We need to transfer any server baseURL to the new language + for i, ol := range oldLangs { + nl := languages2[i] + nl.Set("baseURL", ol.GetString("baseURL")) + } + } + } + + // The defaultContentLanguage is something the user has to decide, but it needs + // to match a language in the language definition list. + langExists := false + for _, lang := range languages2 { + if lang.Lang == defaultLang { + langExists = true + break + } + } + + if !langExists { + return c, fmt.Errorf("site config value %q for defaultContentLanguage does not match any language definition", defaultLang) + } + + c.Languages = languages2 + c.Multihost = languages2.IsMultihost() + c.DefaultContentLanguageInSubdir = c.Multihost + + sortedDefaultFirst := make(Languages, len(c.Languages)) + for i, v := range c.Languages { + sortedDefaultFirst[i] = v + } + sort.Slice(sortedDefaultFirst, func(i, j int) bool { + li, lj := sortedDefaultFirst[i], sortedDefaultFirst[j] + if li.Lang == defaultLang { + return true + } + + if lj.Lang == defaultLang { + return false + } + + return i < j + }) + + cfg.Set("languagesSorted", c.Languages) + cfg.Set("languagesSortedDefaultFirst", sortedDefaultFirst) + cfg.Set("multilingual", len(languages2) > 1) + + multihost := c.Multihost + + if multihost { + cfg.Set("defaultContentLanguageInSubdir", true) + cfg.Set("multihost", true) + } + + if multihost { + // The baseURL may be provided at the language level. If that is true, + // then every language must have a baseURL. In this case we always render + // to a language sub folder, which is then stripped from all the Permalink URLs etc. + for _, l := range languages2 { + burl := l.GetLocal("baseURL") + if burl == nil { + return c, errors.New("baseURL must be set on all or none of the languages") + } + } + + } + + return c, nil +} + +func toSortedLanguages(cfg config.Provider, l map[string]interface{}) (Languages, error) { + languages := make(Languages, len(l)) + i := 0 + + for lang, langConf := range l { + langsMap, err := cast.ToStringMapE(langConf) + + if err != nil { + return nil, fmt.Errorf("Language config is not a map: %T", langConf) + } + + language := NewLanguage(lang, cfg) + + for loki, v := range langsMap { + switch loki { + case "title": + language.Title = cast.ToString(v) + case "languagename": + language.LanguageName = cast.ToString(v) + case "weight": + language.Weight = cast.ToInt(v) + case "contentdir": + language.ContentDir = filepath.Clean(cast.ToString(v)) + case "disabled": + language.Disabled = cast.ToBool(v) + case "params": + m := cast.ToStringMap(v) + // Needed for case insensitive fetching of params values + maps.ToLower(m) + for k, vv := range m { + language.SetParam(k, vv) + } + } + + // Put all into the Params map + language.SetParam(loki, v) + + // Also set it in the configuration map (for baseURL etc.) + language.Set(loki, v) + } + + languages[i] = language + i++ + } + + sort.Sort(languages) + + return languages, nil +} diff --git a/langs/i18n/i18n_test.go b/langs/i18n/i18n_test.go index b67cabc5524..e08210848fe 100644 --- a/langs/i18n/i18n_test.go +++ b/langs/i18n/i18n_test.go @@ -17,11 +17,13 @@ import ( "path/filepath" "testing" + "github.com/gohugoio/hugo/modules" + "github.com/gohugoio/hugo/tpl/tplimpl" "github.com/gohugoio/hugo/common/loggers" - "github.com/gohugoio/hugo/htesting" "github.com/gohugoio/hugo/langs" + "github.com/gohugoio/hugo/resources/page" "github.com/spf13/afero" "github.com/spf13/viper" @@ -199,7 +201,7 @@ func newDepsConfig(tp *TranslationProvider, cfg config.Provider, fs *hugofs.Fs) l.Set("i18nDir", "i18n") return deps.DepsCfg{ Language: l, - Site: htesting.NewTestHugoSite(), + Site: page.NewDummyHugoSite(cfg), Cfg: cfg, Fs: fs, Logger: logger, @@ -219,6 +221,13 @@ func getConfig() *viper.Viper { v.Set("assetDir", "assets") v.Set("resourceDir", "resources") v.Set("publishDir", "public") + langs.LoadLanguageSettings(v, nil) + mod, err := modules.CreateProjectModule(v) + if err != nil { + panic(err) + } + v.Set("allModules", modules.Modules{mod}) + return v } diff --git a/langs/i18n/translationProvider.go b/langs/i18n/translationProvider.go index 74e144007ef..c7b4839eef1 100644 --- a/langs/i18n/translationProvider.go +++ b/langs/i18n/translationProvider.go @@ -40,8 +40,7 @@ func NewTranslationProvider() *TranslationProvider { // Update updates the i18n func in the provided Deps. func (tp *TranslationProvider) Update(d *deps.Deps) error { - sp := source.NewSourceSpec(d.PathSpec, d.BaseFs.SourceFilesystems.I18n.Fs) - src := sp.NewFilesystem("") + spec := source.NewSourceSpec(d.PathSpec, nil) i18nBundle := bundle.New() @@ -51,25 +50,33 @@ func (tp *TranslationProvider) Update(d *deps.Deps) error { } var newLangs []string - for _, r := range src.Files() { - currentSpec := language.GetPluralSpec(r.BaseFileName()) - if currentSpec == nil { - // This may is a language code not supported by go-i18n, it may be - // Klingon or ... not even a fake language. Make sure it works. - newLangs = append(newLangs, r.BaseFileName()) + for _, dir := range d.BaseFs.I18n.Dirs { + src := spec.NewFilesystemFromFileMetaInfo(dir) + + files, err := src.Files() + if err != nil { + return err } - } - if len(newLangs) > 0 { - language.RegisterPluralSpec(newLangs, en) - } + for _, r := range files { + currentSpec := language.GetPluralSpec(r.BaseFileName()) + if currentSpec == nil { + // This may is a language code not supported by go-i18n, it may be + // Klingon or ... not even a fake language. Make sure it works. + newLangs = append(newLangs, r.BaseFileName()) + } + } - // The source files are ordered so the most important comes first. Since this is a - // last key win situation, we have to reverse the iteration order. - files := src.Files() - for i := len(files) - 1; i >= 0; i-- { - if err := addTranslationFile(i18nBundle, files[i]); err != nil { - return err + if len(newLangs) > 0 { + language.RegisterPluralSpec(newLangs, en) + } + + // The source files are ordered so the most important comes first. Since this is a + // last key win situation, we have to reverse the iteration order. + for i := len(files) - 1; i >= 0; i-- { + if err := addTranslationFile(i18nBundle, files[i]); err != nil { + return err + } } } @@ -81,8 +88,8 @@ func (tp *TranslationProvider) Update(d *deps.Deps) error { } -func addTranslationFile(bundle *bundle.Bundle, r source.ReadableFile) error { - f, err := r.Open() +func addTranslationFile(bundle *bundle.Bundle, r source.File) error { + f, err := r.FileInfo().Meta().Open() if err != nil { return _errors.Wrapf(err, "failed to open translations file %q:", r.LogicalName()) } @@ -101,14 +108,15 @@ func (tp *TranslationProvider) Clone(d *deps.Deps) error { return nil } -func errWithFileContext(inerr error, r source.ReadableFile) error { - rfi, ok := r.FileInfo().(hugofs.RealFilenameInfo) +func errWithFileContext(inerr error, r source.File) error { + fim, ok := r.FileInfo().(hugofs.FileMetaInfo) if !ok { return inerr } - realFilename := rfi.RealFilename() - f, err := r.Open() + meta := fim.Meta() + realFilename := meta.Filename() + f, err := meta.Open() if err != nil { return inerr } diff --git a/langs/language.go b/langs/language.go index 14e3263aeb9..f71b0255b38 100644 --- a/langs/language.go +++ b/langs/language.go @@ -78,12 +78,7 @@ func NewLanguage(lang string, cfg config.Provider) *Language { } maps.ToLower(params) - defaultContentDir := cfg.GetString("contentDir") - if defaultContentDir == "" { - panic("contentDir not set") - } - - l := &Language{Lang: lang, ContentDir: defaultContentDir, Cfg: cfg, params: params, settings: make(map[string]interface{})} + l := &Language{Lang: lang, ContentDir: cfg.GetString("contentDir"), Cfg: cfg, params: params, settings: make(map[string]interface{})} return l } @@ -132,6 +127,24 @@ func (l *Language) Params() map[string]interface{} { return l.params } +func (l Languages) AsSet() map[string]bool { + m := make(map[string]bool) + for _, lang := range l { + m[lang.Lang] = true + } + + return m +} + +func (l Languages) AsOrdinalSet() map[string]int { + m := make(map[string]int) + for i, lang := range l { + m[lang.Lang] = i + } + + return m +} + // IsMultihost returns whether there are more than one language and at least one of // the languages has baseURL specificed on the language level. func (l Languages) IsMultihost() bool { diff --git a/magefile.go b/magefile.go index 3b74a7e940e..d0b7c8d98a6 100644 --- a/magefile.go +++ b/magefile.go @@ -143,20 +143,31 @@ func Check() { mg.Deps(TestRace) } +func testGoFlags() string { + if isCI() { + return "" + } + + return "-test.short" +} + // Run tests in 32-bit mode // Note that we don't run with the extended tag. Currently not supported in 32 bit. func Test386() error { - return sh.RunWith(map[string]string{"GOARCH": "386"}, goexe, "test", "./...") + env := map[string]string{"GOARCH": "386", "GOFLAGS": testGoFlags()} + return sh.RunWith(env, goexe, "test", "./...") } // Run tests func Test() error { - return sh.Run(goexe, "test", "./...", "-tags", buildTags()) + env := map[string]string{"GOFLAGS": testGoFlags()} + return sh.RunWith(env, goexe, "test", "./...", "-tags", buildTags()) } // Run tests with race detector func TestRace() error { - return sh.Run(goexe, "test", "-race", "./...", "-tags", buildTags()) + env := map[string]string{"GOFLAGS": testGoFlags()} + return sh.RunWith(env, goexe, "test", "-race", "./...", "-tags", buildTags()) } // Run gofmt linter @@ -296,6 +307,10 @@ func isGoLatest() bool { return strings.Contains(runtime.Version(), "1.11") } +func isCI() bool { + return os.Getenv("CI") != "" +} + func buildTags() string { // To build the extended Hugo SCSS/SASS enabled version, build with // HUGO_BUILD_TAGS=extended mage install etc. diff --git a/modules/client.go b/modules/client.go new file mode 100644 index 00000000000..ac09721dce1 --- /dev/null +++ b/modules/client.go @@ -0,0 +1,570 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package modules + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + + "github.com/gohugoio/hugo/hugofs/files" + + "github.com/gohugoio/hugo/common/loggers" + + "strings" + "time" + + "github.com/gohugoio/hugo/config" + + "github.com/rogpeppe/go-internal/module" + + "github.com/gohugoio/hugo/common/hugio" + + "github.com/pkg/errors" + "github.com/spf13/afero" +) + +var ( + fileSeparator = string(os.PathSeparator) +) + +const ( + goBinaryStatusOK goBinaryStatus = iota + goBinaryStatusNotFound + goBinaryStatusTooOld +) + +// The "vendor" dir is reserved for Go Modules. +const vendord = "_vendor" + +const ( + goModFilename = "go.mod" + goSumFilename = "go.sum" +) + +// NewClient creates a new Client that can be used to manage the Hugo Components +// in a given workingDir. +// The Client will resolve the dependencies recursively, but needs the top +// level imports to start out. +func NewClient(cfg ClientConfig) *Client { + fs := cfg.Fs + + n := filepath.Join(cfg.WorkingDir, goModFilename) + goModEnabled, _ := afero.Exists(fs, n) + var goModFilename string + if goModEnabled { + goModFilename = n + } + + env := os.Environ() + mcfg := cfg.ModuleConfig + + config.SetEnvVars(&env, + "PWD", cfg.WorkingDir, + "GOPROXY", mcfg.Proxy, + "GOPRIVATE", mcfg.Private, + "GONOPROXY", mcfg.NoProxy) + + if cfg.CacheDir != "" { + // Module cache stored below $GOPATH/pkg + config.SetEnvVars(&env, "GOPATH", cfg.CacheDir) + + } + + logger := cfg.Logger + if logger == nil { + logger = loggers.NewWarningLogger() + } + + return &Client{ + fs: fs, + ignoreVendor: cfg.IgnoreVendor, + workingDir: cfg.WorkingDir, + themesDir: cfg.ThemesDir, + logger: logger, + moduleConfig: mcfg, + environ: env, + GoModulesFilename: goModFilename} +} + +// Client contains most of the API provided by this package. +type Client struct { + fs afero.Fs + logger *loggers.Logger + + // Ignore any _vendor directory. + ignoreVendor bool + + // Absolute path to the project dir. + workingDir string + + // Absolute path to the project's themes dir. + themesDir string + + // The top level module config + moduleConfig Config + + // Environment variables used in "go get" etc. + environ []string + + // Set when Go modules are initialized in the current repo, that is: + // a go.mod file exists. + GoModulesFilename string + + // Set if we get a exec.ErrNotFound when running Go, which is most likely + // due to being run on a system without Go installed. We record it here + // so we can give an instructional error at the end if module/theme + // resolution fails. + goBinaryStatus goBinaryStatus +} + +// Graph writes a module dependenchy graph to the given writer. +func (c *Client) Graph(w io.Writer) error { + mc, coll := c.collect(true) + if coll.err != nil { + return coll.err + } + for _, module := range mc.AllModules { + if module.Owner() == nil { + continue + } + + prefix := "" + if module.Disabled() { + prefix = "DISABLED " + } + dep := pathVersion(module.Owner()) + " " + pathVersion(module) + if replace := module.Replace(); replace != nil { + if replace.Version() != "" { + dep += " => " + pathVersion(replace) + } else { + // Local dir. + dep += " => " + replace.Dir() + } + + } + fmt.Fprintln(w, prefix+dep) + } + + return nil +} + +// Tidy can be used to remove unused dependencies from go.mod and go.sum. +func (c *Client) Tidy() error { + tc, coll := c.collect(false) + if coll.err != nil { + return coll.err + } + + if coll.skipTidy { + return nil + } + + return c.tidy(tc.AllModules, false) +} + +// Vendor writes all the module dependencies to a _vendor folder. +// +// Unlike Go, we support it for any level. +// +// We, by default, use the /_vendor folder first, if found. To disable, +// run with +// hugo --ignoreVendor +// +// Given a module tree, Hugo will pick the first module for a given path, +// meaning that if the top-level module is vendored, that will be the full +// set of dependencies. +func (c *Client) Vendor() error { + vendorDir := filepath.Join(c.workingDir, vendord) + if err := c.rmVendorDir(vendorDir); err != nil { + return err + } + + // Write the modules list to modules.txt. + // + // On the form: + // + // # github.com/alecthomas/chroma v0.6.3 + // + // This is how "go mod vendor" does it. Go also lists + // the packages below it, but that is currently not applicable to us. + // + var modulesContent bytes.Buffer + + tc, coll := c.collect(true) + if coll.err != nil { + return coll.err + } + + for _, t := range tc.AllModules { + if t.Owner() == nil { + // This is the project. + continue + } + // We respect the --ignoreVendor flag even for the vendor command. + if !t.IsGoMod() && !t.Vendor() { + // We currently do not vendor components living in the + // theme directory, see https://github.com/gohugoio/hugo/issues/5993 + continue + } + + fmt.Fprintln(&modulesContent, "# "+t.Path()+" "+t.Version()) + + dir := t.Dir() + + for _, mount := range t.Mounts() { + if err := hugio.CopyDir(c.fs, filepath.Join(dir, mount.Source), filepath.Join(vendorDir, t.Path(), mount.Source), nil); err != nil { + return errors.Wrap(err, "failed to copy module to vendor dir") + } + } + + // Include the resource cache if present. + resourcesDir := filepath.Join(dir, files.FolderResources) + _, err := c.fs.Stat(resourcesDir) + if err == nil { + if err := hugio.CopyDir(c.fs, resourcesDir, filepath.Join(vendorDir, t.Path(), files.FolderResources), nil); err != nil { + return errors.Wrap(err, "failed to copy resources to vendor dir") + } + } + + // Also include any theme.toml or config.* files in the root. + configFiles, _ := afero.Glob(c.fs, filepath.Join(dir, "config.*")) + configFiles = append(configFiles, filepath.Join(dir, "theme.toml")) + for _, configFile := range configFiles { + if err := hugio.CopyFile(c.fs, configFile, filepath.Join(vendorDir, t.Path(), filepath.Base(configFile))); err != nil { + if !os.IsNotExist(err) { + return err + } + } + } + } + + if modulesContent.Len() > 0 { + if err := afero.WriteFile(c.fs, filepath.Join(vendorDir, vendorModulesFilename), modulesContent.Bytes(), 0666); err != nil { + return err + } + } + + return nil +} + +// Get runs "go get" with the supplied arguments. +func (c *Client) Get(args ...string) error { + if err := c.runGo(context.Background(), os.Stdout, append([]string{"get"}, args...)...); err != nil { + errors.Wrapf(err, "failed to get %q", args) + } + return nil +} + +// Init initializes this as a Go Module with the given path. +// If path is empty, Go will try to guess. +// If this succeeds, this project will be marked as Go Module. +func (c *Client) Init(path string) error { + err := c.runGo(context.Background(), os.Stdout, "mod", "init", path) + if err != nil { + return errors.Wrap(err, "failed to init modules") + } + + c.GoModulesFilename = filepath.Join(c.workingDir, goModFilename) + + return nil +} + +func (c *Client) isProbablyModule(path string) bool { + return module.CheckPath(path) == nil +} + +func (c *Client) listGoMods() (goModules, error) { + if c.GoModulesFilename == "" { + return nil, nil + } + + out := ioutil.Discard + err := c.runGo(context.Background(), out, "mod", "download") + if err != nil { + return nil, errors.Wrap(err, "failed to download modules") + } + + b := &bytes.Buffer{} + err = c.runGo(context.Background(), b, "list", "-m", "-json", "all") + if err != nil { + return nil, errors.Wrap(err, "failed to list modules") + } + + var modules goModules + + dec := json.NewDecoder(b) + for { + m := &goModule{} + if err := dec.Decode(m); err != nil { + if err == io.EOF { + break + } + return nil, errors.Wrap(err, "failed to decode modules list") + } + + modules = append(modules, m) + } + + return modules, err + +} + +func (c *Client) rewriteGoMod(name string, isGoMod map[string]bool) error { + data, err := c.rewriteGoModRewrite(name, isGoMod) + if err != nil { + return err + } + if data != nil { + if err := afero.WriteFile(c.fs, filepath.Join(c.workingDir, name), data, 0666); err != nil { + return err + } + } + + return nil +} + +func (c *Client) rewriteGoModRewrite(name string, isGoMod map[string]bool) ([]byte, error) { + if name == goModFilename && c.GoModulesFilename == "" { + // Already checked. + return nil, nil + } + + modlineSplitter := getModlineSplitter(name == goModFilename) + + b := &bytes.Buffer{} + f, err := c.fs.Open(filepath.Join(c.workingDir, name)) + if err != nil { + if os.IsNotExist(err) { + // It's been deleted. + return nil, nil + } + return nil, err + } + defer f.Close() + + scanner := bufio.NewScanner(f) + var dirty bool + + for scanner.Scan() { + line := scanner.Text() + var doWrite bool + + if parts := modlineSplitter(line); parts != nil { + modname, modver := parts[0], parts[1] + modver = strings.TrimSuffix(modver, "/"+goModFilename) + modnameVer := modname + " " + modver + doWrite = isGoMod[modnameVer] + } else { + doWrite = true + } + + if doWrite { + fmt.Fprintln(b, line) + } else { + dirty = true + } + } + + if !dirty { + // Nothing changed + return nil, nil + } + + return b.Bytes(), nil + +} + +func (c *Client) rmVendorDir(vendorDir string) error { + modulestxt := filepath.Join(vendorDir, vendorModulesFilename) + + if _, err := c.fs.Stat(vendorDir); err != nil { + return nil + } + + _, err := c.fs.Stat(modulestxt) + if err != nil { + // If we have a _vendor dir without modules.txt it sounds like + // a _vendor dir created by others. + return errors.New("found _vendor dir without modules.txt, skip delete") + } + + return c.fs.RemoveAll(vendorDir) +} + +func (c *Client) runGo( + ctx context.Context, + stdout io.Writer, + args ...string) error { + + if c.goBinaryStatus != 0 { + return nil + } + + stderr := new(bytes.Buffer) + cmd := exec.CommandContext(ctx, "go", args...) + + cmd.Env = c.environ + cmd.Dir = c.workingDir + cmd.Stdout = stdout + cmd.Stderr = io.MultiWriter(stderr, os.Stderr) + + if err := cmd.Run(); err != nil { + if ee, ok := err.(*exec.Error); ok && ee.Err == exec.ErrNotFound { + c.goBinaryStatus = goBinaryStatusNotFound + return nil + } + + _, ok := err.(*exec.ExitError) + if !ok { + return errors.Errorf("failed to execute 'go %v': %s %T", args, err, err) + } + + // Too old Go version + if strings.Contains(stderr.String(), "flag provided but not defined") { + c.goBinaryStatus = goBinaryStatusTooOld + return nil + } + + return errors.Errorf("go command failed: %s", stderr) + + } + + return nil +} + +func (c *Client) tidy(mods Modules, goModOnly bool) error { + isGoMod := make(map[string]bool) + for _, m := range mods { + if m.Owner() == nil { + continue + } + if m.IsGoMod() { + // Matching the format in go.mod + pathVer := m.Path() + " " + m.Version() + isGoMod[pathVer] = true + } + } + + if err := c.rewriteGoMod(goModFilename, isGoMod); err != nil { + return err + } + + if goModOnly { + return nil + } + + if err := c.rewriteGoMod(goSumFilename, isGoMod); err != nil { + return err + } + + return nil +} + +// ClientConfig configures the module Client. +type ClientConfig struct { + Fs afero.Fs + Logger *loggers.Logger + IgnoreVendor bool + WorkingDir string + ThemesDir string // Absolute directory path + CacheDir string // Module cache + ModuleConfig Config +} + +type goBinaryStatus int + +type goModule struct { + Path string // module path + Version string // module version + Versions []string // available module versions (with -versions) + Replace *goModule // replaced by this module + Time *time.Time // time version was created + Update *goModule // available update, if any (with -u) + Main bool // is this the main module? + Indirect bool // is this module only an indirect dependency of main module? + Dir string // directory holding files for this module, if any + GoMod string // path to go.mod file for this module, if any + Error *goModuleError // error loading module +} + +type goModuleError struct { + Err string // the error itself +} + +type goModules []*goModule + +func (modules goModules) GetByPath(p string) *goModule { + if modules == nil { + return nil + } + + for _, m := range modules { + if strings.EqualFold(p, m.Path) { + return m + } + } + + return nil +} + +func (modules goModules) GetMain() *goModule { + for _, m := range modules { + if m.Main { + return m + } + } + + return nil +} + +func getModlineSplitter(isGoMod bool) func(line string) []string { + if isGoMod { + return func(line string) []string { + if strings.HasPrefix(line, "require (") { + return nil + } + if !strings.HasPrefix(line, "require") && !strings.HasPrefix(line, "\t") { + return nil + } + line = strings.TrimPrefix(line, "require") + line = strings.TrimSpace(line) + line = strings.TrimSuffix(line, "// indirect") + + return strings.Fields(line) + } + } + + return func(line string) []string { + return strings.Fields(line) + } +} + +func pathVersion(m Module) string { + versionStr := m.Version() + if m.Vendor() { + versionStr += "+vendor" + } + if versionStr == "" { + return m.Path() + } + return fmt.Sprintf("%s@%s", m.Path(), versionStr) +} diff --git a/modules/client_test.go b/modules/client_test.go new file mode 100644 index 00000000000..d8301514d6b --- /dev/null +++ b/modules/client_test.go @@ -0,0 +1,117 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package modules + +import ( + "bytes" + "testing" + + "github.com/gohugoio/hugo/common/hugo" + + "github.com/gohugoio/hugo/htesting" + + "github.com/gohugoio/hugo/hugofs" + + "github.com/stretchr/testify/require" +) + +func TestClient(t *testing.T) { + if hugo.GoMinorVersion() < 12 { + // https://github.com/golang/go/issues/26794 + // There were some concurrent issues with Go modules in < Go 12. + t.Skip("skip this for Go <= 1.11 due to a bug in Go's stdlib") + } + + t.Parallel() + + modName := "hugo-modules-basic-test" + modPath := "github.com/gohugoio/tests/" + modName + modConfig := DefaultModuleConfig + modConfig.Imports = []Import{Import{Path: "github.com/gohugoio/hugoTestModules1_darwin/modh2_2"}} + + assert := require.New(t) + + workingDir, clean, err := htesting.CreateTempDir(hugofs.Os, modName) + assert.NoError(err) + defer clean() + + client := NewClient(ClientConfig{ + Fs: hugofs.Os, + WorkingDir: workingDir, + ModuleConfig: modConfig, + }) + + // Test Init + assert.NoError(client.Init(modPath)) + + // Test Collect + mc, err := client.Collect() + assert.NoError(err) + assert.Equal(4, len(mc.AllModules)) + for _, m := range mc.AllModules { + assert.NotNil(m) + } + + // Test Graph + var graphb bytes.Buffer + assert.NoError(client.Graph(&graphb)) + + expect := `github.com/gohugoio/tests/hugo-modules-basic-test github.com/gohugoio/hugoTestModules1_darwin/modh2_2@v1.4.0 +github.com/gohugoio/hugoTestModules1_darwin/modh2_2@v1.4.0 github.com/gohugoio/hugoTestModules1_darwin/modh2_2_1v@v1.3.0 +github.com/gohugoio/hugoTestModules1_darwin/modh2_2@v1.4.0 github.com/gohugoio/hugoTestModules1_darwin/modh2_2_2@v1.3.0 +` + + assert.Equal(expect, graphb.String()) + + // Test Vendor + assert.NoError(client.Vendor()) + graphb.Reset() + assert.NoError(client.Graph(&graphb)) + expectVendored := `github.com/gohugoio/tests/hugo-modules-basic-test github.com/gohugoio/hugoTestModules1_darwin/modh2_2@v1.4.0+vendor +github.com/gohugoio/tests/hugo-modules-basic-test github.com/gohugoio/hugoTestModules1_darwin/modh2_2_1v@v1.3.0+vendor +github.com/gohugoio/tests/hugo-modules-basic-test github.com/gohugoio/hugoTestModules1_darwin/modh2_2_2@v1.3.0+vendor +` + assert.Equal(expectVendored, graphb.String()) + + // Test the ignoreVendor setting + clientIgnoreVendor := NewClient(ClientConfig{ + Fs: hugofs.Os, + WorkingDir: workingDir, + ModuleConfig: modConfig, + IgnoreVendor: true, + }) + + graphb.Reset() + assert.NoError(clientIgnoreVendor.Graph(&graphb)) + assert.Equal(expect, graphb.String()) + + // Test Tidy + assert.NoError(client.Tidy()) + +} + +func TestGetModlineSplitter(t *testing.T) { + + assert := require.New(t) + + gomodSplitter := getModlineSplitter(true) + + assert.Equal([]string{"github.com/BurntSushi/toml", "v0.3.1"}, gomodSplitter("\tgithub.com/BurntSushi/toml v0.3.1")) + assert.Equal([]string{"github.com/cpuguy83/go-md2man", "v1.0.8"}, gomodSplitter("\tgithub.com/cpuguy83/go-md2man v1.0.8 // indirect")) + assert.Nil(gomodSplitter("require (")) + + gosumSplitter := getModlineSplitter(false) + assert.Equal([]string{"github.com/BurntSushi/toml", "v0.3.1"}, gosumSplitter("github.com/BurntSushi/toml v0.3.1")) + +} diff --git a/modules/collect.go b/modules/collect.go new file mode 100644 index 00000000000..f57b4d04b36 --- /dev/null +++ b/modules/collect.go @@ -0,0 +1,574 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package modules + +import ( + "bufio" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/spf13/cast" + + "github.com/gohugoio/hugo/common/maps" + + "github.com/gohugoio/hugo/common/hugo" + "github.com/gohugoio/hugo/parser/metadecoders" + + "github.com/gohugoio/hugo/hugofs/files" + + "github.com/rogpeppe/go-internal/module" + + "github.com/pkg/errors" + + "github.com/gohugoio/hugo/config" + "github.com/spf13/afero" +) + +var ErrNotExist = errors.New("module does not exist") + +const vendorModulesFilename = "modules.txt" + +// IsNotExist returns whether an error means that a module could not be found. +func IsNotExist(err error) bool { + return errors.Cause(err) == ErrNotExist +} + +// CreateProjectModule creates modules from the given config. +// This is used in tests only. +func CreateProjectModule(cfg config.Provider) (Module, error) { + workingDir := cfg.GetString("workingDir") + var modConfig Config + + mod := createProjectModule(nil, workingDir, modConfig) + if err := ApplyProjectConfigDefaults(cfg, mod); err != nil { + return nil, err + } + + return mod, nil +} + +func (h *Client) Collect() (ModulesConfig, error) { + mc, coll := h.collect(true) + return mc, coll.err + +} + +func (h *Client) collect(tidy bool) (ModulesConfig, *collector) { + c := &collector{ + Client: h, + } + + c.collect() + if c.err != nil { + return ModulesConfig{}, c + } + + if !c.skipTidy && tidy { + if err := h.tidy(c.modules, true); err != nil { + c.err = err + return ModulesConfig{}, c + } + } + + // TODO(bep) consider --ignoreVendor vs removing from go.mod + var activeMods Modules + for _, mod := range c.modules { + if !mod.Config().HugoVersion.IsValid() { + h.logger.WARN.Printf(`Module %q is not compatible with this Hugo version; run "hugo mod graph" for more information.`, mod.Path()) + } + if !mod.Disabled() { + activeMods = append(activeMods, mod) + } + } + + return ModulesConfig{ + AllModules: c.modules, + ActiveModules: activeMods, + GoModulesFilename: c.GoModulesFilename, + }, c + +} + +type ModulesConfig struct { + // All modules, including any disabled. + AllModules Modules + + // All active modules. + ActiveModules Modules + + // Set if this is a Go modules enabled project. + GoModulesFilename string +} + +type collected struct { + // Pick the first and prevent circular loops. + seen map[string]bool + + // Maps module path to a _vendor dir. These values are fetched from + // _vendor/modules.txt, and the first (top-most) will win. + vendored map[string]vendoredModule + + // Set if a Go modules enabled project. + gomods goModules + + // Ordered list of collected modules, including Go Modules and theme + // components stored below /themes. + modules Modules +} + +// Collects and creates a module tree. +type collector struct { + *Client + + // Store away any non-fatal error and return at the end. + err error + + // Set to disable any Tidy operation in the end. + skipTidy bool + + *collected +} + +func (c *collector) initModules() error { + c.collected = &collected{ + seen: make(map[string]bool), + vendored: make(map[string]vendoredModule), + } + + // We may fail later if we don't find the mods. + return c.loadModules() +} + +func (c *collector) isSeen(path string) bool { + key := pathKey(path) + if c.seen[key] { + return true + } + c.seen[key] = true + return false +} + +func (c *collector) getVendoredDir(path string) (vendoredModule, bool) { + v, found := c.vendored[path] + return v, found +} + +func (c *collector) add(owner *moduleAdapter, moduleImport Import, disabled bool) (*moduleAdapter, error) { + var ( + mod *goModule + moduleDir string + version string + vendored bool + ) + + modulePath := moduleImport.Path + var realOwner Module = owner + + if !c.ignoreVendor { + if err := c.collectModulesTXT(owner); err != nil { + return nil, err + } + + // Try _vendor first. + var vm vendoredModule + vm, vendored = c.getVendoredDir(modulePath) + if vendored { + moduleDir = vm.Dir + realOwner = vm.Owner + version = vm.Version + + if owner.projectMod { + // We want to keep the go.mod intact with the versions and all. + c.skipTidy = true + } + + } + } + + if moduleDir == "" { + mod = c.gomods.GetByPath(modulePath) + if mod != nil { + moduleDir = mod.Dir + } + + if moduleDir == "" { + + if c.GoModulesFilename != "" && c.isProbablyModule(modulePath) { + // Try to "go get" it and reload the module configuration. + if err := c.Get(modulePath); err != nil { + return nil, err + } + if err := c.loadModules(); err != nil { + return nil, err + } + + mod = c.gomods.GetByPath(modulePath) + if mod != nil { + moduleDir = mod.Dir + } + } + + // Fall back to /themes/ + if moduleDir == "" { + moduleDir = filepath.Join(c.themesDir, modulePath) + + if found, _ := afero.Exists(c.fs, moduleDir); !found { + c.err = c.wrapModuleNotFound(errors.Errorf(`module %q not found; either add it as a Hugo Module or store it in %q.`, modulePath, c.themesDir)) + return nil, nil + } + } + } + } + + if found, _ := afero.Exists(c.fs, moduleDir); !found { + c.err = c.wrapModuleNotFound(errors.Errorf("%q not found", moduleDir)) + return nil, nil + } + + if !strings.HasSuffix(moduleDir, fileSeparator) { + moduleDir += fileSeparator + } + + ma := &moduleAdapter{ + dir: moduleDir, + vendor: vendored, + disabled: disabled, + gomod: mod, + version: version, + // This may be the owner of the _vendor dir + owner: realOwner, + } + + if mod == nil { + ma.path = modulePath + } + + if err := ma.validateAndApplyDefaults(c.fs); err != nil { + return nil, err + } + + if !moduleImport.IgnoreConfig { + if err := c.applyThemeConfig(ma); err != nil { + return nil, err + } + } + + if err := c.applyMounts(moduleImport, ma); err != nil { + return nil, err + } + + c.modules = append(c.modules, ma) + return ma, nil + +} + +func (c *collector) addAndRecurse(owner *moduleAdapter, disabled bool) error { + moduleConfig := owner.Config() + if owner.projectMod { + if err := c.applyMounts(Import{}, owner); err != nil { + return err + } + } + + for _, moduleImport := range moduleConfig.Imports { + disabled := disabled || moduleImport.Disabled + + if !c.isSeen(moduleImport.Path) { + tc, err := c.add(owner, moduleImport, disabled) + if err != nil { + return err + } + if tc == nil { + continue + } + if err := c.addAndRecurse(tc, disabled); err != nil { + return err + } + } + } + return nil +} + +func (c *collector) applyMounts(moduleImport Import, mod *moduleAdapter) error { + mounts := moduleImport.Mounts + + if !mod.projectMod && len(mounts) == 0 { + modConfig := mod.Config() + mounts = modConfig.Mounts + if len(mounts) == 0 { + // Create default mount points for every component folder that + // exists in the module. + for _, componentFolder := range files.ComponentFolders { + sourceDir := filepath.Join(mod.Dir(), componentFolder) + _, err := c.fs.Stat(sourceDir) + if err == nil { + mounts = append(mounts, Mount{ + Source: componentFolder, + Target: componentFolder, + }) + } + } + } + } + + var err error + mounts, err = c.normalizeMounts(mod, mounts) + if err != nil { + return err + } + + mod.mounts = mounts + return nil +} + +func (c *collector) applyThemeConfig(tc *moduleAdapter) error { + + var ( + configFilename string + cfg config.Provider + themeCfg map[string]interface{} + hasConfig bool + err error + ) + + // Viper supports more, but this is the sub-set supported by Hugo. + for _, configFormats := range config.ValidConfigFileExtensions { + configFilename = filepath.Join(tc.Dir(), "config."+configFormats) + hasConfig, _ = afero.Exists(c.fs, configFilename) + if hasConfig { + break + } + } + + // The old theme information file. + themeTOML := filepath.Join(tc.Dir(), "theme.toml") + + hasThemeTOML, _ := afero.Exists(c.fs, themeTOML) + if hasThemeTOML { + data, err := afero.ReadFile(c.fs, themeTOML) + if err != nil { + return err + } + themeCfg, err = metadecoders.Default.UnmarshalToMap(data, metadecoders.TOML) + if err != nil { + return errors.Wrapf(err, "failed to read module config for %q in %q", tc.Path(), themeTOML) + } + maps.ToLower(themeCfg) + } + + if hasConfig { + if configFilename != "" { + var err error + cfg, err = config.FromFile(c.fs, configFilename) + if err != nil { + return errors.Wrapf(err, "failed to read module config for %q in %q", tc.Path(), configFilename) + } + } + + tc.configFilename = configFilename + tc.cfg = cfg + } + + config, err := DecodeConfig(cfg) + if err != nil { + return err + } + + const oldVersionKey = "min_version" + + if hasThemeTOML { + + // Merge old with new + if minVersion, found := themeCfg[oldVersionKey]; found { + if config.HugoVersion.Min == "" { + config.HugoVersion.Min = hugo.VersionString(cast.ToString(minVersion)) + } + } + + if config.Params == nil { + config.Params = make(map[string]interface{}) + } + + for k, v := range themeCfg { + if k == oldVersionKey { + continue + } + config.Params[k] = v + } + + } + + tc.config = config + + return nil + +} + +func (c *collector) collect() { + if err := c.initModules(); err != nil { + c.err = err + return + } + + projectMod := createProjectModule(c.gomods.GetMain(), c.workingDir, c.moduleConfig) + + if err := c.addAndRecurse(projectMod, false); err != nil { + c.err = err + return + } + + // Append the project module at the tail. + c.modules = append(c.modules, projectMod) + +} + +func (c *collector) collectModulesTXT(owner Module) error { + vendorDir := filepath.Join(owner.Dir(), vendord) + filename := filepath.Join(vendorDir, vendorModulesFilename) + + f, err := c.fs.Open(filename) + + if err != nil { + if os.IsNotExist(err) { + return nil + } + + return err + } + + defer f.Close() + + scanner := bufio.NewScanner(f) + + for scanner.Scan() { + // # github.com/alecthomas/chroma v0.6.3 + line := scanner.Text() + line = strings.Trim(line, "# ") + line = strings.TrimSpace(line) + parts := strings.Fields(line) + if len(parts) != 2 { + return errors.Errorf("invalid modules list: %q", filename) + } + path := parts[0] + if _, found := c.vendored[path]; !found { + c.vendored[path] = vendoredModule{ + Owner: owner, + Dir: filepath.Join(vendorDir, path), + Version: parts[1], + } + } + + } + return nil +} + +func (c *collector) loadModules() error { + modules, err := c.listGoMods() + if err != nil { + return err + } + c.gomods = modules + return nil +} + +func (c *collector) normalizeMounts(owner Module, mounts []Mount) ([]Mount, error) { + var out []Mount + dir := owner.Dir() + + for _, mnt := range mounts { + errMsg := fmt.Sprintf("invalid module config for %q", owner.Path()) + + if mnt.Source == "" || mnt.Target == "" { + return nil, errors.New(errMsg + ": both source and target must be set") + } + + mnt.Source = filepath.Clean(mnt.Source) + mnt.Target = filepath.Clean(mnt.Target) + + // Verify that Source exists + sourceDir := filepath.Join(dir, mnt.Source) + _, err := c.fs.Stat(sourceDir) + if err != nil { + continue + } + + // Verify that target points to one of the predefined component dirs + targetBase := mnt.Target + idxPathSep := strings.Index(mnt.Target, string(os.PathSeparator)) + if idxPathSep != -1 { + targetBase = mnt.Target[0:idxPathSep] + } + if !files.IsComponentFolder(targetBase) { + return nil, errors.Errorf("%s: mount target must be one of: %v", errMsg, files.ComponentFolders) + } + + out = append(out, mnt) + } + + return out, nil +} + +func (c *collector) wrapModuleNotFound(err error) error { + err = errors.Wrap(ErrNotExist, err.Error()) + if c.GoModulesFilename == "" { + return err + } + + baseMsg := "we found a go.mod file in your project, but" + + switch c.goBinaryStatus { + case goBinaryStatusNotFound: + return errors.Wrap(err, baseMsg+" you need to install Go to use it. See https://golang.org/dl/.") + case goBinaryStatusTooOld: + return errors.Wrap(err, baseMsg+" you need to a newer version of Go to use it. See https://golang.org/dl/.") + } + + return err + +} + +type vendoredModule struct { + Owner Module + Dir string + Version string +} + +func createProjectModule(gomod *goModule, workingDir string, conf Config) *moduleAdapter { + // Create a pseudo module for the main project. + var path string + if gomod == nil { + path = "project" + } + + return &moduleAdapter{ + path: path, + dir: workingDir, + gomod: gomod, + projectMod: true, + config: conf, + } + +} + +// In the first iteration of Hugo Modules, we do not support multiple +// major versions running at the same time, so we pick the first (upper most). +// We will investigate namespaces in future versions. +// TODO(bep) add a warning when the above happens. +func pathKey(p string) string { + prefix, _, _ := module.SplitPathVersion(p) + + return strings.ToLower(prefix) +} diff --git a/modules/collect_test.go b/modules/collect_test.go new file mode 100644 index 00000000000..d76c0b2bbed --- /dev/null +++ b/modules/collect_test.go @@ -0,0 +1,38 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package modules + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestPathKey(t *testing.T) { + assert := require.New(t) + + for _, test := range []struct { + in string + expect string + }{ + {"github.com/foo", "github.com/foo"}, + {"github.com/foo/v2", "github.com/foo"}, + {"github.com/foo/v12", "github.com/foo"}, + {"github.com/foo/v3d", "github.com/foo/v3d"}, + {"MyTheme", "mytheme"}, + } { + assert.Equal(test.expect, pathKey(test.in)) + } + +} diff --git a/modules/config.go b/modules/config.go new file mode 100644 index 00000000000..b084863d4d4 --- /dev/null +++ b/modules/config.go @@ -0,0 +1,335 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package modules + +import ( + "fmt" + "path" + "path/filepath" + "strings" + + "github.com/gohugoio/hugo/common/hugo" + + "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/hugofs/files" + "github.com/gohugoio/hugo/langs" + "github.com/mitchellh/mapstructure" +) + +var DefaultModuleConfig = Config{ + + // Default to direct, which means "git clone" and similar. We + // will investigate proxy settings in more depth later. + // See https://github.com/golang/go/issues/26334 + Proxy: "direct", + + // Comma separated glob list matching paths that should not use the + // proxy configured above. + NoProxy: "none", + + // Comma separated glob list matching paths that should be + // treated as private. + Private: "*.*", +} + +// ApplyProjectConfigDefaults applies default/missing module configuration for +// the main project. +func ApplyProjectConfigDefaults(cfg config.Provider, mod Module) error { + moda := mod.(*moduleAdapter) + + // Map legacy directory config into the new module. + languages := cfg.Get("languagesSortedDefaultFirst").(langs.Languages) + isMultiHost := languages.IsMultihost() + + // To bridge between old and new configuration format we need + // a way to make sure all of the core components are configured on + // the basic level. + componentsConfigured := make(map[string]bool) + for _, mnt := range moda.mounts { + componentsConfigured[mnt.Component()] = true + } + + type dirKeyComponent struct { + key string + component string + multilingual bool + } + + dirKeys := []dirKeyComponent{ + {"contentDir", files.ComponentFolderContent, true}, + {"dataDir", files.ComponentFolderData, false}, + {"layoutDir", files.ComponentFolderLayouts, false}, + {"i18nDir", files.ComponentFolderI18n, false}, + {"archetypeDir", files.ComponentFolderArchetypes, false}, + {"assetDir", files.ComponentFolderAssets, false}, + {"", files.ComponentFolderStatic, isMultiHost}, + } + + createMountsFor := func(d dirKeyComponent, cfg config.Provider) []Mount { + var lang string + if language, ok := cfg.(*langs.Language); ok { + lang = language.Lang + } + + // Static mounts are a little special. + if d.component == files.ComponentFolderStatic { + var mounts []Mount + staticDirs := getStaticDirs(cfg) + if len(staticDirs) > 0 { + componentsConfigured[d.component] = true + } + + for _, dir := range staticDirs { + mounts = append(mounts, Mount{Lang: lang, Source: dir, Target: d.component}) + } + + return mounts + + } + + if cfg.IsSet(d.key) { + source := cfg.GetString(d.key) + componentsConfigured[d.component] = true + + return []Mount{Mount{ + // No lang set for layouts etc. + Source: source, + Target: d.component}} + } + + return nil + } + + createMounts := func(d dirKeyComponent) []Mount { + var mounts []Mount + if d.multilingual { + if d.component == files.ComponentFolderContent { + seen := make(map[string]bool) + hasContentDir := false + for _, language := range languages { + if language.ContentDir != "" { + hasContentDir = true + break + } + } + + if hasContentDir { + for _, language := range languages { + contentDir := language.ContentDir + if contentDir == "" { + contentDir = files.ComponentFolderContent + } + if contentDir == "" || seen[contentDir] { + continue + } + seen[contentDir] = true + mounts = append(mounts, Mount{Lang: language.Lang, Source: contentDir, Target: d.component}) + } + } + + componentsConfigured[d.component] = len(seen) > 0 + + } else { + for _, language := range languages { + mounts = append(mounts, createMountsFor(d, language)...) + } + } + } else { + mounts = append(mounts, createMountsFor(d, cfg)...) + } + + return mounts + } + + var mounts []Mount + for _, dirKey := range dirKeys { + if componentsConfigured[dirKey.component] { + + continue + } + + mounts = append(mounts, createMounts(dirKey)...) + + } + + // Add default configuration + for _, dirKey := range dirKeys { + if componentsConfigured[dirKey.component] { + continue + } + mounts = append(mounts, Mount{Source: dirKey.component, Target: dirKey.component}) + } + + // Remove duplicates + seen := make(map[string]bool) + tmp := mounts[:0] + for _, m := range mounts { + key := path.Join(m.Lang, m.Source, m.Target) + if !seen[key] { + tmp = append(tmp, m) + } + seen[key] = true + } + + moda.mounts = tmp + + return nil +} + +// DecodeConfig creates a modules Config from a given Hugo configuration. +func DecodeConfig(cfg config.Provider) (Config, error) { + c := DefaultModuleConfig + + if cfg == nil { + return c, nil + } + + themeSet := cfg.IsSet("theme") + moduleSet := cfg.IsSet("module") + + if moduleSet { + m := cfg.GetStringMap("module") + if err := mapstructure.WeakDecode(m, &c); err != nil { + return c, err + } + + for i, mnt := range c.Mounts { + mnt.Source = filepath.Clean(mnt.Source) + mnt.Target = filepath.Clean(mnt.Target) + c.Mounts[i] = mnt + } + + } + + if themeSet { + imports := config.GetStringSlicePreserveString(cfg, "theme") + for _, imp := range imports { + c.Imports = append(c.Imports, Import{ + Path: imp, + }) + } + + } + + return c, nil +} + +// Config holds a module config. +type Config struct { + Mounts []Mount + Imports []Import + + // Meta info about this module (license information etc.). + Params map[string]interface{} + + // Will be validated against the running Hugo version. + HugoVersion HugoVersion + + // Configures GOPROXY. + Proxy string + // Configures GONOPROXY. + NoProxy string + // Configures GOPRIVATE. + Private string +} + +// HugoVersion holds Hugo binary version requirements for a module. +type HugoVersion struct { + // The minimum Hugo version that this module works with. + Min hugo.VersionString + + // The maxium Hugo version that this module works with. + Max hugo.VersionString + + // Set if the extended version is needed. + Extended bool +} + +func (v HugoVersion) String() string { + extended := "" + if v.Extended { + extended = " extended" + } + + if v.Min != "" && v.Max != "" { + return fmt.Sprintf("%s/%s%s", v.Min, v.Max, extended) + } + + if v.Min != "" { + return fmt.Sprintf("Min %s%s", v.Min, extended) + } + + if v.Max != "" { + return fmt.Sprintf("Max %s%s", v.Max, extended) + } + + return extended +} + +// IsValid reports whether this version is valid compared to the running +// Hugo binary. +func (v HugoVersion) IsValid() bool { + current := hugo.CurrentVersion.Version() + if v.Extended && !hugo.IsExtended { + return false + } + + isValid := true + + if v.Min != "" && current.Compare(v.Min) > 0 { + isValid = false + } + + if v.Max != "" && current.Compare(v.Max) < 0 { + isValid = false + } + + return isValid +} + +type Import struct { + Path string // Module path + IgnoreConfig bool // Ignore any config.toml found. + Disabled bool // Turn off this module. + Mounts []Mount +} + +type Mount struct { + Source string // relative path in source repo, e.g. "scss" + Target string // relative target path, e.g. "assets/bootstrap/scss" + + Lang string // any language code associated with this mount. +} + +func (m Mount) Component() string { + return strings.Split(m.Target, fileSeparator)[0] +} + +func getStaticDirs(cfg config.Provider) []string { + var staticDirs []string + for i := -1; i <= 10; i++ { + staticDirs = append(staticDirs, getStringOrStringSlice(cfg, "staticDir", i)...) + } + return staticDirs +} + +func getStringOrStringSlice(cfg config.Provider, key string, id int) []string { + + if id >= 0 { + key = fmt.Sprintf("%s%d", key, id) + } + + return config.GetStringSlicePreserveString(cfg, key) + +} diff --git a/modules/config_test.go b/modules/config_test.go new file mode 100644 index 00000000000..e99944b756e --- /dev/null +++ b/modules/config_test.go @@ -0,0 +1,132 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package modules + +import ( + "fmt" + "testing" + + "github.com/gohugoio/hugo/common/hugo" + + "github.com/gohugoio/hugo/config" + + "github.com/stretchr/testify/require" +) + +func TestConfigHugoVersionIsValid(t *testing.T) { + assert := require.New(t) + + for i, test := range []struct { + in HugoVersion + expect bool + }{ + {HugoVersion{Min: "0.33.0"}, true}, + {HugoVersion{Min: "0.56.0-DEV"}, true}, + {HugoVersion{Min: "0.33.0", Max: "0.55.0"}, false}, + {HugoVersion{Min: "0.33.0", Max: "0.99.0"}, true}, + } { + assert.Equal(test.expect, test.in.IsValid(), fmt.Sprintf("test %d", i)) + } +} + +func TestDecodeConfig(t *testing.T) { + assert := require.New(t) + tomlConfig := ` +[module] + +[module.hugoVersion] +min = "0.54.2" +max = "0.99.0" +extended = true + +[[module.mounts]] +source="src/project/blog" +target="content/blog" +lang="en" +[[module.imports]] +path="github.com/bep/mycomponent" +[[module.imports.mounts]] +source="scss" +target="assets/bootstrap/scss" +[[module.imports.mounts]] +source="src/markdown/blog" +target="content/blog" +lang="en" +` + cfg, err := config.FromConfigString(tomlConfig, "toml") + assert.NoError(err) + + mcfg, err := DecodeConfig(cfg) + assert.NoError(err) + + v056 := hugo.VersionString("0.56.0") + + hv := mcfg.HugoVersion + + assert.Equal(-1, v056.Compare(hv.Min)) + assert.Equal(1, v056.Compare(hv.Max)) + assert.True(hv.Extended) + + if hugo.IsExtended { + assert.True(hv.IsValid()) + } + + assert.Len(mcfg.Mounts, 1) + assert.Len(mcfg.Imports, 1) + imp := mcfg.Imports[0] + imp.Path = "github.com/bep/mycomponent" + assert.Equal("src/markdown/blog", imp.Mounts[1].Source) + assert.Equal("content/blog", imp.Mounts[1].Target) + assert.Equal("en", imp.Mounts[1].Lang) + +} + +func TestDecodeConfigBothOldAndNewProvided(t *testing.T) { + assert := require.New(t) + tomlConfig := ` + +theme = ["b", "c"] + +[module] +[[module.imports]] +path="a" + +` + cfg, err := config.FromConfigString(tomlConfig, "toml") + assert.NoError(err) + + modCfg, err := DecodeConfig(cfg) + assert.NoError(err) + assert.Len(modCfg.Imports, 3) + assert.Equal("a", modCfg.Imports[0].Path) + +} + +// Test old style theme import. +func TestDecodeConfigTheme(t *testing.T) { + assert := require.New(t) + tomlConfig := ` + +theme = ["a", "b"] +` + cfg, err := config.FromConfigString(tomlConfig, "toml") + assert.NoError(err) + + mcfg, err := DecodeConfig(cfg) + assert.NoError(err) + + assert.Len(mcfg.Imports, 2) + assert.Equal("a", mcfg.Imports[0].Path) + assert.Equal("b", mcfg.Imports[1].Path) +} diff --git a/modules/module.go b/modules/module.go new file mode 100644 index 00000000000..bba7657c5ea --- /dev/null +++ b/modules/module.go @@ -0,0 +1,196 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package modules provides a client that can be used to manage Hugo Components, +// what's refered to as Hugo Modules. Hugo Modules is built on top of Go Modules, +// but also supports vendoring and components stored directly in the themes dir. +package modules + +import ( + "github.com/gohugoio/hugo/config" + "github.com/spf13/afero" +) + +var _ Module = (*moduleAdapter)(nil) + +type Module interface { + + // Optional config read from the configFilename above. + Cfg() config.Provider + + // The decoded module config and mounts. + Config() Config + + // Optional configuration filename (e.g. "/themes/mytheme/config.json"). + // This will be added to the special configuration watch list when in + // server mode. + ConfigFilename() string + + // Directory holding files for this module. + Dir() string + + // This module is disabled. + Disabled() bool + + // Returns whether this is a Go Module. + IsGoMod() bool + + // Any directory remappings. + Mounts() []Mount + + // In the dependency tree, this is the first module that defines this module + // as a dependency. + Owner() Module + + // Returns the path to this module. + // This will either be the module path, e.g. "github.com/gohugoio/myshortcodes", + // or the path below your /theme folder, e.g. "mytheme". + Path() string + + // Replaced by this module. + Replace() Module + + // Returns whether Dir points below the _vendor dir. + Vendor() bool + + // The module version. + Version() string + + // Whether this module's dir is a watch candidate. + Watch() bool +} + +type Modules []Module + +type moduleAdapter struct { + path string + dir string + version string + vendor bool + disabled bool + projectMod bool + owner Module + + mounts []Mount + + configFilename string + cfg config.Provider + config Config + + // Set if a Go module. + gomod *goModule +} + +func (m *moduleAdapter) Cfg() config.Provider { + return m.cfg +} + +func (m *moduleAdapter) Config() Config { + return m.config +} + +func (m *moduleAdapter) ConfigFilename() string { + return m.configFilename +} + +func (m *moduleAdapter) Dir() string { + // This may point to the _vendor dir. + if !m.IsGoMod() || m.dir != "" { + return m.dir + } + return m.gomod.Dir +} + +func (m *moduleAdapter) Disabled() bool { + return m.disabled +} + +func (m *moduleAdapter) IsGoMod() bool { + return m.gomod != nil +} + +func (m *moduleAdapter) Mounts() []Mount { + return m.mounts +} + +func (m *moduleAdapter) Owner() Module { + return m.owner +} + +func (m *moduleAdapter) Path() string { + if !m.IsGoMod() || m.path != "" { + return m.path + } + return m.gomod.Path +} + +func (m *moduleAdapter) Replace() Module { + if m.IsGoMod() && !m.Vendor() && m.gomod.Replace != nil { + return &moduleAdapter{ + gomod: m.gomod.Replace, + owner: m.owner, + } + } + return nil +} + +func (m *moduleAdapter) Vendor() bool { + return m.vendor +} + +func (m *moduleAdapter) Version() string { + if !m.IsGoMod() || m.version != "" { + return m.version + } + return m.gomod.Version +} + +func (m *moduleAdapter) Watch() bool { + if m.Owner() == nil { + // Main project + return true + } + + if !m.IsGoMod() { + // Module inside /themes + return true + } + + if m.Replace() != nil { + // Version is not set when replaced by a local folder. + return m.Replace().Version() == "" + } + + return false +} + +func (m *moduleAdapter) validateAndApplyDefaults(fs afero.Fs) error { + + /*if len(m.modImport.Mounts) == 0 { + // Create default mount points for every component folder that + // exists in the module. + for _, componentFolder := range files.ComponentFolders { + sourceDir := filepath.Join(dir, componentFolder) + _, err := fs.Stat(sourceDir) + if err == nil { + m.modImport.Mounts = append(m.modImport.Mounts, Mount{ + Source: componentFolder, + Target: componentFolder, + }) + } + } + }*/ + + return nil + +} diff --git a/output/layout.go b/output/layout.go index 5d72938af58..055d742b15f 100644 --- a/output/layout.go +++ b/output/layout.go @@ -71,7 +71,7 @@ func (l *LayoutHandler) For(d LayoutDescriptor, f Format) ([]string, error) { layouts := resolvePageTemplate(d, f) layouts = prependTextPrefixIfNeeded(f, layouts...) - layouts = helpers.UniqueStrings(layouts) + layouts = helpers.UniqueStringsReuse(layouts) l.mu.Lock() l.cache[key] = layouts diff --git a/parser/metadecoders/decoder.go b/parser/metadecoders/decoder.go index e7b0e3c97b8..8af1b49e0ce 100644 --- a/parser/metadecoders/decoder.go +++ b/parser/metadecoders/decoder.go @@ -82,6 +82,30 @@ func (d Decoder) UnmarshalFileToMap(fs afero.Fs, filename string) (map[string]in return d.UnmarshalToMap(data, format) } +// UnmarshalStringTo tries to unmarshal data to a new instance of type typ. +func (d Decoder) UnmarshalStringTo(data string, typ interface{}) (interface{}, error) { + data = strings.TrimSpace(data) + // We only check for the possible types in YAML, JSON and TOML. + switch typ.(type) { + case string: + return data, nil + case map[string]interface{}: + format := d.FormatFromContentString(data) + return d.UnmarshalToMap([]byte(data), format) + case []interface{}: + // A standalone slice. Let YAML handle it. + return d.Unmarshal([]byte(data), YAML) + case int: + return cast.ToIntE(data) + case int64: + return cast.ToInt64E(data) + case float64: + return cast.ToFloat64E(data) + default: + return nil, errors.Errorf("unmarshal: %T not supportedd", typ) + } +} + // Unmarshal will unmarshall data in format f into an interface{}. // This is what's needed for Hugo's /data handling. func (d Decoder) Unmarshal(data []byte, f Format) (interface{}, error) { diff --git a/parser/metadecoders/decoder_test.go b/parser/metadecoders/decoder_test.go index 146df506900..7cb66d7360f 100644 --- a/parser/metadecoders/decoder_test.go +++ b/parser/metadecoders/decoder_test.go @@ -90,6 +90,38 @@ func TestUnmarshalToInterface(t *testing.T) { } +func TestUnmarshalStringTo(t *testing.T) { + assert := require.New(t) + + d := Default + + expectMap := map[string]interface{}{"a": "b"} + + for i, test := range []struct { + data string + to interface{} + expect interface{} + }{ + {"a string", "string", "a string"}, + {`{ "a": "b" }`, make(map[string]interface{}), expectMap}, + {"32", int64(1234), int64(32)}, + {"32", int(1234), int(32)}, + {"3.14159", float64(1), float64(3.14159)}, + {"[3,7,9]", []interface{}{}, []interface{}{3, 7, 9}}, + {"[3.1,7.2,9.3]", []interface{}{}, []interface{}{3.1, 7.2, 9.3}}, + } { + msg := fmt.Sprintf("%d: %T", i, test.to) + m, err := d.UnmarshalStringTo(test.data, test.to) + if b, ok := test.expect.(bool); ok && !b { + assert.Error(err, msg) + } else { + assert.NoError(err, msg) + assert.Equal(test.expect, m, msg) + } + + } +} + func TestStringifyYAMLMapKeys(t *testing.T) { cases := []struct { input interface{} diff --git a/resources/image_cache.go b/resources/image_cache.go index cf1e999badc..4072851e29d 100644 --- a/resources/image_cache.go +++ b/resources/image_cache.go @@ -145,7 +145,7 @@ func (c *imageCache) getOrCreate( } // The file is now stored in this cache. - img.overriddenSourceFs = c.fileCache.Fs + img.sourceFs = c.fileCache.Fs c.mu.Lock() if img2, found := c.store[key]; found { diff --git a/resources/page/page_nop.go b/resources/page/page_nop.go index 229bcb077e4..c3a4819f1f4 100644 --- a/resources/page/page_nop.go +++ b/resources/page/page_nop.go @@ -17,9 +17,10 @@ package page import ( "html/template" - "os" "time" + "github.com/gohugoio/hugo/hugofs" + "github.com/bep/gitmap" "github.com/gohugoio/hugo/navigation" @@ -147,7 +148,7 @@ func (p *nopPage) File() source.File { return nilFile } -func (p *nopPage) FileInfo() os.FileInfo { +func (p *nopPage) FileInfo() hugofs.FileMetaInfo { return nil } diff --git a/resources/page/page_wrappers.autogen.go b/resources/page/page_wrappers.autogen.go index d7fcb520109..d2d14dee6f1 100644 --- a/resources/page/page_wrappers.autogen.go +++ b/resources/page/page_wrappers.autogen.go @@ -18,8 +18,8 @@ package page import ( "github.com/gohugoio/hugo/common/hugo" "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/hugofs" "html/template" - "os" ) // NewDeprecatedWarningPage adds deprecation warnings to the given implementation. @@ -91,7 +91,7 @@ func (p *pageDeprecated) UniqueID() string { helpers.Deprecated("Page", ".UniqueID", "Use .File.UniqueID", false) return p.p.UniqueID() } -func (p *pageDeprecated) FileInfo() os.FileInfo { +func (p *pageDeprecated) FileInfo() hugofs.FileMetaInfo { helpers.Deprecated("Page", ".FileInfo", "Use .File.FileInfo", false) return p.p.FileInfo() } diff --git a/resources/page/pagemeta/page_frontmatter.go b/resources/page/pagemeta/page_frontmatter.go index 1ce3fbee4fc..7b9f13e622e 100644 --- a/resources/page/pagemeta/page_frontmatter.go +++ b/resources/page/pagemeta/page_frontmatter.go @@ -236,7 +236,7 @@ func addDateFieldAliases(values []string) []string { complete = append(complete, aliases...) } } - return helpers.UniqueStrings(complete) + return helpers.UniqueStringsReuse(complete) } func expandDefaultValues(values []string, defaults []string) []string { diff --git a/resources/page/permalinks.go b/resources/page/permalinks.go index 98489231b2c..59f2da91671 100644 --- a/resources/page/permalinks.go +++ b/resources/page/permalinks.go @@ -15,6 +15,7 @@ package page import ( "fmt" + "os" "path/filepath" "regexp" "strconv" @@ -90,7 +91,11 @@ func (l PermalinkExpander) parse(patterns map[string]string) (map[string]func(Pa expanders := make(map[string]func(Page) (string, error)) + // Allow " " and / to represent the root section. + const sectionCutSet = " /" + string(os.PathSeparator) + for k, pattern := range patterns { + k = strings.Trim(k, sectionCutSet) if !l.validate(pattern) { return nil, &permalinkExpandError{pattern: pattern, err: errPermalinkIllFormed} } diff --git a/resources/page/site.go b/resources/page/site.go index 25df063f1b1..9153c855614 100644 --- a/resources/page/site.go +++ b/resources/page/site.go @@ -17,6 +17,8 @@ import ( "html/template" "time" + "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/common/hugo" "github.com/gohugoio/hugo/langs" "github.com/gohugoio/hugo/navigation" @@ -51,3 +53,72 @@ func (s Sites) First() Site { } return s[0] } + +type testSite struct { + h hugo.Info + l *langs.Language +} + +func (t testSite) Hugo() hugo.Info { + return t.h +} + +func (t testSite) ServerPort() int { + return 1313 +} + +func (testSite) LastChange() (t time.Time) { + return +} + +func (t testSite) Title() string { + return "foo" +} + +func (t testSite) Sites() Sites { + return nil +} + +func (t testSite) IsServer() bool { + return false +} + +func (t testSite) Language() *langs.Language { + return t.l +} + +func (t testSite) Pages() Pages { + return nil +} + +func (t testSite) RegularPages() Pages { + return nil +} + +func (t testSite) Menus() navigation.Menus { + return nil +} + +func (t testSite) Taxonomies() interface{} { + return nil +} + +func (t testSite) BaseURL() template.URL { + return "" +} + +func (t testSite) Params() map[string]interface{} { + return nil +} + +func (t testSite) Data() map[string]interface{} { + return nil +} + +// NewDummyHugoSite creates a new minimal test site. +func NewDummyHugoSite(cfg config.Provider) Site { + return testSite{ + h: hugo.NewInfo(hugo.EnvironmentProduction), + l: langs.NewLanguage("en", cfg), + } +} diff --git a/resources/page/testhelpers_test.go b/resources/page/testhelpers_test.go index 60a6c0816fb..fa5f8e9c8a0 100644 --- a/resources/page/testhelpers_test.go +++ b/resources/page/testhelpers_test.go @@ -16,10 +16,11 @@ package page import ( "fmt" "html/template" - "os" "path/filepath" "time" + "github.com/gohugoio/hugo/modules" + "github.com/bep/gitmap" "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/resources/resource" @@ -65,6 +66,12 @@ func newTestPathSpec() *helpers.PathSpec { func newTestPathSpecFor(cfg config.Provider) *helpers.PathSpec { config.SetBaseTestDefaults(cfg) + langs.LoadLanguageSettings(cfg, nil) + mod, err := modules.CreateProjectModule(cfg) + if err != nil { + panic(err) + } + cfg.Set("allModules", modules.Modules{mod}) fs := hugofs.NewMem(cfg) s, err := helpers.NewPathSpec(fs, cfg) if err != nil { @@ -189,7 +196,7 @@ func (p *testPage) File() source.File { return p.file } -func (p *testPage) FileInfo() os.FileInfo { +func (p *testPage) FileInfo() hugofs.FileMetaInfo { panic("not implemented") } diff --git a/resources/page/zero_file.autogen.go b/resources/page/zero_file.autogen.go index eec1dd66dc0..23e36b76490 100644 --- a/resources/page/zero_file.autogen.go +++ b/resources/page/zero_file.autogen.go @@ -17,8 +17,8 @@ package page import ( "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/source" - "os" ) // ZeroFile represents a zero value of source.File with warnings if invoked. @@ -82,7 +82,7 @@ func (z zeroFile) UniqueID() (o0 string) { z.log.Println(".File.UniqueID on zero object. Wrap it in if or with: {{ with .File }}{{ .UniqueID }}{{ end }}") return } -func (z zeroFile) FileInfo() (o0 os.FileInfo) { +func (z zeroFile) FileInfo() (o0 hugofs.FileMetaInfo) { z.log.Println(".File.FileInfo on zero object. Wrap it in if or with: {{ with .File }}{{ .FileInfo }}{{ end }}") return } diff --git a/resources/resource.go b/resources/resource.go index c120a8dd090..0425bb7d8c1 100644 --- a/resources/resource.go +++ b/resources/resource.go @@ -24,6 +24,8 @@ import ( "strings" "sync" + "github.com/gohugoio/hugo/hugofs" + "github.com/gohugoio/hugo/media" "github.com/gohugoio/hugo/output" @@ -133,9 +135,13 @@ type ResourceSourceDescriptor struct { SourceFile source.File OpenReadSeekCloser resource.OpenReadSeekCloser + FileInfo os.FileInfo + // If OpenReadSeekerCloser is not set, we use this to open the file. SourceFilename string + Fs afero.Fs + // The relative target filename without any language code. RelTargetFilename string @@ -157,19 +163,11 @@ func (r ResourceSourceDescriptor) Filename() string { return r.SourceFilename } -func (r *Spec) sourceFs() afero.Fs { - return r.PathSpec.BaseFs.Content.Fs -} - func (r *Spec) New(fd ResourceSourceDescriptor) (resource.Resource, error) { - return r.newResourceForFs(r.sourceFs(), fd) -} - -func (r *Spec) NewForFs(sourceFs afero.Fs, fd ResourceSourceDescriptor) (resource.Resource, error) { - return r.newResourceForFs(sourceFs, fd) + return r.newResourceFor(fd) } -func (r *Spec) newResourceForFs(sourceFs afero.Fs, fd ResourceSourceDescriptor) (resource.Resource, error) { +func (r *Spec) newResourceFor(fd ResourceSourceDescriptor) (resource.Resource, error) { if fd.OpenReadSeekCloser == nil { if fd.SourceFile != nil && fd.SourceFilename != "" { return nil, errors.New("both SourceFile and AbsSourceFilename provided") @@ -187,15 +185,14 @@ func (r *Spec) newResourceForFs(sourceFs afero.Fs, fd ResourceSourceDescriptor) fd.TargetBasePaths = r.MultihostTargetBasePaths } - return r.newResource(sourceFs, fd) + return r.newResource(fd.Fs, fd) } func (r *Spec) newResource(sourceFs afero.Fs, fd ResourceSourceDescriptor) (resource.Resource, error) { - var fi os.FileInfo + fi := fd.FileInfo var sourceFilename string if fd.OpenReadSeekCloser != nil { - } else if fd.SourceFilename != "" { var err error fi, err = sourceFs.Stat(fd.SourceFilename) @@ -207,7 +204,6 @@ func (r *Spec) newResource(sourceFs afero.Fs, fd ResourceSourceDescriptor) (reso } sourceFilename = fd.SourceFilename } else { - fi = fd.SourceFile.FileInfo() sourceFilename = fd.SourceFile.Filename() } @@ -245,8 +241,6 @@ func (r *Spec) newResource(sourceFs afero.Fs, fd ResourceSourceDescriptor) (reso mimeType) if mimeType.MainType == "image" { - ext := strings.ToLower(helpers.Ext(sourceFilename)) - imgFormat, ok := imageFormats[ext] if !ok { // This allows SVG etc. to be used as resources. They will not have the methods of the Image, but @@ -376,7 +370,7 @@ type genericResource struct { // This may be set to tell us to look in another filesystem for this resource. // We, by default, use the sourceFs filesystem in the spec below. - overriddenSourceFs afero.Fs + sourceFs afero.Fs spec *Spec @@ -411,7 +405,11 @@ func (l *genericResource) ReadSeekCloser() (hugio.ReadSeekCloser, error) { if l.openReadSeekerCloser != nil { return l.openReadSeekerCloser() } - f, err := l.sourceFs().Open(l.sourceFilename) + if fim, ok := l.osFileInfo.(hugofs.FileMetaInfo); ok { + return fim.Meta().Open() + } + // TODO(bep) get rid of the below + f, err := l.getSourceFs().Open(l.sourceFilename) if err != nil { return nil, err } @@ -497,11 +495,8 @@ func (l *genericResource) initContent() error { return err } -func (l *genericResource) sourceFs() afero.Fs { - if l.overriddenSourceFs != nil { - return l.overriddenSourceFs - } - return l.spec.sourceFs() +func (l *genericResource) getSourceFs() afero.Fs { + return l.sourceFs } func (l *genericResource) publishIfNeeded() { @@ -711,6 +706,10 @@ func (r *Spec) newGenericResourceWithBase( baseFilename string, mediaType media.Type) *genericResource { + if osFileInfo != nil && osFileInfo.IsDir() { + panic(fmt.Sprintf("dirs nto supported resource types: %v", osFileInfo)) + } + // This value is used both to construct URLs and file paths, but start // with a Unix-styled path. baseFilename = helpers.ToSlashTrimLeading(baseFilename) @@ -724,7 +723,7 @@ func (r *Spec) newGenericResourceWithBase( } pathDescriptor := resourcePathDescriptor{ - baseTargetPathDirs: helpers.UniqueStrings(targetPathBaseDirs), + baseTargetPathDirs: helpers.UniqueStringsReuse(targetPathBaseDirs), targetPathBuilder: targetPathBuilder, relTargetDirFile: dirFile{dir: fpath, file: fname}, } @@ -738,7 +737,7 @@ func (r *Spec) newGenericResourceWithBase( openReadSeekerCloser: openReadSeekerCloser, publishOnce: po, resourcePathDescriptor: pathDescriptor, - overriddenSourceFs: sourceFs, + sourceFs: sourceFs, osFileInfo: osFileInfo, sourceFilename: sourceFilename, mediaType: mediaType, diff --git a/resources/resource/params.go b/resources/resource/params.go index f6ecea35ad1..4cb41715da1 100644 --- a/resources/resource/params.go +++ b/resources/resource/params.go @@ -14,7 +14,7 @@ package resource import ( - "strings" + "github.com/gohugoio/hugo/common/maps" "github.com/spf13/cast" ) @@ -25,65 +25,6 @@ func Param(r ResourceParamsProvider, fallback map[string]interface{}, key interf return nil, err } - keyStr = strings.ToLower(keyStr) - result, _ := traverseDirectParams(r, fallback, keyStr) - if result != nil { - return result, nil - } - - keySegments := strings.Split(keyStr, ".") - if len(keySegments) == 1 { - return nil, nil - } - - return traverseNestedParams(r, fallback, keySegments) -} - -func traverseDirectParams(r ResourceParamsProvider, fallback map[string]interface{}, key string) (interface{}, error) { - keyStr := strings.ToLower(key) - if val, ok := r.Params()[keyStr]; ok { - return val, nil - } - - if fallback == nil { - return nil, nil - } - - return fallback[keyStr], nil -} - -func traverseNestedParams(r ResourceParamsProvider, fallback map[string]interface{}, keySegments []string) (interface{}, error) { - result := traverseParams(keySegments, r.Params()) - if result != nil { - return result, nil - } - - if fallback != nil { - result = traverseParams(keySegments, fallback) - if result != nil { - return result, nil - } - } - - // Didn't find anything, but also no problems. - return nil, nil -} - -func traverseParams(keys []string, m map[string]interface{}) interface{} { - // Shift first element off. - firstKey, rest := keys[0], keys[1:] - result := m[firstKey] - - // No point in continuing here. - if result == nil { - return result - } - - if len(rest) == 0 { - // That was the last key. - return result - } + return maps.GetNestedParam(keyStr, ".", r.Params(), fallback) - // That was not the last key. - return traverseParams(rest, cast.ToStringMap(result)) } diff --git a/resources/resource_factories/bundler/bundler.go b/resources/resource_factories/bundler/bundler.go index 59810e34741..6655ee5c330 100644 --- a/resources/resource_factories/bundler/bundler.go +++ b/resources/resource_factories/bundler/bundler.go @@ -124,9 +124,9 @@ func (c *Client) Concat(targetPath string, r resource.Resources) (resource.Resou return &multiReadSeekCloser{mr: mr, sources: rcsources}, nil } - composite, err := c.rs.NewForFs( - c.rs.FileCaches.AssetsCache().Fs, + composite, err := c.rs.New( resources.ResourceSourceDescriptor{ + Fs: c.rs.FileCaches.AssetsCache().Fs, LazyPublish: true, OpenReadSeekCloser: concatr, RelTargetFilename: filepath.Clean(targetPath)}) diff --git a/resources/resource_factories/create/create.go b/resources/resource_factories/create/create.go index dc565056d8e..36a29e733fb 100644 --- a/resources/resource_factories/create/create.go +++ b/resources/resource_factories/create/create.go @@ -40,10 +40,10 @@ func New(rs *resources.Spec) *Client { func (c *Client) Get(fs afero.Fs, filename string) (resource.Resource, error) { filename = filepath.Clean(filename) return c.rs.ResourceCache.GetOrCreate(resources.ResourceKeyPartition(filename), filename, func() (resource.Resource, error) { - return c.rs.NewForFs(fs, - resources.ResourceSourceDescriptor{ - LazyPublish: true, - SourceFilename: filename}) + return c.rs.New(resources.ResourceSourceDescriptor{ + Fs: fs, + LazyPublish: true, + SourceFilename: filename}) }) } @@ -51,9 +51,9 @@ func (c *Client) Get(fs afero.Fs, filename string) (resource.Resource, error) { // FromString creates a new Resource from a string with the given relative target path. func (c *Client) FromString(targetPath, content string) (resource.Resource, error) { return c.rs.ResourceCache.GetOrCreate(resources.CACHE_OTHER, targetPath, func() (resource.Resource, error) { - return c.rs.NewForFs( - c.rs.FileCaches.AssetsCache().Fs, + return c.rs.New( resources.ResourceSourceDescriptor{ + Fs: c.rs.FileCaches.AssetsCache().Fs, LazyPublish: true, OpenReadSeekCloser: func() (hugio.ReadSeekCloser, error) { return hugio.NewReadSeekerNoOpCloserFromString(content), nil diff --git a/resources/resource_test.go b/resources/resource_test.go index af7867eb1c9..565ae06c433 100644 --- a/resources/resource_test.go +++ b/resources/resource_test.go @@ -21,6 +21,8 @@ import ( "testing" "time" + "github.com/spf13/afero" + "github.com/gohugoio/hugo/resources/resource" "github.com/gohugoio/hugo/media" @@ -61,7 +63,9 @@ func TestNewResourceFromFilename(t *testing.T) { writeSource(t, spec.Fs, "content/a/b/logo.png", "image") writeSource(t, spec.Fs, "content/a/b/data.json", "json") - r, err := spec.New(ResourceSourceDescriptor{SourceFilename: "a/b/logo.png"}) + bfs := afero.NewBasePathFs(spec.Fs.Source, "content") + + r, err := spec.New(ResourceSourceDescriptor{Fs: bfs, SourceFilename: "a/b/logo.png"}) assert.NoError(err) assert.NotNil(r) @@ -69,7 +73,7 @@ func TestNewResourceFromFilename(t *testing.T) { assert.Equal("/a/b/logo.png", r.RelPermalink()) assert.Equal("https://example.com/a/b/logo.png", r.Permalink()) - r, err = spec.New(ResourceSourceDescriptor{SourceFilename: "a/b/data.json"}) + r, err = spec.New(ResourceSourceDescriptor{Fs: bfs, SourceFilename: "a/b/data.json"}) assert.NoError(err) assert.NotNil(r) @@ -85,8 +89,10 @@ func TestNewResourceFromFilenameSubPathInBaseURL(t *testing.T) { spec := newTestResourceSpecForBaseURL(assert, "https://example.com/docs") writeSource(t, spec.Fs, "content/a/b/logo.png", "image") + bfs := afero.NewBasePathFs(spec.Fs.Source, "content") - r, err := spec.New(ResourceSourceDescriptor{SourceFilename: filepath.FromSlash("a/b/logo.png")}) + fmt.Println() + r, err := spec.New(ResourceSourceDescriptor{Fs: bfs, SourceFilename: filepath.FromSlash("a/b/logo.png")}) assert.NoError(err) assert.NotNil(r) diff --git a/resources/resource_transformers/postcss/postcss.go b/resources/resource_transformers/postcss/postcss.go index d26ffad5460..125989a10dd 100644 --- a/resources/resource_transformers/postcss/postcss.go +++ b/resources/resource_transformers/postcss/postcss.go @@ -130,7 +130,7 @@ func (t *postcssTransformation) Transform(ctx *resources.ResourceTransformationC if !filepath.IsAbs(configFile) { // We resolve this against the virtual Work filesystem, to allow // this config file to live in one of the themes if needed. - fi, err := t.rs.BaseFs.Work.Fs.Stat(configFile) + fi, err := t.rs.BaseFs.Work.Stat(configFile) if err != nil { if t.options.Config != "" { // Only fail if the user specificed config file is not found. @@ -138,7 +138,7 @@ func (t *postcssTransformation) Transform(ctx *resources.ResourceTransformationC } configFile = "" } else { - configFile = fi.(hugofs.RealFilenameInfo).RealFilename() + configFile = fi.(hugofs.FileMetaInfo).Meta().Filename() } } diff --git a/resources/resource_transformers/tocss/scss/client.go b/resources/resource_transformers/tocss/scss/client.go index 41ff6743355..e69af2f748f 100644 --- a/resources/resource_transformers/tocss/scss/client.go +++ b/resources/resource_transformers/tocss/scss/client.go @@ -19,6 +19,7 @@ import ( "github.com/gohugoio/hugo/hugolib/filesystems" "github.com/gohugoio/hugo/resources" "github.com/gohugoio/hugo/resources/resource" + "github.com/spf13/afero" "github.com/mitchellh/mapstructure" ) @@ -26,7 +27,7 @@ import ( type Client struct { rs *resources.Spec sfs *filesystems.SourceFilesystem - workFs *filesystems.SourceFilesystem + workFs afero.Fs } func New(fs *filesystems.SourceFilesystem, rs *resources.Spec) (*Client, error) { diff --git a/resources/resource_transformers/tocss/scss/tocss.go b/resources/resource_transformers/tocss/scss/tocss.go index 17c32ea8ece..ad581d681c6 100644 --- a/resources/resource_transformers/tocss/scss/tocss.go +++ b/resources/resource_transformers/tocss/scss/tocss.go @@ -55,7 +55,11 @@ func (t *toCSSTransformation) Transform(ctx *resources.ResourceTransformationCtx // Append any workDir relative include paths for _, ip := range options.from.IncludePaths { - options.to.IncludePaths = append(options.to.IncludePaths, t.c.workFs.RealDirs(filepath.Clean(ip))...) + info, err := t.c.workFs.Stat(filepath.Clean(ip)) + if err == nil { + filename := info.(hugofs.FileMetaInfo).Meta().Filename() + options.to.IncludePaths = append(options.to.IncludePaths, filename) + } } // To allow for overrides of SCSS files anywhere in the project/theme hierarchy, we need @@ -74,6 +78,7 @@ func (t *toCSSTransformation) Transform(ctx *resources.ResourceTransformationCtx prevDir = baseDir } else { prevDir = t.c.sfs.MakePathRelative(filepath.Dir(prev)) + if prevDir == "" { // Not a member of this filesystem. Let LibSASS handle it. return "", "", false @@ -100,8 +105,8 @@ func (t *toCSSTransformation) Transform(ctx *resources.ResourceTransformationCtx filenameToCheck := filepath.Join(basePath, fmt.Sprintf(namePattern, name)) fi, err := t.c.sfs.Fs.Stat(filenameToCheck) if err == nil { - if fir, ok := fi.(hugofs.RealFilenameInfo); ok { - return fir.RealFilename(), "", true + if fim, ok := fi.(hugofs.FileMetaInfo); ok { + return fim.Meta().Filename(), "", true } } } diff --git a/resources/sunset.jpg b/resources/sunset.jpg new file mode 100644 index 00000000000..7d7307bed36 Binary files /dev/null and b/resources/sunset.jpg differ diff --git a/resources/testhelpers_test.go b/resources/testhelpers_test.go index d064fa57052..61a9a6e1e72 100644 --- a/resources/testhelpers_test.go +++ b/resources/testhelpers_test.go @@ -4,7 +4,6 @@ import ( "path/filepath" "testing" - "fmt" "image" "io" "io/ioutil" @@ -12,6 +11,9 @@ import ( "runtime" "strings" + "github.com/gohugoio/hugo/langs" + "github.com/gohugoio/hugo/modules" + "github.com/gohugoio/hugo/cache/filecache" "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/hugofs" @@ -28,9 +30,8 @@ func newTestResourceSpec(assert *require.Assertions) *Spec { return newTestResourceSpecForBaseURL(assert, "https://example.com/") } -func newTestResourceSpecForBaseURL(assert *require.Assertions, baseURL string) *Spec { +func createTestCfg() *viper.Viper { cfg := viper.New() - cfg.Set("baseURL", baseURL) cfg.Set("resourceDir", "resources") cfg.Set("contentDir", "content") cfg.Set("dataDir", "data") @@ -40,6 +41,21 @@ func newTestResourceSpecForBaseURL(assert *require.Assertions, baseURL string) * cfg.Set("archetypeDir", "archetypes") cfg.Set("publishDir", "public") + langs.LoadLanguageSettings(cfg, nil) + mod, err := modules.CreateProjectModule(cfg) + if err != nil { + panic(err) + } + cfg.Set("allModules", modules.Modules{mod}) + + return cfg + +} + +func newTestResourceSpecForBaseURL(assert *require.Assertions, baseURL string) *Spec { + cfg := createTestCfg() + cfg.Set("baseURL", baseURL) + imagingCfg := map[string]interface{}{ "resampleFilter": "linear", "quality": 68, @@ -71,7 +87,7 @@ func newTargetPaths(link string) func() page.TargetPaths { } func newTestResourceOsFs(assert *require.Assertions) *Spec { - cfg := viper.New() + cfg := createTestCfg() cfg.Set("baseURL", "https://example.com") workDir, _ := ioutil.TempDir("", "hugores") @@ -83,14 +99,6 @@ func newTestResourceOsFs(assert *require.Assertions) *Spec { } cfg.Set("workingDir", workDir) - cfg.Set("resourceDir", "resources") - cfg.Set("contentDir", "content") - cfg.Set("dataDir", "data") - cfg.Set("i18nDir", "i18n") - cfg.Set("layoutDir", "layouts") - cfg.Set("assetDir", "assets") - cfg.Set("archetypeDir", "archetypes") - cfg.Set("publishDir", "public") fs := hugofs.NewFrom(hugofs.Os, cfg) fs.Destination = &afero.MemMapFs{} @@ -126,7 +134,7 @@ func fetchResourceForSpec(spec *Spec, assert *require.Assertions, name string) r src, err := os.Open(filepath.FromSlash("testdata/" + name)) assert.NoError(err) - out, err := helpers.OpenFileForWriting(spec.BaseFs.Content.Fs, name) + out, err := helpers.OpenFileForWriting(spec.Fs.Source, name) assert.NoError(err) _, err = io.Copy(out, src) out.Close() @@ -135,7 +143,7 @@ func fetchResourceForSpec(spec *Spec, assert *require.Assertions, name string) r factory := newTargetPaths("/a") - r, err := spec.New(ResourceSourceDescriptor{TargetPaths: factory, LazyPublish: true, SourceFilename: name}) + r, err := spec.New(ResourceSourceDescriptor{Fs: spec.Fs.Source, TargetPaths: factory, LazyPublish: true, SourceFilename: name}) assert.NoError(err) return r.(resource.ContentResource) @@ -144,9 +152,6 @@ func fetchResourceForSpec(spec *Spec, assert *require.Assertions, name string) r func assertImageFile(assert *require.Assertions, fs afero.Fs, filename string, width, height int) { filename = filepath.Clean(filename) f, err := fs.Open(filename) - if err != nil { - printFs(fs, "", os.Stdout) - } assert.NoError(err) defer f.Close() @@ -170,22 +175,3 @@ func writeToFs(t testing.TB, fs afero.Fs, filename, content string) { t.Fatalf("Failed to write file: %s", err) } } - -func printFs(fs afero.Fs, path string, w io.Writer) { - if fs == nil { - return - } - afero.Walk(fs, path, func(path string, info os.FileInfo, err error) error { - if info != nil && !info.IsDir() { - s := path - if lang, ok := info.(hugofs.LanguageAnnouncer); ok { - s = s + "\t" + lang.Lang() - } - if fp, ok := info.(hugofs.FilePather); ok { - s += "\tFilename: " + fp.Filename() + "\tBase: " + fp.BaseDir() - } - fmt.Fprintln(w, " ", s) - } - return nil - }) -} diff --git a/resources/transform.go b/resources/transform.go index 934c713277b..35ae25ec4f7 100644 --- a/resources/transform.go +++ b/resources/transform.go @@ -406,6 +406,7 @@ func (r *transformedResource) transform(setContent, publish bool) (err error) { } if err := tr.transformation.Transform(tctx); err != nil { + if err == herrors.ErrFeatureNotAvailable { // This transformation is not available in this // Hugo installation (scss not compiled in, PostCSS not available etc.) @@ -515,6 +516,9 @@ func (r *transformedResource) initTransform(setContent, publish bool) error { // Copy the file from cache to /public _, src, err := r.cache.fileCache.Get(r.sourceFilename) + if src == nil { + panic(fmt.Sprintf("[BUG] resource cache file not found: %q", r.sourceFilename)) + } if err == nil { defer src.Close() diff --git a/source/fileInfo.go b/source/fileInfo.go index 072b55b6cbb..a4cbf6fe621 100644 --- a/source/fileInfo.go +++ b/source/fileInfo.go @@ -14,11 +14,14 @@ package source import ( - "os" "path/filepath" "strings" "sync" + "github.com/gohugoio/hugo/hugofs/files" + + "github.com/pkg/errors" + "github.com/gohugoio/hugo/common/hugio" "github.com/gohugoio/hugo/hugofs" @@ -28,8 +31,7 @@ import ( // fileInfo implements the File interface. var ( - _ File = (*FileInfo)(nil) - _ ReadableFile = (*FileInfo)(nil) + _ File = (*FileInfo)(nil) ) // File represents a source file. @@ -90,13 +92,7 @@ type FileWithoutOverlap interface { // Hugo content files being one of them, considered to be unique. UniqueID() string - FileInfo() os.FileInfo -} - -// A ReadableFile is a File that is readable. -type ReadableFile interface { - File - Open() (hugio.ReadSeekCloser, error) + FileInfo() hugofs.FileMetaInfo } // FileInfo describes a source file. @@ -107,7 +103,7 @@ type FileInfo struct { sp *SourceSpec - fi os.FileInfo + fi hugofs.FileMetaInfo // Derived from filename ext string // Extension without any "." @@ -179,13 +175,14 @@ func (fi *FileInfo) UniqueID() string { } // FileInfo returns a file's underlying os.FileInfo. -func (fi *FileInfo) FileInfo() os.FileInfo { return fi.fi } +func (fi *FileInfo) FileInfo() hugofs.FileMetaInfo { return fi.fi } func (fi *FileInfo) String() string { return fi.BaseFileName() } // Open implements ReadableFile. func (fi *FileInfo) Open() (hugio.ReadSeekCloser, error) { - f, err := fi.sp.SourceFs.Open(fi.Filename()) + f, err := fi.fi.Meta().Open() + return f, err } @@ -225,38 +222,45 @@ func NewTestFile(filename string) *FileInfo { } } -// NewFileInfo returns a new FileInfo structure. -func (sp *SourceSpec) NewFileInfo(baseDir, filename string, isLeafBundle bool, fi os.FileInfo) *FileInfo { +func (sp *SourceSpec) NewFileInfoFrom(path, filename string) (*FileInfo, error) { + meta := hugofs.FileMeta{ + "filename": filename, + "path": path, + } - var lang, translationBaseName, relPath string + return sp.NewFileInfo(hugofs.NewFileMetaInfo(nil, meta)) +} - if fp, ok := fi.(hugofs.FilePather); ok { - filename = fp.Filename() - baseDir = fp.BaseDir() - relPath = fp.Path() - } +func (sp *SourceSpec) NewFileInfo(fi hugofs.FileMetaInfo) (*FileInfo, error) { - if fl, ok := fi.(hugofs.LanguageAnnouncer); ok { - lang = fl.Lang() - translationBaseName = fl.TranslationBaseName() - } + m := fi.Meta() - dir, name := filepath.Split(filename) - if !strings.HasSuffix(dir, helpers.FilePathSeparator) { - dir = dir + helpers.FilePathSeparator + filename := m.Filename() + relPath := m.Path() + isLeafBundle := m.Classifier() == files.ContentClassLeaf + + if relPath == "" || strings.Contains(relPath, "TODO") { + return nil, errors.Errorf("no Path provided by %v (%T)", m, m.Fs()) } - baseDir = strings.TrimSuffix(baseDir, helpers.FilePathSeparator) + if filename == "" || strings.Contains(filename, "TODO") { + return nil, errors.Errorf("no Filename provided by %v (%T)", m, m.Fs()) + } - relDir := "" - if dir != baseDir { - relDir = strings.TrimPrefix(dir, baseDir) + relDir := filepath.Dir(relPath) + if relDir == "." { + relDir = "" + } + if !strings.HasSuffix(relDir, helpers.FilePathSeparator) { + relDir = relDir + helpers.FilePathSeparator } - relDir = strings.TrimPrefix(relDir, helpers.FilePathSeparator) + lang := m.Lang() + translationBaseName := m.GetString("translationBaseName") - if relPath == "" { - relPath = filepath.Join(relDir, name) + dir, name := filepath.Split(relPath) + if !strings.HasSuffix(dir, helpers.FilePathSeparator) { + dir = dir + helpers.FilePathSeparator } ext := strings.ToLower(strings.TrimPrefix(filepath.Ext(name), ".")) @@ -277,14 +281,14 @@ func (sp *SourceSpec) NewFileInfo(baseDir, filename string, isLeafBundle bool, f lang: lang, ext: ext, dir: dir, - relDir: relDir, - relPath: relPath, + relDir: relDir, // Dir() + relPath: relPath, // Path() name: name, - baseName: baseName, + baseName: baseName, // BaseFileName() translationBaseName: translationBaseName, isLeafBundle: isLeafBundle, } - return f + return f, nil } diff --git a/source/fileInfo_test.go b/source/fileInfo_test.go index 9390c624706..0c024de1882 100644 --- a/source/fileInfo_test.go +++ b/source/fileInfo_test.go @@ -15,12 +15,9 @@ package source import ( "path/filepath" + "strings" "testing" - "github.com/gohugoio/hugo/helpers" - - "github.com/gohugoio/hugo/hugofs" - "github.com/spf13/afero" "github.com/stretchr/testify/require" ) @@ -55,56 +52,10 @@ func TestFileInfo(t *testing.T) { }}, } { - f := s.NewFileInfo(this.base, this.filename, false, nil) + path := strings.TrimPrefix(this.filename, this.base) + f, err := s.NewFileInfoFrom(path, this.filename) + assert.NoError(err) this.assert(f) } } - -func TestFileInfoLanguage(t *testing.T) { - assert := require.New(t) - langs := map[string]bool{ - "sv": true, - "en": true, - } - - m := afero.NewMemMapFs() - lfs := hugofs.NewLanguageFs("sv", langs, m) - v := newTestConfig() - - fs := hugofs.NewFrom(m, v) - - ps, err := helpers.NewPathSpec(fs, v) - assert.NoError(err) - s := SourceSpec{SourceFs: lfs, PathSpec: ps} - s.Languages = map[string]interface{}{ - "en": true, - } - - err = afero.WriteFile(lfs, "page.md", []byte("abc"), 0777) - assert.NoError(err) - err = afero.WriteFile(lfs, "page.en.md", []byte("abc"), 0777) - assert.NoError(err) - - sv, _ := lfs.Stat("page.md") - en, _ := lfs.Stat("page.en.md") - - fiSv := s.NewFileInfo("", "page.md", false, sv) - fiEn := s.NewFileInfo("", "page.en.md", false, en) - - assert.Equal("sv", fiSv.Lang()) - assert.Equal("en", fiEn.Lang()) - - // test contentBaseName implementation - fi := s.NewFileInfo("", "2018-10-01-contentbasename.md", false, nil) - assert.Equal("2018-10-01-contentbasename", fi.ContentBaseName()) - - fi = s.NewFileInfo("", "2018-10-01-contentbasename.en.md", false, nil) - assert.Equal("2018-10-01-contentbasename", fi.ContentBaseName()) - - fi = s.NewFileInfo("", filepath.Join("2018-10-01-contentbasename", "index.en.md"), true, nil) - assert.Equal("2018-10-01-contentbasename", fi.ContentBaseName()) - - fi = s.NewFileInfo("", filepath.Join("2018-10-01-contentbasename", "_index.en.md"), false, nil) - assert.Equal("_index", fi.ContentBaseName()) -} diff --git a/source/filesystem.go b/source/filesystem.go index 0c1a6ac7b9e..ce62c15a4aa 100644 --- a/source/filesystem.go +++ b/source/filesystem.go @@ -14,29 +14,25 @@ package source import ( - "os" "path/filepath" - "runtime" "sync" - "github.com/gohugoio/hugo/helpers" - jww "github.com/spf13/jwalterweatherman" - "golang.org/x/text/unicode/norm" + "github.com/pkg/errors" + + "github.com/gohugoio/hugo/hugofs" ) // Filesystem represents a source filesystem. type Filesystem struct { - files []ReadableFile - filesInit sync.Once + files []File + filesInit sync.Once + filesInitErr error Base string - SourceSpec -} + fi hugofs.FileMetaInfo -// Input describes a source input. -type Input interface { - Files() []ReadableFile + SourceSpec } // NewFilesystem returns a new filesytem for a given source spec. @@ -44,76 +40,74 @@ func (sp SourceSpec) NewFilesystem(base string) *Filesystem { return &Filesystem{SourceSpec: sp, Base: base} } +func (sp SourceSpec) NewFilesystemFromFileMetaInfo(fi hugofs.FileMetaInfo) *Filesystem { + return &Filesystem{SourceSpec: sp, fi: fi} +} + // Files returns a slice of readable files. -func (f *Filesystem) Files() []ReadableFile { +func (f *Filesystem) Files() ([]File, error) { f.filesInit.Do(func() { - f.captureFiles() + err := f.captureFiles() + if err != nil { + f.filesInitErr = errors.Wrap(err, "capture files") + } }) - return f.files + return f.files, f.filesInitErr } // add populates a file in the Filesystem.files -func (f *Filesystem) add(name string, fi os.FileInfo) (err error) { - var file ReadableFile +func (f *Filesystem) add(name string, fi hugofs.FileMetaInfo) (err error) { + var file File - if runtime.GOOS == "darwin" { - // When a file system is HFS+, its filepath is in NFD form. - name = norm.NFC.String(name) + file, err = f.SourceSpec.NewFileInfo(fi) + if err != nil { + return err } - file = f.SourceSpec.NewFileInfo(f.Base, name, false, fi) f.files = append(f.files, file) return err } -func (f *Filesystem) captureFiles() { - walker := func(filePath string, fi os.FileInfo, err error) error { +func (f *Filesystem) captureFiles() error { + walker := func(path string, fi hugofs.FileMetaInfo, err error) error { if err != nil { + return err + } + + if fi.IsDir() { return nil } - b, err := f.shouldRead(filePath, fi) + meta := fi.Meta() + filename := meta.Filename() + + b, err := f.shouldRead(filename, fi) if err != nil { return err } + if b { - f.add(filePath, fi) + err = f.add(filename, fi) } + return err } - if f.SourceFs == nil { - panic("Must have a fs") - } - err := helpers.SymbolicWalk(f.SourceFs, f.Base, walker) + w := hugofs.NewWalkway(hugofs.WalkwayConfig{ + Fs: f.SourceFs, + Info: f.fi, + Root: f.Base, + WalkFn: walker, + }) - if err != nil { - jww.ERROR.Println(err) - } + return w.Walk() } -func (f *Filesystem) shouldRead(filename string, fi os.FileInfo) (bool, error) { - if fi.Mode()&os.ModeSymlink == os.ModeSymlink { - link, err := filepath.EvalSymlinks(filename) - if err != nil { - jww.ERROR.Printf("Cannot read symbolic link '%s', error was: %s", filename, err) - return false, nil - } - linkfi, err := f.SourceFs.Stat(link) - if err != nil { - jww.ERROR.Printf("Cannot stat '%s', error was: %s", link, err) - return false, nil - } - - if !linkfi.Mode().IsRegular() { - jww.ERROR.Printf("Symbolic links for directories not supported, skipping '%s'", filename) - } - return false, nil - } +func (f *Filesystem) shouldRead(filename string, fi hugofs.FileMetaInfo) (bool, error) { - ignore := f.SourceSpec.IgnoreFile(filename) + ignore := f.SourceSpec.IgnoreFile(fi.Meta().Filename()) if fi.IsDir() { if ignore { diff --git a/source/filesystem_test.go b/source/filesystem_test.go index 8c8e30413f2..33007c7e496 100644 --- a/source/filesystem_test.go +++ b/source/filesystem_test.go @@ -14,20 +14,31 @@ package source import ( - "os" + "fmt" + "path/filepath" "runtime" "testing" + "github.com/gohugoio/hugo/modules" + + "github.com/gohugoio/hugo/langs" + + "github.com/spf13/afero" + "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/hugofs" + "github.com/stretchr/testify/require" "github.com/spf13/viper" ) func TestEmptySourceFilesystem(t *testing.T) { + assert := require.New(t) ss := newTestSourceSpec() - src := ss.NewFilesystem("Empty") - if len(src.Files()) != 0 { + src := ss.NewFilesystem("") + files, err := src.Files() + assert.NoError(err) + if len(files) != 0 { t.Errorf("new filesystem should contain 0 files.") } } @@ -38,6 +49,8 @@ func TestUnicodeNorm(t *testing.T) { return } + assert := require.New(t) + paths := []struct { NFC string NFD string @@ -47,14 +60,18 @@ func TestUnicodeNorm(t *testing.T) { } ss := newTestSourceSpec() - var fi os.FileInfo + fi := hugofs.NewFileMetaInfo(nil, hugofs.FileMeta{}) - for _, path := range paths { - src := ss.NewFilesystem("base") + for i, path := range paths { + base := fmt.Sprintf("base%d", i) + assert.NoError(afero.WriteFile(ss.Fs.Source, filepath.Join(base, path.NFD), []byte("some data"), 0777)) + src := ss.NewFilesystem(base) _ = src.add(path.NFD, fi) - f := src.Files()[0] + files, err := src.Files() + assert.NoError(err) + f := files[0] if f.BaseFileName() != path.NFC { - t.Fatalf("file name in NFD form should be normalized (%s)", path.NFC) + t.Fatalf("file %q name in NFD form should be normalized (%s)", f.BaseFileName(), path.NFC) } } @@ -70,12 +87,22 @@ func newTestConfig() *viper.Viper { v.Set("resourceDir", "resources") v.Set("publishDir", "public") v.Set("assetDir", "assets") + _, err := langs.LoadLanguageSettings(v, nil) + if err != nil { + panic(err) + } + mod, err := modules.CreateProjectModule(v) + if err != nil { + panic(err) + } + v.Set("allModules", modules.Modules{mod}) + return v } func newTestSourceSpec() *SourceSpec { v := newTestConfig() - fs := hugofs.NewMem(v) + fs := hugofs.NewFrom(hugofs.NewBaseFileDecorator(afero.NewMemMapFs()), v) ps, err := helpers.NewPathSpec(fs, v) if err != nil { panic(err) diff --git a/source/sourceSpec.go b/source/sourceSpec.go index 9731a8d8d4a..504a3a22da3 100644 --- a/source/sourceSpec.go +++ b/source/sourceSpec.go @@ -17,6 +17,7 @@ import ( "os" "path/filepath" "regexp" + "runtime" "github.com/gohugoio/hugo/langs" "github.com/spf13/afero" @@ -107,6 +108,18 @@ func (s *SourceSpec) IgnoreFile(filename string) bool { } } + if runtime.GOOS == "windows" { + // Also check the forward slash variant if different. + unixFilename := filepath.ToSlash(filename) + if unixFilename != filename { + for _, re := range s.ignoreFilesRe { + if re.MatchString(unixFilename) { + return true + } + } + } + } + return false } diff --git a/tpl/cast/docshelper.go b/tpl/cast/docshelper.go index 6fc35f3c72d..1ee614b10c5 100644 --- a/tpl/cast/docshelper.go +++ b/tpl/cast/docshelper.go @@ -17,7 +17,7 @@ import ( "github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/docshelper" - "github.com/gohugoio/hugo/htesting" + "github.com/gohugoio/hugo/resources/page" "github.com/gohugoio/hugo/tpl/internal" "github.com/spf13/viper" ) @@ -30,7 +30,7 @@ func init() { Cfg: viper.New(), Log: loggers.NewErrorLogger(), BuildStartListeners: &deps.Listeners{}, - Site: htesting.NewTestHugoSite(), + Site: page.NewDummyHugoSite(newTestConfig()), } var namespaces internal.TemplateFuncsNamespaces @@ -47,3 +47,9 @@ func init() { docshelper.AddDocProvider("tpl", docsProvider) } + +func newTestConfig() *viper.Viper { + v := viper.New() + v.Set("contentDir", "content") + return v +} diff --git a/tpl/data/init_test.go b/tpl/data/init_test.go index c4751e8925e..94c8408ea6c 100644 --- a/tpl/data/init_test.go +++ b/tpl/data/init_test.go @@ -16,6 +16,7 @@ package data import ( "testing" + "github.com/gohugoio/hugo/langs" "github.com/gohugoio/hugo/tpl/internal" "github.com/spf13/viper" "github.com/stretchr/testify/require" @@ -27,6 +28,7 @@ func TestInit(t *testing.T) { v := viper.New() v.Set("contentDir", "content") + langs.LoadLanguageSettings(v, nil) for _, nsf := range internal.TemplateFuncsNamespaceRegistry { ns = nsf(newDeps(v)) diff --git a/tpl/data/resources_test.go b/tpl/data/resources_test.go index a42232f940b..39cf6bfa9fd 100644 --- a/tpl/data/resources_test.go +++ b/tpl/data/resources_test.go @@ -23,6 +23,8 @@ import ( "testing" "time" + "github.com/gohugoio/hugo/modules" + "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/cache/filecache" @@ -87,7 +89,7 @@ func getTestServer(handler func(w http.ResponseWriter, r *http.Request)) (*httpt func TestScpGetRemote(t *testing.T) { t.Parallel() fs := new(afero.MemMapFs) - cache := filecache.NewCache(fs, 100) + cache := filecache.NewCache(fs, 100, "") tests := []struct { path string @@ -186,14 +188,19 @@ func newDeps(cfg config.Provider) *deps.Deps { cfg.Set("layoutDir", "layouts") cfg.Set("archetypeDir", "archetypes") - l := langs.NewLanguage("en", cfg) - l.Set("i18nDir", "i18n") - cs, err := helpers.NewContentSpec(l) + langs.LoadLanguageSettings(cfg, nil) + mod, err := modules.CreateProjectModule(cfg) + if err != nil { + panic(err) + } + cfg.Set("allModules", modules.Modules{mod}) + + cs, err := helpers.NewContentSpec(cfg) if err != nil { panic(err) } - fs := hugofs.NewMem(l) + fs := hugofs.NewMem(cfg) logger := loggers.NewErrorLogger() p, err := helpers.NewPathSpec(fs, cfg) diff --git a/tpl/hugo/init_test.go b/tpl/hugo/init_test.go index 128f6fc19f0..f4e31f622cb 100644 --- a/tpl/hugo/init_test.go +++ b/tpl/hugo/init_test.go @@ -16,17 +16,19 @@ package hugo import ( "testing" - "github.com/gohugoio/hugo/htesting" - "github.com/gohugoio/hugo/deps" + "github.com/gohugoio/hugo/resources/page" "github.com/gohugoio/hugo/tpl/internal" + "github.com/spf13/viper" "github.com/stretchr/testify/require" ) func TestInit(t *testing.T) { var found bool var ns *internal.TemplateFuncsNamespace - s := htesting.NewTestHugoSite() + v := viper.New() + v.Set("contentDir", "content") + s := page.NewDummyHugoSite(v) for _, nsf := range internal.TemplateFuncsNamespaceRegistry { ns = nsf(&deps.Deps{Site: s}) diff --git a/tpl/os/os.go b/tpl/os/os.go index 2dab5c4906a..eb31498afbb 100644 --- a/tpl/os/os.go +++ b/tpl/os/os.go @@ -18,6 +18,7 @@ package os import ( "errors" "fmt" + "os" _os "os" "github.com/gohugoio/hugo/deps" @@ -26,23 +27,20 @@ import ( ) // New returns a new instance of the os-namespaced template functions. -func New(deps *deps.Deps) *Namespace { +func New(d *deps.Deps) *Namespace { - // Since Hugo 0.38 we can have multiple content dirs. This can make it hard to - // reason about where the file is placed relative to the project root. - // To make the {{ readFile .Filename }} variant just work, we create a composite - // filesystem that first checks the work dir fs and then the content fs. var rfs afero.Fs - if deps.Fs != nil { - rfs = deps.Fs.WorkingDir - if deps.PathSpec != nil && deps.PathSpec.BaseFs != nil { - rfs = afero.NewReadOnlyFs(afero.NewCopyOnWriteFs(deps.PathSpec.BaseFs.Content.Fs, deps.Fs.WorkingDir)) + if d.Fs != nil { + rfs = d.Fs.WorkingDir + if d.PathSpec != nil && d.PathSpec.BaseFs != nil { + rfs = afero.NewReadOnlyFs(afero.NewCopyOnWriteFs(d.PathSpec.BaseFs.Content.Fs, d.Fs.WorkingDir)) } + } return &Namespace{ readFileFs: rfs, - deps: deps, + deps: d, } } @@ -76,6 +74,9 @@ func readFile(fs afero.Fs, filename string) (string, error) { return "", fmt.Errorf("file %q is too big", filename) } } else { + if os.IsNotExist(err) { + return "", fmt.Errorf("file %q does not exist", filename) + } return "", err } b, err := afero.ReadFile(fs, filename) @@ -96,6 +97,10 @@ func (ns *Namespace) ReadFile(i interface{}) (string, error) { return "", err } + if ns.deps.PathSpec != nil { + s = ns.deps.PathSpec.RelPathify(s) + } + return readFile(ns.readFileFs, s) } diff --git a/tpl/site/init_test.go b/tpl/site/init_test.go index 00704d94355..5ef8856770e 100644 --- a/tpl/site/init_test.go +++ b/tpl/site/init_test.go @@ -16,8 +16,10 @@ package site import ( "testing" + "github.com/spf13/viper" + "github.com/gohugoio/hugo/deps" - "github.com/gohugoio/hugo/htesting" + "github.com/gohugoio/hugo/resources/page" "github.com/gohugoio/hugo/tpl/internal" "github.com/stretchr/testify/require" ) @@ -25,7 +27,9 @@ import ( func TestInit(t *testing.T) { var found bool var ns *internal.TemplateFuncsNamespace - s := htesting.NewTestHugoSite() + v := viper.New() + v.Set("contentDir", "content") + s := page.NewDummyHugoSite(v) for _, nsf := range internal.TemplateFuncsNamespaceRegistry { ns = nsf(&deps.Deps{Site: s}) diff --git a/tpl/template.go b/tpl/template.go index 93577136407..cd00d8061bc 100644 --- a/tpl/template.go +++ b/tpl/template.go @@ -252,12 +252,15 @@ func (t *TemplateAdapter) fileAndFilename(name string) (afero.File, string, erro if err != nil { return nil, "", err } - f, err := fs.Open(filename) + fim := fi.(hugofs.FileMetaInfo) + meta := fim.Meta() + + f, err := meta.Open() if err != nil { return nil, "", errors.Wrapf(err, "failed to open template file %q:", filename) } - return f, fi.(hugofs.RealFilenameInfo).RealFilename(), nil + return f, meta.Filename(), nil } // ExecuteToString executes the current template and returns the result as a diff --git a/tpl/tplimpl/embedded/templates.autogen.go b/tpl/tplimpl/embedded/templates.autogen.go index bdbc842223a..b2c97de6af5 100644 --- a/tpl/tplimpl/embedded/templates.autogen.go +++ b/tpl/tplimpl/embedded/templates.autogen.go @@ -268,8 +268,7 @@ if (!doNotTrack) { {{ end }} -{{ end }} -`}, +{{ end }}`}, {`schema.html`, ` diff --git a/tpl/tplimpl/template.go b/tpl/tplimpl/template.go index f0d3066e2ec..cce867ac2ba 100644 --- a/tpl/tplimpl/template.go +++ b/tpl/tplimpl/template.go @@ -711,7 +711,7 @@ func (t *templateHandler) RebuildClone() { func (t *templateHandler) loadTemplates(prefix string) error { - walker := func(path string, fi os.FileInfo, err error) error { + walker := func(path string, fi hugofs.FileMetaInfo, err error) error { if err != nil || fi.IsDir() { return err } @@ -928,8 +928,8 @@ func (t *templateHandler) addTemplateFile(name, baseTemplatePath, path string) e realFilename := filename if fi, err := fs.Stat(filename); err == nil { - if fir, ok := fi.(hugofs.RealFilenameInfo); ok { - realFilename = fir.RealFilename() + if fim, ok := fi.(hugofs.FileMetaInfo); ok { + realFilename = fim.Meta().Filename() } } diff --git a/tpl/tplimpl/template_funcs_test.go b/tpl/tplimpl/template_funcs_test.go index 449d20fd4c5..faf5b01fe45 100644 --- a/tpl/tplimpl/template_funcs_test.go +++ b/tpl/tplimpl/template_funcs_test.go @@ -21,7 +21,9 @@ import ( "testing" "time" - "github.com/gohugoio/hugo/htesting" + "github.com/gohugoio/hugo/modules" + + "github.com/gohugoio/hugo/resources/page" "github.com/gohugoio/hugo/common/hugo" "github.com/gohugoio/hugo/common/loggers" @@ -52,6 +54,14 @@ func newTestConfig() config.Provider { v.Set("assetDir", "assets") v.Set("resourceDir", "resources") v.Set("publishDir", "public") + + langs.LoadLanguageSettings(v, nil) + mod, err := modules.CreateProjectModule(v) + if err != nil { + panic(err) + } + v.Set("allModules", modules.Modules{mod}) + return v } @@ -59,7 +69,7 @@ func newDepsConfig(cfg config.Provider) deps.DepsCfg { l := langs.NewLanguage("en", cfg) return deps.DepsCfg{ Language: l, - Site: htesting.NewTestHugoSite(), + Site: page.NewDummyHugoSite(cfg), Cfg: cfg, Fs: hugofs.NewMem(l), Logger: logger, diff --git a/transform/livereloadinject/livereloadinject_test.go b/transform/livereloadinject/livereloadinject_test.go index 1058244b4ea..413ca7b430a 100644 --- a/transform/livereloadinject/livereloadinject_test.go +++ b/transform/livereloadinject/livereloadinject_test.go @@ -34,7 +34,7 @@ func doTestLiveReloadInject(t *testing.T, bodyEndTag string) { tr := transform.New(New(1313)) tr.Apply(out, in) - expected := fmt.Sprintf(`%s`, bodyEndTag) + expected := fmt.Sprintf(`%s`, bodyEndTag) if out.String() != expected { t.Errorf("Expected %s got %s", expected, out.String()) }