Add Hugo Modules

This commit implements Hugo Modules.

This is a broad subject, but some keywords include:

* A new `module` configuration section where you can import almost anything. You can configure both your own file mounts nd the file mounts of the modules you import. This is the new recommended way of configuring what you earlier put in `configDir`, `staticDir` etc. And it also allows you to mount folders in non-Hugo-projects, e.g. the `SCSS` folder in the Bootstrap GitHub project.
* A module consists of a set of mounts to the standard 7 component types in Hugo: `static`, `content`, `layouts`, `data`, `assets`, `i18n`, and `archetypes`. Yes, Theme Components can now include content, which should be very useful, especially in bigger multilingual projects.
* Modules not in your local file cache will be downloaded automatically and even "hot replaced" while the server is running.
* Hugo Modules supports and encourages semver versioned modules, and uses the minimal version selection algorithm to resolve versions.
* A new set of CLI commands are provided to manage all of this: `hugo mod init`,  `hugo mod get`,  `hugo mod graph`,  `hugo mod tidy`, and  `hugo mod vendor`.

All of the above is backed by Go Modules.

Fixes #5973
Fixes #5996
Fixes #6010
Fixes #5911
Fixes #5940
Fixes #6074
Fixes #6082
Fixes #6092

benchbep.sh

@@ -1,2 +1 @@
-gobench -package=./hugolib -bench="BenchmarkSiteBuilding/YAML,num_langs=3,num_pages=5000,tags_per_page=5,shortcodes,render" -count=3 > 1.bench
-benchcmp -best 0.bench 1.bench
+gobench -package=./hugolib -bench="BenchmarkSiteNew/Deep_content_tree"

cache/filecache/filecache.go

@@ -44,6 +44,9 @@ type Cache struct {
 	// 0 is effectively turning this cache off.
 	maxAge time.Duration
 
+	// When set, we just remove this entire root directory on expiration.
+	pruneAllRootDir string
+
 	nlocker *lockTracker
 }
 
@@ -77,11 +80,12 @@ type ItemInfo struct {
 }
 
 // NewCache creates a new file cache with the given filesystem and max age.
-func NewCache(fs afero.Fs, maxAge time.Duration) *Cache {
+func NewCache(fs afero.Fs, maxAge time.Duration, pruneAllRootDir string) *Cache {
 	return &Cache{
-		Fs:      fs,
-		nlocker: &lockTracker{Locker: locker.NewLocker(), seen: make(map[string]struct{})},
-		maxAge:  maxAge,
+		Fs:              fs,
+		nlocker:         &lockTracker{Locker: locker.NewLocker(), seen: make(map[string]struct{})},
+		maxAge:          maxAge,
+		pruneAllRootDir: pruneAllRootDir,
 	}
 }
 
@@ -307,9 +311,15 @@ func (f Caches) Get(name string) *Cache {
 // NewCaches creates a new set of file caches from the given
 // configuration.
 func NewCaches(p *helpers.PathSpec) (Caches, error) {
-	dcfg, err := decodeConfig(p)
-	if err != nil {
-		return nil, err
+	var dcfg Configs
+	if c, ok := p.Cfg.Get("filecacheConfigs").(Configs); ok {
+		dcfg = c
+	} else {
+		var err error
+		dcfg, err = DecodeConfig(p.Fs.Source, p.Cfg)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	fs := p.Fs.Source
@@ -319,30 +329,25 @@ func NewCaches(p *helpers.PathSpec) (Caches, error) {
 		var cfs afero.Fs
 
 		if v.isResourceDir {
-			cfs = p.BaseFs.Resources.Fs
+			cfs = p.BaseFs.ResourcesCache
 		} else {
 			cfs = fs
 		}
 
-		var baseDir string
-		if !strings.HasPrefix(v.Dir, "_gen") {
-			// We do cache eviction (file removes) and since the user can set
-			// his/hers own cache directory, we really want to make sure
-			// we do not delete any files that do not belong to this cache.
-			// We do add the cache name as the root, but this is an extra safe
-			// guard. We skip the files inside /resources/_gen/ because
-			// that would be breaking.
-			baseDir = filepath.Join(v.Dir, filecacheRootDirname, k)
-		} else {
-			baseDir = filepath.Join(v.Dir, k)
-		}
-		if err = cfs.MkdirAll(baseDir, 0777); err != nil && !os.IsExist(err) {
+		baseDir := v.Dir
+
+		if err := cfs.MkdirAll(baseDir, 0777); err != nil && !os.IsExist(err) {
 			return nil, err
 		}
 
 		bfs := afero.NewBasePathFs(cfs, baseDir)
 
-		m[k] = NewCache(bfs, v.MaxAge)
+		var pruneAllRootDir string
+		if k == cacheKeyModules {
+			pruneAllRootDir = "pkg"
+		}
+
+		m[k] = NewCache(bfs, v.MaxAge, pruneAllRootDir)
 	}
 
 	return m, nil

cache/filecache/filecache_config.go

@@ -19,6 +19,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/gohugoio/hugo/config"
+
 	"github.com/gohugoio/hugo/helpers"
 
 	"github.com/mitchellh/mapstructure"
@@ -32,7 +34,7 @@ const (
 	resourcesGenDir = ":resourceDir/_gen"
 )
 
-var defaultCacheConfig = cacheConfig{
+var defaultCacheConfig = Config{
 	MaxAge: -1, // Never expire
 	Dir:    ":cacheDir/:project",
 }
@@ -42,9 +44,20 @@ const (
 	cacheKeyGetCSV  = "getcsv"
 	cacheKeyImages  = "images"
 	cacheKeyAssets  = "assets"
+	cacheKeyModules = "modules"
 )
 
-var defaultCacheConfigs = map[string]cacheConfig{
+type Configs map[string]Config
+
+func (c Configs) CacheDirModules() string {
+	return c[cacheKeyModules].Dir
+}
+
+var defaultCacheConfigs = Configs{
+	cacheKeyModules: {
+		MaxAge: -1,
+		Dir:    ":cacheDir/modules",
+	},
 	cacheKeyGetJSON: defaultCacheConfig,
 	cacheKeyGetCSV:  defaultCacheConfig,
 	cacheKeyImages: {
@@ -57,9 +70,7 @@ var defaultCacheConfigs = map[string]cacheConfig{
 	},
 }
 
-type cachesConfig map[string]cacheConfig
-
-type cacheConfig struct {
+type Config struct {
 	// Max age of cache entries in this cache. Any items older than this will
 	// be removed and not returned from the cache.
 	// a negative value means forever, 0 means cache is disabled.
@@ -88,25 +99,28 @@ func (f Caches) ImageCache() *Cache {
 	return f[cacheKeyImages]
 }
 
+// ModulesCache gets the file cache for Hugo Modules.
+func (f Caches) ModulesCache() *Cache {
+	return f[cacheKeyModules]
+}
+
 // AssetsCache gets the file cache for assets (processed resources, SCSS etc.).
 func (f Caches) AssetsCache() *Cache {
 	return f[cacheKeyAssets]
 }
 
-func decodeConfig(p *helpers.PathSpec) (cachesConfig, error) {
-	c := make(cachesConfig)
+func DecodeConfig(fs afero.Fs, cfg config.Provider) (Configs, error) {
+	c := make(Configs)
 	valid := make(map[string]bool)
 	// Add defaults
 	for k, v := range defaultCacheConfigs {
 		c[k] = v
 		valid[k] = true
 	}
 
-	cfg := p.Cfg
-
 	m := cfg.GetStringMap(cachesConfigKey)
 
-	_, isOsFs := p.Fs.Source.(*afero.OsFs)
+	_, isOsFs := fs.(*afero.OsFs)
 
 	for k, v := range m {
 		cc := defaultCacheConfig
@@ -148,7 +162,7 @@ func decodeConfig(p *helpers.PathSpec) (cachesConfig, error) {
 
 		for i, part := range parts {
 			if strings.HasPrefix(part, ":") {
-				resolved, isResource, err := resolveDirPlaceholder(p, part)
+				resolved, isResource, err := resolveDirPlaceholder(fs, cfg, part)
 				if err != nil {
 					return c, err
 				}
@@ -176,6 +190,18 @@ func decodeConfig(p *helpers.PathSpec) (cachesConfig, error) {
 			}
 		}
 
+		if !strings.HasPrefix(v.Dir, "_gen") {
+			// We do cache eviction (file removes) and since the user can set
+			// his/hers own cache directory, we really want to make sure
+			// we do not delete any files that do not belong to this cache.
+			// We do add the cache name as the root, but this is an extra safe
+			// guard. We skip the files inside /resources/_gen/ because
+			// that would be breaking.
+			v.Dir = filepath.Join(v.Dir, filecacheRootDirname, k)
+		} else {
+			v.Dir = filepath.Join(v.Dir, k)
+		}
+
 		if disabled {
 			v.MaxAge = 0
 		}
@@ -187,15 +213,17 @@ func decodeConfig(p *helpers.PathSpec) (cachesConfig, error) {
 }
 
 // Resolves :resourceDir => /myproject/resources etc., :cacheDir => ...
-func resolveDirPlaceholder(p *helpers.PathSpec, placeholder string) (cacheDir string, isResource bool, err error) {
+func resolveDirPlaceholder(fs afero.Fs, cfg config.Provider, placeholder string) (cacheDir string, isResource bool, err error) {
+	workingDir := cfg.GetString("workingDir")
+
 	switch strings.ToLower(placeholder) {
 	case ":resourcedir":
 		return "", true, nil
 	case ":cachedir":
-		d, err := helpers.GetCacheDir(p.Fs.Source, p.Cfg)
+		d, err := helpers.GetCacheDir(fs, cfg)
 		return d, false, err
 	case ":project":
-		return filepath.Base(p.WorkingDir), false, nil
+		return filepath.Base(workingDir), false, nil
 	}
 
 	return "", false, errors.Errorf("%q is not a valid placeholder (valid values are :cacheDir or :resourceDir)", placeholder)

cache/filecache/filecache_config_test.go

@@ -20,10 +20,9 @@ import (
 	"testing"
 	"time"
 
-	"github.com/gohugoio/hugo/helpers"
+	"github.com/spf13/afero"
 
 	"github.com/gohugoio/hugo/config"
-	"github.com/gohugoio/hugo/hugofs"
 
 	"github.com/spf13/viper"
 	"github.com/stretchr/testify/require"
@@ -57,22 +56,19 @@ dir = "/path/to/c3"
 
 	cfg, err := config.FromConfigString(configStr, "toml")
 	assert.NoError(err)
-	fs := hugofs.NewMem(cfg)
-	p, err := helpers.NewPathSpec(fs, cfg)
+	fs := afero.NewMemMapFs()
+	decoded, err := DecodeConfig(fs, cfg)
 	assert.NoError(err)
 
-	decoded, err := decodeConfig(p)
-	assert.NoError(err)
-
-	assert.Equal(4, len(decoded))
+	assert.Equal(5, len(decoded))
 
 	c2 := decoded["getcsv"]
 	assert.Equal("11h0m0s", c2.MaxAge.String())
-	assert.Equal(filepath.FromSlash("/path/to/c2"), c2.Dir)
+	assert.Equal(filepath.FromSlash("/path/to/c2/filecache/getcsv"), c2.Dir)
 
 	c3 := decoded["images"]
 	assert.Equal(time.Duration(-1), c3.MaxAge)
-	assert.Equal(filepath.FromSlash("/path/to/c3"), c3.Dir)
+	assert.Equal(filepath.FromSlash("/path/to/c3/filecache/images"), c3.Dir)
 
 }
 
@@ -105,14 +101,11 @@ dir = "/path/to/c3"
 
 	cfg, err := config.FromConfigString(configStr, "toml")
 	assert.NoError(err)
-	fs := hugofs.NewMem(cfg)
-	p, err := helpers.NewPathSpec(fs, cfg)
-	assert.NoError(err)
-
-	decoded, err := decodeConfig(p)
+	fs := afero.NewMemMapFs()
+	decoded, err := DecodeConfig(fs, cfg)
 	assert.NoError(err)
 
-	assert.Equal(4, len(decoded))
+	assert.Equal(5, len(decoded))
 
 	for _, v := range decoded {
 		assert.Equal(time.Duration(0), v.MaxAge)
@@ -133,24 +126,22 @@ func TestDecodeConfigDefault(t *testing.T) {
 		cfg.Set("cacheDir", "/cache/thecache")
 	}
 
-	fs := hugofs.NewMem(cfg)
-	p, err := helpers.NewPathSpec(fs, cfg)
-	assert.NoError(err)
+	fs := afero.NewMemMapFs()
 
-	decoded, err := decodeConfig(p)
+	decoded, err := DecodeConfig(fs, cfg)
 
 	assert.NoError(err)
 
-	assert.Equal(4, len(decoded))
+	assert.Equal(5, len(decoded))
 
 	imgConfig := decoded[cacheKeyImages]
 	jsonConfig := decoded[cacheKeyGetJSON]
 
 	if runtime.GOOS == "windows" {
-		assert.Equal("_gen", imgConfig.Dir)
+		assert.Equal(filepath.FromSlash("_gen/images"), imgConfig.Dir)
 	} else {
-		assert.Equal("_gen", imgConfig.Dir)
-		assert.Equal("/cache/thecache/hugoproject", jsonConfig.Dir)
+		assert.Equal("_gen/images", imgConfig.Dir)
+		assert.Equal("/cache/thecache/hugoproject/filecache/getjson", jsonConfig.Dir)
 	}
 
 	assert.True(imgConfig.isResourceDir)
@@ -183,11 +174,9 @@ dir = "/"
 
 	cfg, err := config.FromConfigString(configStr, "toml")
 	assert.NoError(err)
-	fs := hugofs.NewMem(cfg)
-	p, err := helpers.NewPathSpec(fs, cfg)
-	assert.NoError(err)
+	fs := afero.NewMemMapFs()
 
-	_, err = decodeConfig(p)
+	_, err = DecodeConfig(fs, cfg)
 	assert.Error(err)
 
 }