Add Hugo Modules

Fixes #5973
Fixes #5996
Fixes #6010
Fixes #5911
Fixes #5940
Fixes #6074
Fixes #6082
Fixes #6092

benchbep.sh

@@ -1,2 +1 @@
-gobench -package=./hugolib -bench="BenchmarkSiteBuilding/YAML,num_langs=3,num_pages=5000,tags_per_page=5,shortcodes,render" -count=3 > 1.bench
-benchcmp -best 0.bench 1.bench
+gobench -package=./hugolib -bench="BenchmarkSiteNew/Deep_content_tree"

cache/filecache/filecache.go

@@ -307,9 +307,15 @@ func (f Caches) Get(name string) *Cache {
 // NewCaches creates a new set of file caches from the given
 // configuration.
 func NewCaches(p *helpers.PathSpec) (Caches, error) {
-	dcfg, err := decodeConfig(p)
-	if err != nil {
-		return nil, err
+	var dcfg Configs
+	if c, ok := p.Cfg.Get("filecacheConfigs").(Configs); ok {
+		dcfg = c
+	} else {
+		var err error
+		dcfg, err = DecodeConfig(p.Fs.Source, p.Cfg)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	fs := p.Fs.Source
@@ -319,7 +325,7 @@ func NewCaches(p *helpers.PathSpec) (Caches, error) {
 		var cfs afero.Fs
 
 		if v.isResourceDir {
-			cfs = p.BaseFs.Resources.Fs
+			cfs = p.BaseFs.ResourcesCache
 		} else {
 			cfs = fs
 		}
@@ -336,7 +342,7 @@ func NewCaches(p *helpers.PathSpec) (Caches, error) {
 		} else {
 			baseDir = filepath.Join(v.Dir, k)
 		}
-		if err = cfs.MkdirAll(baseDir, 0777); err != nil && !os.IsExist(err) {
+		if err := cfs.MkdirAll(baseDir, 0777); err != nil && !os.IsExist(err) {
 			return nil, err
 		}
 

cache/filecache/filecache_config.go

@@ -19,6 +19,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/gohugoio/hugo/config"
+
 	"github.com/gohugoio/hugo/helpers"
 
 	"github.com/mitchellh/mapstructure"
@@ -32,7 +34,7 @@ const (
 	resourcesGenDir = ":resourceDir/_gen"
 )
 
-var defaultCacheConfig = cacheConfig{
+var defaultCacheConfig = Config{
 	MaxAge: -1, // Never expire
 	Dir:    ":cacheDir/:project",
 }
@@ -42,9 +44,20 @@ const (
 	cacheKeyGetCSV  = "getcsv"
 	cacheKeyImages  = "images"
 	cacheKeyAssets  = "assets"
+	cacheKeyModules = "modules"
 )
 
-var defaultCacheConfigs = map[string]cacheConfig{
+type Configs map[string]Config
+
+func (c Configs) CacheDirModules() string {
+	return c[cacheKeyModules].Dir
+}
+
+var defaultCacheConfigs = Configs{
+	cacheKeyModules: {
+		MaxAge: -1,
+		Dir:    ":cacheDir/modules",
+	},
 	cacheKeyGetJSON: defaultCacheConfig,
 	cacheKeyGetCSV:  defaultCacheConfig,
 	cacheKeyImages: {
@@ -57,9 +70,7 @@ var defaultCacheConfigs = map[string]cacheConfig{
 	},
 }
 
-type cachesConfig map[string]cacheConfig
-
-type cacheConfig struct {
+type Config struct {
 	// Max age of cache entries in this cache. Any items older than this will
 	// be removed and not returned from the cache.
 	// a negative value means forever, 0 means cache is disabled.
@@ -93,20 +104,18 @@ func (f Caches) AssetsCache() *Cache {
 	return f[cacheKeyAssets]
 }
 
-func decodeConfig(p *helpers.PathSpec) (cachesConfig, error) {
-	c := make(cachesConfig)
+func DecodeConfig(fs afero.Fs, cfg config.Provider) (Configs, error) {
+	c := make(Configs)
 	valid := make(map[string]bool)
 	// Add defaults
 	for k, v := range defaultCacheConfigs {
 		c[k] = v
 		valid[k] = true
 	}
 
-	cfg := p.Cfg
-
 	m := cfg.GetStringMap(cachesConfigKey)
 
-	_, isOsFs := p.Fs.Source.(*afero.OsFs)
+	_, isOsFs := fs.(*afero.OsFs)
 
 	for k, v := range m {
 		cc := defaultCacheConfig
@@ -148,7 +157,7 @@ func decodeConfig(p *helpers.PathSpec) (cachesConfig, error) {
 
 		for i, part := range parts {
 			if strings.HasPrefix(part, ":") {
-				resolved, isResource, err := resolveDirPlaceholder(p, part)
+				resolved, isResource, err := resolveDirPlaceholder(fs, cfg, part)
 				if err != nil {
 					return c, err
 				}
@@ -187,15 +196,17 @@ func decodeConfig(p *helpers.PathSpec) (cachesConfig, error) {
 }
 
 // Resolves :resourceDir => /myproject/resources etc., :cacheDir => ...
-func resolveDirPlaceholder(p *helpers.PathSpec, placeholder string) (cacheDir string, isResource bool, err error) {
+func resolveDirPlaceholder(fs afero.Fs, cfg config.Provider, placeholder string) (cacheDir string, isResource bool, err error) {
+	workingDir := cfg.GetString("workingDir")
+
 	switch strings.ToLower(placeholder) {
 	case ":resourcedir":
 		return "", true, nil
 	case ":cachedir":
-		d, err := helpers.GetCacheDir(p.Fs.Source, p.Cfg)
+		d, err := helpers.GetCacheDir(fs, cfg)
 		return d, false, err
 	case ":project":
-		return filepath.Base(p.WorkingDir), false, nil
+		return filepath.Base(workingDir), false, nil
 	}
 
 	return "", false, errors.Errorf("%q is not a valid placeholder (valid values are :cacheDir or :resourceDir)", placeholder)

cache/filecache/filecache_config_test.go

@@ -20,10 +20,9 @@ import (
 	"testing"
 	"time"
 
-	"github.com/gohugoio/hugo/helpers"
+	"github.com/spf13/afero"
 
 	"github.com/gohugoio/hugo/config"
-	"github.com/gohugoio/hugo/hugofs"
 
 	"github.com/spf13/viper"
 	"github.com/stretchr/testify/require"
@@ -57,14 +56,11 @@ dir = "/path/to/c3"
 
 	cfg, err := config.FromConfigString(configStr, "toml")
 	assert.NoError(err)
-	fs := hugofs.NewMem(cfg)
-	p, err := helpers.NewPathSpec(fs, cfg)
+	fs := afero.NewMemMapFs()
+	decoded, err := DecodeConfig(fs, cfg)
 	assert.NoError(err)
 
-	decoded, err := decodeConfig(p)
-	assert.NoError(err)
-
-	assert.Equal(4, len(decoded))
+	assert.Equal(5, len(decoded))
 
 	c2 := decoded["getcsv"]
 	assert.Equal("11h0m0s", c2.MaxAge.String())
@@ -105,14 +101,11 @@ dir = "/path/to/c3"
 
 	cfg, err := config.FromConfigString(configStr, "toml")
 	assert.NoError(err)
-	fs := hugofs.NewMem(cfg)
-	p, err := helpers.NewPathSpec(fs, cfg)
-	assert.NoError(err)
-
-	decoded, err := decodeConfig(p)
+	fs := afero.NewMemMapFs()
+	decoded, err := DecodeConfig(fs, cfg)
 	assert.NoError(err)
 
-	assert.Equal(4, len(decoded))
+	assert.Equal(5, len(decoded))
 
 	for _, v := range decoded {
 		assert.Equal(time.Duration(0), v.MaxAge)
@@ -133,15 +126,13 @@ func TestDecodeConfigDefault(t *testing.T) {
 		cfg.Set("cacheDir", "/cache/thecache")
 	}
 
-	fs := hugofs.NewMem(cfg)
-	p, err := helpers.NewPathSpec(fs, cfg)
-	assert.NoError(err)
+	fs := afero.NewMemMapFs()
 
-	decoded, err := decodeConfig(p)
+	decoded, err := DecodeConfig(fs, cfg)
 
 	assert.NoError(err)
 
-	assert.Equal(4, len(decoded))
+	assert.Equal(5, len(decoded))
 
 	imgConfig := decoded[cacheKeyImages]
 	jsonConfig := decoded[cacheKeyGetJSON]
@@ -183,11 +174,9 @@ dir = "/"
 
 	cfg, err := config.FromConfigString(configStr, "toml")
 	assert.NoError(err)
-	fs := hugofs.NewMem(cfg)
-	p, err := helpers.NewPathSpec(fs, cfg)
-	assert.NoError(err)
+	fs := afero.NewMemMapFs()
 
-	_, err = decodeConfig(p)
+	_, err = DecodeConfig(fs, cfg)
 	assert.Error(err)
 
 }

commands/commandeer.go

@@ -16,6 +16,11 @@ package commands
 import (
 	"bytes"
 	"errors"
+	"sync"
+
+	"golang.org/x/sync/semaphore"
+
+	"github.com/gohugoio/hugo/modules"
 
 	"io/ioutil"
 
@@ -27,8 +32,6 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
-	"strings"
-	"sync"
 	"time"
 
 	"github.com/gohugoio/hugo/common/loggers"
@@ -88,6 +91,8 @@ type commandeer struct {
 	configured bool
 	paused     bool
 
+	fullRebuildSem *semaphore.Weighted
+
 	// Any error from the last build.
 	buildErr error
 }
@@ -153,6 +158,7 @@ func newCommandeer(mustHaveConfigFile, running bool, h *hugoBuilderCommon, f fla
 		doWithCommandeer:    doWithCommandeer,
 		visitedURLs:         types.NewEvictingStringQueue(10),
 		debounce:            rebuildDebouncer,
+		fullRebuildSem:      semaphore.NewWeighted(1),
 		// This will be replaced later, but we need something to log to before the configuration is read.
 		logger: loggers.NewLogger(jww.LevelError, jww.LevelError, os.Stdout, ioutil.Discard, running),
 	}
@@ -290,7 +296,7 @@ func (c *commandeer) loadConfig(mustHaveConfigFile, running bool) error {
 		if mustHaveConfigFile {
 			return err
 		}
-		if err != hugolib.ErrNoConfigFile {
+		if err != hugolib.ErrNoConfigFile && !modules.IsNotExist(err) {
 			return err
 		}
 
@@ -388,21 +394,6 @@ func (c *commandeer) loadConfig(mustHaveConfigFile, running bool) error {
 
 	cfg.Logger.INFO.Println("Using config file:", config.ConfigFileUsed())
 
-	themeDir := c.hugo.PathSpec.GetFirstThemeDir()
-	if themeDir != "" {
-		if _, err := sourceFs.Stat(themeDir); os.IsNotExist(err) {
-			return newSystemError("Unable to find theme Directory:", themeDir)
-		}
-	}
-
-	dir, themeVersionMismatch, minVersion := c.isThemeVsHugoVersionMismatch(sourceFs)
-
-	if themeVersionMismatch {
-		name := filepath.Base(dir)
-		cfg.Logger.ERROR.Printf("%s theme does not support Hugo version %s. Minimum version required is %s\n",
-			strings.ToUpper(name), hugo.CurrentVersion.ReleaseVersion(), minVersion)
-	}
-
 	return nil
 
 }

commands/commands.go

@@ -54,6 +54,7 @@ func (b *commandsBuilder) addAll() *commandsBuilder {
 		newImportCmd(),
 		newGenCmd(),
 		createReleaser(),
+		b.newModCmd(),
 	)
 
 	return b
@@ -189,9 +190,10 @@ Complete documentation is available at http://gohugo.io/.`,
 }
 
 type hugoBuilderCommon struct {
-	source      string
-	baseURL     string
-	environment string
+	source       string
+	baseURL      string
+	environment  string
+	ignoreVendor bool
 
 	buildWatch bool
 
@@ -243,20 +245,26 @@ func (cc *hugoBuilderCommon) getEnvironment(isServer bool) string {
 	return hugo.EnvironmentProduction
 }
 
+func (cc *hugoBuilderCommon) handleCommonBuilderFlags(cmd *cobra.Command) {
+	cmd.PersistentFlags().StringVarP(&cc.source, "source", "s", "", "filesystem path to read files relative from")
+	cmd.PersistentFlags().SetAnnotation("source", cobra.BashCompSubdirsInDir, []string{})
+	cmd.PersistentFlags().StringVarP(&cc.environment, "environment", "e", "", "build environment")
+	cmd.PersistentFlags().StringP("themesDir", "", "", "filesystem path to themes directory")
+	cmd.PersistentFlags().BoolP("ignoreVendor", "", false, "ignores any _vendor directory")
+}
+
 func (cc *hugoBuilderCommon) handleFlags(cmd *cobra.Command) {
+	cc.handleCommonBuilderFlags(cmd)
 	cmd.Flags().Bool("cleanDestinationDir", false, "remove files from destination not found in static directories")
 	cmd.Flags().BoolP("buildDrafts", "D", false, "include content marked as draft")
 	cmd.Flags().BoolP("buildFuture", "F", false, "include content with publishdate in the future")
 	cmd.Flags().BoolP("buildExpired", "E", false, "include expired content")
-	cmd.Flags().StringVarP(&cc.source, "source", "s", "", "filesystem path to read files relative from")
-	cmd.Flags().StringVarP(&cc.environment, "environment", "e", "", "build environment")
 	cmd.Flags().StringP("contentDir", "c", "", "filesystem path to content directory")
 	cmd.Flags().StringP("layoutDir", "l", "", "filesystem path to layout directory")
 	cmd.Flags().StringP("cacheDir", "", "", "filesystem path to cache directory. Defaults: $TMPDIR/hugo_cache/")
 	cmd.Flags().BoolP("ignoreCache", "", false, "ignores the cache directory")
 	cmd.Flags().StringP("destination", "d", "", "filesystem path to write files to")
 	cmd.Flags().StringSliceP("theme", "t", []string{}, "themes to use (located in /themes/THEMENAME/)")
-	cmd.Flags().StringP("themesDir", "", "", "filesystem path to themes directory")
 	cmd.Flags().StringVarP(&cc.baseURL, "baseURL", "b", "", "hostname (and path) to the root, e.g. http://spf13.com/")
 	cmd.Flags().Bool("enableGitInfo", false, "add Git revision, date and author info to the pages")
 	cmd.Flags().BoolVar(&cc.gc, "gc", false, "enable to run some cleanup tasks (remove unused cache files) after the build")

commands/convert.go

@@ -30,7 +30,6 @@ import (
 	"github.com/gohugoio/hugo/parser/metadecoders"
 	"github.com/gohugoio/hugo/parser/pageparser"
 
-	src "github.com/gohugoio/hugo/source"
 	"github.com/pkg/errors"
 
 	"github.com/gohugoio/hugo/hugolib"
@@ -152,8 +151,8 @@ func (cc *convertCmd) convertAndSavePage(p page.Page, site *hugolib.Site, target
 
 	site.Log.INFO.Println("Attempting to convert", p.File().Filename())
 
-	f, _ := p.File().(src.ReadableFile)
-	file, err := f.Open()
+	f := p.File()
+	file, err := f.FileInfo().Meta().Open()
 	if err != nil {
 		site.Log.ERROR.Println(errMsg)
 		file.Close()