Add Hugo Modules

Fixes #5973 Fixes #5996 Fixes #6010 Fixes #5911 Fixes #5940 Fixes #6074 Fixes #6082 Fixes #6092
gohugoio · Jul 19, 2019 · 40072a4 · 40072a4
1 parent ac101ab
commit 40072a4
Show file tree

Hide file tree

Showing 148 changed files with 8,834 additions and 5,029 deletions.
diff --git a/benchbep.sh b/benchbep.sh
@@ -1,2 +1 @@
-gobench -package=./hugolib -bench="BenchmarkSiteBuilding/YAML,num_langs=3,num_pages=5000,tags_per_page=5,shortcodes,render" -count=3 > 1.bench
-benchcmp -best 0.bench 1.bench
+gobench -package=./hugolib -bench="BenchmarkSiteNew/Deep_content_tree"
diff --git a/cache/filecache/filecache.go b/cache/filecache/filecache.go
@@ -319,7 +319,7 @@ func NewCaches(p *helpers.PathSpec) (Caches, error) {
 		var cfs afero.Fs
 
 		if v.isResourceDir {
-			cfs = p.BaseFs.Resources.Fs
+			cfs = p.BaseFs.ResourcesCache
 		} else {
 			cfs = fs
 		}

diff --git a/cache/filecache/filecache_config_test.go b/cache/filecache/filecache_config_test.go
@@ -20,6 +20,10 @@ import (
 	"testing"
 	"time"
 
+	"github.com/gohugoio/hugo/modules"
+
+	"github.com/gohugoio/hugo/langs"
+
 	"github.com/gohugoio/hugo/helpers"
 
 	"github.com/gohugoio/hugo/config"
@@ -55,8 +59,9 @@ dir = "/path/to/c3"
 
 `
 
-	cfg, err := config.FromConfigString(configStr, "toml")
+	cfg, err := configFromString(configStr)
 	assert.NoError(err)
+
 	fs := hugofs.NewMem(cfg)
 	p, err := helpers.NewPathSpec(fs, cfg)
 	assert.NoError(err)
@@ -76,6 +81,23 @@ dir = "/path/to/c3"
 
 }
 
+func configFromString(configStr string) (config.Provider, error) {
+	cfg, err := config.FromConfigString(configStr, "toml")
+	if err != nil {
+		return nil, err
+	}
+	if _, err := langs.LoadLanguageSettings(cfg, nil); err != nil {
+		return nil, err
+	}
+	mod, err := modules.CreateProjectModule(cfg)
+	if err != nil {
+		return nil, err
+	}
+	cfg.Set("allModules", modules.Modules{mod})
+
+	return cfg, nil
+}
+
 func TestDecodeConfigIgnoreCache(t *testing.T) {
 	t.Parallel()
 
@@ -103,8 +125,7 @@ dir = "/path/to/c3"
 
 `
 
-	cfg, err := config.FromConfigString(configStr, "toml")
-	assert.NoError(err)
+	cfg, err := configFromString(configStr)
 	fs := hugofs.NewMem(cfg)
 	p, err := helpers.NewPathSpec(fs, cfg)
 	assert.NoError(err)
@@ -181,8 +202,7 @@ dir = "/"
 		configStr = strings.Replace(configStr, "/", "c:\\\\", 1)
 	}
 
-	cfg, err := config.FromConfigString(configStr, "toml")
-	assert.NoError(err)
+	cfg, err := configFromString(configStr)
 	fs := hugofs.NewMem(cfg)
 	p, err := helpers.NewPathSpec(fs, cfg)
 	assert.NoError(err)
@@ -203,5 +223,12 @@ func newTestConfig() *viper.Viper {
 	cfg.Set("archetypeDir", "archetypes")
 	cfg.Set("assetDir", "assets")
 
+	langs.LoadLanguageSettings(cfg, nil)
+	mod, err := modules.CreateProjectModule(cfg)
+	if err != nil {
+		panic(err)
+	}
+	cfg.Set("allModules", modules.Modules{mod})
+
 	return cfg
 }
diff --git a/cache/filecache/filecache_pruner_test.go b/cache/filecache/filecache_pruner_test.go
@@ -18,7 +18,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/gohugoio/hugo/config"
 	"github.com/gohugoio/hugo/helpers"
 	"github.com/gohugoio/hugo/hugofs"
 
@@ -54,7 +53,7 @@ maxAge = "200ms"
 dir = ":resourceDir/_gen"
 `
 
-	cfg, err := config.FromConfigString(configStr, "toml")
+	cfg, err := configFromString(configStr)
 	assert.NoError(err)
 
 	for _, name := range []string{cacheKeyGetCSV, cacheKeyGetJSON, cacheKeyAssets, cacheKeyImages} {

diff --git a/cache/filecache/filecache_test.go b/cache/filecache/filecache_test.go
@@ -26,7 +26,6 @@ import (
 	"time"
 
 	"github.com/gohugoio/hugo/common/hugio"
-	"github.com/gohugoio/hugo/config"
 	"github.com/gohugoio/hugo/helpers"
 
 	"github.com/gohugoio/hugo/hugofs"
@@ -83,8 +82,7 @@ dir = ":cacheDir/c"
 		configStr = replacer.Replace(configStr)
 		configStr = strings.Replace(configStr, "\\", winPathSep, -1)
 
-		cfg, err := config.FromConfigString(configStr, "toml")
-		assert.NoError(err)
+		cfg, err := configFromString(configStr)
 
 		fs := hugofs.NewFrom(osfs, cfg)
 		p, err := helpers.NewPathSpec(fs, cfg)
@@ -207,8 +205,7 @@ dir = "/cache/c"
 
 `
 
-	cfg, err := config.FromConfigString(configStr, "toml")
-	assert.NoError(err)
+	cfg, err := configFromString(configStr)
 	fs := hugofs.NewMem(cfg)
 	p, err := helpers.NewPathSpec(fs, cfg)
 	assert.NoError(err)

diff --git a/commands/commandeer.go b/commands/commandeer.go
@@ -17,6 +17,8 @@ import (
 	"bytes"
 	"errors"
 
+	"github.com/gohugoio/hugo/modules"
+
 	"io/ioutil"
 
 	"github.com/gohugoio/hugo/common/herrors"
@@ -27,7 +29,6 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
-	"strings"
 	"sync"
 	"time"
 
@@ -290,7 +291,7 @@ func (c *commandeer) loadConfig(mustHaveConfigFile, running bool) error {
 		if mustHaveConfigFile {
 			return err
 		}
-		if err != hugolib.ErrNoConfigFile {
+		if err != hugolib.ErrNoConfigFile && !modules.IsNotExist(err) {
 			return err
 		}
 
@@ -388,21 +389,23 @@ func (c *commandeer) loadConfig(mustHaveConfigFile, running bool) error {
 
 	cfg.Logger.INFO.Println("Using config file:", config.ConfigFileUsed())
 
-	themeDir := c.hugo.PathSpec.GetFirstThemeDir()
-	if themeDir != "" {
-		if _, err := sourceFs.Stat(themeDir); os.IsNotExist(err) {
-			return newSystemError("Unable to find theme Directory:", themeDir)
+	// TODO(bep) mod
+	/*
+		themeDir := c.hugo.PathSpec.GetFirstThemeDir()
+		if themeDir != "" {
+			if _, err := sourceFs.Stat(themeDir); os.IsNotExist(err) {
+				return newSystemError("Unable to find theme Directory:", themeDir)
+			}
 		}
-	}
 
-	dir, themeVersionMismatch, minVersion := c.isThemeVsHugoVersionMismatch(sourceFs)
-
-	if themeVersionMismatch {
-		name := filepath.Base(dir)
-		cfg.Logger.ERROR.Printf("%s theme does not support Hugo version %s. Minimum version required is %s\n",
-			strings.ToUpper(name), hugo.CurrentVersion.ReleaseVersion(), minVersion)
-	}
+		dir, themeVersionMismatch, minVersion := c.isThemeVsHugoVersionMismatch(sourceFs)
 
+		if themeVersionMismatch {
+			name := filepath.Base(dir)
+			cfg.Logger.ERROR.Printf("%s theme does not support Hugo version %s. Minimum version required is %s\n",
+				strings.ToUpper(name), hugo.CurrentVersion.ReleaseVersion(), minVersion)
+		}
+	*/
 	return nil
 
 }
diff --git a/commands/commands.go b/commands/commands.go
@@ -54,6 +54,7 @@ func (b *commandsBuilder) addAll() *commandsBuilder {
 		newImportCmd(),
 		newGenCmd(),
 		createReleaser(),
+		b.newModCmd(),
 	)
 
 	return b
@@ -189,9 +190,10 @@ Complete documentation is available at http://gohugo.io/.`,
 }
 
 type hugoBuilderCommon struct {
-	source      string
-	baseURL     string
-	environment string
+	source       string
+	baseURL      string
+	environment  string
+	ignoreVendor bool
 
 	buildWatch bool
 
@@ -243,20 +245,26 @@ func (cc *hugoBuilderCommon) getEnvironment(isServer bool) string {
 	return hugo.EnvironmentProduction
 }
 
+func (cc *hugoBuilderCommon) handleCommonBuilderFlags(cmd *cobra.Command) {
+	cmd.PersistentFlags().StringVarP(&cc.source, "source", "s", "", "filesystem path to read files relative from")
+	cmd.PersistentFlags().SetAnnotation("source", cobra.BashCompSubdirsInDir, []string{})
+	cmd.PersistentFlags().StringVarP(&cc.environment, "environment", "e", "", "build environment")
+	cmd.PersistentFlags().StringP("themesDir", "", "", "filesystem path to themes directory")
+	cmd.PersistentFlags().BoolP("ignoreVendor", "", false, "ignores any _vendor directory")
+}
+
 func (cc *hugoBuilderCommon) handleFlags(cmd *cobra.Command) {
+	cc.handleCommonBuilderFlags(cmd)
 	cmd.Flags().Bool("cleanDestinationDir", false, "remove files from destination not found in static directories")
 	cmd.Flags().BoolP("buildDrafts", "D", false, "include content marked as draft")
 	cmd.Flags().BoolP("buildFuture", "F", false, "include content with publishdate in the future")
 	cmd.Flags().BoolP("buildExpired", "E", false, "include expired content")
-	cmd.Flags().StringVarP(&cc.source, "source", "s", "", "filesystem path to read files relative from")
-	cmd.Flags().StringVarP(&cc.environment, "environment", "e", "", "build environment")
 	cmd.Flags().StringP("contentDir", "c", "", "filesystem path to content directory")
 	cmd.Flags().StringP("layoutDir", "l", "", "filesystem path to layout directory")
 	cmd.Flags().StringP("cacheDir", "", "", "filesystem path to cache directory. Defaults: $TMPDIR/hugo_cache/")
 	cmd.Flags().BoolP("ignoreCache", "", false, "ignores the cache directory")
 	cmd.Flags().StringP("destination", "d", "", "filesystem path to write files to")
 	cmd.Flags().StringSliceP("theme", "t", []string{}, "themes to use (located in /themes/THEMENAME/)")
-	cmd.Flags().StringP("themesDir", "", "", "filesystem path to themes directory")
 	cmd.Flags().StringVarP(&cc.baseURL, "baseURL", "b", "", "hostname (and path) to the root, e.g. http://spf13.com/")
 	cmd.Flags().Bool("enableGitInfo", false, "add Git revision, date and author info to the pages")
 	cmd.Flags().BoolVar(&cc.gc, "gc", false, "enable to run some cleanup tasks (remove unused cache files) after the build")

diff --git a/commands/convert.go b/commands/convert.go
@@ -30,7 +30,6 @@ import (
 	"github.com/gohugoio/hugo/parser/metadecoders"
 	"github.com/gohugoio/hugo/parser/pageparser"
 
-	src "github.com/gohugoio/hugo/source"
 	"github.com/pkg/errors"
 
 	"github.com/gohugoio/hugo/hugolib"
@@ -152,8 +151,8 @@ func (cc *convertCmd) convertAndSavePage(p page.Page, site *hugolib.Site, target
 
 	site.Log.INFO.Println("Attempting to convert", p.File().Filename())
 
-	f, _ := p.File().(src.ReadableFile)
-	file, err := f.Open()
+	f := p.File()
+	file, err := f.FileInfo().Meta().Open()
 	if err != nil {
 		site.Log.ERROR.Println(errMsg)
 		file.Close()