Add support for a content dir set per language

```toml
defaultContentLanguage = "en"
defaultContentLanguageInSubdir = true

[Languages]
[Languages.en]
weight = 10
title = "In English"
languageName = "English"
contentDir = "content/english"

[Languages.nn]
weight = 20
title = "På Norsk"
languageName = "Norsk"
contentDir = "content/norwegian"
```

The value of `contentDir` can be any valid path, even absolute path references. The only restriction is that the content dirs cannot overlap.

The content files will be assigned a language by

1. The placement: `content/norwegian/post/my-post.md` will be read as Norwegian content.
2. The filename: `content/english/post/my-post.nn.md` will be read as Norwegian even if it lives in the English content folder.

The content directories will be merged into a big virtual filesystem with one simple rule: The most specific language file will win.
This means that if both `content/norwegian/post/my-post.md` and `content/english/post/my-post.nn.md` exists, they will be considered duplicates and the version inside `content/norwegian` will win.

Note that translations will be automatically assigned by Hugo by the content file's relative placement, so `content/norwegian/post/my-post.md` will be a translation of `content/english/post/my-post.md`.

If this does not work for you, you can connect the translations together by setting a `translationKey` in the content files' front matter.

Fixes gohugoio#4523
Fixes gohugoio#4552
Fixes gohugoio#4553

Gopkg.toml

@@ -78,7 +78,7 @@
 
 [[constraint]]
   name = "github.com/spf13/afero"
-  version = "^1.0.1"
+  branch = "lstater"
 
 [[constraint]]
   name = "github.com/spf13/cast"

commands/hugo.go

@@ -705,7 +705,7 @@ func (c *commandeer) getDirList() ([]string, error) {
 					c.Logger.ERROR.Printf("Cannot read symbolic link '%s', error was: %s", path, err)
 					return nil
 				}
-				linkfi, err := helpers.LstatIfOs(c.Fs.Source, link)
+				linkfi, err := helpers.LstatIfPossible(c.Fs.Source, link)
 				if err != nil {
 					c.Logger.ERROR.Printf("Cannot stat %q: %s", link, err)
 					return nil
@@ -743,9 +743,13 @@ func (c *commandeer) getDirList() ([]string, error) {
 
 	// SymbolicWalk will log anny ERRORs
 	_ = helpers.SymbolicWalk(c.Fs.Source, dataDir, regularWalker)
-	_ = helpers.SymbolicWalk(c.Fs.Source, c.PathSpec().AbsPathify(c.Cfg.GetString("contentDir")), symLinkWalker)
 	_ = helpers.SymbolicWalk(c.Fs.Source, i18nDir, regularWalker)
 	_ = helpers.SymbolicWalk(c.Fs.Source, layoutDir, regularWalker)
+
+	for _, contentDir := range c.PathSpec().ContentDirs() {
+		_ = helpers.SymbolicWalk(c.Fs.Source, contentDir.Value, symLinkWalker)
+	}
+
 	for _, staticDir := range staticDirs {
 		_ = helpers.SymbolicWalk(c.Fs.Source, staticDir, regularWalker)
 	}

common/types/types.go

@@ -20,6 +20,12 @@ import (
 	"github.com/spf13/cast"
 )
 
+// KeyValueStr is a string tuple.
+type KeyValueStr struct {
+	Key   string
+	Value string
+}
+
 // KeyValues holds an key and a slice of values.
 type KeyValues struct {
 	Key    interface{}

create/content.go

@@ -63,7 +63,22 @@ func NewContent(
 		return err
 	}
 
-	contentPath := s.PathSpec.AbsPathify(filepath.Join(s.Cfg.GetString("contentDir"), targetPath))
+	// The site may have multiple content dirs, and we currently does not know which contentDir the
+	// user wants to create this content in. We should improve on this, but we start by testing if the
+	// provided path points to an existing dir. If that is a fact, use it as is.
+	var contentPath string
+	var exists bool
+	targetDir := filepath.Dir(targetPath)
+
+	if targetDir != "" && targetDir != "." {
+		exists, _ = helpers.Exists(targetDir, ps.Fs.Source)
+	}
+
+	if exists {
+		contentPath = targetPath
+	} else {
+		contentPath = s.PathSpec.AbsPathify(filepath.Join(s.Cfg.GetString("contentDir"), targetPath))
+	}
 
 	if err := helpers.SafeWriteToDisk(contentPath, bytes.NewReader(content), s.Fs.Source); err != nil {
 		return err

create/content_template_handler.go

@@ -88,10 +88,15 @@ func executeArcheTypeAsTemplate(s *hugolib.Site, kind, targetPath, archetypeFile
 		err               error
 	)
 
-	sp := source.NewSourceSpec(s.Deps.Cfg, s.Deps.Fs)
+	ps, err := helpers.NewPathSpec(s.Deps.Fs, s.Deps.Cfg)
+	sp := source.NewSourceSpec(ps, ps.Fs.Source)
+	if err != nil {
+		return nil, err
+	}
 	f := sp.NewFileInfo("", targetPath, false, nil)
 
 	name := f.TranslationBaseName()
+
 	if name == "index" || name == "_index" {
 		// Page bundles; the directory name will hopefully have a better name.
 		dir := strings.TrimSuffix(f.Dir(), helpers.FilePathSeparator)

create/content_test.go

@@ -75,7 +75,7 @@ func TestNewContent(t *testing.T) {
 		for i, v := range c.expected {
 			found := strings.Contains(content, v)
 			if !found {
-				t.Errorf("[%d] %q missing from output:\n%q", i, v, content)
+				t.Fatalf("[%d] %q missing from output:\n%q", i, v, content)
 			}
 		}
 	}

deps/deps.go

@@ -126,7 +126,7 @@ func New(cfg DepsCfg) (*Deps, error) {
 		return nil, err
 	}
 
-	sp := source.NewSourceSpec(cfg.Language, fs)
+	sp := source.NewSourceSpec(ps, fs.Source)
 
 	d := &Deps{
 		Fs:                  fs,

helpers/language.go

@@ -41,6 +41,14 @@ type Language struct {
 	Title        string
 	Weight       int
 
+	Disabled bool
+
+	// If set per language, this tells Hugo that all content files without any
+	// language indicator (e.g. my-page.en.md) is in this language.
+	// This is usually a path relative to the working dir, but it can be an
+	// absolute directory referenece. It is what we get.
+	ContentDir string
+
 	Cfg config.Provider
 
 	// These are params declared in the [params] section of the language merged with the
@@ -66,7 +74,13 @@ func NewLanguage(lang string, cfg config.Provider) *Language {
 		params[k] = v
 	}
 	ToLowerMap(params)
-	l := &Language{Lang: lang, Cfg: cfg, params: params, settings: make(map[string]interface{})}
+
+	defaultContentDir := cfg.GetString("contentDir")
+	if defaultContentDir == "" {
+		panic("contentDir not set")
+	}
+
+	l := &Language{Lang: lang, ContentDir: defaultContentDir, Cfg: cfg, params: params, settings: make(map[string]interface{})}
 	return l
 }
 

helpers/language_test.go

@@ -22,11 +22,12 @@ import (
 
 func TestGetGlobalOnlySetting(t *testing.T) {
 	v := viper.New()
+	v.Set("defaultContentLanguageInSubdir", true)
+	v.Set("contentDir", "content")
+	v.Set("paginatePath", "page")
 	lang := NewDefaultLanguage(v)
 	lang.Set("defaultContentLanguageInSubdir", false)
 	lang.Set("paginatePath", "side")
-	v.Set("defaultContentLanguageInSubdir", true)
-	v.Set("paginatePath", "page")
 
 	require.True(t, lang.GetBool("defaultContentLanguageInSubdir"))
 	require.Equal(t, "side", lang.GetString("paginatePath"))
@@ -37,6 +38,7 @@ func TestLanguageParams(t *testing.T) {
 
 	v := viper.New()
 	v.Set("p1", "p1cfg")
+	v.Set("contentDir", "content")
 
 	lang := NewDefaultLanguage(v)
 	lang.SetParam("p1", "p1p")

helpers/path.go

@@ -33,7 +33,7 @@ var (
 	ErrThemeUndefined = errors.New("no theme set")
 
 	// ErrWalkRootTooShort is returned when the root specified for a file walk is shorter than 4 characters.
-	ErrWalkRootTooShort = errors.New("Path too short. Stop walking.")
+	ErrPathTooShort = errors.New("file path is too short")
 )
 
 // filepathPathBridge is a bridge for common functionality in filepath vs path
@@ -446,7 +446,7 @@ func SymbolicWalk(fs afero.Fs, root string, walker filepath.WalkFunc) error {
 
 	// Sanity check
 	if len(root) < 4 {
-		return ErrWalkRootTooShort
+		return ErrPathTooShort
 	}
 
 	// Handle the root first
@@ -481,7 +481,7 @@ func SymbolicWalk(fs afero.Fs, root string, walker filepath.WalkFunc) error {
 }
 
 func getRealFileInfo(fs afero.Fs, path string) (os.FileInfo, string, error) {
-	fileInfo, err := LstatIfOs(fs, path)
+	fileInfo, err := LstatIfPossible(fs, path)
 	realPath := path
 
 	if err != nil {
@@ -493,7 +493,7 @@ func getRealFileInfo(fs afero.Fs, path string) (os.FileInfo, string, error) {
 		if err != nil {
 			return nil, "", fmt.Errorf("Cannot read symbolic link '%s', error was: %s", path, err)
 		}
-		fileInfo, err = LstatIfOs(fs, link)
+		fileInfo, err = LstatIfPossible(fs, link)
 		if err != nil {
 			return nil, "", fmt.Errorf("Cannot stat '%s', error was: %s", link, err)
 		}
@@ -514,16 +514,14 @@ func GetRealPath(fs afero.Fs, path string) (string, error) {
 	return realPath, nil
 }
 
-// Code copied from Afero's path.go
-// if the filesystem is OsFs use Lstat, else use fs.Stat
-func LstatIfOs(fs afero.Fs, path string) (info os.FileInfo, err error) {
-	_, ok := fs.(*afero.OsFs)
-	if ok {
-		info, err = os.Lstat(path)
-	} else {
-		info, err = fs.Stat(path)
+// LstatIfPossible can be used to call Lstat if possible, else Stat.
+func LstatIfPossible(fs afero.Fs, path string) (os.FileInfo, error) {
+	if lstater, ok := fs.(afero.Lstater); ok {
+		fi, _, err := lstater.LstatIfPossible(path)
+		return fi, err
 	}
-	return
+
+	return fs.Stat(path)
 }
 
 // SafeWriteToDisk is the same as WriteToDisk

helpers/path_test.go

@@ -57,8 +57,10 @@ func TestMakePath(t *testing.T) {
 
 	for _, test := range tests {
 		v := viper.New()
-		l := NewDefaultLanguage(v)
+		v.Set("contentDir", "content")
 		v.Set("removePathAccents", test.removeAccents)
+
+		l := NewDefaultLanguage(v)
 		p, err := NewPathSpec(hugofs.NewMem(v), l)
 		require.NoError(t, err)
 
@@ -71,6 +73,8 @@ func TestMakePath(t *testing.T) {
 
 func TestMakePathSanitized(t *testing.T) {
 	v := viper.New()
+	v.Set("contentDir", "content")
+
 	l := NewDefaultLanguage(v)
 	p, _ := NewPathSpec(hugofs.NewMem(v), l)
 
@@ -98,6 +102,7 @@ func TestMakePathSanitizedDisablePathToLower(t *testing.T) {
 	v := viper.New()
 
 	v.Set("disablePathToLower", true)
+	v.Set("contentDir", "content")
 
 	l := NewDefaultLanguage(v)
 	p, _ := NewPathSpec(hugofs.NewMem(v), l)