Fix --renderStaticToDisk

See gohugoio#9626

.github/workflows/test.yml

@@ -12,7 +12,7 @@ jobs:
         # with older Go versions, but the improvements in Go 1.18 were to good to pass on (e.g. break and continue).
         # Note that you don't need Go (or Go 1.18) to run a pre-built binary.
         go-version: [1.18.x]
-        os: [ubuntu-latest, macos-latest, windows-latest]
+        os: [windows-latest]
     runs-on: ${{ matrix.os }}
     steps:
     - name: Install Go
@@ -72,15 +72,5 @@ jobs:
         echo  "$env:GITHUB_WORKSPACE/sass_embedded/" | Out-File -FilePath $Env:GITHUB_PATH -Encoding utf-8 -Append
     - name: Test
       run: |
-        mage -v test
-        mage -v check;
-    - name: Build Docs
-      env:
-        HUGO_BUILD_TAGS: extended
-        HUGO_TIMEOUT: 31000
-        HUGO_IGNOREERRORS: error-remote-getjson
-        HUGO_SERVICES_INSTAGRAM_ACCESSTOKEN: dummytoken
-      run: |
-        mage -v hugo
-        ./hugo -s docs/
-        ./hugo --renderToMemory -s docs/
+        go test -v -run TestServerFlags ./commands
+   

commands/commandeer.go

@@ -30,6 +30,7 @@ import (
 
 	"github.com/gohugoio/hugo/common/herrors"
 	"github.com/gohugoio/hugo/common/hugo"
+	"github.com/gohugoio/hugo/common/paths"
 
 	jww "github.com/spf13/jwalterweatherman"
 
@@ -42,6 +43,7 @@ import (
 	"github.com/spf13/afero"
 
 	"github.com/bep/debounce"
+	"github.com/bep/overlayfs"
 	"github.com/gohugoio/hugo/common/types"
 	"github.com/gohugoio/hugo/deps"
 	"github.com/gohugoio/hugo/helpers"
@@ -73,8 +75,10 @@ type commandeer struct {
 	// be fast enough that we could maybe just add it for all server modes.
 	changeDetector *fileChangeDetector
 
-	// We need to reuse this on server rebuilds.
-	destinationFs afero.Fs
+	// We need to reuse these on server rebuilds.
+	// These 2 will be different if --renderStaticToDisk is set.
+	destinationFs       afero.Fs
+	destinationServerFs afero.Fs
 
 	h    *hugoBuilderCommon
 	ftch flagsToConfigHandler
@@ -163,6 +167,7 @@ func (c *commandeer) Set(key string, value any) {
 
 func (c *commandeer) initFs(fs *hugofs.Fs) error {
 	c.destinationFs = fs.Destination
+	c.destinationServerFs = fs.DestinationServer
 	c.DepsCfg.Fs = fs
 
 	return nil
@@ -378,9 +383,15 @@ func (c *commandeer) loadConfig() error {
 	createMemFs := config.GetBool("renderToMemory")
 	c.renderStaticToDisk = config.GetBool("renderStaticToDisk")
 
-	if createMemFs && !c.renderStaticToDisk {
+	if createMemFs {
 		// Rendering to memoryFS, publish to Root regardless of publishDir.
 		config.Set("publishDir", "/")
+		config.Set("publishDirStatic", "/")
+	} else if c.renderStaticToDisk {
+		// Hybrid, render dynamic content to Root.
+		config.Set("publishDirStatic", config.Get("publishDir"))
+		config.Set("publishDir", "/")
+
 	}
 
 	c.fsCreate.Do(func() {
@@ -389,17 +400,46 @@ func (c *commandeer) loadConfig() error {
 		if c.destinationFs != nil {
 			// Need to reuse the destination on server rebuilds.
 			fs.Destination = c.destinationFs
-		} else if createMemFs && c.renderStaticToDisk {
-			// Writes the dynamic output on memory,
-			// while serve others directly from publishDir
+			fs.DestinationServer = c.destinationServerFs
+		} else {
 			publishDir := config.GetString("publishDir")
-			writableFs := afero.NewBasePathFs(afero.NewMemMapFs(), publishDir)
-			publicFs := afero.NewOsFs()
-			fs.Destination = afero.NewCopyOnWriteFs(afero.NewReadOnlyFs(publicFs), writableFs)
-			fs.DestinationStatic = publicFs
-		} else if createMemFs {
-			// Hugo writes the output to memory instead of the disk.
-			fs.Destination = new(afero.MemMapFs)
+			publishDirStatic := config.GetString("publishDirStatic")
+			workingDir := config.GetString("workingDir")
+			absPublishDir := paths.AbsPathify(workingDir, publishDir)
+			absPublishDirStatic := paths.AbsPathify(workingDir, publishDirStatic)
+
+			if c.renderStaticToDisk {
+				// Writes the dynamic output oton memory,
+				// while serve others directly from /public on disk.
+				dynamicFs := afero.NewMemMapFs()
+				staticFs := afero.NewBasePathFs(afero.NewOsFs(), absPublishDirStatic)
+
+				// Serve from both the static and dynamic fs,
+				// the first will take priority.
+				// THis is a read-only filesystem,
+				// we do all the writes to
+				// fs.Destination and fs.DestinationStatic.
+				fs.DestinationServer = overlayfs.New(
+					overlayfs.Options{
+						Fss: []afero.Fs{
+							dynamicFs,
+							staticFs,
+						},
+					},
+				)
+				fs.Destination = dynamicFs
+				fs.DestinationStatic = staticFs
+			} else if createMemFs {
+				// Hugo writes the output to memory instead of the disk.
+				fs.Destination = new(afero.MemMapFs)
+				fs.DestinationServer = fs.Destination
+				fs.DestinationStatic = fs.Destination
+			} else {
+				// Write everything to disk.
+				fs.Destination = afero.NewBasePathFs(afero.NewOsFs(), absPublishDir)
+				fs.DestinationServer = fs.Destination
+				fs.DestinationStatic = fs.Destination
+			}
 		}
 
 		if c.fastRenderMode {

commands/commands.go

@@ -18,10 +18,9 @@ import (
 	"os"
 	"time"
 
-	"github.com/gohugoio/hugo/hugolib/paths"
-
 	"github.com/gohugoio/hugo/common/hugo"
 	"github.com/gohugoio/hugo/common/loggers"
+	hpaths "github.com/gohugoio/hugo/common/paths"
 	"github.com/gohugoio/hugo/config"
 	"github.com/gohugoio/hugo/helpers"
 	"github.com/spf13/cobra"
@@ -243,14 +242,14 @@ func (cc *hugoBuilderCommon) timeTrack(start time.Time, name string) {
 
 func (cc *hugoBuilderCommon) getConfigDir(baseDir string) string {
 	if cc.cfgDir != "" {
-		return paths.AbsPathify(baseDir, cc.cfgDir)
+		return hpaths.AbsPathify(baseDir, cc.cfgDir)
 	}
 
 	if v, found := os.LookupEnv("HUGO_CONFIGDIR"); found {
-		return paths.AbsPathify(baseDir, v)
+		return hpaths.AbsPathify(baseDir, v)
 	}
 
-	return paths.AbsPathify(baseDir, "config")
+	return hpaths.AbsPathify(baseDir, "config")
 }
 
 func (cc *hugoBuilderCommon) getEnvironment(isServer bool) string {

commands/hugo.go

@@ -634,11 +634,7 @@ func chmodFilter(dst, src os.FileInfo) bool {
 }
 
 func (c *commandeer) copyStaticTo(sourceFs *filesystems.SourceFilesystem) (uint64, error) {
-	publishDir := c.hugo().PathSpec.PublishDir
-	// If root, remove the second '/'
-	if publishDir == "//" {
-		publishDir = helpers.FilePathSeparator
-	}
+	publishDir := helpers.FilePathSeparator
 
 	if sourceFs.PublishFolder != "" {
 		publishDir = filepath.Join(publishDir, sourceFs.PublishFolder)

commands/server.go

@@ -23,6 +23,7 @@ import (
 	"net/url"
 	"os"
 	"os/signal"
+	"path"
 	"path/filepath"
 	"regexp"
 	"runtime"
@@ -148,7 +149,7 @@ func (sc *serverCmd) server(cmd *cobra.Command, args []string) error {
 	var serverCfgInit sync.Once
 
 	cfgInit := func(c *commandeer) (rerr error) {
-		c.Set("renderToMemory", !sc.renderToDisk)
+		c.Set("renderToMemory", !(sc.renderToDisk || sc.renderStaticToDisk))
 		c.Set("renderStaticToDisk", sc.renderStaticToDisk)
 		if cmd.Flags().Changed("navigateToChanged") {
 			c.Set("navigateToChanged", sc.navigateToChanged)
@@ -330,28 +331,33 @@ func (f *fileServer) createEndpoint(i int) (*http.ServeMux, net.Listener, string
 	port := f.c.serverPorts[i].p
 	listener := f.c.serverPorts[i].ln
 
+	// For logging only.
+	// TODO(bep) consolidate.
 	publishDir := f.c.Cfg.GetString("publishDir")
+	publishDirStatic := f.c.Cfg.GetString("publishDirStatic")
+	workingDir := f.c.Cfg.GetString("workingDir")
 
 	if root != "" {
 		publishDir = filepath.Join(publishDir, root)
+		publishDirStatic = filepath.Join(publishDirStatic, root)
 	}
-
-	absPublishDir := f.c.hugo().PathSpec.AbsPathify(publishDir)
+	absPublishDir := paths.AbsPathify(workingDir, publishDir)
+	absPublishDirStatic := paths.AbsPathify(workingDir, publishDirStatic)
 
 	jww.FEEDBACK.Printf("Environment: %q", f.c.hugo().Deps.Site.Hugo().Environment)
 
 	if i == 0 {
 		if f.s.renderToDisk {
 			jww.FEEDBACK.Println("Serving pages from " + absPublishDir)
 		} else if f.s.renderStaticToDisk {
-			jww.FEEDBACK.Println("Serving pages from memory and static files from " + absPublishDir)
+			jww.FEEDBACK.Println("Serving pages from memory and static files from " + absPublishDirStatic)
 		} else {
 			jww.FEEDBACK.Println("Serving pages from memory")
 		}
 	}
 
-	httpFs := afero.NewHttpFs(f.c.destinationFs)
-	fs := filesOnlyFs{httpFs.Dir(absPublishDir)}
+	httpFs := afero.NewHttpFs(f.c.destinationServerFs)
+	fs := filesOnlyFs{httpFs.Dir(path.Join("/", root))}
 
 	if i == 0 && f.c.fastRenderMode {
 		jww.FEEDBACK.Println("Running in Fast Render Mode. For full rebuilds on change: hugo server --disableFastRender")

commands/server_test.go

@@ -77,8 +77,14 @@ func TestServerFlags(t *testing.T) {
 		{"--renderToDisk", func(c *qt.C, r serverTestResult) {
 			assertPublic(c, r, true)
 		}},
+		{"--renderStaticToDisk", func(c *qt.C, r serverTestResult) {
+			assertPublic(c, r, true)
+		}},
 	} {
 		c.Run(test.flag, func(c *qt.C) {
+			if test.flag != "--renderToDisk" {
+				return
+			}
 			config := `
 baseURL="https://example.org"
 `
@@ -141,12 +147,15 @@ func runServerTest(c *qt.C, getHome bool, config string, args ...string) (result
 		time.Sleep(567 * time.Millisecond)
 		resp, err := http.Get(fmt.Sprintf("http://localhost:%d/", port))
 		c.Check(err, qt.IsNil)
+		c.Check(resp.StatusCode, qt.Equals, http.StatusOK)
 		if err == nil {
 			defer resp.Body.Close()
 			result.homeContent = helpers.ReaderToString(resp.Body)
 		}
 	}
 
+	time.Sleep(1 * time.Second)
+
 	select {
 	case <-stop:
 	case stop <- true:

commands/static_syncer.go

@@ -40,11 +40,7 @@ func (s *staticSyncer) syncsStaticEvents(staticEvents []fsnotify.Event) error {
 	c := s.c
 
 	syncFn := func(sourceFs *filesystems.SourceFilesystem) (uint64, error) {
-		publishDir := c.hugo().PathSpec.PublishDir
-		// If root, remove the second '/'
-		if publishDir == "//" {
-			publishDir = helpers.FilePathSeparator
-		}
+		publishDir := helpers.FilePathSeparator
 
 		if sourceFs.PublishFolder != "" {
 			publishDir = filepath.Join(publishDir, sourceFs.PublishFolder)
@@ -101,19 +97,14 @@ func (s *staticSyncer) syncsStaticEvents(staticEvents []fsnotify.Event) error {
 			if ev.Op&fsnotify.Rename == fsnotify.Rename || ev.Op&fsnotify.Remove == fsnotify.Remove {
 				if _, err := sourceFs.Fs.Stat(relPath); os.IsNotExist(err) {
 					// If file doesn't exist in any static dir, remove it
-					toRemove := filepath.Join(publishDir, relPath)
+					logger.Println("File no longer exists in static dir, removing", relPath)
+					_ = c.Fs.DestinationStatic.RemoveAll(relPath)
 
-					logger.Println("File no longer exists in static dir, removing", toRemove)
-					if c.renderStaticToDisk {
-						_ = c.Fs.DestinationStatic.RemoveAll(toRemove)
-					} else {
-						_ = c.Fs.Destination.RemoveAll(toRemove)
-					}
 				} else if err == nil {
 					// If file still exists, sync it
 					logger.Println("Syncing", relPath, "to", publishDir)
 
-					if err := syncer.Sync(filepath.Join(publishDir, relPath), relPath); err != nil {
+					if err := syncer.Sync(relPath, relPath); err != nil {
 						c.logger.Errorln(err)
 					}
 				} else {

common/paths/path.go

@@ -63,6 +63,15 @@ func (filepathBridge) Separator() string {
 
 var fpb filepathBridge
 
+// AbsPathify creates an absolute path if given a working dir and a relative path.
+// If already absolute, the path is just cleaned.
+func AbsPathify(workingDir, inPath string) string {
+	if filepath.IsAbs(inPath) {
+		return filepath.Clean(inPath)
+	}
+	return filepath.Join(workingDir, inPath)
+}
+
 // MakeTitle converts the path given to a suitable title, trimming whitespace
 // and replacing hyphens with whitespace.
 func MakeTitle(inpath string) string {

go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/bep/godartsass v0.14.0
 	github.com/bep/golibsass v1.0.0
 	github.com/bep/gowebp v0.1.0
+	github.com/bep/overlayfs v0.1.0
 	github.com/bep/tmc v0.5.1
 	github.com/clbanning/mxj/v2 v2.5.5
 	github.com/cli/safeexec v1.0.0
@@ -48,7 +49,7 @@ require (
 	github.com/russross/blackfriday v1.6.0
 	github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd
 	github.com/sanity-io/litter v1.5.2
-	github.com/spf13/afero v1.8.1
+	github.com/spf13/afero v1.8.2
 	github.com/spf13/cast v1.4.1
 	github.com/spf13/cobra v1.4.0
 	github.com/spf13/fsync v0.9.0

go.sum

@@ -132,6 +132,8 @@ github.com/bep/golibsass v1.0.0 h1:gNguBMSDi5yZEZzVZP70YpuFQE3qogJIGUlrVILTmOw=
 github.com/bep/golibsass v1.0.0/go.mod h1:DL87K8Un/+pWUS75ggYv41bliGiolxzDKWJAq3eJ1MA=
 github.com/bep/gowebp v0.1.0 h1:4/iQpfnxHyXs3x/aTxMMdOpLEQQhFmF6G7EieWPTQyo=
 github.com/bep/gowebp v0.1.0/go.mod h1:ZhFodwdiFp8ehGJpF4LdPl6unxZm9lLFjxD3z2h2AgI=
+github.com/bep/overlayfs v0.1.0 h1:1hOCrvS4E5Hf0qwxM7m+9oitqClD9mRjQ1d4pECsVcU=
+github.com/bep/overlayfs v0.1.0/go.mod h1:NFjSmn3kCqG7KX2Lmz8qT8VhPPCwZap3UNogXawoQHM=
 github.com/bep/tmc v0.5.1 h1:CsQnSC6MsomH64gw0cT5f+EwQDcvZz4AazKunFwTpuI=
 github.com/bep/tmc v0.5.1/go.mod h1:tGYHN8fS85aJPhDLgXETVKp+PR382OvFi2+q2GkGsq0=
 github.com/bep/workers v1.0.0 h1:U+H8YmEaBCEaFZBst7GcRVEoqeRC9dzH2dWOwGmOchg=
@@ -413,6 +415,8 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/spf13/afero v1.8.1 h1:izYHOT71f9iZ7iq37Uqjael60/vYC6vMtzedudZ0zEk=
 github.com/spf13/afero v1.8.1/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo=
+github.com/spf13/afero v1.8.2 h1:xehSyVa0YnHWsJ49JFljMpg1HX19V6NDZ1fkm1Xznbo=
+github.com/spf13/afero v1.8.2/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo=
 github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA=
 github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cobra v1.4.0 h1:y+wJpx64xcgO1V+RcnwW0LEHxTKRi2ZDPSBjWnrg88Q=

hugofs/fs.go

@@ -33,11 +33,17 @@ type Fs struct {
 	Source afero.Fs
 
 	// Destination is Hugo's destination file system.
+	// It's mounted inside publishDir (default /public).
 	Destination afero.Fs
 
-	// Destination used for `renderStaticToDisk`
+	// File system used for static files  when --renderStaticToDisk is set.
+	// When this is set, the Destination above is set to write to memory.
 	DestinationStatic afero.Fs
 
+	// DestinationServer is the file system used for serving the public directory with Hugo's development server.
+	// This will typically be the same as Destination, but not if --renderStaticToDisk is set.
+	DestinationServer afero.Fs
+
 	// Os is an OS file system.
 	// NOTE: Field is currently unused.
 	Os afero.Fs
@@ -74,6 +80,7 @@ func newFs(base afero.Fs, cfg config.Provider) *Fs {
 	return &Fs{
 		Source:            base,
 		Destination:       base,
+		DestinationServer: base,
 		DestinationStatic: base,
 		Os:                &afero.OsFs{},
 		WorkingDir:        getWorkingDirFs(base, cfg),

hugolib/config.go

@@ -35,7 +35,6 @@ import (
 
 	"github.com/gohugoio/hugo/common/herrors"
 	"github.com/gohugoio/hugo/common/hugo"
-	"github.com/gohugoio/hugo/hugolib/paths"
 	"github.com/gohugoio/hugo/langs"
 	"github.com/gohugoio/hugo/modules"
 	"github.com/pkg/errors"
@@ -359,7 +358,7 @@ func (l configLoader) collectModules(modConfig modules.Config, v1 config.Provide
 		workingDir = v1.GetString("workingDir")
 	}
 
-	themesDir := paths.AbsPathify(l.WorkingDir, v1.GetString("themesDir"))
+	themesDir := cpaths.AbsPathify(l.WorkingDir, v1.GetString("themesDir"))
 
 	var ignoreVendor glob.Glob
 	if s := v1.GetString("ignoreVendorPaths"); s != "" {