Allow rendering static files to disk and dynamic to memory in server …

…mode

Updates gohugoio#9625

commands/commandeer.go

@@ -92,6 +92,7 @@ type commandeer struct {
 	languagesConfigured bool
 	languages           langs.Languages
 	doLiveReload        bool
+	renderStaticToDisk  bool
 	fastRenderMode      bool
 	showErrorInBrowser  bool
 	wasError            bool
@@ -368,8 +369,9 @@ func (c *commandeer) loadConfig() error {
 	}
 
 	createMemFs := config.GetBool("renderToMemory")
+	c.renderStaticToDisk = config.GetBool("renderStaticToDisk")
 
-	if createMemFs {
+	if createMemFs && !c.renderStaticToDisk {
 		// Rendering to memoryFS, publish to Root regardless of publishDir.
 		config.Set("publishDir", "/")
 	}
@@ -380,6 +382,14 @@ func (c *commandeer) loadConfig() error {
 		if c.destinationFs != nil {
 			// Need to reuse the destination on server rebuilds.
 			fs.Destination = c.destinationFs
+		} else if createMemFs && c.renderStaticToDisk {
+			// Writes the dynamic output on memory,
+			// while serve others directly from publishDir
+			publishDir := config.GetString("publishDir")
+			writableFs := afero.NewBasePathFs(afero.NewMemMapFs(), publishDir)
+			publicFs := afero.NewOsFs()
+			fs.Destination = afero.NewCopyOnWriteFs(afero.NewReadOnlyFs(publicFs), writableFs)
+			fs.DestinationStatic = publicFs
 		} else if createMemFs {
 			// Hugo writes the output to memory instead of the disk.
 			fs.Destination = new(afero.MemMapFs)
@@ -397,11 +407,13 @@ func (c *commandeer) loadConfig() error {
 
 			changeDetector.PrepareNew()
 			fs.Destination = hugofs.NewHashingFs(fs.Destination, changeDetector)
+			fs.DestinationStatic = hugofs.NewHashingFs(fs.DestinationStatic, changeDetector)
 			c.changeDetector = changeDetector
 		}
 
 		if c.Cfg.GetBool("logPathWarnings") {
 			fs.Destination = hugofs.NewCreateCountingFs(fs.Destination)
+			fs.DestinationStatic = hugofs.NewCreateCountingFs(fs.DestinationStatic)
 		}
 
 		// To debug hard-to-find path issues.

commands/hugo.go

@@ -652,6 +652,9 @@ func (c *commandeer) copyStaticTo(sourceFs *filesystems.SourceFilesystem) (uint6
 	syncer.ChmodFilter = chmodFilter
 	syncer.SrcFs = fs
 	syncer.DestFs = c.Fs.Destination
+	if c.renderStaticToDisk {
+		syncer.DestFs = c.Fs.DestinationStatic
+	}
 	// Now that we are using a unionFs for the static directories
 	// We can effectively clean the publishDir on initial sync
 	syncer.Delete = c.Cfg.GetBool("cleanDestinationDir")

commands/server.go

@@ -48,15 +48,16 @@ type serverCmd struct {
 	// Can be used to stop the server. Useful in tests
 	stop <-chan bool
 
-	disableLiveReload bool
-	navigateToChanged bool
-	renderToDisk      bool
-	serverAppend      bool
-	serverInterface   string
-	serverPort        int
-	liveReloadPort    int
-	serverWatch       bool
-	noHTTPCache       bool
+	disableLiveReload  bool
+	navigateToChanged  bool
+	renderToDisk       bool
+	renderStaticToDisk bool
+	serverAppend       bool
+	serverInterface    string
+	serverPort         int
+	liveReloadPort     int
+	serverWatch        bool
+	noHTTPCache        bool
 
 	disableFastRender   bool
 	disableBrowserError bool
@@ -101,6 +102,7 @@ of a second, you will be able to save and see your changes nearly instantly.`,
 	cc.cmd.Flags().BoolVar(&cc.renderToDisk, "renderToDisk", false, "render to Destination path (default is render to memory & serve from there)")
 	cc.cmd.Flags().BoolVar(&cc.disableFastRender, "disableFastRender", false, "enables full re-renders on changes")
 	cc.cmd.Flags().BoolVar(&cc.disableBrowserError, "disableBrowserError", false, "do not show build errors in the browser")
+	cc.cmd.Flags().BoolVar(&cc.renderStaticToDisk, "renderStaticToDisk", false, "render static files to disk but dynamic files render to memory.")
 
 	cc.cmd.Flags().String("memstats", "", "log memory usage to this file")
 	cc.cmd.Flags().String("meminterval", "100ms", "interval to poll memory usage (requires --memstats), valid time units are \"ns\", \"us\" (or \"µs\"), \"ms\", \"s\", \"m\", \"h\".")
@@ -141,6 +143,7 @@ func (sc *serverCmd) server(cmd *cobra.Command, args []string) error {
 
 	cfgInit := func(c *commandeer) error {
 		c.Set("renderToMemory", !sc.renderToDisk)
+		c.Set("renderStaticToDisk", sc.renderStaticToDisk)
 		if cmd.Flags().Changed("navigateToChanged") {
 			c.Set("navigateToChanged", sc.navigateToChanged)
 		}
@@ -332,6 +335,8 @@ func (f *fileServer) createEndpoint(i int) (*http.ServeMux, string, string, erro
 	if i == 0 {
 		if f.s.renderToDisk {
 			jww.FEEDBACK.Println("Serving pages from " + absPublishDir)
+		} else if f.s.renderStaticToDisk {
+			jww.FEEDBACK.Println("Serving pages from memory and static files from " + absPublishDir)
 		} else {
 			jww.FEEDBACK.Println("Serving pages from memory")
 		}

commands/static_syncer.go

@@ -56,6 +56,9 @@ func (s *staticSyncer) syncsStaticEvents(staticEvents []fsnotify.Event) error {
 		syncer.ChmodFilter = chmodFilter
 		syncer.SrcFs = sourceFs.Fs
 		syncer.DestFs = c.Fs.Destination
+		if c.renderStaticToDisk {
+			syncer.DestFs = c.Fs.DestinationStatic
+		}
 
 		// prevent spamming the log on changes
 		logger := helpers.NewDistinctErrorLogger()
@@ -101,7 +104,11 @@ func (s *staticSyncer) syncsStaticEvents(staticEvents []fsnotify.Event) error {
 					toRemove := filepath.Join(publishDir, relPath)
 
 					logger.Println("File no longer exists in static dir, removing", toRemove)
-					_ = c.Fs.Destination.RemoveAll(toRemove)
+					if c.renderStaticToDisk {
+						_ = c.Fs.DestinationStatic.RemoveAll(toRemove)
+					} else {
+						_ = c.Fs.Destination.RemoveAll(toRemove)
+					}
 				} else if err == nil {
 					// If file still exists, sync it
 					logger.Println("Syncing", relPath, "to", publishDir)

hugofs/fs.go

@@ -35,6 +35,9 @@ type Fs struct {
 	// Destination is Hugo's destination file system.
 	Destination afero.Fs
 
+	// Destination used for `renderStaticToDisk`
+	DestinationStatic afero.Fs
+
 	// Os is an OS file system.
 	// NOTE: Field is currently unused.
 	Os afero.Fs
@@ -69,10 +72,11 @@ func NewFrom(fs afero.Fs, cfg config.Provider) *Fs {
 
 func newFs(base afero.Fs, cfg config.Provider) *Fs {
 	return &Fs{
-		Source:      base,
-		Destination: base,
-		Os:          &afero.OsFs{},
-		WorkingDir:  getWorkingDirFs(base, cfg),
+		Source:            base,
+		Destination:       base,
+		DestinationStatic: base,
+		Os:                &afero.OsFs{},
+		WorkingDir:        getWorkingDirFs(base, cfg),
 	}
 }
 

hugolib/filesystems/basefs.go

@@ -71,6 +71,9 @@ type BaseFs struct {
 	// A read-only filesystem starting from the project workDir.
 	WorkDir afero.Fs
 
+	// The filesystem used for renderStaticToDisk.
+	PublishFsStatic afero.Fs
+
 	theBigFs *filesystemsCollector
 
 	// Locks.
@@ -438,15 +441,17 @@ func NewBase(p *paths.Paths, logger loggers.Logger, options ...func(*BaseFs) err
 
 	publishFs := hugofs.NewBaseFileDecorator(afero.NewBasePathFs(fs.Destination, p.AbsPublishDir))
 	sourceFs := hugofs.NewBaseFileDecorator(afero.NewBasePathFs(fs.Source, p.WorkingDir))
+	publishFsStatic := afero.NewBasePathFs(fs.Source, p.AbsPublishDir)
 
 	// Same as sourceFs, but no decoration. This is what's used by os.ReadDir etc.
 	workDir := afero.NewBasePathFs(afero.NewReadOnlyFs(fs.Source), p.WorkingDir)
 
 	b := &BaseFs{
-		SourceFs:  sourceFs,
-		WorkDir:   workDir,
-		PublishFs: publishFs,
-		buildMu:   lockedfile.MutexAt(filepath.Join(p.WorkingDir, lockFileBuild)),
+		SourceFs:        sourceFs,
+		WorkDir:         workDir,
+		PublishFs:       publishFs,
+		PublishFsStatic: publishFsStatic,
+		buildMu:         lockedfile.MutexAt(filepath.Join(p.WorkingDir, lockFileBuild)),
 	}
 
 	for _, opt := range options {

hugolib/pages_process.go

@@ -33,9 +33,10 @@ func newPagesProcessor(h *HugoSites, sp *source.SourceSpec) *pagesProcessor {
 	procs := make(map[string]pagesCollectorProcessorProvider)
 	for _, s := range h.Sites {
 		procs[s.Lang()] = &sitePagesProcessor{
-			m:           s.pageMap,
-			errorSender: s.h,
-			itemChan:    make(chan interface{}, config.GetNumWorkerMultiplier()*2),
+			m:                  s.pageMap,
+			errorSender:        s.h,
+			itemChan:           make(chan interface{}, config.GetNumWorkerMultiplier()*2),
+			renderStaticToDisk: h.Cfg.GetBool("renderStaticToDisk"),
 		}
 	}
 	return &pagesProcessor{
@@ -118,6 +119,8 @@ type sitePagesProcessor struct {
 	ctx       context.Context
 	itemChan  chan interface{}
 	itemGroup *errgroup.Group
+
+	renderStaticToDisk bool
 }
 
 func (p *sitePagesProcessor) Process(item interface{}) error {
@@ -162,7 +165,12 @@ func (p *sitePagesProcessor) copyFile(fim hugofs.FileMetaInfo) error {
 
 	defer f.Close()
 
-	return s.publish(&s.PathSpec.ProcessingStats.Files, target, f)
+	fs := s.PublishFs
+	if p.renderStaticToDisk {
+		fs = s.PublishFsStatic
+	}
+
+	return s.publish(&s.PathSpec.ProcessingStats.Files, target, f, fs)
 }
 
 func (p *sitePagesProcessor) doProcess(item interface{}) error {

hugolib/site.go

@@ -1824,10 +1824,10 @@ func (s *Site) lookupTemplate(layouts ...string) (tpl.Template, bool) {
 	return nil, false
 }
 
-func (s *Site) publish(statCounter *uint64, path string, r io.Reader) (err error) {
+func (s *Site) publish(statCounter *uint64, path string, r io.Reader, fs afero.Fs) (err error) {
 	s.PathSpec.ProcessingStats.Incr(statCounter)
 
-	return helpers.WriteToDisk(filepath.Clean(path), r, s.BaseFs.PublishFs)
+	return helpers.WriteToDisk(filepath.Clean(path), r, fs)
 }
 
 func (s *Site) kindFromFileInfoOrSections(fi *fileInfo, sections []string) string {