From 28733248983c1afc1e4e15dfc30fcf4c442e6ca4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Fri, 2 Feb 2024 16:00:48 +0100 Subject: [PATCH] Misc resource fixes/improvements * Add --pprof flag to server to enable profile debugging. * Don't cache the resource content, it seem to eat memory on bigger sites. * Keep --printMemoryUsag running in server Fixes #11974 --- commands/commandeer.go | 10 +++++-- commands/hugobuilder.go | 20 ++++++-------- commands/server.go | 14 +++++++++- hugolib/hugo_sites_build.go | 2 ++ hugolib/rebuild_test.go | 4 +-- resources/resource.go | 55 ++++++++++--------------------------- resources/resource_spec.go | 18 ++++++------ 7 files changed, 58 insertions(+), 65 deletions(-) diff --git a/commands/commandeer.go b/commands/commandeer.go index 1aac08c42f3..0052d91b4c6 100644 --- a/commands/commandeer.go +++ b/commands/commandeer.go @@ -22,6 +22,7 @@ import ( "os" "os/signal" "path/filepath" + "runtime" "strings" "sync" "sync/atomic" @@ -341,8 +342,12 @@ func (r *rootCommand) Run(ctx context.Context, cd *simplecobra.Commandeer, args if r.buildWatch { defer r.timeTrack(time.Now(), "Built") } - err := b.build() - return err + close, err := b.build() + if err != nil { + return err + } + close() + return nil }() if err != nil { return err @@ -411,6 +416,7 @@ func (r *rootCommand) PreRun(cd, runner *simplecobra.Commandeer) error { MaxEntries: 1, OnEvict: func(key int32, value *hugolib.HugoSites) { value.Close() + runtime.GC() }, }) diff --git a/commands/hugobuilder.go b/commands/hugobuilder.go index 41f42ae6d42..ddc92129c26 100644 --- a/commands/hugobuilder.go +++ b/commands/hugobuilder.go @@ -361,34 +361,32 @@ func (c *hugoBuilder) newWatcher(pollIntervalStr string, dirList ...string) (*wa return watcher, nil } -func (c *hugoBuilder) build() error { +func (c *hugoBuilder) build() (func(), error) { stopProfiling, err := c.initProfiling() if err != nil { - return err + return nil, err } - defer func() { - if stopProfiling != nil { - stopProfiling() - } - }() - if err := c.fullBuild(false); err != nil { - return err + return nil, err } if !c.r.quiet { c.r.Println() h, err := c.hugo() if err != nil { - return err + return nil, err } h.PrintProcessingStats(os.Stdout) c.r.Println() } - return nil + return func() { + if stopProfiling != nil { + stopProfiling() + } + }, nil } func (c *hugoBuilder) buildSites(noBuildLock bool) (err error) { diff --git a/commands/server.go b/commands/server.go index 97cf405b7ef..574f5c3404c 100644 --- a/commands/server.go +++ b/commands/server.go @@ -25,6 +25,7 @@ import ( "io" "net" "net/http" + _ "net/http/pprof" "net/url" "os" "os/signal" @@ -451,6 +452,7 @@ type serverCommand struct { tlsCertFile string tlsKeyFile string tlsAuto bool + pprof bool serverPort int liveReloadPort int serverWatch bool @@ -465,6 +467,11 @@ func (c *serverCommand) Name() string { } func (c *serverCommand) Run(ctx context.Context, cd *simplecobra.Commandeer, args []string) error { + if c.pprof { + go func() { + http.ListenAndServe("localhost:8080", nil) + }() + } // Watch runs its own server as part of the routine if c.serverWatch { @@ -487,15 +494,19 @@ func (c *serverCommand) Run(ctx context.Context, cd *simplecobra.Commandeer, arg } + var close func() err := func() error { defer c.r.timeTrack(time.Now(), "Built") - err := c.build() + var err error + close, err = c.build() return err }() if err != nil { return err } + defer close() + return c.serve() } @@ -520,6 +531,7 @@ of a second, you will be able to save and see your changes nearly instantly.` cmd.Flags().StringVarP(&c.tlsCertFile, "tlsCertFile", "", "", "path to TLS certificate file") cmd.Flags().StringVarP(&c.tlsKeyFile, "tlsKeyFile", "", "", "path to TLS key file") cmd.Flags().BoolVar(&c.tlsAuto, "tlsAuto", false, "generate and use locally-trusted certificates.") + cmd.Flags().BoolVar(&c.pprof, "pprof", false, "enable the pprof server (port 8080)") cmd.Flags().BoolVarP(&c.serverWatch, "watch", "w", true, "watch filesystem for changes and recreate as needed") cmd.Flags().BoolVar(&c.noHTTPCache, "noHTTPCache", false, "prevent HTTP caching") cmd.Flags().BoolVarP(&c.serverAppend, "appendPort", "", true, "append port to baseURL") diff --git a/hugolib/hugo_sites_build.go b/hugolib/hugo_sites_build.go index 257949334c9..a15e1550491 100644 --- a/hugolib/hugo_sites_build.go +++ b/hugolib/hugo_sites_build.go @@ -720,9 +720,11 @@ func (h *HugoSites) processPartial(ctx context.Context, l logg.LevelLogger, conf h.pageTrees.treeTaxonomyEntries.DeletePrefix("") if delete { + _, ok := h.pageTrees.treePages.LongestPrefixAll(pathInfo.Base()) if ok { h.pageTrees.treePages.DeleteAll(pathInfo.Base()) + h.pageTrees.resourceTrees.DeleteAll(pathInfo.Base()) if pathInfo.IsBundle() { // Assume directory removed. h.pageTrees.treePages.DeletePrefixAll(pathInfo.Base() + "/") diff --git a/hugolib/rebuild_test.go b/hugolib/rebuild_test.go index d3ac5665d1f..74b04fe0116 100644 --- a/hugolib/rebuild_test.go +++ b/hugolib/rebuild_test.go @@ -101,10 +101,10 @@ func TestRebuildEditTextFileInBranchBundle(t *testing.T) { func TestRebuildRenameTextFileInLeafBundle(t *testing.T) { b := TestRunning(t, rebuildFilesSimple) - b.AssertFileContent("public/mysection/mysectionbundle/index.html", "My Section Bundle Text 2 Content.") + b.AssertFileContent("public/mysection/mysectionbundle/index.html", "My Section Bundle Text 2 Content.", "Len Resources: 2|") b.RenameFile("content/mysection/mysectionbundle/mysectionbundletext.txt", "content/mysection/mysectionbundle/mysectionbundletext2.txt").Build() - b.AssertFileContent("public/mysection/mysectionbundle/index.html", "mysectionbundletext2", "My Section Bundle Text 2 Content.") + b.AssertFileContent("public/mysection/mysectionbundle/index.html", "mysectionbundletext2", "My Section Bundle Text 2 Content.", "Len Resources: 2|") b.AssertRenderCountPage(3) b.AssertRenderCountContent(3) } diff --git a/resources/resource.go b/resources/resource.go index e78dd12cb91..6bef1b2756e 100644 --- a/resources/resource.go +++ b/resources/resource.go @@ -343,7 +343,7 @@ func GetTestInfoForResource(r resource.Resource) GenericResourceTestInfo { // genericResource represents a generic linkable resource. type genericResource struct { - *resourceContent + publishInit *sync.Once sd ResourceSourceDescriptor paths internal.ResourcePaths @@ -412,11 +412,18 @@ func (l *genericResource) cloneTo(targetPath string) resource.Resource { } func (l *genericResource) Content(context.Context) (any, error) { - if err := l.initContent(); err != nil { - return nil, err + r, err := l.ReadSeekCloser() + if err != nil { + return "", err } + defer r.Close() - return l.content, nil + var b []byte + b, err = io.ReadAll(r) + if err != nil { + return "", err + } + return string(b), nil } func (r *genericResource) Err() resource.ResourceError { @@ -527,28 +534,6 @@ func (l *genericResource) Title() string { return l.title } -func (l *genericResource) initContent() error { - var err error - l.contentInit.Do(func() { - var r hugio.ReadSeekCloser - r, err = l.ReadSeekCloser() - if err != nil { - return - } - defer r.Close() - - var b []byte - b, err = io.ReadAll(r) - if err != nil { - return - } - - l.content = string(b) - }) - - return err -} - func (l *genericResource) getSpec() *Spec { return l.spec } @@ -588,12 +573,9 @@ func (rc *genericResource) cloneWithUpdates(u *transformationUpdate) (baseResour r := rc.clone() if u.content != nil { - r.contentInit.Do(func() { - r.content = *u.content - r.sd.OpenReadSeekCloser = func() (hugio.ReadSeekCloser, error) { - return hugio.NewReadSeekerNoOpCloserFromString(r.content), nil - } - }) + r.sd.OpenReadSeekCloser = func() (hugio.ReadSeekCloser, error) { + return hugio.NewReadSeekerNoOpCloserFromString(*u.content), nil + } } r.sd.MediaType = u.mediaType @@ -620,7 +602,7 @@ func (rc *genericResource) cloneWithUpdates(u *transformationUpdate) (baseResour } func (l genericResource) clone() *genericResource { - l.resourceContent = &resourceContent{} + l.publishInit = &sync.Once{} return &l } @@ -633,13 +615,6 @@ type targetPather interface { TargetPath() string } -type resourceContent struct { - content string - contentInit sync.Once - - publishInit sync.Once -} - type resourceHash struct { value string size int64 diff --git a/resources/resource_spec.go b/resources/resource_spec.go index 66f56d147b3..5fdcdd831f9 100644 --- a/resources/resource_spec.go +++ b/resources/resource_spec.go @@ -163,15 +163,15 @@ func (r *Spec) NewResource(rd ResourceSourceDescriptor) (resource.Resource, erro } gr := &genericResource{ - Staler: &AtomicStaler{}, - h: &resourceHash{}, - paths: rp, - spec: r, - sd: rd, - params: make(map[string]any), - name: rd.Name, - title: rd.Name, - resourceContent: &resourceContent{}, + Staler: &AtomicStaler{}, + h: &resourceHash{}, + publishInit: &sync.Once{}, + paths: rp, + spec: r, + sd: rd, + params: make(map[string]any), + name: rd.Name, + title: rd.Name, } if rd.MediaType.MainType == "image" {