Add support for minification of final output

Hugo Pipes added minification support for resources fetched via ´resources.Get` and similar.

This also adds support for minification of the final output for supported output formats: HTML, XML, SVG, CSS, JavaScript, JSON.

To enable, run Hugo with the `--minify` flag:

```bash
hugo --minify
```

This commit is also a major spring cleaning of the `transform` package to allow the new minification step fit into that processing chain.

Fixes gohugoio#1251

commands/commands.go

@@ -177,6 +177,7 @@ Complete documentation is available at http://gohugo.io/.`,
 	cc.cmd.Flags().BoolVarP(&cc.buildWatch, "watch", "w", false, "watch filesystem for changes and recreate as needed")
 
 	cc.cmd.Flags().Bool("renderToMemory", false, "render to memory (only useful for benchmark testing)")
+	cc.cmd.Flags().Bool("minify", false, "minify any supported output format (HTML, XML etc.)")
 
 	// Set bash-completion
 	_ = cc.cmd.PersistentFlags().SetAnnotation("logFile", cobra.BashCompFilenameExt, []string{})

commands/hugo.go

@@ -209,16 +209,25 @@ func initializeFlags(cmd *cobra.Command, cfg config.Provider) {
 		"verboseLog",
 	}
 
+	// Will set a value even if it is the default.
+	flagKeysForced := []string{
+		"minify",
+	}
+
 	for _, key := range persFlagKeys {
-		setValueFromFlag(cmd.PersistentFlags(), key, cfg, "")
+		setValueFromFlag(cmd.PersistentFlags(), key, cfg, "", false)
 	}
 	for _, key := range flagKeys {
-		setValueFromFlag(cmd.Flags(), key, cfg, "")
+		setValueFromFlag(cmd.Flags(), key, cfg, "", false)
+	}
+
+	for _, key := range flagKeysForced {
+		setValueFromFlag(cmd.Flags(), key, cfg, "", true)
 	}
 
 	// Set some "config aliases"
-	setValueFromFlag(cmd.Flags(), "destination", cfg, "publishDir")
-	setValueFromFlag(cmd.Flags(), "i18n-warnings", cfg, "logI18nWarnings")
+	setValueFromFlag(cmd.Flags(), "destination", cfg, "publishDir", false)
+	setValueFromFlag(cmd.Flags(), "i18n-warnings", cfg, "logI18nWarnings", false)
 
 }
 
@@ -229,9 +238,9 @@ var deprecatedFlags = map[string]bool{
 	strings.ToLower("canonifyURLs"):          true,
 }
 
-func setValueFromFlag(flags *flag.FlagSet, key string, cfg config.Provider, targetKey string) {
+func setValueFromFlag(flags *flag.FlagSet, key string, cfg config.Provider, targetKey string, force bool) {
 	key = strings.TrimSpace(key)
-	if flags.Changed(key) {
+	if (force && flags.Lookup(key) != nil) || flags.Changed(key) {
 		if _, deprecated := deprecatedFlags[strings.ToLower(key)]; deprecated {
 			msg := fmt.Sprintf(`Set "%s = true" in your config.toml.
 If you need to set this configuration value from the command line, set it via an OS environment variable: "HUGO_%s=true hugo"`, key, strings.ToUpper(key))

deps/deps.go

@@ -162,7 +162,7 @@ func New(cfg DepsCfg) (*Deps, error) {
 		return nil, err
 	}
 
-	resourceSpec, err := resource.NewSpec(ps, logger, cfg.MediaTypes)
+	resourceSpec, err := resource.NewSpec(ps, logger, cfg.OutputFormats, cfg.MediaTypes)
 	if err != nil {
 		return nil, err
 	}
@@ -224,7 +224,7 @@ func (d Deps) ForLanguage(cfg DepsCfg) (*Deps, error) {
 	// The resource cache is global so reuse.
 	// TODO(bep) clean up these inits.
 	resourceCache := d.ResourceSpec.ResourceCache
-	d.ResourceSpec, err = resource.NewSpec(d.PathSpec, d.Log, cfg.MediaTypes)
+	d.ResourceSpec, err = resource.NewSpec(d.PathSpec, d.Log, cfg.OutputFormats, cfg.MediaTypes)
 	if err != nil {
 		return nil, err
 	}
@@ -267,6 +267,9 @@ type DepsCfg struct {
 	// The media types configured.
 	MediaTypes media.Types
 
+	// The output formats configured.
+	OutputFormats output.Formats
+
 	// Template handling.
 	TemplateProvider ResourceProvider
 	WithTemplate     func(templ tpl.TemplateHandler) error

helpers/path.go

@@ -515,6 +515,25 @@ func WriteToDisk(inpath string, r io.Reader, fs afero.Fs) (err error) {
 	return afero.WriteReader(fs, inpath, r)
 }
 
+// OpenFileForWriting opens or creates the given file. If the target directory
+// does not exist, it gets created.
+func OpenFileForWriting(fs afero.Fs, filename string) (afero.File, error) {
+	filename = filepath.Clean(filename)
+	// Create will truncate if file already exists.
+	f, err := fs.Create(filename)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return nil, err
+		}
+		if err = fs.MkdirAll(filepath.Dir(filename), 0755); err != nil {
+			return nil, err
+		}
+		f, err = fs.Create(filename)
+	}
+
+	return f, err
+}
+
 // GetTempDir returns a temporary directory with the given sub path.
 func GetTempDir(subPath string, fs afero.Fs) string {
 	return afero.GetTempDir(fs, subPath)

hugolib/alias.go

@@ -22,6 +22,8 @@ import (
 	"runtime"
 	"strings"
 
+	"github.com/gohugoio/hugo/output"
+	"github.com/gohugoio/hugo/publisher"
 	"github.com/gohugoio/hugo/tpl"
 
 	jww "github.com/spf13/jwalterweatherman"
@@ -89,11 +91,11 @@ func (a aliasHandler) renderAlias(isXHTML bool, permalink string, page *Page) (i
 	return buffer, nil
 }
 
-func (s *Site) writeDestAlias(path, permalink string, p *Page) (err error) {
-	return s.publishDestAlias(false, path, permalink, p)
+func (s *Site) writeDestAlias(path, permalink string, outputFormat output.Format, p *Page) (err error) {
+	return s.publishDestAlias(false, path, permalink, outputFormat, p)
 }
 
-func (s *Site) publishDestAlias(allowRoot bool, path, permalink string, p *Page) (err error) {
+func (s *Site) publishDestAlias(allowRoot bool, path, permalink string, outputFormat output.Format, p *Page) (err error) {
 	handler := newAliasHandler(s.Tmpl, s.Log, allowRoot)
 
 	isXHTML := strings.HasSuffix(path, ".xhtml")
@@ -110,7 +112,14 @@ func (s *Site) publishDestAlias(allowRoot bool, path, permalink string, p *Page)
 		return err
 	}
 
-	return s.publish(&s.PathSpec.ProcessingStats.Aliases, targetPath, aliasContent)
+	pd := publisher.Descriptor{
+		Src:          aliasContent,
+		TargetPath:   targetPath,
+		StatCounter:  &s.PathSpec.ProcessingStats.Aliases,
+		OutputFormat: outputFormat,
+	}
+
+	return s.publisher.Publish(pd)
 
 }
 

hugolib/hugo_sites.go

@@ -24,6 +24,7 @@ import (
 	"github.com/gohugoio/hugo/deps"
 	"github.com/gohugoio/hugo/helpers"
 	"github.com/gohugoio/hugo/langs"
+	"github.com/gohugoio/hugo/publisher"
 
 	"github.com/gohugoio/hugo/i18n"
 	"github.com/gohugoio/hugo/tpl"
@@ -182,6 +183,7 @@ func applyDeps(cfg deps.DepsCfg, sites ...*Site) error {
 
 		cfg.Language = s.Language
 		cfg.MediaTypes = s.mediaTypesConfig
+		cfg.OutputFormats = s.outputFormatsConfig
 
 		if d == nil {
 			cfg.WithTemplate = s.withSiteTemplates(cfg.WithTemplate)
@@ -208,6 +210,9 @@ func applyDeps(cfg deps.DepsCfg, sites ...*Site) error {
 			s.Deps = d
 		}
 
+		// Set up the main publishing chain.
+		s.publisher = publisher.NewDestinationPublisher(d.PathSpec.BaseFs.PublishFs, s.outputFormatsConfig, s.mediaTypesConfig, cfg.Cfg.GetBool("minify"))
+
 		if err := s.initializeSiteInfo(); err != nil {
 			return err
 		}

hugolib/minify_publisher_test.go

@@ -0,0 +1,71 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugolib
+
+import (
+	"testing"
+
+	"github.com/spf13/viper"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestMinifyPublisher(t *testing.T) {
+	t.Parallel()
+	assert := require.New(t)
+
+	v := viper.New()
+	v.Set("minify", true)
+	v.Set("baseURL", "https://example.org/")
+
+	htmlTemplate := `
+<!DOCTYPE html>
+<html lang="en">
+<head>
+	<meta charset="utf-8">
+	<title>HTML5 boilerplate – all you really need…</title>
+	<link rel="stylesheet" href="css/style.css">
+	<!--[if IE]>
+		<script src="http://html5shiv.googlecode.com/svn/trunk/html5.js"></script>
+	<![endif]-->
+</head>
+
+<body id="home">
+
+	<h1>{{ .Page.Title }}</h1>
+
+</body>
+</html>
+`
+
+	b := newTestSitesBuilder(t)
+	b.WithViper(v).WithContent("page.md", pageWithAlias)
+	b.WithTemplates("_default/list.html", htmlTemplate, "_default/single.html", htmlTemplate, "alias.html", htmlTemplate)
+	b.CreateSites().Build(BuildCfg{})
+
+	assert.Equal(1, len(b.H.Sites))
+	require.Len(t, b.H.Sites[0].RegularPages, 1)
+
+	// Check minification
+	// HTML
+	b.AssertFileContent("public/page/index.html", "<!doctype html><html lang=en><head><meta charset=utf-8><title>HTML5 boilerplate – all you really need…</title><link rel=stylesheet href=css/style.css></head><body id=home><h1>Has Alias</h1></body></html>")
+	// HTML alias. Note the custom template which does no redirect.
+	b.AssertFileContent("public/foo/bar/index.html", "<!doctype html><html lang=en><head><meta charset=utf-8><title>HTML5 boilerplate ")
+
+	// RSS
+	b.AssertFileContent("public/index.xml", "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"yes\"?><rss version=\"2.0\" xmlns:atom=\"http://www.w3.org/2005/Atom\"><channel><title/><link>https://example.org/</link>")
+
+	// Sitemap
+	b.AssertFileContent("public/sitemap.xml", "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"yes\"?><urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\" xmlns:xhtml=\"http://www.w3.org/1999/xhtml\"><url><loc>https://example.org/</loc><priority>0</priority></url><url>")
+}

hugolib/resource_chain_test.go

@@ -223,7 +223,7 @@ Min HTML: {{ ( resources.Get "mydata/html1.html" | resources.Minify ).Content |
 			b.AssertFileContent("public/index.html", `Min XML: <hello><world>Hugo Rocks!</<world></hello>`)
 			b.AssertFileContent("public/index.html", `Min SVG: <svg height="100" width="100"><path d="M5 10 20 40z"/></svg>`)
 			b.AssertFileContent("public/index.html", `Min SVG again: <svg height="100" width="100"><path d="M5 10 20 40z"/></svg>`)
-			b.AssertFileContent("public/index.html", `Min HTML: <a href=#>Cool</a>`)
+			b.AssertFileContent("public/index.html", `Min HTML: <html><a href=#>Cool</a></html>`)
 		}},
 
 		{"concat", func() bool { return true }, func(b *sitesBuilder) {