Simplify the content map key structure to ease finding stuff

Fixes gohugoio#8307
Fixes gohugoio#8498
Fixes gohugoio#8927
Fixes gohugoio#9192

benchbep.sh

@@ -1 +1 @@
-gobench -package=./hugolib -bench="BenchmarkSiteNew/Deep_content_tree"
+gobench --package ./hugolib --bench "BenchmarkSiteNew/Regular_Deep" -base v0.89.4

commands/server.go

@@ -511,7 +511,7 @@ func (c *commandeer) serve(s *serverCmd) error {
 		mu, serverURL, endpoint, err := srv.createEndpoint(i)
 
 		if doLiveReload {
-			u, err := url.Parse(helpers.SanitizeURL(baseURLs[i]))
+			u, err := url.Parse(baseURLs[i])
 			if err != nil {
 				return err
 			}

common/herrors/errors.go

@@ -65,6 +65,7 @@ type ErrorSender interface {
 // Recover is a helper function that can be used to capture panics.
 // Put this at the top of a method/function that crashes in a template:
 //     defer herrors.Recover()
+// TODO1 check usage
 func Recover(args ...interface{}) {
 	if r := recover(); r != nil {
 		fmt.Println("ERR:", r)

common/paths/path.go

@@ -16,6 +16,7 @@ package paths
 import (
 	"errors"
 	"fmt"
+	"net/url"
 	"os"
 	"path"
 	"path/filepath"
@@ -250,6 +251,17 @@ func PrettifyPath(in string) string {
 	return prettifyPath(in, fpb)
 }
 
+var slashFunc = func(r rune) bool {
+	return r == '/'
+}
+
+// FieldsSlash cuts s into fields separated with '/'.
+// TODO1 add some tests, consider leading/trailing slashes.
+func FieldsSlash(s string) []string {
+	f := strings.FieldsFunc(s, slashFunc)
+	return f
+}
+
 func prettifyPath(in string, b filepathPathBridge) string {
 	if filepath.Ext(in) == "" {
 		// /section/name/  -> /section/name/index.html
@@ -310,3 +322,16 @@ func AddTrailingSlash(path string) string {
 	}
 	return path
 }
+
+// PathEscape escapes unicode letters in pth.
+// Use URLEscape to escape full URLs including scheme, query etc.
+// This is slightly faster for the common case.
+// Note, there is a url.PathEscape function, but that also
+// escapes /.
+func PathEscape(pth string) string {
+	u, err := url.Parse(pth)
+	if err != nil {
+		panic(err)
+	}
+	return u.EscapedPath()
+}

common/paths/url.go

@@ -18,8 +18,6 @@ import (
 	"net/url"
 	"path"
 	"strings"
-
-	"github.com/PuerkitoBio/purell"
 )
 
 type pathBridge struct {
@@ -51,51 +49,6 @@ func (pathBridge) Separator() string {
 
 var pb pathBridge
 
-func sanitizeURLWithFlags(in string, f purell.NormalizationFlags) string {
-	s, err := purell.NormalizeURLString(in, f)
-	if err != nil {
-		return in
-	}
-
-	// Temporary workaround for the bug fix and resulting
-	// behavioral change in purell.NormalizeURLString():
-	// a leading '/' was inadvertently added to relative links,
-	// but no longer, see #878.
-	//
-	// I think the real solution is to allow Hugo to
-	// make relative URL with relative path,
-	// e.g. "../../post/hello-again/", as wished by users
-	// in issues #157, #622, etc., without forcing
-	// relative URLs to begin with '/'.
-	// Once the fixes are in, let's remove this kludge
-	// and restore SanitizeURL() to the way it was.
-	//                         -- @anthonyfok, 2015-02-16
-	//
-	// Begin temporary kludge
-	u, err := url.Parse(s)
-	if err != nil {
-		panic(err)
-	}
-	if len(u.Path) > 0 && !strings.HasPrefix(u.Path, "/") {
-		u.Path = "/" + u.Path
-	}
-	return u.String()
-	// End temporary kludge
-
-	// return s
-
-}
-
-// SanitizeURL sanitizes the input URL string.
-func SanitizeURL(in string) string {
-	return sanitizeURLWithFlags(in, purell.FlagsSafe|purell.FlagRemoveTrailingSlash|purell.FlagRemoveDotSegments|purell.FlagRemoveDuplicateSlashes|purell.FlagRemoveUnnecessaryHostDots|purell.FlagRemoveEmptyPortSeparator)
-}
-
-// SanitizeURLKeepTrailingSlash is the same as SanitizeURL, but will keep any trailing slash.
-func SanitizeURLKeepTrailingSlash(in string) string {
-	return sanitizeURLWithFlags(in, purell.FlagsSafe|purell.FlagRemoveDotSegments|purell.FlagRemoveDuplicateSlashes|purell.FlagRemoveUnnecessaryHostDots|purell.FlagRemoveEmptyPortSeparator)
-}
-
 // MakePermalink combines base URL with content path to create full URL paths.
 // Example
 //    base:   http://spf13.com/
@@ -210,3 +163,13 @@ func Uglify(in string) string {
 	// /section/name.html -> /section/name.html
 	return path.Clean(in)
 }
+
+// URLEscape escapes unicode letters.
+func URLEscape(uri string) string {
+	// escape unicode letters
+	u, err := url.Parse(uri)
+	if err != nil {
+		panic(err)
+	}
+	return u.String()
+}

common/paths/url_test.go

@@ -14,41 +14,11 @@
 package paths
 
 import (
-	"strings"
 	"testing"
 
 	qt "github.com/frankban/quicktest"
 )
 
-func TestSanitizeURL(t *testing.T) {
-	tests := []struct {
-		input    string
-		expected string
-	}{
-		{"http://foo.bar/", "http://foo.bar"},
-		{"http://foo.bar", "http://foo.bar"},          // issue #1105
-		{"http://foo.bar/zoo/", "http://foo.bar/zoo"}, // issue #931
-	}
-
-	for i, test := range tests {
-		o1 := SanitizeURL(test.input)
-		o2 := SanitizeURLKeepTrailingSlash(test.input)
-
-		expected2 := test.expected
-
-		if strings.HasSuffix(test.input, "/") && !strings.HasSuffix(expected2, "/") {
-			expected2 += "/"
-		}
-
-		if o1 != test.expected {
-			t.Errorf("[%d] 1: Expected %#v, got %#v\n", i, test.expected, o1)
-		}
-		if o2 != expected2 {
-			t.Errorf("[%d] 2: Expected %#v, got %#v\n", i, expected2, o2)
-		}
-	}
-}
-
 func TestMakePermalink(t *testing.T) {
 	type test struct {
 		host, link, output string

common/types/types.go

@@ -90,3 +90,15 @@ func IsNil(v interface{}) bool {
 type DevMarker interface {
 	DevOnly()
 }
+
+// Identifier identifies a resource.
+type Identifier interface {
+	Key() string
+}
+
+// KeyString is a string that implements Identifier.
+type KeyString string
+
+func (k KeyString) Key() string {
+	return string(k)
+}

go.mod

@@ -2,8 +2,6 @@ module github.com/gohugoio/hugo
 
 require (
 	github.com/BurntSushi/locker v0.0.0-20171006230638-a6e239ea1c69
-	github.com/PuerkitoBio/purell v1.1.1
-	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
 	github.com/alecthomas/chroma v0.9.4
 	github.com/armon/go-radix v1.0.0
 	github.com/aws/aws-sdk-go v1.41.14

go.sum

@@ -95,10 +95,6 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym
 github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0=
 github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0=
 github.com/GoogleCloudPlatform/cloudsql-proxy v0.0.0-20191009163259-e802c2cb94ae/go.mod h1:mjwGPas4yKduTyubHvD1Atl9r1rUq8DfVy+gkVvZ+oo=
-github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI=
-github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
-github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
-github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c=
 github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38/go.mod h1:r7bzyVFMNntcxPZXK3/+KdruV1H5KSlyVY0gc+NgInI=
 github.com/alecthomas/chroma v0.7.2-0.20200305040604-4f3623dce67a/go.mod h1:fv5SzZPFJbwp2NXJWpFIX7DZS4HgV1K4ew4Pc2OZD9s=

helpers/path.go

@@ -44,7 +44,11 @@ var ErrThemeUndefined = errors.New("no theme set")
 // whilst preserving the original casing of the string.
 // E.g. Social Media -> Social-Media
 func (p *PathSpec) MakePath(s string) string {
-	return p.UnicodeSanitize(s)
+	s = p.UnicodeSanitize(s)
+	if p.RemovePathAccents {
+		s = text.RemoveAccentsString(s)
+	}
+	return s
 }
 
 // MakePathsSanitized applies MakePathSanitized on every item in the slice
@@ -74,7 +78,7 @@ func MakeTitle(inpath string) string {
 }
 
 // From https://golang.org/src/net/url/url.go
-func ishex(c rune) bool {
+func ishex(c byte) bool {
 	switch {
 	case '0' <= c && c <= '9':
 		return true
@@ -88,22 +92,26 @@ func ishex(c rune) bool {
 
 // UnicodeSanitize sanitizes string to be used in Hugo URL's, allowing only
 // a predefined set of special Unicode characters.
-// If RemovePathAccents configuration flag is enabled, Unicode accents
-// are also removed.
 // Spaces will be replaced with a single hyphen, and sequential hyphens will be reduced to one.
 func (p *PathSpec) UnicodeSanitize(s string) string {
-	if p.RemovePathAccents {
-		s = text.RemoveAccentsString(s)
+	var willChange bool
+	for i, r := range s {
+		willChange = !p.isAllowedPathCharacter(s, i, r)
+		if willChange {
+			break
+		}
+	}
+
+	if !willChange {
+		// Prevent allocation when nothing changes.
+		return s
 	}
 
-	source := []rune(s)
-	target := make([]rune, 0, len(source))
+	target := make([]rune, 0, len(s))
 	var prependHyphen bool
 
-	for i, r := range source {
-		isAllowed := r == '.' || r == '/' || r == '\\' || r == '_' || r == '#' || r == '+' || r == '~'
-		isAllowed = isAllowed || unicode.IsLetter(r) || unicode.IsDigit(r) || unicode.IsMark(r)
-		isAllowed = isAllowed || (r == '%' && i+2 < len(source) && ishex(source[i+1]) && ishex(source[i+2]))
+	for i, r := range s {
+		isAllowed := p.isAllowedPathCharacter(s, i, r)
 
 		if isAllowed {
 			if prependHyphen {
@@ -119,6 +127,13 @@ func (p *PathSpec) UnicodeSanitize(s string) string {
 	return string(target)
 }
 
+func (p *PathSpec) isAllowedPathCharacter(s string, i int, r rune) bool {
+	isAllowed := r == '.' || r == '/' || r == '\\' || r == '_' || r == '#' || r == '+' || r == '~'
+	isAllowed = isAllowed || unicode.IsLetter(r) || unicode.IsDigit(r) || unicode.IsMark(r)
+	isAllowed = isAllowed || (r == '%' && i+2 < len(s) && ishex(s[i+1]) && ishex(s[i+2]))
+	return isAllowed
+}
+
 func makePathRelative(inPath string, possibleDirectories ...string) (string, error) {
 	for _, currentPath := range possibleDirectories {
 		if strings.HasPrefix(inPath, currentPath) {
@@ -479,3 +494,18 @@ func AddTrailingSlash(path string) string {
 	}
 	return path
 }
+
+// AddLeadingSlash adds a leading Unix styled slash (/) if not already
+// there.
+func AddLeadingSlash(path string) string {
+	if !strings.HasPrefix(path, "/") {
+		path = "/" + path
+	}
+	return path
+}
+
+// AddLeadingAndTrailingSlash adds a leading and trailing Unix styled slash (/)
+// if not already there.
+func AddLeadingAndTrailingSlash(path string) string {
+	return AddTrailingSlash(AddLeadingSlash(path))
+}

helpers/path_test.go

@@ -44,6 +44,8 @@ func TestMakePath(t *testing.T) {
 		{"Foo.Bar/foo_Bar-Foo", "Foo.Bar/foo_Bar-Foo", true},
 		{"fOO,bar:foobAR", "fOObarfoobAR", true},
 		{"FOo/BaR.html", "FOo/BaR.html", true},
+		{"FOo/Ba---R.html", "FOo/Ba-R.html", true},
+		{"FOo/Ba       R.html", "FOo/Ba-R.html", true},
 		{"трям/трям", "трям/трям", true},
 		{"은행", "은행", true},
 		{"Банковский кассир", "Банковскии-кассир", true},