parser/metadecoders: Consolidate the metadata decoders

See gohugoio#5324

commands/convert.go

@@ -14,16 +14,26 @@
 package commands
 
 import (
+	"bytes"
+	"fmt"
+	"strings"
 	"time"
 
+	"github.com/gohugoio/hugo/hugofs"
+
+	"github.com/gohugoio/hugo/helpers"
+
+	"github.com/gohugoio/hugo/parser"
+	"github.com/gohugoio/hugo/parser/metadecoders"
+	"github.com/gohugoio/hugo/parser/pageparser"
+
 	src "github.com/gohugoio/hugo/source"
 	"github.com/pkg/errors"
 
 	"github.com/gohugoio/hugo/hugolib"
 
 	"path/filepath"
 
-	"github.com/gohugoio/hugo/parser"
 	"github.com/spf13/cast"
 	"github.com/spf13/cobra"
 )
@@ -60,7 +70,7 @@ See convert's subcommands toJSON, toTOML and toYAML for more information.`,
 			Long: `toJSON converts all front matter in the content directory
 to use JSON for the front matter.`,
 			RunE: func(cmd *cobra.Command, args []string) error {
-				return cc.convertContents(rune([]byte(parser.JSONLead)[0]))
+				return cc.convertContents(metadecoders.JSON)
 			},
 		},
 		&cobra.Command{
@@ -69,7 +79,7 @@ to use JSON for the front matter.`,
 			Long: `toTOML converts all front matter in the content directory
 to use TOML for the front matter.`,
 			RunE: func(cmd *cobra.Command, args []string) error {
-				return cc.convertContents(rune([]byte(parser.TOMLLead)[0]))
+				return cc.convertContents(metadecoders.TOML)
 			},
 		},
 		&cobra.Command{
@@ -78,7 +88,7 @@ to use TOML for the front matter.`,
 			Long: `toYAML converts all front matter in the content directory
 to use YAML for the front matter.`,
 			RunE: func(cmd *cobra.Command, args []string) error {
-				return cc.convertContents(rune([]byte(parser.YAMLLead)[0]))
+				return cc.convertContents(metadecoders.YAML)
 			},
 		},
 	)
@@ -91,7 +101,7 @@ to use YAML for the front matter.`,
 	return cc
 }
 
-func (cc *convertCmd) convertContents(mark rune) error {
+func (cc *convertCmd) convertContents(format metadecoders.Format) error {
 	if cc.outputDir == "" && !cc.unsafe {
 		return newUserError("Unsafe operation not allowed, use --unsafe or set a different output path")
 	}
@@ -114,17 +124,17 @@ func (cc *convertCmd) convertContents(mark rune) error {
 
 	site.Log.FEEDBACK.Println("processing", len(site.AllPages), "content files")
 	for _, p := range site.AllPages {
-		if err := cc.convertAndSavePage(p, site, mark); err != nil {
+		if err := cc.convertAndSavePage(p, site, format); err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
-func (cc *convertCmd) convertAndSavePage(p *hugolib.Page, site *hugolib.Site, mark rune) error {
+func (cc *convertCmd) convertAndSavePage(p *hugolib.Page, site *hugolib.Site, targetFormat metadecoders.Format) error {
 	// The resources are not in .Site.AllPages.
 	for _, r := range p.Resources.ByType("page") {
-		if err := cc.convertAndSavePage(r.(*hugolib.Page), site, mark); err != nil {
+		if err := cc.convertAndSavePage(r.(*hugolib.Page), site, targetFormat); err != nil {
 			return err
 		}
 	}
@@ -134,37 +144,56 @@ func (cc *convertCmd) convertAndSavePage(p *hugolib.Page, site *hugolib.Site, ma
 		return nil
 	}
 
+	errMsg := fmt.Errorf("Error processing file %q", p.Path())
+
 	site.Log.INFO.Println("Attempting to convert", p.LogicalName())
-	newPage, err := site.NewPage(p.LogicalName())
-	if err != nil {
-		return err
-	}
 
 	f, _ := p.File.(src.ReadableFile)
 	file, err := f.Open()
 	if err != nil {
-		site.Log.ERROR.Println("Error reading file:", p.Path())
+		site.Log.ERROR.Println(errMsg)
 		file.Close()
 		return nil
 	}
 
-	psr, err := parser.ReadFrom(file)
+	psr, err := pageparser.Parse(file)
 	if err != nil {
-		site.Log.ERROR.Println("Error processing file:", p.Path())
+		site.Log.ERROR.Println(errMsg)
 		file.Close()
 		return err
 	}
 
 	file.Close()
 
-	metadata, err := psr.Metadata()
+	var sourceFormat, sourceContent []byte
+	var fromFormat metadecoders.Format
+
+	iter := psr.Iterator()
+
+	walkFn := func(item pageparser.Item) bool {
+		if sourceFormat != nil {
+			// The rest is content.
+			sourceContent = psr.Input()[item.Pos:]
+			// Done
+			return false
+		} else if item.IsFrontMatter() {
+			fromFormat = metadecoders.FormatFromFrontMatterType(item.Type)
+			sourceFormat = item.Val
+		}
+		return true
+
+	}
+
+	iter.PeekWalk(walkFn)
+
+	metadata, err := metadecoders.UnmarshalToMap(sourceFormat, fromFormat)
 	if err != nil {
-		site.Log.ERROR.Println("Error processing file:", p.Path())
+		site.Log.ERROR.Println(errMsg)
 		return err
 	}
 
 	// better handling of dates in formats that don't have support for them
-	if mark == parser.FormatToLeadRune("json") || mark == parser.FormatToLeadRune("yaml") || mark == parser.FormatToLeadRune("toml") {
+	if fromFormat == metadecoders.JSON || fromFormat == metadecoders.YAML || fromFormat == metadecoders.TOML {
 		newMetadata := cast.ToStringMap(metadata)
 		for k, v := range newMetadata {
 			switch vv := v.(type) {
@@ -175,18 +204,26 @@ func (cc *convertCmd) convertAndSavePage(p *hugolib.Page, site *hugolib.Site, ma
 		metadata = newMetadata
 	}
 
-	newPage.SetSourceContent(psr.Content())
-	if err = newPage.SetSourceMetaData(metadata, mark); err != nil {
-		site.Log.ERROR.Printf("Failed to set source metadata for file %q: %s. For more info see For more info see https://github.com/gohugoio/hugo/issues/2458", newPage.FullFilePath(), err)
-		return nil
+	var newContent bytes.Buffer
+	err = parser.InterfaceToFrontMatter2(metadata, targetFormat, &newContent)
+	if err != nil {
+		site.Log.ERROR.Println(errMsg)
+		return err
 	}
 
+	newContent.Write(sourceContent)
+
 	newFilename := p.Filename()
+
 	if cc.outputDir != "" {
-		newFilename = filepath.Join(cc.outputDir, p.Dir(), newPage.LogicalName())
+		contentDir := strings.TrimSuffix(newFilename, p.Path())
+		contentDir = filepath.Base(contentDir)
+
+		newFilename = filepath.Join(cc.outputDir, contentDir, p.Path())
 	}
 
-	if err = newPage.SaveSourceAs(newFilename); err != nil {
+	fs := hugofs.Os
+	if err := helpers.WriteToDisk(newFilename, &newContent, fs); err != nil {
 		return errors.Wrapf(err, "Failed to save file %q:", newFilename)
 	}
 

commands/hugo.go

@@ -42,7 +42,7 @@ import (
 
 	"github.com/gohugoio/hugo/config"
 
-	"github.com/gohugoio/hugo/parser"
+	"github.com/gohugoio/hugo/parser/metadecoders"
 	flag "github.com/spf13/pflag"
 
 	"github.com/fsnotify/fsnotify"
@@ -1017,7 +1017,7 @@ func (c *commandeer) isThemeVsHugoVersionMismatch(fs afero.Fs) (dir string, mism
 
 		b, err := afero.ReadFile(fs, path)
 
-		tomlMeta, err := parser.HandleTOMLMetaData(b)
+		tomlMeta, err := metadecoders.UnmarshalToMap(b, metadecoders.TOML)
 
 		if err != nil {
 			continue

commands/import_jekyll.go

@@ -25,6 +25,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/gohugoio/hugo/parser/metadecoders"
+
 	"github.com/gohugoio/hugo/helpers"
 	"github.com/gohugoio/hugo/hugofs"
 	"github.com/gohugoio/hugo/hugolib"
@@ -253,7 +255,7 @@ func (i *importCmd) loadJekyllConfig(fs afero.Fs, jekyllRoot string) map[string]
 		return nil
 	}
 
-	c, err := parser.HandleYAMLMetaData(b)
+	c, err := metadecoders.UnmarshalToMap(b, metadecoders.YAML)
 
 	if err != nil {
 		return nil

go.mod

@@ -51,7 +51,7 @@ require (
 	github.com/spf13/nitro v0.0.0-20131003134307-24d7ef30a12d
 	github.com/spf13/pflag v1.0.2
 	github.com/spf13/viper v1.2.0
-	github.com/stretchr/testify v1.2.2
+	github.com/stretchr/testify v1.2.3-0.20181002232621-f2347ac6c9c9
 	github.com/tdewolff/minify v2.3.5+incompatible
 	github.com/tdewolff/parse v2.3.3+incompatible // indirect
 	github.com/tdewolff/test v0.0.0-20171106182207-265427085153 // indirect
@@ -63,6 +63,7 @@ require (
 	golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e // indirect
 	golang.org/x/text v0.3.0
 	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
+	gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0
 	gopkg.in/yaml.v2 v2.2.1
 )
 

go.sum

@@ -65,6 +65,7 @@ github.com/magefile/mage v1.4.0 h1:RI7B1CgnPAuu2O9lWszwya61RLmfL0KCdo+QyyI/Bhk=
 github.com/magefile/mage v1.4.0/go.mod h1:IUDi13rsHje59lecXokTfGX0QIzO45uVPlXnJYsXepA=
 github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/markbates/inflect v0.0.0-20171215194931-a12c3aec81a6 h1:LZhVjIISSbj8qLf2qDPP0D8z0uvOWAW5C85ly5mJW6c=
 github.com/markbates/inflect v0.0.0-20171215194931-a12c3aec81a6/go.mod h1:oTeZL2KHA7CUX6X+fovmK9OvIOFuqu0TwdQrZjLTh88=
 github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs=
 github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
@@ -118,6 +119,8 @@ github.com/spf13/viper v1.2.0 h1:M4Rzxlu+RgU4pyBRKhKaVN1VeYOm8h2jgyXnAseDgCc=
 github.com/spf13/viper v1.2.0/go.mod h1:P4AexN0a+C9tGAnUFNwDMYYZv3pjFuvmeiMyKRaNVlI=
 github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.2.3-0.20181002232621-f2347ac6c9c9 h1:kcVw9CGDqYBy0TTpIq2+BNR4W9poqiwEPBh/OYX5CaU=
+github.com/stretchr/testify v1.2.3-0.20181002232621-f2347ac6c9c9/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/tdewolff/minify v2.3.5+incompatible h1:oFxBKxTIY1F/1DEJhLeh/T507W56JqZtWVrawFcdadI=
 github.com/tdewolff/minify v2.3.5+incompatible/go.mod h1:9Ov578KJUmAWpS6NeZwRZyT56Uf6o3Mcz9CEsg8USYs=
 github.com/tdewolff/parse v2.3.3+incompatible h1:q6OSjvHtvBucLb34z24OH1xl5wGdw1mI9Vd38Qj9evs=
@@ -143,5 +146,7 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0 h1:POO/ycCATvegFmVuPpQzZFJ+pGZeX22Ufu6fibxDVjU=
+gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0/go.mod h1:WDnlLJ4WF5VGsH/HVa3CI79GS0ol3YnhVnKP89i0kNg=
 gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

hugolib/site.go

@@ -44,6 +44,7 @@ import (
 	"github.com/gohugoio/hugo/config"
 
 	"github.com/gohugoio/hugo/media"
+	"github.com/gohugoio/hugo/parser/metadecoders"
 
 	"github.com/markbates/inflect"
 
@@ -53,7 +54,6 @@ import (
 	"github.com/gohugoio/hugo/helpers"
 	"github.com/gohugoio/hugo/hugolib/pagemeta"
 	"github.com/gohugoio/hugo/output"
-	"github.com/gohugoio/hugo/parser"
 	"github.com/gohugoio/hugo/related"
 	"github.com/gohugoio/hugo/source"
 	"github.com/gohugoio/hugo/tpl"
@@ -949,16 +949,8 @@ func (s *Site) readData(f source.ReadableFile) (interface{}, error) {
 	defer file.Close()
 	content := helpers.ReaderToBytes(file)
 
-	switch f.Extension() {
-	case "yaml", "yml":
-		return parser.HandleYAMLData(content)
-	case "json":
-		return parser.HandleJSONData(content)
-	case "toml":
-		return parser.HandleTOMLMetaData(content)
-	default:
-		return nil, fmt.Errorf("Data not supported for extension '%s'", f.Extension())
-	}
+	format := metadecoders.FormatFromString(f.Extension())
+	return metadecoders.Unmarshal(content, format)
 }
 
 func (s *Site) readDataFromSourceFS() error {