Convert the rest to new page parser code paths

And remove some now unused code.

See #5324

commands/convert.go

@@ -16,6 +16,7 @@ package commands
 import (
 	"bytes"
 	"fmt"
+	"io"
 	"strings"
 	"time"
 
@@ -34,7 +35,6 @@ import (
 
 	"path/filepath"
 
-	"github.com/spf13/cast"
 	"github.com/spf13/cobra"
 )
 
@@ -156,7 +156,7 @@ func (cc *convertCmd) convertAndSavePage(p *hugolib.Page, site *hugolib.Site, ta
 		return nil
 	}
 
-	psr, err := pageparser.Parse(file)
+	pf, err := parseContentFile(file)
 	if err != nil {
 		site.Log.ERROR.Println(errMsg)
 		file.Close()
@@ -165,53 +165,24 @@ func (cc *convertCmd) convertAndSavePage(p *hugolib.Page, site *hugolib.Site, ta
 
 	file.Close()
 
-	var sourceFormat, sourceContent []byte
-	var fromFormat metadecoders.Format
-
-	iter := psr.Iterator()
-
-	walkFn := func(item pageparser.Item) bool {
-		if sourceFormat != nil {
-			// The rest is content.
-			sourceContent = psr.Input()[item.Pos:]
-			// Done
-			return false
-		} else if item.IsFrontMatter() {
-			fromFormat = metadecoders.FormatFromFrontMatterType(item.Type)
-			sourceFormat = item.Val
-		}
-		return true
-
-	}
-
-	iter.PeekWalk(walkFn)
-
-	metadata, err := metadecoders.UnmarshalToMap(sourceFormat, fromFormat)
-	if err != nil {
-		site.Log.ERROR.Println(errMsg)
-		return err
-	}
-
 	// better handling of dates in formats that don't have support for them
-	if fromFormat == metadecoders.JSON || fromFormat == metadecoders.YAML || fromFormat == metadecoders.TOML {
-		newMetadata := cast.ToStringMap(metadata)
-		for k, v := range newMetadata {
+	if pf.frontMatterFormat == metadecoders.JSON || pf.frontMatterFormat == metadecoders.YAML || pf.frontMatterFormat == metadecoders.TOML {
+		for k, v := range pf.frontMatter {
 			switch vv := v.(type) {
 			case time.Time:
-				newMetadata[k] = vv.Format(time.RFC3339)
+				pf.frontMatter[k] = vv.Format(time.RFC3339)
 			}
 		}
-		metadata = newMetadata
 	}
 
 	var newContent bytes.Buffer
-	err = parser.InterfaceToFrontMatter2(metadata, targetFormat, &newContent)
+	err = parser.InterfaceToFrontMatter(pf.frontMatter, targetFormat, &newContent)
 	if err != nil {
 		site.Log.ERROR.Println(errMsg)
 		return err
 	}
 
-	newContent.Write(sourceContent)
+	newContent.Write(pf.content)
 
 	newFilename := p.Filename()
 
@@ -229,3 +200,48 @@ func (cc *convertCmd) convertAndSavePage(p *hugolib.Page, site *hugolib.Site, ta
 
 	return nil
 }
+
+type parsedFile struct {
+	frontMatterFormat metadecoders.Format
+	frontMatterSource []byte
+	frontMatter       map[string]interface{}
+
+	// Everything after Front Matter
+	content []byte
+}
+
+func parseContentFile(r io.Reader) (parsedFile, error) {
+	var pf parsedFile
+
+	psr, err := pageparser.Parse(r)
+	if err != nil {
+		return pf, err
+	}
+
+	iter := psr.Iterator()
+
+	walkFn := func(item pageparser.Item) bool {
+		if pf.frontMatterSource != nil {
+			// The rest is content.
+			pf.content = psr.Input()[item.Pos:]
+			// Done
+			return false
+		} else if item.IsFrontMatter() {
+			pf.frontMatterFormat = metadecoders.FormatFromFrontMatterType(item.Type)
+			pf.frontMatterSource = item.Val
+		}
+		return true
+
+	}
+
+	iter.PeekWalk(walkFn)
+
+	metadata, err := metadecoders.UnmarshalToMap(pf.frontMatterSource, pf.frontMatterFormat)
+	if err != nil {
+		return pf, err
+	}
+	pf.frontMatter = metadata
+
+	return pf, nil
+
+}

commands/import_jekyll.go

@@ -16,6 +16,7 @@ package commands
 import (
 	"bytes"
 	"errors"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
@@ -264,7 +265,7 @@ func (i *importCmd) loadJekyllConfig(fs afero.Fs, jekyllRoot string) map[string]
 	return c
 }
 
-func (i *importCmd) createConfigFromJekyll(fs afero.Fs, inpath string, kind string, jekyllConfig map[string]interface{}) (err error) {
+func (i *importCmd) createConfigFromJekyll(fs afero.Fs, inpath string, kind metadecoders.Format, jekyllConfig map[string]interface{}) (err error) {
 	title := "My New Hugo Site"
 	baseURL := "http://example.org/"
 
@@ -290,15 +291,14 @@ func (i *importCmd) createConfigFromJekyll(fs afero.Fs, inpath string, kind stri
 		"languageCode":       "en-us",
 		"disablePathToLower": true,
 	}
-	kind = parser.FormatSanitize(kind)
 
 	var buf bytes.Buffer
-	err = parser.InterfaceToConfig(in, parser.FormatToLeadRune(kind), &buf)
+	err = parser.InterfaceToConfig(in, kind, &buf)
 	if err != nil {
 		return err
 	}
 
-	return helpers.WriteToDisk(filepath.Join(inpath, "config."+kind), &buf, fs)
+	return helpers.WriteToDisk(filepath.Join(inpath, "config."+string(kind)), &buf, fs)
 }
 
 func copyFile(source string, dest string) error {
@@ -447,39 +447,25 @@ func convertJekyllPost(s *hugolib.Site, path, relPath, targetDir string, draft b
 		return err
 	}
 
-	psr, err := parser.ReadFrom(bytes.NewReader(contentBytes))
+	pf, err := parseContentFile(bytes.NewReader(contentBytes))
 	if err != nil {
 		jww.ERROR.Println("Parse file error:", path)
 		return err
 	}
 
-	metadata, err := psr.Metadata()
-	if err != nil {
-		jww.ERROR.Println("Processing file error:", path)
-		return err
-	}
-
-	newmetadata, err := convertJekyllMetaData(metadata, postName, postDate, draft)
+	newmetadata, err := convertJekyllMetaData(pf.frontMatter, postName, postDate, draft)
 	if err != nil {
 		jww.ERROR.Println("Convert metadata error:", path)
 		return err
 	}
 
-	jww.TRACE.Println(newmetadata)
-	content := convertJekyllContent(newmetadata, string(psr.Content()))
+	content := convertJekyllContent(newmetadata, string(pf.content))
 
-	page, err := s.NewPage(filename)
-	if err != nil {
-		jww.ERROR.Println("New page error", filename)
-		return err
+	fs := hugofs.Os
+	if err := helpers.WriteToDisk(targetFile, strings.NewReader(content), fs); err != nil {
+		return fmt.Errorf("Failed to save file %q:", filename)
 	}
 
-	page.SetSourceContent([]byte(content))
-	page.SetSourceMetaData(newmetadata, parser.FormatToLeadRune("yaml"))
-	page.SaveSourceAs(targetFile)
-
-	jww.TRACE.Println("Target file:", targetFile)
-
 	return nil
 }
 

commands/new_site.go

@@ -19,6 +19,8 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/gohugoio/hugo/parser/metadecoders"
+
 	_errors "github.com/pkg/errors"
 
 	"github.com/gohugoio/hugo/create"
@@ -131,10 +133,9 @@ func createConfig(fs *hugofs.Fs, inpath string, kind string) (err error) {
 		"title":        "My New Hugo Site",
 		"languageCode": "en-us",
 	}
-	kind = parser.FormatSanitize(kind)
 
 	var buf bytes.Buffer
-	err = parser.InterfaceToConfig(in, parser.FormatToLeadRune(kind), &buf)
+	err = parser.InterfaceToConfig(in, metadecoders.FormatFromString(kind), &buf)
 	if err != nil {
 		return err
 	}

hugolib/page.go

@@ -36,7 +36,6 @@ import (
 	"github.com/gohugoio/hugo/resource"
 
 	"github.com/gohugoio/hugo/output"
-	"github.com/gohugoio/hugo/parser"
 	"github.com/mitchellh/mapstructure"
 
 	"html/template"
@@ -485,6 +484,7 @@ func (p *Page) MediaType() media.Type {
 	return media.OctetType
 }
 
+// TODO(bep) 2errors remove
 type Source struct {
 	Frontmatter []byte
 	Content     []byte
@@ -1725,36 +1725,6 @@ func (p *Page) SetSourceContent(content []byte) {
 	p.Source.Content = content
 }
 
-func (p *Page) SetSourceMetaData(in interface{}, mark rune) (err error) {
-	// See https://github.com/gohugoio/hugo/issues/2458
-	defer func() {
-		if r := recover(); r != nil {
-			var ok bool
-			err, ok = r.(error)
-			if !ok {
-				err = fmt.Errorf("error from marshal: %v", r)
-			}
-		}
-	}()
-
-	buf := bp.GetBuffer()
-	defer bp.PutBuffer(buf)
-
-	err = parser.InterfaceToFrontMatter(in, mark, buf)
-	if err != nil {
-		return
-	}
-
-	_, err = buf.WriteRune('\n')
-	if err != nil {
-		return
-	}
-
-	p.Source.Frontmatter = buf.Bytes()
-
-	return
-}
-
 func (p *Page) SafeSaveSourceAs(path string) error {
 	return p.saveSourceAs(path, true)
 }