diff --git a/commands/convert.go b/commands/convert.go index dc6b8fe1511..29714301f0b 100644 --- a/commands/convert.go +++ b/commands/convert.go @@ -14,8 +14,19 @@ package commands import ( + "bytes" + "fmt" + "strings" "time" + "github.com/gohugoio/hugo/hugofs" + + "github.com/gohugoio/hugo/helpers" + + "github.com/gohugoio/hugo/parser" + "github.com/gohugoio/hugo/parser/metadecoders" + "github.com/gohugoio/hugo/parser/pageparser" + src "github.com/gohugoio/hugo/source" "github.com/pkg/errors" @@ -23,7 +34,6 @@ import ( "path/filepath" - "github.com/gohugoio/hugo/parser" "github.com/spf13/cast" "github.com/spf13/cobra" ) @@ -60,7 +70,7 @@ See convert's subcommands toJSON, toTOML and toYAML for more information.`, Long: `toJSON converts all front matter in the content directory to use JSON for the front matter.`, RunE: func(cmd *cobra.Command, args []string) error { - return cc.convertContents(rune([]byte(parser.JSONLead)[0])) + return cc.convertContents(metadecoders.JSON) }, }, &cobra.Command{ @@ -69,7 +79,7 @@ to use JSON for the front matter.`, Long: `toTOML converts all front matter in the content directory to use TOML for the front matter.`, RunE: func(cmd *cobra.Command, args []string) error { - return cc.convertContents(rune([]byte(parser.TOMLLead)[0])) + return cc.convertContents(metadecoders.TOML) }, }, &cobra.Command{ @@ -78,7 +88,7 @@ to use TOML for the front matter.`, Long: `toYAML converts all front matter in the content directory to use YAML for the front matter.`, RunE: func(cmd *cobra.Command, args []string) error { - return cc.convertContents(rune([]byte(parser.YAMLLead)[0])) + return cc.convertContents(metadecoders.YAML) }, }, ) @@ -91,7 +101,7 @@ to use YAML for the front matter.`, return cc } -func (cc *convertCmd) convertContents(mark rune) error { +func (cc *convertCmd) convertContents(format metadecoders.Format) error { if cc.outputDir == "" && !cc.unsafe { return newUserError("Unsafe operation not allowed, use --unsafe or set a different output path") } @@ -114,17 +124,17 @@ func (cc *convertCmd) convertContents(mark rune) error { site.Log.FEEDBACK.Println("processing", len(site.AllPages), "content files") for _, p := range site.AllPages { - if err := cc.convertAndSavePage(p, site, mark); err != nil { + if err := cc.convertAndSavePage(p, site, format); err != nil { return err } } return nil } -func (cc *convertCmd) convertAndSavePage(p *hugolib.Page, site *hugolib.Site, mark rune) error { +func (cc *convertCmd) convertAndSavePage(p *hugolib.Page, site *hugolib.Site, targetFormat metadecoders.Format) error { // The resources are not in .Site.AllPages. for _, r := range p.Resources.ByType("page") { - if err := cc.convertAndSavePage(r.(*hugolib.Page), site, mark); err != nil { + if err := cc.convertAndSavePage(r.(*hugolib.Page), site, targetFormat); err != nil { return err } } @@ -134,37 +144,56 @@ func (cc *convertCmd) convertAndSavePage(p *hugolib.Page, site *hugolib.Site, ma return nil } + errMsg := fmt.Errorf("Error processing file %q", p.Path()) + site.Log.INFO.Println("Attempting to convert", p.LogicalName()) - newPage, err := site.NewPage(p.LogicalName()) - if err != nil { - return err - } f, _ := p.File.(src.ReadableFile) file, err := f.Open() if err != nil { - site.Log.ERROR.Println("Error reading file:", p.Path()) + site.Log.ERROR.Println(errMsg) file.Close() return nil } - psr, err := parser.ReadFrom(file) + psr, err := pageparser.Parse(file) if err != nil { - site.Log.ERROR.Println("Error processing file:", p.Path()) + site.Log.ERROR.Println(errMsg) file.Close() return err } file.Close() - metadata, err := psr.Metadata() + var sourceFormat, sourceContent []byte + var fromFormat metadecoders.Format + + iter := psr.Iterator() + + walkFn := func(item pageparser.Item) bool { + if sourceFormat != nil { + // The rest is content. + sourceContent = psr.Input()[item.Pos:] + // Done + return false + } else if item.IsFrontMatter() { + fromFormat = metadecoders.FormatFromFrontMatterType(item.Type) + sourceFormat = item.Val + } + return true + + } + + iter.PeekWalk(walkFn) + + metadata, err := metadecoders.UnmarshalToMap(sourceFormat, fromFormat) if err != nil { - site.Log.ERROR.Println("Error processing file:", p.Path()) + site.Log.ERROR.Println(errMsg) return err } // better handling of dates in formats that don't have support for them - if mark == parser.FormatToLeadRune("json") || mark == parser.FormatToLeadRune("yaml") || mark == parser.FormatToLeadRune("toml") { + if fromFormat == metadecoders.JSON || fromFormat == metadecoders.YAML || fromFormat == metadecoders.TOML { newMetadata := cast.ToStringMap(metadata) for k, v := range newMetadata { switch vv := v.(type) { @@ -175,18 +204,26 @@ func (cc *convertCmd) convertAndSavePage(p *hugolib.Page, site *hugolib.Site, ma metadata = newMetadata } - newPage.SetSourceContent(psr.Content()) - if err = newPage.SetSourceMetaData(metadata, mark); err != nil { - site.Log.ERROR.Printf("Failed to set source metadata for file %q: %s. For more info see For more info see https://github.com/gohugoio/hugo/issues/2458", newPage.FullFilePath(), err) - return nil + var newContent bytes.Buffer + err = parser.InterfaceToFrontMatter2(metadata, targetFormat, &newContent) + if err != nil { + site.Log.ERROR.Println(errMsg) + return err } + newContent.Write(sourceContent) + newFilename := p.Filename() + if cc.outputDir != "" { - newFilename = filepath.Join(cc.outputDir, p.Dir(), newPage.LogicalName()) + contentDir := strings.TrimSuffix(newFilename, p.Path()) + contentDir = filepath.Base(contentDir) + + newFilename = filepath.Join(cc.outputDir, contentDir, p.Path()) } - if err = newPage.SaveSourceAs(newFilename); err != nil { + fs := hugofs.Os + if err := helpers.WriteToDisk(newFilename, &newContent, fs); err != nil { return errors.Wrapf(err, "Failed to save file %q:", newFilename) } diff --git a/commands/hugo.go b/commands/hugo.go index deaa1f7ff02..2204ae9f35b 100644 --- a/commands/hugo.go +++ b/commands/hugo.go @@ -42,7 +42,7 @@ import ( "github.com/gohugoio/hugo/config" - "github.com/gohugoio/hugo/parser" + "github.com/gohugoio/hugo/parser/metadecoders" flag "github.com/spf13/pflag" "github.com/fsnotify/fsnotify" @@ -1017,7 +1017,7 @@ func (c *commandeer) isThemeVsHugoVersionMismatch(fs afero.Fs) (dir string, mism b, err := afero.ReadFile(fs, path) - tomlMeta, err := parser.HandleTOMLMetaData(b) + tomlMeta, err := metadecoders.UnmarshalToMap(b, metadecoders.TOML) if err != nil { continue diff --git a/commands/import_jekyll.go b/commands/import_jekyll.go index 6d88a7fd82a..2a86840d62e 100644 --- a/commands/import_jekyll.go +++ b/commands/import_jekyll.go @@ -25,6 +25,8 @@ import ( "strings" "time" + "github.com/gohugoio/hugo/parser/metadecoders" + "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/hugolib" @@ -253,7 +255,7 @@ func (i *importCmd) loadJekyllConfig(fs afero.Fs, jekyllRoot string) map[string] return nil } - c, err := parser.HandleYAMLMetaData(b) + c, err := metadecoders.UnmarshalToMap(b, metadecoders.YAML) if err != nil { return nil diff --git a/go.mod b/go.mod index aa73284e97c..b931009767e 100644 --- a/go.mod +++ b/go.mod @@ -51,7 +51,7 @@ require ( github.com/spf13/nitro v0.0.0-20131003134307-24d7ef30a12d github.com/spf13/pflag v1.0.2 github.com/spf13/viper v1.2.0 - github.com/stretchr/testify v1.2.2 + github.com/stretchr/testify v1.2.3-0.20181002232621-f2347ac6c9c9 github.com/tdewolff/minify v2.3.5+incompatible github.com/tdewolff/parse v2.3.3+incompatible // indirect github.com/tdewolff/test v0.0.0-20171106182207-265427085153 // indirect @@ -63,6 +63,7 @@ require ( golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e // indirect golang.org/x/text v0.3.0 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect + gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0 gopkg.in/yaml.v2 v2.2.1 ) diff --git a/go.sum b/go.sum index c41cacfb322..806cdc98a98 100644 --- a/go.sum +++ b/go.sum @@ -65,6 +65,7 @@ github.com/magefile/mage v1.4.0 h1:RI7B1CgnPAuu2O9lWszwya61RLmfL0KCdo+QyyI/Bhk= github.com/magefile/mage v1.4.0/go.mod h1:IUDi13rsHje59lecXokTfGX0QIzO45uVPlXnJYsXepA= github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/markbates/inflect v0.0.0-20171215194931-a12c3aec81a6 h1:LZhVjIISSbj8qLf2qDPP0D8z0uvOWAW5C85ly5mJW6c= github.com/markbates/inflect v0.0.0-20171215194931-a12c3aec81a6/go.mod h1:oTeZL2KHA7CUX6X+fovmK9OvIOFuqu0TwdQrZjLTh88= github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= @@ -118,6 +119,8 @@ github.com/spf13/viper v1.2.0 h1:M4Rzxlu+RgU4pyBRKhKaVN1VeYOm8h2jgyXnAseDgCc= github.com/spf13/viper v1.2.0/go.mod h1:P4AexN0a+C9tGAnUFNwDMYYZv3pjFuvmeiMyKRaNVlI= github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.2.3-0.20181002232621-f2347ac6c9c9 h1:kcVw9CGDqYBy0TTpIq2+BNR4W9poqiwEPBh/OYX5CaU= +github.com/stretchr/testify v1.2.3-0.20181002232621-f2347ac6c9c9/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/tdewolff/minify v2.3.5+incompatible h1:oFxBKxTIY1F/1DEJhLeh/T507W56JqZtWVrawFcdadI= github.com/tdewolff/minify v2.3.5+incompatible/go.mod h1:9Ov578KJUmAWpS6NeZwRZyT56Uf6o3Mcz9CEsg8USYs= github.com/tdewolff/parse v2.3.3+incompatible h1:q6OSjvHtvBucLb34z24OH1xl5wGdw1mI9Vd38Qj9evs= @@ -143,5 +146,7 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0 h1:POO/ycCATvegFmVuPpQzZFJ+pGZeX22Ufu6fibxDVjU= +gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0/go.mod h1:WDnlLJ4WF5VGsH/HVa3CI79GS0ol3YnhVnKP89i0kNg= gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/hugolib/site.go b/hugolib/site.go index 7f6ddce6c3a..cb980e8ab51 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -44,6 +44,7 @@ import ( "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/media" + "github.com/gohugoio/hugo/parser/metadecoders" "github.com/markbates/inflect" @@ -53,7 +54,6 @@ import ( "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/hugolib/pagemeta" "github.com/gohugoio/hugo/output" - "github.com/gohugoio/hugo/parser" "github.com/gohugoio/hugo/related" "github.com/gohugoio/hugo/source" "github.com/gohugoio/hugo/tpl" @@ -949,16 +949,8 @@ func (s *Site) readData(f source.ReadableFile) (interface{}, error) { defer file.Close() content := helpers.ReaderToBytes(file) - switch f.Extension() { - case "yaml", "yml": - return parser.HandleYAMLData(content) - case "json": - return parser.HandleJSONData(content) - case "toml": - return parser.HandleTOMLMetaData(content) - default: - return nil, fmt.Errorf("Data not supported for extension '%s'", f.Extension()) - } + format := metadecoders.FormatFromString(f.Extension()) + return metadecoders.Unmarshal(content, format) } func (s *Site) readDataFromSourceFS() error { diff --git a/parser/frontmatter.go b/parser/frontmatter.go index 284d3f955da..a42db0ccab2 100644 --- a/parser/frontmatter.go +++ b/parser/frontmatter.go @@ -19,16 +19,12 @@ import ( "bytes" "encoding/json" "errors" - "fmt" "io" "strings" - "github.com/gohugoio/hugo/helpers" - - "github.com/spf13/cast" + "github.com/gohugoio/hugo/parser/metadecoders" "github.com/BurntSushi/toml" - "github.com/chaseadamsio/goorgeous" "gopkg.in/yaml.v2" ) @@ -79,6 +75,82 @@ func InterfaceToConfig(in interface{}, mark rune, w io.Writer) error { } } +func InterfaceToConfig2(in interface{}, format metadecoders.Format, w io.Writer) error { + if in == nil { + return errors.New("input was nil") + } + + switch format { + case metadecoders.YAML: + b, err := yaml.Marshal(in) + if err != nil { + return err + } + + _, err = w.Write(b) + return err + + case metadecoders.TOML: + return toml.NewEncoder(w).Encode(in) + case metadecoders.JSON: + b, err := json.MarshalIndent(in, "", " ") + if err != nil { + return err + } + + _, err = w.Write(b) + if err != nil { + return err + } + + _, err = w.Write([]byte{'\n'}) + return err + + default: + return errors.New("Unsupported Format provided") + } +} + +func InterfaceToFrontMatter2(in interface{}, format metadecoders.Format, w io.Writer) error { + if in == nil { + return errors.New("input was nil") + } + + switch format { + case metadecoders.YAML: + _, err := w.Write([]byte(YAMLDelimUnix)) + if err != nil { + return err + } + + err = InterfaceToConfig2(in, format, w) + if err != nil { + return err + } + + _, err = w.Write([]byte(YAMLDelimUnix)) + return err + + case metadecoders.TOML: + _, err := w.Write([]byte(TOMLDelimUnix)) + if err != nil { + return err + } + + err = InterfaceToConfig2(in, format, w) + + if err != nil { + return err + } + + _, err = w.Write([]byte("\n" + TOMLDelimUnix)) + return err + + default: + return InterfaceToConfig2(in, format, w) + } +} + // InterfaceToFrontMatter encodes a given input into a frontmatter // representation based upon the mark with the appropriate front matter delimiters // surrounding the output, which is written to w. @@ -155,34 +227,6 @@ func FormatSanitize(kind string) string { } } -// DetectFrontMatter detects the type of frontmatter analysing its first character. -func DetectFrontMatter(mark rune) (f *FrontmatterType) { - switch mark { - case '-': - return &FrontmatterType{HandleYAMLMetaData, []byte(YAMLDelim), []byte(YAMLDelim), false} - case '+': - return &FrontmatterType{HandleTOMLMetaData, []byte(TOMLDelim), []byte(TOMLDelim), false} - case '{': - return &FrontmatterType{HandleJSONMetaData, []byte{'{'}, []byte{'}'}, true} - case '#': - return &FrontmatterType{HandleOrgMetaData, []byte("#+"), []byte("\n"), false} - default: - return nil - } -} - -// HandleTOMLMetaData unmarshals TOML-encoded datum and returns a Go interface -// representing the encoded data structure. -func HandleTOMLMetaData(datum []byte) (map[string]interface{}, error) { - m := map[string]interface{}{} - datum = removeTOMLIdentifier(datum) - - _, err := toml.Decode(string(datum), &m) - - return m, err - -} - // removeTOMLIdentifier removes, if necessary, beginning and ending TOML // frontmatter delimiters from a byte slice. func removeTOMLIdentifier(datum []byte) []byte { @@ -200,125 +244,3 @@ func removeTOMLIdentifier(datum []byte) []byte { b = bytes.Trim(b, "\r\n") return bytes.TrimSuffix(b, []byte(TOMLDelim)) } - -// HandleYAMLMetaData unmarshals YAML-encoded datum and returns a Go interface -// representing the encoded data structure. -// TODO(bep) 2errors remove these handlers (and hopefully package) -func HandleYAMLMetaData(datum []byte) (map[string]interface{}, error) { - m := map[string]interface{}{} - err := yaml.Unmarshal(datum, &m) - - // To support boolean keys, the `yaml` package unmarshals maps to - // map[interface{}]interface{}. Here we recurse through the result - // and change all maps to map[string]interface{} like we would've - // gotten from `json`. - if err == nil { - for k, v := range m { - if vv, changed := stringifyMapKeys(v); changed { - m[k] = vv - } - } - } - - return m, err -} - -// HandleYAMLData unmarshals YAML-encoded datum and returns a Go interface -// representing the encoded data structure. -func HandleYAMLData(datum []byte) (interface{}, error) { - var m interface{} - err := yaml.Unmarshal(datum, &m) - if err != nil { - return nil, err - } - - // To support boolean keys, the `yaml` package unmarshals maps to - // map[interface{}]interface{}. Here we recurse through the result - // and change all maps to map[string]interface{} like we would've - // gotten from `json`. - if mm, changed := stringifyMapKeys(m); changed { - return mm, nil - } - - return m, nil -} - -// stringifyMapKeys recurses into in and changes all instances of -// map[interface{}]interface{} to map[string]interface{}. This is useful to -// work around the impedence mismatch between JSON and YAML unmarshaling that's -// described here: https://github.com/go-yaml/yaml/issues/139 -// -// Inspired by https://github.com/stripe/stripe-mock, MIT licensed -func stringifyMapKeys(in interface{}) (interface{}, bool) { - switch in := in.(type) { - case []interface{}: - for i, v := range in { - if vv, replaced := stringifyMapKeys(v); replaced { - in[i] = vv - } - } - case map[interface{}]interface{}: - res := make(map[string]interface{}) - var ( - ok bool - err error - ) - for k, v := range in { - var ks string - - if ks, ok = k.(string); !ok { - ks, err = cast.ToStringE(k) - if err != nil { - ks = fmt.Sprintf("%v", k) - } - // TODO(bep) added in Hugo 0.37, remove some time in the future. - helpers.DistinctFeedbackLog.Printf("WARNING: YAML data/frontmatter with keys of type %T is since Hugo 0.37 converted to strings", k) - } - if vv, replaced := stringifyMapKeys(v); replaced { - res[ks] = vv - } else { - res[ks] = v - } - } - return res, true - } - - return nil, false -} - -// HandleJSONMetaData unmarshals JSON-encoded datum and returns a Go interface -// representing the encoded data structure. -func HandleJSONMetaData(datum []byte) (map[string]interface{}, error) { - m := make(map[string]interface{}) - - if datum == nil { - // Package json returns on error on nil input. - // Return an empty map to be consistent with our other supported - // formats. - return m, nil - } - - err := json.Unmarshal(datum, &m) - return m, err -} - -// HandleJSONData unmarshals JSON-encoded datum and returns a Go interface -// representing the encoded data structure. -func HandleJSONData(datum []byte) (interface{}, error) { - if datum == nil { - // Package json returns on error on nil input. - // Return an empty map to be consistent with our other supported - // formats. - return make(map[string]interface{}), nil - } - - var f interface{} - err := json.Unmarshal(datum, &f) - return f, err -} - -// HandleOrgMetaData unmarshals org-mode encoded datum and returns a Go -// interface representing the encoded data structure. -func HandleOrgMetaData(datum []byte) (map[string]interface{}, error) { - return goorgeous.OrgHeaders(datum) -} diff --git a/parser/frontmatter_test.go b/parser/frontmatter_test.go index 7281ca3368f..d6e6e79c341 100644 --- a/parser/frontmatter_test.go +++ b/parser/frontmatter_test.go @@ -132,116 +132,6 @@ func TestInterfaceToFrontMatter(t *testing.T) { } } -func TestHandleTOMLMetaData(t *testing.T) { - cases := []struct { - input []byte - want interface{} - isErr bool - }{ - {nil, map[string]interface{}{}, false}, - {[]byte("title = \"test 1\""), map[string]interface{}{"title": "test 1"}, false}, - {[]byte("a = [1, 2, 3]"), map[string]interface{}{"a": []interface{}{int64(1), int64(2), int64(3)}}, false}, - {[]byte("b = [\n[1, 2],\n[3, 4]\n]"), map[string]interface{}{"b": []interface{}{[]interface{}{int64(1), int64(2)}, []interface{}{int64(3), int64(4)}}}, false}, - // errors - {[]byte("z = [\n[1, 2]\n[3, 4]\n]"), nil, true}, - } - - for i, c := range cases { - res, err := HandleTOMLMetaData(c.input) - if err != nil { - if c.isErr { - continue - } - t.Fatalf("[%d] unexpected error value: %v", i, err) - } - - if !reflect.DeepEqual(res, c.want) { - t.Errorf("[%d] not equal: given %q\nwant %#v,\n got %#v", i, c.input, c.want, res) - } - } -} - -func TestHandleYAMLMetaData(t *testing.T) { - cases := []struct { - input []byte - want interface{} - isErr bool - }{ - {nil, map[string]interface{}{}, false}, - {[]byte("title: test 1"), map[string]interface{}{"title": "test 1"}, false}, - {[]byte("a: Easy!\nb:\n c: 2\n d: [3, 4]"), map[string]interface{}{"a": "Easy!", "b": map[string]interface{}{"c": 2, "d": []interface{}{3, 4}}}, false}, - {[]byte("a:\n true: 1\n false: 2"), map[string]interface{}{"a": map[string]interface{}{"true": 1, "false": 2}}, false}, - // errors - {[]byte("z = not toml"), nil, true}, - } - - for i, c := range cases { - res, err := HandleYAMLMetaData(c.input) - if err != nil { - if c.isErr { - continue - } - t.Fatalf("[%d] unexpected error value: %v", i, err) - } - - if !reflect.DeepEqual(res, c.want) { - t.Errorf("[%d] not equal: given %q\nwant %#v,\n got %#v", i, c.input, c.want, res) - } - } -} - -func TestHandleJSONMetaData(t *testing.T) { - cases := []struct { - input []byte - want interface{} - isErr bool - }{ - {nil, map[string]interface{}{}, false}, - {[]byte("{\"title\": \"test 1\"}"), map[string]interface{}{"title": "test 1"}, false}, - // errors - {[]byte("{noquotes}"), nil, true}, - } - - for i, c := range cases { - res, err := HandleJSONMetaData(c.input) - if err != nil { - if c.isErr { - continue - } - t.Fatalf("[%d] unexpected error value: %v", i, err) - } - - if !reflect.DeepEqual(res, c.want) { - t.Errorf("[%d] not equal: given %q\nwant %#v,\n got %#v", i, c.input, c.want, res) - } - } -} - -func TestHandleOrgMetaData(t *testing.T) { - cases := []struct { - input []byte - want interface{} - isErr bool - }{ - {nil, map[string]interface{}{}, false}, - {[]byte("#+title: test 1\n"), map[string]interface{}{"title": "test 1"}, false}, - } - - for i, c := range cases { - res, err := HandleOrgMetaData(c.input) - if err != nil { - if c.isErr { - continue - } - t.Fatalf("[%d] unexpected error value: %v", i, err) - } - - if !reflect.DeepEqual(res, c.want) { - t.Errorf("[%d] not equal: given %q\nwant %#v,\n got %#v", i, c.input, c.want, res) - } - } -} - func TestFormatToLeadRune(t *testing.T) { for i, this := range []struct { kind string @@ -264,41 +154,6 @@ func TestFormatToLeadRune(t *testing.T) { } } -func TestDetectFrontMatter(t *testing.T) { - cases := []struct { - mark rune - want *FrontmatterType - }{ - // funcs are uncomparable, so we ignore FrontmatterType.Parse in these tests - {'-', &FrontmatterType{nil, []byte(YAMLDelim), []byte(YAMLDelim), false}}, - {'+', &FrontmatterType{nil, []byte(TOMLDelim), []byte(TOMLDelim), false}}, - {'{', &FrontmatterType{nil, []byte("{"), []byte("}"), true}}, - {'#', &FrontmatterType{nil, []byte("#+"), []byte("\n"), false}}, - {'$', nil}, - } - - for _, c := range cases { - res := DetectFrontMatter(c.mark) - if res == nil { - if c.want == nil { - continue - } - - t.Fatalf("want %v, got %v", *c.want, res) - } - - if !reflect.DeepEqual(res.markstart, c.want.markstart) { - t.Errorf("markstart mismatch: want %v, got %v", c.want.markstart, res.markstart) - } - if !reflect.DeepEqual(res.markend, c.want.markend) { - t.Errorf("markend mismatch: want %v, got %v", c.want.markend, res.markend) - } - if !reflect.DeepEqual(res.includeMark, c.want.includeMark) { - t.Errorf("includeMark mismatch: want %v, got %v", c.want.includeMark, res.includeMark) - } - } -} - func TestRemoveTOMLIdentifier(t *testing.T) { cases := []struct { input string @@ -321,64 +176,6 @@ func TestRemoveTOMLIdentifier(t *testing.T) { } } -func TestStringifyYAMLMapKeys(t *testing.T) { - cases := []struct { - input interface{} - want interface{} - replaced bool - }{ - { - map[interface{}]interface{}{"a": 1, "b": 2}, - map[string]interface{}{"a": 1, "b": 2}, - true, - }, - { - map[interface{}]interface{}{"a": []interface{}{1, map[interface{}]interface{}{"b": 2}}}, - map[string]interface{}{"a": []interface{}{1, map[string]interface{}{"b": 2}}}, - true, - }, - { - map[interface{}]interface{}{true: 1, "b": false}, - map[string]interface{}{"true": 1, "b": false}, - true, - }, - { - map[interface{}]interface{}{1: "a", 2: "b"}, - map[string]interface{}{"1": "a", "2": "b"}, - true, - }, - { - map[interface{}]interface{}{"a": map[interface{}]interface{}{"b": 1}}, - map[string]interface{}{"a": map[string]interface{}{"b": 1}}, - true, - }, - { - map[string]interface{}{"a": map[string]interface{}{"b": 1}}, - map[string]interface{}{"a": map[string]interface{}{"b": 1}}, - false, - }, - { - []interface{}{map[interface{}]interface{}{1: "a", 2: "b"}}, - []interface{}{map[string]interface{}{"1": "a", "2": "b"}}, - false, - }, - } - - for i, c := range cases { - res, replaced := stringifyMapKeys(c.input) - - if c.replaced != replaced { - t.Fatalf("[%d] Replaced mismatch: %t", i, replaced) - } - if !c.replaced { - res = c.input - } - if !reflect.DeepEqual(res, c.want) { - t.Errorf("[%d] given %q\nwant: %q\n got: %q", i, c.input, c.want, res) - } - } -} - func BenchmarkFrontmatterTags(b *testing.B) { for _, frontmatter := range []string{"JSON", "YAML", "YAML2", "TOML"} { @@ -388,69 +185,6 @@ func BenchmarkFrontmatterTags(b *testing.B) { } } -func BenchmarkStringifyMapKeysStringsOnlyInterfaceMaps(b *testing.B) { - maps := make([]map[interface{}]interface{}, b.N) - for i := 0; i < b.N; i++ { - maps[i] = map[interface{}]interface{}{ - "a": map[interface{}]interface{}{ - "b": 32, - "c": 43, - "d": map[interface{}]interface{}{ - "b": 32, - "c": 43, - }, - }, - "b": []interface{}{"a", "b"}, - "c": "d", - } - } - b.ResetTimer() - for i := 0; i < b.N; i++ { - stringifyMapKeys(maps[i]) - } -} - -func BenchmarkStringifyMapKeysStringsOnlyStringMaps(b *testing.B) { - m := map[string]interface{}{ - "a": map[string]interface{}{ - "b": 32, - "c": 43, - "d": map[string]interface{}{ - "b": 32, - "c": 43, - }, - }, - "b": []interface{}{"a", "b"}, - "c": "d", - } - - b.ResetTimer() - for i := 0; i < b.N; i++ { - stringifyMapKeys(m) - } -} - -func BenchmarkStringifyMapKeysIntegers(b *testing.B) { - maps := make([]map[interface{}]interface{}, b.N) - for i := 0; i < b.N; i++ { - maps[i] = map[interface{}]interface{}{ - 1: map[interface{}]interface{}{ - 4: 32, - 5: 43, - 6: map[interface{}]interface{}{ - 7: 32, - 8: 43, - }, - }, - 2: []interface{}{"a", "b"}, - 3: "d", - } - } - b.ResetTimer() - for i := 0; i < b.N; i++ { - stringifyMapKeys(maps[i]) - } -} func doBenchmarkFrontmatter(b *testing.B, fileformat string, numTags int) { yamlTemplate := `--- name: "Tags" diff --git a/parser/metadecoders/decoder.go b/parser/metadecoders/decoder.go index 280361a8411..e409b76653d 100644 --- a/parser/metadecoders/decoder.go +++ b/parser/metadecoders/decoder.go @@ -15,81 +15,139 @@ package metadecoders import ( "encoding/json" + "fmt" "github.com/BurntSushi/toml" "github.com/chaseadamsio/goorgeous" - "github.com/gohugoio/hugo/parser/pageparser" "github.com/pkg/errors" + "github.com/spf13/cast" yaml "gopkg.in/yaml.v2" ) -type Format string - -const ( - // These are the supported metdata formats in Hugo. Most of these are also - // supported as /data formats. - ORG Format = "org" - JSON Format = "json" - TOML Format = "toml" - YAML Format = "yaml" -) - -// FormatFromFrontMatterType will return empty if not supported. -func FormatFromFrontMatterType(typ pageparser.ItemType) Format { - switch typ { - case pageparser.TypeFrontMatterJSON: - return JSON - case pageparser.TypeFrontMatterORG: - return ORG - case pageparser.TypeFrontMatterTOML: - return TOML - case pageparser.TypeFrontMatterYAML: - return YAML - default: - return "" - } -} - // UnmarshalToMap will unmarshall data in format f into a new map. This is // what's needed for Hugo's front matter decoding. func UnmarshalToMap(data []byte, f Format) (map[string]interface{}, error) { m := make(map[string]interface{}) - if data == nil { return m, nil } + err := unmarshal(data, f, &m) + + return m, err + +} + +// Unmarshal will unmarshall data in format f into an interface{}. +// This is what's needed for Hugo's /data handling. +func Unmarshal(data []byte, f Format) (interface{}, error) { + if data == nil { + return make(map[string]interface{}), nil + } + var v interface{} + err := unmarshal(data, f, &v) + + return v, err +} + +// unmarshal unmarshals data in format f into v. +func unmarshal(data []byte, f Format, v interface{}) error { + var err error switch f { case ORG: - m, err = goorgeous.OrgHeaders(data) + vv, err := goorgeous.OrgHeaders(data) + if err != nil { + return err + } + switch v.(type) { + case *map[string]interface{}: + *v.(*map[string]interface{}) = vv + default: + *v.(*interface{}) = vv + } case JSON: - err = json.Unmarshal(data, &m) + err = json.Unmarshal(data, v) case TOML: - _, err = toml.Decode(string(data), &m) + err = toml.Unmarshal(data, v) case YAML: - err = yaml.Unmarshal(data, &m) + err = yaml.Unmarshal(data, v) - // To support boolean keys, the `yaml` package unmarshals maps to + // To support boolean keys, the YAML package unmarshals maps to // map[interface{}]interface{}. Here we recurse through the result // and change all maps to map[string]interface{} like we would've // gotten from `json`. - if err == nil { - for k, v := range m { - if vv, changed := stringifyMapKeys(v); changed { - m[k] = vv - } + var ptr interface{} + switch v.(type) { + case *map[string]interface{}: + ptr = *v.(*map[string]interface{}) + case *interface{}: + ptr = *v.(*interface{}) + default: + return errors.Errorf("unknown type %T in YAML unmarshal", v) + } + + if mm, changed := stringifyMapKeys(ptr); changed { + switch v.(type) { + case *map[string]interface{}: + *v.(*map[string]interface{}) = mm.(map[string]interface{}) + case *interface{}: + *v.(*interface{}) = mm } } default: - return nil, errors.Errorf("unmarshal of format %q is not supported", f) + return errors.Errorf("unmarshal of format %q is not supported", f) } - if err != nil { - return nil, errors.Wrapf(err, "unmarshal failed for format %q", f) - } + return err - return m, nil +} + +// stringifyMapKeys recurses into in and changes all instances of +// map[interface{}]interface{} to map[string]interface{}. This is useful to +// work around the impedence mismatch between JSON and YAML unmarshaling that's +// described here: https://github.com/go-yaml/yaml/issues/139 +// +// Inspired by https://github.com/stripe/stripe-mock, MIT licensed +func stringifyMapKeys(in interface{}) (interface{}, bool) { + + switch in := in.(type) { + case []interface{}: + for i, v := range in { + if vv, replaced := stringifyMapKeys(v); replaced { + in[i] = vv + } + } + case map[string]interface{}: + for k, v := range in { + if vv, changed := stringifyMapKeys(v); changed { + in[k] = vv + } + } + case map[interface{}]interface{}: + res := make(map[string]interface{}) + var ( + ok bool + err error + ) + for k, v := range in { + var ks string + + if ks, ok = k.(string); !ok { + ks, err = cast.ToStringE(k) + if err != nil { + ks = fmt.Sprintf("%v", k) + } + } + if vv, replaced := stringifyMapKeys(v); replaced { + res[ks] = vv + } else { + res[ks] = v + } + } + return res, true + } + return nil, false } diff --git a/parser/metadecoders/decoder_test.go b/parser/metadecoders/decoder_test.go new file mode 100644 index 00000000000..94cfd5a9a2d --- /dev/null +++ b/parser/metadecoders/decoder_test.go @@ -0,0 +1,207 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metadecoders + +import ( + "fmt" + "reflect" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestUnmarshalToMap(t *testing.T) { + assert := require.New(t) + + expect := map[string]interface{}{"a": "b"} + + for i, test := range []struct { + data string + format Format + expect interface{} + }{ + {`a = "b"`, TOML, expect}, + {`a: "b"`, YAML, expect}, + // Make sure we get all string keys, even for YAML + {"a: Easy!\nb:\n c: 2\n d: [3, 4]", YAML, map[string]interface{}{"a": "Easy!", "b": map[string]interface{}{"c": 2, "d": []interface{}{3, 4}}}}, + {"a:\n true: 1\n false: 2", YAML, map[string]interface{}{"a": map[string]interface{}{"true": 1, "false": 2}}}, + {`{ "a": "b" }`, JSON, expect}, + {`#+a: b`, ORG, expect}, + // errors + {`a = b`, TOML, false}, + } { + msg := fmt.Sprintf("%d: %s", i, test.format) + m, err := UnmarshalToMap([]byte(test.data), test.format) + if b, ok := test.expect.(bool); ok && !b { + assert.Error(err, msg) + } else { + assert.NoError(err, msg) + assert.Equal(test.expect, m, msg) + } + } +} + +func TestUnmarshalToInterface(t *testing.T) { + assert := require.New(t) + + expect := map[string]interface{}{"a": "b"} + + for i, test := range []struct { + data string + format Format + expect interface{} + }{ + {`[ "Brecker", "Blake", "Redman" ]`, JSON, []interface{}{"Brecker", "Blake", "Redman"}}, + {`{ "a": "b" }`, JSON, expect}, + {`#+a: b`, ORG, expect}, + {`a = "b"`, TOML, expect}, + {`a: "b"`, YAML, expect}, + {"a: Easy!\nb:\n c: 2\n d: [3, 4]", YAML, map[string]interface{}{"a": "Easy!", "b": map[string]interface{}{"c": 2, "d": []interface{}{3, 4}}}}, + // errors + {`a = "`, TOML, false}, + } { + msg := fmt.Sprintf("%d: %s", i, test.format) + m, err := Unmarshal([]byte(test.data), test.format) + if b, ok := test.expect.(bool); ok && !b { + assert.Error(err, msg) + } else { + assert.NoError(err, msg) + assert.Equal(test.expect, m, msg) + } + + } + +} + +func TestStringifyYAMLMapKeys(t *testing.T) { + cases := []struct { + input interface{} + want interface{} + replaced bool + }{ + { + map[interface{}]interface{}{"a": 1, "b": 2}, + map[string]interface{}{"a": 1, "b": 2}, + true, + }, + { + map[interface{}]interface{}{"a": []interface{}{1, map[interface{}]interface{}{"b": 2}}}, + map[string]interface{}{"a": []interface{}{1, map[string]interface{}{"b": 2}}}, + true, + }, + { + map[interface{}]interface{}{true: 1, "b": false}, + map[string]interface{}{"true": 1, "b": false}, + true, + }, + { + map[interface{}]interface{}{1: "a", 2: "b"}, + map[string]interface{}{"1": "a", "2": "b"}, + true, + }, + { + map[interface{}]interface{}{"a": map[interface{}]interface{}{"b": 1}}, + map[string]interface{}{"a": map[string]interface{}{"b": 1}}, + true, + }, + { + map[string]interface{}{"a": map[string]interface{}{"b": 1}}, + map[string]interface{}{"a": map[string]interface{}{"b": 1}}, + false, + }, + { + []interface{}{map[interface{}]interface{}{1: "a", 2: "b"}}, + []interface{}{map[string]interface{}{"1": "a", "2": "b"}}, + false, + }, + } + + for i, c := range cases { + res, replaced := stringifyMapKeys(c.input) + + if c.replaced != replaced { + t.Fatalf("[%d] Replaced mismatch: %t", i, replaced) + } + if !c.replaced { + res = c.input + } + if !reflect.DeepEqual(res, c.want) { + t.Errorf("[%d] given %q\nwant: %q\n got: %q", i, c.input, c.want, res) + } + } +} + +func BenchmarkStringifyMapKeysStringsOnlyInterfaceMaps(b *testing.B) { + maps := make([]map[interface{}]interface{}, b.N) + for i := 0; i < b.N; i++ { + maps[i] = map[interface{}]interface{}{ + "a": map[interface{}]interface{}{ + "b": 32, + "c": 43, + "d": map[interface{}]interface{}{ + "b": 32, + "c": 43, + }, + }, + "b": []interface{}{"a", "b"}, + "c": "d", + } + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + stringifyMapKeys(maps[i]) + } +} + +func BenchmarkStringifyMapKeysStringsOnlyStringMaps(b *testing.B) { + m := map[string]interface{}{ + "a": map[string]interface{}{ + "b": 32, + "c": 43, + "d": map[string]interface{}{ + "b": 32, + "c": 43, + }, + }, + "b": []interface{}{"a", "b"}, + "c": "d", + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + stringifyMapKeys(m) + } +} + +func BenchmarkStringifyMapKeysIntegers(b *testing.B) { + maps := make([]map[interface{}]interface{}, b.N) + for i := 0; i < b.N; i++ { + maps[i] = map[interface{}]interface{}{ + 1: map[interface{}]interface{}{ + 4: 32, + 5: 43, + 6: map[interface{}]interface{}{ + 7: 32, + 8: 43, + }, + }, + 2: []interface{}{"a", "b"}, + 3: "d", + } + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + stringifyMapKeys(maps[i]) + } +} diff --git a/parser/metadecoders/format.go b/parser/metadecoders/format.go new file mode 100644 index 00000000000..b9f7f691905 --- /dev/null +++ b/parser/metadecoders/format.go @@ -0,0 +1,66 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metadecoders + +import ( + "strings" + + "github.com/gohugoio/hugo/parser/pageparser" +) + +type Format string + +const ( + // These are the supported metdata formats in Hugo. Most of these are also + // supported as /data formats. + ORG Format = "org" + JSON Format = "json" + TOML Format = "toml" + YAML Format = "yaml" +) + +// FormatFromString turns formatStr, typically a file extension without any ".", +// into a Format. It returns an empty string for unknown formats. +func FormatFromString(formatStr string) Format { + formatStr = strings.ToLower(formatStr) + switch formatStr { + case "yaml", "yml": + return YAML + case "json": + return JSON + case "toml": + return TOML + case "org": + return ORG + } + + return "" + +} + +// FormatFromFrontMatterType will return empty if not supported. +func FormatFromFrontMatterType(typ pageparser.ItemType) Format { + switch typ { + case pageparser.TypeFrontMatterJSON: + return JSON + case pageparser.TypeFrontMatterORG: + return ORG + case pageparser.TypeFrontMatterTOML: + return TOML + case pageparser.TypeFrontMatterYAML: + return YAML + default: + return "" + } +} diff --git a/parser/metadecoders/format_test.go b/parser/metadecoders/format_test.go new file mode 100644 index 00000000000..46b4e434af6 --- /dev/null +++ b/parser/metadecoders/format_test.go @@ -0,0 +1,57 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metadecoders + +import ( + "fmt" + "testing" + + "github.com/gohugoio/hugo/parser/pageparser" + + "github.com/stretchr/testify/require" +) + +func TestFormatFromString(t *testing.T) { + assert := require.New(t) + for i, test := range []struct { + s string + expect Format + }{ + {"json", JSON}, + {"yaml", YAML}, + {"yml", YAML}, + {"toml", TOML}, + {"tOMl", TOML}, + {"org", ORG}, + {"foo", ""}, + } { + assert.Equal(test.expect, FormatFromString(test.s), fmt.Sprintf("t%d", i)) + } +} + +func TestFormatFromFrontMatterType(t *testing.T) { + assert := require.New(t) + for i, test := range []struct { + typ pageparser.ItemType + expect Format + }{ + {pageparser.TypeFrontMatterJSON, JSON}, + {pageparser.TypeFrontMatterTOML, TOML}, + {pageparser.TypeFrontMatterYAML, YAML}, + {pageparser.TypeFrontMatterORG, ORG}, + {pageparser.TypeIgnore, ""}, + } { + assert.Equal(test.expect, FormatFromFrontMatterType(test.typ), fmt.Sprintf("t%d", i)) + } +} diff --git a/parser/metadecoders/json.go b/parser/metadecoders/json.go deleted file mode 100644 index 21ca8a3b9c0..00000000000 --- a/parser/metadecoders/json.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package metadecoders - -import "encoding/json" - -// HandleJSONData unmarshals JSON-encoded datum and returns a Go interface -// representing the encoded data structure. -func HandleJSONData(datum []byte) (interface{}, error) { - if datum == nil { - // Package json returns on error on nil input. - // Return an empty map to be consistent with our other supported - // formats. - return make(map[string]interface{}), nil - } - - var f interface{} - err := json.Unmarshal(datum, &f) - return f, err -} diff --git a/parser/metadecoders/yaml.go b/parser/metadecoders/yaml.go deleted file mode 100644 index 21b23a9fd0a..00000000000 --- a/parser/metadecoders/yaml.go +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// The metadecoders package contains functions to decode metadata (e.g. page front matter) -// from different formats: TOML, YAML, JSON. -package metadecoders - -import ( - "fmt" - - "github.com/spf13/cast" - yaml "gopkg.in/yaml.v2" -) - -// HandleYAMLData unmarshals YAML-encoded datum and returns a Go interface -// representing the encoded data structure. -func HandleYAMLData(datum []byte) (interface{}, error) { - var m interface{} - err := yaml.Unmarshal(datum, &m) - if err != nil { - return nil, err - } - - // To support boolean keys, the `yaml` package unmarshals maps to - // map[interface{}]interface{}. Here we recurse through the result - // and change all maps to map[string]interface{} like we would've - // gotten from `json`. - if mm, changed := stringifyMapKeys(m); changed { - return mm, nil - } - - return m, nil -} - -// stringifyMapKeys recurses into in and changes all instances of -// map[interface{}]interface{} to map[string]interface{}. This is useful to -// work around the impedence mismatch between JSON and YAML unmarshaling that's -// described here: https://github.com/go-yaml/yaml/issues/139 -// -// Inspired by https://github.com/stripe/stripe-mock, MIT licensed -func stringifyMapKeys(in interface{}) (interface{}, bool) { - switch in := in.(type) { - case []interface{}: - for i, v := range in { - if vv, replaced := stringifyMapKeys(v); replaced { - in[i] = vv - } - } - case map[interface{}]interface{}: - res := make(map[string]interface{}) - var ( - ok bool - err error - ) - for k, v := range in { - var ks string - - if ks, ok = k.(string); !ok { - ks, err = cast.ToStringE(k) - if err != nil { - ks = fmt.Sprintf("%v", k) - } - } - if vv, replaced := stringifyMapKeys(v); replaced { - res[ks] = vv - } else { - res[ks] = v - } - } - return res, true - } - - return nil, false -} diff --git a/parser/page.go b/parser/page.go index 84a59f186d6..68a5426698f 100644 --- a/parser/page.go +++ b/parser/page.go @@ -101,15 +101,8 @@ func (p *page) IsRenderable() bool { // Metadata returns the unmarshalled frontmatter data. func (p *page) Metadata() (meta map[string]interface{}, err error) { - frontmatter := p.FrontMatter() - if len(frontmatter) != 0 { - fm := DetectFrontMatter(rune(frontmatter[0])) - if fm != nil { - meta, err = fm.Parse(frontmatter) - } - } - return + return nil, nil } // ReadFrom reads the content from an io.Reader and constructs a page. diff --git a/parser/page_test.go b/parser/page_test.go index 07d7660d427..0bfe2c257e4 100644 --- a/parser/page_test.go +++ b/parser/page_test.go @@ -1,130 +1 @@ package parser - -import ( - "fmt" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestPage(t *testing.T) { - cases := []struct { - raw string - - content string - frontmatter string - renderable bool - metadata map[string]interface{} - }{ - { - testPageLeader + jsonPageFrontMatter + "\n" + testPageTrailer + jsonPageContent, - jsonPageContent, - jsonPageFrontMatter, - true, - map[string]interface{}{ - "title": "JSON Test 1", - "social": []interface{}{ - []interface{}{"a", "#"}, - []interface{}{"b", "#"}, - }, - }, - }, - { - testPageLeader + tomlPageFrontMatter + testPageTrailer + tomlPageContent, - tomlPageContent, - tomlPageFrontMatter, - true, - map[string]interface{}{ - "title": "TOML Test 1", - "social": []interface{}{ - []interface{}{"a", "#"}, - []interface{}{"b", "#"}, - }, - }, - }, - { - testPageLeader + yamlPageFrontMatter + testPageTrailer + yamlPageContent, - yamlPageContent, - yamlPageFrontMatter, - true, - map[string]interface{}{ - "title": "YAML Test 1", - "social": []interface{}{ - []interface{}{"a", "#"}, - []interface{}{"b", "#"}, - }, - }, - }, - { - testPageLeader + orgPageFrontMatter + orgPageContent, - orgPageContent, - orgPageFrontMatter, - true, - map[string]interface{}{ - "TITLE": "Org Test 1", - "categories": []string{"a", "b"}, - }, - }, - } - - for i, c := range cases { - p := pageMust(ReadFrom(strings.NewReader(c.raw))) - meta, err := p.Metadata() - - mesg := fmt.Sprintf("[%d]", i) - - require.Nil(t, err, mesg) - assert.Equal(t, c.content, string(p.Content()), mesg+" content") - assert.Equal(t, c.frontmatter, string(p.FrontMatter()), mesg+" frontmatter") - assert.Equal(t, c.renderable, p.IsRenderable(), mesg+" renderable") - assert.Equal(t, c.metadata, meta, mesg+" metadata") - } -} - -var ( - testWhitespace = "\t\t\n\n" - testPageLeader = "\ufeff" + testWhitespace + "\n" - - jsonPageContent = "# JSON Test\n" - jsonPageFrontMatter = `{ - "title": "JSON Test 1", - "social": [ - ["a", "#"], - ["b", "#"] - ] -}` - - tomlPageContent = "# TOML Test\n" - tomlPageFrontMatter = `+++ -title = "TOML Test 1" -social = [ - ["a", "#"], - ["b", "#"], -] -+++ -` - - yamlPageContent = "# YAML Test\n" - yamlPageFrontMatter = `--- -title: YAML Test 1 -social: - - - "a" - - "#" - - - "b" - - "#" ---- -` - - orgPageContent = "* Org Test\n" - orgPageFrontMatter = `#+TITLE: Org Test 1 -#+categories: a b -` - - pageHTMLComment = ` -` -) diff --git a/parser/pageparser/item.go b/parser/pageparser/item.go index afc3b5fab32..c6f6c3f3860 100644 --- a/parser/pageparser/item.go +++ b/parser/pageparser/item.go @@ -20,7 +20,7 @@ import ( type Item struct { Type ItemType - pos pos + Pos Pos Val []byte } diff --git a/parser/pageparser/pagelexer.go b/parser/pageparser/pagelexer.go index a6a26016b01..d3fc11bf2c4 100644 --- a/parser/pageparser/pagelexer.go +++ b/parser/pageparser/pagelexer.go @@ -25,7 +25,7 @@ import ( ) // position (in bytes) -type pos int +type Pos int const eof = -1 @@ -47,9 +47,9 @@ type pageLexer struct { input []byte stateStart stateFunc state stateFunc - pos pos // input position - start pos // item start position - width pos // width of last element + pos Pos // input position + start Pos // item start position + width Pos // width of last element // Set when we have parsed any summary divider summaryDividerChecked bool @@ -73,7 +73,7 @@ func (l *pageLexer) Input() []byte { // note: the input position here is normally 0 (start), but // can be set if position of first shortcode is known // TODO(bep) 2errors byte -func newPageLexer(input []byte, inputPosition pos, stateStart stateFunc) *pageLexer { +func newPageLexer(input []byte, inputPosition Pos, stateStart stateFunc) *pageLexer { lexer := &pageLexer{ input: input, pos: inputPosition, @@ -131,7 +131,7 @@ func (l *pageLexer) next() rune { } runeValue, runeWidth := utf8.DecodeRune(l.input[l.pos:]) - l.width = pos(runeWidth) + l.width = Pos(runeWidth) l.pos += l.width return runeValue } @@ -210,7 +210,7 @@ func lexMainSection(l *pageLexer) stateFunc { l3 = l.index(leftDelimSc) skip := minPositiveIndex(l1, l2, l3) if skip > 0 { - l.pos += pos(skip) + l.pos += Pos(skip) } for { @@ -234,7 +234,7 @@ func lexMainSection(l *pageLexer) stateFunc { l.emit(tText) } l.summaryDividerChecked = true - l.pos += pos(len(summaryDivider)) + l.pos += Pos(len(summaryDivider)) //l.consumeCRLF() l.emit(TypeLeadSummaryDivider) } else if l.hasPrefix(summaryDividerOrg) { @@ -242,7 +242,7 @@ func lexMainSection(l *pageLexer) stateFunc { l.emit(tText) } l.summaryDividerChecked = true - l.pos += pos(len(summaryDividerOrg)) + l.pos += Pos(len(summaryDividerOrg)) //l.consumeCRLF() l.emit(TypeSummaryDividerOrg) } @@ -291,12 +291,12 @@ LOOP: if right == -1 { return l.errorf("starting HTML comment with no end") } - l.pos += pos(right) + pos(len(htmlCOmmentEnd)) + l.pos += Pos(right) + Pos(len(htmlCOmmentEnd)) l.emit(TypeHTMLComment) } else { // Not need to look further. Hugo treats this as plain HTML, // no front matter, no shortcodes, no nothing. - l.pos = pos(len(l.input)) + l.pos = Pos(len(l.input)) l.emit(TypeHTMLDocument) } } @@ -434,7 +434,7 @@ func (l *pageLexer) lexFrontMatterSection(tp ItemType, delimr rune, name string, } func lexShortcodeLeftDelim(l *pageLexer) stateFunc { - l.pos += pos(len(l.currentLeftShortcodeDelim())) + l.pos += Pos(len(l.currentLeftShortcodeDelim())) if l.hasPrefix(leftComment) { return lexShortcodeComment } @@ -451,20 +451,20 @@ func lexShortcodeComment(l *pageLexer) stateFunc { } // we emit all as text, except the comment markers l.emit(tText) - l.pos += pos(len(leftComment)) + l.pos += Pos(len(leftComment)) l.ignore() - l.pos += pos(posRightComment - len(leftComment)) + l.pos += Pos(posRightComment - len(leftComment)) l.emit(tText) - l.pos += pos(len(rightComment)) + l.pos += Pos(len(rightComment)) l.ignore() - l.pos += pos(len(l.currentRightShortcodeDelim())) + l.pos += Pos(len(l.currentRightShortcodeDelim())) l.emit(tText) return lexMainSection } func lexShortcodeRightDelim(l *pageLexer) stateFunc { l.closingState = 0 - l.pos += pos(len(l.currentRightShortcodeDelim())) + l.pos += Pos(len(l.currentRightShortcodeDelim())) l.emit(l.currentRightShortcodeDelimItem()) return lexMainSection } diff --git a/parser/pageparser/pageparser.go b/parser/pageparser/pageparser.go index bc6f55dd81c..0d32c0e89a3 100644 --- a/parser/pageparser/pageparser.go +++ b/parser/pageparser/pageparser.go @@ -48,7 +48,7 @@ func Parse(r io.Reader) (Result, error) { } func parseMainSection(input []byte, from int) Result { - lexer := newPageLexer(input, pos(from), lexMainSection) // TODO(bep) 2errors + lexer := newPageLexer(input, Pos(from), lexMainSection) // TODO(bep) 2errors lexer.run() return lexer } @@ -57,7 +57,7 @@ func parseMainSection(input []byte, from int) Result { // if needed. type Iterator struct { l *pageLexer - lastPos pos // position of the last item returned by nextItem + lastPos Pos // position of the last item returned by nextItem } // consumes and returns the next item @@ -69,7 +69,7 @@ func (t *Iterator) Next() Item { var errIndexOutOfBounds = Item{tError, 0, []byte("no more tokens")} func (t *Iterator) current() Item { - if t.lastPos >= pos(len(t.l.items)) { + if t.lastPos >= Pos(len(t.l.items)) { return errIndexOutOfBounds } return t.l.items[t.lastPos] @@ -98,7 +98,7 @@ func (t *Iterator) Peek() Item { // PeekWalk will feed the next items in the iterator to walkFn // until it returns false. func (t *Iterator) PeekWalk(walkFn func(item Item) bool) { - for i := t.lastPos + 1; i < pos(len(t.l.items)); i++ { + for i := t.lastPos + 1; i < Pos(len(t.l.items)); i++ { item := t.l.items[i] if !walkFn(item) { break @@ -120,5 +120,5 @@ func (t *Iterator) Consume(cnt int) { // LineNumber returns the current line number. Used for logging. func (t *Iterator) LineNumber() int { - return bytes.Count(t.l.input[:t.current().pos], lf) + 1 + return bytes.Count(t.l.input[:t.current().Pos], lf) + 1 } diff --git a/parser/pageparser/pageparser_intro_test.go b/parser/pageparser/pageparser_intro_test.go index 850254ac7b6..1a8c2d23775 100644 --- a/parser/pageparser/pageparser_intro_test.go +++ b/parser/pageparser/pageparser_intro_test.go @@ -59,9 +59,7 @@ var frontMatterTests = []lexerTest{ {"No front matter", "\nSome text.\n", []Item{tstSomeText, tstEOF}}, {"YAML front matter", "---\nfoo: \"bar\"\n---\n\nSome text.\n", []Item{tstFrontMatterYAML, tstSomeText, tstEOF}}, {"YAML empty front matter", "---\n---\n\nSome text.\n", []Item{nti(TypeFrontMatterYAML, "\n"), tstSomeText, tstEOF}}, - {"YAML commented out front matter", "\nSome text.\n", []Item{nti(TypeHTMLComment, ""), tstSomeText, tstEOF}}, - // Note that we keep all bytes as they are, but we need to handle CRLF {"YAML front matter CRLF", "---\r\nfoo: \"bar\"\r\n---\n\nSome text.\n", []Item{tstFrontMatterYAMLCRLF, tstSomeText, tstEOF}}, {"TOML front matter", "+++\nfoo = \"bar\"\n+++\n\nSome text.\n", []Item{tstFrontMatterTOML, tstSomeText, tstEOF}}, diff --git a/tpl/transform/remarshal.go b/tpl/transform/remarshal.go index 490def5f30f..d5fe96ac69a 100644 --- a/tpl/transform/remarshal.go +++ b/tpl/transform/remarshal.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/gohugoio/hugo/parser" + "github.com/gohugoio/hugo/parser/metadecoders" "github.com/spf13/cast" ) @@ -38,21 +39,7 @@ func (ns *Namespace) Remarshal(format string, data interface{}) (string, error) return "", err } - var metaHandler func(d []byte) (map[string]interface{}, error) - - switch fromFormat { - case "yaml": - metaHandler = parser.HandleYAMLMetaData - case "toml": - metaHandler = parser.HandleTOMLMetaData - case "json": - metaHandler = parser.HandleJSONMetaData - } - - meta, err := metaHandler([]byte(from)) - if err != nil { - return "", err - } + meta, err := metadecoders.UnmarshalToMap([]byte(from), fromFormat) var result bytes.Buffer if err := parser.InterfaceToConfig(meta, mark, &result); err != nil { @@ -76,21 +63,21 @@ func toFormatMark(format string) (rune, error) { return 0, errors.New("failed to detect target data serialization format") } -func detectFormat(data string) (string, error) { +func detectFormat(data string) (metadecoders.Format, error) { jsonIdx := strings.Index(data, "{") yamlIdx := strings.Index(data, ":") tomlIdx := strings.Index(data, "=") if jsonIdx != -1 && (yamlIdx == -1 || jsonIdx < yamlIdx) && (tomlIdx == -1 || jsonIdx < tomlIdx) { - return "json", nil + return metadecoders.JSON, nil } if yamlIdx != -1 && (tomlIdx == -1 || yamlIdx < tomlIdx) { - return "yaml", nil + return metadecoders.YAML, nil } if tomlIdx != -1 { - return "toml", nil + return metadecoders.TOML, nil } return "", errors.New("failed to detect data serialization format") diff --git a/tpl/transform/remarshal_test.go b/tpl/transform/remarshal_test.go index 07c51c3b097..1416afff356 100644 --- a/tpl/transform/remarshal_test.go +++ b/tpl/transform/remarshal_test.go @@ -18,6 +18,7 @@ import ( "testing" "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/parser/metadecoders" "github.com/spf13/viper" "github.com/stretchr/testify/require" ) @@ -179,12 +180,12 @@ func TestRemarshalDetectFormat(t *testing.T) { data string expect interface{} }{ - {`foo = "bar"`, "toml"}, - {` foo = "bar"`, "toml"}, - {`foo="bar"`, "toml"}, - {`foo: "bar"`, "yaml"}, - {`foo:"bar"`, "yaml"}, - {`{ "foo": "bar"`, "json"}, + {`foo = "bar"`, metadecoders.TOML}, + {` foo = "bar"`, metadecoders.TOML}, + {`foo="bar"`, metadecoders.TOML}, + {`foo: "bar"`, metadecoders.YAML}, + {`foo:"bar"`, metadecoders.YAML}, + {`{ "foo": "bar"`, metadecoders.JSON}, {`asdfasdf`, false}, {``, false}, } { @@ -198,6 +199,6 @@ func TestRemarshalDetectFormat(t *testing.T) { } assert.NoError(err, errMsg) - assert.Equal(test.expect, result, errMsg) + assert.Equal(test.expect, result) } }