Skip to content

Commit

Permalink
parser/metadecoders: Add CSV lazyQuotes option to transform.Unmarshal
Browse files Browse the repository at this point in the history
If true, a quote may appear in an unquoted field and a non-doubled quote
may appear in a quoted field. It defaults to false.

Closes #11884
  • Loading branch information
jmooring authored and bep committed Jan 16, 2024
1 parent 911bc60 commit 912c657
Show file tree
Hide file tree
Showing 3 changed files with 36 additions and 0 deletions.
3 changes: 3 additions & 0 deletions docs/content/en/functions/transform/Unmarshal.md
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,9 @@ delimiter
comment
: (`string`) The comment character used in the CSV. If set, lines beginning with the comment character without preceding whitespace are ignored.

lazyQuotes
: (`bool`) If true, a quote may appear in an unquoted field and a non-doubled quote may appear in a quoted field. Default is `false`.

```go-html-template
{{ $csv := "a;b;c" | transform.Unmarshal (dict "delimiter" ";") }}
```
Expand Down
7 changes: 7 additions & 0 deletions parser/metadecoders/decoder.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"fmt"
"log"
"regexp"
"strconv"
"strings"

"github.com/gohugoio/hugo/common/herrors"
Expand All @@ -41,13 +42,18 @@ type Decoder struct {
// Comment, if not 0, is the comment character used in the CSV decoder. Lines beginning with the
// Comment character without preceding whitespace are ignored.
Comment rune

// If true, a quote may appear in an unquoted field and a non-doubled quote
// may appear in a quoted field. It defaults to false.
LazyQuotes bool
}

// OptionsKey is used in cache keys.
func (d Decoder) OptionsKey() string {
var sb strings.Builder
sb.WriteRune(d.Delimiter)
sb.WriteRune(d.Comment)
sb.WriteString(strconv.FormatBool(d.LazyQuotes))
return sb.String()
}

Expand Down Expand Up @@ -205,6 +211,7 @@ func (d Decoder) unmarshalCSV(data []byte, v any) error {
r := csv.NewReader(bytes.NewReader(data))
r.Comma = d.Delimiter
r.Comment = d.Comment
r.LazyQuotes = d.LazyQuotes

records, err := r.ReadAll()
if err != nil {
Expand Down
26 changes: 26 additions & 0 deletions tpl/transform/integration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -107,3 +107,29 @@ disableKinds = ['page','rss','section','sitemap','taxonomy','term']
_, err := b.BuildE()
b.Assert(err.Error(), qt.Contains, "error calling highlight: invalid Highlight option: 0")
}

// Issue #11884
func TestUnmarshalCSVLazyDecoding(t *testing.T) {
t.Parallel()

files := `
-- hugo.toml --
disableKinds = ['page','rss','section','sitemap','taxonomy','term']
-- assets/pets.csv --
name,description,age
Spot,a nice dog,3
Rover,"a big dog",5
Felix,a "malicious" cat,7
Bella,"an "evil" cat",9
Scar,"a "dead cat",11
-- layouts/index.html --
{{ $opts := dict "lazyQuotes" true }}
{{ $data := resources.Get "pets.csv" | transform.Unmarshal $opts }}
{{ printf "%v" $data | safeHTML }}
`
b := hugolib.Test(t, files)

b.AssertFileContent("public/index.html", `
[[name description age] [Spot a nice dog 3] [Rover a big dog 5] [Felix a "malicious" cat 7] [Bella an "evil" cat 9] [Scar a "dead cat 11]]
`)
}

0 comments on commit 912c657

Please sign in to comment.