Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Work In Progress: hugolib: Extract date and slug from filename #4436

Closed
wants to merge 13 commits into from
Closed

Work In Progress: hugolib: Extract date and slug from filename #4436

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
22c493f
hugolib: Extract date and slug from filename
bep Feb 21, 2018
1d9e1e8
Work
bep Feb 21, 2018
f3db894
Work
bep Feb 22, 2018
8d07454
Work
bep Feb 22, 2018
69b829e
Work
bep Feb 22, 2018
51574fd
Work
bep Feb 22, 2018
3d6d4eb
Working
bep Feb 22, 2018
2b61ea7
Rename
bep Feb 22, 2018
69b97b2
Adjust pubdate
bep Feb 23, 2018
247a82f
Test work
bep Feb 23, 2018
0bb9de7
More
bep Feb 23, 2018
352e787
More
bep Feb 24, 2018
dbd5c97
Move to pagemeta package
bep Feb 24, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
129 changes: 41 additions & 88 deletions hugolib/page.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2016 The Hugo Authors. All rights reserved.
// Copyright 2018 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand All @@ -25,6 +25,7 @@ import (
"github.com/bep/gitmap"

"github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/hugolib/pagemeta"
"github.com/gohugoio/hugo/resource"

"github.com/gohugoio/hugo/output"
Expand Down Expand Up @@ -140,9 +141,6 @@ type Page struct {
Draft bool
Status string

PublishDate time.Time
ExpiryDate time.Time

// PageMeta contains page stats such as word count etc.
PageMeta

Expand Down Expand Up @@ -223,11 +221,12 @@ type Page struct {
Keywords []string
Data map[string]interface{}

Date time.Time
Lastmod time.Time
pagemeta.PageDates

Sitemap Sitemap
URLPath
pagemeta.URLPath
frontMatterURL string

permalink string
relPermalink string

Expand Down Expand Up @@ -1115,12 +1114,44 @@ func (p *Page) update(frontmatter map[string]interface{}) error {
// Needed for case insensitive fetching of params values
helpers.ToLowerMap(frontmatter)

var modified time.Time
var mtime time.Time
if p.Source.FileInfo() != nil {
mtime = p.Source.FileInfo().ModTime()
}

descriptor := pagemeta.FrontMatterDescriptor{
Frontmatter: frontmatter,
Params: p.params,
Dates: &p.PageDates,
PageURLs: &p.URLPath,
BaseFilename: p.BaseFileName(),
ModTime: mtime}

// Handle the date separately
// TODO(bep) we need to "do more" in this area so this can be split up and
// more easily tested without the Page, but the coupling is strong.
err := p.s.frontmatterHandler.HandleDates(descriptor)
if err != nil {
p.s.Log.ERROR.Printf("Failed to handle dates for page %q: %s", p.Path(), err)
}

var err error
var draft, published, isCJKLanguage *bool
for k, v := range frontmatter {
loki := strings.ToLower(k)

if loki == "published" { // Intentionally undocumented
vv, err := cast.ToBoolE(v)
if err == nil {
published = &vv
}
// published may also be a date
continue
}

if p.s.frontmatterHandler.IsDateKey(loki) {
continue
}

switch loki {
case "title":
p.title = cast.ToString(v)
Expand All @@ -1139,7 +1170,7 @@ func (p *Page) update(frontmatter map[string]interface{}) error {
return fmt.Errorf("Only relative URLs are supported, %v provided", url)
}
p.URLPath.URL = cast.ToString(v)
p.URLPath.frontMatterURL = p.URLPath.URL
p.frontMatterURL = p.URLPath.URL
p.params[loki] = p.URLPath.URL
case "type":
p.contentType = cast.ToString(v)
Expand All @@ -1150,32 +1181,13 @@ func (p *Page) update(frontmatter map[string]interface{}) error {
case "keywords":
p.Keywords = cast.ToStringSlice(v)
p.params[loki] = p.Keywords
case "date":
p.Date, err = cast.ToTimeE(v)
if err != nil {
p.s.Log.ERROR.Printf("Failed to parse date '%v' in page %s", v, p.File.Path())
}
p.params[loki] = p.Date
case "headless":
// For now, only the leaf bundles ("index.md") can be headless (i.e. produce no output).
// We may expand on this in the future, but that gets more complex pretty fast.
if p.TranslationBaseName() == "index" {
p.headless = cast.ToBool(v)
}
p.params[loki] = p.headless
case "lastmod":
p.Lastmod, err = cast.ToTimeE(v)
if err != nil {
p.s.Log.ERROR.Printf("Failed to parse lastmod '%v' in page %s", v, p.File.Path())
}
case "modified":
vv, err := cast.ToTimeE(v)
if err == nil {
p.params[loki] = vv
modified = vv
} else {
p.params[loki] = cast.ToString(v)
}
case "outputs":
o := cast.ToStringSlice(v)
if len(o) > 0 {
Expand All @@ -1190,34 +1202,9 @@ func (p *Page) update(frontmatter map[string]interface{}) error {
}

}
case "publishdate", "pubdate":
p.PublishDate, err = cast.ToTimeE(v)
if err != nil {
p.s.Log.ERROR.Printf("Failed to parse publishdate '%v' in page %s", v, p.File.Path())
}
p.params[loki] = p.PublishDate
case "expirydate", "unpublishdate":
p.ExpiryDate, err = cast.ToTimeE(v)
if err != nil {
p.s.Log.ERROR.Printf("Failed to parse expirydate '%v' in page %s", v, p.File.Path())
}
case "draft":
draft = new(bool)
*draft = cast.ToBool(v)
case "published": // Intentionally undocumented
vv, err := cast.ToBoolE(v)
if err == nil {
published = &vv
} else {
// Some sites use this as the publishdate
vv, err := cast.ToTimeE(v)
if err == nil {
p.PublishDate = vv
p.params[loki] = p.PublishDate
} else {
p.params[loki] = cast.ToString(v)
}
}
case "layout":
p.Layout = cast.ToString(v)
p.params[loki] = p.Layout
Expand Down Expand Up @@ -1333,32 +1320,6 @@ func (p *Page) update(frontmatter map[string]interface{}) error {
}
p.params["draft"] = p.Draft

if p.Date.IsZero() {
p.Date = p.PublishDate
}

if p.PublishDate.IsZero() {
p.PublishDate = p.Date
}

if p.Date.IsZero() && p.s.Cfg.GetBool("useModTimeAsFallback") {
p.Date = p.Source.FileInfo().ModTime()
}

if p.Lastmod.IsZero() {
if !modified.IsZero() {
p.Lastmod = modified
} else {
p.Lastmod = p.Date
}

}

p.params["date"] = p.Date
p.params["lastmod"] = p.Lastmod
p.params["publishdate"] = p.PublishDate
p.params["expirydate"] = p.ExpiryDate

if isCJKLanguage != nil {
p.isCJKLanguage = *isCJKLanguage
} else if p.s.Cfg.GetBool("hasCJKLanguage") {
Expand Down Expand Up @@ -1865,14 +1826,6 @@ func (p *Page) String() string {
return fmt.Sprintf("Page(%q)", p.title)
}

type URLPath struct {
URL string
frontMatterURL string
Permalink string
Slug string
Section string
}

// Scratch returns the writable context associated with this Page.
func (p *Page) Scratch() *Scratch {
if p.scratch == nil {
Expand Down
2 changes: 1 addition & 1 deletion hugolib/page_paths.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ func (p *Page) initTargetPathDescriptor() error {
Sections: p.sections,
UglyURLs: p.s.Info.uglyURLs(p),
Dir: filepath.ToSlash(p.Source.Dir()),
URL: p.URLPath.frontMatterURL,
URL: p.frontMatterURL,
IsMultihost: p.s.owner.IsMultihost(),
}

Expand Down
109 changes: 83 additions & 26 deletions hugolib/page_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2015 The Hugo Authors. All rights reserved.
// Copyright 2018 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -27,8 +27,6 @@ import (

"github.com/gohugoio/hugo/deps"
"github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/hugofs"
"github.com/gohugoio/hugo/source"
"github.com/spf13/cast"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
Expand Down Expand Up @@ -728,6 +726,7 @@ func TestPageWithDelimiterForMarkdownThatCrossesBorder(t *testing.T) {
}

// Issue #3854
// Also see https://github.com/gohugoio/hugo/issues/3977
func TestPageWithDateFields(t *testing.T) {
assert := require.New(t)
pageWithDate := `---
Expand All @@ -737,8 +736,8 @@ weight: %d
---
Simple Page With Some Date`

hasBothDates := func(p *Page) bool {
return p.Date.Year() == 2017 && p.PublishDate.Year() == 2017
hasDate := func(p *Page) bool {
return p.Date.Year() == 2017
}

datePage := func(field string, weight int) string {
Expand All @@ -749,7 +748,7 @@ Simple Page With Some Date`
assertFunc := func(t *testing.T, ext string, pages Pages) {
assert.True(len(pages) > 0)
for _, p := range pages {
assert.True(hasBothDates(p))
assert.True(hasDate(p))
}

}
Expand Down Expand Up @@ -985,8 +984,64 @@ Page With empty front matter`
zero_FM = "Page With empty front matter"
)

func TestPageWithFilenameDateAsFallback(t *testing.T) {
t.Parallel()

for _, useFilename := range []bool{false, true} {
t.Run(fmt.Sprintf("useFilename=%t", useFilename), func(t *testing.T) {
ass := require.New(t)
cfg, fs := newTestCfg()

pageTemplate := `
---
title: Page
weight: %d
%s
---
Content
`

if useFilename {
cfg.Set("frontmatter", map[string]interface{}{
"defaultDate": []string{"filename"},
})
}

writeSource(t, fs, filepath.Join("content", "section", "2012-02-21-noslug.md"), fmt.Sprintf(pageTemplate, 1, ""))
writeSource(t, fs, filepath.Join("content", "section", "2012-02-22-slug.md"), fmt.Sprintf(pageTemplate, 2, "slug: aslug"))

s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{SkipRender: true})

ass.Len(s.RegularPages, 2)

noSlug := s.RegularPages[0]
slug := s.RegularPages[1]

if useFilename {
ass.False(noSlug.Date.IsZero())
ass.False(slug.Date.IsZero())
ass.Equal(2012, noSlug.Date.Year())
ass.Equal(2012, slug.Date.Year())
ass.Equal("noslug", noSlug.Slug)
ass.Equal("aslug", slug.Slug)

} else {
ass.True(noSlug.Date.IsZero())
ass.True(slug.Date.IsZero())
ass.Equal("", noSlug.Slug)
ass.Equal("aslug", slug.Slug)
}

})
}

}

func TestMetadataDates(t *testing.T) {
t.Parallel()

assert := require.New(t)

var tests = []struct {
text string
filename string
Expand Down Expand Up @@ -1016,41 +1071,40 @@ func TestMetadataDates(t *testing.T) {
//
// ------- inputs --------|--- outputs ---|
//content filename modfb? D P L M E
{p_D____, "test.md", false, D, D, D, x, x}, // date copied across
{p_D____, "testy.md", true, D, D, D, x, x},
{p_D____, "test.md", false, D, o, D, x, x},
{p_D____, "testy.md", true, D, o, D, x, x},
{p__P___, "test.md", false, P, P, P, x, x}, // pubdate copied across
{p__P___, "testy.md", true, P, P, P, x, x},
//{p__P___, "testy.md", true, P, P, P, x, x}, // TODO(bep) date from modTime
{p_DP___, "test.md", false, D, P, D, x, x}, // date -> lastMod
{p_DP___, "testy.md", true, D, P, D, x, x},
{p_DP___, "testy.md", true, D, P, D, x, x}, // TODO(bep) date from modTime
{p__PL__, "test.md", false, P, P, L, x, x}, // pub -> date overrides lastMod -> date code (inconsistent?)
{p__PL__, "testy.md", true, P, P, L, x, x},
//{p__PL__, "testy.md", true, P, P, L, x, x},
{p_DPL__, "test.md", false, D, P, L, x, x}, // three dates
{p_DPL__, "testy.md", true, D, P, L, x, x},
{p_DPL_E, "testy.md", true, D, P, L, x, E}, // lastMod NOT copied to mod (inconsistent?)
{p_DP_ME, "testy.md", true, D, P, M, M, E}, // mod copied to lastMod
{p_DPLME, "testy.md", true, D, P, L, M, E}, // all dates

{emptyFM, "test.md", false, o, o, o, x, x}, // 3 year-one dates, 2 empty dates
{zero_FM, "test.md", false, o, o, o, x, x},
{emptyFM, "testy.md", true, s, o, s, x, x}, // 2 filesys, 1 year-one, 2 empty
{zero_FM, "testy.md", true, s, o, s, x, x},
//{emptyFM, "testy.md", true, s, o, s, x, x}, // 2 filesys, 1 year-one, 2 empty TODO(bep) date from modTime
//{zero_FM, "testy.md", true, s, o, s, x, x}, // TODO(bep) date from modTime
}

for i, test := range tests {
s := newTestSite(t)
s.Cfg.Set("useModTimeAsFallback", test.modFallback)
fs := hugofs.NewMem(s.Cfg)

writeToFs(t, fs.Source, test.filename, test.text)
file, err := fs.Source.Open(test.filename)
if err != nil {
t.Fatal("failed to write test file to test filesystem")
}
fi, _ := fs.Source.Stat(test.filename)
var (
cfg, fs = newTestCfg()
)

writeToFs(t, fs.Source, filepath.Join("content", test.filename), test.text)

cfg.Set("useModTimeAsFallback", test.modFallback)

s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{SkipRender: true})

sp := source.NewSourceSpec(s.Cfg, fs)
p := s.newPageFromFile(newFileInfo(sp, "", test.filename, fi, bundleNot))
p.ReadFrom(file)
assert.Equal(1, len(s.RegularPages))
p := s.RegularPages[0]
fi := p.Source.FileInfo()

// check Page Variables
checkDate(t, i+1, "Date", p.Date, test.expDate, fi)
Expand All @@ -1059,6 +1113,9 @@ func TestMetadataDates(t *testing.T) {
checkDate(t, i+1, "LastMod", p.ExpiryDate, test.expExp, fi)

// check Page Params
// TODO(bep) we need to rewrite these date tests to more unit style.
// The params checks below are currently flawed, as they don't check for the
// absense (nil) of a date.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I fixed this in PR #4340. Please see my comment on why you really should just merge it.

I'm not sure what you mean by "more unit style". I don't think you mean tests of smaller "units" such as an individual functions. What I did with these tests is to cover "the complex" that governs date values (front matter, filenames, file timestamps)... the "unit" is the blackbox the is Hugo's content processing. Or perhaps the table of inputs and outputs don't seem unit style to you? But that table is the data for data-driven unit tests. Inputs and outputs stripped of all the scaffolding code.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A creating a site, then do a full build, and check the result will by most people categorised as an integration test. We do that a lot in Hugo, because building all is often cheaper/easier than creating testable and smaller units. My new frontmatterHandler has no dependencies to Page or Site and can be tested as an unit.

checkDate(t, i+1, "param date", cast.ToTime(p.params["date"]), test.expDate, fi)
checkDate(t, i+1, "param publishdate", cast.ToTime(p.params["publishdate"]), test.expPub, fi)
checkDate(t, i+1, "param modified", cast.ToTime(p.params["modified"]), test.expMod, fi)
Expand Down
Loading