Skip to content

Commit

Permalink
Implement resources.FromRemote
Browse files Browse the repository at this point in the history 
Closes gohugoio#5255
Supports gohugoio#9044
  • Loading branch information
@vanbroup
vanbroup committed Nov 18, 2021
1 parent f676b47 commit 188e8c7
Show file tree
Hide file tree
Showing 4 changed files with 239 additions and 8 deletions.
102 changes: 94 additions & 8 deletions hugolib/resource_chain_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ import (
"fmt"
"io"
"math/rand"
"net/http"
"net/http/httptest"
"os"

"github.com/gohugoio/hugo/config"
Expand All @@ -35,6 +37,7 @@ import (

"github.com/gohugoio/hugo/common/herrors"

"github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/htesting"

qt "github.com/frankban/quicktest"
Expand Down Expand Up @@ -385,8 +388,11 @@ T1: {{ $r.Content }}
func TestResourceChainBasic(t *testing.T) {
t.Parallel()

ts := httptest.NewServer(http.FileServer(http.Dir("testdata/")))
defer ts.Close()

b := newTestSitesBuilder(t)
b.WithTemplatesAdded("index.html", `
b.WithTemplatesAdded("index.html", fmt.Sprintf(`
{{ $hello := "<h1> Hello World! </h1>" | resources.FromString "hello.html" | fingerprint "sha512" | minify | fingerprint }}
{{ $cssFingerprinted1 := "body { background-color: lightblue; }" | resources.FromString "styles.css" | minify | fingerprint }}
{{ $cssFingerprinted2 := "body { background-color: orange; }" | resources.FromString "styles2.css" | minify | fingerprint }}
Expand All @@ -403,7 +409,14 @@ FIT: {{ $fit.Name }}|{{ $fit.RelPermalink }}|{{ $fit.Width }}
CSS integrity Data first: {{ $cssFingerprinted1.Data.Integrity }} {{ $cssFingerprinted1.RelPermalink }}
CSS integrity Data last: {{ $cssFingerprinted2.RelPermalink }} {{ $cssFingerprinted2.Data.Integrity }}
`)
{{ $rimg := resources.FromRemote "%[1]s/sunset.jpg" }}
{{ $rfit := $rimg.Fit "200x200" }}
{{ $rfit2 := $rfit.Fit "100x200" }}
{{ $rimg = $rimg | fingerprint }}
SUNSET REMOTE: {{ $rimg.Name }}|{{ $rimg.RelPermalink }}|{{ $rimg.Width }}|{{ len $rimg.Content }}
FIT REMOTE: {{ $rfit.Name }}|{{ $rfit.RelPermalink }}|{{ $rfit.Width }}
`, ts.URL))

fs := b.Fs.Source

Expand All @@ -424,12 +437,16 @@ CSS integrity Data last: {{ $cssFingerprinted2.RelPermalink }} {{ $cssFingerpri
b.Build(BuildCfg{})

b.AssertFileContent("public/index.html",
`
fmt.Sprintf(`
SUNSET: images/sunset.jpg|/images/sunset.a9bf1d944e19c0f382e0d8f51de690f7d0bc8fa97390c4242a86c3e5c0737e71.jpg|900|90587
FIT: images/sunset.jpg|/images/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_200x200_fit_q75_box.jpg|200
CSS integrity Data first: sha256-od9YaHw8nMOL8mUy97Sy8sKwMV3N4hI3aVmZXATxH&#43;8= /styles.min.a1df58687c3c9cc38bf26532f7b4b2f2c2b0315dcde212376959995c04f11fef.css
CSS integrity Data last: /styles2.min.1cfc52986836405d37f9998a63fd6dd8608e8c410e5e3db1daaa30f78bc273ba.css sha256-HPxSmGg2QF03&#43;ZmKY/1t2GCOjEEOXj2x2qow94vCc7o=
`)
SUNSET REMOTE: %[1]s.jpg|/%[1]s.a9bf1d944e19c0f382e0d8f51de690f7d0bc8fa97390c4242a86c3e5c0737e71.jpg|900|90587
FIT REMOTE: %[1]s.jpg|/%[1]s_hu59e56ffff1bc1d8d122b1403d34e039f_09044_200x200_fit_q75_box.jpg|200
`, helpers.MD5String(ts.URL+"/sunset.jpg")))

b.AssertFileContent("public/styles.min.a1df58687c3c9cc38bf26532f7b4b2f2c2b0315dcde212376959995c04f11fef.css", "body{background-color:#add8e6}")
b.AssertFileContent("public//styles2.min.1cfc52986836405d37f9998a63fd6dd8608e8c410e5e3db1daaa30f78bc273ba.css", "body{background-color:orange}")
Expand Down Expand Up @@ -537,6 +554,65 @@ func TestResourceChains(t *testing.T) {

c := qt.New(t)

ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/css/styles1.css":
w.Header().Set("Content-Type", "text/css")
w.Write([]byte(`h1 {
font-style: bold;
}`))
return

case "/js/script1.js":
w.Write([]byte(`var x; x = 5, document.getElementById("demo").innerHTML = x * 10`))
return

case "/mydata/json1.json":
w.Write([]byte(`{
"employees": [
{
"firstName": "John",
"lastName": "Doe"
},
{
"firstName": "Anna",
"lastName": "Smith"
},
{
"firstName": "Peter",
"lastName": "Jones"
}
]
}`))
return

case "/mydata/xml1.xml":
w.Write([]byte(`
<hello>
<world>Hugo Rocks!</<world>
</hello>`))
return

case "/mydata/svg1.svg":
w.Write([]byte(`
<svg height="100" width="100">
<path d="M1e2 1e2H3e2 2e2z"/>
</svg>`))
return

case "/mydata/html1.html":
w.Write([]byte(`
<html>
<a href=#>Cool</a>
</html>`))
return
}

http.Error(w, "Not found", http.StatusNotFound)
return
}))
defer ts.Close()

tests := []struct {
name string
shouldRun func() bool
Expand Down Expand Up @@ -578,25 +654,35 @@ T6: {{ $bundle1.Permalink }}
[minify.tdewolff.html]
keepWhitespace = false
`)
b.WithTemplates("home.html", `
b.WithTemplates("home.html", fmt.Sprintf(`
Min CSS: {{ ( resources.Get "css/styles1.css" | minify ).Content }}
Min CSS Remote: {{ ( resources.FromRemote "%[1]s/css/styles1.css" | minify ).Content }}
Min JS: {{ ( resources.Get "js/script1.js" | resources.Minify ).Content | safeJS }}
Min JS Remote: {{ ( resources.FromRemote "%[1]s/js/script1.js" | minify ).Content }}
Min JSON: {{ ( resources.Get "mydata/json1.json" | resources.Minify ).Content | safeHTML }}
Min JSON Remote: {{ ( resources.FromRemote "%[1]s/mydata/json1.json" | resources.Minify ).Content | safeHTML }}
Min XML: {{ ( resources.Get "mydata/xml1.xml" | resources.Minify ).Content | safeHTML }}
Min XML Remote: {{ ( resources.FromRemote "%[1]s/mydata/xml1.xml" | resources.Minify ).Content | safeHTML }}
Min SVG: {{ ( resources.Get "mydata/svg1.svg" | resources.Minify ).Content | safeHTML }}
Min SVG Remote: {{ ( resources.FromRemote "%[1]s/mydata/svg1.svg" | resources.Minify ).Content | safeHTML }}
Min SVG again: {{ ( resources.Get "mydata/svg1.svg" | resources.Minify ).Content | safeHTML }}
Min HTML: {{ ( resources.Get "mydata/html1.html" | resources.Minify ).Content | safeHTML }}
`)
Min HTML Remote: {{ ( resources.FromRemote "%[1]s/mydata/html1.html" | resources.Minify ).Content | safeHTML }}
`, ts.URL))
}, func(b *sitesBuilder) {
b.AssertFileContent("public/index.html", `Min CSS: h1{font-style:bold}`)
b.AssertFileContent("public/index.html", `Min CSS Remote: h1{font-style:bold}`)
b.AssertFileContent("public/index.html", `Min JS: var x;x=5,document.getElementById(&#34;demo&#34;).innerHTML=x*10`)
b.AssertFileContent("public/index.html", `Min JS Remote: var x;x=5,document.getElementById(&#34;demo&#34;).innerHTML=x*10`)
b.AssertFileContent("public/index.html", `Min JSON: {"employees":[{"firstName":"John","lastName":"Doe"},{"firstName":"Anna","lastName":"Smith"},{"firstName":"Peter","lastName":"Jones"}]}`)
b.AssertFileContent("public/index.html", `Min JSON Remote: {"employees":[{"firstName":"John","lastName":"Doe"},{"firstName":"Anna","lastName":"Smith"},{"firstName":"Peter","lastName":"Jones"}]}`)
b.AssertFileContent("public/index.html", `Min XML: <hello><world>Hugo Rocks!</<world></hello>`)
b.AssertFileContent("public/index.html", `Min XML Remote: <hello><world>Hugo Rocks!</<world></hello>`)
b.AssertFileContent("public/index.html", `Min SVG: <svg height="100" width="100"><path d="M1e2 1e2H3e2 2e2z"/></svg>`)
b.AssertFileContent("public/index.html", `Min SVG Remote: <svg height="100" width="100"><path d="M1e2 1e2H3e2 2e2z"/></svg>`)
b.AssertFileContent("public/index.html", `Min SVG again: <svg height="100" width="100"><path d="M1e2 1e2H3e2 2e2z"/></svg>`)
b.AssertFileContent("public/index.html", `Min HTML: <html><a href=#>Cool</a></html>`)
b.AssertFileContent("public/index.html", `Min HTML Remote: <html><a href=#>Cool</a></html>`)
}},

{"concat", func() bool { return true }, func(b *sitesBuilder) {
Expand Down
135 changes: 135 additions & 0 deletions resources/resource_factories/create/create.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,11 @@
package create

import (
"bytes"
"io/ioutil"
"mime"
"net/http"
"net/url"
"path"
"path/filepath"
"strings"
Expand All @@ -25,8 +30,14 @@ import (
"github.com/gohugoio/hugo/hugofs"

"github.com/gohugoio/hugo/common/hugio"
"github.com/gohugoio/hugo/common/maps"
"github.com/gohugoio/hugo/common/types"
"github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/resources"
"github.com/gohugoio/hugo/resources/resource"

"github.com/pkg/errors"
"github.com/spf13/cast"
)

// Client contains methods to create Resource objects.
Expand Down Expand Up @@ -126,3 +137,127 @@ func (c *Client) FromString(targetPath, content string) (resource.Resource, erro
})
})
}

// FromRemote expects one or n-parts of a URL to a resource
// If you provide multiple parts they will be joined together to the final URL.
func (c *Client) FromRemote(args ...interface{}) (resource.Resource, error) {
uri, headers := toURLAndHeaders(args)
resourceID := helpers.MD5String(uri)

return c.rs.ResourceCache.GetOrCreate(path.Join(resources.CACHE_OTHER, resourceID), func() (resource.Resource, error) {
req, err := http.NewRequest("GET", uri, nil)
if err != nil {
return nil, errors.Wrapf(err, "Failed to create request for resource %s", uri)
}
addUserProvidedHeaders(headers, req)
res, err := c.rs.HTTPClient.Do(req)
if err != nil {
return nil, err
}

if res.StatusCode < 200 || res.StatusCode > 299 {
return nil, errors.Errorf("Failed to retrieve remote resource: %s", http.StatusText(res.StatusCode))
}

body, err := ioutil.ReadAll(res.Body)
if err != nil {
return nil, errors.Wrapf(err, "Failed to read remote resource %s", uri)
}

var contentType string
if arr, _ := mime.ExtensionsByType(res.Header.Get("Content-Type")); len(arr) == 1 {
contentType = arr[0]
}

// If content type was not determined by header, look for a file extention
if contentType == "" {
if u, err := url.Parse(uri); err == nil {
c.rs.Logger.Printf("uri parsed %#v", u)
if ext := filepath.Ext(u.Path); ext != "" {
contentType = ext
}
}
}

// If content type was not determined by header or file extention, try using content itself
if contentType == "" {
if ct := http.DetectContentType(body); ct != "application/octet-stream" {
if arr, _ := mime.ExtensionsByType(ct); arr != nil {
contentType = arr[0]
}
}
}

resourceID = resourceID + contentType

return c.rs.New(
resources.ResourceSourceDescriptor{
Fs: c.rs.FileCaches.AssetsCache().Fs,
LazyPublish: true,
OpenReadSeekCloser: func() (hugio.ReadSeekCloser, error) {
return hugio.NewReadSeekerNoOpCloser(bytes.NewReader(body)), nil
},
RelTargetFilename: filepath.Clean(resourceID),
})
})
}

func addDefaultHeaders(req *http.Request, accepts ...string) {
for _, accept := range accepts {
if !hasHeaderValue(req.Header, "Accept", accept) {
req.Header.Add("Accept", accept)
}
}
if !hasHeaderKey(req.Header, "User-Agent") {
req.Header.Add("User-Agent", "Hugo Static Site Generator")
}
}

func addUserProvidedHeaders(headers map[string]interface{}, req *http.Request) {
if headers == nil {
return
}
for key, val := range headers {
vals := types.ToStringSlicePreserveString(val)
for _, s := range vals {
req.Header.Add(key, s)
}
}
}

func hasHeaderValue(m http.Header, key, value string) bool {
var s []string
var ok bool

if s, ok = m[key]; !ok {
return false
}

for _, v := range s {
if v == value {
return true
}
}
return false
}

func hasHeaderKey(m http.Header, key string) bool {
_, ok := m[key]
return ok
}

func toURLAndHeaders(urlParts []interface{}) (string, map[string]interface{}) {
if len(urlParts) == 0 {
return "", nil
}

// The last argument may be a map.
headers, err := maps.ToStringMapE(urlParts[len(urlParts)-1])
if err == nil {
urlParts = urlParts[:len(urlParts)-1]
} else {
headers = nil
}

return strings.Join(cast.ToStringSlice(urlParts), ""), headers
}
4 changes: 4 additions & 0 deletions resources/resource_spec.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
"errors"
"fmt"
"mime"
"net/http"
"os"
"path"
"path/filepath"
Expand Down Expand Up @@ -80,6 +81,7 @@ func NewSpec(
PathSpec: s,
Logger: logger,
ErrorSender: errorHandler,
HTTPClient: http.DefaultClient,
imaging: imaging,
incr: incr,
MediaTypes: mimeTypes,
Expand Down Expand Up @@ -112,6 +114,8 @@ type Spec struct {
Logger loggers.Logger
ErrorSender herrors.ErrorSender

HTTPClient *http.Client

TextTemplates tpl.TemplateParseFinder

Permalinks page.PermalinkExpander
Expand Down
6 changes: 6 additions & 0 deletions tpl/resources/resources.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,12 @@ func (ns *Namespace) FromString(targetPathIn, contentIn interface{}) (resource.R
return ns.createClient.FromString(targetPath, content)
}

// FromRemote creates a Resource from a URL.
// If you provide multiple parts they will be joined together to the final URL.
func (ns *Namespace) FromRemote(args ...interface{}) (resource.Resource, error) {
return ns.createClient.FromRemote(args...)
}

// ExecuteAsTemplate creates a Resource from a Go template, parsed and executed with
// the given data, and published to the relative target path.
func (ns *Namespace) ExecuteAsTemplate(args ...interface{}) (resource.Resource, error) {
Expand Down

0 comments on commit 188e8c7

Please sign in to comment.