diff --git a/commands/mod.go b/commands/mod.go index 81f660f4362..b390d1e75d0 100644 --- a/commands/mod.go +++ b/commands/mod.go @@ -1,4 +1,4 @@ -// Copyright 2019 The Hugo Authors. All rights reserved. +// Copyright 2020 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -20,6 +20,8 @@ import ( "path/filepath" "regexp" + "github.com/gohugoio/hugo/hugolib" + "github.com/gohugoio/hugo/modules" "github.com/spf13/cobra" ) @@ -114,6 +116,8 @@ This is not needed if you only operate on modules inside /themes or if you have RunE: nil, } + cmd.AddCommand(newModNPMCmd(c)) + cmd.AddCommand( &cobra.Command{ Use: "get", @@ -272,6 +276,15 @@ func (c *modCmd) withModsClient(failOnMissingConfig bool, f func(*modules.Client return f(com.hugo().ModulesClient) } +func (c *modCmd) withHugo(f func(*hugolib.HugoSites) error) error { + com, err := c.initConfig(true) + if err != nil { + return err + } + + return f(com.hugo()) +} + func (c *modCmd) initConfig(failOnNoConfig bool) (*commandeer, error) { com, err := initializeConfig(failOnNoConfig, false, &c.hugoBuilderCommon, c, nil) if err != nil { diff --git a/commands/npm.go b/commands/npm.go new file mode 100644 index 00000000000..47582183189 --- /dev/null +++ b/commands/npm.go @@ -0,0 +1,62 @@ +// Copyright 2020 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package commands + +import ( + "fmt" + + "github.com/gohugoio/hugo/hugolib" + "github.com/spf13/cobra" +) + +func newModNPMCmd(c *modCmd) *cobra.Command { + + cmd := &cobra.Command{ + Use: "npm", + Short: "Various npm helpers.", + Long: `Various npm (Node package manager) helpers.`, + RunE: func(cmd *cobra.Command, args []string) error { + return c.withHugo(func(h *hugolib.HugoSites) error { + return nil + }) + }, + } + + cmd.AddCommand(&cobra.Command{ + Use: "pack", + Short: "Experimental: Prepares and writes a composite package.json file for your project.", + Long: `Prepares and writes a composite package.json file for your project. + +Note that this command will fail if there is already a "package.json" file in the project. + +If there is a file named "package.hugo.json", that file will be used as a template file +with the base dependency set. + +This command is marked as 'Experimental'. We think it's a great idea, so it's not likely to be +removed from Hugo, but we need to test this out in "real life" to get a feel of it, +so this may/will change in future versions of Hugo. +`, + RunE: func(cmd *cobra.Command, args []string) error { + + return c.withHugo(func(h *hugolib.HugoSites) error { + fmt.Println("PACK", h) + + return nil + + }) + }, + }) + + return cmd +} diff --git a/hugofs/files/classifier.go b/hugofs/files/classifier.go index 5e26bbac029..87ef43274e9 100644 --- a/hugofs/files/classifier.go +++ b/hugofs/files/classifier.go @@ -154,6 +154,7 @@ func isHTMLContent(r io.Reader) bool { } const ( + ComponentFolderRoot = "root" // Pseudo folder for files in the root of the project/module. ComponentFolderArchetypes = "archetypes" ComponentFolderStatic = "static" ComponentFolderLayouts = "layouts" diff --git a/hugofs/rootmapping_fs.go b/hugofs/rootmapping_fs.go index ea3ef003ecc..cfd5a0ac34c 100644 --- a/hugofs/rootmapping_fs.go +++ b/hugofs/rootmapping_fs.go @@ -42,9 +42,6 @@ func NewRootMappingFs(fs afero.Fs, rms ...RootMapping) (*RootMappingFs, error) { (&rm).clean() fromBase := files.ResolveComponentFolder(rm.From) - if fromBase == "" { - panic("unrecognised component folder in" + rm.From) - } if len(rm.To) < 2 { panic(fmt.Sprintf("invalid root mapping; from/to: %s/%s", rm.From, rm.To)) @@ -113,7 +110,7 @@ func newRootMappingFsFromFromTo( ToBasedir: baseDir, } } - + return NewRootMappingFs(fs, rms...) } diff --git a/hugolib/filesystems/basefs.go b/hugolib/filesystems/basefs.go index 57a95a03713..39245ca84f8 100644 --- a/hugolib/filesystems/basefs.go +++ b/hugolib/filesystems/basefs.go @@ -49,6 +49,9 @@ type BaseFs struct { // SourceFilesystems contains the different source file systems. *SourceFilesystems + // The project source. + SourceFs afero.Fs + // The filesystem used to publish the rendered site. // This usually maps to /my-project/public. PublishFs afero.Fs @@ -111,6 +114,10 @@ type SourceFilesystems struct { Archetypes *SourceFilesystem Assets *SourceFilesystem + // A special filesystem with a set of files only, e.g. package.json, + // postcss.config.js etc. + Root *SourceFilesystem + // Writable filesystem on top the project's resources directory, // with any sub module's resource fs layered below. ResourcesCache afero.Fs @@ -346,8 +353,10 @@ func NewBase(p *paths.Paths, logger *loggers.Logger, options ...func(*BaseFs) er } publishFs := hugofs.NewBaseFileDecorator(afero.NewBasePathFs(fs.Destination, p.AbsPublishDir)) + sourceFs := hugofs.NewBaseFileDecorator(afero.NewBasePathFs(fs.Source, p.WorkingDir)) b := &BaseFs{ + SourceFs: sourceFs, PublishFs: publishFs, } @@ -422,6 +431,7 @@ func (b *sourceFilesystemsBuilder) Build() (*SourceFilesystems, error) { b.result.Archetypes = createView(files.ComponentFolderArchetypes) b.result.Layouts = createView(files.ComponentFolderLayouts) b.result.Assets = createView(files.ComponentFolderAssets) + b.result.Root = createView(files.ComponentFolderRoot) b.result.ResourcesCache = b.theBigFs.overlayResources // Data, i18n and content cannot use the overlay fs @@ -696,11 +706,18 @@ type filesystemsCollector struct { func (c *filesystemsCollector) addDirs(rfs *hugofs.RootMappingFs) { for _, componentFolder := range files.ComponentFolders { - dirs, err := rfs.Dirs(componentFolder) + c.addDir(rfs, componentFolder) + } - if err == nil { - c.overlayDirs[componentFolder] = append(c.overlayDirs[componentFolder], dirs...) - } + // Also add the special root folder. + c.addDir(rfs, files.ComponentFolderRoot) +} + +func (c *filesystemsCollector) addDir(rfs *hugofs.RootMappingFs, componentFolder string) { + dirs, err := rfs.Dirs(componentFolder) + + if err == nil { + c.overlayDirs[componentFolder] = append(c.overlayDirs[componentFolder], dirs...) } } diff --git a/hugolib/filesystems/basefs_test.go b/hugolib/filesystems/basefs_test.go index 633c8fe08cf..05d3c1c7b55 100644 --- a/hugolib/filesystems/basefs_test.go +++ b/hugolib/filesystems/basefs_test.go @@ -400,6 +400,42 @@ func TestMakePathRelative(t *testing.T) { } +func TestRootFs(t *testing.T) { + c := qt.New(t) + v := createConfig() + fs := hugofs.NewMem(v) + workDir := "mywork" + v.Set("workingDir", workDir) + + c.Assert(afero.WriteFile(fs.Source, filepath.Join(workDir, "package.json"), []byte(`my packages`), 0777), qt.IsNil) + + moduleCfg := map[string]interface{}{ + "mounts": []interface{}{ + map[string]interface{}{ + "source": "package.json", + "target": "package.json", + }, + }, + } + + v.Set("module", moduleCfg) + + c.Assert(initConfig(fs.Source, v), qt.IsNil) + + p, err := paths.New(fs, v) + c.Assert(err, qt.IsNil) + bfs, err := NewBase(p, nil) + c.Assert(err, qt.IsNil) + + rfs := bfs.Root + c.Assert(rfs, qt.Not(qt.IsNil)) + + c.Assert(rfs.Dirs, qt.HasLen, 1) + checkFileCount(rfs.Fs, "", c, 1) + checkFileContent(rfs.Fs, "package.json", c, "my packages") + +} + func checkFileCount(fs afero.Fs, dirname string, c *qt.C, expected int) { count, _, err := countFilesAndGetFilenames(fs, dirname) c.Assert(err, qt.IsNil) diff --git a/hugolib/hugo_modules_test.go b/hugolib/hugo_modules_test.go index 0ed4fceb01d..90bcd6fe11b 100644 --- a/hugolib/hugo_modules_test.go +++ b/hugolib/hugo_modules_test.go @@ -22,6 +22,8 @@ import ( "testing" "time" + "github.com/gohugoio/hugo/modules/npm" + "github.com/gohugoio/hugo/common/loggers" "github.com/spf13/afero" @@ -38,7 +40,6 @@ import ( "github.com/spf13/viper" ) -// https://github.com/gohugoio/hugo/issues/6730 func TestHugoModulesVariants(t *testing.T) { if !isCI() { t.Skip("skip (relative) long running modules test when running locally") @@ -129,6 +130,57 @@ JS imported in module: | `) }) + t.Run("Create package.json", func(t *testing.T) { + + b, clean := newTestBuilder(t, "ignoreImports=true") + defer clean() + + b.WithSourceFile("package.json", `{ + "name": "mypack", + "version": "1.2.3", + "scripts": {}, + "dependencies": { + "nonon": "error" + } +}`) + + b.WithSourceFile("package.hugo.json", `{ + "name": "mypack", + "version": "1.2.3", + "scripts": {}, + "dependencies": { + "foo": "1.2.3" + }, + "devDependencies": { + "postcss-cli": "7.8.0", + "tailwindcss": "1.8.0" + + } +}`) + + b.Build(BuildCfg{}) + b.Assert(npm.Pack(b.H.BaseFs.SourceFs, b.H.BaseFs.Root.Dirs), qt.IsNil) + + b.AssertFileContentFn("package.json", func(s string) bool { + return s == `{ + "dependencies": { + "foo": "1.2.3", + "react-dom": "^16.13.1" + }, + "devDependencies": { + "@babel/cli": "7.8.4", + "@babel/core": "7.9.0", + "@babel/preset-env": "7.9.5", + "postcss-cli": "7.8.0", + "tailwindcss": "1.8.0" + }, + "name": "mypack", + "scripts": {}, + "version": "1.2.3" +}` + }) + }) + } // TODO(bep) this fails when testmodBuilder is also building ... diff --git a/modules/collect.go b/modules/collect.go index b82d395fd0e..1a72625dbe5 100644 --- a/modules/collect.go +++ b/modules/collect.go @@ -382,6 +382,11 @@ func (c *collector) applyMounts(moduleImport Import, mod *moduleAdapter) error { return err } + mounts, err = c.addRootMounts(mod, mounts) + if err != nil { + return err + } + mod.mounts = mounts return nil } @@ -549,6 +554,43 @@ func (c *collector) loadModules() error { return nil } +func (c *collector) addRootMounts(owner *moduleAdapter, mounts []Mount) ([]Mount, error) { + if m, created := c.createRootMountIfNotExists(owner, mounts, "package.json"); created { + mounts = append(mounts, m) + } + if owner.projectMod { + if m, created := c.createRootMountIfNotExists(owner, mounts, "package.hugo.json"); created { + mounts = append(mounts, m) + } + } + return mounts, nil +} + +func (c *collector) createRootMountIfNotExists(owner *moduleAdapter, mounts []Mount, nameToCreate string) (Mount, bool) { + var isMounted bool + for _, m := range mounts { + component, name := m.ComponentAndName() + if component == files.ComponentFolderRoot && name == nameToCreate { + isMounted = true + break + } + } + if !isMounted { + // If it exists, mount it. + filename := filepath.Join(owner.Dir(), nameToCreate) + _, err := c.fs.Stat(filename) + + if err == nil { + return Mount{ + Source: nameToCreate, + Target: filepath.Join(files.ComponentFolderRoot, nameToCreate), + }, true + } + } + + return Mount{}, false +} + func (c *collector) normalizeMounts(owner *moduleAdapter, mounts []Mount) ([]Mount, error) { var out []Mount dir := owner.Dir() @@ -563,17 +605,17 @@ func (c *collector) normalizeMounts(owner *moduleAdapter, mounts []Mount) ([]Mou mnt.Source = filepath.Clean(mnt.Source) mnt.Target = filepath.Clean(mnt.Target) - var sourceDir string + var sourceFilename string if owner.projectMod && filepath.IsAbs(mnt.Source) { // Abs paths in the main project is allowed. - sourceDir = mnt.Source + sourceFilename = mnt.Source } else { - sourceDir = filepath.Join(dir, mnt.Source) + sourceFilename = filepath.Join(dir, mnt.Source) } // Verify that Source exists - _, err := c.fs.Stat(sourceDir) + fi, err := c.fs.Stat(sourceFilename) if err != nil { continue } @@ -581,11 +623,18 @@ func (c *collector) normalizeMounts(owner *moduleAdapter, mounts []Mount) ([]Mou // Verify that target points to one of the predefined component dirs targetBase := mnt.Target idxPathSep := strings.Index(mnt.Target, string(os.PathSeparator)) - if idxPathSep != -1 { + var isRoot bool + if idxPathSep == -1 && !fi.IsDir() { + isRoot = true + mnt.Target = filepath.Join(files.ComponentFolderRoot, mnt.Target) + } else if idxPathSep != -1 { targetBase = mnt.Target[0:idxPathSep] + } else if fi.IsDir() { + targetBase = mnt.Target } - if !files.IsComponentFolder(targetBase) { - return nil, errors.Errorf("%s: mount target must be one of: %v", errMsg, files.ComponentFolders) + + if !isRoot && !files.IsComponentFolder(targetBase) { + return nil, errors.Errorf("%s: mount target must be a filename in the root or one of these directories: %v", errMsg, files.ComponentFolders) } out = append(out, mnt) diff --git a/modules/config.go b/modules/config.go index 1964479f46c..c37a775d6de 100644 --- a/modules/config.go +++ b/modules/config.go @@ -318,12 +318,21 @@ type Mount struct { Target string // relative target path, e.g. "assets/bootstrap/scss" Lang string // any language code associated with this mount. + } func (m Mount) Component() string { return strings.Split(m.Target, fileSeparator)[0] } +func (m Mount) ComponentAndName() (string, string) { + k := strings.Index(m.Target, fileSeparator) + if k == -1 { + return m.Target, "" + } + return m.Target[:k], m.Target[k+1:] +} + func getStaticDirs(cfg config.Provider) []string { var staticDirs []string for i := -1; i <= 10; i++ { diff --git a/modules/npm/package_builder.go b/modules/npm/package_builder.go new file mode 100644 index 00000000000..c1b72bd1f2b --- /dev/null +++ b/modules/npm/package_builder.go @@ -0,0 +1,188 @@ +// Copyright 2020 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package npm + +import ( + "encoding/json" + "io" + "path/filepath" + "strings" + + "github.com/pkg/errors" + + "github.com/gohugoio/hugo/hugofs" + "github.com/spf13/afero" + + "github.com/spf13/cast" + + "github.com/gohugoio/hugo/helpers" +) + +const ( + dependenciesKey = "dependencies" + devDependenciesKey = "devDependencies" + + packageJSONName = "package.json" + packageHugoJSONName = "package.hugo.json" +) + +func Pack(fs afero.Fs, files []hugofs.FileMetaInfo) error { + + var b *packageBuilder + var workDir string + +Loop: + for _, fi := range files { + switch fi.Name() { + case packageHugoJSONName: + meta := fi.(hugofs.FileMetaInfo).Meta() + f, err := meta.Open() + if err != nil { + return errors.Wrap(err, "npm pack: failed to open package file") + } + b = newPackageBuilder(f) + workDir = filepath.Dir(meta.Filename()) + f.Close() + break Loop + + } + + } + +Loop2: + for _, fi := range files { + switch fi.Name() { + case packageJSONName: + if b == nil { + return errors.New("npm pack: no package.hugo.json found") + } + meta := fi.(hugofs.FileMetaInfo).Meta() + if strings.HasPrefix(meta.Filename(), workDir) { + continue Loop2 + } + f, err := meta.Open() + if err != nil { + return errors.Wrap(err, "npm pack: failed to open package file") + } + b.Add(f) + f.Close() + } + + } + + if b.Err() != nil { + return errors.Wrap(b.Err(), "npm pack: failed to build") + } + + b.originalPackageJSON[dependenciesKey] = b.dependencies + b.originalPackageJSON[devDependenciesKey] = b.devDependencies + + // Write it out to the project package.json + packageJSONData, err := json.MarshalIndent(b.originalPackageJSON, "", " ") + if err != nil { + return errors.Wrap(err, "npm pack: failed to marshal JSON") + } + + if err := afero.WriteFile(fs, "package.json", packageJSONData, 0666); err != nil { + return errors.Wrap(err, "npm pack: failed to write package.json") + } + + return nil + +} + +// TODO(bep) comments, see https://dev.to/napolux/how-to-add-comments-to-packagejson-5doi +func newPackageBuilder(first io.Reader) *packageBuilder { + b := &packageBuilder{ + devDependencies: make(map[string]interface{}), + dependencies: make(map[string]interface{}), + } + + m := b.unmarshal(first) + if b.err != nil { + return b + } + + b.addm(m) + b.originalPackageJSON = m + + return b +} + +type packageBuilder struct { + err error + + // The original package.hugo.json, + // may be nil. + originalPackageJSON map[string]interface{} + + devDependencies map[string]interface{} + dependencies map[string]interface{} +} + +func (b *packageBuilder) Add(r io.Reader) *packageBuilder { + if b.err != nil { + return b + } + + m := b.unmarshal(r) + if b.err != nil { + return b + } + + b.addm(m) + + return b +} + +func (b *packageBuilder) addm(m map[string]interface{}) { + // The version selection is currently very simple. + // We may consider minimal version selection or something + // after testing this out. + // + // But for now, the first version string for a given dependency wins. + // These packages will be added by order of import (project, module1, module2...), + // so that should at least give the project control over the situation. + if devDeps, found := m[devDependenciesKey]; found { + mm := cast.ToStringMapString(devDeps) + for k, v := range mm { + if _, added := b.devDependencies[k]; !added { + b.devDependencies[k] = v + } + } + } + + if deps, found := m[dependenciesKey]; found { + mm := cast.ToStringMapString(deps) + for k, v := range mm { + if _, added := b.dependencies[k]; !added { + b.dependencies[k] = v + } + } + } + +} + +func (b *packageBuilder) unmarshal(r io.Reader) map[string]interface{} { + m := make(map[string]interface{}) + err := json.Unmarshal(helpers.ReaderToBytes(r), &m) + if err != nil { + b.err = err + } + return m +} + +func (b *packageBuilder) Err() error { + return b.err +} diff --git a/modules/npm/package_builder_test.go b/modules/npm/package_builder_test.go new file mode 100644 index 00000000000..5061dfacbfc --- /dev/null +++ b/modules/npm/package_builder_test.go @@ -0,0 +1,95 @@ +// Copyright 2020 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package npm + +import ( + "strings" + "testing" + + qt "github.com/frankban/quicktest" +) + +const templ = `{ + "name": "foo", + "version": "0.1.1", + "scripts": {}, + "dependencies": { + "react-dom": "1.1.1", + "tailwindcss": "1.2.0", + "@babel/cli": "7.8.4", + "@babel/core": "7.9.0", + "@babel/preset-env": "7.9.5" + }, + "devDependencies": { + "postcss-cli": "7.1.0", + "tailwindcss": "1.2.0", + "@babel/cli": "7.8.4", + "@babel/core": "7.9.0", + "@babel/preset-env": "7.9.5" + } +}` + +func TestPackageBuilder(t *testing.T) { + c := qt.New(t) + + b := newPackageBuilder(strings.NewReader(templ)) + c.Assert(b.Err(), qt.IsNil) + + b.Add(strings.NewReader(`{ +"dependencies": { + "react-dom": "9.1.1", + "add1": "1.1.1" +}, +"devDependencies": { + "tailwindcss": "error", + "add2": "2.1.1" +} +}`)) + + b.Add(strings.NewReader(`{ +"dependencies": { + "react-dom": "error", + "add1": "error", + "add3": "3.1.1" +}, +"devDependencies": { + "tailwindcss": "error", + "add2": "error", + "add4": "4.1.1" + +} +}`)) + + c.Assert(b.Err(), qt.IsNil) + + c.Assert(b.dependencies, qt.DeepEquals, map[string]interface{}{ + "@babel/cli": "7.8.4", + "add1": "1.1.1", + "add3": "3.1.1", + "@babel/core": "7.9.0", + "@babel/preset-env": "7.9.5", + "react-dom": "1.1.1", + "tailwindcss": "1.2.0", + }) + + c.Assert(b.devDependencies, qt.DeepEquals, map[string]interface{}{ + "tailwindcss": "1.2.0", + "@babel/cli": "7.8.4", + "@babel/core": "7.9.0", + "add2": "2.1.1", + "add4": "4.1.1", + "@babel/preset-env": "7.9.5", + "postcss-cli": "7.1.0", + }) +}