From bb5808805ee14a9b513ff100a2f88adc8c7e8cae Mon Sep 17 00:00:00 2001 From: Hariom Verma Date: Fri, 24 Mar 2023 19:16:02 +0530 Subject: [PATCH] feat(mod): fetch indirect dependencies (#602) --- cmd/gnodev/mod.go | 6 +- pkgs/gnolang/gnomod/file.go | 59 +++++++++++++--- pkgs/gnolang/gnomod/file_test.go | 118 +++++++++++++++++++++++++++++++ pkgs/gnolang/gnomod/gnomod.go | 49 ++++++++----- pkgs/gnolang/precompile.go | 30 ++++---- 5 files changed, 220 insertions(+), 42 deletions(-) create mode 100644 pkgs/gnolang/gnomod/file_test.go diff --git a/cmd/gnodev/mod.go b/cmd/gnodev/mod.go index d4c6e2ce9a6..861386234c2 100644 --- a/cmd/gnodev/mod.go +++ b/cmd/gnodev/mod.go @@ -92,8 +92,12 @@ func execModDownload(cfg *modDownloadCfg, args []string, io *commands.IO) error return fmt.Errorf("validate: %w", err) } + gnoModPath, err := gnomod.GetGnoModPath() + if err != nil { + return fmt.Errorf("get gno.mod path: %w", err) + } // fetch dependencies - if err := gnoMod.FetchDeps(cfg.remote); err != nil { + if err := gnoMod.FetchDeps(gnoModPath, cfg.remote); err != nil { return fmt.Errorf("fetch: %w", err) } diff --git a/pkgs/gnolang/gnomod/file.go b/pkgs/gnolang/gnomod/file.go index c0a55a0a1f5..9b97bd2e587 100644 --- a/pkgs/gnolang/gnomod/file.go +++ b/pkgs/gnolang/gnomod/file.go @@ -6,7 +6,9 @@ import ( "log" "os" "path/filepath" + "strings" + "github.com/gnolang/gno/pkgs/gnolang" "golang.org/x/mod/modfile" "golang.org/x/mod/module" ) @@ -32,12 +34,7 @@ func (f *File) Validate() error { // FetchDeps fetches and writes gno.mod packages // in GOPATH/pkg/gnomod/ -func (f *File) FetchDeps(remote string) error { - gnoModPath, err := GetGnoModPath() - if err != nil { - return fmt.Errorf("get gno.mod path: %w", err) - } - +func (f *File) FetchDeps(path string, remote string) error { for _, r := range f.Require { mod, replaced := isReplaced(r.Mod, f.Replace) if replaced { @@ -46,21 +43,63 @@ func (f *File) FetchDeps(remote string) error { } r.Mod = *mod } - log.Println("fetching", r.Mod.Path) - err := writePackage(remote, gnoModPath, r.Mod.Path) + indirect := "" + if r.Indirect { + indirect = "// indirect" + } + + _, err := os.Stat(filepath.Join(path, r.Mod.Path)) + if !os.IsNotExist(err) { + log.Println("cached", r.Mod.Path, indirect) + continue + } + log.Println("fetching", r.Mod.Path, indirect) + requirements, err := writePackage(remote, path, r.Mod.Path) if err != nil { return fmt.Errorf("writepackage: %w", err) } - f := &File{ + modFile := &File{ Module: &modfile.Module{ Mod: module.Version{ Path: r.Mod.Path, }, }, } + for _, req := range requirements { + path := req[1 : len(req)-1] // trim leading and trailing `"` + if strings.HasSuffix(path, modFile.Module.Mod.Path) { + continue + } + // skip if `std`, special case. + if path == gnolang.GnoStdPkgAfter { + continue + } - f.WriteToPath(filepath.Join(gnoModPath, r.Mod.Path)) + if strings.HasPrefix(path, gnolang.ImportPrefix) { + path = strings.TrimPrefix(path, gnolang.ImportPrefix+"/examples/") + modFile.Require = append(modFile.Require, &modfile.Require{ + Mod: module.Version{ + Path: path, + Version: "v0.0.0", // TODO: Use latest? + }, + Indirect: true, + }) + } + } + + err = modFile.FetchDeps(path, remote) + if err != nil { + return err + } + goMod, err := GnoToGoMod(*modFile) + if err != nil { + return err + } + err = goMod.WriteToPath(filepath.Join(path, r.Mod.Path)) + if err != nil { + return err + } } return nil diff --git a/pkgs/gnolang/gnomod/file_test.go b/pkgs/gnolang/gnomod/file_test.go new file mode 100644 index 00000000000..2729dba5a4c --- /dev/null +++ b/pkgs/gnolang/gnomod/file_test.go @@ -0,0 +1,118 @@ +package gnomod + +import ( + "bytes" + "log" + "os" + "path/filepath" + "testing" + + "github.com/gnolang/gno/pkgs/testutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/mod/modfile" + "golang.org/x/mod/module" +) + +const testRemote string = "test3.gno.land:36657" + +func TestFetchDeps(t *testing.T) { + for _, tc := range []struct { + desc string + modFile File + requirements []string + stdOutContains []string + cachedStdOutContains []string + }{ + { + desc: "fetch_gno.land/p/demo/avl", + modFile: File{ + Module: &modfile.Module{ + Mod: module.Version{ + Path: "testFetchDeps", + }, + }, + Require: []*modfile.Require{ + { + Mod: module.Version{ + Path: "gno.land/p/demo/avl", + Version: "v0.0.0", + }, + }, + }, + }, + requirements: []string{"avl"}, + stdOutContains: []string{ + "fetching gno.land/p/demo/avl", + }, + cachedStdOutContains: []string{ + "cached gno.land/p/demo/avl", + }, + }, { + desc: "fetch_gno.land/p/demo/blog", + modFile: File{ + Module: &modfile.Module{ + Mod: module.Version{ + Path: "testFetchDeps", + }, + }, + Require: []*modfile.Require{ + { + Mod: module.Version{ + Path: "gno.land/p/demo/blog", + Version: "v0.0.0", + }, + }, + }, + }, + requirements: []string{"avl", "blog", "ufmt"}, + stdOutContains: []string{ + "fetching gno.land/p/demo/blog", + "fetching gno.land/p/demo/avl // indirect", + "fetching gno.land/p/demo/ufmt // indirect", + }, + cachedStdOutContains: []string{ + "cached gno.land/p/demo/blog", + }, + }, + } { + t.Run(tc.desc, func(t *testing.T) { + var buf bytes.Buffer + log.SetOutput(&buf) + defer func() { + log.SetOutput(os.Stderr) + }() + + // Create test dir + dirPath, cleanUpFn := testutils.NewTestCaseDir(t) + assert.NotNil(t, dirPath) + defer cleanUpFn() + + // Fetching dependencies + tc.modFile.FetchDeps(dirPath, testRemote) + + // Read dir + entries, err := os.ReadDir(filepath.Join(dirPath, "gno.land", "p", "demo")) + require.Nil(t, err) + + // Check dir entries + assert.Equal(t, len(tc.requirements), len(entries)) + for _, e := range entries { + assert.Contains(t, tc.requirements, e.Name()) + } + + // Check logs + for _, c := range tc.stdOutContains { + assert.Contains(t, buf.String(), c) + } + + buf.Reset() + + // Try fetching again. Should be cached + tc.modFile.FetchDeps(dirPath, testRemote) + for _, c := range tc.cachedStdOutContains { + assert.Contains(t, buf.String(), c) + } + }) + } +} diff --git a/pkgs/gnolang/gnomod/gnomod.go b/pkgs/gnolang/gnomod/gnomod.go index d817536db0f..d47e91398f0 100644 --- a/pkgs/gnolang/gnomod/gnomod.go +++ b/pkgs/gnolang/gnomod/gnomod.go @@ -25,10 +25,10 @@ func GetGnoModPath() (string, error) { return filepath.Join(goPath, "pkg", "gnomod"), nil } -func writePackage(remote, basePath, pkgPath string) error { +func writePackage(remote, basePath, pkgPath string) (requirements []string, err error) { res, err := queryChain(remote, queryPathFile, []byte(pkgPath)) if err != nil { - return fmt.Errorf("querychain: %w", err) + return nil, fmt.Errorf("querychain: %w", err) } dirPath, fileName := std.SplitFilepath(pkgPath) @@ -36,17 +36,19 @@ func writePackage(remote, basePath, pkgPath string) error { // Is Dir // Create Dir if not exists dirPath := filepath.Join(basePath, dirPath) - if _, err := os.Stat(dirPath); os.IsNotExist(err) { - if err := os.MkdirAll(dirPath, 0o755); err != nil { - return fmt.Errorf("mkdir %q: %w", dirPath, err) + if _, err = os.Stat(dirPath); os.IsNotExist(err) { + if err = os.MkdirAll(dirPath, 0o755); err != nil { + return nil, fmt.Errorf("mkdir %q: %w", dirPath, err) } } files := strings.Split(string(res.Data), "\n") for _, file := range files { - if err := writePackage(remote, basePath, filepath.Join(pkgPath, file)); err != nil { - return fmt.Errorf("writepackage: %w", err) + reqs, err := writePackage(remote, basePath, filepath.Join(pkgPath, file)) + if err != nil { + return nil, fmt.Errorf("writepackage: %w", err) } + requirements = append(requirements, reqs...) } } else { // Is File @@ -55,17 +57,21 @@ func writePackage(remote, basePath, pkgPath string) error { targetFilename, _ := gnolang.GetPrecompileFilenameAndTags(filePath) precompileRes, err := gnolang.Precompile(string(res.Data), "", fileName) if err != nil { - return fmt.Errorf("precompile: %w", err) + return nil, fmt.Errorf("precompile: %w", err) + } + + for _, i := range precompileRes.Imports { + requirements = append(requirements, i.Path.Value) } fileNameWithPath := filepath.Join(basePath, dirPath, targetFilename) err = os.WriteFile(fileNameWithPath, []byte(precompileRes.Translated), 0o644) if err != nil { - return fmt.Errorf("writefile %q: %w", fileNameWithPath, err) + return nil, fmt.Errorf("writefile %q: %w", fileNameWithPath, err) } } - return nil + return removeDuplicateStr(requirements), nil } // GnoToGoMod make necessary modifications in the gno.mod @@ -76,9 +82,9 @@ func GnoToGoMod(f File) (*File, error) { return nil, err } - if strings.HasPrefix(f.Module.Mod.Path, "gno.land/r/") || - strings.HasPrefix(f.Module.Mod.Path, "gno.land/p/demo/") { - f.Module.Mod.Path = "github.com/gnolang/gno/examples/" + f.Module.Mod.Path + if strings.HasPrefix(f.Module.Mod.Path, gnolang.GnoRealmPkgsPrefixBefore) || + strings.HasPrefix(f.Module.Mod.Path, gnolang.GnoPackagePrefixBefore) { + f.Module.Mod.Path = gnolang.ImportPrefix + "/examples/" + f.Module.Mod.Path } for i := range f.Require { @@ -89,9 +95,9 @@ func GnoToGoMod(f File) (*File, error) { } } path := f.Require[i].Mod.Path - if strings.HasPrefix(f.Require[i].Mod.Path, "gno.land/r/") || - strings.HasPrefix(f.Require[i].Mod.Path, "gno.land/p/demo/") { - f.Require[i].Mod.Path = "github.com/gnolang/gno/examples/" + f.Require[i].Mod.Path + if strings.HasPrefix(f.Require[i].Mod.Path, gnolang.GnoRealmPkgsPrefixBefore) || + strings.HasPrefix(f.Require[i].Mod.Path, gnolang.GnoPackagePrefixBefore) { + f.Require[i].Mod.Path = gnolang.ImportPrefix + "/examples/" + f.Require[i].Mod.Path } f.Replace = append(f.Replace, &modfile.Replace{ @@ -153,3 +159,14 @@ func isReplaced(module module.Version, repl []*modfile.Replace) (*module.Version } return nil, false } + +func removeDuplicateStr(str []string) (res []string) { + m := make(map[string]struct{}, len(str)) + for _, s := range str { + if _, ok := m[s]; !ok { + m[s] = struct{}{} + res = append(res, s) + } + } + return +} diff --git a/pkgs/gnolang/precompile.go b/pkgs/gnolang/precompile.go index 3a4e2016a0c..705caa11534 100644 --- a/pkgs/gnolang/precompile.go +++ b/pkgs/gnolang/precompile.go @@ -20,12 +20,12 @@ import ( ) const ( - gnoRealmPkgsPrefixBefore = "gno.land/r/" - gnoRealmPkgsPrefixAfter = "github.com/gnolang/gno/examples/gno.land/r/" - gnoPackagePrefixBefore = "gno.land/p/demo/" - gnoPackagePrefixAfter = "github.com/gnolang/gno/examples/gno.land/p/demo/" - gnoStdPkgBefore = "std" - gnoStdPkgAfter = "github.com/gnolang/gno/stdlibs/stdshim" + GnoRealmPkgsPrefixBefore = "gno.land/r/" + GnoRealmPkgsPrefixAfter = "github.com/gnolang/gno/examples/gno.land/r/" + GnoPackagePrefixBefore = "gno.land/p/demo/" + GnoPackagePrefixAfter = "github.com/gnolang/gno/examples/gno.land/p/demo/" + GnoStdPkgBefore = "std" + GnoStdPkgAfter = "github.com/gnolang/gno/stdlibs/stdshim" ) var stdlibWhitelist = []string{ @@ -263,11 +263,11 @@ func precompileAST(fset *token.FileSet, f *ast.File, checkWhitelist bool) (ast.N for _, importSpec := range paragraph { importPath := strings.TrimPrefix(strings.TrimSuffix(importSpec.Path.Value, `"`), `"`) - if strings.HasPrefix(importPath, gnoRealmPkgsPrefixBefore) { + if strings.HasPrefix(importPath, GnoRealmPkgsPrefixBefore) { continue } - if strings.HasPrefix(importPath, gnoPackagePrefixBefore) { + if strings.HasPrefix(importPath, GnoPackagePrefixBefore) { continue } @@ -303,15 +303,15 @@ func precompileAST(fset *token.FileSet, f *ast.File, checkWhitelist bool) (ast.N importPath := strings.TrimPrefix(strings.TrimSuffix(importSpec.Path.Value, `"`), `"`) // std package - if importPath == gnoStdPkgBefore { - if !astutil.RewriteImport(fset, f, gnoStdPkgBefore, gnoStdPkgAfter) { - errs = multierr.Append(errs, fmt.Errorf("failed to replace the %q package with %q", gnoStdPkgBefore, gnoStdPkgAfter)) + if importPath == GnoStdPkgBefore { + if !astutil.RewriteImport(fset, f, GnoStdPkgBefore, GnoStdPkgAfter) { + errs = multierr.Append(errs, fmt.Errorf("failed to replace the %q package with %q", GnoStdPkgBefore, GnoStdPkgAfter)) } } // p/pkg packages - if strings.HasPrefix(importPath, gnoPackagePrefixBefore) { - target := gnoPackagePrefixAfter + strings.TrimPrefix(importPath, gnoPackagePrefixBefore) + if strings.HasPrefix(importPath, GnoPackagePrefixBefore) { + target := GnoPackagePrefixAfter + strings.TrimPrefix(importPath, GnoPackagePrefixBefore) if !astutil.RewriteImport(fset, f, importPath, target) { errs = multierr.Append(errs, fmt.Errorf("failed to replace the %q package with %q", importPath, target)) @@ -319,8 +319,8 @@ func precompileAST(fset *token.FileSet, f *ast.File, checkWhitelist bool) (ast.N } // r/realm packages - if strings.HasPrefix(importPath, gnoRealmPkgsPrefixBefore) { - target := gnoRealmPkgsPrefixAfter + strings.TrimPrefix(importPath, gnoRealmPkgsPrefixBefore) + if strings.HasPrefix(importPath, GnoRealmPkgsPrefixBefore) { + target := GnoRealmPkgsPrefixAfter + strings.TrimPrefix(importPath, GnoRealmPkgsPrefixBefore) if !astutil.RewriteImport(fset, f, importPath, target) { errs = multierr.Append(errs, fmt.Errorf("failed to replace the %q package with %q", importPath, target))