Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(mod): fetch indirect dependencies #602

Merged
merged 6 commits into from
Mar 24, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion cmd/gnodev/mod.go
Original file line number Diff line number Diff line change
Expand Up @@ -92,8 +92,12 @@ func execModDownload(cfg *modDownloadCfg, args []string, io *commands.IO) error
return fmt.Errorf("validate: %w", err)
}

gnoModPath, err := gnomod.GetGnoModPath()
if err != nil {
return fmt.Errorf("get gno.mod path: %w", err)
}
// fetch dependencies
if err := gnoMod.FetchDeps(cfg.remote); err != nil {
if err := gnoMod.FetchDeps(gnoModPath, cfg.remote); err != nil {
return fmt.Errorf("fetch: %w", err)
}

Expand Down
59 changes: 49 additions & 10 deletions pkgs/gnolang/gnomod/file.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,9 @@ import (
"log"
"os"
"path/filepath"
"strings"

"github.com/gnolang/gno/pkgs/gnolang"
"golang.org/x/mod/modfile"
"golang.org/x/mod/module"
)
Expand All @@ -32,12 +34,7 @@ func (f *File) Validate() error {

// FetchDeps fetches and writes gno.mod packages
// in GOPATH/pkg/gnomod/
func (f *File) FetchDeps(remote string) error {
gnoModPath, err := GetGnoModPath()
if err != nil {
return fmt.Errorf("get gno.mod path: %w", err)
}

func (f *File) FetchDeps(path string, remote string) error {
for _, r := range f.Require {
mod, replaced := isReplaced(r.Mod, f.Replace)
if replaced {
Expand All @@ -46,21 +43,63 @@ func (f *File) FetchDeps(remote string) error {
}
r.Mod = *mod
}
log.Println("fetching", r.Mod.Path)
err := writePackage(remote, gnoModPath, r.Mod.Path)
indirect := ""
if r.Indirect {
indirect = "// indirect"
}

_, err := os.Stat(filepath.Join(path, r.Mod.Path))
if !os.IsNotExist(err) {
log.Println("cached", r.Mod.Path, indirect)
continue
}
log.Println("fetching", r.Mod.Path, indirect)
requirements, err := writePackage(remote, path, r.Mod.Path)
if err != nil {
return fmt.Errorf("writepackage: %w", err)
}

f := &File{
modFile := &File{
Module: &modfile.Module{
Mod: module.Version{
Path: r.Mod.Path,
},
},
}
for _, req := range requirements {
path := req[1 : len(req)-1] // trim leading and trailing `"`
if strings.HasSuffix(path, modFile.Module.Mod.Path) {
continue
}
// skip if `std`, special case.
if path == gnolang.GnoStdPkgAfter {
continue
moul marked this conversation as resolved.
Show resolved Hide resolved
}

f.WriteToPath(filepath.Join(gnoModPath, r.Mod.Path))
if strings.HasPrefix(path, gnolang.ImportPrefix) {
path = strings.TrimPrefix(path, gnolang.ImportPrefix+"/examples/")
modFile.Require = append(modFile.Require, &modfile.Require{
Mod: module.Version{
Path: path,
Version: "v0.0.0", // TODO: Use latest?
moul marked this conversation as resolved.
Show resolved Hide resolved
},
Indirect: true,
})
}
}

err = modFile.FetchDeps(path, remote)
if err != nil {
return err
}
goMod, err := GnoToGoMod(*modFile)
if err != nil {
return err
}
err = goMod.WriteToPath(filepath.Join(path, r.Mod.Path))
if err != nil {
return err
}
}

return nil
Expand Down
118 changes: 118 additions & 0 deletions pkgs/gnolang/gnomod/file_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
package gnomod

import (
"bytes"
"log"
"os"
"path/filepath"
"testing"

"github.com/gnolang/gno/pkgs/testutils"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/mod/modfile"
"golang.org/x/mod/module"
)

const testRemote string = "test3.gno.land:36657"

func TestFetchDeps(t *testing.T) {
for _, tc := range []struct {
desc string
modFile File
requirements []string
stdOutContains []string
cachedStdOutContains []string
}{
{
desc: "fetch_gno.land/p/demo/avl",
modFile: File{
Module: &modfile.Module{
Mod: module.Version{
Path: "testFetchDeps",
},
},
Require: []*modfile.Require{
{
Mod: module.Version{
Path: "gno.land/p/demo/avl",
Version: "v0.0.0",
},
},
},
},
requirements: []string{"avl"},
stdOutContains: []string{
"fetching gno.land/p/demo/avl",
},
cachedStdOutContains: []string{
"cached gno.land/p/demo/avl",
},
}, {
desc: "fetch_gno.land/p/demo/blog",
modFile: File{
Module: &modfile.Module{
Mod: module.Version{
Path: "testFetchDeps",
},
},
Require: []*modfile.Require{
{
Mod: module.Version{
Path: "gno.land/p/demo/blog",
Version: "v0.0.0",
},
},
},
},
requirements: []string{"avl", "blog", "ufmt"},
stdOutContains: []string{
"fetching gno.land/p/demo/blog",
"fetching gno.land/p/demo/avl // indirect",
"fetching gno.land/p/demo/ufmt // indirect",
},
cachedStdOutContains: []string{
"cached gno.land/p/demo/blog",
},
},
} {
t.Run(tc.desc, func(t *testing.T) {
var buf bytes.Buffer
log.SetOutput(&buf)
defer func() {
log.SetOutput(os.Stderr)
}()

// Create test dir
dirPath, cleanUpFn := testutils.NewTestCaseDir(t)
assert.NotNil(t, dirPath)
defer cleanUpFn()

// Fetching dependencies
tc.modFile.FetchDeps(dirPath, testRemote)

// Read dir
entries, err := os.ReadDir(filepath.Join(dirPath, "gno.land", "p", "demo"))
require.Nil(t, err)

// Check dir entries
assert.Equal(t, len(tc.requirements), len(entries))
for _, e := range entries {
assert.Contains(t, tc.requirements, e.Name())
}

// Check logs
for _, c := range tc.stdOutContains {
assert.Contains(t, buf.String(), c)
}

buf.Reset()

// Try fetching again. Should be cached
tc.modFile.FetchDeps(dirPath, testRemote)
for _, c := range tc.cachedStdOutContains {
assert.Contains(t, buf.String(), c)
}
})
}
}
49 changes: 33 additions & 16 deletions pkgs/gnolang/gnomod/gnomod.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,28 +25,30 @@ func GetGnoModPath() (string, error) {
return filepath.Join(goPath, "pkg", "gnomod"), nil
}

func writePackage(remote, basePath, pkgPath string) error {
func writePackage(remote, basePath, pkgPath string) (requirements []string, err error) {
moul marked this conversation as resolved.
Show resolved Hide resolved
res, err := queryChain(remote, queryPathFile, []byte(pkgPath))
if err != nil {
return fmt.Errorf("querychain: %w", err)
return nil, fmt.Errorf("querychain: %w", err)
}

dirPath, fileName := std.SplitFilepath(pkgPath)
if fileName == "" {
// Is Dir
// Create Dir if not exists
dirPath := filepath.Join(basePath, dirPath)
if _, err := os.Stat(dirPath); os.IsNotExist(err) {
if err := os.MkdirAll(dirPath, 0o755); err != nil {
return fmt.Errorf("mkdir %q: %w", dirPath, err)
if _, err = os.Stat(dirPath); os.IsNotExist(err) {
if err = os.MkdirAll(dirPath, 0o755); err != nil {
return nil, fmt.Errorf("mkdir %q: %w", dirPath, err)
}
}

files := strings.Split(string(res.Data), "\n")
for _, file := range files {
if err := writePackage(remote, basePath, filepath.Join(pkgPath, file)); err != nil {
return fmt.Errorf("writepackage: %w", err)
reqs, err := writePackage(remote, basePath, filepath.Join(pkgPath, file))
if err != nil {
return nil, fmt.Errorf("writepackage: %w", err)
}
requirements = append(requirements, reqs...)
}
} else {
// Is File
Expand All @@ -55,17 +57,21 @@ func writePackage(remote, basePath, pkgPath string) error {
targetFilename, _ := gnolang.GetPrecompileFilenameAndTags(filePath)
precompileRes, err := gnolang.Precompile(string(res.Data), "", fileName)
if err != nil {
return fmt.Errorf("precompile: %w", err)
return nil, fmt.Errorf("precompile: %w", err)
}

for _, i := range precompileRes.Imports {
requirements = append(requirements, i.Path.Value)
}

fileNameWithPath := filepath.Join(basePath, dirPath, targetFilename)
err = os.WriteFile(fileNameWithPath, []byte(precompileRes.Translated), 0o644)
if err != nil {
return fmt.Errorf("writefile %q: %w", fileNameWithPath, err)
return nil, fmt.Errorf("writefile %q: %w", fileNameWithPath, err)
}
}

return nil
return removeDuplicateStr(requirements), nil
}

// GnoToGoMod make necessary modifications in the gno.mod
Expand All @@ -76,9 +82,9 @@ func GnoToGoMod(f File) (*File, error) {
return nil, err
}

if strings.HasPrefix(f.Module.Mod.Path, "gno.land/r/") ||
strings.HasPrefix(f.Module.Mod.Path, "gno.land/p/demo/") {
f.Module.Mod.Path = "github.com/gnolang/gno/examples/" + f.Module.Mod.Path
if strings.HasPrefix(f.Module.Mod.Path, gnolang.GnoRealmPkgsPrefixBefore) ||
strings.HasPrefix(f.Module.Mod.Path, gnolang.GnoPackagePrefixBefore) {
f.Module.Mod.Path = gnolang.ImportPrefix + "/examples/" + f.Module.Mod.Path
}

for i := range f.Require {
Expand All @@ -89,9 +95,9 @@ func GnoToGoMod(f File) (*File, error) {
}
}
path := f.Require[i].Mod.Path
if strings.HasPrefix(f.Require[i].Mod.Path, "gno.land/r/") ||
strings.HasPrefix(f.Require[i].Mod.Path, "gno.land/p/demo/") {
f.Require[i].Mod.Path = "github.com/gnolang/gno/examples/" + f.Require[i].Mod.Path
if strings.HasPrefix(f.Require[i].Mod.Path, gnolang.GnoRealmPkgsPrefixBefore) ||
strings.HasPrefix(f.Require[i].Mod.Path, gnolang.GnoPackagePrefixBefore) {
f.Require[i].Mod.Path = gnolang.ImportPrefix + "/examples/" + f.Require[i].Mod.Path
}

f.Replace = append(f.Replace, &modfile.Replace{
Expand Down Expand Up @@ -153,3 +159,14 @@ func isReplaced(module module.Version, repl []*modfile.Replace) (*module.Version
}
return nil, false
}

func removeDuplicateStr(str []string) (res []string) {
moul marked this conversation as resolved.
Show resolved Hide resolved
m := make(map[string]struct{}, len(str))
for _, s := range str {
if _, ok := m[s]; !ok {
m[s] = struct{}{}
res = append(res, s)
}
}
return
}
30 changes: 15 additions & 15 deletions pkgs/gnolang/precompile.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,12 @@ import (
)

const (
gnoRealmPkgsPrefixBefore = "gno.land/r/"
gnoRealmPkgsPrefixAfter = "github.com/gnolang/gno/examples/gno.land/r/"
gnoPackagePrefixBefore = "gno.land/p/demo/"
gnoPackagePrefixAfter = "github.com/gnolang/gno/examples/gno.land/p/demo/"
gnoStdPkgBefore = "std"
gnoStdPkgAfter = "github.com/gnolang/gno/stdlibs/stdshim"
GnoRealmPkgsPrefixBefore = "gno.land/r/"
GnoRealmPkgsPrefixAfter = "github.com/gnolang/gno/examples/gno.land/r/"
GnoPackagePrefixBefore = "gno.land/p/demo/"
GnoPackagePrefixAfter = "github.com/gnolang/gno/examples/gno.land/p/demo/"
GnoStdPkgBefore = "std"
GnoStdPkgAfter = "github.com/gnolang/gno/stdlibs/stdshim"
)

var stdlibWhitelist = []string{
Expand Down Expand Up @@ -263,11 +263,11 @@ func precompileAST(fset *token.FileSet, f *ast.File, checkWhitelist bool) (ast.N
for _, importSpec := range paragraph {
importPath := strings.TrimPrefix(strings.TrimSuffix(importSpec.Path.Value, `"`), `"`)

if strings.HasPrefix(importPath, gnoRealmPkgsPrefixBefore) {
if strings.HasPrefix(importPath, GnoRealmPkgsPrefixBefore) {
continue
}

if strings.HasPrefix(importPath, gnoPackagePrefixBefore) {
if strings.HasPrefix(importPath, GnoPackagePrefixBefore) {
continue
}

Expand Down Expand Up @@ -303,24 +303,24 @@ func precompileAST(fset *token.FileSet, f *ast.File, checkWhitelist bool) (ast.N
importPath := strings.TrimPrefix(strings.TrimSuffix(importSpec.Path.Value, `"`), `"`)

// std package
if importPath == gnoStdPkgBefore {
if !astutil.RewriteImport(fset, f, gnoStdPkgBefore, gnoStdPkgAfter) {
errs = multierr.Append(errs, fmt.Errorf("failed to replace the %q package with %q", gnoStdPkgBefore, gnoStdPkgAfter))
if importPath == GnoStdPkgBefore {
if !astutil.RewriteImport(fset, f, GnoStdPkgBefore, GnoStdPkgAfter) {
errs = multierr.Append(errs, fmt.Errorf("failed to replace the %q package with %q", GnoStdPkgBefore, GnoStdPkgAfter))
}
}

// p/pkg packages
if strings.HasPrefix(importPath, gnoPackagePrefixBefore) {
target := gnoPackagePrefixAfter + strings.TrimPrefix(importPath, gnoPackagePrefixBefore)
if strings.HasPrefix(importPath, GnoPackagePrefixBefore) {
target := GnoPackagePrefixAfter + strings.TrimPrefix(importPath, GnoPackagePrefixBefore)

if !astutil.RewriteImport(fset, f, importPath, target) {
errs = multierr.Append(errs, fmt.Errorf("failed to replace the %q package with %q", importPath, target))
}
}

// r/realm packages
if strings.HasPrefix(importPath, gnoRealmPkgsPrefixBefore) {
target := gnoRealmPkgsPrefixAfter + strings.TrimPrefix(importPath, gnoRealmPkgsPrefixBefore)
if strings.HasPrefix(importPath, GnoRealmPkgsPrefixBefore) {
target := GnoRealmPkgsPrefixAfter + strings.TrimPrefix(importPath, GnoRealmPkgsPrefixBefore)

if !astutil.RewriteImport(fset, f, importPath, target) {
errs = multierr.Append(errs, fmt.Errorf("failed to replace the %q package with %q", importPath, target))
Expand Down