Skip to content

Commit

Permalink
feat(mod): fetch indirect dependencies (#602)
Browse files Browse the repository at this point in the history
  • Loading branch information
@harry-hov
harry-hov authored Mar 24, 2023
1 parent a2966a7 commit bb58088
Show file tree
Hide file tree
Showing 5 changed files with 220 additions and 42 deletions.
6 changes: 5 additions & 1 deletion cmd/gnodev/mod.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,8 +92,12 @@ func execModDownload(cfg *modDownloadCfg, args []string, io *commands.IO) error
return fmt.Errorf("validate: %w", err)
}

gnoModPath, err := gnomod.GetGnoModPath()
if err != nil {
return fmt.Errorf("get gno.mod path: %w", err)
}
// fetch dependencies
if err := gnoMod.FetchDeps(cfg.remote); err != nil {
if err := gnoMod.FetchDeps(gnoModPath, cfg.remote); err != nil {
return fmt.Errorf("fetch: %w", err)
}

Expand Down
59 changes: 49 additions & 10 deletions pkgs/gnolang/gnomod/file.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,9 @@ import (
"log"
"os"
"path/filepath"
"strings"

"github.com/gnolang/gno/pkgs/gnolang"
"golang.org/x/mod/modfile"
"golang.org/x/mod/module"
)
Expand All @@ -32,12 +34,7 @@ func (f *File) Validate() error {

// FetchDeps fetches and writes gno.mod packages
// in GOPATH/pkg/gnomod/
func (f *File) FetchDeps(remote string) error {
gnoModPath, err := GetGnoModPath()
if err != nil {
return fmt.Errorf("get gno.mod path: %w", err)
}

func (f *File) FetchDeps(path string, remote string) error {
for _, r := range f.Require {
mod, replaced := isReplaced(r.Mod, f.Replace)
if replaced {
Expand All @@ -46,21 +43,63 @@ func (f *File) FetchDeps(remote string) error {
}
r.Mod = *mod
}
log.Println("fetching", r.Mod.Path)
err := writePackage(remote, gnoModPath, r.Mod.Path)
indirect := ""
if r.Indirect {
indirect = "// indirect"
}

_, err := os.Stat(filepath.Join(path, r.Mod.Path))
if !os.IsNotExist(err) {
log.Println("cached", r.Mod.Path, indirect)
continue
}
log.Println("fetching", r.Mod.Path, indirect)
requirements, err := writePackage(remote, path, r.Mod.Path)
if err != nil {
return fmt.Errorf("writepackage: %w", err)
}

f := &File{
modFile := &File{
Module: &modfile.Module{
Mod: module.Version{
Path: r.Mod.Path,
},
},
}
for _, req := range requirements {
path := req[1 : len(req)-1] // trim leading and trailing `"`
if strings.HasSuffix(path, modFile.Module.Mod.Path) {
continue
}
// skip if `std`, special case.
if path == gnolang.GnoStdPkgAfter {
continue
}

f.WriteToPath(filepath.Join(gnoModPath, r.Mod.Path))
if strings.HasPrefix(path, gnolang.ImportPrefix) {
path = strings.TrimPrefix(path, gnolang.ImportPrefix+"/examples/")
modFile.Require = append(modFile.Require, &modfile.Require{
Mod: module.Version{
Path: path,
Version: "v0.0.0", // TODO: Use latest?
},
Indirect: true,
})
}
}

err = modFile.FetchDeps(path, remote)
if err != nil {
return err
}
goMod, err := GnoToGoMod(*modFile)
if err != nil {
return err
}
err = goMod.WriteToPath(filepath.Join(path, r.Mod.Path))
if err != nil {
return err
}
}

return nil
Expand Down
118 changes: 118 additions & 0 deletions pkgs/gnolang/gnomod/file_test.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
package gnomod

import (
"bytes"
"log"
"os"
"path/filepath"
"testing"

"github.com/gnolang/gno/pkgs/testutils"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/mod/modfile"
"golang.org/x/mod/module"
)

const testRemote string = "test3.gno.land:36657"

func TestFetchDeps(t *testing.T) {
for _, tc := range []struct {
desc string
modFile File
requirements []string
stdOutContains []string
cachedStdOutContains []string
}{
{
desc: "fetch_gno.land/p/demo/avl",
modFile: File{
Module: &modfile.Module{
Mod: module.Version{
Path: "testFetchDeps",
},
},
Require: []*modfile.Require{
{
Mod: module.Version{
Path: "gno.land/p/demo/avl",
Version: "v0.0.0",
},
},
},
},
requirements: []string{"avl"},
stdOutContains: []string{
"fetching gno.land/p/demo/avl",
},
cachedStdOutContains: []string{
"cached gno.land/p/demo/avl",
},
}, {
desc: "fetch_gno.land/p/demo/blog",
modFile: File{
Module: &modfile.Module{
Mod: module.Version{
Path: "testFetchDeps",
},
},
Require: []*modfile.Require{
{
Mod: module.Version{
Path: "gno.land/p/demo/blog",
Version: "v0.0.0",
},
},
},
},
requirements: []string{"avl", "blog", "ufmt"},
stdOutContains: []string{
"fetching gno.land/p/demo/blog",
"fetching gno.land/p/demo/avl // indirect",
"fetching gno.land/p/demo/ufmt // indirect",
},
cachedStdOutContains: []string{
"cached gno.land/p/demo/blog",
},
},
} {
t.Run(tc.desc, func(t *testing.T) {
var buf bytes.Buffer
log.SetOutput(&buf)
defer func() {
log.SetOutput(os.Stderr)
}()

// Create test dir
dirPath, cleanUpFn := testutils.NewTestCaseDir(t)
assert.NotNil(t, dirPath)
defer cleanUpFn()

// Fetching dependencies
tc.modFile.FetchDeps(dirPath, testRemote)

// Read dir
entries, err := os.ReadDir(filepath.Join(dirPath, "gno.land", "p", "demo"))
require.Nil(t, err)

// Check dir entries
assert.Equal(t, len(tc.requirements), len(entries))
for _, e := range entries {
assert.Contains(t, tc.requirements, e.Name())
}

// Check logs
for _, c := range tc.stdOutContains {
assert.Contains(t, buf.String(), c)
}

buf.Reset()

// Try fetching again. Should be cached
tc.modFile.FetchDeps(dirPath, testRemote)
for _, c := range tc.cachedStdOutContains {
assert.Contains(t, buf.String(), c)
}
})
}
}
49 changes: 33 additions & 16 deletions pkgs/gnolang/gnomod/gnomod.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,28 +25,30 @@ func GetGnoModPath() (string, error) {
return filepath.Join(goPath, "pkg", "gnomod"), nil
}

func writePackage(remote, basePath, pkgPath string) error {
func writePackage(remote, basePath, pkgPath string) (requirements []string, err error) {
res, err := queryChain(remote, queryPathFile, []byte(pkgPath))
if err != nil {
return fmt.Errorf("querychain: %w", err)
return nil, fmt.Errorf("querychain: %w", err)
}

dirPath, fileName := std.SplitFilepath(pkgPath)
if fileName == "" {
// Is Dir
// Create Dir if not exists
dirPath := filepath.Join(basePath, dirPath)
if _, err := os.Stat(dirPath); os.IsNotExist(err) {
if err := os.MkdirAll(dirPath, 0o755); err != nil {
return fmt.Errorf("mkdir %q: %w", dirPath, err)
if _, err = os.Stat(dirPath); os.IsNotExist(err) {
if err = os.MkdirAll(dirPath, 0o755); err != nil {
return nil, fmt.Errorf("mkdir %q: %w", dirPath, err)
}
}

files := strings.Split(string(res.Data), "\n")
for _, file := range files {
if err := writePackage(remote, basePath, filepath.Join(pkgPath, file)); err != nil {
return fmt.Errorf("writepackage: %w", err)
reqs, err := writePackage(remote, basePath, filepath.Join(pkgPath, file))
if err != nil {
return nil, fmt.Errorf("writepackage: %w", err)
}
requirements = append(requirements, reqs...)
}
} else {
// Is File
Expand All @@ -55,17 +57,21 @@ func writePackage(remote, basePath, pkgPath string) error {
targetFilename, _ := gnolang.GetPrecompileFilenameAndTags(filePath)
precompileRes, err := gnolang.Precompile(string(res.Data), "", fileName)
if err != nil {
return fmt.Errorf("precompile: %w", err)
return nil, fmt.Errorf("precompile: %w", err)
}

for _, i := range precompileRes.Imports {
requirements = append(requirements, i.Path.Value)
}

fileNameWithPath := filepath.Join(basePath, dirPath, targetFilename)
err = os.WriteFile(fileNameWithPath, []byte(precompileRes.Translated), 0o644)
if err != nil {
return fmt.Errorf("writefile %q: %w", fileNameWithPath, err)
return nil, fmt.Errorf("writefile %q: %w", fileNameWithPath, err)
}
}

return nil
return removeDuplicateStr(requirements), nil
}

// GnoToGoMod make necessary modifications in the gno.mod
Expand All @@ -76,9 +82,9 @@ func GnoToGoMod(f File) (*File, error) {
return nil, err
}

if strings.HasPrefix(f.Module.Mod.Path, "gno.land/r/") ||
strings.HasPrefix(f.Module.Mod.Path, "gno.land/p/demo/") {
f.Module.Mod.Path = "github.com/gnolang/gno/examples/" + f.Module.Mod.Path
if strings.HasPrefix(f.Module.Mod.Path, gnolang.GnoRealmPkgsPrefixBefore) ||
strings.HasPrefix(f.Module.Mod.Path, gnolang.GnoPackagePrefixBefore) {
f.Module.Mod.Path = gnolang.ImportPrefix + "/examples/" + f.Module.Mod.Path
}

for i := range f.Require {
Expand All @@ -89,9 +95,9 @@ func GnoToGoMod(f File) (*File, error) {
}
}
path := f.Require[i].Mod.Path
if strings.HasPrefix(f.Require[i].Mod.Path, "gno.land/r/") ||
strings.HasPrefix(f.Require[i].Mod.Path, "gno.land/p/demo/") {
f.Require[i].Mod.Path = "github.com/gnolang/gno/examples/" + f.Require[i].Mod.Path
if strings.HasPrefix(f.Require[i].Mod.Path, gnolang.GnoRealmPkgsPrefixBefore) ||
strings.HasPrefix(f.Require[i].Mod.Path, gnolang.GnoPackagePrefixBefore) {
f.Require[i].Mod.Path = gnolang.ImportPrefix + "/examples/" + f.Require[i].Mod.Path
}

f.Replace = append(f.Replace, &modfile.Replace{
Expand Down Expand Up @@ -153,3 +159,14 @@ func isReplaced(module module.Version, repl []*modfile.Replace) (*module.Version
}
return nil, false
}

func removeDuplicateStr(str []string) (res []string) {
m := make(map[string]struct{}, len(str))
for _, s := range str {
if _, ok := m[s]; !ok {
m[s] = struct{}{}
res = append(res, s)
}
}
return
}
30 changes: 15 additions & 15 deletions pkgs/gnolang/precompile.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,12 @@ import (
)

const (
gnoRealmPkgsPrefixBefore = "gno.land/r/"
gnoRealmPkgsPrefixAfter = "github.com/gnolang/gno/examples/gno.land/r/"
gnoPackagePrefixBefore = "gno.land/p/demo/"
gnoPackagePrefixAfter = "github.com/gnolang/gno/examples/gno.land/p/demo/"
gnoStdPkgBefore = "std"
gnoStdPkgAfter = "github.com/gnolang/gno/stdlibs/stdshim"
GnoRealmPkgsPrefixBefore = "gno.land/r/"
GnoRealmPkgsPrefixAfter = "github.com/gnolang/gno/examples/gno.land/r/"
GnoPackagePrefixBefore = "gno.land/p/demo/"
GnoPackagePrefixAfter = "github.com/gnolang/gno/examples/gno.land/p/demo/"
GnoStdPkgBefore = "std"
GnoStdPkgAfter = "github.com/gnolang/gno/stdlibs/stdshim"
)

var stdlibWhitelist = []string{
Expand Down Expand Up @@ -263,11 +263,11 @@ func precompileAST(fset *token.FileSet, f *ast.File, checkWhitelist bool) (ast.N
for _, importSpec := range paragraph {
importPath := strings.TrimPrefix(strings.TrimSuffix(importSpec.Path.Value, `"`), `"`)

if strings.HasPrefix(importPath, gnoRealmPkgsPrefixBefore) {
if strings.HasPrefix(importPath, GnoRealmPkgsPrefixBefore) {
continue
}

if strings.HasPrefix(importPath, gnoPackagePrefixBefore) {
if strings.HasPrefix(importPath, GnoPackagePrefixBefore) {
continue
}

Expand Down Expand Up @@ -303,24 +303,24 @@ func precompileAST(fset *token.FileSet, f *ast.File, checkWhitelist bool) (ast.N
importPath := strings.TrimPrefix(strings.TrimSuffix(importSpec.Path.Value, `"`), `"`)

// std package
if importPath == gnoStdPkgBefore {
if !astutil.RewriteImport(fset, f, gnoStdPkgBefore, gnoStdPkgAfter) {
errs = multierr.Append(errs, fmt.Errorf("failed to replace the %q package with %q", gnoStdPkgBefore, gnoStdPkgAfter))
if importPath == GnoStdPkgBefore {
if !astutil.RewriteImport(fset, f, GnoStdPkgBefore, GnoStdPkgAfter) {
errs = multierr.Append(errs, fmt.Errorf("failed to replace the %q package with %q", GnoStdPkgBefore, GnoStdPkgAfter))
}
}

// p/pkg packages
if strings.HasPrefix(importPath, gnoPackagePrefixBefore) {
target := gnoPackagePrefixAfter + strings.TrimPrefix(importPath, gnoPackagePrefixBefore)
if strings.HasPrefix(importPath, GnoPackagePrefixBefore) {
target := GnoPackagePrefixAfter + strings.TrimPrefix(importPath, GnoPackagePrefixBefore)

if !astutil.RewriteImport(fset, f, importPath, target) {
errs = multierr.Append(errs, fmt.Errorf("failed to replace the %q package with %q", importPath, target))
}
}

// r/realm packages
if strings.HasPrefix(importPath, gnoRealmPkgsPrefixBefore) {
target := gnoRealmPkgsPrefixAfter + strings.TrimPrefix(importPath, gnoRealmPkgsPrefixBefore)
if strings.HasPrefix(importPath, GnoRealmPkgsPrefixBefore) {
target := GnoRealmPkgsPrefixAfter + strings.TrimPrefix(importPath, GnoRealmPkgsPrefixBefore)

if !astutil.RewriteImport(fset, f, importPath, target) {
errs = multierr.Append(errs, fmt.Errorf("failed to replace the %q package with %q", importPath, target))
Expand Down

0 comments on commit bb58088

Please sign in to comment.