Skip to content

Commit

Permalink
Add magefiles directory support (#405)
Browse files Browse the repository at this point in the history 
* Use magefolder if no directory set and it exists.

If no directory was passed by the user as an explicit option and there is a folder named "magefolder" use that.

Workdir is kept as it is likely still "."

* Remove the default . for -d flag

Also correct os.Stat error checking to expect no error

* Add tests and test data for magefolder

* Rename magefolder and accept untagged files

Magefolder was renamed to magefiles
We now accept files that are not tagged too when using a magefiles directory

* Assume tagging when mix tagging is present

When using magefiles directory, if there are mixed tagging files assume tagging is used for mage files

* Update error format to %v

We support building for older go versions so error formatting should use %v

* sort outputs

* Accept mixed tagging in magefiles folder

When mixed tagging is found within a magefiles folder, opt to use all files

* little tweak to only do go list once when using magefiles directory

* Add magefiles directory information to the website

* Add a preference for mage files over directories

Add a temporary preference for mage files over magefiles directories and warn users this is a temporary functionality leading to a change where directory will be preferred.

Co-authored-by: Nate Finch <[email protected]>
  • Loading branch information
@perrito666 @natefinch
perrito666 and natefinch authored Mar 16, 2022
1 parent 526bf47 commit 3504e09
Show file tree
Hide file tree
Showing 15 changed files with 314 additions and 48 deletions.
140 changes: 97 additions & 43 deletions mage/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,15 @@ type Invocation struct {
HashFast bool // don't rely on GOCACHE, just hash the magefiles
}

// MagefilesDirName is the name of the default folder to look for if no directory was specified,
// if this folder exists it will be assumed mage package lives inside it.
const MagefilesDirName = "magefiles"

// UsesMagefiles returns true if we are getting our mage files from a magefiles directory.
func (i Invocation) UsesMagefiles() bool {
return i.Dir == MagefilesDirName
}

// ParseAndRun parses the command line, and then compiles and runs the mage
// files in the given directory with the given args (do not include the command
// name in the args).
Expand Down Expand Up @@ -180,7 +189,7 @@ func Parse(stderr, stdout io.Writer, args []string) (inv Invocation, cmd Command
fs.BoolVar(&inv.Help, "h", false, "show this help")
fs.DurationVar(&inv.Timeout, "t", 0, "timeout in duration parsable format (e.g. 5m30s)")
fs.BoolVar(&inv.Keep, "keep", false, "keep intermediate mage files around after running")
fs.StringVar(&inv.Dir, "d", ".", "directory to read magefiles from")
fs.StringVar(&inv.Dir, "d", "", "directory to read magefiles from")
fs.StringVar(&inv.WorkDir, "w", "", "working directory where magefiles will run")
fs.StringVar(&inv.GoCmd, "gocmd", mg.GoCmd(), "use the given go binary to compile the output")
fs.StringVar(&inv.GOOS, "goos", "", "set GOOS for binary produced with -compile")
Expand Down Expand Up @@ -216,7 +225,7 @@ Commands:
Options:
-d <string>
directory to read magefiles from (default ".")
directory to read magefiles from (default "." or "magefiles" if exists)
-debug turn on debug messages
-f force recreation of compiled magefile
-goarch sets the GOARCH for the binary created by -compile (default: current arch)
Expand Down Expand Up @@ -296,23 +305,50 @@ Options:
return inv, cmd, err
}

const dotDirectory = "."

// Invoke runs Mage with the given arguments.
func Invoke(inv Invocation) int {
errlog := log.New(inv.Stderr, "", 0)
if inv.GoCmd == "" {
inv.GoCmd = "go"
}
var noDir bool
if inv.Dir == "" {
inv.Dir = "."
noDir = true
inv.Dir = dotDirectory
// . will be default unless we find a mage folder.
mfSt, err := os.Stat(MagefilesDirName)
if err == nil {
if mfSt.IsDir() {
stderrBuf := &bytes.Buffer{}
inv.Dir = MagefilesDirName // preemptive assignment
// TODO: Remove this fallback and the above Magefiles invocation when the bw compatibility is removed.
files, err := Magefiles(dotDirectory, inv.GOOS, inv.GOARCH, inv.GoCmd, stderrBuf, false, inv.Debug)
if err == nil {
if len(files) != 0 {
errlog.Println("[WARNING] You have both a magefiles directory and mage files in the " +
"current directory, in future versions the files will be ignored in favor of the directory")
inv.Dir = dotDirectory
}
}
}
}
}

if inv.WorkDir == "" {
inv.WorkDir = inv.Dir
if noDir {
inv.WorkDir = dotDirectory
} else {
inv.WorkDir = inv.Dir
}
}

if inv.CacheDir == "" {
inv.CacheDir = mg.CacheDir()
}

files, err := Magefiles(inv.Dir, inv.GOOS, inv.GOARCH, inv.GoCmd, inv.Stderr, inv.Debug)
files, err := Magefiles(inv.Dir, inv.GOOS, inv.GOARCH, inv.GoCmd, inv.Stderr, inv.UsesMagefiles(), inv.Debug)
if err != nil {
errlog.Println("Error determining list of magefiles:", err)
return 1
Expand Down Expand Up @@ -432,69 +468,87 @@ type mainfileTemplateData struct {
BinaryName string
}

func listGoFiles(magePath, goCmd, tags string, env []string) ([]string, error) {
args := []string{"list"}
if tags != "" {
args = append(args, fmt.Sprintf("-tags=%s", tags))
}
args = append(args, "-e", "-f", `{{join .GoFiles "||"}}`)
cmd := exec.Command(goCmd, args...)
cmd.Env = env
buf := &bytes.Buffer{}
cmd.Stderr = buf
cmd.Dir = magePath
b, err := cmd.Output()
if err != nil {
stderr := buf.String()
// if the error is "cannot find module", that can mean that there's no
// non-mage files, which is fine, so ignore it.
if !strings.Contains(stderr, "cannot find module for path") {
if tags == "" {
return nil, fmt.Errorf("failed to list un-tagged gofiles: %v: %s", err, stderr)
}
return nil, fmt.Errorf("failed to list gofiles tagged with %q: %v: %s", tags, err, stderr)
}
}
out := strings.TrimSpace(string(b))
list := strings.Split(out, "||")
for i := range list {
list[i] = filepath.Join(magePath, list[i])
}
return list, nil
}

// Magefiles returns the list of magefiles in dir.
func Magefiles(magePath, goos, goarch, goCmd string, stderr io.Writer, isDebug bool) ([]string, error) {
func Magefiles(magePath, goos, goarch, goCmd string, stderr io.Writer, isMagefilesDirectory, isDebug bool) ([]string, error) {
start := time.Now()
defer func() {
debug.Println("time to scan for Magefiles:", time.Since(start))
}()
fail := func(err error) ([]string, error) {
return nil, err
}

env, err := internal.EnvWithGOOS(goos, goarch)
if err != nil {
return nil, err
}

debug.Println("getting all non-mage files in", magePath)
debug.Println("getting all files including those with mage tag in", magePath)
mageFiles, err := listGoFiles(magePath, goCmd, "mage", env)
if err != nil {
return nil, fmt.Errorf("listing mage files: %v", err)
}

// // first, grab all the files with no build tags specified.. this is actually
// // our exclude list of things without the mage build tag.
cmd := exec.Command(goCmd, "list", "-e", "-f", `{{join .GoFiles "||"}}`)
cmd.Env = env
buf := &bytes.Buffer{}
cmd.Stderr = buf
cmd.Dir = magePath
b, err := cmd.Output()
if isMagefilesDirectory {
// For the magefiles directory, we always use all go files, both with
// and without the mage tag, as per normal go build tag rules.
debug.Println("using all go files in magefiles directory", magePath)
return mageFiles, nil
}

// For folders other than the magefiles directory, we only consider files
// that have the mage build tag and ignore those that don't.

debug.Println("getting all files without mage tag in", magePath)
nonMageFiles, err := listGoFiles(magePath, goCmd, "", env)
if err != nil {
stderr := buf.String()
// if the error is "cannot find module", that can mean that there's no
// non-mage files, which is fine, so ignore it.
if !strings.Contains(stderr, "cannot find module for path") {
return fail(fmt.Errorf("failed to list non-mage gofiles: %v: %s", err, stderr))
}
return nil, fmt.Errorf("listing non-mage files: %v", err)
}
list := strings.TrimSpace(string(b))
debug.Println("found non-mage files", list)

// convert non-Mage list to a map of files to exclude.
exclude := map[string]bool{}
for _, f := range strings.Split(list, "||") {
for _, f := range nonMageFiles {
if f != "" {
debug.Printf("marked file as non-mage: %q", f)
exclude[f] = true
}
}
debug.Println("getting all files plus mage files")
cmd = exec.Command(goCmd, "list", "-tags=mage", "-e", "-f", `{{join .GoFiles "||"}}`)
cmd.Env = env

buf.Reset()
cmd.Dir = magePath
b, err = cmd.Output()
if err != nil {
return fail(fmt.Errorf("failed to list mage gofiles: %v: %s", err, buf.Bytes()))
}

list = strings.TrimSpace(string(b))
files := []string{}
for _, f := range strings.Split(list, "||") {
// filter out the non-mage files from the mage files.
var files []string
for _, f := range mageFiles {
if f != "" && !exclude[f] {
files = append(files, f)
}
}
for i := range files {
files[i] = filepath.Join(magePath, files[i])
}
return files, nil
}

Expand Down
144 changes: 139 additions & 5 deletions mage/main_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,7 @@ func TestTransitiveHashFast(t *testing.T) {

func TestListMagefilesMain(t *testing.T) {
buf := &bytes.Buffer{}
files, err := Magefiles("testdata/mixed_main_files", "", "", "go", buf, false)
files, err := Magefiles("testdata/mixed_main_files", "", "", "go", buf, false, false)
if err != nil {
t.Errorf("error from magefile list: %v: %s", err, buf)
}
Expand All @@ -210,7 +210,7 @@ func TestListMagefilesIgnoresGOOS(t *testing.T) {
os.Setenv("GOOS", "windows")
}
defer os.Setenv("GOOS", runtime.GOOS)
files, err := Magefiles("testdata/goos_magefiles", "", "", "go", buf, false)
files, err := Magefiles("testdata/goos_magefiles", "", "", "go", buf, false, false)
if err != nil {
t.Errorf("error from magefile list: %v: %s", err, buf)
}
Expand All @@ -234,7 +234,7 @@ func TestListMagefilesIgnoresRespectsGOOSArg(t *testing.T) {
goos = "windows"
}
// Set GOARCH as amd64 because windows is not on all non-x86 architectures.
files, err := Magefiles("testdata/goos_magefiles", goos, "amd64", "go", buf, false)
files, err := Magefiles("testdata/goos_magefiles", goos, "amd64", "go", buf, false, false)
if err != nil {
t.Errorf("error from magefile list: %v: %s", err, buf)
}
Expand Down Expand Up @@ -308,7 +308,7 @@ func TestCompileDiffGoosGoarch(t *testing.T) {

func TestListMagefilesLib(t *testing.T) {
buf := &bytes.Buffer{}
files, err := Magefiles("testdata/mixed_lib_files", "", "", "go", buf, false)
files, err := Magefiles("testdata/mixed_lib_files", "", "", "go", buf, false, false)
if err != nil {
t.Errorf("error from magefile list: %v: %s", err, buf)
}
Expand Down Expand Up @@ -342,6 +342,140 @@ func TestMixedMageImports(t *testing.T) {
}
}

func TestMagefilesFolder(t *testing.T) {
resetTerm()
wd, err := os.Getwd()
t.Log(wd)
if err != nil {
t.Fatalf("finding current working directory: %v", err)
}
if err := os.Chdir("testdata/with_magefiles_folder"); err != nil {
t.Fatalf("changing to magefolders tests data: %v", err)
}
// restore previous state
defer os.Chdir(wd)

stderr := &bytes.Buffer{}
stdout := &bytes.Buffer{}
inv := Invocation{
Dir: "",
Stdout: stdout,
Stderr: stderr,
List: true,
}
code := Invoke(inv)
if code != 0 {
t.Errorf("expected to exit with code 0, but got %v, stderr: %s", code, stderr)
}
expected := "Targets:\n build \n"
actual := stdout.String()
if actual != expected {
t.Fatalf("expected %q but got %q", expected, actual)
}
}

func TestMagefilesFolderMixedWithMagefiles(t *testing.T) {
resetTerm()
wd, err := os.Getwd()
t.Log(wd)
if err != nil {
t.Fatalf("finding current working directory: %v", err)
}
if err := os.Chdir("testdata/with_magefiles_folder_and_mage_files_in_dot"); err != nil {
t.Fatalf("changing to magefolders tests data: %v", err)
}
// restore previous state
defer os.Chdir(wd)

stderr := &bytes.Buffer{}
stdout := &bytes.Buffer{}
inv := Invocation{
Dir: "",
Stdout: stdout,
Stderr: stderr,
List: true,
}
code := Invoke(inv)
if code != 0 {
t.Errorf("expected to exit with code 0, but got %v, stderr: %s", code, stderr)
}
expected := "Targets:\n build \n"
actual := stdout.String()
if actual != expected {
t.Fatalf("expected %q but got %q", expected, actual)
}

expectedErr := "[WARNING] You have both a magefiles directory and mage files in the current directory, in future versions the files will be ignored in favor of the directory\n"
actualErr := stderr.String()
if actualErr != expectedErr {
t.Fatalf("expected Warning %q but got %q", expectedErr, actualErr)
}
}

func TestUntaggedMagefilesFolder(t *testing.T) {
resetTerm()
wd, err := os.Getwd()
t.Log(wd)
if err != nil {
t.Fatalf("finding current working directory: %v", err)
}
if err := os.Chdir("testdata/with_untagged_magefiles_folder"); err != nil {
t.Fatalf("changing to magefolders tests data: %v", err)
}
// restore previous state
defer os.Chdir(wd)

stderr := &bytes.Buffer{}
stdout := &bytes.Buffer{}
inv := Invocation{
Dir: "",
Stdout: stdout,
Stderr: stderr,
List: true,
}
code := Invoke(inv)
if code != 0 {
t.Errorf("expected to exit with code 0, but got %v, stderr: %s", code, stderr)
}
expected := "Targets:\n build \n"
actual := stdout.String()
if actual != expected {
t.Fatalf("expected %q but got %q", expected, actual)
}
}

func TestMixedTaggingMagefilesFolder(t *testing.T) {
resetTerm()
wd, err := os.Getwd()
t.Log(wd)
if err != nil {
t.Fatalf("finding current working directory: %v", err)
}
if err := os.Chdir("testdata/with_mixtagged_magefiles_folder"); err != nil {
t.Fatalf("changing to magefolders tests data: %v", err)
}
// restore previous state
defer os.Chdir(wd)

stderr := &bytes.Buffer{}
stdout := &bytes.Buffer{}
inv := Invocation{
Dir: "",
Stdout: stdout,
Stderr: stderr,
List: true,
}
code := Invoke(inv)
if code != 0 {
t.Errorf("expected to exit with code 0, but got %v, stderr: %s", code, stderr)
}
expected := "Targets:\n build \n untaggedBuild \n"
actual := stdout.String()
if actual != expected {
t.Fatalf("expected %q but got %q", expected, actual)
}
}

func TestGoRun(t *testing.T) {
c := exec.Command("go", "run", "main.go")
c.Dir = "./testdata"
Expand Down Expand Up @@ -1615,7 +1749,7 @@ func TestWrongDependency(t *testing.T) {
}
}

/// This code liberally borrowed from https://github.com/rsc/goversion/blob/master/version/exe.go
// / This code liberally borrowed from https://github.com/rsc/goversion/blob/master/version/exe.go

type (
exeType int
Expand Down
Loading

0 comments on commit 3504e09

Please sign in to comment.