Skip to content

Commit

Permalink
feat: copy commands into volume, not into project, for ddev#4416 (dde…
Browse files Browse the repository at this point in the history
  • Loading branch information
GuySartorelli authored Apr 5, 2024
1 parent 02b40b3 commit 59fb0bc
Show file tree
Hide file tree
Showing 11 changed files with 93 additions and 79 deletions.
8 changes: 0 additions & 8 deletions cmd/ddev/cmd/autocompletion_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -239,7 +239,6 @@ func TestAutocompletionForCustomCmds(t *testing.T) {
ddevapp.StopMutagenDaemon()
_ = os.RemoveAll(tmpHome)
_ = fileutil.PurgeDirectory(filepath.Join(site.Dir, ".ddev", "commands"))
_ = fileutil.PurgeDirectory(filepath.Join(site.Dir, ".ddev", ".global_commands"))
})
err = app.Start()
require.NoError(t, err)
Expand All @@ -251,15 +250,12 @@ func TestAutocompletionForCustomCmds(t *testing.T) {

tmpHomeGlobalCommandsDir := filepath.Join(tmpHome, ".ddev", "commands")
projectCommandsDir := app.GetConfigPath("commands")
projectGlobalCommandsCopy := app.GetConfigPath(".global_commands")

// Remove existing commands
err = os.RemoveAll(tmpHomeGlobalCommandsDir)
assert.NoError(err)
err = os.RemoveAll(projectCommandsDir)
assert.NoError(err)
err = os.RemoveAll(projectGlobalCommandsCopy)
assert.NoError(err)
// Copy project and global commands into project
err = fileutil.CopyDir(filepath.Join(testdataCustomCommandsDir, "project_commands"), projectCommandsDir)
assert.NoError(err)
Expand Down Expand Up @@ -317,7 +313,6 @@ func TestAutocompleteTermsForCustomCmds(t *testing.T) {
ddevapp.StopMutagenDaemon()
_ = os.RemoveAll(tmpHome)
_ = fileutil.PurgeDirectory(filepath.Join(site.Dir, ".ddev", "commands"))
_ = fileutil.PurgeDirectory(filepath.Join(site.Dir, ".ddev", ".global_commands"))
})
err = app.Start()
require.NoError(t, err)
Expand All @@ -329,15 +324,12 @@ func TestAutocompleteTermsForCustomCmds(t *testing.T) {

tmpHomeGlobalCommandsDir := filepath.Join(tmpHome, ".ddev", "commands")
projectCommandsDir := app.GetConfigPath("commands")
projectGlobalCommandsCopy := app.GetConfigPath(".global_commands")

// Remove existing commands
err = os.RemoveAll(tmpHomeGlobalCommandsDir)
assert.NoError(err)
err = os.RemoveAll(projectCommandsDir)
assert.NoError(err)
err = os.RemoveAll(projectGlobalCommandsCopy)
assert.NoError(err)
// Copy project and global commands into project
err = fileutil.CopyDir(filepath.Join(testdataCustomCommandsDir, "project_commands"), projectCommandsDir)
assert.NoError(err)
Expand Down
24 changes: 15 additions & 9 deletions cmd/ddev/cmd/commands.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,13 +66,13 @@ func addCustomCommands(rootCmd *cobra.Command) error {

projectCommandPath := app.GetConfigPath("commands")
// Make sure our target global command directory is empty
copiedGlobalCommandPath := app.GetConfigPath(".global_commands")
err = os.MkdirAll(copiedGlobalCommandPath, 0755)
globalCommandPath := filepath.Join(globalconfig.GetGlobalDdevDir(), "commands")
err = os.MkdirAll(globalCommandPath, 0755)
if err != nil {
return err
}

for _, commandSet := range []string{projectCommandPath, copiedGlobalCommandPath} {
for _, commandSet := range []string{projectCommandPath, globalCommandPath} {
commandDirs, err := fileutil.ListFilesInDirFullPath(commandSet)
if err != nil {
return err
Expand All @@ -86,7 +86,7 @@ func addCustomCommands(rootCmd *cobra.Command) error {
if err != nil {
return err
}
err = addCustomCommandsFromDir(rootCmd, app, serviceDirOnHost, commandFiles, commandSet == copiedGlobalCommandPath, commandsAdded)
err = addCustomCommandsFromDir(rootCmd, app, serviceDirOnHost, commandFiles, commandSet == globalCommandPath, commandsAdded)
if err != nil {
return err
}
Expand All @@ -102,9 +102,6 @@ func addCustomCommandsFromDir(rootCmd *cobra.Command, app *ddevapp.DdevApp, serv
var err error

for _, commandName := range commandFiles {
// Use path.Join() for the inContainerFullPath because it's about the path in the container, not on the
// host; a Windows path is not useful here.
inContainerFullPath := path.Join("/mnt/ddev_config", filepath.Base(filepath.Dir(serviceDirOnHost)), service, commandName)
onHostFullPath := filepath.Join(serviceDirOnHost, commandName)

if strings.HasSuffix(commandName, ".example") || strings.HasPrefix(commandName, "README") || strings.HasPrefix(commandName, ".") || fileutil.IsDirectory(onHostFullPath) {
Expand All @@ -117,6 +114,8 @@ func addCustomCommandsFromDir(rootCmd *cobra.Command, app *ddevapp.DdevApp, serv
}

// Any command we find will want to be executable on Linux
// Note that this only affects host commands and project-level commands.
// Global container commands are made executable on `ddev start` instead.
_ = os.Chmod(onHostFullPath, 0755)
if hasCR, _ := fileutil.FgrepStringInFile(onHostFullPath, "\r\n"); hasCR {
util.Warning("Command '%s' contains CRLF, please convert to Linux-style linefeeds with dos2unix or another tool, skipping %s", commandName, onHostFullPath)
Expand Down Expand Up @@ -275,16 +274,23 @@ func addCustomCommandsFromDir(rootCmd *cobra.Command, app *ddevapp.DdevApp, serv
commandToAdd.ValidArgsFunction = makeHostCompletionFunc(autocompletePathOnHost, commandToAdd)
}
} else {
// Use path.Join() for the container path because it's about the path in the container, not on the
// host; a Windows path is not useful here.
containerBasePath := path.Join("/mnt/ddev_config", filepath.Base(filepath.Dir(serviceDirOnHost)), service)
if strings.HasPrefix(serviceDirOnHost, globalconfig.GetGlobalDdevDir()) {
containerBasePath = path.Join("/mnt/ddev-global-cache/global-commands/", service)
}
inContainerFullPath := path.Join(containerBasePath, commandName)
commandToAdd.Run = makeContainerCmd(app, inContainerFullPath, commandName, service, execRaw, relative)
if fileutil.FileExists(autocompletePathOnHost) {
// Make sure autocomplete script can be executed
_ = os.Chmod(autocompletePathOnHost, 0755)
if hasCR, _ := fileutil.FgrepStringInFile(autocompletePathOnHost, "\r\n"); hasCR {
util.Warning("Command '%s' contains CRLF, please convert to Linux-style linefeeds with dos2unix or another tool, skipping %s", commandName, onHostFullPath)
util.Warning("Autocomplete script for command '%s' contains CRLF, please convert to Linux-style linefeeds with dos2unix or another tool, skipping %s", commandName, autocompletePathOnHost)
continue
}
// Add autocomplete script
autocompletePathInContainer := path.Join("/mnt/ddev_config", filepath.Base(filepath.Dir(serviceDirOnHost)), service, "autocomplete", commandName)
autocompletePathInContainer := path.Join(containerBasePath, "autocomplete", commandName)
commandToAdd.ValidArgsFunction = makeContainerCompletionFunc(autocompletePathInContainer, service, app, commandToAdd)
}
}
Expand Down
24 changes: 12 additions & 12 deletions cmd/ddev/cmd/commands_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"fmt"
"os"
osexec "os/exec"
"path"
"path/filepath"
"runtime"
"strings"
Expand Down Expand Up @@ -53,8 +54,8 @@ func TestCustomCommands(t *testing.T) {
assert.NoError(err)
_ = os.RemoveAll(tmpHome)
_ = fileutil.PurgeDirectory(filepath.Join(site.Dir, ".ddev", "commands"))
_ = fileutil.PurgeDirectory(filepath.Join(site.Dir, ".ddev", ".global_commands"))
})
// We must start the app before copying commands, so they don't get copied over
err = app.Start()
require.NoError(t, err)

Expand All @@ -67,8 +68,6 @@ func TestCustomCommands(t *testing.T) {
assert.NoError(err)

projectCommandsDir := app.GetConfigPath("commands")
projectGlobalCommandsCopy := app.GetConfigPath(".global_commands")
_ = os.RemoveAll(projectGlobalCommandsCopy)
err = fileutil.CopyDir(filepath.Join(testdataCustomCommandsDir, "global_commands"), tmpHomeGlobalCommandsDir)
require.NoError(t, err)

Expand Down Expand Up @@ -122,8 +121,6 @@ func TestCustomCommands(t *testing.T) {

err = os.RemoveAll(projectCommandsDir)
assert.NoError(err)
err = os.RemoveAll(projectGlobalCommandsCopy)
assert.NoError(err)

// Now copy a project commands and global commands and make sure they show up and execute properly
err = fileutil.CopyDir(filepath.Join(testdataCustomCommandsDir, "project_commands"), projectCommandsDir)
Expand Down Expand Up @@ -156,7 +153,7 @@ func TestCustomCommands(t *testing.T) {
homeEnv := os.Getenv("HOME")
t.Errorf("userHome=%s, globalDdevDir=%s, homeEnv=%s", userHome, globalDdevDir, homeEnv)
t.Errorf("Failed to run ddev %s: %v, home=%s output=%s", c, err, userHome, out)
out, err = exec.RunHostCommand("ls", "-lR", globalDdevDir, "comamnds")
out, err = exec.RunHostCommand("ls", "-lR", globalDdevDir, "commands")
assert.NoError(err)
t.Errorf("Commands dir: %s", out)
}
Expand Down Expand Up @@ -287,10 +284,14 @@ func TestCustomCommands(t *testing.T) {
assert.NoError(err)
}

// Make sure that the non-command stuff we installed has been copied into projectGlobalCommandsCopy
for _, f := range []string{".gitattributes", "db/mysqldump.example", "db/README.txt", "host/heidisql", "host/mysqlworkbench.example", "host/phpstorm.example", "host/README.txt", "host/sequelace", "host/sequelpro", "host/tableplus", "host/dbeaver", "host/querious", "web/README.txt"} {
assert.FileExists(filepath.Join(projectGlobalCommandsCopy, f))
// Make sure that the non-command stuff we installed has been copied into /mnt/ddev-global-cache
commandDirInVolume := "/mnt/ddev-global-cache/global-commands/"
for _, f := range []string{".gitattributes", "db/mysqldump.example", "db/README.txt", "web/README.txt"} {
filePathInVolume := path.Join(commandDirInVolume, f)
out, err = exec.RunHostCommand(DdevBin, "exec", "[ -f "+filePathInVolume+" ] && exit 0 || exit 1")
assert.NoError(err, filePathInVolume+" does not exist, output=%s", out)
}

// Make sure that the non-command stuff we installed is in project commands dir
for _, f := range []string{".gitattributes", "db/README.txt", "host/README.txt", "host/solrtail.example", "solr/README.txt", "solr/solrtail.example", "web/README.txt"} {
assert.FileExists(filepath.Join(projectCommandsDir, f))
Expand All @@ -305,7 +306,6 @@ func TestCustomCommands(t *testing.T) {
cmdPath := app.GetConfigPath(filepath.Join("commands", command))
assert.False(fileutil.FileExists(cmdPath), "file %s exists but it should not", cmdPath)
}

}

// TestLaunchCommand tests that the launch command behaves all the ways it should behave
Expand Down Expand Up @@ -396,7 +396,7 @@ func TestMysqlCommand(t *testing.T) {
require.NoError(t, err)

// This populates the project's
// .ddev/.global_commands which otherwise doesn't get done until ddev start
// /mnt/ddev-global-cache/global-commands/ which otherwise doesn't get done until ddev start
// This matters when --no-bind-mount=true
_, err = exec.RunHostCommand("ddev")
assert.NoError(err)
Expand Down Expand Up @@ -439,7 +439,7 @@ func TestPsqlCommand(t *testing.T) {
require.NoError(t, err)

// This populates the project's
// .ddev/.global_commands which otherwise doesn't get done until ddev start
// /mnt/ddev-global-cache/global-commands/ which otherwise doesn't get done until ddev start
// This matters when --no-bind-mount=true
_, err = exec.RunHostCommand("ddev")
assert.NoError(err)
Expand Down
18 changes: 3 additions & 15 deletions cmd/ddev/cmd/debug-fixcommands.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,31 +2,19 @@ package cmd

import (
"github.com/ddev/ddev/pkg/ddevapp"
"github.com/ddev/ddev/pkg/globalconfig"
"github.com/ddev/ddev/pkg/util"
"github.com/spf13/cobra"
)

// DebugFixCommandsCmd fixes up custom/shell commands without having to do a ddev start
// DebugFixCommandsCmd fixes up global container commands without having to do a ddev start
var DebugFixCommandsCmd = &cobra.Command{
Use: "fix-commands",
Short: "Fix up custom/shell commands without running ddev start",
Short: "Fix up global container commands without running ddev start",
Run: func(_ *cobra.Command, _ []string) {
app, err := ddevapp.GetActiveApp("")
if err != nil {
util.Failed("Can't find active project: %v", err)
}
err = ddevapp.PopulateCustomCommandFiles(app)
err := ddevapp.PopulateGlobalCustomCommandFiles()
if err != nil {
util.Warning("Failed to populate custom command files: %v", err)
}
// If no-bind-mounts we have to do a start to push the commands back in there again.
if globalconfig.DdevGlobalConfig.NoBindMounts {
err = app.Start()
if err != nil {
util.Failed("Failed to restart with NoBindMounts set: %v", err)
}
}
},
}

Expand Down
4 changes: 2 additions & 2 deletions docs/content/users/extend/custom-commands.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,11 +36,11 @@ open -a PhpStorm.app ${DDEV_APPROOT}

To provide a command which will execute in a container, add a Bash script to `.ddev/commands/<container_name>`, for example, `.ddev/commands/web/mycommand`. The Bash script will be executed inside the named container. For example, see the [several standard DDEV script-based web container commands](https://github.com/ddev/ddev/blob/master/pkg/ddevapp/global_dotddev_assets/commands/web).

You can run commands in custom containers as well as standard DDEV `web` and `db` containers. Use the service name, like `.ddev/commands/solr/<command>`. The only catch with a custom container is that your service must mount `/mnt/ddev_config` like the `web` and `db` containers do; the `volumes` section of `docker-compose.<servicename>.yaml` needs:
You can run commands in custom containers as well as standard DDEV `web` and `db` containers. Use the service name, like `.ddev/commands/solr/<command>`. The only catch with a custom container is that your service must mount `/mnt/ddev-global-cache` like the `web` and `db` containers do; the `volumes` section of `docker-compose.<servicename>.yaml` needs:

```
volumes:
- ".:/mnt/ddev_config"
- ddev-global-cache:/mnt/ddev-global-cache
```

For example, to add a `solrtail` command that runs in a Solr service, add `.ddev/commands/solr/solrtail` with:
Expand Down
1 change: 1 addition & 0 deletions docs/content/users/extend/custom-compose-files.md
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,7 @@ services:
- .:/mnt/ddev_config
# `ddev-global-cache` gets mounted so we have the CAROOT
# This is required so that the CA is available for `mkcert` to install
# and for custom commands to work
- ddev-global-cache:/mnt/ddev-global-cache
```
Expand Down
3 changes: 0 additions & 3 deletions docs/content/users/usage/architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -92,9 +92,6 @@ Files beginning with `.` are hidden because they shouldn’t be fiddled with; mo
`.gitignore`
: The `.gitignore` is generated by DDEV and should generally not be edited or checked in. (It gitignores itself to make sure you don’t check it in.) It’s generated on every `ddev start` and will change as DDEV versions change, so if you check it in by accident it will always be showing changes that you don’t need to see in `git status`.

`.global_commands`
: Temporary directory used to get global commands available inside a project. You shouldn’t ever have to look there.

`.homeadditions`
: Temporary directory used to consolidate global `homeadditions` with project-level `homeadditions`. You shouldn’t ever have to look here.

Expand Down
65 changes: 43 additions & 22 deletions pkg/ddevapp/commands.go
Original file line number Diff line number Diff line change
@@ -1,47 +1,68 @@
package ddevapp

import (
"github.com/ddev/ddev/pkg/fileutil"
"github.com/ddev/ddev/pkg/globalconfig"
"github.com/ddev/ddev/pkg/util"
copy2 "github.com/otiai10/copy"
"fmt"
"os"
"path/filepath"

dockerImages "github.com/ddev/ddev/pkg/docker"
"github.com/ddev/ddev/pkg/dockerutil"
"github.com/ddev/ddev/pkg/globalconfig"
"github.com/ddev/ddev/pkg/nodeps"
"github.com/ddev/ddev/pkg/util"
)

// PopulateCustomCommandFiles sets up the custom command files in the project
// PopulateGlobalCustomCommandFiles sets up the custom command files in the project
// directories where they need to go.
func PopulateCustomCommandFiles(app *DdevApp) error {

func PopulateGlobalCustomCommandFiles() error {
sourceGlobalCommandPath := filepath.Join(globalconfig.GetGlobalDdevDir(), "commands")
err := os.MkdirAll(sourceGlobalCommandPath, 0755)
if err != nil {
return nil
}

projectCommandPath := app.GetConfigPath("commands")
// Make sure our target global command directory is empty
copiedGlobalCommandPath := app.GetConfigPath(".global_commands")
err = os.MkdirAll(copiedGlobalCommandPath, 0755)
// Remove contents of the directory, if the directory exists and has some contents
commandDirInVolume := "/mnt/ddev-global-cache/global-commands/"
_, _, err = performTaskInContainer([]string{"rm", "-rf", commandDirInVolume})
if err != nil {
util.Error("Unable to create directory %s: %v", copiedGlobalCommandPath, err)
return nil
return fmt.Errorf("unable to rm %s: %v", commandDirInVolume, err)
}

// Make sure it's empty
err = fileutil.PurgeDirectory(copiedGlobalCommandPath)
// Copy commands into container (this will create the directory if it's not there already)
uid, _, _ := util.GetContainerUIDGid()
err = dockerutil.CopyIntoVolume(sourceGlobalCommandPath, "ddev-global-cache", "global-commands", uid, "host", false)
if err != nil {
util.Error("Unable to remove %s: %v", copiedGlobalCommandPath, err)
return nil
return err
}

err = copy2.Copy(sourceGlobalCommandPath, copiedGlobalCommandPath)
// Make sure all commands can be executed
_, stderr, err := performTaskInContainer([]string{"sh", "-c", "chmod -R u+rwx " + commandDirInVolume})
if err != nil {
return err
return fmt.Errorf("unable to chmod %s: %v (stderr=%s)", commandDirInVolume, err, stderr)
}

if !fileutil.FileExists(projectCommandPath) || !fileutil.IsDirectory(projectCommandPath) {
return nil
}
return nil
}

// performTaskInContainer runs a command in the web container if it's available,
// but uses an anonymous container otherwise.
func performTaskInContainer(command []string) (string, string, error) {
app, err := GetActiveApp("")
if err == nil {
status, _ := app.SiteStatus()
if status == SiteRunning {
// Prepare docker exec command
opts := &ExecOpts{
RawCmd: command,
Tty: false,
NoCapture: false,
}
return app.Exec(opts)
}
}

// If there is no running active site, use an anonymous container instead.
containerName := "performTaskInContainer" + nodeps.RandomString(12)
uid, _, _ := util.GetContainerUIDGid()
return dockerutil.RunSimpleContainer(dockerImages.GetWebImage(), containerName, command, nil, nil, []string{"ddev-global-cache:/mnt/ddev-global-cache"}, uid, true, false, map[string]string{"com.ddev.site-name": ""}, nil)
}
13 changes: 11 additions & 2 deletions pkg/ddevapp/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -667,7 +667,6 @@ func (app *DdevApp) CheckDeprecations() {

// FixObsolete removes files that may be obsolete, etc.
func (app *DdevApp) FixObsolete() {

// Remove old in-project commands (which have been moved to global)
for _, command := range []string{"db/mysql", "host/launch", "web/xdebug"} {
cmdPath := app.GetConfigPath(filepath.Join("commands", command))
Expand Down Expand Up @@ -713,6 +712,16 @@ func (app *DdevApp) FixObsolete() {
}
}
}

// Remove old .global_commands directory
legacyCommandDir := app.GetConfigPath(".global_commands")
if fileutil.IsDirectory(legacyCommandDir) {
err := os.RemoveAll(legacyCommandDir)
if err != nil {
util.Warning("attempted to remove %s but failed, you may want to remove it manually: %v", legacyCommandDir, err)
}
}

}

type composeYAMLVars struct {
Expand Down Expand Up @@ -1351,7 +1360,7 @@ func PrepDdevDirectory(app *DdevApp) error {
return err
}

err = CreateGitIgnore(dir, "**/*.example", ".dbimageBuild", ".dbimageExtra", ".ddev-docker-*.yaml", ".*downloads", ".global_commands", ".homeadditions", ".importdb*", ".sshimageBuild", ".venv", ".webimageBuild", ".webimageExtra", "apache/apache-site.conf", "commands/.gitattributes", "commands/db/mysql", "commands/host/launch", "commands/web/xdebug", "commands/web/live", "config.local.y*ml", "db_snapshots", "import-db", "import.yaml", "mutagen/mutagen.yml", "mutagen/.start-synced", "nginx_full/nginx-site.conf", "postgres/postgresql.conf", "providers/acquia.yaml", "providers/lagoon.yaml", "providers/platform.yaml", "providers/upsun.yaml", "sequelpro.spf", "settings/settings.ddev.py", fmt.Sprintf("traefik/config/%s.yaml", app.Name), fmt.Sprintf("traefik/certs/%s.crt", app.Name), fmt.Sprintf("traefik/certs/%s.key", app.Name), "xhprof/xhprof_prepend.php", "**/README.*")
err = CreateGitIgnore(dir, "**/*.example", ".dbimageBuild", ".dbimageExtra", ".ddev-docker-*.yaml", ".*downloads", ".homeadditions", ".importdb*", ".sshimageBuild", ".venv", ".webimageBuild", ".webimageExtra", "apache/apache-site.conf", "commands/.gitattributes", "commands/db/mysql", "commands/host/launch", "commands/web/xdebug", "commands/web/live", "config.local.y*ml", "db_snapshots", "import-db", "import.yaml", "mutagen/mutagen.yml", "mutagen/.start-synced", "nginx_full/nginx-site.conf", "postgres/postgresql.conf", "providers/acquia.yaml", "providers/lagoon.yaml", "providers/platform.yaml", "providers/upsun.yaml", "sequelpro.spf", "settings/settings.ddev.py", fmt.Sprintf("traefik/config/%s.yaml", app.Name), fmt.Sprintf("traefik/certs/%s.crt", app.Name), fmt.Sprintf("traefik/certs/%s.key", app.Name), "xhprof/xhprof_prepend.php", "**/README.*")
if err != nil {
return fmt.Errorf("failed to create gitignore in %s: %v", dir, err)
}
Expand Down
Loading

0 comments on commit 59fb0bc

Please sign in to comment.