From f454c31697312780a3edcb36d39d5b374deb415b Mon Sep 17 00:00:00 2001 From: pmahindrakar-oss Date: Mon, 7 Nov 2022 23:27:42 +0530 Subject: [PATCH] Added dryRun flag for printing docker command for demo and sandbox (#367) * Added printCommand flag for printing docker command for demo and sandbox Signed-off-by: pmahindrakar-oss * test fixes Signed-off-by: pmahindrakar-oss * Added pull image command aswell Signed-off-by: pmahindrakar-oss * using dryRun Signed-off-by: pmahindrakar-oss * flytectl config export Signed-off-by: pmahindrakar-oss Signed-off-by: pmahindrakar-oss --- cmd/config/subcommand/sandbox/config_flags.go | 1 + .../subcommand/sandbox/config_flags_test.go | 14 +++++ .../subcommand/sandbox/sandbox_config.go | 3 +- cmd/demo/start.go | 6 ++ cmd/sandbox/sandbox.go | 6 ++ pkg/docker/docker_util.go | 62 +++++++++++++++++-- pkg/docker/docker_util_test.go | 16 ++--- pkg/sandbox/start.go | 40 ++++++------ pkg/util/util.go | 27 +++++--- pkg/util/util_test.go | 2 +- 10 files changed, 136 insertions(+), 41 deletions(-) diff --git a/cmd/config/subcommand/sandbox/config_flags.go b/cmd/config/subcommand/sandbox/config_flags.go index eaa7c88091..e9fbd5a60e 100755 --- a/cmd/config/subcommand/sandbox/config_flags.go +++ b/cmd/config/subcommand/sandbox/config_flags.go @@ -59,5 +59,6 @@ func (cfg Config) GetPFlagSet(prefix string) *pflag.FlagSet { cmdFlags.StringVar(&DefaultConfig.ImagePullOptions.RegistryAuth, fmt.Sprintf("%v%v", prefix, "imagePullOptions.registryAuth"), DefaultConfig.ImagePullOptions.RegistryAuth, "The base64 encoded credentials for the registry.") cmdFlags.StringVar(&DefaultConfig.ImagePullOptions.Platform, fmt.Sprintf("%v%v", prefix, "imagePullOptions.platform"), DefaultConfig.ImagePullOptions.Platform, "Forces a specific platform's image to be pulled.'") cmdFlags.BoolVar(&DefaultConfig.Dev, fmt.Sprintf("%v%v", prefix, "dev"), DefaultConfig.Dev, "Optional. Only start minio and postgres in the sandbox.") + cmdFlags.BoolVar(&DefaultConfig.DryRun, fmt.Sprintf("%v%v", prefix, "dryRun"), DefaultConfig.DryRun, "Optional. Only print the docker commands to bring up flyte sandbox/demo container.This will still call github api's to get the latest flyte release to use'") return cmdFlags } diff --git a/cmd/config/subcommand/sandbox/config_flags_test.go b/cmd/config/subcommand/sandbox/config_flags_test.go index 39dc790cfe..0d7021b76d 100755 --- a/cmd/config/subcommand/sandbox/config_flags_test.go +++ b/cmd/config/subcommand/sandbox/config_flags_test.go @@ -223,4 +223,18 @@ func TestConfig_SetFlags(t *testing.T) { } }) }) + t.Run("Test_dryRun", func(t *testing.T) { + + t.Run("Override", func(t *testing.T) { + testValue := "1" + + cmdFlags.Set("dryRun", testValue) + if vBool, err := cmdFlags.GetBool("dryRun"); err == nil { + testDecodeJson_Config(t, fmt.Sprintf("%v", vBool), &actual.DryRun) + + } else { + assert.FailNow(t, err.Error()) + } + }) + }) } diff --git a/cmd/config/subcommand/sandbox/sandbox_config.go b/cmd/config/subcommand/sandbox/sandbox_config.go index c2d37c6407..463f751ecc 100644 --- a/cmd/config/subcommand/sandbox/sandbox_config.go +++ b/cmd/config/subcommand/sandbox/sandbox_config.go @@ -29,7 +29,8 @@ type Config struct { ImagePullOptions docker.ImagePullOptions `json:"imagePullOptions" pflag:",Optional. Defines image pull options (e.g. auth)"` // It's used for development. Users are able to start flyte locally via single binary and save the data to the minio or postgres in the sandbox. - Dev bool `json:"dev" pflag:",Optional. Only start minio and postgres in the sandbox."` + Dev bool `json:"dev" pflag:",Optional. Only start minio and postgres in the sandbox."` + DryRun bool `json:"dryRun" pflag:",Optional. Only print the docker commands to bring up flyte sandbox/demo container.This will still call github api's to get the latest flyte release to use'"` } //go:generate pflags Config --default-var DefaultConfig --bind-default-var diff --git a/cmd/demo/start.go b/cmd/demo/start.go index 1300496a2a..a800ed1795 100644 --- a/cmd/demo/start.go +++ b/cmd/demo/start.go @@ -75,6 +75,12 @@ eg : for passing multiple environment variables flytectl demo start --env USER=foo --env PASSWORD=bar +For just printing the docker commands for bringingup the demo container +:: + + flytectl demo start --dryRun + + Usage ` ) diff --git a/cmd/sandbox/sandbox.go b/cmd/sandbox/sandbox.go index 2e342c41e4..0e20df4312 100644 --- a/cmd/sandbox/sandbox.go +++ b/cmd/sandbox/sandbox.go @@ -32,6 +32,12 @@ To execute commands inside the sandbox container, use exec: :: flytectl sandbox exec -- pwd + +For just printing the docker commands for bringingup the demo container +:: + + flytectl demo start --dryRun + ` ) diff --git a/pkg/docker/docker_util.go b/pkg/docker/docker_util.go index 404db17fbc..a3ae226bc8 100644 --- a/pkg/docker/docker_util.go +++ b/pkg/docker/docker_util.go @@ -138,8 +138,12 @@ func GetDemoPorts() (map[nat.Port]struct{}, map[nat.Port][]nat.PortBinding, erro // PullDockerImage will Pull docker image func PullDockerImage(ctx context.Context, cli Docker, image string, pullPolicy ImagePullPolicy, - imagePullOptions ImagePullOptions) error { - + imagePullOptions ImagePullOptions, dryRun bool) error { + if dryRun { + PrintPullImage(image, imagePullOptions) + return nil + } + fmt.Printf("%v pulling docker image for release %s\n", emoji.Whale, image) if pullPolicy == ImagePullPolicyAlways || pullPolicy == ImagePullPolicyIfNotPresent { if pullPolicy == ImagePullPolicyIfNotPresent { imageSummary, err := cli.ImageList(ctx, types.ImageListOptions{}) @@ -169,11 +173,61 @@ func PullDockerImage(ctx context.Context, cli Docker, image string, pullPolicy I return nil } -//StartContainer will create and start docker container +// PrintPullImage helper function to print the sandbox pull image command +func PrintPullImage(image string, pullOptions ImagePullOptions) { + fmt.Printf("%v Run the following command to pull the sandbox image from registry.\n", emoji.Sparkle) + var sb strings.Builder + sb.WriteString("docker pull ") + if len(pullOptions.Platform) > 0 { + sb.WriteString(fmt.Sprintf("--platform %v ", pullOptions.Platform)) + } + sb.WriteString(fmt.Sprintf("%v", image)) + fmt.Printf(" %v \n", sb.String()) +} + +// PrintRemoveContainer helper function to remove sandbox container +func PrintRemoveContainer(name string) { + fmt.Printf("%v Run the following command to remove the existing sandbox\n", emoji.Sparkle) + fmt.Printf(" docker container rm %v --force\n", name) +} + +// PrintCreateContainer helper function to print the docker command to run +func PrintCreateContainer(volumes []mount.Mount, portBindings map[nat.Port][]nat.PortBinding, name, image string, environment []string) { + var sb strings.Builder + fmt.Printf("%v Run the following command to create new sandbox container\n", emoji.Sparkle) + sb.WriteString(" docker create --privileged ") + for portProto, bindings := range portBindings { + srcPort := portProto.Port() + for _, binding := range bindings { + sb.WriteString(fmt.Sprintf("-p %v:%v:%v ", binding.HostIP, srcPort, binding.HostPort)) + } + } + for _, env := range environment { + sb.WriteString(fmt.Sprintf("--env %v ", env)) + } + + for _, volume := range volumes { + sb.WriteString(fmt.Sprintf("--mount type=%v,source=%v,target=%v ", volume.Type, volume.Source, volume.Target)) + } + sb.WriteString(fmt.Sprintf("--name %v ", name)) + sb.WriteString(fmt.Sprintf("%v", image)) + fmt.Printf("%v\n", sb.String()) + fmt.Printf("%v Run the following command to start the sandbox container\n", emoji.Sparkle) + fmt.Printf(" docker start %v\n", name) + fmt.Printf("%v Run the following command to check the logs and monitor the sandbox container and make sure there are no error during startup and then visit flyteconsole\n", emoji.EightSpokedAsterisk) + fmt.Printf(" docker logs -f %v\n", name) +} + +// StartContainer will create and start docker container func StartContainer(ctx context.Context, cli Docker, volumes []mount.Mount, exposedPorts map[nat.Port]struct{}, - portBindings map[nat.Port][]nat.PortBinding, name, image string, additionalEnvVars []string) (string, error) { + portBindings map[nat.Port][]nat.PortBinding, name, image string, additionalEnvVars []string, dryRun bool) (string, error) { // Append the additional env variables to the default list of env Environment = append(Environment, additionalEnvVars...) + if dryRun { + PrintCreateContainer(volumes, portBindings, name, image, Environment) + return "", nil + } + fmt.Printf("%v booting Flyte-sandbox container\n", emoji.FactoryWorker) resp, err := cli.ContainerCreate(ctx, &container.Config{ Env: Environment, Image: image, diff --git a/pkg/docker/docker_util_test.go b/pkg/docker/docker_util_test.go index 8524443641..7b33be9425 100644 --- a/pkg/docker/docker_util_test.go +++ b/pkg/docker/docker_util_test.go @@ -104,7 +104,7 @@ func TestPullDockerImage(t *testing.T) { ctx := context.Background() // Verify the attributes mockDocker.OnImagePullMatch(ctx, mock.Anything, types.ImagePullOptions{}).Return(os.Stdin, nil) - err := PullDockerImage(ctx, mockDocker, "nginx:latest", ImagePullPolicyAlways, ImagePullOptions{}) + err := PullDockerImage(ctx, mockDocker, "nginx:latest", ImagePullPolicyAlways, ImagePullOptions{}, false) assert.Nil(t, err) }) @@ -114,7 +114,7 @@ func TestPullDockerImage(t *testing.T) { ctx := context.Background() // Verify the attributes mockDocker.OnImagePullMatch(ctx, mock.Anything, types.ImagePullOptions{}).Return(os.Stdin, fmt.Errorf("error")) - err := PullDockerImage(ctx, mockDocker, "nginx:latest", ImagePullPolicyAlways, ImagePullOptions{}) + err := PullDockerImage(ctx, mockDocker, "nginx:latest", ImagePullPolicyAlways, ImagePullOptions{}, false) assert.NotNil(t, err) }) @@ -125,7 +125,7 @@ func TestPullDockerImage(t *testing.T) { // Verify the attributes mockDocker.OnImagePullMatch(ctx, mock.Anything, types.ImagePullOptions{}).Return(os.Stdin, nil) mockDocker.OnImageListMatch(ctx, types.ImageListOptions{}).Return([]types.ImageSummary{}, nil) - err := PullDockerImage(ctx, mockDocker, "nginx:latest", ImagePullPolicyIfNotPresent, ImagePullOptions{}) + err := PullDockerImage(ctx, mockDocker, "nginx:latest", ImagePullPolicyIfNotPresent, ImagePullOptions{}, false) assert.Nil(t, err) }) @@ -133,7 +133,7 @@ func TestPullDockerImage(t *testing.T) { setupSandbox() mockDocker := &mocks.Docker{} ctx := context.Background() - err := PullDockerImage(ctx, mockDocker, "nginx:latest", ImagePullPolicyNever, ImagePullOptions{}) + err := PullDockerImage(ctx, mockDocker, "nginx:latest", ImagePullPolicyNever, ImagePullOptions{}, false) assert.Nil(t, err) }) } @@ -160,7 +160,7 @@ func TestStartContainer(t *testing.T) { ID: "Hello", }, nil) mockDocker.OnContainerStart(ctx, "Hello", types.ContainerStartOptions{}).Return(nil) - id, err := StartContainer(ctx, mockDocker, Volumes, p1, p2, "nginx", imageName, nil) + id, err := StartContainer(ctx, mockDocker, Volumes, p1, p2, "nginx", imageName, nil, false) assert.Nil(t, err) assert.Greater(t, len(id), 0) assert.Equal(t, id, "Hello") @@ -189,7 +189,7 @@ func TestStartContainer(t *testing.T) { ID: "Hello", }, nil) mockDocker.OnContainerStart(ctx, "Hello", types.ContainerStartOptions{}).Return(nil) - id, err := StartContainer(ctx, mockDocker, Volumes, p1, p2, "nginx", imageName, additionalEnv) + id, err := StartContainer(ctx, mockDocker, Volumes, p1, p2, "nginx", imageName, additionalEnv, false) assert.Nil(t, err) assert.Greater(t, len(id), 0) assert.Equal(t, id, "Hello") @@ -215,7 +215,7 @@ func TestStartContainer(t *testing.T) { ID: "", }, fmt.Errorf("error")) mockDocker.OnContainerStart(ctx, "Hello", types.ContainerStartOptions{}).Return(nil) - id, err := StartContainer(ctx, mockDocker, Volumes, p1, p2, "nginx", imageName, nil) + id, err := StartContainer(ctx, mockDocker, Volumes, p1, p2, "nginx", imageName, nil, false) assert.NotNil(t, err) assert.Equal(t, len(id), 0) assert.Equal(t, id, "") @@ -240,7 +240,7 @@ func TestStartContainer(t *testing.T) { ID: "Hello", }, nil) mockDocker.OnContainerStart(ctx, "Hello", types.ContainerStartOptions{}).Return(fmt.Errorf("error")) - id, err := StartContainer(ctx, mockDocker, Volumes, p1, p2, "nginx", imageName, nil) + id, err := StartContainer(ctx, mockDocker, Volumes, p1, p2, "nginx", imageName, nil, false) assert.NotNil(t, err) assert.Equal(t, len(id), 0) assert.Equal(t, id, "") diff --git a/pkg/sandbox/start.go b/pkg/sandbox/start.go index 9260c7f32c..3a53923ca1 100644 --- a/pkg/sandbox/start.go +++ b/pkg/sandbox/start.go @@ -150,16 +150,18 @@ func UpdateLocalKubeContext(k8sCtxMgr k8s.ContextOps, dockerCtx string, contextN func startSandbox(ctx context.Context, cli docker.Docker, g github.GHRepoService, reader io.Reader, sandboxConfig *sandboxCmdConfig.Config, defaultImageName string, defaultImagePrefix string, exposedPorts map[nat.Port]struct{}, portBindings map[nat.Port][]nat.PortBinding, consolePort int) (*bufio.Scanner, error) { fmt.Printf("%v Bootstrapping a brand new flyte cluster... %v %v\n", emoji.FactoryWorker, emoji.Hammer, emoji.Wrench) - - if err := docker.RemoveSandbox(ctx, cli, reader); err != nil { - if err.Error() != clierrors.ErrSandboxExists { - return nil, err + if sandboxConfig.DryRun { + docker.PrintRemoveContainer(docker.FlyteSandboxClusterName) + } else { + if err := docker.RemoveSandbox(ctx, cli, reader); err != nil { + if err.Error() != clierrors.ErrSandboxExists { + return nil, err + } + fmt.Printf("Existing details of your sandbox") + util.PrintSandboxMessage(consolePort, sandboxConfig.DryRun) + return nil, nil } - fmt.Printf("Existing details of your sandbox") - util.PrintSandboxMessage(consolePort) - return nil, nil } - if err := util.SetupFlyteDir(); err != nil { return nil, err } @@ -186,11 +188,9 @@ func startSandbox(ctx context.Context, cli docker.Docker, g github.GHRepoService return nil, err } sandboxImage = image - fmt.Printf("%s Fully Qualified image\n", image) - fmt.Printf("%v Running Flyte %s release\n", emoji.Whale, version) + fmt.Printf("%v Going to use Flyte %s release with image %s \n", emoji.Whale, version, image) } - fmt.Printf("%v pulling docker image for release %s\n", emoji.Whale, sandboxImage) - if err := docker.PullDockerImage(ctx, cli, sandboxImage, sandboxConfig.ImagePullPolicy, sandboxConfig.ImagePullOptions); err != nil { + if err := docker.PullDockerImage(ctx, cli, sandboxImage, sandboxConfig.ImagePullPolicy, sandboxConfig.ImagePullOptions, sandboxConfig.DryRun); err != nil { return nil, err } sandboxEnv := sandboxConfig.Env @@ -198,18 +198,20 @@ func startSandbox(ctx context.Context, cli docker.Docker, g github.GHRepoService sandboxEnv = append(sandboxEnv, "FLYTE_DEV=True") } - fmt.Printf("%v booting Flyte-sandbox container\n", emoji.FactoryWorker) ID, err := docker.StartContainer(ctx, cli, volumes, exposedPorts, portBindings, docker.FlyteSandboxClusterName, - sandboxImage, sandboxEnv) + sandboxImage, sandboxEnv, sandboxConfig.DryRun) if err != nil { fmt.Printf("%v Something went wrong: Failed to start Sandbox container %v, Please check your docker client and try again. \n", emoji.GrimacingFace, emoji.Whale) return nil, err } - logReader, err := docker.ReadLogs(ctx, cli, ID) - if err != nil { - return nil, err + var logReader *bufio.Scanner + if !sandboxConfig.DryRun { + logReader, err = docker.ReadLogs(ctx, cli, ID) + if err != nil { + return nil, err + } } return logReader, nil @@ -301,7 +303,7 @@ func StartDemoCluster(ctx context.Context, args []string, sandboxConfig *sandbox if err != nil { return err } - util.PrintSandboxMessage(util.DemoConsolePort) + util.PrintSandboxMessage(util.DemoConsolePort, sandboxConfig.DryRun) return nil } @@ -316,6 +318,6 @@ func StartSandboxCluster(ctx context.Context, args []string, sandboxConfig *sand if err != nil { return err } - util.PrintSandboxMessage(util.SandBoxConsolePort) + util.PrintSandboxMessage(util.SandBoxConsolePort, sandboxConfig.DryRun) return nil } diff --git a/pkg/util/util.go b/pkg/util/util.go index 5fc54438ec..afdb35967b 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -17,9 +17,10 @@ import ( ) const ( - ProgressSuccessMessage = "Flyte is ready! Flyte UI is available at" - SandBoxConsolePort = 30081 - DemoConsolePort = 30080 + ProgressSuccessMessage = "Flyte is ready! Flyte UI is available at" + ProgressSuccessMessagePending = "Flyte would be ready after this! Flyte UI would be available at" + SandBoxConsolePort = 30081 + DemoConsolePort = 30080 ) var Ext string @@ -52,17 +53,27 @@ func SetupFlyteDir() error { } // PrintSandboxMessage will print sandbox success message -func PrintSandboxMessage(flyteConsolePort int) { +func PrintSandboxMessage(flyteConsolePort int, dryRun bool) { kubeconfig := strings.Join([]string{ "$KUBECONFIG", f.FilePathJoin(f.UserHomeDir(), ".kube", "config"), docker.Kubeconfig, }, ":") - successMsg := fmt.Sprintf("%v http://localhost:%v/console", ProgressSuccessMessage, flyteConsolePort) + + var successMsg string + if dryRun { + successMsg = fmt.Sprintf("%v http://localhost:%v/console", ProgressSuccessMessagePending, flyteConsolePort) + } else { + successMsg = fmt.Sprintf("%v http://localhost:%v/console", ProgressSuccessMessage, flyteConsolePort) + + } fmt.Printf("%v %v %v %v %v \n", emoji.ManTechnologist, successMsg, emoji.Rocket, emoji.Rocket, emoji.PartyPopper) - fmt.Printf("Add KUBECONFIG and FLYTECTL_CONFIG to your environment variable \n") - fmt.Printf("export KUBECONFIG=%v \n", kubeconfig) - fmt.Printf("export FLYTECTL_CONFIG=%v \n", configutil.FlytectlConfig) + fmt.Printf("%v Run the following command to export sandbox environment variables for accessing flytectl\n", emoji.Sparkle) + fmt.Printf(" export FLYTECTL_CONFIG=%v \n", configutil.FlytectlConfig) + if dryRun { + fmt.Printf("%v Run the following command to export kubeconfig variables for accessing flyte pods locally\n", emoji.Sparkle) + fmt.Printf(" export KUBECONFIG=%v \n", kubeconfig) + } } // SendRequest will create request and return the response diff --git a/pkg/util/util_test.go b/pkg/util/util_test.go index 718a244d9c..9bbb89964f 100644 --- a/pkg/util/util_test.go +++ b/pkg/util/util_test.go @@ -25,7 +25,7 @@ func TestSetupFlyteDir(t *testing.T) { func TestPrintSandboxMessage(t *testing.T) { t.Run("Print Sandbox Message", func(t *testing.T) { - PrintSandboxMessage(SandBoxConsolePort) + PrintSandboxMessage(SandBoxConsolePort, false) }) }