Add support dev cluster (flyteorg#353)

* Add support dev cluster Signed-off-by: Kevin Su <[email protected]> * nit Signed-off-by: Kevin Su <[email protected]> * nit Signed-off-by: Kevin Su <[email protected]> * nit Signed-off-by: Kevin Su <[email protected]> * make generate Signed-off-by: Kevin Su <[email protected]> * lint Signed-off-by: Kevin Su <[email protected]> Signed-off-by: Kevin Su <[email protected]>
austin362667 · May 7, 2024 · 9849967 · 9849967
1 parent 2c39143
commit 9849967
Show file tree

Hide file tree

Showing 11 changed files with 50 additions and 10 deletions.
diff --git a/flytectl/cmd/compile/compile.go b/flytectl/cmd/compile/compile.go
@@ -29,8 +29,8 @@ func compileTasks(tasks []*core.TaskTemplate) ([]*core.CompiledTask, error) {
 }
 
 /*
- Utility to compile a packaged workflow locally.
- compilation is done locally so no flyte cluster is required.
+Utility to compile a packaged workflow locally.
+compilation is done locally so no flyte cluster is required.
 */
 func compileFromPackage(packagePath string) error {
 	args := []string{packagePath}

diff --git a/flytectl/cmd/completion.go b/flytectl/cmd/completion.go
@@ -5,7 +5,7 @@ Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
 
-    http://www.apache.org/licenses/LICENSE-2.0
+	http://www.apache.org/licenses/LICENSE-2.0
 
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,

diff --git a/flytectl/cmd/config/subcommand/config/init_flags.go b/flytectl/cmd/config/subcommand/config/init_flags.go
@@ -7,7 +7,7 @@ var (
 	}
 )
 
-//Configs
+// Configs
 type Config struct {
 	Host     string `json:"host" pflag:",Endpoint of flyte admin"`
 	Insecure bool   `json:"insecure" pflag:",Enable insecure mode"`

diff --git a/flytectl/cmd/config/subcommand/project/project_config.go b/flytectl/cmd/config/subcommand/project/project_config.go
@@ -44,7 +44,7 @@ var DefaultProjectConfig = &ConfigProject{
 	Labels:      map[string]string{},
 }
 
-//GetProjectSpec return project spec from a file/flags
+// GetProjectSpec return project spec from a file/flags
 func (c *ConfigProject) GetProjectSpec(id string) (*admin.Project, error) {
 	projectSpec := admin.Project{}
 	if len(c.File) > 0 {
@@ -71,7 +71,7 @@ func (c *ConfigProject) GetProjectSpec(id string) (*admin.Project, error) {
 	return &projectSpec, nil
 }
 
-//MapToAdminState return project spec from a file/flags
+// MapToAdminState return project spec from a file/flags
 func (c *ConfigProject) MapToAdminState() (admin.Project_ProjectState, error) {
 	if c.ActivateProject {
 		c.Activate = c.ActivateProject

diff --git a/flytectl/cmd/config/subcommand/sandbox/config_flags.go b/flytectl/cmd/config/subcommand/sandbox/config_flags.go
diff --git a/flytectl/cmd/config/subcommand/sandbox/config_flags_test.go b/flytectl/cmd/config/subcommand/sandbox/config_flags_test.go
diff --git a/flytectl/cmd/config/subcommand/sandbox/sandbox_config.go b/flytectl/cmd/config/subcommand/sandbox/sandbox_config.go
@@ -2,7 +2,7 @@ package sandbox
 
 import "github.com/flyteorg/flytectl/pkg/docker"
 
-//Config holds configuration flags for sandbox command.
+// Config holds configuration flags for sandbox command.
 type Config struct {
 	Source string `json:"source" pflag:",Path of your source code"`
 
@@ -27,6 +27,9 @@ type Config struct {
 	ImagePullPolicy docker.ImagePullPolicy `json:"imagePullPolicy" pflag:",Optional. Defines the image pull behavior [Always/IfNotPresent/Never]"`
 
 	ImagePullOptions docker.ImagePullOptions `json:"imagePullOptions" pflag:",Optional. Defines image pull options (e.g. auth)"`
+
+	// It's used for development. Users are able to start flyte locally via single binary and save the data to the minio or postgres in the sandbox.
+	Dev bool `json:"dev" pflag:",Optional. Only start minio and postgres in the sandbox."`
 }
 
 //go:generate pflags Config --default-var DefaultConfig --bind-default-var

diff --git a/flytectl/cmd/demo/start.go b/flytectl/cmd/demo/start.go
@@ -20,6 +20,11 @@ Starts the demo cluster without any source code:
 
  flytectl demo start
 
+Runs a dev cluster, which only has minio and postgres pod.
+::
+
+ flytectl demo start --dev
+
 Mounts your source code repository inside the demo cluster:
 ::
 
@@ -45,7 +50,7 @@ Runs a specific version of Flyte. Flytectl demo only supports Flyte version avai
 .. note::
 	  Flytectl demo is only supported for Flyte versions >= v1.0.0
 
-Runs the latest pre release of  Flyte.
+Runs the latest pre release of Flyte.
 ::
 
  flytectl demo start --pre

diff --git a/flytectl/cmd/upgrade/upgrade_test.go b/flytectl/cmd/upgrade/upgrade_test.go
@@ -44,7 +44,6 @@ func TestUpgradeCommand(t *testing.T) {
 	assert.Equal(t, cmdNouns[0].Long, upgradeCmdLong)
 }
 
-//
 func TestUpgrade(t *testing.T) {
 	_ = util.WriteIntoFile([]byte("data"), tempExt)
 	stdlibversion.Version = version

diff --git a/flytectl/pkg/docker/docker_util.go b/flytectl/pkg/docker/docker_util.go
@@ -96,6 +96,17 @@ func RemoveSandbox(ctx context.Context, cli Docker, reader io.Reader) error {
 	return nil
 }
 
+// GetDevPorts will return dev cluster (minio + postgres) ports
+func GetDevPorts() (map[nat.Port]struct{}, map[nat.Port][]nat.PortBinding, error) {
+	return nat.ParsePortSpecs([]string{
+		"0.0.0.0:30082:30082", // K8s Dashboard Port
+		"0.0.0.0:30084:30084", // Minio API Port
+		"0.0.0.0:30086:30086", // K8s cluster
+		"0.0.0.0:30088:30088", // Minio Console Port
+		"0.0.0.0:30089:30089", // Postgres Port
+	})
+}
+
 // GetSandboxPorts will return sandbox ports
 func GetSandboxPorts() (map[nat.Port]struct{}, map[nat.Port][]nat.PortBinding, error) {
 	return nat.ParsePortSpecs([]string{

diff --git a/flytectl/pkg/sandbox/start.go b/flytectl/pkg/sandbox/start.go
@@ -193,10 +193,14 @@ func startSandbox(ctx context.Context, cli docker.Docker, g github.GHRepoService
 	if err := docker.PullDockerImage(ctx, cli, sandboxImage, sandboxConfig.ImagePullPolicy, sandboxConfig.ImagePullOptions); err != nil {
 		return nil, err
 	}
+	sandboxEnv := sandboxConfig.Env
+	if sandboxConfig.Dev {
+		sandboxEnv = append(sandboxEnv, "FLYTE_DEV=True")
+	}
 
 	fmt.Printf("%v booting Flyte-sandbox container\n", emoji.FactoryWorker)
 	ID, err := docker.StartContainer(ctx, cli, volumes, exposedPorts, portBindings, docker.FlyteSandboxClusterName,
-		sandboxImage, sandboxConfig.Env)
+		sandboxImage, sandboxEnv)
 
 	if err != nil {
 		fmt.Printf("%v Something went wrong: Failed to start Sandbox container %v, Please check your docker client and try again. \n", emoji.GrimacingFace, emoji.Whale)
@@ -281,6 +285,9 @@ func StartDemoCluster(ctx context.Context, args []string, sandboxConfig *sandbox
 	primePod := true
 	sandboxImagePrefix := "sha"
 	exposedPorts, portBindings, err := docker.GetDemoPorts()
+	if sandboxConfig.Dev {
+		exposedPorts, portBindings, err = docker.GetDevPorts()
+	}
 	if err != nil {
 		return err
 	}