Added env flag for sandbox start (flyteorg#273)

* Added env flag for sandbox start

Signed-off-by: Prafulla Mahindrakar <[email protected]>

* ADDED docs for FLYTE_TIMEOUT

Signed-off-by: Prafulla Mahindrakar <[email protected]>

* Fixed validation logic for launch plan schedule registration (flyteorg#280)

Signed-off-by: Prafulla Mahindrakar <[email protected]>

Signed-off-by: pmahindrakar-oss <[email protected]>

* boilerplate updates

Signed-off-by: Prafulla Mahindrakar <[email protected]>

* fixed unit test after stdlib upgrade

Signed-off-by: Prafulla Mahindrakar <[email protected]>

* fixed unit test

Signed-off-by: Prafulla Mahindrakar <[email protected]>

boilerplate/flyte/golang_support_tools/go.mod

@@ -1,13 +1,191 @@
 module github.com/flyteorg/boilerplate
 
-go 1.16
+go 1.17
 
 require (
 	github.com/alvaroloes/enumer v1.1.2
 	github.com/flyteorg/flytestdlib v0.4.7
 	github.com/golangci/golangci-lint v1.38.0
+	github.com/pseudomuto/protoc-gen-doc v1.4.1
 	github.com/vektra/mockery v0.0.0-20181123154057-e78b021dcbb5
-	github.com/pseudomuto/protoc-gen-doc v0.1.1 // indirect
+)
+
+require (
+	4d63.com/gochecknoglobals v0.0.0-20201008074935-acfc0b28355a // indirect
+	cloud.google.com/go v0.75.0 // indirect
+	cloud.google.com/go/storage v1.12.0 // indirect
+	github.com/Azure/azure-sdk-for-go v51.0.0+incompatible // indirect
+	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
+	github.com/Azure/go-autorest/autorest v0.11.17 // indirect
+	github.com/Azure/go-autorest/autorest/adal v0.9.10 // indirect
+	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
+	github.com/Azure/go-autorest/logger v0.2.0 // indirect
+	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
+	github.com/BurntSushi/toml v0.3.1 // indirect
+	github.com/Djarvur/go-err113 v0.0.0-20210108212216-aea10b59be24 // indirect
+	github.com/Masterminds/semver v1.5.0 // indirect
+	github.com/Masterminds/sprig v2.15.0+incompatible // indirect
+	github.com/OpenPeeDeeP/depguard v1.0.1 // indirect
+	github.com/alexkohler/prealloc v1.0.0 // indirect
+	github.com/aokoli/goutils v1.0.1 // indirect
+	github.com/ashanbrown/forbidigo v1.1.0 // indirect
+	github.com/ashanbrown/makezero v0.0.0-20201205152432-7b7cdbb3025a // indirect
+	github.com/aws/aws-sdk-go v1.37.1 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/bkielbasa/cyclop v1.2.0 // indirect
+	github.com/bombsimon/wsl/v3 v3.2.0 // indirect
+	github.com/cespare/xxhash v1.1.0 // indirect
+	github.com/cespare/xxhash/v2 v2.1.1 // indirect
+	github.com/charithe/durationcheck v0.0.6 // indirect
+	github.com/coocood/freecache v1.1.1 // indirect
+	github.com/daixiang0/gci v0.2.8 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/denis-tingajkin/go-header v0.4.2 // indirect
+	github.com/envoyproxy/protoc-gen-validate v0.3.0-java // indirect
+	github.com/ernesto-jimenez/gogen v0.0.0-20180125220232-d7d4131e6607 // indirect
+	github.com/esimonov/ifshort v1.0.1 // indirect
+	github.com/fatih/color v1.10.0 // indirect
+	github.com/fatih/structtag v1.2.0 // indirect
+	github.com/form3tech-oss/jwt-go v3.2.2+incompatible // indirect
+	github.com/fsnotify/fsnotify v1.4.9 // indirect
+	github.com/fzipp/gocyclo v0.3.1 // indirect
+	github.com/ghodss/yaml v1.0.0 // indirect
+	github.com/go-critic/go-critic v0.5.4 // indirect
+	github.com/go-logr/logr v0.4.0 // indirect
+	github.com/go-toolsmith/astcast v1.0.0 // indirect
+	github.com/go-toolsmith/astcopy v1.0.0 // indirect
+	github.com/go-toolsmith/astequal v1.0.0 // indirect
+	github.com/go-toolsmith/astfmt v1.0.0 // indirect
+	github.com/go-toolsmith/astp v1.0.0 // indirect
+	github.com/go-toolsmith/strparse v1.0.0 // indirect
+	github.com/go-toolsmith/typep v1.0.2 // indirect
+	github.com/go-xmlfmt/xmlfmt v0.0.0-20191208150333-d5b6f63a941b // indirect
+	github.com/gobwas/glob v0.2.3 // indirect
+	github.com/gofrs/flock v0.8.0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
+	github.com/golang/protobuf v1.4.3 // indirect
+	github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 // indirect
+	github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a // indirect
+	github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613 // indirect
+	github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a // indirect
+	github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0 // indirect
+	github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca // indirect
+	github.com/golangci/misspell v0.3.5 // indirect
+	github.com/golangci/revgrep v0.0.0-20210208091834-cd28932614b5 // indirect
+	github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4 // indirect
+	github.com/google/go-cmp v0.5.4 // indirect
+	github.com/google/uuid v1.1.2 // indirect
+	github.com/googleapis/gax-go/v2 v2.0.5 // indirect
+	github.com/gordonklaus/ineffassign v0.0.0-20210225214923-2e10b2664254 // indirect
+	github.com/gostaticanalysis/analysisutil v0.4.1 // indirect
+	github.com/gostaticanalysis/comment v1.4.1 // indirect
+	github.com/gostaticanalysis/forcetypeassert v0.0.0-20200621232751-01d4955beaa5 // indirect
+	github.com/gostaticanalysis/nilerr v0.1.1 // indirect
+	github.com/graymeta/stow v0.2.7 // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/huandu/xstrings v1.0.0 // indirect
+	github.com/imdario/mergo v0.3.5 // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/jgautheron/goconst v1.4.0 // indirect
+	github.com/jingyugao/rowserrcheck v0.0.0-20210130005344-c6a0c12dd98d // indirect
+	github.com/jirfag/go-printf-func-name v0.0.0-20200119135958-7558a9eaa5af // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/jstemmer/go-junit-report v0.9.1 // indirect
+	github.com/julz/importas v0.0.0-20210226073942-60b4fa260dd0 // indirect
+	github.com/kisielk/errcheck v1.6.0 // indirect
+	github.com/kisielk/gotool v1.0.0 // indirect
+	github.com/kulti/thelper v0.4.0 // indirect
+	github.com/kunwardeep/paralleltest v1.0.2 // indirect
+	github.com/kyoh86/exportloopref v0.1.8 // indirect
+	github.com/magefile/mage v1.10.0 // indirect
+	github.com/magiconair/properties v1.8.4 // indirect
+	github.com/maratori/testpackage v1.0.1 // indirect
+	github.com/matoous/godox v0.0.0-20210227103229-6504466cf951 // indirect
+	github.com/mattn/go-colorable v0.1.8 // indirect
+	github.com/mattn/go-isatty v0.0.12 // indirect
+	github.com/mattn/go-runewidth v0.0.7 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/mbilski/exhaustivestruct v1.2.0 // indirect
+	github.com/mgechev/dots v0.0.0-20190921121421-c36f7dcfbb81 // indirect
+	github.com/mgechev/revive v1.0.3 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/mitchellh/mapstructure v1.4.1 // indirect
+	github.com/moricho/tparallel v0.2.1 // indirect
+	github.com/mwitkow/go-proto-validators v0.0.0-20180403085117-0950a7990007 // indirect
+	github.com/nakabonne/nestif v0.3.0 // indirect
+	github.com/nbutton23/zxcvbn-go v0.0.0-20201221231540-e56b841a3c88 // indirect
+	github.com/ncw/swift v1.0.53 // indirect
+	github.com/nishanths/exhaustive v0.1.0 // indirect
+	github.com/nishanths/predeclared v0.2.1 // indirect
+	github.com/olekukonko/tablewriter v0.0.4 // indirect
+	github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1 // indirect
+	github.com/pelletier/go-toml v1.8.1 // indirect
+	github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/polyfloyd/go-errorlint v0.0.0-20201127212506-19bd8db6546f // indirect
+	github.com/prometheus/client_golang v1.9.0 // indirect
+	github.com/prometheus/client_model v0.2.0 // indirect
+	github.com/prometheus/common v0.15.0 // indirect
+	github.com/prometheus/procfs v0.3.0 // indirect
+	github.com/pseudomuto/protokit v0.2.0 // indirect
+	github.com/quasilyte/go-ruleguard v0.3.0 // indirect
+	github.com/quasilyte/regex/syntax v0.0.0-20200407221936-30656e2c4a95 // indirect
+	github.com/ryancurrah/gomodguard v1.2.0 // indirect
+	github.com/ryanrolds/sqlclosecheck v0.3.0 // indirect
+	github.com/sanposhiho/wastedassign v0.1.3 // indirect
+	github.com/satori/go.uuid v1.2.0 // indirect
+	github.com/securego/gosec/v2 v2.6.1 // indirect
+	github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c // indirect
+	github.com/sirupsen/logrus v1.8.0 // indirect
+	github.com/sonatard/noctx v0.0.1 // indirect
+	github.com/sourcegraph/go-diff v0.6.1 // indirect
+	github.com/spf13/afero v1.5.1 // indirect
+	github.com/spf13/cast v1.3.1 // indirect
+	github.com/spf13/cobra v1.1.3 // indirect
+	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/spf13/viper v1.7.1 // indirect
+	github.com/ssgreg/nlreturn/v2 v2.1.0 // indirect
+	github.com/stretchr/objx v0.3.0 // indirect
+	github.com/stretchr/testify v1.7.0 // indirect
+	github.com/subosito/gotenv v1.2.0 // indirect
+	github.com/tdakkota/asciicheck v0.0.0-20200416200610-e657995f937b // indirect
+	github.com/tetafro/godot v1.4.4 // indirect
+	github.com/timakin/bodyclose v0.0.0-20200424151742-cb6215831a94 // indirect
+	github.com/tomarrell/wrapcheck v0.0.0-20201130113247-1683564d9756 // indirect
+	github.com/tommy-muehle/go-mnd/v2 v2.3.1 // indirect
+	github.com/ultraware/funlen v0.0.3 // indirect
+	github.com/ultraware/whitespace v0.0.4 // indirect
+	github.com/uudashr/gocognit v1.0.1 // indirect
+	go.opencensus.io v0.22.6 // indirect
+	golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad // indirect
+	golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 // indirect
+	golang.org/x/mod v0.4.1 // indirect
+	golang.org/x/net v0.0.0-20210119194325-5f4716e94777 // indirect
+	golang.org/x/oauth2 v0.0.0-20210126194326-f9ce19ea3013 // indirect
+	golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c // indirect
+	golang.org/x/text v0.3.5 // indirect
+	golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect
+	golang.org/x/tools v0.1.0 // indirect
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	google.golang.org/api v0.38.0 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/genproto v0.0.0-20210126160654-44e461bb6506 // indirect
+	google.golang.org/grpc v1.35.0 // indirect
+	google.golang.org/protobuf v1.25.0 // indirect
+	gopkg.in/ini.v1 v1.62.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	honnef.co/go/tools v0.1.2 // indirect
+	k8s.io/apimachinery v0.20.2 // indirect
+	k8s.io/client-go v0.0.0-20210217172142-7279fc64d847 // indirect
+	k8s.io/klog/v2 v2.5.0 // indirect
+	mvdan.cc/gofumpt v0.1.0 // indirect
+	mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed // indirect
+	mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b // indirect
+	mvdan.cc/unparam v0.0.0-20210104141923-aac4ce9116a7 // indirect
 )
 
 replace github.com/vektra/mockery => github.com/enghabu/mockery v0.0.0-20191009061720-9d0c8670c2f0

boilerplate/flyte/golang_support_tools/tools.go

@@ -7,5 +7,5 @@ import (
 	_ "github.com/flyteorg/flytestdlib/cli/pflags"
 	_ "github.com/golangci/golangci-lint/cmd/golangci-lint"
 	_ "github.com/vektra/mockery/cmd/mockery"
-	- "github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc"
+	_ "github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc"
 )

boilerplate/flyte/golang_test_targets/download_tooling.sh

@@ -17,8 +17,8 @@ set -e
 # In the format of "<cli>:<package>" or ":<package>" if no cli
 tools=(
   "github.com/vektra/mockery/cmd/mockery"
-  "github.com/flyteorg/flytestdlib/cli/pflags"
-  "github.com/golangci/golangci-lint/cmd/golangci-lint"
+  "github.com/flyteorg/flytestdlib/cli/pflags@latest"
+  "github.com/golangci/golangci-lint/cmd/golangci-lint@latest"
   "github.com/alvaroloes/enumer"
   "github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc"
 )

cmd/config/subcommand/sandbox/sandbox_config.go

@@ -47,6 +47,9 @@ type Config struct {
 	// Default value false represents that flytectl will not use latest pre release if exist
 	Prerelease bool `json:"pre" pflag:",Optional. Pre release Version of flyte will be used for sandbox."`
 
+	// Optionally it is possible to pass in environment variables to sandbox container.
+	Env []string `json:"env" pflag:",Optional. Provide Env variable in key=value format which can be passed to sandbox container."`
+
 	// Optionally it is possible to use local sandbox image
 	// If local flag pass then flytectl will not pull image from registry. Usually useful, if you want to test your local images without pushing them to a registry
 	ImagePullPolicy ImagePullPolicy `json:"imagePullPolicy" pflag:",Optional. Defines the image pull behavior [Always/IfNotPresent/Never]"`

cmd/configuration/configuration_test.go

@@ -23,19 +23,22 @@ func TestCreateInitCommand(t *testing.T) {
 	assert.Equal(t, configCmd.Use, "config")
 	assert.Equal(t, configCmd.Short, "Runs various config commands, look at the help of this command to get a list of available commands..")
 	fmt.Println(configCmd.Commands())
-	assert.Equal(t, len(configCmd.Commands()), 3)
+	assert.Equal(t, 4, len(configCmd.Commands()))
 	cmdNouns := configCmd.Commands()
 	// Sort by Use value.
 	sort.Slice(cmdNouns, func(i, j int) bool {
 		return cmdNouns[i].Use < cmdNouns[j].Use
 	})
 
-	assert.Equal(t, cmdNouns[0].Use, "discover")
-	assert.Equal(t, cmdNouns[0].Short, "Searches for a config in one of the default search paths.")
-	assert.Equal(t, cmdNouns[1].Use, "init")
-	assert.Equal(t, cmdNouns[1].Short, initCmdShort)
-	assert.Equal(t, cmdNouns[2].Use, "validate")
-	assert.Equal(t, cmdNouns[2].Short, "Validates the loaded config.")
+	assert.Equal(t, "discover", cmdNouns[0].Use)
+	assert.Equal(t, "Searches for a config in one of the default search paths.", cmdNouns[0].Short)
+	assert.Equal(t, "docs", cmdNouns[1].Use)
+	assert.Equal(t, "Generate configuration documetation in rst format", cmdNouns[1].Short)
+
+	assert.Equal(t, "init", cmdNouns[2].Use)
+	assert.Equal(t, initCmdShort, cmdNouns[2].Short)
+	assert.Equal(t, "validate", cmdNouns[3].Use)
+	assert.Equal(t, "Validates the loaded config.", cmdNouns[3].Short)
 
 }
 

cmd/register/register_util_test.go

@@ -276,8 +276,7 @@ func TestRegisterFile(t *testing.T) {
 		results, err := registerFile(ctx, args[0], "", registerResults, cmdCtx, *rconfig.DefaultFilesConfig)
 		assert.Equal(t, 1, len(results))
 		assert.Equal(t, "Failed", results[0].Status)
-		assert.Equal(t, "Error hydrating spec due to param values are missing on scheduled workflow for"+
-			" the following params [var1 var2]. Either specify them having a default or fixed value", results[0].Info)
+		assert.Contains(t, results[0].Info, "param values are missing on scheduled workflow for the following params")
 		assert.NotNil(t, err)
 	})
 	t.Run("Non existent file", func(t *testing.T) {
@@ -709,8 +708,7 @@ func TestValidateLaunchSpec(t *testing.T) {
 		}
 		err := validateLaunchSpec(ctx, lpSpec, cmdCtx)
 		assert.NotNil(t, err)
-		assert.Equal(t, "param values are missing on scheduled workflow for the following params [var1 var2]."+
-			" Either specify them having a default or fixed value", err.Error())
+		assert.Contains(t, err.Error(), "param values are missing on scheduled workflow for the following params")
 	})
 	t.Run("launchplan spec non empty schedule required param success", func(t *testing.T) {
 		setup()

cmd/sandbox/start.go

@@ -70,6 +70,28 @@ Specify a Flyte Sandbox image pull policy. Possible pull policy values are Alway
 ::
 
  flytectl sandbox start  --image docker.io/my-override:latest --imagePullPolicy Always
+
+Start sandbox cluster passing environment variables. This can be used to pass docker specific env variables or flyte specific env variables.
+eg : for passing timeout value in secs for the sandbox container use the following.
+::
+
+ flytectl sandbox start --env FLYTE_TIMEOUT=700
+
+
+The DURATION can be a positive integer or a floating-point number, followed by an optional unit suffix::
+s - seconds (default)
+m - minutes
+h - hours
+d - days
+When no unit is used, it defaults to seconds. If the duration is set to zero, the associated timeout is disabled.
+
+
+eg : for passing multiple environment variables
+::
+
+ flytectl sandbox start --env USER=foo --env PASSWORD=bar
+
+
 Usage
 `
 	k8sEndpoint          = "https://127.0.0.1:30086"
@@ -161,7 +183,8 @@ func startSandbox(ctx context.Context, cli docker.Docker, reader io.Reader) (*bu
 	}
 
 	volumes := docker.Volumes
-	if vol, err := mountVolume(sandboxConfig.DefaultConfig.Source, docker.Source); err != nil {
+	sandboxDefaultConfig := sandboxConfig.DefaultConfig
+	if vol, err := mountVolume(sandboxDefaultConfig.Source, docker.Source); err != nil {
 		return nil, err
 	} else if vol != nil {
 		volumes = append(volumes, *vol)
@@ -182,7 +205,9 @@ func startSandbox(ctx context.Context, cli docker.Docker, reader io.Reader) (*bu
 
 	fmt.Printf("%v booting Flyte-sandbox container\n", emoji.FactoryWorker)
 	exposedPorts, portBindings, _ := docker.GetSandboxPorts()
-	ID, err := docker.StartContainer(ctx, cli, volumes, exposedPorts, portBindings, docker.FlyteSandboxClusterName, sandboxImage)
+	ID, err := docker.StartContainer(ctx, cli, volumes, exposedPorts, portBindings, docker.FlyteSandboxClusterName,
+		sandboxImage, sandboxDefaultConfig.Env)
+
 	if err != nil {
 		fmt.Printf("%v Something went wrong: Failed to start Sandbox container %v, Please check your docker client and try again. \n", emoji.GrimacingFace, emoji.Whale)
 		return nil, err

go.mod

@@ -10,7 +10,7 @@ require (
 	github.com/docker/go-connections v0.4.0
 	github.com/enescakir/emoji v1.0.0
 	github.com/flyteorg/flyteidl v0.21.24
-	github.com/flyteorg/flytestdlib v0.4.0
+	github.com/flyteorg/flytestdlib v0.4.14
 	github.com/ghodss/yaml v1.0.0
 	github.com/go-ozzo/ozzo-validation/v4 v4.3.0
 	github.com/golang/protobuf v1.5.0