From 936134ee7e693ecf050ae5bd3cda217af026fe5c Mon Sep 17 00:00:00 2001 From: pmahindrakar-oss Date: Fri, 4 Mar 2022 19:50:37 +0530 Subject: [PATCH] Added env flag for sandbox start (#273) * Added env flag for sandbox start Signed-off-by: Prafulla Mahindrakar * ADDED docs for FLYTE_TIMEOUT Signed-off-by: Prafulla Mahindrakar * Fixed validation logic for launch plan schedule registration (#280) Signed-off-by: Prafulla Mahindrakar Signed-off-by: pmahindrakar-oss * boilerplate updates Signed-off-by: Prafulla Mahindrakar * fixed unit test after stdlib upgrade Signed-off-by: Prafulla Mahindrakar * fixed unit test Signed-off-by: Prafulla Mahindrakar --- boilerplate/flyte/golang_support_tools/go.mod | 182 +++++++++++++++++- .../flyte/golang_support_tools/tools.go | 2 +- .../golang_test_targets/download_tooling.sh | 4 +- cmd/config/subcommand/sandbox/config_flags.go | 1 + .../subcommand/sandbox/config_flags_test.go | 14 ++ .../subcommand/sandbox/sandbox_config.go | 3 + cmd/configuration/configuration_test.go | 17 +- cmd/register/register_util_test.go | 6 +- cmd/sandbox/start.go | 29 ++- docs/source/gen/flytectl_sandbox_start.rst | 23 +++ go.mod | 2 +- go.sum | 4 +- pkg/docker/docker_util.go | 5 +- pkg/docker/docker_util_test.go | 36 +++- 14 files changed, 303 insertions(+), 25 deletions(-) diff --git a/boilerplate/flyte/golang_support_tools/go.mod b/boilerplate/flyte/golang_support_tools/go.mod index 13941936c5..307398c89e 100644 --- a/boilerplate/flyte/golang_support_tools/go.mod +++ b/boilerplate/flyte/golang_support_tools/go.mod @@ -1,13 +1,191 @@ module github.com/flyteorg/boilerplate -go 1.16 +go 1.17 require ( github.com/alvaroloes/enumer v1.1.2 github.com/flyteorg/flytestdlib v0.4.7 github.com/golangci/golangci-lint v1.38.0 + github.com/pseudomuto/protoc-gen-doc v1.4.1 github.com/vektra/mockery v0.0.0-20181123154057-e78b021dcbb5 - github.com/pseudomuto/protoc-gen-doc v0.1.1 // indirect +) + +require ( + 4d63.com/gochecknoglobals v0.0.0-20201008074935-acfc0b28355a // indirect + cloud.google.com/go v0.75.0 // indirect + cloud.google.com/go/storage v1.12.0 // indirect + github.com/Azure/azure-sdk-for-go v51.0.0+incompatible // indirect + github.com/Azure/go-autorest v14.2.0+incompatible // indirect + github.com/Azure/go-autorest/autorest v0.11.17 // indirect + github.com/Azure/go-autorest/autorest/adal v0.9.10 // indirect + github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect + github.com/Azure/go-autorest/logger v0.2.0 // indirect + github.com/Azure/go-autorest/tracing v0.6.0 // indirect + github.com/BurntSushi/toml v0.3.1 // indirect + github.com/Djarvur/go-err113 v0.0.0-20210108212216-aea10b59be24 // indirect + github.com/Masterminds/semver v1.5.0 // indirect + github.com/Masterminds/sprig v2.15.0+incompatible // indirect + github.com/OpenPeeDeeP/depguard v1.0.1 // indirect + github.com/alexkohler/prealloc v1.0.0 // indirect + github.com/aokoli/goutils v1.0.1 // indirect + github.com/ashanbrown/forbidigo v1.1.0 // indirect + github.com/ashanbrown/makezero v0.0.0-20201205152432-7b7cdbb3025a // indirect + github.com/aws/aws-sdk-go v1.37.1 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/bkielbasa/cyclop v1.2.0 // indirect + github.com/bombsimon/wsl/v3 v3.2.0 // indirect + github.com/cespare/xxhash v1.1.0 // indirect + github.com/cespare/xxhash/v2 v2.1.1 // indirect + github.com/charithe/durationcheck v0.0.6 // indirect + github.com/coocood/freecache v1.1.1 // indirect + github.com/daixiang0/gci v0.2.8 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/denis-tingajkin/go-header v0.4.2 // indirect + github.com/envoyproxy/protoc-gen-validate v0.3.0-java // indirect + github.com/ernesto-jimenez/gogen v0.0.0-20180125220232-d7d4131e6607 // indirect + github.com/esimonov/ifshort v1.0.1 // indirect + github.com/fatih/color v1.10.0 // indirect + github.com/fatih/structtag v1.2.0 // indirect + github.com/form3tech-oss/jwt-go v3.2.2+incompatible // indirect + github.com/fsnotify/fsnotify v1.4.9 // indirect + github.com/fzipp/gocyclo v0.3.1 // indirect + github.com/ghodss/yaml v1.0.0 // indirect + github.com/go-critic/go-critic v0.5.4 // indirect + github.com/go-logr/logr v0.4.0 // indirect + github.com/go-toolsmith/astcast v1.0.0 // indirect + github.com/go-toolsmith/astcopy v1.0.0 // indirect + github.com/go-toolsmith/astequal v1.0.0 // indirect + github.com/go-toolsmith/astfmt v1.0.0 // indirect + github.com/go-toolsmith/astp v1.0.0 // indirect + github.com/go-toolsmith/strparse v1.0.0 // indirect + github.com/go-toolsmith/typep v1.0.2 // indirect + github.com/go-xmlfmt/xmlfmt v0.0.0-20191208150333-d5b6f63a941b // indirect + github.com/gobwas/glob v0.2.3 // indirect + github.com/gofrs/flock v0.8.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect + github.com/golang/protobuf v1.4.3 // indirect + github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 // indirect + github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a // indirect + github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613 // indirect + github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a // indirect + github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0 // indirect + github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca // indirect + github.com/golangci/misspell v0.3.5 // indirect + github.com/golangci/revgrep v0.0.0-20210208091834-cd28932614b5 // indirect + github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4 // indirect + github.com/google/go-cmp v0.5.4 // indirect + github.com/google/uuid v1.1.2 // indirect + github.com/googleapis/gax-go/v2 v2.0.5 // indirect + github.com/gordonklaus/ineffassign v0.0.0-20210225214923-2e10b2664254 // indirect + github.com/gostaticanalysis/analysisutil v0.4.1 // indirect + github.com/gostaticanalysis/comment v1.4.1 // indirect + github.com/gostaticanalysis/forcetypeassert v0.0.0-20200621232751-01d4955beaa5 // indirect + github.com/gostaticanalysis/nilerr v0.1.1 // indirect + github.com/graymeta/stow v0.2.7 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/huandu/xstrings v1.0.0 // indirect + github.com/imdario/mergo v0.3.5 // indirect + github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/jgautheron/goconst v1.4.0 // indirect + github.com/jingyugao/rowserrcheck v0.0.0-20210130005344-c6a0c12dd98d // indirect + github.com/jirfag/go-printf-func-name v0.0.0-20200119135958-7558a9eaa5af // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/jstemmer/go-junit-report v0.9.1 // indirect + github.com/julz/importas v0.0.0-20210226073942-60b4fa260dd0 // indirect + github.com/kisielk/errcheck v1.6.0 // indirect + github.com/kisielk/gotool v1.0.0 // indirect + github.com/kulti/thelper v0.4.0 // indirect + github.com/kunwardeep/paralleltest v1.0.2 // indirect + github.com/kyoh86/exportloopref v0.1.8 // indirect + github.com/magefile/mage v1.10.0 // indirect + github.com/magiconair/properties v1.8.4 // indirect + github.com/maratori/testpackage v1.0.1 // indirect + github.com/matoous/godox v0.0.0-20210227103229-6504466cf951 // indirect + github.com/mattn/go-colorable v0.1.8 // indirect + github.com/mattn/go-isatty v0.0.12 // indirect + github.com/mattn/go-runewidth v0.0.7 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/mbilski/exhaustivestruct v1.2.0 // indirect + github.com/mgechev/dots v0.0.0-20190921121421-c36f7dcfbb81 // indirect + github.com/mgechev/revive v1.0.3 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/mitchellh/mapstructure v1.4.1 // indirect + github.com/moricho/tparallel v0.2.1 // indirect + github.com/mwitkow/go-proto-validators v0.0.0-20180403085117-0950a7990007 // indirect + github.com/nakabonne/nestif v0.3.0 // indirect + github.com/nbutton23/zxcvbn-go v0.0.0-20201221231540-e56b841a3c88 // indirect + github.com/ncw/swift v1.0.53 // indirect + github.com/nishanths/exhaustive v0.1.0 // indirect + github.com/nishanths/predeclared v0.2.1 // indirect + github.com/olekukonko/tablewriter v0.0.4 // indirect + github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1 // indirect + github.com/pelletier/go-toml v1.8.1 // indirect + github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/polyfloyd/go-errorlint v0.0.0-20201127212506-19bd8db6546f // indirect + github.com/prometheus/client_golang v1.9.0 // indirect + github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/common v0.15.0 // indirect + github.com/prometheus/procfs v0.3.0 // indirect + github.com/pseudomuto/protokit v0.2.0 // indirect + github.com/quasilyte/go-ruleguard v0.3.0 // indirect + github.com/quasilyte/regex/syntax v0.0.0-20200407221936-30656e2c4a95 // indirect + github.com/ryancurrah/gomodguard v1.2.0 // indirect + github.com/ryanrolds/sqlclosecheck v0.3.0 // indirect + github.com/sanposhiho/wastedassign v0.1.3 // indirect + github.com/satori/go.uuid v1.2.0 // indirect + github.com/securego/gosec/v2 v2.6.1 // indirect + github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c // indirect + github.com/sirupsen/logrus v1.8.0 // indirect + github.com/sonatard/noctx v0.0.1 // indirect + github.com/sourcegraph/go-diff v0.6.1 // indirect + github.com/spf13/afero v1.5.1 // indirect + github.com/spf13/cast v1.3.1 // indirect + github.com/spf13/cobra v1.1.3 // indirect + github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/viper v1.7.1 // indirect + github.com/ssgreg/nlreturn/v2 v2.1.0 // indirect + github.com/stretchr/objx v0.3.0 // indirect + github.com/stretchr/testify v1.7.0 // indirect + github.com/subosito/gotenv v1.2.0 // indirect + github.com/tdakkota/asciicheck v0.0.0-20200416200610-e657995f937b // indirect + github.com/tetafro/godot v1.4.4 // indirect + github.com/timakin/bodyclose v0.0.0-20200424151742-cb6215831a94 // indirect + github.com/tomarrell/wrapcheck v0.0.0-20201130113247-1683564d9756 // indirect + github.com/tommy-muehle/go-mnd/v2 v2.3.1 // indirect + github.com/ultraware/funlen v0.0.3 // indirect + github.com/ultraware/whitespace v0.0.4 // indirect + github.com/uudashr/gocognit v1.0.1 // indirect + go.opencensus.io v0.22.6 // indirect + golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad // indirect + golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 // indirect + golang.org/x/mod v0.4.1 // indirect + golang.org/x/net v0.0.0-20210119194325-5f4716e94777 // indirect + golang.org/x/oauth2 v0.0.0-20210126194326-f9ce19ea3013 // indirect + golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c // indirect + golang.org/x/text v0.3.5 // indirect + golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect + golang.org/x/tools v0.1.0 // indirect + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + google.golang.org/api v0.38.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/genproto v0.0.0-20210126160654-44e461bb6506 // indirect + google.golang.org/grpc v1.35.0 // indirect + google.golang.org/protobuf v1.25.0 // indirect + gopkg.in/ini.v1 v1.62.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect + honnef.co/go/tools v0.1.2 // indirect + k8s.io/apimachinery v0.20.2 // indirect + k8s.io/client-go v0.0.0-20210217172142-7279fc64d847 // indirect + k8s.io/klog/v2 v2.5.0 // indirect + mvdan.cc/gofumpt v0.1.0 // indirect + mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed // indirect + mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b // indirect + mvdan.cc/unparam v0.0.0-20210104141923-aac4ce9116a7 // indirect ) replace github.com/vektra/mockery => github.com/enghabu/mockery v0.0.0-20191009061720-9d0c8670c2f0 diff --git a/boilerplate/flyte/golang_support_tools/tools.go b/boilerplate/flyte/golang_support_tools/tools.go index eee691d8c3..d970d2106a 100644 --- a/boilerplate/flyte/golang_support_tools/tools.go +++ b/boilerplate/flyte/golang_support_tools/tools.go @@ -7,5 +7,5 @@ import ( _ "github.com/flyteorg/flytestdlib/cli/pflags" _ "github.com/golangci/golangci-lint/cmd/golangci-lint" _ "github.com/vektra/mockery/cmd/mockery" - - "github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc" + _ "github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc" ) diff --git a/boilerplate/flyte/golang_test_targets/download_tooling.sh b/boilerplate/flyte/golang_test_targets/download_tooling.sh index c0ab06b063..02fb004881 100755 --- a/boilerplate/flyte/golang_test_targets/download_tooling.sh +++ b/boilerplate/flyte/golang_test_targets/download_tooling.sh @@ -17,8 +17,8 @@ set -e # In the format of ":" or ":" if no cli tools=( "github.com/vektra/mockery/cmd/mockery" - "github.com/flyteorg/flytestdlib/cli/pflags" - "github.com/golangci/golangci-lint/cmd/golangci-lint" + "github.com/flyteorg/flytestdlib/cli/pflags@latest" + "github.com/golangci/golangci-lint/cmd/golangci-lint@latest" "github.com/alvaroloes/enumer" "github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc" ) diff --git a/cmd/config/subcommand/sandbox/config_flags.go b/cmd/config/subcommand/sandbox/config_flags.go index 5e4451b922..0355ee08f5 100755 --- a/cmd/config/subcommand/sandbox/config_flags.go +++ b/cmd/config/subcommand/sandbox/config_flags.go @@ -54,6 +54,7 @@ func (cfg Config) GetPFlagSet(prefix string) *pflag.FlagSet { cmdFlags.StringVar(&DefaultConfig.Version, fmt.Sprintf("%v%v", prefix, "version"), DefaultConfig.Version, "Version of flyte. Only supports flyte releases greater than v0.10.0") cmdFlags.StringVar(&DefaultConfig.Image, fmt.Sprintf("%v%v", prefix, "image"), DefaultConfig.Image, "Optional. Provide a fully qualified path to a Flyte compliant docker image.") cmdFlags.BoolVar(&DefaultConfig.Prerelease, fmt.Sprintf("%v%v", prefix, "pre"), DefaultConfig.Prerelease, "Optional. Pre release Version of flyte will be used for sandbox.") + cmdFlags.StringSliceVar(&DefaultConfig.Env, fmt.Sprintf("%v%v", prefix, "env"), DefaultConfig.Env, "Optional. Provide Env variable in key=value format which can be passed to sandbox container.") cmdFlags.Var(&DefaultConfig.ImagePullPolicy, fmt.Sprintf("%v%v", prefix, "imagePullPolicy"), "Optional. Defines the image pull behavior [Always/IfNotPresent/Never]") return cmdFlags } diff --git a/cmd/config/subcommand/sandbox/config_flags_test.go b/cmd/config/subcommand/sandbox/config_flags_test.go index da98f59320..6433a3e647 100755 --- a/cmd/config/subcommand/sandbox/config_flags_test.go +++ b/cmd/config/subcommand/sandbox/config_flags_test.go @@ -155,6 +155,20 @@ func TestConfig_SetFlags(t *testing.T) { } }) }) + t.Run("Test_env", func(t *testing.T) { + + t.Run("Override", func(t *testing.T) { + testValue := join_Config(DefaultConfig.Env, ",") + + cmdFlags.Set("env", testValue) + if vStringSlice, err := cmdFlags.GetStringSlice("env"); err == nil { + testDecodeRaw_Config(t, join_Config(vStringSlice, ","), &actual.Env) + + } else { + assert.FailNow(t, err.Error()) + } + }) + }) t.Run("Test_imagePullPolicy", func(t *testing.T) { t.Run("Override", func(t *testing.T) { diff --git a/cmd/config/subcommand/sandbox/sandbox_config.go b/cmd/config/subcommand/sandbox/sandbox_config.go index 77246f09d0..be553a4efc 100644 --- a/cmd/config/subcommand/sandbox/sandbox_config.go +++ b/cmd/config/subcommand/sandbox/sandbox_config.go @@ -47,6 +47,9 @@ type Config struct { // Default value false represents that flytectl will not use latest pre release if exist Prerelease bool `json:"pre" pflag:",Optional. Pre release Version of flyte will be used for sandbox."` + // Optionally it is possible to pass in environment variables to sandbox container. + Env []string `json:"env" pflag:",Optional. Provide Env variable in key=value format which can be passed to sandbox container."` + // Optionally it is possible to use local sandbox image // If local flag pass then flytectl will not pull image from registry. Usually useful, if you want to test your local images without pushing them to a registry ImagePullPolicy ImagePullPolicy `json:"imagePullPolicy" pflag:",Optional. Defines the image pull behavior [Always/IfNotPresent/Never]"` diff --git a/cmd/configuration/configuration_test.go b/cmd/configuration/configuration_test.go index 4f6e6a6186..bb2d256229 100644 --- a/cmd/configuration/configuration_test.go +++ b/cmd/configuration/configuration_test.go @@ -23,19 +23,22 @@ func TestCreateInitCommand(t *testing.T) { assert.Equal(t, configCmd.Use, "config") assert.Equal(t, configCmd.Short, "Runs various config commands, look at the help of this command to get a list of available commands..") fmt.Println(configCmd.Commands()) - assert.Equal(t, len(configCmd.Commands()), 3) + assert.Equal(t, 4, len(configCmd.Commands())) cmdNouns := configCmd.Commands() // Sort by Use value. sort.Slice(cmdNouns, func(i, j int) bool { return cmdNouns[i].Use < cmdNouns[j].Use }) - assert.Equal(t, cmdNouns[0].Use, "discover") - assert.Equal(t, cmdNouns[0].Short, "Searches for a config in one of the default search paths.") - assert.Equal(t, cmdNouns[1].Use, "init") - assert.Equal(t, cmdNouns[1].Short, initCmdShort) - assert.Equal(t, cmdNouns[2].Use, "validate") - assert.Equal(t, cmdNouns[2].Short, "Validates the loaded config.") + assert.Equal(t, "discover", cmdNouns[0].Use) + assert.Equal(t, "Searches for a config in one of the default search paths.", cmdNouns[0].Short) + assert.Equal(t, "docs", cmdNouns[1].Use) + assert.Equal(t, "Generate configuration documetation in rst format", cmdNouns[1].Short) + + assert.Equal(t, "init", cmdNouns[2].Use) + assert.Equal(t, initCmdShort, cmdNouns[2].Short) + assert.Equal(t, "validate", cmdNouns[3].Use) + assert.Equal(t, "Validates the loaded config.", cmdNouns[3].Short) } diff --git a/cmd/register/register_util_test.go b/cmd/register/register_util_test.go index 15a14a8887..491c0c1f98 100644 --- a/cmd/register/register_util_test.go +++ b/cmd/register/register_util_test.go @@ -276,8 +276,7 @@ func TestRegisterFile(t *testing.T) { results, err := registerFile(ctx, args[0], "", registerResults, cmdCtx, *rconfig.DefaultFilesConfig) assert.Equal(t, 1, len(results)) assert.Equal(t, "Failed", results[0].Status) - assert.Equal(t, "Error hydrating spec due to param values are missing on scheduled workflow for"+ - " the following params [var1 var2]. Either specify them having a default or fixed value", results[0].Info) + assert.Contains(t, results[0].Info, "param values are missing on scheduled workflow for the following params") assert.NotNil(t, err) }) t.Run("Non existent file", func(t *testing.T) { @@ -709,8 +708,7 @@ func TestValidateLaunchSpec(t *testing.T) { } err := validateLaunchSpec(ctx, lpSpec, cmdCtx) assert.NotNil(t, err) - assert.Equal(t, "param values are missing on scheduled workflow for the following params [var1 var2]."+ - " Either specify them having a default or fixed value", err.Error()) + assert.Contains(t, err.Error(), "param values are missing on scheduled workflow for the following params") }) t.Run("launchplan spec non empty schedule required param success", func(t *testing.T) { setup() diff --git a/cmd/sandbox/start.go b/cmd/sandbox/start.go index a946601c5c..ece842beb8 100644 --- a/cmd/sandbox/start.go +++ b/cmd/sandbox/start.go @@ -70,6 +70,28 @@ Specify a Flyte Sandbox image pull policy. Possible pull policy values are Alway :: flytectl sandbox start --image docker.io/my-override:latest --imagePullPolicy Always + +Start sandbox cluster passing environment variables. This can be used to pass docker specific env variables or flyte specific env variables. +eg : for passing timeout value in secs for the sandbox container use the following. +:: + + flytectl sandbox start --env FLYTE_TIMEOUT=700 + + +The DURATION can be a positive integer or a floating-point number, followed by an optional unit suffix:: +s - seconds (default) +m - minutes +h - hours +d - days +When no unit is used, it defaults to seconds. If the duration is set to zero, the associated timeout is disabled. + + +eg : for passing multiple environment variables +:: + + flytectl sandbox start --env USER=foo --env PASSWORD=bar + + Usage ` k8sEndpoint = "https://127.0.0.1:30086" @@ -161,7 +183,8 @@ func startSandbox(ctx context.Context, cli docker.Docker, reader io.Reader) (*bu } volumes := docker.Volumes - if vol, err := mountVolume(sandboxConfig.DefaultConfig.Source, docker.Source); err != nil { + sandboxDefaultConfig := sandboxConfig.DefaultConfig + if vol, err := mountVolume(sandboxDefaultConfig.Source, docker.Source); err != nil { return nil, err } else if vol != nil { volumes = append(volumes, *vol) @@ -182,7 +205,9 @@ func startSandbox(ctx context.Context, cli docker.Docker, reader io.Reader) (*bu fmt.Printf("%v booting Flyte-sandbox container\n", emoji.FactoryWorker) exposedPorts, portBindings, _ := docker.GetSandboxPorts() - ID, err := docker.StartContainer(ctx, cli, volumes, exposedPorts, portBindings, docker.FlyteSandboxClusterName, sandboxImage) + ID, err := docker.StartContainer(ctx, cli, volumes, exposedPorts, portBindings, docker.FlyteSandboxClusterName, + sandboxImage, sandboxDefaultConfig.Env) + if err != nil { fmt.Printf("%v Something went wrong: Failed to start Sandbox container %v, Please check your docker client and try again. \n", emoji.GrimacingFace, emoji.Whale) return nil, err diff --git a/docs/source/gen/flytectl_sandbox_start.rst b/docs/source/gen/flytectl_sandbox_start.rst index ec6fa871cf..941cb57926 100644 --- a/docs/source/gen/flytectl_sandbox_start.rst +++ b/docs/source/gen/flytectl_sandbox_start.rst @@ -47,6 +47,28 @@ Specify a Flyte Sandbox image pull policy. Possible pull policy values are Alway :: flytectl sandbox start --image docker.io/my-override:latest --imagePullPolicy Always + +Start sandbox cluster passing environment variables. This can be used to pass docker specific env variables or flyte specific env variables. +eg : for passing timeout value in secs for the sandbox container use the following. +:: + + flytectl sandbox start --env FLYTE_TIMEOUT=700 + + +The DURATION can be a positive integer or a floating-point number, followed by an optional unit suffix:: +s - seconds (default) +m - minutes +h - hours +d - days +When no unit is used, it defaults to seconds. If the duration is set to zero, the associated timeout is disabled. + + +eg : for passing multiple environment variables +:: + + flytectl sandbox start --env USER=foo --env PASSWORD=bar + + Usage @@ -59,6 +81,7 @@ Options :: + --env strings Optional. Provide Env variable in key=value format which can be passed to sandbox container. -h, --help help for start --image string Optional. Provide a fully qualified path to a Flyte compliant docker image. --imagePullPolicy ImagePullPolicy Optional. Defines the image pull behavior [Always/IfNotPresent/Never] (default Always) diff --git a/go.mod b/go.mod index b34924a30d..096232e2c3 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/docker/go-connections v0.4.0 github.com/enescakir/emoji v1.0.0 github.com/flyteorg/flyteidl v0.21.24 - github.com/flyteorg/flytestdlib v0.4.0 + github.com/flyteorg/flytestdlib v0.4.14 github.com/ghodss/yaml v1.0.0 github.com/go-ozzo/ozzo-validation/v4 v4.3.0 github.com/golang/protobuf v1.5.0 diff --git a/go.sum b/go.sum index 9787249411..a3a5753e53 100644 --- a/go.sum +++ b/go.sum @@ -359,8 +359,8 @@ github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4 github.com/flyteorg/flyteidl v0.21.24 h1:e2wPBK4aiLE+fw2zmhUDNg39QoJk6Lf5lQRvj8XgtFk= github.com/flyteorg/flyteidl v0.21.24/go.mod h1:576W2ViEyjTpT+kEVHAGbrTP3HARNUZ/eCwrNPmdx9U= github.com/flyteorg/flytestdlib v0.3.13/go.mod h1:Tz8JCECAbX6VWGwFT6cmEQ+RJpZ/6L9pswu3fzWs220= -github.com/flyteorg/flytestdlib v0.4.0 h1:cEMkNfjocCuBSLzM9tKjsODhkr5gXTZAGl6k62FrT60= -github.com/flyteorg/flytestdlib v0.4.0/go.mod h1:7cDWkY3v7xsoesFcDdu6DSW5Q2U2W5KlHUbUHSwBG1Q= +github.com/flyteorg/flytestdlib v0.4.14 h1:qpPwvJ+DqM1fI/y5uPKAP8p8VONz8oRp9Fz0jFl/5aI= +github.com/flyteorg/flytestdlib v0.4.14/go.mod h1:fv1ar34LJLMTaf0tbfetisLykUlARi7rP+NQTUn6QQs= github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= diff --git a/pkg/docker/docker_util.go b/pkg/docker/docker_util.go index a1d952475f..cf8f96a2c6 100644 --- a/pkg/docker/docker_util.go +++ b/pkg/docker/docker_util.go @@ -131,7 +131,10 @@ func PullDockerImage(ctx context.Context, cli Docker, image string, pullPolicy s } //StartContainer will create and start docker container -func StartContainer(ctx context.Context, cli Docker, volumes []mount.Mount, exposedPorts map[nat.Port]struct{}, portBindings map[nat.Port][]nat.PortBinding, name, image string) (string, error) { +func StartContainer(ctx context.Context, cli Docker, volumes []mount.Mount, exposedPorts map[nat.Port]struct{}, + portBindings map[nat.Port][]nat.PortBinding, name, image string, additionalEnvVars []string) (string, error) { + // Append the additional env variables to the default list of env + Environment = append(Environment, additionalEnvVars...) resp, err := cli.ContainerCreate(ctx, &container.Config{ Env: Environment, Image: image, diff --git a/pkg/docker/docker_util_test.go b/pkg/docker/docker_util_test.go index cc27fd8af4..1915722640 100644 --- a/pkg/docker/docker_util_test.go +++ b/pkg/docker/docker_util_test.go @@ -166,12 +166,42 @@ func TestStartContainer(t *testing.T) { ID: "Hello", }, nil) mockDocker.OnContainerStart(context, "Hello", types.ContainerStartOptions{}).Return(nil) - id, err := StartContainer(context, mockDocker, Volumes, p1, p2, "nginx", imageName) + id, err := StartContainer(context, mockDocker, Volumes, p1, p2, "nginx", imageName, nil) assert.Nil(t, err) assert.Greater(t, len(id), 0) assert.Equal(t, id, "Hello") }) + t.Run("Successfully create a container with Env", func(t *testing.T) { + setupSandbox() + mockDocker := &mocks.Docker{} + context := context.Background() + // Setup additional env + additionalEnv := []string{"a=1", "b=2"} + expectedEnv := append(Environment, "a=1") + expectedEnv = append(expectedEnv, "b=2") + + // Verify the attributes + mockDocker.OnContainerCreate(context, &container.Config{ + Env: expectedEnv, + Image: imageName, + Tty: false, + ExposedPorts: p1, + }, &container.HostConfig{ + Mounts: Volumes, + PortBindings: p2, + Privileged: true, + }, nil, nil, mock.Anything).Return(container.ContainerCreateCreatedBody{ + ID: "Hello", + }, nil) + mockDocker.OnContainerStart(context, "Hello", types.ContainerStartOptions{}).Return(nil) + id, err := StartContainer(context, mockDocker, Volumes, p1, p2, "nginx", imageName, additionalEnv) + assert.Nil(t, err) + assert.Greater(t, len(id), 0) + assert.Equal(t, id, "Hello") + assert.Equal(t, expectedEnv, Environment) + }) + t.Run("Error in creating container", func(t *testing.T) { setupSandbox() mockDocker := &mocks.Docker{} @@ -191,7 +221,7 @@ func TestStartContainer(t *testing.T) { ID: "", }, fmt.Errorf("error")) mockDocker.OnContainerStart(context, "Hello", types.ContainerStartOptions{}).Return(nil) - id, err := StartContainer(context, mockDocker, Volumes, p1, p2, "nginx", imageName) + id, err := StartContainer(context, mockDocker, Volumes, p1, p2, "nginx", imageName, nil) assert.NotNil(t, err) assert.Equal(t, len(id), 0) assert.Equal(t, id, "") @@ -216,7 +246,7 @@ func TestStartContainer(t *testing.T) { ID: "Hello", }, nil) mockDocker.OnContainerStart(context, "Hello", types.ContainerStartOptions{}).Return(fmt.Errorf("error")) - id, err := StartContainer(context, mockDocker, Volumes, p1, p2, "nginx", imageName) + id, err := StartContainer(context, mockDocker, Volumes, p1, p2, "nginx", imageName, nil) assert.NotNil(t, err) assert.Equal(t, len(id), 0) assert.Equal(t, id, "")