From 6148a7ab68586e1b33edfc11aae0a307bc3e1a2a Mon Sep 17 00:00:00 2001 From: Yuvraj Date: Tue, 25 Jan 2022 11:53:43 +0530 Subject: [PATCH] Always use sha for sandbox image (#266) * Always use sha for sandbox image Signed-off-by: Yuvraj * Upgrade go 1.17 Signed-off-by: Yuvraj --- flytectl/.github/workflows/build.yaml | 40 +++++- flytectl/.github/workflows/generate-docs.yaml | 2 +- flytectl/.github/workflows/release.yml | 2 +- .../workflows/update-flyte-component.yml | 2 +- .../subcommand/project/project_config_test.go | 41 ++++++ .../config/subcommand/sandbox/config_flags.go | 1 + .../subcommand/sandbox/config_flags_test.go | 14 +++ .../subcommand/sandbox/sandbox_config.go | 3 + flytectl/cmd/register/examples.go | 2 +- flytectl/cmd/register/register_util.go | 2 +- flytectl/cmd/sandbox/start.go | 65 +++------- flytectl/cmd/sandbox/start_test.go | 96 ++++++++------ flytectl/cmd/upgrade/upgrade.go | 1 + flytectl/go.mod | 118 +++++++++++++++--- flytectl/go.sum | 14 +-- flytectl/pkg/docker/docker_util.go | 37 +++--- flytectl/pkg/docker/docker_util_test.go | 13 +- flytectl/pkg/util/githubutil/githubutil.go | 92 +++++++++++--- flytectl/pkg/util/util.go | 29 ++--- flytectl/pkg/util/util_test.go | 48 +++---- 20 files changed, 427 insertions(+), 195 deletions(-) diff --git a/flytectl/.github/workflows/build.yaml b/flytectl/.github/workflows/build.yaml index 148963bd0c..736cd5e0bb 100644 --- a/flytectl/.github/workflows/build.yaml +++ b/flytectl/.github/workflows/build.yaml @@ -6,8 +6,8 @@ on: - master jobs: - build: - name: Run tests and lint + dry-run-goreleaser: + name: Dry Run Goreleaser runs-on: ubuntu-latest steps: - name: Checkout @@ -22,16 +22,35 @@ jobs: key: ${{ runner.os }}-go-${{ hashFiles('go.sum') }} - uses: actions/setup-go@v2 with: - go-version: '1.16' + go-version: '1.17' - name: Run GoReleaser dry run uses: goreleaser/goreleaser-action@v2 with: version: latest args: --snapshot --skip-publish --rm-dist + + unit-test: + name: Run unit test + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: "2" + - uses: actions/cache@v2 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('go.sum') }} + - uses: actions/setup-go@v2 + with: + go-version: '1.17' - name: Unit Tests env: GO111MODULE: "on" CI_ENV: "true" + GITHUB_TOKEN: ${{ secrets.FLYTE_BOT_PAT }} run: make install && make test_unit_without_flag - name: Push CodeCov uses: codecov/codecov-action@v1.0.5 @@ -39,6 +58,19 @@ jobs: file: coverage.txt flags: unittests fail_ci_if_error: true + + + lint: + name: Run Lint + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: "2" + - uses: actions/setup-go@v2 + with: + go-version: '1.17' - name: Lint env: GO111MODULE: "on" @@ -50,6 +82,6 @@ jobs: - uses: actions/checkout@v1 - uses: actions/setup-go@v2 with: - go-version: '1.16' + go-version: '1.17' - name: Go generate and diff run: DELTA_CHECK=true make generate \ No newline at end of file diff --git a/flytectl/.github/workflows/generate-docs.yaml b/flytectl/.github/workflows/generate-docs.yaml index 1c442ea4f8..853aefe30e 100644 --- a/flytectl/.github/workflows/generate-docs.yaml +++ b/flytectl/.github/workflows/generate-docs.yaml @@ -15,7 +15,7 @@ jobs: lfs: true - uses: actions/setup-go@v1 with: - go-version: '1.16' + go-version: '1.17' - uses: actions/setup-python@v1 with: python-version: 3.8 diff --git a/flytectl/.github/workflows/release.yml b/flytectl/.github/workflows/release.yml index 7137ab353c..47d6863674 100644 --- a/flytectl/.github/workflows/release.yml +++ b/flytectl/.github/workflows/release.yml @@ -16,7 +16,7 @@ jobs: fetch-depth: "0" - uses: actions/setup-go@v2 with: - go-version: '1.16' + go-version: '1.17' - name: Run GoReleaser dry run uses: goreleaser/goreleaser-action@v2 with: diff --git a/flytectl/.github/workflows/update-flyte-component.yml b/flytectl/.github/workflows/update-flyte-component.yml index 7893e4fb8b..ec765da89f 100644 --- a/flytectl/.github/workflows/update-flyte-component.yml +++ b/flytectl/.github/workflows/update-flyte-component.yml @@ -13,7 +13,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v2 with: - go-version: 1.16 + go-version: 1.17 - uses: crazy-max/ghaction-import-gpg@v3 with: gpg-private-key: ${{ secrets.FLYTE_BOT_GPG_PRIVATE_KEY }} diff --git a/flytectl/cmd/config/subcommand/project/project_config_test.go b/flytectl/cmd/config/subcommand/project/project_config_test.go index fdad3cc33a..a44cd0b423 100644 --- a/flytectl/cmd/config/subcommand/project/project_config_test.go +++ b/flytectl/cmd/config/subcommand/project/project_config_test.go @@ -1,8 +1,12 @@ package project import ( + "errors" "testing" + "github.com/flyteorg/flytectl/clierrors" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" + "github.com/stretchr/testify/assert" ) @@ -25,3 +29,40 @@ func TestGetProjectSpec(t *testing.T) { assert.Equal(t, "flytesnacks test", response.Description) }) } + +func TestMapToAdminState(t *testing.T) { + t.Run("Successful mapToAdminState with archive", func(t *testing.T) { + c := &ConfigProject{ + Archive: true, + } + state, err := c.MapToAdminState() + assert.Nil(t, err) + assert.Equal(t, admin.Project_ARCHIVED, state) + }) + t.Run("Successful mapToAdminState with activate", func(t *testing.T) { + c := &ConfigProject{ + Activate: true, + } + state, err := c.MapToAdminState() + assert.Nil(t, err) + assert.Equal(t, admin.Project_ACTIVE, state) + }) + t.Run("Invalid state", func(t *testing.T) { + c := &ConfigProject{ + Activate: true, + Archive: true, + } + state, err := c.MapToAdminState() + assert.NotNil(t, err) + assert.Equal(t, errors.New(clierrors.ErrInvalidStateUpdate), err) + assert.Equal(t, admin.Project_ACTIVE, state) + }) + t.Run("deprecated Flags Test", func(t *testing.T) { + c := &ConfigProject{ + ActivateProject: true, + } + state, err := c.MapToAdminState() + assert.Nil(t, err) + assert.Equal(t, admin.Project_ACTIVE, state) + }) +} diff --git a/flytectl/cmd/config/subcommand/sandbox/config_flags.go b/flytectl/cmd/config/subcommand/sandbox/config_flags.go index 5339211026..5e4451b922 100755 --- a/flytectl/cmd/config/subcommand/sandbox/config_flags.go +++ b/flytectl/cmd/config/subcommand/sandbox/config_flags.go @@ -53,6 +53,7 @@ func (cfg Config) GetPFlagSet(prefix string) *pflag.FlagSet { cmdFlags.StringVar(&DefaultConfig.Source, fmt.Sprintf("%v%v", prefix, "source"), DefaultConfig.Source, "Path of your source code") cmdFlags.StringVar(&DefaultConfig.Version, fmt.Sprintf("%v%v", prefix, "version"), DefaultConfig.Version, "Version of flyte. Only supports flyte releases greater than v0.10.0") cmdFlags.StringVar(&DefaultConfig.Image, fmt.Sprintf("%v%v", prefix, "image"), DefaultConfig.Image, "Optional. Provide a fully qualified path to a Flyte compliant docker image.") + cmdFlags.BoolVar(&DefaultConfig.Prerelease, fmt.Sprintf("%v%v", prefix, "pre"), DefaultConfig.Prerelease, "Optional. Pre release Version of flyte will be used for sandbox.") cmdFlags.Var(&DefaultConfig.ImagePullPolicy, fmt.Sprintf("%v%v", prefix, "imagePullPolicy"), "Optional. Defines the image pull behavior [Always/IfNotPresent/Never]") return cmdFlags } diff --git a/flytectl/cmd/config/subcommand/sandbox/config_flags_test.go b/flytectl/cmd/config/subcommand/sandbox/config_flags_test.go index 4e1410c228..da98f59320 100755 --- a/flytectl/cmd/config/subcommand/sandbox/config_flags_test.go +++ b/flytectl/cmd/config/subcommand/sandbox/config_flags_test.go @@ -141,6 +141,20 @@ func TestConfig_SetFlags(t *testing.T) { } }) }) + t.Run("Test_pre", func(t *testing.T) { + + t.Run("Override", func(t *testing.T) { + testValue := "1" + + cmdFlags.Set("pre", testValue) + if vBool, err := cmdFlags.GetBool("pre"); err == nil { + testDecodeJson_Config(t, fmt.Sprintf("%v", vBool), &actual.Prerelease) + + } else { + assert.FailNow(t, err.Error()) + } + }) + }) t.Run("Test_imagePullPolicy", func(t *testing.T) { t.Run("Override", func(t *testing.T) { diff --git a/flytectl/cmd/config/subcommand/sandbox/sandbox_config.go b/flytectl/cmd/config/subcommand/sandbox/sandbox_config.go index ff7559b562..77246f09d0 100644 --- a/flytectl/cmd/config/subcommand/sandbox/sandbox_config.go +++ b/flytectl/cmd/config/subcommand/sandbox/sandbox_config.go @@ -44,6 +44,9 @@ type Config struct { // from there. Image string `json:"image" pflag:",Optional. Provide a fully qualified path to a Flyte compliant docker image."` + // Default value false represents that flytectl will not use latest pre release if exist + Prerelease bool `json:"pre" pflag:",Optional. Pre release Version of flyte will be used for sandbox."` + // Optionally it is possible to use local sandbox image // If local flag pass then flytectl will not pull image from registry. Usually useful, if you want to test your local images without pushing them to a registry ImagePullPolicy ImagePullPolicy `json:"imagePullPolicy" pflag:",Optional. Defines the image pull behavior [Always/IfNotPresent/Never]"` diff --git a/flytectl/cmd/register/examples.go b/flytectl/cmd/register/examples.go index d19641fd85..845dcc417c 100644 --- a/flytectl/cmd/register/examples.go +++ b/flytectl/cmd/register/examples.go @@ -6,7 +6,7 @@ import ( "github.com/flyteorg/flytestdlib/logger" - "github.com/google/go-github/v37/github" + "github.com/google/go-github/v42/github" rconfig "github.com/flyteorg/flytectl/cmd/config/subcommand/register" cmdCore "github.com/flyteorg/flytectl/cmd/core" diff --git a/flytectl/cmd/register/register_util.go b/flytectl/cmd/register/register_util.go index 30f2ae8350..29e83c698d 100644 --- a/flytectl/cmd/register/register_util.go +++ b/flytectl/cmd/register/register_util.go @@ -30,7 +30,7 @@ import ( "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" "github.com/flyteorg/flytestdlib/logger" "github.com/flyteorg/flytestdlib/storage" - "github.com/google/go-github/v37/github" + "github.com/google/go-github/v42/github" "github.com/golang/protobuf/jsonpb" "github.com/golang/protobuf/proto" diff --git a/flytectl/cmd/sandbox/start.go b/flytectl/cmd/sandbox/start.go index 2023806dd5..00c445d509 100644 --- a/flytectl/cmd/sandbox/start.go +++ b/flytectl/cmd/sandbox/start.go @@ -9,10 +9,10 @@ import ( "path/filepath" "time" - "github.com/flyteorg/flytectl/clierrors" "github.com/flyteorg/flytectl/pkg/util/githubutil" "github.com/avast/retry-go" + "github.com/flyteorg/flytectl/clierrors" "github.com/olekukonko/tablewriter" corev1api "k8s.io/api/core/v1" corev1 "k8s.io/client-go/kubernetes/typed/core/v1" @@ -64,15 +64,13 @@ Specify a Flyte Sandbox image pull policy. Possible pull policy values are Alway flytectl sandbox start --image docker.io/my-override:latest --imagePullPolicy Always Usage ` - k8sEndpoint = "https://127.0.0.1:30086" - flyteNamespace = "flyte" - flyteRepository = "flyte" - dind = "dind" - sandboxSupportedVersion = "v0.10.0" - diskPressureTaint = "node.kubernetes.io/disk-pressure" - taintEffect = "NoSchedule" - sandboxContextName = "flyte-sandbox" - sandboxDockerContext = "default" + k8sEndpoint = "https://127.0.0.1:30086" + flyteNamespace = "flyte" + diskPressureTaint = "node.kubernetes.io/disk-pressure" + taintEffect = "NoSchedule" + sandboxContextName = "flyte-sandbox" + sandboxDockerContext = "default" + imageName = "cr.flyte.org/flyteorg/flyte-sandbox" ) type ExecResult struct { @@ -160,20 +158,23 @@ func startSandbox(ctx context.Context, cli docker.Docker, reader io.Reader) (*bu } else if vol != nil { volumes = append(volumes, *vol) } - - image, err := getSandboxImage(sandboxConfig.DefaultConfig.Version, sandboxConfig.DefaultConfig.Image) - if err != nil { - return nil, err + sandboxImage := sandboxConfig.DefaultConfig.Image + if len(sandboxConfig.DefaultConfig.Image) == 0 { + image, version, err := githubutil.GetSandboxImage(sandboxConfig.DefaultConfig.Version, imageName) + if err != nil { + return nil, err + } + sandboxImage = image + fmt.Printf("%v Running Flyte %s release\n", emoji.Whale, version) } - fmt.Printf("%v pulling docker image for release %s\n", emoji.Whale, image) - - if err := docker.PullDockerImage(ctx, cli, image, sandboxConfig.DefaultConfig.ImagePullPolicy); err != nil { + fmt.Printf("%v pulling docker image for release %s\n", emoji.Whale, sandboxImage) + if err := docker.PullDockerImage(ctx, cli, sandboxImage, sandboxConfig.DefaultConfig.ImagePullPolicy); err != nil { return nil, err } fmt.Printf("%v booting Flyte-sandbox container\n", emoji.FactoryWorker) exposedPorts, portBindings, _ := docker.GetSandboxPorts() - ID, err := docker.StartContainer(ctx, cli, volumes, exposedPorts, portBindings, docker.FlyteSandboxClusterName, image) + ID, err := docker.StartContainer(ctx, cli, volumes, exposedPorts, portBindings, docker.FlyteSandboxClusterName, sandboxImage) if err != nil { fmt.Printf("%v Something went wrong: Failed to start Sandbox container %v, Please check your docker client and try again. \n", emoji.GrimacingFace, emoji.Whale) return nil, err @@ -187,34 +188,6 @@ func startSandbox(ctx context.Context, cli docker.Docker, reader io.Reader) (*bu return logReader, nil } -// Returns the alternate image if specified, else -// if no version is specified then the Latest release of cr.flyte.org/flyteorg/flyte-sandbox:dind is used -// else cr.flyte.org/flyteorg/flyte-sandbox:dind-{SHA}, where sha is derived from the version. -func getSandboxImage(version string, alternateImage string) (string, error) { - - if len(alternateImage) > 0 { - return alternateImage, nil - } - - var tag = dind - if len(version) > 0 { - isGreater, err := util.IsVersionGreaterThan(version, sandboxSupportedVersion) - if err != nil { - return "", err - } - if !isGreater { - return "", fmt.Errorf("version flag only supported with flyte %s+ release", sandboxSupportedVersion) - } - sha, err := githubutil.GetSHAFromVersion(version, flyteRepository) - if err != nil { - return "", err - } - tag = fmt.Sprintf("%s-%s", dind, sha) - } - - return docker.GetSandboxImage(tag), nil -} - func mountVolume(file, destination string) (*mount.Mount, error) { if len(file) > 0 { source, err := filepath.Abs(file) diff --git a/flytectl/cmd/sandbox/start_test.go b/flytectl/cmd/sandbox/start_test.go index ca64917914..71c3e45ffb 100644 --- a/flytectl/cmd/sandbox/start_test.go +++ b/flytectl/cmd/sandbox/start_test.go @@ -87,11 +87,13 @@ func TestStartSandboxFunc(t *testing.T) { ctx := context.Background() mockDocker := &mocks.Docker{} errCh := make(chan error) - sandboxConfig.DefaultConfig.Version = "" + sandboxConfig.DefaultConfig.Version = "v0.19.1" bodyStatus := make(chan container.ContainerWaitOKBody) + image, _, err := githubutil.GetSandboxImage("", imageName) + assert.Nil(t, err) mockDocker.OnContainerCreate(ctx, &container.Config{ Env: docker.Environment, - Image: docker.GetSandboxImage(dind), + Image: image, Tty: false, ExposedPorts: p1, }, &container.HostConfig{ @@ -111,17 +113,19 @@ func TestStartSandboxFunc(t *testing.T) { Follow: true, }).Return(nil, nil) mockDocker.OnContainerWaitMatch(ctx, mock.Anything, container.WaitConditionNotRunning).Return(bodyStatus, errCh) - _, err := startSandbox(ctx, mockDocker, os.Stdin) + _, err = startSandbox(ctx, mockDocker, os.Stdin) assert.Nil(t, err) }) t.Run("Successfully exit when sandbox cluster exist", func(t *testing.T) { ctx := context.Background() mockDocker := &mocks.Docker{} errCh := make(chan error) + image, _, err := githubutil.GetSandboxImage("", imageName) + assert.Nil(t, err) bodyStatus := make(chan container.ContainerWaitOKBody) mockDocker.OnContainerCreate(ctx, &container.Config{ Env: docker.Environment, - Image: docker.GetSandboxImage(dind), + Image: image, Tty: false, ExposedPorts: p1, }, &container.HostConfig{ @@ -165,9 +169,11 @@ func TestStartSandboxFunc(t *testing.T) { Source: sandboxConfig.DefaultConfig.Source, Target: docker.Source, }) + image, _, err := githubutil.GetSandboxImage("", imageName) + assert.Nil(t, err) mockDocker.OnContainerCreate(ctx, &container.Config{ Env: docker.Environment, - Image: docker.GetSandboxImage(dind), + Image: image, Tty: false, ExposedPorts: p1, }, &container.HostConfig{ @@ -187,7 +193,7 @@ func TestStartSandboxFunc(t *testing.T) { Follow: true, }).Return(nil, nil) mockDocker.OnContainerWaitMatch(ctx, mock.Anything, container.WaitConditionNotRunning).Return(bodyStatus, errCh) - _, err := startSandbox(ctx, mockDocker, os.Stdin) + _, err = startSandbox(ctx, mockDocker, os.Stdin) assert.Nil(t, err) }) t.Run("Successfully run sandbox cluster with abs path of source code", func(t *testing.T) { @@ -205,9 +211,11 @@ func TestStartSandboxFunc(t *testing.T) { Source: absPath, Target: docker.Source, }) + image, _, err := githubutil.GetSandboxImage("", imageName) + assert.Nil(t, err) mockDocker.OnContainerCreate(ctx, &container.Config{ Env: docker.Environment, - Image: docker.GetSandboxImage(dind), + Image: image, Tty: false, ExposedPorts: p1, }, &container.HostConfig{ @@ -235,16 +243,15 @@ func TestStartSandboxFunc(t *testing.T) { errCh := make(chan error) bodyStatus := make(chan container.ContainerWaitOKBody) mockDocker := &mocks.Docker{} - sandboxConfig.DefaultConfig.Version = "v0.15.0" + sandboxConfig.DefaultConfig.Version = "v0.18.0" sandboxConfig.DefaultConfig.Source = "" - sha, err := githubutil.GetSHAFromVersion(sandboxConfig.DefaultConfig.Version, "flyte") + image, _, err := githubutil.GetSandboxImage(sandboxConfig.DefaultConfig.Version, imageName) assert.Nil(t, err) - volumes := docker.Volumes mockDocker.OnContainerCreate(ctx, &container.Config{ Env: docker.Environment, - Image: docker.GetSandboxImage(fmt.Sprintf("%s-%s", dind, sha)), + Image: image, Tty: false, ExposedPorts: p1, }, &container.HostConfig{ @@ -274,10 +281,12 @@ func TestStartSandboxFunc(t *testing.T) { mockDocker := &mocks.Docker{} sandboxConfig.DefaultConfig.Version = "v0.1444.0" sandboxConfig.DefaultConfig.Source = "" + image, _, err := githubutil.GetSandboxImage("", imageName) + assert.Nil(t, err) volumes := docker.Volumes mockDocker.OnContainerCreate(ctx, &container.Config{ Env: docker.Environment, - Image: docker.GetSandboxImage(dind), + Image: image, Tty: false, ExposedPorts: p1, }, &container.HostConfig{ @@ -297,7 +306,7 @@ func TestStartSandboxFunc(t *testing.T) { Follow: true, }).Return(nil, nil) mockDocker.OnContainerWaitMatch(ctx, mock.Anything, container.WaitConditionNotRunning).Return(bodyStatus, errCh) - _, err := startSandbox(ctx, mockDocker, os.Stdin) + _, err = startSandbox(ctx, mockDocker, os.Stdin) assert.NotNil(t, err) }) t.Run("Error in pulling image", func(t *testing.T) { @@ -305,7 +314,8 @@ func TestStartSandboxFunc(t *testing.T) { errCh := make(chan error) bodyStatus := make(chan container.ContainerWaitOKBody) mockDocker := &mocks.Docker{} - + image, _, err := githubutil.GetSandboxImage("", imageName) + assert.Nil(t, err) sandboxConfig.DefaultConfig.Source = f.UserHomeDir() volumes := docker.Volumes volumes = append(volumes, mount.Mount{ @@ -315,7 +325,7 @@ func TestStartSandboxFunc(t *testing.T) { }) mockDocker.OnContainerCreate(ctx, &container.Config{ Env: docker.Environment, - Image: docker.GetSandboxImage(dind), + Image: image, Tty: false, ExposedPorts: p1, }, &container.HostConfig{ @@ -335,7 +345,7 @@ func TestStartSandboxFunc(t *testing.T) { Follow: true, }).Return(nil, nil) mockDocker.OnContainerWaitMatch(ctx, mock.Anything, container.WaitConditionNotRunning).Return(bodyStatus, errCh) - _, err := startSandbox(ctx, mockDocker, os.Stdin) + _, err = startSandbox(ctx, mockDocker, os.Stdin) assert.NotNil(t, err) }) t.Run("Error in removing existing cluster", func(t *testing.T) { @@ -350,9 +360,11 @@ func TestStartSandboxFunc(t *testing.T) { Source: sandboxConfig.DefaultConfig.Source, Target: docker.Source, }) + image, _, err := githubutil.GetSandboxImage("", imageName) + assert.Nil(t, err) mockDocker.OnContainerCreate(ctx, &container.Config{ Env: docker.Environment, - Image: docker.GetSandboxImage(dind), + Image: image, Tty: false, ExposedPorts: p1, }, &container.HostConfig{ @@ -380,7 +392,7 @@ func TestStartSandboxFunc(t *testing.T) { }).Return(nil, nil) mockDocker.OnContainerRemove(ctx, mock.Anything, types.ContainerRemoveOptions{Force: true}).Return(fmt.Errorf("error")) mockDocker.OnContainerWaitMatch(ctx, mock.Anything, container.WaitConditionNotRunning).Return(bodyStatus, errCh) - _, err := startSandbox(ctx, mockDocker, strings.NewReader("y")) + _, err = startSandbox(ctx, mockDocker, strings.NewReader("y")) assert.NotNil(t, err) }) t.Run("Error in start container", func(t *testing.T) { @@ -390,9 +402,11 @@ func TestStartSandboxFunc(t *testing.T) { mockDocker := &mocks.Docker{} sandboxConfig.DefaultConfig.Source = "" sandboxConfig.DefaultConfig.Version = "" + image, _, err := githubutil.GetSandboxImage("", imageName) + assert.Nil(t, err) mockDocker.OnContainerCreate(ctx, &container.Config{ Env: docker.Environment, - Image: docker.GetSandboxImage(dind), + Image: image, Tty: false, ExposedPorts: p1, }, &container.HostConfig{ @@ -412,7 +426,7 @@ func TestStartSandboxFunc(t *testing.T) { Follow: true, }).Return(nil, nil) mockDocker.OnContainerWaitMatch(ctx, mock.Anything, container.WaitConditionNotRunning).Return(bodyStatus, errCh) - _, err := startSandbox(ctx, mockDocker, os.Stdin) + _, err = startSandbox(ctx, mockDocker, os.Stdin) assert.NotNil(t, err) }) t.Run("Error in reading logs", func(t *testing.T) { @@ -427,9 +441,11 @@ func TestStartSandboxFunc(t *testing.T) { Source: sandboxConfig.DefaultConfig.Source, Target: docker.Source, }) + image, _, err := githubutil.GetSandboxImage("", imageName) + assert.Nil(t, err) mockDocker.OnContainerCreate(ctx, &container.Config{ Env: docker.Environment, - Image: docker.GetSandboxImage(dind), + Image: image, Tty: false, ExposedPorts: p1, }, &container.HostConfig{ @@ -449,7 +465,7 @@ func TestStartSandboxFunc(t *testing.T) { Follow: true, }).Return(nil, fmt.Errorf("error")) mockDocker.OnContainerWaitMatch(ctx, mock.Anything, container.WaitConditionNotRunning).Return(bodyStatus, errCh) - _, err := startSandbox(ctx, mockDocker, os.Stdin) + _, err = startSandbox(ctx, mockDocker, os.Stdin) assert.NotNil(t, err) }) t.Run("Error in list container", func(t *testing.T) { @@ -465,9 +481,11 @@ func TestStartSandboxFunc(t *testing.T) { Source: sandboxConfig.DefaultConfig.Source, Target: docker.Source, }) + image, _, err := githubutil.GetSandboxImage("", imageName) + assert.Nil(t, err) mockDocker.OnContainerCreate(ctx, &container.Config{ Env: docker.Environment, - Image: docker.GetSandboxImage(dind), + Image: image, Tty: false, ExposedPorts: p1, }, &container.HostConfig{ @@ -487,7 +505,7 @@ func TestStartSandboxFunc(t *testing.T) { Follow: true, }).Return(nil, nil) mockDocker.OnContainerWaitMatch(ctx, mock.Anything, container.WaitConditionNotRunning).Return(bodyStatus, errCh) - _, err := startSandbox(ctx, mockDocker, os.Stdin) + _, err = startSandbox(ctx, mockDocker, os.Stdin) assert.Nil(t, err) }) t.Run("Successfully run sandbox cluster command", func(t *testing.T) { @@ -507,10 +525,12 @@ func TestStartSandboxFunc(t *testing.T) { if err != nil { t.Error(err) } + image, _, err := githubutil.GetSandboxImage("", imageName) + assert.Nil(t, err) bodyStatus := make(chan container.ContainerWaitOKBody) mockDocker.OnContainerCreate(ctx, &container.Config{ Env: docker.Environment, - Image: docker.GetSandboxImage(dind), + Image: image, Tty: false, ExposedPorts: p1, }, &container.HostConfig{ @@ -545,9 +565,11 @@ func TestStartSandboxFunc(t *testing.T) { mockDocker := &mocks.Docker{} errCh := make(chan error) bodyStatus := make(chan container.ContainerWaitOKBody) + image, _, err := githubutil.GetSandboxImage("", imageName) + assert.Nil(t, err) mockDocker.OnContainerCreate(ctx, &container.Config{ Env: docker.Environment, - Image: docker.GetSandboxImage(dind), + Image: image, Tty: false, ExposedPorts: p1, }, &container.HostConfig{ @@ -571,7 +593,7 @@ func TestStartSandboxFunc(t *testing.T) { mockDocker.OnContainerWaitMatch(ctx, mock.Anything, container.WaitConditionNotRunning).Return(bodyStatus, errCh) docker.Client = mockDocker sandboxConfig.DefaultConfig.Source = "" - err := startSandboxCluster(ctx, []string{}, cmdCtx) + err = startSandboxCluster(ctx, []string{}, cmdCtx) assert.NotNil(t, err) }) } @@ -678,33 +700,27 @@ func TestGetNodeTaintStatus(t *testing.T) { func TestGetSandboxImage(t *testing.T) { t.Run("Get Latest sandbox", func(t *testing.T) { - image, err := getSandboxImage("", "") + image, _, err := githubutil.GetSandboxImage("", imageName) assert.Nil(t, err) - assert.Equal(t, docker.GetSandboxImage(dind), image) + assert.Equal(t, true, strings.HasPrefix(image, "cr.flyte.org/flyteorg/flyte-sandbox:dind-")) }) t.Run("Get sandbox image with version ", func(t *testing.T) { - image, err := getSandboxImage("v0.14.0", "") + image, _, err := githubutil.GetSandboxImage("v0.14.0", imageName) assert.Nil(t, err) - assert.Equal(t, true, strings.HasPrefix(image, docker.ImageName)) + assert.Equal(t, true, strings.HasPrefix(image, imageName)) }) t.Run("Get sandbox image with wrong version ", func(t *testing.T) { - _, err := getSandboxImage("v100.1.0", "") + _, _, err := githubutil.GetSandboxImage("v100.1.0", imageName) assert.NotNil(t, err) }) t.Run("Get sandbox image with wrong version ", func(t *testing.T) { - _, err := getSandboxImage("aaaaaa", "") + _, _, err := githubutil.GetSandboxImage("aaaaaa", imageName) assert.NotNil(t, err) }) t.Run("Get sandbox image with version that is not supported", func(t *testing.T) { - _, err := getSandboxImage("v0.10.0", "") + _, _, err := githubutil.GetSandboxImage("v0.10.0", imageName) assert.NotNil(t, err) }) - t.Run("Get sandbox image with version that is not supported", func(t *testing.T) { - img := "docker.io/my-override:latest" - i, err := getSandboxImage("v0.11.0", img) - assert.Nil(t, err) - assert.Equal(t, i, img) - }) } diff --git a/flytectl/cmd/upgrade/upgrade.go b/flytectl/cmd/upgrade/upgrade.go index 7fdd6d5f82..b0965f20d9 100644 --- a/flytectl/cmd/upgrade/upgrade.go +++ b/flytectl/cmd/upgrade/upgrade.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/flyteorg/flytectl/pkg/util" + stdlibversion "github.com/flyteorg/flytestdlib/version" "github.com/flyteorg/flytectl/pkg/util/githubutil" diff --git a/flytectl/go.mod b/flytectl/go.mod index cc8a7bb50f..3481e540f0 100644 --- a/flytectl/go.mod +++ b/flytectl/go.mod @@ -1,12 +1,10 @@ module github.com/flyteorg/flytectl -go 1.16 +go 1.17 require ( - github.com/Microsoft/go-winio v0.5.0 // indirect github.com/avast/retry-go v3.0.0+incompatible github.com/awalterschulze/gographviz v2.0.3+incompatible - github.com/containerd/containerd v1.5.9 // indirect github.com/disiqueira/gotree v1.0.0 github.com/docker/docker v20.10.7+incompatible github.com/docker/go-connections v0.4.0 @@ -16,20 +14,13 @@ require ( github.com/ghodss/yaml v1.0.0 github.com/go-ozzo/ozzo-validation/v4 v4.3.0 github.com/golang/protobuf v1.5.0 - github.com/google/go-github v17.0.0+incompatible - github.com/google/go-github/v37 v37.0.0 - github.com/google/go-querystring v1.1.0 // indirect + github.com/google/go-github/v42 v42.0.0 github.com/google/uuid v1.2.0 - github.com/gorilla/mux v1.8.0 // indirect github.com/hashicorp/go-version v1.3.0 github.com/kataras/tablewriter v0.0.0-20180708051242-e063d29b7c23 github.com/landoop/tableprinter v0.0.0-20180806200924-8bd8c2576d27 github.com/manifoldco/promptui v0.8.0 - github.com/mattn/go-isatty v0.0.13 // indirect - github.com/mattn/go-runewidth v0.0.13 // indirect github.com/mitchellh/mapstructure v1.4.1 - github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 // indirect - github.com/morikuni/aec v1.0.0 // indirect github.com/mouuff/go-rocket-update v1.5.1 github.com/olekukonko/tablewriter v0.0.5 github.com/opencontainers/image-spec v1.0.2 @@ -41,10 +32,7 @@ require ( github.com/stretchr/testify v1.7.0 github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 github.com/zalando/go-keyring v0.1.1 - golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 // indirect golang.org/x/oauth2 v0.0.0-20210126194326-f9ce19ea3013 - golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect - golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect google.golang.org/grpc v1.36.0 google.golang.org/protobuf v1.27.1 gopkg.in/yaml.v2 v2.4.0 @@ -55,3 +43,105 @@ require ( k8s.io/client-go v0.21.3 sigs.k8s.io/yaml v1.2.0 ) + +require ( + cloud.google.com/go v0.75.0 // indirect + cloud.google.com/go/storage v1.12.0 // indirect + github.com/Azure/azure-sdk-for-go v51.0.0+incompatible // indirect + github.com/Azure/go-autorest v14.2.0+incompatible // indirect + github.com/Azure/go-autorest/autorest v0.11.17 // indirect + github.com/Azure/go-autorest/autorest/adal v0.9.10 // indirect + github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect + github.com/Azure/go-autorest/logger v0.2.0 // indirect + github.com/Azure/go-autorest/tracing v0.6.0 // indirect + github.com/Microsoft/go-winio v0.5.0 // indirect + github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496 // indirect + github.com/aws/aws-sdk-go v1.37.1 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash v1.1.0 // indirect + github.com/cespare/xxhash/v2 v2.1.1 // indirect + github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect + github.com/containerd/containerd v1.5.9 // indirect + github.com/coocood/freecache v1.1.1 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect + github.com/danieljoos/wincred v1.1.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/docker/distribution v2.7.1+incompatible // indirect + github.com/docker/go-units v0.4.0 // indirect + github.com/dustin/go-humanize v1.0.0 // indirect + github.com/evanphx/json-patch v4.9.0+incompatible // indirect + github.com/fatih/color v1.10.0 // indirect + github.com/form3tech-oss/jwt-go v3.2.2+incompatible // indirect + github.com/fsnotify/fsnotify v1.4.9 // indirect + github.com/go-logr/logr v0.4.0 // indirect + github.com/godbus/dbus/v5 v5.0.4 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect + github.com/google/go-cmp v0.5.6 // indirect + github.com/google/go-querystring v1.1.0 // indirect + github.com/google/gofuzz v1.1.0 // indirect + github.com/googleapis/gax-go/v2 v2.0.5 // indirect + github.com/googleapis/gnostic v0.4.1 // indirect + github.com/gorilla/mux v1.8.0 // indirect + github.com/graymeta/stow v0.2.7 // indirect + github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 // indirect + github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect + github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect + github.com/hashicorp/golang-lru v0.5.4 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/imdario/mergo v0.3.12 // indirect + github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/json-iterator/go v1.1.10 // indirect + github.com/jstemmer/go-junit-report v0.9.1 // indirect + github.com/juju/ansiterm v0.0.0-20180109212912-720a0952cc2a // indirect + github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect + github.com/lunixbochs/vtclean v0.0.0-20180621232353-2d01aacdc34a // indirect + github.com/magiconair/properties v1.8.4 // indirect + github.com/mattn/go-colorable v0.1.8 // indirect + github.com/mattn/go-isatty v0.0.13 // indirect + github.com/mattn/go-runewidth v0.0.13 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect + github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.1 // indirect + github.com/morikuni/aec v1.0.0 // indirect + github.com/ncw/swift v1.0.53 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/pelletier/go-toml v1.8.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_golang v1.9.0 // indirect + github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/common v0.15.0 // indirect + github.com/prometheus/procfs v0.6.0 // indirect + github.com/rivo/uniseg v0.2.0 // indirect + github.com/russross/blackfriday/v2 v2.0.1 // indirect + github.com/satori/go.uuid v1.2.0 // indirect + github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect + github.com/spf13/afero v1.5.1 // indirect + github.com/spf13/cast v1.3.1 // indirect + github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/spf13/viper v1.7.1 // indirect + github.com/stretchr/objx v0.3.0 // indirect + github.com/subosito/gotenv v1.2.0 // indirect + go.opencensus.io v0.22.6 // indirect + golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect + golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 // indirect + golang.org/x/mod v0.4.1 // indirect + golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 // indirect + golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect + golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect + golang.org/x/text v0.3.5 // indirect + golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect + golang.org/x/tools v0.1.0 // indirect + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + google.golang.org/api v0.38.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/genproto v0.0.0-20210126160654-44e461bb6506 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/ini.v1 v1.62.0 // indirect + k8s.io/klog/v2 v2.8.0 // indirect + k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 // indirect + k8s.io/utils v0.0.0-20201110183641-67b214c5f920 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect +) diff --git a/flytectl/go.sum b/flytectl/go.sum index 9945f3089d..a1093f10c1 100644 --- a/flytectl/go.sum +++ b/flytectl/go.sum @@ -149,6 +149,7 @@ github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJm github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= +github.com/bradleyfalzon/ghinstallation/v2 v2.0.3/go.mod h1:tlgi+JWCXnKFx/Y4WtnDbZEINo31N5bcvnCoqieefmk= github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk= github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= @@ -417,6 +418,7 @@ github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -463,11 +465,9 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= -github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= -github.com/google/go-github/v37 v37.0.0 h1:rCspN8/6kB1BAJWZfuafvHhyfIo5fkAulaP/3bOQ/tM= -github.com/google/go-github/v37 v37.0.0/go.mod h1:LM7in3NmXDrX58GbEHy7FtNLbI2JijX93RnMKvWG3m4= -github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/go-github/v39 v39.0.0/go.mod h1:C1s8C5aCC9L+JXIYpJM5GYytdX52vC1bLvHEF1IhBrE= +github.com/google/go-github/v42 v42.0.0 h1:YNT0FwjPrEysRkLIiKuEfSvBPCGKphW5aS5PxwaoLec= +github.com/google/go-github/v42 v42.0.0/go.mod h1:jgg/jvyI0YlDOM1/ps6XYh04HNQ3vKf0CVko62/EhRg= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -985,8 +985,8 @@ golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 h1:/UOmuWzQfxxo9UtlXMwuQU8CMgg1eZXqTRwkSQJWKOI= -golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= diff --git a/flytectl/pkg/docker/docker_util.go b/flytectl/pkg/docker/docker_util.go index e0edaeb148..a1d952475f 100644 --- a/flytectl/pkg/docker/docker_util.go +++ b/flytectl/pkg/docker/docker_util.go @@ -9,6 +9,9 @@ import ( "os" "strings" + "github.com/docker/docker/client" + "github.com/enescakir/emoji" + sandboxConfig "github.com/flyteorg/flytectl/cmd/config/subcommand/sandbox" "github.com/flyteorg/flytectl/clierrors" @@ -16,10 +19,8 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/mount" - "github.com/docker/docker/client" "github.com/docker/docker/pkg/stdcopy" "github.com/docker/go-connections/nat" - "github.com/enescakir/emoji" cmdUtil "github.com/flyteorg/flytectl/pkg/commandutils" f "github.com/flyteorg/flytectl/pkg/filesystemutils" ) @@ -27,7 +28,6 @@ import ( var ( Kubeconfig = f.FilePathJoin(f.UserHomeDir(), ".flyte", "k3s", "k3s.yaml") SuccessMessage = "Deploying Flyte..." - ImageName = "cr.flyte.org/flyteorg/flyte-sandbox" FlyteSandboxClusterName = "flyte-sandbox" Environment = []string{"SANDBOX=1", "KUBERNETES_API_PORT=30086", "FLYTE_HOST=localhost:30081", "FLYTE_AWS_ENDPOINT=http://localhost:30084"} Source = "/root" @@ -51,6 +51,19 @@ var ( StartingBufLen = 32*1024 + StdWriterPrefixLen + 1 ) +// GetDockerClient will returns the docker client +func GetDockerClient() (Docker, error) { + if Client == nil { + cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) + if err != nil { + fmt.Printf("%v Please Check your docker client %v \n", emoji.GrimacingFace, emoji.Whale) + return nil, err + } + return cli, nil + } + return Client, nil +} + // GetSandbox will return sandbox container if it exist func GetSandbox(ctx context.Context, cli Docker) *types.Container { containers, _ := cli.ContainerList(ctx, types.ContainerListOptions{ @@ -166,19 +179,6 @@ func WaitForSandbox(reader *bufio.Scanner, message string) bool { return false } -// GetDockerClient will returns the docker client -func GetDockerClient() (Docker, error) { - if Client == nil { - cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) - if err != nil { - fmt.Printf("%v Please Check your docker client %v \n", emoji.GrimacingFace, emoji.Whale) - return nil, err - } - return cli, nil - } - return Client, nil -} - // ExecCommend will execute a command in container and returns an execution id func ExecCommend(ctx context.Context, cli Docker, containerID string, command []string) (types.IDResponse, error) { ExecConfig.Cmd = command @@ -200,8 +200,3 @@ func InspectExecResp(ctx context.Context, cli Docker, containerID string) error } return nil } - -// GetSandboxImage will return the sandbox image with tag -func GetSandboxImage(tag string) string { - return fmt.Sprintf("%s:%s", ImageName, tag) -} diff --git a/flytectl/pkg/docker/docker_util_test.go b/flytectl/pkg/docker/docker_util_test.go index 0a40d55ea2..cc27fd8af4 100644 --- a/flytectl/pkg/docker/docker_util_test.go +++ b/flytectl/pkg/docker/docker_util_test.go @@ -26,6 +26,7 @@ import ( var ( cmdCtx cmdCore.CommandContext containers []types.Container + imageName = "cr.flyte.org/flyteorg/flyte-sandbox" ) func setupSandbox() { @@ -154,7 +155,7 @@ func TestStartContainer(t *testing.T) { // Verify the attributes mockDocker.OnContainerCreate(context, &container.Config{ Env: Environment, - Image: ImageName, + Image: imageName, Tty: false, ExposedPorts: p1, }, &container.HostConfig{ @@ -165,7 +166,7 @@ func TestStartContainer(t *testing.T) { ID: "Hello", }, nil) mockDocker.OnContainerStart(context, "Hello", types.ContainerStartOptions{}).Return(nil) - id, err := StartContainer(context, mockDocker, Volumes, p1, p2, "nginx", ImageName) + id, err := StartContainer(context, mockDocker, Volumes, p1, p2, "nginx", imageName) assert.Nil(t, err) assert.Greater(t, len(id), 0) assert.Equal(t, id, "Hello") @@ -179,7 +180,7 @@ func TestStartContainer(t *testing.T) { // Verify the attributes mockDocker.OnContainerCreate(context, &container.Config{ Env: Environment, - Image: ImageName, + Image: imageName, Tty: false, ExposedPorts: p1, }, &container.HostConfig{ @@ -190,7 +191,7 @@ func TestStartContainer(t *testing.T) { ID: "", }, fmt.Errorf("error")) mockDocker.OnContainerStart(context, "Hello", types.ContainerStartOptions{}).Return(nil) - id, err := StartContainer(context, mockDocker, Volumes, p1, p2, "nginx", ImageName) + id, err := StartContainer(context, mockDocker, Volumes, p1, p2, "nginx", imageName) assert.NotNil(t, err) assert.Equal(t, len(id), 0) assert.Equal(t, id, "") @@ -204,7 +205,7 @@ func TestStartContainer(t *testing.T) { // Verify the attributes mockDocker.OnContainerCreate(context, &container.Config{ Env: Environment, - Image: ImageName, + Image: imageName, Tty: false, ExposedPorts: p1, }, &container.HostConfig{ @@ -215,7 +216,7 @@ func TestStartContainer(t *testing.T) { ID: "Hello", }, nil) mockDocker.OnContainerStart(context, "Hello", types.ContainerStartOptions{}).Return(fmt.Errorf("error")) - id, err := StartContainer(context, mockDocker, Volumes, p1, p2, "nginx", ImageName) + id, err := StartContainer(context, mockDocker, Volumes, p1, p2, "nginx", imageName) assert.NotNil(t, err) assert.Equal(t, len(id), 0) assert.Equal(t, id, "") diff --git a/flytectl/pkg/util/githubutil/githubutil.go b/flytectl/pkg/util/githubutil/githubutil.go index 109b6daec1..075b1cc8df 100644 --- a/flytectl/pkg/util/githubutil/githubutil.go +++ b/flytectl/pkg/util/githubutil/githubutil.go @@ -3,34 +3,42 @@ package githubutil import ( "context" "net/http" + "os" "path/filepath" "runtime" "strings" + sandboxConfig "github.com/flyteorg/flytectl/cmd/config/subcommand/sandbox" + + "github.com/flyteorg/flytectl/pkg/util" + + "github.com/flyteorg/flytestdlib/logger" + + "golang.org/x/oauth2" + "github.com/flyteorg/flytectl/pkg/util/platformutil" stdlibversion "github.com/flyteorg/flytestdlib/version" "github.com/mouuff/go-rocket-update/pkg/provider" "github.com/mouuff/go-rocket-update/pkg/updater" - "github.com/flyteorg/flytectl/pkg/util" - "fmt" - "github.com/google/go-github/v37/github" + "github.com/google/go-github/v42/github" ) const ( - owner = "flyteorg" - flyte = "flyte" - sandboxManifest = "flyte_sandbox_manifest.yaml" - flytectl = "flytectl" - flytectlRepository = "github.com/flyteorg/flytectl" - commonMessage = "\n A new release of flytectl is available: %s → %s \n" - brewMessage = "To upgrade, run: brew update && brew upgrade flytectl \n" - linuxMessage = "To upgrade, run: flytectl upgrade \n" - darwinMessage = "To upgrade, run: flytectl upgrade \n" - releaseURL = "https://github.com/flyteorg/flytectl/releases/tag/%s \n" - brewInstallDirectory = "/Cellar/flytectl" + owner = "flyteorg" + flyte = "flyte" + sandboxManifest = "flyte_sandbox_manifest.yaml" + flytectl = "flytectl" + sandboxSupportedVersion = "v0.10.0" + flytectlRepository = "github.com/flyteorg/flytectl" + commonMessage = "\n A new release of flytectl is available: %s → %s \n" + brewMessage = "To upgrade, run: brew update && brew upgrade flytectl \n" + linuxMessage = "To upgrade, run: flytectl upgrade \n" + darwinMessage = "To upgrade, run: flytectl upgrade \n" + releaseURL = "https://github.com/flyteorg/flytectl/releases/tag/%s \n" + brewInstallDirectory = "/Cellar/flytectl" ) // FlytectlReleaseConfig represent the updater config for flytectl binary @@ -49,6 +57,11 @@ var ( //GetGHClient will return github client func GetGHClient() *github.Client { + if len(os.Getenv("GITHUB_TOKEN")) > 0 { + return github.NewClient(oauth2.NewClient(context.Background(), oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: os.Getenv("GITHUB_TOKEN")}, + ))) + } return github.NewClient(&http.Client{}) } @@ -74,6 +87,45 @@ func GetListRelease(repository string) ([]*github.RepositoryRelease, error) { return releases, err } +// GetSandboxImageSha returns the sha as per input +func GetSandboxImageSha(version string, pre bool) (string, string, error) { + var release *github.RepositoryRelease + if len(version) == 0 { + releases, err := GetListRelease(flyte) + if err != nil { + return "", release.GetTagName(), err + } + for _, v := range releases { + if *v.Prerelease && pre { + release = v + break + } else if !*v.Prerelease && !pre { + release = v + break + } + } + logger.Infof(context.Background(), "sandbox started with release %s", release.GetTagName()) + } else if len(version) > 0 { + r, err := CheckVersionExist(version, flyte) + if err != nil { + return "", r.GetTagName(), err + } + release = r + } + isGreater, err := util.IsVersionGreaterThan(release.GetTagName(), sandboxSupportedVersion) + if err != nil { + return "", release.GetTagName(), err + } + if !isGreater { + return "", release.GetTagName(), fmt.Errorf("version flag only supported with flyte %s+ release", sandboxSupportedVersion) + } + sha, err := GetSHAFromVersion(release.GetTagName(), flyte) + if err != nil { + return "", release.GetTagName(), err + } + return sha, release.GetTagName(), nil +} + func getFlytectlAssetName() string { if arch == platformutil.ArchAmd64 { arch = platformutil.ArchX86 @@ -159,3 +211,15 @@ func CheckBrewInstall(goos platformutil.Platform) (string, error) { } return "", nil } + +// GetSandboxImage Returns the alternate image if specified, else +// if no version is specified then the Latest release of cr.flyte.org/flyteorg/flyte-sandbox:dind-{SHA} is used +// else cr.flyte.org/flyteorg/flyte-sandbox:dind-{SHA}, where sha is derived from the version. +func GetSandboxImage(version, image string) (string, string, error) { + sha, version, err := GetSandboxImageSha(version, sandboxConfig.DefaultConfig.Prerelease) + if err != nil { + return "", version, err + } + + return fmt.Sprintf("%s:%s", image, fmt.Sprintf("dind-%s", sha)), version, nil +} diff --git a/flytectl/pkg/util/util.go b/flytectl/pkg/util/util.go index 31c01db225..c5f5470dc0 100644 --- a/flytectl/pkg/util/util.go +++ b/flytectl/pkg/util/util.go @@ -9,11 +9,12 @@ import ( "os" "strings" + hversion "github.com/hashicorp/go-version" + "github.com/enescakir/emoji" "github.com/flyteorg/flytectl/pkg/configutil" "github.com/flyteorg/flytectl/pkg/docker" f "github.com/flyteorg/flytectl/pkg/filesystemutils" - hversion "github.com/hashicorp/go-version" ) const ( @@ -49,19 +50,6 @@ func SetupFlyteDir() error { return nil } -// IsVersionGreaterThan check version if it's greater then other -func IsVersionGreaterThan(version1, version2 string) (bool, error) { - semanticVersion1, err := hversion.NewVersion(version1) - if err != nil { - return false, err - } - semanticVersion2, err := hversion.NewVersion(version2) - if err != nil { - return false, err - } - return semanticVersion1.GreaterThan(semanticVersion2), nil -} - // PrintSandboxMessage will print sandbox success message func PrintSandboxMessage() { kubeconfig := strings.Join([]string{ @@ -89,3 +77,16 @@ func SendRequest(method, url string, option io.Reader) (*http.Response, error) { } return response, nil } + +// IsVersionGreaterThan check version if it's greater then other +func IsVersionGreaterThan(version1, version2 string) (bool, error) { + semanticVersion1, err := hversion.NewVersion(version1) + if err != nil { + return false, err + } + semanticVersion2, err := hversion.NewVersion(version2) + if err != nil { + return false, err + } + return semanticVersion1.GreaterThan(semanticVersion2), nil +} diff --git a/flytectl/pkg/util/util_test.go b/flytectl/pkg/util/util_test.go index b97b11804e..185b20db82 100644 --- a/flytectl/pkg/util/util_test.go +++ b/flytectl/pkg/util/util_test.go @@ -23,6 +23,30 @@ func TestSetupFlyteDir(t *testing.T) { assert.Nil(t, SetupFlyteDir()) } +func TestPrintSandboxMessage(t *testing.T) { + t.Run("Print Sandbox Message", func(t *testing.T) { + PrintSandboxMessage() + }) +} + +func TestSendRequest(t *testing.T) { + t.Run("Successful get request", func(t *testing.T) { + response, err := SendRequest("GET", "https://github.com", nil) + assert.Nil(t, err) + assert.NotNil(t, response) + }) + t.Run("Successful get request failed", func(t *testing.T) { + response, err := SendRequest("GET", "htp://github.com", nil) + assert.NotNil(t, err) + assert.Nil(t, response) + }) + t.Run("Successful get request failed", func(t *testing.T) { + response, err := SendRequest("GET", "https://github.com/evalsocket/flyte/archive/refs/tags/source-code.zip", nil) + assert.NotNil(t, err) + assert.Nil(t, response) + }) +} + func TestIsVersionGreaterThan(t *testing.T) { t.Run("Compare FlyteCTL version when upgrade available", func(t *testing.T) { _, err := IsVersionGreaterThan("v1.1.21", testVersion) @@ -56,27 +80,3 @@ func TestIsVersionGreaterThan(t *testing.T) { assert.NotNil(t, err) }) } - -func TestPrintSandboxMessage(t *testing.T) { - t.Run("Print Sandbox Message", func(t *testing.T) { - PrintSandboxMessage() - }) -} - -func TestSendRequest(t *testing.T) { - t.Run("Successful get request", func(t *testing.T) { - response, err := SendRequest("GET", "https://github.com", nil) - assert.Nil(t, err) - assert.NotNil(t, response) - }) - t.Run("Successful get request failed", func(t *testing.T) { - response, err := SendRequest("GET", "htp://github.com", nil) - assert.NotNil(t, err) - assert.Nil(t, response) - }) - t.Run("Successful get request failed", func(t *testing.T) { - response, err := SendRequest("GET", "https://github.com/evalsocket/flyte/archive/refs/tags/source-code.zip", nil) - assert.NotNil(t, err) - assert.Nil(t, response) - }) -}