From da2267fa3d9b34adb5e3e5d7c53db26054869902 Mon Sep 17 00:00:00 2001 From: Daniel Jiang Date: Thu, 18 Apr 2024 13:41:35 +0800 Subject: [PATCH 01/13] Bump up the version of KinD and k8s in github actions Signed-off-by: Daniel Jiang --- .github/workflows/crds-verify-kind.yaml | 16 ++++++++-------- .github/workflows/e2e-test-kind.yaml | 16 ++++++++-------- changelogs/unreleased/7702-reasonerjt | 1 + 3 files changed, 17 insertions(+), 16 deletions(-) create mode 100644 changelogs/unreleased/7702-reasonerjt diff --git a/.github/workflows/crds-verify-kind.yaml b/.github/workflows/crds-verify-kind.yaml index 9ed9936990..9255f0582a 100644 --- a/.github/workflows/crds-verify-kind.yaml +++ b/.github/workflows/crds-verify-kind.yaml @@ -57,13 +57,13 @@ jobs: matrix: # Latest k8s versions. There's no series-based tag, nor is there a latest tag. k8s: - - 1.19.7 - - 1.20.2 - - 1.21.1 - - 1.22.0 - - 1.23.6 - - 1.24.2 - - 1.25.3 + - 1.23.17 + - 1.24.17 + - 1.25.16 + - 1.26.13 + - 1.27.10 + - 1.28.6 + - 1.29.1 # All steps run in parallel unless otherwise specified. # See https://docs.github.com/en/actions/learn-github-actions/managing-complex-workflows#creating-dependent-jobs steps: @@ -81,7 +81,7 @@ jobs: velero-${{ github.event.pull_request.number }}- - uses: engineerd/setup-kind@v0.5.0 with: - version: "v0.17.0" + version: "v0.21.0" image: "kindest/node:v${{ matrix.k8s }}" - name: Install CRDs run: | diff --git a/.github/workflows/e2e-test-kind.yaml b/.github/workflows/e2e-test-kind.yaml index fa39dcaa3c..1dd7274ac2 100644 --- a/.github/workflows/e2e-test-kind.yaml +++ b/.github/workflows/e2e-test-kind.yaml @@ -60,13 +60,13 @@ jobs: strategy: matrix: k8s: - - 1.19.16 - - 1.20.15 - - 1.21.12 - - 1.22.9 - - 1.23.6 - - 1.24.0 - - 1.25.3 + - 1.23.17 + - 1.24.17 + - 1.25.16 + - 1.26.13 + - 1.27.10 + - 1.28.6 + - 1.29.1 focus: # tests to focus on, use `|` to concatenate multiple regexes to run on the same job # ordered according to e2e_suite_test.go order @@ -91,7 +91,7 @@ jobs: docker run -d --rm -p 9000:9000 -e "MINIO_ACCESS_KEY=minio" -e "MINIO_SECRET_KEY=minio123" -e "MINIO_DEFAULT_BUCKETS=bucket,additional-bucket" bitnami/minio:2021.6.17-debian-10-r7 - uses: engineerd/setup-kind@v0.5.0 with: - version: "v0.17.0" + version: "v0.21.0" image: "kindest/node:v${{ matrix.k8s }}" - name: Fetch built CLI id: cli-cache diff --git a/changelogs/unreleased/7702-reasonerjt b/changelogs/unreleased/7702-reasonerjt new file mode 100644 index 0000000000..a970ac8e2c --- /dev/null +++ b/changelogs/unreleased/7702-reasonerjt @@ -0,0 +1 @@ +Bump up the version of KinD and k8s in github actions \ No newline at end of file From 01f25db1b494941018c2d82bbe345c91371dc5d5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Apr 2024 19:38:03 +0000 Subject: [PATCH 02/13] Bump jpmcb/prow-github-actions from 1.1.2 to 1.1.3 Bumps [jpmcb/prow-github-actions](https://github.com/jpmcb/prow-github-actions) from 1.1.2 to 1.1.3. - [Release notes](https://github.com/jpmcb/prow-github-actions/releases) - [Commits](https://github.com/jpmcb/prow-github-actions/compare/v1.1.2...v1.1.3) --- updated-dependencies: - dependency-name: jpmcb/prow-github-actions dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/prow-action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/prow-action.yml b/.github/workflows/prow-action.yml index d6d68bb042..bfddf23b2b 100644 --- a/.github/workflows/prow-action.yml +++ b/.github/workflows/prow-action.yml @@ -9,7 +9,7 @@ jobs: execute: runs-on: ubuntu-latest steps: - - uses: jpmcb/prow-github-actions@v1.1.2 + - uses: jpmcb/prow-github-actions@v1.1.3 with: # Only support /kind command for now. # TODO: before allowing the /lgtm command, see if we can block merging if changelog labels are missing. From 2c9ff8b6d196bf3b740df3beed769668d31eb414 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Apr 2024 19:38:07 +0000 Subject: [PATCH 03/13] Bump actions/stale from 6.0.1 to 9.0.0 Bumps [actions/stale](https://github.com/actions/stale) from 6.0.1 to 9.0.0. - [Release notes](https://github.com/actions/stale/releases) - [Changelog](https://github.com/actions/stale/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/stale/compare/v6.0.1...v9.0.0) --- updated-dependencies: - dependency-name: actions/stale dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/stale-issues.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stale-issues.yml b/.github/workflows/stale-issues.yml index 8167e03833..7b29970fce 100644 --- a/.github/workflows/stale-issues.yml +++ b/.github/workflows/stale-issues.yml @@ -7,7 +7,7 @@ jobs: stale: runs-on: ubuntu-latest steps: - - uses: actions/stale@v6.0.1 + - uses: actions/stale@v9.0.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} stale-issue-message: "This issue is stale because it has been open 60 days with no activity. Remove stale label or comment or this will be closed in 14 days. If a Velero team member has requested log or more information, please provide the output of the shared commands." From 8c7f7590026a494345a93b0b9c91d3e516fedc8c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Apr 2024 19:38:14 +0000 Subject: [PATCH 04/13] Bump github/codeql-action from 2 to 3 Bumps [github/codeql-action](https://github.com/github/codeql-action) from 2 to 3. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/v2...v3) --- updated-dependencies: - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/nightly-trivy-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nightly-trivy-scan.yml b/.github/workflows/nightly-trivy-scan.yml index f4b2d9694b..d9e9076e5a 100644 --- a/.github/workflows/nightly-trivy-scan.yml +++ b/.github/workflows/nightly-trivy-scan.yml @@ -31,6 +31,6 @@ jobs: output: 'trivy-results.sarif' - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v2 + uses: github/codeql-action/upload-sarif@v3 with: sarif_file: 'trivy-results.sarif' \ No newline at end of file From eed655dddd0cdda3796d05e02e87a035fc827174 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Apr 2024 19:38:20 +0000 Subject: [PATCH 05/13] Bump actions/checkout from 2 to 4 Bumps [actions/checkout](https://github.com/actions/checkout) from 2 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v2...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/crds-verify-kind.yaml | 2 +- .github/workflows/e2e-test-kind.yaml | 4 ++-- .github/workflows/nightly-trivy-scan.yml | 2 +- .github/workflows/pr-changelog-check.yml | 2 +- .github/workflows/pr-ci-check.yml | 2 +- .github/workflows/pr-codespell.yml | 2 +- .github/workflows/pr-containers.yml | 2 +- .github/workflows/pr-goreleaser.yml | 2 +- .github/workflows/push-builder.yml | 2 +- .github/workflows/push.yml | 2 +- .github/workflows/rebase.yml | 2 +- 11 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/crds-verify-kind.yaml b/.github/workflows/crds-verify-kind.yaml index 9ed9936990..7b67ff88df 100644 --- a/.github/workflows/crds-verify-kind.yaml +++ b/.github/workflows/crds-verify-kind.yaml @@ -40,7 +40,7 @@ jobs: ${{ runner.os }}-go- - name: Check out the code - uses: actions/checkout@v2 + uses: actions/checkout@v4 if: steps.cache.outputs.cache-hit != 'true' # If no binaries were built for this PR, build it now. diff --git a/.github/workflows/e2e-test-kind.yaml b/.github/workflows/e2e-test-kind.yaml index fa39dcaa3c..efff3a8da9 100644 --- a/.github/workflows/e2e-test-kind.yaml +++ b/.github/workflows/e2e-test-kind.yaml @@ -40,7 +40,7 @@ jobs: restore-keys: | ${{ runner.os }}-go- - name: Check out the code - uses: actions/checkout@v2 + uses: actions/checkout@v4 if: steps.cli-cache.outputs.cache-hit != 'true' || steps.image-cache.outputs.cache-hit != 'true' # If no binaries were built for this PR, build it now. - name: Build Velero CLI @@ -85,7 +85,7 @@ jobs: go-version: '1.22' id: go - name: Check out the code - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Install MinIO run: docker run -d --rm -p 9000:9000 -e "MINIO_ACCESS_KEY=minio" -e "MINIO_SECRET_KEY=minio123" -e "MINIO_DEFAULT_BUCKETS=bucket,additional-bucket" bitnami/minio:2021.6.17-debian-10-r7 diff --git a/.github/workflows/nightly-trivy-scan.yml b/.github/workflows/nightly-trivy-scan.yml index f4b2d9694b..2503242c4d 100644 --- a/.github/workflows/nightly-trivy-scan.yml +++ b/.github/workflows/nightly-trivy-scan.yml @@ -19,7 +19,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Run Trivy vulnerability scanner uses: aquasecurity/trivy-action@master diff --git a/.github/workflows/pr-changelog-check.yml b/.github/workflows/pr-changelog-check.yml index 308e07d2d9..96052fc2d6 100644 --- a/.github/workflows/pr-changelog-check.yml +++ b/.github/workflows/pr-changelog-check.yml @@ -12,7 +12,7 @@ jobs: steps: - name: Check out the code - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Changelog check if: ${{ !(contains(github.event.pull_request.labels.*.name, 'kind/changelog-not-required') || contains(github.event.pull_request.labels.*.name, 'Design') || contains(github.event.pull_request.labels.*.name, 'Website') || contains(github.event.pull_request.labels.*.name, 'Documentation'))}} diff --git a/.github/workflows/pr-ci-check.yml b/.github/workflows/pr-ci-check.yml index 286ca370b5..102cb0ba7c 100644 --- a/.github/workflows/pr-ci-check.yml +++ b/.github/workflows/pr-ci-check.yml @@ -13,7 +13,7 @@ jobs: go-version: '1.22' id: go - name: Check out the code - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Fetch cached go modules uses: actions/cache@v2 with: diff --git a/.github/workflows/pr-codespell.yml b/.github/workflows/pr-codespell.yml index fd4b88c4f1..b1aa46f904 100644 --- a/.github/workflows/pr-codespell.yml +++ b/.github/workflows/pr-codespell.yml @@ -8,7 +8,7 @@ jobs: steps: - name: Check out the code - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Codespell uses: codespell-project/actions-codespell@master diff --git a/.github/workflows/pr-containers.yml b/.github/workflows/pr-containers.yml index 2e21ea8aaa..345f24362b 100644 --- a/.github/workflows/pr-containers.yml +++ b/.github/workflows/pr-containers.yml @@ -13,7 +13,7 @@ jobs: name: Build runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 name: Checkout - name: Set up QEMU diff --git a/.github/workflows/pr-goreleaser.yml b/.github/workflows/pr-goreleaser.yml index 8b5eec5bd9..aed88ab14a 100644 --- a/.github/workflows/pr-goreleaser.yml +++ b/.github/workflows/pr-goreleaser.yml @@ -14,7 +14,7 @@ jobs: name: Build runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 name: Checkout - name: Verify .goreleaser.yml and try a dryrun release. diff --git a/.github/workflows/push-builder.yml b/.github/workflows/push-builder.yml index 513937a44f..d4461bbdde 100644 --- a/.github/workflows/push-builder.yml +++ b/.github/workflows/push-builder.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: # The default value is "1" which fetches only a single commit. If we merge PR without squash or rebase, # there are at least two commits: the first one is the merge commit and the second one is the real commit diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index b6551ff044..51ea81baca 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -21,7 +21,7 @@ jobs: go-version: '1.22' id: go - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 # Fix issue of setup-gcloud - run: | diff --git a/.github/workflows/rebase.yml b/.github/workflows/rebase.yml index c63426a01a..69c8259a20 100644 --- a/.github/workflows/rebase.yml +++ b/.github/workflows/rebase.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout the latest code - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: fetch-depth: 0 - name: Automatic Rebase From 6550fc94bca7014b087cd0858d5c8ec08b448bda Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Apr 2024 19:38:24 +0000 Subject: [PATCH 06/13] Bump cirrus-actions/rebase from 1.3.1 to 1.8 Bumps [cirrus-actions/rebase](https://github.com/cirrus-actions/rebase) from 1.3.1 to 1.8. - [Release notes](https://github.com/cirrus-actions/rebase/releases) - [Commits](https://github.com/cirrus-actions/rebase/compare/1.3.1...1.8) --- updated-dependencies: - dependency-name: cirrus-actions/rebase dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/rebase.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rebase.yml b/.github/workflows/rebase.yml index c63426a01a..f08bbb59e2 100644 --- a/.github/workflows/rebase.yml +++ b/.github/workflows/rebase.yml @@ -13,6 +13,6 @@ jobs: with: fetch-depth: 0 - name: Automatic Rebase - uses: cirrus-actions/rebase@1.3.1 + uses: cirrus-actions/rebase@1.8 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 8a3f2f41e46134a455236d303ee084d87464d43d Mon Sep 17 00:00:00 2001 From: danfengl Date: Tue, 23 Apr 2024 05:45:33 +0000 Subject: [PATCH 07/13] Fix 1.14 nightly issues 1. Add sleep for native snapshot tests when using test.go interface; 2. Add --confirm for velero plugin add CLI as new feature introduced. Signed-off-by: danfengl --- test/e2e/test/test.go | 9 +++++++++ test/util/velero/velero_utils.go | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/test/e2e/test/test.go b/test/e2e/test/test.go index 15bde3958c..89a8fa71d1 100644 --- a/test/e2e/test/test.go +++ b/test/e2e/test/test.go @@ -145,6 +145,15 @@ func (t *TestCase) Backup() error { RunDebug(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, t.BackupName, "") return errors.Wrapf(err, "Failed to backup resources") } + + // the snapshots of AWS may be still in pending status when do the restore, wait for a while + // to avoid this https://github.com/vmware-tanzu/velero/issues/1799 + // TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed + if t.UseVolumeSnapshots { + fmt.Println("Waiting 5 minutes to make sure the snapshots are ready...") + time.Sleep(5 * time.Minute) + } + return nil } diff --git a/test/util/velero/velero_utils.go b/test/util/velero/velero_utils.go index 3976480e7b..21f895fddf 100644 --- a/test/util/velero/velero_utils.go +++ b/test/util/velero/velero_utils.go @@ -706,7 +706,7 @@ func VeleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNa stdoutBuf := new(bytes.Buffer) stderrBuf := new(bytes.Buffer) - installPluginCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "plugin", "add", plugin) + installPluginCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "plugin", "add", plugin, "--confirm") fmt.Printf("installPluginCmd cmd =%v\n", installPluginCmd) installPluginCmd.Stdout = stdoutBuf installPluginCmd.Stderr = stderrBuf From 8d2bef2486db8a4d2c253dc684a4ad8c49e4d7c8 Mon Sep 17 00:00:00 2001 From: Shubham Pampattiwar Date: Tue, 9 Apr 2024 13:46:10 -0700 Subject: [PATCH 08/13] Extend Volume Policies feature to support more actions Signed-off-by: Shubham Pampattiwar fix volume policy action execution Signed-off-by: Shubham Pampattiwar remove unused files Signed-off-by: Shubham Pampattiwar add changelog file Signed-off-by: Shubham Pampattiwar fix CI linter errors fix linter errors address pr review comments Signed-off-by: Shubham Pampattiwar fix via make update cmd Signed-off-by: Shubham Pampattiwar address PR feedback and add tests Signed-off-by: Shubham Pampattiwar fix codespell Signed-off-by: Shubham Pampattiwar fix ci linter checks Signed-off-by: Shubham Pampattiwar remove volsToExclude processing from volume policy logic and add tests Signed-off-by: Shubham Pampattiwar fix ci linter issue Signed-off-by: Shubham Pampattiwar --- .../unreleased/7664-shubham-pampattiwar | 1 + go.mod | 2 +- .../resourcepolicies/resource_policies.go | 23 +- .../resource_policies_test.go | 20 +- .../volume_resources_validator.go | 6 +- .../volume_resources_validator_test.go | 113 ++- internal/volumehelper/volume_policy_helper.go | 197 +++++ .../volumehelper/volume_policy_helper_test.go | 701 ++++++++++++++++++ pkg/backup/item_backupper.go | 70 +- pkg/cmd/server/server.go | 2 +- pkg/util/kube/pvc_pv.go | 18 + pkg/util/kube/pvc_pv_test.go | 122 +++ pkg/util/podvolume/pod_volume.go | 40 +- pkg/util/podvolume/pod_volume_test.go | 96 ++- 14 files changed, 1348 insertions(+), 63 deletions(-) create mode 100644 changelogs/unreleased/7664-shubham-pampattiwar create mode 100644 internal/volumehelper/volume_policy_helper.go create mode 100644 internal/volumehelper/volume_policy_helper_test.go diff --git a/changelogs/unreleased/7664-shubham-pampattiwar b/changelogs/unreleased/7664-shubham-pampattiwar new file mode 100644 index 0000000000..d77305c4dc --- /dev/null +++ b/changelogs/unreleased/7664-shubham-pampattiwar @@ -0,0 +1 @@ +Implementation for Extending VolumePolicies to support more actions \ No newline at end of file diff --git a/go.mod b/go.mod index cdc0d80df9..8212ebf568 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/vmware-tanzu/velero -go 1.22 +go 1.22.0 require ( cloud.google.com/go/storage v1.40.0 diff --git a/internal/resourcepolicies/resource_policies.go b/internal/resourcepolicies/resource_policies.go index 6666370864..ebfe01a322 100644 --- a/internal/resourcepolicies/resource_policies.go +++ b/internal/resourcepolicies/resource_policies.go @@ -27,8 +27,13 @@ type VolumeActionType string const ( // currently only support configmap type of resource config - ConfigmapRefType string = "configmap" - Skip VolumeActionType = "skip" + ConfigmapRefType string = "configmap" + // skip action implies the volume would be skipped from the backup operation + Skip VolumeActionType = "skip" + // fs-backup action implies that the volume would be backed up via file system copy method using the uploader(kopia/restic) configured by the user + FSBackup VolumeActionType = "fs-backup" + // snapshot action can have 3 different meaning based on velero configuration and backup spec - cloud provider based snapshots, local csi snapshots and datamover snapshots + Snapshot VolumeActionType = "snapshot" ) // Action defined as one action for a specific way of backup @@ -40,16 +45,16 @@ type Action struct { } // volumePolicy defined policy to conditions to match Volumes and related action to handle matched Volumes -type volumePolicy struct { +type VolumePolicy struct { // Conditions defined list of conditions to match Volumes Conditions map[string]interface{} `yaml:"conditions"` Action Action `yaml:"action"` } // resourcePolicies currently defined slice of volume policies to handle backup -type resourcePolicies struct { +type ResourcePolicies struct { Version string `yaml:"version"` - VolumePolicies []volumePolicy `yaml:"volumePolicies"` + VolumePolicies []VolumePolicy `yaml:"volumePolicies"` // we may support other resource policies in the future, and they could be added separately // OtherResourcePolicies []OtherResourcePolicy } @@ -60,8 +65,8 @@ type Policies struct { // OtherPolicies } -func unmarshalResourcePolicies(yamlData *string) (*resourcePolicies, error) { - resPolicies := &resourcePolicies{} +func unmarshalResourcePolicies(yamlData *string) (*ResourcePolicies, error) { + resPolicies := &ResourcePolicies{} err := decodeStruct(strings.NewReader(*yamlData), resPolicies) if err != nil { return nil, fmt.Errorf("failed to decode yaml data into resource policies %v", err) @@ -69,7 +74,7 @@ func unmarshalResourcePolicies(yamlData *string) (*resourcePolicies, error) { return resPolicies, nil } -func (p *Policies) buildPolicy(resPolicies *resourcePolicies) error { +func (p *Policies) BuildPolicy(resPolicies *ResourcePolicies) error { for _, vp := range resPolicies.VolumePolicies { con, err := unmarshalVolConditions(vp.Conditions) if err != nil { @@ -162,7 +167,7 @@ func GetResourcePoliciesFromConfig(cm *v1.ConfigMap) (*Policies, error) { } policies := &Policies{} - if err := policies.buildPolicy(resPolicies); err != nil { + if err := policies.BuildPolicy(resPolicies); err != nil { return nil, errors.WithStack(err) } diff --git a/internal/resourcepolicies/resource_policies_test.go b/internal/resourcepolicies/resource_policies_test.go index 1ff89a92ef..3476874378 100644 --- a/internal/resourcepolicies/resource_policies_test.go +++ b/internal/resourcepolicies/resource_policies_test.go @@ -121,9 +121,9 @@ func TestLoadResourcePolicies(t *testing.T) { } func TestGetResourceMatchedAction(t *testing.T) { - resPolicies := &resourcePolicies{ + resPolicies := &ResourcePolicies{ Version: "v1", - VolumePolicies: []volumePolicy{ + VolumePolicies: []VolumePolicy{ { Action: Action{Type: "skip"}, Conditions: map[string]interface{}{ @@ -136,7 +136,7 @@ func TestGetResourceMatchedAction(t *testing.T) { }, }, { - Action: Action{Type: "volume-snapshot"}, + Action: Action{Type: "snapshot"}, Conditions: map[string]interface{}{ "capacity": "10,100Gi", "storageClass": []string{"gp2", "ebs-sc"}, @@ -147,7 +147,7 @@ func TestGetResourceMatchedAction(t *testing.T) { }, }, { - Action: Action{Type: "file-system-backup"}, + Action: Action{Type: "fs-backup"}, Conditions: map[string]interface{}{ "storageClass": []string{"gp2", "ebs-sc"}, "csi": interface{}( @@ -179,7 +179,7 @@ func TestGetResourceMatchedAction(t *testing.T) { storageClass: "ebs-sc", csi: &csiVolumeSource{Driver: "aws.efs.csi.driver"}, }, - expectedAction: &Action{Type: "volume-snapshot"}, + expectedAction: &Action{Type: "snapshot"}, }, { name: "dismatch all policies", @@ -195,7 +195,7 @@ func TestGetResourceMatchedAction(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { policies := &Policies{} - err := policies.buildPolicy(resPolicies) + err := policies.BuildPolicy(resPolicies) if err != nil { t.Errorf("Failed to build policy with error %v", err) } @@ -237,9 +237,9 @@ func TestGetResourcePoliciesFromConfig(t *testing.T) { // Check that the returned resourcePolicies object contains the expected data assert.Equal(t, "v1", resPolicies.version) assert.Len(t, resPolicies.volumePolicies, 1) - policies := resourcePolicies{ + policies := ResourcePolicies{ Version: "v1", - VolumePolicies: []volumePolicy{ + VolumePolicies: []VolumePolicy{ { Conditions: map[string]interface{}{ "capacity": "0,10Gi", @@ -251,7 +251,7 @@ func TestGetResourcePoliciesFromConfig(t *testing.T) { }, } p := &Policies{} - err = p.buildPolicy(&policies) + err = p.BuildPolicy(&policies) if err != nil { t.Fatalf("failed to build policy with error %v", err) } @@ -424,7 +424,7 @@ volumePolicies: } assert.Nil(t, err) policies := &Policies{} - err = policies.buildPolicy(resPolicies) + err = policies.BuildPolicy(resPolicies) assert.Nil(t, err) action, err := policies.GetMatchAction(tc.vol) assert.Nil(t, err) diff --git a/internal/resourcepolicies/volume_resources_validator.go b/internal/resourcepolicies/volume_resources_validator.go index 79ccbb3dc7..cf9a40c0fc 100644 --- a/internal/resourcepolicies/volume_resources_validator.go +++ b/internal/resourcepolicies/volume_resources_validator.go @@ -82,7 +82,11 @@ func decodeStruct(r io.Reader, s interface{}) error { // validate check action format func (a *Action) validate() error { // validate Type - if a.Type != Skip { + valid := false + if a.Type == Skip || a.Type == Snapshot || a.Type == FSBackup { + valid = true + } + if !valid { return fmt.Errorf("invalid action type %s", a.Type) } diff --git a/internal/resourcepolicies/volume_resources_validator_test.go b/internal/resourcepolicies/volume_resources_validator_test.go index 8518065807..1cbc6d7325 100644 --- a/internal/resourcepolicies/volume_resources_validator_test.go +++ b/internal/resourcepolicies/volume_resources_validator_test.go @@ -84,14 +84,14 @@ func TestCapacityConditionValidate(t *testing.T) { func TestValidate(t *testing.T) { testCases := []struct { name string - res *resourcePolicies + res *ResourcePolicies wantErr bool }{ { name: "unknown key in yaml", - res: &resourcePolicies{ + res: &ResourcePolicies{ Version: "v1", - VolumePolicies: []volumePolicy{ + VolumePolicies: []VolumePolicy{ { Action: Action{Type: "skip"}, Conditions: map[string]interface{}{ @@ -110,9 +110,9 @@ func TestValidate(t *testing.T) { }, { name: "error format of capacity", - res: &resourcePolicies{ + res: &ResourcePolicies{ Version: "v1", - VolumePolicies: []volumePolicy{ + VolumePolicies: []VolumePolicy{ { Action: Action{Type: "skip"}, Conditions: map[string]interface{}{ @@ -130,9 +130,9 @@ func TestValidate(t *testing.T) { }, { name: "error format of storageClass", - res: &resourcePolicies{ + res: &ResourcePolicies{ Version: "v1", - VolumePolicies: []volumePolicy{ + VolumePolicies: []VolumePolicy{ { Action: Action{Type: "skip"}, Conditions: map[string]interface{}{ @@ -150,9 +150,9 @@ func TestValidate(t *testing.T) { }, { name: "error format of csi", - res: &resourcePolicies{ + res: &ResourcePolicies{ Version: "v1", - VolumePolicies: []volumePolicy{ + VolumePolicies: []VolumePolicy{ { Action: Action{Type: "skip"}, Conditions: map[string]interface{}{ @@ -167,9 +167,9 @@ func TestValidate(t *testing.T) { }, { name: "unsupported version", - res: &resourcePolicies{ + res: &ResourcePolicies{ Version: "v2", - VolumePolicies: []volumePolicy{ + VolumePolicies: []VolumePolicy{ { Action: Action{Type: "skip"}, Conditions: map[string]interface{}{ @@ -186,9 +186,9 @@ func TestValidate(t *testing.T) { }, { name: "unsupported action", - res: &resourcePolicies{ + res: &ResourcePolicies{ Version: "v1", - VolumePolicies: []volumePolicy{ + VolumePolicies: []VolumePolicy{ { Action: Action{Type: "unsupported"}, Conditions: map[string]interface{}{ @@ -205,9 +205,9 @@ func TestValidate(t *testing.T) { }, { name: "error format of nfs", - res: &resourcePolicies{ + res: &ResourcePolicies{ Version: "v1", - VolumePolicies: []volumePolicy{ + VolumePolicies: []VolumePolicy{ { Action: Action{Type: "skip"}, Conditions: map[string]interface{}{ @@ -221,10 +221,10 @@ func TestValidate(t *testing.T) { wantErr: true, }, { - name: "supported formart volume policies", - res: &resourcePolicies{ + name: "supported format volume policies", + res: &ResourcePolicies{ Version: "v1", - VolumePolicies: []volumePolicy{ + VolumePolicies: []VolumePolicy{ { Action: Action{Type: "skip"}, Conditions: map[string]interface{}{ @@ -245,11 +245,86 @@ func TestValidate(t *testing.T) { }, wantErr: false, }, + { + name: "supported format volume policies, action type snapshot", + res: &ResourcePolicies{ + Version: "v1", + VolumePolicies: []VolumePolicy{ + { + Action: Action{Type: "snapshot"}, + Conditions: map[string]interface{}{ + "capacity": "0,10Gi", + "storageClass": []string{"gp2", "ebs-sc"}, + "csi": interface{}( + map[string]interface{}{ + "driver": "aws.efs.csi.driver", + }), + "nfs": interface{}( + map[string]interface{}{ + "server": "192.168.20.90", + "path": "/mnt/data/", + }), + }, + }, + }, + }, + wantErr: false, + }, + { + name: "supported format volume policies, action type fs-backup", + res: &ResourcePolicies{ + Version: "v1", + VolumePolicies: []VolumePolicy{ + { + Action: Action{Type: "fs-backup"}, + Conditions: map[string]interface{}{ + "capacity": "0,10Gi", + "storageClass": []string{"gp2", "ebs-sc"}, + "csi": interface{}( + map[string]interface{}{ + "driver": "aws.efs.csi.driver", + }), + "nfs": interface{}( + map[string]interface{}{ + "server": "192.168.20.90", + "path": "/mnt/data/", + }), + }, + }, + }, + }, + wantErr: false, + }, + { + name: "supported format volume policies, action type fs-backup and snapshot", + res: &ResourcePolicies{ + Version: "v1", + VolumePolicies: []VolumePolicy{ + { + Action: Action{Type: Snapshot}, + Conditions: map[string]interface{}{ + "storageClass": []string{"gp2"}, + }, + }, + { + Action: Action{Type: FSBackup}, + Conditions: map[string]interface{}{ + "nfs": interface{}( + map[string]interface{}{ + "server": "192.168.20.90", + "path": "/mnt/data/", + }), + }, + }, + }, + }, + wantErr: false, + }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { policies := &Policies{} - err1 := policies.buildPolicy(tc.res) + err1 := policies.BuildPolicy(tc.res) err2 := policies.Validate() if tc.wantErr { diff --git a/internal/volumehelper/volume_policy_helper.go b/internal/volumehelper/volume_policy_helper.go new file mode 100644 index 0000000000..a1929846f2 --- /dev/null +++ b/internal/volumehelper/volume_policy_helper.go @@ -0,0 +1,197 @@ +package volumehelper + +import ( + "fmt" + "strings" + + "github.com/vmware-tanzu/velero/internal/resourcepolicies" + + kbclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/vmware-tanzu/velero/pkg/util/boolptr" + kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube" + + "github.com/sirupsen/logrus" + corev1api "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + + "github.com/vmware-tanzu/velero/pkg/kuberesource" + pdvolumeutil "github.com/vmware-tanzu/velero/pkg/util/podvolume" +) + +type VolumeHelper interface { + GetVolumesForFSBackup(pod *corev1api.Pod, defaultVolumesToFsBackup, backupExcludePVC bool, kbclient kbclient.Client) ([]string, []string, error) + ShouldPerformSnapshot(obj runtime.Unstructured, groupResource schema.GroupResource, kbclient kbclient.Client) (bool, error) +} + +type VolumeHelperImpl struct { + VolumePolicy *resourcepolicies.Policies + SnapshotVolumes *bool + Logger logrus.FieldLogger +} + +func (v *VolumeHelperImpl) ShouldPerformSnapshot(obj runtime.Unstructured, groupResource schema.GroupResource, kbclient kbclient.Client) (bool, error) { + // check if volume policy exists and also check if the object(pv/pvc) fits a volume policy criteria and see if the associated action is snapshot + // if it is not snapshot then skip the code path for snapshotting the PV/PVC + pvc := new(corev1api.PersistentVolumeClaim) + pv := new(corev1api.PersistentVolume) + var err error + + if groupResource == kuberesource.PersistentVolumeClaims { + if err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), &pvc); err != nil { + return false, err + } + + pv, err = kubeutil.GetPVForPVC(pvc, kbclient) + if err != nil { + return false, err + } + } + + if groupResource == kuberesource.PersistentVolumes { + if err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), &pv); err != nil { + return false, err + } + } + + if v.VolumePolicy != nil { + action, err := v.VolumePolicy.GetMatchAction(pv) + if err != nil { + return false, err + } + + // Also account for SnapshotVolumes flag on backup CR + if action != nil && action.Type == resourcepolicies.Snapshot && boolptr.IsSetToTrue(v.SnapshotVolumes) { + v.Logger.Infof(fmt.Sprintf("performing snapshot action for pv %s as it satisfies the volume policy criteria and snapshotVolumes is set to true", pv.Name)) + return true, nil + } + v.Logger.Infof(fmt.Sprintf("skipping snapshot action for pv %s possibly due to not satisfying the volume policy criteria or snapshotVolumes is not true", pv.Name)) + return false, nil + } + + return false, nil +} + +func (v *VolumeHelperImpl) GetVolumesForFSBackup(pod *corev1api.Pod, defaultVolumesToFsBackup, backupExcludePVC bool, kbclient kbclient.Client) ([]string, []string, error) { + // Check if there is a fs-backup/snapshot volume policy specified by the user, if yes then use the volume policy approach to + // get the list volumes for fs-backup else go via the legacy annotation based approach + var fsBackupVolumePolicyVols, nonFsBackupVolumePolicyVols, volsToProcessByLegacyApproach = make([]string, 0), make([]string, 0), make([]string, 0) + var err error + + if v.VolumePolicy != nil { + // Get the list of volumes to back up using pod volume backup for the given pod matching fs-backup volume policy action + v.Logger.Infof("Volume Policy specified by the user, using volume policy approach to segregate pod volumes for fs-backup") + // GetVolumesMatchingFSBackupAction return 3 list of Volumes: + // fsBackupVolumePolicyVols: Volumes that have a matching fs-backup action from the volume policy specified by the user + // nonFsBackupVolumePolicyVols: Volumes that have an action matching from the volume policy specified by the user, but it is not fs-backup action + // volsToProcessByLegacyApproach: Volumes that did not have any matching action i.e. action was nil from the volume policy specified by the user, these volumes will be processed via the legacy annotations based approach (fallback option) + fsBackupVolumePolicyVols, nonFsBackupVolumePolicyVols, volsToProcessByLegacyApproach, err = v.GetVolumesMatchingFSBackupAction(pod, v.VolumePolicy, backupExcludePVC, kbclient) + if err != nil { + return fsBackupVolumePolicyVols, nonFsBackupVolumePolicyVols, err + } + // if volsToProcessByLegacyApproach is empty then no need to sue legacy approach as fallback option return from here + if len(volsToProcessByLegacyApproach) == 0 { + return fsBackupVolumePolicyVols, nonFsBackupVolumePolicyVols, nil + } + } + + // process legacy annotation based approach, this will done when: + // 1. volume policy os specified by the user + // 2. And there are some volumes for which the volume policy approach did not get any supported matching actions + if v.VolumePolicy != nil && len(volsToProcessByLegacyApproach) > 0 { + v.Logger.Infof("volume policy specified by the user but there are volumes with no matching action, using legacy approach based on annotations as a fallback for those volumes") + includedVolumesFromLegacyFallBack, optedOutVolumesFromLegacyFallBack := pdvolumeutil.GetVolumesByPod(pod, defaultVolumesToFsBackup, backupExcludePVC, volsToProcessByLegacyApproach) + // merge the volumePolicy approach and legacy Fallback lists + fsBackupVolumePolicyVols = append(fsBackupVolumePolicyVols, includedVolumesFromLegacyFallBack...) + nonFsBackupVolumePolicyVols = append(nonFsBackupVolumePolicyVols, optedOutVolumesFromLegacyFallBack...) + return fsBackupVolumePolicyVols, nonFsBackupVolumePolicyVols, nil + } + // Normal legacy workflow + // Get the list of volumes to back up using pod volume backup from the pod's annotations. + // We will also pass the list of volume that did not have any supported volume policy action matched in legacy approach so that + // those volumes get processed via legacy annotation based approach, this is a fallback option on annotation based legacy approach + v.Logger.Infof("fs-backup or snapshot Volume Policy not specified by the user, using legacy approach based on annotations") + includedVolumes, optedOutVolumes := pdvolumeutil.GetVolumesByPod(pod, defaultVolumesToFsBackup, backupExcludePVC, volsToProcessByLegacyApproach) + return includedVolumes, optedOutVolumes, nil +} + +// GetVolumesMatchingFSBackupAction returns a list of volume names to backup for the provided pod having fs-backup volume policy action +func (v *VolumeHelperImpl) GetVolumesMatchingFSBackupAction(pod *corev1api.Pod, volumePolicies *resourcepolicies.Policies, backupExcludePVC bool, kbclient kbclient.Client) ([]string, []string, []string, error) { + FSBackupActionMatchingVols := []string{} + FSBackupNonActionMatchingVols := []string{} + NoActionMatchingVols := []string{} + + for i, vol := range pod.Spec.Volumes { + if !v.ShouldIncludeVolumeInBackup(vol, backupExcludePVC) { + continue + } + + if vol.PersistentVolumeClaim != nil { + // fetch the associated PVC first + pvc, err := kubeutil.GetPVCForPodVolume(&pod.Spec.Volumes[i], pod, kbclient) + if err != nil { + return FSBackupActionMatchingVols, FSBackupNonActionMatchingVols, NoActionMatchingVols, err + } + // now fetch the PV and call GetMatchAction on it + pv, err := kubeutil.GetPVForPVC(pvc, kbclient) + if err != nil { + return FSBackupActionMatchingVols, FSBackupNonActionMatchingVols, NoActionMatchingVols, err + } + // now get the action for pv + action, err := volumePolicies.GetMatchAction(pv) + if err != nil { + return FSBackupActionMatchingVols, FSBackupNonActionMatchingVols, NoActionMatchingVols, err + } + + // Record volume list having no matched action so that they are processed in legacy fallback option + if action == nil { + NoActionMatchingVols = append(NoActionMatchingVols, vol.Name) + } + + // Now if the matched action is not nil and is `fs-backup` then add that Volume to the FSBackupActionMatchingVols + // else add that volume to FSBackupNonActionMatchingVols + // we already tracked the volume not matching any kind actions supported by volume policy in NoActionMatchingVols + // The NoActionMatchingVols list will be processed via legacy annotation based approach as a fallback option + if action != nil && action.Type == resourcepolicies.FSBackup { + FSBackupActionMatchingVols = append(FSBackupActionMatchingVols, vol.Name) + } else if action != nil { + FSBackupNonActionMatchingVols = append(FSBackupNonActionMatchingVols, vol.Name) + } + } + } + return FSBackupActionMatchingVols, FSBackupNonActionMatchingVols, NoActionMatchingVols, nil +} + +func (v *VolumeHelperImpl) ShouldIncludeVolumeInBackup(vol corev1api.Volume, backupExcludePVC bool) bool { + includeVolumeInBackup := true + // cannot backup hostpath volumes as they are not mounted into /var/lib/kubelet/pods + // and therefore not accessible to the node agent daemon set. + if vol.HostPath != nil { + includeVolumeInBackup = false + } + // don't backup volumes mounting secrets. Secrets will be backed up separately. + if vol.Secret != nil { + includeVolumeInBackup = false + } + // don't backup volumes mounting ConfigMaps. ConfigMaps will be backed up separately. + if vol.ConfigMap != nil { + includeVolumeInBackup = false + } + // don't backup volumes mounted as projected volumes, all data in those come from kube state. + if vol.Projected != nil { + includeVolumeInBackup = false + } + // don't backup DownwardAPI volumes, all data in those come from kube state. + if vol.DownwardAPI != nil { + includeVolumeInBackup = false + } + if vol.PersistentVolumeClaim != nil && backupExcludePVC { + includeVolumeInBackup = false + } + // don't include volumes that mount the default service account token. + if strings.HasPrefix(vol.Name, "default-token") { + includeVolumeInBackup = false + } + return includeVolumeInBackup +} diff --git a/internal/volumehelper/volume_policy_helper_test.go b/internal/volumehelper/volume_policy_helper_test.go new file mode 100644 index 0000000000..b66cb2dfaf --- /dev/null +++ b/internal/volumehelper/volume_policy_helper_test.go @@ -0,0 +1,701 @@ +package volumehelper + +import ( + "testing" + + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/utils/pointer" + + "github.com/vmware-tanzu/velero/internal/resourcepolicies" + "github.com/vmware-tanzu/velero/pkg/kuberesource" + velerotest "github.com/vmware-tanzu/velero/pkg/test" +) + +func TestVolumeHelperImpl_ShouldPerformSnapshot(t *testing.T) { + PVObjectGP2 := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "PersistentVolume", + "metadata": map[string]interface{}{ + "name": "example-pv", + }, + "spec": map[string]interface{}{ + "capacity": map[string]interface{}{ + "storage": "1Gi", + }, + "volumeMode": "Filesystem", + "accessModes": []interface{}{"ReadWriteOnce"}, + "persistentVolumeReclaimPolicy": "Retain", + "storageClassName": "gp2-csi", + "hostPath": map[string]interface{}{ + "path": "/mnt/data", + }, + }, + }, + } + + PVObjectGP3 := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "PersistentVolume", + "metadata": map[string]interface{}{ + "name": "example-pv", + }, + "spec": map[string]interface{}{ + "capacity": map[string]interface{}{ + "storage": "1Gi", + }, + "volumeMode": "Filesystem", + "accessModes": []interface{}{"ReadWriteOnce"}, + "persistentVolumeReclaimPolicy": "Retain", + "storageClassName": "gp3-csi", + "hostPath": map[string]interface{}{ + "path": "/mnt/data", + }, + }, + }, + } + + testCases := []struct { + name string + obj runtime.Unstructured + groupResource schema.GroupResource + resourcePolicies resourcepolicies.ResourcePolicies + snapshotVolumesFlag *bool + shouldSnapshot bool + expectedErr bool + }{ + { + name: "Given PV object matches volume policy snapshot action snapshotVolumes flags is true returns true and no error", + obj: PVObjectGP2, + groupResource: kuberesource.PersistentVolumes, + resourcePolicies: resourcepolicies.ResourcePolicies{ + Version: "v1", + VolumePolicies: []resourcepolicies.VolumePolicy{ + { + Conditions: map[string]interface{}{ + "storageClass": []string{"gp2-csi"}, + }, + Action: resourcepolicies.Action{ + Type: resourcepolicies.Snapshot, + }, + }, + }, + }, + snapshotVolumesFlag: pointer.Bool(true), + shouldSnapshot: true, + expectedErr: false, + }, + { + name: "Given PV object matches volume policy snapshot action snapshotVolumes flags is false returns false and no error", + obj: PVObjectGP2, + groupResource: kuberesource.PersistentVolumes, + resourcePolicies: resourcepolicies.ResourcePolicies{ + Version: "v1", + VolumePolicies: []resourcepolicies.VolumePolicy{ + { + Conditions: map[string]interface{}{ + "storageClass": []string{"gp2-csi"}, + }, + Action: resourcepolicies.Action{ + Type: resourcepolicies.Snapshot, + }, + }, + }, + }, + snapshotVolumesFlag: pointer.Bool(false), + shouldSnapshot: false, + expectedErr: false, + }, + { + name: "Given PV object matches volume policy snapshot action snapshotVolumes flags is true returns false and no error", + obj: PVObjectGP3, + groupResource: kuberesource.PersistentVolumes, + resourcePolicies: resourcepolicies.ResourcePolicies{ + Version: "v1", + VolumePolicies: []resourcepolicies.VolumePolicy{ + { + Conditions: map[string]interface{}{ + "storageClass": []string{"gp2-csi"}, + }, + Action: resourcepolicies.Action{ + Type: resourcepolicies.Snapshot, + }, + }, + }, + }, + snapshotVolumesFlag: pointer.Bool(true), + shouldSnapshot: false, + expectedErr: false, + }, + { + name: "Given PVC object matches volume policy snapshot action snapshotVolumes flags is true return false and error case PVC not found", + obj: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "metadata": map[string]interface{}{ + "name": "example-pvc", + "namespace": "default", + }, + "spec": map[string]interface{}{ + "accessModes": []string{"ReadWriteOnce"}, + "resources": map[string]interface{}{ + "requests": map[string]interface{}{ + "storage": "1Gi", + }, + }, + "storageClassName": "gp2-csi", + }, + }, + }, + groupResource: kuberesource.PersistentVolumeClaims, + resourcePolicies: resourcepolicies.ResourcePolicies{ + Version: "v1", + VolumePolicies: []resourcepolicies.VolumePolicy{ + { + Conditions: map[string]interface{}{ + "storageClass": []string{"gp2-csi"}, + }, + Action: resourcepolicies.Action{ + Type: resourcepolicies.Snapshot, + }, + }, + }, + }, + snapshotVolumesFlag: pointer.Bool(true), + shouldSnapshot: false, + expectedErr: true, + }, + } + + mockedPV := &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "example-pv", + }, + Spec: corev1.PersistentVolumeSpec{ + Capacity: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimRetain, + StorageClassName: "gp2-csi", + ClaimRef: &corev1.ObjectReference{ + Name: "example-pvc", + Namespace: "default", + }, + }, + } + + objs := []runtime.Object{mockedPV} + fakeClient := velerotest.NewFakeControllerRuntimeClient(t, objs...) + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + policies := tc.resourcePolicies + p := &resourcepolicies.Policies{} + err := p.BuildPolicy(&policies) + if err != nil { + t.Fatalf("failed to build policy with error %v", err) + } + vh := &VolumeHelperImpl{ + VolumePolicy: p, + SnapshotVolumes: tc.snapshotVolumesFlag, + Logger: velerotest.NewLogger(), + } + ActualShouldSnapshot, actualError := vh.ShouldPerformSnapshot(tc.obj, tc.groupResource, fakeClient) + if tc.expectedErr { + assert.NotNil(t, actualError, "Want error; Got nil error") + return + } + + assert.Equalf(t, ActualShouldSnapshot, tc.shouldSnapshot, "Want shouldSnapshot as %v; Got shouldSnapshot as %v", tc.shouldSnapshot, ActualShouldSnapshot) + }) + } +} + +func TestVolumeHelperImpl_ShouldIncludeVolumeInBackup(t *testing.T) { + testCases := []struct { + name string + vol corev1.Volume + backupExcludePVC bool + shouldInclude bool + }{ + { + name: "volume has host path so do not include", + vol: corev1.Volume{ + Name: "sample-volume", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "some-path", + }, + }, + }, + backupExcludePVC: false, + shouldInclude: false, + }, + { + name: "volume has secret mounted so do not include", + vol: corev1.Volume{ + Name: "sample-volume", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "sample-secret", + Items: []corev1.KeyToPath{ + { + Key: "username", + Path: "my-username", + }, + }, + }, + }, + }, + backupExcludePVC: false, + shouldInclude: false, + }, + { + name: "volume has configmap so do not include", + vol: corev1.Volume{ + Name: "sample-volume", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "sample-cm", + }, + }, + }, + }, + backupExcludePVC: false, + shouldInclude: false, + }, + { + name: "volume is mounted as project volume so do not include", + vol: corev1.Volume{ + Name: "sample-volume", + VolumeSource: corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ + Sources: []corev1.VolumeProjection{}, + }, + }, + }, + backupExcludePVC: false, + shouldInclude: false, + }, + { + name: "volume has downwardAPI so do not include", + vol: corev1.Volume{ + Name: "sample-volume", + VolumeSource: corev1.VolumeSource{ + DownwardAPI: &corev1.DownwardAPIVolumeSource{ + Items: []corev1.DownwardAPIVolumeFile{ + { + Path: "labels", + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.labels", + }, + }, + }, + }, + }, + }, + backupExcludePVC: false, + shouldInclude: false, + }, + { + name: "volume has pvc and backupExcludePVC is true so do not include", + vol: corev1.Volume{ + Name: "sample-volume", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "sample-pvc", + }, + }, + }, + backupExcludePVC: true, + shouldInclude: false, + }, + { + name: "volume name has prefix default-token so do not include", + vol: corev1.Volume{ + Name: "default-token-vol-name", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "sample-pvc", + }, + }, + }, + backupExcludePVC: false, + shouldInclude: false, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + resourcePolicies := resourcepolicies.ResourcePolicies{ + Version: "v1", + VolumePolicies: []resourcepolicies.VolumePolicy{ + { + Conditions: map[string]interface{}{ + "storageClass": []string{"gp2-csi"}, + }, + Action: resourcepolicies.Action{ + Type: resourcepolicies.Snapshot, + }, + }, + }, + } + policies := resourcePolicies + p := &resourcepolicies.Policies{} + err := p.BuildPolicy(&policies) + if err != nil { + t.Fatalf("failed to build policy with error %v", err) + } + vh := &VolumeHelperImpl{ + VolumePolicy: p, + SnapshotVolumes: pointer.Bool(true), + Logger: velerotest.NewLogger(), + } + actualShouldInclude := vh.ShouldIncludeVolumeInBackup(tc.vol, tc.backupExcludePVC) + assert.Equalf(t, actualShouldInclude, tc.shouldInclude, "Want shouldInclude as %v; Got actualShouldInclude as %v", tc.shouldInclude, actualShouldInclude) + }) + } +} + +var ( + gp2csi = "gp2-csi" + gp3csi = "gp3-csi" +) +var ( + samplePod = &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sample-pod", + Namespace: "sample-ns", + }, + + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "sample-container", + Image: "sample-image", + VolumeMounts: []corev1.VolumeMount{ + { + Name: "sample-vm", + MountPath: "/etc/pod-info", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "sample-volume-1", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "sample-pvc-1", + }, + }, + }, + { + Name: "sample-volume-2", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "sample-pvc-2", + }, + }, + }, + }, + }, + } + + samplePVC1 = &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sample-pvc-1", + Namespace: "sample-ns", + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{}, + }, + StorageClassName: &gp2csi, + VolumeName: "sample-pv-1", + }, + Status: corev1.PersistentVolumeClaimStatus{ + Phase: corev1.ClaimBound, + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Capacity: corev1.ResourceList{}, + }, + } + + samplePVC2 = &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sample-pvc-2", + Namespace: "sample-ns", + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{}, + }, + StorageClassName: &gp3csi, + VolumeName: "sample-pv-2", + }, + Status: corev1.PersistentVolumeClaimStatus{ + Phase: corev1.ClaimBound, + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Capacity: corev1.ResourceList{}, + }, + } + + samplePV1 = &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sample-pv-1", + }, + Spec: corev1.PersistentVolumeSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Capacity: corev1.ResourceList{}, + ClaimRef: &corev1.ObjectReference{ + Kind: "PersistentVolumeClaim", + Name: "sample-pvc-1", + Namespace: "sample-ns", + ResourceVersion: "1027", + UID: "7d28e566-ade7-4ed6-9e15-2e44d2fbcc08", + }, + PersistentVolumeSource: corev1.PersistentVolumeSource{ + CSI: &corev1.CSIPersistentVolumeSource{ + Driver: "ebs.csi.aws.com", + FSType: "ext4", + VolumeAttributes: map[string]string{ + "storage.kubernetes.io/csiProvisionerIdentity": "1582049697841-8081-hostpath.csi.k8s.io", + }, + VolumeHandle: "e61f2b48-527a-11ea-b54f-cab6317018f1", + }, + }, + PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimDelete, + StorageClassName: gp2csi, + }, + Status: corev1.PersistentVolumeStatus{ + Phase: corev1.VolumeBound, + }, + } + + samplePV2 = &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sample-pv-2", + }, + Spec: corev1.PersistentVolumeSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Capacity: corev1.ResourceList{}, + ClaimRef: &corev1.ObjectReference{ + Kind: "PersistentVolumeClaim", + Name: "sample-pvc-2", + Namespace: "sample-ns", + ResourceVersion: "1027", + UID: "7d28e566-ade7-4ed6-9e15-2e44d2fbcc08", + }, + PersistentVolumeSource: corev1.PersistentVolumeSource{ + CSI: &corev1.CSIPersistentVolumeSource{ + Driver: "ebs.csi.aws.com", + FSType: "ext4", + VolumeAttributes: map[string]string{ + "storage.kubernetes.io/csiProvisionerIdentity": "1582049697841-8081-hostpath.csi.k8s.io", + }, + VolumeHandle: "e61f2b48-527a-11ea-b54f-cab6317018f1", + }, + }, + PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimDelete, + StorageClassName: gp3csi, + }, + Status: corev1.PersistentVolumeStatus{ + Phase: corev1.VolumeBound, + }, + } + resourcePolicies1 = resourcepolicies.ResourcePolicies{ + Version: "v1", + VolumePolicies: []resourcepolicies.VolumePolicy{ + { + Conditions: map[string]interface{}{ + "storageClass": []string{"gp2-csi"}, + }, + Action: resourcepolicies.Action{ + Type: resourcepolicies.FSBackup, + }, + }, + { + Conditions: map[string]interface{}{ + "storageClass": []string{"gp3-csi"}, + }, + Action: resourcepolicies.Action{ + Type: resourcepolicies.Snapshot, + }, + }, + }, + } + + resourcePolicies2 = resourcepolicies.ResourcePolicies{ + Version: "v1", + VolumePolicies: []resourcepolicies.VolumePolicy{ + { + Conditions: map[string]interface{}{ + "storageClass": []string{"gp2-csi"}, + }, + Action: resourcepolicies.Action{ + Type: resourcepolicies.FSBackup, + }, + }, + }, + } + + resourcePolicies3 = resourcepolicies.ResourcePolicies{ + Version: "v1", + VolumePolicies: []resourcepolicies.VolumePolicy{ + { + Conditions: map[string]interface{}{ + "storageClass": []string{"gp4-csi"}, + }, + Action: resourcepolicies.Action{ + Type: resourcepolicies.FSBackup, + }, + }, + }, + } +) + +func TestVolumeHelperImpl_GetVolumesMatchingFSBackupAction(t *testing.T) { + testCases := []struct { + name string + backupExcludePVC bool + resourcepoliciesApplied resourcepolicies.ResourcePolicies + FSBackupActionMatchingVols []string + FSBackupNonActionMatchingVols []string + NoActionMatchingVols []string + expectedErr bool + }{ + { + name: "For a given pod with 2 volumes and volume policy we get one fs-backup action matching volume, one fs-back action non-matching volume but has some matching action and zero no action matching volumes", + backupExcludePVC: false, + resourcepoliciesApplied: resourcePolicies1, + FSBackupActionMatchingVols: []string{"sample-volume-1"}, + FSBackupNonActionMatchingVols: []string{"sample-volume-2"}, + NoActionMatchingVols: []string{}, + expectedErr: false, + }, + { + name: "For a given pod with 2 volumes and volume policy we get one fs-backup action matching volume, zero fs-backup action non-matching volume and one no action matching volumes", + backupExcludePVC: false, + resourcepoliciesApplied: resourcePolicies2, + FSBackupActionMatchingVols: []string{"sample-volume-1"}, + FSBackupNonActionMatchingVols: []string{}, + NoActionMatchingVols: []string{"sample-volume-2"}, + expectedErr: false, + }, + { + name: "For a given pod with 2 volumes and volume policy we get one fs-backup action matching volume, one fs-backup action non-matching volume and one no action matching volumes but backupExcludePVC is true so all returned list should be empty", + backupExcludePVC: true, + resourcepoliciesApplied: resourcePolicies2, + FSBackupActionMatchingVols: []string{}, + FSBackupNonActionMatchingVols: []string{}, + NoActionMatchingVols: []string{}, + expectedErr: false, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + policies := tc.resourcepoliciesApplied + p := &resourcepolicies.Policies{} + err := p.BuildPolicy(&policies) + if err != nil { + t.Fatalf("failed to build policy with error %v", err) + } + vh := &VolumeHelperImpl{ + VolumePolicy: p, + SnapshotVolumes: pointer.Bool(true), + Logger: velerotest.NewLogger(), + } + objs := []runtime.Object{samplePod, samplePVC1, samplePVC2, samplePV1, samplePV2} + fakeClient := velerotest.NewFakeControllerRuntimeClient(t, objs...) + gotFSBackupActionMatchingVols, gotFSBackupNonActionMatchingVols, gotNoActionMatchingVols, actualError := vh.GetVolumesMatchingFSBackupAction(samplePod, vh.VolumePolicy, tc.backupExcludePVC, fakeClient) + if tc.expectedErr { + assert.NotNil(t, actualError, "Want error; Got nil error") + return + } + assert.Nilf(t, actualError, "Want: nil error; Got: %v", actualError) + assert.Equalf(t, gotFSBackupActionMatchingVols, tc.FSBackupActionMatchingVols, "Want FSBackupActionMatchingVols as %v; Got gotFSBackupActionMatchingVols as %v", tc.FSBackupActionMatchingVols, gotFSBackupActionMatchingVols) + assert.Equalf(t, gotFSBackupNonActionMatchingVols, tc.FSBackupNonActionMatchingVols, "Want FSBackupNonActionMatchingVols as %v; Got gotFSBackupNonActionMatchingVols as %v", tc.FSBackupNonActionMatchingVols, gotFSBackupNonActionMatchingVols) + assert.Equalf(t, gotNoActionMatchingVols, tc.NoActionMatchingVols, "Want NoActionMatchingVols as %v; Got gotNoActionMatchingVols as %v", tc.NoActionMatchingVols, gotNoActionMatchingVols) + }) + } +} + +func TestVolumeHelperImpl_GetVolumesForFSBackup(t *testing.T) { + testCases := []struct { + name string + backupExcludePVC bool + defaultVolumesToFsBackup bool + resourcepoliciesApplied resourcepolicies.ResourcePolicies + includedVolumes []string + optedOutVolumes []string + expectedErr bool + }{ + { + name: "For a given pod with 2 volumes and volume policy we get one fs-backup action matching volume, one fs-back action non-matching volume but matches snapshot action so no volumes for legacy fallback process, defaultvolumestofsbackup is false but no effect", + backupExcludePVC: false, + defaultVolumesToFsBackup: false, + resourcepoliciesApplied: resourcePolicies1, + includedVolumes: []string{"sample-volume-1"}, + optedOutVolumes: []string{"sample-volume-2"}, + }, + { + name: "For a given pod with 2 volumes and volume policy we get one fs-backup action matching volume, one fs-back action non-matching volume but matches snapshot action so no volumes for legacy fallback process, defaultvolumestofsbackup is true but no effect", + backupExcludePVC: false, + defaultVolumesToFsBackup: true, + resourcepoliciesApplied: resourcePolicies1, + includedVolumes: []string{"sample-volume-1"}, + optedOutVolumes: []string{"sample-volume-2"}, + }, + { + name: "For a given pod with 2 volumes and volume policy we get no volume matching fs-backup action defaultvolumesToFSBackup is false, no annotations, using legacy as fallback for non-action matching volumes", + backupExcludePVC: false, + defaultVolumesToFsBackup: false, + resourcepoliciesApplied: resourcePolicies3, + includedVolumes: []string{}, + optedOutVolumes: []string{}, + }, + { + name: "For a given pod with 2 volumes and volume policy we get no volume matching fs-backup action defaultvolumesToFSBackup is true, no annotations, using legacy as fallback for non-action matching volumes", + backupExcludePVC: false, + defaultVolumesToFsBackup: true, + resourcepoliciesApplied: resourcePolicies3, + includedVolumes: []string{"sample-volume-1", "sample-volume-2"}, + optedOutVolumes: []string{}, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + policies := tc.resourcepoliciesApplied + p := &resourcepolicies.Policies{} + err := p.BuildPolicy(&policies) + if err != nil { + t.Fatalf("failed to build policy with error %v", err) + } + vh := &VolumeHelperImpl{ + VolumePolicy: p, + SnapshotVolumes: pointer.Bool(true), + Logger: velerotest.NewLogger(), + } + objs := []runtime.Object{samplePod, samplePVC1, samplePVC2, samplePV1, samplePV2} + fakeClient := velerotest.NewFakeControllerRuntimeClient(t, objs...) + gotIncludedVolumes, gotOptedOutVolumes, actualError := vh.GetVolumesForFSBackup(samplePod, tc.defaultVolumesToFsBackup, tc.backupExcludePVC, fakeClient) + if tc.expectedErr { + assert.NotNil(t, actualError, "Want error; Got nil error") + return + } + assert.Nilf(t, actualError, "Want: nil error; Got: %v", actualError) + assert.Equalf(t, tc.includedVolumes, gotIncludedVolumes, "Want includedVolumes as %v; Got gotIncludedVolumes as %v", tc.includedVolumes, gotIncludedVolumes) + assert.Equalf(t, tc.optedOutVolumes, gotOptedOutVolumes, "Want optedOutVolumes as %v; Got gotOptedOutVolumes as %v", tc.optedOutVolumes, gotOptedOutVolumes) + }) + } +} diff --git a/pkg/backup/item_backupper.go b/pkg/backup/item_backupper.go index 953b3191e2..1e35ea3856 100644 --- a/pkg/backup/item_backupper.go +++ b/pkg/backup/item_backupper.go @@ -24,6 +24,8 @@ import ( "strings" "time" + "github.com/vmware-tanzu/velero/internal/volumehelper" + "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" @@ -52,7 +54,6 @@ import ( "github.com/vmware-tanzu/velero/pkg/podvolume" "github.com/vmware-tanzu/velero/pkg/util/boolptr" csiutil "github.com/vmware-tanzu/velero/pkg/util/csi" - pdvolumeutil "github.com/vmware-tanzu/velero/pkg/util/podvolume" ) const ( @@ -187,6 +188,16 @@ func (ib *itemBackupper) backupItemInternal(logger logrus.FieldLogger, obj runti ib.trackSkippedPV(obj, groupResource, podVolumeApproach, fmt.Sprintf("opted out due to annotation in pod %s", podName), log) } + // Instantiate volumepolicyhelper struct here + vh := &volumehelper.VolumeHelperImpl{ + SnapshotVolumes: ib.backupRequest.Spec.SnapshotVolumes, + Logger: logger, + } + + if ib.backupRequest.ResPolicies != nil { + vh.VolumePolicy = ib.backupRequest.ResPolicies + } + if groupResource == kuberesource.Pods { // pod needs to be initialized for the unstructured converter pod = new(corev1api.Pod) @@ -195,14 +206,13 @@ func (ib *itemBackupper) backupItemInternal(logger logrus.FieldLogger, obj runti // nil it on error since it's not valid pod = nil } else { - // Get the list of volumes to back up using pod volume backup from the pod's annotations. Remove from this list + // Get the list of volumes to back up using pod volume backup from the pod's annotations or volume policy approach. Remove from this list // any volumes that use a PVC that we've already backed up (this would be in a read-write-many scenario, // where it's been backed up from another pod), since we don't need >1 backup per PVC. - includedVolumes, optedOutVolumes := pdvolumeutil.GetVolumesByPod( - pod, - boolptr.IsSetToTrue(ib.backupRequest.Spec.DefaultVolumesToFsBackup), - !ib.backupRequest.ResourceIncludesExcludes.ShouldInclude(kuberesource.PersistentVolumeClaims.String()), - ) + includedVolumes, optedOutVolumes, err := vh.GetVolumesForFSBackup(pod, boolptr.IsSetToTrue(ib.backupRequest.Spec.DefaultVolumesToFsBackup), !ib.backupRequest.ResourceIncludesExcludes.ShouldInclude(kuberesource.PersistentVolumeClaims.String()), ib.kbClient) + if err != nil { + backupErrs = append(backupErrs, errors.WithStack(err)) + } for _, volume := range includedVolumes { // track the volumes that are PVCs using the PVC snapshot tracker, so that when we backup PVCs/PVs @@ -229,7 +239,7 @@ func (ib *itemBackupper) backupItemInternal(logger logrus.FieldLogger, obj runti // the group version of the object. versionPath := resourceVersion(obj) - updatedObj, additionalItemFiles, err := ib.executeActions(log, obj, groupResource, name, namespace, metadata, finalize) + updatedObj, additionalItemFiles, err := ib.executeActions(log, obj, groupResource, name, namespace, metadata, finalize, vh) if err != nil { backupErrs = append(backupErrs, err) @@ -255,7 +265,7 @@ func (ib *itemBackupper) backupItemInternal(logger logrus.FieldLogger, obj runti backupErrs = append(backupErrs, err) } - if err := ib.takePVSnapshot(obj, log); err != nil { + if err := ib.takePVSnapshot(obj, log, vh); err != nil { backupErrs = append(backupErrs, err) } } @@ -351,6 +361,7 @@ func (ib *itemBackupper) executeActions( name, namespace string, metadata metav1.Object, finalize bool, + vh *volumehelper.VolumeHelperImpl, ) (runtime.Unstructured, []FileForArchive, error) { var itemFiles []FileForArchive for _, action := range ib.backupRequest.ResolvedActions { @@ -374,6 +385,19 @@ func (ib *itemBackupper) executeActions( continue } + if groupResource == kuberesource.PersistentVolumeClaims && actionName == csiBIAPluginName && vh.VolumePolicy != nil { + snapshotVolume, err := vh.ShouldPerformSnapshot(obj, kuberesource.PersistentVolumeClaims, ib.kbClient) + if err != nil { + return nil, itemFiles, errors.WithStack(err) + } + + if !snapshotVolume { + log.Info(fmt.Sprintf("skipping csi volume snapshot for PVC %s as it does not fit the volume policy criteria specified by the user for snapshot action", namespace+"/"+name)) + ib.trackSkippedPV(obj, kuberesource.PersistentVolumeClaims, volumeSnapshotApproach, "does not satisfy the criteria for volume policy based snapshot action", log) + continue + } + } + updatedItem, additionalItemIdentifiers, operationID, postOperationItems, err := action.Execute(obj, ib.backupRequest.Backup) if err != nil { return nil, itemFiles, errors.Wrapf(err, "error executing custom action (groupResource=%s, namespace=%s, name=%s)", groupResource.String(), namespace, name) @@ -504,15 +528,9 @@ const ( // takePVSnapshot triggers a snapshot for the volume/disk underlying a PersistentVolume if the provided // backup has volume snapshots enabled and the PV is of a compatible type. Also records cloud // disk type and IOPS (if applicable) to be able to restore to current state later. -func (ib *itemBackupper) takePVSnapshot(obj runtime.Unstructured, log logrus.FieldLogger) error { +func (ib *itemBackupper) takePVSnapshot(obj runtime.Unstructured, log logrus.FieldLogger, vh *volumehelper.VolumeHelperImpl) error { log.Info("Executing takePVSnapshot") - if boolptr.IsSetToFalse(ib.backupRequest.Spec.SnapshotVolumes) { - log.Info("Backup has volume snapshots disabled; skipping volume snapshot action.") - ib.trackSkippedPV(obj, kuberesource.PersistentVolumes, volumeSnapshotApproach, "backup has volume snapshots disabled", log) - return nil - } - pv := new(corev1api.PersistentVolume) if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), pv); err != nil { return errors.WithStack(err) @@ -520,6 +538,26 @@ func (ib *itemBackupper) takePVSnapshot(obj runtime.Unstructured, log logrus.Fie log = log.WithField("persistentVolume", pv.Name) + if vh.VolumePolicy != nil { + snapshotVolume, err := vh.ShouldPerformSnapshot(obj, kuberesource.PersistentVolumes, ib.kbClient) + + if err != nil { + return err + } + + if !snapshotVolume { + log.Info(fmt.Sprintf("skipping volume snapshot for PV %s as it does not fit the volume policy criteria specified by the user for snapshot action", pv.Name)) + ib.trackSkippedPV(obj, kuberesource.PersistentVolumes, volumeSnapshotApproach, "does not satisfy the criteria for volume policy based snapshot action", log) + return nil + } + } + + if boolptr.IsSetToFalse(ib.backupRequest.Spec.SnapshotVolumes) { + log.Info("Backup has volume snapshots disabled; skipping volume snapshot action.") + ib.trackSkippedPV(obj, kuberesource.PersistentVolumes, volumeSnapshotApproach, "backup has volume snapshots disabled", log) + return nil + } + // If this PV is claimed, see if we've already taken a (pod volume backup) snapshot of the contents // of this PV. If so, don't take a snapshot. if pv.Spec.ClaimRef != nil { diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index 4aa6a3ade9..692f4d96ef 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -758,7 +758,7 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string if _, ok := enabledRuntimeControllers[controller.Backup]; ok { backupper, err := backup.NewKubernetesBackupper( - s.mgr.GetClient(), + s.crClient, s.discoveryHelper, client.NewDynamicFactory(s.dynamicClient), podexec.NewPodCommandExecutor(s.kubeClientConfig, s.kubeClient.CoreV1().RESTClient()), diff --git a/pkg/util/kube/pvc_pv.go b/pkg/util/kube/pvc_pv.go index 8dfa8d1ebc..12daa0e3ee 100644 --- a/pkg/util/kube/pvc_pv.go +++ b/pkg/util/kube/pvc_pv.go @@ -386,3 +386,21 @@ func GetPVForPVC( } return pv, nil } + +func GetPVCForPodVolume(vol *corev1api.Volume, pod *corev1api.Pod, crClient crclient.Client) (*corev1api.PersistentVolumeClaim, error) { + if vol.PersistentVolumeClaim == nil { + return nil, errors.Errorf("volume %s/%s has no PVC associated with it", pod.Namespace, vol.Name) + } + pvc := &corev1api.PersistentVolumeClaim{} + err := crClient.Get( + context.TODO(), + crclient.ObjectKey{Name: vol.PersistentVolumeClaim.ClaimName, Namespace: pod.Namespace}, + pvc, + ) + if err != nil { + return nil, errors.Wrapf(err, "failed to get PVC %s for Volume %s/%s", + vol.PersistentVolumeClaim.ClaimName, pod.Namespace, vol.Name) + } + + return pvc, nil +} diff --git a/pkg/util/kube/pvc_pv_test.go b/pkg/util/kube/pvc_pv_test.go index a2eeea3460..113c28e230 100644 --- a/pkg/util/kube/pvc_pv_test.go +++ b/pkg/util/kube/pvc_pv_test.go @@ -1256,3 +1256,125 @@ func TestGetPVForPVC(t *testing.T) { }) } } + +func TestGetPVCForPodVolume(t *testing.T) { + sampleVol := &corev1api.Volume{ + Name: "sample-volume", + VolumeSource: corev1api.VolumeSource{ + PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ + ClaimName: "sample-pvc", + }, + }, + } + + sampleVol2 := &corev1api.Volume{ + Name: "sample-volume", + VolumeSource: corev1api.VolumeSource{ + PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ + ClaimName: "sample-pvc-1", + }, + }, + } + + sampleVol3 := &corev1api.Volume{ + Name: "sample-volume", + VolumeSource: corev1api.VolumeSource{}, + } + + samplePod := &corev1api.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sample-pod", + Namespace: "sample-ns", + }, + + Spec: corev1api.PodSpec{ + Containers: []corev1api.Container{ + { + Name: "sample-container", + Image: "sample-image", + VolumeMounts: []corev1api.VolumeMount{ + { + Name: "sample-vm", + MountPath: "/etc/pod-info", + }, + }, + }, + }, + Volumes: []corev1api.Volume{ + { + Name: "sample-volume", + VolumeSource: corev1api.VolumeSource{ + PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ + ClaimName: "sample-pvc", + }, + }, + }, + }, + }, + } + + matchingPVC := &corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sample-pvc", + Namespace: "sample-ns", + }, + Spec: v1.PersistentVolumeClaimSpec{ + AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, + Resources: v1.VolumeResourceRequirements{ + Requests: v1.ResourceList{}, + }, + StorageClassName: &csiStorageClass, + VolumeName: "test-csi-7d28e566-ade7-4ed6-9e15-2e44d2fbcc08", + }, + Status: v1.PersistentVolumeClaimStatus{ + Phase: v1.ClaimBound, + AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, + Capacity: v1.ResourceList{}, + }, + } + + testCases := []struct { + name string + vol *corev1api.Volume + pod *corev1api.Pod + expectedPVC *corev1api.PersistentVolumeClaim + expectedError bool + }{ + { + name: "should find PVC for volume", + vol: sampleVol, + pod: samplePod, + expectedPVC: matchingPVC, + expectedError: false, + }, + { + name: "should not find PVC for volume not found error case", + vol: sampleVol2, + pod: samplePod, + expectedPVC: nil, + expectedError: true, + }, + { + name: "should not find PVC vol has no PVC, error case", + vol: sampleVol3, + pod: samplePod, + expectedPVC: nil, + expectedError: true, + }, + } + + objs := []runtime.Object{matchingPVC} + fakeClient := velerotest.NewFakeControllerRuntimeClient(t, objs...) + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actualPVC, actualError := GetPVCForPodVolume(tc.vol, samplePod, fakeClient) + if tc.expectedError { + assert.NotNil(t, actualError, "Want error; Got nil error") + assert.Nilf(t, actualPVC, "Want PV: nil; Got PV: %q", actualPVC) + return + } + assert.Nilf(t, actualError, "Want: nil error; Got: %v", actualError) + assert.Equalf(t, actualPVC.Name, tc.expectedPVC.Name, "Want PVC with name %q; Got PVC with name %q", tc.expectedPVC.Name, actualPVC) + }) + } +} diff --git a/pkg/util/podvolume/pod_volume.go b/pkg/util/podvolume/pod_volume.go index 20f991930e..bd99202c99 100644 --- a/pkg/util/podvolume/pod_volume.go +++ b/pkg/util/podvolume/pod_volume.go @@ -30,7 +30,7 @@ import ( ) // GetVolumesByPod returns a list of volume names to backup for the provided pod. -func GetVolumesByPod(pod *corev1api.Pod, defaultVolumesToFsBackup, backupExcludePVC bool) ([]string, []string) { +func GetVolumesByPod(pod *corev1api.Pod, defaultVolumesToFsBackup, backupExcludePVC bool, volsToProcessByLegacyApproach []string) ([]string, []string) { // tracks the volumes that have been explicitly opted out of backup via the annotation in the pod optedOutVolumes := make([]string, 0) @@ -38,9 +38,13 @@ func GetVolumesByPod(pod *corev1api.Pod, defaultVolumesToFsBackup, backupExclude return GetVolumesToBackup(pod), optedOutVolumes } - volsToExclude := getVolumesToExclude(pod) + volsToExclude := GetVolumesToExclude(pod) podVolumes := []string{} - for _, pv := range pod.Spec.Volumes { + // Identify volume to process + // For normal case all the pod volume will be processed + // For case when volsToProcessByLegacyApproach is non-empty then only those volume will be processed + volsToProcess := GetVolumesToProcess(pod.Spec.Volumes, volsToProcessByLegacyApproach) + for _, pv := range volsToProcess { // cannot backup hostpath volumes as they are not mounted into /var/lib/kubelet/pods // and therefore not accessible to the node agent daemon set. if pv.HostPath != nil { @@ -96,7 +100,7 @@ func GetVolumesToBackup(obj metav1.Object) []string { return strings.Split(backupsValue, ",") } -func getVolumesToExclude(obj metav1.Object) []string { +func GetVolumesToExclude(obj metav1.Object) []string { annotations := obj.GetAnnotations() if annotations == nil { return nil @@ -112,7 +116,7 @@ func IsPVCDefaultToFSBackup(pvcNamespace, pvcName string, crClient crclient.Clie } for index := range pods { - vols, _ := GetVolumesByPod(&pods[index], defaultVolumesToFsBackup, false) + vols, _ := GetVolumesByPod(&pods[index], defaultVolumesToFsBackup, false, []string{}) if len(vols) > 0 { volName, err := getPodVolumeNameForPVC(pods[index], pvcName) if err != nil { @@ -160,3 +164,29 @@ func getPodsUsingPVC( return podsUsingPVC, nil } + +func GetVolumesToProcess(volumes []corev1api.Volume, volsToProcessByLegacyApproach []string) []corev1api.Volume { + volsToProcess := make([]corev1api.Volume, 0) + + // return empty list when no volumes associated with pod + if len(volumes) == 0 { + return volsToProcess + } + + // legacy approach as a fallback option case + if len(volsToProcessByLegacyApproach) > 0 { + for _, vol := range volumes { + // don't process volumes that are already matched for supported action in volume policy approach + if !util.Contains(volsToProcessByLegacyApproach, vol.Name) { + continue + } + + // add volume that is not processed in volume policy approach + volsToProcess = append(volsToProcess, vol) + } + + return volsToProcess + } + // normal case return the list as in + return volumes +} diff --git a/pkg/util/podvolume/pod_volume_test.go b/pkg/util/podvolume/pod_volume_test.go index 3bebb3cf6a..31ea813ac9 100644 --- a/pkg/util/podvolume/pod_volume_test.go +++ b/pkg/util/podvolume/pod_volume_test.go @@ -369,7 +369,7 @@ func TestGetVolumesByPod(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - actualIncluded, actualOptedOut := GetVolumesByPod(tc.pod, tc.defaultVolumesToFsBackup, tc.backupExcludePVC) + actualIncluded, actualOptedOut := GetVolumesByPod(tc.pod, tc.defaultVolumesToFsBackup, tc.backupExcludePVC, []string{}) sort.Strings(tc.expected.included) sort.Strings(actualIncluded) @@ -792,3 +792,97 @@ func TestGetPodsUsingPVC(t *testing.T) { }) } } + +func TestGetVolumesToProcess(t *testing.T) { + testCases := []struct { + name string + volumes []corev1api.Volume + volsToProcessByLegacyApproach []string + expectedVolumes []corev1api.Volume + }{ + { + name: "pod has 2 volumes empty volsToProcessByLegacyApproach list return 2 volumes", + volumes: []corev1api.Volume{ + { + Name: "sample-volume-1", + VolumeSource: corev1api.VolumeSource{ + PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ + ClaimName: "sample-pvc-1", + }, + }, + }, + { + Name: "sample-volume-2", + VolumeSource: corev1api.VolumeSource{ + PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ + ClaimName: "sample-pvc-2", + }, + }, + }, + }, + volsToProcessByLegacyApproach: []string{}, + expectedVolumes: []corev1api.Volume{ + { + Name: "sample-volume-1", + VolumeSource: corev1api.VolumeSource{ + PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ + ClaimName: "sample-pvc-1", + }, + }, + }, + { + Name: "sample-volume-2", + VolumeSource: corev1api.VolumeSource{ + PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ + ClaimName: "sample-pvc-2", + }, + }, + }, + }, + }, + { + name: "pod has 2 volumes non-empty volsToProcessByLegacyApproach list returns 1 volumes", + volumes: []corev1api.Volume{ + { + Name: "sample-volume-1", + VolumeSource: corev1api.VolumeSource{ + PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ + ClaimName: "sample-pvc-1", + }, + }, + }, + { + Name: "sample-volume-2", + VolumeSource: corev1api.VolumeSource{ + PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ + ClaimName: "sample-pvc-2", + }, + }, + }, + }, + volsToProcessByLegacyApproach: []string{"sample-volume-2"}, + expectedVolumes: []corev1api.Volume{ + { + Name: "sample-volume-2", + VolumeSource: corev1api.VolumeSource{ + PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ + ClaimName: "sample-pvc-2", + }, + }, + }, + }, + }, + { + name: "empty case, return empty list", + volumes: []corev1api.Volume{}, + volsToProcessByLegacyApproach: []string{}, + expectedVolumes: []corev1api.Volume{}, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actualVolumes := GetVolumesToProcess(tc.volumes, tc.volsToProcessByLegacyApproach) + assert.Equal(t, tc.expectedVolumes, actualVolumes, "Want Volumes List %v; Got Volumes List %v", tc.expectedVolumes, actualVolumes) + }) + } +} From c894b4bff128bd6d6da210ecb07bda5885638b98 Mon Sep 17 00:00:00 2001 From: Jiaolin Yang Date: Wed, 24 Apr 2024 13:50:13 +0800 Subject: [PATCH 09/13] Update hackmd link for community meeting notes Update hackmd link for community meeting notes. Signed-off-by: Jiaolin Yang --- site/content/community/_index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/content/community/_index.md b/site/content/community/_index.md index 19304273d1..c9eebef314 100644 --- a/site/content/community/_index.md +++ b/site/content/community/_index.md @@ -18,6 +18,6 @@ You can follow the work we do, see our milestones, and our backlog on our [GitHu Bi-weekly community meeting alternating every week between Beijing Friendly timezone and EST/Europe Friendly Timezone * Beijing/US friendly - we start at 8am Beijing Time(bound to CST) / 8pm EDT(7pm EST) / 5pm PDT(4pm PST) / 2am CEST(1am CET) - [Convert to your time zone](https://dateful.com/convert/beijing-china?t=8am) * US/Europe friendly - we start at 10am ET(bound to ET) / 7am PT / 3pm CET / 10pm(11pm) CST - [Convert to your time zone](https://dateful.com/convert/est-edt-eastern-time?t=10) -* Read and comment on the [meeting notes](https://hackmd.io/Jq6F5zqZR7S80CeDWUklkA?view) +* Read and comment on the [meeting notes](https://hackmd.io/bxrvgewUQ5ORH10BKUFpxw) * See previous community meetings on our [YouTube Channel](https://www.youtube.com/playlist?list=PL7bmigfV0EqQRysvqvqOtRNk4L5S7uqwM) * Have a question to discuss in the community meeting? Please add it to our [Q&A Discussion board](https://github.com/vmware-tanzu/velero/discussions/categories/community-support-q-a) From bffe4f9f56ba16230441f1f16231ae48f4f80866 Mon Sep 17 00:00:00 2001 From: Tiger Kaovilai Date: Thu, 25 Apr 2024 12:28:53 -0400 Subject: [PATCH 10/13] Add Schedule.spec.pause to docs Signed-off-by: Tiger Kaovilai --- site/content/docs/main/api-types/schedule.md | 2 ++ site/content/docs/v1.10/api-types/schedule.md | 2 ++ site/content/docs/v1.11/api-types/schedule.md | 2 ++ site/content/docs/v1.12/api-types/schedule.md | 2 ++ site/content/docs/v1.13/api-types/schedule.md | 2 ++ 5 files changed, 10 insertions(+) diff --git a/site/content/docs/main/api-types/schedule.md b/site/content/docs/main/api-types/schedule.md index 2797f70269..e9afea460d 100644 --- a/site/content/docs/main/api-types/schedule.md +++ b/site/content/docs/main/api-types/schedule.md @@ -30,6 +30,8 @@ metadata: namespace: velero # Parameters about the scheduled backup. Required. spec: + # Paused specifies whether the schedule is paused or not + paused: false # Schedule is a Cron expression defining when to run the Backup schedule: 0 7 * * * # Specifies whether to use OwnerReferences on backups created by this Schedule. diff --git a/site/content/docs/v1.10/api-types/schedule.md b/site/content/docs/v1.10/api-types/schedule.md index 4b759501af..623dbf77ee 100644 --- a/site/content/docs/v1.10/api-types/schedule.md +++ b/site/content/docs/v1.10/api-types/schedule.md @@ -30,6 +30,8 @@ metadata: namespace: velero # Parameters about the scheduled backup. Required. spec: + # Paused specifies whether the schedule is paused or not + paused: false # Schedule is a Cron expression defining when to run the Backup schedule: 0 7 * * * # Specifies whether to use OwnerReferences on backups created by this Schedule. diff --git a/site/content/docs/v1.11/api-types/schedule.md b/site/content/docs/v1.11/api-types/schedule.md index 58a4c454b1..278f383aab 100644 --- a/site/content/docs/v1.11/api-types/schedule.md +++ b/site/content/docs/v1.11/api-types/schedule.md @@ -30,6 +30,8 @@ metadata: namespace: velero # Parameters about the scheduled backup. Required. spec: + # Paused specifies whether the schedule is paused or not + paused: false # Schedule is a Cron expression defining when to run the Backup schedule: 0 7 * * * # Specifies whether to use OwnerReferences on backups created by this Schedule. diff --git a/site/content/docs/v1.12/api-types/schedule.md b/site/content/docs/v1.12/api-types/schedule.md index eb8e8fbd80..37f671b5cc 100644 --- a/site/content/docs/v1.12/api-types/schedule.md +++ b/site/content/docs/v1.12/api-types/schedule.md @@ -30,6 +30,8 @@ metadata: namespace: velero # Parameters about the scheduled backup. Required. spec: + # Paused specifies whether the schedule is paused or not + paused: false # Schedule is a Cron expression defining when to run the Backup schedule: 0 7 * * * # Specifies whether to use OwnerReferences on backups created by this Schedule. diff --git a/site/content/docs/v1.13/api-types/schedule.md b/site/content/docs/v1.13/api-types/schedule.md index 2797f70269..e9afea460d 100644 --- a/site/content/docs/v1.13/api-types/schedule.md +++ b/site/content/docs/v1.13/api-types/schedule.md @@ -30,6 +30,8 @@ metadata: namespace: velero # Parameters about the scheduled backup. Required. spec: + # Paused specifies whether the schedule is paused or not + paused: false # Schedule is a Cron expression defining when to run the Backup schedule: 0 7 * * * # Specifies whether to use OwnerReferences on backups created by this Schedule. From 5eae54276224f83c5b0996157b187067a475d934 Mon Sep 17 00:00:00 2001 From: Ming Qiu Date: Fri, 26 Apr 2024 09:45:07 +0000 Subject: [PATCH 11/13] Fix maintenance job launched immediately after prune error Signed-off-by: Ming Qiu --- pkg/cmd/cli/install/install.go | 3 --- pkg/controller/backup_repository_controller.go | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/pkg/cmd/cli/install/install.go b/pkg/cmd/cli/install/install.go index 48629252d9..c51cee6586 100644 --- a/pkg/cmd/cli/install/install.go +++ b/pkg/cmd/cli/install/install.go @@ -164,9 +164,6 @@ func NewInstallOptions() *Options { DefaultSnapshotMoveData: false, DisableInformerCache: false, ScheduleSkipImmediately: false, - MaintenanceCfg: repository.MaintenanceConfig{ - KeepLatestMaitenanceJobs: repository.DefaultKeepLatestMaitenanceJobs, - }, } } diff --git a/pkg/controller/backup_repository_controller.go b/pkg/controller/backup_repository_controller.go index e4d68d9998..7e298d48dc 100644 --- a/pkg/controller/backup_repository_controller.go +++ b/pkg/controller/backup_repository_controller.go @@ -72,7 +72,7 @@ func (r *BackupRepoReconciler) SetupWithManager(mgr ctrl.Manager) error { s := kube.NewPeriodicalEnqueueSource(r.logger, mgr.GetClient(), &velerov1api.BackupRepositoryList{}, repoSyncPeriod, kube.PeriodicalEnqueueSourceOption{}) return ctrl.NewControllerManagedBy(mgr). - For(&velerov1api.BackupRepository{}). + For(&velerov1api.BackupRepository{}, builder.WithPredicates(kube.SpecChangePredicate{})). WatchesRawSource(s, nil). Watches(&velerov1api.BackupStorageLocation{}, kube.EnqueueRequestsFromMapUpdateFunc(r.invalidateBackupReposForBSL), builder.WithPredicates( From 82fc557bd112ae492506798cf58693a7c6998cc3 Mon Sep 17 00:00:00 2001 From: danfengl Date: Fri, 29 Mar 2024 02:29:27 +0000 Subject: [PATCH 12/13] Add checkpoint of VSC for data movement migration test 1. In data movement scenario, volumesnapshotcontent by Velero backup will be deleted instead of retained in CSI scenaito, so add a checkpoint for data movement scenario to verify no volumesnapshotcontent left after Velero backup; 2. Fix global context varaible issue, context varaible is not effective due to it's initialized right after the very beginning of all tests instead of beginning of each test, so if someone script a new E2E test and did not overwrite it in the test body, then it will fail the test if it was triggerd one hour later; 3. Due to CSI plugin is deprecated, it breaked down migration tests, because v1.13 still needs to install CSI plugin for the test. Signed-off-by: danfengl --- test/e2e/backups/ttl.go | 2 +- test/e2e/basic/backup-volume-info/base.go | 42 ++++--- .../backup-volume-info/native_snapshot.go | 5 +- test/e2e/basic/namespace-mapping.go | 2 - test/e2e/basic/nodeport.go | 4 +- test/e2e/basic/pvc-selected-node-changing.go | 2 - test/e2e/basic/resources-check/namespaces.go | 1 - .../resources-check/namespaces_annotation.go | 3 - test/e2e/basic/resources-check/rbac.go | 3 - test/e2e/basic/storage-class-changing.go | 3 +- test/e2e/migration/migration.go | 103 ++++++++++-------- test/e2e/pv-backup/pv-backup-filter.go | 2 - test/e2e/resource-filtering/base.go | 2 - test/e2e/resource-filtering/exclude_label.go | 3 - .../resource-filtering/exclude_namespaces.go | 3 - .../resource-filtering/include_namespaces.go | 3 - test/e2e/resource-filtering/label_selector.go | 3 - .../resourcemodifiers/resource_modifiers.go | 5 - .../e2e/resourcepolicies/resource_policies.go | 5 - test/e2e/schedule/ordered_resources.go | 3 - test/e2e/schedule/schedule-backup-creation.go | 1 - test/e2e/schedule/schedule.go | 2 - test/e2e/test/test.go | 91 +++++++--------- test/e2e/upgrade/upgrade.go | 9 +- test/types.go | 4 +- test/util/csi/common.go | 94 +++++++--------- test/util/k8s/common.go | 3 +- test/util/k8s/namespace.go | 7 +- test/util/k8s/rbac.go | 2 +- test/util/k8s/serviceaccount.go | 11 +- test/util/kibishii/kibishii_utils.go | 16 ++- test/util/providers/common.go | 2 +- test/util/velero/install.go | 5 +- test/util/velero/velero_utils.go | 27 ++--- 34 files changed, 203 insertions(+), 270 deletions(-) diff --git a/test/e2e/backups/ttl.go b/test/e2e/backups/ttl.go index 409773264f..0e5a4d653d 100644 --- a/test/e2e/backups/ttl.go +++ b/test/e2e/backups/ttl.go @@ -139,7 +139,7 @@ func TTLTest() { fmt.Sprintf("Failed to delete namespace %s", BackupCfg.BackupName)) }) - if veleroCfg.CloudProvider == Aws && useVolumeSnapshots { + if veleroCfg.CloudProvider == AWS && useVolumeSnapshots { fmt.Println("Waiting 7 minutes to make sure the snapshots are ready...") time.Sleep(7 * time.Minute) } diff --git a/test/e2e/basic/backup-volume-info/base.go b/test/e2e/basic/backup-volume-info/base.go index b23fe15671..e6ba12ce1a 100644 --- a/test/e2e/basic/backup-volume-info/base.go +++ b/test/e2e/basic/backup-volume-info/base.go @@ -17,7 +17,6 @@ limitations under the License. package basic import ( - "context" "fmt" "strconv" "strings" @@ -45,26 +44,6 @@ type BackupVolumeInfo struct { func (v *BackupVolumeInfo) Init() error { v.TestCase.Init() - - BeforeEach(func() { - if v.VeleroCfg.CloudProvider == Vsphere && (!strings.Contains(v.CaseBaseName, "fs-upload") && !strings.Contains(v.CaseBaseName, "skipped")) { - fmt.Printf("Skip snapshot case %s for vsphere environment.\n", v.CaseBaseName) - Skip("Skip snapshot case due to vsphere environment doesn't cover the CSI test, and it doesn't have a Velero native snapshot plugin.") - } - - if strings.Contains(v.VeleroCfg.Features, FeatureCSI) { - if strings.Contains(v.CaseBaseName, "native-snapshot") { - fmt.Printf("Skip native snapshot case %s when the CSI feature is enabled.\n", v.CaseBaseName) - Skip("Skip native snapshot case due to CSI feature is enabled.") - } - } else { - if strings.Contains(v.CaseBaseName, "csi") { - fmt.Printf("Skip CSI related case %s when the CSI feature is not enabled.\n", v.CaseBaseName) - Skip("Skip CSI cases due to CSI feature is not enabled.") - } - } - }) - v.CaseBaseName = v.CaseBaseName + v.UUIDgen v.BackupName = "backup-" + v.CaseBaseName v.RestoreName = "restore-" + v.CaseBaseName @@ -98,8 +77,27 @@ func (v *BackupVolumeInfo) Init() error { return nil } +func (v *BackupVolumeInfo) Start() error { + if v.VeleroCfg.CloudProvider == Vsphere && (!strings.Contains(v.CaseBaseName, "fs-upload") && !strings.Contains(v.CaseBaseName, "skipped")) { + fmt.Printf("Skip snapshot case %s for vsphere environment.\n", v.CaseBaseName) + Skip("Skip snapshot case due to vsphere environment doesn't cover the CSI test, and it doesn't have a Velero native snapshot plugin.") + } + + if strings.Contains(v.VeleroCfg.Features, FeatureCSI) { + if strings.Contains(v.CaseBaseName, "native-snapshot") { + fmt.Printf("Skip native snapshot case %s when the CSI feature is enabled.\n", v.CaseBaseName) + Skip("Skip native snapshot case due to CSI feature is enabled.") + } + } else { + if strings.Contains(v.CaseBaseName, "csi") { + fmt.Printf("Skip CSI related case %s when the CSI feature is not enabled.\n", v.CaseBaseName) + Skip("Skip CSI cases due to CSI feature is not enabled.") + } + } + v.TestCase.Start() + return nil +} func (v *BackupVolumeInfo) CreateResources() error { - v.Ctx, v.CtxCancel = context.WithTimeout(context.Background(), v.TimeoutDuration) labels := map[string]string{ "volume-info": "true", } diff --git a/test/e2e/basic/backup-volume-info/native_snapshot.go b/test/e2e/basic/backup-volume-info/native_snapshot.go index fb3a6f3d8c..c8ec0be1b5 100644 --- a/test/e2e/basic/backup-volume-info/native_snapshot.go +++ b/test/e2e/basic/backup-volume-info/native_snapshot.go @@ -30,7 +30,8 @@ var NativeSnapshotVolumeInfoTest func() = TestFunc(&NativeSnapshotVolumeInfo{ BackupVolumeInfo{ SnapshotVolumes: true, TestCase: TestCase{ - CaseBaseName: "native-snapshot-volumeinfo", + UseVolumeSnapshots: true, + CaseBaseName: "native-snapshot-volumeinfo", TestMsg: &TestMSG{ Desc: "Test backup's VolumeInfo metadata content for native snapshot case.", Text: "The VolumeInfo should be generated, and the NativeSnapshotInfo structure should not be nil.", @@ -54,7 +55,7 @@ func (n *NativeSnapshotVolumeInfo) Verify() error { BackupObjectsPrefix+"/"+n.BackupName, ) - Expect(err).ShouldNot(HaveOccurred(), fmt.Sprintf("Fail to get VolumeInfo metadata in the Backup Repository.")) + Expect(err).ShouldNot(HaveOccurred(), "Fail to get VolumeInfo metadata in the Backup Repository.") fmt.Printf("The VolumeInfo metadata content: %+v\n", *volumeInfo[0]) Expect(len(volumeInfo) > 0).To(BeIdenticalTo(true)) diff --git a/test/e2e/basic/namespace-mapping.go b/test/e2e/basic/namespace-mapping.go index c85fe247e0..48aa0cdd5c 100644 --- a/test/e2e/basic/namespace-mapping.go +++ b/test/e2e/basic/namespace-mapping.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "strings" - "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -86,7 +85,6 @@ func (n *NamespaceMapping) Init() error { } func (n *NamespaceMapping) CreateResources() error { - n.Ctx, n.CtxCancel = context.WithTimeout(context.Background(), 60*time.Minute) for index, ns := range *n.NSIncluded { n.kibishiiData.Levels = len(*n.NSIncluded) + index By(fmt.Sprintf("Creating namespaces ...%s\n", ns), func() { diff --git a/test/e2e/basic/nodeport.go b/test/e2e/basic/nodeport.go index 846edb9a2a..a127d0d391 100644 --- a/test/e2e/basic/nodeport.go +++ b/test/e2e/basic/nodeport.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "strings" - "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -61,9 +60,8 @@ func (n *NodePort) Init() error { } return nil } -func (n *NodePort) CreateResources() error { - n.Ctx, n.CtxCancel = context.WithTimeout(context.Background(), 60*time.Minute) +func (n *NodePort) CreateResources() error { for _, ns := range *n.NSIncluded { By(fmt.Sprintf("Creating service %s in namespaces %s ......\n", n.serviceName, ns), func() { Expect(CreateNamespace(n.Ctx, n.Client, ns)).To(Succeed(), fmt.Sprintf("Failed to create namespace %s", ns)) diff --git a/test/e2e/basic/pvc-selected-node-changing.go b/test/e2e/basic/pvc-selected-node-changing.go index e073946158..073c874df6 100644 --- a/test/e2e/basic/pvc-selected-node-changing.go +++ b/test/e2e/basic/pvc-selected-node-changing.go @@ -3,7 +3,6 @@ package basic import ( "context" "fmt" - "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -64,7 +63,6 @@ func (p *PVCSelectedNodeChanging) Init() error { } func (p *PVCSelectedNodeChanging) CreateResources() error { - p.Ctx, p.CtxCancel = context.WithTimeout(context.Background(), 60*time.Minute) By(fmt.Sprintf("Create namespace %s", p.namespace), func() { Expect(CreateNamespace(p.Ctx, p.Client, p.namespace)).To(Succeed(), fmt.Sprintf("Failed to create namespace %s", p.namespace)) diff --git a/test/e2e/basic/resources-check/namespaces.go b/test/e2e/basic/resources-check/namespaces.go index e4a88d4a77..b99154a563 100644 --- a/test/e2e/basic/resources-check/namespaces.go +++ b/test/e2e/basic/resources-check/namespaces.go @@ -84,7 +84,6 @@ func (m *MultiNSBackup) Init() error { } func (m *MultiNSBackup) CreateResources() error { - m.Ctx, m.CtxCancel = context.WithTimeout(context.Background(), m.TimeoutDuration) fmt.Printf("Creating namespaces ...\n") labels := map[string]string{ "ns-test": "true", diff --git a/test/e2e/basic/resources-check/namespaces_annotation.go b/test/e2e/basic/resources-check/namespaces_annotation.go index c8642c22c7..e698d48185 100644 --- a/test/e2e/basic/resources-check/namespaces_annotation.go +++ b/test/e2e/basic/resources-check/namespaces_annotation.go @@ -17,10 +17,8 @@ limitations under the License. package basic import ( - "context" "fmt" "strings" - "time" "github.com/pkg/errors" @@ -63,7 +61,6 @@ func (n *NSAnnotationCase) Init() error { } func (n *NSAnnotationCase) CreateResources() error { - n.Ctx, n.CtxCancel = context.WithTimeout(context.Background(), 60*time.Minute) for nsNum := 0; nsNum < n.NamespacesTotal; nsNum++ { createNSName := fmt.Sprintf("%s-%00000d", n.CaseBaseName, nsNum) createAnnotationName := fmt.Sprintf("annotation-%s-%00000d", n.CaseBaseName, nsNum) diff --git a/test/e2e/basic/resources-check/rbac.go b/test/e2e/basic/resources-check/rbac.go index 37e2ce5fbc..7afedd13c6 100644 --- a/test/e2e/basic/resources-check/rbac.go +++ b/test/e2e/basic/resources-check/rbac.go @@ -33,10 +33,8 @@ limitations under the License. package basic import ( - "context" "fmt" "strings" - "time" "github.com/pkg/errors" @@ -79,7 +77,6 @@ func (r *RBACCase) Init() error { } func (r *RBACCase) CreateResources() error { - r.Ctx, r.CtxCancel = context.WithTimeout(context.Background(), 10*time.Minute) for nsNum := 0; nsNum < r.NamespacesTotal; nsNum++ { createNSName := fmt.Sprintf("%s-%00000d", r.CaseBaseName, nsNum) fmt.Printf("Creating namespaces ...%s\n", createNSName) diff --git a/test/e2e/basic/storage-class-changing.go b/test/e2e/basic/storage-class-changing.go index fae10c9f89..8bd344700c 100644 --- a/test/e2e/basic/storage-class-changing.go +++ b/test/e2e/basic/storage-class-changing.go @@ -3,7 +3,6 @@ package basic import ( "context" "fmt" - "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -67,11 +66,11 @@ func (s *StorageClasssChanging) Init() error { } return nil } + func (s *StorageClasssChanging) CreateResources() error { label := map[string]string{ "app": "test", } - s.Ctx, s.CtxCancel = context.WithTimeout(context.Background(), 10*time.Minute) By(("Installing storage class..."), func() { Expect(InstallTestStorageClasses(fmt.Sprintf("../testdata/storage-class/%s.yaml", s.VeleroCfg.CloudProvider))).To(Succeed(), "Failed to install storage class") diff --git a/test/e2e/migration/migration.go b/test/e2e/migration/migration.go index f84a3f7a54..c57b04b469 100644 --- a/test/e2e/migration/migration.go +++ b/test/e2e/migration/migration.go @@ -27,6 +27,7 @@ import ( . "github.com/onsi/gomega" . "github.com/vmware-tanzu/velero/test" + util "github.com/vmware-tanzu/velero/test/util/csi" . "github.com/vmware-tanzu/velero/test/util/k8s" . "github.com/vmware-tanzu/velero/test/util/kibishii" . "github.com/vmware-tanzu/velero/test/util/providers" @@ -54,9 +55,11 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) var ( backupName, restoreName string backupScName, restoreScName string + kibishiiWorkerCount int err error ) BeforeEach(func() { + kibishiiWorkerCount = 3 veleroCfg = VeleroCfg UUIDgen, err = uuid.NewRandom() migrationNamespace = "migration-" + UUIDgen.String() @@ -116,10 +119,6 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) if !useVolumeSnapshots { Skip("FSB migration test is not needed in data mover scenario") } - // TODO: remove this block once Velero version in cluster A is great than V1.11 for all migration path. - if veleroCLI2Version.VeleroVersion != "self" { - Skip(fmt.Sprintf("Only V1.12 support data mover scenario instead of %s", veleroCLI2Version.VeleroVersion)) - } } oneHourTimeout, ctxCancel := context.WithTimeout(context.Background(), time.Minute*60) defer ctxCancel() @@ -128,20 +127,46 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) Expect(err).To(Succeed()) supportUploaderType, err := IsSupportUploaderType(veleroCLI2Version.VeleroVersion) Expect(err).To(Succeed()) + + OriginVeleroCfg := veleroCfg if veleroCLI2Version.VeleroCLI == "" { //Assume tag of velero server image is identical to velero CLI version //Download velero CLI if it's empty according to velero CLI version By(fmt.Sprintf("Install the expected version Velero CLI (%s) for installing Velero", veleroCLI2Version.VeleroVersion), func() { + //"self" represents 1.14.x and future versions if veleroCLI2Version.VeleroVersion == "self" { veleroCLI2Version.VeleroCLI = veleroCfg.VeleroCLI } else { + fmt.Printf("Using default images address of Velero CLI %s\n", veleroCLI2Version.VeleroVersion) + OriginVeleroCfg.VeleroImage = "" + OriginVeleroCfg.RestoreHelperImage = "" + OriginVeleroCfg.Plugins = "" + + // It is for v1.13.x migration scenario only, because since v1.14, nightly CI won't + // pass velero-plugin-for-csi to E2E test anymore, and velero installation will not + // fetch velero-plugin-for-csi automatically, so add it as hardcode below. + // TODO: remove this section from future version like v1.15, e.g. + if OriginVeleroCfg.CloudProvider == Azure { + OriginVeleroCfg.Plugins = "velero/velero-plugin-for-microsoft-azure:v1.9.0" + } + if OriginVeleroCfg.CloudProvider == AWS { + OriginVeleroCfg.Plugins = "velero/velero-plugin-for-aws:v1.9.0" + } + if strings.Contains(OriginVeleroCfg.Features, FeatureCSI) { + OriginVeleroCfg.Plugins = OriginVeleroCfg.Plugins + ",velero/velero-plugin-for-csi:v0.7.0" + } + if OriginVeleroCfg.SnapshotMoveData { + if OriginVeleroCfg.CloudProvider == Azure { + OriginVeleroCfg.Plugins = OriginVeleroCfg.Plugins + ",velero/velero-plugin-for-aws:v1.9.0" + } + } veleroCLI2Version.VeleroCLI, err = InstallVeleroCLI(veleroCLI2Version.VeleroVersion) Expect(err).To(Succeed()) } }) } - OriginVeleroCfg := veleroCfg + By(fmt.Sprintf("Install Velero in cluster-A (%s) to backup workload", veleroCfg.DefaultClusterContext), func() { Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultClusterContext)).To(Succeed()) OriginVeleroCfg.MigrateFromVeleroVersion = veleroCLI2Version.VeleroVersion @@ -156,21 +181,10 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) Expect(err).To(Succeed(), "Fail to get Velero version") OriginVeleroCfg.VeleroVersion = version - // self represents v1.12 - if veleroCLI2Version.VeleroVersion == "self" { - if OriginVeleroCfg.SnapshotMoveData { - OriginVeleroCfg.UseNodeAgent = true - } - } else { - Expect(err).To(Succeed()) - fmt.Printf("Using default images address of Velero CLI %s\n", veleroCLI2Version.VeleroVersion) - OriginVeleroCfg.VeleroImage = "" - OriginVeleroCfg.RestoreHelperImage = "" - OriginVeleroCfg.Plugins = "" - //TODO: Remove this setting when migration path is from 1.13 to higher version - //TODO: or self, because version 1.12 and older versions have no this parameter. - OriginVeleroCfg.WithoutDisableInformerCacheParam = true + if OriginVeleroCfg.SnapshotMoveData { + OriginVeleroCfg.UseNodeAgent = true } + Expect(VeleroInstall(context.Background(), &OriginVeleroCfg, false)).To(Succeed()) if veleroCLI2Version.VeleroVersion != "self" { Expect(CheckVeleroVersion(context.Background(), OriginVeleroCfg.VeleroCLI, @@ -190,10 +204,7 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) KibishiiData := *DefaultKibishiiData By("Deploy sample workload of Kibishii", func() { - if OriginVeleroCfg.SnapshotMoveData { - KibishiiData.ExpectedNodes = 3 - } - + KibishiiData.ExpectedNodes = kibishiiWorkerCount Expect(KibishiiPrepareBeforeBackup(oneHourTimeout, *veleroCfg.DefaultClient, veleroCfg.CloudProvider, migrationNamespace, veleroCfg.RegistryCredentialFile, veleroCfg.Features, veleroCfg.KibishiiDirectory, useVolumeSnapshots, &KibishiiData)).To(Succeed()) @@ -238,54 +249,50 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) // TODO - remove after upload progress monitoring is implemented By("Waiting for vSphere uploads to complete", func() { Expect(WaitForVSphereUploadCompletion(context.Background(), time.Hour, - migrationNamespace, 2)).To(Succeed()) + migrationNamespace, kibishiiWorkerCount)).To(Succeed()) }) } var snapshotCheckPoint SnapshotCheckPoint snapshotCheckPoint.NamespaceBackedUp = migrationNamespace - if !OriginVeleroCfg.SnapshotMoveData { - By("Snapshot should be created in cloud object store", func() { - snapshotCheckPoint, err := GetSnapshotCheckPoint(*veleroCfg.DefaultClient, veleroCfg, 2, - migrationNamespace, backupName, KibishiiPVCNameList) + if OriginVeleroCfg.SnapshotMoveData { + //VolumeSnapshotContent should be deleted after data movement + _, err := util.CheckVolumeSnapshotCR(*veleroCfg.DefaultClient, map[string]string{"namespace": migrationNamespace}, 0) + Expect(err).NotTo(HaveOccurred(), "VSC count is not as expected 0") + } else { + // the snapshots of AWS may be still in pending status when do the restore, wait for a while + // to avoid this https://github.com/vmware-tanzu/velero/issues/1799 + // TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed + if veleroCfg.CloudProvider == Azure && strings.EqualFold(veleroCfg.Features, FeatureCSI) || veleroCfg.CloudProvider == AWS { + By("Sleep 5 minutes to avoid snapshot recreated by unknown reason ", func() { + time.Sleep(5 * time.Minute) + }) + } + + By("Snapshot should be created in cloud object store with retain policy", func() { + snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.DefaultClient, veleroCfg, kibishiiWorkerCount, + migrationNamespace, backupName, GetKibishiiPVCNameList(kibishiiWorkerCount)) Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint") Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLConfig, backupName, snapshotCheckPoint)).To(Succeed()) }) - } else { - //TODO: checkpoint for datamover } } - if useVolumeSnapshots && veleroCfg.CloudProvider == Azure && - strings.EqualFold(veleroCfg.Features, FeatureCSI) && - !OriginVeleroCfg.SnapshotMoveData { - By("Sleep 5 minutes to avoid snapshot recreated by unknown reason ", func() { - time.Sleep(5 * time.Minute) - }) - } - // the snapshots of AWS may be still in pending status when do the restore, wait for a while - // to avoid this https://github.com/vmware-tanzu/velero/issues/1799 - // TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed - if veleroCfg.CloudProvider == Aws && useVolumeSnapshots && !OriginVeleroCfg.SnapshotMoveData { - fmt.Println("Waiting 5 minutes to make sure the snapshots are ready...") - time.Sleep(5 * time.Minute) - } - By(fmt.Sprintf("Install Velero in cluster-B (%s) to restore workload", veleroCfg.StandbyClusterContext), func() { //Ensure workload of "migrationNamespace" existed in cluster-A ns, err := GetNamespace(context.Background(), *veleroCfg.DefaultClient, migrationNamespace) Expect(ns.Name).To(Equal(migrationNamespace)) - Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("get namespace in cluster-B err: %v", err)) + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("get namespace in source cluster err: %v", err)) //Ensure cluster-B is the target cluster Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyClusterContext)).To(Succeed()) _, err = GetNamespace(context.Background(), *veleroCfg.StandbyClient, migrationNamespace) - Expect(err).To(HaveOccurred()) - strings.Contains(fmt.Sprint(err), "namespaces \""+migrationNamespace+"\" not found") + Expect(err).To(HaveOccurred(), fmt.Sprintf("get namespace in dst cluster successfully, it's not as expected: %s", migrationNamespace)) fmt.Println(err) + Expect(strings.Contains(fmt.Sprint(err), "namespaces \""+migrationNamespace+"\" not found")).Should(BeTrue()) veleroCfg.ClientToInstallVelero = veleroCfg.StandbyClient veleroCfg.ClusterToInstallVelero = veleroCfg.StandbyClusterName diff --git a/test/e2e/pv-backup/pv-backup-filter.go b/test/e2e/pv-backup/pv-backup-filter.go index a858743f95..90c5cfdea6 100644 --- a/test/e2e/pv-backup/pv-backup-filter.go +++ b/test/e2e/pv-backup/pv-backup-filter.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "strings" - "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -64,7 +63,6 @@ func (p *PVBackupFiltering) Init() error { } func (p *PVBackupFiltering) CreateResources() error { - p.Ctx, p.CtxCancel = context.WithTimeout(context.Background(), 30*time.Minute) err := InstallStorageClass(p.Ctx, fmt.Sprintf("../testdata/storage-class/%s.yaml", p.VeleroCfg.CloudProvider)) if err != nil { return errors.Wrapf(err, "failed to install storage class for pv backup filtering test") diff --git a/test/e2e/resource-filtering/base.go b/test/e2e/resource-filtering/base.go index 61be331c37..8cfce438d5 100644 --- a/test/e2e/resource-filtering/base.go +++ b/test/e2e/resource-filtering/base.go @@ -19,7 +19,6 @@ package filtering import ( "context" "fmt" - "time" "github.com/pkg/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -61,7 +60,6 @@ func (f *FilteringCase) Init() error { } func (f *FilteringCase) CreateResources() error { - f.Ctx, f.CtxCancel = context.WithTimeout(context.Background(), 30*time.Minute) for nsNum := 0; nsNum < f.NamespacesTotal; nsNum++ { namespace := fmt.Sprintf("%s-%00000d", f.CaseBaseName, nsNum) fmt.Printf("Creating resources in namespace ...%s\n", namespace) diff --git a/test/e2e/resource-filtering/exclude_label.go b/test/e2e/resource-filtering/exclude_label.go index ecf26878ef..27f7f53e6c 100644 --- a/test/e2e/resource-filtering/exclude_label.go +++ b/test/e2e/resource-filtering/exclude_label.go @@ -17,9 +17,7 @@ limitations under the License. package filtering import ( - "context" "fmt" - "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -76,7 +74,6 @@ func (e *ExcludeFromBackup) Init() error { } func (e *ExcludeFromBackup) CreateResources() error { - e.Ctx, e.CtxCancel = context.WithTimeout(context.Background(), 10*time.Minute) namespace := e.CaseBaseName // These 2 labels for resources to be included label1 := map[string]string{ diff --git a/test/e2e/resource-filtering/exclude_namespaces.go b/test/e2e/resource-filtering/exclude_namespaces.go index f4e22e852f..1b8e5da550 100644 --- a/test/e2e/resource-filtering/exclude_namespaces.go +++ b/test/e2e/resource-filtering/exclude_namespaces.go @@ -17,10 +17,8 @@ limitations under the License. package filtering import ( - "context" "fmt" "strings" - "time" "github.com/pkg/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -108,7 +106,6 @@ func (e *ExcludeNamespaces) Init() error { } func (e *ExcludeNamespaces) CreateResources() error { - e.Ctx, e.CtxCancel = context.WithTimeout(context.Background(), 10*time.Minute) for nsNum := 0; nsNum < e.NamespacesTotal; nsNum++ { createNSName := fmt.Sprintf("%s-%00000d", e.CaseBaseName, nsNum) fmt.Printf("Creating namespaces ...%s\n", createNSName) diff --git a/test/e2e/resource-filtering/include_namespaces.go b/test/e2e/resource-filtering/include_namespaces.go index d39c2acb94..d511de2123 100644 --- a/test/e2e/resource-filtering/include_namespaces.go +++ b/test/e2e/resource-filtering/include_namespaces.go @@ -17,10 +17,8 @@ limitations under the License. package filtering import ( - "context" "fmt" "strings" - "time" "github.com/pkg/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -103,7 +101,6 @@ func (i *IncludeNamespaces) Init() error { } func (i *IncludeNamespaces) CreateResources() error { - i.Ctx, i.CtxCancel = context.WithTimeout(context.Background(), 10*time.Minute) for nsNum := 0; nsNum < i.NamespacesTotal; nsNum++ { createNSName := fmt.Sprintf("%s-%00000d", i.CaseBaseName, nsNum) fmt.Printf("Creating namespaces ...%s\n", createNSName) diff --git a/test/e2e/resource-filtering/label_selector.go b/test/e2e/resource-filtering/label_selector.go index 21610ec040..1d566b5853 100644 --- a/test/e2e/resource-filtering/label_selector.go +++ b/test/e2e/resource-filtering/label_selector.go @@ -17,10 +17,8 @@ limitations under the License. package filtering import ( - "context" "fmt" "strings" - "time" "github.com/pkg/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -75,7 +73,6 @@ func (l *LabelSelector) Init() error { } func (l *LabelSelector) CreateResources() error { - l.Ctx, l.CtxCancel = context.WithTimeout(context.Background(), 10*time.Minute) for nsNum := 0; nsNum < l.NamespacesTotal; nsNum++ { namespace := fmt.Sprintf("%s-%00000d", l.CaseBaseName, nsNum) fmt.Printf("Creating resources in namespace ...%s\n", namespace) diff --git a/test/e2e/resourcemodifiers/resource_modifiers.go b/test/e2e/resourcemodifiers/resource_modifiers.go index f87965d2fa..cc1d6d3f18 100644 --- a/test/e2e/resourcemodifiers/resource_modifiers.go +++ b/test/e2e/resourcemodifiers/resource_modifiers.go @@ -17,10 +17,8 @@ limitations under the License. package resourcemodifiers import ( - "context" "fmt" "strings" - "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -97,9 +95,6 @@ func (r *ResourceModifiersCase) Init() error { } func (r *ResourceModifiersCase) CreateResources() error { - // It's better to set a global timeout in CreateResources function which is the real beginning of one e2e test - r.Ctx, r.CtxCancel = context.WithTimeout(context.Background(), 10*time.Minute) - By(fmt.Sprintf("Create configmap %s in namespaces %s for workload\n", r.cmName, r.VeleroCfg.VeleroNamespace), func() { Expect(CreateConfigMapFromYAMLData(r.Client.ClientGo, r.yamlConfig, r.cmName, r.VeleroCfg.VeleroNamespace)).To(Succeed(), fmt.Sprintf("Failed to create configmap %s in namespaces %s for workload\n", r.cmName, r.VeleroCfg.VeleroNamespace)) }) diff --git a/test/e2e/resourcepolicies/resource_policies.go b/test/e2e/resourcepolicies/resource_policies.go index c98adfdb3b..7502f8a0d5 100644 --- a/test/e2e/resourcepolicies/resource_policies.go +++ b/test/e2e/resourcepolicies/resource_policies.go @@ -17,10 +17,8 @@ limitations under the License. package filtering import ( - "context" "fmt" "strings" - "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -103,9 +101,6 @@ func (r *ResourcePoliciesCase) Init() error { } func (r *ResourcePoliciesCase) CreateResources() error { - // It's better to set a global timeout in CreateResources function which is the real beginning of one e2e test - r.Ctx, r.CtxCancel = context.WithTimeout(context.Background(), 10*time.Minute) - By(("Installing storage class..."), func() { Expect(InstallTestStorageClasses(fmt.Sprintf("../testdata/storage-class/%s.yaml", r.VeleroCfg.CloudProvider))).To(Succeed(), "Failed to install storage class") }) diff --git a/test/e2e/schedule/ordered_resources.go b/test/e2e/schedule/ordered_resources.go index 4724c7d7c7..8ae7fb56d3 100644 --- a/test/e2e/schedule/ordered_resources.go +++ b/test/e2e/schedule/ordered_resources.go @@ -18,7 +18,6 @@ limitations under the License. //the ordered resources test related to https://github.com/vmware-tanzu/velero/issues/4561 import ( - "context" "fmt" "strings" "time" @@ -70,9 +69,7 @@ func (o *OrderedResources) Init() error { return nil } - func (o *OrderedResources) CreateResources() error { - o.Ctx, o.CtxCancel = context.WithTimeout(context.Background(), 10*time.Minute) label := map[string]string{ "orderedresources": "true", } diff --git a/test/e2e/schedule/schedule-backup-creation.go b/test/e2e/schedule/schedule-backup-creation.go index 437afde75f..b87169df66 100644 --- a/test/e2e/schedule/schedule-backup-creation.go +++ b/test/e2e/schedule/schedule-backup-creation.go @@ -63,7 +63,6 @@ func (n *ScheduleBackupCreation) Init() error { } func (p *ScheduleBackupCreation) CreateResources() error { - p.Ctx, p.CtxCancel = context.WithTimeout(context.Background(), 60*time.Minute) By(fmt.Sprintf("Create namespace %s", p.namespace), func() { Expect(CreateNamespace(p.Ctx, p.Client, p.namespace)).To(Succeed(), fmt.Sprintf("Failed to create namespace %s", p.namespace)) diff --git a/test/e2e/schedule/schedule.go b/test/e2e/schedule/schedule.go index 959000c29a..99e4c01ddb 100644 --- a/test/e2e/schedule/schedule.go +++ b/test/e2e/schedule/schedule.go @@ -47,9 +47,7 @@ func (n *ScheduleBackup) Init() error { Expect(n.Period).To(BeNumerically("<", 30)) return nil } - func (n *ScheduleBackup) CreateResources() error { - n.Ctx, n.CtxCancel = context.WithTimeout(context.Background(), 60*time.Minute) for _, ns := range *n.NSIncluded { By(fmt.Sprintf("Creating namespaces %s ......\n", ns), func() { Expect(CreateNamespace(n.Ctx, n.Client, ns)).To(Succeed(), fmt.Sprintf("Failed to create namespace %s", ns)) diff --git a/test/e2e/test/test.go b/test/e2e/test/test.go index 89a8fa71d1..9f64f3b1f4 100644 --- a/test/e2e/test/test.go +++ b/test/e2e/test/test.go @@ -18,7 +18,6 @@ package test import ( "context" - "flag" "fmt" "math/rand" "strings" @@ -48,6 +47,7 @@ type VeleroBackupRestoreTest interface { Restore() error Verify() error Clean() error + Start() error GetTestMsg() *TestMSG GetTestCase() *TestCase } @@ -78,52 +78,26 @@ type TestCase struct { func TestFunc(test VeleroBackupRestoreTest) func() { return func() { - Expect(test.Init()).To(Succeed(), "Failed to instantiate test cases") - BeforeEach(func() { - flag.Parse() - // Using the global velero config which covered the installation for most common cases - veleroCfg := test.GetTestCase().VeleroCfg - // TODO: Skip nodeport test until issue https://github.com/kubernetes/kubernetes/issues/114384 fixed - // TODO: Although this issue is closed, but it's not fixed. - // TODO: After bump up k8s version in AWS pipeline, this issue also apply for AWS pipeline. - if (veleroCfg.CloudProvider == Azure || veleroCfg.CloudProvider == Aws) && strings.Contains(test.GetTestCase().CaseBaseName, "nodeport") { - Skip("Skip due to issue https://github.com/kubernetes/kubernetes/issues/114384 on AKS") - } - if InstallVelero { - Expect(PrepareVelero(context.Background(), test.GetTestCase().CaseBaseName, veleroCfg)).To(Succeed()) - } - }) - It(test.GetTestMsg().Text, func() { - Expect(RunTestCase(test)).To(Succeed(), test.GetTestMsg().FailedMSG) - }) + TestIt(test) } } func TestFuncWithMultiIt(tests []VeleroBackupRestoreTest) func() { return func() { for k := range tests { - Expect(tests[k].Init()).To(Succeed(), fmt.Sprintf("Failed to instantiate test %s case", tests[k].GetTestMsg().Desc)) - defer tests[k].GetTestCase().CtxCancel() - } - - BeforeEach(func() { - flag.Parse() - if InstallVelero { - Expect(PrepareVelero(context.Background(), tests[0].GetTestCase().CaseBaseName, tests[0].GetTestCase().VeleroCfg)).To(Succeed()) - } - }) - - for k := range tests { - curTest := tests[k] - It(curTest.GetTestMsg().Text, func() { - Expect(RunTestCase(curTest)).To(Succeed(), curTest.GetTestMsg().FailedMSG) - }) + TestIt(tests[k]) } } } +func TestIt(test VeleroBackupRestoreTest) error { + test.Init() + It(test.GetTestMsg().Text, func() { + Expect(RunTestCase(test)).To(Succeed(), test.GetTestMsg().FailedMSG) + }) + return nil +} func (t *TestCase) Init() error { - t.Ctx, t.CtxCancel = context.WithTimeout(context.Background(), 1*time.Hour) t.UUIDgen = t.GenerateUUID() t.VeleroCfg = VeleroCfg t.Client = *t.VeleroCfg.ClientToInstallVelero @@ -141,18 +115,13 @@ func (t *TestCase) CreateResources() error { func (t *TestCase) Backup() error { veleroCfg := t.GetTestCase().VeleroCfg - if err := VeleroBackupExec(t.Ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, t.BackupName, t.BackupArgs); err != nil { - RunDebug(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, t.BackupName, "") - return errors.Wrapf(err, "Failed to backup resources") - } - // the snapshots of AWS may be still in pending status when do the restore, wait for a while - // to avoid this https://github.com/vmware-tanzu/velero/issues/1799 - // TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed - if t.UseVolumeSnapshots { - fmt.Println("Waiting 5 minutes to make sure the snapshots are ready...") - time.Sleep(5 * time.Minute) - } + By("Start to backup ......", func() { + Expect(VeleroBackupExec(t.Ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, t.BackupName, t.BackupArgs)).To(Succeed(), func() string { + RunDebug(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, t.BackupName, "") + return "Failed to backup resources" + }) + }) return nil } @@ -170,13 +139,15 @@ func (t *TestCase) Restore() error { } veleroCfg := t.GetTestCase().VeleroCfg + // the snapshots of AWS may be still in pending status when do the restore, wait for a while // to avoid this https://github.com/vmware-tanzu/velero/issues/1799 // TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed - if t.UseVolumeSnapshots && veleroCfg.CloudProvider != Vsphere { - fmt.Println("Waiting 5 minutes to make sure the snapshots are ready...") - time.Sleep(5 * time.Minute) - } + By("Waiting 5 minutes to make sure the snapshots are ready...", func() { + if t.UseVolumeSnapshots && veleroCfg.CloudProvider != Vsphere { + time.Sleep(5 * time.Minute) + } + }) By("Start to restore ......", func() { if t.RestorePhaseExpect == "" { @@ -194,6 +165,15 @@ func (t *TestCase) Verify() error { return nil } +func (t *TestCase) Start() error { + t.Ctx, t.CtxCancel = context.WithTimeout(context.Background(), 1*time.Hour) + veleroCfg := t.GetTestCase().VeleroCfg + if (veleroCfg.CloudProvider == Azure || veleroCfg.CloudProvider == AWS) && strings.Contains(t.GetTestCase().CaseBaseName, "nodeport") { + Skip("Skip due to issue https://github.com/kubernetes/kubernetes/issues/114384 on AKS") + } + return nil +} + func (t *TestCase) Clean() error { veleroCfg := t.GetTestCase().VeleroCfg if !veleroCfg.Debug { @@ -217,10 +197,17 @@ func (t *TestCase) GetTestCase() *TestCase { } func RunTestCase(test VeleroBackupRestoreTest) error { - fmt.Printf("Running test case %s %s\n", test.GetTestMsg().Desc, time.Now().Format("2006-01-02 15:04:05")) if test == nil { return errors.New("No case should be tested") } + test.Start() + defer test.GetTestCase().CtxCancel() + + fmt.Printf("Running test case %s %s\n", test.GetTestMsg().Desc, time.Now().Format("2006-01-02 15:04:05")) + + if InstallVelero { + Expect(PrepareVelero(context.Background(), test.GetTestCase().CaseBaseName, test.GetTestCase().VeleroCfg)).To(Succeed()) + } defer test.Clean() diff --git a/test/e2e/upgrade/upgrade.go b/test/e2e/upgrade/upgrade.go index 149ae76253..ef9e31c217 100644 --- a/test/e2e/upgrade/upgrade.go +++ b/test/e2e/upgrade/upgrade.go @@ -33,10 +33,7 @@ import ( . "github.com/vmware-tanzu/velero/test/util/velero" ) -const ( - upgradeNamespace = "upgrade-workload" -) - +var upgradeNamespace string var veleroCfg VeleroConfig func BackupUpgradeRestoreWithSnapshots() { @@ -62,6 +59,8 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC BeforeEach(func() { veleroCfg = VeleroCfg veleroCfg.IsUpgradeTest = true + UUIDgen, err = uuid.NewRandom() + upgradeNamespace = "upgrade-" + UUIDgen.String() if !InstallVelero { Skip("Upgrade test should not be triggered if veleroCfg.InstallVelero is set to false") } @@ -223,7 +222,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC // the snapshots of AWS may be still in pending status when do the restore, wait for a while // to avoid this https://github.com/vmware-tanzu/velero/issues/1799 // TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed - if tmpCfg.CloudProvider == Aws && useVolumeSnapshots { + if tmpCfg.CloudProvider == AWS && useVolumeSnapshots { fmt.Println("Waiting 5 minutes to make sure the snapshots are ready...") time.Sleep(5 * time.Minute) } diff --git a/test/types.go b/test/types.go index e64c6d677a..6ba28b549b 100644 --- a/test/types.go +++ b/test/types.go @@ -34,11 +34,11 @@ const Kind = "kind" const Azure = "azure" const AzureCSI = "azure-csi" const AwsCSI = "aws-csi" -const Aws = "aws" +const AWS = "aws" const Gcp = "gcp" const Vsphere = "vsphere" -var PublicCloudProviders = []string{Aws, Azure, Gcp, Vsphere} +var PublicCloudProviders = []string{AWS, Azure, Gcp, Vsphere} var LocalCloudProviders = []string{Kind, VanillaZFS} var CloudProviders = append(PublicCloudProviders, LocalCloudProviders...) diff --git a/test/util/csi/common.go b/test/util/csi/common.go index e336336ffc..9c10e962d9 100644 --- a/test/util/csi/common.go +++ b/test/util/csi/common.go @@ -21,6 +21,7 @@ import ( "fmt" "strings" + volumeSnapshotV1 "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" snapshotterClientSet "github.com/kubernetes-csi/external-snapshotter/client/v7/clientset/versioned" "github.com/pkg/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -52,77 +53,47 @@ func GetClients() (*kubernetes.Clientset, *snapshotterClientSet.Clientset, error return client, snapshotterClient, nil } -func GetCsiSnapshotHandle(client TestClient, backupName string) ([]string, error) { +func GetCsiSnapshotHandle(client TestClient, apiVersion string, index map[string]string) ([]string, error) { _, snapshotClient, err := GetClients() + + var vscList *volumeSnapshotV1.VolumeSnapshotContentList if err != nil { return nil, err } - vscList, err1 := snapshotClient.SnapshotV1().VolumeSnapshotContents().List(context.TODO(), metav1.ListOptions{}) - if err1 != nil { - return nil, err + switch apiVersion { + // case "v1beta1": + // vscList, err = snapshotterClientSetV1beta1.SnapshotV1beta1().VolumeSnapshotContents().List(context.TODO(), metav1.ListOptions{}) + case "v1": + vscList, err = snapshotClient.SnapshotV1().VolumeSnapshotContents().List(context.TODO(), metav1.ListOptions{}) + default: + errors.New(fmt.Sprintf("API version %s is not valid", apiVersion)) } - var snapshotHandleList []string - for _, i := range vscList.Items { - if i.Status == nil { - fmt.Println("SnapshotHandle Status s nil") - continue - } - if i.Status.SnapshotHandle == nil { - fmt.Println("SnapshotHandle is nil") - continue - } - - if i.Labels == nil { - fmt.Println("VolumeSnapshotContents label is nil") - continue - } - - if i.Labels["velero.io/backup-name"] == backupName { - tmp := strings.Split(*i.Status.SnapshotHandle, "/") - snapshotHandleList = append(snapshotHandleList, tmp[len(tmp)-1]) - } - } - - if len(snapshotHandleList) == 0 { - fmt.Printf("No VolumeSnapshotContent from backup %s\n", backupName) - } - return snapshotHandleList, nil -} -func GetCsiSnapshotHandleV1(client TestClient, backupName string) ([]string, error) { - _, snapshotClient, err := GetClients() if err != nil { return nil, err } - vscList, err1 := snapshotClient.SnapshotV1().VolumeSnapshotContents().List(context.TODO(), metav1.ListOptions{}) - if err1 != nil { - return nil, err - } var snapshotHandleList []string for _, i := range vscList.Items { if i.Status == nil { - fmt.Println("SnapshotHandle Status s nil") + fmt.Println("VolumeSnapshotContent status is nil") continue } if i.Status.SnapshotHandle == nil { fmt.Println("SnapshotHandle is nil") continue } - - if i.Labels == nil { - fmt.Println("VolumeSnapshotContents label is nil") - continue - } - - if i.Labels["velero.io/backup-name"] == backupName { + if (index["backupNameLabel"] != "" && i.Labels != nil && i.Labels["velero.io/backup-name"] == index["backupNameLabel"]) || + (index["namespace"] != "" && i.Spec.VolumeSnapshotRef.Namespace == index["namespace"]) { tmp := strings.Split(*i.Status.SnapshotHandle, "/") snapshotHandleList = append(snapshotHandleList, tmp[len(tmp)-1]) } } if len(snapshotHandleList) == 0 { - fmt.Printf("No VolumeSnapshotContent from backup %s\n", backupName) + fmt.Printf("No VolumeSnapshotContent from key %v\n", index) + } else { + fmt.Printf("Volume snapshot content list: %v\n", snapshotHandleList) } return snapshotHandleList, nil } @@ -164,22 +135,35 @@ func GetVolumeSnapshotContentNameByPod(client TestClient, podName, namespace, ba return "", errors.New(fmt.Sprintf("Fail to get VolumeSnapshotContentName for pod %s under namespace %s", podName, namespace)) } -func CheckVolumeSnapshotCR(client TestClient, backupName string, expectedCount int, apiVersion string) ([]string, error) { +func CheckVolumeSnapshotCR(client TestClient, index map[string]string, expectedCount int) ([]string, error) { var err error var snapshotContentNameList []string - if apiVersion == "v1beta1" { - if snapshotContentNameList, err = GetCsiSnapshotHandle(client, backupName); err != nil { - return nil, errors.Wrap(err, "Fail to get Azure CSI snapshot content") - } - } else if apiVersion == "v1" { - if snapshotContentNameList, err = GetCsiSnapshotHandleV1(client, backupName); err != nil { + + resourceName := "snapshot.storage.k8s.io" + + apiVersion, err := GetAPIVersions(&client, resourceName) + if err != nil { + return nil, err + } + if len(apiVersion) == 0 { + return nil, errors.New("Fail to get APIVersion") + } + // if apiVersion[0] == "v1beta1" { + // if snapshotContentNameList, err = GetCsiSnapshotHandle(client, apiVersion[0], index); err != nil { + // return nil, errors.Wrap(err, "Fail to get Azure CSI snapshot content") + // } + // } else + if apiVersion[0] == "v1" { + if snapshotContentNameList, err = GetCsiSnapshotHandle(client, apiVersion[0], index); err != nil { return nil, errors.Wrap(err, "Fail to get Azure CSI snapshot content") } } else { return nil, errors.New("API version is invalid") } - if len(snapshotContentNameList) != expectedCount { - return nil, errors.New(fmt.Sprintf("Snapshot content count %d is not as expect %d", len(snapshotContentNameList), expectedCount)) + if expectedCount >= 0 { + if len(snapshotContentNameList) != expectedCount { + return nil, errors.New(fmt.Sprintf("Snapshot content count %d is not as expect %d", len(snapshotContentNameList), expectedCount)) + } } fmt.Printf("snapshotContentNameList: %v \n", snapshotContentNameList) return snapshotContentNameList, nil diff --git a/test/util/k8s/common.go b/test/util/k8s/common.go index 097e3be2fd..9335815629 100644 --- a/test/util/k8s/common.go +++ b/test/util/k8s/common.go @@ -250,7 +250,6 @@ func GetAPIVersions(client *TestClient, name string) ([]string, error) { return nil, errors.Wrap(err, "Fail to get server API groups") } for _, group := range APIGroup.Groups { - fmt.Println(group.Name) if group.Name == name { for _, v := range group.Versions { fmt.Println(v.Version) @@ -259,7 +258,7 @@ func GetAPIVersions(client *TestClient, name string) ([]string, error) { return version, nil } } - return nil, errors.New("Server API groups is empty") + return nil, errors.New("Fail to get server API groups") } func GetPVByPVCName(client TestClient, namespace, pvcName string) (string, error) { diff --git a/test/util/k8s/namespace.go b/test/util/k8s/namespace.go index e747a4d152..b2a5cae0e6 100644 --- a/test/util/k8s/namespace.go +++ b/test/util/k8s/namespace.go @@ -42,7 +42,7 @@ func CreateNamespace(ctx context.Context, client TestClient, namespace string) e "pod-security.kubernetes.io/enforce": "baseline", "pod-security.kubernetes.io/enforce-version": "latest", } - _, err := client.ClientGo.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) + _, err := client.ClientGo.CoreV1().Namespaces().Create(context.TODO(), ns, metav1.CreateOptions{}) if apierrors.IsAlreadyExists(err) { return nil } @@ -70,7 +70,7 @@ func CreateNamespaceWithAnnotation(ctx context.Context, client TestClient, names "pod-security.kubernetes.io/enforce-version": "latest", } ns.ObjectMeta.Annotations = annotation - _, err := client.ClientGo.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) + _, err := client.ClientGo.CoreV1().Namespaces().Create(context.TODO(), ns, metav1.CreateOptions{}) if apierrors.IsAlreadyExists(err) { return nil } @@ -169,6 +169,9 @@ func CleanupNamespacesFiterdByExcludes(ctx context.Context, client TestClient, e } func CleanupNamespaces(ctx context.Context, client TestClient, CaseBaseName string) error { + if ctx == nil { + fmt.Println("ctx is nil ....") + } namespaces, err := client.ClientGo.CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) if err != nil { return errors.Wrap(err, "Could not retrieve namespaces") diff --git a/test/util/k8s/rbac.go b/test/util/k8s/rbac.go index c6fa2559c6..634fdf84f1 100644 --- a/test/util/k8s/rbac.go +++ b/test/util/k8s/rbac.go @@ -34,7 +34,7 @@ func CreateRBACWithBindingSA(ctx context.Context, client TestClient, namespace s }, } - _, err = client.ClientGo.RbacV1().ClusterRoles().Create(ctx, role, metav1.CreateOptions{}) + _, err = client.ClientGo.RbacV1().ClusterRoles().Create(context.TODO(), role, metav1.CreateOptions{}) if err != nil && !apierrors.IsAlreadyExists(err) { return err diff --git a/test/util/k8s/serviceaccount.go b/test/util/k8s/serviceaccount.go index 24d748aeaf..1af56a798a 100644 --- a/test/util/k8s/serviceaccount.go +++ b/test/util/k8s/serviceaccount.go @@ -23,14 +23,13 @@ import ( "time" "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "github.com/vmware-tanzu/velero/pkg/builder" - - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func WaitUntilServiceAccountCreated(ctx context.Context, client TestClient, namespace, serviceAccount string, timeout time.Duration) error { @@ -81,7 +80,7 @@ func CreateServiceAccount(ctx context.Context, client TestClient, namespace stri AutomountServiceAccountToken: nil, } - _, err = client.ClientGo.CoreV1().ServiceAccounts(namespace).Create(ctx, sa, metav1.CreateOptions{}) + _, err = client.ClientGo.CoreV1().ServiceAccounts(namespace).Create(context.TODO(), sa, metav1.CreateOptions{}) if err != nil && !apierrors.IsAlreadyExists(err) { return err @@ -90,5 +89,5 @@ func CreateServiceAccount(ctx context.Context, client TestClient, namespace stri } func GetServiceAccount(ctx context.Context, client TestClient, namespace string, serviceAccount string) (*corev1.ServiceAccount, error) { - return client.ClientGo.CoreV1().ServiceAccounts(namespace).Get(ctx, serviceAccount, metav1.GetOptions{}) + return client.ClientGo.CoreV1().ServiceAccounts(namespace).Get(context.TODO(), serviceAccount, metav1.GetOptions{}) } diff --git a/test/util/kibishii/kibishii_utils.go b/test/util/kibishii/kibishii_utils.go index 5046f4f21c..2fe223ea96 100644 --- a/test/util/kibishii/kibishii_utils.go +++ b/test/util/kibishii/kibishii_utils.go @@ -55,6 +55,14 @@ var KibishiiPodNameList = []string{"kibishii-deployment-0", "kibishii-deployment var KibishiiPVCNameList = []string{"kibishii-data-kibishii-deployment-0", "kibishii-data-kibishii-deployment-1"} var KibishiiStorageClassName = "kibishii-storage-class" +func GetKibishiiPVCNameList(workerCount int) []string { + var kibishiiPVCNameList []string + for i := 0; i < workerCount; i++ { + kibishiiPVCNameList = append(kibishiiPVCNameList, fmt.Sprintf("kibishii-data-kibishii-deployment-%d", i)) + } + return kibishiiPVCNameList +} + // RunKibishiiTests runs kibishii tests on the provider. func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLocation, kibishiiNamespace string, useVolumeSnapshots, defaultVolumesToFsBackup bool) error { @@ -211,10 +219,10 @@ func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLoc } } - // the snapshots of AWS may be still in pending status when do the restore, wait for a while - // to avoid this https://github.com/vmware-tanzu/velero/issues/1799 - // TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed if useVolumeSnapshots { + // the snapshots of AWS may be still in pending status when do the restore, wait for a while + // to avoid this https://github.com/vmware-tanzu/velero/issues/1799 + // TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed fmt.Println("Waiting 5 minutes to make sure the snapshots are ready...") time.Sleep(5 * time.Minute) } @@ -248,7 +256,7 @@ func installKibishii(ctx context.Context, namespace string, cloudPlatform, veler strings.EqualFold(veleroFeatures, FeatureCSI) { cloudPlatform = AzureCSI } - if strings.EqualFold(cloudPlatform, Aws) && + if strings.EqualFold(cloudPlatform, AWS) && strings.EqualFold(veleroFeatures, FeatureCSI) { cloudPlatform = AwsCSI } diff --git a/test/util/providers/common.go b/test/util/providers/common.go index 1b39f22712..99441d268e 100644 --- a/test/util/providers/common.go +++ b/test/util/providers/common.go @@ -75,7 +75,7 @@ func ObjectsShouldNotBeInBucket(objectStoreProvider, cloudCredentialsFile, bslBu func getProvider(cloudProvider string) (ObjectsInStorage, error) { var s ObjectsInStorage switch cloudProvider { - case Aws, Vsphere: + case AWS, Vsphere: aws := AWSStorage("") s = &aws case Gcp: diff --git a/test/util/velero/install.go b/test/util/velero/install.go index 09fc7f3f77..48e2faf1d9 100644 --- a/test/util/velero/install.go +++ b/test/util/velero/install.go @@ -104,7 +104,7 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, isStandbyCluste // backup, but needed to pick up the provider plugins earlier. vSphere plugin no longer needs a Volume // Snapshot location specified if veleroCfg.ObjectStoreProvider == "" { - veleroCfg.ObjectStoreProvider = Aws + veleroCfg.ObjectStoreProvider = AWS } if err := configvSpherePlugin(veleroCfg); err != nil { return errors.WithMessagef(err, "Failed to config vsphere plugin") @@ -118,7 +118,7 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, isStandbyCluste // For AWS IRSA credential test, AWS IAM service account is required, so if ServiceAccountName and EKSPolicyARN // are both provided, we assume IRSA test is running, otherwise skip this IAM service account creation part. - if veleroCfg.CloudProvider == Aws && veleroInstallOptions.ServiceAccountName != "" { + if veleroCfg.CloudProvider == AWS && veleroInstallOptions.ServiceAccountName != "" { if veleroCfg.EKSPolicyARN == "" { return errors.New("Please provide EKSPolicyARN for IRSA test.") } @@ -155,6 +155,7 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, isStandbyCluste }) if err != nil { + time.Sleep(9 * time.Hour) RunDebug(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, "", "") return errors.WithMessagef(err, "Failed to install Velero in the cluster") } diff --git a/test/util/velero/velero_utils.go b/test/util/velero/velero_utils.go index 21f895fddf..ac55a7c776 100644 --- a/test/util/velero/velero_utils.go +++ b/test/util/velero/velero_utils.go @@ -98,11 +98,12 @@ var pluginsMatrix = map[string]map[string][]string{ "csi": {"velero/velero-plugin-for-csi:v0.6.0"}, }, "v1.13": { - "aws": {"velero/velero-plugin-for-aws:v1.9.0"}, - "azure": {"velero/velero-plugin-for-microsoft-azure:v1.9.0"}, - "vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.5.2"}, - "gcp": {"velero/velero-plugin-for-gcp:v1.9.0"}, - "csi": {"velero/velero-plugin-for-csi:v0.7.0"}, + "aws": {"velero/velero-plugin-for-aws:v1.9.0"}, + "azure": {"velero/velero-plugin-for-microsoft-azure:v1.9.0"}, + "vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.5.2"}, + "gcp": {"velero/velero-plugin-for-gcp:v1.9.0"}, + "csi": {"velero/velero-plugin-for-csi:v0.7.0"}, + "datamover": {"velero/velero-plugin-for-aws:v1.9.0"}, }, "main": { "aws": {"velero/velero-plugin-for-aws:main"}, @@ -129,7 +130,7 @@ func getPluginsByVersion(version, cloudProvider, objectStoreProvider string, nee var ok bool if slices.Contains(LocalCloudProviders, cloudProvider) { - plugins, ok = cloudMap[Aws] + plugins, ok = cloudMap[AWS] if !ok { return nil, errors.Errorf("fail to get plugins by version: %s and provider %s", version, cloudProvider) } @@ -1231,24 +1232,16 @@ func GetRepositories(ctx context.Context, veleroNamespace, targetNamespace strin } func GetSnapshotCheckPoint(client TestClient, veleroCfg VeleroConfig, expectCount int, namespaceBackedUp, backupName string, KibishiiPVCNameList []string) (SnapshotCheckPoint, error) { + var err error var snapshotCheckPoint SnapshotCheckPoint snapshotCheckPoint.ExpectCount = expectCount snapshotCheckPoint.NamespaceBackedUp = namespaceBackedUp snapshotCheckPoint.PodName = KibishiiPVCNameList - if (veleroCfg.CloudProvider == Azure || veleroCfg.CloudProvider == Aws) && strings.EqualFold(veleroCfg.Features, FeatureCSI) { + if (veleroCfg.CloudProvider == Azure || veleroCfg.CloudProvider == AWS) && strings.EqualFold(veleroCfg.Features, FeatureCSI) { snapshotCheckPoint.EnableCSI = true - resourceName := "snapshot.storage.k8s.io" - - srcVersions, err := GetAPIVersions(veleroCfg.DefaultClient, resourceName) - if err != nil { - return snapshotCheckPoint, err - } - if len(srcVersions) == 0 { - return snapshotCheckPoint, errors.New("Fail to get APIVersion") - } - if snapshotCheckPoint.SnapshotIDList, err = util.CheckVolumeSnapshotCR(client, backupName, expectCount, srcVersions[0]); err != nil { + if snapshotCheckPoint.SnapshotIDList, err = util.CheckVolumeSnapshotCR(client, map[string]string{"backupNameLabel": backupName}, expectCount); err != nil { return snapshotCheckPoint, errors.Wrapf(err, "Fail to get Azure CSI snapshot content") } } From 4d48273a24969b748b6c2c68f376928053188102 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 30 Apr 2024 09:43:16 -0400 Subject: [PATCH 13/13] Bump golangci/golangci-lint-action from 4 to 5 (#7756) Bumps [golangci/golangci-lint-action](https://github.com/golangci/golangci-lint-action) from 4 to 5. - [Release notes](https://github.com/golangci/golangci-lint-action/releases) - [Commits](https://github.com/golangci/golangci-lint-action/compare/v4...v5) --- updated-dependencies: - dependency-name: golangci/golangci-lint-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/pr-linter-check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-linter-check.yml b/.github/workflows/pr-linter-check.yml index 36badcaa14..077753c73e 100644 --- a/.github/workflows/pr-linter-check.yml +++ b/.github/workflows/pr-linter-check.yml @@ -16,7 +16,7 @@ jobs: uses: actions/checkout@v4 - name: golangci-lint - uses: golangci/golangci-lint-action@v4 + uses: golangci/golangci-lint-action@v5 with: version: v1.57.2 args: --out-format=colored-line-number