Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add CA rotation TM test #285

Merged
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions .test-defs/TestSuiteRegistryCacheDisruptiveSerial.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
apiVersion: testmachinery.sapcloud.io
kind: TestDefinition
metadata:
name: registry-cache-beta-disruptive-test-suite
spec:
owner: [email protected]
description: registry-cache extension test suite that includes all disruptive beta tests

activeDeadlineSeconds: 9000
labels: ["shoot", "beta"]
behavior:
- disruptive

command: [bash, -c]
args:
- >-
go test -timeout=0 ./test/testmachinery/shoot
--v -ginkgo.v -ginkgo.show-node-events -ginkgo.no-color
-kubecfg=$TM_KUBECONFIG_PATH/gardener.config
-project-namespace=$PROJECT_NAMESPACE
-shoot-name=$SHOOT_NAME
-ginkgo.focus="\[BETA\].*\[DISRUPTIVE\]"
-ginkgo.skip="\[SERIAL\]"
ialidzhikov marked this conversation as resolved.
Show resolved Hide resolved
-ginkgo.timeout=9000s

image: golang:1.23.4
6 changes: 6 additions & 0 deletions test/common/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,12 @@ const (
// - gcr.io, registry.k8s.io, quay.io, europe-docker.pkg.dev: These are all registries used in the Gardener's local setup. Avoid using them to do not have conflicts with provider-local in some corner cases.
// - Amazon ECR: The Distribution project does not support image pulls from Amazon ECR. Ref https://github.com/distribution/distribution/issues/4383.

// GithubRegistryJitesoftAlpine31511Image is the ghcr.io/jitesoft/alpine:3.15.11 image.
GithubRegistryJitesoftAlpine31511Image = "ghcr.io/jitesoft/alpine:3.15.11"
// GithubRegistryJitesoftAlpine3169Image is the ghcr.io/jitesoft/alpine:3.16.9 image.
GithubRegistryJitesoftAlpine3169Image = "ghcr.io/jitesoft/alpine:3.16.9"
// GithubRegistryJitesoftAlpine3179Image is the ghcr.io/jitesoft/alpine:3.17.9 image.
GithubRegistryJitesoftAlpine3179Image = "ghcr.io/jitesoft/alpine:3.17.9"
// GithubRegistryJitesoftAlpine3189Image is the ghcr.io/jitesoft/alpine:3.18.9 image.
GithubRegistryJitesoftAlpine3189Image = "ghcr.io/jitesoft/alpine:3.18.9"
// GithubRegistryJitesoftAlpine3194Image is the ghcr.io/jitesoft/alpine:3.19.4 image.
Expand Down
107 changes: 107 additions & 0 deletions test/testmachinery/shoot/enable_rotate_certs_disable_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors
ialidzhikov marked this conversation as resolved.
Show resolved Hide resolved
//
// SPDX-License-Identifier: Apache-2.0

package shoot

import (
"context"
"time"

gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1"
"github.com/gardener/gardener/pkg/apis/core/v1beta1/constants"
dimitar-kostadinov marked this conversation as resolved.
Show resolved Hide resolved
"github.com/gardener/gardener/test/framework"
"github.com/gardener/gardener/test/utils/access"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"k8s.io/apimachinery/pkg/api/resource"

"github.com/gardener/gardener-extension-registry-cache/pkg/apis/registry/v1alpha3"
"github.com/gardener/gardener-extension-registry-cache/test/common"
)

const (
rotateCertTestTimeout = 60 * time.Minute
rotateCertTestCleanupTimeout = 20 * time.Minute
ialidzhikov marked this conversation as resolved.
Show resolved Hide resolved
)

var _ = Describe("Registry Cache Extension Tests", Label("cache"), func() {
f := framework.NewShootFramework(nil)

f.Disruptive().Beta().CIt("should enable extension, rotate Certificate, disable extension", func(parentCtx context.Context) {
ialidzhikov marked this conversation as resolved.
Show resolved Hide resolved
ialidzhikov marked this conversation as resolved.
Show resolved Hide resolved
By("Enable the registry-cache extension")
ctx, cancel := context.WithTimeout(parentCtx, 10*time.Minute)
defer cancel()
Expect(f.UpdateShoot(ctx, func(shoot *gardencorev1beta1.Shoot) error {
size := resource.MustParse("2Gi")
if shoot.Spec.Provider.Type == "alicloud" {
// On AliCloud the minimum size for SSD volumes is 20Gi.
size = resource.MustParse("20Gi")
}

common.AddOrUpdateRegistryCacheExtension(shoot, []v1alpha3.RegistryCache{
{Upstream: "ghcr.io", Volume: &v1alpha3.Volume{Size: &size}},
})

return nil
})).To(Succeed())

By("Verify registry-cache works")
// Images ghcr.io/jitesoft/alpine:3.18.9, ghcr.io/jitesoft/alpine:3.19.4 and ghcr.io/jitesoft/alpine:3.20.3 are already used by the
// "should enable and disable the registry-cache extension" and "should enable extension, hibernate Shoot, reconcile Shoot, wake up Shoot, disable extension"
// tests and may present in the Node.
ialidzhikov marked this conversation as resolved.
Show resolved Hide resolved
// So the current test will use ghcr.io/jitesoft/alpine:3.15.11 ghcr.io/jitesoft/alpine:3.16.9 and ghcr.io/jitesoft/alpine:3.17.9 images.
ialidzhikov marked this conversation as resolved.
Show resolved Hide resolved
common.VerifyRegistryCache(parentCtx, f.Logger, f.ShootClient, common.GithubRegistryJitesoftAlpine31511Image, common.SleepInfinity)

By("Rotate certificates phase Preparing")
ialidzhikov marked this conversation as resolved.
Show resolved Hide resolved
ctx, cancel = context.WithTimeout(parentCtx, 25*time.Minute)
defer cancel()
Expect(f.AnnotateShoot(ctx, f.Shoot, map[string]string{constants.GardenerOperation: constants.OperationRotateCAStart})).To(Succeed())
Expect(f.WaitForShootToBeReconciled(ctx, f.Shoot)).To(Succeed())
Expect(f.Shoot.Status.Credentials.Rotation.CertificateAuthorities.Phase).To(Equal(gardencorev1beta1.RotationPrepared))
ialidzhikov marked this conversation as resolved.
Show resolved Hide resolved

By("Verify registry-cache works when phase is Prepared")
ialidzhikov marked this conversation as resolved.
Show resolved Hide resolved
common.VerifyRegistryCache(parentCtx, f.Logger, f.ShootClient, common.GithubRegistryJitesoftAlpine3169Image, common.SleepInfinity)

By("Rotate certificates phase Completing")
ialidzhikov marked this conversation as resolved.
Show resolved Hide resolved
ctx, cancel = context.WithTimeout(parentCtx, 10*time.Minute)
defer cancel()
Expect(f.AnnotateShoot(ctx, f.Shoot, map[string]string{constants.GardenerOperation: constants.OperationRotateCAComplete})).To(Succeed())
Expect(f.WaitForShootToBeReconciled(ctx, f.Shoot)).To(Succeed())
Expect(f.Shoot.Status.Credentials.Rotation.CertificateAuthorities.Phase).To(Equal(gardencorev1beta1.RotationCompleted))
ialidzhikov marked this conversation as resolved.
Show resolved Hide resolved

By("Verify registry-cache works after CA rotation")
ialidzhikov marked this conversation as resolved.
Show resolved Hide resolved
// After rotating the CA, the shoot client must be recreated, otherwise any attempt to use it will fail with::
// "tls: failed to verify certificate: x509: certificate signed by unknown authority"
ialidzhikov marked this conversation as resolved.
Show resolved Hide resolved
shootClient, err := access.CreateShootClientFromAdminKubeconfig(ctx, f.GardenClient, f.Shoot)
Expect(err).NotTo(HaveOccurred())
common.VerifyRegistryCache(parentCtx, f.Logger, shootClient, common.GithubRegistryJitesoftAlpine3179Image, common.SleepInfinity)

By("Disable the registry-cache extension")
ctx, cancel = context.WithTimeout(parentCtx, 10*time.Minute)
defer cancel()
Expect(f.UpdateShoot(ctx, func(shoot *gardencorev1beta1.Shoot) error {
common.RemoveExtension(shoot, "registry-cache")

return nil
})).To(Succeed())

By("Verify registry configuration is removed")
ctx, cancel = context.WithTimeout(parentCtx, 2*time.Minute)
defer cancel()
common.VerifyHostsTOMLFilesDeletedForAllNodes(ctx, f.Logger, shootClient, []string{"ghcr.io"})
}, rotateCertTestTimeout, framework.WithCAfterTest(func(ctx context.Context) {
if f.Shoot.Status.Credentials != nil && f.Shoot.Status.Credentials.Rotation.CertificateAuthorities.Phase != gardencorev1beta1.RotationCompleted {
ialidzhikov marked this conversation as resolved.
Show resolved Hide resolved
ialidzhikov marked this conversation as resolved.
Show resolved Hide resolved
Expect(f.AnnotateShoot(ctx, f.Shoot, map[string]string{constants.GardenerOperation: constants.OperationRotateCAComplete})).To(Succeed())
Expect(f.WaitForShootToBeReconciled(ctx, f.Shoot)).To(Succeed())
}
if common.HasRegistryCacheExtension(f.Shoot) {
By("Disable the registry-cache extension")
Expect(f.UpdateShoot(ctx, func(shoot *gardencorev1beta1.Shoot) error {
common.RemoveExtension(shoot, "registry-cache")

return nil
})).To(Succeed())
}
}, rotateCertTestCleanupTimeout))
})