From 4b011cb81a0cc824c7e6280968ccaa66b2b0781b Mon Sep 17 00:00:00 2001 From: Krzysztof Siedlecki Date: Thu, 6 Feb 2020 16:00:20 +0100 Subject: [PATCH 1/2] e2e vendor update --- vertical-pod-autoscaler/e2e/go.mod | 2 +- .../pkg/utils/status/status_object.go | 201 ++++++++++++++++++ .../pkg/utils/status/status_updater.go | 61 ++++++ .../e2e/vendor/modules.txt | 3 +- 4 files changed, 265 insertions(+), 2 deletions(-) create mode 100644 vertical-pod-autoscaler/e2e/vendor/k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/status/status_object.go create mode 100644 vertical-pod-autoscaler/e2e/vendor/k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/status/status_updater.go diff --git a/vertical-pod-autoscaler/e2e/go.mod b/vertical-pod-autoscaler/e2e/go.mod index feb5115d4f7..f596714950f 100644 --- a/vertical-pod-autoscaler/e2e/go.mod +++ b/vertical-pod-autoscaler/e2e/go.mod @@ -26,7 +26,7 @@ require ( gopkg.in/yaml.v2 v2.2.5 // indirect k8s.io/api v0.0.0 k8s.io/apimachinery v0.0.0 - k8s.io/autoscaler/vertical-pod-autoscaler v0.0.0-20200204104326-563f253401ee + k8s.io/autoscaler/vertical-pod-autoscaler v0.0.0-20200207133143-e7988a6dd041 k8s.io/client-go v11.0.0+incompatible k8s.io/component-base v0.0.0 k8s.io/klog v1.0.0 diff --git a/vertical-pod-autoscaler/e2e/vendor/k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/status/status_object.go b/vertical-pod-autoscaler/e2e/vendor/k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/status/status_object.go new file mode 100644 index 00000000000..0f6a31c708e --- /dev/null +++ b/vertical-pod-autoscaler/e2e/vendor/k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/status/status_object.go @@ -0,0 +1,201 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package status + +import ( + "net" + "time" + + apicoordinationv1 "k8s.io/api/coordination/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + utilnet "k8s.io/apimachinery/pkg/util/net" + "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" + typedcoordinationv1 "k8s.io/client-go/kubernetes/typed/coordination/v1" + "k8s.io/utils/pointer" +) + +const ( + // AdmissionControllerStatusName is the name of + // the Admission Controller status object. + AdmissionControllerStatusName = "vpa-admission-controller" + // AdmissionControllerStatusNamespace is the namespace of + // the Admission Controller status object. + AdmissionControllerStatusNamespace = "kube-system" + // AdmissionControllerStatusTimeout is a time after which + // if not updated the Admission Controller status is no longer valid. + AdmissionControllerStatusTimeout = 1 * time.Minute + + // Parameters for retrying with exponential backoff. + retryBackoffInitialDuration = 100 * time.Millisecond + retryBackoffFactor = 3 + retryBackoffJitter = 0 + retryBackoffSteps = 3 +) + +// Client for the status object. +type Client struct { + client typedcoordinationv1.LeaseInterface + leaseName string + leaseNamespace string + leaseDurationSeconds int32 + holderIdentity string +} + +// Validator for the status object. +type Validator interface { + IsStatusValid(statusTimeout time.Duration) (bool, error) +} + +// NewClient returns a client for the status object. +func NewClient(c clientset.Interface, leaseName, leaseNamespace string, leaseDuration time.Duration, holderIdentity string) *Client { + return &Client{ + client: c.CoordinationV1().Leases(leaseNamespace), + leaseName: leaseName, + leaseNamespace: leaseNamespace, + leaseDurationSeconds: int32(leaseDuration.Seconds()), + holderIdentity: holderIdentity, + } +} + +// NewValidator returns a validator for the status object. +func NewValidator(c clientset.Interface, leaseName, leaseNamespace string) Validator { + return &Client{ + client: c.CoordinationV1().Leases(leaseNamespace), + leaseName: leaseName, + leaseNamespace: leaseNamespace, + leaseDurationSeconds: 0, + holderIdentity: "", + } +} + +// UpdateStatus renews status object lease. +// Status object will be created if it doesn't exist. +func (c *Client) UpdateStatus() error { + updateFn := func() error { + lease, err := c.client.Get(c.leaseName, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + // Create lease if it doesn't exist. + return c.create() + } else if err != nil { + return err + } + lease.Spec.RenewTime = &metav1.MicroTime{Time: time.Now()} + lease.Spec.HolderIdentity = pointer.StringPtr(c.holderIdentity) + _, err = c.client.Update(lease) + if apierrors.IsConflict(err) { + // Lease was updated by an another replica of the component. + // No error should be returned. + return nil + } + return err + } + return retryWithExponentialBackOff(updateFn) +} + +func (c *Client) create() error { + _, err := c.client.Create(c.newLease()) + if apierrors.IsAlreadyExists(err) { + // Lease was created by an another replica of the component. + // No error should be returned. + return nil + } + return err +} + +func (c *Client) newLease() *apicoordinationv1.Lease { + return &apicoordinationv1.Lease{ + ObjectMeta: metav1.ObjectMeta{ + Name: c.leaseName, + Namespace: c.leaseNamespace, + }, + Spec: apicoordinationv1.LeaseSpec{ + HolderIdentity: pointer.StringPtr(c.holderIdentity), + LeaseDurationSeconds: pointer.Int32Ptr(c.leaseDurationSeconds), + }, + } +} + +// IsStatusValid verifies if the current status object +// was updated before lease timing out. +func (c *Client) IsStatusValid(statusTimeout time.Duration) (bool, error) { + status, err := c.getStatus() + if err != nil { + return false, err + } + return isStatusValid(status, statusTimeout, time.Now()), nil +} + +func (c *Client) getStatus() (*apicoordinationv1.Lease, error) { + var lease *apicoordinationv1.Lease + getFn := func() error { + var err error + lease, err = c.client.Get(c.leaseName, metav1.GetOptions{}) + return err + } + err := retryWithExponentialBackOff(getFn) + return lease, err +} + +func isStatusValid(status *apicoordinationv1.Lease, leaseTimeout time.Duration, now time.Time) bool { + return status.CreationTimestamp.Add(leaseTimeout).After(now) || + (status.Spec.RenewTime != nil && + status.Spec.RenewTime.Add(leaseTimeout).After(now)) +} + +func isRetryableAPIError(err error) bool { + // These errors may indicate a transient error that we can retry. + if apierrors.IsInternalError(err) || apierrors.IsTimeout(err) || apierrors.IsServerTimeout(err) || + apierrors.IsTooManyRequests(err) || utilnet.IsProbableEOF(err) || utilnet.IsConnectionReset(err) { + return true + } + // If the error sends the Retry-After header, we respect it as an explicit confirmation we should retry. + if _, shouldRetry := apierrors.SuggestsClientDelay(err); shouldRetry { + return true + } + return false +} + +func isRetryableNetError(err error) bool { + if netError, ok := err.(net.Error); ok { + return netError.Temporary() || netError.Timeout() + } + return false +} + +func retryWithExponentialBackOff(fn func() error) error { + backoff := wait.Backoff{ + Duration: retryBackoffInitialDuration, + Factor: retryBackoffFactor, + Jitter: retryBackoffJitter, + Steps: retryBackoffSteps, + } + retryFn := func(fn func() error) func() (bool, error) { + return func() (bool, error) { + err := fn() + if err == nil { + return true, nil + } + if isRetryableAPIError(err) || isRetryableNetError(err) { + return false, nil + } + return false, err + } + } + return wait.ExponentialBackoff(backoff, retryFn(fn)) +} diff --git a/vertical-pod-autoscaler/e2e/vendor/k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/status/status_updater.go b/vertical-pod-autoscaler/e2e/vendor/k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/status/status_updater.go new file mode 100644 index 00000000000..cf955484b44 --- /dev/null +++ b/vertical-pod-autoscaler/e2e/vendor/k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/status/status_updater.go @@ -0,0 +1,61 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package status + +import ( + "time" + + clientset "k8s.io/client-go/kubernetes" + "k8s.io/klog" +) + +// Updater periodically updates status object. +type Updater struct { + client *Client + updateInterval time.Duration +} + +// NewUpdater returns a new status updater. +func NewUpdater(c clientset.Interface, statusName, statusNamespace string, + updateInterval time.Duration, holderIdentity string) *Updater { + return &Updater{ + client: NewClient( + c, + statusName, + statusNamespace, + updateInterval, + holderIdentity, + ), + updateInterval: updateInterval, + } +} + +// Run starts status updates. +func (su *Updater) Run(stopCh <-chan struct{}) { + go func() { + for { + select { + case <-stopCh: + return + case <-time.After(su.updateInterval): + if err := su.client.UpdateStatus(); err != nil { + klog.Errorf("Status update by %s failed: %v", su.client.holderIdentity, err) + } + } + } + }() +} diff --git a/vertical-pod-autoscaler/e2e/vendor/modules.txt b/vertical-pod-autoscaler/e2e/vendor/modules.txt index 6af25ea5fc9..00db9e8bf23 100644 --- a/vertical-pod-autoscaler/e2e/vendor/modules.txt +++ b/vertical-pod-autoscaler/e2e/vendor/modules.txt @@ -923,7 +923,7 @@ k8s.io/apiserver/pkg/util/webhook k8s.io/apiserver/pkg/util/wsstream k8s.io/apiserver/plugin/pkg/authenticator/token/webhook k8s.io/apiserver/plugin/pkg/authorizer/webhook -# k8s.io/autoscaler/vertical-pod-autoscaler v0.0.0-20200204104326-563f253401ee => ../ +# k8s.io/autoscaler/vertical-pod-autoscaler v0.0.0-20200207133143-e7988a6dd041 => ../ k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1 k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1beta1 k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1beta2 @@ -935,6 +935,7 @@ k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/clientset/versioned/typed/a k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1beta2 k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/clientset/versioned/typed/poc.autoscaling.k8s.io/v1alpha1 k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/annotations +k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/status # k8s.io/client-go v11.0.0+incompatible => k8s.io/client-go v0.0.0-20191016111102-bec269661e48 k8s.io/client-go/discovery k8s.io/client-go/discovery/cached/memory From da62c4b6d6844448850e4fea8ed35dbf218b9950 Mon Sep 17 00:00:00 2001 From: Krzysztof Siedlecki Date: Thu, 6 Feb 2020 16:02:17 +0100 Subject: [PATCH 2/2] adding updater e2e test --- vertical-pod-autoscaler/e2e/v1/updater.go | 90 +++++++++++++++++++ .../e2e/v1beta2/updater.go | 90 +++++++++++++++++++ 2 files changed, 180 insertions(+) create mode 100644 vertical-pod-autoscaler/e2e/v1/updater.go create mode 100644 vertical-pod-autoscaler/e2e/v1beta2/updater.go diff --git a/vertical-pod-autoscaler/e2e/v1/updater.go b/vertical-pod-autoscaler/e2e/v1/updater.go new file mode 100644 index 00000000000..9329ba3229d --- /dev/null +++ b/vertical-pod-autoscaler/e2e/v1/updater.go @@ -0,0 +1,90 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package autoscaling + +import ( + "fmt" + "time" + + autoscaling "k8s.io/api/autoscaling/v1" + apiv1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/status" + "k8s.io/kubernetes/test/e2e/framework" + + "github.com/onsi/ginkgo" + "github.com/onsi/gomega" +) + +var _ = UpdaterE2eDescribe("Updater", func() { + f := framework.NewDefaultFramework("vertical-pod-autoscaling") + + ginkgo.It("evicts pods when Admission Controller status available", func() { + const statusUpdateInterval = 10 * time.Second + + ginkgo.By("Setting up the Admission Controller status") + stopCh := make(chan struct{}) + statusUpdater := status.NewUpdater( + f.ClientSet, + status.AdmissionControllerStatusName, + status.AdmissionControllerStatusNamespace, + statusUpdateInterval, + "e2e test", + ) + defer func() { + // Schedule a cleanup of the Admission Controller status. + // Status is created outside the test namespace. + ginkgo.By("Deleting the Admission Controller status") + close(stopCh) + err := f.ClientSet.CoordinationV1().Leases(status.AdmissionControllerStatusNamespace). + Delete(status.AdmissionControllerStatusName, &metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }() + statusUpdater.Run(stopCh) + + podList := setupPodsForEviction(f) + + ginkgo.By("Waiting for pods to be evicted") + err := WaitForPodsEvicted(f, podList) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + + ginkgo.It("doesn't evict pods when Admission Controller status unavailable", func() { + podList := setupPodsForEviction(f) + + ginkgo.By(fmt.Sprintf("Waiting for pods to be evicted, hoping it won't happen, sleep for %s", VpaEvictionTimeout.String())) + CheckNoPodsEvicted(f, MakePodSet(podList)) + }) +}) + +func setupPodsForEviction(f *framework.Framework) *apiv1.PodList { + controller := &autoscaling.CrossVersionObjectReference{ + APIVersion: "apps/v1", + Kind: "Deployment", + Name: "hamster-deployment", + } + ginkgo.By(fmt.Sprintf("Setting up a hamster %v", controller.Kind)) + setupHamsterController(f, controller.Kind, "100m", "100Mi", defaultHamsterReplicas) + podList, err := GetHamsterPods(f) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Setting up a VPA CRD") + SetupVPA(f, "200m", vpa_types.UpdateModeAuto, controller) + + return podList +} diff --git a/vertical-pod-autoscaler/e2e/v1beta2/updater.go b/vertical-pod-autoscaler/e2e/v1beta2/updater.go new file mode 100644 index 00000000000..3d4d0ddfc45 --- /dev/null +++ b/vertical-pod-autoscaler/e2e/v1beta2/updater.go @@ -0,0 +1,90 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package autoscaling + +import ( + "fmt" + "time" + + autoscaling "k8s.io/api/autoscaling/v1" + apiv1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1beta2" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/status" + "k8s.io/kubernetes/test/e2e/framework" + + "github.com/onsi/ginkgo" + "github.com/onsi/gomega" +) + +var _ = UpdaterE2eDescribe("Updater", func() { + f := framework.NewDefaultFramework("vertical-pod-autoscaling") + + ginkgo.It("evicts pods when Admission Controller status available", func() { + const statusUpdateInterval = 10 * time.Second + + ginkgo.By("Setting up the Admission Controller status") + stopCh := make(chan struct{}) + statusUpdater := status.NewUpdater( + f.ClientSet, + status.AdmissionControllerStatusName, + status.AdmissionControllerStatusNamespace, + statusUpdateInterval, + "e2e test", + ) + defer func() { + // Schedule a cleanup of the Admission Controller status. + // Status is created outside the test namespace. + ginkgo.By("Deleting the Admission Controller status") + close(stopCh) + err := f.ClientSet.CoordinationV1().Leases(status.AdmissionControllerStatusNamespace). + Delete(status.AdmissionControllerStatusName, &metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }() + statusUpdater.Run(stopCh) + + podList := setupPodsForEviction(f) + + ginkgo.By("Waiting for pods to be evicted") + err := WaitForPodsEvicted(f, podList) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + + ginkgo.It("doesn't evict pods when Admission Controller status unavailable", func() { + podList := setupPodsForEviction(f) + + ginkgo.By(fmt.Sprintf("Waiting for pods to be evicted, hoping it won't happen, sleep for %s", VpaEvictionTimeout.String())) + CheckNoPodsEvicted(f, MakePodSet(podList)) + }) +}) + +func setupPodsForEviction(f *framework.Framework) *apiv1.PodList { + controller := &autoscaling.CrossVersionObjectReference{ + APIVersion: "apps/v1", + Kind: "Deployment", + Name: "hamster-deployment", + } + ginkgo.By(fmt.Sprintf("Setting up a hamster %v", controller.Kind)) + setupHamsterController(f, controller.Kind, "100m", "100Mi", defaultHamsterReplicas) + podList, err := GetHamsterPods(f) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Setting up a VPA CRD") + SetupVPA(f, "200m", vpa_types.UpdateModeAuto, controller) + + return podList +}