diff --git a/pkg/controller/release/release_controller.go b/pkg/controller/release/release_controller.go index ebf01d71f..34ba90833 100644 --- a/pkg/controller/release/release_controller.go +++ b/pkg/controller/release/release_controller.go @@ -29,7 +29,6 @@ import ( "github.com/bookingcom/shipper/pkg/controller" shippercontroller "github.com/bookingcom/shipper/pkg/controller" shippererrors "github.com/bookingcom/shipper/pkg/errors" - conditions "github.com/bookingcom/shipper/pkg/util/conditions" diffutil "github.com/bookingcom/shipper/pkg/util/diff" releaseutil "github.com/bookingcom/shipper/pkg/util/release" rolloutblock "github.com/bookingcom/shipper/pkg/util/rolloutblock" @@ -37,8 +36,10 @@ import ( ) const ( - AgentName = "release-controller" - ClustersNotReady = "ClustersNotReady" + AgentName = "release-controller" + + ClustersNotReady = "ClustersNotReady" + StrategyExecutionFailed = "StrategyExecutionFailed" ) // Controller is a Kubernetes controller whose role is to pick up a newly created @@ -49,9 +50,6 @@ type Controller struct { clientset shipperclient.Interface store clusterclientstore.Interface - applicationLister shipperlisters.ApplicationLister - applicationsSynced cache.InformerSynced - releaseLister shipperlisters.ReleaseLister releasesSynced cache.InformerSynced @@ -70,7 +68,7 @@ type Controller struct { rolloutBlockLister shipperlisters.RolloutBlockLister rolloutBlockSynced cache.InformerSynced - releaseWorkqueue workqueue.RateLimitingInterface + workqueue workqueue.RateLimitingInterface chartFetcher shipperrepo.ChartFetcher @@ -98,7 +96,6 @@ func NewController( recorder record.EventRecorder, ) *Controller { - applicationInformer := informerFactory.Shipper().V1alpha1().Applications() releaseInformer := informerFactory.Shipper().V1alpha1().Releases() clusterInformer := informerFactory.Shipper().V1alpha1().Clusters() installationTargetInformer := informerFactory.Shipper().V1alpha1().InstallationTargets() @@ -112,9 +109,6 @@ func NewController( clientset: clientset, store: store, - applicationLister: applicationInformer.Lister(), - applicationsSynced: applicationInformer.Informer().HasSynced, - releaseLister: releaseInformer.Lister(), releasesSynced: releaseInformer.Informer().HasSynced, @@ -133,7 +127,7 @@ func NewController( rolloutBlockLister: rolloutBlockInformer.Lister(), rolloutBlockSynced: rolloutBlockInformer.Informer().HasSynced, - releaseWorkqueue: workqueue.NewNamedRateLimitingQueue( + workqueue: workqueue.NewNamedRateLimitingQueue( shipperworkqueue.NewDefaultControllerRateLimiter(), "release_controller_releases", ), @@ -177,14 +171,13 @@ func NewController( // Run starts Release Controller workers and waits until stopCh is closed. func (c *Controller) Run(threadiness int, stopCh <-chan struct{}) { defer runtime.HandleCrash() - defer c.releaseWorkqueue.ShutDown() + defer c.workqueue.ShutDown() klog.V(2).Info("Starting Release controller") defer klog.V(2).Info("Shutting down Release controller") if ok := cache.WaitForCacheSync( stopCh, - c.applicationsSynced, c.releasesSynced, c.clustersSynced, c.installationTargetsSynced, @@ -206,20 +199,20 @@ func (c *Controller) Run(threadiness int, stopCh <-chan struct{}) { } func (c *Controller) runReleaseWorker() { - for c.processNextReleaseWorkItem() { + for c.processNextWorkItem() { } } // processNextReleaseWorkItem pops an element from the head of the workqueue and // passes to the sync release handler. It returns bool indicating if the // execution process should go on. -func (c *Controller) processNextReleaseWorkItem() bool { - obj, shutdown := c.releaseWorkqueue.Get() +func (c *Controller) processNextWorkItem() bool { + obj, shutdown := c.workqueue.Get() if shutdown { return false } - defer c.releaseWorkqueue.Done(obj) + defer c.workqueue.Done(obj) var ( key string @@ -227,7 +220,7 @@ func (c *Controller) processNextReleaseWorkItem() bool { ) if key, ok = obj.(string); !ok { - c.releaseWorkqueue.Forget(obj) + c.workqueue.Forget(obj) runtime.HandleError(fmt.Errorf("invalid object key (will retry: false): %#v", obj)) return true } @@ -241,12 +234,12 @@ func (c *Controller) processNextReleaseWorkItem() bool { } if shouldRetry { - c.releaseWorkqueue.AddRateLimited(key) + c.workqueue.AddRateLimited(key) return true } klog.V(4).Infof("Successfully synced Release %q", key) - c.releaseWorkqueue.Forget(obj) + c.workqueue.Forget(obj) return true } @@ -419,8 +412,8 @@ func (c *Controller) processRelease(rel *shipper.Release) (*shipper.Release, []S releaseStrategyExecutedCond := releaseutil.NewReleaseCondition( shipper.ReleaseConditionTypeStrategyExecuted, corev1.ConditionFalse, - conditions.StrategyExecutionFailed, - fmt.Sprintf("failed to execute strategy: %q", err), + StrategyExecutionFailed, + err.Error(), ) diff.Append(releaseutil.SetReleaseCondition(&rel.Status, *releaseStrategyExecutedCond)) @@ -684,7 +677,7 @@ func (c *Controller) enqueueRelease(obj interface{}) { return } - c.releaseWorkqueue.Add(key) + c.workqueue.Add(key) } func (c *Controller) enqueueReleaseFromRolloutBlock(obj interface{}) { diff --git a/pkg/controller/release/release_controller_test.go b/pkg/controller/release/release_controller_test.go index d9d610f50..c3d393c3a 100644 --- a/pkg/controller/release/release_controller_test.go +++ b/pkg/controller/release/release_controller_test.go @@ -1,3078 +1,480 @@ package release import ( - "encoding/json" "fmt" - "math/rand" "sort" - "strconv" "strings" "testing" "time" corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - - "k8s.io/apimachinery/pkg/util/wait" - kubetesting "k8s.io/client-go/testing" - "k8s.io/client-go/tools/record" shipper "github.com/bookingcom/shipper/pkg/apis/shipper/v1alpha1" - shipperfake "github.com/bookingcom/shipper/pkg/client/clientset/versioned/fake" - shipperinformers "github.com/bookingcom/shipper/pkg/client/informers/externalversions" shippertesting "github.com/bookingcom/shipper/pkg/testing" - apputil "github.com/bookingcom/shipper/pkg/util/application" "github.com/bookingcom/shipper/pkg/util/conditions" releaseutil "github.com/bookingcom/shipper/pkg/util/release" - targetutil "github.com/bookingcom/shipper/pkg/util/target" ) -func init() { - apputil.ConditionsShouldDiscardTimestamps = true - releaseutil.ConditionsShouldDiscardTimestamps = true - conditions.StrategyConditionsShouldDiscardTimestamps = true -} - -var vanguard = shipper.RolloutStrategy{ - Steps: []shipper.RolloutStrategyStep{ - { - Name: "staging", - Capacity: shipper.RolloutStrategyStepValue{Incumbent: 100, Contender: 1}, - Traffic: shipper.RolloutStrategyStepValue{Incumbent: 100, Contender: 0}, - }, - { - Name: "50/50", - Capacity: shipper.RolloutStrategyStepValue{Incumbent: 50, Contender: 50}, - Traffic: shipper.RolloutStrategyStepValue{Incumbent: 50, Contender: 50}, - }, - { - Name: "full on", - Capacity: shipper.RolloutStrategyStepValue{Incumbent: 0, Contender: 100}, - Traffic: shipper.RolloutStrategyStepValue{Incumbent: 0, Contender: 100}, - }, - }, -} - -var fullon = shipper.RolloutStrategy{ - Steps: []shipper.RolloutStrategyStep{ - { - Name: "full on", - Capacity: shipper.RolloutStrategyStepValue{Incumbent: 0, Contender: 100}, - Traffic: shipper.RolloutStrategyStepValue{Incumbent: 0, Contender: 100}, - }, - }, -} - -type role int - const ( - Contender role = iota - Incumbent - testRolloutBlockName = "test-rollout-block" -) + StepStaging int32 = iota + StepVanguard + StepFullOn -type releaseInfoPair struct { - incumbent *releaseInfo - contender *releaseInfo -} - -type actionfilter struct { - verbs []string - resources []string -} + GenerationInactive string = "1" + GenerationIncumbent string = "2" + GenerationContender string = "3" +) -func (af actionfilter) Extend(ext actionfilter) actionfilter { - verbset := make(map[string]struct{}) - newverbs := make([]string, 0) - for _, verb := range append(af.verbs, ext.verbs...) { - if _, ok := verbset[verb]; !ok { - newverbs = append(newverbs, verb) - verbset[verb] = struct{}{} - } +var ( + ReleaseConditionUnblocked = shipper.ReleaseCondition{ + Type: shipper.ReleaseConditionTypeBlocked, + Status: corev1.ConditionFalse, } - resourceset := make(map[string]struct{}) - newresources := make([]string, 0) - for _, resource := range append(af.resources, ext.resources...) { - if _, ok := resourceset[resource]; !ok { - newresources = append(newresources, resource) - resourceset[resource] = struct{}{} - } + ReleaseConditionScheduled = shipper.ReleaseCondition{ + Type: shipper.ReleaseConditionTypeScheduled, + Status: corev1.ConditionTrue, } - sort.Strings(newverbs) - sort.Strings(newresources) - return actionfilter{ - verbs: newverbs, - resources: newresources, + ReleaseConditionStrategyExecuted = shipper.ReleaseCondition{ + Type: shipper.ReleaseConditionTypeStrategyExecuted, + Status: corev1.ConditionTrue, } -} - -func (af actionfilter) IsEmpty() bool { - return len(af.verbs) == 0 && len(af.resources) == 0 -} - -func (af actionfilter) DoFilter(actions []kubetesting.Action) []kubetesting.Action { - if af.IsEmpty() { - return actions + ReleaseConditionComplete = shipper.ReleaseCondition{ + Type: shipper.ReleaseConditionTypeComplete, + Status: corev1.ConditionTrue, } - ignore := func(action kubetesting.Action) bool { - for _, v := range af.verbs { - for _, r := range af.resources { - if action.Matches(v, r) { - return false - } - } - } - return true + StateWaitingForCommand = shipper.ReleaseStrategyState{ + WaitingForInstallation: shipper.StrategyStateFalse, + WaitingForCapacity: shipper.StrategyStateFalse, + WaitingForTraffic: shipper.StrategyStateFalse, + WaitingForCommand: shipper.StrategyStateTrue, } - - var ret []kubetesting.Action - for _, action := range actions { - if ignore(action) { - continue - } - - ret = append(ret, action) + StateWaitingForNone = shipper.ReleaseStrategyState{ + WaitingForInstallation: shipper.StrategyStateFalse, + WaitingForCapacity: shipper.StrategyStateFalse, + WaitingForTraffic: shipper.StrategyStateFalse, + WaitingForCommand: shipper.StrategyStateFalse, } - return ret -} - -type fixture struct { - initialized bool - t *testing.T - cycles int - objects []runtime.Object - clientset *shipperfake.Clientset - store *shippertesting.FakeClusterClientStore - informerFactory shipperinformers.SharedInformerFactory - recorder *record.FakeRecorder - - actions []kubetesting.Action - filter actionfilter - receivedEvents []string - expectedEvents []string -} - -func newFixture(t *testing.T, objects ...runtime.Object) *fixture { - return &fixture{ - initialized: false, - t: t, - cycles: -1, - objects: objects, - - actions: make([]kubetesting.Action, 0), - filter: actionfilter{}, - receivedEvents: make([]string, 0), - expectedEvents: make([]string, 0), + StrategyConditionContenderAchievedInstallation = shipper.ReleaseStrategyCondition{ + Type: shipper.StrategyConditionContenderAchievedInstallation, + Status: corev1.ConditionTrue, } -} + StrategyConditionContenderAchievedCapacity = shipper.ReleaseStrategyCondition{ + Type: shipper.StrategyConditionContenderAchievedCapacity, + Status: corev1.ConditionTrue, + } + StrategyConditionContenderAchievedTraffic = shipper.ReleaseStrategyCondition{ + Type: shipper.StrategyConditionContenderAchievedTraffic, + Status: corev1.ConditionTrue, + } +) -func (f *fixture) addObjects(objects ...runtime.Object) { - f.objects = append(f.objects, objects...) +func init() { + releaseutil.ConditionsShouldDiscardTimestamps = true + conditions.StrategyConditionsShouldDiscardTimestamps = true } -func (f *fixture) run() { - f.clientset = shipperfake.NewSimpleClientset(f.objects...) - - const syncPeriod time.Duration = 0 - informerFactory := shipperinformers.NewSharedInformerFactory(f.clientset, syncPeriod) - - f.informerFactory = informerFactory - f.recorder = record.NewFakeRecorder(42) - - controller := f.newController() - - stopCh := make(chan struct{}) - defer close(stopCh) - - f.informerFactory.Start(stopCh) - f.informerFactory.WaitForCacheSync(stopCh) +type releaseControllerTestExpectation struct { + release *shipper.Release + status shipper.ReleaseStatus + clusters []string +} - wait.PollUntil( - 10*time.Millisecond, - func() (bool, error) { - return controller.releaseWorkqueue.Len() > 0, nil - }, - stopCh, +// TestRolloutsBlocked tests that a Release will not progress whenever there's +// a relevant RolloutBlock present, and that it will have a Blocked condition +// set to True. +func TestRolloutsBlocked(t *testing.T) { + rel := buildRelease( + shippertesting.TestNamespace, + shippertesting.TestApp, + "blocked", + 0, ) + rb := buildRolloutBlock(shippertesting.TestNamespace, "block-rollouts") - readyCh := make(chan struct{}) - go func() { - for e := range f.recorder.Events { - f.receivedEvents = append(f.receivedEvents, e) - } - close(readyCh) - }() + mgmtClusterObjects := []runtime.Object{rel, rb} + appClusterObjects := map[string][]runtime.Object{} - cycles := 0 - for (f.cycles < 0 || cycles < f.cycles) && controller.releaseWorkqueue.Len() > 0 { - controller.processNextReleaseWorkItem() - cycles++ + expectedStatus := shipper.ReleaseStatus{ + Conditions: []shipper.ReleaseCondition{ + { + Type: shipper.ReleaseConditionTypeBlocked, + Status: corev1.ConditionTrue, + Reason: shipper.RolloutBlockReason, + Message: fmt.Sprintf( + "rollout block(s) with name(s) %s/%s exist", + shippertesting.TestNamespace, rb.Name, + ), + }, + }, } - close(f.recorder.Events) - <-readyCh - - actual := shippertesting.FilterActions(f.clientset.Actions()) - actual = f.filter.DoFilter(actual) - shippertesting.CheckActions(f.actions, actual, f.t) - shippertesting.CheckEvents(f.expectedEvents, f.receivedEvents, f.t) + runReleaseControllerTest(t, mgmtClusterObjects, appClusterObjects, + []releaseControllerTestExpectation{ + { + release: rel, + status: expectedStatus, + }, + }) } -func (f *fixture) newController() *Controller { - return NewController( - f.clientset, - f.store, - f.informerFactory, - shippertesting.LocalFetchChart, - f.recorder, +// TestUnschedulable tests that a Release will not progress when it can't +// choose clusters based on the its requirements. +func TestUnschedulable(t *testing.T) { + var replicas int32 = 1 + rel := buildRelease( + shippertesting.TestNamespace, + shippertesting.TestApp, + "unschedulable", + replicas, ) -} -func newRolloutBlock(name string, namespace string) *shipper.RolloutBlock { - return &shipper.RolloutBlock{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Spec: shipper.RolloutBlockSpec{ - Message: "Simple test rollout block", - Author: shipper.RolloutBlockAuthor{ - Type: "user", - Name: "testUser", - }, - }, - } -} + mgmtClusterObjects := []runtime.Object{rel} + appClusterObjects := map[string][]runtime.Object{} -func buildApplication(namespace string, appName string) *shipper.Application { - return &shipper.Application{ - ObjectMeta: metav1.ObjectMeta{ - Name: appName, - Namespace: namespace, - UID: "foobarbaz", - }, - Status: shipper.ApplicationStatus{ - History: []string{}, - }, - } -} + // Our release requires one cluster, but we have none available in + // mgmtClusterObjects -func buildCluster(name string) *shipper.Cluster { - return &shipper.Cluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - }, - Spec: shipper.ClusterSpec{ - APIMaster: "https://127.0.0.1", - Capabilities: []string{}, - Region: shippertesting.TestRegion, + expectedStatus := shipper.ReleaseStatus{ + Conditions: []shipper.ReleaseCondition{ + ReleaseConditionUnblocked, + { + Type: shipper.ReleaseConditionTypeScheduled, + Status: corev1.ConditionFalse, + Reason: "NotEnoughClustersInRegion", + Message: fmt.Sprintf( + "Not enough clusters in region %q. Required: %d / Available: 0", + shippertesting.TestRegion, replicas, + ), + }, }, } + + runReleaseControllerTest(t, mgmtClusterObjects, appClusterObjects, + []releaseControllerTestExpectation{ + { + release: rel, + status: expectedStatus, + }, + }) } -func (f *fixture) buildIncumbent(namespace string, relName string, replicaCount int32) *releaseInfo { - var app *shipper.Application - for _, object := range f.objects { - if conv, ok := object.(*shipper.Application); ok { - app = conv - break - } - } - if app == nil { - f.t.Fatalf("The fixture is missing an Application object") - } +// TestInvalidStrategy tests that a Release will not progress when it +// can't execute its strategy. +func TestInvalidStrategy(t *testing.T) { + rel := buildRelease( + shippertesting.TestNamespace, + shippertesting.TestApp, + "invalid-strategy", + 1, + ) - clusterNames := make([]string, 0) - for _, obj := range f.objects { - if cluster, ok := obj.(*shipper.Cluster); ok { - clusterNames = append(clusterNames, cluster.GetName()) - } - } - if len(clusterNames) == 0 { - f.t.Fatalf("The fixture is missing at least 1 Cluster object") - } + // we set the Release's TargetStep to something that's very obviously + // invalid, making its strategy not executable + rel.Spec.TargetStep = 999 - step, stepName := int32(2), "full on" + cluster := buildCluster("cluster-a") + mgmtClusterObjects := []runtime.Object{rel, cluster} - rolloutblocksOverrides := app.Annotations[shipper.RolloutBlocksOverrideAnnotation] - rel := &shipper.Release{ - TypeMeta: metav1.TypeMeta{ - APIVersion: shipper.SchemeGroupVersion.String(), - Kind: "Release", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: relName, - Namespace: namespace, - OwnerReferences: []metav1.OwnerReference{ - metav1.OwnerReference{ - APIVersion: shipper.SchemeGroupVersion.String(), - Kind: "Application", - Name: app.GetName(), - UID: app.GetUID(), - }, - }, - Labels: map[string]string{ - shipper.ReleaseLabel: relName, - shipper.AppLabel: app.GetName(), - }, - Annotations: map[string]string{ - shipper.ReleaseGenerationAnnotation: "0", - shipper.ReleaseClustersAnnotation: strings.Join(clusterNames, ","), - shipper.RolloutBlocksOverrideAnnotation: rolloutblocksOverrides, - }, - }, - Status: shipper.ReleaseStatus{ - AchievedStep: &shipper.AchievedStep{ - Step: step, - Name: stepName, - }, - Conditions: []shipper.ReleaseCondition{ - {Type: shipper.ReleaseConditionTypeBlocked, Status: corev1.ConditionFalse}, - {Type: shipper.ReleaseConditionTypeComplete, Status: corev1.ConditionTrue}, - {Type: shipper.ReleaseConditionTypeScheduled, Status: corev1.ConditionTrue}, - {Type: shipper.ReleaseConditionTypeStrategyExecuted, Status: corev1.ConditionTrue}, - }, - Strategy: &shipper.ReleaseStrategyStatus{ - State: shipper.ReleaseStrategyState{ - WaitingForInstallation: shipper.StrategyStateFalse, - WaitingForTraffic: shipper.StrategyStateFalse, - WaitingForCapacity: shipper.StrategyStateFalse, - WaitingForCommand: shipper.StrategyStateFalse, - }, - Conditions: []shipper.ReleaseStrategyCondition{ - { - Type: shipper.StrategyConditionContenderAchievedCapacity, - Status: corev1.ConditionTrue, - Step: step, - }, - { - Type: shipper.StrategyConditionContenderAchievedInstallation, - Status: corev1.ConditionTrue, - Step: step, - }, - { - Type: shipper.StrategyConditionContenderAchievedTraffic, - Status: corev1.ConditionTrue, - Step: step, - }, - }, - }, - }, - Spec: shipper.ReleaseSpec{ - TargetStep: step, - Environment: shipper.ReleaseEnvironment{ - Strategy: &vanguard, - Chart: shipper.Chart{ - Name: "simple", - Version: "0.0.1", - }, - ClusterRequirements: shipper.ClusterRequirements{ - Regions: []shipper.RegionRequirement{{Name: shippertesting.TestRegion}}, - }, - }, - }, + // we need to set the cluster as a key here so it gets its own client, + // otherwise the release won't be scheduled properly. + appClusterObjects := map[string][]runtime.Object{ + cluster.Name: []runtime.Object{}, } - installationTarget := &shipper.InstallationTarget{ - TypeMeta: metav1.TypeMeta{ - APIVersion: shipper.SchemeGroupVersion.String(), - Kind: "InstallationTarget", - }, - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: relName, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: shipper.SchemeGroupVersion.String(), - Name: relName, - Kind: "Release", - UID: rel.GetUID(), - }, - }, - }, - Status: shipper.InstallationTargetStatus{ - Conditions: []shipper.TargetCondition{ - { - Type: shipper.TargetConditionTypeReady, - Status: corev1.ConditionTrue, - }, + expectedStatus := shipper.ReleaseStatus{ + Conditions: []shipper.ReleaseCondition{ + ReleaseConditionUnblocked, + ReleaseConditionScheduled, + { + Type: shipper.ReleaseConditionTypeStrategyExecuted, + Status: corev1.ConditionFalse, + Reason: StrategyExecutionFailed, + Message: fmt.Sprintf( + "no step %d in strategy for Release %q", + rel.Spec.TargetStep, fmt.Sprintf("%s/%s", rel.Namespace, rel.Name), + ), }, }, - Spec: shipper.InstallationTargetSpec{ - Clusters: clusterNames, - }, } - capacityTargetSpecClusters := make([]shipper.ClusterCapacityTarget, 0, len(clusterNames)) - for _, clusterName := range clusterNames { - capacityTargetSpecClusters = append(capacityTargetSpecClusters, shipper.ClusterCapacityTarget{ - Name: clusterName, - Percent: 100, - TotalReplicaCount: replicaCount, + runReleaseControllerTest(t, mgmtClusterObjects, appClusterObjects, + []releaseControllerTestExpectation{ + { + release: rel, + status: expectedStatus, + clusters: []string{cluster.Name}, + }, }) - } +} - capacityTarget := &shipper.CapacityTarget{ - TypeMeta: metav1.TypeMeta{ - APIVersion: shipper.SchemeGroupVersion.String(), - Kind: "CapacityTarget", - }, - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: relName, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: shipper.SchemeGroupVersion.String(), - Name: rel.GetName(), - Kind: "Release", - UID: rel.GetUID(), - }, - }, - }, - Status: shipper.CapacityTargetStatus{ - Conditions: []shipper.TargetCondition{ - { - Type: shipper.TargetConditionTypeReady, - Status: corev1.ConditionTrue, - }, - }, - }, - Spec: shipper.CapacityTargetSpec{ - Clusters: capacityTargetSpecClusters, - }, - } +// TestIntermediateStep tests that a Release will have all of its conditions +// set appropriately for an intermediate achieved step (as in, not a final +// step). This expects most conditions to be true, except for Blocked and +// Complete. +func TestIntermediateStep(t *testing.T) { + rel := buildRelease( + shippertesting.TestNamespace, + shippertesting.TestApp, + "intermediate-step", + 1, + ) - trafficTargetSpecClusters := make([]shipper.ClusterTrafficTarget, 0, len(clusterNames)) - for _, clusterName := range clusterNames { - trafficTargetSpecClusters = append(trafficTargetSpecClusters, shipper.ClusterTrafficTarget{ - Name: clusterName, - Weight: 100, - }) - } + targetStep := StepVanguard + achievedStep := StepVanguard - trafficTarget := &shipper.TrafficTarget{ - TypeMeta: metav1.TypeMeta{ - APIVersion: shipper.SchemeGroupVersion.String(), - Kind: "TrafficTarget", - }, - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: relName, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: shipper.SchemeGroupVersion.String(), - Name: rel.GetName(), - Kind: "Release", - UID: rel.GetUID(), - }, - }, + // rel has a three-step strategy, so StepVanguard should be an + // intermediate step + rel.Spec.TargetStep = targetStep + + cluster := buildCluster("cluster-a") + it, tt, ct := buildAssociatedObjectsWithStatus(rel, []*shipper.Cluster{cluster}, &achievedStep) + + mgmtClusterObjects := []runtime.Object{rel, cluster, it, ct, tt} + appClusterObjects := map[string][]runtime.Object{} + + expectedStatus := shipper.ReleaseStatus{ + AchievedStep: &shipper.AchievedStep{ + Step: achievedStep, + Name: rel.Spec.Environment.Strategy.Steps[achievedStep].Name, }, - Status: shipper.TrafficTargetStatus{ - Conditions: []shipper.TargetCondition{ - { - Type: shipper.TargetConditionTypeReady, - Status: corev1.ConditionTrue, - }, - }, + Conditions: []shipper.ReleaseCondition{ + ReleaseConditionUnblocked, + ReleaseConditionScheduled, + ReleaseConditionStrategyExecuted, }, - Spec: shipper.TrafficTargetSpec{ - Clusters: trafficTargetSpecClusters, + Strategy: &shipper.ReleaseStrategyStatus{ + Conditions: stepify(achievedStep, []shipper.ReleaseStrategyCondition{ + StrategyConditionContenderAchievedCapacity, + StrategyConditionContenderAchievedInstallation, + StrategyConditionContenderAchievedTraffic, + }), + State: StateWaitingForCommand, }, } - return &releaseInfo{ - release: rel, - installationTarget: installationTarget, - capacityTarget: capacityTarget, - trafficTarget: trafficTarget, - } + runReleaseControllerTest(t, mgmtClusterObjects, appClusterObjects, + []releaseControllerTestExpectation{ + { + release: rel, + status: expectedStatus, + clusters: []string{cluster.Name}, + }, + }) } -func (f *fixture) buildContender(namespace string, relName string, replicaCount int32) *releaseInfo { - var app *shipper.Application - for _, object := range f.objects { - if conv, ok := object.(*shipper.Application); ok { - app = conv - break - } - } - if app == nil { - f.t.Fatalf("The fixture is missing an Application object") - } +// TestLastStep tests that a Release will have all of its conditions set +// appropriately for a final achieved step. This expects most conditions to be +// true (except for Blocked) as in TestIntermediateStep, but now also Complete. +func TestLastStep(t *testing.T) { + rel := buildRelease( + shippertesting.TestNamespace, + shippertesting.TestApp, + "last-step", + 1, + ) - clusterNames := make([]string, 0) - for _, obj := range f.objects { - if cluster, ok := obj.(*shipper.Cluster); ok { - clusterNames = append(clusterNames, cluster.GetName()) - } - } - if len(clusterNames) == 0 { - f.t.Fatalf("The fixture is missing at least 1 Cluster object") - } + targetStep := StepFullOn + achievedStep := StepFullOn - rolloutblocksOverrides := app.Annotations[shipper.RolloutBlocksOverrideAnnotation] - rel := &shipper.Release{ - TypeMeta: metav1.TypeMeta{ - APIVersion: shipper.SchemeGroupVersion.String(), - Kind: "Release", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: relName, - Namespace: namespace, - OwnerReferences: []metav1.OwnerReference{ - metav1.OwnerReference{ - APIVersion: shipper.SchemeGroupVersion.String(), - Kind: "Application", - Name: app.GetName(), - UID: app.GetUID(), - }, - }, - Labels: map[string]string{ - shipper.ReleaseLabel: relName, - shipper.AppLabel: app.GetName(), - }, - Annotations: map[string]string{ - shipper.ReleaseGenerationAnnotation: "1", - shipper.RolloutBlocksOverrideAnnotation: rolloutblocksOverrides, - shipper.ReleaseClustersAnnotation: strings.Join(clusterNames, ","), - }, - }, - Status: shipper.ReleaseStatus{ - Conditions: []shipper.ReleaseCondition{ - {Type: shipper.ReleaseConditionTypeBlocked, Status: corev1.ConditionFalse}, - }, - Strategy: &shipper.ReleaseStrategyStatus{}, - }, - Spec: shipper.ReleaseSpec{ - TargetStep: 0, - Environment: shipper.ReleaseEnvironment{ - Strategy: &vanguard, - Chart: shipper.Chart{ - Name: "simple", - Version: "0.0.1", - }, - ClusterRequirements: shipper.ClusterRequirements{ - Regions: []shipper.RegionRequirement{{Name: shippertesting.TestRegion}}, - }, - }, - }, - } + // rel has a three-step strategy, so StepVanguard should be an + // intermediate step + rel.Spec.TargetStep = targetStep - installationTarget := &shipper.InstallationTarget{ - TypeMeta: metav1.TypeMeta{ - APIVersion: shipper.SchemeGroupVersion.String(), - Kind: "InstallationTarget", - }, - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: relName, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: shipper.SchemeGroupVersion.String(), - Name: relName, - Kind: "Release", - UID: rel.GetUID(), - }, - }, - }, - Status: shipper.InstallationTargetStatus{ - Conditions: []shipper.TargetCondition{ - { - Type: shipper.TargetConditionTypeReady, - Status: corev1.ConditionTrue, - }, - }, - }, - Spec: shipper.InstallationTargetSpec{ - Clusters: clusterNames, - }, - } + cluster := buildCluster("cluster-a") + it, tt, ct := buildAssociatedObjectsWithStatus(rel, []*shipper.Cluster{cluster}, &achievedStep) - capacityTargetSpecClusters := make([]shipper.ClusterCapacityTarget, 0, len(clusterNames)) - for _, clusterName := range clusterNames { - capacityTargetSpecClusters = append(capacityTargetSpecClusters, shipper.ClusterCapacityTarget{ - Name: clusterName, - Percent: 0, - TotalReplicaCount: replicaCount, - }) - } + mgmtClusterObjects := []runtime.Object{rel, cluster, it, ct, tt} + appClusterObjects := map[string][]runtime.Object{} - capacityTarget := &shipper.CapacityTarget{ - TypeMeta: metav1.TypeMeta{ - APIVersion: shipper.SchemeGroupVersion.String(), - Kind: "CapacityTarget", + expectedStatus := shipper.ReleaseStatus{ + AchievedStep: &shipper.AchievedStep{ + Step: achievedStep, + Name: rel.Spec.Environment.Strategy.Steps[achievedStep].Name, }, - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: relName, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: shipper.SchemeGroupVersion.String(), - Name: rel.GetName(), - Kind: "Release", - UID: rel.GetUID(), - }, - }, - }, - Status: shipper.CapacityTargetStatus{ - Conditions: []shipper.TargetCondition{ - { - Type: shipper.TargetConditionTypeReady, - Status: corev1.ConditionTrue, - }, - }, + Conditions: []shipper.ReleaseCondition{ + ReleaseConditionUnblocked, + ReleaseConditionComplete, + ReleaseConditionScheduled, + ReleaseConditionStrategyExecuted, }, - Spec: shipper.CapacityTargetSpec{ - Clusters: capacityTargetSpecClusters, + Strategy: &shipper.ReleaseStrategyStatus{ + Conditions: stepify(achievedStep, []shipper.ReleaseStrategyCondition{ + StrategyConditionContenderAchievedCapacity, + StrategyConditionContenderAchievedInstallation, + StrategyConditionContenderAchievedTraffic, + }), + State: StateWaitingForNone, }, } - trafficTargetSpecClusters := make([]shipper.ClusterTrafficTarget, 0, len(clusterNames)) - for _, clusterName := range clusterNames { - trafficTargetSpecClusters = append(trafficTargetSpecClusters, shipper.ClusterTrafficTarget{ - Name: clusterName, - Weight: 0, + runReleaseControllerTest(t, mgmtClusterObjects, appClusterObjects, + []releaseControllerTestExpectation{ + { + release: rel, + status: expectedStatus, + clusters: []string{cluster.Name}, + }, }) - } +} - trafficTarget := &shipper.TrafficTarget{ - TypeMeta: metav1.TypeMeta{ - APIVersion: shipper.SchemeGroupVersion.String(), - Kind: "TrafficTarget", - }, - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: relName, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: shipper.SchemeGroupVersion.String(), - Name: rel.GetName(), - Kind: "Release", - UID: rel.GetUID(), - }, - }, - }, - Status: shipper.TrafficTargetStatus{ - Conditions: []shipper.TargetCondition{ - { - Type: shipper.TargetConditionTypeReady, - Status: corev1.ConditionTrue, - }, - }, - }, - Spec: shipper.TrafficTargetSpec{ - Clusters: trafficTargetSpecClusters, - }, - } +// TestIncumbentNotOnLastStep tests that a release that has a contender (as in, +// not a head release) won't have its strategy executed if its target step is +// not the final step in the strategy, to prevent historical releases from +// being woken up by user changes. +func TestIncumbentNotOnLastStep(t *testing.T) { + incumbent := buildRelease( + shippertesting.TestNamespace, + shippertesting.TestApp, + "incumbent-not-on-last-step-incumbent", + 1, + ) - return &releaseInfo{ - release: rel, - installationTarget: installationTarget, - capacityTarget: capacityTarget, - trafficTarget: trafficTarget, - } -} + contender := buildRelease( + shippertesting.TestNamespace, + shippertesting.TestApp, + "incumbent-not-on-last-step-contender", + 1, + ) -func init() { - rand.Seed(time.Now().UnixNano()) -} + contender.Annotations[shipper.ReleaseGenerationAnnotation] = GenerationContender + incumbent.Annotations[shipper.ReleaseGenerationAnnotation] = GenerationIncumbent + incumbent.Spec.TargetStep = StepVanguard -func addCluster(ri *releaseInfo, cluster *shipper.Cluster) { - clusters := getReleaseClusters(ri.release) - exists := false - for _, cl := range clusters { - if cl == cluster.Name { - exists = true - break - } - } - if !exists { - clusters = append(clusters, cluster.Name) - sort.Strings(clusters) - ri.release.ObjectMeta.Annotations[shipper.ReleaseClustersAnnotation] = strings.Join(clusters, ",") - } + cluster := buildCluster("cluster-a") + mgmtClusterObjects := []runtime.Object{incumbent, contender, cluster} - ri.installationTarget.Spec.Clusters = append(ri.installationTarget.Spec.Clusters, - cluster.Name, - ) - ri.capacityTarget.Spec.Clusters = append(ri.capacityTarget.Spec.Clusters, - shipper.ClusterCapacityTarget{Name: cluster.Name, Percent: 0}, - ) - ri.trafficTarget.Spec.Clusters = append(ri.trafficTarget.Spec.Clusters, - shipper.ClusterTrafficTarget{Name: cluster.Name, Weight: 0}, - ) -} + // we need to set the cluster as a key here so it gets its own client, + // otherwise the release won't be scheduled properly. + appClusterObjects := map[string][]runtime.Object{ + cluster.Name: []runtime.Object{}, + } -func (f *fixture) expectReleaseWaitingForCommand(rel *shipper.Release, step int32) { - f.filter = f.filter.Extend(actionfilter{ - []string{"patch"}, - []string{"releases"}, - }) - - gvr := shipper.SchemeGroupVersion.WithResource("releases") - newStatus := map[string]interface{}{ - "status": shipper.ReleaseStatus{ - Strategy: &shipper.ReleaseStrategyStatus{ - State: shipper.ReleaseStrategyState{ - WaitingForInstallation: shipper.StrategyStateFalse, - WaitingForCommand: shipper.StrategyStateTrue, - WaitingForTraffic: shipper.StrategyStateFalse, - WaitingForCapacity: shipper.StrategyStateFalse, - }, - Conditions: []shipper.ReleaseStrategyCondition{ - { - Type: shipper.StrategyConditionContenderAchievedCapacity, - Status: corev1.ConditionTrue, - Step: step, - }, - { - Type: shipper.StrategyConditionContenderAchievedInstallation, - Status: corev1.ConditionTrue, - Step: step, - }, - { - Type: shipper.StrategyConditionContenderAchievedTraffic, - Status: corev1.ConditionTrue, - Step: step, - }, - { - Type: shipper.StrategyConditionIncumbentAchievedCapacity, - Status: corev1.ConditionTrue, - Step: step, - }, - { - Type: shipper.StrategyConditionIncumbentAchievedTraffic, - Status: corev1.ConditionTrue, - Step: step, - }, - }, + expectedStatus := shipper.ReleaseStatus{ + Conditions: []shipper.ReleaseCondition{ + ReleaseConditionUnblocked, + ReleaseConditionScheduled, + { + Type: shipper.ReleaseConditionTypeStrategyExecuted, + Status: corev1.ConditionFalse, + Reason: StrategyExecutionFailed, + Message: fmt.Sprintf( + "Release %q target step is inconsistent: unexpected value %d (expected: %d)", + fmt.Sprintf("%s/%s", incumbent.Namespace, incumbent.Name), + incumbent.Spec.TargetStep, StepFullOn, + ), }, }, } - patch, _ := json.Marshal(newStatus) - action := kubetesting.NewPatchAction(gvr, rel.GetNamespace(), rel.GetName(), types.MergePatchType, patch) - f.actions = append(f.actions, action) - - relKey := fmt.Sprintf("%s/%s", rel.GetNamespace(), rel.GetName()) - f.expectedEvents = []string{ - fmt.Sprintf("Normal StrategyApplied step [%d] finished", step), - fmt.Sprintf(`Normal ReleaseStateTransitioned Release "%s" had its state "WaitingForCapacity" transitioned to "False"`, relKey), - fmt.Sprintf(`Normal ReleaseStateTransitioned Release "%s" had its state "WaitingForCommand" transitioned to "True"`, relKey), - fmt.Sprintf(`Normal ReleaseStateTransitioned Release "%s" had its state "WaitingForInstallation" transitioned to "False"`, relKey), - fmt.Sprintf(`Normal ReleaseStateTransitioned Release "%s" had its state "WaitingForTraffic" transitioned to "False"`, relKey), - `Normal ReleaseConditionChanged [] -> [Scheduled True], [] -> [StrategyExecuted True]`, + // NOTE(jgreff): although we include the contender in + // mgmtClusterObjects, we don't check its status, as we actually don't + // care about it in this particular test case + runReleaseControllerTest(t, mgmtClusterObjects, appClusterObjects, + []releaseControllerTestExpectation{ + { + release: incumbent, + status: expectedStatus, + clusters: []string{cluster.Name}, + }, + }) +} + +func stepify(step int32, conditions []shipper.ReleaseStrategyCondition) []shipper.ReleaseStrategyCondition { + for i, _ := range conditions { + conditions[i].Step = step } + + return conditions } -func buildExpectedActions(release *shipper.Release, clusters []*shipper.Cluster) []kubetesting.Action { +func runReleaseControllerTest( + t *testing.T, + mgmtClusterObjects []runtime.Object, + appClusterObjects map[string][]runtime.Object, + expectations []releaseControllerTestExpectation, +) { + f := shippertesting.NewControllerTestFixture() - clusterNames := make([]string, 0, len(clusters)) - for _, cluster := range clusters { - clusterNames = append(clusterNames, cluster.GetName()) - } - sort.Strings(clusterNames) + clusterNames := []string{} + for clusterName, objects := range appClusterObjects { + cluster := f.AddNamedCluster(clusterName) + clusterNames = append(clusterNames, clusterName) - installationTarget := &shipper.InstallationTarget{ - ObjectMeta: metav1.ObjectMeta{ - Name: release.GetName(), - Namespace: release.GetNamespace(), - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: release.APIVersion, - Kind: release.Kind, - Name: release.Name, - UID: release.UID, - }, - }, - Labels: map[string]string{ - shipper.AppLabel: release.OwnerReferences[0].Name, - shipper.ReleaseLabel: release.GetName(), - }, - }, - Spec: shipper.InstallationTargetSpec{ - Clusters: clusterNames, - CanOverride: true, - Chart: release.Spec.Environment.Chart, - Values: release.Spec.Environment.Values, - }, - } + for _, object := range objects { + cluster.ShipperClient.Tracker().Add(object) + } - clusterCapacityTargets := make([]shipper.ClusterCapacityTarget, 0, len(clusters)) - for _, cluster := range clusters { - clusterCapacityTargets = append( - clusterCapacityTargets, - shipper.ClusterCapacityTarget{ - Name: cluster.GetName(), - Percent: 0, - TotalReplicaCount: 12, - }) + _, err := f.ClusterClientStore.GetApplicationClusterClientset(clusterName, AgentName) + if err != nil { + panic(err) + } } - capacityTarget := &shipper.CapacityTarget{ - ObjectMeta: metav1.ObjectMeta{ - Name: release.Name, - Namespace: release.GetNamespace(), - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: release.APIVersion, - Kind: release.Kind, - Name: release.Name, - UID: release.UID, - }, - }, - Labels: map[string]string{ - shipper.AppLabel: release.OwnerReferences[0].Name, - shipper.ReleaseLabel: release.GetName(), - }, - }, - Spec: shipper.CapacityTargetSpec{ - Clusters: clusterCapacityTargets, - }, - } + sort.Strings(clusterNames) - clusterTrafficTargets := make([]shipper.ClusterTrafficTarget, 0, len(clusters)) - for _, cluster := range clusters { - clusterTrafficTargets = append( - clusterTrafficTargets, - shipper.ClusterTrafficTarget{ - Name: cluster.GetName(), - }) + for _, object := range mgmtClusterObjects { + f.ShipperClient.Tracker().Add(object) } - trafficTarget := &shipper.TrafficTarget{ - ObjectMeta: metav1.ObjectMeta{ - Name: release.Name, - Namespace: release.GetNamespace(), - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: release.APIVersion, - Kind: release.Kind, - Name: release.Name, - UID: release.UID, - }, - }, - Labels: map[string]string{ - shipper.AppLabel: release.OwnerReferences[0].Name, - shipper.ReleaseLabel: release.GetName(), - }, - }, - Spec: shipper.TrafficTargetSpec{ - Clusters: clusterTrafficTargets, - }, - } + runController(f) - actions := []kubetesting.Action{ - kubetesting.NewCreateAction( - shipper.SchemeGroupVersion.WithResource("installationtargets"), - release.GetNamespace(), - installationTarget), - kubetesting.NewCreateAction( - shipper.SchemeGroupVersion.WithResource("traffictargets"), - release.GetNamespace(), - trafficTarget), - kubetesting.NewCreateAction( - shipper.SchemeGroupVersion.WithResource("capacitytargets"), - release.GetNamespace(), - capacityTarget, - ), - } - return actions -} + relGVR := shipper.SchemeGroupVersion.WithResource("releases") + for _, expectation := range expectations { + initialRel := expectation.release + relKey := fmt.Sprintf("%s/%s", initialRel.Namespace, initialRel.Name) + object, err := f.ShipperClient.Tracker().Get(relGVR, initialRel.Namespace, initialRel.Name) + if err != nil { + t.Errorf("could not Get Release %q: %s", relKey, err) + continue + } -func (f *fixture) expectAssociatedObjectsCreated(release *shipper.Release, clusters []*shipper.Cluster) { - f.filter = f.filter.Extend( - actionfilter{ - []string{"create"}, - []string{"installationtargets", "traffictargets", "capacitytargets"}, - }) + rel := object.(*shipper.Release) - relKey := fmt.Sprintf("%s/%s", release.GetNamespace(), release.GetName()) - f.expectedEvents = append(f.expectedEvents, - fmt.Sprintf( - "Normal ReleaseScheduled Created InstallationTarget \"%s\"", - relKey, - ), - fmt.Sprintf( - "Normal ReleaseScheduled Created TrafficTarget \"%s\"", - relKey, - ), - fmt.Sprintf( - "Normal ReleaseScheduled Created CapacityTarget \"%s\"", - relKey, - ), - "Normal ReleaseConditionChanged [] -> [Scheduled True], [] -> [StrategyExecuted True]", - ) -} + expectedSelectedClusters := strings.Join(expectation.clusters, ",") + selectedClusters, ok := rel.Annotations[shipper.ReleaseClustersAnnotation] + if !ok && len(expectation.clusters) > 0 { + t.Errorf("whoops") + } -func (f *fixture) expectReleaseScheduled(release *shipper.Release, clusters []*shipper.Cluster) { - clusterNames := make([]string, 0, len(clusters)) - for _, cluster := range clusters { - clusterNames = append(clusterNames, cluster.GetName()) - } - sort.Strings(clusterNames) - clusterNamesStr := strings.Join(clusterNames, ",") - - expected := release.DeepCopy() - expected.Annotations[shipper.ReleaseClustersAnnotation] = clusterNamesStr - expected.Status.Conditions = []shipper.ReleaseCondition{ - {Type: shipper.ReleaseConditionTypeBlocked, Status: corev1.ConditionFalse}, - {Type: shipper.ReleaseConditionTypeScheduled, Status: corev1.ConditionTrue}, - {Type: shipper.ReleaseConditionTypeStrategyExecuted, Status: corev1.ConditionTrue}, + if expectedSelectedClusters != selectedClusters { + t.Errorf("expected release %q to have clusters %q but got %q", + relKey, expectedSelectedClusters, selectedClusters) + } + + actualStatus := rel.Status + eq, diff := shippertesting.DeepEqualDiff(expectation.status, actualStatus) + if !eq { + t.Errorf( + "Release %q has Status different from expected:\n%s", + relKey, diff) + continue + } } +} - f.filter = f.filter.Extend(actionfilter{[]string{"update"}, []string{"releases"}}) - f.actions = append(f.actions, buildExpectedActions(expected, clusters)...) - f.actions = append(f.actions, kubetesting.NewUpdateAction( - shipper.SchemeGroupVersion.WithResource("releases"), - release.GetNamespace(), - expected)) - - relKey := fmt.Sprintf("%s/%s", release.GetNamespace(), release.GetName()) - f.expectedEvents = append(f.expectedEvents, - fmt.Sprintf( - "Normal ClustersSelected Set clusters for \"%s\" to %s", - relKey, - clusterNamesStr, - ), +func runController(f *shippertesting.ControllerTestFixture) { + controller := NewController( + f.ShipperClient, + f.ClusterClientStore, + f.ShipperInformerFactory, + shippertesting.LocalFetchChart, + f.Recorder, ) -} -func (f *fixture) expectCapacityStatusPatch(step int32, ct *shipper.CapacityTarget, r *shipper.Release, value uint, totalReplicaCount uint, role role) { - f.filter = f.filter.Extend(actionfilter{ - []string{"patch"}, - []string{"releases", "capacitytargets"}, - }) - - gvr := shipper.SchemeGroupVersion.WithResource("capacitytargets") - newSpec := map[string]interface{}{ - "spec": shipper.CapacityTargetSpec{ - Clusters: []shipper.ClusterCapacityTarget{ - {Name: "minikube", Percent: int32(value), TotalReplicaCount: int32(totalReplicaCount)}, - }, - }, - } - patch, _ := json.Marshal(newSpec) - action := kubetesting.NewPatchAction(gvr, ct.GetNamespace(), ct.GetName(), types.MergePatchType, patch) - f.actions = append(f.actions, action) + stopCh := make(chan struct{}) + defer close(stopCh) - var strategyConditions conditions.StrategyConditionsMap + f.Run(stopCh) - if role == Contender { - strategyConditions = conditions.NewStrategyConditions( - shipper.ReleaseStrategyCondition{ - Type: shipper.StrategyConditionContenderAchievedInstallation, - Status: corev1.ConditionTrue, - Step: step, - }, - shipper.ReleaseStrategyCondition{ - Type: shipper.StrategyConditionContenderAchievedCapacity, - Status: corev1.ConditionFalse, - Step: step, - Reason: ClustersNotReady, - Message: fmt.Sprintf( - "release %q hasn't achieved capacity in clusters: [%s]. for more details try `kubectl describe ct %s`", - ct.Name, - "minikube", - ct.Name, - ), - }, - ) - } else { - strategyConditions = conditions.NewStrategyConditions( - shipper.ReleaseStrategyCondition{ - Type: shipper.StrategyConditionContenderAchievedInstallation, - Status: corev1.ConditionTrue, - Step: step, - }, - shipper.ReleaseStrategyCondition{ - Type: shipper.StrategyConditionContenderAchievedCapacity, - Status: corev1.ConditionTrue, - Step: step, - }, - shipper.ReleaseStrategyCondition{ - Type: shipper.StrategyConditionContenderAchievedTraffic, - Status: corev1.ConditionTrue, - Step: step, - }, - shipper.ReleaseStrategyCondition{ - Type: shipper.StrategyConditionIncumbentAchievedTraffic, - Status: corev1.ConditionTrue, - Step: step, - }, - shipper.ReleaseStrategyCondition{ - Type: shipper.StrategyConditionIncumbentAchievedCapacity, - Status: corev1.ConditionFalse, - Step: step, - Reason: ClustersNotReady, - Message: fmt.Sprintf( - "release %q hasn't achieved capacity in clusters: [%s]. for more details try `kubectl describe ct %s`", - ct.Name, - "minikube", - ct.Name, - ), - }, - ) - } - - newStatus := map[string]interface{}{ - "status": shipper.ReleaseStatus{ - Strategy: &shipper.ReleaseStrategyStatus{ - Conditions: strategyConditions.AsReleaseStrategyConditions(), - State: strategyConditions.AsReleaseStrategyState(step, true, false, true), - }, - }, - } - patch, _ = json.Marshal(newStatus) - action = kubetesting.NewPatchAction( - shipper.SchemeGroupVersion.WithResource("releases"), - r.GetNamespace(), - r.GetName(), - types.MergePatchType, - patch) - f.actions = append(f.actions, action) - - f.expectedEvents = []string{ - "Normal ReleaseConditionChanged [] -> [Scheduled True], [] -> [StrategyExecuted True]", - } -} - -func (f *fixture) expectTrafficStatusPatch(step int32, tt *shipper.TrafficTarget, r *shipper.Release, value uint32, role role) { - f.filter = f.filter.Extend(actionfilter{ - []string{"patch"}, - []string{"releases", "traffictargets"}, - }) - - gvr := shipper.SchemeGroupVersion.WithResource("traffictargets") - newSpec := map[string]interface{}{ - "spec": shipper.TrafficTargetSpec{ - Clusters: []shipper.ClusterTrafficTarget{ - {Name: "minikube", Weight: value}, - }, - }, - } - patch, _ := json.Marshal(newSpec) - action := kubetesting.NewPatchAction(gvr, tt.GetNamespace(), tt.GetName(), types.MergePatchType, patch) - f.actions = append(f.actions, action) - - var strategyConditions conditions.StrategyConditionsMap - - if role == Contender { - strategyConditions = conditions.NewStrategyConditions( - shipper.ReleaseStrategyCondition{ - Type: shipper.StrategyConditionContenderAchievedInstallation, - Status: corev1.ConditionTrue, - Step: step, - }, - shipper.ReleaseStrategyCondition{ - Type: shipper.StrategyConditionContenderAchievedCapacity, - Status: corev1.ConditionTrue, - Step: step, - }, - shipper.ReleaseStrategyCondition{ - Type: shipper.StrategyConditionContenderAchievedTraffic, - Status: corev1.ConditionFalse, - Step: step, - Reason: ClustersNotReady, - Message: fmt.Sprintf( - "release %q hasn't achieved traffic in clusters: [%s]. for more details try `kubectl describe tt %s`", - tt.Name, - "minikube", - tt.Name, - ), - }, - ) - } else { - strategyConditions = conditions.NewStrategyConditions( - shipper.ReleaseStrategyCondition{ - Type: shipper.StrategyConditionContenderAchievedInstallation, - Status: corev1.ConditionTrue, - Step: step, - }, - shipper.ReleaseStrategyCondition{ - Type: shipper.StrategyConditionContenderAchievedCapacity, - Status: corev1.ConditionTrue, - Step: step, - }, - shipper.ReleaseStrategyCondition{ - Type: shipper.StrategyConditionContenderAchievedTraffic, - Status: corev1.ConditionTrue, - Step: step, - }, - shipper.ReleaseStrategyCondition{ - Type: shipper.StrategyConditionIncumbentAchievedTraffic, - Status: corev1.ConditionFalse, - Step: step, - Reason: ClustersNotReady, - Message: fmt.Sprintf( - "release %q hasn't achieved traffic in clusters: [%s]. for more details try `kubectl describe tt %s`", - tt.Name, - "minikube", - tt.Name, - ), - }, - ) - } - - newStatus := map[string]interface{}{ - "status": shipper.ReleaseStatus{ - Strategy: &shipper.ReleaseStrategyStatus{ - Conditions: strategyConditions.AsReleaseStrategyConditions(), - State: strategyConditions.AsReleaseStrategyState(step, true, false, true), - }, - }, - } - patch, _ = json.Marshal(newStatus) - action = kubetesting.NewPatchAction( - shipper.SchemeGroupVersion.WithResource("releases"), - r.GetNamespace(), - r.GetName(), - types.MergePatchType, - patch) - f.actions = append(f.actions, action) - - f.expectedEvents = []string{ - "Normal ReleaseConditionChanged [] -> [Scheduled True], [] -> [StrategyExecuted True]", - } -} - -func (f *fixture) expectReleaseReleased(rel *shipper.Release, targetStep int32) { - f.filter = f.filter.Extend(actionfilter{ - []string{"patch"}, - []string{"releases"}, - }) - - gvr := shipper.SchemeGroupVersion.WithResource("releases") - newStatus := map[string]interface{}{ - "status": shipper.ReleaseStatus{ - Strategy: &shipper.ReleaseStrategyStatus{ - State: shipper.ReleaseStrategyState{ - WaitingForInstallation: shipper.StrategyStateFalse, - WaitingForCommand: shipper.StrategyStateFalse, - WaitingForTraffic: shipper.StrategyStateFalse, - WaitingForCapacity: shipper.StrategyStateFalse, - }, - // The following conditions are sorted alphabetically by Type - Conditions: []shipper.ReleaseStrategyCondition{ - { - Type: shipper.StrategyConditionContenderAchievedCapacity, - Status: corev1.ConditionTrue, - Step: targetStep, - }, - { - Type: shipper.StrategyConditionContenderAchievedInstallation, - Status: corev1.ConditionTrue, - Step: targetStep, - }, - { - Type: shipper.StrategyConditionContenderAchievedTraffic, - Status: corev1.ConditionTrue, - Step: targetStep, - }, - { - Type: shipper.StrategyConditionIncumbentAchievedCapacity, - Status: corev1.ConditionTrue, - Step: targetStep, - }, - { - Type: shipper.StrategyConditionIncumbentAchievedTraffic, - Status: corev1.ConditionTrue, - Step: targetStep, - }, - }, - }, - }, - } - - patch, _ := json.Marshal(newStatus) - action := kubetesting.NewPatchAction(gvr, rel.GetNamespace(), rel.GetName(), types.MergePatchType, patch) - - f.actions = append(f.actions, action) - - f.expectedEvents = []string{ - fmt.Sprintf("Normal StrategyApplied step [%d] finished", targetStep), - fmt.Sprintf(`Normal ReleaseStateTransitioned Release "%s/%s" had its state "WaitingForCapacity" transitioned to "False"`, rel.GetNamespace(), rel.GetName()), - fmt.Sprintf(`Normal ReleaseStateTransitioned Release "%s/%s" had its state "WaitingForCommand" transitioned to "False"`, rel.GetNamespace(), rel.GetName()), - fmt.Sprintf(`Normal ReleaseStateTransitioned Release "%s/%s" had its state "WaitingForInstallation" transitioned to "False"`, rel.GetNamespace(), rel.GetName()), - fmt.Sprintf(`Normal ReleaseStateTransitioned Release "%s/%s" had its state "WaitingForTraffic" transitioned to "False"`, rel.GetNamespace(), rel.GetName()), - "Normal ReleaseConditionChanged [] -> [Scheduled True], [] -> [StrategyExecuted True], [] -> [Complete True]", - } -} - -// NOTE(btyler): when we add tests to use this function with a wider set of use -// cases, we'll need a "pint32(int32) *int32" func to let us take pointers to literals -func (f *fixture) expectInstallationNotReady(rel *shipper.Release, achievedStepIndex *int32, targetStepIndex int32, role role) { - f.filter = f.filter.Extend(actionfilter{ - []string{"patch"}, - []string{"releases"}, - }) - - gvr := shipper.SchemeGroupVersion.WithResource("releases") - - // var achievedStep *shipper.AchievedStep - // if achievedStepIndex != nil { - // achievedStep = &shipper.AchievedStep{ - // Step: *achievedStepIndex, - // Name: rel.Spec.Environment.Strategy.Steps[*achievedStepIndex].Name, - // } - // } - - newStatus := map[string]interface{}{ - "status": shipper.ReleaseStatus{ - Strategy: &shipper.ReleaseStrategyStatus{ - State: shipper.ReleaseStrategyState{ - WaitingForInstallation: shipper.StrategyStateTrue, - WaitingForCommand: shipper.StrategyStateFalse, - WaitingForTraffic: shipper.StrategyStateFalse, - WaitingForCapacity: shipper.StrategyStateFalse, - }, - Conditions: []shipper.ReleaseStrategyCondition{ - { - Type: shipper.StrategyConditionContenderAchievedInstallation, - Status: corev1.ConditionFalse, - Reason: ClustersNotReady, - Step: targetStepIndex, - Message: fmt.Sprintf("clusters pending installation: [broken-installation-cluster]. for more details try `kubectl describe it %s`", rel.Name), - }, - }, - }, - }, - } - - patch, _ := json.Marshal(newStatus) - action := kubetesting.NewPatchAction(gvr, rel.GetNamespace(), rel.GetName(), types.MergePatchType, patch) - - f.actions = append(f.actions, action) - - f.expectedEvents = []string{ - fmt.Sprintf(`Normal ReleaseConditionChanged [] -> [Scheduled True], [] -> [StrategyExecuted True]`), - } -} - -func (f *fixture) expectCapacityNotReady(relpair releaseInfoPair, targetStep, achievedStepIndex int32, role role, brokenClusterName string) { - f.filter = f.filter.Extend(actionfilter{ - []string{"patch"}, - []string{"releases"}, - }) - - gvr := shipper.SchemeGroupVersion.WithResource("releases") - - var newStatus map[string]interface{} - - // var achievedStep *shipper.AchievedStep - // if achievedStepIndex != 0 { - // achievedStep = &shipper.AchievedStep{ - // Step: achievedStepIndex, - // Name: rel.Spec.Environment.Strategy.Steps[achievedStepIndex].Name, - // } - // } - - rel := relpair.contender.release - - if role == Contender { - newStatus = map[string]interface{}{ - "status": shipper.ReleaseStatus{ - Strategy: &shipper.ReleaseStrategyStatus{ - State: shipper.ReleaseStrategyState{ - WaitingForInstallation: shipper.StrategyStateFalse, - WaitingForCommand: shipper.StrategyStateFalse, - WaitingForTraffic: shipper.StrategyStateFalse, - WaitingForCapacity: shipper.StrategyStateTrue, - }, - Conditions: []shipper.ReleaseStrategyCondition{ - { - Type: shipper.StrategyConditionContenderAchievedCapacity, - Status: corev1.ConditionFalse, - Reason: ClustersNotReady, - Message: fmt.Sprintf( - "release %q hasn't achieved capacity in clusters: [%s]. for more details try `kubectl describe ct %s`", - relpair.contender.release.Name, - brokenClusterName, - relpair.contender.capacityTarget.Name, - ), - Step: targetStep, - }, - { - Type: shipper.StrategyConditionContenderAchievedInstallation, - Status: corev1.ConditionTrue, - Step: targetStep, - }, - }, - }, - }, - } - } else { - newStatus = map[string]interface{}{ - "status": shipper.ReleaseStatus{ - Strategy: &shipper.ReleaseStrategyStatus{ - State: shipper.ReleaseStrategyState{ - WaitingForInstallation: shipper.StrategyStateFalse, - WaitingForCommand: shipper.StrategyStateFalse, - WaitingForTraffic: shipper.StrategyStateFalse, - WaitingForCapacity: shipper.StrategyStateTrue, - }, - Conditions: []shipper.ReleaseStrategyCondition{ - { - Type: shipper.StrategyConditionContenderAchievedCapacity, - Status: corev1.ConditionTrue, - Step: targetStep, - }, - { - Type: shipper.StrategyConditionContenderAchievedInstallation, - Status: corev1.ConditionTrue, - Step: targetStep, - }, - { - Type: shipper.StrategyConditionContenderAchievedTraffic, - Status: corev1.ConditionTrue, - Step: targetStep, - }, - { - Type: shipper.StrategyConditionIncumbentAchievedCapacity, - Status: corev1.ConditionFalse, - Reason: ClustersNotReady, - Step: targetStep, - Message: fmt.Sprintf( - "release %q hasn't achieved capacity in clusters: [%s]. for more details try `kubectl describe ct %s`", - relpair.incumbent.release.Name, - brokenClusterName, - relpair.incumbent.capacityTarget.Name, - ), - }, - { - Type: shipper.StrategyConditionIncumbentAchievedTraffic, - Status: corev1.ConditionTrue, - Step: targetStep, - }, - }, - }, - }, - } - } - - patch, _ := json.Marshal(newStatus) - action := kubetesting.NewPatchAction(gvr, rel.GetNamespace(), rel.GetName(), types.MergePatchType, patch) - - f.actions = append(f.actions, action) - - f.expectedEvents = []string{ - fmt.Sprintf(`Normal ReleaseConditionChanged [] -> [Scheduled True], [] -> [StrategyExecuted True]`), - } -} - -func (f *fixture) expectTrafficNotReady(relpair releaseInfoPair, targetStep, achievedStepIndex int32, role role, brokenClusterName string) { - gvr := shipper.SchemeGroupVersion.WithResource("releases") - var newStatus map[string]interface{} - - // var achievedStep *shipper.AchievedStep - // if achievedStepIndex != 0 { - // achievedStep = &shipper.AchievedStep{ - // Step: achievedStepIndex, - // Name: rel.Spec.Environment.Strategy.Steps[achievedStepIndex].Name, - // } - // } - - rel := relpair.contender.release - - if role == Contender { - newStatus = map[string]interface{}{ - "status": shipper.ReleaseStatus{ - Strategy: &shipper.ReleaseStrategyStatus{ - State: shipper.ReleaseStrategyState{ - WaitingForInstallation: shipper.StrategyStateFalse, - WaitingForCommand: shipper.StrategyStateFalse, - WaitingForTraffic: shipper.StrategyStateTrue, - WaitingForCapacity: shipper.StrategyStateFalse, - }, - Conditions: []shipper.ReleaseStrategyCondition{ - { - Type: shipper.StrategyConditionContenderAchievedCapacity, - Status: corev1.ConditionTrue, - Step: targetStep, - }, - { - Type: shipper.StrategyConditionContenderAchievedInstallation, - Status: corev1.ConditionTrue, - Step: targetStep, - }, - { - Type: shipper.StrategyConditionContenderAchievedTraffic, - Status: corev1.ConditionFalse, - Reason: ClustersNotReady, - Message: fmt.Sprintf( - "release %q hasn't achieved traffic in clusters: [%s]. for more details try `kubectl describe tt %s`", - relpair.contender.release.Name, - brokenClusterName, - relpair.contender.capacityTarget.Name, - ), - Step: targetStep, - }, - }, - }, - }, + for controller.processNextWorkItem() { + if controller.workqueue.Len() == 0 { + time.Sleep(200 * time.Millisecond) } - } else { - newStatus = map[string]interface{}{ - "status": shipper.ReleaseStatus{ - Strategy: &shipper.ReleaseStrategyStatus{ - State: shipper.ReleaseStrategyState{ - WaitingForInstallation: shipper.StrategyStateFalse, - WaitingForCommand: shipper.StrategyStateFalse, - WaitingForTraffic: shipper.StrategyStateTrue, - WaitingForCapacity: shipper.StrategyStateFalse, - }, - Conditions: []shipper.ReleaseStrategyCondition{ - { - Type: shipper.StrategyConditionContenderAchievedCapacity, - Status: corev1.ConditionTrue, - Step: targetStep, - }, - { - Type: shipper.StrategyConditionContenderAchievedInstallation, - Status: corev1.ConditionTrue, - Step: targetStep, - }, - { - Type: shipper.StrategyConditionContenderAchievedTraffic, - Status: corev1.ConditionTrue, - Step: targetStep, - }, - { - Type: shipper.StrategyConditionIncumbentAchievedTraffic, - Status: corev1.ConditionFalse, - Reason: ClustersNotReady, - Message: fmt.Sprintf( - "release %q hasn't achieved traffic in clusters: [%s]. for more details try `kubectl describe tt %s`", - relpair.incumbent.release.Name, - brokenClusterName, - relpair.incumbent.capacityTarget.Name, - ), - Step: targetStep, - }, - }, - }, - }, + if controller.workqueue.Len() == 0 { + return } } - - patch, _ := json.Marshal(newStatus) - action := kubetesting.NewPatchAction(gvr, rel.GetNamespace(), rel.GetName(), types.MergePatchType, patch) - - f.actions = append(f.actions, action) - - f.expectedEvents = []string{} -} - -func TestContenderReleasePhaseIsWaitingForCommandForInitialStepState(t *testing.T) { - namespace := "test-namespace" - app := buildApplication(namespace, "test-app") - cluster := buildCluster("minikube") - - incumbentName, contenderName := "test-incumbent", "test-contender" - app.Status.History = []string{incumbentName, contenderName} - f := newFixture(t, app.DeepCopy(), cluster.DeepCopy()) - f.cycles = 1 - - totalReplicaCount := int32(10) - incumbent := f.buildIncumbent(namespace, incumbentName, totalReplicaCount) - contender := f.buildContender(namespace, contenderName, totalReplicaCount) - - contender.capacityTarget.Spec.Clusters[0].Percent = 1 - incumbent.capacityTarget.Spec.Clusters[0].Percent = 100 - - f.addObjects( - contender.release.DeepCopy(), - contender.installationTarget.DeepCopy(), - contender.capacityTarget.DeepCopy(), - contender.trafficTarget.DeepCopy(), - - incumbent.release.DeepCopy(), - incumbent.installationTarget.DeepCopy(), - incumbent.capacityTarget.DeepCopy(), - incumbent.trafficTarget.DeepCopy(), - ) - var step int32 = 0 - f.expectReleaseWaitingForCommand(contender.release, step) - f.run() -} - -func TestContenderDoNothingClusterInstallationNotReady(t *testing.T) { - namespace := "test-namespace" - incumbentName, contenderName := "test-incumbent", "test-contender" - app := buildApplication(namespace, "test-app") - cluster := buildCluster("minikube") - brokenCluster := buildCluster("broken-installation-cluster") - - f := newFixture(t, app.DeepCopy(), cluster.DeepCopy()) - f.cycles = 1 - - totalReplicaCount := int32(10) - contender := f.buildContender(namespace, contenderName, totalReplicaCount) - incumbent := f.buildIncumbent(namespace, incumbentName, totalReplicaCount) - - addCluster(contender, brokenCluster) - - contender.release.Spec.TargetStep = 0 - - // the fixture creates installation targets in 'installation succeeded' - // status, so we'll break one - contender.installationTarget.Status.Conditions, _ = targetutil.SetTargetCondition( - contender.installationTarget.Status.Conditions, - targetutil.NewTargetCondition( - shipper.TargetConditionTypeReady, - corev1.ConditionFalse, - ClustersNotReady, "[broken-installation-cluster]")) - - f.addObjects( - contender.release.DeepCopy(), - contender.installationTarget.DeepCopy(), - contender.capacityTarget.DeepCopy(), - contender.trafficTarget.DeepCopy(), - - incumbent.release.DeepCopy(), - incumbent.installationTarget.DeepCopy(), - incumbent.capacityTarget.DeepCopy(), - incumbent.trafficTarget.DeepCopy(), - ) - - r := contender.release.DeepCopy() - f.expectInstallationNotReady(r, nil, 0, Contender) - f.run() -} - -func TestContenderDoNothingClusterCapacityNotReady(t *testing.T) { - namespace := "test-namespace" - incumbentName, contenderName := "test-incumbent", "test-contender" - app := buildApplication(namespace, "test-app") - cluster := buildCluster("minikube") - brokenCluster := buildCluster("broken-capacity-cluster") - - f := newFixture(t, app.DeepCopy(), cluster.DeepCopy()) - f.cycles = 1 - - totalReplicaCount := int32(10) - contender := f.buildContender(namespace, contenderName, totalReplicaCount) - incumbent := f.buildIncumbent(namespace, incumbentName, totalReplicaCount) - - addCluster(contender, brokenCluster) - - // We'll set cluster 0 to be all set, but make cluster 1 broken. - contender.release.Spec.TargetStep = 1 - contender.capacityTarget.Spec.Clusters[0].Percent = 50 - contender.capacityTarget.Spec.Clusters[0].TotalReplicaCount = int32(totalReplicaCount) - contender.trafficTarget.Spec.Clusters[0].Weight = 50 - - // No capacity yet. - contender.capacityTarget.Spec.Clusters[1].Percent = 50 - contender.capacityTarget.Spec.Clusters[1].TotalReplicaCount = totalReplicaCount - contender.trafficTarget.Spec.Clusters[1].Weight = 50 - contender.capacityTarget.Status.Conditions, _ = targetutil.SetTargetCondition( - contender.capacityTarget.Status.Conditions, - targetutil.NewTargetCondition( - shipper.TargetConditionTypeReady, - corev1.ConditionFalse, - ClustersNotReady, "[broken-capacity-cluster]")) - - incumbent.trafficTarget.Spec.Clusters[0].Weight = 50 - incumbent.capacityTarget.Spec.Clusters[0].Percent = 50 - incumbent.capacityTarget.Spec.Clusters[0].TotalReplicaCount = totalReplicaCount - - f.addObjects( - brokenCluster.DeepCopy(), - - contender.release.DeepCopy(), - contender.installationTarget.DeepCopy(), - contender.capacityTarget.DeepCopy(), - contender.trafficTarget.DeepCopy(), - - incumbent.release.DeepCopy(), - incumbent.installationTarget.DeepCopy(), - incumbent.capacityTarget.DeepCopy(), - incumbent.trafficTarget.DeepCopy(), - ) - - relpair := releaseInfoPair{ - contender: contender, - incumbent: incumbent, - } - f.expectCapacityNotReady(relpair, 1, 0, Contender, brokenCluster.Name) - f.run() -} - -func TestContenderDoNothingClusterTrafficNotReady(t *testing.T) { - namespace := "test-namespace" - incumbentName, contenderName := "test-incumbent", "test-contender" - app := buildApplication(namespace, "test-app") - cluster := buildCluster("minikube") - brokenCluster := buildCluster("broken-traffic-cluster") - - f := newFixture(t, app.DeepCopy(), cluster.DeepCopy()) - f.cycles = 1 - - totalReplicaCount := int32(10) - contender := f.buildContender(namespace, contenderName, totalReplicaCount) - incumbent := f.buildIncumbent(namespace, incumbentName, totalReplicaCount) - - contender.release.Annotations[shipper.ReleaseClustersAnnotation] = cluster.GetName() - condScheduled := releaseutil.NewReleaseCondition(shipper.ReleaseConditionTypeScheduled, corev1.ConditionTrue, "", "") - releaseutil.SetReleaseCondition(&contender.release.Status, *condScheduled) - condStrategyExecuted := releaseutil.NewReleaseCondition(shipper.ReleaseConditionTypeStrategyExecuted, corev1.ConditionTrue, "", "") - releaseutil.SetReleaseCondition(&contender.release.Status, *condStrategyExecuted) - - addCluster(contender, brokenCluster) - - // We'll set cluster 0 to be all set, but make cluster 1 broken. - contender.release.Spec.TargetStep = 1 - contender.capacityTarget.Spec.Clusters[0].Percent = 50 - contender.capacityTarget.Spec.Clusters[0].TotalReplicaCount = totalReplicaCount - contender.trafficTarget.Spec.Clusters[0].Weight = 50 - - contender.capacityTarget.Spec.Clusters[1].Percent = 50 - contender.capacityTarget.Spec.Clusters[1].TotalReplicaCount = totalReplicaCount - - // No traffic yet. - contender.trafficTarget.Spec.Clusters[1].Weight = 50 - contender.trafficTarget.Status.Conditions, _ = targetutil.SetTargetCondition( - contender.trafficTarget.Status.Conditions, - targetutil.NewTargetCondition( - shipper.TargetConditionTypeReady, - corev1.ConditionFalse, - ClustersNotReady, "[broken-traffic-cluster]")) - - incumbent.trafficTarget.Spec.Clusters[0].Weight = 50 - incumbent.capacityTarget.Spec.Clusters[0].Percent = 50 - incumbent.capacityTarget.Spec.Clusters[0].TotalReplicaCount = totalReplicaCount - - f.addObjects( - contender.release.DeepCopy(), - contender.installationTarget.DeepCopy(), - contender.capacityTarget.DeepCopy(), - contender.trafficTarget.DeepCopy(), - - incumbent.release.DeepCopy(), - incumbent.installationTarget.DeepCopy(), - incumbent.capacityTarget.DeepCopy(), - incumbent.trafficTarget.DeepCopy(), - ) - - relpair := releaseInfoPair{ - contender: contender, - incumbent: incumbent, - } - f.expectTrafficNotReady(relpair, 1, 0, Contender, brokenCluster.Name) - f.run() -} - -func TestContenderCapacityShouldIncrease(t *testing.T) { - namespace := "test-namespace" - incumbentName, contenderName := "test-incumbent", "test-contender" - app := buildApplication(namespace, "test-app") - cluster := buildCluster("minikube") - - f := newFixture(t, app.DeepCopy(), cluster.DeepCopy()) - f.cycles = 1 - - totalReplicaCount := int32(10) - contender := f.buildContender(namespace, contenderName, totalReplicaCount) - incumbent := f.buildIncumbent(namespace, incumbentName, totalReplicaCount) - - contender.release.Spec.TargetStep = 1 - - f.addObjects( - contender.release.DeepCopy(), - contender.installationTarget.DeepCopy(), - contender.capacityTarget.DeepCopy(), - contender.trafficTarget.DeepCopy(), - - incumbent.release.DeepCopy(), - incumbent.installationTarget.DeepCopy(), - incumbent.capacityTarget.DeepCopy(), - incumbent.trafficTarget.DeepCopy(), - ) - - ct := contender.capacityTarget.DeepCopy() - r := contender.release.DeepCopy() - f.expectCapacityStatusPatch(contender.release.Spec.TargetStep, ct, r, 50, uint(totalReplicaCount), Contender) - f.run() -} - -func TestContenderCapacityShouldIncreaseWithRolloutBlockOverride(t *testing.T) { - namespace := "test-namespace" - incumbentName, contenderName := "test-incumbent", "test-contender" - app := buildApplication(namespace, "test-app") - rolloutBlock := newRolloutBlock(testRolloutBlockName, namespace) - rolloutBlockKey := fmt.Sprintf("%s/%s", namespace, testRolloutBlockName) - cluster := buildCluster("minikube") - - totalReplicaCount := int32(10) - f := newFixture(t, app.DeepCopy(), cluster.DeepCopy(), rolloutBlock.DeepCopy()) - f.cycles = 1 - - contender := f.buildContender(namespace, contenderName, totalReplicaCount) - contender.release.Annotations[shipper.RolloutBlocksOverrideAnnotation] = rolloutBlockKey - incumbent := f.buildIncumbent(namespace, incumbentName, totalReplicaCount) - incumbent.release.Annotations[shipper.RolloutBlocksOverrideAnnotation] = rolloutBlockKey - - contender.release.Spec.TargetStep = 1 - - incumbent.capacityTarget.Spec.Clusters[0].Percent = 50 - incumbent.capacityTarget.Spec.Clusters[0].TotalReplicaCount = totalReplicaCount - incumbent.trafficTarget.Spec.Clusters[0].Weight = 50 - - f.addObjects( - contender.release.DeepCopy(), - contender.installationTarget.DeepCopy(), - contender.capacityTarget.DeepCopy(), - contender.trafficTarget.DeepCopy(), - - incumbent.release.DeepCopy(), - incumbent.installationTarget.DeepCopy(), - incumbent.capacityTarget.DeepCopy(), - incumbent.trafficTarget.DeepCopy(), - ) - - ct := contender.capacityTarget.DeepCopy() - r := contender.release.DeepCopy() - f.expectCapacityStatusPatch(contender.release.Spec.TargetStep, ct, r, 50, uint(totalReplicaCount), Contender) - overrideEvent := fmt.Sprintf("%s RolloutBlockOverridden %s", corev1.EventTypeNormal, rolloutBlockKey) - f.expectedEvents = append([]string{overrideEvent}, f.expectedEvents...) - f.run() -} - -func TestContenderCapacityShouldNotIncreaseWithRolloutBlock(t *testing.T) { - namespace := "test-namespace" - contenderName := "test-contender-bimbambom" - app := buildApplication(namespace, "test-app") - rolloutBlock := newRolloutBlock(testRolloutBlockName, namespace) - rolloutBlockKey := fmt.Sprintf("%s/%s", namespace, testRolloutBlockName) - cluster := buildCluster("minikube") - - f := newFixture(t, app.DeepCopy(), cluster.DeepCopy(), rolloutBlock.DeepCopy()) - f.cycles = 1 - - totalReplicaCount := int32(3) - contender := f.buildContender(namespace, contenderName, totalReplicaCount) - contender.release.Annotations[shipper.RolloutBlocksOverrideAnnotation] = "" - - contender.release.Spec.TargetStep = 1 - - f.addObjects( - contender.release.DeepCopy(), - contender.installationTarget.DeepCopy(), - contender.capacityTarget.DeepCopy(), - contender.trafficTarget.DeepCopy(), - ) - - expectedContender := contender.release.DeepCopy() - rolloutBlockMessage := fmt.Sprintf("rollout block(s) with name(s) %s exist", rolloutBlockKey) - condBlocked := releaseutil.NewReleaseCondition( - shipper.ReleaseConditionTypeBlocked, - corev1.ConditionTrue, - "RolloutsBlocked", - rolloutBlockMessage) - releaseutil.SetReleaseCondition(&expectedContender.Status, *condBlocked) - - action := kubetesting.NewUpdateAction( - shipper.SchemeGroupVersion.WithResource("releases"), - namespace, - expectedContender) - f.actions = append(f.actions, action) - - f.expectedEvents = append(f.expectedEvents, - fmt.Sprintf("%s RolloutBlocked %s", corev1.EventTypeWarning, rolloutBlockKey), - fmt.Sprintf("Normal ReleaseConditionChanged [Blocked False] -> [Blocked True RolloutsBlocked %s]", rolloutBlockMessage), - ) - f.run() -} - -func TestContenderTrafficShouldIncrease(t *testing.T) { - namespace := "test-namespace" - incumbentName, contenderName := "test-incumbent", "test-contender" - app := buildApplication(namespace, "test-app") - cluster := buildCluster("minikube") - - f := newFixture(t, app.DeepCopy(), cluster.DeepCopy()) - f.cycles = 1 // It only runs a single cycle of processNextReleaseWorkItem - - totalReplicaCount := int32(10) - contender := f.buildContender(namespace, contenderName, totalReplicaCount) - incumbent := f.buildIncumbent(namespace, incumbentName, totalReplicaCount) - - contender.release.Spec.TargetStep = 1 - contender.capacityTarget.Spec.Clusters[0].Percent = 50 - contender.capacityTarget.Spec.Clusters[0].TotalReplicaCount = totalReplicaCount - - incumbent.trafficTarget.Spec.Clusters[0].Weight = 50 - incumbent.capacityTarget.Spec.Clusters[0].Percent = 50 - incumbent.capacityTarget.Spec.Clusters[0].TotalReplicaCount = totalReplicaCount - - f.addObjects( - contender.release.DeepCopy(), - contender.installationTarget.DeepCopy(), - contender.capacityTarget.DeepCopy(), - contender.trafficTarget.DeepCopy(), - - incumbent.release.DeepCopy(), - incumbent.installationTarget.DeepCopy(), - incumbent.capacityTarget.DeepCopy(), - incumbent.trafficTarget.DeepCopy(), - ) - - tt := contender.trafficTarget.DeepCopy() - r := contender.release.DeepCopy() - f.expectTrafficStatusPatch(contender.release.Spec.TargetStep, tt, r, 50, Contender) - f.run() -} - -func TestContenderTrafficShouldIncreaseWithRolloutBlockOverride(t *testing.T) { - namespace := "test-namespace" - incumbentName, contenderName := "test-incumbent", "test-contender" - app := buildApplication(namespace, "test-app") - rolloutBlock := newRolloutBlock(testRolloutBlockName, namespace) - rolloutBlockKey := fmt.Sprintf("%s/%s", namespace, testRolloutBlockName) - cluster := buildCluster("minikube") - - f := newFixture(t, app.DeepCopy(), cluster.DeepCopy(), rolloutBlock.DeepCopy()) - f.cycles = 1 - - totalReplicaCount := int32(10) - contender := f.buildContender(namespace, contenderName, totalReplicaCount) - incumbent := f.buildIncumbent(namespace, incumbentName, totalReplicaCount) - - contender.release.Annotations[shipper.RolloutBlocksOverrideAnnotation] = rolloutBlockKey - incumbent.release.Annotations[shipper.RolloutBlocksOverrideAnnotation] = rolloutBlockKey - - contender.release.Spec.TargetStep = 1 - contender.capacityTarget.Spec.Clusters[0].Percent = 50 - contender.capacityTarget.Spec.Clusters[0].TotalReplicaCount = totalReplicaCount - - incumbent.trafficTarget.Spec.Clusters[0].Weight = 50 - incumbent.capacityTarget.Spec.Clusters[0].Percent = 50 - incumbent.capacityTarget.Spec.Clusters[0].TotalReplicaCount = totalReplicaCount - - f.addObjects( - contender.release.DeepCopy(), - contender.installationTarget.DeepCopy(), - contender.capacityTarget.DeepCopy(), - contender.trafficTarget.DeepCopy(), - - incumbent.release.DeepCopy(), - incumbent.installationTarget.DeepCopy(), - incumbent.capacityTarget.DeepCopy(), - incumbent.trafficTarget.DeepCopy(), - ) - - tt := contender.trafficTarget.DeepCopy() - r := contender.release.DeepCopy() - f.expectTrafficStatusPatch(contender.release.Spec.TargetStep, tt, r, 50, Contender) - overrideEvent := fmt.Sprintf("%s RolloutBlockOverridden %s", corev1.EventTypeNormal, rolloutBlockKey) - f.expectedEvents = append([]string{overrideEvent}, f.expectedEvents...) - f.run() -} - -func TestContenderTrafficShouldNotIncreaseWithRolloutBlock(t *testing.T) { - namespace := "test-namespace" - app := buildApplication(namespace, "test-app") - rolloutBlock := newRolloutBlock(testRolloutBlockName, namespace) - rolloutBlockKey := fmt.Sprintf("%s/%s", namespace, testRolloutBlockName) - cluster := buildCluster("minikube") - contenderName := "contender" - - f := newFixture(t, app.DeepCopy(), cluster.DeepCopy(), rolloutBlock.DeepCopy()) - f.cycles = 1 - - totalReplicaCount := int32(10) - contender := f.buildContender(namespace, contenderName, totalReplicaCount) - - contender.release.Spec.TargetStep = 1 - contender.capacityTarget.Spec.Clusters[0].Percent = 50 - contender.capacityTarget.Spec.Clusters[0].TotalReplicaCount = totalReplicaCount - - f.addObjects( - contender.release.DeepCopy(), - contender.installationTarget.DeepCopy(), - contender.capacityTarget.DeepCopy(), - contender.trafficTarget.DeepCopy(), - ) - - expectedContender := contender.release.DeepCopy() - rolloutBlockMessage := fmt.Sprintf("rollout block(s) with name(s) %s exist", rolloutBlockKey) - condBlocked := releaseutil.NewReleaseCondition( - shipper.ReleaseConditionTypeBlocked, - corev1.ConditionTrue, - "RolloutsBlocked", - rolloutBlockMessage) - releaseutil.SetReleaseCondition(&expectedContender.Status, *condBlocked) - - action := kubetesting.NewUpdateAction( - shipper.SchemeGroupVersion.WithResource("releases"), - namespace, - expectedContender) - f.actions = append(f.actions, action) - - f.expectedEvents = append(f.expectedEvents, - fmt.Sprintf("%s RolloutBlocked %s", corev1.EventTypeWarning, rolloutBlockKey), - fmt.Sprintf("Normal ReleaseConditionChanged [Blocked False] -> [Blocked True RolloutsBlocked %s]", rolloutBlockMessage)) - f.run() -} - -func TestIncumbentTrafficShouldDecrease(t *testing.T) { - namespace := "test-namespace" - incumbentName, contenderName := "test-incumbent", "test-contender" - app := buildApplication(namespace, "test-app") - cluster := buildCluster("minikube") - - f := newFixture(t, app.DeepCopy(), cluster.DeepCopy()) - f.cycles = 1 - - totalReplicaCount := int32(10) - contender := f.buildContender(namespace, contenderName, totalReplicaCount) - incumbent := f.buildIncumbent(namespace, incumbentName, totalReplicaCount) - - contender.release.Spec.TargetStep = 1 - contender.capacityTarget.Spec.Clusters[0].Percent = 50 - contender.capacityTarget.Spec.Clusters[0].TotalReplicaCount = totalReplicaCount - contender.trafficTarget.Spec.Clusters[0].Weight = 50 - contender.release.Status.AchievedStep = &shipper.AchievedStep{Step: 1} - - f.addObjects( - contender.release.DeepCopy(), - contender.installationTarget.DeepCopy(), - contender.capacityTarget.DeepCopy(), - contender.trafficTarget.DeepCopy(), - - incumbent.release.DeepCopy(), - incumbent.installationTarget.DeepCopy(), - incumbent.capacityTarget.DeepCopy(), - incumbent.trafficTarget.DeepCopy(), - ) - - tt := incumbent.trafficTarget.DeepCopy() - r := contender.release.DeepCopy() - f.expectTrafficStatusPatch(contender.release.Spec.TargetStep, tt, r, 50, Incumbent) - f.run() -} - -func TestIncumbentTrafficShouldDecreaseWithRolloutBlockOverride(t *testing.T) { - namespace := "test-namespace" - incumbentName, contenderName := "test-incumbent", "test-contender" - app := buildApplication(namespace, "test-app") - rolloutBlock := newRolloutBlock(testRolloutBlockName, namespace) - rolloutBlockKey := fmt.Sprintf("%s/%s", namespace, testRolloutBlockName) - cluster := buildCluster("minikube") - - f := newFixture(t, app.DeepCopy(), cluster.DeepCopy(), rolloutBlock.DeepCopy()) - f.cycles = 1 // we're looking at a single-step progression - - totalReplicaCount := int32(10) - contender := f.buildContender(namespace, contenderName, totalReplicaCount) - incumbent := f.buildIncumbent(namespace, incumbentName, totalReplicaCount) - - contender.release.Annotations[shipper.RolloutBlocksOverrideAnnotation] = rolloutBlockKey - incumbent.release.Annotations[shipper.RolloutBlocksOverrideAnnotation] = rolloutBlockKey - - contender.release.Spec.TargetStep = 1 - contender.capacityTarget.Spec.Clusters[0].Percent = 50 - contender.capacityTarget.Spec.Clusters[0].TotalReplicaCount = totalReplicaCount - contender.trafficTarget.Spec.Clusters[0].Weight = 50 - - f.addObjects( - contender.release.DeepCopy(), - contender.installationTarget.DeepCopy(), - contender.capacityTarget.DeepCopy(), - contender.trafficTarget.DeepCopy(), - - incumbent.release.DeepCopy(), - incumbent.installationTarget.DeepCopy(), - incumbent.capacityTarget.DeepCopy(), - incumbent.trafficTarget.DeepCopy(), - ) - - tt := incumbent.trafficTarget.DeepCopy() - r := contender.release.DeepCopy() - f.expectTrafficStatusPatch(contender.release.Spec.TargetStep, tt, r, 50, Incumbent) - overrideEvent := fmt.Sprintf("%s RolloutBlockOverridden %s", corev1.EventTypeNormal, rolloutBlockKey) - f.expectedEvents = append([]string{overrideEvent}, f.expectedEvents...) - f.run() -} - -func TestIncumbentTrafficShouldNotDecreaseWithRolloutBlock(t *testing.T) { - namespace := "test-namespace" - app := buildApplication(namespace, "test-app") - rolloutBlock := newRolloutBlock(testRolloutBlockName, namespace) - rolloutBlockKey := fmt.Sprintf("%s/%s", namespace, testRolloutBlockName) - cluster := buildCluster("minikube") - - totalReplicaCount := int32(3) - f := newFixture(t, app.DeepCopy(), cluster.DeepCopy(), rolloutBlock.DeepCopy()) - f.cycles = 1 - - contenderName := "contender" - contender := f.buildContender(namespace, contenderName, totalReplicaCount) - - contender.release.Spec.TargetStep = 1 - contender.capacityTarget.Spec.Clusters[0].Percent = 50 - contender.capacityTarget.Spec.Clusters[0].TotalReplicaCount = totalReplicaCount - contender.trafficTarget.Spec.Clusters[0].Weight = 50 - - f.addObjects( - contender.release.DeepCopy(), - contender.installationTarget.DeepCopy(), - contender.capacityTarget.DeepCopy(), - contender.trafficTarget.DeepCopy(), - ) - - expectedContender := contender.release.DeepCopy() - rolloutBlockMessage := fmt.Sprintf("rollout block(s) with name(s) %s exist", rolloutBlockKey) - condBlocked := releaseutil.NewReleaseCondition( - shipper.ReleaseConditionTypeBlocked, - corev1.ConditionTrue, - "RolloutsBlocked", - rolloutBlockMessage) - releaseutil.SetReleaseCondition(&expectedContender.Status, *condBlocked) - - action := kubetesting.NewUpdateAction( - shipper.SchemeGroupVersion.WithResource("releases"), - namespace, - expectedContender) - f.actions = append(f.actions, action) - - f.expectedEvents = append(f.expectedEvents, - fmt.Sprintf("%s RolloutBlocked %s", corev1.EventTypeWarning, rolloutBlockKey), - fmt.Sprintf("Normal ReleaseConditionChanged [Blocked False] -> [Blocked True RolloutsBlocked %s]", rolloutBlockMessage)) - f.run() -} - -func TestIncumbentCapacityShouldDecrease(t *testing.T) { - namespace := "test-namespace" - incumbentName, contenderName := "test-incumbent", "test-contender" - app := buildApplication(namespace, "test-app") - cluster := buildCluster("minikube") - - f := newFixture(t, app.DeepCopy(), cluster.DeepCopy()) - f.cycles = 1 - - totalReplicaCount := int32(10) - contender := f.buildContender(namespace, contenderName, totalReplicaCount) - incumbent := f.buildIncumbent(namespace, incumbentName, totalReplicaCount) - - contender.release.Spec.TargetStep = 1 - contender.capacityTarget.Spec.Clusters[0].Percent = 50 - contender.capacityTarget.Spec.Clusters[0].TotalReplicaCount = totalReplicaCount - contender.trafficTarget.Spec.Clusters[0].Weight = 50 - contender.release.Status.AchievedStep = &shipper.AchievedStep{Step: 1} - - incumbent.trafficTarget.Spec.Clusters[0].Weight = 50 - - f.addObjects( - contender.release.DeepCopy(), - contender.installationTarget.DeepCopy(), - contender.capacityTarget.DeepCopy(), - contender.trafficTarget.DeepCopy(), - - incumbent.release.DeepCopy(), - incumbent.installationTarget.DeepCopy(), - incumbent.capacityTarget.DeepCopy(), - incumbent.trafficTarget.DeepCopy(), - ) - - tt := incumbent.capacityTarget.DeepCopy() - r := contender.release.DeepCopy() - f.expectCapacityStatusPatch(contender.release.Spec.TargetStep, tt, r, 50, uint(totalReplicaCount), Incumbent) - f.run() -} - -func TestIncumbentCapacityShouldDecreaseWithRolloutBlockOverride(t *testing.T) { - namespace := "test-namespace" - incumbentName, contenderName := "test-incumbent", "test-contender" - app := buildApplication(namespace, "test-app") - rolloutBlock := newRolloutBlock(testRolloutBlockName, namespace) - rolloutBlockKey := fmt.Sprintf("%s/%s", namespace, testRolloutBlockName) - cluster := buildCluster("minikube") - - f := newFixture(t, app.DeepCopy(), cluster.DeepCopy(), rolloutBlock.DeepCopy()) - f.cycles = 1 - - totalReplicaCount := int32(3) - contender := f.buildContender(namespace, contenderName, totalReplicaCount) - incumbent := f.buildIncumbent(namespace, incumbentName, totalReplicaCount) - - contender.release.Annotations[shipper.RolloutBlocksOverrideAnnotation] = rolloutBlockKey - incumbent.release.Annotations[shipper.RolloutBlocksOverrideAnnotation] = rolloutBlockKey - - contender.release.Spec.TargetStep = 1 - contender.capacityTarget.Spec.Clusters[0].Percent = 50 - contender.capacityTarget.Spec.Clusters[0].TotalReplicaCount = totalReplicaCount - contender.trafficTarget.Spec.Clusters[0].Weight = 50 - - incumbent.trafficTarget.Spec.Clusters[0].Weight = 50 - - f.addObjects( - contender.release.DeepCopy(), - contender.installationTarget.DeepCopy(), - contender.capacityTarget.DeepCopy(), - contender.trafficTarget.DeepCopy(), - - incumbent.release.DeepCopy(), - incumbent.installationTarget.DeepCopy(), - incumbent.capacityTarget.DeepCopy(), - incumbent.trafficTarget.DeepCopy(), - ) - - tt := incumbent.capacityTarget.DeepCopy() - r := contender.release.DeepCopy() - f.expectCapacityStatusPatch(contender.release.Spec.TargetStep, tt, r, 50, uint(totalReplicaCount), Incumbent) - overrideEvent := fmt.Sprintf("%s RolloutBlockOverridden %s", corev1.EventTypeNormal, rolloutBlockKey) - f.expectedEvents = append([]string{overrideEvent}, f.expectedEvents...) - f.run() -} - -func TestIncumbentCapacityShouldNotDecreaseWithRolloutBlock(t *testing.T) { - namespace := "test-namespace" - contenderName := "test-contender" - app := buildApplication(namespace, "test-app") - rolloutBlock := newRolloutBlock(testRolloutBlockName, namespace) - rolloutBlockKey := fmt.Sprintf("%s/%s", namespace, testRolloutBlockName) - cluster := buildCluster("minikube") - - f := newFixture(t, app.DeepCopy(), cluster.DeepCopy(), rolloutBlock.DeepCopy()) - f.cycles = 1 - - totalReplicaCount := int32(3) - contender := f.buildContender(namespace, contenderName, totalReplicaCount) - - contender.release.Spec.TargetStep = 1 - contender.capacityTarget.Spec.Clusters[0].Percent = 50 - contender.capacityTarget.Spec.Clusters[0].TotalReplicaCount = totalReplicaCount - contender.trafficTarget.Spec.Clusters[0].Weight = 50 - - f.addObjects( - contender.release.DeepCopy(), - contender.installationTarget.DeepCopy(), - contender.capacityTarget.DeepCopy(), - contender.trafficTarget.DeepCopy(), - ) - - expectedContender := contender.release.DeepCopy() - rolloutBlockMessage := fmt.Sprintf("rollout block(s) with name(s) %s exist", rolloutBlockKey) - condBlocked := releaseutil.NewReleaseCondition( - shipper.ReleaseConditionTypeBlocked, - corev1.ConditionTrue, - "RolloutsBlocked", - rolloutBlockMessage) - releaseutil.SetReleaseCondition(&expectedContender.Status, *condBlocked) - - action := kubetesting.NewUpdateAction( - shipper.SchemeGroupVersion.WithResource("releases"), - namespace, - expectedContender) - f.actions = append(f.actions, action) - - f.expectedEvents = append(f.expectedEvents, - fmt.Sprintf("%s RolloutBlocked %s", corev1.EventTypeWarning, rolloutBlockKey), - fmt.Sprintf("Normal ReleaseConditionChanged [Blocked False] -> [Blocked True RolloutsBlocked %s]", rolloutBlockMessage), - ) - f.run() -} - -func TestContenderReleasePhaseIsWaitingForCommandForFinalStepState(t *testing.T) { - namespace := "test-namespace" - incumbentName, contenderName := "test-incumbent", "test-contender" - app := buildApplication(namespace, "test-app") - cluster := buildCluster("minikube") - - f := newFixture(t, app.DeepCopy(), cluster.DeepCopy()) - f.cycles = 1 - - totalReplicaCount := int32(10) - contender := f.buildContender(namespace, contenderName, totalReplicaCount) - incumbent := f.buildIncumbent(namespace, incumbentName, totalReplicaCount) - - contender.release.Spec.TargetStep = 1 - contender.capacityTarget.Spec.Clusters[0].Percent = 50 - contender.capacityTarget.Spec.Clusters[0].TotalReplicaCount = totalReplicaCount - contender.trafficTarget.Spec.Clusters[0].Weight = 50 - incumbent.trafficTarget.Spec.Clusters[0].Weight = 50 - incumbent.capacityTarget.Spec.Clusters[0].Percent = 50 - incumbent.capacityTarget.Spec.Clusters[0].TotalReplicaCount = totalReplicaCount - - f.addObjects( - contender.release.DeepCopy(), - contender.installationTarget.DeepCopy(), - contender.capacityTarget.DeepCopy(), - contender.trafficTarget.DeepCopy(), - - incumbent.release.DeepCopy(), - incumbent.installationTarget.DeepCopy(), - incumbent.capacityTarget.DeepCopy(), - incumbent.trafficTarget.DeepCopy(), - ) - - f.expectReleaseWaitingForCommand(contender.release, 1) - f.run() -} - -func TestContenderReleaseIsInstalled(t *testing.T) { - namespace := "test-namespace" - incumbentName, contenderName := "test-incumbent", "test-contender" - app := buildApplication(namespace, "test-app") - cluster := buildCluster("minikube") - - f := newFixture(t, app.DeepCopy(), cluster.DeepCopy()) - f.cycles = 1 - - totalReplicaCount := int32(10) - contender := f.buildContender(namespace, contenderName, totalReplicaCount) - incumbent := f.buildIncumbent(namespace, incumbentName, totalReplicaCount) - - contender.release.Spec.TargetStep = 2 - contender.capacityTarget.Spec.Clusters[0].Percent = 100 - contender.capacityTarget.Spec.Clusters[0].TotalReplicaCount = totalReplicaCount - contender.trafficTarget.Spec.Clusters[0].Weight = 100 - - incumbent.trafficTarget.Spec.Clusters[0].Weight = 0 - incumbent.capacityTarget.Spec.Clusters[0].Percent = 0 - - f.addObjects( - contender.release.DeepCopy(), - contender.installationTarget.DeepCopy(), - contender.capacityTarget.DeepCopy(), - contender.trafficTarget.DeepCopy(), - - incumbent.release.DeepCopy(), - incumbent.installationTarget.DeepCopy(), - incumbent.capacityTarget.DeepCopy(), - incumbent.trafficTarget.DeepCopy(), - ) - - f.expectReleaseReleased(contender.release, 2) - - f.run() -} - -func TestApplicationExposesStrategyFailureIndexOutOfBounds(t *testing.T) { - namespace := "test-namespace" - incumbentName, contenderName := "test-incumbent", "test-contender" - app := buildApplication(namespace, "test-app") - - cluster := buildCluster("minikube") - - f := newFixture(t, app.DeepCopy(), cluster.DeepCopy()) - f.cycles = 1 - - totalReplicaCount := int32(1) - contender := f.buildContender(namespace, contenderName, totalReplicaCount) - incumbent := f.buildIncumbent(namespace, incumbentName, totalReplicaCount) - - missingStepMsg := fmt.Sprintf("failed to execute strategy: \"no step 2 in strategy for Release \\\"%s/%s\\\"\"", contender.release.Namespace, contender.release.Name) - - // We define 2 steps and will intentionally set target step index out of this bound - strategy := shipper.RolloutStrategy{ - Steps: []shipper.RolloutStrategyStep{ - { - Name: "staging", - Capacity: shipper.RolloutStrategyStepValue{Incumbent: 100, Contender: 1}, - Traffic: shipper.RolloutStrategyStepValue{Incumbent: 100, Contender: 0}, - }, - { - Name: "full on", - Capacity: shipper.RolloutStrategyStepValue{Incumbent: 0, Contender: 100}, - Traffic: shipper.RolloutStrategyStepValue{Incumbent: 0, Contender: 100}, - }, - }, - } - - contender.release.Spec.Environment.Strategy = &strategy - contender.release.Spec.TargetStep = 2 // out of bound index - - expectedRel := contender.release.DeepCopy() - expectedRel.Status.Conditions = []shipper.ReleaseCondition{ - { - Type: shipper.ReleaseConditionTypeBlocked, - Status: corev1.ConditionFalse, - }, - { - Type: shipper.ReleaseConditionTypeScheduled, - Status: corev1.ConditionTrue, - }, - { - Type: shipper.ReleaseConditionTypeStrategyExecuted, - Status: corev1.ConditionFalse, - Reason: conditions.StrategyExecutionFailed, - Message: missingStepMsg, - }, - } - - f.addObjects( - contender.release.DeepCopy(), - contender.installationTarget.DeepCopy(), - contender.capacityTarget.DeepCopy(), - contender.trafficTarget.DeepCopy(), - - incumbent.release.DeepCopy(), - incumbent.installationTarget.DeepCopy(), - incumbent.capacityTarget.DeepCopy(), - incumbent.trafficTarget.DeepCopy(), - ) - - f.actions = append(f.actions, kubetesting.NewUpdateAction( - shipper.SchemeGroupVersion.WithResource("releases"), - namespace, - expectedRel)) - - f.filter = f.filter.Extend(actionfilter{ - []string{"update"}, - []string{"releases"}, - }) - f.expectedEvents = append(f.expectedEvents, - fmt.Sprintf("Normal ReleaseConditionChanged [] -> [Scheduled True], [] -> [StrategyExecuted False StrategyExecutionFailed %s]", missingStepMsg)) - - f.run() -} - -func TestApplicationExposesStrategyFailureSuccessorIndexOutOfBounds(t *testing.T) { - namespace := "test-namespace" - incumbentName, contenderName := "test-incumbent", "test-contender" - app := buildApplication(namespace, "test-app") - - cluster := buildCluster("minikube") - - f := newFixture(t, app.DeepCopy(), cluster.DeepCopy()) - // we're testing 2 cycles because we expect contender and incumbent patches - // to be issued independently - f.cycles = 2 - - totalReplicaCount := int32(1) - contender := f.buildContender(namespace, contenderName, totalReplicaCount) - incumbent := f.buildIncumbent(namespace, incumbentName, totalReplicaCount) - - // We define 2 steps and will intentionally set target step index out of this bound - strategyStaging := shipper.RolloutStrategy{ - Steps: []shipper.RolloutStrategyStep{ - { - Name: "staging", - Capacity: shipper.RolloutStrategyStepValue{Incumbent: 100, Contender: 1}, - Traffic: shipper.RolloutStrategyStepValue{Incumbent: 100, Contender: 0}, - }, - { - Name: "full on", - Capacity: shipper.RolloutStrategyStepValue{Incumbent: 0, Contender: 100}, - Traffic: shipper.RolloutStrategyStepValue{Incumbent: 0, Contender: 100}, - }, - }, - } - - // clean up conditions as incumbent helper gives us some extras - incumbent.release.Status.Conditions = []shipper.ReleaseCondition{ - { - Type: shipper.ReleaseConditionTypeBlocked, - Status: corev1.ConditionFalse, - }, - } - - contender.release.Spec.Environment.Strategy = &strategyStaging - contender.release.Spec.TargetStep = 2 // out of bound index - - expectedIncumbent := incumbent.release.DeepCopy() - expectedIncumbent.Status.Conditions = []shipper.ReleaseCondition{ - { - Type: shipper.ReleaseConditionTypeBlocked, - Status: corev1.ConditionFalse, - }, - { - Type: shipper.ReleaseConditionTypeScheduled, - Status: corev1.ConditionTrue, - }, - { - Type: shipper.ReleaseConditionTypeStrategyExecuted, - Status: corev1.ConditionFalse, - Reason: conditions.StrategyExecutionFailed, - Message: fmt.Sprintf(`failed to execute strategy: "no step 2 in strategy for Release \"%s/%s\""`, namespace, incumbentName), - }, - } - - expectedContender := contender.release.DeepCopy() - expectedContender.Status.Conditions = []shipper.ReleaseCondition{ - { - Type: shipper.ReleaseConditionTypeBlocked, - Status: corev1.ConditionFalse, - }, - { - Type: shipper.ReleaseConditionTypeScheduled, - Status: corev1.ConditionTrue, - }, - { - Type: shipper.ReleaseConditionTypeStrategyExecuted, - Status: corev1.ConditionFalse, - Reason: conditions.StrategyExecutionFailed, - Message: fmt.Sprintf(`failed to execute strategy: "no step 2 in strategy for Release \"%s/%s\""`, namespace, contenderName), - }, - } - - // we change the order of incumbent and contender here: we want to - // ensure we're safe when an incumbent steps in first and then triggers - // it's successor processing. - f.addObjects( - incumbent.release.DeepCopy(), - incumbent.installationTarget.DeepCopy(), - incumbent.capacityTarget.DeepCopy(), - incumbent.trafficTarget.DeepCopy(), - - contender.release.DeepCopy(), - contender.installationTarget.DeepCopy(), - contender.capacityTarget.DeepCopy(), - contender.trafficTarget.DeepCopy(), - ) - - f.actions = append(f.actions, - kubetesting.NewUpdateAction( - shipper.SchemeGroupVersion.WithResource("releases"), - namespace, - expectedIncumbent, - ), - kubetesting.NewUpdateAction( - shipper.SchemeGroupVersion.WithResource("releases"), - namespace, - expectedContender, - ), - ) - - f.filter = f.filter.Extend(actionfilter{ - []string{"update"}, - []string{"releases"}, - }) - f.expectedEvents = append(f.expectedEvents, - fmt.Sprintf(`Normal ReleaseConditionChanged [] -> [Scheduled True], [] -> [StrategyExecuted False StrategyExecutionFailed %s]`, - fmt.Sprintf(`failed to execute strategy: "no step 2 in strategy for Release \"%s/%s\""`, namespace, incumbentName)), - fmt.Sprintf(`Normal ReleaseConditionChanged [] -> [Scheduled True], [] -> [StrategyExecuted False StrategyExecutionFailed %s]`, - fmt.Sprintf(`failed to execute strategy: "no step 2 in strategy for Release \"%s/%s\""`, namespace, contenderName)), - ) - - f.run() -} - -func TestWaitingOnContenderCapacityProducesNoPatches(t *testing.T) { - namespace := "test-namespace" - incumbentName, contenderName := "test-incumbent", "test-contender" - - app := buildApplication(namespace, "test-app") - cluster := buildCluster("minikube") - - f := newFixture(t, app.DeepCopy(), cluster.DeepCopy()) - f.cycles = 1 - - totalReplicaCount := int32(10) - contender := f.buildContender(namespace, contenderName, totalReplicaCount) - incumbent := f.buildIncumbent(namespace, incumbentName, totalReplicaCount) - - contender.release.Spec.TargetStep = 1 - - // Working on contender capacity. - contender.capacityTarget.Spec.Clusters[0].Percent = 50 - contender.capacityTarget.Spec.Clusters[0].TotalReplicaCount = totalReplicaCount - contender.capacityTarget.Status.Conditions, _ = targetutil.SetTargetCondition( - contender.capacityTarget.Status.Conditions, - targetutil.NewTargetCondition( - shipper.TargetConditionTypeReady, - corev1.ConditionFalse, - ClustersNotReady, "[minikube]")) - - f.addObjects( - contender.release.DeepCopy(), - contender.installationTarget.DeepCopy(), - contender.capacityTarget.DeepCopy(), - contender.trafficTarget.DeepCopy(), - - incumbent.release.DeepCopy(), - incumbent.installationTarget.DeepCopy(), - incumbent.capacityTarget.DeepCopy(), - incumbent.trafficTarget.DeepCopy(), - ) - - relpair := releaseInfoPair{ - contender: contender, - incumbent: incumbent, - } - f.expectCapacityNotReady(relpair, 1, 0, Contender, "minikube") - f.run() -} - -func TestWaitingOnContenderTrafficProducesNoPatches(t *testing.T) { - namespace := "test-namespace" - incumbentName, contenderName := "test-incumbent", "test-contender" - - app := buildApplication(namespace, "test-app") - cluster := buildCluster("minikube") - - f := newFixture(t, app.DeepCopy(), cluster.DeepCopy()) - f.cycles = 1 - - totalReplicaCount := int32(10) - contender := f.buildContender(namespace, contenderName, totalReplicaCount) - incumbent := f.buildIncumbent(namespace, incumbentName, totalReplicaCount) - - contender.release.Annotations[shipper.ReleaseClustersAnnotation] = cluster.GetName() - condScheduled := releaseutil.NewReleaseCondition(shipper.ReleaseConditionTypeScheduled, corev1.ConditionTrue, "", "") - releaseutil.SetReleaseCondition(&contender.release.Status, *condScheduled) - condStrategyExecuted := releaseutil.NewReleaseCondition(shipper.ReleaseConditionTypeStrategyExecuted, corev1.ConditionTrue, "", "") - releaseutil.SetReleaseCondition(&contender.release.Status, *condStrategyExecuted) - - contender.release.Spec.TargetStep = 1 - - // Desired contender capacity achieved. - contender.capacityTarget.Spec.Clusters[0].Percent = 50 - contender.capacityTarget.Spec.Clusters[0].TotalReplicaCount = totalReplicaCount - - // Working on contender traffic. - contender.trafficTarget.Spec.Clusters[0].Weight = 50 - contender.trafficTarget.Status.Conditions, _ = targetutil.SetTargetCondition( - contender.trafficTarget.Status.Conditions, - targetutil.NewTargetCondition( - shipper.TargetConditionTypeReady, - corev1.ConditionFalse, - ClustersNotReady, "[minikube]")) - - f.addObjects( - contender.release.DeepCopy(), - contender.installationTarget.DeepCopy(), - contender.capacityTarget.DeepCopy(), - contender.trafficTarget.DeepCopy(), - - incumbent.release.DeepCopy(), - incumbent.installationTarget.DeepCopy(), - incumbent.capacityTarget.DeepCopy(), - incumbent.trafficTarget.DeepCopy(), - ) - - relpair := releaseInfoPair{ - contender: contender, - incumbent: incumbent, - } - f.expectTrafficNotReady(relpair, 1, 0, Contender, "minikube") - f.run() - -} - -func TestWaitingOnIncumbentTrafficProducesNoPatches(t *testing.T) { - namespace := "test-namespace" - incumbentName, contenderName := "test-incumbent", "test-contender" - - app := buildApplication(namespace, "test-app") - cluster := buildCluster("minikube") - - f := newFixture(t, app.DeepCopy(), cluster.DeepCopy()) - f.cycles = 1 - - totalReplicaCount := int32(10) - contender := f.buildContender(namespace, contenderName, totalReplicaCount) - incumbent := f.buildIncumbent(namespace, incumbentName, totalReplicaCount) - - contender.release.Annotations[shipper.ReleaseClustersAnnotation] = cluster.GetName() - condScheduled := releaseutil.NewReleaseCondition(shipper.ReleaseConditionTypeScheduled, corev1.ConditionTrue, "", "") - releaseutil.SetReleaseCondition(&contender.release.Status, *condScheduled) - condStrategyExecuted := releaseutil.NewReleaseCondition(shipper.ReleaseConditionTypeStrategyExecuted, corev1.ConditionTrue, "", "") - releaseutil.SetReleaseCondition(&contender.release.Status, *condStrategyExecuted) - - contender.release.Spec.TargetStep = 1 - - // Desired contender capacity achieved. - contender.capacityTarget.Spec.Clusters[0].Percent = 50 - contender.capacityTarget.Spec.Clusters[0].TotalReplicaCount = totalReplicaCount - - // Desired contender traffic achieved. - contender.trafficTarget.Spec.Clusters[0].Weight = 50 - - // Working on incumbent traffic. - incumbent.trafficTarget.Spec.Clusters[0].Weight = 50 - incumbent.trafficTarget.Status.Conditions, _ = targetutil.SetTargetCondition( - incumbent.trafficTarget.Status.Conditions, - targetutil.NewTargetCondition( - shipper.TargetConditionTypeReady, - corev1.ConditionFalse, - ClustersNotReady, "[minikube]")) - - f.addObjects( - contender.release.DeepCopy(), - contender.installationTarget.DeepCopy(), - contender.capacityTarget.DeepCopy(), - contender.trafficTarget.DeepCopy(), - - incumbent.release.DeepCopy(), - incumbent.installationTarget.DeepCopy(), - incumbent.capacityTarget.DeepCopy(), - incumbent.trafficTarget.DeepCopy(), - ) - - relpair := releaseInfoPair{ - contender: contender, - incumbent: incumbent, - } - f.expectTrafficNotReady(relpair, 1, 0, Incumbent, "minikube") - f.run() -} - -func TestWaitingOnIncumbentCapacityProducesNoPatches(t *testing.T) { - namespace := "test-namespace" - incumbentName, contenderName := "test-incumbent", "test-contender" - - app := buildApplication(namespace, "test-app") - cluster := buildCluster("minikube") - - f := newFixture(t, app.DeepCopy(), cluster.DeepCopy()) - f.cycles = 1 - - totalReplicaCount := int32(10) - contender := f.buildContender(namespace, contenderName, totalReplicaCount) - incumbent := f.buildIncumbent(namespace, incumbentName, totalReplicaCount) - - contender.release.Spec.TargetStep = 1 - - // Desired contender capacity achieved. - contender.capacityTarget.Spec.Clusters[0].Percent = 50 - contender.capacityTarget.Spec.Clusters[0].TotalReplicaCount = totalReplicaCount - - // Desired contender traffic achieved. - contender.trafficTarget.Spec.Clusters[0].Weight = 50 - - // Desired incumbent traffic achieved. - incumbent.trafficTarget.Spec.Clusters[0].Weight = 50 - - // Working on incumbent capacity. - incumbent.capacityTarget.Spec.Clusters[0].Percent = 50 - incumbent.capacityTarget.Spec.Clusters[0].TotalReplicaCount = totalReplicaCount - incumbent.capacityTarget.Status.Conditions, _ = targetutil.SetTargetCondition( - incumbent.capacityTarget.Status.Conditions, - targetutil.NewTargetCondition( - shipper.TargetConditionTypeReady, - corev1.ConditionFalse, - ClustersNotReady, "[minikube]")) - - f.addObjects( - contender.release.DeepCopy(), - contender.installationTarget.DeepCopy(), - contender.capacityTarget.DeepCopy(), - contender.trafficTarget.DeepCopy(), - - incumbent.release.DeepCopy(), - incumbent.installationTarget.DeepCopy(), - incumbent.capacityTarget.DeepCopy(), - incumbent.trafficTarget.DeepCopy(), - ) - - relpair := releaseInfoPair{ - contender: contender, - incumbent: incumbent, - } - f.expectCapacityNotReady(relpair, 1, 0, Incumbent, "minikube") - f.run() -} - -func TestIncumbentOutOfRangeTargetStep(t *testing.T) { - namespace := "test-namespace" - incumbentName, contenderName := "test-incumbent", "test-contender" - app := buildApplication(namespace, "test-app") - cluster := buildCluster("minikube") - - f := newFixture(t, app.DeepCopy(), cluster.DeepCopy()) - f.cycles = 2 - - totalReplicaCount := int32(10) - contender := f.buildContender(namespace, contenderName, totalReplicaCount) - incumbent := f.buildIncumbent(namespace, incumbentName, totalReplicaCount) - - // Incumbent spec contains only 1 strategy step but we intentionally - // specify an out-of-range index in order to test if it carefully - // handles indices. - incumbent.release.Spec.TargetStep = 2 - incumbent.release.Spec.Environment.Strategy = &fullon - incumbent.release.Status.AchievedStep.Step = 0 - incumbent.trafficTarget.Spec.Clusters[0].Weight = 50 - incumbent.capacityTarget.Spec.Clusters[0].Percent = 50 - incumbent.capacityTarget.Spec.Clusters[0].TotalReplicaCount = totalReplicaCount - - step := int32(1) - contender.release.Spec.TargetStep = step - contender.capacityTarget.Spec.Clusters[0].Percent = 50 - contender.capacityTarget.Spec.Clusters[0].TotalReplicaCount = totalReplicaCount - contender.trafficTarget.Spec.Clusters[0].Weight = 50 - - f.addObjects( - contender.release.DeepCopy(), - contender.installationTarget.DeepCopy(), - contender.capacityTarget.DeepCopy(), - contender.trafficTarget.DeepCopy(), - - incumbent.release.DeepCopy(), - incumbent.installationTarget.DeepCopy(), - incumbent.capacityTarget.DeepCopy(), - incumbent.trafficTarget.DeepCopy(), - ) - - f.expectReleaseWaitingForCommand(contender.release, step) - f.expectedEvents = append(f.expectedEvents, - fmt.Sprintf(`Normal ReleaseConditionChanged [StrategyExecuted True] -> [StrategyExecuted False StrategyExecutionFailed failed to execute strategy: "Release %s/%s target step is inconsistent: unexpected value %d (expected: 0)"]`, - namespace, incumbentName, 2)) - - f.run() -} - -func TestUnhealthyTrafficAndCapacityIncumbentConvergesConsistently(t *testing.T) { - namespace := "test-namespace" - incumbentName, contenderName := "test-incumbent", "test-contender" - app := buildApplication(namespace, "test-app") - cluster := buildCluster("minikube") - brokenCluster := buildCluster("broken-cluster") - - f := newFixture(t, app.DeepCopy(), cluster.DeepCopy()) - replicaCount := int32(4) - - contender := f.buildContender(namespace, contenderName, replicaCount) - incumbent := f.buildIncumbent(namespace, incumbentName, replicaCount) - - addCluster(incumbent, brokenCluster) - - // Mark contender as fully healthy - var step int32 = 2 - contender.release.Spec.TargetStep = step - contender.release.Status.AchievedStep = &shipper.AchievedStep{Step: 2} - contender.release.Status = shipper.ReleaseStatus{ - Conditions: []shipper.ReleaseCondition{ - {Type: shipper.ReleaseConditionTypeBlocked, Status: corev1.ConditionFalse}, - {Type: shipper.ReleaseConditionTypeScheduled, Status: corev1.ConditionTrue}, - }, - Strategy: &shipper.ReleaseStrategyStatus{ - State: shipper.ReleaseStrategyState{ - WaitingForInstallation: shipper.StrategyStateFalse, - WaitingForCommand: shipper.StrategyStateFalse, - WaitingForTraffic: shipper.StrategyStateFalse, - WaitingForCapacity: shipper.StrategyStateFalse, - }, - Conditions: []shipper.ReleaseStrategyCondition{ - shipper.ReleaseStrategyCondition{ - Type: shipper.StrategyConditionContenderAchievedCapacity, - Status: corev1.ConditionTrue, - Step: step, - }, - shipper.ReleaseStrategyCondition{ - Type: shipper.StrategyConditionContenderAchievedInstallation, - Status: corev1.ConditionTrue, - Step: step, - }, - shipper.ReleaseStrategyCondition{ - Type: shipper.StrategyConditionContenderAchievedTraffic, - Status: corev1.ConditionTrue, - Step: step, - }, - }, - }, - } - - contender.capacityTarget.Spec.Clusters[0].Percent = 100 - contender.capacityTarget.Spec.Clusters[0].TotalReplicaCount = replicaCount - contender.trafficTarget.Spec.Clusters[0].Weight = 100 - - incumbent.trafficTarget.Spec.Clusters[0].Weight = 0 - incumbent.trafficTarget.Spec.Clusters[1].Weight = 0 - incumbent.trafficTarget.Status.Clusters = []*shipper.ClusterTrafficStatus{ - { - AchievedTraffic: 50, - }, - { - AchievedTraffic: 0, - }, - } - incumbent.trafficTarget.Status.Conditions, _ = targetutil.SetTargetCondition( - incumbent.trafficTarget.Status.Conditions, - targetutil.NewTargetCondition( - shipper.TargetConditionTypeReady, - corev1.ConditionFalse, - ClustersNotReady, "[broken-cluster]", - ), - ) - incumbent.capacityTarget.Spec.Clusters[0].Name = "broken-cluster" - incumbent.capacityTarget.Spec.Clusters[0].Percent = 0 - incumbent.capacityTarget.Spec.Clusters[0].TotalReplicaCount = replicaCount - incumbent.capacityTarget.Spec.Clusters[1].Name = "minikube" - incumbent.capacityTarget.Spec.Clusters[1].Percent = 0 - incumbent.capacityTarget.Spec.Clusters[1].TotalReplicaCount = 0 - incumbent.capacityTarget.Status.Conditions, _ = targetutil.SetTargetCondition( - incumbent.capacityTarget.Status.Conditions, - targetutil.NewTargetCondition( - shipper.TargetConditionTypeReady, - corev1.ConditionFalse, - ClustersNotReady, "[broken-cluster]")) - incumbent.capacityTarget.Status.Clusters = []shipper.ClusterCapacityStatus{ - { - AvailableReplicas: 42, // anything but spec-matching - AchievedPercent: 42, // anything but spec-matching - }, - { - AvailableReplicas: 0, - AchievedPercent: 0, - }, - } - - expected := contender.release.DeepCopy() - condScheduled := releaseutil.NewReleaseCondition(shipper.ReleaseConditionTypeScheduled, corev1.ConditionTrue, "", "") - releaseutil.SetReleaseCondition(&expected.Status, *condScheduled) - condStrategyExecuted := releaseutil.NewReleaseCondition(shipper.ReleaseConditionTypeStrategyExecuted, corev1.ConditionTrue, "", "") - releaseutil.SetReleaseCondition(&expected.Status, *condStrategyExecuted) - - f.addObjects( - contender.release.DeepCopy(), - contender.installationTarget.DeepCopy(), - contender.capacityTarget.DeepCopy(), - contender.trafficTarget.DeepCopy(), - - incumbent.release.DeepCopy(), - incumbent.installationTarget.DeepCopy(), - incumbent.capacityTarget.DeepCopy(), - incumbent.trafficTarget.DeepCopy(), - ) - - f.actions = append(f.actions, - kubetesting.NewUpdateAction( - shipper.SchemeGroupVersion.WithResource("releases"), - contender.release.GetNamespace(), - expected)) - - var patch []byte - - newContenderStatus := map[string]interface{}{ - "status": shipper.ReleaseStatus{ - Strategy: &shipper.ReleaseStrategyStatus{ - State: shipper.ReleaseStrategyState{ - WaitingForInstallation: shipper.StrategyStateFalse, - WaitingForCommand: shipper.StrategyStateFalse, - WaitingForTraffic: shipper.StrategyStateTrue, - WaitingForCapacity: shipper.StrategyStateFalse, - }, - Conditions: []shipper.ReleaseStrategyCondition{ - shipper.ReleaseStrategyCondition{ - Type: shipper.StrategyConditionContenderAchievedCapacity, - Status: corev1.ConditionTrue, - Step: step, - }, - shipper.ReleaseStrategyCondition{ - Type: shipper.StrategyConditionContenderAchievedInstallation, - Status: corev1.ConditionTrue, - Step: step, - }, - shipper.ReleaseStrategyCondition{ - Type: shipper.StrategyConditionContenderAchievedTraffic, - Status: corev1.ConditionTrue, - Step: step, - }, - shipper.ReleaseStrategyCondition{ - Type: shipper.StrategyConditionIncumbentAchievedTraffic, - Status: corev1.ConditionFalse, - Step: step, - Reason: ClustersNotReady, - Message: fmt.Sprintf("release \"test-incumbent\" hasn't achieved traffic in clusters: [broken-cluster]. for more details try `kubectl describe tt test-incumbent`"), - }, - }, - }, - }, - } - patch, _ = json.Marshal(newContenderStatus) - - f.actions = append(f.actions, kubetesting.NewPatchAction( - shipper.SchemeGroupVersion.WithResource("releases"), - contender.release.GetNamespace(), - contender.release.GetName(), - types.MergePatchType, - patch, - )) - - f.expectedEvents = append(f.expectedEvents, - `Normal ReleaseConditionChanged [] -> [StrategyExecuted True]`) - - f.run() -} - -func TestControllerDetectInconsistentTargetStep(t *testing.T) { - namespace := "test-namespace" - app := buildApplication(namespace, "test-app") - incumbentName, contenderName := "test-incumbent", "test-contender" - - cluster := buildCluster("minikube") - - f := newFixture(t, app.DeepCopy(), cluster.DeepCopy()) - f.cycles = 1 - - totalReplicaCount := int32(10) - contender := f.buildContender(namespace, contenderName, totalReplicaCount) - incumbent := f.buildIncumbent(namespace, incumbentName, totalReplicaCount) - - contender.release.Spec.TargetStep = 2 - incumbent.release.Spec.TargetStep = 1 - - f.addObjects( - incumbent.release.DeepCopy(), - incumbent.installationTarget.DeepCopy(), - incumbent.capacityTarget.DeepCopy(), - incumbent.trafficTarget.DeepCopy(), - - contender.release.DeepCopy(), - contender.installationTarget.DeepCopy(), - contender.capacityTarget.DeepCopy(), - contender.trafficTarget.DeepCopy(), - ) - - expected := incumbent.release.DeepCopy() - condStrategyExecuted := releaseutil.NewReleaseCondition( - shipper.ReleaseConditionTypeStrategyExecuted, - corev1.ConditionFalse, - "StrategyExecutionFailed", - "failed to execute strategy: \"Release test-namespace/test-incumbent target step is inconsistent: unexpected value 1 (expected: 2)\"", - ) - releaseutil.SetReleaseCondition(&expected.Status, *condStrategyExecuted) - - f.actions = []kubetesting.Action{ - kubetesting.NewUpdateAction( - shipper.SchemeGroupVersion.WithResource("releases"), - incumbent.release.GetNamespace(), - expected), - } - - f.expectedEvents = append(f.expectedEvents, - fmt.Sprintf("Normal ReleaseConditionChanged [StrategyExecuted True] -> [StrategyExecuted False StrategyExecutionFailed failed to execute strategy: \"Release test-namespace/test-incumbent target step is inconsistent: unexpected value 1 (expected: 2)\"]"), - ) - - f.run() -} - -// This test ensures we review historical release strategy conditions -func TestUpdatesHistoricalReleaseStrategyStateConditions(t *testing.T) { - namespace := "test-namespace" - app := buildApplication(namespace, "test-app") - totalReplicaCount := int32(4) - - cluster := buildCluster("minikube") - f := newFixture(t, app.DeepCopy(), cluster.DeepCopy()) - - step := int32(2) - - // we are not interested in all updates happening in the loop and we expect - // the first one to update the very first release in the chain. - f.cycles = 1 - - // we instantiate 3 releases: in this case the first one will be left out of - // the "extended" (contender-incumbent) strategy executor loop and reviewed - // with a reduced amount of ensurer steps - relinfos := []*releaseInfo{ - f.buildIncumbent(namespace, "pre-incumbent", totalReplicaCount), - f.buildIncumbent(namespace, "incumbent", totalReplicaCount), - // we create a full-on release intentionally, therefore we use - // buildIncumbent helper - f.buildContender(namespace, "contender", totalReplicaCount), - } - - // make sure generation-sorted releases preserve the original order - for i, relinfo := range relinfos { - relinfo.release.ObjectMeta.Annotations[shipper.ReleaseGenerationAnnotation] = strconv.Itoa(i) - } - - relinfos[0].capacityTarget.Spec.Clusters = []shipper.ClusterCapacityTarget{ - { - Name: cluster.Name, - Percent: 0, - TotalReplicaCount: totalReplicaCount, - }, - } - relinfos[0].trafficTarget.Spec.Clusters = []shipper.ClusterTrafficTarget{ - { - Name: cluster.Name, - Weight: 0, - }, - } - - relinfos[2].capacityTarget.Spec.Clusters = []shipper.ClusterCapacityTarget{ - { - Name: cluster.Name, - Percent: 1, - TotalReplicaCount: totalReplicaCount, - }, - } - - // we intenitonally set one of the strategy conditions to an unready state - // and expect it to get fixed by the controller - preincumbent := relinfos[0].release - cond := conditions.NewStrategyConditions(preincumbent.Status.Strategy.Conditions...) - cond.SetFalse( - shipper.StrategyConditionContenderAchievedCapacity, - conditions.StrategyConditionsUpdate{ - Reason: ClustersNotReady, - // this message is incomplete but it doesn't matter in the context - // of this test - Message: fmt.Sprintf("release %q hasn't achieved capacity in clusters: %s", - preincumbent.Name, cluster.Name), - Step: step, - LastTransitionTime: time.Now(), - }, - ) - preincumbent.Status.Strategy = &shipper.ReleaseStrategyStatus{ - Conditions: cond.AsReleaseStrategyConditions(), - State: cond.AsReleaseStrategyState(step, false, true, false), - } - - for _, relinfo := range relinfos { - f.addObjects( - relinfo.release.DeepCopy(), - relinfo.installationTarget.DeepCopy(), - relinfo.capacityTarget.DeepCopy(), - relinfo.trafficTarget.DeepCopy(), - ) - } - - expectedStatus := map[string]interface{}{ - "status": shipper.ReleaseStatus{ - Strategy: &shipper.ReleaseStrategyStatus{ - State: shipper.ReleaseStrategyState{ - WaitingForInstallation: shipper.StrategyStateFalse, - WaitingForCommand: shipper.StrategyStateFalse, - WaitingForTraffic: shipper.StrategyStateFalse, - WaitingForCapacity: shipper.StrategyStateFalse, - }, - Conditions: []shipper.ReleaseStrategyCondition{ - shipper.ReleaseStrategyCondition{ - Type: shipper.StrategyConditionContenderAchievedCapacity, - Status: corev1.ConditionTrue, - Step: step, - }, - shipper.ReleaseStrategyCondition{ - Type: shipper.StrategyConditionContenderAchievedInstallation, - Status: corev1.ConditionTrue, - Step: step, - }, - shipper.ReleaseStrategyCondition{ - Type: shipper.StrategyConditionContenderAchievedTraffic, - Status: corev1.ConditionTrue, - Step: step, - }, - }, - }, - }, - } - patch, _ := json.Marshal(expectedStatus) - f.actions = append(f.actions, kubetesting.NewPatchAction( - shipper.SchemeGroupVersion.WithResource("releases"), - preincumbent.GetNamespace(), - preincumbent.GetName(), - types.MergePatchType, - patch, - )) - - key := fmt.Sprintf("%s/%s", preincumbent.GetNamespace(), preincumbent.GetName()) - - f.expectedEvents = append(f.expectedEvents, - fmt.Sprintf( - "Normal ReleaseStateTransitioned Release %q had its state \"WaitingForCapacity\" transitioned to \"False\"", - key)) - - f.run() } diff --git a/pkg/controller/release/scheduler_test.go b/pkg/controller/release/scheduler_test.go index 86fbf7f0f..e9bb7b133 100644 --- a/pkg/controller/release/scheduler_test.go +++ b/pkg/controller/release/scheduler_test.go @@ -1,7 +1,6 @@ package release import ( - "sort" "testing" "time" @@ -18,104 +17,17 @@ import ( shippertesting "github.com/bookingcom/shipper/pkg/testing" ) -func buildAssociatedObjects(release *shipper.Release, clusters []*shipper.Cluster) (*shipper.InstallationTarget, *shipper.TrafficTarget, *shipper.CapacityTarget) { - clusterNames := make([]string, 0, len(clusters)) - for _, cluster := range clusters { - clusterNames = append(clusterNames, cluster.GetName()) - } - sort.Strings(clusterNames) - - installationTarget := &shipper.InstallationTarget{ - ObjectMeta: metav1.ObjectMeta{ - Name: release.GetName(), - Namespace: release.GetNamespace(), - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: release.APIVersion, - Kind: release.Kind, - Name: release.Name, - UID: release.UID, - }, - }, - Labels: map[string]string{ - shipper.AppLabel: release.OwnerReferences[0].Name, - shipper.ReleaseLabel: release.GetName(), - }, - }, - Spec: shipper.InstallationTargetSpec{ - Clusters: clusterNames, - CanOverride: true, - Chart: release.Spec.Environment.Chart, - Values: release.Spec.Environment.Values, - }, - } +func buildReleaseForSchedulerTest(clusters []*shipper.Cluster) *shipper.Release { + rel := buildRelease( + shippertesting.TestNamespace, + shippertesting.TestApp, + "contender", + 0, + ) - clusterTrafficTargets := make([]shipper.ClusterTrafficTarget, 0, len(clusters)) - for _, cluster := range clusters { - clusterTrafficTargets = append( - clusterTrafficTargets, - shipper.ClusterTrafficTarget{ - Name: cluster.GetName(), - }) - } - - trafficTarget := &shipper.TrafficTarget{ - ObjectMeta: metav1.ObjectMeta{ - Name: release.Name, - Namespace: release.GetNamespace(), - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: release.APIVersion, - Kind: release.Kind, - Name: release.Name, - UID: release.UID, - }, - }, - Labels: map[string]string{ - shipper.AppLabel: release.OwnerReferences[0].Name, - shipper.ReleaseLabel: release.GetName(), - }, - }, - Spec: shipper.TrafficTargetSpec{ - Clusters: clusterTrafficTargets, - }, - } - - clusterCapacityTargets := make([]shipper.ClusterCapacityTarget, 0, len(clusters)) - for _, cluster := range clusters { - clusterCapacityTargets = append( - clusterCapacityTargets, - shipper.ClusterCapacityTarget{ - Name: cluster.GetName(), - Percent: 0, - TotalReplicaCount: 12, - }) - } - - capacityTarget := &shipper.CapacityTarget{ - ObjectMeta: metav1.ObjectMeta{ - Name: release.Name, - Namespace: release.GetNamespace(), - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: release.APIVersion, - Kind: release.Kind, - Name: release.Name, - UID: release.UID, - }, - }, - Labels: map[string]string{ - shipper.AppLabel: release.OwnerReferences[0].Name, - shipper.ReleaseLabel: release.GetName(), - }, - }, - Spec: shipper.CapacityTargetSpec{ - Clusters: clusterCapacityTargets, - }, - } - - return installationTarget, trafficTarget, capacityTarget + setReleaseClusters(rel, clusters) + return rel } func newScheduler( @@ -150,23 +62,15 @@ func newScheduler( // objects do not exist by the moment of scheduling, therefore 3 extra create // actions are expected. func TestCreateAssociatedObjects(t *testing.T) { - cluster := buildCluster("minikube-a") - release := buildRelease() - release.Annotations[shipper.ReleaseClustersAnnotation] = cluster.GetName() - fixtures := []runtime.Object{release, cluster} + clusters := []*shipper.Cluster{buildCluster("minikube-a")} + release := buildReleaseForSchedulerTest(clusters) - // Expected release and actions. The release should have, at the end of the - // business logic, a list of clusters containing the sole cluster we've added - // to the client, and also a Scheduled condition with True status. Expected - // actions contain the intent to create all the associated target objects. - expected := release.DeepCopy() - expected.Status.Conditions = []shipper.ReleaseCondition{ - {Type: shipper.ReleaseConditionTypeScheduled, Status: corev1.ConditionTrue}, - } - expectedActions := buildExpectedActions(expected.DeepCopy(), []*shipper.Cluster{cluster.DeepCopy()}) + fixtures := []runtime.Object{release} + + expectedActions := buildExpectedActions(release, clusters) c, clientset := newScheduler(fixtures) - if _, err := c.ScheduleRelease(release.DeepCopy()); err != nil { + if _, err := c.ScheduleRelease(release); err != nil { t.Fatal(err) } @@ -181,7 +85,7 @@ func TestCreateAssociatedObjects(t *testing.T) { // be updated. func TestCreateAssociatedObjectsDuplicateInstallationTargetMismatchingClusters(t *testing.T) { cluster := buildCluster("minikube-a") - release := buildRelease() + release := buildReleaseForSchedulerTest([]*shipper.Cluster{cluster}) release.Annotations[shipper.ReleaseClustersAnnotation] = cluster.GetName() installationtarget := &shipper.InstallationTarget{ @@ -244,9 +148,8 @@ func TestCreateAssociatedObjectsDuplicateInstallationTargetMismatchingClusters(t // proceed normally. Instead of creating a new object, the existing one should // be updated. func TestCreateAssociatedObjectsDuplicateTrafficTargetMismatchingClusters(t *testing.T) { - cluster := buildCluster("minikube-a") - release := buildRelease() - release.Annotations[shipper.ReleaseClustersAnnotation] = cluster.GetName() + clusters := []*shipper.Cluster{buildCluster("minikube-a")} + release := buildReleaseForSchedulerTest(clusters) traffictarget := &shipper.TrafficTarget{ ObjectMeta: metav1.ObjectMeta{ @@ -262,16 +165,11 @@ func TestCreateAssociatedObjectsDuplicateTrafficTargetMismatchingClusters(t *tes }, } - fixtures := []runtime.Object{release, traffictarget, cluster} - - expected := release.DeepCopy() - expected.Status.Conditions = []shipper.ReleaseCondition{ - {Type: shipper.ReleaseConditionTypeScheduled, Status: corev1.ConditionTrue}, - } + fixtures := []runtime.Object{traffictarget} // traffictarget already exists, expect an update ection. The rest // does not exist yet, therefore 2 more create actions. - it, tt, ct := buildAssociatedObjects(expected.DeepCopy(), []*shipper.Cluster{cluster.DeepCopy()}) + it, tt, ct := buildAssociatedObjects(release, clusters) expectedActions := []kubetesting.Action{ kubetesting.NewCreateAction( shipper.SchemeGroupVersion.WithResource("installationtargets"), @@ -289,7 +187,7 @@ func TestCreateAssociatedObjectsDuplicateTrafficTargetMismatchingClusters(t *tes } c, clientset := newScheduler(fixtures) - if _, err := c.ScheduleRelease(release.DeepCopy()); err != nil { + if _, err := c.ScheduleRelease(release); err != nil { t.Fatal(err) } @@ -303,9 +201,9 @@ func TestCreateAssociatedObjectsDuplicateTrafficTargetMismatchingClusters(t *tes // proceed normally. Instead of creating a new object, the existing one should // be updated. func TestCreateAssociatedObjectsDuplicateCapacityTargetMismatchingClusters(t *testing.T) { - cluster := buildCluster("minikube-a") - release := buildRelease() - release.Annotations[shipper.ReleaseClustersAnnotation] = cluster.GetName() + clusters := []*shipper.Cluster{buildCluster("minikube-a")} + release := buildReleaseForSchedulerTest(clusters) + release.Annotations[shipper.ReleaseClustersAnnotation] = clusters[0].GetName() capacitytarget := &shipper.CapacityTarget{ ObjectMeta: metav1.ObjectMeta{ @@ -321,16 +219,11 @@ func TestCreateAssociatedObjectsDuplicateCapacityTargetMismatchingClusters(t *te }, } - fixtures := []runtime.Object{release, capacitytarget, cluster} - - expected := release.DeepCopy() - expected.Status.Conditions = []shipper.ReleaseCondition{ - {Type: shipper.ReleaseConditionTypeScheduled, Status: corev1.ConditionTrue}, - } + fixtures := []runtime.Object{capacitytarget} - it, tt, ct := buildAssociatedObjects(expected.DeepCopy(), []*shipper.Cluster{cluster.DeepCopy()}) // capacitytarget already exists, expect an update ection. The rest // does not exist yet, therefore 2 more create actions. + it, tt, ct := buildAssociatedObjects(release, clusters) expectedActions := []kubetesting.Action{ kubetesting.NewCreateAction( shipper.SchemeGroupVersion.WithResource("installationtargets"), @@ -348,7 +241,7 @@ func TestCreateAssociatedObjectsDuplicateCapacityTargetMismatchingClusters(t *te } c, clientset := newScheduler(fixtures) - if _, err := c.ScheduleRelease(release.DeepCopy()); err != nil { + if _, err := c.ScheduleRelease(release); err != nil { t.Fatal(err) } @@ -362,7 +255,7 @@ func TestCreateAssociatedObjectsDuplicateCapacityTargetMismatchingClusters(t *te // create the missing objects and proceed normally. func TestCreateAssociatedObjectsDuplicateInstallationTargetSameOwner(t *testing.T) { cluster := buildCluster("minikube-a") - release := buildRelease() + release := buildReleaseForSchedulerTest([]*shipper.Cluster{cluster}) release.Annotations[shipper.ReleaseClustersAnnotation] = cluster.GetName() installationtarget := &shipper.InstallationTarget{ @@ -415,7 +308,7 @@ func TestCreateAssociatedObjectsDuplicateInstallationTargetSameOwner(t *testing. // error to be returned. func TestCreateAssociatedObjectsDuplicateInstallationTargetNoOwner(t *testing.T) { cluster := buildCluster("minikube-a") - release := buildRelease() + release := buildReleaseForSchedulerTest([]*shipper.Cluster{cluster}) release.Annotations[shipper.ReleaseClustersAnnotation] = cluster.GetName() installationtarget := &shipper.InstallationTarget{ @@ -451,10 +344,8 @@ func TestCreateAssociatedObjectsDuplicateInstallationTargetNoOwner(t *testing.T) // case we expect the missing asiociated objects to be created and the release // to be scheduled. func TestCreateAssociatedObjectsDuplicateTrafficTargetSameOwner(t *testing.T) { - // Fixtures - cluster := buildCluster("minikube-a") - release := buildRelease() - release.Annotations[shipper.ReleaseClustersAnnotation] = cluster.GetName() + clusters := []*shipper.Cluster{buildCluster("minikube-a")} + release := buildReleaseForSchedulerTest(clusters) traffictarget := &shipper.TrafficTarget{ ObjectMeta: metav1.ObjectMeta{ @@ -465,21 +356,12 @@ func TestCreateAssociatedObjectsDuplicateTrafficTargetSameOwner(t *testing.T) { }, }, } - setTrafficTargetClusters(traffictarget, []string{cluster.Name}) - fixtures := []runtime.Object{cluster, release, traffictarget} + setTrafficTargetClusters(traffictarget, []string{clusters[0].Name}) - // Expected release and actions. Even with an existing traffictarget - // object for this release, at the end of the business logic the expected - // release should have its .status.phase set to "WaitingForStrategy". Expected - // actions contain the intent to create the missing associated target - // objects. - expected := release.DeepCopy() - expected.Status.Conditions = []shipper.ReleaseCondition{ - {Type: shipper.ReleaseConditionTypeScheduled, Status: corev1.ConditionTrue}, - } + fixtures := []runtime.Object{traffictarget} - it, _, ct := buildAssociatedObjects(expected.DeepCopy(), []*shipper.Cluster{cluster.DeepCopy()}) // 2 create actions: installationtarget and capacitytarget + it, _, ct := buildAssociatedObjects(release, clusters) expectedActions := []kubetesting.Action{ kubetesting.NewCreateAction( shipper.SchemeGroupVersion.WithResource("installationtargets"), @@ -493,7 +375,7 @@ func TestCreateAssociatedObjectsDuplicateTrafficTargetSameOwner(t *testing.T) { } c, clientset := newScheduler(fixtures) - if _, err := c.ScheduleRelease(release.DeepCopy()); err != nil { + if _, err := c.ScheduleRelease(release); err != nil { t.Fatal(err) } @@ -505,10 +387,8 @@ func TestCreateAssociatedObjectsDuplicateTrafficTargetSameOwner(t *testing.T) { // and existing traffictarget object exists but has a wrong owner reference. // It's an exception case and we expect the appropriate error to be returned. func TestCreateAssociatedObjectsDuplicateTrafficTargetNoOwner(t *testing.T) { - // Fixtures - cluster := buildCluster("minikube-a") - release := buildRelease() - release.Annotations[shipper.ReleaseClustersAnnotation] = cluster.GetName() + clusters := []*shipper.Cluster{buildCluster("minikube-a")} + release := buildReleaseForSchedulerTest(clusters) traffictarget := &shipper.TrafficTarget{ ObjectMeta: metav1.ObjectMeta{ @@ -517,19 +397,12 @@ func TestCreateAssociatedObjectsDuplicateTrafficTargetNoOwner(t *testing.T) { // No explicit owner reference here }, } - fixtures := []runtime.Object{cluster, release, traffictarget} - // Expected a release but no actions. With an existing traffictarget - // object but no explicit reference, it's a no-go. Expected an - // already-exists error. - expected := release.DeepCopy() - expected.Status.Conditions = []shipper.ReleaseCondition{ - {Type: shipper.ReleaseConditionTypeScheduled, Status: corev1.ConditionTrue}, - } + fixtures := []runtime.Object{traffictarget} c, _ := newScheduler(fixtures) - _, err := c.CreateOrUpdateTrafficTarget(release.DeepCopy()) + _, err := c.CreateOrUpdateTrafficTarget(release) if err == nil { t.Fatalf("Expected an error here, none received") } @@ -543,10 +416,8 @@ func TestCreateAssociatedObjectsDuplicateTrafficTargetNoOwner(t *testing.T) { // where a capacitytarget object already exists and has a right owner reference. // In this case we expect the missing objects to be created. func TestCreateAssociatedObjectsDuplicateCapacityTargetSameOwner(t *testing.T) { - // Fixtures - cluster := buildCluster("minikube-a") - release := buildRelease() - release.Annotations[shipper.ReleaseClustersAnnotation] = cluster.GetName() + clusters := []*shipper.Cluster{buildCluster("minikube-a")} + release := buildReleaseForSchedulerTest(clusters) var totalReplicaCount int32 = 1 @@ -559,20 +430,12 @@ func TestCreateAssociatedObjectsDuplicateCapacityTargetSameOwner(t *testing.T) { }, }, } - setCapacityTargetClusters(capacitytarget, []string{cluster.Name}, totalReplicaCount) - fixtures := []runtime.Object{cluster, release, capacitytarget} + setCapacityTargetClusters(capacitytarget, []string{clusters[0].Name}, totalReplicaCount) - // Expected release and actions. Even with an existing capacitytarget object - // for this release, at the end of the business logic the expected release - // should have its .status.phase set to "WaitingForStrategy". Expected actions - // contain the intent to create all the associated target objects. - expected := release.DeepCopy() - expected.Status.Conditions = []shipper.ReleaseCondition{ - {Type: shipper.ReleaseConditionTypeScheduled, Status: corev1.ConditionTrue}, - } + fixtures := []runtime.Object{capacitytarget} - it, tt, _ := buildAssociatedObjects(expected.DeepCopy(), []*shipper.Cluster{cluster.DeepCopy()}) // 2 create actions: installationtarget and traffictarget + it, tt, _ := buildAssociatedObjects(release, clusters) expectedActions := []kubetesting.Action{ kubetesting.NewCreateAction( shipper.SchemeGroupVersion.WithResource("installationtargets"), @@ -586,7 +449,7 @@ func TestCreateAssociatedObjectsDuplicateCapacityTargetSameOwner(t *testing.T) { } c, clientset := newScheduler(fixtures) - if _, err := c.ScheduleRelease(release.DeepCopy()); err != nil { + if _, err := c.ScheduleRelease(release); err != nil { t.Fatal(err) } @@ -598,10 +461,8 @@ func TestCreateAssociatedObjectsDuplicateCapacityTargetSameOwner(t *testing.T) { // a capacitytarget object already exists but it has a wrong owner reference. // It's an exception and we expect the appropriate error to be returned. func TestCreateAssociatedObjectsDuplicateCapacityTargetNoOwner(t *testing.T) { - // Fixtures - cluster := buildCluster("minikube-a") - release := buildRelease() - release.Annotations[shipper.ReleaseClustersAnnotation] = cluster.GetName() + clusters := []*shipper.Cluster{buildCluster("minikube-a")} + release := buildReleaseForSchedulerTest(clusters) capacitytarget := &shipper.CapacityTarget{ ObjectMeta: metav1.ObjectMeta{ @@ -609,19 +470,12 @@ func TestCreateAssociatedObjectsDuplicateCapacityTargetNoOwner(t *testing.T) { Namespace: release.GetNamespace(), }, } - fixtures := []runtime.Object{cluster, release, capacitytarget} - // Expected a release but no actions. With an existing capacitytarget - // object but no explicit reference, it's a no-go. Expected a - // conflict error. - expected := release.DeepCopy() - expected.Status.Conditions = []shipper.ReleaseCondition{ - {Type: shipper.ReleaseConditionTypeScheduled, Status: corev1.ConditionTrue}, - } + fixtures := []runtime.Object{capacitytarget} c, _ := newScheduler(fixtures) - _, err := c.CreateOrUpdateCapacityTarget(release.DeepCopy(), 1) + _, err := c.CreateOrUpdateCapacityTarget(release, 1) if err == nil { t.Fatalf("Expected an error here, none received") } diff --git a/pkg/controller/release/utils_test.go b/pkg/controller/release/utils_test.go index c909685c4..d6ee12cf5 100644 --- a/pkg/controller/release/utils_test.go +++ b/pkg/controller/release/utils_test.go @@ -1,39 +1,81 @@ package release import ( + "fmt" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kubetesting "k8s.io/client-go/testing" shipper "github.com/bookingcom/shipper/pkg/apis/shipper/v1alpha1" shippertesting "github.com/bookingcom/shipper/pkg/testing" ) +var ( + TargetConditionOperational = shipper.TargetCondition{ + Type: shipper.TargetConditionTypeOperational, + Status: corev1.ConditionTrue, + } + TargetConditionReady = shipper.TargetCondition{ + Type: shipper.TargetConditionTypeReady, + Status: corev1.ConditionTrue, + } + + vanguard = shipper.RolloutStrategy{ + Steps: []shipper.RolloutStrategyStep{ + { + Name: "staging", + Capacity: shipper.RolloutStrategyStepValue{Incumbent: 100, Contender: 1}, + Traffic: shipper.RolloutStrategyStepValue{Incumbent: 100, Contender: 0}, + }, + { + Name: "50/50", + Capacity: shipper.RolloutStrategyStepValue{Incumbent: 50, Contender: 50}, + Traffic: shipper.RolloutStrategyStepValue{Incumbent: 50, Contender: 50}, + }, + { + Name: "full on", + Capacity: shipper.RolloutStrategyStepValue{Incumbent: 0, Contender: 100}, + Traffic: shipper.RolloutStrategyStepValue{Incumbent: 0, Contender: 100}, + }, + }, + } +) + func pint32(i int32) *int32 { return &i } -func buildRelease() *shipper.Release { - return &shipper.Release{ - TypeMeta: metav1.TypeMeta{ - APIVersion: shipper.SchemeGroupVersion.String(), - Kind: "Release", +func buildRelease( + namespace, app, name string, + replicaCount int32, +) *shipper.Release { + relName := fmt.Sprintf("%s-%s", app, name) + ownerRef := metav1.OwnerReference{ + APIVersion: shipper.SchemeGroupVersion.String(), + Kind: "Application", + Name: app, + } + clusterRequirements := shipper.ClusterRequirements{ + Regions: []shipper.RegionRequirement{ + { + Name: shippertesting.TestRegion, + Replicas: pint32(replicaCount), + }, }, + } + + return &shipper.Release{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-release", - Namespace: shippertesting.TestNamespace, + Namespace: namespace, + Name: relName, + OwnerReferences: []metav1.OwnerReference{ownerRef}, Annotations: map[string]string{ - shipper.ReleaseGenerationAnnotation: "1", - }, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: shipper.SchemeGroupVersion.String(), - Kind: "Application", - Name: "test-application", - }, + shipper.ReleaseGenerationAnnotation: GenerationInactive, }, Labels: map[string]string{ - shipper.ReleaseLabel: "test-release", - shipper.AppLabel: "test-application", + shipper.ReleaseLabel: relName, + shipper.AppLabel: app, }, }, Spec: shipper.ReleaseSpec{ @@ -43,16 +85,164 @@ func buildRelease() *shipper.Release { Name: "simple", Version: "0.0.1", }, - ClusterRequirements: shipper.ClusterRequirements{ - Regions: []shipper.RegionRequirement{{Name: shippertesting.TestRegion}}, - }, + ClusterRequirements: clusterRequirements, }, }, - Status: shipper.ReleaseStatus{ - Conditions: []shipper.ReleaseCondition{ - {Type: shipper.ReleaseConditionTypeBlocked, Status: corev1.ConditionFalse}, - {Type: shipper.ReleaseConditionTypeScheduled, Status: corev1.ConditionFalse}, + } +} + +func buildCluster(name string) *shipper.Cluster { + return &shipper.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: shipper.ClusterSpec{ + Capabilities: []string{}, + Region: shippertesting.TestRegion, + }, + } +} + +func buildRolloutBlock(namespace, name string) *shipper.RolloutBlock { + return &shipper.RolloutBlock{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: shipper.RolloutBlockSpec{ + Message: "Simple test rollout block", + Author: shipper.RolloutBlockAuthor{ + Type: "user", + Name: "testUser", }, }, } } + +func buildAssociatedObjects(release *shipper.Release, clusters []*shipper.Cluster) (*shipper.InstallationTarget, *shipper.TrafficTarget, *shipper.CapacityTarget) { + ownerReferences := []metav1.OwnerReference{ + createOwnerRefFromRelease(release), + } + labels := map[string]string{ + shipper.AppLabel: release.OwnerReferences[0].Name, + shipper.ReleaseLabel: release.GetName(), + } + + clusterInstallationTargets := make([]string, 0, len(clusters)) + clusterCapacityTargets := make([]shipper.ClusterCapacityTarget, 0, len(clusters)) + clusterTrafficTargets := make([]shipper.ClusterTrafficTarget, 0, len(clusters)) + + for _, cluster := range clusters { + clusterInstallationTargets = append( + clusterInstallationTargets, + cluster.GetName()) + + clusterCapacityTargets = append( + clusterCapacityTargets, + shipper.ClusterCapacityTarget{ + Name: cluster.GetName(), + Percent: 0, + TotalReplicaCount: 12, + }) + + clusterTrafficTargets = append( + clusterTrafficTargets, + shipper.ClusterTrafficTarget{ + Name: cluster.GetName(), + }) + } + + installationTarget := &shipper.InstallationTarget{ + ObjectMeta: metav1.ObjectMeta{ + Name: release.GetName(), + Namespace: release.GetNamespace(), + OwnerReferences: ownerReferences, + Labels: labels, + }, + Spec: shipper.InstallationTargetSpec{ + Clusters: clusterInstallationTargets, + CanOverride: true, + Chart: release.Spec.Environment.Chart, + Values: release.Spec.Environment.Values, + }, + } + + trafficTarget := &shipper.TrafficTarget{ + ObjectMeta: metav1.ObjectMeta{ + Name: release.Name, + Namespace: release.GetNamespace(), + OwnerReferences: ownerReferences, + Labels: labels, + }, + Spec: shipper.TrafficTargetSpec{ + Clusters: clusterTrafficTargets, + }, + } + + capacityTarget := &shipper.CapacityTarget{ + ObjectMeta: metav1.ObjectMeta{ + Name: release.Name, + Namespace: release.GetNamespace(), + OwnerReferences: ownerReferences, + Labels: labels, + }, + Spec: shipper.CapacityTargetSpec{ + Clusters: clusterCapacityTargets, + }, + } + + return installationTarget, trafficTarget, capacityTarget +} + +func buildAssociatedObjectsWithStatus( + release *shipper.Release, + clusters []*shipper.Cluster, + achievedStep *int32, +) (*shipper.InstallationTarget, *shipper.TrafficTarget, *shipper.CapacityTarget) { + it, tt, ct := buildAssociatedObjects(release, clusters) + + if achievedStep != nil { + it.Status = shipper.InstallationTargetStatus{ + Conditions: []shipper.TargetCondition{ + TargetConditionOperational, + TargetConditionReady, + }, + } + + ct.Status = shipper.CapacityTargetStatus{ + Conditions: []shipper.TargetCondition{ + TargetConditionOperational, + TargetConditionReady, + }, + } + + tt.Status = shipper.TrafficTargetStatus{ + Conditions: []shipper.TargetCondition{ + TargetConditionOperational, + TargetConditionReady, + }, + } + } + + return it, tt, ct +} + +func buildExpectedActions(release *shipper.Release, clusters []*shipper.Cluster) []kubetesting.Action { + installationTarget, trafficTarget, capacityTarget := buildAssociatedObjects(release, clusters) + + return []kubetesting.Action{ + kubetesting.NewCreateAction( + shipper.SchemeGroupVersion.WithResource("installationtargets"), + release.GetNamespace(), + installationTarget), + kubetesting.NewCreateAction( + shipper.SchemeGroupVersion.WithResource("traffictargets"), + release.GetNamespace(), + trafficTarget), + kubetesting.NewCreateAction( + shipper.SchemeGroupVersion.WithResource("capacitytargets"), + release.GetNamespace(), + capacityTarget, + ), + } +} diff --git a/pkg/errors/release.go b/pkg/errors/release.go index 1072557a8..c843ddfbb 100644 --- a/pkg/errors/release.go +++ b/pkg/errors/release.go @@ -177,24 +177,6 @@ func NewDuplicateCapabilityRequirementError(capability string) DuplicateCapabili } } -type NotWorkingOnStrategyError struct { - contenderReleaseKey string -} - -func (e NotWorkingOnStrategyError) Error() string { - return fmt.Sprintf("found %s as a contender, but it is not currently working on any strategy", e.contenderReleaseKey) -} - -func (e NotWorkingOnStrategyError) ShouldRetry() bool { - return false -} - -func NewNotWorkingOnStrategyError(contenderReleaseKey string) NotWorkingOnStrategyError { - return NotWorkingOnStrategyError{ - contenderReleaseKey: contenderReleaseKey, - } -} - type InconsistentReleaseTargetStep struct { relKey string gotTargetStep int32 @@ -202,7 +184,7 @@ type InconsistentReleaseTargetStep struct { } func (e InconsistentReleaseTargetStep) Error() string { - return fmt.Sprintf("Release %s target step is inconsistent: unexpected value %d (expected: %d)", + return fmt.Sprintf("Release %q target step is inconsistent: unexpected value %d (expected: %d)", e.relKey, e.gotTargetStep, e.wantTargetStep) } diff --git a/pkg/testing/util.go b/pkg/testing/util.go index e8b4a3c33..7ede060b8 100644 --- a/pkg/testing/util.go +++ b/pkg/testing/util.go @@ -124,9 +124,15 @@ func CheckAction(expected, actual kubetesting.Action, t *testing.T) { prettyExpected := prettyPrintAction(expected) prettyActual := prettyPrintAction(actual) - diff, err := YamlDiff(prettyActual, prettyExpected) + diff, err := difflib.GetUnifiedDiffString(difflib.UnifiedDiff{ + A: difflib.SplitLines(string(prettyExpected)), + B: difflib.SplitLines(string(prettyActual)), + FromFile: "Expected", + ToFile: "Actual", + Context: ContextLines, + }) if err != nil { - panic(fmt.Sprintf("couldn't generate yaml diff: %s", err)) + panic(fmt.Sprintf("couldn't generate diff: %s", err)) } t.Errorf("expected action is different from actual:\n%s", diff)