From dd65f8996753f181449e8fce4e8e19e6995396f0 Mon Sep 17 00:00:00 2001 From: hari rongali Date: Tue, 10 Aug 2021 11:58:09 -0700 Subject: [PATCH 1/3] fix: fix analysis metrics to wait for all metrics to complete Signed-off-by: hari rongali --- analysis/analysis.go | 8 ++++++++ test/e2e/analysis_test.go | 6 +++++- .../analysistemplate-sleep-job.yaml | 17 +++++++++++++++- .../functional/rollout-inline-analysis.yaml | 4 +--- test/fixtures/then.go | 20 +++++++++++++++++++ 5 files changed, 50 insertions(+), 5 deletions(-) diff --git a/analysis/analysis.go b/analysis/analysis.go index 265406ef60..ab2d6b9741 100644 --- a/analysis/analysis.go +++ b/analysis/analysis.go @@ -396,6 +396,14 @@ func (c *Controller) assessRunStatus(run *v1alpha1.AnalysisRun) (v1alpha1.Analys } } } + } else { + // metric hasn't started running. possible cases where some of the metrics starts with delay + everythingCompleted = false + if terminating { + // we have yet to take a single measurement, but have already been instructed to stop + log.Infof("metric assessed %s: run terminated", v1alpha1.AnalysisPhaseSuccessful) + return v1alpha1.AnalysisPhaseSuccessful, worstMessage + } } } if !everythingCompleted { diff --git a/test/e2e/analysis_test.go b/test/e2e/analysis_test.go index 317977c2cb..6cc2ca313b 100644 --- a/test/e2e/analysis_test.go +++ b/test/e2e/analysis_test.go @@ -74,14 +74,18 @@ func (s *AnalysisSuite) TestCanaryInlineAnalysis() { When(). UpdateSpec(). WaitForRolloutStatus("Paused"). + PromoteRollout(). + Sleep(5 * time.Second). Then(). ExpectAnalysisRunCount(1). + ExpectInlineAnalysisRunPhase("Running"). When(). WaitForInlineAnalysisRunPhase("Successful"). + WaitForRolloutStatus("Paused"). PromoteRollout(). WaitForRolloutStatus("Healthy"). Then(). - ExpectAnalysisRunCount(3) + ExpectAnalysisRunCount(2) } // TestBlueGreenAnalysis tests blue-green with pre/post analysis and then fast-tracked rollback diff --git a/test/e2e/functional/analysistemplate-sleep-job.yaml b/test/e2e/functional/analysistemplate-sleep-job.yaml index 462dda8d7a..87fc51f14f 100644 --- a/test/e2e/functional/analysistemplate-sleep-job.yaml +++ b/test/e2e/functional/analysistemplate-sleep-job.yaml @@ -13,7 +13,8 @@ spec: value: "1" metrics: - name: sleep-job - count: "{{args.count}}" + initialDelay: 10s + count: 1 provider: job: spec: @@ -26,3 +27,17 @@ spec: args: ["sleep {{args.duration}} && exit {{args.exit-code}}"] restartPolicy: Never backoffLimit: 0 + - name: sleep-job-rep + count: 1 + provider: + job: + spec: + template: + spec: + containers: + - name: sleep-job + image: nginx:1.19-alpine + command: [sh, -c, -x] + args: ["sleep {{args.duration}} && exit {{args.exit-code}}"] + restartPolicy: Never + backoffLimit: 0 \ No newline at end of file diff --git a/test/e2e/functional/rollout-inline-analysis.yaml b/test/e2e/functional/rollout-inline-analysis.yaml index d933cd5d62..b37f54703f 100644 --- a/test/e2e/functional/rollout-inline-analysis.yaml +++ b/test/e2e/functional/rollout-inline-analysis.yaml @@ -7,13 +7,11 @@ spec: canary: steps: - setWeight: 10 - - analysis: - templates: - - templateName: sleep-job - pause: {} - analysis: templates: - templateName: sleep-job + - pause: {} - analysis: templates: - templateName: sleep-job diff --git a/test/fixtures/then.go b/test/fixtures/then.go index 77a04b2faf..b99d65347c 100644 --- a/test/fixtures/then.go +++ b/test/fixtures/then.go @@ -257,6 +257,26 @@ func (t *Then) ExpectBackgroundAnalysisRunPhase(phase string) *Then { ) } +func (t *Then) ExpectInlineAnalysisRun(expectation string, expectFunc AnalysisRunExpectation) *Then { + t.t.Helper() + bgArun := t.GetInlineAnalysisRun() + if !expectFunc(bgArun) { + t.log.Errorf("Inline AnalysisRun expectation '%s' failed", expectation) + t.t.FailNow() + } + t.log.Infof("Inline AnalysisRun expectation '%s' met", expectation) + return t +} + +func (t *Then) ExpectInlineAnalysisRunPhase(phase string) *Then { + t.t.Helper() + return t.ExpectInlineAnalysisRun(fmt.Sprintf("inline analysis phase == %s", phase), + func(run *rov1.AnalysisRun) bool { + return string(run.Status.Phase) == phase + }, + ) +} + // ExpectStableRevision verifies the ReplicaSet with the specified revision is marked stable func (t *Then) ExpectStableRevision(revision string) *Then { t.t.Helper() From b6105e7027d8ca929683551ff0556070404032a0 Mon Sep 17 00:00:00 2001 From: hari rongali Date: Tue, 10 Aug 2021 13:00:51 -0700 Subject: [PATCH 2/3] fix: add more e2e tests Signed-off-by: hari rongali --- test/e2e/analysis_test.go | 27 ++++++++++-- .../analysistemplate-multiple-job.yaml | 43 +++++++++++++++++++ .../analysistemplate-sleep-job.yaml | 17 +------- .../functional/rollout-inline-analysis.yaml | 4 +- .../rollout-inline-multiple-analysis.yaml | 28 ++++++++++++ 5 files changed, 98 insertions(+), 21 deletions(-) create mode 100644 test/e2e/functional/analysistemplate-multiple-job.yaml create mode 100644 test/e2e/functional/rollout-inline-multiple-analysis.yaml diff --git a/test/e2e/analysis_test.go b/test/e2e/analysis_test.go index 6cc2ca313b..f0a9758a1a 100644 --- a/test/e2e/analysis_test.go +++ b/test/e2e/analysis_test.go @@ -25,6 +25,7 @@ func (s *AnalysisSuite) SetupSuite() { // shared analysis templates for suite s.ApplyManifests("@functional/analysistemplate-web-background.yaml") s.ApplyManifests("@functional/analysistemplate-sleep-job.yaml") + s.ApplyManifests("@functional/analysistemplate-multiple-job.yaml") } // convenience to generate a new service with a given name @@ -74,6 +75,27 @@ func (s *AnalysisSuite) TestCanaryInlineAnalysis() { When(). UpdateSpec(). WaitForRolloutStatus("Paused"). + Then(). + ExpectAnalysisRunCount(1). + When(). + WaitForInlineAnalysisRunPhase("Successful"). + PromoteRollout(). + WaitForRolloutStatus("Healthy"). + Then(). + ExpectAnalysisRunCount(3) +} + +func (s *AnalysisSuite) TestCanaryInlineMultipleAnalysis() { + s.Given(). + RolloutObjects("@functional/rollout-inline-multiple-analysis.yaml"). + When(). + ApplyManifests(). + WaitForRolloutStatus("Healthy"). + Then(). + ExpectAnalysisRunCount(0). + When(). + UpdateSpec(). + WaitForRolloutStatus("Paused"). PromoteRollout(). Sleep(5 * time.Second). Then(). @@ -81,13 +103,10 @@ func (s *AnalysisSuite) TestCanaryInlineAnalysis() { ExpectInlineAnalysisRunPhase("Running"). When(). WaitForInlineAnalysisRunPhase("Successful"). - WaitForRolloutStatus("Paused"). - PromoteRollout(). WaitForRolloutStatus("Healthy"). Then(). - ExpectAnalysisRunCount(2) + ExpectAnalysisRunCount(1) } - // TestBlueGreenAnalysis tests blue-green with pre/post analysis and then fast-tracked rollback func (s *AnalysisSuite) TestBlueGreenAnalysis() { original := ` diff --git a/test/e2e/functional/analysistemplate-multiple-job.yaml b/test/e2e/functional/analysistemplate-multiple-job.yaml new file mode 100644 index 0000000000..fabfeef6a2 --- /dev/null +++ b/test/e2e/functional/analysistemplate-multiple-job.yaml @@ -0,0 +1,43 @@ +# AnalysisTemplate which sleeps for a specified duration and exits with a specified exit-code +kind: AnalysisTemplate +apiVersion: argoproj.io/v1alpha1 +metadata: + name: multiple-job +spec: + args: + - name: duration + value: 0s + - name: exit-code + value: "0" + - name: count + value: "1" + metrics: + - name: sleep-job + initialDelay: 10s + count: 1 + provider: + job: + spec: + template: + spec: + containers: + - name: sleep-job + image: nginx:1.19-alpine + command: [sh, -c, -x] + args: ["sleep {{args.duration}} && exit {{args.exit-code}}"] + restartPolicy: Never + backoffLimit: 0 + - name: sleep-job-rep + count: 1 + provider: + job: + spec: + template: + spec: + containers: + - name: sleep-job + image: nginx:1.19-alpine + command: [sh, -c, -x] + args: ["sleep {{args.duration}} && exit {{args.exit-code}}"] + restartPolicy: Never + backoffLimit: 0 \ No newline at end of file diff --git a/test/e2e/functional/analysistemplate-sleep-job.yaml b/test/e2e/functional/analysistemplate-sleep-job.yaml index 87fc51f14f..39b242af99 100644 --- a/test/e2e/functional/analysistemplate-sleep-job.yaml +++ b/test/e2e/functional/analysistemplate-sleep-job.yaml @@ -13,22 +13,7 @@ spec: value: "1" metrics: - name: sleep-job - initialDelay: 10s - count: 1 - provider: - job: - spec: - template: - spec: - containers: - - name: sleep-job - image: nginx:1.19-alpine - command: [sh, -c, -x] - args: ["sleep {{args.duration}} && exit {{args.exit-code}}"] - restartPolicy: Never - backoffLimit: 0 - - name: sleep-job-rep - count: 1 + count: "{{args.count}}" provider: job: spec: diff --git a/test/e2e/functional/rollout-inline-analysis.yaml b/test/e2e/functional/rollout-inline-analysis.yaml index b37f54703f..d933cd5d62 100644 --- a/test/e2e/functional/rollout-inline-analysis.yaml +++ b/test/e2e/functional/rollout-inline-analysis.yaml @@ -7,7 +7,6 @@ spec: canary: steps: - setWeight: 10 - - pause: {} - analysis: templates: - templateName: sleep-job @@ -15,6 +14,9 @@ spec: - analysis: templates: - templateName: sleep-job + - analysis: + templates: + - templateName: sleep-job selector: matchLabels: app: rollout-inline-analysis diff --git a/test/e2e/functional/rollout-inline-multiple-analysis.yaml b/test/e2e/functional/rollout-inline-multiple-analysis.yaml new file mode 100644 index 0000000000..521424f1c3 --- /dev/null +++ b/test/e2e/functional/rollout-inline-multiple-analysis.yaml @@ -0,0 +1,28 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Rollout +metadata: + name: rollout-inline-analysis +spec: + strategy: + canary: + steps: + - setWeight: 10 + - pause: {} + - analysis: + templates: + - templateName: multiple-job + selector: + matchLabels: + app: rollout-inline-analysis + template: + metadata: + labels: + app: rollout-inline-analysis + spec: + containers: + - name: rollouts-demo + image: nginx:1.19-alpine + resources: + requests: + memory: 16Mi + cpu: 5m From 86d3e8899bac69f4243de6b21d7af50cfac77b45 Mon Sep 17 00:00:00 2001 From: hari rongali Date: Tue, 10 Aug 2021 21:26:11 -0700 Subject: [PATCH 3/3] Trigger Build Signed-off-by: hari rongali