From d00f97b410d170ee591e4e8836e90d39e2231b14 Mon Sep 17 00:00:00 2001 From: geobk Date: Wed, 11 Nov 2020 16:16:32 -0500 Subject: [PATCH] Allow custom releases in ci-operator validations --- pkg/api/config.go | 56 +++++++++++++++++++++++------------------ pkg/api/config_test.go | 57 ++++++++++++++++++++++++++++++------------ 2 files changed, 73 insertions(+), 40 deletions(-) diff --git a/pkg/api/config.go b/pkg/api/config.go index 73d2c392de4..0b1e741562d 100644 --- a/pkg/api/config.go +++ b/pkg/api/config.go @@ -69,7 +69,11 @@ func (config *ReleaseBuildConfiguration) validate(org, repo string, resolved boo validationErrors = append(validationErrors, validateReleaseBuildConfiguration(config, org, repo)...) validationErrors = append(validationErrors, validateBuildRootImageConfiguration("build_root", config.InputConfiguration.BuildRootImage, len(config.Images) > 0)) - validationErrors = append(validationErrors, validateTestStepConfiguration("tests", config.Tests, config.ReleaseTagConfiguration, resolved)...) + releases := sets.NewString() + for name := range releases { + releases.Insert(name) + } + validationErrors = append(validationErrors, validateTestStepConfiguration("tests", config.Tests, config.ReleaseTagConfiguration, releases, resolved)...) // this validation brings together a large amount of data from separate // parts of the configuration, so it's written as a standalone method @@ -334,7 +338,7 @@ func validateOperator(fieldRoot string, input *OperatorStepConfiguration, linkFo return validationErrors } -func validateTestStepConfiguration(fieldRoot string, input []TestStepConfiguration, release *ReleaseTagConfiguration, resolved bool) []error { +func validateTestStepConfiguration(fieldRoot string, input []TestStepConfiguration, release *ReleaseTagConfiguration, releases sets.String, resolved bool) []error { var validationErrors []error // check for test.As duplicates @@ -390,7 +394,7 @@ func validateTestStepConfiguration(fieldRoot string, input []TestStepConfigurati } } - validationErrors = append(validationErrors, validateTestConfigurationType(fieldRootN, test, release, resolved)...) + validationErrors = append(validationErrors, validateTestConfigurationType(fieldRootN, test, release, releases, resolved)...) } return validationErrors } @@ -473,7 +477,7 @@ func searchForTestDuplicates(tests []TestStepConfiguration) []error { return nil } -func validateTestConfigurationType(fieldRoot string, test TestStepConfiguration, release *ReleaseTagConfiguration, resolved bool) []error { +func validateTestConfigurationType(fieldRoot string, test TestStepConfiguration, release *ReleaseTagConfiguration, releases sets.String, resolved bool) []error { var validationErrors []error typeCount := 0 if testConfig := test.ContainerTestConfiguration; testConfig != nil { @@ -546,9 +550,9 @@ func validateTestConfigurationType(fieldRoot string, test TestStepConfiguration, validationErrors = append(validationErrors, validateClusterProfile(fieldRoot, testConfig.ClusterProfile)...) } seen := sets.NewString() - validationErrors = append(validationErrors, validateTestStepsPre(fmt.Sprintf("%s.Pre", fieldRoot), testConfig.Pre, seen, testConfig.Environment)...) - validationErrors = append(validationErrors, validateTestStepsTest(fmt.Sprintf("%s.Test", fieldRoot), testConfig.Test, seen, testConfig.Environment)...) - validationErrors = append(validationErrors, validateTestStepsPost(fmt.Sprintf("%s.Post", fieldRoot), testConfig.Post, seen, testConfig.Environment)...) + validationErrors = append(validationErrors, validateTestStepsPre(fmt.Sprintf("%s.Pre", fieldRoot), testConfig.Pre, seen, testConfig.Environment, releases)...) + validationErrors = append(validationErrors, validateTestStepsTest(fmt.Sprintf("%s.Test", fieldRoot), testConfig.Test, seen, testConfig.Environment, releases)...) + validationErrors = append(validationErrors, validateTestStepsPost(fmt.Sprintf("%s.Post", fieldRoot), testConfig.Post, seen, testConfig.Environment, releases)...) } if testConfig := test.MultiStageTestConfigurationLiteral; testConfig != nil { typeCount++ @@ -558,15 +562,15 @@ func validateTestConfigurationType(fieldRoot string, test TestStepConfiguration, seen := sets.NewString() for i, s := range testConfig.Pre { fieldRootI := fmt.Sprintf("%s.Pre[%d]", fieldRoot, i) - validationErrors = append(validationErrors, validateLiteralTestStepPre(fieldRootI, s, seen, testConfig.Environment)...) + validationErrors = append(validationErrors, validateLiteralTestStepPre(fieldRootI, s, seen, testConfig.Environment, releases)...) } for i, s := range testConfig.Test { fieldRootI := fmt.Sprintf("%s.Test[%d]", fieldRoot, i) - validationErrors = append(validationErrors, validateLiteralTestStepTest(fieldRootI, s, seen, testConfig.Environment)...) + validationErrors = append(validationErrors, validateLiteralTestStepTest(fieldRootI, s, seen, testConfig.Environment, releases)...) } for i, s := range testConfig.Post { fieldRootI := fmt.Sprintf("%s.Post[%d]", fieldRoot, i) - validationErrors = append(validationErrors, validateLiteralTestStepPost(fieldRootI, s, seen, testConfig.Environment)...) + validationErrors = append(validationErrors, validateLiteralTestStepPost(fieldRootI, s, seen, testConfig.Environment, releases)...) } } if test.OpenshiftInstallerRandomClusterTestConfiguration != nil { @@ -585,34 +589,34 @@ func validateTestConfigurationType(fieldRoot string, test TestStepConfiguration, return validationErrors } -func validateTestStepsPre(fieldRoot string, steps []TestStep, seen sets.String, env TestEnvironment) (ret []error) { +func validateTestStepsPre(fieldRoot string, steps []TestStep, seen sets.String, env TestEnvironment, releases sets.String) (ret []error) { for i, s := range steps { fieldRootI := fmt.Sprintf("%s[%d]", fieldRoot, i) ret = validateTestStep(fieldRootI, s, seen) if s.LiteralTestStep != nil { - ret = append(ret, validateLiteralTestStepPre(fieldRootI, *s.LiteralTestStep, seen, env)...) + ret = append(ret, validateLiteralTestStepPre(fieldRootI, *s.LiteralTestStep, seen, env, releases)...) } } return } -func validateTestStepsTest(fieldRoot string, steps []TestStep, seen sets.String, env TestEnvironment) (ret []error) { +func validateTestStepsTest(fieldRoot string, steps []TestStep, seen sets.String, env TestEnvironment, releases sets.String) (ret []error) { for i, s := range steps { fieldRootI := fmt.Sprintf("%s[%d]", fieldRoot, i) ret = validateTestStep(fieldRootI, s, seen) if s.LiteralTestStep != nil { - ret = append(ret, validateLiteralTestStepTest(fieldRootI, *s.LiteralTestStep, seen, env)...) + ret = append(ret, validateLiteralTestStepTest(fieldRootI, *s.LiteralTestStep, seen, env, releases)...) } } return } -func validateTestStepsPost(fieldRoot string, steps []TestStep, seen sets.String, env TestEnvironment) (ret []error) { +func validateTestStepsPost(fieldRoot string, steps []TestStep, seen sets.String, env TestEnvironment, releases sets.String) (ret []error) { for i, s := range steps { fieldRootI := fmt.Sprintf("%s[%d]", fieldRoot, i) ret = validateTestStep(fieldRootI, s, seen) if s.LiteralTestStep != nil { - ret = append(ret, validateLiteralTestStepPost(fieldRootI, *s.LiteralTestStep, seen, env)...) + ret = append(ret, validateLiteralTestStepPost(fieldRootI, *s.LiteralTestStep, seen, env, releases)...) } } return @@ -650,7 +654,7 @@ func validateTestStep(fieldRootI string, step TestStep, seen sets.String) (ret [ return } -func validateLiteralTestStepCommon(fieldRoot string, step LiteralTestStep, seen sets.String, env TestEnvironment) (ret []error) { +func validateLiteralTestStepCommon(fieldRoot string, step LiteralTestStep, seen sets.String, env TestEnvironment, releases sets.String) (ret []error) { if len(step.As) == 0 { ret = append(ret, fmt.Errorf("%s: `as` is required", fieldRoot)) } else if seen.Has(step.As) { @@ -684,7 +688,11 @@ func validateLiteralTestStepCommon(fieldRoot string, step LiteralTestStep, seen switch obj { case PipelineImageStream, ReleaseStreamFor(LatestReleaseName), ReleaseStreamFor(InitialReleaseName), ReleaseImageStream: default: - ret = append(ret, fmt.Errorf("%s.from: unknown imagestream '%s'", fieldRoot, imageParts[0])) + releaseName := ReleaseNameFrom(obj) + if !releases.Has(releaseName) { + ret = append(ret, fmt.Errorf("%s.from: unknown imagestream '%s'", fieldRoot, imageParts[0])) + } + } } } @@ -701,21 +709,21 @@ func validateLiteralTestStepCommon(fieldRoot string, step LiteralTestStep, seen return } -func validateLiteralTestStepPre(fieldRoot string, step LiteralTestStep, seen sets.String, env TestEnvironment) (ret []error) { - ret = validateLiteralTestStepCommon(fieldRoot, step, seen, env) +func validateLiteralTestStepPre(fieldRoot string, step LiteralTestStep, seen sets.String, env TestEnvironment, releases sets.String) (ret []error) { + ret = validateLiteralTestStepCommon(fieldRoot, step, seen, env, releases) if step.OptionalOnSuccess != nil { ret = append(ret, fmt.Errorf("%s: `optional_on_success` is only allowed for Post steps", fieldRoot)) } return } -func validateLiteralTestStepTest(fieldRoot string, step LiteralTestStep, seen sets.String, env TestEnvironment) (ret []error) { - ret = validateLiteralTestStepPre(fieldRoot, step, seen, env) +func validateLiteralTestStepTest(fieldRoot string, step LiteralTestStep, seen sets.String, env TestEnvironment, releases sets.String) (ret []error) { + ret = validateLiteralTestStepPre(fieldRoot, step, seen, env, releases) return } -func validateLiteralTestStepPost(fieldRoot string, step LiteralTestStep, seen sets.String, env TestEnvironment) (ret []error) { - ret = validateLiteralTestStepCommon(fieldRoot, step, seen, env) +func validateLiteralTestStepPost(fieldRoot string, step LiteralTestStep, seen sets.String, env TestEnvironment, releases sets.String) (ret []error) { + ret = validateLiteralTestStepCommon(fieldRoot, step, seen, env, releases) return } diff --git a/pkg/api/config_test.go b/pkg/api/config_test.go index d2c3bd5171c..db8ec950c5f 100644 --- a/pkg/api/config_test.go +++ b/pkg/api/config_test.go @@ -15,6 +15,7 @@ func TestValidateTests(t *testing.T) { for _, tc := range []struct { id string release *ReleaseTagConfiguration + releases sets.String tests []TestStepConfiguration resolved bool expectedValid bool @@ -387,7 +388,7 @@ func TestValidateTests(t *testing.T) { }, } { t.Run(tc.id, func(t *testing.T) { - if errs := validateTestStepConfiguration("tests", tc.tests, tc.release, tc.resolved); len(errs) > 0 && tc.expectedValid { + if errs := validateTestStepConfiguration("tests", tc.tests, tc.release, tc.releases, tc.resolved); len(errs) > 0 && tc.expectedValid { t.Errorf("expected to be valid, got: %v", errs) } else if !tc.expectedValid && len(errs) == 0 { t.Error("expected to be invalid, but returned valid") @@ -527,10 +528,11 @@ func TestValidateTestSteps(t *testing.T) { asReference := "as" yes := true for _, tc := range []struct { - name string - steps []TestStep - seen sets.String - errs []error + name string + steps []TestStep + seen sets.String + errs []error + releases sets.String }{{ name: "valid step", steps: []TestStep{{ @@ -686,6 +688,27 @@ func TestValidateTestSteps(t *testing.T) { Resources: resources}, }}, errs: []error{errors.New("test[0].from: unknown imagestream 'no-such-imagestream'")}, + }, { + name: "custom imagestream", + steps: []TestStep{{ + LiteralTestStep: &LiteralTestStep{ + As: "as", + From: "stable-previous:base", + Commands: "commands", + Resources: resources}, + }}, + releases: sets.NewString("previous"), + }, { + name: "invalid image 4", + steps: []TestStep{{ + LiteralTestStep: &LiteralTestStep{ + As: "as", + From: "stable-nonexistent:base", + Commands: "commands", + Resources: resources}, + }}, + releases: sets.NewString("previous"), + errs: []error{errors.New("test[0].from: unknown imagestream 'stable-nonexistent'")}, }, { name: "no commands", steps: []TestStep{{ @@ -758,7 +781,7 @@ func TestValidateTestSteps(t *testing.T) { if seen == nil { seen = sets.NewString() } - ret := validateTestStepsTest("test", tc.steps, seen, nil) + ret := validateTestStepsTest("test", tc.steps, seen, nil, tc.releases) if !errListMessagesEqual(ret, tc.errs) { t.Fatal(diff.ObjectReflectDiff(ret, tc.errs)) } @@ -773,10 +796,11 @@ func TestValidatePostSteps(t *testing.T) { } yes := true for _, tc := range []struct { - name string - steps []TestStep - seen sets.String - errs []error + name string + steps []TestStep + seen sets.String + errs []error + releases sets.String }{{ name: "Valid Post steps", @@ -794,7 +818,7 @@ func TestValidatePostSteps(t *testing.T) { if seen == nil { seen = sets.NewString() } - ret := validateTestStepsPost("test", tc.steps, seen, nil) + ret := validateTestStepsPost("test", tc.steps, seen, nil, tc.releases) if !errListMessagesEqual(ret, tc.errs) { t.Fatal(diff.ObjectReflectDiff(ret, tc.errs)) } @@ -805,10 +829,11 @@ func TestValidatePostSteps(t *testing.T) { func TestValidateParameters(t *testing.T) { defaultStr := "default" for _, tc := range []struct { - name string - params []StepParameter - env TestEnvironment - err []error + name string + params []StepParameter + env TestEnvironment + err []error + releases sets.String }{{ name: "no parameters", }, { @@ -834,7 +859,7 @@ func TestValidateParameters(t *testing.T) { Limits: ResourceList{"memory": "1m"}, }, Environment: tc.params, - }, sets.NewString(), tc.env) + }, sets.NewString(), tc.env, tc.releases) if diff := diff.ObjectReflectDiff(err, tc.err); diff != "" { t.Errorf("incorrect error: %s", diff) }