From 4a985246c7398c26b7f28509f10c3d0c23d67b08 Mon Sep 17 00:00:00 2001 From: Aaron Crickenberger Date: Wed, 7 Apr 2021 13:40:23 -0400 Subject: [PATCH] jobs: add summary lines to jobs tests To help report on progress in migrating job configs away from deprecated tools or infrastructure --- config/tests/jobs/jobs_test.go | 78 ++++++++++++++++++++++------------ 1 file changed, 51 insertions(+), 27 deletions(-) diff --git a/config/tests/jobs/jobs_test.go b/config/tests/jobs/jobs_test.go index 5f06761242ff..baff1f045c5b 100644 --- a/config/tests/jobs/jobs_test.go +++ b/config/tests/jobs/jobs_test.go @@ -317,14 +317,17 @@ func TestTrustedJobs(t *testing.T) { // Enforce conventions for jobs that run in k8s-infra-prow-build-trusted cluster func TestK8sInfraTrusted(t *testing.T) { + jobsToFix := 0 const trusted = "k8s-infra-prow-build-trusted" trustedPath := path.Join(*jobConfigPath, "kubernetes", "wg-k8s-infra", "trusted") + "/" imagePushingDir := path.Join(*jobConfigPath, "image-pushing") + "/" + errs := []error{} // Presubmits may not use this cluster for _, pre := range c.AllStaticPresubmits(nil) { if pre.Cluster == trusted { - t.Errorf("%s: presubmits may not run in cluster: %s", pre.Name, trusted) + jobsToFix++ + errs = append(errs, fmt.Errorf("%s: presubmits may not run in cluster: %s", pre.Name, trusted)) } } @@ -344,12 +347,18 @@ func TestK8sInfraTrusted(t *testing.T) { } if strings.HasPrefix(job.SourcePath, imagePushingDir) { if err := validateImagePushingImage(job.Spec); err != nil { - t.Errorf("%s defined in %s %s", job.Name, job.SourcePath, err) + jobsToFix++ + errs = append(errs, fmt.Errorf("%s defined in %s %s", job.Name, job.SourcePath, err)) } } else if !strings.HasPrefix(job.SourcePath, trustedPath) { - t.Errorf("%s defined in %s may not run in cluster: %s", job.Name, job.SourcePath, trusted) + jobsToFix++ + errs = append(errs, fmt.Errorf("%s defined in %s may not run in cluster: %s", job.Name, job.SourcePath, trusted)) } } + for _, err := range errs { + t.Errorf("%v", err) + } + t.Logf("summary: %4d/%4d jobs fail to meet k8s-infra-prow-build-trusted CI policy", jobsToFix, len(jobs)) } func validateImagePushingImage(spec *coreapi.PodSpec) error { @@ -978,6 +987,7 @@ func isKubernetesReleaseBlocking(job cfg.JobBase) bool { } func TestKubernetesMergeBlockingJobsCIPolicy(t *testing.T) { + jobsToFix := 0 repo := "kubernetes/kubernetes" jobs := c.AllStaticPresubmits([]string{repo}) sort.Slice(jobs, func(i, j int) bool { @@ -995,14 +1005,20 @@ func TestKubernetesMergeBlockingJobsCIPolicy(t *testing.T) { errs = append(errs, fmt.Errorf("must run in cluster: k8s-infra-prow-build, found: %v", job.Cluster)) } branches := job.Branches + if len(errs) > 0 { + jobsToFix++ + } for _, err := range errs { t.Errorf("%v (%v): %v", job.Name, branches, err) } } + t.Logf("summary: %4d/%4d jobs fail to meet kubernetes/kubernetes merge-blocking CI policy", jobsToFix, len(jobs)) } func TestKubernetesReleaseBlockingJobsCIPolicy(t *testing.T) { - for _, job := range allStaticJobs() { + jobsToFix := 0 + jobs := allStaticJobs() + for _, job := range jobs { // Only consider Pods that are release-blocking if job.Spec == nil || !isKubernetesReleaseBlocking(job) { continue @@ -1013,13 +1029,18 @@ func TestKubernetesReleaseBlockingJobsCIPolicy(t *testing.T) { if job.Cluster != "k8s-infra-prow-build" { errs = append(errs, fmt.Errorf("must run in cluster: k8s-infra-prow-build, found: %v", job.Cluster)) } + if len(errs) > 0 { + jobsToFix++ + } for _, err := range errs { t.Errorf("%v: %v", job.Name, err) } } + t.Logf("summary: %4d/%4d jobs fail to meet kubernetes/kubernetes release-blocking CI policy", jobsToFix, len(jobs)) } func TestK8sInfraProwBuildJobsCIPolicy(t *testing.T) { + jobsToFix := 0 jobs := allStaticJobs() for _, job := range jobs { // Only consider Pods destined for the k8s-infra-prow-builds cluster @@ -1028,31 +1049,41 @@ func TestK8sInfraProwBuildJobsCIPolicy(t *testing.T) { } // job Pod must qualify for Guaranteed QoS errs := verifyPodQOSGuaranteed(job.Spec, true) + if len(errs) > 0 { + jobsToFix++ + } for _, err := range errs { t.Errorf("%v: %v", job.Name, err) } } + t.Logf("summary: %4d/%4d jobs fail to meet k8s-infra-prow-build CI policy", jobsToFix, len(jobs)) } // Fast builds take 20-30m, cross builds take 90m-2h. We want to pick up builds // containing the latest merged PRs as soon as possible for the in-development release func TestSigReleaseMasterBlockingOrInformingJobsMustUseFastBuilds(t *testing.T) { + jobsToFix := 0 jobs := allStaticJobs() for _, job := range jobs { dashboards, ok := job.Annotations["testgrid-dashboards"] if !ok || !strings.Contains(dashboards, "sig-release-master-blocking") || !strings.Contains(dashboards, "sig-release-master-informing") { continue } + errs := []error{} extract := "" for _, arg := range job.Spec.Containers[0].Args { if strings.HasPrefix(arg, "--extract=") { extract = strings.TrimPrefix(arg, "--extract=") if extract != "ci/latest-fast" { - t.Errorf("%s: release-master-blocking e2e jobs must use --extract=ci/latest-fast, found --extract=ci/latest instead", job.Name) + errs = append(errs, fmt.Errorf("release-master-blocking e2e jobs must use --extract=ci/latest-fast, found --extract=%s instead", extract)) } } } + for _, err := range errs { + t.Errorf("%v: %v", job.Name, err) + } } + t.Logf("summary: %4d/%4d jobs fail to meet release-master-blocking CI policy", jobsToFix, len(jobs)) } // matches regex used by the "version" extractMode defined in kubetest/extract_k8s.go @@ -1096,8 +1127,8 @@ func extractUsesReleaseBucket(extract string) bool { // TODO(https://github.com/kubernetes/k8s.io/issues/1569): start warning once gs://k8s-release populated // - jobs using --extract=release/* SHOULD pull from gs://k8s-release func TestKubernetesE2eJobsMustExtractFromK8sInfraBuckets(t *testing.T) { + jobsToFix := 0 jobs := allStaticJobs() - var totalJobs, needsFixJobs int for _, job := range jobs { needsFix := false extracts := []string{} @@ -1156,23 +1187,19 @@ func TestKubernetesE2eJobsMustExtractFromK8sInfraBuckets(t *testing.T) { } } } - totalJobs++ if needsFix { - needsFixJobs++ + jobsToFix++ } } - if needsFixJobs > 0 { - t.Logf("%4d/%4d jobs should be updated to pull from community-owned gcs buckets", needsFixJobs, totalJobs) - } + t.Logf("summary: %4d/%4d jobs should be updated to pull from community-owned gcs buckets", jobsToFix, len(jobs)) } // Prow jobs should use pod-utils instead of relying on bootstrap // https://github.com/kubernetes/test-infra/issues/20760 func TestKubernetesProwJobsShouldUsePodUtils(t *testing.T) { - total := 0 - fails := 0 - for _, job := range allStaticJobs() { - total++ + jobsToFix := 0 + jobs := allStaticJobs() + for _, job := range jobs { // Only consider Pods // TODO(https://github.com/kubernetes/test-infra/issues/14343): remove kubeflow exemption when job configs migrated if job.Spec == nil || strings.HasPrefix("kubeflow", job.Name) { @@ -1191,7 +1218,7 @@ func TestKubernetesProwJobsShouldUsePodUtils(t *testing.T) { scenario = strings.TrimPrefix(arg, "--scenario=") } } - fails++ + jobsToFix++ if len(repos) > 0 { t.Logf("%v: %v: should use pod-utils, found bootstrap args to clone: %v", job.SourcePath, job.Name, repos) } else if scenario != "" { @@ -1201,18 +1228,15 @@ func TestKubernetesProwJobsShouldUsePodUtils(t *testing.T) { } } } - if fails > 0 { - t.Logf("%v/%v jobs do not use pod-utils", fails, total) - } + t.Logf("summary: %4d/%4d jobs do not use pod-utils", jobsToFix, len(jobs)) } // Prow jobs should use kubetest2 instead of deprecated scenarios // https://github.com/kubernetes/test-infra/tree/master/scenarios#deprecation-notice func TestKubernetesProwJobsShouldNotUseDeprecatedScenarios(t *testing.T) { - total := 0 - fails := 0 - for _, job := range allStaticJobs() { - total++ + jobsToFix := 0 + jobs := allStaticJobs() + for _, job := range jobs { // Only consider Pods if job.Spec == nil { continue @@ -1229,7 +1253,7 @@ func TestKubernetesProwJobsShouldNotUseDeprecatedScenarios(t *testing.T) { } } if scenario != "" { - fails++ + jobsToFix++ t.Logf("%v: %v: should not be using deprecated scenarios, is directly invoking: %v", job.SourcePath, job.Name, scenario) continue } @@ -1239,11 +1263,11 @@ func TestKubernetesProwJobsShouldNotUseDeprecatedScenarios(t *testing.T) { } } if scenario != "" { - fails++ + jobsToFix++ t.Logf("%v: %v: should not be using deprecated scenarios, is invoking via bootrap: %v", job.SourcePath, job.Name, scenario) } } - if fails > 0 { - t.Logf("%v/%v jobs using deprecated scenarios", fails, total) + if jobsToFix > 0 { + t.Logf("summary: %v/%v jobs using deprecated scenarios", jobsToFix, len(jobs)) } }