Skip to content

Commit

Permalink
Merge pull request #22838 from spiffxp/summary-tests
Browse files Browse the repository at this point in the history 
jobs: add summary lines to jobs tests
  • Loading branch information
@k8s-ci-robot
k8s-ci-robot authored Jul 9, 2021
2 parents 07b76df + 4a98524 commit 064a817
Showing 1 changed file with 51 additions and 27 deletions.
78 changes: 51 additions & 27 deletions config/tests/jobs/jobs_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -317,14 +317,17 @@ func TestTrustedJobs(t *testing.T) {

// Enforce conventions for jobs that run in k8s-infra-prow-build-trusted cluster
func TestK8sInfraTrusted(t *testing.T) {
jobsToFix := 0
const trusted = "k8s-infra-prow-build-trusted"
trustedPath := path.Join(*jobConfigPath, "kubernetes", "wg-k8s-infra", "trusted") + "/"
imagePushingDir := path.Join(*jobConfigPath, "image-pushing") + "/"

errs := []error{}
// Presubmits may not use this cluster
for _, pre := range c.AllStaticPresubmits(nil) {
if pre.Cluster == trusted {
t.Errorf("%s: presubmits may not run in cluster: %s", pre.Name, trusted)
jobsToFix++
errs = append(errs, fmt.Errorf("%s: presubmits may not run in cluster: %s", pre.Name, trusted))
}
}

Expand All @@ -344,12 +347,18 @@ func TestK8sInfraTrusted(t *testing.T) {
}
if strings.HasPrefix(job.SourcePath, imagePushingDir) {
if err := validateImagePushingImage(job.Spec); err != nil {
t.Errorf("%s defined in %s %s", job.Name, job.SourcePath, err)
jobsToFix++
errs = append(errs, fmt.Errorf("%s defined in %s %s", job.Name, job.SourcePath, err))
}
} else if !strings.HasPrefix(job.SourcePath, trustedPath) {
t.Errorf("%s defined in %s may not run in cluster: %s", job.Name, job.SourcePath, trusted)
jobsToFix++
errs = append(errs, fmt.Errorf("%s defined in %s may not run in cluster: %s", job.Name, job.SourcePath, trusted))
}
}
for _, err := range errs {
t.Errorf("%v", err)
}
t.Logf("summary: %4d/%4d jobs fail to meet k8s-infra-prow-build-trusted CI policy", jobsToFix, len(jobs))
}

func validateImagePushingImage(spec *coreapi.PodSpec) error {
Expand Down Expand Up @@ -978,6 +987,7 @@ func isKubernetesReleaseBlocking(job cfg.JobBase) bool {
}

func TestKubernetesMergeBlockingJobsCIPolicy(t *testing.T) {
jobsToFix := 0
repo := "kubernetes/kubernetes"
jobs := c.AllStaticPresubmits([]string{repo})
sort.Slice(jobs, func(i, j int) bool {
Expand All @@ -995,14 +1005,20 @@ func TestKubernetesMergeBlockingJobsCIPolicy(t *testing.T) {
errs = append(errs, fmt.Errorf("must run in cluster: k8s-infra-prow-build, found: %v", job.Cluster))
}
branches := job.Branches
if len(errs) > 0 {
jobsToFix++
}
for _, err := range errs {
t.Errorf("%v (%v): %v", job.Name, branches, err)
}
}
t.Logf("summary: %4d/%4d jobs fail to meet kubernetes/kubernetes merge-blocking CI policy", jobsToFix, len(jobs))
}

func TestKubernetesReleaseBlockingJobsCIPolicy(t *testing.T) {
for _, job := range allStaticJobs() {
jobsToFix := 0
jobs := allStaticJobs()
for _, job := range jobs {
// Only consider Pods that are release-blocking
if job.Spec == nil || !isKubernetesReleaseBlocking(job) {
continue
Expand All @@ -1013,13 +1029,18 @@ func TestKubernetesReleaseBlockingJobsCIPolicy(t *testing.T) {
if job.Cluster != "k8s-infra-prow-build" {
errs = append(errs, fmt.Errorf("must run in cluster: k8s-infra-prow-build, found: %v", job.Cluster))
}
if len(errs) > 0 {
jobsToFix++
}
for _, err := range errs {
t.Errorf("%v: %v", job.Name, err)
}
}
t.Logf("summary: %4d/%4d jobs fail to meet kubernetes/kubernetes release-blocking CI policy", jobsToFix, len(jobs))
}

func TestK8sInfraProwBuildJobsCIPolicy(t *testing.T) {
jobsToFix := 0
jobs := allStaticJobs()
for _, job := range jobs {
// Only consider Pods destined for the k8s-infra-prow-builds cluster
Expand All @@ -1028,31 +1049,41 @@ func TestK8sInfraProwBuildJobsCIPolicy(t *testing.T) {
}
// job Pod must qualify for Guaranteed QoS
errs := verifyPodQOSGuaranteed(job.Spec, true)
if len(errs) > 0 {
jobsToFix++
}
for _, err := range errs {
t.Errorf("%v: %v", job.Name, err)
}
}
t.Logf("summary: %4d/%4d jobs fail to meet k8s-infra-prow-build CI policy", jobsToFix, len(jobs))
}

// Fast builds take 20-30m, cross builds take 90m-2h. We want to pick up builds
// containing the latest merged PRs as soon as possible for the in-development release
func TestSigReleaseMasterBlockingOrInformingJobsMustUseFastBuilds(t *testing.T) {
jobsToFix := 0
jobs := allStaticJobs()
for _, job := range jobs {
dashboards, ok := job.Annotations["testgrid-dashboards"]
if !ok || !strings.Contains(dashboards, "sig-release-master-blocking") || !strings.Contains(dashboards, "sig-release-master-informing") {
continue
}
errs := []error{}
extract := ""
for _, arg := range job.Spec.Containers[0].Args {
if strings.HasPrefix(arg, "--extract=") {
extract = strings.TrimPrefix(arg, "--extract=")
if extract != "ci/latest-fast" {
t.Errorf("%s: release-master-blocking e2e jobs must use --extract=ci/latest-fast, found --extract=ci/latest instead", job.Name)
errs = append(errs, fmt.Errorf("release-master-blocking e2e jobs must use --extract=ci/latest-fast, found --extract=%s instead", extract))
}
}
}
for _, err := range errs {
t.Errorf("%v: %v", job.Name, err)
}
}
t.Logf("summary: %4d/%4d jobs fail to meet release-master-blocking CI policy", jobsToFix, len(jobs))
}

// matches regex used by the "version" extractMode defined in kubetest/extract_k8s.go
Expand Down Expand Up @@ -1096,8 +1127,8 @@ func extractUsesReleaseBucket(extract string) bool {
// TODO(https://github.com/kubernetes/k8s.io/issues/1569): start warning once gs://k8s-release populated
// - jobs using --extract=release/* SHOULD pull from gs://k8s-release
func TestKubernetesE2eJobsMustExtractFromK8sInfraBuckets(t *testing.T) {
jobsToFix := 0
jobs := allStaticJobs()
var totalJobs, needsFixJobs int
for _, job := range jobs {
needsFix := false
extracts := []string{}
Expand Down Expand Up @@ -1156,23 +1187,19 @@ func TestKubernetesE2eJobsMustExtractFromK8sInfraBuckets(t *testing.T) {
}
}
}
totalJobs++
if needsFix {
needsFixJobs++
jobsToFix++
}
}
if needsFixJobs > 0 {
t.Logf("%4d/%4d jobs should be updated to pull from community-owned gcs buckets", needsFixJobs, totalJobs)
}
t.Logf("summary: %4d/%4d jobs should be updated to pull from community-owned gcs buckets", jobsToFix, len(jobs))
}

// Prow jobs should use pod-utils instead of relying on bootstrap
// https://github.com/kubernetes/test-infra/issues/20760
func TestKubernetesProwJobsShouldUsePodUtils(t *testing.T) {
total := 0
fails := 0
for _, job := range allStaticJobs() {
total++
jobsToFix := 0
jobs := allStaticJobs()
for _, job := range jobs {
// Only consider Pods
// TODO(https://github.com/kubernetes/test-infra/issues/14343): remove kubeflow exemption when job configs migrated
if job.Spec == nil || strings.HasPrefix("kubeflow", job.Name) {
Expand All @@ -1191,7 +1218,7 @@ func TestKubernetesProwJobsShouldUsePodUtils(t *testing.T) {
scenario = strings.TrimPrefix(arg, "--scenario=")
}
}
fails++
jobsToFix++
if len(repos) > 0 {
t.Logf("%v: %v: should use pod-utils, found bootstrap args to clone: %v", job.SourcePath, job.Name, repos)
} else if scenario != "" {
Expand All @@ -1201,18 +1228,15 @@ func TestKubernetesProwJobsShouldUsePodUtils(t *testing.T) {
}
}
}
if fails > 0 {
t.Logf("%v/%v jobs do not use pod-utils", fails, total)
}
t.Logf("summary: %4d/%4d jobs do not use pod-utils", jobsToFix, len(jobs))
}

// Prow jobs should use kubetest2 instead of deprecated scenarios
// https://github.com/kubernetes/test-infra/tree/master/scenarios#deprecation-notice
func TestKubernetesProwJobsShouldNotUseDeprecatedScenarios(t *testing.T) {
total := 0
fails := 0
for _, job := range allStaticJobs() {
total++
jobsToFix := 0
jobs := allStaticJobs()
for _, job := range jobs {
// Only consider Pods
if job.Spec == nil {
continue
Expand All @@ -1229,7 +1253,7 @@ func TestKubernetesProwJobsShouldNotUseDeprecatedScenarios(t *testing.T) {
}
}
if scenario != "" {
fails++
jobsToFix++
t.Logf("%v: %v: should not be using deprecated scenarios, is directly invoking: %v", job.SourcePath, job.Name, scenario)
continue
}
Expand All @@ -1239,11 +1263,11 @@ func TestKubernetesProwJobsShouldNotUseDeprecatedScenarios(t *testing.T) {
}
}
if scenario != "" {
fails++
jobsToFix++
t.Logf("%v: %v: should not be using deprecated scenarios, is invoking via bootrap: %v", job.SourcePath, job.Name, scenario)
}
}
if fails > 0 {
t.Logf("%v/%v jobs using deprecated scenarios", fails, total)
if jobsToFix > 0 {
t.Logf("summary: %v/%v jobs using deprecated scenarios", jobsToFix, len(jobs))
}
}

0 comments on commit 064a817

Please sign in to comment.