Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

jobs: add summary lines to jobs tests #22838

Merged
merged 1 commit into from
Jul 9, 2021
Merged

jobs: add summary lines to jobs tests #22838

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 51 additions & 27 deletions config/tests/jobs/jobs_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -317,14 +317,17 @@ func TestTrustedJobs(t *testing.T) {

// Enforce conventions for jobs that run in k8s-infra-prow-build-trusted cluster
func TestK8sInfraTrusted(t *testing.T) {
jobsToFix := 0
const trusted = "k8s-infra-prow-build-trusted"
trustedPath := path.Join(*jobConfigPath, "kubernetes", "wg-k8s-infra", "trusted") + "/"
imagePushingDir := path.Join(*jobConfigPath, "image-pushing") + "/"

errs := []error{}
// Presubmits may not use this cluster
for _, pre := range c.AllStaticPresubmits(nil) {
if pre.Cluster == trusted {
t.Errorf("%s: presubmits may not run in cluster: %s", pre.Name, trusted)
jobsToFix++
errs = append(errs, fmt.Errorf("%s: presubmits may not run in cluster: %s", pre.Name, trusted))
}
}

Expand All @@ -344,12 +347,18 @@ func TestK8sInfraTrusted(t *testing.T) {
}
if strings.HasPrefix(job.SourcePath, imagePushingDir) {
if err := validateImagePushingImage(job.Spec); err != nil {
t.Errorf("%s defined in %s %s", job.Name, job.SourcePath, err)
jobsToFix++
errs = append(errs, fmt.Errorf("%s defined in %s %s", job.Name, job.SourcePath, err))
}
} else if !strings.HasPrefix(job.SourcePath, trustedPath) {
t.Errorf("%s defined in %s may not run in cluster: %s", job.Name, job.SourcePath, trusted)
jobsToFix++
errs = append(errs, fmt.Errorf("%s defined in %s may not run in cluster: %s", job.Name, job.SourcePath, trusted))
}
}
for _, err := range errs {
t.Errorf("%v", err)
}
t.Logf("summary: %4d/%4d jobs fail to meet k8s-infra-prow-build-trusted CI policy", jobsToFix, len(jobs))
}

func validateImagePushingImage(spec *coreapi.PodSpec) error {
Expand Down Expand Up @@ -978,6 +987,7 @@ func isKubernetesReleaseBlocking(job cfg.JobBase) bool {
}

func TestKubernetesMergeBlockingJobsCIPolicy(t *testing.T) {
jobsToFix := 0
repo := "kubernetes/kubernetes"
jobs := c.AllStaticPresubmits([]string{repo})
sort.Slice(jobs, func(i, j int) bool {
Expand All @@ -995,14 +1005,20 @@ func TestKubernetesMergeBlockingJobsCIPolicy(t *testing.T) {
errs = append(errs, fmt.Errorf("must run in cluster: k8s-infra-prow-build, found: %v", job.Cluster))
}
branches := job.Branches
if len(errs) > 0 {
jobsToFix++
}
for _, err := range errs {
t.Errorf("%v (%v): %v", job.Name, branches, err)
}
}
t.Logf("summary: %4d/%4d jobs fail to meet kubernetes/kubernetes merge-blocking CI policy", jobsToFix, len(jobs))
}

func TestKubernetesReleaseBlockingJobsCIPolicy(t *testing.T) {
for _, job := range allStaticJobs() {
jobsToFix := 0
jobs := allStaticJobs()
for _, job := range jobs {
// Only consider Pods that are release-blocking
if job.Spec == nil || !isKubernetesReleaseBlocking(job) {
continue
Expand All @@ -1013,13 +1029,18 @@ func TestKubernetesReleaseBlockingJobsCIPolicy(t *testing.T) {
if job.Cluster != "k8s-infra-prow-build" {
errs = append(errs, fmt.Errorf("must run in cluster: k8s-infra-prow-build, found: %v", job.Cluster))
}
if len(errs) > 0 {
jobsToFix++
}
for _, err := range errs {
t.Errorf("%v: %v", job.Name, err)
}
}
t.Logf("summary: %4d/%4d jobs fail to meet kubernetes/kubernetes release-blocking CI policy", jobsToFix, len(jobs))
}

func TestK8sInfraProwBuildJobsCIPolicy(t *testing.T) {
jobsToFix := 0
jobs := allStaticJobs()
for _, job := range jobs {
// Only consider Pods destined for the k8s-infra-prow-builds cluster
Expand All @@ -1028,31 +1049,41 @@ func TestK8sInfraProwBuildJobsCIPolicy(t *testing.T) {
}
// job Pod must qualify for Guaranteed QoS
errs := verifyPodQOSGuaranteed(job.Spec, true)
if len(errs) > 0 {
jobsToFix++
}
for _, err := range errs {
t.Errorf("%v: %v", job.Name, err)
}
}
t.Logf("summary: %4d/%4d jobs fail to meet k8s-infra-prow-build CI policy", jobsToFix, len(jobs))
}

// Fast builds take 20-30m, cross builds take 90m-2h. We want to pick up builds
// containing the latest merged PRs as soon as possible for the in-development release
func TestSigReleaseMasterBlockingOrInformingJobsMustUseFastBuilds(t *testing.T) {
jobsToFix := 0
jobs := allStaticJobs()
for _, job := range jobs {
dashboards, ok := job.Annotations["testgrid-dashboards"]
if !ok || !strings.Contains(dashboards, "sig-release-master-blocking") || !strings.Contains(dashboards, "sig-release-master-informing") {
continue
}
errs := []error{}
extract := ""
for _, arg := range job.Spec.Containers[0].Args {
if strings.HasPrefix(arg, "--extract=") {
extract = strings.TrimPrefix(arg, "--extract=")
if extract != "ci/latest-fast" {
t.Errorf("%s: release-master-blocking e2e jobs must use --extract=ci/latest-fast, found --extract=ci/latest instead", job.Name)
errs = append(errs, fmt.Errorf("release-master-blocking e2e jobs must use --extract=ci/latest-fast, found --extract=%s instead", extract))
}
}
}
for _, err := range errs {
t.Errorf("%v: %v", job.Name, err)
}
}
t.Logf("summary: %4d/%4d jobs fail to meet release-master-blocking CI policy", jobsToFix, len(jobs))
}

// matches regex used by the "version" extractMode defined in kubetest/extract_k8s.go
Expand Down Expand Up @@ -1096,8 +1127,8 @@ func extractUsesReleaseBucket(extract string) bool {
// TODO(https://github.com/kubernetes/k8s.io/issues/1569): start warning once gs://k8s-release populated
// - jobs using --extract=release/* SHOULD pull from gs://k8s-release
func TestKubernetesE2eJobsMustExtractFromK8sInfraBuckets(t *testing.T) {
jobsToFix := 0
jobs := allStaticJobs()
var totalJobs, needsFixJobs int
for _, job := range jobs {
needsFix := false
extracts := []string{}
Expand Down Expand Up @@ -1156,23 +1187,19 @@ func TestKubernetesE2eJobsMustExtractFromK8sInfraBuckets(t *testing.T) {
}
}
}
totalJobs++
if needsFix {
needsFixJobs++
jobsToFix++
}
}
if needsFixJobs > 0 {
t.Logf("%4d/%4d jobs should be updated to pull from community-owned gcs buckets", needsFixJobs, totalJobs)
}
t.Logf("summary: %4d/%4d jobs should be updated to pull from community-owned gcs buckets", jobsToFix, len(jobs))
}

// Prow jobs should use pod-utils instead of relying on bootstrap
// https://github.com/kubernetes/test-infra/issues/20760
func TestKubernetesProwJobsShouldUsePodUtils(t *testing.T) {
total := 0
fails := 0
for _, job := range allStaticJobs() {
total++
jobsToFix := 0
jobs := allStaticJobs()
for _, job := range jobs {
// Only consider Pods
// TODO(https://github.com/kubernetes/test-infra/issues/14343): remove kubeflow exemption when job configs migrated
if job.Spec == nil || strings.HasPrefix("kubeflow", job.Name) {
Expand All @@ -1191,7 +1218,7 @@ func TestKubernetesProwJobsShouldUsePodUtils(t *testing.T) {
scenario = strings.TrimPrefix(arg, "--scenario=")
}
}
fails++
jobsToFix++
if len(repos) > 0 {
t.Logf("%v: %v: should use pod-utils, found bootstrap args to clone: %v", job.SourcePath, job.Name, repos)
} else if scenario != "" {
Expand All @@ -1201,18 +1228,15 @@ func TestKubernetesProwJobsShouldUsePodUtils(t *testing.T) {
}
}
}
if fails > 0 {
t.Logf("%v/%v jobs do not use pod-utils", fails, total)
}
t.Logf("summary: %4d/%4d jobs do not use pod-utils", jobsToFix, len(jobs))
}

// Prow jobs should use kubetest2 instead of deprecated scenarios
// https://github.com/kubernetes/test-infra/tree/master/scenarios#deprecation-notice
func TestKubernetesProwJobsShouldNotUseDeprecatedScenarios(t *testing.T) {
total := 0
fails := 0
for _, job := range allStaticJobs() {
total++
jobsToFix := 0
jobs := allStaticJobs()
for _, job := range jobs {
// Only consider Pods
if job.Spec == nil {
continue
Expand All @@ -1229,7 +1253,7 @@ func TestKubernetesProwJobsShouldNotUseDeprecatedScenarios(t *testing.T) {
}
}
if scenario != "" {
fails++
jobsToFix++
t.Logf("%v: %v: should not be using deprecated scenarios, is directly invoking: %v", job.SourcePath, job.Name, scenario)
continue
}
Expand All @@ -1239,11 +1263,11 @@ func TestKubernetesProwJobsShouldNotUseDeprecatedScenarios(t *testing.T) {
}
}
if scenario != "" {
fails++
jobsToFix++
t.Logf("%v: %v: should not be using deprecated scenarios, is invoking via bootrap: %v", job.SourcePath, job.Name, scenario)
}
}
if fails > 0 {
t.Logf("%v/%v jobs using deprecated scenarios", fails, total)
if jobsToFix > 0 {
t.Logf("summary: %v/%v jobs using deprecated scenarios", jobsToFix, len(jobs))
}
}