Resync 20211223

README.md

@@ -24,6 +24,7 @@ The kube-scheduler binary includes the below list of plugins. They can be config
 * [Coscheduling](pkg/coscheduling/README.md)
 * [Node Resources](pkg/noderesources/README.md)
 * [Node Resource Topology](pkg/noderesourcetopology/README.md)
+* [Preemption Toleration](pkg/preemptiontoleration/README.md)
 * [Trimaran](pkg/trimaran/README.md)
 
 Additionally the kube-scheduler binary includes the below list of sample plugins. These plugins are not intended for use in production

RESYNC.md

@@ -39,8 +39,10 @@ at the beginning of the commit message.
 
 For the sake of transparency, for every resync process we should update the following table:
 
-| Resync Date | Merge With Upstream Tag/Commit                                                                       | Author |
-|-------------|------------------------------------------------------------------------------------------------------|--------|
-| 12.11.2021  | https://github.com/kubernetes-sigs/scheduler-plugins/commit/b8d13e17a3e1f633d72d71276a3da6fecf89f2e3 | Tal-or |
+  | Resync Date  | Merge With Upstream Tag/Commit                                                                        | Author|
+  |--------------|-------------------------------------------------------------------------------------------------------|-------|
+  | 23.12.2021   | https://github.com/kubernetes-sigs/scheduler-plugins/commit/5f63e9cc7af2fda28ff1f0ba0dff0cb847c36179  | Tal-or|
+  | 12.11.2021   | https://github.com/kubernetes-sigs/scheduler-plugins/commit/b8d13e17a3e1f633d72d71276a3da6fecf89f2e3  | Tal-or|
+
+The newest resync should appear in the first row.
 
-The newest resync should appear in the first row. 

cloudbuild.yaml

@@ -7,7 +7,7 @@ timeout: 2400s
 options:
   substitution_option: ALLOW_LOOSE
 steps:
-  - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20190906-745fed4'
+  - name: 'gcr.io/k8s-staging-test-infra/gcb-docker-gcloud:v20211118-2f2d816b90'
     entrypoint: make
     env:
     - DOCKER_CLI_EXPERIMENTAL=enabled

cmd/scheduler/main.go

@@ -30,9 +30,11 @@ import (
 	"sigs.k8s.io/scheduler-plugins/pkg/noderesources"
 	"sigs.k8s.io/scheduler-plugins/pkg/noderesourcetopology"
 	"sigs.k8s.io/scheduler-plugins/pkg/podstate"
+	"sigs.k8s.io/scheduler-plugins/pkg/preemptiontoleration"
 	"sigs.k8s.io/scheduler-plugins/pkg/qos"
 	"sigs.k8s.io/scheduler-plugins/pkg/trimaran/loadvariationriskbalancing"
 	"sigs.k8s.io/scheduler-plugins/pkg/trimaran/targetloadpacking"
+
 	// Ensure scheme package is initialized.
 	_ "sigs.k8s.io/scheduler-plugins/pkg/apis/config/scheme"
 )
@@ -49,6 +51,7 @@ func main() {
 		app.WithPlugin(loadvariationriskbalancing.Name, loadvariationriskbalancing.New),
 		app.WithPlugin(noderesources.AllocatableName, noderesources.NewAllocatable),
 		app.WithPlugin(noderesourcetopology.Name, noderesourcetopology.New),
+		app.WithPlugin(preemptiontoleration.Name, preemptiontoleration.New),
 		app.WithPlugin(targetloadpacking.Name, targetloadpacking.New),
 		// Sample plugins below.
 		app.WithPlugin(crossnodepreemption.Name, crossnodepreemption.New),

kep/205-preemption-toleration/README.md

@@ -111,9 +111,6 @@ kind: PriorityClass
 metadata:
   name: toleration-policy-sample
   annotation:
-    # this key is needed to enable preemption toleration policy for distinguishing 
-    # between no toleration policy and empty toleration policy (all fields will be default)
-    preemption-toleration.scheduling.sigs.k8s.io/enabled: ""
     preemption-toleration.scheduling.sigs.k8s.io/minimum-preemptable-priority: "10000"
     preemption-toleration.scheduling.sigs.k8s.io/toleration-seconds: "3600"
 value: 8000
@@ -155,7 +152,6 @@ kind: PriorityClass
 metadata:
   name: low-non-preempted
   annotation:
-    preemption-toleration.scheduling.sigs.k8s.io/enabled: ""
     # This priority class can tolerate preemption by priority with p < 10000.
     preemption-toleration.scheduling.sigs.k8s.io/minimum-preemptable-priority: "10000"
     # This priority class can tolerate preemption forever by priority with p < 10000(=minimum-preemptable-priority)
@@ -189,7 +185,6 @@ kind: PriorityClass
 metadata:
   name: low-non-preempted-10min
   annotation:
-    preemption-toleration.scheduling.sigs.k8s.io/enabled: ""
     # This priority class can tolerate preemption by priority with p < 10000.
     preemption-toleration.scheduling.sigs.k8s.io/minimum-preemptable-priority: "10000"
     # This priority class can tolerate preemption for 10 minutes (600 seconds) 

manifests/install/charts/as-a-second-scheduler/templates/configmap.yaml

@@ -5,7 +5,7 @@ metadata:
   namespace: {{ .Values.scheduler.namespace }}
 data:
   scheduler-config.yaml: |
-    apiVersion: kubescheduler.config.k8s.io/v1beta2
+    apiVersion: kubescheduler.config.k8s.io/v1beta1
     kind: KubeSchedulerConfiguration
     leaderElection:
       leaderElect: false

pkg/apis/config/register.go

@@ -42,6 +42,7 @@ func addKnownTypes(scheme *runtime.Scheme) error {
 		&TargetLoadPackingArgs{},
 		&LoadVariationRiskBalancingArgs{},
 		&NodeResourceTopologyMatchArgs{},
+		&PreemptionTolerationArgs{},
 	)
 	return nil
 }

pkg/apis/config/scheme/scheme_test.go

@@ -33,6 +33,7 @@ import (
 	"sigs.k8s.io/scheduler-plugins/pkg/apis/config/v1beta1"
 	"sigs.k8s.io/scheduler-plugins/pkg/coscheduling"
 	"sigs.k8s.io/scheduler-plugins/pkg/noderesources"
+	"sigs.k8s.io/scheduler-plugins/pkg/preemptiontoleration"
 	"sigs.k8s.io/scheduler-plugins/pkg/trimaran/loadvariationriskbalancing"
 	"sigs.k8s.io/scheduler-plugins/pkg/trimaran/targetloadpacking"
 	"sigs.k8s.io/yaml"
@@ -357,6 +358,10 @@ profiles:
       safeVarianceMargin: 1.0
       safeVarianceSensitivity: 1.0
       watcherAddress: http://deadbeef:2020
+  - name: PreemptionToleration
+    args:
+      minCandidateNodesPercentage: 20
+      minCandidateNodesAbsolute: 200
 `),
 			wantProfiles: []defaultconfig.KubeSchedulerProfile{
 				{
@@ -407,6 +412,10 @@ profiles:
 								},
 							},
 						},
+						{
+							Name: preemptiontoleration.Name,
+							Args: &config.PreemptionTolerationArgs{MinCandidateNodesPercentage: 20, MinCandidateNodesAbsolute: 200},
+						},
 						{
 							Name: "DefaultPreemption",
 							Args: &defaultconfig.DefaultPreemptionArgs{MinCandidateNodesPercentage: 10, MinCandidateNodesAbsolute: 100},
@@ -460,6 +469,8 @@ profiles:
     args:
   - name: LoadVariationRiskBalancing
     args:
+  - name: PreemptionToleration
+    args:
 `),
 			wantProfiles: []defaultconfig.KubeSchedulerProfile{
 				{
@@ -512,6 +523,10 @@ profiles:
 								},
 							},
 						},
+						{
+							Name: preemptiontoleration.Name,
+							Args: &config.PreemptionTolerationArgs{MinCandidateNodesPercentage: 10, MinCandidateNodesAbsolute: 100},
+						},
 						{
 							Name: "DefaultPreemption",
 							Args: &defaultconfig.DefaultPreemptionArgs{MinCandidateNodesPercentage: 10, MinCandidateNodesAbsolute: 100},

pkg/apis/config/types.go

@@ -19,7 +19,10 @@ package config
 import (
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	// TODO: eliminate the "versioned" import, i.e., schedulerconfig.ResourceSpec should be unversioned.ResourceSpec.
 	schedulerconfig "k8s.io/kube-scheduler/config/v1"
+	unversioned "k8s.io/kubernetes/pkg/scheduler/apis/config"
 )
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
@@ -145,3 +148,8 @@ type NodeResourceTopologyMatchArgs struct {
 	// ScoringStrategy a scoring model that determine how the plugin will score the nodes.
 	ScoringStrategy ScoringStrategy
 }
+
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
+// PreemptionTolerationArgs reuses DefaultPluginArgs.
+type PreemptionTolerationArgs unversioned.DefaultPreemptionArgs

pkg/apis/config/v1beta2/defaults.go

@@ -24,7 +24,9 @@ import (
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	schedulerconfig "k8s.io/kube-scheduler/config/v1"
+	schedulerconfigv1beta2 "k8s.io/kube-scheduler/config/v1beta2"
 
+	k8sschedulerconfigv1beta2 "k8s.io/kubernetes/pkg/scheduler/apis/config/v1beta2"
 	pluginConfig "sigs.k8s.io/scheduler-plugins/pkg/apis/config"
 )
 
@@ -143,3 +145,8 @@ func SetDefaultsNodeResourceTopologyMatchArgs(obj *NodeResourceTopologyMatchArgs
 		}
 	}
 }
+
+// PreemptionTolerationArgs reuses SetDefaults_DefaultPreemptionArgs
+func SetDefaultsPreemptionTolerationArgs(obj *PreemptionTolerationArgs) {
+	k8sschedulerconfigv1beta2.SetDefaults_DefaultPreemptionArgs((*schedulerconfigv1beta2.DefaultPreemptionArgs)(obj))
+}

pkg/apis/config/v1beta2/defaults_test.go

@@ -146,6 +146,14 @@ func TestSchedulingDefaults(t *testing.T) {
 				},
 			},
 		},
+		{
+			name:   "empty config PreeemptionTolerationArgs",
+			config: &PreemptionTolerationArgs{},
+			expect: &PreemptionTolerationArgs{
+				MinCandidateNodesPercentage: pointer.Int32Ptr(10),
+				MinCandidateNodesAbsolute:   pointer.Int32Ptr(100),
+			},
+		},
 	}
 
 	for _, tc := range tests {

pkg/apis/config/v1beta2/register.go

@@ -42,6 +42,7 @@ func addKnownTypes(scheme *runtime.Scheme) error {
 		&TargetLoadPackingArgs{},
 		&LoadVariationRiskBalancingArgs{},
 		&NodeResourceTopologyMatchArgs{},
+		&PreemptionTolerationArgs{},
 	)
 	return nil
 }

pkg/apis/config/v1beta2/types.go

@@ -20,6 +20,7 @@ import (
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	schedulerconfig "k8s.io/kube-scheduler/config/v1"
+	schedulerconfigv1beta2 "k8s.io/kube-scheduler/config/v1beta2"
 )
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
@@ -142,3 +143,9 @@ type NodeResourceTopologyMatchArgs struct {
 
 	ScoringStrategy *ScoringStrategy `json:"scoringStrategy,omitempty"`
 }
+
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+// +k8s:defaulter-gen=true
+
+// PreemptionTolerationArgs reuses DefaultPluginArgs.
+type PreemptionTolerationArgs schedulerconfigv1beta2.DefaultPreemptionArgs

pkg/controller/podgroup.go

@@ -243,6 +243,11 @@ func (ctrl *PodGroupController) syncHandler(key string) error {
 		pgCopy.Status.Succeeded = succeeded
 		pgCopy.Status.Running = running
 
+		if len(pods) == 0 {
+			pgCopy.Status.Phase = schedv1alpha1.PodGroupPending
+			break
+		}
+
 		if pgCopy.Status.Scheduled >= pgCopy.Spec.MinMember && pgCopy.Status.Phase == schedv1alpha1.PodGroupScheduling {
 			pgCopy.Status.Phase = schedv1alpha1.PodGroupScheduled
 		}