Skip to content

Commit

Permalink
feat: add TopologySpreadConstraints to PodSpec (#2362)
Browse files Browse the repository at this point in the history 
* feat: add TopologySpreadConstraints to PodSpec

Signed-off-by: Shahar Harari <[email protected]>

* add RL test

Signed-off-by: Shahar Harari <[email protected]>

* fix lint error

Signed-off-by: Shahar Harari <[email protected]>

* update generated files

Signed-off-by: Shahar Harari <[email protected]>

---------

Signed-off-by: Shahar Harari <[email protected]>
  • Loading branch information
@shahar-h
shahar-h authored Dec 29, 2023
1 parent 987ce64 commit a723843
Show file tree
Hide file tree
Showing 10 changed files with 602 additions and 0 deletions.
7 changes: 7 additions & 0 deletions api/v1alpha1/shared_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,13 @@ type KubernetesPodSpec struct {
//
// +optional
NodeSelector map[string]string `json:"nodeSelector,omitempty"`

// TopologySpreadConstraints describes how a group of pods ought to spread across topology
// domains. Scheduler will schedule pods in a way which abides by the constraints.
// All topologySpreadConstraints are ANDed.
//
// +optional
TopologySpreadConstraints []corev1.TopologySpreadConstraint `json:"topologySpreadConstraints,omitempty"`
}

// KubernetesContainerSpec defines the desired state of the Kubernetes container resource.
Expand Down
7 changes: 7 additions & 0 deletions api/v1alpha1/zz_generated.deepcopy.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

215 changes: 215 additions & 0 deletions charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml
View file

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions internal/infrastructure/kubernetes/proxy/resource_provider.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -241,6 +241,7 @@ func (r *ResourceRender) Deployment() (*appsv1.Deployment, error) {
Volumes: expectedDeploymentVolumes(r.infra.Name, deploymentConfig),
ImagePullSecrets: deploymentConfig.Pod.ImagePullSecrets,
NodeSelector: deploymentConfig.Pod.NodeSelector,
TopologySpreadConstraints: deploymentConfig.Pod.TopologySpreadConstraints,
},
},
RevisionHistoryLimit: ptr.To[int32](10),
Expand Down
20 changes: 20 additions & 0 deletions internal/infrastructure/kubernetes/proxy/resource_provider_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
autoscalingv2 "k8s.io/api/autoscaling/v2"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/utils/ptr"
"sigs.k8s.io/yaml"

Expand Down Expand Up @@ -403,6 +404,25 @@ func TestDeployment(t *testing.T) {
},
},
},
{
caseName: "with-topology-spread-constraints",
infra: newTestInfra(),
deploy: &egv1a1.KubernetesDeploymentSpec{
Pod: &egv1a1.KubernetesPodSpec{
TopologySpreadConstraints: []corev1.TopologySpreadConstraint{
{
MaxSkew: 1,
TopologyKey: "kubernetes.io/hostname",
WhenUnsatisfiable: corev1.DoNotSchedule,
LabelSelector: &metav1.LabelSelector{
MatchLabels: map[string]string{"app": "foo"},
},
MatchLabelKeys: []string{"pod-template-hash"},
},
},
},
},
},
}
for _, tc := range cases {
t.Run(tc.caseName, func(t *testing.T) {
Expand Down
208 changes: 208 additions & 0 deletions ...nfrastructure/kubernetes/proxy/testdata/deployments/with-topology-spread-constraints.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,208 @@
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app.kubernetes.io/name: envoy
app.kubernetes.io/component: proxy
app.kubernetes.io/managed-by: envoy-gateway
gateway.envoyproxy.io/owning-gateway-name: default
gateway.envoyproxy.io/owning-gateway-namespace: default
name: envoy-default-37a8eec1
namespace: envoy-gateway-system
spec:
replicas: 1
strategy:
type: RollingUpdate
selector:
matchLabels:
app.kubernetes.io/name: envoy
app.kubernetes.io/component: proxy
app.kubernetes.io/managed-by: envoy-gateway
gateway.envoyproxy.io/owning-gateway-name: default
gateway.envoyproxy.io/owning-gateway-namespace: default
template:
metadata:
labels:
app.kubernetes.io/name: envoy
app.kubernetes.io/component: proxy
app.kubernetes.io/managed-by: envoy-gateway
gateway.envoyproxy.io/owning-gateway-name: default
gateway.envoyproxy.io/owning-gateway-namespace: default
spec:
automountServiceAccountToken: false
containers:
- args:
- --service-cluster default
- --service-node $(ENVOY_POD_NAME)
- |
--config-yaml admin:
access_log:
- name: envoy.access_loggers.file
typed_config:
"@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog
path: /dev/null
address:
socket_address:
address: 127.0.0.1
port_value: 19000
dynamic_resources:
ads_config:
api_type: DELTA_GRPC
transport_api_version: V3
grpc_services:
- envoy_grpc:
cluster_name: xds_cluster
set_node_on_first_message_only: true
lds_config:
ads: {}
resource_api_version: V3
cds_config:
ads: {}
resource_api_version: V3
static_resources:
listeners:
- name: envoy-gateway-proxy-ready-0.0.0.0-19001
address:
socket_address:
address: 0.0.0.0
port_value: 19001
protocol: TCP
filter_chains:
- filters:
- name: envoy.filters.network.http_connection_manager
typed_config:
"@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
stat_prefix: eg-ready-http
route_config:
name: local_route
http_filters:
- name: envoy.filters.http.health_check
typed_config:
"@type": type.googleapis.com/envoy.extensions.filters.http.health_check.v3.HealthCheck
pass_through_mode: false
headers:
- name: ":path"
string_match:
exact: /ready
- name: envoy.filters.http.router
typed_config:
"@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
clusters:
- connect_timeout: 10s
load_assignment:
cluster_name: xds_cluster
endpoints:
- load_balancing_weight: 1
lb_endpoints:
- load_balancing_weight: 1
endpoint:
address:
socket_address:
address: envoy-gateway
port_value: 18000
typed_extension_protocol_options:
envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
"@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions"
explicit_http_config:
http2_protocol_options:
connection_keepalive:
interval: 30s
timeout: 5s
name: xds_cluster
type: STRICT_DNS
transport_socket:
name: envoy.transport_sockets.tls
typed_config:
"@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext
common_tls_context:
tls_params:
tls_maximum_protocol_version: TLSv1_3
tls_certificate_sds_secret_configs:
- name: xds_certificate
sds_config:
path_config_source:
path: "/sds/xds-certificate.json"
resource_api_version: V3
validation_context_sds_secret_config:
name: xds_trusted_ca
sds_config:
path_config_source:
path: "/sds/xds-trusted-ca.json"
resource_api_version: V3
- --log-level warn
- --cpuset-threads
command:
- envoy
env:
- name: ENVOY_GATEWAY_NAMESPACE
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
- name: ENVOY_POD_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.name
image: envoyproxy/envoy-dev:latest
imagePullPolicy: IfNotPresent
name: envoy
ports:
- containerPort: 8080
name: EnvoyH-d76a15e2
protocol: TCP
- containerPort: 8443
name: EnvoyH-6658f727
protocol: TCP
resources:
requests:
cpu: 100m
memory: 512Mi
readinessProbe:
httpGet:
path: /ready
port: 19001
scheme: HTTP
timeoutSeconds: 1
periodSeconds: 10
successThreshold: 1
failureThreshold: 3
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /certs
name: certs
readOnly: true
- mountPath: /sds
name: sds
dnsPolicy: ClusterFirst
restartPolicy: Always
schedulerName: default-scheduler
serviceAccountName: envoy-default-37a8eec1
terminationGracePeriodSeconds: 300
topologySpreadConstraints:
- maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: DoNotSchedule
labelSelector:
matchLabels:
app: foo
matchLabelKeys:
- pod-template-hash
volumes:
- name: certs
secret:
secretName: envoy
defaultMode: 420
- configMap:
defaultMode: 420
items:
- key: xds-trusted-ca.json
path: xds-trusted-ca.json
- key: xds-certificate.json
path: xds-certificate.json
name: envoy-default-37a8eec1
optional: false
name: sds
revisionHistoryLimit: 10
progressDeadlineSeconds: 600
1 change: 1 addition & 0 deletions internal/infrastructure/kubernetes/ratelimit/resource_provider.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,7 @@ func (r *ResourceRender) Deployment() (*appsv1.Deployment, error) {
Tolerations: r.rateLimitDeployment.Pod.Tolerations,
ImagePullSecrets: r.rateLimitDeployment.Pod.ImagePullSecrets,
NodeSelector: r.rateLimitDeployment.Pod.NodeSelector,
TopologySpreadConstraints: r.rateLimitDeployment.Pod.TopologySpreadConstraints,
},
},
RevisionHistoryLimit: ptr.To[int32](10),
Expand Down
20 changes: 20 additions & 0 deletions internal/infrastructure/kubernetes/ratelimit/resource_provider_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import (
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/utils/ptr"
gwapiv1 "sigs.k8s.io/gateway-api/apis/v1"
Expand Down Expand Up @@ -493,6 +494,25 @@ func TestDeployment(t *testing.T) {
},
},
},
{
caseName: "with-topology-spread-constraints",
rateLimit: rateLimit,
deploy: &egv1a1.KubernetesDeploymentSpec{
Pod: &egv1a1.KubernetesPodSpec{
TopologySpreadConstraints: []corev1.TopologySpreadConstraint{
{
MaxSkew: 1,
TopologyKey: "kubernetes.io/hostname",
WhenUnsatisfiable: corev1.DoNotSchedule,
LabelSelector: &metav1.LabelSelector{
MatchLabels: map[string]string{"app": "foo"},
},
MatchLabelKeys: []string{"pod-template-hash"},
},
},
},
},
},
}
for _, tc := range cases {
t.Run(tc.caseName, func(t *testing.T) {
Expand Down
Loading

0 comments on commit a723843

Please sign in to comment.