From ccca1c312f118d9bebcda2003836a10ffb677aab Mon Sep 17 00:00:00 2001 From: Ewout Prangsma Date: Fri, 16 Feb 2018 15:04:07 +0100 Subject: [PATCH] Added pod affinity --- pkg/deployment/pods.go | 9 +++-- pkg/util/k8sutil/affinity.go | 76 +++++++++++++++++++++++++++++++++++ pkg/util/k8sutil/constants.go | 1 + pkg/util/k8sutil/pods.go | 18 ++++++--- 4 files changed, 95 insertions(+), 9 deletions(-) create mode 100644 pkg/util/k8sutil/affinity.go diff --git a/pkg/deployment/pods.go b/pkg/deployment/pods.go index 21d9ae9af..b55d6c8ca 100644 --- a/pkg/deployment/pods.go +++ b/pkg/deployment/pods.go @@ -285,7 +285,6 @@ func (d *Deployment) createReadinessProbe(apiObject *api.ArangoDeployment, group // ensurePods creates all Pods listed in member status func (d *Deployment) ensurePods(apiObject *api.ArangoDeployment) error { kubecli := d.deps.KubeCli - owner := apiObject.AsOwner() if err := apiObject.ForeachServerGroup(func(group api.ServerGroup, spec api.ServerGroupSpec, status *api.MemberStatusList) error { for _, m := range *status { @@ -305,7 +304,7 @@ func (d *Deployment) ensurePods(apiObject *api.ArangoDeployment) error { if err != nil { return maskAny(err) } - if err := k8sutil.CreateArangodPod(kubecli, apiObject, role, m.ID, m.PersistentVolumeClaimName, apiObject.Spec.Image, apiObject.Spec.ImagePullPolicy, args, env, livenessProbe, readinessProbe, owner); err != nil { + if err := k8sutil.CreateArangodPod(kubecli, apiObject.Spec.IsDevelopment(), apiObject, role, m.ID, m.PersistentVolumeClaimName, apiObject.Spec.Image, apiObject.Spec.ImagePullPolicy, args, env, livenessProbe, readinessProbe); err != nil { return maskAny(err) } } else if group.IsArangosync() { @@ -315,7 +314,11 @@ func (d *Deployment) ensurePods(apiObject *api.ArangoDeployment) error { if err != nil { return maskAny(err) } - if err := k8sutil.CreateArangoSyncPod(kubecli, apiObject, role, m.ID, apiObject.Spec.Sync.Image, apiObject.Spec.Sync.ImagePullPolicy, args, env, livenessProbe, owner); err != nil { + affinityWithRole := "" + if group == api.ServerGroupSyncWorkers { + affinityWithRole = api.ServerGroupDBServers.AsRole() + } + if err := k8sutil.CreateArangoSyncPod(kubecli, apiObject.Spec.IsDevelopment(), apiObject, role, m.ID, apiObject.Spec.Sync.Image, apiObject.Spec.Sync.ImagePullPolicy, args, env, livenessProbe, affinityWithRole); err != nil { return maskAny(err) } } diff --git a/pkg/util/k8sutil/affinity.go b/pkg/util/k8sutil/affinity.go new file mode 100644 index 000000000..4be4f68b5 --- /dev/null +++ b/pkg/util/k8sutil/affinity.go @@ -0,0 +1,76 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package k8sutil + +import ( + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// createAffinity creates pod anti-affinity for the given role. +// role contains the name of the role to configure any-affinity with. +// affinityWithRole contains the role to configure affinity with. +func createAffinity(deploymentName, role string, required bool, affinityWithRole string) *v1.Affinity { + a := &v1.Affinity{ + PodAntiAffinity: &v1.PodAntiAffinity{}, + } + labels := LabelsForDeployment(deploymentName, role) + labelSelector := &metav1.LabelSelector{ + MatchLabels: labels, + } + if required { + a.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution = append(a.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution, v1.PodAffinityTerm{ + LabelSelector: labelSelector, + TopologyKey: TopologyKeyHostname, + }) + } else { + a.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution = append(a.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution, v1.WeightedPodAffinityTerm{ + Weight: 1, + PodAffinityTerm: v1.PodAffinityTerm{ + LabelSelector: labelSelector, + TopologyKey: TopologyKeyHostname, + }, + }) + } + if affinityWithRole != "" { + a.PodAffinity = &v1.PodAffinity{} + labelSelector := &metav1.LabelSelector{ + MatchLabels: LabelsForDeployment(deploymentName, affinityWithRole), + } + if required { + a.PodAffinity.RequiredDuringSchedulingIgnoredDuringExecution = append(a.PodAffinity.RequiredDuringSchedulingIgnoredDuringExecution, v1.PodAffinityTerm{ + LabelSelector: labelSelector, + TopologyKey: TopologyKeyHostname, + }) + } else { + a.PodAffinity.PreferredDuringSchedulingIgnoredDuringExecution = append(a.PodAffinity.PreferredDuringSchedulingIgnoredDuringExecution, v1.WeightedPodAffinityTerm{ + Weight: 1, + PodAffinityTerm: v1.PodAffinityTerm{ + LabelSelector: labelSelector, + TopologyKey: TopologyKeyHostname, + }, + }) + } + } + return a +} diff --git a/pkg/util/k8sutil/constants.go b/pkg/util/k8sutil/constants.go index c65bc748e..28fc22cdf 100644 --- a/pkg/util/k8sutil/constants.go +++ b/pkg/util/k8sutil/constants.go @@ -29,4 +29,5 @@ const ( // K8s constants ClusterIPNone = "None" TolerateUnreadyEndpointsAnnotation = "service.alpha.kubernetes.io/tolerate-unready-endpoints" + TopologyKeyHostname = "kubernetes.io/hostname" ) diff --git a/pkg/util/k8sutil/pods.go b/pkg/util/k8sutil/pods.go index 2985ea4d1..b66457059 100644 --- a/pkg/util/k8sutil/pods.go +++ b/pkg/util/k8sutil/pods.go @@ -125,8 +125,8 @@ func newPod(deploymentName, ns, role, id string) v1.Pod { // CreateArangodPod creates a Pod that runs `arangod`. // If the pod already exists, nil is returned. // If another error occurs, that error is returned. -func CreateArangodPod(kubecli kubernetes.Interface, deployment metav1.Object, role, id, pvcName, image string, imagePullPolicy v1.PullPolicy, - args []string, env map[string]string, livenessProbe *HTTPProbeConfig, readinessProbe *HTTPProbeConfig, owner metav1.OwnerReference) error { +func CreateArangodPod(kubecli kubernetes.Interface, developmentMode bool, deployment APIObject, role, id, pvcName, image string, imagePullPolicy v1.PullPolicy, + args []string, env map[string]string, livenessProbe *HTTPProbeConfig, readinessProbe *HTTPProbeConfig) error { // Prepare basic pod p := newPod(deployment.GetName(), deployment.GetNamespace(), role, id) @@ -157,7 +157,10 @@ func CreateArangodPod(kubecli kubernetes.Interface, deployment metav1.Object, ro p.Spec.Volumes = append(p.Spec.Volumes, vol) } - if err := createPod(kubecli, &p, deployment.GetNamespace(), owner); err != nil { + // Add (anti-)affinity + p.Spec.Affinity = createAffinity(deployment.GetName(), role, !developmentMode, "") + + if err := createPod(kubecli, &p, deployment.GetNamespace(), deployment.AsOwner()); err != nil { return maskAny(err) } return nil @@ -166,8 +169,8 @@ func CreateArangodPod(kubecli kubernetes.Interface, deployment metav1.Object, ro // CreateArangoSyncPod creates a Pod that runs `arangosync`. // If the pod already exists, nil is returned. // If another error occurs, that error is returned. -func CreateArangoSyncPod(kubecli kubernetes.Interface, deployment metav1.Object, role, id, image string, imagePullPolicy v1.PullPolicy, - args []string, env map[string]string, livenessProbe *HTTPProbeConfig, owner metav1.OwnerReference) error { +func CreateArangoSyncPod(kubecli kubernetes.Interface, developmentMode bool, deployment APIObject, role, id, image string, imagePullPolicy v1.PullPolicy, + args []string, env map[string]string, livenessProbe *HTTPProbeConfig, affinityWithRole string) error { // Prepare basic pod p := newPod(deployment.GetName(), deployment.GetNamespace(), role, id) @@ -175,7 +178,10 @@ func CreateArangoSyncPod(kubecli kubernetes.Interface, deployment metav1.Object, c := arangosyncContainer(p.GetName(), image, imagePullPolicy, args, env, livenessProbe) p.Spec.Containers = append(p.Spec.Containers, c) - if err := createPod(kubecli, &p, deployment.GetNamespace(), owner); err != nil { + // Add (anti-)affinity + p.Spec.Affinity = createAffinity(deployment.GetName(), role, !developmentMode, affinityWithRole) + + if err := createPod(kubecli, &p, deployment.GetNamespace(), deployment.AsOwner()); err != nil { return maskAny(err) } return nil