Skip to content

Commit

Permalink
Extract code to ba shared between updater and admission controller
Browse files Browse the repository at this point in the history 
To be squashed
  • Loading branch information
@jbartosik
jbartosik committed May 29, 2019
1 parent 79b031d commit b7be892
Show file tree
Hide file tree
Showing 10 changed files with 301 additions and 146 deletions.
4 changes: 2 additions & 2 deletions vertical-pod-autoscaler/pkg/admission-controller/logic/limit_range_calculator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,8 @@ import (
listers "k8s.io/client-go/listers/core/v1"
)

// LimitsRangeCalculator calculates limit range items that has the same effect as all limit range items present in the cluster.
type LimitsRangeCalculator interface {
// LimitRangeCalculator calculates limit range items that has the same effect as all limit range items present in the cluster.
type LimitRangeCalculator interface {
// GetContainerLimitRangeItem returns LimitRangeItem that describes limitation on container limits in the given namespace.
GetContainerLimitRangeItem(namespace string) (*v1.LimitRangeItem, error)
}
Expand Down
131 changes: 20 additions & 111 deletions vertical-pod-autoscaler/pkg/admission-controller/logic/recommendation_provider.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,6 @@ import (
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
"k8s.io/apimachinery/pkg/labels"
"math"
"math/big"

vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1beta2"
vpa_lister "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/listers/autoscaling.k8s.io/v1beta2"
Expand All @@ -31,33 +29,20 @@ import (
"k8s.io/klog"
)

// ContainerResources holds resources request for container
type ContainerResources struct {
Limits v1.ResourceList
Requests v1.ResourceList
}

func newContainerResources() ContainerResources {
return ContainerResources{
Requests: v1.ResourceList{},
Limits: v1.ResourceList{},
}
}

// RecommendationProvider gets current recommendation, annotations and vpaName for the given pod.
type RecommendationProvider interface {
GetContainersResourcesForPod(pod *v1.Pod) ([]ContainerResources, vpa_api_util.ContainerToAnnotationsMap, string, error)
GetContainersResourcesForPod(pod *v1.Pod) ([]vpa_api_util.ContainerResources, vpa_api_util.ContainerToAnnotationsMap, string, error)
}

type recommendationProvider struct {
limitsRangeCalculator LimitsRangeCalculator
limitsRangeCalculator LimitRangeCalculator
recommendationProcessor vpa_api_util.RecommendationProcessor
selectorFetcher target.VpaTargetSelectorFetcher
vpaLister vpa_lister.VerticalPodAutoscalerLister
}

// NewRecommendationProvider constructs the recommendation provider that list VPAs and can be used to determine recommendations for pods.
func NewRecommendationProvider(calculator LimitsRangeCalculator, recommendationProcessor vpa_api_util.RecommendationProcessor,
func NewRecommendationProvider(calculator LimitRangeCalculator, recommendationProcessor vpa_api_util.RecommendationProcessor,
selectorFetcher target.VpaTargetSelectorFetcher, vpaLister vpa_lister.VerticalPodAutoscalerLister) *recommendationProvider {
return &recommendationProvider{
limitsRangeCalculator: calculator,
Expand All @@ -67,108 +52,32 @@ func NewRecommendationProvider(calculator LimitsRangeCalculator, recommendationP
}
}

// scaleQuantityProportionally returns value which has the same proportion to scaledQuantity as scaleResult has to scaleBase
// It also returns a bool indicating if it had to cap result to MaxInt64 milliunits.
func scaleQuantityProportionally(scaledQuantity, scaleBase, scaleResult *resource.Quantity) (*resource.Quantity, bool) {
originalMilli := big.NewInt(scaledQuantity.MilliValue())
scaleBaseMilli := big.NewInt(scaleBase.MilliValue())
scaleResultMilli := big.NewInt(scaleResult.MilliValue())
var scaledOriginal big.Int
scaledOriginal.Mul(originalMilli, scaleResultMilli)
scaledOriginal.Div(&scaledOriginal, scaleBaseMilli)
if scaledOriginal.IsInt64() {
return resource.NewMilliQuantity(scaledOriginal.Int64(), scaledQuantity.Format), false
}
return resource.NewMilliQuantity(math.MaxInt64, scaledQuantity.Format), true
}

func getProportionalLimit(originalLimit, originalRequest, recommendedRequest, defaultLimit *resource.Quantity) (limit *resource.Quantity, capped bool) {
if originalLimit == nil || originalLimit.Value() == 0 && defaultLimit != nil {
originalLimit = defaultLimit
}
// originalLimit not set, don't set limit.
if originalLimit == nil || originalLimit.Value() == 0 {
return nil, false
}
// originalLimit set but originalRequest not set - K8s will treat the pod as if they were equal,
// recommend limit equal to request
if originalRequest == nil || originalRequest.Value() == 0 {
result := *recommendedRequest
return &result, false
}
// originalLimit and originalRequest are set. If they are equal recommend limit equal to request.
if originalRequest.MilliValue() == originalLimit.MilliValue() {
result := *recommendedRequest
return &result, false
}

// Input and output milli values should fit in int64 but intermediate values might be bigger.
return scaleQuantityProportionally( /*scaledQuantity=*/ originalLimit /*scaleBase=*/, originalRequest /*scaleResult=*/, recommendedRequest)
}

func proportionallyCapLimitToMax(recommendedRequest, recommendedLimit, maxLimit *resource.Quantity) (request, limit *resource.Quantity) {
if recommendedLimit == nil || maxLimit == nil || maxLimit.IsZero() {
return recommendedRequest, recommendedLimit
}
if recommendedLimit.Cmp(*maxLimit) <= 0 {
return recommendedRequest, recommendedLimit
}
scaledRequest, _ := scaleQuantityProportionally(recommendedRequest, recommendedLimit, maxLimit)
return scaledRequest, maxLimit
}

func proportionallyCapResourcesToMaxLimit(recommendedRequests v1.ResourceList, cpuLimit, memLimit, maxCpuLimit, maxMemLimit *resource.Quantity) ContainerResources {
scaledCpuRequest, scaledCpuLimit := proportionallyCapLimitToMax(recommendedRequests.Cpu(), cpuLimit, maxCpuLimit)
scaledMemRequest, scaledMemLimit := proportionallyCapLimitToMax(recommendedRequests.Memory(), memLimit, maxMemLimit)
result := newContainerResources()

result.Requests[v1.ResourceCPU] = *scaledCpuRequest
result.Requests[v1.ResourceMemory] = *scaledMemRequest
if scaledCpuLimit != nil {
result.Limits[v1.ResourceCPU] = *scaledCpuLimit
}
if scaledMemLimit != nil {
result.Limits[v1.ResourceMemory] = *scaledMemLimit
}
return result
}

// GetContainersResources returns the recommended resources for each container in the given pod in the same order they are specified in the pod.Spec.
func GetContainersResources(pod *v1.Pod, podRecommendation vpa_types.RecommendedPodResources, limitRange *v1.LimitRangeItem,
annotations vpa_api_util.ContainerToAnnotationsMap) []ContainerResources {
resources := make([]ContainerResources, len(pod.Spec.Containers))
annotations vpa_api_util.ContainerToAnnotationsMap) []vpa_api_util.ContainerResources {
resources := make([]vpa_api_util.ContainerResources, len(pod.Spec.Containers))
var defaultCpu, defaultMem, maxCpuLimit, maxMemLimit *resource.Quantity
if limitRange != nil {
defaultCpu = limitRange.Default.Cpu()
defaultMem = limitRange.Default.Memory()
maxCpuLimit = limitRange.Max.Cpu()
maxMemLimit = limitRange.Max.Memory()
}
for i, container := range pod.Spec.Containers {

recommendation := vpa_api_util.GetRecommendationForContainer(container.Name, &podRecommendation)
if recommendation == nil {
klog.V(2).Infof("no matching recommendation found for container %s", container.Name)
continue
}

var defaultCpu, defaultMem, maxCpuLimit, maxMemLimit *resource.Quantity
if limitRange != nil {
defaultCpu = limitRange.Default.Cpu()
defaultMem = limitRange.Default.Memory()
maxCpuLimit = limitRange.Max.Cpu()
maxMemLimit = limitRange.Max.Memory()
}
cpuLimit, capped := getProportionalLimit(container.Resources.Limits.Cpu(), container.Resources.Requests.Cpu(), recommendation.Target.Cpu(), defaultCpu)
if capped {
annotations[container.Name] = append(
annotations[container.Name],
fmt.Sprintf(
"Failed to keep CPU limit to request proportion of %d to %d with recommended request of %d milliCPU; doesn't fit in int64. Capping limit to MaxInt64",
container.Resources.Limits.Cpu().MilliValue(), container.Resources.Requests.Cpu().MilliValue(), recommendation.Target.Cpu().MilliValue()))
cpuLimit, annotation := vpa_api_util.GetProportionalLimit(container.Resources.Limits.Cpu(), container.Resources.Requests.Cpu(), recommendation.Target.Cpu(), defaultCpu)
if annotation != "" {
annotations[container.Name] = append(annotations[container.Name], fmt.Sprintf("CPU: %s", annotation))
}
memLimit, capped := getProportionalLimit(container.Resources.Limits.Memory(), container.Resources.Requests.Memory(), recommendation.Target.Memory(), defaultMem)
if capped {
annotations[container.Name] = append(
annotations[container.Name],
fmt.Sprintf(
"Failed to keep memory limit to request proportion of %d to %d with recommended request of %d milliBytes; doesn't fit in int64. Capping limit to MaxInt64",
container.Resources.Limits.Memory().MilliValue(), container.Resources.Requests.Memory().MilliValue(), recommendation.Target.Memory().MilliValue()))
memLimit, annotation := vpa_api_util.GetProportionalLimit(container.Resources.Limits.Memory(), container.Resources.Requests.Memory(), recommendation.Target.Memory(), defaultMem)
if annotation != "" {
annotations[container.Name] = append(annotations[container.Name], fmt.Sprintf("memory: %s", annotation))
}
resources[i] = proportionallyCapResourcesToMaxLimit(recommendation.Target, cpuLimit, memLimit, maxCpuLimit, maxMemLimit)
resources[i] = vpa_api_util.ProportionallyCapResourcesToMaxLimit(recommendation.Target, cpuLimit, memLimit, maxCpuLimit, maxMemLimit)
}
return resources
}
Expand Down Expand Up @@ -204,7 +113,7 @@ func (p *recommendationProvider) getMatchingVPA(pod *v1.Pod) *vpa_types.Vertical

// GetContainersResourcesForPod returns recommended request for a given pod, annotations and name of controlling VPA.
// The returned slice corresponds 1-1 to containers in the Pod.
func (p *recommendationProvider) GetContainersResourcesForPod(pod *v1.Pod) ([]ContainerResources, vpa_api_util.ContainerToAnnotationsMap, string, error) {
func (p *recommendationProvider) GetContainersResourcesForPod(pod *v1.Pod) ([]vpa_api_util.ContainerResources, vpa_api_util.ContainerToAnnotationsMap, string, error) {
klog.V(2).Infof("updating requirements for pod %s.", pod.Name)
vpaConfig := p.getMatchingVPA(pod)
if vpaConfig == nil {
Expand Down
4 changes: 2 additions & 2 deletions vertical-pod-autoscaler/pkg/admission-controller/logic/recommendation_provider_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -261,8 +261,8 @@ func TestUpdateResourceRequests(t *testing.T) {
labelSelector: "app = testingApp",
annotations: vpa_api_util.ContainerToAnnotationsMap{
containerName: []string{
"Failed to keep CPU limit to request proportion of 10000 to 1000 with recommended request of -9223372036854775808 milliCPU; doesn't fit in int64. Capping limit to MaxInt64",
"Failed to keep memory limit to request proportion of 1048576000000 to 104857600000 with recommended request of -9223372036854775808 milliBytes; doesn't fit in int64. Capping limit to MaxInt64",
"CPU: failed to keep limit to request proportion of 10 to 1 with recommended request of 1Ei; doesn't fit in int64. Capping limit to MaxInt64 milliunits",
"memory: failed to keep limit to request proportion of 1000Mi to 100Mi with recommended request of 1Ei; doesn't fit in int64. Capping limit to MaxInt64 milliunits",
},
},
},
Expand Down
6 changes: 3 additions & 3 deletions vertical-pod-autoscaler/pkg/admission-controller/logic/server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,11 +39,11 @@ type AdmissionServer struct {
recommendationProvider RecommendationProvider
podPreProcessor PodPreProcessor
vpaPreProcessor VpaPreProcessor
limitsChecker LimitsRangeCalculator
limitsChecker LimitRangeCalculator
}

// NewAdmissionServer constructs new AdmissionServer
func NewAdmissionServer(recommendationProvider RecommendationProvider, podPreProcessor PodPreProcessor, vpaPreProcessor VpaPreProcessor, limitsChecker LimitsRangeCalculator) *AdmissionServer {
func NewAdmissionServer(recommendationProvider RecommendationProvider, podPreProcessor PodPreProcessor, vpaPreProcessor VpaPreProcessor, limitsChecker LimitRangeCalculator) *AdmissionServer {
return &AdmissionServer{recommendationProvider, podPreProcessor, vpaPreProcessor, limitsChecker}
}

Expand Down Expand Up @@ -122,7 +122,7 @@ func getAddResourceRequirementValuePatch(i int, kind string, resource v1.Resourc
Value: quantity.String()}
}

func (s *AdmissionServer) getContainerPatch(pod v1.Pod, i int, annotationsPerContainer vpa_api_util.ContainerToAnnotationsMap, containerResources ContainerResources) ([]patchRecord, string) {
func (s *AdmissionServer) getContainerPatch(pod v1.Pod, i int, annotationsPerContainer vpa_api_util.ContainerToAnnotationsMap, containerResources vpa_api_util.ContainerResources) ([]patchRecord, string) {
var patches []patchRecord
// Add empty resources object if missing
if pod.Spec.Containers[i].Resources.Limits == nil &&
Expand Down
22 changes: 11 additions & 11 deletions vertical-pod-autoscaler/pkg/admission-controller/logic/server_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,13 +53,13 @@ func (fvp *fakeVpaPreProcessor) Process(vpa *vpa_types.VerticalPodAutoscaler, is
}

type fakeRecommendationProvider struct {
resources []ContainerResources
resources []vpa_api_util.ContainerResources
containerToAnnotations vpa_api_util.ContainerToAnnotationsMap
name string
e error
}

func (frp *fakeRecommendationProvider) GetContainersResourcesForPod(pod *apiv1.Pod) ([]ContainerResources, vpa_api_util.ContainerToAnnotationsMap, string, error) {
func (frp *fakeRecommendationProvider) GetContainersResourcesForPod(pod *apiv1.Pod) ([]vpa_api_util.ContainerResources, vpa_api_util.ContainerToAnnotationsMap, string, error) {
return frp.resources, frp.containerToAnnotations, frp.name, frp.e
}

Expand Down Expand Up @@ -135,7 +135,7 @@ func TestGetPatchesForResourceRequest(t *testing.T) {
podJson []byte
namespace string
podPreProcessorError error
recommendResources []ContainerResources
recommendResources []vpa_api_util.ContainerResources
recommendAnnotations vpa_api_util.ContainerToAnnotationsMap
recommendName string
recommendError error
Expand All @@ -147,7 +147,7 @@ func TestGetPatchesForResourceRequest(t *testing.T) {
podJson: []byte("{"),
namespace: "default",
podPreProcessorError: nil,
recommendResources: []ContainerResources{},
recommendResources: []vpa_api_util.ContainerResources{},
recommendAnnotations: vpa_api_util.ContainerToAnnotationsMap{},
recommendName: "name",
expectError: fmt.Errorf("unexpected end of JSON input"),
Expand All @@ -157,7 +157,7 @@ func TestGetPatchesForResourceRequest(t *testing.T) {
podJson: []byte("{}"),
namespace: "default",
podPreProcessorError: fmt.Errorf("bad pod"),
recommendResources: []ContainerResources{},
recommendResources: []vpa_api_util.ContainerResources{},
recommendAnnotations: vpa_api_util.ContainerToAnnotationsMap{},
recommendName: "name",
expectError: fmt.Errorf("bad pod"),
Expand All @@ -171,7 +171,7 @@ func TestGetPatchesForResourceRequest(t *testing.T) {
}
}`),
namespace: "default",
recommendResources: []ContainerResources{
recommendResources: []vpa_api_util.ContainerResources{
{
Requests: apiv1.ResourceList{
cpu: resource.MustParse("1"),
Expand Down Expand Up @@ -204,7 +204,7 @@ func TestGetPatchesForResourceRequest(t *testing.T) {
}
}`),
namespace: "default",
recommendResources: []ContainerResources{
recommendResources: []vpa_api_util.ContainerResources{
{
Requests: apiv1.ResourceList{
cpu: resource.MustParse("1"),
Expand Down Expand Up @@ -236,7 +236,7 @@ func TestGetPatchesForResourceRequest(t *testing.T) {
}
}`),
namespace: "default",
recommendResources: []ContainerResources{
recommendResources: []vpa_api_util.ContainerResources{
{
Requests: apiv1.ResourceList{
cpu: resource.MustParse("1"),
Expand Down Expand Up @@ -267,7 +267,7 @@ func TestGetPatchesForResourceRequest(t *testing.T) {
}
}`),
namespace: "default",
recommendResources: []ContainerResources{
recommendResources: []vpa_api_util.ContainerResources{
{
Limits: apiv1.ResourceList{
cpu: resource.MustParse("1"),
Expand Down Expand Up @@ -300,7 +300,7 @@ func TestGetPatchesForResourceRequest(t *testing.T) {
}
}`),
namespace: "default",
recommendResources: []ContainerResources{
recommendResources: []vpa_api_util.ContainerResources{
{
Limits: apiv1.ResourceList{
cpu: resource.MustParse("1"),
Expand Down Expand Up @@ -344,7 +344,7 @@ func TestGetPatchesForResourceRequest(t *testing.T) {
func TestGetPatchesForResourceRequest_TwoReplacementResources(t *testing.T) {
fppp := fakePodPreProcessor{}
fvpp := fakeVpaPreProcessor{}
recommendResources := []ContainerResources{
recommendResources := []vpa_api_util.ContainerResources{
{
Requests: apiv1.ResourceList{
cpu: resource.MustParse("1"),
Expand Down
25 changes: 10 additions & 15 deletions vertical-pod-autoscaler/pkg/admission-controller/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,13 +48,12 @@ var (
tlsPrivateKey: flag.String("tls-private-key", "/etc/tls-certs/serverKey.pem", "Path to server certificate key PEM file."),
}

port = flag.Int("port", 8000, "The port to listen on.")
address = flag.String("address", ":8944", "The address to expose Prometheus metrics.")
namespace = os.Getenv("NAMESPACE")
webhookAddress = flag.String("webhook-address", "", "Address under which webhook is registered. Used when registerByURL is set to true.")
webhookPort = flag.String("webhook-port", "", "Server Port for Webhook")
registerByURL = flag.Bool("register-by-url", false, "If set to true, admission webhook will be registered by URL (webhookAddress:webhookPort) instead of by service name")
allowToAdjustLimits = flag.Bool("allow-to-adjust-limits", false, "If set to true, admission webhook will set limits per container too if needed")
port = flag.Int("port", 8000, "The port to listen on.")
address = flag.String("address", ":8944", "The address to expose Prometheus metrics.")
namespace = os.Getenv("NAMESPACE")
webhookAddress = flag.String("webhook-address", "", "Address under which webhook is registered. Used when registerByURL is set to true.")
webhookPort = flag.String("webhook-port", "", "Server Port for Webhook")
registerByURL = flag.Bool("register-by-url", false, "If set to true, admission webhook will be registered by URL (webhookAddress:webhookPort) instead of by service name")
)

func main() {
Expand Down Expand Up @@ -83,14 +82,10 @@ func main() {
)
podPreprocessor := logic.NewDefaultPodPreProcessor()
vpaPreprocessor := logic.NewDefaultVpaPreProcessor()
var limitsChecker logic.LimitsRangeCalculator
if *allowToAdjustLimits {
limitsChecker, err = logic.NewLimitsRangeCalculator(factory)
if err != nil {
klog.Errorf("Failed to create limitsChecker, falling back to not checking limits. Error message: %s", err)
limitsChecker = logic.NewNoopLimitsCalculator()
}
} else {
var limitsChecker logic.LimitRangeCalculator
limitsChecker, err = logic.NewLimitsRangeCalculator(factory)
if err != nil {
klog.Errorf("Failed to create limitsChecker, falling back to not checking limits. Error message: %s", err)
limitsChecker = logic.NewNoopLimitsCalculator()
}
recommendationProvider := logic.NewRecommendationProvider(limitsChecker, vpa_api_util.NewCappingRecommendationProcessor(), targetSelectorFetcher, vpaLister)
Expand Down
Loading

0 comments on commit b7be892

Please sign in to comment.