Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Expose Horizontal Pod Autoscaler Behavior and add hpa scaledown test #1077

Merged
merged 17 commits into from
Sep 15, 2022
Merged

Expose Horizontal Pod Autoscaler Behavior and add hpa scaledown test #1077

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
f4abf28
Add scaledown test for autoscaling
kevinearls Sep 6, 2022
7b51fee
Fix nits
kevinearls Sep 6, 2022
43710b9
Appease the linter
kevinearls Sep 6, 2022
7070f4b
Don't use a default, only set scaleUp/scaleDown if they are in the CR
kevinearls Sep 6, 2022
ae83264
Removed commented out code
kevinearls Sep 6, 2022
19154be
Change defaults for scaleUp/scaleDown
kevinearls Sep 6, 2022
108d0f1
Update autoscaling scaleup/scaledown
kevinearls Sep 8, 2022
037aa23
Run generate
kevinearls Sep 8, 2022
bdc6eb3
Ran make api-docs
kevinearls Sep 8, 2022
60cdfe7
Update HPA implementation to embed HorizontalPodAutoscalerBehavior
kevinearls Sep 15, 2022
54ff0d3
Merge branch 'main' into add-hpa-scaledown-test
kevinearls Sep 15, 2022
bcf71e7
Only set behavior if it exists in the collector CR
kevinearls Sep 15, 2022
0f8e029
Aded some unit tests
kevinearls Sep 15, 2022
a42084c
Merge branch 'main' into add-hpa-scaledown-test
kevinearls Sep 15, 2022
f3a682c
Merge branch 'main' into add-hpa-scaledown-test
kevinearls Sep 15, 2022
c14096d
Add kuttl assertion that hpa scaled down
kevinearls Sep 15, 2022
89a2f48
added whitespace to rerun tests
kevinearls Sep 15, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 16 additions & 6 deletions apis/v1alpha1/opentelemetrycollector_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -204,17 +204,27 @@ type OpenTelemetryCollectorList struct {

// AutoscalerSpec defines the OpenTelemetryCollector's pod autoscaling specification.
type AutoscalerSpec struct {
// ScaleUp is the minimum number of seconds to wait before scaling up
//
// +optional
// +kubebuilder:validation:Minimum=0
ScaleUp *int32 `json:"scaleUp,omitempty"`
Behavior *AutoscalerBehaviorSpec `json:"behavior,omitempty"`
}

// AutoscalerBehaviorSpec configures the scaling behavior of the target.
type AutoscalerBehaviorSpec struct {
// ScaleUp is the policy for scaling up
// +optional
ScaleUp *AutoScalerRules `json:"scaleUp,omitempty"`

// ScaleDown is the minimum number of seconds to wait before scaling down
// ScaleDown is policy for scaling down
//
// +optional
ScaleDown *AutoScalerRules `json:"scaleDown,omitempty"`
}

// AutoScalerRules configures the scaling behavior for one direction.
type AutoScalerRules struct {
// +optional
// +kubebuilder:validation:Minimum=0
ScaleDown *int32 `json:"scaleDown,omitempty"`
StabilizationWindowSeconds *int32 `json:"stabilizationWindowSeconds,omitempty"`
}

func init() {
Expand Down
6 changes: 3 additions & 3 deletions apis/v1alpha1/opentelemetrycollector_webhook.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,12 +129,12 @@ func (r *OpenTelemetryCollector) validateCRDSpec() error {
return fmt.Errorf("the OpenTelemetry Spec autoscale configuration is incorrect, minReplicas should be one or more")
}

if r.Spec.Autoscaler != nil {
if r.Spec.Autoscaler.ScaleDown != nil && *r.Spec.Autoscaler.ScaleDown < int32(1) {
if r.Spec.Autoscaler != nil && r.Spec.Autoscaler.Behavior != nil {
if r.Spec.Autoscaler.Behavior.ScaleDown != nil && *r.Spec.Autoscaler.Behavior.ScaleDown.StabilizationWindowSeconds < int32(1) {
return fmt.Errorf("the OpenTelemetry Spec autoscale configuration is incorrect, scaleDown should be one or more")
}

if r.Spec.Autoscaler.ScaleUp != nil && *r.Spec.Autoscaler.ScaleUp < int32(1) {
if r.Spec.Autoscaler.Behavior.ScaleUp != nil && *r.Spec.Autoscaler.Behavior.ScaleUp.StabilizationWindowSeconds < int32(1) {
return fmt.Errorf("the OpenTelemetry Spec autoscale configuration is incorrect, scaleUp should be one or more")
}
}
Expand Down
60 changes: 50 additions & 10 deletions apis/v1alpha1/zz_generated.deepcopy.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

33 changes: 21 additions & 12 deletions bundle/manifests/opentelemetry.io_opentelemetrycollectors.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,18 +63,27 @@ spec:
description: Autoscaler specifies the pod autoscaling configuration
to use for the OpenTelemetryCollector workload.
properties:
scaleDown:
description: ScaleDown is the minimum number of seconds to wait
before scaling down
format: int32
minimum: 0
type: integer
scaleUp:
description: ScaleUp is the minimum number of seconds to wait
before scaling up
format: int32
minimum: 0
type: integer
behavior:
description: AutoscalerBehaviorSpec configures the scaling behavior
of the target
properties:
scaleDown:
description: ScaleDown is policy for scaling down
properties:
stabilizationWindowSeconds:
format: int32
minimum: 0
type: integer
type: object
scaleUp:
description: ScaleUp is the policy for scaling up
properties:
stabilizationWindowSeconds:
format: int32
minimum: 0
type: integer
type: object
type: object
type: object
config:
description: Config is the raw JSON to be used as the collector's
Expand Down
33 changes: 21 additions & 12 deletions config/crd/bases/opentelemetry.io_opentelemetrycollectors.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,18 +61,27 @@ spec:
description: Autoscaler specifies the pod autoscaling configuration
to use for the OpenTelemetryCollector workload.
properties:
scaleDown:
description: ScaleDown is the minimum number of seconds to wait
before scaling down
format: int32
minimum: 0
type: integer
scaleUp:
description: ScaleUp is the minimum number of seconds to wait
before scaling up
format: int32
minimum: 0
type: integer
behavior:
description: AutoscalerBehaviorSpec configures the scaling behavior
of the target
properties:
scaleDown:
description: ScaleDown is policy for scaling down
properties:
stabilizationWindowSeconds:
format: int32
minimum: 0
type: integer
type: object
scaleUp:
description: ScaleUp is the policy for scaling up
properties:
stabilizationWindowSeconds:
format: int32
minimum: 0
type: integer
type: object
type: object
type: object
config:
description: Config is the raw JSON to be used as the collector's
Expand Down
91 changes: 86 additions & 5 deletions docs/api.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1890,20 +1890,101 @@ Autoscaler specifies the pod autoscaling configuration to use for the OpenTeleme
</tr>
</thead>
<tbody><tr>
<td><b>scaleDown</b></td>
<td><b><a href="#opentelemetrycollectorspecautoscalerbehavior">behavior</a></b></td>
<td>object</td>
<td>
AutoscalerBehaviorSpec configures the scaling behavior of the target<br/>
</td>
<td>false</td>
</tr></tbody>
</table>


### OpenTelemetryCollector.spec.autoscaler.behavior
<sup><sup>[↩ Parent](#opentelemetrycollectorspecautoscaler)</sup></sup>



AutoscalerBehaviorSpec configures the scaling behavior of the target

<table>
<thead>
<tr>
<th>Name</th>
<th>Type</th>
<th>Description</th>
<th>Required</th>
</tr>
</thead>
<tbody><tr>
<td><b><a href="#opentelemetrycollectorspecautoscalerbehaviorscaledown">scaleDown</a></b></td>
<td>object</td>
<td>
ScaleDown is policy for scaling down<br/>
</td>
<td>false</td>
</tr><tr>
<td><b><a href="#opentelemetrycollectorspecautoscalerbehaviorscaleup">scaleUp</a></b></td>
<td>object</td>
<td>
ScaleUp is the policy for scaling up<br/>
</td>
<td>false</td>
</tr></tbody>
</table>


### OpenTelemetryCollector.spec.autoscaler.behavior.scaleDown
<sup><sup>[↩ Parent](#opentelemetrycollectorspecautoscalerbehavior)</sup></sup>



ScaleDown is policy for scaling down

<table>
<thead>
<tr>
<th>Name</th>
<th>Type</th>
<th>Description</th>
<th>Required</th>
</tr>
</thead>
<tbody><tr>
<td><b>stabilizationWindowSeconds</b></td>
<td>integer</td>
<td>
ScaleDown is the minimum number of seconds to wait before scaling down<br/>
<br/>
<br/>
<i>Format</i>: int32<br/>
<i>Minimum</i>: 0<br/>
</td>
<td>false</td>
</tr><tr>
<td><b>scaleUp</b></td>
</tr></tbody>
</table>


### OpenTelemetryCollector.spec.autoscaler.behavior.scaleUp
<sup><sup>[↩ Parent](#opentelemetrycollectorspecautoscalerbehavior)</sup></sup>



ScaleUp is the policy for scaling up

<table>
<thead>
<tr>
<th>Name</th>
<th>Type</th>
<th>Description</th>
<th>Required</th>
</tr>
</thead>
<tbody><tr>
<td><b>stabilizationWindowSeconds</b></td>
<td>integer</td>
<td>
ScaleUp is the minimum number of seconds to wait before scaling up<br/>
<br/>
<br/>
<i>Format</i>: int32<br/>
<i>Minimum</i>: 0<br/>
Expand Down
17 changes: 10 additions & 7 deletions pkg/collector/horizontalpodautoscaler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,12 +50,15 @@ func HorizontalPodAutoscaler(cfg config.Config, logger logr.Logger, otelcol v1al
}

scaleUpTime := defaultScaleUpTime
if otelcol.Spec.Autoscaler != nil && otelcol.Spec.Autoscaler.ScaleDown != nil {
scaleUpTime = *otelcol.Spec.Autoscaler.ScaleUp
}
scaleDownTime := defaultScaleDownTime
if otelcol.Spec.Autoscaler != nil && otelcol.Spec.Autoscaler.ScaleDown != nil {
scaleDownTime = *otelcol.Spec.Autoscaler.ScaleDown
if otelcol.Spec.Autoscaler != nil && otelcol.Spec.Autoscaler.Behavior != nil {
if otelcol.Spec.Autoscaler.Behavior.ScaleUp != nil {
scaleUpTime = *otelcol.Spec.Autoscaler.Behavior.ScaleUp.StabilizationWindowSeconds
}

if otelcol.Spec.Autoscaler.Behavior.ScaleDown != nil {
scaleDownTime = *otelcol.Spec.Autoscaler.Behavior.ScaleDown.StabilizationWindowSeconds
}
}

if autoscalingVersion == autodetect.AutoscalingVersionV2Beta2 {
Expand Down Expand Up @@ -85,7 +88,7 @@ func HorizontalPodAutoscaler(cfg config.Config, logger logr.Logger, otelcol v1al
},
}

if otelcol.Spec.Autoscaler != nil {
if otelcol.Spec.Autoscaler != nil && otelcol.Spec.Autoscaler.Behavior != nil {
scaleUpRules := &autoscalingv2beta2.HPAScalingRules{
StabilizationWindowSeconds: &scaleUpTime,
}
Expand Down Expand Up @@ -126,7 +129,7 @@ func HorizontalPodAutoscaler(cfg config.Config, logger logr.Logger, otelcol v1al
},
}

if otelcol.Spec.Autoscaler != nil {
if otelcol.Spec.Autoscaler != nil && otelcol.Spec.Autoscaler.Behavior != nil {
scaleUpRules := &autoscalingv2.HPAScalingRules{
StabilizationWindowSeconds: &scaleUpTime,
}
Expand Down
3 changes: 3 additions & 0 deletions tests/e2e/autoscale/00-assert.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,7 @@ metadata:
spec:
minReplicas: 1
maxReplicas: 2
# This is not neccesarily exact. We really just want to wait until this is no longer <unknown>
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a really good "hack"

status:
currentCPUUtilizationPercentage: 20

7 changes: 5 additions & 2 deletions tests/e2e/autoscale/00-install.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,11 @@ spec:
minReplicas: 1
maxReplicas: 2
autoscaler:
scaleUp: 10
scaleDown: 15
behavior:
scaleUp:
stabilizationWindowSeconds: 10
scaleDown:
stabilizationWindowSeconds: 15
resources:
limits:
cpu: 500m
Expand Down