feat(memory_limiter): switch to percentage based limits

Signed-off-by: Szilard Parrag <[email protected]>

api/telemetry/v1alpha1/collector_types.go

@@ -15,18 +15,67 @@
 package v1alpha1
 
 import (
-	corev1 "k8s.io/api/core/v1"
+	"time"
+
+	"github.com/cisco-open/operator-tools/pkg/typeoverride"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
+type MemoryLimiter struct {
+	// From 	go.opentelemetry.io/collector/processor/memorylimiterprocessor
+
+	// CheckInterval is the time between measurements of memory usage for the
+	// purposes of avoiding going over the limits. Defaults to zero, so no
+	// checks will be performed.
+	CheckInterval time.Duration `json:"check_interval"`
+
+	// MemoryLimitMiB is the maximum amount of memory, in MiB, targeted to be
+	// allocated by the process.
+	MemoryLimitMiB uint32 `json:"limit_mib"`
+
+	// MemorySpikeLimitMiB is the maximum, in MiB, spike expected between the
+	// measurements of memory usage.
+	MemorySpikeLimitMiB uint32 `json:"spike_limit_mib"`
+
+	// MemoryLimitPercentage is the maximum amount of memory, in %, targeted to be
+	// allocated by the process. The fixed memory settings MemoryLimitMiB has a higher precedence.
+	MemoryLimitPercentage uint32 `json:"limit_percentage"`
+
+	// MemorySpikePercentage is the maximum, in percents against the total memory,
+	// spike expected between the measurements of memory usage.
+	MemorySpikePercentage uint32 `json:"spike_limit_percentage"`
+}
+
 // CollectorSpec defines the desired state of Collector
 type CollectorSpec struct {
 	TenantSelector metav1.LabelSelector `json:"tenantSelector,omitempty"`
 	// Namespace where OTel collector DaemonSet is deployed
 	ControlNamespace string `json:"controlNamespace"`
 	// Enables debug logging for the collector
-	Debug     bool                        `json:"debug,omitempty"`
-	Resources corev1.ResourceRequirements `json:"resources,omitempty"`
+	Debug bool `json:"debug,omitempty"`
+	// Setting memory limits for the Collector
+	MemoryLimiter      *MemoryLimiter          `json:"memoryLimiter,omitempty"`
+	DaemonSetOverrides *typeoverride.DaemonSet `json:"daemonSet,omitempty"`
+}
+
+func (c *CollectorSpec) SetDefaults() {
+	if c.MemoryLimiter == nil {
+		c.MemoryLimiter = &MemoryLimiter{
+			CheckInterval:         1 * time.Second,
+			MemoryLimitPercentage: 75,
+			MemorySpikeLimitMiB:   25,
+		}
+	}
+}
+
+func (c CollectorSpec) GetMemoryLimit() *resource.Quantity {
+	if c.DaemonSetOverrides != nil && len(c.DaemonSetOverrides.Spec.Template.Spec.Containers) > 0 {
+		if memoryLimit := c.DaemonSetOverrides.Spec.Template.Spec.Containers[0].Resources.Limits.Memory(); !memoryLimit.IsZero() {
+			return memoryLimit
+		}
+	}
+	return nil
 }
 
 // CollectorStatus defines the observed state of Collector

go.mod

@@ -20,115 +20,22 @@ require (
 require go.opentelemetry.io/collector/config/configopaque v1.4.0
 
 require (
-	cloud.google.com/go/compute v1.23.4 // indirect
-	cloud.google.com/go/compute/metadata v0.2.3 // indirect
-	dario.cat/mergo v1.0.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.10.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.5.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4 v4.3.0 // indirect
-	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 // indirect
-	github.com/BurntSushi/toml v1.3.2 // indirect
-	github.com/Code-Hex/go-generics-cache v1.3.1 // indirect
-	github.com/Masterminds/semver/v3 v3.2.1 // indirect
-	github.com/Microsoft/go-winio v0.6.1 // indirect
-	github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 // indirect
-	github.com/armon/go-metrics v0.4.1 // indirect
-	github.com/aws/aws-sdk-go v1.50.32 // indirect
 	github.com/briandowns/spinner v1.23.0 // indirect
 	github.com/cisco-open/k8s-objectmatcher v1.9.0 // indirect
-	github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa // indirect
 	github.com/cppforlife/go-patch v0.2.0 // indirect
-	github.com/digitalocean/godo v1.109.0 // indirect
-	github.com/distribution/reference v0.5.0 // indirect
-	github.com/docker/docker v25.0.5+incompatible // indirect
-	github.com/docker/go-connections v0.4.0 // indirect
-	github.com/docker/go-units v0.5.0 // indirect
-	github.com/envoyproxy/go-control-plane v0.12.0 // indirect
-	github.com/envoyproxy/protoc-gen-validate v1.0.4 // indirect
 	github.com/evanphx/json-patch v5.9.0+incompatible // indirect
 	github.com/fatih/color v1.16.0 // indirect
-	github.com/felixge/httpsnoop v1.0.4 // indirect
-	github.com/go-kit/log v0.2.1 // indirect
-	github.com/go-logfmt/logfmt v0.6.0 // indirect
-	github.com/go-logr/stdr v1.2.2 // indirect
-	github.com/go-resty/resty/v2 v2.11.0 // indirect
-	github.com/go-zookeeper/zk v1.0.3 // indirect
-	github.com/golang-jwt/jwt/v5 v5.2.0 // indirect
-	github.com/google/go-querystring v1.1.0 // indirect
-	github.com/google/s2a-go v0.1.7 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
-	github.com/googleapis/gax-go/v2 v2.12.2 // indirect
-	github.com/gophercloud/gophercloud v1.8.0 // indirect
-	github.com/gorilla/websocket v1.5.1 // indirect
-	github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd // indirect
-	github.com/hashicorp/consul/api v1.28.2 // indirect
-	github.com/hashicorp/cronexpr v1.1.2 // indirect
-	github.com/hashicorp/errwrap v1.1.0 // indirect
-	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
-	github.com/hashicorp/go-hclog v1.5.0 // indirect
-	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
-	github.com/hashicorp/go-multierror v1.1.1 // indirect
-	github.com/hashicorp/go-retryablehttp v0.7.4 // indirect
-	github.com/hashicorp/go-rootcerts v1.0.2 // indirect
 	github.com/hashicorp/go-version v1.6.0 // indirect
-	github.com/hashicorp/golang-lru v0.6.0 // indirect
-	github.com/hashicorp/nomad/api v0.0.0-20240306004928-3e7191ccb702 // indirect
-	github.com/hashicorp/serf v0.10.1 // indirect
-	github.com/hetznercloud/hcloud-go/v2 v2.6.0 // indirect
 	github.com/iancoleman/orderedmap v0.3.0 // indirect
-	github.com/inconshreveable/mousetrap v1.1.0 // indirect
-	github.com/ionos-cloud/sdk-go/v6 v6.1.11 // indirect
-	github.com/jmespath/go-jmespath v0.4.0 // indirect
-	github.com/jpillora/backoff v1.0.0 // indirect
-	github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b // indirect
-	github.com/kylelemons/godebug v1.1.0 // indirect
-	github.com/linode/linodego v1.29.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
-	github.com/miekg/dns v1.1.58 // indirect
-	github.com/mitchellh/go-homedir v1.1.0 // indirect
-	github.com/mitchellh/mapstructure v1.5.0 // indirect
-	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
+	github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c // indirect
 	github.com/nxadm/tail v1.4.8 // indirect
-	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/opencontainers/image-spec v1.1.0-rc5 // indirect
-	github.com/openshift/api v0.0.0-20240124164020-e2ce40831f2e // indirect
-	github.com/operator-framework/operator-lib v0.12.0 // indirect
-	github.com/ovh/go-ovh v1.4.3 // indirect
-	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
-	github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.72.0 // indirect
-	github.com/prometheus/common/sigv4 v0.1.0 // indirect
-	github.com/prometheus/prometheus v0.51.1 // indirect
-	github.com/scaleway/scaleway-sdk-go v1.0.0-beta.25 // indirect
+	github.com/rogpeppe/go-internal v1.12.0 // indirect
 	github.com/spf13/cast v1.6.0 // indirect
-	github.com/spf13/cobra v1.7.0 // indirect
-	github.com/vultr/govultr/v2 v2.17.2 // indirect
 	github.com/wayneashleyberry/terminal-dimensions v1.1.0 // indirect
-	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/collector/featuregate v1.3.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
-	go.opentelemetry.io/otel v1.24.0 // indirect
-	go.opentelemetry.io/otel/metric v1.24.0 // indirect
-	go.opentelemetry.io/otel/trace v1.24.0 // indirect
-	golang.org/x/crypto v0.22.0 // indirect
-	golang.org/x/exp/typeparams v0.0.0-20221212164502-fae10dda9338 // indirect
-	golang.org/x/mod v0.17.0 // indirect
-	golang.org/x/sync v0.7.0 // indirect
-	golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2 // indirect
-	golang.org/x/tools/gopls v0.15.1 // indirect
-	golang.org/x/vuln v1.0.1 // indirect
-	google.golang.org/api v0.168.0 // indirect
-	google.golang.org/genproto v0.0.0-20240205150955-31a09d347014 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20240304212257-790db918fca8 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20240304161311-37d4d3c04a78 // indirect
-	google.golang.org/grpc v1.62.1 // indirect
-	gopkg.in/ini.v1 v1.67.0 // indirect
-	honnef.co/go/tools v0.4.6 // indirect
-	mvdan.cc/gofumpt v0.6.0 // indirect
-	mvdan.cc/xurls/v2 v2.5.0 // indirect
 
 )