Issue 130: Ability to specify container resources (#129)

* Specify container resources for testing Signed-off-by: Adrián Moreno <[email protected]> * Set Xmx and Xms Java opts to SS and C Signed-off-by: Adrián Moreno <[email protected]> * Store BK index directory in a PVC Signed-off-by: Adrián Moreno <[email protected]> * enable custom resource configuration Signed-off-by: wenqi <[email protected]> * Generate deepcopy and go fmt Signed-off-by: wenqi <[email protected]> * Fix golang errors Signed-off-by: Adrián Moreno <[email protected]> * Adjust resource values Signed-off-by: Adrián Moreno <[email protected]> * Set Xmx value based on memory limit Signed-off-by: Adrián Moreno <[email protected]> * Reduce default resource consumption Signed-off-by: Adrián Moreno <[email protected]> * Further reduce resources Signed-off-by: Adrián Moreno <[email protected]> * Add unit test to check resource is working Signed-off-by: wenqi <[email protected]> * Refractor test Signed-off-by: wenqi <[email protected]> * Use literals instead of constants on unit test assertions Signed-off-by: Adrián Moreno <[email protected]> * Wait after cluster destoryed Signed-off-by: wenqi <[email protected]> * Go fmt Signed-off-by: wenqi <[email protected]> * Wait for PVCs to be deleted when destroying a cluster Signed-off-by: Adrián Moreno <[email protected]>
pravega · Mar 22, 2019 · 78d9944 · 78d9944
1 parent f58ca1e
commit 78d9944
Show file tree

Hide file tree

Showing 9 changed files with 432 additions and 55 deletions.
diff --git a/example/cr-detailed.yaml b/example/cr-detailed.yaml
@@ -12,6 +12,13 @@ spec:
       pullPolicy: IfNotPresent
 
     replicas: 3
+    resources:
+      requests:
+        memory: "3Gi"
+        cpu: "1000m"
+      limits:
+        memory: "5Gi"
+        cpu: "2000m"
 
     storage:
       ledgerVolumeClaimTemplate:
@@ -51,7 +58,22 @@ spec:
 
   pravega:
     controllerReplicas: 1
+    controllerResources:
+      requests:
+        memory: "1Gi"
+        cpu: "1000m"
+      limits:
+        memory: "3Gi"
+        cpu: "2000m"
+
     segmentStoreReplicas: 3
+    segmentStoreResources:
+      requests:
+        memory: "3Gi"
+        cpu: "1000m"
+      limits:
+        memory: "5Gi"
+        cpu: "2000m"
 
     # Turn on Pravega Debug Logging
     debugLogging: false

diff --git a/pkg/apis/pravega/v1alpha1/bookkeeper.go b/pkg/apis/pravega/v1alpha1/bookkeeper.go
@@ -46,6 +46,18 @@ const (
 	// MinimumBookkeeperReplicas is the minimum number of Bookkeeper replicas
 	// accepted
 	MinimumBookkeeperReplicas = 3
+
+	// DefaultBookkeeperRequestCPU is the default CPU request for BookKeeper
+	DefaultBookkeeperRequestCPU = "500m"
+
+	// DefaultBookkeeperLimitCPU is the default CPU limit for BookKeeper
+	DefaultBookkeeperLimitCPU = "1"
+
+	// DefaultBookkeeperRequestMemory is the default memory request for BookKeeper
+	DefaultBookkeeperRequestMemory = "1Gi"
+
+	// DefaultBookkeeperLimitMemory is the limit memory limit for BookKeeper
+	DefaultBookkeeperLimitMemory = "2Gi"
 )
 
 // BookkeeperSpec defines the configuration of BookKeeper
@@ -68,6 +80,10 @@ type BookkeeperSpec struct {
 	// ServiceAccountName configures the service account used on BookKeeper instances
 	ServiceAccountName string `json:"serviceAccountName,omitempty"`
 
+	// BookieResources specifies the request and limit of resources that bookie can have.
+	// BookieResources includes CPU and memory resources
+	Resources *v1.ResourceRequirements `json:"resources,omitempty"`
+
 	// Options is the Bookkeeper configuration that is to override the bk_server.conf
 	// in bookkeeper. Some examples can be found here
 	// https://github.com/apache/bookkeeper/blob/master/docker/README.md
@@ -102,9 +118,24 @@ func (s *BookkeeperSpec) withDefaults() (changed bool) {
 		s.AutoRecovery = &boolTrue
 	}
 
+	if s.Resources == nil {
+		changed = true
+		s.Resources = &v1.ResourceRequirements{
+			Requests: v1.ResourceList{
+				v1.ResourceCPU:    resource.MustParse(DefaultBookkeeperRequestCPU),
+				v1.ResourceMemory: resource.MustParse(DefaultBookkeeperRequestMemory),
+			},
+			Limits: v1.ResourceList{
+				v1.ResourceCPU:    resource.MustParse(DefaultBookkeeperLimitCPU),
+				v1.ResourceMemory: resource.MustParse(DefaultBookkeeperLimitMemory),
+			},
+		}
+	}
+
 	if s.Options == nil {
 		s.Options = map[string]string{}
 	}
+
 	return changed
 }
 

diff --git a/pkg/apis/pravega/v1alpha1/pravega.go b/pkg/apis/pravega/v1alpha1/pravega.go
@@ -45,6 +45,30 @@ const (
 	// DefaultSegmentStoreReplicas is the default number of replicas for the Pravega
 	// Segment Store component
 	DefaultSegmentStoreReplicas = 1
+
+	// DefaultControllerRequestCPU is the default CPU request for Pravega
+	DefaultControllerRequestCPU = "250m"
+
+	// DefaultControllerLimitCPU is the default CPU limit for Pravega
+	DefaultControllerLimitCPU = "500m"
+
+	// DefaultControllerRequestMemory is the default memory request for Pravega
+	DefaultControllerRequestMemory = "512Mi"
+
+	// DefaultControllerLimitMemory is the default memory limit for Pravega
+	DefaultControllerLimitMemory = "1Gi"
+
+	// DefaultSegmentStoreRequestCPU is the default CPU request for Pravega
+	DefaultSegmentStoreRequestCPU = "500m"
+
+	// DefaultSegmentStoreLimitCPU is the default CPU limit for Pravega
+	DefaultSegmentStoreLimitCPU = "1"
+
+	// DefaultSegmentStoreRequestMemory is the default memory request for Pravega
+	DefaultSegmentStoreRequestMemory = "1Gi"
+
+	// DefaultSegmentStoreLimitMemory is the default memory limit for Pravega
+	DefaultSegmentStoreLimitMemory = "2Gi"
 )
 
 // PravegaSpec defines the configuration of Pravega
@@ -87,6 +111,14 @@ type PravegaSpec struct {
 	// SegmentStoreServiceAccountName configures the service account used on segment store instances.
 	// If not specified, Kubernetes will automatically assign the default service account in the namespace
 	SegmentStoreServiceAccountName string `json:"segmentStoreServiceAccountName,omitempty"`
+
+	// ControllerResources specifies the request and limit of resources that controller can have.
+	// ControllerResources includes CPU and memory resources
+	ControllerResources *v1.ResourceRequirements `json:"controllerResources,omitempty"`
+
+	// SegmentStoreResources specifies the request and limit of resources that segmentStore can have.
+	// SegmentStoreResources includes CPU and memory resources
+	SegmentStoreResources *v1.ResourceRequirements `json:"segmentStoreResources,omitempty"`
 }
 
 func (s *PravegaSpec) withDefaults() (changed bool) {
@@ -129,10 +161,39 @@ func (s *PravegaSpec) withDefaults() (changed bool) {
 		changed = true
 		s.Tier2 = &Tier2Spec{}
 	}
+
 	if s.Tier2.withDefaults() {
 		changed = true
 	}
 
+	if s.ControllerResources == nil {
+		changed = true
+		s.ControllerResources = &v1.ResourceRequirements{
+			Requests: v1.ResourceList{
+				v1.ResourceCPU:    resource.MustParse(DefaultControllerRequestCPU),
+				v1.ResourceMemory: resource.MustParse(DefaultControllerRequestMemory),
+			},
+			Limits: v1.ResourceList{
+				v1.ResourceCPU:    resource.MustParse(DefaultControllerLimitCPU),
+				v1.ResourceMemory: resource.MustParse(DefaultControllerLimitMemory),
+			},
+		}
+	}
+
+	if s.SegmentStoreResources == nil {
+		changed = true
+		s.SegmentStoreResources = &v1.ResourceRequirements{
+			Requests: v1.ResourceList{
+				v1.ResourceCPU:    resource.MustParse(DefaultSegmentStoreRequestCPU),
+				v1.ResourceMemory: resource.MustParse(DefaultSegmentStoreRequestMemory),
+			},
+			Limits: v1.ResourceList{
+				v1.ResourceCPU:    resource.MustParse(DefaultSegmentStoreLimitCPU),
+				v1.ResourceMemory: resource.MustParse(DefaultSegmentStoreLimitMemory),
+			},
+		}
+	}
+
 	return changed
 }
 

diff --git a/pkg/apis/pravega/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/pravega/v1alpha1/zz_generated.deepcopy.go
diff --git a/pkg/controller/pravega/bookie.go b/pkg/controller/pravega/bookie.go
@@ -12,13 +12,13 @@ package pravega
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/pravega/pravega-operator/pkg/apis/pravega/v1alpha1"
 	"github.com/pravega/pravega-operator/pkg/util"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	policyv1beta1 "k8s.io/api/policy/v1beta1"
-	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 )
@@ -122,16 +122,7 @@ func makeBookiePodSpec(clusterName string, bookkeeperSpec *v1alpha1.BookkeeperSp
 						MountPath: "/bk/index",
 					},
 				},
-				Resources: corev1.ResourceRequirements{
-					Requests: corev1.ResourceList{
-						corev1.ResourceCPU:    resource.MustParse("1000m"),
-						corev1.ResourceMemory: resource.MustParse("3Gi"),
-					},
-					Limits: corev1.ResourceList{
-						corev1.ResourceCPU:    resource.MustParse("2000m"),
-						corev1.ResourceMemory: resource.MustParse("5Gi"),
-					},
-				},
+				Resources: *bookkeeperSpec.Resources,
 				ReadinessProbe: &corev1.Probe{
 					Handler: corev1.Handler{
 						Exec: &corev1.ExecAction{
@@ -193,11 +184,31 @@ func makeBookieVolumeClaimTemplates(spec *v1alpha1.BookkeeperSpec) []corev1.Pers
 }
 
 func MakeBookieConfigMap(pravegaCluster *v1alpha1.PravegaCluster) *corev1.ConfigMap {
+	javaOpts := []string{
+		"-Xms1g",
+		"-XX:+UnlockExperimentalVMOptions",
+		"-XX:+UseCGroupMemoryLimitForHeap",
+		"-XX:MaxRAMFraction=1",
+		"-XX:MaxDirectMemorySize=1g",
+		"-XX:+UseG1GC",
+		"-XX:MaxGCPauseMillis=10",
+		"-XX:+ParallelRefProcEnabled",
+		"-XX:+AggressiveOpts",
+		"-XX:+DoEscapeAnalysis",
+		"-XX:ParallelGCThreads=32",
+		"-XX:ConcGCThreads=32",
+		"-XX:G1NewSizePercent=50",
+		"-XX:+DisableExplicitGC",
+		"-XX:-ResizePLAB",
+	}
+
 	configData := map[string]string{
-		"BK_BOOKIE_EXTRA_OPTS": "-Xms1g -Xmx4g -XX:MaxDirectMemorySize=1g -XX:+UseG1GC  -XX:MaxGCPauseMillis=10 -XX:+ParallelRefProcEnabled -XX:+UnlockExperimentalVMOptions -XX:+AggressiveOpts -XX:+DoEscapeAnalysis -XX:ParallelGCThreads=32 -XX:ConcGCThreads=32 -XX:G1NewSizePercent=50 -XX:+DisableExplicitGC -XX:-ResizePLAB",
+		"BK_BOOKIE_EXTRA_OPTS": strings.Join(javaOpts, " "),
 		"ZK_URL":               pravegaCluster.Spec.ZookeeperUri,
 		// Set useHostNameAsBookieID to false until BookKeeper Docker
 		// image is updated to 4.7
+		// This value can be explicitly overridden when using the operator
+		// with images based on BookKeeper 4.7 or newer
 		"BK_useHostNameAsBookieID": "false",
 		"PRAVEGA_CLUSTER_NAME":     pravegaCluster.ObjectMeta.Name,
 		"WAIT_FOR":                 pravegaCluster.Spec.ZookeeperUri,

diff --git a/pkg/controller/pravega/pravega_controller.go b/pkg/controller/pravega/pravega_controller.go
@@ -20,7 +20,6 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	policyv1beta1 "k8s.io/api/policy/v1beta1"
-	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 )
@@ -79,16 +78,7 @@ func makeControllerPodSpec(name string, pravegaSpec *api.PravegaSpec) *corev1.Po
 						},
 					},
 				},
-				Resources: corev1.ResourceRequirements{
-					Requests: corev1.ResourceList{
-						corev1.ResourceCPU:    resource.MustParse("1000m"),
-						corev1.ResourceMemory: resource.MustParse("1Gi"),
-					},
-					Limits: corev1.ResourceList{
-						corev1.ResourceCPU:    resource.MustParse("2000m"),
-						corev1.ResourceMemory: resource.MustParse("3Gi"),
-					},
-				},
+				Resources: *pravegaSpec.ControllerResources,
 				ReadinessProbe: &corev1.Probe{
 					Handler: corev1.Handler{
 						Exec: &corev1.ExecAction{
@@ -127,7 +117,10 @@ func makeControllerPodSpec(name string, pravegaSpec *api.PravegaSpec) *corev1.Po
 
 func MakeControllerConfigMap(p *api.PravegaCluster) *corev1.ConfigMap {
 	var javaOpts = []string{
-		"-Xms1g -Xmx2g -Dpravegaservice.clusterName=" + p.Name,
+		"-Xms1g",
+		"-XX:+UnlockExperimentalVMOptions",
+		"-XX:+UseCGroupMemoryLimitForHeap",
+		"-Dpravegaservice.clusterName=" + p.Name,
 	}
 
 	for name, value := range p.Spec.Pravega.Options {
@@ -147,10 +140,6 @@ func MakeControllerConfigMap(p *api.PravegaCluster) *corev1.ConfigMap {
 		"WAIT_FOR":               p.Spec.ZookeeperUri,
 	}
 
-	for name, value := range p.Spec.Pravega.Options {
-		configData[name] = value
-	}
-
 	configMap := &corev1.ConfigMap{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "ConfigMap",