Issue 134,137: Fix BookKeeper errors (#135)

* Store BK index directory in a PVC Signed-off-by: Adrián Moreno <[email protected]> * fix operator version to 0.2.1 Signed-off-by: Adrián Moreno <[email protected]> * Re-enable tests Signed-off-by: Adrián Moreno <[email protected]> * Update example CRs Signed-off-by: Adrián Moreno <[email protected]> * Use IP as bookie ID Signed-off-by: Adrián Moreno <[email protected]> * Set BK id to IP address Signed-off-by: Adrián Moreno <[email protected]> * Specify container resources for testing Signed-off-by: Adrián Moreno <[email protected]> * Set Xmx and Xms Java opts to SS and C Signed-off-by: Adrián Moreno <[email protected]> * Adjust CPU and memory limits Signed-off-by: Adrián Moreno <[email protected]>
pravega · Mar 20, 2019 · f58ca1e · f58ca1e
1 parent 9048fee
commit f58ca1e
Show file tree

Hide file tree

Showing 6 changed files with 82 additions and 45 deletions.
diff --git a/example/cr-detailed.yaml b/example/cr-detailed.yaml
@@ -3,12 +3,12 @@ kind: "PravegaCluster"
 metadata:
   name: "example"
 spec:
-  zookeeperUri: example-client:2181
+  zookeeperUri: zk-client:2181
 
   bookkeeper:
     image:
       repository: pravega/bookkeeper
-      tag: 0.3.2
+      tag: 0.4.0
       pullPolicy: IfNotPresent
 
     replicas: 3
@@ -28,6 +28,13 @@ spec:
           requests:
             storage: 10Gi
 
+      indexVolumeClaimTemplate:
+        accessModes: [ "ReadWriteOnce" ]
+        storageClassName: "standard"
+        resources:
+          requests:
+            storage: 10Gi
+
     # Turns on automatic recovery
     # see https://bookkeeper.apache.org/docs/latest/admin/autorecovery/
     autoRecovery: true
@@ -51,7 +58,7 @@ spec:
 
     image:
       repository: pravega/pravega
-      tag: 0.3.2
+      tag: 0.4.0
       pullPolicy: IfNotPresent
 
     cacheVolumeClaimTemplate:

diff --git a/example/cr.yaml b/example/cr.yaml
@@ -16,48 +16,11 @@ metadata:
   name: "example"
 spec:
   zookeeperUri: zk-client:2181
-
   bookkeeper:
-    image:
-      repository: pravega/bookkeeper
-      tag: latest
-      pullPolicy: Always
-
     replicas: 3
-
-    storage:
-      ledgerVolumeClaimTemplate:
-        accessModes: [ "ReadWriteOnce" ]
-        storageClassName: "standard"
-        resources:
-          requests:
-            storage: 10Gi
-
-      journalVolumeClaimTemplate:
-        accessModes: [ "ReadWriteOnce" ]
-        storageClassName: "standard"
-        resources:
-          requests:
-            storage: 10Gi
-
-    autoRecovery: true
-
   pravega:
     controllerReplicas: 1
     segmentStoreReplicas: 3
-
-    cacheVolumeClaimTemplate:
-      accessModes: [ "ReadWriteOnce" ]
-      storageClassName: "standard"
-      resources:
-        requests:
-          storage: 20Gi
-
-    image:
-      repository: pravega/pravega
-      tag: latest
-      pullPolicy: Always
-
     tier2:
       filesystem:
         persistentVolumeClaim:

diff --git a/pkg/apis/pravega/v1alpha1/bookkeeper.go b/pkg/apis/pravega/v1alpha1/bookkeeper.go
@@ -39,6 +39,10 @@ const (
 	// Bookkeeper journal volume
 	DefaultBookkeeperJournalVolumeSize = "10Gi"
 
+	// DefaultBookkeeperIndexVolumeSize is the default volume size for the
+	// Bookkeeper index volume
+	DefaultBookkeeperIndexVolumeSize = "10Gi"
+
 	// MinimumBookkeeperReplicas is the minimum number of Bookkeeper replicas
 	// accepted
 	MinimumBookkeeperReplicas = 3
@@ -144,6 +148,11 @@ type BookkeeperStorageSpec struct {
 	// This field is optional. If no PVC spec and there is no default storage class,
 	// stateful containers will use emptyDir as volume
 	JournalVolumeClaimTemplate *v1.PersistentVolumeClaimSpec `json:"journalVolumeClaimTemplate"`
+
+	// IndexVolumeClaimTemplate is the spec to describe PVC for the BookKeeper index
+	// This field is optional. If no PVC spec and there is no default storage class,
+	// stateful containers will use emptyDir as volume
+	IndexVolumeClaimTemplate *v1.PersistentVolumeClaimSpec `json:"indexVolumeClaimTemplate"`
 }
 
 func (s *BookkeeperStorageSpec) withDefaults() (changed bool) {
@@ -171,5 +180,17 @@ func (s *BookkeeperStorageSpec) withDefaults() (changed bool) {
 		}
 	}
 
+	if s.IndexVolumeClaimTemplate == nil {
+		changed = true
+		s.IndexVolumeClaimTemplate = &v1.PersistentVolumeClaimSpec{
+			AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce},
+			Resources: v1.ResourceRequirements{
+				Requests: v1.ResourceList{
+					v1.ResourceStorage: resource.MustParse(DefaultBookkeeperIndexVolumeSize),
+				},
+			},
+		}
+	}
+
 	return changed
 }
diff --git a/pkg/controller/pravega/bookie.go b/pkg/controller/pravega/bookie.go
@@ -18,13 +18,15 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	policyv1beta1 "k8s.io/api/policy/v1beta1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 )
 
 const (
 	LedgerDiskName  = "ledger"
 	JournalDiskName = "journal"
+	IndexDiskName   = "index"
 )
 
 func MakeBookieHeadlessService(pravegaCluster *v1alpha1.PravegaCluster) *corev1.Service {
@@ -115,6 +117,20 @@ func makeBookiePodSpec(clusterName string, bookkeeperSpec *v1alpha1.BookkeeperSp
 						Name:      JournalDiskName,
 						MountPath: "/bk/ledgers",
 					},
+					{
+						Name:      IndexDiskName,
+						MountPath: "/bk/index",
+					},
+				},
+				Resources: corev1.ResourceRequirements{
+					Requests: corev1.ResourceList{
+						corev1.ResourceCPU:    resource.MustParse("1000m"),
+						corev1.ResourceMemory: resource.MustParse("3Gi"),
+					},
+					Limits: corev1.ResourceList{
+						corev1.ResourceCPU:    resource.MustParse("2000m"),
+						corev1.ResourceMemory: resource.MustParse("5Gi"),
+					},
 				},
 				ReadinessProbe: &corev1.Probe{
 					Handler: corev1.Handler{
@@ -167,14 +183,22 @@ func makeBookieVolumeClaimTemplates(spec *v1alpha1.BookkeeperSpec) []corev1.Pers
 			},
 			Spec: *spec.Storage.LedgerVolumeClaimTemplate,
 		},
+		{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: IndexDiskName,
+			},
+			Spec: *spec.Storage.IndexVolumeClaimTemplate,
+		},
 	}
 }
 
 func MakeBookieConfigMap(pravegaCluster *v1alpha1.PravegaCluster) *corev1.ConfigMap {
 	configData := map[string]string{
-		"BK_BOOKIE_EXTRA_OPTS":     "\"-Xms1g -Xmx1g -XX:MaxDirectMemorySize=1g -XX:+UseG1GC  -XX:MaxGCPauseMillis=10 -XX:+ParallelRefProcEnabled -XX:+UnlockExperimentalVMOptions -XX:+AggressiveOpts -XX:+DoEscapeAnalysis -XX:ParallelGCThreads=32 -XX:ConcGCThreads=32 -XX:G1NewSizePercent=50 -XX:+DisableExplicitGC -XX:-ResizePLAB\"",
-		"ZK_URL":                   pravegaCluster.Spec.ZookeeperUri,
-		"BK_useHostNameAsBookieID": "true",
+		"BK_BOOKIE_EXTRA_OPTS": "-Xms1g -Xmx4g -XX:MaxDirectMemorySize=1g -XX:+UseG1GC  -XX:MaxGCPauseMillis=10 -XX:+ParallelRefProcEnabled -XX:+UnlockExperimentalVMOptions -XX:+AggressiveOpts -XX:+DoEscapeAnalysis -XX:ParallelGCThreads=32 -XX:ConcGCThreads=32 -XX:G1NewSizePercent=50 -XX:+DisableExplicitGC -XX:-ResizePLAB",
+		"ZK_URL":               pravegaCluster.Spec.ZookeeperUri,
+		// Set useHostNameAsBookieID to false until BookKeeper Docker
+		// image is updated to 4.7
+		"BK_useHostNameAsBookieID": "false",
 		"PRAVEGA_CLUSTER_NAME":     pravegaCluster.ObjectMeta.Name,
 		"WAIT_FOR":                 pravegaCluster.Spec.ZookeeperUri,
 	}

diff --git a/pkg/controller/pravega/pravega_controller.go b/pkg/controller/pravega/pravega_controller.go
@@ -20,6 +20,7 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	policyv1beta1 "k8s.io/api/policy/v1beta1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 )
@@ -78,6 +79,16 @@ func makeControllerPodSpec(name string, pravegaSpec *api.PravegaSpec) *corev1.Po
 						},
 					},
 				},
+				Resources: corev1.ResourceRequirements{
+					Requests: corev1.ResourceList{
+						corev1.ResourceCPU:    resource.MustParse("1000m"),
+						corev1.ResourceMemory: resource.MustParse("1Gi"),
+					},
+					Limits: corev1.ResourceList{
+						corev1.ResourceCPU:    resource.MustParse("2000m"),
+						corev1.ResourceMemory: resource.MustParse("3Gi"),
+					},
+				},
 				ReadinessProbe: &corev1.Probe{
 					Handler: corev1.Handler{
 						Exec: &corev1.ExecAction{
@@ -116,7 +127,7 @@ func makeControllerPodSpec(name string, pravegaSpec *api.PravegaSpec) *corev1.Po
 
 func MakeControllerConfigMap(p *api.PravegaCluster) *corev1.ConfigMap {
 	var javaOpts = []string{
-		"-Dpravegaservice.clusterName=" + p.Name,
+		"-Xms1g -Xmx2g -Dpravegaservice.clusterName=" + p.Name,
 	}
 
 	for name, value := range p.Spec.Pravega.Options {

diff --git a/pkg/controller/pravega/pravega_segmentstore.go b/pkg/controller/pravega/pravega_segmentstore.go
@@ -20,6 +20,7 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	policyv1beta1 "k8s.io/api/policy/v1beta1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 )
@@ -98,6 +99,16 @@ func makeSegmentstorePodSpec(pravegaCluster *api.PravegaCluster) corev1.PodSpec
 						MountPath: cacheVolumeMountPoint,
 					},
 				},
+				Resources: corev1.ResourceRequirements{
+					Requests: corev1.ResourceList{
+						corev1.ResourceCPU:    resource.MustParse("1000m"),
+						corev1.ResourceMemory: resource.MustParse("3Gi"),
+					},
+					Limits: corev1.ResourceList{
+						corev1.ResourceCPU:    resource.MustParse("2000m"),
+						corev1.ResourceMemory: resource.MustParse("5Gi"),
+					},
+				},
 				ReadinessProbe: &corev1.Probe{
 					Handler: corev1.Handler{
 						Exec: &corev1.ExecAction{
@@ -142,7 +153,7 @@ func makeSegmentstorePodSpec(pravegaCluster *api.PravegaCluster) corev1.PodSpec
 
 func MakeSegmentstoreConfigMap(pravegaCluster *api.PravegaCluster) *corev1.ConfigMap {
 	javaOpts := []string{
-		"-Dpravegaservice.clusterName=" + pravegaCluster.Name,
+		"-Xms1g -Xmx4g -XX:MaxDirectMemorySize=1g -Dpravegaservice.clusterName=" + pravegaCluster.Name,
 	}
 
 	for name, value := range pravegaCluster.Spec.Pravega.Options {