From d7beb941510b5b0d2ad71cedba5f0d77db9a7161 Mon Sep 17 00:00:00 2001 From: Ewout Prangsma Date: Fri, 30 Mar 2018 14:19:25 +0200 Subject: [PATCH] Check contents of persisted volume when dbserver is restarting --- pkg/apis/deployment/v1alpha/member_status.go | 3 +++ pkg/deployment/images.go | 2 +- pkg/deployment/resources/pod_creator.go | 5 ++++- pkg/deployment/resources/pod_inspector.go | 1 + pkg/util/k8sutil/pods.go | 23 +++++++++++++++++--- 5 files changed, 29 insertions(+), 5 deletions(-) diff --git a/pkg/apis/deployment/v1alpha/member_status.go b/pkg/apis/deployment/v1alpha/member_status.go index 20831c25b..01028ab70 100644 --- a/pkg/apis/deployment/v1alpha/member_status.go +++ b/pkg/apis/deployment/v1alpha/member_status.go @@ -44,6 +44,9 @@ type MemberStatus struct { // RecentTerminatons holds the times when this member was recently terminated. // First entry is the oldest. (do not add omitempty, since we want to be able to switch from a list to an empty list) RecentTerminations []metav1.Time `json:"recent-terminations"` + // IsInitialized is set after the very first time a pod was created for this member. + // After that, DBServers must have a UUID field or fail. + IsInitialized bool `json:"initialized"` } // RemoveTerminationsBefore removes all recent terminations before the given timestamp. diff --git a/pkg/deployment/images.go b/pkg/deployment/images.go index 323c1f4c5..504a4ca7b 100644 --- a/pkg/deployment/images.go +++ b/pkg/deployment/images.go @@ -166,7 +166,7 @@ func (ib *imagesBuilder) fetchArangoDBImageIDAndVersion(ctx context.Context, ima "--server.authentication=false", fmt.Sprintf("--server.endpoint=tcp://[::]:%d", k8sutil.ArangoPort), } - if err := k8sutil.CreateArangodPod(ib.KubeCli, true, ib.APIObject, role, id, podName, "", image, ib.Spec.GetImagePullPolicy(), args, nil, nil, nil, "", ""); err != nil { + if err := k8sutil.CreateArangodPod(ib.KubeCli, true, ib.APIObject, role, id, podName, "", image, ib.Spec.GetImagePullPolicy(), "", false, args, nil, nil, nil, "", ""); err != nil { log.Debug().Err(err).Msg("Failed to create image ID pod") return true, maskAny(err) } diff --git a/pkg/deployment/resources/pod_creator.go b/pkg/deployment/resources/pod_creator.go index 373f98757..dfb87f7e5 100644 --- a/pkg/deployment/resources/pod_creator.go +++ b/pkg/deployment/resources/pod_creator.go @@ -366,7 +366,10 @@ func (r *Resources) createPodForMember(spec api.DeploymentSpec, group api.Server SecretKey: constants.SecretKeyJWT, } } - if err := k8sutil.CreateArangodPod(kubecli, spec.IsDevelopment(), apiObject, role, m.ID, m.PodName, m.PersistentVolumeClaimName, info.ImageID, spec.GetImagePullPolicy(), args, env, livenessProbe, readinessProbe, tlsKeyfileSecretName, rocksdbEncryptionSecretName); err != nil { + engine := string(spec.GetStorageEngine()) + requireUUID := group == api.ServerGroupDBServers && m.IsInitialized + if err := k8sutil.CreateArangodPod(kubecli, spec.IsDevelopment(), apiObject, role, m.ID, m.PodName, m.PersistentVolumeClaimName, info.ImageID, spec.GetImagePullPolicy(), + engine, requireUUID, args, env, livenessProbe, readinessProbe, tlsKeyfileSecretName, rocksdbEncryptionSecretName); err != nil { return maskAny(err) } log.Debug().Str("pod-name", m.PodName).Msg("Created pod") diff --git a/pkg/deployment/resources/pod_inspector.go b/pkg/deployment/resources/pod_inspector.go index b419b3143..1ae157e0c 100644 --- a/pkg/deployment/resources/pod_inspector.go +++ b/pkg/deployment/resources/pod_inspector.go @@ -100,6 +100,7 @@ func (r *Resources) InspectPods() error { // Pod is now ready if memberStatus.Conditions.Update(api.ConditionTypeReady, true, "Pod Ready", "") { log.Debug().Str("pod-name", p.GetName()).Msg("Updating member condition Ready to true") + memberStatus.IsInitialized = true // Require future pods for this member to have an existing UUID (in case of dbserver). updateMemberStatusNeeded = true } } else { diff --git a/pkg/util/k8sutil/pods.go b/pkg/util/k8sutil/pods.go index 1fc53917d..8700e0f5d 100644 --- a/pkg/util/k8sutil/pods.go +++ b/pkg/util/k8sutil/pods.go @@ -25,6 +25,7 @@ package k8sutil import ( "fmt" "path/filepath" + "strings" "time" "k8s.io/api/core/v1" @@ -170,13 +171,28 @@ func rocksdbEncryptionVolumeMounts() []v1.VolumeMount { // arangodInitContainer creates a container configured to // initalize a UUID file. -func arangodInitContainer(name, id string) v1.Container { +func arangodInitContainer(name, id, engine string, requireUUID bool) v1.Container { uuidFile := filepath.Join(ArangodVolumeMountDir, "UUID") + engineFile := filepath.Join(ArangodVolumeMountDir, "ENGINE") + var command string + if requireUUID { + command = strings.Join([]string{ + // Files must exist + fmt.Sprintf("test -f %s", uuidFile), + fmt.Sprintf("test -f %s", engineFile), + // Content must match + fmt.Sprintf("grep -q %s %s", id, uuidFile), + fmt.Sprintf("grep -q %s %s", engine, engineFile), + }, " && ") + + } else { + command = fmt.Sprintf("test -f %s || echo '%s' > %s", uuidFile, id, uuidFile) + } c := v1.Container{ Command: []string{ "/bin/sh", "-c", - fmt.Sprintf("test -f %s || echo '%s' > %s", uuidFile, id, uuidFile), + command, }, Name: name, Image: alpineImage, @@ -261,6 +277,7 @@ func newPod(deploymentName, ns, role, id, podName string) v1.Pod { // If another error occurs, that error is returned. func CreateArangodPod(kubecli kubernetes.Interface, developmentMode bool, deployment APIObject, role, id, podName, pvcName, image string, imagePullPolicy v1.PullPolicy, + engine string, requireUUID bool, args []string, env map[string]EnvValue, livenessProbe *HTTPProbeConfig, readinessProbe *HTTPProbeConfig, tlsKeyfileSecretName, rocksdbEncryptionSecretName string) error { @@ -278,7 +295,7 @@ func CreateArangodPod(kubecli kubernetes.Interface, developmentMode bool, deploy p.Spec.Containers = append(p.Spec.Containers, c) // Add UUID init container - p.Spec.InitContainers = append(p.Spec.InitContainers, arangodInitContainer("uuid", id)) + p.Spec.InitContainers = append(p.Spec.InitContainers, arangodInitContainer("uuid", id, engine, requireUUID)) // Add volume if pvcName != "" {