From 317cb4876b507f74f5ff800f5ee3a0284890c84b Mon Sep 17 00:00:00 2001
From: dougfales <doug@getharvest.com>
Date: Fri, 8 Nov 2019 10:38:27 -0700
Subject: [PATCH] Adding a specific case for cluster initialization.

When no services are available, we want to die and let k8s
re-initiialize the pod, *unless* this is the initial pod, in which case
it is valid to allow initialization to coninue without cloning from a
service or bucket.
---
 pkg/sidecar/appclone.go      | 10 ++++++++--
 pkg/sidecar/appclone_test.go | 13 ++++++++++++-
 pkg/sidecar/configs.go       |  5 +++++
 3 files changed, 25 insertions(+), 3 deletions(-)

diff --git a/pkg/sidecar/appclone.go b/pkg/sidecar/appclone.go
index 1d9e3ac9e..e0d934985 100644
--- a/pkg/sidecar/appclone.go
+++ b/pkg/sidecar/appclone.go
@@ -52,6 +52,10 @@ import (
 // master; bucket URL exists | The assumption is that this is the bootstrap case: the
 //                           | very first mysql pod is being initialized.
 // ------------------------------------------------------------------------------------
+// No healthy replcias; no   | If this is the first pod in the cluster, then allow it
+// master; no bucket URL     | to initialize as an empty instance, otherwise, return an
+//                           | error to allow k8s to kill and restart the pod.
+// ------------------------------------------------------------------------------------
 func RunCloneCommand(cfg *Config) error {
 	log.Info("cloning command", "host", cfg.Hostname)
 
@@ -79,9 +83,11 @@ func RunCloneCommand(cfg *Config) error {
 		if err := cloneFromBucket(cfg.InitBucketURL); err != nil {
 			return fmt.Errorf("failed to clone from bucket, err: %s", err)
 		}
-	} else {
-		log.Info("nothing to clone from: no existing data found, no replicas and no master available, and no clone bucket url found")
+	} else if cfg.IsFirstPodInSet() {
+		log.Info("nothing to clone from: empty cluster initializing")
 		return nil
+	} else {
+		return fmt.Errorf("nothing to clone from: no existing data found, no replicas and no master available, and no clone bucket url found")
 	}
 
 	// prepare backup
diff --git a/pkg/sidecar/appclone_test.go b/pkg/sidecar/appclone_test.go
index 1df96b00b..e7abfbbea 100644
--- a/pkg/sidecar/appclone_test.go
+++ b/pkg/sidecar/appclone_test.go
@@ -255,7 +255,8 @@ var _ = Describe("Test RunCloneCommand cloning logic", func() {
 			stopFakeMasterService()
 		})
 
-		It("should return nil", func() {
+		It("should return nil for first pod", func() {
+			cfg.Hostname = "mysql-mysql-0"
 			Expect(fakeBackupFile).ShouldNot(BeAnExistingFile())
 
 			err := RunCloneCommand(cfg)
@@ -264,6 +265,16 @@ var _ = Describe("Test RunCloneCommand cloning logic", func() {
 			Expect(fakeBackupFile).ShouldNot(BeAnExistingFile())
 		})
 
+		It("should return an error for subsequent pods", func() {
+			cfg.Hostname = "mysql-mysql-1"
+			Expect(fakeBackupFile).ShouldNot(BeAnExistingFile())
+
+			err := RunCloneCommand(cfg)
+			Expect(err).To(HaveOccurred())
+
+			Expect(fakeBackupFile).ShouldNot(BeAnExistingFile())
+		})
+
 	})
 
 })
diff --git a/pkg/sidecar/configs.go b/pkg/sidecar/configs.go
index 1c5804f4f..df6104c18 100644
--- a/pkg/sidecar/configs.go
+++ b/pkg/sidecar/configs.go
@@ -120,6 +120,11 @@ func (cfg *Config) MysqlDSN() string {
 	)
 }
 
+func (cfg *Config) IsFirstPodInSet() bool {
+	ordinal := getOrdinalFromHostname(cfg.Hostname)
+	return ordinal == 0
+}
+
 // ShouldCloneFromBucket returns true if it's time to initialize from a bucket URL provided
 func (cfg *Config) ShouldCloneFromBucket() bool {
 	return !cfg.ExistsMySQLData && cfg.ServerID() == cfg.MyServerIDOffset && len(cfg.InitBucketURL) != 0