Add the field spec.readOnly and the logic required for this

cmd/mysql-helper/apphelper/apphelper.go

@@ -148,11 +148,9 @@ func waitForMysqlReady() error {
 
 func configReadOnly() error {
 	var query string
-	if tb.NodeRole() == "master" {
-		query = "SET GLOBAL READ_ONLY = 0"
-	} else {
-		query = "SET GLOBAL SUPER_READ_ONLY = 1"
-	}
+
+	query = "SET GLOBAL SUPER_READ_ONLY = 1"
+
 	if err := tb.RunQuery(query); err != nil {
 		return fmt.Errorf("failed to set read_only config, err: %s", err)
 	}

deploy/mysqlclusters.yaml

@@ -115,6 +115,8 @@ spec:
                   type: integer
               required:
               - maxQueryTime
+            readOnly:
+              type: boolean
             replicas:
               description: The number of pods. This updates replicas filed Defaults
                 to 0

hack/charts/mysql-operator/values.yaml

@@ -54,7 +54,7 @@ orchestrator:
     RecoverMasterClusterFilters: ['.*']
     RecoverIntermediateMasterClusterFilters: ['.*']
     # `reset slave all` and `set read_only=0` on promoted master
-    ApplyMySQLPromotionAfterMasterFailover: true
+    ApplyMySQLPromotionAfterMasterFailover: false
     # set downtime on the failed master
     MasterFailoverLostInstancesDowntimeMinutes: 10
     # https://github.com/github/orchestrator/blob/master/docs/configuration-recovery.md#promotion-actions

pkg/apis/mysql/v1alpha1/types.go

@@ -107,6 +107,8 @@ type ClusterSpec struct {
 	// QueryLimits represents limits for a query
 	// +optional
 	QueryLimits *QueryLimits `json:"queryLimits,omitempty"`
+
+	ReadOnly bool `json:"readOnly,omitempty"`
 }
 
 type MysqlConf map[string]string
@@ -140,6 +142,7 @@ const (
 	ClusterConditionReady       ClusterConditionType = "Ready"
 	ClusterConditionConfig                           = "ConfigReady"
 	ClusterConditionFailoverAck                      = "PendingFailoverAck"
+	ClusterConditionReadOnly                         = "ReadOnly"
 )
 
 type NodeStatus struct {
@@ -159,6 +162,7 @@ const (
 	NodeConditionLagged      NodeConditionType = "Lagged"
 	NodeConditionReplicating                   = "Replicating"
 	NodeConditionMaster                        = "Master"
+	NodeConditionReadOnly                      = "ReadOnly"
 )
 
 type PodSpec struct {

pkg/mysqlcluster/orc_reconciliation.go

@@ -43,7 +43,9 @@ func (f *cFactory) SyncOrchestratorStatus(ctx context.Context) error {
 	}
 
 	if insts, err := f.orcClient.Cluster(f.getClusterAlias()); err == nil {
+
 		f.updateStatusFromOrc(insts)
+
 	} else {
 		glog.Errorf("Fail to get cluster from orchestrator: %s. Now tries to register nodes.", err)
 		return f.registerNodesInOrc()
@@ -72,8 +74,101 @@ func (f *cFactory) SyncOrchestratorStatus(ctx context.Context) error {
 	return nil
 }
 
+func getInstance(hostname string, insts []orc.Instance) (*orc.Instance, error) {
+
+	for _, node := range insts {
+		host := node.Key.Hostname
+
+		if host == hostname {
+			return &node, nil
+		}
+	}
+
+	return nil, fmt.Errorf("the element was not found")
+}
+
+func getMaster(node *orc.Instance, insts []orc.Instance) (string, error) {
+
+	if len(node.MasterKey.Hostname) != 0 && node.IsCoMaster == false {
+		next, err := getInstance(node.MasterKey.Hostname, insts)
+		if err == nil {
+			return getMaster(next, insts)
+		} else {
+			return "", err
+		}
+	}
+
+	if node.IsCoMaster == true {
+		next, err := getInstance(node.MasterKey.Hostname, insts)
+		if err == nil {
+			return next.Key.Hostname, nil
+		} else {
+			return "", err
+		}
+	}
+
+	return node.Key.Hostname, nil
+}
+
+func determineMasterFor(insts []orc.Instance) (string, error) {
+
+	var masterForNode []string
+
+	for _, node := range insts {
+		master, err := getMaster(&node, insts)
+		if err == nil {
+			masterForNode = append(masterForNode, master)
+		} else {
+			return "", fmt.Errorf("not able to retrieve the root of this node %s", node.Key.Hostname)
+		}
+	}
+
+	if len(masterForNode) != 0 {
+		masterHostName := masterForNode[0]
+		var check bool = true
+		for _, node := range masterForNode {
+			if node != masterHostName {
+				check = false
+			}
+
+		}
+		if check == true {
+			return masterHostName, nil
+		} else {
+			return "", fmt.Errorf("multiple masters")
+		}
+	} else {
+		return "", fmt.Errorf("0 elements in instance array")
+	}
+
+}
+
+func (f *cFactory) setMasterWritable(masterHostName string, insts []orc.Instance) error {
+	master, err := getInstance(masterHostName, insts)
+
+	if err != nil {
+		return err
+	}
+
+	if f.cluster.Spec.ReadOnly == false {
+		err := f.orcClient.SetHostWritable(master.Key.Hostname, master.Key.Port)
+		if err != nil {
+			return err
+		}
+	} else {
+		err = f.orcClient.SetHostReadOnly(master.Key.Hostname, master.Key.Port)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 func (f *cFactory) updateStatusFromOrc(insts []orc.Instance) {
 	updatedNodes := []string{}
+
+	var isReadOnly bool = true
 	for _, node := range insts {
 		host := node.Key.Hostname
 		updatedNodes = append(updatedNodes, host)
@@ -106,11 +201,36 @@ func (f *cFactory) updateStatusFromOrc(insts []orc.Instance) {
 			f.updateNodeCondition(host, api.NodeConditionReplicating, core.ConditionFalse)
 		}
 
-		if !node.ReadOnly {
-			f.updateNodeCondition(host, api.NodeConditionMaster, core.ConditionTrue)
+		f.updateNodeCondition(host, api.NodeConditionMaster, core.ConditionFalse)
+
+		isReadOnly = isReadOnly && node.ReadOnly
+
+		if node.ReadOnly == true {
+			f.updateNodeCondition(host, api.NodeConditionReadOnly, core.ConditionTrue)
 		} else {
-			f.updateNodeCondition(host, api.NodeConditionMaster, core.ConditionFalse)
+			f.updateNodeCondition(host, api.NodeConditionReadOnly, core.ConditionFalse)
 		}
+
+	}
+
+	masterHostName, err := determineMasterFor(insts)
+	if err == nil {
+		f.updateNodeCondition(masterHostName, api.NodeConditionMaster, core.ConditionTrue)
+	} else {
+		f.updateNodeCondition(masterHostName, api.NodeConditionMaster, core.ConditionFalse)
+	}
+
+	err = f.setMasterWritable(masterHostName, insts)
+	if err != nil {
+		glog.Infof("Error setting Master readOnly/writable %s", err)
+	}
+
+	if isReadOnly == true {
+		f.cluster.UpdateStatusCondition(api.ClusterConditionReadOnly,
+			core.ConditionTrue, "initializedTrue", "settingReadOnlyTrue")
+	} else {
+		f.cluster.UpdateStatusCondition(api.ClusterConditionReadOnly,
+			core.ConditionFalse, "initializedFalse", "settingReadOnlyFalse")
 	}
 
 	f.removeNodeConditionNotIn(updatedNodes)

pkg/mysqlcluster/orc_reconciliation_test.go

@@ -18,21 +18,20 @@ package mysqlcluster
 
 import (
 	"context"
-	"testing"
-	"time"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-	core "k8s.io/api/core/v1"
-	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/client-go/kubernetes/fake"
-	"k8s.io/client-go/tools/record"
-
 	api "github.com/presslabs/mysql-operator/pkg/apis/mysql/v1alpha1"
 	fakeMyClient "github.com/presslabs/mysql-operator/pkg/generated/clientset/versioned/fake"
 	"github.com/presslabs/mysql-operator/pkg/util/options"
 	fakeOrc "github.com/presslabs/mysql-operator/pkg/util/orchestrator/fake"
 	tutil "github.com/presslabs/mysql-operator/pkg/util/test"
+	core "k8s.io/api/core/v1"
+	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes/fake"
+	"k8s.io/client-go/tools/record"
+	"testing"
+	"time"
 )
 
 func TestReconciliation(t *testing.T) {
@@ -99,6 +98,8 @@ var _ = Describe("Mysql cluster reconcilation", func() {
 				Ω(rec.Events).Should(Receive(&event))
 				Expect(event).To(ContainSubstring("ReplicationStopped"))
 				Ω(rec.Events).Should(Receive(&event))
+				Expect(event).To(ContainSubstring("DemoteMaster"))
+				Ω(rec.Events).Should(Receive(&event))
 				Expect(event).To(ContainSubstring("PromoteMaster"))
 			})
 
@@ -148,13 +149,13 @@ var _ = Describe("Mysql cluster reconcilation", func() {
 				Expect(event).To(ContainSubstring("RecoveryAcked"))
 			})
 
-			It("node not uptodate in orc", func() {
+			It("master is in orc", func() {
 				orcClient.AddInstance("asd.default", cluster.GetPodHostname(0),
 					true, -1, false, false)
 				Ω(factory.SyncOrchestratorStatus(ctx)).Should(Succeed())
 
 				Expect(cluster.Status.Nodes[0].GetCondition(api.NodeConditionMaster).Status).To(
-					Equal(core.ConditionUnknown))
+					Equal(core.ConditionTrue))
 			})
 
 			It("node not in orc", func() {
@@ -173,6 +174,103 @@ var _ = Describe("Mysql cluster reconcilation", func() {
 
 			})
 
+			It("existence of a single master", func() {
+
+				orcClient.AddInstanceInTopology("asd.default", "foo122-mysql-0",
+					3306, false, -1, false, true, "", 0, false)
+
+				orcClient.AddInstanceInTopology("asd.default", "foo122-mysql-1",
+					3307, false, -1, false, true, "foo122-mysql-0", 3306, false)
+
+				orcClient.AddInstanceInTopology("asd.default", "foo122-mysql-2",
+					3308, false, -1, false, true, "foo122-mysql-0", 3306, false)
+
+				orcClient.AddInstanceInTopology("asd.default", "foo122-mysql-3",
+					3309, false, -1, false, true, "foo122-mysql-2", 3308, false)
+
+				orcClient.AddInstanceInTopology("asd.default", "foo122-mysql-4",
+					3310, false, -1, false, true, "foo122-mysql-3", 3309, false)
+
+				insts, _ := orcClient.Cluster("asd.default")
+
+				_, err := determineMasterFor(insts)
+				Expect(err).To(BeNil())
+
+			})
+
+			It("existence of multiple masters", func() {
+
+				orcClient.AddInstanceInTopology("asd.default", "foo122-mysql-0",
+					3306, false, -1, false, true, "foo122-mysql-5", 0, true)
+
+				orcClient.AddInstanceInTopology("asd.default", "foo122-mysql-1",
+					3307, false, -1, false, true, "foo122-mysql-0", 3306, false)
+
+				orcClient.AddInstanceInTopology("asd.default", "foo122-mysql-2",
+					3308, false, -1, false, true, "foo122-mysql-0", 3306, false)
+
+				orcClient.AddInstanceInTopology("asd.default", "foo122-mysql-3",
+					3309, false, -1, false, true, "foo122-mysql-2", 3308, false)
+
+				orcClient.AddInstanceInTopology("asd.default", "foo122-mysql-4",
+					3310, false, -1, false, true, "foo122-mysql-3", 3309, false)
+
+				orcClient.AddInstanceInTopology("asd.default", "foo122-mysql-5",
+					3311, false, -1, false, true, "foo122-mysql-0", 3309, true)
+
+				insts, _ := orcClient.Cluster("asd.default")
+
+				_, err := determineMasterFor(insts)
+				Expect(err).ToNot(BeNil())
+
+			})
+
+			It("no instances", func() {
+
+				insts, _ := orcClient.Cluster("asd.default")
+
+				_, err := determineMasterFor(insts)
+				Expect(err).ToNot(BeNil())
+
+			})
+
+			It("set master readOnly/Writable", func() {
+
+				//Set ReadOnly to true in order to get master ReadOnly
+
+				orcClient.AddInstance("asd.default", cluster.GetPodHostname(0),
+					true, -1, false, true)
+
+				factory.cluster.Spec.ReadOnly = true
+
+				insts, _ := orcClient.Cluster("asd.default")
+
+				err := factory.setMasterWritable(cluster.GetPodHostname(0), insts)
+				Expect(err).To(BeNil())
+
+				for _, instance := range insts {
+					if instance.Key.Hostname == cluster.GetPodHostname(0) && instance.Key.Port == 3306 {
+						Expect(instance.ReadOnly).To(Equal(true))
+					}
+				}
+
+				//Set ReadOnly to false in order to get the master Writable
+
+				factory.cluster.Spec.ReadOnly = false
+
+				insts, _ = orcClient.Cluster("asd.default")
+
+				err = factory.setMasterWritable(cluster.GetPodHostname(0), insts)
+				Expect(err).To(BeNil())
+
+				for _, instance := range insts {
+					if instance.Key.Hostname == cluster.GetPodHostname(0) && instance.Key.Port == 3306 {
+						Expect(instance.ReadOnly).To(Equal(false))
+					}
+				}
+
+			})
+
 		})
 	})
 })

pkg/openapi/openapi_generated.go

@@ -363,6 +363,12 @@ func schema_pkg_apis_mysql_v1alpha1_ClusterSpec(ref common.ReferenceCallback) co
 							Ref:         ref("github.com/presslabs/mysql-operator/pkg/apis/mysql/v1alpha1.QueryLimits"),
 						},
 					},
+					"readOnly": {
+						SchemaProps: spec.SchemaProps{
+							Type:   []string{"boolean"},
+							Format: "",
+						},
+					},
 				},
 			},
 		},