Add option to disable read-only/writable setting in operator

Changelog.md

@@ -14,6 +14,7 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [0.4.0] - TBD
 ### Added
+ * Added `ignoreReadOnly` option to prevent the operator from changing R/W state of mysql instances.
  * Added a test suite for RunCloneCommand logic, along with a mock backup server.
  * Added checks for service availability when cloning.
  * Added "fail fast" logic when unexpected errors occur during cloning/download.

charts/mysql-operator/values.yaml

@@ -139,7 +139,9 @@ orchestrator:
     # Recover both, masters and intermediate masters
     RecoverMasterClusterFilters: ['.*']
     RecoverIntermediateMasterClusterFilters: ['.*']
-    # `reset slave all` and `set read_only=0` on promoted master
+    # `reset slave all` and `set read_only=0` on promoted master; if you use the ignoreReadOnly setting in your mysql
+    # cluster config, then you may want to set this to true, as the operator will not automatically make newly promoted
+    # master instances writable when ignoreReadOnly=true
     ApplyMySQLPromotionAfterMasterFailover: false
     MasterFailoverDetachReplicaMasterHost: true
     # https://github.com/github/orchestrator/blob/master/docs/configuration-recovery.md#promotion-actions

config/crds/mysql_v1alpha1_mysqlcluster.yaml

@@ -66,6 +66,14 @@ spec:
             backupURL:
               description: Represents an URL to the location where to put backups.
               type: string
+            ignoreReadOnly:
+              description: Makes the operator ignore the ReadOnly setting completely.
+                This is an advanced setting that implies you take responsibility for
+                managing the R/W status of your instances. You will need to make sure
+                Orchestrator is configured to set promoted master instances to writable
+                (for example, by setting ApplyMySQLPromotionAfterMasterFailover to
+                true in your Orchestrator configuration).
+              type: boolean
             image:
               description: To specify the image that will be used for mysql server
                 container. If this is specified then the mysqlVersion is used as source

docs/deploy-mysql-cluster.md

@@ -93,6 +93,7 @@ Some important fields of `MySQLCluster` resource from `spec` are described in th
 | `maxSlaveLatency`                | The allowed slave lag until it's removed from read service. (in seconds)                    | `30`                       | nil                     |
 | `queryLimits`                    | Parameters for pt-kill to ensure some query run limits. (e.g. idle time)                    | `idelTime: 60`             | nil                     |
 | `readOnly`                       | A Boolean value that sets the cluster in read-only state.                                   | `True`                     | False                   |
+| `ignoreReadOnly`                 | A Boolean value that prevents the operator from changing the R/W status of mysql instances. | `False`                    | False                   |
 
 
 For more detailed information about cluster structure and configuration fields can be found in [godoc](https://godoc.org/github.com/presslabs/mysql-operator/pkg/apis/mysql/v1alpha1#MysqlClusterSpec).

pkg/apis/mysql/v1alpha1/mysqlcluster_types.go

@@ -126,6 +126,13 @@ type MysqlClusterSpec struct {
 	// +optional
 	ReadOnly bool `json:"readOnly,omitempty"`
 
+	// Makes the operator ignore the ReadOnly setting completely. This is an advanced setting that implies you take
+	// responsibility for managing the R/W status of your instances. You will need to make sure Orchestrator is configured
+	// to set promoted master instances to writable (for example, by setting ApplyMySQLPromotionAfterMasterFailover to
+	// true in your Orchestrator configuration).
+	// +optional
+	IgnoreReadOnly bool `json:"ignoreReadOnly,omitempty"`
+
 	// Set a custom offset for Server IDs.  ServerID for each node will be the index of the statefulset, plus offset
 	// +optional
 	ServerIDOffset *int `json:"serverIDOffset,omitempty"`

pkg/controller/orchestrator/orchestrator_reconcile.go

@@ -502,6 +502,14 @@ func (ou *orcUpdater) setReadOnlyNode(inst orc.Instance) error {
 
 // nolint: gocyclo
 func (ou *orcUpdater) markReadOnlyNodesInOrc(insts InstancesSet, master *orc.Instance) {
+	// If the user has set IgnoreReadOnly option to true, we will not interfere with Orchestrator's control of RW status
+	if ou.cluster.Spec.IgnoreReadOnly {
+		if ou.cluster.Spec.ReadOnly {
+			log.Info("IgnoreReadOnly=true takes precedence over ReadOnly=true, will not change RW status of any instances in Orchestrator",
+				"IgnoreReadOnly", ou.cluster.Spec.IgnoreReadOnly, "ReadOnly", ou.cluster.Spec.ReadOnly)
+		}
+		return
+	}
 	// If there is an in-progress failover, we will not interfere with readable/writable status on this iteration.
 	fip := ou.cluster.GetClusterCondition(api.ClusterConditionFailoverInProgress)
 	if fip != nil && fip.Status == core.ConditionTrue {

pkg/controller/orchestrator/orchestrator_reconcile_test.go

@@ -467,6 +467,23 @@ var _ = Describe("Orchestrator reconciler", func() {
 			Expect(node1.ReadOnly).To(Equal(true))
 		})
 
+		It("should not change read/write status when ReadOnly is Ignored", func() {
+			// Ignore the ReadOnly management; we will never tell Orchestrator what to do in this case
+			cluster.Spec.IgnoreReadOnly = true
+
+			insts, _ := orcClient.Cluster(cluster.GetClusterAlias())
+			node0 := InstancesSet(insts).GetInstance(cluster.GetPodHostname(0))
+
+			node0.ReadOnly = true
+			updater.markReadOnlyNodesInOrc(insts, node0)
+			Expect(node0.ReadOnly).To(Equal(true))
+
+			node0.ReadOnly = false
+			updater.markReadOnlyNodesInOrc(insts, node0)
+			Expect(node0.ReadOnly).To(Equal(false))
+
+		})
+
 		It("should remove old nodes from orchestrator", func() {
 			cluster.Spec.Replicas = &one
 			cluster.Status.ReadyNodes = 1