diff --git a/Changelog.md b/Changelog.md index 037e63cb8..18745d758 100644 --- a/Changelog.md +++ b/Changelog.md @@ -14,6 +14,7 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [0.4.0] - TBD ### Added + * Added `ignoreReadOnly` option to prevent the operator from changing R/W state of mysql instances. * Added a test suite for RunCloneCommand logic, along with a mock backup server. * Added checks for service availability when cloning. * Added "fail fast" logic when unexpected errors occur during cloning/download. diff --git a/charts/mysql-operator/values.yaml b/charts/mysql-operator/values.yaml index 689de5bdf..b1464a278 100644 --- a/charts/mysql-operator/values.yaml +++ b/charts/mysql-operator/values.yaml @@ -139,7 +139,9 @@ orchestrator: # Recover both, masters and intermediate masters RecoverMasterClusterFilters: ['.*'] RecoverIntermediateMasterClusterFilters: ['.*'] - # `reset slave all` and `set read_only=0` on promoted master + # `reset slave all` and `set read_only=0` on promoted master; if you use the ignoreReadOnly setting in your mysql + # cluster config, then you may want to set this to true, as the operator will not automatically make newly promoted + # master instances writable when ignoreReadOnly=true ApplyMySQLPromotionAfterMasterFailover: false MasterFailoverDetachReplicaMasterHost: true # https://github.com/github/orchestrator/blob/master/docs/configuration-recovery.md#promotion-actions diff --git a/config/crds/mysql_v1alpha1_mysqlcluster.yaml b/config/crds/mysql_v1alpha1_mysqlcluster.yaml index f8e734684..da7eb63b0 100644 --- a/config/crds/mysql_v1alpha1_mysqlcluster.yaml +++ b/config/crds/mysql_v1alpha1_mysqlcluster.yaml @@ -66,6 +66,14 @@ spec: backupURL: description: Represents an URL to the location where to put backups. type: string + ignoreReadOnly: + description: Makes the operator ignore the ReadOnly setting completely. + This is an advanced setting that implies you take responsibility for + managing the R/W status of your instances. You will need to make sure + Orchestrator is configured to set promoted master instances to writable + (for example, by setting ApplyMySQLPromotionAfterMasterFailover to + true in your Orchestrator configuration). + type: boolean image: description: To specify the image that will be used for mysql server container. If this is specified then the mysqlVersion is used as source diff --git a/docs/deploy-mysql-cluster.md b/docs/deploy-mysql-cluster.md index 1c9fa302a..7457e6c70 100644 --- a/docs/deploy-mysql-cluster.md +++ b/docs/deploy-mysql-cluster.md @@ -93,6 +93,7 @@ Some important fields of `MySQLCluster` resource from `spec` are described in th | `maxSlaveLatency` | The allowed slave lag until it's removed from read service. (in seconds) | `30` | nil | | `queryLimits` | Parameters for pt-kill to ensure some query run limits. (e.g. idle time) | `idelTime: 60` | nil | | `readOnly` | A Boolean value that sets the cluster in read-only state. | `True` | False | +| `ignoreReadOnly` | A Boolean value that prevents the operator from changing the R/W status of mysql instances. | `False` | False | For more detailed information about cluster structure and configuration fields can be found in [godoc](https://godoc.org/github.com/presslabs/mysql-operator/pkg/apis/mysql/v1alpha1#MysqlClusterSpec). diff --git a/pkg/apis/mysql/v1alpha1/mysqlcluster_types.go b/pkg/apis/mysql/v1alpha1/mysqlcluster_types.go index 4ffcf4259..0cdb046a1 100644 --- a/pkg/apis/mysql/v1alpha1/mysqlcluster_types.go +++ b/pkg/apis/mysql/v1alpha1/mysqlcluster_types.go @@ -126,6 +126,13 @@ type MysqlClusterSpec struct { // +optional ReadOnly bool `json:"readOnly,omitempty"` + // Makes the operator ignore the ReadOnly setting completely. This is an advanced setting that implies you take + // responsibility for managing the R/W status of your instances. You will need to make sure Orchestrator is configured + // to set promoted master instances to writable (for example, by setting ApplyMySQLPromotionAfterMasterFailover to + // true in your Orchestrator configuration). + // +optional + IgnoreReadOnly bool `json:"ignoreReadOnly,omitempty"` + // Set a custom offset for Server IDs. ServerID for each node will be the index of the statefulset, plus offset // +optional ServerIDOffset *int `json:"serverIDOffset,omitempty"` diff --git a/pkg/controller/orchestrator/orchestrator_reconcile.go b/pkg/controller/orchestrator/orchestrator_reconcile.go index 3c8e8b9d6..0676ff7e3 100644 --- a/pkg/controller/orchestrator/orchestrator_reconcile.go +++ b/pkg/controller/orchestrator/orchestrator_reconcile.go @@ -502,6 +502,14 @@ func (ou *orcUpdater) setReadOnlyNode(inst orc.Instance) error { // nolint: gocyclo func (ou *orcUpdater) markReadOnlyNodesInOrc(insts InstancesSet, master *orc.Instance) { + // If the user has set IgnoreReadOnly option to true, we will not interfere with Orchestrator's control of RW status + if ou.cluster.Spec.IgnoreReadOnly { + if ou.cluster.Spec.ReadOnly { + log.Info("IgnoreReadOnly=true takes precedence over ReadOnly=true, will not change RW status of any instances in Orchestrator", + "IgnoreReadOnly", ou.cluster.Spec.IgnoreReadOnly, "ReadOnly", ou.cluster.Spec.ReadOnly) + } + return + } // If there is an in-progress failover, we will not interfere with readable/writable status on this iteration. fip := ou.cluster.GetClusterCondition(api.ClusterConditionFailoverInProgress) if fip != nil && fip.Status == core.ConditionTrue { diff --git a/pkg/controller/orchestrator/orchestrator_reconcile_test.go b/pkg/controller/orchestrator/orchestrator_reconcile_test.go index f67e91d4c..70750ec97 100644 --- a/pkg/controller/orchestrator/orchestrator_reconcile_test.go +++ b/pkg/controller/orchestrator/orchestrator_reconcile_test.go @@ -467,6 +467,23 @@ var _ = Describe("Orchestrator reconciler", func() { Expect(node1.ReadOnly).To(Equal(true)) }) + It("should not change read/write status when ReadOnly is Ignored", func() { + // Ignore the ReadOnly management; we will never tell Orchestrator what to do in this case + cluster.Spec.IgnoreReadOnly = true + + insts, _ := orcClient.Cluster(cluster.GetClusterAlias()) + node0 := InstancesSet(insts).GetInstance(cluster.GetPodHostname(0)) + + node0.ReadOnly = true + updater.markReadOnlyNodesInOrc(insts, node0) + Expect(node0.ReadOnly).To(Equal(true)) + + node0.ReadOnly = false + updater.markReadOnlyNodesInOrc(insts, node0) + Expect(node0.ReadOnly).To(Equal(false)) + + }) + It("should remove old nodes from orchestrator", func() { cluster.Spec.Replicas = &one cluster.Status.ReadyNodes = 1