Update user creation logic

src/charm.py

@@ -63,7 +63,7 @@
     ServiceInfo,
 )
 from pymongo.errors import PyMongoError
-from tenacity import before_log, retry, stop_after_attempt, wait_fixed
+from tenacity import Retrying, before_log, retry, stop_after_attempt, wait_fixed
 
 from config import Config
 from exceptions import AdminUserCreationError, MissingSecretError
@@ -76,6 +76,19 @@
 UNIT_SCOPE = Config.Relations.UNIT_SCOPE
 Scopes = Config.Relations.Scopes
 
+USER_CREATING_MAX_ATTEMPTS = 5
+USER_CREATION_COOLDOWN = 30
+
+
+def _before_sleep_user_creation(retry_state) -> None:
+    logger.error(
+        f"Attempt {retry_state.attempt_number} failed. {USER_CREATING_MAX_ATTEMPTS - retry_state.attempt_number} attempts left. Retrying after {USER_CREATION_COOLDOWN} seconds."
+    )
+
+
+def _user_creation_stop_condition(retry_state) -> bool:
+    return retry_state.attempt_number >= USER_CREATION_COOLDOWN
+
 
 class MongoDBCharm(CharmBase):
     """A Juju Charm to deploy MongoDB on Kubernetes."""
@@ -271,21 +284,6 @@ def db_initialised(self, value):
                 f"'db_initialised' must be a boolean value. Proivded: {value} is of type {type(value)}"
             )
 
-    @property
-    def users_initialized(self) -> bool:
-        """Check if MongoDB users are created."""
-        return "users_initialized" in self.app_peer_data
-
-    @users_initialized.setter
-    def users_initialized(self, value):
-        """Set the users_initialized flag."""
-        if isinstance(value, bool):
-            self.app_peer_data["users_initialized"] = str(value)
-        else:
-            raise ValueError(
-                f"'users_initialized' must be a boolean value. Proivded: {value} is of type {type(value)}"
-            )
-
     # END: properties
 
     # BEGIN: generic helper methods
@@ -402,7 +400,6 @@ def _on_start(self, event) -> None:
             return
 
         self._initialise_replica_set(event)
-        self._initialise_users(event)
 
         # mongod is now active
         self.unit.status = ActiveStatus()
@@ -606,48 +603,6 @@ def _on_secret_changed(self, event):
     # END: actions
 
     # BEGIN: user management
-    @retry(
-        stop=stop_after_attempt(3),
-        wait=wait_fixed(5),
-        reraise=True,
-        before=before_log(logger, logging.DEBUG),
-    )
-    def _initialise_users(self, event: StartEvent) -> None:
-        """Create users.
-
-        User creation can only be completed after the replica set has
-        been initialised which requires some time.
-        In race conditions this can lead to failure to initialise users.
-        To prevent these race conditions from breaking the code, retry on failure.
-        """
-        if not self.db_initialised:
-            return
-
-        if self.users_initialized:
-            return
-
-        # only leader should create users
-        if not self.unit.is_leader():
-            return
-
-        logger.info("User initialization")
-
-        try:
-            self._init_operator_user()
-            self._init_backup_user()
-            self._init_monitor_user()
-            logger.info("Reconcile relations")
-            self.client_relations.oversee_users(None, event)
-            self.users_initialized = True
-        except ExecError as e:
-            logger.error("Deferring on_start: exit code: %i, stderr: %s", e.exit_code, e.stderr)
-            event.defer()
-            return
-        except PyMongoError as e:
-            logger.error("Deferring on_start since: error=%r", e)
-            event.defer()
-            return
-
     @retry(
         stop=stop_after_attempt(3),
         wait=wait_fixed(5),
@@ -815,6 +770,26 @@ def _initialise_replica_set(self, event: StartEvent) -> None:
             try:
                 logger.info("Replica Set initialization")
                 direct_mongo.init_replset()
+
+                # Check replica set status before creating users
+                while not direct_mongo.client.admin.command("hello")["isWritablePrimary"]:
+                    time.sleep(10)
+                logger.info("User initialization")
+
+                for attempt in Retrying(
+                    stop=_user_creation_stop_condition,
+                    wait=wait_fixed(USER_CREATION_COOLDOWN),
+                    reraise=True,
+                    before_sleep=_before_sleep_user_creation,
+                ):
+                    with attempt:
+                        logger.error(
+                            "Initializing users. Attempt #%s", attempt.retry_state.attempt_number
+                        )
+                        self._init_operator_user()
+                        self._init_monitor_user()
+                        self._init_backup_user()
+                        self.client_relations.oversee_users(None, event)
             except ExecError as e:
                 logger.error(
                     "Deferring on_start: exit code: %i, stderr: %s", e.exit_code, e.stderr

tests/unit/test_charm.py

@@ -295,7 +295,6 @@ def test_start_already_initialised(self, connection, init_user, provider, defer)
         self.harness.charm.unit.get_container = mock_container
 
         self.harness.charm.app_peer_data["db_initialised"] = "True"
-        self.harness.charm.app_peer_data["users_initialized"] = "True"
 
         self.harness.charm.on.start.emit()
 
@@ -395,7 +394,7 @@ def test_start_mongod_error_initalising_user(self, connection, init_user, provid
         defer.assert_called()
 
         # verify app data
-        self.assertEqual("users_initialized" in self.harness.charm.app_peer_data, False)
+        self.assertEqual("db_initialised" in self.harness.charm.app_peer_data, False)
 
     @patch("ops.framework.EventBase.defer")
     @patch("charm.MongoDBProvider")
@@ -424,7 +423,7 @@ def test_start_mongod_error_overseeing_users(self, connection, init_user, provid
             defer.assert_called()
 
             # verify app data
-            self.assertEqual("users_initialized" in self.harness.charm.app_peer_data, False)
+            self.assertEqual("db_initialised" in self.harness.charm.app_peer_data, False)
 
     @patch("ops.framework.EventBase.defer")
     @patch("charm.MongoDBConnection")