Skip to content

Commit

Permalink
Merge pull request #745 from microbiomedata/672-migrations-update-not…
Browse files Browse the repository at this point in the history
…ebook-to-omit-mongo-credentials-from-commands-when-authentication-disabled

Migrations: Update notebook to omit Mongo credentials from commands when authentication is disabled
  • Loading branch information
eecavanna authored Oct 31, 2024
2 parents 9ec7abb + 75274d1 commit 14d8ded
Show file tree
Hide file tree
Showing 3 changed files with 93 additions and 53 deletions.
26 changes: 26 additions & 0 deletions demo/metadata_migration/notebooks/helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,32 @@
class Config:
"""Wrapper class for configuration values related to database migration."""

@staticmethod
def make_mongo_cli_base_options(
mongo_host: str,
mongo_port: str,
mongo_username: Optional[str],
mongo_password: Optional[str],
) -> str:
r"""Compile common Mongo CLI options into a reusable string."""

# Always include the CLI options that specify the server host and port.
base_options: List[str] = [
f"--host='{mongo_host}'",
f"--port='{mongo_port}'",
]

# If the server uses authentication, include authentication-related CLI options.
server_uses_authentication: bool = mongo_username not in ["", None]
if server_uses_authentication:
base_options.extend([
f"--username='{mongo_username}'",
f"--password='{mongo_password}'",
f"--authenticationDatabase='admin'",
])

return " ".join(base_options)

def parse_and_validate_notebook_config_file(
self, notebook_config_file_path: str
) -> Dict[str, str]:
Expand Down
86 changes: 33 additions & 53 deletions demo/metadata_migration/notebooks/migrate_10_9_1_to_11_0_0.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,20 @@
"mongorestore = cfg.mongorestore_path\n",
"mongosh = cfg.mongosh_path\n",
"\n",
"# Make the base CLI options for Mongo shell commands.\n",
"origin_mongo_cli_base_options = Config.make_mongo_cli_base_options(\n",
" mongo_host=cfg.origin_mongo_host, \n",
" mongo_port=cfg.origin_mongo_port, \n",
" mongo_username=cfg.origin_mongo_username, \n",
" mongo_password=cfg.origin_mongo_password,\n",
")\n",
"transformer_mongo_cli_base_options = Config.make_mongo_cli_base_options(\n",
" mongo_host=cfg.transformer_mongo_host,\n",
" mongo_port=cfg.transformer_mongo_port, \n",
" mongo_username=cfg.transformer_mongo_username, \n",
" mongo_password=cfg.transformer_mongo_password,\n",
")\n",
"\n",
"# Perform a sanity test of the application paths.\n",
"!{mongodump} --version\n",
"!{mongorestore} --version\n",
Expand Down Expand Up @@ -261,11 +275,7 @@
"# Note: I run this command via Python's `subprocess` module instead of via an IPython magic `!` command\n",
"# because I expect to eventually use regular Python scripts—not Python notebooks—for migrations.\n",
"shell_command = f\"\"\"\n",
" {cfg.mongosh_path} \\\n",
" --host='{cfg.transformer_mongo_host}' \\\n",
" --port='{cfg.transformer_mongo_port}' \\\n",
" --username='{cfg.transformer_mongo_username}' \\\n",
" --password='{cfg.transformer_mongo_password}' \\\n",
" {cfg.mongosh_path} {transformer_mongo_cli_base_options} \\\n",
" --quiet \\\n",
" --eval 'use nmdc' \\\n",
" --eval 'db.dropDatabase()'\n",
Expand Down Expand Up @@ -365,11 +375,7 @@
"metadata": {},
"source": [
"shell_command = f\"\"\"\n",
" {cfg.mongosh_path} \\\n",
" --host='{cfg.origin_mongo_host}' \\\n",
" --port='{cfg.origin_mongo_port}' \\\n",
" --username='{cfg.origin_mongo_username}' \\\n",
" --password='{cfg.origin_mongo_password}' \\\n",
" {cfg.mongosh_path} {origin_mongo_cli_base_options} \\\n",
" --quiet \\\n",
" --file='mongosh-scripts/revoke-privileges.mongo.js'\n",
"\"\"\"\n",
Expand Down Expand Up @@ -421,12 +427,7 @@
"source": [
"# Dump all collections from the \"origin\" database.\n",
"shell_command = f\"\"\"\n",
" {mongodump} \\\n",
" --host='{cfg.origin_mongo_host}' \\\n",
" --port='{cfg.origin_mongo_port}' \\\n",
" --username='{cfg.origin_mongo_username}' \\\n",
" --password='{cfg.origin_mongo_password}' \\\n",
" --authenticationDatabase='admin' \\\n",
" {mongodump} {origin_mongo_cli_base_options} \\\n",
" --db='nmdc' \\\n",
" --gzip \\\n",
" --out='{cfg.origin_dump_folder_path}'\n",
Expand Down Expand Up @@ -456,12 +457,7 @@
"source": [
"# Restore the dumped collections to the \"transformer\" MongoDB server.\n",
"shell_command = f\"\"\"\n",
" {mongorestore} \\\n",
" --host='{cfg.transformer_mongo_host}' \\\n",
" --port='{cfg.transformer_mongo_port}' \\\n",
" --username='{cfg.transformer_mongo_username}' \\\n",
" --password='{cfg.transformer_mongo_password}' \\\n",
" --authenticationDatabase='admin' \\\n",
" {mongorestore} {transformer_mongo_cli_base_options} \\\n",
" --gzip \\\n",
" --drop \\\n",
" --preserveUUID \\\n",
Expand Down Expand Up @@ -545,6 +541,7 @@
" num_documents_in_collection = collection.count_documents({})\n",
" print(f\"Validating collection {collection_name} ({num_documents_in_collection} documents)\", end=\"\\t\") # no newline\n",
"\n",
" num_documents_in_collection_validated = 0\n",
" for document in collection.find():\n",
" # Validate the transformed document.\n",
" #\n",
Expand All @@ -558,10 +555,19 @@
" # Note: `root_to_validate` is a dictionary having the shape: { \"some_collection_name\": [ some_document ] }\n",
" # Reference: https://docs.python.org/3/library/stdtypes.html#dict (see the \"type constructor\" section)\n",
" #\n",
" # TODO: Use `linkml.validator` — instead of `jsonschema` — to validate the document.\n",
" # Reference: https://linkml.io/linkml/data/validating-data.html\n",
" #\n",
" document_without_underscore_id_key = {key: value for key, value in document.items() if key != \"_id\"}\n",
" root_to_validate = dict([(collection_name, [document_without_underscore_id_key])])\n",
" nmdc_jsonschema_validator.validate(root_to_validate) # raises exception if invalid\n",
"\n",
" # Show a \"sign of life\" each time we validate an additional 10% of the documents in the collection.\n",
" # Note: The `//` operator performs \"floor division,\" returning the largest integer <= the real quotient.\n",
" num_documents_in_collection_validated += 1\n",
" if num_documents_in_collection_validated % (num_documents_in_collection // 10) == 0:\n",
" print(\".\", end=\"\") # no newline\n",
" \n",
" print(f\"Done\")"
],
"outputs": [],
Expand All @@ -586,12 +592,7 @@
"source": [
"# Dump the database from the \"transformer\" MongoDB server.\n",
"shell_command = f\"\"\"\n",
" {mongodump} \\\n",
" --host='{cfg.transformer_mongo_host}' \\\n",
" --port='{cfg.transformer_mongo_port}' \\\n",
" --username='{cfg.transformer_mongo_username}' \\\n",
" --password='{cfg.transformer_mongo_password}' \\\n",
" --authenticationDatabase='admin' \\\n",
" {mongodump} {transformer_mongo_cli_base_options} \\\n",
" --db='nmdc' \\\n",
" --gzip \\\n",
" --out='{cfg.transformer_dump_folder_path}'\n",
Expand Down Expand Up @@ -663,11 +664,7 @@
"metadata": {},
"source": [
"shell_command = f\"\"\"\n",
" {cfg.mongosh_path} \\\n",
" --host='{cfg.origin_mongo_host}' \\\n",
" --port='{cfg.origin_mongo_port}' \\\n",
" --username='{cfg.origin_mongo_username}' \\\n",
" --password='{cfg.origin_mongo_password}' \\\n",
" {cfg.mongosh_path} {origin_mongo_cli_base_options} \\\n",
" --eval 'use nmdc' \\\n",
" --eval 'db.dropDatabase()'\n",
"\"\"\"\n",
Expand Down Expand Up @@ -696,12 +693,7 @@
"source": [
"# Load the transformed collections into the origin server, replacing any same-named ones that are there.\n",
"shell_command = f\"\"\"\n",
" {mongorestore} \\\n",
" --host='{cfg.origin_mongo_host}' \\\n",
" --port='{cfg.origin_mongo_port}' \\\n",
" --username='{cfg.origin_mongo_username}' \\\n",
" --password='{cfg.origin_mongo_password}' \\\n",
" --authenticationDatabase='admin' \\\n",
" {mongorestore} {origin_mongo_cli_base_options} \\\n",
" --gzip \\\n",
" --verbose \\\n",
" --dir='{cfg.transformer_dump_folder_path}' \\\n",
Expand Down Expand Up @@ -761,11 +753,7 @@
"metadata": {},
"source": [
"shell_command = f\"\"\"\n",
" {cfg.mongosh_path} \\\n",
" --host='{cfg.origin_mongo_host}' \\\n",
" --port='{cfg.origin_mongo_port}' \\\n",
" --username='{cfg.origin_mongo_username}' \\\n",
" --password='{cfg.origin_mongo_password}' \\\n",
" {cfg.mongosh_path} {origin_mongo_cli_base_options} \\\n",
" --quiet \\\n",
" --file='mongosh-scripts/restore-privileges.mongo.js'\n",
"\"\"\"\n",
Expand All @@ -774,19 +762,11 @@
],
"outputs": [],
"execution_count": null
},
{
"cell_type": "code",
"id": "037db214-ea76-46bf-bb6a-bf1ff9b28a72",
"metadata": {},
"source": [],
"outputs": [],
"execution_count": null
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"display_name": ".venv",
"language": "python",
"name": "python3"
},
Expand Down
34 changes: 34 additions & 0 deletions demo/metadata_migration/notebooks/test_helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,40 @@ class TestConfig(unittest.TestCase):
Reference: https://docs.python.org/3/library/unittest.html#basic-example
"""

def test_make_mongo_cli_base_options(self):
# Case 1: Input includes username.
s = Config.make_mongo_cli_base_options(
mongo_host="thehost",
mongo_port="12345",
mongo_username="alice",
mongo_password="shhh",
)
assert s == f"""
--host='thehost' --port='12345' --username='alice' --password='shhh' --authenticationDatabase='admin'
""".strip()

# Case 2: Input username is empty string.
s = Config.make_mongo_cli_base_options(
mongo_host="thehost",
mongo_port="12345",
mongo_username="",
mongo_password="shhh",
)
assert s == f"""
--host='thehost' --port='12345'
""".strip()

# Case 2: Input username is `None`.
s = Config.make_mongo_cli_base_options(
mongo_host="thehost",
mongo_port="12345",
mongo_username=None,
mongo_password="shhh",
)
assert s == f"""
--host='thehost' --port='12345'
""".strip()

def test_init_method(self):
with (TempFile() as notebook_config_file,
TempFile() as origin_mongo_config_file,
Expand Down

0 comments on commit 14d8ded

Please sign in to comment.