From b08b68de3ac41057ea3be72f1535ea97eca2395e Mon Sep 17 00:00:00 2001 From: annaCPR Date: Mon, 14 Oct 2024 09:14:47 +0100 Subject: [PATCH] Feature/pdct 1538 bulk import make async (#231) * Add navigator-notify dependency and update python * Fix upgrade to python 3.10 issues * Add notification service * Make integration tests use the aws s3 mock * Rename * Fix test and change bcrypt version * Fix logging * Save local bulk import results to a dir that is ignored by git * Bump python in Dockerfile * Remove print --- .env.example | 7 +- .gitignore | 1 + Dockerfile | 2 +- app/api/api_v1/routers/ingest.py | 2 +- app/clients/aws/s3bucket.py | 9 +- app/service/ingest.py | 15 ++- app/service/notification.py | 16 +++ poetry.lock | 120 +++++++++++++----- pyproject.toml | 10 +- tests/integration_tests/ingest/test_ingest.py | 6 + tests/unit_tests/clients/aws/test_client.py | 6 +- .../service/ingest/test_ingest_service.py | 37 ++++-- .../notification/test_notification_service.py | 50 ++++++++ 13 files changed, 226 insertions(+), 55 deletions(-) create mode 100644 app/service/notification.py create mode 100644 tests/unit_tests/service/notification/test_notification_service.py diff --git a/.env.example b/.env.example index 319d2235..bd7cfffb 100644 --- a/.env.example +++ b/.env.example @@ -8,4 +8,9 @@ SECRET_KEY=secret_test_key API_HOST=http://navigator-admin-backend:8888 # AWS -INGEST_JSON_BUCKET=test_bucket +BULK_IMPORT_BUCKET=skip +AWS_ACCESS_KEY_ID=test +_AWS_SECRET_ACCESS_KEY=test + +# Slack Notifications +SLACK_WEBHOOK_URL=skip diff --git a/.gitignore b/.gitignore index 53cf8311..246e6ec9 100644 --- a/.gitignore +++ b/.gitignore @@ -53,6 +53,7 @@ coverage.xml *.py,cover .hypothesis/ .pytest_cache/ +bulk_import_results/ # Translations *.mo diff --git a/Dockerfile b/Dockerfile index a3c07fa8..4dda3c4e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9-slim +FROM python:3.10-slim WORKDIR /usr/src ENV PYTHONPATH=/usr/src diff --git a/app/api/api_v1/routers/ingest.py b/app/api/api_v1/routers/ingest.py index 6e3e8d7e..2e9f724c 100644 --- a/app/api/api_v1/routers/ingest.py +++ b/app/api/api_v1/routers/ingest.py @@ -251,7 +251,7 @@ async def ingest( :param UploadFile new_data: file containing json representation of data to ingest. :return Json: json representation of the data to ingest. """ - _LOGGER.info(f"Starting bulk import for corpus: {corpus_import_id}") + _LOGGER.info(f"Received bulk import request for corpus: {corpus_import_id}") try: content = await new_data.read() diff --git a/app/clients/aws/s3bucket.py b/app/clients/aws/s3bucket.py index a734678b..1a111c02 100644 --- a/app/clients/aws/s3bucket.py +++ b/app/clients/aws/s3bucket.py @@ -113,13 +113,16 @@ def upload_ingest_json_to_s3( :param str corpus_import_id: The import_id of the corpus the ingest data belongs to. :param dict[str, Any] json_data: The ingest json data to be uploaded to S3. """ - ingest_upload_bucket = os.environ["INGEST_JSON_BUCKET"] + _LOGGER.info(os.getenv("BULK_IMPORT_BUCKET", "Not there")) + ingest_upload_bucket = os.environ["BULK_IMPORT_BUCKET"] current_timestamp = datetime.now().strftime("%m-%d-%YT%H:%M:%S") + filename = f"{ingest_id}-{corpus_import_id}-{current_timestamp}.json" if ingest_upload_bucket == "skip": - with open(filename, "w") as f: - json.dump(data, f, indent=4) + os.makedirs("bulk_import_results", exist_ok=True) + with open(f"bulk_import_results/{filename}", "w") as file: + json.dump(data, file, indent=4) return s3_client = get_s3_client() diff --git a/app/service/ingest.py b/app/service/ingest.py index 5a611958..739f8bcb 100644 --- a/app/service/ingest.py +++ b/app/service/ingest.py @@ -23,6 +23,7 @@ import app.repository.family as family_repository import app.service.corpus as corpus import app.service.geography as geography +import app.service.notification as notification_service import app.service.validation as validation from app.clients.aws.s3bucket import upload_ingest_json_to_s3 from app.model.ingest import ( @@ -224,6 +225,10 @@ def import_data(data: dict[str, Any], corpus_import_id: str) -> None: :raises RepositoryError: raised on a database error. :raises ValidationError: raised should the data be invalid. """ + notification_service.send_notification( + f"🚀 Bulk import for corpus: {corpus_import_id} has started." + ) + ingest_uuid = uuid4() upload_ingest_json_to_s3(f"{ingest_uuid}-request", corpus_import_id, data) @@ -254,16 +259,18 @@ def import_data(data: dict[str, Any], corpus_import_id: str) -> None: _LOGGER.info("Saving events") result["events"] = save_events(event_data, corpus_import_id, db) - _LOGGER.info( - f"Bulk import for corpus: {corpus_import_id} successfully completed" - ) - upload_ingest_json_to_s3(f"{ingest_uuid}-result", corpus_import_id, result) + notification_service.send_notification( + f"🎉 Bulk import for corpus: {corpus_import_id} successfully completed." + ) except Exception as e: _LOGGER.error( f"Rolling back transaction due to the following error: {e}", exc_info=True ) db.rollback() + notification_service.send_notification( + f"💥 Bulk import for corpus: {corpus_import_id} has failed." + ) finally: db.commit() diff --git a/app/service/notification.py b/app/service/notification.py new file mode 100644 index 00000000..6af05dc5 --- /dev/null +++ b/app/service/notification.py @@ -0,0 +1,16 @@ +import logging +import os + +from notify.slack import slack_message + +_LOGGER = logging.getLogger(__name__) +_LOGGER.setLevel(logging.DEBUG) + + +def send_notification(notification: str): + try: + _LOGGER.info(notification) + if os.environ["SLACK_WEBHOOK_URL"] != "skip": + slack_message(notification) + except Exception as e: + _LOGGER.error(f"Error sending notification caused by: {e}") diff --git a/poetry.lock b/poetry.lock index 5a860330..6a486096 100644 --- a/poetry.lock +++ b/poetry.lock @@ -88,6 +88,40 @@ files = [ [package.dependencies] typing-extensions = {version = ">=4.0.0", markers = "python_version < \"3.11\""} +[[package]] +name = "bcrypt" +version = "4.0.1" +description = "Modern password hashing for your software and your servers" +optional = false +python-versions = ">=3.6" +files = [ + {file = "bcrypt-4.0.1-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:b1023030aec778185a6c16cf70f359cbb6e0c289fd564a7cfa29e727a1c38f8f"}, + {file = "bcrypt-4.0.1-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:08d2947c490093a11416df18043c27abe3921558d2c03e2076ccb28a116cb6d0"}, + {file = "bcrypt-4.0.1-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0eaa47d4661c326bfc9d08d16debbc4edf78778e6aaba29c1bc7ce67214d4410"}, + {file = "bcrypt-4.0.1-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ae88eca3024bb34bb3430f964beab71226e761f51b912de5133470b649d82344"}, + {file = "bcrypt-4.0.1-cp36-abi3-manylinux_2_24_x86_64.whl", hash = "sha256:a522427293d77e1c29e303fc282e2d71864579527a04ddcfda6d4f8396c6c36a"}, + {file = "bcrypt-4.0.1-cp36-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:fbdaec13c5105f0c4e5c52614d04f0bca5f5af007910daa8b6b12095edaa67b3"}, + {file = "bcrypt-4.0.1-cp36-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:ca3204d00d3cb2dfed07f2d74a25f12fc12f73e606fcaa6975d1f7ae69cacbb2"}, + {file = "bcrypt-4.0.1-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:089098effa1bc35dc055366740a067a2fc76987e8ec75349eb9484061c54f535"}, + {file = "bcrypt-4.0.1-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:e9a51bbfe7e9802b5f3508687758b564069ba937748ad7b9e890086290d2f79e"}, + {file = "bcrypt-4.0.1-cp36-abi3-win32.whl", hash = "sha256:2caffdae059e06ac23fce178d31b4a702f2a3264c20bfb5ff541b338194d8fab"}, + {file = "bcrypt-4.0.1-cp36-abi3-win_amd64.whl", hash = "sha256:8a68f4341daf7522fe8d73874de8906f3a339048ba406be6ddc1b3ccb16fc0d9"}, + {file = "bcrypt-4.0.1-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf4fa8b2ca74381bb5442c089350f09a3f17797829d958fad058d6e44d9eb83c"}, + {file = "bcrypt-4.0.1-pp37-pypy37_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:67a97e1c405b24f19d08890e7ae0c4f7ce1e56a712a016746c8b2d7732d65d4b"}, + {file = "bcrypt-4.0.1-pp37-pypy37_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:b3b85202d95dd568efcb35b53936c5e3b3600c7cdcc6115ba461df3a8e89f38d"}, + {file = "bcrypt-4.0.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cbb03eec97496166b704ed663a53680ab57c5084b2fc98ef23291987b525cb7d"}, + {file = "bcrypt-4.0.1-pp38-pypy38_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:5ad4d32a28b80c5fa6671ccfb43676e8c1cc232887759d1cd7b6f56ea4355215"}, + {file = "bcrypt-4.0.1-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:b57adba8a1444faf784394de3436233728a1ecaeb6e07e8c22c8848f179b893c"}, + {file = "bcrypt-4.0.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:705b2cea8a9ed3d55b4491887ceadb0106acf7c6387699fca771af56b1cdeeda"}, + {file = "bcrypt-4.0.1-pp39-pypy39_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:2b3ac11cf45161628f1f3733263e63194f22664bf4d0c0f3ab34099c02134665"}, + {file = "bcrypt-4.0.1-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:3100851841186c25f127731b9fa11909ab7b1df6fc4b9f8353f4f1fd952fbf71"}, + {file = "bcrypt-4.0.1.tar.gz", hash = "sha256:27d375903ac8261cfe4047f6709d16f7d18d39b1ec92aaf72af989552a650ebd"}, +] + +[package.extras] +tests = ["pytest (>=3.2.1,!=3.3.0)"] +typecheck = ["mypy"] + [[package]] name = "black" version = "24.10.0" @@ -136,17 +170,17 @@ uvloop = ["uvloop (>=0.15.2)"] [[package]] name = "boto3" -version = "1.35.36" +version = "1.35.37" description = "The AWS SDK for Python" optional = false python-versions = ">=3.8" files = [ - {file = "boto3-1.35.36-py3-none-any.whl", hash = "sha256:33735b9449cd2ef176531ba2cb2265c904a91244440b0e161a17da9d24a1e6d1"}, - {file = "boto3-1.35.36.tar.gz", hash = "sha256:586524b623e4fbbebe28b604c6205eb12f263cc4746bccb011562d07e217a4cb"}, + {file = "boto3-1.35.37-py3-none-any.whl", hash = "sha256:385ca77bf8ea4ab2d97f6e2435bdb29f77d9301e2f7ac796c2f465753c2adf3c"}, + {file = "boto3-1.35.37.tar.gz", hash = "sha256:470d981583885859fed2fd1c185eeb01cc03e60272d499bafe41b12625b158c8"}, ] [package.dependencies] -botocore = ">=1.35.36,<1.36.0" +botocore = ">=1.35.37,<1.36.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.10.0,<0.11.0" @@ -155,22 +189,19 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.35.36" +version = "1.35.37" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">=3.8" files = [ - {file = "botocore-1.35.36-py3-none-any.whl", hash = "sha256:64241c778bf2dc863d93abab159e14024d97a926a5715056ef6411418cb9ead3"}, - {file = "botocore-1.35.36.tar.gz", hash = "sha256:354ec1b766f0029b5d6ff0c45d1a0f9e5007b7d2f3ec89bcdd755b208c5bc797"}, + {file = "botocore-1.35.37-py3-none-any.whl", hash = "sha256:64f965d4ba7adb8d79ce044c3aef7356e05dd74753cf7e9115b80f477845d920"}, + {file = "botocore-1.35.37.tar.gz", hash = "sha256:b2b4d29bafd95b698344f2f0577bb67064adbf1735d8a0e3c7473daa59c23ba6"}, ] [package.dependencies] jmespath = ">=0.7.1,<2.0.0" python-dateutil = ">=2.1,<3.0.0" -urllib3 = [ - {version = ">=1.25.4,<1.27", markers = "python_version < \"3.10\""}, - {version = ">=1.25.4,<2.2.0 || >2.2.0,<3", markers = "python_version >= \"3.10\""}, -] +urllib3 = {version = ">=1.25.4,<2.2.0 || >2.2.0,<3", markers = "python_version >= \"3.10\""} [package.extras] crt = ["awscrt (==0.22.0)"] @@ -578,13 +609,13 @@ profile = ["gprof2dot (>=2022.7.29)"] [[package]] name = "distlib" -version = "0.3.8" +version = "0.3.9" description = "Distribution utilities" optional = false python-versions = "*" files = [ - {file = "distlib-0.3.8-py2.py3-none-any.whl", hash = "sha256:034db59a0b96f8ca18035f36290806a9a6e6bd9d1ff91e45a7f172eb17e51784"}, - {file = "distlib-0.3.8.tar.gz", hash = "sha256:1530ea13e350031b6312d8580ddb6b27a104275a31106523b8f123787f494f64"}, + {file = "distlib-0.3.9-py2.py3-none-any.whl", hash = "sha256:47f8c22fd27c27e25a65601af709b38e4f0a45ea4fc2e710f65755fa8caaaf87"}, + {file = "distlib-0.3.9.tar.gz", hash = "sha256:a60f20dea646b8a33f3e7772f74dc0b2d0772d2837ee1342a00645c81edf9403"}, ] [[package]] @@ -1409,6 +1440,26 @@ files = [ {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, ] +[[package]] +name = "navigator-notify" +version = "0.1.0" +description = "" +optional = false +python-versions = "^3.10" +files = [] +develop = false + +[package.dependencies] +click = "^8.1.7" +pre-commit = "^3.8.0" +slack-sdk = "^3.31.0" + +[package.source] +type = "git" +url = "https://github.com/climatepolicyradar/navigator-notify.git" +reference = "v0.0.2-beta" +resolved_reference = "875e548e0b9a9c1dff21389a5017bde1eae81a71" + [[package]] name = "nodeenv" version = "1.9.1" @@ -1557,7 +1608,6 @@ cleo = ">=2.1.0,<3.0.0" crashtest = ">=0.4.1,<0.5.0" dulwich = ">=0.21.2,<0.22.0" fastjsonschema = ">=2.18.0,<3.0.0" -importlib-metadata = {version = ">=4.4", markers = "python_version < \"3.10\""} installer = ">=0.7.0,<0.8.0" keyring = ">=24.0.0,<25.0.0" packaging = ">=20.5" @@ -1604,13 +1654,13 @@ poetry-core = ">=1.7.0,<2.0.0" [[package]] name = "pre-commit" -version = "2.21.0" +version = "3.8.0" description = "A framework for managing and maintaining multi-language pre-commit hooks." optional = false -python-versions = ">=3.7" +python-versions = ">=3.9" files = [ - {file = "pre_commit-2.21.0-py2.py3-none-any.whl", hash = "sha256:e2f91727039fc39a92f58a588a25b87f936de6567eed4f0e673e0507edc75bad"}, - {file = "pre_commit-2.21.0.tar.gz", hash = "sha256:31ef31af7e474a8d8995027fefdfcf509b5c913ff31f2015b4ec4beb26a6f658"}, + {file = "pre_commit-3.8.0-py2.py3-none-any.whl", hash = "sha256:9a90a53bf82fdd8778d58085faf8d83df56e40dfe18f45b19446e26bf1b3a63f"}, + {file = "pre_commit-3.8.0.tar.gz", hash = "sha256:8bb6494d4a20423842e198980c9ecf9f96607a07ea29549e180eef9ae80fe7af"}, ] [package.dependencies] @@ -1738,8 +1788,8 @@ files = [ annotated-types = ">=0.6.0" pydantic-core = "2.23.4" typing-extensions = [ - {version = ">=4.6.1", markers = "python_version < \"3.13\""}, {version = ">=4.12.2", markers = "python_version >= \"3.13\""}, + {version = ">=4.6.1", markers = "python_version < \"3.13\""}, ] [package.extras] @@ -1879,8 +1929,8 @@ files = [ astroid = ">=3.3.4,<=3.4.0-dev0" colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""} dill = [ - {version = ">=0.2", markers = "python_version < \"3.11\""}, {version = ">=0.3.7", markers = "python_version >= \"3.12\""}, + {version = ">=0.2", markers = "python_version < \"3.11\""}, {version = ">=0.3.6", markers = "python_version >= \"3.11\" and python_version < \"3.12\""}, ] isort = ">=4.2.5,<5.13.0 || >5.13.0,<6" @@ -1888,7 +1938,6 @@ mccabe = ">=0.6,<0.8" platformdirs = ">=2.2.0" tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} tomlkit = ">=0.10.1" -typing-extensions = {version = ">=3.10.0", markers = "python_version < \"3.10\""} [package.extras] spelling = ["pyenchant (>=3.2,<4.0)"] @@ -2362,6 +2411,20 @@ files = [ {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] +[[package]] +name = "slack-sdk" +version = "3.33.1" +description = "The Slack API Platform SDK for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "slack_sdk-3.33.1-py2.py3-none-any.whl", hash = "sha256:ef93beec3ce9c8f64da02fd487598a05ec4bc9c92ceed58f122dbe632691cbe2"}, + {file = "slack_sdk-3.33.1.tar.gz", hash = "sha256:e328bb661d95db5f66b993b1d64288ac7c72201a745b4c7cf8848dafb7b74e40"}, +] + +[package.extras] +optional = ["SQLAlchemy (>=1.4,<3)", "aiodns (>1.0)", "aiohttp (>=3.7.3,<4)", "boto3 (<=2)", "websocket-client (>=1,<2)", "websockets (>=9.1,<14)"] + [[package]] name = "sniffio" version = "1.3.1" @@ -2491,7 +2554,6 @@ files = [ [package.dependencies] anyio = ">=3.4.0,<5" -typing-extensions = {version = ">=3.10.0", markers = "python_version < \"3.10\""} [package.extras] full = ["httpx (>=0.22.0)", "itsdangerous", "jinja2", "python-multipart", "pyyaml"] @@ -2961,13 +3023,13 @@ cffi = ">=1.0" [[package]] name = "xmltodict" -version = "0.14.0" +version = "0.14.1" description = "Makes working with XML feel like you are working with JSON" optional = false -python-versions = ">=3.4" +python-versions = ">=3.6" files = [ - {file = "xmltodict-0.14.0-py2.py3-none-any.whl", hash = "sha256:6dd20b8de8d0eb84d175ec706cc17b53df236615b0980de33537736319e5ee85"}, - {file = "xmltodict-0.14.0.tar.gz", hash = "sha256:8b39b25b564fd466be566c9e8a869cc4b5083c2fec7f98665f47bf0853f6cc77"}, + {file = "xmltodict-0.14.1-py2.py3-none-any.whl", hash = "sha256:3ef4a7b71c08f19047fcbea572e1d7f4207ab269da1565b5d40e9823d3894e63"}, + {file = "xmltodict-0.14.1.tar.gz", hash = "sha256:338c8431e4fc554517651972d62f06958718f6262b04316917008e8fd677a6b0"}, ] [[package]] @@ -2991,5 +3053,5 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" -python-versions = "^3.9" -content-hash = "2e261a48d27dc0ed25db78c0778446f59359948cf76a588ab8e04879e0e6a697" +python-versions = "^3.10" +content-hash = "ab42faf3a5801424a9ef88e0dc61b879497283eee96403c397e1f578ffc94648" diff --git a/pyproject.toml b/pyproject.toml index 8097edde..e36bf259 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,12 +1,12 @@ [tool.poetry] name = "admin_backend" -version = "2.17.0" +version = "2.17.1" description = "" authors = ["CPR-dev-team "] packages = [{ include = "app" }, { include = "tests" }] [tool.poetry.dependencies] -python = "^3.9" +python = "^3.10" fastapi = "^0.103.0" fastapi-health = "^0.4.0" fastapi-pagination = "^0.12.9" @@ -30,9 +30,11 @@ moto = "^4.2.2" types-sqlalchemy = "^1.4.53.38" urllib3 = "^1.26.17" db-client = { git = "https://github.com/climatepolicyradar/navigator-db-client.git", tag = "v3.8.18" } +navigator-notify = { git = "https://github.com/climatepolicyradar/navigator-notify.git", tag = "v0.0.2-beta" } +bcrypt = "4.0.1" [tool.poetry.dev-dependencies] -pre-commit = "^2.17.0" +pre-commit = "^3.8.0" python-dotenv = "^0.19.2" pytest = "^8.3.2" pytest-dotenv = "^0.5.2" @@ -67,5 +69,5 @@ exclude = "^/alembic/versions/" [tool.pyright] include = ["app", "tests"] exclude = ["**/__pycache__"] -pythonVersion = "3.9" +pythonVersion = "3.10" venv = "admin-backend" diff --git a/tests/integration_tests/ingest/test_ingest.py b/tests/integration_tests/ingest/test_ingest.py index 84310482..43e90653 100644 --- a/tests/integration_tests/ingest/test_ingest.py +++ b/tests/integration_tests/ingest/test_ingest.py @@ -2,6 +2,7 @@ import json import logging import os +from unittest.mock import patch from db_client.models.dfce import FamilyEvent from db_client.models.dfce.collection import Collection @@ -14,6 +15,7 @@ from tests.integration_tests.setup_db import setup_db +@patch.dict(os.environ, {"BULK_IMPORT_BUCKET": "test_bucket"}) def test_ingest_when_ok( data_db: Session, client: TestClient, user_header_token, basic_s3_client ): @@ -84,6 +86,7 @@ def test_ingest_when_ok( assert ev.family_import_id in expected_family_import_ids +@patch.dict(os.environ, {"BULK_IMPORT_BUCKET": "test_bucket"}) def test_import_data_rollback( caplog, data_db: Session, @@ -122,6 +125,7 @@ def test_import_data_rollback( assert actual_collection is None +@patch.dict(os.environ, {"BULK_IMPORT_BUCKET": "test_bucket"}) def test_ingest_idempotency( caplog, data_db: Session, client: TestClient, user_header_token, basic_s3_client ): @@ -234,6 +238,7 @@ def test_ingest_idempotency( ) +@patch.dict(os.environ, {"BULK_IMPORT_BUCKET": "test_bucket"}) def test_ingest_when_corpus_import_id_invalid( caplog, data_db: Session, client: TestClient, user_header_token, basic_s3_client ): @@ -261,6 +266,7 @@ def test_ingest_when_corpus_import_id_invalid( assert f"No organisation associated with corpus {invalid_corpus}" in caplog.text +@patch.dict(os.environ, {"BULK_IMPORT_BUCKET": "test_bucket"}) def test_ingest_events_when_event_type_invalid( caplog, data_db: Session, client: TestClient, user_header_token, basic_s3_client ): diff --git a/tests/unit_tests/clients/aws/test_client.py b/tests/unit_tests/clients/aws/test_client.py index 5dbf4b65..cb763861 100644 --- a/tests/unit_tests/clients/aws/test_client.py +++ b/tests/unit_tests/clients/aws/test_client.py @@ -40,7 +40,7 @@ def test_upload_json_to_s3_when_error(basic_s3_client): assert e.value.response["Error"]["Code"] == "NoSuchBucket" -@patch.dict(os.environ, {"INGEST_JSON_BUCKET": "test_bucket"}) +@patch.dict(os.environ, {"BULK_IMPORT_BUCKET": "test_bucket"}) def test_upload_ingest_json_to_s3_success(basic_s3_client): json_data = {"test": "test"} upload_ingest_json_to_s3("1111-1111", "test_corpus_id", json_data) @@ -58,7 +58,7 @@ def test_upload_ingest_json_to_s3_success(basic_s3_client): assert json.loads(body) == json_data -@patch.dict(os.environ, {"INGEST_JSON_BUCKET": "skip"}) +@patch.dict(os.environ, {"BULK_IMPORT_BUCKET": "skip"}) def test_do_not_save_ingest_json_to_s3_when_in_local_development(basic_s3_client): json_data = {"test": "test"} @@ -69,4 +69,4 @@ def test_do_not_save_ingest_json_to_s3_when_in_local_development(basic_s3_client ) assert "Contents" not in find_response - cleanup_local_files("1111-1111-test_corpus_id*") + cleanup_local_files("bulk_import_results/1111-1111-test_corpus_id*") diff --git a/tests/unit_tests/service/ingest/test_ingest_service.py b/tests/unit_tests/service/ingest/test_ingest_service.py index 7c086c2c..1d6599e2 100644 --- a/tests/unit_tests/service/ingest/test_ingest_service.py +++ b/tests/unit_tests/service/ingest/test_ingest_service.py @@ -10,7 +10,7 @@ @patch("app.service.ingest._exists_in_db", Mock(return_value=False)) -@patch.dict(os.environ, {"INGEST_JSON_BUCKET": "test_bucket"}) +@patch.dict(os.environ, {"BULK_IMPORT_BUCKET": "test_bucket"}) def test_ingest_when_ok( basic_s3_client, corpus_repo_mock, @@ -71,9 +71,14 @@ def test_ingest_when_ok( } try: - with patch( - "app.service.ingest.uuid4", return_value="1111-1111" - ) as mock_uuid_generator: + with ( + patch( + "app.service.ingest.uuid4", return_value="1111-1111" + ) as mock_uuid_generator, + patch( + "app.service.ingest.notification_service.send_notification" + ) as mock_notification_service, + ): ingest_service.import_data(test_data, "test_corpus_id") response = basic_s3_client.list_objects_v2( @@ -81,7 +86,11 @@ def test_ingest_when_ok( ) mock_uuid_generator.assert_called_once() - assert "Contents" in response + assert 2 == mock_notification_service.call_count + mock_notification_service.assert_called_with( + "🎉 Bulk import for corpus: test_corpus_id successfully completed." + ) + objects = response["Contents"] assert len(objects) == 1 @@ -93,7 +102,7 @@ def test_ingest_when_ok( assert False, f"import_data in ingest service raised an exception: {e}" -@patch.dict(os.environ, {"INGEST_JSON_BUCKET": "test_bucket"}) +@patch.dict(os.environ, {"BULK_IMPORT_BUCKET": "test_bucket"}) def test_import_data_when_data_invalid(caplog, basic_s3_client): test_data = { "collections": [ @@ -112,7 +121,7 @@ def test_import_data_when_data_invalid(caplog, basic_s3_client): @patch("app.service.ingest._exists_in_db", Mock(return_value=False)) -@patch.dict(os.environ, {"INGEST_JSON_BUCKET": "test_bucket"}) +@patch.dict(os.environ, {"BULK_IMPORT_BUCKET": "test_bucket"}) def test_ingest_when_db_error( caplog, basic_s3_client, corpus_repo_mock, collection_repo_mock ): @@ -128,15 +137,25 @@ def test_ingest_when_db_error( ] } - with caplog.at_level(logging.ERROR): + with ( + caplog.at_level(logging.ERROR), + patch( + "app.service.ingest.notification_service.send_notification" + ) as mock_notification_service, + ): ingest_service.import_data(test_data, "test") + + assert 2 == mock_notification_service.call_count + mock_notification_service.assert_called_with( + "💥 Bulk import for corpus: test has failed." + ) assert ( "Rolling back transaction due to the following error: bad collection repo" in caplog.text ) -@patch.dict(os.environ, {"INGEST_JSON_BUCKET": "test_bucket"}) +@patch.dict(os.environ, {"BULK_IMPORT_BUCKET": "test_bucket"}) def test_request_json_saved_to_s3_on_ingest(basic_s3_client): bucket_name = "test_bucket" json_data = {"key": "value"} diff --git a/tests/unit_tests/service/notification/test_notification_service.py b/tests/unit_tests/service/notification/test_notification_service.py new file mode 100644 index 00000000..803e7553 --- /dev/null +++ b/tests/unit_tests/service/notification/test_notification_service.py @@ -0,0 +1,50 @@ +import logging +import os +from unittest.mock import patch + +from app.service.notification import send_notification + + +@patch.dict(os.environ, {"SLACK_WEBHOOK_URL": "test"}) +def test_send_notification_success(caplog): + notification = "Hello World!" + + with ( + caplog.at_level(logging.INFO), + patch("app.service.notification.slack_message") as slack_mock, + ): + send_notification(notification) + + slack_mock.assert_called_once_with(notification) + assert notification in caplog.text + + +@patch.dict(os.environ, {"SLACK_WEBHOOK_URL": "test"}) +def test_send_notification_error(caplog): + + exception_message = "Test error" + + with ( + caplog.at_level(logging.ERROR), + patch( + "app.service.notification.slack_message", + side_effect=Exception(exception_message), + ), + ): + send_notification("Hello World!") + + assert f"Error sending notification caused by: {exception_message}" in caplog.text + + +@patch.dict(os.environ, {"SLACK_WEBHOOK_URL": "skip"}) +def test_do_not_send_notification_when_in_local_development(caplog): + notification = "Hello World!" + + with ( + caplog.at_level(logging.INFO), + patch("app.service.notification.slack_message") as slack_mock, + ): + send_notification(notification) + + slack_mock.assert_not_called() + assert notification in caplog.text