From 80cd88e45376cded17d4102330cc5060d18df53f Mon Sep 17 00:00:00 2001 From: Katy Baulch <46493669+katybaulch@users.noreply.github.com> Date: Thu, 16 Nov 2023 17:28:28 +0000 Subject: [PATCH] Added created and last_modified columns to family documents. (#181) --- .gitignore | 1 + alembic.ini | 7 +- alembic/env.py | 37 ++++++- ...dded_created_and_last_modified_columns_.py | 96 ++++++++++++++++++ app/core/config.py | 6 +- app/db/models/law_policy/family.py | 10 ++ poetry.lock | 98 ++++++++++++++++++- pyproject.toml | 5 + tests/core/ingestion/conftest.py | 43 ++++++++ tests/core/ingestion/test_family.py | 66 +++++++++---- 10 files changed, 343 insertions(+), 26 deletions(-) create mode 100644 alembic/versions/0020_added_created_and_last_modified_columns_.py create mode 100644 tests/core/ingestion/conftest.py diff --git a/.gitignore b/.gitignore index ffc5f8b3..53e048c1 100644 --- a/.gitignore +++ b/.gitignore @@ -190,3 +190,4 @@ backend/models # PyCharm .idea/ +alembic/replaceable_objects/** diff --git a/alembic.ini b/alembic.ini index e7212240..3cb94197 100644 --- a/alembic.ini +++ b/alembic.ini @@ -48,7 +48,7 @@ script_location = alembic # Logging configuration [loggers] -keys = root,sqlalchemy,alembic +keys = root,sqlalchemy,alembic,alembic_utils [handlers] keys = console @@ -71,6 +71,11 @@ level = INFO handlers = qualname = alembic +[logger_alembic_utils] +level = INFO +handlers = +qualname = alembic_utils + [handler_console] class = StreamHandler args = (sys.stderr,) diff --git a/alembic/env.py b/alembic/env.py index 3ef8e6fa..470de78d 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -1,10 +1,13 @@ import logging import os -from alembic import context from logging.config import fileConfig -from sqlalchemy import engine_from_config -from sqlalchemy import pool +from alembic_utils.pg_function import PGFunction +from alembic_utils.pg_trigger import PGTrigger +from alembic_utils.replaceable_entity import register_entities +from sqlalchemy import engine_from_config, pool + +from alembic import context from app.db.models import Base logger = logging.getLogger(__name__) @@ -30,6 +33,31 @@ # my_important_option = config.get_main_option("my_important_option") # ... etc. +# Add any custom DDL statements here. +last_modified_procedure = PGFunction( + schema="public", + signature="update_last_modified()", + definition=""" +RETURNS TRIGGER AS $$ +BEGIN + NEW.last_modified = NOW(); + RETURN NEW; +END; +$$ language 'plpgsql' +""", +) + +last_modified_trigger = PGTrigger( + schema="public", + signature="update_last_modified", + on_entity="public.family_document", + definition=""" + BEFORE INSERT OR UPDATE ON public.family_document + FOR EACH ROW + EXECUTE PROCEDURE public.update_last_modified() +""", +) + def get_url(): db_url = os.environ["DATABASE_URL"] @@ -103,6 +131,9 @@ def run_migrations_online(): context.run_migrations() +register_entities([last_modified_procedure, last_modified_trigger]) + + if context.is_offline_mode(): run_migrations_offline() else: diff --git a/alembic/versions/0020_added_created_and_last_modified_columns_.py b/alembic/versions/0020_added_created_and_last_modified_columns_.py new file mode 100644 index 00000000..7ff9aa89 --- /dev/null +++ b/alembic/versions/0020_added_created_and_last_modified_columns_.py @@ -0,0 +1,96 @@ +""" +Added created and last modified columns on documents. + +Revision ID: 0020 +Revises: 0019 +Create Date: 2023-11-16 17:01:21.091516 + +""" +import sqlalchemy as sa +from alembic_utils.pg_function import PGFunction +from alembic_utils.pg_trigger import PGTrigger + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "0020" +down_revision = "0019" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column( + "family_document", + sa.Column( + "created", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + ) + op.add_column( + "family_document", + sa.Column("last_modified", sa.DateTime(timezone=True), nullable=False), + ) + public_update_last_modified = PGFunction( + schema="public", + signature="update_last_modified()", + definition=""" + RETURNS TRIGGER AS $$ + BEGIN + NEW.last_modified = NOW(); + RETURN NEW; + END; + $$ language 'plpgsql' +""", + ) + op.create_entity(public_update_last_modified) # type: ignore + + public_family_document_update_last_modified = PGTrigger( + schema="public", + signature="update_last_modified", + on_entity="public.family_document", + definition=""" + BEFORE INSERT OR UPDATE ON public.family_document + FOR EACH ROW + EXECUTE PROCEDURE public.update_last_modified() +""", + ) + op.create_entity(public_family_document_update_last_modified) # type: ignore + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + public_family_document_update_last_modified = PGTrigger( + schema="public", + signature="update_last_modified", + on_entity="public.family_document", + is_constraint=False, + definition=""" + BEFORE INSERT OR UPDATE ON public.family_document + FOR EACH ROW + EXECUTE PROCEDURE public.update_last_modified() +""", + ) + op.drop_entity(public_family_document_update_last_modified) # type: ignore + + public_update_last_modified = PGFunction( + schema="public", + signature="update_last_modified()", + definition=""" + RETURNS TRIGGER AS $$ + BEGIN + NEW.last_modified = NOW(); + RETURN NEW; + END; + $$ language 'plpgsql' +""", + ) + op.drop_entity(public_update_last_modified) # type: ignore + + op.drop_column("family_document", "last_modified") + op.drop_column("family_document", "created") + # ### end Alembic commands ### diff --git a/app/core/config.py b/app/core/config.py index 605f3500..44220807 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -68,9 +68,7 @@ os.getenv("VESPA_SEARCH_MAX_MATCHES_PER_DOC", "20") ) VESPA_SECRETS_LOCATION: str = os.getenv("VESPA_SECRETS_LOCATION", "/secrets") -VESPA_URL: str = os.environ["VESPA_URL"] +VESPA_URL: str = os.getenv("VESPA_URL", "") # Shared search config -INDEX_ENCODER_CACHE_FOLDER: str = os.getenv( - "INDEX_ENCODER_CACHE_FOLDER", "/models" -) +INDEX_ENCODER_CACHE_FOLDER: str = os.getenv("INDEX_ENCODER_CACHE_FOLDER", "/models") diff --git a/app/db/models/law_policy/family.py b/app/db/models/law_policy/family.py index 58927848..adbaa2ce 100644 --- a/app/db/models/law_policy/family.py +++ b/app/db/models/law_policy/family.py @@ -9,6 +9,7 @@ from app.db.models.app.enum import BaseModelEnum from app.db.models.document import PhysicalDocument from app.db.session import Base + from .geography import Geography @@ -202,6 +203,15 @@ class FamilyDocument(Base): ) document_type = sa.Column(sa.ForeignKey(FamilyDocumentType.name), nullable=True) document_role = sa.Column(sa.ForeignKey(FamilyDocumentRole.name), nullable=True) + created = sa.Column( + sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ) + last_modified = sa.Column( + sa.DateTime(timezone=True), + default=sa.func.now(), + onupdate=sa.func.now(), + nullable=False, + ) slugs: list["Slug"] = relationship("Slug", lazy="joined") physical_document: PhysicalDocument = relationship( diff --git a/poetry.lock b/poetry.lock index 9268b131..98631b8c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -141,6 +141,28 @@ typing-extensions = ">=4" [package.extras] tz = ["python-dateutil"] +[[package]] +name = "alembic-utils" +version = "0.8.2" +description = "A sqlalchemy/alembic extension for migrating procedures and views" +optional = false +python-versions = ">=3.7" +files = [ + {file = "alembic_utils-0.8.2.tar.gz", hash = "sha256:d7b3f179dd750a4d457f298642871e36fd85e20af97cf526df86fbc5ebecfd3a"}, +] + +[package.dependencies] +alembic = ">=1.9" +flupy = "*" +parse = ">=1.8.4" +sqlalchemy = ">=1.4" +typing_extensions = "*" + +[package.extras] +dev = ["black", "mkdocs", "mypy", "pre-commit", "psycopg2-binary", "pylint", "pytest", "pytest-cov"] +docs = ["mkautodoc", "mkdocs", "pygments", "pymdown-extensions"] +nvim = ["neovim", "python-language-server"] + [[package]] name = "anyio" version = "3.7.1" @@ -842,6 +864,36 @@ docs = ["furo (>=2023.9.10)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1 testing = ["covdefaults (>=2.3)", "coverage (>=7.3.2)", "diff-cover (>=8)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)", "pytest-timeout (>=2.2)"] typing = ["typing-extensions (>=4.8)"] +[[package]] +name = "flupy" +version = "1.2.0" +description = "Method chaining built on generators" +optional = false +python-versions = "*" +files = [ + {file = "flupy-1.2.0.tar.gz", hash = "sha256:12487a008e9744cd35d0f6ea3cfa06f4b2b27cb138bf57d0788f5c26e57afe69"}, +] + +[package.dependencies] +typing_extensions = "*" + +[package.extras] +dev = ["black", "mypy", "pre-commit", "pylint", "pytest", "pytest-benchmark", "pytest-cov"] + +[[package]] +name = "freezegun" +version = "1.2.2" +description = "Let your Python tests travel through time" +optional = false +python-versions = ">=3.6" +files = [ + {file = "freezegun-1.2.2-py3-none-any.whl", hash = "sha256:ea1b963b993cb9ea195adbd893a48d573fda951b0da64f60883d7e988b606c9f"}, + {file = "freezegun-1.2.2.tar.gz", hash = "sha256:cd22d1ba06941384410cd967d8a99d5ae2442f57dfafeff2fda5de8dc5c05446"}, +] + +[package.dependencies] +python-dateutil = ">=2.7" + [[package]] name = "frozenlist" version = "1.4.0" @@ -1325,6 +1377,16 @@ files = [ {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, @@ -1874,6 +1936,17 @@ pytz = ">=2020.1" [package.extras] test = ["hypothesis (>=5.5.3)", "pytest (>=6.0)", "pytest-xdist (>=1.31)"] +[[package]] +name = "parse" +version = "1.19.1" +description = "parse() is the opposite of format()" +optional = false +python-versions = "*" +files = [ + {file = "parse-1.19.1-py2.py3-none-any.whl", hash = "sha256:371ed3800dc63983832159cc9373156613947707bc448b5215473a219dbd4362"}, + {file = "parse-1.19.1.tar.gz", hash = "sha256:cc3a47236ff05da377617ddefa867b7ba983819c664e1afe46249e5b469be464"}, +] + [[package]] name = "passlib" version = "1.7.4" @@ -2050,6 +2123,7 @@ files = [ {file = "psycopg2_binary-2.9.9-cp311-cp311-win32.whl", hash = "sha256:dc4926288b2a3e9fd7b50dc6a1909a13bbdadfc67d93f3374d984e56f885579d"}, {file = "psycopg2_binary-2.9.9-cp311-cp311-win_amd64.whl", hash = "sha256:b76bedd166805480ab069612119ea636f5ab8f8771e640ae103e05a4aae3e417"}, {file = "psycopg2_binary-2.9.9-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:8532fd6e6e2dc57bcb3bc90b079c60de896d2128c5d9d6f24a63875a95a088cf"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b0605eaed3eb239e87df0d5e3c6489daae3f7388d455d0c0b4df899519c6a38d"}, {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f8544b092a29a6ddd72f3556a9fcf249ec412e10ad28be6a0c0d948924f2212"}, {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2d423c8d8a3c82d08fe8af900ad5b613ce3632a1249fd6a223941d0735fce493"}, {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2e5afae772c00980525f6d6ecf7cbca55676296b580c0e6abb407f15f3706996"}, @@ -2058,6 +2132,8 @@ files = [ {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:cb16c65dcb648d0a43a2521f2f0a2300f40639f6f8c1ecbc662141e4e3e1ee07"}, {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:911dda9c487075abd54e644ccdf5e5c16773470a6a5d3826fda76699410066fb"}, {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:57fede879f08d23c85140a360c6a77709113efd1c993923c59fde17aa27599fe"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-win32.whl", hash = "sha256:64cf30263844fa208851ebb13b0732ce674d8ec6a0c86a4e160495d299ba3c93"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-win_amd64.whl", hash = "sha256:81ff62668af011f9a48787564ab7eded4e9fb17a4a6a74af5ffa6a457400d2ab"}, {file = "psycopg2_binary-2.9.9-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:2293b001e319ab0d869d660a704942c9e2cce19745262a8aba2115ef41a0a42a"}, {file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:03ef7df18daf2c4c07e2695e8cfd5ee7f748a1d54d802330985a78d2a5a6dca9"}, {file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a602ea5aff39bb9fac6308e9c9d82b9a35c2bf288e184a816002c9fae930b77"}, @@ -2439,6 +2515,7 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -2446,8 +2523,15 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -2464,6 +2548,7 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -2471,6 +2556,7 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -3109,6 +3195,16 @@ typing-extensions = {version = ">=3.10.0", markers = "python_version < \"3.10\"" [package.extras] full = ["httpx (>=0.22.0)", "itsdangerous", "jinja2", "python-multipart", "pyyaml"] +[[package]] +name = "surrogate" +version = "0.1" +description = "A Python micro-lib to create stubs for non-existing modules." +optional = false +python-versions = "*" +files = [ + {file = "surrogate-0.1.tar.gz", hash = "sha256:edebec660d728325be1d52cab40d778d4c75ba04f927f4aba12d35f730b2df03"}, +] + [[package]] name = "sympy" version = "1.12" @@ -4060,4 +4156,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "d13d4ba85ac08c0ef43690965c084b7900d605a7dc9e34cb509f8d92b1a41945" +content-hash = "bb0c95f79a976156d60794947fbc7839c566d88aad7cbf70907a36fc30322071" diff --git a/pyproject.toml b/pyproject.toml index d9ae3a54..7695c145 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,7 @@ SQLAlchemy-Utils = "^0.38.2" starlette = "^0.22.0" tenacity = "^8.0.1" uvicorn = { extras = ["standard"], version = "^0.20.0" } +alembic-utils = "^0.8.2" [tool.poetry.dev-dependencies] black = "^23.1.0" @@ -45,6 +46,10 @@ pyright = "^1.1.294" ruff = "^0.0.291" types-SQLAlchemy = "^1.4.31" +[tool.poetry.group.dev.dependencies] +surrogate = "^0.1" +freezegun = "^1.2.2" + [build-system] requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" diff --git a/tests/core/ingestion/conftest.py b/tests/core/ingestion/conftest.py new file mode 100644 index 00000000..74e642fd --- /dev/null +++ b/tests/core/ingestion/conftest.py @@ -0,0 +1,43 @@ +import datetime +from contextlib import contextmanager + +import pytest +from freezegun import freeze_time +from sqlalchemy import event +from surrogate import surrogate + +from app.db.models.law_policy.family import FamilyDocument + + +@pytest.fixture() +def patch_current_time(): + return patch_time + + +@contextmanager +def patch_time(time_to_freeze, tick=True): + with freeze_time(time_to_freeze, tick=tick) as frozen_time: + + def set_initial_timestamp(mapper, connection, target): + now = datetime.datetime.now() + if hasattr(target, "created"): + target.created = now + + event.listen( + FamilyDocument, "before_insert", set_initial_timestamp, propagate=True + ) + yield frozen_time + event.remove(FamilyDocument, "before_insert", set_initial_timestamp) + + +@pytest.fixture(scope="session", autouse=True) +def stub_freezegun_dynamic_imports(): + with surrogate("transformers.models.deprecated.open_llama.tokenization_open_llama"): + with surrogate( + "transformers.models.deprecated.open_llama.tokenization_open_llama_fast" + ): + with surrogate("transformers.models.open_llama.tokenization_open_llama"): + with surrogate( + "transformers.models.open_llama.tokenization_open_llama_fast" + ): + yield diff --git a/tests/core/ingestion/test_family.py b/tests/core/ingestion/test_family.py index 2ca71eb8..fdca5c80 100644 --- a/tests/core/ingestion/test_family.py +++ b/tests/core/ingestion/test_family.py @@ -1,9 +1,12 @@ +from datetime import datetime, timezone + from sqlalchemy.orm import Session + +from app.core.ingestion.cclw.ingest_row_cclw import CCLWDocumentIngestRow from app.core.ingestion.family import ( handle_family_document_from_params, handle_family_from_params, ) -from app.core.ingestion.cclw.ingest_row_cclw import CCLWDocumentIngestRow from app.core.ingestion.physical_document import create_physical_document_from_params from app.core.ingestion.processor import build_params_from_cclw from app.db.models.law_policy.family import ( @@ -122,16 +125,22 @@ def test_family_document_from_row__creates(test_db: Session): ) assert db_family_doc == family_document assert db_family_doc.physical_document.title == row.document_title + assert db_family_doc.created is not None + assert db_family_doc.last_modified is not None + assert db_family_doc.created == db_family_doc.last_modified -def test_family_document_from_row__updates(test_db: Session): - populate_for_ingest(test_db) - row = CCLWDocumentIngestRow.from_row(1, get_doc_ingest_row_data(0)) - family = add_a_family(test_db) - result = {} - handle_family_document_from_params( - test_db, build_params_from_cclw(row), family, result=result - ) +def test_family_document_from_row__updates(test_db: Session, patch_current_time): + with patch_current_time("2000-01-01 00:00:00.0Z", tick=False): + populate_for_ingest(test_db) + + row = CCLWDocumentIngestRow.from_row(1, get_doc_ingest_row_data(0)) + family = add_a_family(test_db) + result = {} + handle_family_document_from_params( + test_db, build_params_from_cclw(row), family, result=result + ) + result = {} row.document_title = "test-title" row.document_role = "PRESS RELEASE" @@ -139,6 +148,14 @@ def test_family_document_from_row__updates(test_db: Session): test_db, build_params_from_cclw(row), family, result=result ) + assert family_document.created is not None + assert family_document.created == datetime( + 2000, 1, 1, 0, 0, 0, 0, tzinfo=timezone.utc + ) + assert family_document.last_modified is not None + assert family_document.created != family_document.last_modified + assert family_document.created < family_document.last_modified + actual_keys = set(result.keys()) expected_keys = set( [ @@ -157,15 +174,24 @@ def test_family_document_from_row__updates(test_db: Session): assert db_family_doc.physical_document.title == "test-title" assert db_family_doc.document_role == "PRESS RELEASE" - -def test_family_document_from_row__updates_status(test_db: Session): - populate_for_ingest(test_db) - row = CCLWDocumentIngestRow.from_row(1, get_doc_ingest_row_data(0)) - family = add_a_family(test_db) - result = {} - handle_family_document_from_params( - test_db, build_params_from_cclw(row), family, result=result + assert db_family_doc.created == datetime( + 2000, 1, 1, 0, 0, 0, 0, tzinfo=timezone.utc ) + assert db_family_doc.last_modified is not None + assert db_family_doc.created != db_family_doc.last_modified + assert db_family_doc.created < db_family_doc.last_modified + + +def test_family_document_from_row__updates_status(test_db: Session, patch_current_time): + with patch_current_time("2000-01-01 00:00:00.0Z", tick=False): + populate_for_ingest(test_db) + row = CCLWDocumentIngestRow.from_row(1, get_doc_ingest_row_data(0)) + family = add_a_family(test_db) + result = {} + handle_family_document_from_params( + test_db, build_params_from_cclw(row), family, result=result + ) + result = {} row.cpr_document_status = "DELETED" family_document = handle_family_document_from_params( @@ -181,6 +207,12 @@ def test_family_document_from_row__updates_status(test_db: Session): ) assert db_family_doc == family_document assert db_family_doc.document_status == "DELETED" + assert db_family_doc.created == datetime( + 2000, 1, 1, 0, 0, 0, 0, tzinfo=timezone.utc + ) + assert db_family_doc.last_modified is not None + assert db_family_doc.created != db_family_doc.last_modified + assert db_family_doc.created < db_family_doc.last_modified def add_a_family(test_db: Session) -> Family: