From a255f81ba1bddf4be806c7ddc0558bfc45955b4b Mon Sep 17 00:00:00 2001 From: Dan Cardin Date: Mon, 31 Oct 2022 09:45:57 -0400 Subject: [PATCH] fix: Refresh alembic history to enable tests generate new revisions to be aware of those revisions. --- .github/workflows/build.yml | 6 +- .gitignore | 3 + CHANGELOG.md | 122 ++++++++++++------ examples/test_generate_revision/alembic.ini | 36 ++++++ examples/test_generate_revision/conftest.py | 0 .../test_generate_revision/migrations/env.py | 25 ++++ .../migrations/script.py.mako | 24 ++++ .../migrations/versions/.gitkeep | 0 examples/test_generate_revision/models.py | 10 ++ examples/test_generate_revision/setup.cfg | 2 + .../test_generate_revision/test_migration.py | 16 +++ pyproject.toml | 2 +- src/pytest_alembic/runner.py | 57 ++++++-- src/pytest_alembic/tests/default.py | 1 + tests/test_runner.py | 5 + 15 files changed, 249 insertions(+), 60 deletions(-) create mode 100644 examples/test_generate_revision/alembic.ini create mode 100644 examples/test_generate_revision/conftest.py create mode 100644 examples/test_generate_revision/migrations/env.py create mode 100644 examples/test_generate_revision/migrations/script.py.mako create mode 100644 examples/test_generate_revision/migrations/versions/.gitkeep create mode 100644 examples/test_generate_revision/models.py create mode 100644 examples/test_generate_revision/setup.cfg create mode 100644 examples/test_generate_revision/test_migration.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 038b96c..160e9ea 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -28,8 +28,8 @@ jobs: sqlalchemy-version: ["1.3", "1.4"] steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Install poetry @@ -38,7 +38,7 @@ jobs: poetry-version: "1.2" - name: Set up cache - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ~/.cache/pypoetry/virtualenvs key: venv-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.pytest-version }}-${{ matrix.pytest-asyncio-version }}-${{ matrix.sqlalchemy-version }}-${{ hashFiles('**/poetry.lock') }} diff --git a/.gitignore b/.gitignore index 351edff..acf8603 100644 --- a/.gitignore +++ b/.gitignore @@ -59,3 +59,6 @@ shell.nix # Documentation docs/_* + +# Generated by test runs. +examples/test_generate_revision/test_migration.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 9819753..b423778 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,136 +1,172 @@ # Changelog -## [Unreleased](https://github.com/schireson/pytest-alembic/compare/v0.7.0...HEAD) (2022-02-08) +## [Unreleased](https://github.com/schireson/pytest-alembic/compare/v0.8.3...HEAD) (2022-10-31) ### Fixes -* (Huge speed optimization) Avoid the use of the high-level alembic command interface in most cases. 1ae311f +* Refresh alembic history to enable tests generate new revisions to be aware of +those revisions. + ([a81b5f8](https://github.com/schireson/pytest-alembic/commit/a81b5f82cbf0243f045cc41263a56bb788fc260d)) +* Compatibility with newer versions of pytest and pytest-asyncio. + ([4ed809b](https://github.com/schireson/pytest-alembic/commit/4ed809b8b059091cbd55aa68d57d398c129a7d3f)) +* Correctly insert the root package during metaadata detection. + ([d719608](https://github.com/schireson/pytest-alembic/commit/d71960884a6e47176d21e64e14d987bdc09715f0)) +### [v0.8.3](https://github.com/schireson/pytest-alembic/compare/v0.8.2...v0.8.3) (2022-07-20) + +### [v0.8.2](https://github.com/schireson/pytest-alembic/compare/v0.8.1...v0.8.2) (2022-04-10) + +#### Fixes + +* Add missing connection param to table_at_revision. + ([a20d16e](https://github.com/schireson/pytest-alembic/commit/a20d16e42c9cec5f1062e2b7d3072eae42ef5534)) +* Improve test options for all_models_register_on_metadata. + ([28b7f59](https://github.com/schireson/pytest-alembic/commit/28b7f5950e5239f81c6b46a0b4265b0ed73fcb10)) + +### [v0.8.1](https://github.com/schireson/pytest-alembic/compare/v0.8.0...v0.8.1) (2022-03-12) + +#### Fixes + +* Add missing explicit reexports. + ([d5375ad](https://github.com/schireson/pytest-alembic/commit/d5375ad3cba6066826c2ac4df3220d20433d381e)) + +## [v0.8.0](https://github.com/schireson/pytest-alembic/compare/v0.7.0...v0.8.0) (2022-02-08) + +### Fixes + +* (Huge speed optimization) Avoid the use of the high-level alembic command +interface in most cases. + ([d616ffa](https://github.com/schireson/pytest-alembic/commit/d616ffaacc83acdd48b6ace0b517ceb35aaf0172)) ## [v0.7.0](https://github.com/schireson/pytest-alembic/compare/v0.6.1...v0.7.0) (2021-12-21) ### ⚠ BREAKING CHANGE -* Starting with this release, python 3.6 will no longer be tested or officially supported. In this specific release, only the new official support for asyncio-based engine with alembic and pytest-alembic is incompatible with 3.6. Any existing usage should remain at least provisionally compatible until later releases which may or may not further break compatibility. +* Starting with this release, python 3.6 will no longer be tested or officiallysupported. In this specific release, only the new official support forasyncio-based engine with alembic and pytest-alembic is incompatible with 3.6.Any existing usage should remain at least provisionally compatible until laterreleases which may or may not further break compatibility. ### Features -* Enable in-test insertion of data in async contexts. e9f8d97 +* Enable in-test insertion of data in async contexts. + ([e9f8d97](https://github.com/schireson/pytest-alembic/commit/e9f8d9726e1a6a9032aa773db8dc1b69cc81cc5a)) ### Fixes -* asynchronous engine tests which perform transaction manipulation. 245f9ef - +* asynchronous engine tests which perform transaction manipulation. + ([245f9ef](https://github.com/schireson/pytest-alembic/commit/245f9ef4e94f82d5d7742407451bcd0ad12762ac)) ### [v0.6.1](https://github.com/schireson/pytest-alembic/compare/v0.6.0...v0.6.1) (2021-12-02) #### Fixes -* Add missing alembic Config options. c3cab87 - +* Add missing alembic Config options. + ([c3cab87](https://github.com/schireson/pytest-alembic/commit/c3cab870677ebe690fb2e82170f2af3981e2ebeb)) ## [v0.6.0](https://github.com/schireson/pytest-alembic/compare/v0.5.1...v0.6.0) (2021-11-30) ### Features -* Add ability to set a minimum bound downgrade migration cda6937 -* Add new test which asserts parity between upgrade and downgrade detectable effects. ab9b645 -* Add new test for roundtrip downgrade isolation. 2fb20d0 +* Add ability to set a minimum bound downgrade migration + ([cda6937](https://github.com/schireson/pytest-alembic/commit/cda69378272a70efc40535e13546f50b5fdc7d74)) +* Add new test which asserts parity between upgrade and downgrade detectable +effects. + ([ab9b645](https://github.com/schireson/pytest-alembic/commit/ab9b6450988ff000899ff8ee193a309a3ff6c9a3)) +* Add new test for roundtrip downgrade isolation. + ([2fb20d0](https://github.com/schireson/pytest-alembic/commit/2fb20d0b8d17a70d84252832ee36fad020b06a68)) ### Fixes -* Run pytest tests inline (faster and easier coverage). ea9b59d - +* Run pytest tests inline (faster and easier coverage). + ([ea9b59d](https://github.com/schireson/pytest-alembic/commit/ea9b59dc61ac537fa5648273878c628094dbae71)) ### [v0.5.1](https://github.com/schireson/pytest-alembic/compare/v0.5.0...v0.5.1) (2021-11-23) #### Fixes -* Increase minimum python version to 3.6+ (this was already true!). e6bdfe6 -* Incompatibility of branched history downgrade strategy with alembic 1.6+. 192686b -* ensure the up-down consistency test actually verifies migrations a2e9d13 - +* Increase minimum python version to 3.6+ (this was already true!). + ([e6bdfe6](https://github.com/schireson/pytest-alembic/commit/e6bdfe67f7d0bf8e675eeefa38cd44a06847799f)) +* Incompatibility of branched history downgrade strategy with alembic 1.6+. + ([192686b](https://github.com/schireson/pytest-alembic/commit/192686b9f3eaf43e8109c9376b9a806352f3a8c7)) +* ensure the up-down consistency test actually verifies migrations + ([a2e9d13](https://github.com/schireson/pytest-alembic/commit/a2e9d1321b378036e19af8e9525d78eddac09a37)) ## [v0.5.0](https://github.com/schireson/pytest-alembic/compare/v0.4.0...v0.5.0) (2021-09-03) ### Features -* Add experimental test to identify tables which alembic will not recognize. d12e342 +* Add experimental test to identify tables which alembic will not recognize. + ([d12e342](https://github.com/schireson/pytest-alembic/commit/d12e3422f2123eb0395e3b4a4535fdf9d2676f4a)) ### Fixes -* Add back missing lint job. 80242f3 - +* Add back missing lint job. + ([80242f3](https://github.com/schireson/pytest-alembic/commit/80242f3e4c4fc7e0120b44a4a03a4eecead2c51e)) ## [v0.4.0](https://github.com/schireson/pytest-alembic/compare/v0.3.3...v0.4.0) (2021-08-16) ### Features -* Create a mechanism in which to create multiple alembic runner fixtures. ef1d5da -* Allow alembic Config to be used directly in alembic_config fixture. 3b00103 +* Create a mechanism in which to create multiple alembic runner fixtures. + ([ef1d5da](https://github.com/schireson/pytest-alembic/commit/ef1d5daec9d66e256a4b1b8a742d6889fbbbc44d)) +* Allow alembic Config to be used directly in alembic_config fixture. + ([3b00103](https://github.com/schireson/pytest-alembic/commit/3b0010398fd245a44e6ce16f9765a2e4c0c45c66)) ### Fixes -* Run covtest on all branches. f1bd6ac - +* Run covtest on all branches. + ([f1bd6ac](https://github.com/schireson/pytest-alembic/commit/f1bd6aca6196cbea4674f4b6d1c1eee204cee387)) ### [v0.3.3](https://github.com/schireson/pytest-alembic/compare/v0.3.2...v0.3.3) (2021-08-04) #### Fixes -* Conditionally set script_location. a26f59b - +* Conditionally set script_location. + ([a26f59b](https://github.com/schireson/pytest-alembic/commit/a26f59b8b737eff8e77e663f23623024377e5371)) ### [v0.3.2](https://github.com/schireson/pytest-alembic/compare/v0.3.1...v0.3.2) (2021-08-04) - ### [v0.3.1](https://github.com/schireson/pytest-alembic/compare/v0.3.0...v0.3.1) (2021-05-10) - ## [v0.3.0](https://github.com/schireson/pytest-alembic/compare/v0.2.6...v0.3.0) (2021-05-10) - ### [v0.2.6](https://github.com/schireson/pytest-alembic/compare/v0.2.5...v0.2.6) (2021-04-26) - ### [v0.2.5](https://github.com/schireson/pytest-alembic/compare/v0.2.4...v0.2.5) (2020-07-13) #### Features -* Allow the customization of the location at which the built in tests are executed. 255c95c - +* Allow the customization of the location at which the built in tests are +executed. + ([255c95c](https://github.com/schireson/pytest-alembic/commit/255c95c8edf0055f9d97aa671590449600b3e2a4)) ### [v0.2.4](https://github.com/schireson/pytest-alembic/compare/v0.2.3...v0.2.4) (2020-07-01) #### Fixes -* Require dataclasses only below 3.7, as it is included in stdlib 3.7 onward. 0b30fb4 - +* Require dataclasses only below 3.7, as it is included in stdlib 3.7 onward. + ([0b30fb4](https://github.com/schireson/pytest-alembic/commit/0b30fb41bebf702102b09c55bba18931158d94ef)) ### [v0.2.3](https://github.com/schireson/pytest-alembic/compare/v0.2.2...v0.2.3) (2020-06-26) #### Features -* Reduce the multiple pages of traceback output to a few lines of context that are actually meaningful to a failed test. d9bcfcc - +* Reduce the multiple pages of traceback output to a few lines of context that are +actually meaningful to a failed test. + ([d9bcfcc](https://github.com/schireson/pytest-alembic/commit/d9bcfcc709421734e14f3d034bfa77f74c15729e)) ### [v0.2.2](https://github.com/schireson/pytest-alembic/compare/v0.2.1...v0.2.2) (2020-06-25) #### Features -* Add rendered migration body to failed model-sync test. 108db31 - +* Add rendered migration body to failed model-sync test. + ([108db31](https://github.com/schireson/pytest-alembic/commit/108db31b874cc199418a012f314daa47d87b310a)) ### [v0.2.1](https://github.com/schireson/pytest-alembic/compare/v0.1.1...v0.2.1) (2020-03-23) #### Fixes -* Fix deprecation pytest warning in 3.4. f15a86b - +* Fix deprecation pytest warning in 3.4. + ([f15a86b](https://github.com/schireson/pytest-alembic/commit/f15a86bd0620606203732a3f13d454b786d21a50)) ### [v0.1.1](https://github.com/schireson/pytest-alembic/compare/v0.1.0...v0.1.1) (2020-03-09) - ## v0.1.0 (2020-03-09) - - diff --git a/examples/test_generate_revision/alembic.ini b/examples/test_generate_revision/alembic.ini new file mode 100644 index 0000000..1ceb413 --- /dev/null +++ b/examples/test_generate_revision/alembic.ini @@ -0,0 +1,36 @@ +[alembic] +script_location = migrations + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/examples/test_generate_revision/conftest.py b/examples/test_generate_revision/conftest.py new file mode 100644 index 0000000..e69de29 diff --git a/examples/test_generate_revision/migrations/env.py b/examples/test_generate_revision/migrations/env.py new file mode 100644 index 0000000..283a00b --- /dev/null +++ b/examples/test_generate_revision/migrations/env.py @@ -0,0 +1,25 @@ +from logging.config import fileConfig + +from alembic import context +from sqlalchemy import engine_from_config, pool + +from models import Base + +fileConfig(context.config.config_file_name) +target_metadata = Base.metadata + + +connectable = context.config.attributes.get("connection", None) + +if connectable is None: + connectable = engine_from_config( + context.config.get_section(context.config.config_ini_section), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + +with connectable.connect() as connection: + context.configure(connection=connection, target_metadata=target_metadata) + + with context.begin_transaction(): + context.run_migrations() diff --git a/examples/test_generate_revision/migrations/script.py.mako b/examples/test_generate_revision/migrations/script.py.mako new file mode 100644 index 0000000..2c01563 --- /dev/null +++ b/examples/test_generate_revision/migrations/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/examples/test_generate_revision/migrations/versions/.gitkeep b/examples/test_generate_revision/migrations/versions/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/examples/test_generate_revision/models.py b/examples/test_generate_revision/models.py new file mode 100644 index 0000000..021f150 --- /dev/null +++ b/examples/test_generate_revision/models.py @@ -0,0 +1,10 @@ +from sqlalchemy import Column, types +from sqlalchemy.ext.declarative import declarative_base + +Base = declarative_base() + + +class Foo(Base): + __tablename__ = "foo" + + id = Column(types.Integer(), primary_key=True) diff --git a/examples/test_generate_revision/setup.cfg b/examples/test_generate_revision/setup.cfg new file mode 100644 index 0000000..2508f0b --- /dev/null +++ b/examples/test_generate_revision/setup.cfg @@ -0,0 +1,2 @@ +[tool:pytest] +pytest_alembic_exclude = model_definitions_match_ddl, single_head_revision diff --git a/examples/test_generate_revision/test_migration.py b/examples/test_generate_revision/test_migration.py new file mode 100644 index 0000000..4982424 --- /dev/null +++ b/examples/test_generate_revision/test_migration.py @@ -0,0 +1,16 @@ +import pytest + +from pytest_alembic.runner import MigrationContext + + +def test_generate_revision(alembic_runner: MigrationContext, alembic_engine): + with pytest.raises(Exception): + alembic_engine.execute("SELECT * FROM foo").fetchall() + + alembic_runner.generate_revision(autogenerate=True, prevent_file_generation=False) + alembic_runner.migrate_up_one() + + alembic_engine.execute("INSERT INTO foo (id) VALUES (100)") + + result = alembic_engine.execute("SELECT * FROM foo").fetchall() + assert len(result) == 1 diff --git a/pyproject.toml b/pyproject.toml index 02dcb34..c1bc58b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pytest-alembic" -version = "0.9.0" +version = "0.9.1" description = "A pytest plugin for verifying alembic migrations." authors = [ "Dan Cardin ", diff --git a/src/pytest_alembic/runner.py b/src/pytest_alembic/runner.py index 560651c..b01a26f 100644 --- a/src/pytest_alembic/runner.py +++ b/src/pytest_alembic/runner.py @@ -5,6 +5,7 @@ import alembic.command import alembic.migration +from alembic.script.revision import RevisionMap from sqlalchemy.engine import Engine from pytest_alembic.config import Config @@ -36,8 +37,8 @@ class MigrationContext: command_executor: CommandExecutor revision_data: RevisionData connection_executor: ConnectionExecutor - config: Config history: AlembicHistory + config: Config connection: Engine = None @classmethod @@ -48,15 +49,14 @@ def from_config( connection_executor: ConnectionExecutor, connection: Engine, ): - raw_history = command_executor.script.revision_map - history = AlembicHistory.parse(raw_history) + history = AlembicHistory.parse(command_executor.script.revision_map) return cls( command_executor=command_executor, revision_data=RevisionData.from_config(config), connection_executor=connection_executor, - config=config, history=history, + config=config, connection=connection, ) @@ -84,22 +84,53 @@ def get_current(conn): return current return "base" - def generate_revision(self, process_revision_directives=None, **kwargs): + def refresh_history(self) -> AlembicHistory: + """Refresh the context's version of the alembic history. + + Note this is not done automatically to avoid the expensive reevaluation + step which can make long histories take seconds longer to evaluate for + each test. + """ + script = self.command_executor.script + script.revision_map = RevisionMap(script._load_revisions) + self.history = AlembicHistory.parse(self.command_executor.script.revision_map) + return self.history + + def generate_revision( + self, + process_revision_directives=None, + prevent_file_generation=True, + autogenerate=False, + **kwargs + ): """Generate a test revision. - The final act of this process raises a `RevisionSuccess`, which is used as a sentinal - to indicate the revision was generated successfully, while not actually finishing the - generation of the revision file. + If `prevent_file_generation` is `True`, the final act of this process raises a + `RevisionSuccess`, which is used as a sentinal to indicate the revision was + generated successfully, while not actually finishing the generation of the + revision file on disk. """ alembic_config = self.command_executor.alembic_config config_directive = alembic_config.attributes["process_revision_directives"] - fn = RevisionSuccess.process_revision_directives( - _sequence_directives(config_directive, process_revision_directives) - ) + + directive = _sequence_directives(config_directive, process_revision_directives) + + if prevent_file_generation: + directive = RevisionSuccess.process_revision_directives(directive) + try: - return self.command_executor.run_command( - "revision", process_revision_directives=fn, **kwargs + result = self.command_executor.run_command( + "revision", + process_revision_directives=directive, + autogenerate=autogenerate, + **kwargs, ) + + # The history will only have changed if we didn't aritifically prevent it from failing. + if not prevent_file_generation: + self.refresh_history() + + return result except RevisionSuccess: pass diff --git a/src/pytest_alembic/tests/default.py b/src/pytest_alembic/tests/default.py index fec9075..56a9bd4 100644 --- a/src/pytest_alembic/tests/default.py +++ b/src/pytest_alembic/tests/default.py @@ -85,6 +85,7 @@ def verify_is_empty_revision(migration_context, __, directives): alembic_runner.generate_revision( message="test revision", autogenerate=True, + prevent_file_generation=True, process_revision_directives=verify_is_empty_revision, ) diff --git a/tests/test_runner.py b/tests/test_runner.py index 4b7b618..16083a4 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -250,3 +250,8 @@ def test_experimental_all_models_register_namespace_package(pytester): """Assert all_models_register_on_metadata with namespace packages.""" pytester.syspathinsert(pytester.path) run_pytest(pytester, passed=5) + + +def test_generate_revision(pytester): + """Assert history is refreshed when generating a revision in a test.""" + run_pytest(pytester, passed=3)