From 552c90d080d10d7c4afcf2dd6cee67864608aaeb Mon Sep 17 00:00:00 2001 From: Michael Genson <71845777+michael-genson@users.noreply.github.com> Date: Mon, 7 Oct 2024 15:40:54 +0000 Subject: [PATCH 1/8] add migration fix for dangling data --- mealie/db/fixes/fix_migration_data.py | 42 +++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/mealie/db/fixes/fix_migration_data.py b/mealie/db/fixes/fix_migration_data.py index 6b27a0cb393..866b9357fcf 100644 --- a/mealie/db/fixes/fix_migration_data.py +++ b/mealie/db/fixes/fix_migration_data.py @@ -1,6 +1,7 @@ from uuid import uuid4 from slugify import slugify +from sqlalchemy import update from sqlalchemy.orm import Session from mealie.core import root_logger @@ -9,10 +10,49 @@ from mealie.db.models.labels import MultiPurposeLabel from mealie.db.models.recipe.ingredient import IngredientFoodModel, IngredientUnitModel from mealie.db.models.recipe.recipe import RecipeModel +from mealie.db.models.users.users import User logger = root_logger.get_logger("init_db") +def fix_dangling_refs(session: Session): + REASSIGN_REF_TABLES = ["GroupMealPlan", "RecipeModel", "ShoppingList"] + DELETE_REF_TABLES = ["LongLiveToken", "PasswordResetModel", "RecipeComment", "RecipeTimelineEvent"] + + engine = session.get_bind() + groups = session.query(Group).all() + for group in groups: + default_user = session.query(User).filter(User.group_id == group.id).first() + if not default_user: + continue + + valid_user_ids = {user.id for user in group.users} + + for table_name in REASSIGN_REF_TABLES: + table = engine.metadata.tables[table_name] + update_stmt = update(table).where(~table.c.user_id.in_(valid_user_ids)).values(user_id=default_user.id) + result = session.execute(update_stmt) + + if result.rowcount: + logger.info( + f'Reassigned {result.rowcount} {"row" if result.rowcount == 1 else "rows"}' + f'in "{table_name}" table to default user ({default_user.id})' + ) + + for table_name in DELETE_REF_TABLES: + table = engine.metadata.tables[table_name] + delete_stmt = table.delete().where(~table.c.user_id.in_(valid_user_ids)) + result = session.execute(delete_stmt) + + if result.rowcount: + logger.info( + f'Deleted {result.rowcount} {"row" if result.rowcount == 1 else "rows"}' + f'in "{table_name}" table with invalid user ids' + ) + + session.commit() + + def fix_recipe_normalized_search_properties(session: Session): recipes = session.query(RecipeModel).all() recipes_fixed = False @@ -144,6 +184,8 @@ def fix_normalized_unit_and_food_names(session: Session): def fix_migration_data(session: Session): logger.info("Checking for migration data fixes") + + fix_dangling_refs(session) fix_recipe_normalized_search_properties(session) fix_shopping_list_label_settings(session) fix_group_slugs(session) From 7d22b400e7e2e1d75e0d1478c90f1ca49468561b Mon Sep 17 00:00:00 2001 From: Michael Genson <71845777+michael-genson@users.noreply.github.com> Date: Mon, 7 Oct 2024 15:46:04 +0000 Subject: [PATCH 2/8] run migration data fixes before db export --- mealie/services/backups_v2/alchemy_exporter.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/mealie/services/backups_v2/alchemy_exporter.py b/mealie/services/backups_v2/alchemy_exporter.py index 48f026bcae6..3e4b0877ebf 100644 --- a/mealie/services/backups_v2/alchemy_exporter.py +++ b/mealie/services/backups_v2/alchemy_exporter.py @@ -14,6 +14,7 @@ from alembic import command from alembic.config import Config from mealie.db import init_db +from mealie.db.fixes.fix_migration_data import fix_migration_data from mealie.db.models._model_utils.guid import GUID from mealie.services._base_service import BaseService @@ -137,6 +138,14 @@ def dump(self) -> dict[str, list[dict]]: Returns the entire SQLAlchemy database as a python dictionary. This dictionary is wrapped by jsonable_encoder to ensure that the object can be converted to a json string. """ + + # run database fixes first so we aren't backing up bad data + with self.session_maker() as session: + try: + fix_migration_data(session) + except Exception: + self.logger.error("Error fixing migration data during export; continuing anyway") + with self.engine.connect() as connection: self.meta.reflect(bind=self.engine) # http://docs.sqlalchemy.org/en/rel_0_9/core/reflection.html From c3027f3d60fc9280b624788f05d5bbb0aa8fc8c8 Mon Sep 17 00:00:00 2001 From: Michael Genson <71845777+michael-genson@users.noreply.github.com> Date: Mon, 7 Oct 2024 15:59:09 +0000 Subject: [PATCH 3/8] fixes --- mealie/db/fixes/fix_migration_data.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mealie/db/fixes/fix_migration_data.py b/mealie/db/fixes/fix_migration_data.py index 866b9357fcf..b13bc588844 100644 --- a/mealie/db/fixes/fix_migration_data.py +++ b/mealie/db/fixes/fix_migration_data.py @@ -5,6 +5,7 @@ from sqlalchemy.orm import Session from mealie.core import root_logger +from mealie.db.models._model_base import SqlAlchemyBase from mealie.db.models.group.group import Group from mealie.db.models.household.shopping_list import ShoppingList, ShoppingListMultiPurposeLabel from mealie.db.models.labels import MultiPurposeLabel @@ -16,10 +17,9 @@ def fix_dangling_refs(session: Session): - REASSIGN_REF_TABLES = ["GroupMealPlan", "RecipeModel", "ShoppingList"] - DELETE_REF_TABLES = ["LongLiveToken", "PasswordResetModel", "RecipeComment", "RecipeTimelineEvent"] + REASSIGN_REF_TABLES = ["group_meal_plans", "recipes", "shopping_lists"] + DELETE_REF_TABLES = ["long_live_tokens", "password_reset_tokens", "recipe_comments", "recipe_timeline_events"] - engine = session.get_bind() groups = session.query(Group).all() for group in groups: default_user = session.query(User).filter(User.group_id == group.id).first() @@ -29,18 +29,18 @@ def fix_dangling_refs(session: Session): valid_user_ids = {user.id for user in group.users} for table_name in REASSIGN_REF_TABLES: - table = engine.metadata.tables[table_name] + table = SqlAlchemyBase.metadata.tables[table_name] update_stmt = update(table).where(~table.c.user_id.in_(valid_user_ids)).values(user_id=default_user.id) result = session.execute(update_stmt) if result.rowcount: logger.info( - f'Reassigned {result.rowcount} {"row" if result.rowcount == 1 else "rows"}' + f'Reassigned {result.rowcount} {"row" if result.rowcount == 1 else "rows"} ' f'in "{table_name}" table to default user ({default_user.id})' ) for table_name in DELETE_REF_TABLES: - table = engine.metadata.tables[table_name] + table = SqlAlchemyBase.metadata.tables[table_name] delete_stmt = table.delete().where(~table.c.user_id.in_(valid_user_ids)) result = session.execute(delete_stmt) From b5328ad6106e4b795e37d35faacebbc4ad0c350a Mon Sep 17 00:00:00 2001 From: Michael Genson <71845777+michael-genson@users.noreply.github.com> Date: Mon, 7 Oct 2024 16:12:48 +0000 Subject: [PATCH 4/8] attempt to use an admin user as the default user --- mealie/db/fixes/fix_migration_data.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/mealie/db/fixes/fix_migration_data.py b/mealie/db/fixes/fix_migration_data.py index b13bc588844..366b22d4428 100644 --- a/mealie/db/fixes/fix_migration_data.py +++ b/mealie/db/fixes/fix_migration_data.py @@ -22,9 +22,15 @@ def fix_dangling_refs(session: Session): groups = session.query(Group).all() for group in groups: - default_user = session.query(User).filter(User.group_id == group.id).first() + # Find an arbitrary admin user in the group + default_user = session.query(User).filter(User.group_id == group.id, User.admin == True).first() # noqa: E712 - required for SQLAlchemy comparison if not default_user: - continue + # If there is no admin user, just pick the first user + default_user = session.query(User).filter(User.group_id == group.id).first() + + # If there are no users in the group, we can't do anything + if not default_user: + continue valid_user_ids = {user.id for user in group.users} From a5e9af5ef159adaba4e750c800e69c368d8a33e5 Mon Sep 17 00:00:00 2001 From: Michael Genson <71845777+michael-genson@users.noreply.github.com> Date: Mon, 7 Oct 2024 16:23:27 +0000 Subject: [PATCH 5/8] added missing group_id filter --- mealie/db/fixes/fix_migration_data.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/mealie/db/fixes/fix_migration_data.py b/mealie/db/fixes/fix_migration_data.py index 366b22d4428..fb835ffe1c7 100644 --- a/mealie/db/fixes/fix_migration_data.py +++ b/mealie/db/fixes/fix_migration_data.py @@ -1,7 +1,7 @@ from uuid import uuid4 from slugify import slugify -from sqlalchemy import update +from sqlalchemy import and_, update from sqlalchemy.orm import Session from mealie.core import root_logger @@ -36,7 +36,16 @@ def fix_dangling_refs(session: Session): for table_name in REASSIGN_REF_TABLES: table = SqlAlchemyBase.metadata.tables[table_name] - update_stmt = update(table).where(~table.c.user_id.in_(valid_user_ids)).values(user_id=default_user.id) + update_stmt = ( + update(table) + .where( + and_( + ~table.c.user_id.in_(valid_user_ids), + table.c.group_id == group.id, + ) + ) + .values(user_id=default_user.id) + ) result = session.execute(update_stmt) if result.rowcount: From c9145af4d04b2f0a06ad411ac08d7e980e86987b Mon Sep 17 00:00:00 2001 From: Michael Genson <71845777+michael-genson@users.noreply.github.com> Date: Mon, 7 Oct 2024 16:30:42 +0000 Subject: [PATCH 6/8] missing space in log --- mealie/db/fixes/fix_migration_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mealie/db/fixes/fix_migration_data.py b/mealie/db/fixes/fix_migration_data.py index fb835ffe1c7..41df9d6c653 100644 --- a/mealie/db/fixes/fix_migration_data.py +++ b/mealie/db/fixes/fix_migration_data.py @@ -61,7 +61,7 @@ def fix_dangling_refs(session: Session): if result.rowcount: logger.info( - f'Deleted {result.rowcount} {"row" if result.rowcount == 1 else "rows"}' + f'Deleted {result.rowcount} {"row" if result.rowcount == 1 else "rows"} ' f'in "{table_name}" table with invalid user ids' ) From 51f87203958282e7df8ca1dbb29cbcf73d5c829f Mon Sep 17 00:00:00 2001 From: Michael Genson <71845777+michael-genson@users.noreply.github.com> Date: Mon, 14 Oct 2024 09:09:09 -0500 Subject: [PATCH 7/8] Update mealie/db/fixes/fix_migration_data.py Co-authored-by: Kuchenpirat <24235032+Kuchenpirat@users.noreply.github.com> --- mealie/db/fixes/fix_migration_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mealie/db/fixes/fix_migration_data.py b/mealie/db/fixes/fix_migration_data.py index 41df9d6c653..0cdd036b123 100644 --- a/mealie/db/fixes/fix_migration_data.py +++ b/mealie/db/fixes/fix_migration_data.py @@ -56,7 +56,7 @@ def fix_dangling_refs(session: Session): for table_name in DELETE_REF_TABLES: table = SqlAlchemyBase.metadata.tables[table_name] - delete_stmt = table.delete().where(~table.c.user_id.in_(valid_user_ids)) + delete_stmt = table.delete().where(table.c.user_id.notin_(valid_user_ids)) result = session.execute(delete_stmt) if result.rowcount: From 14033375c7cb20c1bb195c43741dc804d4210323 Mon Sep 17 00:00:00 2001 From: Michael Genson <71845777+michael-genson@users.noreply.github.com> Date: Mon, 14 Oct 2024 09:09:14 -0500 Subject: [PATCH 8/8] Update mealie/db/fixes/fix_migration_data.py Co-authored-by: Kuchenpirat <24235032+Kuchenpirat@users.noreply.github.com> --- mealie/db/fixes/fix_migration_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mealie/db/fixes/fix_migration_data.py b/mealie/db/fixes/fix_migration_data.py index 0cdd036b123..d594fce14a7 100644 --- a/mealie/db/fixes/fix_migration_data.py +++ b/mealie/db/fixes/fix_migration_data.py @@ -40,7 +40,7 @@ def fix_dangling_refs(session: Session): update(table) .where( and_( - ~table.c.user_id.in_(valid_user_ids), + table.c.user_id.notin_(valid_user_ids), table.c.group_id == group.id, ) )