From 6dcf74c86096c6bfdb4b8e4b2bfbb356834d03d4 Mon Sep 17 00:00:00 2001 From: Paul Sanders Date: Mon, 18 Jul 2022 09:42:10 -0400 Subject: [PATCH] Add fideslib models auth (#915) * Add fideslib models * Fix migration * Fix migration issues * Update CHANGELOG * Run isort * Merge alembic heads to fix error after merging main * Run black and isort * Add new tables to annotations Co-authored-by: Paul Sanders --- .fides/dataset.yml | 234 +++++++++ CHANGELOG.md | 2 + src/fidesctl/api/database/crud.py | 16 +- src/fidesctl/api/database/database.py | 5 +- src/fidesctl/api/migrations/env.py | 4 +- .../155fd8e51d9d_add_fideslib_models.py | 480 ++++++++++++++++++ ...51d8a102a_add_created_at_and_updated_at.py | 54 +- .../versions/f53e04e5b7f5_merge_heads.py | 23 + src/fidesctl/api/sql_models.py | 40 +- 9 files changed, 800 insertions(+), 58 deletions(-) create mode 100644 src/fidesctl/api/migrations/versions/155fd8e51d9d_add_fideslib_models.py create mode 100644 src/fidesctl/api/migrations/versions/f53e04e5b7f5_merge_heads.py diff --git a/.fides/dataset.yml b/.fides/dataset.yml index 9ee660f796..0843bd084a 100644 --- a/.fides/dataset.yml +++ b/.fides/dataset.yml @@ -26,6 +26,123 @@ dataset: data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified retention: null fields: null + - name: auditlog + description: null + data_categories: null + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + retention: null + fields: + - name: created_at + description: null + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + retention: null + fields: null + - name: action + description: null + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + retention: null + fields: null + - name: id + description: null + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + retention: null + fields: null + - name: message + description: null + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + retention: null + fields: null + - name: privacy_request_id + description: null + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + retention: null + fields: null + - name: updated_at + description: null + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + retention: null + fields: null + - name: user_id + description: null + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + retention: null + fields: null + - name: client + description: null + data_categories: null + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + retention: null + fields: + - name: created_at + description: null + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + retention: null + fields: null + - name: fides_key + description: null + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + retention: null + fields: null + - name: hashed_secret + description: null + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + retention: null + fields: null + - name: id + description: null + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + retention: null + fields: null + - name: salt + description: null + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + retention: null + fields: null + - name: scopes + description: null + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + retention: null + fields: null + - name: updated_at + description: null + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + retention: null + fields: null + - name: user_id + description: null + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + retention: null + fields: null - name: data_categories description: null data_categories: null @@ -551,6 +668,123 @@ dataset: data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified retention: null fields: null + - name: fidesuser + description: null + data_categories: null + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + retention: null + fields: + - name: created_at + description: null + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + retention: null + fields: null + - name: first_name + description: null + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + retention: null + fields: null + - name: hashed_password + description: null + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + retention: null + fields: null + - name: id + description: null + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + retention: null + fields: null + - name: last_login_at + description: null + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + retention: null + fields: null + - name: last_name + description: null + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + retention: null + fields: null + - name: password_reset_at + description: null + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + retention: null + fields: null + - name: salt + description: null + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + retention: null + fields: null + - name: updated_at + description: The timestamp of when the row was created + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + retention: null + fields: null + - name: username + description: null + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + retention: null + fields: null + - name: fidesuserpermissions + description: null + data_categories: null + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + retention: null + fields: + - name: created_at + description: null + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + retention: null + fields: null + - name: id + description: null + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + retention: null + fields: null + - name: scopes + description: null + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + retention: null + fields: null + - name: updated_at + description: null + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + retention: null + fields: null + - name: user_id + description: null + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + retention: null + fields: null - name: organizations description: null data_categories: null diff --git a/CHANGELOG.md b/CHANGELOG.md index f892aaec8b..a81fc06250 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ The types of changes are: * Add db support to `/generate` endpoint [849](https://github.com/ethyca/fides/pull/849) * Add BigQuery support for the `generate` cli command [#814](https://github.com/ethyca/fides/pull/814) * Added OpenAPI TypeScript client generation for the UI app. See the [README](/clients/admin-ui/src/types/api/README.md) for more details. +* Added user auth tables [915](https://github.com/ethyca/fides/pull/915) * Standardized API error parsing under `~/types/errors` ### Changed @@ -44,6 +45,7 @@ The types of changes are: * Webserver dependencies now come as a standard part of the package [#881](https://github.com/ethyca/fides/pull/881) * Initial configuration wizard UI view * Refactored step & form results management to use Redux Toolkit slice. +* Change `id` field in tables from an integer to a string [915](https://github.com/ethyca/fides/pull/915) * Update `fideslang` to `1.1.0`, simplifying the default taxonomy and adding `tags` for resources [#865](https://github.com/ethyca/fides/pull/865) * Remove the `obscure` requirement from the `generate` endpoint [#819](https://github.com/ethyca/fides/pull/819) diff --git a/src/fidesctl/api/database/crud.py b/src/fidesctl/api/database/crud.py index d72d90c100..451ce84d95 100644 --- a/src/fidesctl/api/database/crud.py +++ b/src/fidesctl/api/database/crud.py @@ -4,6 +4,7 @@ """ from typing import Dict, List, Tuple +from fideslib.db.base import Base from loguru import logger as log from sqlalchemy import column from sqlalchemy import delete as _delete @@ -13,15 +14,14 @@ from sqlalchemy.future import select from fidesctl.api.database.session import async_session -from fidesctl.api.sql_models import SqlAlchemyBase from fidesctl.api.utils import errors # CRUD Functions async def create_resource( - sql_model: SqlAlchemyBase, + sql_model: Base, resource_dict: Dict, -) -> SqlAlchemyBase: +) -> Base: """Create a resource in the database.""" with log.contextualize( sql_model=sql_model.__name__, fides_key=resource_dict["fides_key"] @@ -53,7 +53,7 @@ async def create_resource( return await get_resource(sql_model, resource_dict["fides_key"]) -async def get_resource(sql_model: SqlAlchemyBase, fides_key: str) -> SqlAlchemyBase: +async def get_resource(sql_model: Base, fides_key: str) -> Base: """ Get a resource from the databse by its FidesKey. @@ -80,7 +80,7 @@ async def get_resource(sql_model: SqlAlchemyBase, fides_key: str) -> SqlAlchemyB return sql_resource -async def list_resource(sql_model: SqlAlchemyBase) -> List[SqlAlchemyBase]: +async def list_resource(sql_model: Base) -> List[Base]: """ Get a list of all of the resources of this type from the database. @@ -104,7 +104,7 @@ async def list_resource(sql_model: SqlAlchemyBase) -> List[SqlAlchemyBase]: return sql_resources -async def update_resource(sql_model: SqlAlchemyBase, resource_dict: Dict) -> Dict: +async def update_resource(sql_model: Base, resource_dict: Dict) -> Dict: """Update a resource in the database by its fides_key.""" with log.contextualize( @@ -131,7 +131,7 @@ async def update_resource(sql_model: SqlAlchemyBase, resource_dict: Dict) -> Dic async def upsert_resources( - sql_model: SqlAlchemyBase, + sql_model: Base, resource_dicts: List[Dict], ) -> Tuple[int, int]: """ @@ -180,7 +180,7 @@ async def upsert_resources( raise error -async def delete_resource(sql_model: SqlAlchemyBase, fides_key: str) -> SqlAlchemyBase: +async def delete_resource(sql_model: Base, fides_key: str) -> Base: """Delete a resource by its fides_key.""" with log.contextualize(sql_model=sql_model.__name__, fides_key=fides_key): diff --git a/src/fidesctl/api/database/database.py b/src/fidesctl/api/database/database.py index 16ffa1de8f..4c3b7ab4fe 100644 --- a/src/fidesctl/api/database/database.py +++ b/src/fidesctl/api/database/database.py @@ -7,11 +7,12 @@ from alembic.config import Config from alembic.runtime import migration from fideslang import DEFAULT_TAXONOMY +from fideslib.db.base import Base from loguru import logger as log from sqlalchemy import create_engine from sqlalchemy_utils.functions import create_database, database_exists -from fidesctl.api.sql_models import SqlAlchemyBase, sql_model_map +from fidesctl.api.sql_models import sql_model_map from fidesctl.api.utils.errors import ( AlreadyExistsError, QueryError, @@ -107,7 +108,7 @@ def reset_db(database_url: str) -> None: log.info("Resetting database") engine = get_db_engine(database_url) connection = engine.connect() - SqlAlchemyBase.metadata.drop_all(connection) + Base.metadata.drop_all(connection) migration_context = migration.MigrationContext.configure(connection) version = migration_context._version # pylint: disable=protected-access diff --git a/src/fidesctl/api/migrations/env.py b/src/fidesctl/api/migrations/env.py index bb58fa9f1e..ab35c89a68 100644 --- a/src/fidesctl/api/migrations/env.py +++ b/src/fidesctl/api/migrations/env.py @@ -24,9 +24,9 @@ # for 'autogenerate' support # from myapp import mymodel # target_metadata = mymodel.Base.metadata -from fidesctl.api.sql_models import SqlAlchemyBase +from fidesctl.api.sql_models import Base -target_metadata = SqlAlchemyBase.metadata +target_metadata = Base.metadata # other values from the config, defined by the needs of env.py, # can be acquired: diff --git a/src/fidesctl/api/migrations/versions/155fd8e51d9d_add_fideslib_models.py b/src/fidesctl/api/migrations/versions/155fd8e51d9d_add_fideslib_models.py new file mode 100644 index 0000000000..93f6ce01ff --- /dev/null +++ b/src/fidesctl/api/migrations/versions/155fd8e51d9d_add_fideslib_models.py @@ -0,0 +1,480 @@ +"""Add fideslib models + +Revision ID: 155fd8e51d9d +Revises: be432bd23596 +Create Date: 2022-07-09 01:13:23.440193 + +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "155fd8e51d9d" +down_revision = "be432bd23596" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "auditlog", + sa.Column("id", sa.String(length=255), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=True, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=True, + ), + sa.Column("user_id", sa.String(), nullable=True), + sa.Column("privacy_request_id", sa.String(), nullable=True), + sa.Column( + "action", + sa.Enum("approved", "denied", name="auditlogaction"), + nullable=False, + ), + sa.Column("message", sa.String(), nullable=True), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index(op.f("ix_auditlog_action"), "auditlog", ["action"], unique=False) + op.create_index(op.f("ix_auditlog_id"), "auditlog", ["id"], unique=False) + op.create_index( + op.f("ix_auditlog_privacy_request_id"), + "auditlog", + ["privacy_request_id"], + unique=False, + ) + op.create_index(op.f("ix_auditlog_user_id"), "auditlog", ["user_id"], unique=False) + op.create_table( + "fidesuser", + sa.Column("id", sa.String(length=255), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=True, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=True, + ), + sa.Column("username", sa.String(), nullable=True), + sa.Column("first_name", sa.String(), nullable=True), + sa.Column("last_name", sa.String(), nullable=True), + sa.Column("hashed_password", sa.String(), nullable=False), + sa.Column("salt", sa.String(), nullable=False), + sa.Column("last_login_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("password_reset_at", sa.DateTime(timezone=True), nullable=True), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index(op.f("ix_fidesuser_id"), "fidesuser", ["id"], unique=False) + op.create_index( + op.f("ix_fidesuser_username"), "fidesuser", ["username"], unique=True + ) + op.create_table( + "client", + sa.Column("id", sa.String(length=255), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=True, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=True, + ), + sa.Column("hashed_secret", sa.String(), nullable=False), + sa.Column("salt", sa.String(), nullable=False), + sa.Column("scopes", sa.ARRAY(sa.String()), nullable=False), + sa.Column("fides_key", sa.String(), nullable=True), + sa.Column("user_id", sa.String(), nullable=True), + sa.ForeignKeyConstraint( + ["user_id"], + ["fidesuser.id"], + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("user_id"), + ) + op.create_index(op.f("ix_client_fides_key"), "client", ["fides_key"], unique=True) + op.create_index(op.f("ix_client_id"), "client", ["id"], unique=False) + op.create_table( + "fidesuserpermissions", + sa.Column("id", sa.String(length=255), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=True, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=True, + ), + sa.Column("user_id", sa.String(), nullable=False), + sa.Column("scopes", sa.ARRAY(sa.String()), nullable=False), + sa.ForeignKeyConstraint( + ["user_id"], + ["fidesuser.id"], + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("user_id"), + ) + op.create_index( + op.f("ix_fidesuserpermissions_id"), "fidesuserpermissions", ["id"], unique=False + ) + + # data_categories + op.execute("DROP SEQUENCE data_categories_id_seq CASCADE") + op.execute( + "ALTER TABLE data_categories DROP CONSTRAINT data_categories_pkey CASCADE" + ) + op.alter_column( + table_name="data_categories", + column_name="id", + existing_type=sa.Integer, + type_=sa.String(255), + nullable=False, + ) + op.create_primary_key( + table_name="data_categories", + constraint_name="data_categories_pkey", + columns=["id"], + ) + + # data_qualifiers + op.execute("DROP SEQUENCE data_qualifiers_id_seq CASCADE") + op.execute( + "ALTER TABLE data_qualifiers DROP CONSTRAINT data_qualifiers_pkey CASCADE" + ) + op.alter_column( + table_name="data_qualifiers", + column_name="id", + existing_type=sa.Integer, + type_=sa.String(255), + nullable=False, + ) + op.create_primary_key( + table_name="data_qualifiers", + constraint_name="data_qualifiers_pkey", + columns=["id"], + ) + + # data_subjects + op.execute("DROP SEQUENCE data_subjects_id_seq CASCADE") + op.execute("ALTER TABLE data_subjects DROP CONSTRAINT data_subjects_pkey CASCADE") + op.alter_column( + table_name="data_subjects", + column_name="id", + existing_type=sa.Integer, + type_=sa.String(255), + nullable=False, + ) + op.create_primary_key( + table_name="data_subjects", + constraint_name="data_subjects_pkey", + columns=["id"], + ) + + # data_uses + op.execute("DROP SEQUENCE data_uses_id_seq CASCADE") + op.execute("ALTER TABLE data_uses DROP CONSTRAINT data_uses_pkey CASCADE") + op.alter_column( + table_name="data_uses", + column_name="id", + existing_type=sa.Integer, + type_=sa.String(255), + nullable=False, + ) + op.create_primary_key( + table_name="data_uses", constraint_name="data_uses_pkey", columns=["id"] + ) + + # datasets + op.execute("DROP SEQUENCE datasets_id_seq CASCADE") + op.execute("ALTER TABLE datasets DROP CONSTRAINT datasets_pkey CASCADE") + op.alter_column( + table_name="datasets", + column_name="id", + existing_type=sa.Integer, + type_=sa.String(255), + nullable=False, + ) + op.create_primary_key( + table_name="datasets", + constraint_name="data_sets_pkey", + columns=["id"], + ) + + # evaluations + op.add_column( + "evaluations", + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=True, + ), + ) + op.add_column( + "evaluations", + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=True, + ), + ) + op.execute("DROP SEQUENCE evaluations_id_seq CASCADE") + op.execute("ALTER TABLE evaluations DROP CONSTRAINT evaluations_pkey CASCADE") + op.alter_column( + table_name="evaluations", + column_name="id", + existing_type=sa.Integer, + type_=sa.String(255), + nullable=False, + ) + op.create_primary_key( + table_name="evaluations", + constraint_name="evaluations_pkey", + columns=["id"], + ) + + # organizations + op.execute("DROP SEQUENCE organizations_id_seq CASCADE") + op.execute("ALTER TABLE organizations DROP CONSTRAINT organizations_pkey CASCADE") + op.alter_column( + table_name="organizations", + column_name="id", + existing_type=sa.Integer, + type_=sa.String(255), + nullable=False, + ) + op.create_primary_key( + table_name="organizations", constraint_name="organizations_pkey", columns=["id"] + ) + + # policies + op.execute("DROP SEQUENCE policies_id_seq CASCADE") + op.execute("ALTER TABLE policies DROP CONSTRAINT policies_pkey CASCADE") + op.alter_column( + table_name="policies", + column_name="id", + existing_type=sa.Integer, + type_=sa.String(255), + nullable=False, + ) + op.create_primary_key( + table_name="policies", constraint_name="policies_pkey", columns=["id"] + ) + + # registries + op.execute("DROP SEQUENCE registries_id_seq CASCADE") + op.execute("ALTER TABLE registries DROP CONSTRAINT registries_pkey CASCADE") + op.alter_column( + table_name="registries", + column_name="id", + existing_type=sa.Integer, + type_=sa.String(255), + nullable=False, + ) + op.create_primary_key( + table_name="registries", constraint_name="registries_pkey", columns=["id"] + ) + + # systems + op.execute("DROP SEQUENCE systems_id_seq CASCADE") + op.execute("ALTER TABLE systems DROP CONSTRAINT systems_pkey CASCADE") + op.alter_column( + table_name="systems", + column_name="id", + existing_type=sa.Integer, + type_=sa.String(255), + nullable=False, + ) + op.create_primary_key( + table_name="systems", constraint_name="systems_pkey", columns=["id"] + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + + # systems + op.execute("ALTER TABLE systems DROP CONSTRAINT systems_pkey CASCADE") + op.execute("CREATE SEQUENCE systems_id_seq") + op.alter_column( + table_name="systems", + column_name="id", + existing_type=sa.String(255), + type_=sa.Integer, + nullable=False, + ) + op.create_primary_key( + table_name="systems", constraint_name="systems_pkey", columns=["id"] + ) + + # registries + op.execute("ALTER TABLE registries DROP CONSTRAINT registries_pkey CASCADE") + op.execute("CREATE SEQUENCE registries_id_seq") + op.alter_column( + table_name="registries", + column_name="id", + existing_type=sa.String(255), + type_=sa.Integer, + nullable=False, + ) + op.create_primary_key( + table_name="registries", constraint_name="registries_pkey", columns=["id"] + ) + + # policies + + op.execute("ALTER TABLE policies DROP CONSTRAINT policies_pkey CASCADE") + op.execute("CREATE SEQUENCE policies_id_seq") + op.alter_column( + table_name="policies", + column_name="id", + existing_type=sa.String(255), + type_=sa.Integer, + nullable=False, + ) + op.create_primary_key( + table_name="policies", constraint_name="policies_pkey", columns=["id"] + ) + + # organizations + op.execute("ALTER TABLE organizations DROP CONSTRAINT organizations_pkey CASCADE") + op.execute("CREATE SEQUENCE policies_id_seq") + op.alter_column( + table_name="organizations", + column_name="id", + existing_type=sa.String(255), + type_=sa.Integer, + nullable=False, + ) + op.create_primary_key( + table_name="organizations", constraint_name="organizations_pkey", columns=["id"] + ) + + # evaluations + op.execute("ALTER TABLE evaluations DROP CONSTRAINT evaluations_pkey CASCADE") + op.execute("CREATE SEQUENCE policies_id_seq") + op.alter_column( + table_name="evaluations", + column_name="id", + existing_type=sa.String(255), + type_=sa.Integer, + nullable=False, + ) + op.create_primary_key( + table_name="evaluations", constraint_name="evaluations_pkey", columns=["id"] + ) + op.drop_column("evaluations", "updated_at") + op.drop_column("evaluations", "created_at") + + # datasets + op.execute("ALTER TABLE datasets DROP CONSTRAINT datasets_pkey CASCADE") + op.execute("CREATE SEQUENCE datasets_id_seq") + op.alter_column( + table_name="datasets", + column_name="id", + existing_type=sa.String(255), + type_=sa.Integer, + nullable=False, + ) + op.create_primary_key( + table_name="datasets", constraint_name="datasets_pkey", columns=["id"] + ) + + # data_uses + op.execute("ALTER TABLE data_uses DROP CONSTRAINT data_uses_pkey CASCADE") + op.execute("CREATE SEQUENCE data_uses_id_seq") + op.alter_column( + table_name="data_uses", + column_name="id", + existing_type=sa.String(255), + type_=sa.Integer, + nullable=False, + ) + op.create_primary_key( + table_name="data_uses", constraint_name="data_uses_pkey", columns=["id"] + ) + + # data_subjects + op.execute("ALTER TABLE data_subjects DROP CONSTRAINT data_subjects_pkey CASCADE") + op.execute("CREATE SEQUENCE data_uses_id_seq") + op.alter_column( + table_name="data_subjects", + column_name="id", + existing_type=sa.String(255), + type_=sa.Integer, + nullable=False, + ) + op.create_primary_key( + table_name="data_subjects", constraint_name="data_subjects_pkey", columns=["id"] + ) + + # data_qualifiers + op.execute( + "ALTER TABLE data_qualifiers DROP CONSTRAINT data_qualifiers_pkey CASCADE" + ) + op.execute("CREATE SEQUENCE data_uses_id_seq") + op.alter_column( + table_name="data_qualifiers", + column_name="id", + existing_type=sa.String(255), + type_=sa.Integer, + nullable=False, + ) + op.create_primary_key( + table_name="data_qualifiers", + constraint_name="data_qualifiers_pkey", + columns=["id"], + ) + + # data_categories + op.execute( + "ALTER TABLE data_categories DROP CONSTRAINT data_categories_pkey CASCADE" + ) + op.execute("CREATE SEQUENCE data_categories_id_seq") + op.alter_column( + table_name="data_categories", + column_name="id", + existing_type=sa.String(255), + type_=sa.Integer, + nullable=False, + ) + op.create_primary_key( + table_name="data_categories", + constraint_name="data_categories_pkey", + columns=["id"], + ) + + op.drop_index(op.f("ix_fidesuserpermissions_id"), table_name="fidesuserpermissions") + op.drop_table("fidesuserpermissions") + op.drop_index(op.f("ix_client_id"), table_name="client") + op.drop_index(op.f("ix_client_fides_key"), table_name="client") + op.drop_table("client") + op.drop_index(op.f("ix_fidesuser_username"), table_name="fidesuser") + op.drop_index(op.f("ix_fidesuser_id"), table_name="fidesuser") + op.drop_table("fidesuser") + op.drop_index(op.f("ix_auditlog_user_id"), table_name="auditlog") + op.drop_index(op.f("ix_auditlog_privacy_request_id"), table_name="auditlog") + op.drop_index(op.f("ix_auditlog_id"), table_name="auditlog") + op.drop_index(op.f("ix_auditlog_action"), table_name="auditlog") + op.drop_table("auditlog") + # ### end Alembic commands ### diff --git a/src/fidesctl/api/migrations/versions/7c851d8a102a_add_created_at_and_updated_at.py b/src/fidesctl/api/migrations/versions/7c851d8a102a_add_created_at_and_updated_at.py index 9ad9c77a6f..2524d16842 100644 --- a/src/fidesctl/api/migrations/versions/7c851d8a102a_add_created_at_and_updated_at.py +++ b/src/fidesctl/api/migrations/versions/7c851d8a102a_add_created_at_and_updated_at.py @@ -18,28 +18,38 @@ def upgrade(): - for model in sql_model_map.values(): - op.add_column( - model.__tablename__, - Column( - "created_at", - DateTime(timezone=True), - server_default=text("now()"), - nullable=True, - ), - ) - op.add_column( - model.__tablename__, - Column( - "updated_at", - DateTime(timezone=True), - server_default=text("now()"), - nullable=True, - ), - ) + for k, model in sql_model_map.items(): + if k not in ( + "client_detail", + "fides_user", + "fides_user_permissions", + ): + op.add_column( + model.__tablename__, + Column( + "created_at", + DateTime(timezone=True), + server_default=text("now()"), + nullable=True, + ), + ) + op.add_column( + model.__tablename__, + Column( + "updated_at", + DateTime(timezone=True), + server_default=text("now()"), + nullable=True, + ), + ) def downgrade(): - for model in sql_model_map.values(): - op.drop_column(model.__tablename__, "created_at") - op.drop_column(model.__tablename__, "updated_at") + for k, model in sql_model_map.items(): + if k not in ( + "client_detail", + "fides_user", + "fides_user_permissions", + ): + op.drop_column(model.__tablename__, "created_at") + op.drop_column(model.__tablename__, "updated_at") diff --git a/src/fidesctl/api/migrations/versions/f53e04e5b7f5_merge_heads.py b/src/fidesctl/api/migrations/versions/f53e04e5b7f5_merge_heads.py new file mode 100644 index 0000000000..6659feeb7c --- /dev/null +++ b/src/fidesctl/api/migrations/versions/f53e04e5b7f5_merge_heads.py @@ -0,0 +1,23 @@ +"""Merge heads + +Revision ID: f53e04e5b7f5 +Revises: 155fd8e51d9d, d8f8bd52754f +Create Date: 2022-07-16 02:48:43.622664 + +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "f53e04e5b7f5" +down_revision = ("155fd8e51d9d", "d8f8bd52754f") +branch_labels = None +depends_on = None + + +def upgrade(): + pass + + +def downgrade(): + pass diff --git a/src/fidesctl/api/sql_models.py b/src/fidesctl/api/sql_models.py index 22d499b30c..3f214a6d00 100644 --- a/src/fidesctl/api/sql_models.py +++ b/src/fidesctl/api/sql_models.py @@ -6,12 +6,13 @@ from typing import Dict +from fideslib.db.base import Base, ClientDetail, FidesUser, FidesUserPermissions +from fideslib.db.base_class import FidesBase as FideslibBase from sqlalchemy import ( ARRAY, BOOLEAN, JSON, Column, - Integer, String, Text, TypeDecorator, @@ -19,7 +20,6 @@ type_coerce, ) from sqlalchemy.dialects.postgresql import BYTEA -from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.sql import func from sqlalchemy.sql.sqltypes import DateTime @@ -28,15 +28,7 @@ CONFIG: FidesctlConfig = get_config() -class SqlModelBase: - """ - This is the base class used to describe columns that every object should have. - """ - - id = Column(Integer, primary_key=True, index=True, unique=True, autoincrement=True) - - -class FidesBase(SqlModelBase): +class FidesBase(FideslibBase): """ The base SQL model for all top-level Fides Resources. """ @@ -54,9 +46,6 @@ class FidesBase(SqlModelBase): ) -SqlAlchemyBase = declarative_base(cls=SqlModelBase) - - class PGEncryptedString(TypeDecorator): """ This TypeDecorator handles encrypting and decrypting values at rest @@ -97,7 +86,7 @@ def process_result_value(self, value, dialect): # Privacy Types -class DataCategory(SqlAlchemyBase, FidesBase): +class DataCategory(Base, FidesBase): """ The SQL model for the DataCategory resource. """ @@ -107,7 +96,7 @@ class DataCategory(SqlAlchemyBase, FidesBase): parent_key = Column(Text) -class DataQualifier(SqlAlchemyBase, FidesBase): +class DataQualifier(Base, FidesBase): """ The SQL model for the DataQualifier resource. """ @@ -117,7 +106,7 @@ class DataQualifier(SqlAlchemyBase, FidesBase): parent_key = Column(Text) -class DataSubject(SqlAlchemyBase, FidesBase): +class DataSubject(Base, FidesBase): """ The SQL model for the DataSubject resource. """ @@ -127,7 +116,7 @@ class DataSubject(SqlAlchemyBase, FidesBase): automated_decisions_or_profiling = Column(BOOLEAN, nullable=True) -class DataUse(SqlAlchemyBase, FidesBase): +class DataUse(Base, FidesBase): """ The SQL model for the DataUse resource. """ @@ -143,7 +132,7 @@ class DataUse(SqlAlchemyBase, FidesBase): # Dataset -class Dataset(SqlAlchemyBase, FidesBase): +class Dataset(Base, FidesBase): """ The SQL model for the Dataset resource. """ @@ -161,7 +150,7 @@ class Dataset(SqlAlchemyBase, FidesBase): # Evaluation -class Evaluation(SqlAlchemyBase): +class Evaluation(Base): """ The SQL model for the Evaluation resource. """ @@ -175,7 +164,7 @@ class Evaluation(SqlAlchemyBase): # Organization -class Organization(SqlAlchemyBase, FidesBase): +class Organization(Base, FidesBase): """ The SQL model for the Organization resource. """ @@ -192,7 +181,7 @@ class Organization(SqlAlchemyBase, FidesBase): # Policy -class Policy(SqlAlchemyBase, FidesBase): +class Policy(Base, FidesBase): """ The SQL model for the Policy resource. """ @@ -203,7 +192,7 @@ class Policy(SqlAlchemyBase, FidesBase): # Registry -class Registry(SqlAlchemyBase, FidesBase): +class Registry(Base, FidesBase): """ The SQL model for the Registry resource. """ @@ -212,7 +201,7 @@ class Registry(SqlAlchemyBase, FidesBase): # System -class System(SqlAlchemyBase, FidesBase): +class System(Base, FidesBase): """ The SQL model for the system resource. """ @@ -233,11 +222,14 @@ class System(SqlAlchemyBase, FidesBase): sql_model_map: Dict = { + "client_detail": ClientDetail, "data_category": DataCategory, "data_qualifier": DataQualifier, "data_subject": DataSubject, "data_use": DataUse, "dataset": Dataset, + "fides_user": FidesUser, + "fides_user_permissions": FidesUserPermissions, "organization": Organization, "policy": Policy, "registry": Registry,