From 6b1baaf0fa5f4df5eba29543a0c7f079d3404348 Mon Sep 17 00:00:00 2001 From: Catherine Smith Date: Mon, 10 Jan 2022 18:20:59 -0500 Subject: [PATCH] Fidesops 78 mssql support (#151) Co-authored-by: Dawn Pattison Co-authored-by: catherinesmith Co-authored-by: Catherine Smith --- Dockerfile | 16 +- Makefile | 14 +- data/sql/mssql_example.sql | 84 ++++++ docker-compose.integration-test.yml | 11 + docs/fidesops/docs/development/overview.md | 9 +- .../docs/guides/database_connectors.md | 19 +- .../postman/Fidesops.postman_collection.json | 79 ++++++ fidesops-integration.toml | 10 +- requirements.txt | 1 + .../api/v1/endpoints/connection_endpoints.py | 4 +- src/fidesops/models/connectionconfig.py | 7 +- .../connection_configuration/__init__.py | 6 + .../connection_secrets_mssql.py | 27 ++ src/fidesops/service/connectors/__init__.py | 2 + .../service/connectors/sql_connector.py | 43 ++- .../versions/f3841942d90c_add_mssql.py | 35 +++ tests/fixtures.py | 23 ++ tests/integration_tests/mssql_setup.py | 21 ++ ...st_connection_configuration_integration.py | 260 +++++++++++++++++- .../test_integration_mssql_example.py | 52 ++++ 20 files changed, 706 insertions(+), 17 deletions(-) create mode 100644 data/sql/mssql_example.sql create mode 100644 src/fidesops/schemas/connection_configuration/connection_secrets_mssql.py create mode 100644 src/migrations/versions/f3841942d90c_add_mssql.py create mode 100644 tests/integration_tests/mssql_setup.py create mode 100644 tests/integration_tests/test_integration_mssql_example.py diff --git a/Dockerfile b/Dockerfile index 52eea6893..bbb8432a2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9.6-slim-buster +FROM --platform=linux/amd64 python:3.9.6-slim-buster # Install auxiliary software RUN apt-get update && \ @@ -8,8 +8,22 @@ RUN apt-get update && \ ipython \ vim \ curl \ + g++ \ + gnupg \ gcc +# SQL Server (MS SQL) +# https://docs.microsoft.com/en-us/sql/connect/odbc/linux-mac/installing-the-microsoft-odbc-driver-for-sql-server?view=sql-server-ver15 +RUN apt-get install apt-transport-https +RUN curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - +RUN curl https://packages.microsoft.com/config/debian/10/prod.list | tee /etc/apt/sources.list.d/msprod.list +RUN apt-get update +ENV ACCEPT_EULA=y DEBIAN_FRONTEND=noninteractive +RUN apt-get -y install \ + unixodbc-dev \ + msodbcsql17 \ + mssql-tools + # Update pip and install requirements COPY requirements.txt dev-requirements.txt ./ RUN pip install -U pip \ diff --git a/Makefile b/Makefile index 0d21b115b..786af0503 100644 --- a/Makefile +++ b/Makefile @@ -48,7 +48,13 @@ integration-shell: compose-build @docker-compose -f docker-compose.yml -f docker-compose.integration-test.yml run $(IMAGE_NAME) /bin/bash integration-env: compose-build - @docker-compose -f docker-compose.yml -f docker-compose.integration-test.yml up + @echo "Bringing up main image and images for integration testing" + @docker-compose -f docker-compose.yml -f docker-compose.integration-test.yml up -d + @echo "Waiting 15s for integration containers to be ready..." + @sleep 15 + @echo "Running additional setup for mssql integration tests" + @docker exec -it fidesops python tests/integration_tests/mssql_setup.py + @docker-compose -f docker-compose.yml -f docker-compose.integration-test.yml logs -f -t quickstart: compose-build @docker-compose -f docker-compose.yml -f docker-compose.integration-test.yml up -d @@ -107,8 +113,10 @@ pytest-integration-access: compose-build @docker-compose -f docker-compose.yml -f docker-compose.integration-test.yml build @echo "Bringing up the integration environment..." @docker-compose -f docker-compose.yml -f docker-compose.integration-test.yml up -d - @echo "Waiting 10s for integration containers to be ready..." - @sleep 10 + @echo "Waiting 15s for integration containers to be ready..." + @sleep 15 + @echo "Running additional setup for mssql integration tests" + @docker exec fidesops python tests/integration_tests/mssql_setup.py @echo "Running pytest integration tests..." @docker-compose -f docker-compose.yml -f docker-compose.integration-test.yml \ run $(IMAGE_NAME) \ diff --git a/data/sql/mssql_example.sql b/data/sql/mssql_example.sql new file mode 100644 index 000000000..46e99a850 --- /dev/null +++ b/data/sql/mssql_example.sql @@ -0,0 +1,84 @@ +USE master; + +-- CREATE USER IF NOT EXISTS 'sa'@'mssql_example' IDENTIFIED BY 'Mssql_pw1'; +-- GRANT ALL PRIVILEGES ON *.* TO 'sa'@'mssql_example' ; +-- GRANT ALL PRIVILEGES ON *.* TO 'sa'@'%' ; +-- FLUSH PRIVILEGES; + +DROP DATABASE IF EXISTS mssql_example; +CREATE DATABASE mssql_example; +USE mssql_example; + +DROP TABLE IF EXISTS report; +DROP TABLE IF EXISTS service_request; +DROP TABLE IF EXISTS login; +DROP TABLE IF EXISTS visit; +DROP TABLE IF EXISTS order_item; +DROP TABLE IF EXISTS orders; +DROP TABLE IF EXISTS payment_card; +DROP TABLE IF EXISTS employee; +DROP TABLE IF EXISTS customer; +DROP TABLE IF EXISTS address; +DROP TABLE IF EXISTS product; +DROP TABLE IF EXISTS composite_pk_test; +DROP TABLE IF EXISTS type_link_test; + + +CREATE TABLE product ( id INT PRIMARY KEY, name CHARACTER VARYING(100), price MONEY); + +CREATE TABLE address ( id BIGINT PRIMARY KEY, house INT, street CHARACTER VARYING(100), city CHARACTER VARYING(100), state CHARACTER VARYING(100), zip CHARACTER VARYING(100)); + +CREATE TABLE customer ( id INT PRIMARY KEY, email CHARACTER VARYING(100), name CHARACTER VARYING(100), created DATETIME, address_id BIGINT); + +CREATE TABLE employee ( id INT PRIMARY KEY, email CHARACTER VARYING(100), name CHARACTER VARYING(100), address_id BIGINT); + +CREATE TABLE payment_card ( id CHARACTER VARYING(100) PRIMARY KEY, name CHARACTER VARYING(100), ccn BIGINT, code SMALLINT, preferred BIT, customer_id INT, billing_address_id BIGINT); + +CREATE TABLE orders ( id CHARACTER VARYING(100) PRIMARY KEY, customer_id INT, shipping_address_id BIGINT, payment_card_id CHARACTER VARYING(100)); + +CREATE TABLE order_item ( order_id CHARACTER VARYING(100), item_no SMALLINT, product_id INT, quantity SMALLINT, CONSTRAINT order_item_pk PRIMARY KEY (order_id, item_no)); + +CREATE TABLE visit ( email CHARACTER VARYING(100), last_visit DATETIME, CONSTRAINT visit_pk PRIMARY KEY (email, last_visit)); + +CREATE TABLE login ( id INT PRIMARY KEY, customer_id INT, time DATETIME); + +CREATE TABLE service_request ( id CHARACTER VARYING(100) PRIMARY KEY, email CHARACTER VARYING(100), alt_email CHARACTER VARYING(100), opened DATE, closed DATE, employee_id INT); + +CREATE TABLE report ( id INT PRIMARY KEY, email CHARACTER VARYING(100), name CHARACTER VARYING(100), year INT, month INT, total_visits INT); + +CREATE TABLE composite_pk_test ( id_a INT NOT NULL, id_b INT NOT NULL, description VARCHAR(100), customer_id INT, PRIMARY KEY(id_a, id_b)); + +INSERT INTO composite_pk_test VALUES (1,10,'linked to customer 1',1), (1,11,'linked to customer 2',2), (2,10,'linked to customer 3',3); + +CREATE TABLE type_link_test ( id CHARACTER VARYING(100) PRIMARY KEY, name CHARACTER VARYING(100)); + +-- Populate tables with some public data +INSERT INTO product VALUES (1, 'Example Product 1', '$10.00'), (2, 'Example Product 2', '$20.00'), (3, 'Example Product 3', '$50.00'); + +INSERT INTO address VALUES (1, '123', 'Example Street', 'Exampletown', 'NY', '12345'), (2, '4', 'Example Lane', 'Exampletown', 'NY', '12321'), (3, '555', 'Example Ave', 'Example City', 'NY', '12000'), (4, '1111', 'Example Place', 'Example Mountain', 'TX', '54321'); + + +INSERT INTO customer VALUES (1, 'customer-1@example.com', 'John Customer', '2020-04-01 11:47:42', 1), (2, 'customer-2@example.com', 'Jill Customer', '2020-04-01 11:47:42', 2), (3, 'jane@example.com', 'Jane Customer', '2020-04-01 11:47:42', 4); + + +INSERT INTO employee VALUES (1, 'employee-1@example.com', 'Jack Employee', 3), (2, 'employee-2@example.com', 'Jane Employee', 3); + +INSERT INTO payment_card VALUES ('pay_aaa-aaa', 'Example Card 1', 123456789, 321, 1, 1, 1), ('pay_bbb-bbb', 'Example Card 2', 987654321, 123, 0, 2, 1), ('pay_ccc-ccc', 'Example Card 3', 373719391, 222, 0, 3, 4); + + +INSERT INTO orders VALUES ('ord_aaa-aaa', 1, 2, 'pay_aaa-aaa'), ('ord_bbb-bbb', 2, 1, 'pay_bbb-bbb'), ('ord_ccc-ccc', 1, 1, 'pay_aaa-aaa'), ('ord_ddd-ddd', 1, 1, 'pay_bbb-bbb'), ('ord_ddd-eee', 3, 4, 'pay-ccc-ccc'); + + +INSERT INTO order_item VALUES ('ord_aaa-aaa', 1, 1, 1), ('ord_bbb-bbb', 1, 1, 1), ('ord_ccc-ccc', 1, 1, 1), ('ord_ccc-ccc', 2, 2, 1), ('ord_ddd-ddd', 1, 1, 1), ('ord_eee-eee', 3, 4, 3); + + +INSERT INTO visit VALUES ('customer-1@example.com', '2021-01-06 01:00:00'), ('customer-2@example.com', '2021-01-06 01:00:00'); + +INSERT INTO login VALUES (1, 1, '2021-01-01 01:00:00'), (2, 1, '2021-01-02 01:00:00'), (3, 1, '2021-01-03 01:00:00'), (4, 1, '2021-01-04 01:00:00'), (5, 1, '2021-01-05 01:00:00'), (6, 1, '2021-01-06 01:00:00'), (7, 2, '2021-01-06 01:00:00'), (8, 3, '2021-01-06 01:00:00'); + + +INSERT INTO service_request VALUES ('ser_aaa-aaa', 'customer-1@example.com', 'customer-1-alt@example.com', '2021-01-01', '2021-01-03', 1), ('ser_bbb-bbb', 'customer-2@example.com', null, '2021-01-04', null, 1), ('ser_ccc-ccc', 'customer-3@example.com', null, '2021-01-05', '2020-01-07', 1), ('ser_ddd-ddd', 'customer-3@example.com', null, '2021-05-05', '2020-05-08', 2); + +INSERT INTO report VALUES (1, 'admin-account@example.com', 'Monthly Report', 2021, 8, 100), (2, 'admin-account@example.com', 'Monthly Report', 2021, 9, 100), (3, 'admin-account@example.com', 'Monthly Report', 2021, 10, 100), (4, 'admin-account@example.com', 'Monthly Report', 2021, 11, 100); + +INSERT INTO type_link_test VALUES ('1', 'name1'), ('2', 'name2'); \ No newline at end of file diff --git a/docker-compose.integration-test.yml b/docker-compose.integration-test.yml index ad10d8ca3..14956d9d9 100644 --- a/docker-compose.integration-test.yml +++ b/docker-compose.integration-test.yml @@ -4,6 +4,7 @@ services: - postgres_example - mongodb_example - mysql_example + - mssql_example postgres_example: image: postgres:12 @@ -50,3 +51,13 @@ services: - "3306:3306" volumes: - ./data/sql/mysql_example.sql:/docker-entrypoint-initdb.d/mysql_example.sql + + mssql_example: + image: mcr.microsoft.com/azure-sql-edge:latest # Equivalent to SQL Server 2016 + ports: + - 1433:1433 + environment: + - SA_PASSWORD=Mssql_pw1 + - ACCEPT_EULA="Y" + + diff --git a/docs/fidesops/docs/development/overview.md b/docs/fidesops/docs/development/overview.md index a795948be..32e270594 100644 --- a/docs/fidesops/docs/development/overview.md +++ b/docs/fidesops/docs/development/overview.md @@ -29,8 +29,8 @@ commands to give you different functionality. - `make server-shell`- opens a shell on the Docker container, from here you can run useful commands like: - `ipython` - open a Python shell - `make pytest` - runs all unit tests except those that talk to integration databases -- `make pytest-integration-access` - runs access integration tests -- `make pytest-integration-erasure` - runs erasure integration tests +- `make pytest-integration-access` - runs access integration tests. +- `make pytest-integration-erasure` - runs erasure integration tests. - `make reset-db` - tears down the Fideops postgres db, then recreates and re-runs migrations. - `make quickstart` - runs a quick, five second quickstart that talks to the Fidesops API to execute privacy requests - `make check-migrations` - verifies there are no un-run migrations @@ -40,7 +40,10 @@ See [Makefile](https://github.com/ethyca/fidesops/blob/main/Makefile) for more o #### Issues -When running `make server`, if you get a `importlib.metadata.PackageNotFoundError: fidesops`, do `make server-shell`, + +- MSSQL: Known issues around connecting to MSSQL exist today for Apple M1 users. M1 users that wish to install `pyodbc` locally, please reference the workaround [here](https://github.com/mkleehammer/pyodbc/issues/846). + +- Package not found: When running `make server`, if you get a `importlib.metadata.PackageNotFoundError: fidesops`, do `make server-shell`, and then run `pip install -e .`. Verify Fidesops is installed with `pip list`. diff --git a/docs/fidesops/docs/guides/database_connectors.md b/docs/fidesops/docs/guides/database_connectors.md index b8ef5ecab..596b4a9a3 100644 --- a/docs/fidesops/docs/guides/database_connectors.md +++ b/docs/fidesops/docs/guides/database_connectors.md @@ -24,6 +24,7 @@ Fidesops supports connections to the following databases: * PostgreSQL * MongoDB * MySQL +* Microsoft SQLServer * Amazon Redshift * Snowflake @@ -33,7 +34,7 @@ Other platforms will be added in future releases. The connection between Fidesops and your database is represented by a _ConnectionConfig_ object. To create a ConnectionConfig, you issue a request to the [Create a ConnectionConfig](/fidesops/api/#operations-Connections-put_connections_api_v1_connection_put) operation, passing a payload that contains the properties listed below. -* `name` is a a human-readable name for your database. +* `name` is a human-readable name for your database. * `key` is a string token that uniquely identifies your ConnectionConfig object. If you don't supply a `key`, the `name` value, converted to snake-case, is used. For example, if the `name` is `Application PostgreSQL DB`, the converted key is `application_postgresql_db`. @@ -90,10 +91,24 @@ PATCH api/v1/connection ] ``` +#### Example 4: MsSQL ConnectionConfig + +``` +PATCH api/v1/connection +[ + { + "name": "My MsSQL DB", + "key": "my_mssql_db", + "connection_type": "mssql", + "access": "write" + } +] +``` + ### Set the ConnectionConfig's Secrets -After you create a ConnectionConfig, you explain how to connect to it by setting its "secrets": host, port, user, and password. You do this by creating a ConnectionConfig Secrets object by calling the [Set a ConnectionConfig's Secrets](/fidesops/api#operations-Connections-put_connection_config_secrets_api_v1_connection__connection_key__secret_put) operation. You can set the object's attributes separately, or supply a single `url` string that encodes them all. +After you create a ConnectionConfig, you explain how to connect to it by setting its "secrets": host, port, user, and password (note that the secrets used are specific to the DB connector). You do this by creating a ConnectionConfig Secrets object by calling the [Set a ConnectionConfig's Secrets](/fidesops/api#operations-Connections-put_connection_config_secrets_api_v1_connection__connection_key__secret_put) operation. You can set the object's attributes separately, or supply a single `url` string that encodes them all. If you set the `verify` query parameter to `true`, the operation will test the connection by issuing a trivial request to the database. The `test_status` response property announces the success of the connection attempt as `succeeded` or `failed`. If the attempt has failed, the `failure_reason` property gives further details about the failure. diff --git a/docs/fidesops/docs/postman/Fidesops.postman_collection.json b/docs/fidesops/docs/postman/Fidesops.postman_collection.json index 5536930c4..30d8cf07e 100644 --- a/docs/fidesops/docs/postman/Fidesops.postman_collection.json +++ b/docs/fidesops/docs/postman/Fidesops.postman_collection.json @@ -336,6 +336,43 @@ }, "response": [] }, + { + "name": "Create/Update Connection Configs: MsSQL", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "[\n {\"name\": \"Application MsSQL DB\",\n \"key\": \"{{mssql_key}}\",\n \"connection_type\": \"mssql\",\n \"access\": \"read\"\n}]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/", + "host": [ + "{{host}}" + ], + "path": [ + "connection", + "" + ] + } + }, + "response": [] + }, { "name": "Create/Update Connection Configs: Mongo", "request": { @@ -411,6 +448,44 @@ }, "response": [] }, + { + "name": "Update Connection Secrets: MsSQL", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PUT", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"host\": \"mssql_example\",\n \"port\": 1433,\n \"dbname\": \"mssql_example\",\n \"username\": \"sa\",\n \"password\": \"Ms_sql1234\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/{{mssql_key}}/secret", + "host": [ + "{{host}}" + ], + "path": [ + "connection", + "{{mssql_key}}", + "secret" + ] + } + }, + "response": [] + }, { "name": "Update Connection Secrets: Mongo", "request": { @@ -1870,6 +1945,10 @@ "key": "postgres_key", "value": "app_postgres_db" }, + { + "key": "mssql_key", + "value": "app_mssql_db" + }, { "key": "mongo_key", "value": "app_mongo_db" diff --git a/fidesops-integration.toml b/fidesops-integration.toml index a7f910108..796a2e0e6 100644 --- a/fidesops-integration.toml +++ b/fidesops-integration.toml @@ -17,10 +17,18 @@ SERVER="mysql_example" USER="mysql_user" PASSWORD="mysql_pw" DB="mysql_example" -PORT= 3306 +PORT=3306 [redshift] external_uri="" [snowflake] external_uri="" + +[mssql_example] +SERVER="mssql_example" +USER="sa" +PASSWORD="Mssql_pw1" +DB="mssql_example" +PORT=1433 + diff --git a/requirements.txt b/requirements.txt index 54b98868f..b086bb7b9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,5 +28,6 @@ pymongo==3.12.0 pandas==1.3.3 click==7.1.2 PyMySQL==1.0.2 +pyodbc==4.0.32 sqlalchemy-redshift==0.8.8 snowflake-sqlalchemy==1.3.2 diff --git a/src/fidesops/api/v1/endpoints/connection_endpoints.py b/src/fidesops/api/v1/endpoints/connection_endpoints.py index 4d34a79cb..bf3d356cb 100644 --- a/src/fidesops/api/v1/endpoints/connection_endpoints.py +++ b/src/fidesops/api/v1/endpoints/connection_endpoints.py @@ -195,7 +195,9 @@ def connection_status( status: ConnectionTestStatus = connector.test_connection() except ConnectionException as exc: logger.warning( - "Connection test failed on %s: %s", NotPii(connection_config.key), str(exc) + "Connection test failed on %s: %s", + NotPii(connection_config.key), + str(exc), ) connection_config.update_test_status( test_status=ConnectionTestStatus.failed, db=db diff --git a/src/fidesops/models/connectionconfig.py b/src/fidesops/models/connectionconfig.py index ae92a931e..98a26d53b 100644 --- a/src/fidesops/models/connectionconfig.py +++ b/src/fidesops/models/connectionconfig.py @@ -35,7 +35,7 @@ class ConnectionTestStatus(enum.Enum): class ConnectionType(enum.Enum): """ - Supported types to which we can connect fides-ops. + Supported types to which we can connect fidesops. """ postgres = "postgres" @@ -44,11 +44,12 @@ class ConnectionType(enum.Enum): https = "https" redshift = "redshift" snowflake = "snowflake" + mssql = "mssql" class AccessLevel(enum.Enum): """ - Perms given to the ConnectionConfig. For example, with "read" permissions, fides-ops promises + Perms given to the ConnectionConfig. For example, with "read" permissions, fidesops promises to not modify the data on a connected application database in any way. "Write" perms mean we can update/delete items in the connected database. @@ -60,7 +61,7 @@ class AccessLevel(enum.Enum): class ConnectionConfig(Base): """ - Stores credentials to connect fides-ops to an engineer's application databases. + Stores credentials to connect fidesops to an engineer's application databases. """ name = Column(String, index=True, unique=True, nullable=False) diff --git a/src/fidesops/schemas/connection_configuration/__init__.py b/src/fidesops/schemas/connection_configuration/__init__.py index e1ac88943..cdac9c70c 100644 --- a/src/fidesops/schemas/connection_configuration/__init__.py +++ b/src/fidesops/schemas/connection_configuration/__init__.py @@ -7,6 +7,10 @@ from fidesops.schemas.connection_configuration.connection_secrets import ( ConnectionConfigSecretsSchema, ) +from fidesops.schemas.connection_configuration.connection_secrets_mssql import ( + MicrosoftSQLServerSchema, + MSSQLDocsSchema, +) from fidesops.schemas.connection_configuration.connection_secrets_mysql import ( MySQLSchema, MySQLDocsSchema, @@ -35,6 +39,7 @@ ConnectionType.mysql.value: MySQLSchema, ConnectionType.redshift.value: RedshiftSchema, ConnectionType.snowflake.value: SnowflakeSchema, + ConnectionType.mssql.value: MicrosoftSQLServerSchema, } @@ -61,4 +66,5 @@ def get_connection_secrets_validator( MySQLDocsSchema, RedshiftDocsSchema, SnowflakeDocsSchema, + MSSQLDocsSchema, ] diff --git a/src/fidesops/schemas/connection_configuration/connection_secrets_mssql.py b/src/fidesops/schemas/connection_configuration/connection_secrets_mssql.py new file mode 100644 index 000000000..5cfaa0db3 --- /dev/null +++ b/src/fidesops/schemas/connection_configuration/connection_secrets_mssql.py @@ -0,0 +1,27 @@ +from typing import Optional, List + +from fidesops.schemas.base_class import NoValidationSchema +from fidesops.schemas.connection_configuration.connection_secrets import ( + ConnectionConfigSecretsSchema, +) + + +class MicrosoftSQLServerSchema(ConnectionConfigSecretsSchema): + """Schema to validate the secrets needed to connect to a MS SQL Database + + connection string takes the format: + mssql+pyodbc://[username]:[password]@[host]:[port]/[dbname]?driver=ODBC+Driver+17+for+SQL+Server + + """ + + username: Optional[str] = None + password: Optional[str] = None + host: Optional[str] = None + port: Optional[int] = None + dbname: Optional[str] = None + + _required_components: List[str] = ["host"] + + +class MSSQLDocsSchema(MicrosoftSQLServerSchema, NoValidationSchema): + """MS SQL Secrets Schema for API Docs""" diff --git a/src/fidesops/service/connectors/__init__.py b/src/fidesops/service/connectors/__init__.py index d0560b452..6bc4c63d5 100644 --- a/src/fidesops/service/connectors/__init__.py +++ b/src/fidesops/service/connectors/__init__.py @@ -9,6 +9,7 @@ MySQLConnector, RedshiftConnector, SnowflakeConnector, + MicrosoftSQLServerConnector, ) supported_connectors: Dict[str, Any] = { @@ -18,6 +19,7 @@ ConnectionType.redshift.value: RedshiftConnector, ConnectionType.snowflake.value: SnowflakeConnector, ConnectionType.https.value: HTTPSConnector, + ConnectionType.mssql.value: MicrosoftSQLServerConnector, } diff --git a/src/fidesops/service/connectors/sql_connector.py b/src/fidesops/service/connectors/sql_connector.py index 440535f3c..22f7fd5d2 100644 --- a/src/fidesops/service/connectors/sql_connector.py +++ b/src/fidesops/service/connectors/sql_connector.py @@ -4,6 +4,7 @@ from sqlalchemy import Column, text from sqlalchemy.engine import ( + URL, Engine, create_engine, CursorResult, @@ -11,7 +12,7 @@ Connection, ) from sqlalchemy.exc import OperationalError, InternalError -from snowflake.sqlalchemy import URL +from snowflake.sqlalchemy import URL as Snowflake_URL from fidesops.common_exceptions import ConnectionException from fidesops.graph.traversal import Row, TraversalNode @@ -22,6 +23,7 @@ PostgreSQLSchema, RedshiftSchema, SnowflakeSchema, + MicrosoftSQLServerSchema, ) from fidesops.schemas.connection_configuration.connection_secrets_mysql import ( MySQLSchema, @@ -290,7 +292,7 @@ def build_uri(self) -> str: if config.role_name: kwargs["role"] = config.role_name - url: str = URL(**kwargs) + url: str = Snowflake_URL(**kwargs) return url def create_client(self) -> Engine: @@ -306,3 +308,40 @@ def create_client(self) -> Engine: def query_config(self, node: TraversalNode) -> SQLQueryConfig: """Query wrapper corresponding to the input traversal_node.""" return SnowflakeQueryConfig(node) + + +class MicrosoftSQLServerConnector(SQLConnector): + """ + Connector specific to Microsoft SQL Server + """ + + def build_uri(self) -> URL: + """ + Build URI of format + mssql+pyodbc://[username]:[password]@[host]:[port]/[dbname]?driver=ODBC+Driver+17+for+SQL+Server + Returns URL obj, since SQLAlchemy's create_engine method accepts either a URL obj or a string + """ + + config = MicrosoftSQLServerSchema(**self.configuration.secrets or {}) + + url = URL.create( + "mssql+pyodbc", + username=config.username, + password=config.password, + host=config.host, + port=config.port, + database=config.dbname, + query={"driver": "ODBC Driver 17 for SQL Server"}, + ) + + return url + + def create_client(self) -> Engine: + """Returns a SQLAlchemy Engine that can be used to interact with a MicrosoftSQLServer database""" + config = MicrosoftSQLServerSchema(**self.configuration.secrets or {}) + uri = config.url or self.build_uri() + return create_engine( + uri, + hide_parameters=self.hide_parameters, + echo=not self.hide_parameters, + ) diff --git a/src/migrations/versions/f3841942d90c_add_mssql.py b/src/migrations/versions/f3841942d90c_add_mssql.py new file mode 100644 index 000000000..394ce195a --- /dev/null +++ b/src/migrations/versions/f3841942d90c_add_mssql.py @@ -0,0 +1,35 @@ +"""add mssql (Microsoft SQL Server) + +Revision ID: f3841942d90c +Revises: d65e7e921814 +Create Date: 2021-12-13 22:15:25.043952 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "f3841942d90c" +down_revision = "d65e7e921814" +branch_labels = None +depends_on = None + + +def upgrade(): + op.execute("alter type connectiontype add value 'mssql'") + + +def downgrade(): + op.execute("delete from connectionconfig where connection_type in ('mssql')") + op.execute("alter type connectiontype rename to connectiontype_old") + op.execute( + "create type connectiontype as enum('postgres', 'mongodb', 'mysql', 'https', 'snowflake', 'redshift')" + ) + op.execute( + ( + "alter table connectionconfig alter column connection_type type connectiontype using " + "connection_type::text::connectiontype" + ) + ) + op.execute("drop type connectiontype_old") diff --git a/tests/fixtures.py b/tests/fixtures.py index 1a6c2b96f..5fc191101 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -82,6 +82,13 @@ "username": pydash.get(integration_config, "mysql_example.USER"), "password": pydash.get(integration_config, "mysql_example.PASSWORD"), }, + "mssql_example": { + "host": pydash.get(integration_config, "mssql_example.SERVER"), + "port": pydash.get(integration_config, "mssql_example.PORT"), + "dbname": pydash.get(integration_config, "mssql_example.DB"), + "username": pydash.get(integration_config, "mssql_example.USER"), + "password": pydash.get(integration_config, "mssql_example.PASSWORD"), + }, } @@ -199,6 +206,22 @@ def connection_config_mysql(db: Session) -> Generator: connection_config.delete(db) +@pytest.fixture(scope="function") +def connection_config_mssql(db: Session) -> Generator: + connection_config = ConnectionConfig.create( + db=db, + data={ + "name": str(uuid4()), + "key": "my_mssql_db_1", + "connection_type": ConnectionType.mssql, + "access": AccessLevel.write, + "secrets": integration_secrets["mssql_example"], + }, + ) + yield connection_config + connection_config.delete(db) + + @pytest.fixture(scope="function") def connection_config_dry_run(db: Session) -> Generator: connection_config = ConnectionConfig.create( diff --git a/tests/integration_tests/mssql_setup.py b/tests/integration_tests/mssql_setup.py new file mode 100644 index 000000000..dc7efe46a --- /dev/null +++ b/tests/integration_tests/mssql_setup.py @@ -0,0 +1,21 @@ +import sqlalchemy + +MSSQL_URL_TEMPLATE = "mssql+pyodbc://sa:Mssql_pw1@mssql_example:1433/{}?driver=ODBC+Driver+17+for+SQL+Server" +MASTER_MSSQL_URL = MSSQL_URL_TEMPLATE.format("master") + "&autocommit=True" + + +def mssql_setup(): + """ + Set up the SQL Server Database for testing. + The query file must have each query on a separate line. + Initial connection must be done to the master database. + """ + engine = sqlalchemy.create_engine(MASTER_MSSQL_URL) + with open("data/sql/mssql_example.sql", "r") as query_file: + queries = [query for query in query_file.read().splitlines() if query != ""] + for query in queries: + engine.execute(sqlalchemy.sql.text(query)) + + +if __name__ == "__main__": + mssql_setup() diff --git a/tests/integration_tests/test_connection_configuration_integration.py b/tests/integration_tests/test_connection_configuration_integration.py index 0f3206b68..8292df2e1 100644 --- a/tests/integration_tests/test_connection_configuration_integration.py +++ b/tests/integration_tests/test_connection_configuration_integration.py @@ -9,7 +9,7 @@ from fidesops.models.client import ClientDetail from fidesops.models.connectionconfig import ConnectionTestStatus from fidesops.service.connectors import MongoDBConnector -from fidesops.service.connectors.sql_connector import MySQLConnector +from fidesops.service.connectors.sql_connector import MySQLConnector, MicrosoftSQLServerConnector from fidesops.common_exceptions import ConnectionException from fidesops.service.connectors import PostgreSQLConnector from fidesops.service.connectors import get_connector @@ -528,6 +528,264 @@ def test_mysql_db_connector( connector.test_connection() +class TestMicrosoftSQLServerConnection: + + @pytest.fixture(scope="function") + def url_put_secret(self, oauth_client, policy, connection_config_mssql) -> str: + return f"{V1_URL_PREFIX}{CONNECTIONS}/{connection_config_mssql.key}/secret" + + @pytest.mark.integration + def test_mssql_db_connection_incorrect_secrets( + self, + api_client: TestClient, + db: Session, + generate_auth_header, + connection_config_mssql, + url_put_secret, + ) -> None: + auth_header = generate_auth_header(scopes=[CONNECTION_CREATE_OR_UPDATE]) + payload = { + "username": "sa", + "password": "incorrect", + "host": "mssql_example", + "port": 1433, + "dbname": "mssql_example", + "url": None + } + resp = api_client.put( + url_put_secret, + headers=auth_header, + json=payload, + ) + assert resp.status_code == 200 + body = json.loads(resp.text) + assert ( + body["msg"] + == f"Secrets updated for ConnectionConfig with key: {connection_config_mssql.key}." + ) + assert body["test_status"] == "failed" + assert "Connection error." == body["failure_reason"] + db.refresh(connection_config_mssql) + + assert connection_config_mssql.secrets == { + "username": "sa", + "password": "incorrect", + "host": "mssql_example", + "port": 1433, + "dbname": "mssql_example", + "url": None, + } + assert connection_config_mssql.last_test_timestamp is not None + assert connection_config_mssql.last_test_succeeded is False + + @pytest.mark.integration + def test_mssql_db_connection_connect_with_components( + self, + url_put_secret, + api_client: TestClient, + db: Session, + generate_auth_header, + connection_config_mssql, + ) -> None: + payload = { + "username": "sa", + "password": "Mssql_pw1", + "host": "mssql_example", + "port": 1433, + "dbname": "mssql_example" + } + + auth_header = generate_auth_header(scopes=[CONNECTION_CREATE_OR_UPDATE]) + resp = api_client.put( + url_put_secret, + headers=auth_header, + json=payload, + ) + assert resp.status_code == 200 + body = resp.json() + + assert ( + body["msg"] + == f"Secrets updated for ConnectionConfig with key: {connection_config_mssql.key}." + ) + assert body["test_status"] == "succeeded" + assert body["failure_reason"] is None + db.refresh(connection_config_mssql) + assert connection_config_mssql.secrets == { + "username": "sa", + "password": "Mssql_pw1", + "host": "mssql_example", + "port": 1433, + "dbname": "mssql_example", + "url": None + } + assert connection_config_mssql.last_test_timestamp is not None + assert connection_config_mssql.last_test_succeeded is True + + @pytest.mark.integration + def test_mssql_db_connection_connect_with_url( + self, + url_put_secret, + api_client: TestClient, + db: Session, + generate_auth_header, + connection_config_mssql, + ) -> None: + payload = { + "url": "mssql+pyodbc://sa:Mssql_pw1@mssql_example:1433/mssql_example?driver=ODBC+Driver+17+for+SQL+Server" + } + + auth_header = generate_auth_header(scopes=[CONNECTION_CREATE_OR_UPDATE]) + resp = api_client.put( + url_put_secret, + headers=auth_header, + json=payload, + ) + assert resp.status_code == 200 + body = json.loads(resp.text) + + assert ( + body["msg"] + == f"Secrets updated for ConnectionConfig with key: {connection_config_mssql.key}." + ) + assert body["failure_reason"] is None + assert body["test_status"] == "succeeded" + db.refresh(connection_config_mssql) + assert connection_config_mssql.secrets == { + "username": None, + "password": None, + "host": None, + "port": None, + "dbname": None, + "url": payload["url"], + } + assert connection_config_mssql.last_test_timestamp is not None + assert connection_config_mssql.last_test_succeeded is True + + @pytest.fixture(scope="function") + def url_test_secrets(self, oauth_client, policy, connection_config_mssql) -> str: + return f"{V1_URL_PREFIX}{CONNECTIONS}/{connection_config_mssql.key}/test" + + @pytest.mark.integration + def test_connection_configuration_test_not_authenticated( + self, + url_test_secrets, + api_client: TestClient, + db: Session, + generate_auth_header, + connection_config_mssql, + ) -> None: + assert connection_config_mssql.last_test_timestamp is None + + resp = api_client.get(url_test_secrets) + assert resp.status_code == 401 + db.refresh(connection_config_mssql) + assert connection_config_mssql.last_test_timestamp is None + assert connection_config_mssql.last_test_succeeded is None + + @pytest.mark.integration + def test_connection_configuration_test_incorrect_scopes( + self, + url_test_secrets, + api_client: TestClient, + db: Session, + generate_auth_header, + connection_config_mssql, + ) -> None: + assert connection_config_mssql.last_test_timestamp is None + + auth_header = generate_auth_header(scopes=[STORAGE_READ]) + resp = api_client.get( + url_test_secrets, + headers=auth_header, + ) + assert resp.status_code == 403 + db.refresh(connection_config_mssql) + assert connection_config_mssql.last_test_timestamp is None + assert connection_config_mssql.last_test_succeeded is None + + @pytest.mark.integration + def test_connection_configuration_test_failed_response( + self, + url_test_secrets, + api_client: TestClient, + db: Session, + generate_auth_header, + connection_config_mssql, + ) -> None: + assert connection_config_mssql.last_test_timestamp is None + connection_config_mssql.secrets = {"host": "invalid_host"} + connection_config_mssql.save(db) + + auth_header = generate_auth_header(scopes=[CONNECTION_READ]) + resp = api_client.get( + url_test_secrets, + headers=auth_header, + ) + assert resp.status_code == 200 + body = json.loads(resp.text) + + db.refresh(connection_config_mssql) + assert connection_config_mssql.last_test_timestamp is not None + assert connection_config_mssql.last_test_succeeded is False + assert body["test_status"] == "failed" + assert "Connection error." == body["failure_reason"] + assert ( + body["msg"] + == f"Test completed for ConnectionConfig with key: {connection_config_mssql.key}." + ) + + @pytest.mark.integration + def test_connection_configuration_test( + self, + url_test_secrets, + api_client: TestClient, + db: Session, + generate_auth_header, + connection_config_mssql, + ) -> None: + assert connection_config_mssql.last_test_timestamp is None + + auth_header = generate_auth_header(scopes=[CONNECTION_READ]) + resp = api_client.get( + url_test_secrets, + headers=auth_header, + ) + assert resp.status_code == 200 + body = json.loads(resp.text) + + assert ( + body["msg"] + == f"Test completed for ConnectionConfig with key: {connection_config_mssql.key}." + ) + assert body["failure_reason"] is None + assert body["test_status"] == "succeeded" + db.refresh(connection_config_mssql) + assert connection_config_mssql.last_test_timestamp is not None + assert connection_config_mssql.last_test_succeeded is True + + @pytest.mark.integration + def test_mssql_db_connector( + self, + api_client: TestClient, + db: Session, + generate_auth_header, + connection_config_mssql, + ) -> None: + connector = get_connector(connection_config_mssql) + assert connector.__class__ == MicrosoftSQLServerConnector + + client = connector.client() + assert client.__class__ == Engine + assert connector.test_connection() == ConnectionTestStatus.succeeded + + connection_config_mssql.secrets = {"host": "bad_host"} + connection_config_mssql.save(db) + connector = get_connector(connection_config_mssql) + with pytest.raises(ConnectionException): + connector.test_connection() + + class TestMongoConnector: @pytest.mark.integration def test_mongo_db_connector( diff --git a/tests/integration_tests/test_integration_mssql_example.py b/tests/integration_tests/test_integration_mssql_example.py new file mode 100644 index 000000000..dd29f3dcc --- /dev/null +++ b/tests/integration_tests/test_integration_mssql_example.py @@ -0,0 +1,52 @@ +import logging +from typing import Generator + +import pytest +from sqlalchemy import func, select, table + +from fidesops.db.session import get_db_session, get_db_engine + +logger = logging.getLogger(__name__) + +MSSQL_URL_TEMPLATE = "mssql+pyodbc://sa:Mssql_pw1@mssql_example:1433/{}?driver=ODBC+Driver+17+for+SQL+Server" +MSSQL_URL = MSSQL_URL_TEMPLATE.format("mssql_example") + + +@pytest.fixture(scope="module") +def mssql_example_db() -> Generator: + """Return a connection to the MsSQL example DB""" + engine = get_db_engine(database_uri=MSSQL_URL) + logger.debug(f"Connecting to MsSQL example database at: {engine.url}") + SessionLocal = get_db_session(engine=engine) + the_session = SessionLocal() + # Setup above... + yield the_session + # Teardown below... + the_session.close() + engine.dispose() + + +@pytest.mark.integration +def test_mssql_example_data(mssql_example_db): + """Confirm that the example database is populated with simulated data""" + expected_counts = { + "product": 3, + "address": 4, + "customer": 3, + "employee": 2, + "payment_card": 3, + "orders": 5, + "order_item": 6, + "visit": 2, + "login": 8, + "service_request": 4, + "report": 4, + "type_link_test": 2 + } + + for table_name, expected_count in expected_counts.items(): + # NOTE: we could use text() here, but we want to avoid SQL string + # templating as much as possible. instead, use the table() helper to + # dynamically generate the FROM clause for each table_name + count_sql = select(func.count()).select_from(table(table_name)) + assert mssql_example_db.execute(count_sql).scalar() == expected_count