Fidesops 78 mssql support (#151)

Co-authored-by: Dawn Pattison <[email protected]> Co-authored-by: catherinesmith <[email protected]> Co-authored-by: Catherine Smith <[email protected]>
ethyca · Jan 10, 2022 · 6b1baaf · 6b1baaf
1 parent 7f2be16
commit 6b1baaf
Show file tree

Hide file tree

Showing 20 changed files with 706 additions and 17 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.9.6-slim-buster
+FROM --platform=linux/amd64 python:3.9.6-slim-buster
 
 # Install auxiliary software
 RUN apt-get update && \
@@ -8,8 +8,22 @@ RUN apt-get update && \
     ipython \
     vim \
     curl \
+    g++ \
+    gnupg \
     gcc
 
+# SQL Server (MS SQL)
+# https://docs.microsoft.com/en-us/sql/connect/odbc/linux-mac/installing-the-microsoft-odbc-driver-for-sql-server?view=sql-server-ver15
+RUN apt-get install apt-transport-https
+RUN curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add -
+RUN curl https://packages.microsoft.com/config/debian/10/prod.list | tee /etc/apt/sources.list.d/msprod.list
+RUN apt-get update
+ENV ACCEPT_EULA=y DEBIAN_FRONTEND=noninteractive
+RUN apt-get -y install \
+    unixodbc-dev \
+    msodbcsql17 \
+    mssql-tools
+
 # Update pip and install requirements
 COPY requirements.txt dev-requirements.txt ./
 RUN pip install -U pip  \

diff --git a/Makefile b/Makefile
@@ -48,7 +48,13 @@ integration-shell: compose-build
 	@docker-compose -f docker-compose.yml -f docker-compose.integration-test.yml run $(IMAGE_NAME) /bin/bash
 
 integration-env: compose-build
-	@docker-compose -f docker-compose.yml -f docker-compose.integration-test.yml up
+	@echo "Bringing up main image and images for integration testing"
+	@docker-compose -f docker-compose.yml -f docker-compose.integration-test.yml up -d
+	@echo "Waiting 15s for integration containers to be ready..."
+	@sleep 15
+	@echo "Running additional setup for mssql integration tests"
+	@docker exec -it fidesops python tests/integration_tests/mssql_setup.py
+	@docker-compose -f docker-compose.yml -f docker-compose.integration-test.yml logs -f -t
 
 quickstart: compose-build
 	@docker-compose -f docker-compose.yml -f docker-compose.integration-test.yml up -d
@@ -107,8 +113,10 @@ pytest-integration-access: compose-build
 	@docker-compose -f docker-compose.yml -f docker-compose.integration-test.yml build
 	@echo "Bringing up the integration environment..."
 	@docker-compose -f docker-compose.yml -f docker-compose.integration-test.yml up -d
-	@echo "Waiting 10s for integration containers to be ready..."
-	@sleep 10
+	@echo "Waiting 15s for integration containers to be ready..."
+	@sleep 15
+	@echo "Running additional setup for mssql integration tests"
+	@docker exec fidesops python tests/integration_tests/mssql_setup.py
 	@echo "Running pytest integration tests..."
 	@docker-compose -f docker-compose.yml -f docker-compose.integration-test.yml \
 		run $(IMAGE_NAME) \

diff --git a/data/sql/mssql_example.sql b/data/sql/mssql_example.sql
@@ -0,0 +1,84 @@
+USE master;
+
+-- CREATE USER IF NOT EXISTS 'sa'@'mssql_example' IDENTIFIED BY 'Mssql_pw1';
+-- GRANT ALL PRIVILEGES ON *.* TO 'sa'@'mssql_example' ;
+-- GRANT ALL PRIVILEGES ON *.* TO 'sa'@'%' ;
+-- FLUSH PRIVILEGES;
+
+DROP DATABASE IF EXISTS mssql_example;
+CREATE DATABASE mssql_example;
+USE mssql_example;
+
+DROP TABLE IF EXISTS report;
+DROP TABLE IF EXISTS service_request;
+DROP TABLE IF EXISTS login;
+DROP TABLE IF EXISTS visit;
+DROP TABLE IF EXISTS order_item;
+DROP TABLE IF EXISTS orders;
+DROP TABLE IF EXISTS payment_card;
+DROP TABLE IF EXISTS employee;
+DROP TABLE IF EXISTS customer;
+DROP TABLE IF EXISTS address;
+DROP TABLE IF EXISTS product;
+DROP TABLE IF EXISTS composite_pk_test;
+DROP TABLE IF EXISTS type_link_test;
+
+
+CREATE TABLE product ( id INT PRIMARY KEY, name CHARACTER VARYING(100), price MONEY);
+
+CREATE TABLE address ( id BIGINT PRIMARY KEY, house INT, street CHARACTER VARYING(100), city CHARACTER VARYING(100), state CHARACTER VARYING(100), zip CHARACTER VARYING(100));
+
+CREATE TABLE customer ( id INT PRIMARY KEY, email CHARACTER VARYING(100), name CHARACTER VARYING(100), created DATETIME, address_id BIGINT);
+
+CREATE TABLE employee ( id INT PRIMARY KEY, email CHARACTER VARYING(100), name CHARACTER VARYING(100), address_id BIGINT);
+
+CREATE TABLE payment_card ( id CHARACTER VARYING(100) PRIMARY KEY, name CHARACTER VARYING(100), ccn BIGINT, code SMALLINT, preferred BIT, customer_id INT, billing_address_id BIGINT);
+
+CREATE TABLE orders ( id CHARACTER VARYING(100) PRIMARY KEY, customer_id INT, shipping_address_id BIGINT, payment_card_id CHARACTER VARYING(100));
+
+CREATE TABLE order_item ( order_id CHARACTER VARYING(100), item_no SMALLINT, product_id INT, quantity SMALLINT, CONSTRAINT order_item_pk PRIMARY KEY (order_id, item_no));
+
+CREATE TABLE visit ( email CHARACTER VARYING(100), last_visit DATETIME, CONSTRAINT visit_pk PRIMARY KEY (email, last_visit));
+
+CREATE TABLE login ( id INT PRIMARY KEY, customer_id INT, time DATETIME);
+
+CREATE TABLE service_request ( id CHARACTER VARYING(100) PRIMARY KEY, email CHARACTER VARYING(100), alt_email CHARACTER VARYING(100), opened DATE, closed DATE, employee_id INT);
+
+CREATE TABLE report ( id INT PRIMARY KEY, email CHARACTER VARYING(100), name CHARACTER VARYING(100), year INT, month INT, total_visits INT);
+
+CREATE TABLE composite_pk_test ( id_a INT NOT NULL, id_b INT NOT NULL, description VARCHAR(100), customer_id INT, PRIMARY KEY(id_a, id_b));
+
+INSERT INTO composite_pk_test VALUES (1,10,'linked to customer 1',1), (1,11,'linked to customer 2',2), (2,10,'linked to customer 3',3);
+
+CREATE TABLE type_link_test ( id CHARACTER VARYING(100) PRIMARY KEY, name CHARACTER VARYING(100));
+
+-- Populate tables with some public data
+INSERT INTO product VALUES (1, 'Example Product 1', '$10.00'), (2, 'Example Product 2', '$20.00'), (3, 'Example Product 3', '$50.00');
+
+INSERT INTO address VALUES (1, '123', 'Example Street', 'Exampletown', 'NY', '12345'), (2, '4', 'Example Lane', 'Exampletown', 'NY', '12321'), (3, '555', 'Example Ave', 'Example City', 'NY', '12000'), (4, '1111', 'Example Place', 'Example Mountain', 'TX', '54321');
+
+
+INSERT INTO customer VALUES (1, '[email protected]', 'John Customer', '2020-04-01 11:47:42', 1), (2, '[email protected]', 'Jill Customer', '2020-04-01 11:47:42', 2), (3, '[email protected]', 'Jane Customer', '2020-04-01 11:47:42', 4);
+
+
+INSERT INTO employee VALUES (1, '[email protected]', 'Jack Employee', 3), (2, '[email protected]', 'Jane Employee', 3);
+
+INSERT INTO payment_card VALUES ('pay_aaa-aaa', 'Example Card 1', 123456789, 321, 1, 1, 1), ('pay_bbb-bbb', 'Example Card 2', 987654321, 123, 0, 2, 1), ('pay_ccc-ccc', 'Example Card 3', 373719391, 222, 0, 3, 4);
+
+
+INSERT INTO orders VALUES ('ord_aaa-aaa', 1, 2, 'pay_aaa-aaa'), ('ord_bbb-bbb', 2, 1, 'pay_bbb-bbb'), ('ord_ccc-ccc', 1, 1, 'pay_aaa-aaa'), ('ord_ddd-ddd', 1, 1, 'pay_bbb-bbb'), ('ord_ddd-eee', 3, 4, 'pay-ccc-ccc');
+
+
+INSERT INTO order_item VALUES ('ord_aaa-aaa', 1, 1, 1), ('ord_bbb-bbb', 1, 1, 1), ('ord_ccc-ccc', 1, 1, 1), ('ord_ccc-ccc', 2, 2, 1), ('ord_ddd-ddd', 1, 1, 1), ('ord_eee-eee', 3, 4, 3);
+
+
+INSERT INTO visit VALUES ('[email protected]', '2021-01-06 01:00:00'), ('[email protected]', '2021-01-06 01:00:00');
+
+INSERT INTO login VALUES (1, 1, '2021-01-01 01:00:00'), (2, 1, '2021-01-02 01:00:00'), (3, 1, '2021-01-03 01:00:00'), (4, 1, '2021-01-04 01:00:00'), (5, 1, '2021-01-05 01:00:00'), (6, 1, '2021-01-06 01:00:00'), (7, 2, '2021-01-06 01:00:00'), (8, 3, '2021-01-06 01:00:00');
+
+
+INSERT INTO service_request VALUES ('ser_aaa-aaa', '[email protected]', '[email protected]', '2021-01-01', '2021-01-03', 1), ('ser_bbb-bbb', '[email protected]', null, '2021-01-04', null, 1), ('ser_ccc-ccc', '[email protected]', null, '2021-01-05', '2020-01-07', 1), ('ser_ddd-ddd', '[email protected]', null, '2021-05-05', '2020-05-08', 2);
+
+INSERT INTO report VALUES (1, '[email protected]', 'Monthly Report', 2021, 8, 100), (2, '[email protected]', 'Monthly Report', 2021, 9, 100), (3, '[email protected]', 'Monthly Report', 2021, 10, 100), (4, '[email protected]', 'Monthly Report', 2021, 11, 100);
+
+INSERT INTO type_link_test VALUES ('1', 'name1'), ('2', 'name2');
diff --git a/docker-compose.integration-test.yml b/docker-compose.integration-test.yml
@@ -4,6 +4,7 @@ services:
       - postgres_example
       - mongodb_example
       - mysql_example
+      - mssql_example
 
   postgres_example:
     image: postgres:12
@@ -50,3 +51,13 @@ services:
       - "3306:3306"
     volumes:
       - ./data/sql/mysql_example.sql:/docker-entrypoint-initdb.d/mysql_example.sql
+
+  mssql_example:
+    image: mcr.microsoft.com/azure-sql-edge:latest # Equivalent to SQL Server 2016
+    ports:
+      - 1433:1433
+    environment:
+      - SA_PASSWORD=Mssql_pw1
+      - ACCEPT_EULA="Y"
+
+
diff --git a/docs/fidesops/docs/development/overview.md b/docs/fidesops/docs/development/overview.md
@@ -29,8 +29,8 @@ commands to give you different functionality.
 - `make server-shell`-  opens a shell on the Docker container, from here you can run useful commands like:
   - `ipython` - open a Python shell
 - `make pytest` - runs all unit tests except those that talk to integration databases
-- `make pytest-integration-access` - runs access integration tests
-- `make pytest-integration-erasure` - runs erasure integration tests
+- `make pytest-integration-access` - runs access integration tests.
+- `make pytest-integration-erasure` - runs erasure integration tests.
 - `make reset-db` - tears down the Fideops postgres db, then recreates and re-runs migrations.
 - `make quickstart` - runs a quick, five second quickstart that talks to the Fidesops API to execute privacy requests
 - `make check-migrations` - verifies there are no un-run migrations 
@@ -40,7 +40,10 @@ See [Makefile](https://github.com/ethyca/fidesops/blob/main/Makefile) for more o
 
 
 #### Issues 
-When running `make server`, if you get a `importlib.metadata.PackageNotFoundError: fidesops`, do `make server-shell`,
+
+- MSSQL: Known issues around connecting to MSSQL exist today for Apple M1 users. M1 users that wish to install `pyodbc` locally, please reference the workaround [here](https://github.com/mkleehammer/pyodbc/issues/846).
+
+- Package not found: When running `make server`, if you get a `importlib.metadata.PackageNotFoundError: fidesops`, do `make server-shell`,
 and then run `pip install -e .`. Verify Fidesops is installed with `pip list`. 
 
 

diff --git a/docs/fidesops/docs/guides/database_connectors.md b/docs/fidesops/docs/guides/database_connectors.md
@@ -24,6 +24,7 @@ Fidesops supports connections to the following databases:
 * PostgreSQL
 * MongoDB
 * MySQL
+* Microsoft SQLServer
 * Amazon Redshift
 * Snowflake
 
@@ -33,7 +34,7 @@ Other platforms will be added in future releases.
 
 The connection between Fidesops and your database is represented by a _ConnectionConfig_ object. To create a ConnectionConfig, you issue a request to the [Create a ConnectionConfig](/fidesops/api/#operations-Connections-put_connections_api_v1_connection_put) operation, passing a payload that contains the properties listed below. 
 
-* `name`  is a  a human-readable name for your database.
+* `name`  is a human-readable name for your database.
 
 * `key`  is a string token that uniquely identifies your ConnectionConfig object. If you don't supply a `key`, the `name` value, converted to snake-case, is used. For example, if the `name` is `Application PostgreSQL DB`, the converted key is `application_postgresql_db`.
 
@@ -90,10 +91,24 @@ PATCH api/v1/connection
 ]
 ``` 
 
+#### Example 4: MsSQL ConnectionConfig
+
+```
+PATCH api/v1/connection 
+[
+  { 
+    "name": "My MsSQL DB",
+    "key": "my_mssql_db",
+    "connection_type": "mssql",
+    "access": "write"
+  }
+]
+``` 
+
 
 ### Set the ConnectionConfig's Secrets
 
-After you create a ConnectionConfig, you explain how to connect to it by setting its "secrets": host, port, user, and password. You do this by creating a ConnectionConfig Secrets object by calling the [Set a ConnectionConfig's Secrets](/fidesops/api#operations-Connections-put_connection_config_secrets_api_v1_connection__connection_key__secret_put) operation. You can set the object's attributes separately, or supply a single `url` string that encodes them all.
+After you create a ConnectionConfig, you explain how to connect to it by setting its "secrets": host, port, user, and password (note that the secrets used are specific to the DB connector). You do this by creating a ConnectionConfig Secrets object by calling the [Set a ConnectionConfig's Secrets](/fidesops/api#operations-Connections-put_connection_config_secrets_api_v1_connection__connection_key__secret_put) operation. You can set the object's attributes separately, or supply a single `url` string that encodes them all.
 
 If you set the `verify` query parameter to `true`, the operation  will  test the connection by issuing a trivial request to the database. The `test_status` response property announces the success of the connection attempt as `succeeded` or `failed`. If the attempt has failed, the `failure_reason` property gives further details about the failure.
 

diff --git a/docs/fidesops/docs/postman/Fidesops.postman_collection.json b/docs/fidesops/docs/postman/Fidesops.postman_collection.json
@@ -336,6 +336,43 @@
 					},
 					"response": []
 				},
+				{
+					"name": "Create/Update Connection Configs: MsSQL",
+					"request": {
+						"auth": {
+							"type": "bearer",
+							"bearer": [
+								{
+									"key": "token",
+									"value": "{{client_token}}",
+									"type": "string"
+								}
+							]
+						},
+						"method": "PATCH",
+						"header": [],
+						"body": {
+							"mode": "raw",
+							"raw": "[\n    {\"name\": \"Application MsSQL DB\",\n    \"key\": \"{{mssql_key}}\",\n    \"connection_type\": \"mssql\",\n    \"access\": \"read\"\n}]",
+							"options": {
+								"raw": {
+									"language": "json"
+								}
+							}
+						},
+						"url": {
+							"raw": "{{host}}/connection/",
+							"host": [
+								"{{host}}"
+							],
+							"path": [
+								"connection",
+								""
+							]
+						}
+					},
+					"response": []
+				},
 				{
 					"name": "Create/Update Connection Configs: Mongo",
 					"request": {
@@ -411,6 +448,44 @@
 					},
 					"response": []
 				},
+				{
+					"name": "Update Connection Secrets: MsSQL",
+					"request": {
+						"auth": {
+							"type": "bearer",
+							"bearer": [
+								{
+									"key": "token",
+									"value": "{{client_token}}",
+									"type": "string"
+								}
+							]
+						},
+						"method": "PUT",
+						"header": [],
+						"body": {
+							"mode": "raw",
+							"raw": "{\n    \"host\": \"mssql_example\",\n    \"port\": 1433,\n    \"dbname\": \"mssql_example\",\n    \"username\": \"sa\",\n    \"password\": \"Ms_sql1234\"\n}",
+							"options": {
+								"raw": {
+									"language": "json"
+								}
+							}
+						},
+						"url": {
+							"raw": "{{host}}/connection/{{mssql_key}}/secret",
+							"host": [
+								"{{host}}"
+							],
+							"path": [
+								"connection",
+								"{{mssql_key}}",
+								"secret"
+							]
+						}
+					},
+					"response": []
+				},
 				{
 					"name": "Update Connection Secrets: Mongo",
 					"request": {
@@ -1870,6 +1945,10 @@
 			"key": "postgres_key",
 			"value": "app_postgres_db"
 		},
+		{
+			"key": "mssql_key",
+			"value": "app_mssql_db"
+		},
 		{
 			"key": "mongo_key",
 			"value": "app_mongo_db"

diff --git a/fidesops-integration.toml b/fidesops-integration.toml
@@ -17,10 +17,18 @@ SERVER="mysql_example"
 USER="mysql_user"
 PASSWORD="mysql_pw"
 DB="mysql_example"
-PORT= 3306
+PORT=3306
 
 [redshift]
 external_uri=""
 
 [snowflake]
 external_uri=""
+
+[mssql_example]
+SERVER="mssql_example"
+USER="sa"
+PASSWORD="Mssql_pw1"
+DB="mssql_example"
+PORT=1433
+
diff --git a/requirements.txt b/requirements.txt
@@ -28,5 +28,6 @@ pymongo==3.12.0
 pandas==1.3.3
 click==7.1.2
 PyMySQL==1.0.2
+pyodbc==4.0.32
 sqlalchemy-redshift==0.8.8
 snowflake-sqlalchemy==1.3.2
diff --git a/src/fidesops/api/v1/endpoints/connection_endpoints.py b/src/fidesops/api/v1/endpoints/connection_endpoints.py
@@ -195,7 +195,9 @@ def connection_status(
         status: ConnectionTestStatus = connector.test_connection()
     except ConnectionException as exc:
         logger.warning(
-            "Connection test failed on %s: %s", NotPii(connection_config.key), str(exc)
+            "Connection test failed on %s: %s",
+            NotPii(connection_config.key),
+            str(exc),
         )
         connection_config.update_test_status(
             test_status=ConnectionTestStatus.failed, db=db