Snowflake Query Execution [#73] (#104)

data/dataset/snowflake_example_test_dataset.yml

@@ -0,0 +1,229 @@
+dataset:
+  - fides_key: snowflake_example_test_dataset
+    name: Snowflake Example Test Dataset
+    description: Example of a Snowflake dataset containing a variety of related tables like customers, products, addresses, etc.
+    collections:
+      - name: address
+        fields:
+          - name: city
+            data_categories: [user.provided.identifiable.contact.city]
+          - name: house
+            data_categories: [user.provided.identifiable.contact.street]
+          - name: id
+            data_categories: [system.operations]
+            fidesops_meta:
+              primary_key: True
+          - name: state
+            data_categories: [user.provided.identifiable.contact.state]
+          - name: street
+            data_categories: [user.provided.identifiable.contact.street]
+          - name: zip
+            data_categories: [user.provided.identifiable.contact.postal_code]
+
+      - name: customer
+        fields:
+          - name: address_id
+            data_categories: [system.operations]
+            fidesops_meta:
+              references:
+                - dataset: snowflake_example_test_dataset
+                  field: address.id
+                  direction: to
+          - name: created
+            data_categories: [system.operations]
+          - name: email
+            data_categories: [user.provided.identifiable.contact.email]
+            fidesops_meta:
+              identity: email
+              data_type: string
+          - name: id
+            data_categories: [user.derived.identifiable.unique_id]
+            fidesops_meta:
+              primary_key: True
+          - name: name
+            data_categories: [user.provided.identifiable.name]
+            fidesops_meta:
+              data_type: string
+              length: 40
+          - name: variant_eg
+            # We use this data category so we can target this column from
+            # our Snowflake tests
+            data_categories: [user.provided.identifiable.name]
+
+      - name: employee
+        fields:
+          - name: address_id
+            data_categories: [system.operations]
+            fidesops_meta:
+              references:
+                - dataset: snowflake_example_test_dataset
+                  field: address.id
+                  direction: to
+          - name: email
+            data_categories: [user.provided.identifiable.contact.email]
+            fidesops_meta:
+              identity: email
+              data_type: string
+          - name: id
+            data_categories: [user.derived.identifiable.unique_id]
+            fidesops_meta:
+              primary_key: True
+          - name: name
+            data_categories: [user.provided.identifiable.name]
+            fidesops_meta:
+              data_type: string
+
+      - name: login
+        fields:
+          - name: customer_id
+            data_categories: [user.derived.identifiable.unique_id]
+            fidesops_meta:
+              references:
+                - dataset: snowflake_example_test_dataset
+                  field: customer.id
+                  direction: from
+          - name: id
+            data_categories: [system.operations]
+            fidesops_meta:
+              primary_key: True
+          - name: time
+            data_categories: [user.derived.nonidentifiable.sensor]
+
+      - name: order
+        fields:
+          - name: customer_id
+            data_categories: [user.derived.identifiable.unique_id]
+            fidesops_meta:
+              references:
+                - dataset: snowflake_example_test_dataset
+                  field: customer.id
+                  direction: from
+          - name: id
+            data_categories: [system.operations]
+            fidesops_meta:
+              primary_key: True
+          - name: shipping_address_id
+            data_categories: [system.operations]
+            fidesops_meta:
+              references:
+                - dataset: snowflake_example_test_dataset
+                  field: address.id
+                  direction: to
+
+      # order_item
+      - name: order_item
+        fields:
+          - name: order_id
+            data_categories: [system.operations]
+            fidesops_meta:
+              references:
+                - dataset: snowflake_example_test_dataset
+                  field: order.id
+                  direction: from
+          - name: product_id
+            data_categories: [system.operations]
+            fidesops_meta:
+              references:
+                - dataset: snowflake_example_test_dataset
+                  field: product.id
+                  direction: to
+          - name: quantity
+            data_categories: [system.operations]
+
+      - name: payment_card
+        fields:
+          - name: billing_address_id
+            data_categories: [system.operations]
+            fidesops_meta:
+              references:
+                - dataset: snowflake_example_test_dataset
+                  field: address.id
+                  direction: to
+          - name: ccn
+            data_categories: [user.provided.identifiable.financial.account_number]
+          - name: code
+            data_categories: [user.provided.identifiable.financial]
+          - name: customer_id
+            data_categories: [user.derived.identifiable.unique_id]
+            fidesops_meta:
+              references:
+                - dataset: snowflake_example_test_dataset
+                  field: customer.id
+                  direction: from
+          - name: id
+            data_categories: [system.operations]
+            fidesops_meta:
+              primary_key: True
+          - name: name
+            data_categories: [user.provided.identifiable.financial]
+          - name: preferred
+            data_categories: [user.provided.nonidentifiable]
+
+      - name: product
+        fields:
+          - name: id
+            data_categories: [system.operations]
+            fidesops_meta:
+              primary_key: True
+          - name: name
+            data_categories: [system.operations]
+          - name: price
+            data_categories: [system.operations]
+
+      - name: report
+        fields:
+          - name: email
+            data_categories: [user.provided.identifiable.contact.email]
+            fidesops_meta:
+              identity: email
+              data_type: string
+          - name: id
+            data_categories: [system.operations]
+            fidesops_meta:
+              primary_key: True
+          - name: month
+            data_categories: [system.operations]
+          - name: name
+            data_categories: [system.operations]
+          - name: total_visits
+            data_categories: [system.operations]
+          - name: year
+            data_categories: [system.operations]
+
+      - name: service_request
+        fields:
+          - name: alt_email
+            data_categories: [user.provided.identifiable.contact.email]
+            fidesops_meta:
+              identity: email
+              data_type: string
+          - name: closed
+            data_categories: [system.operations]
+          - name: email
+            data_categories: [system.operations]
+            fidesops_meta:
+              identity: email
+              data_type: string
+          - name: employee_id
+            data_categories: [user.derived.identifiable.unique_id]
+            fidesops_meta:
+              references:
+                - dataset: snowflake_example_test_dataset
+                  field: employee.id
+                  direction: from
+          - name: id
+            data_categories: [system.operations]
+            fidesops_meta:
+              primary_key: True
+          - name: opened
+            data_categories: [system.operations]
+
+      - name: visit
+        fields:
+          - name: email
+            data_categories: [user.provided.identifiable.contact.email]
+            fidesops_meta:
+              identity: email
+              data_type: string
+          - name: last_visit
+            data_categories: [system.operations]

src/fidesops/core/config.py

@@ -174,6 +174,8 @@ class FidesopsConfig(FidesSettings):
     security: SecuritySettings
     execution: ExecutionSettings
 
+    is_test_mode: bool = os.getenv("TESTING") == "True"
+
     class Config:  # pylint: disable=C0115
         case_sensitive = True
 

src/fidesops/db/session.py

@@ -1,5 +1,4 @@
 import logging
-import os
 from typing import Optional
 
 from sqlalchemy import create_engine
@@ -20,7 +19,7 @@ def get_db_engine(database_uri: Optional[str] = None) -> Engine:
     """
     if database_uri is None:
         # Don't override any database_uri explicity passed in
-        if os.getenv("TESTING"):
+        if config.is_test_mode:
             database_uri = config.database.SQLALCHEMY_TEST_DATABASE_URI
         else:
             database_uri = config.database.SQLALCHEMY_DATABASE_URI

src/fidesops/models/connectionconfig.py

@@ -1,5 +1,6 @@
 import enum
 from datetime import datetime
+from typing import Optional
 
 from sqlalchemy import (
     Column,
@@ -18,7 +19,10 @@
 
 
 from fidesops.core.config import config
-from fidesops.db.base_class import Base, JSONTypeOverride
+from fidesops.db.base_class import (
+    Base,
+    JSONTypeOverride,
+)
 
 
 class TestStatus(enum.Enum):
@@ -88,3 +92,10 @@ def update_test_status(self, test_status: TestStatus, db: Session) -> None:
         self.last_test_timestamp = datetime.now()
         self.last_test_succeeded = test_status == TestStatus.succeeded
         self.save(db)
+
+    def delete(self, db: Session) -> Optional[Base]:
+        """Hard deletes datastores that map this ConnectionConfig."""
+        for dataset in self.datasets:
+            dataset.delete(db=db)
+
+        return super().delete(db=db)

src/fidesops/service/connectors/base_connector.py

@@ -2,6 +2,7 @@
 from abc import abstractmethod, ABC
 from typing import Any, Dict, List, Optional, TypeVar, Generic
 
+from fidesops.core.config import config
 from fidesops.graph.traversal import Row, TraversalNode
 from fidesops.models.connectionconfig import ConnectionConfig, TestStatus
 from fidesops.models.policy import Policy
@@ -29,6 +30,11 @@ class BaseConnector(Generic[DB_CONNECTOR_TYPE], ABC):
 
     def __init__(self, configuration: ConnectionConfig):
         self.configuration = configuration
+        # If Fidesops is running in test mode, it's OK to show
+        # parameters inside queries for debugging purposes. By
+        # default we assume that Fidesops is not running in test
+        # mode.
+        self.hide_parameters = not config.is_test_mode
         self.db_client: Optional[DB_CONNECTOR_TYPE] = None
 
     @abstractmethod