Add ConnectionConfig Search [#609] (#641)

* Add search on connectionconfig field that examines name, key, and description fields.

- Add field ConnectionConfig.description
- Allow ConnectionConfig.description to be updated via PATCH /connectionconfig

* Add search to the guides and update changelog.

* New description key returned in response.

* Bump downrev.

* Add ordering assertion to test.

CHANGELOG.md

@@ -28,6 +28,7 @@ The types of changes are:
 * Build and deploy Admin UI from webserver [#625](https://github.com/ethyca/fidesops/pull/625)
 * Allow disabling a ConnectionConfig [#637](https://github.com/ethyca/fidesops/pull/637)
 * Erasure support for Outreach connector [#619](https://github.com/ethyca/fidesops/pull/619)
+* Adds searching of ConnectionConfigs [#641](https://github.com/ethyca/fidesops/pull/641)
 
 ### Changed
 

docs/fidesops/docs/guides/database_connectors.md

@@ -7,6 +7,7 @@ In this section we'll cover:
 - How do you create a ConnectionConfig object?
 - How do you identify the database that a ConnectionConfig connects to?
 - How do you test and update a ConnectionConfig's Secrets?
+- How do you search your ConnectionConfigs?
 - How does a ConnectionConfig differ from a Dataset?
 
 
@@ -46,6 +47,8 @@ The connection between Fidesops and your database is represented by a _Connectio
 
 * `disabled` determines whether the ConnectionConfig is active.  If True, we skip running queries for any collection associated with that ConnectionConfig.
 
+* `description` is an extra field to add further details about your connection. 
+
 While the ConnectionConfig object contains meta information about the database, you'll notice that it doesn't actually identify the database itself. We'll get to that when we set the ConnectionConfig's "secrets".
 
 
@@ -137,7 +140,8 @@ PATCH api/v1/connection
     "key": "manual_connector",
     "connection_type": "manual",
     "access": "read",
-    "disabled": false
+    "disabled": false,
+    "description": "Connector describing manual actions"
   }
 ]
 ``` 
@@ -273,6 +277,32 @@ Once you have a working ConnectionConfig, it can be associated to an existing [d
 }]
 ```
 
+## Searching ConnectionConfigs
+
+You can search the `name`, `key`, and `description` fields of your ConnectionConfigs with the `search` query parameter.
+
+### Example 1
+```json title="<code>GET /api/v1/connection/?search=application mysql</code>"
+{
+    "items": [
+        {
+            "name": "Application MySQL DB",
+            "key": "app_mysql_db",
+            "description": "My Backup MySQL DB",
+            "connection_type": "mysql",
+            "access": "read",
+            "created_at": "2022-06-13T18:03:28.404091+00:00",
+            "updated_at": "2022-06-13T18:03:28.404091+00:00",
+            "last_test_timestamp": null,
+            "last_test_succeeded": null
+        }
+    ],
+    "total": 1,
+    "page": 1,
+    "size": 50
+}
+```
+
 ## How do ConnectionConfigs differ from Datasets?
 
 A Dataset is an annotation of your database schema; it describes the PII category (or Data Categories) for each field that the database contains. A ConnectionConfig holds the secrets to connect to the database. Each Dataset has a foreign key to a ConnectionConfig.

src/fidesops/api/v1/endpoints/connection_endpoints.py

@@ -7,7 +7,9 @@
 from fastapi_pagination.bases import AbstractPage
 from fastapi_pagination.ext.sqlalchemy import paginate
 from pydantic import ValidationError, conlist
+from sqlalchemy import or_
 from sqlalchemy.orm import Session
+from sqlalchemy_utils import escape_like
 from starlette.status import (
     HTTP_200_OK,
     HTTP_204_NO_CONTENT,
@@ -80,14 +82,28 @@ def get_connection_config_or_error(
     response_model=Page[ConnectionConfigurationResponse],
 )
 def get_connections(
-    *, db: Session = Depends(deps.get_db), params: Params = Depends()
+    *,
+    db: Session = Depends(deps.get_db),
+    params: Params = Depends(),
+    search: Optional[str] = None,
 ) -> AbstractPage[ConnectionConfig]:
-    """Returns all connection configurations in the database."""
+    """Returns all connection configurations in the database.
+    Optionally filter the key, name, and description with a search query param
+    """
     logger.info(
-        f"Finding all connection configurations with pagination params {params}"
+        f"Finding all connection configurations with pagination params {params} and search query: '{search if search else ''}'."
     )
+    query = ConnectionConfig.query(db)
+    if search:
+        query = query.filter(
+            or_(
+                ConnectionConfig.key.ilike(f"%{escape_like(search)}%"),
+                ConnectionConfig.name.ilike(f"%{escape_like(search)}%"),
+                ConnectionConfig.description.ilike(f"%{escape_like(search)}%"),
+            )
+        )
     return paginate(
-        ConnectionConfig.query(db).order_by(ConnectionConfig.created_at.desc()),
+        query.order_by(ConnectionConfig.created_at.desc()),
         params=params,
     )
 

src/fidesops/migrations/versions/b3b68c87c4a0_add_connection_config_description.py

@@ -0,0 +1,34 @@
+"""add connection config description
+
+Revision ID: b3b68c87c4a0
+Revises: c3472d75c80e
+Create Date: 2022-06-13 17:24:56.889227
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "b3b68c87c4a0"
+down_revision = "c3472d75c80e"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column(
+        "connectionconfig", sa.Column("description", sa.String(), nullable=True)
+    )
+    op.create_index(
+        op.f("ix_connectionconfig_description"),
+        "connectionconfig",
+        ["description"],
+        unique=False,
+    )
+
+
+def downgrade():
+    op.drop_index(
+        op.f("ix_connectionconfig_description"), table_name="connectionconfig"
+    )
+    op.drop_column("connectionconfig", "description")

src/fidesops/models/connectionconfig.py

@@ -63,6 +63,7 @@ class ConnectionConfig(Base):
 
     name = Column(String, index=True, unique=True, nullable=False)
     key = Column(String, index=True, unique=True, nullable=False)
+    description = Column(String, index=True, nullable=True)
     connection_type = Column(Enum(ConnectionType), nullable=False)
     access = Column(Enum(AccessLevel), nullable=False)
     secrets = Column(

src/fidesops/schemas/connection_configuration/connection_config.py

@@ -20,6 +20,7 @@ class CreateConnectionConfiguration(BaseModel):
     connection_type: ConnectionType
     access: AccessLevel
     disabled: Optional[bool] = False
+    description: Optional[str]
 
     class Config:
         """Restrict adding other fields through this schema and set orm_mode to support mapping to ConnectionConfig"""
@@ -38,6 +39,7 @@ class ConnectionConfigurationResponse(BaseModel):
 
     name: str
     key: FidesOpsKey
+    description: Optional[str]
     connection_type: ConnectionType
     access: AccessLevel
     created_at: datetime

tests/api/v1/endpoints/test_connection_config_endpoints.py

@@ -7,6 +7,7 @@
 from fastapi import HTTPException
 from fastapi_pagination import Params
 from sqlalchemy.orm import Session
+from sqlalchemy.testing import db
 from starlette.testclient import TestClient
 
 from fidesops.api.v1.scope_registry import (
@@ -238,6 +239,7 @@ def test_patch_connections_bulk_update(
                 "name": "Snowflake Warehouse",
                 "connection_type": "snowflake",
                 "access": "write",
+                "description": "Backup snowflake db",
             },
         ]
 
@@ -315,10 +317,12 @@ def test_patch_connections_bulk_update(
         snowflake_connection = response_body["succeeded"][7]
         assert snowflake_connection["access"] == "write"
         assert snowflake_connection["updated_at"] is not None
+        assert snowflake_connection["description"] == "Backup snowflake db"
         snowflake_resource = (
             db.query(ConnectionConfig).filter_by(key="my_snowflake").first()
         )
         assert snowflake_resource.access.value == "write"
+        assert snowflake_resource.description == "Backup snowflake db"
         assert "secrets" not in snowflake_connection
 
         postgres_resource.delete(db)
@@ -365,13 +369,15 @@ def test_patch_connections_failed_response(
             "connection_type": "postgres",
             "access": "write",
             "disabled": False,
+            "description": None,
         }
         assert response_body["failed"][1]["data"] == {
             "name": "My Mongo DB",
             "key": None,
             "connection_type": "mongodb",
             "access": "read",
             "disabled": False,
+            "description": None,
         }
 
 
@@ -414,6 +420,7 @@ def test_get_connection_configs(
             "key",
             "created_at",
             "disabled",
+            "description",
         }
 
         assert connection["key"] == "my_postgres_db_1"
@@ -426,6 +433,50 @@ def test_get_connection_configs(
         assert response_body["page"] == 1
         assert response_body["size"] == page_size
 
+    def test_search_connections(
+        self,
+        db,
+        connection_config,
+        read_connection_config,
+        api_client: TestClient,
+        generate_auth_header,
+        url,
+    ):
+        auth_header = generate_auth_header(scopes=[CONNECTION_READ])
+
+        resp = api_client.get(url + "?search=primary", headers=auth_header)
+        assert resp.status_code == 200
+        assert len(resp.json()["items"]) == 1
+        assert "primary" in resp.json()["items"][0]["description"].lower()
+
+        resp = api_client.get(url + "?search=read", headers=auth_header)
+        assert resp.status_code == 200
+        assert len(resp.json()["items"]) == 1
+        assert "read" in resp.json()["items"][0]["description"].lower()
+
+        resp = api_client.get(url + "?search=nonexistent", headers=auth_header)
+        assert resp.status_code == 200
+        assert len(resp.json()["items"]) == 0
+
+        resp = api_client.get(url + "?search=postgres", headers=auth_header)
+        assert resp.status_code == 200
+        items = resp.json()["items"]
+        assert len(items) == 2
+
+        ordered = (
+            db.query(ConnectionConfig)
+            .filter(
+                ConnectionConfig.key.in_(
+                    [read_connection_config.key, connection_config.key]
+                )
+            )
+            .order_by(ConnectionConfig.created_at.desc())
+            .all()
+        )
+        assert len(ordered) == 2
+        assert ordered[0].key == items[0]["key"]
+        assert ordered[1].key == items[1]["key"]
+
 
 class TestGetConnection:
     @pytest.fixture(scope="function")
@@ -473,6 +524,7 @@ def test_get_connection_config(
             "key",
             "created_at",
             "disabled",
+            "description",
         }
 
         assert response_body["key"] == "my_postgres_db_1"

tests/api/v1/endpoints/test_policy_webhook_endpoints.py

@@ -34,6 +34,7 @@ def embedded_http_connection_config(connection_config: ConnectionConfig) -> Dict
         "last_test_timestamp": None,
         "last_test_succeeded": None,
         "disabled": False,
+        "description": None,
     }
 
 

tests/fixtures/postgres_fixtures.py

@@ -141,6 +141,7 @@ def connection_config(
             "access": AccessLevel.write,
             "secrets": integration_secrets["postgres_example"],
             "disabled": False,
+            "description": "Primary postgres connection",
         },
     )
     yield connection_config
@@ -159,6 +160,7 @@ def read_connection_config(
             "connection_type": ConnectionType.postgres,
             "access": AccessLevel.read,
             "secrets": integration_secrets["postgres_example"],
+            "description": "Read-only connection config",
         },
     )
     yield connection_config