559-endpoint-log-events - Adds middleware for calling analytics events for each endpoint

CHANGELOG.md

@@ -33,6 +33,8 @@ The types of changes are:
 * Prettier formatting CI check for frontend code [#655](https://github.com/ethyca/fidesops/pull/655)
 * Adds default policies [#654](https://github.com/ethyca/fidesops/pull/654)
 * Added ConnectionConfig `connection_type` and `disabled` filters [#675](https://github.com/ethyca/fidesops/pull/675)
+* Adds Fideslog integration [#541](https://github.com/ethyca/fidesops/pull/541)
+* Adds endpoint analytics events [#622](https://github.com/ethyca/fidesops/pull/622)
 
 ### Changed
 
@@ -143,7 +145,6 @@ The types of changes are:
 * Frontend for privacy request denial reaons [#480](https://github.com/ethyca/fidesops/pull/480)
 * Publish Fidesops to Pypi [#491](https://github.com/ethyca/fidesops/pull/491)
 * DRP data rights endpoint [#526](https://github.com/ethyca/fidesops/pull/526)
-* ADDS Fideslog integration [#541](https://github.com/ethyca/fidesops/pull/541)
 
 
 ### Changed

clients/admin-ui/.eslintrc.js

@@ -15,6 +15,7 @@ module.exports = {
     // causes bug in re-exporting default exports, see
     // https://github.com/eslint/eslint/issues/15617
     'no-restricted-exports': [0],
+    'import/prefer-default-export': 'off',
     'react/function-component-definition': [
       2,
       {

clients/admin-ui/src/features/auth/auth.slice.ts

@@ -7,6 +7,7 @@ import { createApi, fetchBaseQuery } from "@reduxjs/toolkit/query/react";
 
 import type { RootState } from "../../app/store";
 import { BASE_API_URN, STORED_CREDENTIALS_KEY } from "../../constants";
+import { addCommonHeaders } from "../common/CommonHeaders";
 import { User } from "../user-management/types";
 import { LoginRequest, LoginResponse } from "./types";
 
@@ -71,17 +72,13 @@ credentialStorage.startListening({
 });
 
 // Auth API
-export const authApi = createApi({
+export const authApi: any = createApi({
   reducerPath: "authApi",
   baseQuery: fetchBaseQuery({
     baseUrl: BASE_API_URN,
     prepareHeaders: (headers, { getState }) => {
-      const token = selectToken(getState() as RootState);
-      headers.set("Access-Control-Allow-Origin", "*");
-      if (token) {
-        headers.set("authorization", `Bearer ${token}`);
-      }
-      return headers;
+      const token: string | null = selectToken(getState() as RootState);
+      return addCommonHeaders(headers, token);
     },
   }),
   tagTypes: ["Auth"],

clients/admin-ui/src/features/common/CommonHeaders.ts

@@ -0,0 +1,11 @@
+/**
+ * Adds common headers to all api calls to fidesops
+ */
+export function addCommonHeaders(headers: Headers, token: string | null) {
+  headers.set("Access-Control-Allow-Origin", "*");
+  headers.set("X-Fides-Source", "fidesops-admin-ui");
+  if (token) {
+    headers.set("authorization", `Bearer ${token}`);
+  }
+  return headers;
+}

clients/admin-ui/src/features/privacy-requests/RequestTable.tsx

@@ -75,7 +75,7 @@ const RequestTable: React.FC<RequestTableProps> = () => {
           </Tr>
         </Thead>
         <Tbody>
-          {requests.map((request) => (
+          {requests.map((request: any) => (
             <RequestRow request={request} key={request.id} />
           ))}
         </Tbody>

clients/admin-ui/src/features/privacy-requests/privacy-requests.slice.ts

@@ -4,6 +4,7 @@ import { createApi, fetchBaseQuery } from "@reduxjs/toolkit/query/react";
 import type { RootState } from "../../app/store";
 import { BASE_API_URN } from "../../constants";
 import { selectToken } from "../auth";
+import { addCommonHeaders } from "../common/CommonHeaders";
 import {
   DenyPrivacyRequest,
   PrivacyRequest,
@@ -47,17 +48,13 @@ export function mapFiltersToSearchParams({
 }
 
 // Subject requests API
-export const privacyRequestApi = createApi({
+export const privacyRequestApi: any = createApi({
   reducerPath: "privacyRequestApi",
   baseQuery: fetchBaseQuery({
     baseUrl: BASE_API_URN,
     prepareHeaders: (headers, { getState }) => {
-      const token = selectToken(getState() as RootState);
-      headers.set("Access-Control-Allow-Origin", "*");
-      if (token) {
-        headers.set("authorization", `Bearer ${token}`);
-      }
-      return headers;
+      const token: string | null = selectToken(getState() as RootState);
+      return addCommonHeaders(headers, token);
     },
   }),
   tagTypes: ["Request"],
@@ -130,6 +127,7 @@ export const requestCSVDownload = async ({
       headers: {
         "Access-Control-Allow-Origin": "*",
         Authorization: `Bearer ${token}`,
+        "X-Fides-Source": "fidesops-admin-ui",
       },
     }
   )

clients/admin-ui/src/features/user-management/UserManagementTable.tsx

@@ -66,7 +66,7 @@ const UserManagementTable: React.FC<UsersTableProps> = () => {
           </Tr>
         </Thead>
         <Tbody>
-          {users?.map((user) => (
+          {users?.map((user: any) => (
             <UserManagementRow user={user} key={user.id} />
           ))}
         </Tbody>

clients/admin-ui/src/features/user-management/user-management.slice.ts

@@ -4,6 +4,7 @@ import { createApi, fetchBaseQuery } from "@reduxjs/toolkit/query/react";
 import type { RootState } from "../../app/store";
 import { BASE_API_URN } from "../../constants";
 import { selectToken } from "../auth";
+import { addCommonHeaders } from "../common/CommonHeaders";
 import {
   User,
   UserPasswordUpdate,
@@ -76,17 +77,13 @@ export const mapFiltersToSearchParams = ({
   ...(username ? { username } : {}),
 });
 
-export const userApi = createApi({
+export const userApi: any = createApi({
   reducerPath: "userApi",
   baseQuery: fetchBaseQuery({
     baseUrl: BASE_API_URN,
     prepareHeaders: (headers, { getState }) => {
-      const token = selectToken(getState() as RootState);
-      headers.set("Access-Control-Allow-Origin", "*");
-      if (token) {
-        headers.set("authorization", `Bearer ${token}`);
-      }
-      return headers;
+      const token: string | null = selectToken(getState() as RootState);
+      return addCommonHeaders(headers, token);
     },
   }),
   tagTypes: ["User"],

clients/admin-ui/src/pages/api/auth/[...nextauth].ts

@@ -0,0 +1 @@
+export {};

clients/privacy-center/components/RequestModal.tsx

@@ -91,6 +91,7 @@ const useRequestForm = ({
           headers: {
             Accept: "application/json",
             "Content-Type": "application/json",
+            "X-Fides-Source": "fidesops-privacy-center",
           },
           body: JSON.stringify(body),
         });

docker-compose.yml

@@ -24,7 +24,7 @@ services:
         read_only: False
       - /fidesops/src/fidesops.egg-info
     environment:
-      - FIDESOPS__DEV_MODE=${FIDESOPS__DEV_MODE}
+      - FIDESOPS__DEV_MODE=True
       - FIDESOPS__LOG_PII=${FIDESOPS__LOG_PII}
       - FIDESOPS__HOT_RELOAD=${FIDESOPS__HOT_RELOAD}
       - FIDESOPS__ROOT_USER__ANALYTICS_ID=${FIDESOPS__ROOT_USER__ANALYTICS_ID}

requirements.txt

@@ -11,7 +11,7 @@ fastapi-pagination[sqlalchemy]~= 0.8.3
 fastapi[all]==0.78.0
 fideslang==1.0.0
 fideslib==2.0.4
-fideslog==1.1.5
+fideslog==1.2.0
 multidimensional_urlencode==0.0.4
 pandas==1.3.3
 passlib[bcrypt]==1.7.4

src/fidesops/main.py

@@ -1,11 +1,13 @@
 import logging
 from datetime import datetime, timezone
 from pathlib import Path
+from typing import Callable, Optional
 
 import uvicorn
-from fastapi import FastAPI
+from fastapi import FastAPI, Request, Response
 from fastapi.staticfiles import StaticFiles
 from fideslog.sdk.python.event import AnalyticsEvent
+from starlette.background import BackgroundTask
 from starlette.middleware.cors import CORSMiddleware
 
 from fidesops.analytics import (
@@ -19,7 +21,7 @@
 from fidesops.common_exceptions import FunctionalityNotConfigured
 from fidesops.core.config import config
 from fidesops.db.database import init_db
-from fidesops.schemas.analytics import EVENT
+from fidesops.schemas.analytics import Event, ExtraData
 from fidesops.tasks.scheduled.scheduler import scheduler
 from fidesops.tasks.scheduled.tasks import initiate_scheduled_request_intake
 from fidesops.util.logger import get_fides_log_record_factory
@@ -40,6 +42,65 @@
         allow_headers=["*"],
     )
 
+
+@app.middleware("http")
+async def dispatch_log_request(request: Request, call_next: Callable) -> Response:
+    fides_source: Optional[str] = request.headers.get("X-Fides-Source")
+    now: datetime = datetime.now(tz=timezone.utc)
+    valid_hostname = request.url.hostname
+    # only needed until we implement validator fix in fideslog
+    if valid_hostname == "0.0.0.0":
+        valid_hostname = "localhost"
+    endpoint = (
+        f"{request.method}: {request.url.scheme}://{valid_hostname}{request.url.path}"
+    )
+
+    try:
+        response = await call_next(request)
+        # HTTPExceptions are considered a handled err by default so are not thrown here.
+        # Accepted workaround is to inspect status code of response.
+        # More context- https://github.com/tiangolo/fastapi/issues/1840
+        response.background = BackgroundTask(
+            prepare_and_log_request,
+            endpoint,
+            response.status_code,
+            now,
+            fides_source,
+            "HTTPException" if response.status_code >= 400 else None,
+        )
+        return response
+
+    except Exception as e:
+        prepare_and_log_request(endpoint, 500, now, fides_source, e.__class__.__name__)
+        raise
+
+
+def prepare_and_log_request(
+    endpoint: str,
+    status_code: int,
+    event_created_at: datetime,
+    fides_source: Optional[str],
+    error_class: Optional[str],
+) -> None:
+    # this check prevents AnalyticsEvent from being called with invalid endpoint during unit tests
+    if config.root_user.ANALYTICS_OPT_OUT:
+        return
+    send_analytics_event(
+        AnalyticsEvent(
+            docker=in_docker_container(),
+            event=Event.endpoint_call.value,
+            event_created_at=event_created_at,
+            local_host=running_on_local_host(),
+            endpoint=endpoint,
+            status_code=status_code,
+            error=error_class or None,
+            extra_data={ExtraData.fides_source.value: fides_source}
+            if fides_source
+            else None,
+        )
+    )
+
+
 app.include_router(api_router)
 for handler in ExceptionHandlers.get_handlers():
     app.add_exception_handler(FunctionalityNotConfigured, handler)
@@ -84,7 +145,7 @@ def start_webserver() -> None:
     send_analytics_event(
         AnalyticsEvent(
             docker=in_docker_container(),
-            event=EVENT.server_start.value,
+            event=Event.server_start.value,
             event_created_at=datetime.now(tz=timezone.utc),
             local_host=running_on_local_host(),
         )

src/fidesops/schemas/analytics.py

@@ -1,7 +1,14 @@
 from enum import Enum
 
 
-class EVENT(str, Enum):
+class Event(str, Enum):
     """Enum to hold analytics event names"""
 
     server_start = "server_start"
+    endpoint_call = "endpoint_call"
+
+
+class ExtraData(str, Enum):
+    """Enum to hold keys for extra data"""
+
+    fides_source = "fides_source"

src/fidesops/task/graph_task.py

@@ -63,9 +63,6 @@ def result(*args: Any, **kwargs: Any) -> List[Optional[Row]]:
             self = args[0]
 
             raised_ex = None
-            if config.dev_mode:
-                # If dev mode, return here so exception isn't caught
-                return func(*args, **kwargs)
             for attempt in range(config.execution.TASK_RETRY_COUNT + 1):
                 try:
                     self.skip_if_disabled()

tests/api/v1/test_main.py

@@ -3,6 +3,12 @@
 from fidesops.api.v1.urn_registry import V1_URL_PREFIX
 
 
+class CustomTestException(BaseException):
+    """Mock Non-HTTP Exception"""
+
+    pass
+
+
 def test_read_autogenerated_docs(api_client: TestClient):
     """Test to ensure automatically generated docs build properly"""
     response = api_client.get(f"{V1_URL_PREFIX}/openapi.json")

tests/conftest.py

@@ -162,3 +162,12 @@ def integration_config() -> MutableMapping[str, Any]:
 def celery_enable_logging():
     """Turns on celery output logs."""
     return True
+
+
+@pytest.fixture(autouse=True, scope="session")
+def analytics_opt_out():
+    """Disable sending analytics when running tests."""
+    original_value = config.root_user.ANALYTICS_OPT_OUT
+    config.root_user.ANALYTICS_OPT_OUT = True
+    yield
+    config.root_user.ANALYTICS_OPT_OUT = original_value

tests/integration_tests/test_execution.py

@@ -5,6 +5,7 @@
 from pydantic import ValidationError
 from sqlalchemy.exc import InvalidRequestError
 
+from fidesops.core.config import config
 from fidesops.db.session import get_db_session
 from fidesops.graph.config import CollectionAddress
 from fidesops.graph.graph import DatasetGraph

tests/integration_tests/test_manual_task.py

@@ -3,6 +3,7 @@
 import pytest
 
 from fidesops.common_exceptions import PrivacyRequestPaused
+from fidesops.core.config import config
 from fidesops.graph.config import CollectionAddress
 from fidesops.models.policy import PausedStep
 from fidesops.models.privacy_request import (