Adds fidesops worker command to start the Celery worker [#663]

Dockerfile.worker

@@ -0,0 +1,53 @@
+FROM node:16 as frontend
+
+FROM --platform=linux/amd64 python:3.9.13-slim-buster as backend
+
+ARG MSSQL_REQUIRED
+
+# Install auxiliary software
+RUN apt-get update && \
+    apt-get install -y \
+    git \
+    make \
+    ipython \
+    vim \
+    curl \
+    g++ \
+    gnupg \
+    gcc
+
+
+RUN echo "Building fidesops celery worker"
+RUN echo "ENVIRONMENT VAR:  $MSSQL_REQUIRED"
+
+# SQL Server (MS SQL)
+# https://docs.microsoft.com/en-us/sql/connect/odbc/linux-mac/installing-the-microsoft-odbc-driver-for-sql-server?view=sql-server-ver15
+RUN if [ "$MSSQL_REQUIRED" = "true" ] ; then apt-get install apt-transport-https ; fi
+RUN if [ "$MSSQL_REQUIRED" = "true" ] ; then curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - ; fi
+RUN if [ "$MSSQL_REQUIRED" = "true" ] ; then curl https://packages.microsoft.com/config/debian/10/prod.list | tee /etc/apt/sources.list.d/msprod.list ; fi
+RUN if [ "$MSSQL_REQUIRED" = "true" ] ; then apt-get update ; fi
+ENV ACCEPT_EULA=y DEBIAN_FRONTEND=noninteractive
+RUN if [ "$MSSQL_REQUIRED" = "true" ] ; then apt-get -y install \
+    unixodbc-dev \
+    msodbcsql17 \
+    mssql-tools ; fi
+
+# Update pip and install requirements
+COPY requirements.txt dev-requirements.txt mssql-requirements.txt ./
+RUN pip install -U pip  \
+    && pip install 'cryptography~=3.4.8' \
+    && pip install snowflake-connector-python --no-use-pep517  \
+    && pip install -r requirements.txt -r dev-requirements.txt
+
+RUN if [ "$MSSQL_REQUIRED" = "true" ] ; then pip install -U pip -r mssql-requirements.txt ; fi
+
+
+# Copy in the application files and install it locally
+COPY . /fidesops
+WORKDIR /fidesops
+RUN pip install -e .
+
+# Enable detection of running within Docker
+ENV RUNNING_IN_DOCKER=true
+
+CMD [ "fidesops", "worker" ]

Makefile

@@ -7,7 +7,7 @@
 REGISTRY := ethyca
 IMAGE_TAG := $(shell git fetch --force --tags && git describe --tags --dirty --always)
 
-IMAGE_NAME := fidesops
+IMAGE_NAME := webserver
 IMAGE := $(REGISTRY)/$(IMAGE_NAME):$(IMAGE_TAG)
 IMAGE_LATEST := $(REGISTRY)/$(IMAGE_NAME):latest
 

docker-compose.integration-mariadb.yml

@@ -1,5 +1,5 @@
 services:
-  fidesops:
+  webserver:
     depends_on:
       - mariadb_example
 

docker-compose.integration-mongodb.yml

@@ -1,5 +1,5 @@
 services:
-  fidesops:
+  webserver:
     depends_on:
       - mongodb_example
 

docker-compose.integration-mssql.yml

@@ -1,5 +1,5 @@
 services:
-  fidesops:
+  webserver:
     depends_on:
       - mssql_example
 

docker-compose.integration-mysql.yml

@@ -1,5 +1,5 @@
 services:
-  fidesops:
+  webserver:
     depends_on:
       - mysql_example
 

docker-compose.integration-postgres.yml

@@ -1,5 +1,5 @@
 services:
-  fidesops:
+  webserver:
     depends_on:
       - postgres_example
 

docker-compose.no-db.yml

@@ -1,9 +1,8 @@
 services:
-  fidesops:
-    container_name: fidesops
+  webserver:
     build:
       context: .
-      dockerfile: Dockerfile
+      dockerfile: Dockerfile.app
     expose:
       - 8080
     healthcheck:
@@ -29,7 +28,7 @@ services:
   docs:
     build:
       context: docs/fidesops/
-      dockerfile: Dockerfile
+      dockerfile: Dockerfile.docs
     volumes:
       - ./docs/fidesops:/docs
     expose:

docker-compose.yml

@@ -1,11 +1,10 @@
 services:
-  fidesops:
-    container_name: fidesops
+  webserver:
     build:
       context: .
-      dockerfile: Dockerfile
+      dockerfile: Dockerfile.app
     depends_on:
-      - celery
+      - worker
       - db
       - redis
     expose:
@@ -59,11 +58,11 @@ services:
     ports:
       - "0.0.0.0:6379:6379"
 
-  celery:
+  worker:
     build:
       context: .
-      dockerfile: Dockerfile
-    command: celery -A fidesops.tasks.celery_app worker --loglevel=info
+      dockerfile: Dockerfile.worker
+    command: fidesops worker
     depends_on:
       redis:
         condition: service_started
@@ -77,7 +76,7 @@ services:
   docs:
     build:
       context: docs/fidesops/
-      dockerfile: Dockerfile
+      dockerfile: Dockerfile.docs
     volumes:
       - ./docs/fidesops:/docs
     expose:

run_infrastructure.py

@@ -24,7 +24,7 @@
     "bigquery": ["BIGQUERY_KEYFILE_CREDS", "BIGQUERY_DATASET"],
 }
 EXTERNAL_DATASTORES = list(EXTERNAL_DATASTORE_CONFIG.keys())
-IMAGE_NAME = "fidesops"
+IMAGE_NAME = "webserver"
 
 
 def run_infrastructure(

src/fidesops/cli.py

@@ -2,6 +2,7 @@
 import click
 
 from fidesops.main import start_webserver
+from fidesops.tasks import start_worker
 
 
 @click.group()
@@ -18,3 +19,13 @@ def webserver(ctx: click.Context) -> None:
     Runs any pending DB migrations and starts the webserver.
     """
     start_webserver()
+
+
+@cli.command()
+@click.pass_context
+def worker(ctx: click.Context) -> None:
+    """
+    Starts a Celery worker
+    """
+    celery_args = ["worker", "--loglevel=info"]
+    start_worker(celery_args)

src/fidesops/tasks/__init__.py

@@ -1,3 +1,5 @@
+from typing import List, Optional
+
 from celery import Celery
 from celery.utils.log import get_task_logger
 
@@ -28,6 +30,10 @@ def _create_celery() -> Celery:
 celery_app = _create_celery()
 
 
-if __name__ == "__main__":
+def start_worker(argv: Optional[List[str]] = None) -> None:
     logger.info("Running Celery worker...")
-    celery_app.worker_main()
+    celery_app.worker_main(argv)
+
+
+if __name__ == "__main__":
+    start_worker()