diff --git a/CHANGELOG.md b/CHANGELOG.md index c7445fe6d..9c6cac05f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,10 @@ The types of changes are: ## [Unreleased](https://github.com/ethyca/fidesops/compare/1.6.0...main) + +### Added +* `fidesops worker` command for running a Celery worker [#673](https://github.com/ethyca/fidesops/pull/673/) + ### Developer Experience * Reduce the size of the docker image [#707](https://github.com/ethyca/fidesops/pull/707) * Parallelize CI safe checks to reduce run time [#717](https://github.com/ethyca/fidesops/pull/717) diff --git a/Dockerfile b/Dockerfile.app similarity index 100% rename from Dockerfile rename to Dockerfile.app diff --git a/Dockerfile.worker b/Dockerfile.worker new file mode 100644 index 000000000..cda7f7f77 --- /dev/null +++ b/Dockerfile.worker @@ -0,0 +1,55 @@ +FROM node:16 as frontend + +FROM --platform=linux/amd64 python:3.9.13-slim-buster as backend + +ARG SKIP_MSSQL_INSTALLATION + +# Install auxiliary software +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + git \ + make \ + ipython \ + vim \ + curl \ + g++ \ + gnupg \ + gcc \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + + +RUN echo "Building fidesops celery worker" +RUN echo "ENVIRONMENT VAR: SKIP_MSSQL_INSTALLATION $SKIP_MSSQL_INSTALLATION" + +# SQL Server (MS SQL) +# https://docs.microsoft.com/en-us/sql/connect/odbc/linux-mac/installing-the-microsoft-odbc-driver-for-sql-server?view=sql-server-ver15 +RUN if [ "$SKIP_MSSQL_INSTALLATION" != "true" ] ; then apt-get install apt-transport-https ; fi +RUN if [ "$SKIP_MSSQL_INSTALLATION" != "true" ] ; then curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - ; fi +RUN if [ "$SKIP_MSSQL_INSTALLATION" != "true" ] ; then curl https://packages.microsoft.com/config/debian/10/prod.list | tee /etc/apt/sources.list.d/msprod.list ; fi +RUN if [ "$SKIP_MSSQL_INSTALLATION" != "true" ] ; then apt-get update ; fi +ENV ACCEPT_EULA=y DEBIAN_FRONTEND=noninteractive +RUN if [ "$SKIP_MSSQL_INSTALLATION" != "true" ] ; then apt-get -y install \ + unixodbc-dev \ + msodbcsql17 \ + mssql-tools ; fi + +# Update pip and install requirements +COPY requirements.txt dev-requirements.txt mssql-requirements.txt ./ +RUN pip install -U pip \ + && pip install 'cryptography~=3.4.8' \ + && pip install snowflake-connector-python --no-use-pep517 \ + && pip install -r requirements.txt -r dev-requirements.txt + +RUN if [ "$SKIP_MSSQL_INSTALLATION" != "true" ] ; then pip install -U pip -r mssql-requirements.txt ; fi + + +# Copy in the application files and install it locally +COPY . /fidesops +WORKDIR /fidesops +RUN pip install -e . + +# Enable detection of running within Docker +ENV RUNNING_IN_DOCKER=true + +CMD [ "fidesops", "worker" ] diff --git a/Makefile b/Makefile index 464faef65..799bafe57 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ REGISTRY := ethyca IMAGE_TAG := $(shell git fetch --force --tags && git describe --tags --dirty --always) -IMAGE_NAME := fidesops +IMAGE_NAME := webserver IMAGE := $(REGISTRY)/$(IMAGE_NAME):$(IMAGE_TAG) IMAGE_LATEST := $(REGISTRY)/$(IMAGE_NAME):latest @@ -69,7 +69,7 @@ quickstart: #################### docker-build: - docker build --tag $(IMAGE) . + docker build --tag $(IMAGE) -f Dockerfile.app . docker-push: docker tag $(IMAGE) $(IMAGE_LATEST) diff --git a/docker-compose.integration-mariadb.yml b/docker-compose.integration-mariadb.yml index cd59b8fa9..0deba0a47 100644 --- a/docker-compose.integration-mariadb.yml +++ b/docker-compose.integration-mariadb.yml @@ -1,5 +1,5 @@ services: - fidesops: + webserver: depends_on: - mariadb_example diff --git a/docker-compose.integration-mongodb.yml b/docker-compose.integration-mongodb.yml index 53267c696..9bee0c83f 100644 --- a/docker-compose.integration-mongodb.yml +++ b/docker-compose.integration-mongodb.yml @@ -1,5 +1,5 @@ services: - fidesops: + webserver: depends_on: - mongodb_example diff --git a/docker-compose.integration-mssql.yml b/docker-compose.integration-mssql.yml index 8320c4a4a..ecdba5650 100644 --- a/docker-compose.integration-mssql.yml +++ b/docker-compose.integration-mssql.yml @@ -1,5 +1,5 @@ services: - fidesops: + webserver: depends_on: - mssql_example diff --git a/docker-compose.integration-mysql.yml b/docker-compose.integration-mysql.yml index e00bb76b8..937a453fd 100644 --- a/docker-compose.integration-mysql.yml +++ b/docker-compose.integration-mysql.yml @@ -1,5 +1,5 @@ services: - fidesops: + webserver: depends_on: - mysql_example diff --git a/docker-compose.integration-postgres.yml b/docker-compose.integration-postgres.yml index fde35c1a1..c6e01c7b8 100644 --- a/docker-compose.integration-postgres.yml +++ b/docker-compose.integration-postgres.yml @@ -1,5 +1,5 @@ services: - fidesops: + webserver: depends_on: - postgres_example diff --git a/docker-compose.no-db.yml b/docker-compose.no-db.yml index ba8aea237..2cc7cf429 100644 --- a/docker-compose.no-db.yml +++ b/docker-compose.no-db.yml @@ -1,9 +1,8 @@ services: - fidesops: - container_name: fidesops + webserver: build: context: . - dockerfile: Dockerfile + dockerfile: Dockerfile.app expose: - 8080 healthcheck: @@ -29,7 +28,7 @@ services: docs: build: context: docs/fidesops/ - dockerfile: Dockerfile + dockerfile: Dockerfile.docs volumes: - ./docs/fidesops:/docs expose: diff --git a/docker-compose.yml b/docker-compose.yml index d6b6f27d4..705830c44 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,11 +1,10 @@ services: - fidesops: - container_name: fidesops + webserver: build: context: . - dockerfile: Dockerfile + dockerfile: Dockerfile.app depends_on: - - celery + - worker - db - redis expose: @@ -59,11 +58,11 @@ services: ports: - "0.0.0.0:6379:6379" - celery: + worker: build: context: . - dockerfile: Dockerfile - command: celery -A fidesops.tasks.celery_app worker --loglevel=info + dockerfile: Dockerfile.worker + command: fidesops worker depends_on: redis: condition: service_started @@ -73,11 +72,12 @@ services: source: ./ target: /fidesops read_only: False + - /fidesops/src/fidesops.egg-info docs: build: context: docs/fidesops/ - dockerfile: Dockerfile + dockerfile: Dockerfile.docs volumes: - ./docs/fidesops:/docs expose: diff --git a/docs/fidesops/Dockerfile b/docs/fidesops/Dockerfile.docs similarity index 100% rename from docs/fidesops/Dockerfile rename to docs/fidesops/Dockerfile.docs diff --git a/run_infrastructure.py b/run_infrastructure.py index d9d4f9bec..523df7bca 100644 --- a/run_infrastructure.py +++ b/run_infrastructure.py @@ -24,7 +24,7 @@ "bigquery": ["BIGQUERY_KEYFILE_CREDS", "BIGQUERY_DATASET"], } EXTERNAL_DATASTORES = list(EXTERNAL_DATASTORE_CONFIG.keys()) -IMAGE_NAME = "fidesops" +IMAGE_NAME = "webserver" def run_infrastructure( diff --git a/src/fidesops/cli.py b/src/fidesops/cli.py index 85f480155..757d94096 100644 --- a/src/fidesops/cli.py +++ b/src/fidesops/cli.py @@ -1,7 +1,12 @@ """Sets up a simple fidesops CLI""" +import logging + import click from fidesops.main import start_webserver +from fidesops.tasks import start_worker + +logger = logging.getLogger(__name__) @click.group() @@ -18,3 +23,13 @@ def webserver(ctx: click.Context) -> None: Runs any pending DB migrations and starts the webserver. """ start_webserver() + + +@cli.command() +@click.pass_context +def worker(ctx: click.Context) -> None: + """ + Starts a Celery worker + """ + logger.info("Running Celery worker from CLI...") + start_worker() diff --git a/src/fidesops/tasks/__init__.py b/src/fidesops/tasks/__init__.py index 3dc16da95..850c3232c 100644 --- a/src/fidesops/tasks/__init__.py +++ b/src/fidesops/tasks/__init__.py @@ -28,6 +28,10 @@ def _create_celery() -> Celery: celery_app = _create_celery() -if __name__ == "__main__": +def start_worker() -> None: logger.info("Running Celery worker...") - celery_app.worker_main() + celery_app.worker_main(argv=["worker", "--loglevel=info"]) + + +if __name__ == "__main__": + start_worker()