From 1d841b966be62df44d7307ef1c28f60987c76ff1 Mon Sep 17 00:00:00 2001 From: sarayourfriend <24264157+sarayourfriend@users.noreply.github.com> Date: Thu, 17 Nov 2022 13:46:12 +1100 Subject: [PATCH] Add Nginx target to API Dockerfile (#990) * Get nginx target building and running Still having a networking issue, but I think it comes down to some gap in my understanding of how the nginx image works and how to forward ports to it properly * Fix CI; publish api-nginx image on release * Simplify CI collectstatic step * Update docker/setup-buildx-action version Co-authored-by: Krystle Salazar * Update actions * Revert "Update actions" This reverts commit 9d8d419ff594418220e9958ee1d350382456f08f. Co-authored-by: Krystle Salazar --- .github/workflows/ci_cd.yml | 48 +++++++++++++++++++++ api/.dockerignore | 3 ++ api/.gitignore | 1 + api/Dockerfile | 16 +++++++ api/catalog/settings.py | 2 +- api/nginx.conf.template | 85 +++++++++++++++++++++++++++++++++++++ docker-compose.yml | 3 ++ justfile | 7 +++ 8 files changed, 164 insertions(+), 1 deletion(-) create mode 100644 api/.gitignore create mode 100644 api/nginx.conf.template diff --git a/.github/workflows/ci_cd.yml b/.github/workflows/ci_cd.yml index 6a1607fe8..adbbbd52a 100644 --- a/.github/workflows/ci_cd.yml +++ b/.github/workflows/ci_cd.yml @@ -74,6 +74,52 @@ jobs: name: ${{ matrix.image }} path: /tmp/${{ matrix.image }}.tar + build-nginx: + # This requires a separate job due to the dependency on the other image builds + name: Build `nginx` Dockerfile target + runs-on: ubuntu-latest + needs: + - build-images + + steps: + - uses: actions/checkout@v3 + - uses: extractions/setup-just@v1 + + - name: Download all images + uses: actions/download-artifact@v2 + with: + path: /tmp + + - name: Load API and ingestion server images + run: | + docker load --input /tmp/api/api.tar + docker load --input /tmp/ingestion_server/ingestion_server.tar + + - name: collectstatic + run: just collectstatic + + - name: Setup Docker Buildx + uses: docker/setup-buildx-action@v2 + with: + install: true + + - name: Build image `nginx` + uses: docker/build-push-action@v2 + with: + context: api + target: nginx + push: false + tags: openverse-api-nginx + cache-from: type=gha,scope=nginx + cache-to: type=gha,scope=nginx + outputs: type=docker,dest=/tmp/api-nginx.tar + + - name: Upload image `api-nginx` + uses: actions/upload-artifact@v2 + with: + name: api-nginx + path: /tmp/api-nginx.tar + test-ing: name: Run tests for ingestion-server runs-on: ubuntu-latest @@ -281,6 +327,7 @@ jobs: needs: - test-ing - test-api + - build-nginx permissions: packages: write contents: read @@ -289,6 +336,7 @@ jobs: image: - api - ingestion_server + - api-nginx steps: - name: Log in to GitHub Docker Registry uses: docker/login-action@v1 diff --git a/api/.dockerignore b/api/.dockerignore index 782533e2b..c591dc985 100644 --- a/api/.dockerignore +++ b/api/.dockerignore @@ -4,3 +4,6 @@ !manage.py !Pipfile* !run.sh +!nginx.conf.template +!static +!nginx-entrypoint.sh diff --git a/api/.gitignore b/api/.gitignore new file mode 100644 index 000000000..5551fd0b9 --- /dev/null +++ b/api/.gitignore @@ -0,0 +1 @@ +static/* diff --git a/api/Dockerfile b/api/Dockerfile index a0fdbfad2..6aeb15128 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -34,6 +34,22 @@ COPY Pipfile Pipfile.lock ./ # Install Python dependencies system-wide (uses the active virtualenv) RUN pipenv install --system --deploy --dev +######### +# Nginx # +######### + +# This target assumes that the build host has run `manage.py collectstatic` +# `just collectstatic` is provided as a convenient alias for running it the expected way + +FROM nginx:1.23.2-alpine as nginx + +WORKDIR /app + +COPY nginx.conf.template /etc/nginx/templates/openverse-api.conf.template +COPY /static /app/static + +ENV NGINX_ENVSUBST_FILTER="DJANGO_UPSTREAM_URL" + ####### # API # ####### diff --git a/api/catalog/settings.py b/api/catalog/settings.py index 268ebb623..d8eee58b6 100644 --- a/api/catalog/settings.py +++ b/api/catalog/settings.py @@ -31,7 +31,7 @@ BASE_DIR = Path(__file__).resolve().parent.parent # Where to collect static files in production/development deployments -STATIC_ROOT = "/var/api_static_content/static" +STATIC_ROOT = config("STATIC_ROOT", default="/var/api_static_content/static") # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/2.0/howto/deployment/checklist/ diff --git a/api/nginx.conf.template b/api/nginx.conf.template new file mode 100644 index 000000000..78b556dfe --- /dev/null +++ b/api/nginx.conf.template @@ -0,0 +1,85 @@ +error_log /var/log/nginx/error.log; + +log_format json_combined escape=json + '{' + '"time_local":"\$time_local",' + '"remote_addr":"\$remote_addr",' + '"remote_user":"\$remote_user",' + '"request":"\$request",' + '"status": "\$status",' + '"host_header": "\$host",' + '"body_bytes_sent":\$body_bytes_sent,' + '"request_time":"\$request_time",' + '"http_referrer":"\$http_referer",' + '"http_user_agent":"\$http_user_agent",' + '"upstream_response_time":\$upstream_response_time,' + '"http_x_forwarded_for":"\$http_x_forwarded_for"' + '}'; + +access_log /var/log/nginx/access.log json_combined; + +tcp_nopush on; +tcp_nodelay on; +types_hash_max_size 2048; + +# Compress large responses to save bandwidth and improve latency +gzip on; +gzip_min_length 860; +gzip_vary on; +gzip_proxied expired private auth; +gzip_types application/json text/plain application/javascript; +gzip_disable "MSIE [1-6]\."; + +upstream django { + # DJANGO_UPSTREAM_URL must be substituted using a tool like envsubst + server $DJANGO_UPSTREAM_URL; +} + +server { + listen 8080; + server_name _; + charset utf-8; + client_max_body_size 75M; + error_page 500 /500.json; + + location /static { + alias /app/static; + } + + location /media { + alias /app/media; + } + + location / { + proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto \$scheme; + proxy_set_header Host \$http_host; + proxy_redirect off; + proxy_pass http://django; + error_page 500 /500.json; + } + + location ~ ^/(v1|admin)/ { + proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto \$scheme; + proxy_set_header Host \$http_host; + proxy_redirect off; + proxy_pass http://django; + error_page 500 /500.json; + } + + location /500.json { + default_type application/json; + return 500 '{"detail": "An internal server error occurred."}'; + } + + location /terms_of_service.html { + root /var/api/api; + index terms_of_service.html; + } + + location /version { + default_type "application/json"; + alias /var/api/version.json; + } +} diff --git a/docker-compose.yml b/docker-compose.yml index 7f0929cd5..5cbe021b9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -82,6 +82,9 @@ services: - cache env_file: - api/env.docker + environment: + STATIC_ROOT: ${STATIC_ROOT:-} + MEDIA_ROOT: ${MEDIA_ROOT:-} stdin_open: true tty: true user: ${DOCKER_USER_ID}:${DOCKER_GROUP_ID} diff --git a/justfile b/justfile index 138e7ed1e..f1049e0ce 100644 --- a/justfile +++ b/justfile @@ -230,6 +230,13 @@ stats media="images": ipython: just dj shell +# Run `collectstatic` to prepare for building the `nginx` Dockerfile target. +@collectstatic: _api-up + # The `STATIC_ROOT` setting is relative to the directory in which the Django + # container runs (i.e., the `api` directory at the root of the repository). + # The resulting output will be at `api/static` and is git ignored for convenience. + @STATIC_ROOT="./static" just dj collectstatic --noinput + ########## # Sphinx #