From 981a9443b50c7575415067fd5eb3c4535e9783b1 Mon Sep 17 00:00:00 2001 From: Vincent Vallaeys Date: Sat, 27 Apr 2024 16:17:41 +0200 Subject: [PATCH] chore: deployment improvements (#372) * chore: reduce image size * feat: tool-versions * chore: networking * feat: dns servers * chore: remove unnecessary copy's * chore: I hate fixtures * chore: remove unneccessary rm * chore: added back container names --- .dev.env | 3 +- .gitignore | 1 - .prod.env | 6 ++-- .tool-versions | 2 ++ backend/Dockerfile | 19 +++++++++---- backend/gunicorn_config.py | 2 +- backend/setup.sh | 11 +++----- backend/ypovoli/settings.py | 1 - data/nginx/nginx.dev.conf | 2 +- data/nginx/nginx.prod.conf | 2 +- data/nginx/nginx.test.conf | 6 ++-- development.sh | 4 +-- development.yml | 17 +++++------- frontend/Dockerfile.dev | 2 -- frontend/Dockerfile.prod | 2 +- frontend/cypress.config.ts | 2 +- production.yml | 36 +++++++++--------------- test.sh | 16 +++++------ test.yml | 55 ++++++++++++++++--------------------- 19 files changed, 84 insertions(+), 105 deletions(-) create mode 100644 .tool-versions diff --git a/.dev.env b/.dev.env index 1738e1ad..7d96f8b7 100644 --- a/.dev.env +++ b/.dev.env @@ -10,7 +10,6 @@ FRONTEND_DIR="./frontend" SSL_DIR="./data/nginx/ssl" # Redis -REDIS_IP=192.168.90.10 REDIS_PORT=6379 # Django @@ -20,5 +19,5 @@ DJANGO_DOMAIN_NAME=localhost DJANGO_CAS_URL_PREFIX="" DJANGO_CAS_PORT=8080 DJANGO_DB_ENGINE="django.db.backends.sqlite3" -DJANGO_REDIS_HOST=${REDIS_IP} +DJANGO_REDIS_HOST="redis" DJANGO_REDIS_PORT=${REDIS_PORT} diff --git a/.gitignore b/.gitignore index 4feb86b2..4e8c4ed7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ -.tool-versions .env .venv .idea diff --git a/.prod.env b/.prod.env index 4506903a..031297c4 100644 --- a/.prod.env +++ b/.prod.env @@ -10,14 +10,12 @@ FRONTEND_DIR="./frontend" SSL_DIR="" # Postgress DB -POSTGRES_IP=192.168.90.9 POSTGRES_PORT=5432 POSTGRES_DB=selab POSTGRES_USER=selab_user POSTGRES_PASSWORD="" # Redis -REDIS_IP=192.168.90.10 REDIS_PORT=6379 # Django @@ -30,7 +28,7 @@ DJANGO_DB_ENGINE=django.db.backends.postgresql DJANGO_DB_NAME=${POSTGRES_DB} DJANGO_DB_USER=${POSTGRES_USER} DJANGO_DB_PASSWORD=${POSTGRES_PASSWORD} -DJANGO_DB_HOST=${POSTGRES_IP} +DJANGO_DB_HOST="postgres" DJANGO_DB_PORT=${POSTGRES_PORT} -DJANGO_REDIS_HOST=${REDIS_IP} +DJANGO_REDIS_HOST="redis" DJANGO_REDIS_PORT=${REDIS_PORT} diff --git a/.tool-versions b/.tool-versions new file mode 100644 index 00000000..ca7334bd --- /dev/null +++ b/.tool-versions @@ -0,0 +1,2 @@ +python 3.11.8 +nodejs 18.17.1 diff --git a/backend/Dockerfile b/backend/Dockerfile index f95477dd..653faddb 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,11 +1,20 @@ -FROM python:3.11.4-alpine3.18 +FROM python:3.11.4-alpine3.18 as requirements -RUN apk update && apk add --no-cache gettext libintl && pip install -U poetry -RUN poetry config virtualenvs.create false +RUN pip install poetry-plugin-export WORKDIR /code COPY pyproject.toml poetry.lock ./ -RUN poetry install --only main -COPY . ./ \ No newline at end of file +RUN poetry export --without-hashes --format=requirements.txt > requirements.txt + + +FROM python:3.11.4-alpine3.18 + +RUN apk add --no-cache gettext libintl + +WORKDIR /code + +COPY --from=requirements /code/requirements.txt . + +RUN pip install -r requirements.txt --no-cache-dir diff --git a/backend/gunicorn_config.py b/backend/gunicorn_config.py index 6ec33776..158f4679 100644 --- a/backend/gunicorn_config.py +++ b/backend/gunicorn_config.py @@ -1,4 +1,4 @@ workers = 4 -bind = "0.0.0.0:8080" +bind = "0.0.0.0:8000" chdir = "/code/" module = "ypovoli.wsgi:application" diff --git a/backend/setup.sh b/backend/setup.sh index 2e684df6..197b21f9 100755 --- a/backend/setup.sh +++ b/backend/setup.sh @@ -1,10 +1,7 @@ -echo "Installing dependencies..." -pip install poetry > /dev/null 2>&1 -poetry install > /dev/null echo "Migrating database..." -python manage.py migrate > /dev/null +python manage.py migrate > /dev/null 2>&1 echo "Compiling translations..." -django-admin compilemessages > /dev/null +django-admin compilemessages > /dev/null 2>&1 echo "Generating Swagger documentation..." -echo "yes" | python manage.py collectstatic > /dev/null -echo "Done" +echo "yes" | python manage.py collectstatic > /dev/null 2>&1 +echo "Setup complete" diff --git a/backend/ypovoli/settings.py b/backend/ypovoli/settings.py index 1381adab..fa9aa0e2 100644 --- a/backend/ypovoli/settings.py +++ b/backend/ypovoli/settings.py @@ -21,7 +21,6 @@ BASE_DIR = Path(__file__).resolve().parent.parent MEDIA_ROOT = os.path.normpath(os.path.join(BASE_DIR, "data/production")) -# TODO: What does this do? TESTING_BASE_LINK = "http://testserver" # SECURITY WARNING: keep the secret key used in production secret! diff --git a/data/nginx/nginx.dev.conf b/data/nginx/nginx.dev.conf index b6bec79a..ff0e992f 100644 --- a/data/nginx/nginx.dev.conf +++ b/data/nginx/nginx.dev.conf @@ -4,7 +4,7 @@ events { http { upstream backend { - server backend:8080; + server backend:8000; } upstream frontend { diff --git a/data/nginx/nginx.prod.conf b/data/nginx/nginx.prod.conf index 8d59a46f..c21ea310 100644 --- a/data/nginx/nginx.prod.conf +++ b/data/nginx/nginx.prod.conf @@ -4,7 +4,7 @@ events { http { upstream backend { - server backend:8080; + server backend:8000; } upstream frontend { diff --git a/data/nginx/nginx.test.conf b/data/nginx/nginx.test.conf index e8fcaae2..ff0e992f 100644 --- a/data/nginx/nginx.test.conf +++ b/data/nginx/nginx.test.conf @@ -4,15 +4,15 @@ events { http { upstream backend { - server test_backend:8080; + server backend:8000; } upstream frontend { - server test_frontend:5173; + server frontend:5173; } upstream hmr { - server test_frontend:443; + server frontend:443; } server { diff --git a/development.sh b/development.sh index 5fd7eb47..3e83e009 100755 --- a/development.sh +++ b/development.sh @@ -79,8 +79,8 @@ if [ "$data" != "" ]; then echo "This can take some time..." echo "--------------------------" - rm -f backend/db.sqlite3 > /dev/null 2>&1 - docker run -v backend:/code ugent-7_backend sh -c "python manage.py migrate; python manage.py loaddata */fixtures/$data/*" > /dev/null + docker run -v backend:/code ugent-7_backend sh -c "python manage.py flush --no-input; python manage.py migrate; python manage.py loaddata */fixtures/$data/*" > /dev/null 2>&1 + echo "Database filled." fi # Build Docker images diff --git a/development.yml b/development.yml index 9cb3a448..16f6592f 100644 --- a/development.yml +++ b/development.yml @@ -8,7 +8,7 @@ networks: driver: bridge ipam: config: - - subnet: 192.168.90.0/24 + - subnet: 192.168.91.0/24 ############################# EXTENSIONS @@ -24,14 +24,18 @@ x-common-keys-selab: &common-keys-selab PGID: $PGID env_file: - .env + dns: + - 1.1.1.1 + - 4.4.4.4 + - 8.8.8.8 ############################# SERVICES services: nginx: <<: *common-keys-selab - image: nginx:latest container_name: nginx + image: nginx:latest ports: - 80:80 - 443:443 @@ -49,9 +53,7 @@ services: build: context: $BACKEND_DIR dockerfile: Dockerfile - command: sh -c "./setup.sh; python manage.py runsslserver 192.168.90.2:8080" - expose: - - 8080 + command: sh -c "./setup.sh; python manage.py runsslserver 0.0.0.0:8000" volumes: - ${BACKEND_DIR}:/code @@ -76,8 +78,6 @@ services: context: $FRONTEND_DIR dockerfile: Dockerfile.dev command: sh -c "npm install && npm run host" - expose: - - 5173 volumes: - ${FRONTEND_DIR}:/app depends_on: @@ -87,9 +87,6 @@ services: <<: *common-keys-selab container_name: redis image: redis:latest - networks: - selab_network: - ipv4_address: $REDIS_IP expose: - $REDIS_PORT entrypoint: redis-server --appendonly yes --maxmemory 512mb --maxmemory-policy allkeys-lru diff --git a/frontend/Dockerfile.dev b/frontend/Dockerfile.dev index 7dd77795..c31cf82b 100644 --- a/frontend/Dockerfile.dev +++ b/frontend/Dockerfile.dev @@ -5,5 +5,3 @@ WORKDIR /app COPY package*.json ./ RUN npm install - -COPY . /app/ diff --git a/frontend/Dockerfile.prod b/frontend/Dockerfile.prod index 6311db3e..3a1c107d 100644 --- a/frontend/Dockerfile.prod +++ b/frontend/Dockerfile.prod @@ -5,7 +5,7 @@ RUN npm install COPY ./ . RUN npm run build -FROM nginx as production-stage +FROM nginx:alpine-slim as production-stage EXPOSE 3000 RUN mkdir /app COPY nginx.conf /etc/nginx/conf.d/default.conf diff --git a/frontend/cypress.config.ts b/frontend/cypress.config.ts index 686e5ac2..6b759d73 100644 --- a/frontend/cypress.config.ts +++ b/frontend/cypress.config.ts @@ -2,7 +2,7 @@ import { defineConfig } from 'cypress'; export default defineConfig({ e2e: { - baseUrl: 'http://test_nginx', + baseUrl: 'http://nginx', specPattern: 'src/test/e2e/**/*.cy.{js,jsx,ts,tsx}', }, }); diff --git a/production.yml b/production.yml index 13aacc0b..e72f63e0 100644 --- a/production.yml +++ b/production.yml @@ -3,8 +3,8 @@ version: "3.9" ############################# NETWORKS networks: - selab_network: - name: selab_network + selab_network_prod: + name: selab_network_prod driver: bridge ipam: config: @@ -14,7 +14,7 @@ networks: x-common-keys-selab: &common-keys-selab networks: - - selab_network + - selab_network_prod security_opt: - no-new-privileges:true restart: unless-stopped @@ -24,6 +24,10 @@ x-common-keys-selab: &common-keys-selab PGID: $PGID env_file: - .env + dns: + - 1.1.1.1 + - 4.4.4.4 + - 8.8.8.8 ############################# SERVICES @@ -31,7 +35,7 @@ services: nginx: <<: *common-keys-selab image: nginx:latest - container_name: nginx + container_name: nginx_prod ports: - 80:80 - 443:443 @@ -45,47 +49,35 @@ services: postgres: <<: *common-keys-selab image: postgres:15.2 - container_name: postgres - networks: - selab_network: - ipv4_address: $POSTGRES_IP + container_name: postgres_prod environment: POSTGRES_DB: $POSTGRES_DB POSTGRES_USER: $POSTGRES_USER POSTGRES_PASSWORD: $POSTGRES_PASSWORD - expose: - - $POSTGRES_PORT volumes: - ${DATA_DIR}/postgres:/var/lib/postgresql/data backend: <<: *common-keys-selab - container_name: backend + container_name: backend_prod build: context: $BACKEND_DIR dockerfile: Dockerfile command: sh -c "./setup.sh && gunicorn --config gunicorn_config.py ypovoli.wsgi:application" - expose: - - 8080 depends_on: - postgres redis: <<: *common-keys-selab - container_name: redis + container_name: redis_prod image: redis:latest - networks: - selab_network: - ipv4_address: $REDIS_IP - expose: - - $REDIS_PORT entrypoint: redis-server --appendonly yes --maxmemory 512mb --maxmemory-policy allkeys-lru volumes: - ${DATA_DIR}/redis:/data celery: <<: *common-keys-selab - container_name: celery + container_name: celery_prod build: context: $BACKEND_DIR dockerfile: Dockerfile @@ -98,11 +90,9 @@ services: frontend: <<: *common-keys-selab - container_name: frontend + container_name: frontend_prod build: context: $FRONTEND_DIR dockerfile: Dockerfile.prod - expose: - - 3000 depends_on: - backend diff --git a/test.sh b/test.sh index b6b7b363..500e6073 100755 --- a/test.sh +++ b/test.sh @@ -58,7 +58,7 @@ if [ "$build" = true ]; then fi echo "Starting services..." -docker-compose -f test.yml up -d --scale test_cypress=0 +docker-compose -f test.yml up -d --scale cypress=0 cypress_exit=0 vitest_exit=0 @@ -67,25 +67,25 @@ django_exit=0 if [ "$frontend" = true ]; then echo "Running frontend tests..." echo "Running Cypress tests..." - docker-compose -f test.yml up --exit-code-from test_cypress --abort-on-container-exit test_cypress + docker-compose -f test.yml up --exit-code-from cypress --abort-on-container-exit cypress cypress_exit=$? echo "Running Vitest tests..." - docker exec test_frontend npm run test + docker exec frontend npm run test vitest_exit=$? elif [ "$backend" = true ]; then echo "Running backend tests..." - docker exec test_backend python manage.py test + docker exec backend python manage.py test django_exit=$? else echo "Running backend tests..." - docker exec test_backend python manage.py test + docker exec backend python manage.py test django_exit=$? - echo "Running frontend tests..." + echo "Running frontend_test tests..." echo "Running Cypress tests..." - docker-compose -f test.yml up --exit-code-from test_cypress --abort-on-container-exit test_cypress + docker-compose -f test.yml up --exit-code-from cypress --abort-on-container-exit cypress cypress_exit=$? echo "Running Vitest tests..." - docker exec test_frontend npm run test + docker exec frontend npm run test vitest_exit=$? fi diff --git a/test.yml b/test.yml index 9ca22c67..4d90cf7c 100644 --- a/test.yml +++ b/test.yml @@ -8,7 +8,7 @@ networks: driver: bridge ipam: config: - - subnet: 192.168.91.0/24 + - subnet: 192.168.92.0/24 ############################# EXTENSIONS @@ -24,40 +24,38 @@ x-common-keys-selab_test: &common-keys-selab_test PGID: $PGID env_file: - .env + dns: + - 1.1.1.1 + - 4.4.4.4 + - 8.8.8.8 ############################# SERVICES services: - test_nginx: + nginx: <<: *common-keys-selab_test image: nginx:latest - container_name: test_nginx - expose: - - 80 - - 443 - - 8080 + container_name: nginx volumes: - ${DATA_DIR}/nginx/nginx.test.conf:/etc/nginx/nginx.conf:ro - ${SSL_DIR}:/etc/nginx/ssl:ro depends_on: - - test_backend - - test_frontend + - backend + - frontend - test_backend: + backend: <<: *common-keys-selab_test - container_name: test_backend + container_name: backend build: context: $BACKEND_DIR dockerfile: Dockerfile - command: sh -c "./setup.sh && python manage.py runsslserver 192.168.91.2:8080" - expose: - - 8080 + command: sh -c "./setup.sh && python manage.py runsslserver 0.0.0.0:8000" volumes: - $BACKEND_DIR:/code - test_celery: + celery: <<: *common-keys-selab_test - container_name: test_celery + container_name: celery build: context: $BACKEND_DIR dockerfile: Dockerfile @@ -65,39 +63,32 @@ services: volumes: - $BACKEND_DIR:/code depends_on: - - test_backend - - test_redis + - backend + - redis - test_frontend: + frontend: <<: *common-keys-selab_test - container_name: test_frontend + container_name: frontend build: context: $FRONTEND_DIR dockerfile: Dockerfile.dev command: sh -c "npm install && npm run host" - expose: - - 5173 volumes: - $FRONTEND_DIR:/app depends_on: - - test_backend + - backend - test_redis: + redis: <<: *common-keys-selab_test - container_name: test_redis + container_name: redis image: redis:latest - networks: - test_selab_network: - ipv4_address: 192.168.91.10 - expose: - - 6379 entrypoint: redis-server --appendonly yes --maxmemory 512mb --maxmemory-policy allkeys-lru volumes: - ${DATA_DIR}/redis:/data - test_cypress: + cypress: <<: *common-keys-selab_test - container_name: test_cypress + container_name: cypress image: cypress/included:cypress-12.17.3-node-18.16.0-chrome-114.0.5735.133-1-ff-114.0.2-edge-114.0.1823.51-1 working_dir: /e2e volumes: