From d431fe4ea6ecec3ccf0ac55b6419fa1174ebe405 Mon Sep 17 00:00:00 2001 From: Antti Viitala Date: Mon, 18 Dec 2023 15:56:53 +0800 Subject: [PATCH] feat: init commit --- .github/workflows/build-container.yaml | 44 ++++++ .gitignore | 177 +++++++++++++++++++++++++ Dockerfile | 47 +++++++ README.md | 58 ++++++++ deployment/server-entrypoint.sh | 15 +++ deployment/worker-entrypoint.sh | 2 + docker-compose.yaml | 57 ++++++++ makefile | 25 ++++ manage.py | 22 +++ railway_django_stack/__init__.py | 3 + railway_django_stack/asgi.py | 16 +++ railway_django_stack/celery.py | 17 +++ railway_django_stack/settings.py | 143 ++++++++++++++++++++ railway_django_stack/urls.py | 22 +++ railway_django_stack/wsgi.py | 16 +++ requirements-dev.txt | 7 + requirements.txt | 8 ++ 17 files changed, 679 insertions(+) create mode 100644 .github/workflows/build-container.yaml create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 README.md create mode 100755 deployment/server-entrypoint.sh create mode 100644 deployment/worker-entrypoint.sh create mode 100644 docker-compose.yaml create mode 100644 makefile create mode 100755 manage.py create mode 100644 railway_django_stack/__init__.py create mode 100644 railway_django_stack/asgi.py create mode 100644 railway_django_stack/celery.py create mode 100644 railway_django_stack/settings.py create mode 100644 railway_django_stack/urls.py create mode 100644 railway_django_stack/wsgi.py create mode 100644 requirements-dev.txt create mode 100644 requirements.txt diff --git a/.github/workflows/build-container.yaml b/.github/workflows/build-container.yaml new file mode 100644 index 0000000..a477d1e --- /dev/null +++ b/.github/workflows/build-container.yaml @@ -0,0 +1,44 @@ +name: "Build Django image" + +concurrency: + group: ${{ github.ref }} + cancel-in-progress: true + +on: + push: + branches: + - "main" + workflow_dispatch: + +permissions: + contents: read + packages: write + +jobs: + build-push: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + + - name: Build and push + uses: docker/build-push-action@v4 + id: build-image + env: + REGISTRY: ghcr.io/${{ github.repository }} + with: + context: ./. + file: ./Dockerfile + push: true + tags: ${{env.REGISTRY}}:${{github.run_number}} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1c3ab93 --- /dev/null +++ b/.gitignore @@ -0,0 +1,177 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal +test_emails/ +test_backups/ + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +# Ignore 'real' .env files, but ensure that .erb ending env files are committed as these are templates +.env +.env.* +!.env*.erb +*.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +.DS_Store + +# Django +*.sql + +# Linters +**.ruff_cache + +# Profilers +importtime.txt diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..69b8ff1 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,47 @@ +ARG PYTHON_BUILD_VERSION=3.11 + +FROM python:$PYTHON_BUILD_VERSION-slim AS BASE +WORKDIR /app + +ENV PIP_DEFAULT_TIMEOUT=100 \ + # Allow statements and log messages to immediately appear + PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + # disable a pip version check to reduce run-time & log-spam + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + # cache is useless in docker image, so disable it to reduce image size + PIP_NO_CACHE_DIR=1 + +# Update and install dependencies +RUN apt-get update && \ + apt-get -y upgrade && \ + apt-get -y install --no-install-recommends python3-dev gcc libc-dev vim curl postgresql-client && \ + # Clean up + apt-get autoremove -y && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* && \ + # Add a non-root user + groupadd -g 999 python && \ + useradd -r -u 999 -g python python + +WORKDIR /app + +# Python Dependencies +COPY --chown=python:python ./requirements.txt /app/ +RUN pip install --upgrade pip \ + && pip install -r requirements.txt --no-cache-dir --compile + +# Clean up +RUN apt-get -y purge gcc libc-dev python3-dev + +# Add all application code from this folder, including deployment entrypoints +COPY --chown=python:python ./ /app + +# Make entrypoints executable +RUN chmod +x /app/deployment/server-entrypoint.sh && \ + chmod +x /app/deployment/worker-entrypoint.sh + +USER 999 + +EXPOSE 8000 +CMD [ "/app/deployment/server-entrypoint.sh" ] diff --git a/README.md b/README.md new file mode 100644 index 0000000..11e83dc --- /dev/null +++ b/README.md @@ -0,0 +1,58 @@ +# `railway_django_stack` + + + +## Resources + +This template deploys: + +- 1 container running Django +- 1 container running Celery (same as container #1 but with different startup command) +- 1 service running Redis +- 1 service running Postgres + +You can test the setup locally with docker compose: + +```bash +git clone https://github.com/Antvirf/railway_django_stack +cd railway_django_stack +docker-compose up +``` + +## Service diagram + +```mermaid +flowchart LR + +subgraph rwp["Your Railway Project"] + subgraph public["Publicly exposed services"] + django["App container\n(Django server)"] + end + subgraph private["Private services"] + celery["App container\n(Celery worker)"] + psql["PostgreSQL"] + redis["Redis"] + end +end + +users["Users"] --> django +django --> celery +django --> psql + +celery --> psql +celery --> redis +django --> redis + +``` + +## Django project setup + +This is a barebones Django-project with the following additions/updates: + +- Configures a PostgreSQL database +- Configures a Redis cache +- Configures Celery, and installs the following add-on apps: + - [`django-celery-beat`](https://github.com/celery/django-celery-beat) for periodic task management + - [`django-celery-results`](https://github.com/celery/django-celery-results) for viewing results of Celery tasks in Django Admin +- Uses [`python-decouple`](https://github.com/HBNetwork/python-decouple) to manage settings via environment varialbes +- Installs and runs with [`gunicorn`](https://github.com/benoitc/gunicorn) diff --git a/deployment/server-entrypoint.sh b/deployment/server-entrypoint.sh new file mode 100755 index 0000000..e810668 --- /dev/null +++ b/deployment/server-entrypoint.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +if [ "$RUN_MIGRATIONS" = "True" ]; then + echo "Running migrations..." + until python manage.py migrate + do + echo "Waiting for db to be ready..." + sleep 2 + done +fi + +echo "Starting app server..." +python -m gunicorn railway_django_stack.wsgi:application \ + --bind 0.0.0.0:8000 \ + --log-level info diff --git a/deployment/worker-entrypoint.sh b/deployment/worker-entrypoint.sh new file mode 100644 index 0000000..a19b222 --- /dev/null +++ b/deployment/worker-entrypoint.sh @@ -0,0 +1,2 @@ +#!/bin/sh +celery -A railway_django_stack worker --beat -l INFO \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..c92eaf2 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,57 @@ +version: "3" +services: + server: + build: &app-build + context: . + dockerfile: Dockerfile + args: + PYTHON_BUILD_VERSION: "3.11" + environment: &app-environment + DEBUG: "True" + RUN_MIGRATIONS: "True" + DJANGO_SECRET_KEY: "insecure-key-fill-with-your-own" + REDIS_URL: "redis://redis:6379" + POSTGRES_HOST: "db" + POSTGRES_DB: "postgres" + POSTGRES_USER: "postgres" + POSTGRES_PASSWORD: "postgres" + POSTGRES_PORT: "5432" + depends_on: &app-depends-on + - db + - redis + command: /app/deployment/server-entrypoint.sh + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/"] + interval: 5s + timeout: 2s + retries: 5 + start_period: 5s + ports: + - 8000:8000 + + worker: + build: *app-build + environment: *app-environment + depends_on: *app-depends-on + command: /app/deployment/worker-entrypoint.sh + + redis: + restart: unless-stopped + image: redis:latest + expose: + - 6379 + + db: + image: postgres:14.0-alpine + restart: unless-stopped + volumes: + - postgres_data:/var/lib/postgresql/data/ + environment: + POSTGRES_DB: postgres + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + expose: + - 5432 + +volumes: + postgres_data: {} diff --git a/makefile b/makefile new file mode 100644 index 0000000..9c9a7a6 --- /dev/null +++ b/makefile @@ -0,0 +1,25 @@ +.PHONY: venv-create install + +## ENV SETUP +venv-create: + python -m venv venv + +install: + pip install -r requirements.txt + +install-dev: + pip install -r requirements-dev.txt + +## DEVELOPMENT helpers +lint: + black . + ruff check --fix . --ignore E501 + djlint . --reformat + isort . --profile black + +celery: + celery -A railway_django_stack worker --beat -l INFO + +redis: + docker run -p 6379:6379 redis + diff --git a/manage.py b/manage.py new file mode 100755 index 0000000..b940c8b --- /dev/null +++ b/manage.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +"""Django's command-line utility for administrative tasks.""" +import os +import sys + + +def main(): + """Run administrative tasks.""" + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "railway_django_stack.settings") + try: + from django.core.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc + execute_from_command_line(sys.argv) + + +if __name__ == "__main__": + main() diff --git a/railway_django_stack/__init__.py b/railway_django_stack/__init__.py new file mode 100644 index 0000000..53f4ccb --- /dev/null +++ b/railway_django_stack/__init__.py @@ -0,0 +1,3 @@ +from .celery import app as celery_app + +__all__ = ("celery_app",) diff --git a/railway_django_stack/asgi.py b/railway_django_stack/asgi.py new file mode 100644 index 0000000..4dfd761 --- /dev/null +++ b/railway_django_stack/asgi.py @@ -0,0 +1,16 @@ +""" +ASGI config for railway_django_stack project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/4.2/howto/deployment/asgi/ +""" + +import os + +from django.core.asgi import get_asgi_application + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "railway_django_stack.settings") + +application = get_asgi_application() diff --git a/railway_django_stack/celery.py b/railway_django_stack/celery.py new file mode 100644 index 0000000..13cd2e6 --- /dev/null +++ b/railway_django_stack/celery.py @@ -0,0 +1,17 @@ +import os + +from celery import Celery + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "railway_django_stack.settings") + +app = Celery("railway_django_stack") + +app.config_from_object("django.conf:settings", namespace="CELERY") + +# Load task modules from all registered Django apps. +app.autodiscover_tasks() + + +@app.task(bind=True, ignore_result=True) +def example_task(self): + print("You've triggered the example task!") diff --git a/railway_django_stack/settings.py b/railway_django_stack/settings.py new file mode 100644 index 0000000..c0d6b1d --- /dev/null +++ b/railway_django_stack/settings.py @@ -0,0 +1,143 @@ +""" +Django settings for railway_django_stack project. + +Generated by 'django-admin startproject' using Django 4.2.2. + +For more information on this file, see +https://docs.djangoproject.com/en/4.2/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/4.2/ref/settings/ +""" + +from pathlib import Path + +from decouple import config + +# Build paths inside the project like this: BASE_DIR / 'subdir'. +BASE_DIR = Path(__file__).resolve().parent.parent + + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = config("DJANGO_SECRET_KEY", default="insecure-key-fill-with-your-own") + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = config("DEBUG", default=False, cast=bool) + +ALLOWED_HOSTS = [] + + +# Application definition + +INSTALLED_APPS = [ + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.messages", + "django.contrib.staticfiles", + "django_celery_beat", + "django_celery_results", + # Your custom apps would come here +] + +MIDDLEWARE = [ + "django.middleware.security.SecurityMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", +] + +ROOT_URLCONF = "railway_django_stack.urls" + +TEMPLATES = [ + { + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [], + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", + ], + }, + }, +] + +WSGI_APPLICATION = "railway_django_stack.wsgi.application" + + +# DATABASE CONFIG +DATABASES = { + "default": { + "ENGINE": "django.db.backends.postgresql", + "HOST": config("POSTGRES_HOST", default="db"), + "USER": config("POSTGRES_USER", default="postgres"), + "NAME": config("POSTGRES_DB", default="postgres"), + "PASSWORD": config("POSTGRES_PASSWORD", default="postgres"), + "PORT": config("POSTGRES_PORT", default=5432), + "CONN_MAX_AGE": 60, + } +} + +# CELERY CONFIG +CELERY_BROKER_URL = config("REDIS_URL", default="redis://redis:6379") +CELERY_RESULT_BACKEND = config("CELERY_RESULT_BACKEND", default="django-db") +CELERY_BEAT_SCHEDULER = config( + "CELERY_BEAT_SCHEDULER", default="django_celery_beat.schedulers.DatabaseScheduler" +) + +# REDIS CACHE CONFIG +CACHES = { + "default": { + "BACKEND": "django.core.cache.backends.redis.RedisCache", + "LOCATION": config("REDIS_URL", default="redis://redis:6379"), + } +} + + +# Password validation +# https://docs.djangoproject.com/en/4.2/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/4.2/topics/i18n/ + +LANGUAGE_CODE = "en-us" + +TIME_ZONE = "UTC" + +USE_I18N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/4.2/howto/static-files/ + +STATIC_URL = "static/" + +# Default primary key field type +# https://docs.djangoproject.com/en/4.2/ref/settings/#default-auto-field + +DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" diff --git a/railway_django_stack/urls.py b/railway_django_stack/urls.py new file mode 100644 index 0000000..a3cb6dc --- /dev/null +++ b/railway_django_stack/urls.py @@ -0,0 +1,22 @@ +""" +URL configuration for railway_django_stack project. + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/4.2/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: path('', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.urls import include, path + 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) +""" +from django.contrib import admin +from django.urls import path + +urlpatterns = [ + path("admin/", admin.site.urls), +] diff --git a/railway_django_stack/wsgi.py b/railway_django_stack/wsgi.py new file mode 100644 index 0000000..74b8db0 --- /dev/null +++ b/railway_django_stack/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for railway_django_stack project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/4.2/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "railway_django_stack.settings") + +application = get_wsgi_application() diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..a8fdd5b --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,7 @@ +isort==5.12.0 +djlint==1.32.1 +EditorConfig==0.12.3 +black==23.11.0 +ruff==0.0.287 +mypy-extensions==1.0.0 +coverage==7.3.2 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..a53ee1b --- /dev/null +++ b/requirements.txt @@ -0,0 +1,8 @@ +celery==5.3.1 +Django==4.2.8 +django-celery-beat==2.5.0 +django-celery-results==2.5.1 +gunicorn==21.2.0 +psycopg[binary]==3.1.15 +python-decouple==3.8 +redis==5.0.1 \ No newline at end of file