From 95b3b5b8f8255a6adc1e24c6ff1fcd19d15b715c Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Tue, 3 Oct 2023 17:18:30 +0545 Subject: [PATCH 1/2] Added support for docker CPU --- backend/Dockerfile_CPU | 32 ++++++++++++++++++ docker-compose-cpu.yml | 76 ++++++++++++++++++++++++++++++++++++++++++ docker-compose.yml | 7 +--- 3 files changed, 109 insertions(+), 6 deletions(-) create mode 100644 backend/Dockerfile_CPU create mode 100644 docker-compose-cpu.yml diff --git a/backend/Dockerfile_CPU b/backend/Dockerfile_CPU new file mode 100644 index 00000000..5fa52037 --- /dev/null +++ b/backend/Dockerfile_CPU @@ -0,0 +1,32 @@ +FROM tensorflow/tensorflow:2.9.2 + +RUN apt-get update && apt-get install -y python3-opencv +RUN add-apt-repository ppa:ubuntugis/ppa && apt-get update +RUN apt-get update +RUN apt-get install -y gdal-bin +RUN apt-get install -y libgdal-dev +ENV CPLUS_INCLUDE_PATH=/usr/include/gdal +ENV C_INCLUDE_PATH=/usr/include/gdal + +#install numpy before gdal +RUN pip install numpy==1.23.5 + +# pip install dependencies. +RUN pip install --global-option=build_ext --global-option="-I/usr/include/gdal" GDAL==`gdal-config --version` + +COPY docker/ramp/docker-requirements.txt docker-requirements.txt +RUN pip install -r docker-requirements.txt + +# pip install solaris -- try with tmp-free build +COPY docker/ramp/solaris /tmp/solaris +RUN pip install /tmp/solaris --use-feature=in-tree-build + +RUN pip install scikit-fmm --use-feature=in-tree-build + +RUN pip install setuptools --upgrade + +COPY requirements.txt requirements.txt +RUN pip install -r requirements.txt + +WORKDIR /app +COPY . /app \ No newline at end of file diff --git a/docker-compose-cpu.yml b/docker-compose-cpu.yml new file mode 100644 index 00000000..f54319b0 --- /dev/null +++ b/docker-compose-cpu.yml @@ -0,0 +1,76 @@ +version: "3.8" + +services: + postgres: + restart: always + image: postgis/postgis + container_name: pgsql + environment: + - POSTGRES_DB=ai + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=admin + volumes: + - ./postgres-data:/var/lib/postgresql/data + ports: + - "5434:5432" + + redis: + image: redis + container_name: redis + ports: + - "6379:6379" + + backend-api: + build: + context: ./backend + dockerfile: Dockerfile + container_name: api + command: python manage.py runserver 0.0.0.0:8000 + + ports: + - 8000:8000 + volumes: + - ./backend:/app + - ${RAMP_HOME}:/RAMP_HOME + - ${TRAINING_WORKSPACE}:/TRAINING_WORKSPACE + depends_on: + - redis + - postgres + + backend-worker: + build: + context: ./backend + dockerfile: Dockerfile + container_name: worker + command: celery -A aiproject worker --loglevel=INFO --concurrency=1 + + volumes: + - ./backend:/app + - ${RAMP_HOME}:/RAMP_HOME + - ${TRAINING_WORKSPACE}:/TRAINING_WORKSPACE + depends_on: + - backend-api + - redis + - postgres + + worker-dashboard: + image: mher/flower + container_name: flower + command: celery --broker=redis://redis:6379// flower --address=0.0.0.0 --port=5000 + ports: + - 5500:5000 + depends_on: + - backend-api + - redis + - backend-worker + + frontend: + build: + context: ./frontend + dockerfile: Dockerfile.frontend + container_name: frontend + command: npm start -- --host 0.0.0.0 --port 3000 + ports: + - 3000:3000 + depends_on: + - backend-api diff --git a/docker-compose.yml b/docker-compose.yml index 7d949280..ac37dbee 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,12 +26,7 @@ services: dockerfile: Dockerfile container_name: api command: python manage.py runserver 0.0.0.0:8000 - deploy: - resources: - reservations: - devices: - - driver: nvidia - capabilities: [gpu] + ports: - 8000:8000 volumes: From 08f3402c42db0ba9dc48bd74ff9caaadaa2f8562 Mon Sep 17 00:00:00 2001 From: kshtiijrajsharma Date: Tue, 3 Oct 2023 22:24:33 +0545 Subject: [PATCH 2/2] Updated requirements alongside with dockerfile comments and migrations bash script --- .gitignore | 10 ++++++--- backend/Dockerfile | 44 ++++++++++++++++++++---------------- backend/Dockerfile_CPU | 45 +++++++++++++++++++++---------------- backend/requirements.txt | 2 +- docker-compose-cpu.yml | 4 ++-- docs/Docker-installation.md | 18 ++++++++++----- run_migrations.sh | 6 +++++ 7 files changed, 79 insertions(+), 50 deletions(-) create mode 100644 run_migrations.sh diff --git a/.gitignore b/.gitignore index 60d4c644..8b0ba420 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,11 @@ frontend/node_modules frontend/.pnp frontend/.pnp.js +#standard +.env +**/.vscode/** +postgres-data/ + # testing frontend/coverage postgres-data/* @@ -28,7 +33,7 @@ frontend/package-lock* frontend/.env -# for +# for backend backend/static backend/api_static backend/env @@ -40,8 +45,7 @@ backend/media backend/data/* backend/log/* backend/training/* +trainings/* backend/.env backend/config.txt backend/postgres-data -**/.vscode/** -postgres-data/ \ No newline at end of file diff --git a/backend/Dockerfile b/backend/Dockerfile index 2f67c2be..210d1127 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,32 +1,38 @@ +# Use a specific base image with GPU support FROM tensorflow/tensorflow:2.9.2-gpu -RUN apt-get update && apt-get install -y python3-opencv -RUN add-apt-repository ppa:ubuntugis/ppa && apt-get update -RUN apt-get update -RUN apt-get install -y gdal-bin -RUN apt-get install -y libgdal-dev +# Update package lists and install necessary dependencies in a single step +RUN apt-get update && \ + apt-get install -y python3-opencv gdal-bin libgdal-dev && \ + add-apt-repository ppa:ubuntugis/ppa && \ + apt-get update && \ + rm -rf /var/lib/apt/lists/* + +# Set environment variables for GDAL ENV CPLUS_INCLUDE_PATH=/usr/include/gdal ENV C_INCLUDE_PATH=/usr/include/gdal -#install numpy before gdal +# Install specific version of NumPy RUN pip install numpy==1.23.5 -# pip install dependencies. -RUN pip install --global-option=build_ext --global-option="-I/usr/include/gdal" GDAL==`gdal-config --version` - -COPY docker/ramp/docker-requirements.txt docker-requirements.txt -RUN pip install -r docker-requirements.txt +# Install GDAL with specified options +RUN pip install --global-option=build_ext --global-option="-I/usr/include/gdal" GDAL==$(gdal-config --version) -# pip install solaris -- try with tmp-free build -COPY docker/ramp/solaris /tmp/solaris -RUN pip install /tmp/solaris --use-feature=in-tree-build +# Copy and install Python requirements +COPY docker/ramp/docker-requirements.txt /tmp/docker-requirements.txt +RUN pip install -r /tmp/docker-requirements.txt -RUN pip install scikit-fmm --use-feature=in-tree-build +# Upgrade setuptools +RUN pip install --upgrade setuptools -RUN pip install setuptools --upgrade +COPY requirements.txt /tmp/requirements.txt +RUN pip install -r /tmp/requirements.txt -COPY requirements.txt requirements.txt -RUN pip install -r requirements.txt +# Install solaris and scikit-fmm packages +COPY docker/ramp/solaris /tmp/solaris +RUN pip install /tmp/solaris --use-feature=in-tree-build && \ + pip install scikit-fmm --use-feature=in-tree-build +# Set working directory and copy the application code WORKDIR /app -COPY . /app \ No newline at end of file +COPY . /app diff --git a/backend/Dockerfile_CPU b/backend/Dockerfile_CPU index 5fa52037..75377dd5 100644 --- a/backend/Dockerfile_CPU +++ b/backend/Dockerfile_CPU @@ -1,32 +1,39 @@ +# Use a specific base image with the desired TensorFlow version FROM tensorflow/tensorflow:2.9.2 -RUN apt-get update && apt-get install -y python3-opencv -RUN add-apt-repository ppa:ubuntugis/ppa && apt-get update -RUN apt-get update -RUN apt-get install -y gdal-bin -RUN apt-get install -y libgdal-dev +# Update package lists and install necessary dependencies in a single step +RUN apt-get update && \ + apt-get install -y software-properties-common python3-opencv && \ + add-apt-repository ppa:ubuntugis/ppa && \ + apt-get update && \ + apt-get install -y gdal-bin libgdal-dev && \ + rm -rf /var/lib/apt/lists/* + +# Set environment variables for GDAL ENV CPLUS_INCLUDE_PATH=/usr/include/gdal ENV C_INCLUDE_PATH=/usr/include/gdal -#install numpy before gdal +# Install specific version of NumPy RUN pip install numpy==1.23.5 -# pip install dependencies. -RUN pip install --global-option=build_ext --global-option="-I/usr/include/gdal" GDAL==`gdal-config --version` - -COPY docker/ramp/docker-requirements.txt docker-requirements.txt -RUN pip install -r docker-requirements.txt +# Install GDAL with specified options +RUN pip install --global-option=build_ext --global-option="-I/usr/include/gdal" GDAL==$(gdal-config --version) -# pip install solaris -- try with tmp-free build -COPY docker/ramp/solaris /tmp/solaris -RUN pip install /tmp/solaris --use-feature=in-tree-build +# Copy and install Python requirements +COPY docker/ramp/docker-requirements.txt /tmp/docker-requirements.txt +RUN pip install -r /tmp/docker-requirements.txt -RUN pip install scikit-fmm --use-feature=in-tree-build +# Upgrade setuptools +RUN pip install --upgrade setuptools -RUN pip install setuptools --upgrade +COPY requirements.txt /tmp/requirements.txt +RUN pip install -r /tmp/requirements.txt -COPY requirements.txt requirements.txt -RUN pip install -r requirements.txt +# Install solaris and scikit-fmm packages +COPY docker/ramp/solaris /tmp/solaris +RUN pip install /tmp/solaris --use-feature=in-tree-build && \ + pip install scikit-fmm --use-feature=in-tree-build +# Set working directory and copy the application code WORKDIR /app -COPY . /app \ No newline at end of file +COPY . /app diff --git a/backend/requirements.txt b/backend/requirements.txt index e78eeb7e..037a5e0f 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -17,7 +17,7 @@ django_celery_results==2.4.0 flower==1.2.0 validators==0.20.0 gpxpy==1.5.0 -hot-fair-utilities +hot-fair-utilities==1.2.2 geojson2osm==0.0.1 osmconflator orthogonalizer \ No newline at end of file diff --git a/docker-compose-cpu.yml b/docker-compose-cpu.yml index f54319b0..5e2272b1 100644 --- a/docker-compose-cpu.yml +++ b/docker-compose-cpu.yml @@ -23,7 +23,7 @@ services: backend-api: build: context: ./backend - dockerfile: Dockerfile + dockerfile: Dockerfile_CPU container_name: api command: python manage.py runserver 0.0.0.0:8000 @@ -40,7 +40,7 @@ services: backend-worker: build: context: ./backend - dockerfile: Dockerfile + dockerfile: Dockerfile_CPU container_name: worker command: celery -A aiproject worker --loglevel=INFO --concurrency=1 diff --git a/docs/Docker-installation.md b/docs/Docker-installation.md index a801ae38..ece44360 100644 --- a/docs/Docker-installation.md +++ b/docs/Docker-installation.md @@ -17,11 +17,11 @@ Docker Compose is created with redis , worker , postgis database , api and fron 3. Check your Graphics - fAIr works best with graphics card. It is highly recommended to use graphics card . It might not work with CPU only . Nvidia Graphics cards are tested + fAIr works best with graphics card. It is highly recommended to use graphics card . It might not work with CPU only (You can setup and test from bottom of this document). Nvidia Graphics cards are tested You need to make sure you can see your graphics card details and can be accessed through docker by installing necessary drivers - By following command you can see your graphics and graphics driver details + By following command you can see your graphics and graphics driver details & nvidia container toolkit is installed More details [here](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html) ``` nvidia-smi ``` @@ -75,6 +75,7 @@ Docker Compose is created with redis , worker , postgis database , api and fron 6. Create Env variables - Create a file ```.env``` in backend with [docker_sample_env](../backend/docker_sample_env) content ``` + cd backend cp docker_sample_env .env ``` - Fill out the details of ```OSM_CLIENT_ID``` &```OSM_CLIENT_SECRET``` in .env file and generate a unique key & paste it to ```OSM_SECRET_KEY``` (It can be random for dev setup) @@ -83,6 +84,7 @@ Docker Compose is created with redis , worker , postgis database , api and fron - Create ```.env``` in /frontend ``` + cd frontend cp .env_sample .env ``` You can leave it as it is for dev setup @@ -99,13 +101,17 @@ Docker Compose is created with redis , worker , postgis database , api and fron 8. Run Migrations - See Running containers grab their ID and launch bash to make migrations (This is needed for the first time to set database) + Run directly bash script : - docker container ps + ``` + bash run_migrations.sh + ``` + + OR - Grab Container ID & Open Bash + Grab API container & Open Bash - docker exec -it CONTAINER_ID bash + docker exec -it api bash Once Bash is promoted hit following commands diff --git a/run_migrations.sh b/run_migrations.sh new file mode 100644 index 00000000..215db9d8 --- /dev/null +++ b/run_migrations.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +docker exec -it api bash -c "python manage.py makemigrations" +docker exec -it api bash -c "python manage.py makemigrations login" +docker exec -it api bash -c "python manage.py makemigrations core" +docker exec -it api bash -c "python manage.py migrate"