Merge pull request hotosm#167 from hotosm/enhance/docker_cpu

Enhance : Added support for docker CPU

.gitignore

@@ -6,6 +6,11 @@ frontend/node_modules
 frontend/.pnp
 frontend/.pnp.js
 
+#standard
+.env
+**/.vscode/**
+postgres-data/
+
 # testing
 frontend/coverage
 postgres-data/*
@@ -28,7 +33,7 @@ frontend/package-lock*
 
 frontend/.env
 
-# for 
+# for backend 
 backend/static
 backend/api_static
 backend/env
@@ -40,8 +45,7 @@ backend/media
 backend/data/*
 backend/log/*
 backend/training/*
+trainings/*
 backend/.env
 backend/config.txt
 backend/postgres-data
-**/.vscode/**
-postgres-data/

backend/Dockerfile

@@ -1,32 +1,38 @@
+# Use a specific base image with GPU support
 FROM tensorflow/tensorflow:2.9.2-gpu
 
-RUN apt-get update && apt-get install -y python3-opencv
-RUN add-apt-repository ppa:ubuntugis/ppa && apt-get update
-RUN apt-get update
-RUN apt-get install -y gdal-bin
-RUN apt-get install -y libgdal-dev
+# Update package lists and install necessary dependencies in a single step
+RUN apt-get update && \
+    apt-get install -y python3-opencv gdal-bin libgdal-dev && \
+    add-apt-repository ppa:ubuntugis/ppa && \
+    apt-get update && \
+    rm -rf /var/lib/apt/lists/*
+
+# Set environment variables for GDAL
 ENV CPLUS_INCLUDE_PATH=/usr/include/gdal
 ENV C_INCLUDE_PATH=/usr/include/gdal
 
-#install numpy before gdal 
+# Install specific version of NumPy
 RUN pip install numpy==1.23.5
 
-# pip install dependencies.
-RUN pip install --global-option=build_ext --global-option="-I/usr/include/gdal" GDAL==`gdal-config --version`
-
-COPY docker/ramp/docker-requirements.txt docker-requirements.txt
-RUN pip install -r docker-requirements.txt
+# Install GDAL with specified options
+RUN pip install --global-option=build_ext --global-option="-I/usr/include/gdal" GDAL==$(gdal-config --version)
 
-# pip install solaris -- try with tmp-free build
-COPY docker/ramp/solaris /tmp/solaris
-RUN pip install /tmp/solaris --use-feature=in-tree-build
+# Copy and install Python requirements
+COPY docker/ramp/docker-requirements.txt /tmp/docker-requirements.txt
+RUN pip install -r /tmp/docker-requirements.txt
 
-RUN pip install scikit-fmm --use-feature=in-tree-build
+# Upgrade setuptools
+RUN pip install --upgrade setuptools
 
-RUN pip install setuptools --upgrade 
+COPY requirements.txt /tmp/requirements.txt
+RUN pip install -r /tmp/requirements.txt
 
-COPY requirements.txt requirements.txt
-RUN pip install -r requirements.txt
+# Install solaris and scikit-fmm packages
+COPY docker/ramp/solaris /tmp/solaris
+RUN pip install /tmp/solaris --use-feature=in-tree-build && \
+    pip install scikit-fmm --use-feature=in-tree-build
 
+# Set working directory and copy the application code
 WORKDIR /app
-COPY . /app
+COPY . /app

backend/Dockerfile_CPU

@@ -0,0 +1,39 @@
+# Use a specific base image with the desired TensorFlow version
+FROM tensorflow/tensorflow:2.9.2
+
+# Update package lists and install necessary dependencies in a single step
+RUN apt-get update && \
+    apt-get install -y software-properties-common python3-opencv && \
+    add-apt-repository ppa:ubuntugis/ppa && \
+    apt-get update && \
+    apt-get install -y gdal-bin libgdal-dev && \
+    rm -rf /var/lib/apt/lists/*
+
+# Set environment variables for GDAL
+ENV CPLUS_INCLUDE_PATH=/usr/include/gdal
+ENV C_INCLUDE_PATH=/usr/include/gdal
+
+# Install specific version of NumPy
+RUN pip install numpy==1.23.5
+
+# Install GDAL with specified options
+RUN pip install --global-option=build_ext --global-option="-I/usr/include/gdal" GDAL==$(gdal-config --version)
+
+# Copy and install Python requirements
+COPY docker/ramp/docker-requirements.txt /tmp/docker-requirements.txt
+RUN pip install -r /tmp/docker-requirements.txt
+
+# Upgrade setuptools
+RUN pip install --upgrade setuptools
+
+COPY requirements.txt /tmp/requirements.txt
+RUN pip install -r /tmp/requirements.txt
+
+# Install solaris and scikit-fmm packages
+COPY docker/ramp/solaris /tmp/solaris
+RUN pip install /tmp/solaris --use-feature=in-tree-build && \
+    pip install scikit-fmm --use-feature=in-tree-build
+
+# Set working directory and copy the application code
+WORKDIR /app
+COPY . /app

backend/requirements.txt

@@ -17,7 +17,7 @@ django_celery_results==2.4.0
 flower==1.2.0
 validators==0.20.0
 gpxpy==1.5.0
-hot-fair-utilities
+hot-fair-utilities==1.2.2
 geojson2osm==0.0.1
 osmconflator
 orthogonalizer

docker-compose-cpu.yml

@@ -0,0 +1,76 @@
+version: "3.8"
+
+services:
+  postgres:
+    restart: always
+    image: postgis/postgis
+    container_name: pgsql
+    environment:
+      - POSTGRES_DB=ai
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=admin
+    volumes:
+      - ./postgres-data:/var/lib/postgresql/data
+    ports:
+      - "5434:5432"
+
+  redis:
+    image: redis
+    container_name: redis
+    ports:
+      - "6379:6379"
+
+  backend-api:
+    build:
+      context: ./backend
+      dockerfile: Dockerfile_CPU
+    container_name: api
+    command: python manage.py runserver 0.0.0.0:8000
+
+    ports:
+      - 8000:8000
+    volumes:
+      - ./backend:/app
+      - ${RAMP_HOME}:/RAMP_HOME
+      - ${TRAINING_WORKSPACE}:/TRAINING_WORKSPACE
+    depends_on:
+      - redis
+      - postgres
+
+  backend-worker:
+    build:
+      context: ./backend
+      dockerfile: Dockerfile_CPU
+    container_name: worker
+    command: celery -A aiproject worker --loglevel=INFO --concurrency=1
+
+    volumes:
+      - ./backend:/app
+      - ${RAMP_HOME}:/RAMP_HOME
+      - ${TRAINING_WORKSPACE}:/TRAINING_WORKSPACE
+    depends_on:
+      - backend-api
+      - redis
+      - postgres
+
+  worker-dashboard:
+    image: mher/flower
+    container_name: flower
+    command: celery --broker=redis://redis:6379// flower --address=0.0.0.0 --port=5000
+    ports:
+      - 5500:5000
+    depends_on:
+      - backend-api
+      - redis
+      - backend-worker
+
+  frontend:
+    build:
+      context: ./frontend
+      dockerfile: Dockerfile.frontend
+    container_name: frontend
+    command: npm start  -- --host 0.0.0.0 --port 3000
+    ports:
+      - 3000:3000
+    depends_on:
+      - backend-api

docker-compose.yml

@@ -26,12 +26,7 @@ services:
       dockerfile: Dockerfile
     container_name: api
     command: python manage.py runserver 0.0.0.0:8000
-    deploy:
-      resources:
-        reservations:
-          devices:
-            - driver: nvidia
-              capabilities: [gpu]
+
     ports:
       - 8000:8000
     volumes:

docs/Docker-installation.md

@@ -17,11 +17,11 @@ Docker Compose is created with redis , worker , postgis database ,  api and fron
 
 3. Check your Graphics 
 
-    fAIr works best with graphics card. It is highly recommended to use graphics card . It might not work with CPU only . Nvidia Graphics cards are tested 
+    fAIr works best with graphics card. It is highly recommended to use graphics card . It might not work with CPU only (You can setup and test from bottom of this document). Nvidia Graphics cards are tested 
 
     You need to make sure you can see your graphics card details and can be accessed through docker by installing necessary drivers
 
-    By following command you can see your graphics and graphics driver details 
+    By following command you can see your graphics and graphics driver details & nvidia container toolkit is installed More details [here](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html)
     ```
     nvidia-smi
     ```
@@ -75,6 +75,7 @@ Docker Compose is created with redis , worker , postgis database ,  api and fron
 6. Create Env variables 
     - Create a file ```.env``` in backend with [docker_sample_env](../backend/docker_sample_env) content 
         ```
+        cd backend
         cp docker_sample_env .env
         ```
     - Fill out the details of ```OSM_CLIENT_ID``` &```OSM_CLIENT_SECRET``` in .env file and generate a unique key & paste it to ```OSM_SECRET_KEY``` (It can be random for dev setup)
@@ -83,6 +84,7 @@ Docker Compose is created with redis , worker , postgis database ,  api and fron
 
     - Create ```.env``` in /frontend
         ```
+        cd frontend
         cp .env_sample .env
         ```
         You can leave it as it is for dev setup
@@ -99,13 +101,17 @@ Docker Compose is created with redis , worker , postgis database ,  api and fron
 
 8. Run Migrations 
 
-    See Running containers grab their ID and launch bash to make migrations (This is needed for the first time to set database)
+    Run directly bash script : 
 
-        docker container ps
+    ```
+    bash run_migrations.sh
+    ```
+
+    OR 
 
-    Grab Container ID & Open Bash
+    Grab API container & Open Bash
 
-        docker exec -it CONTAINER_ID bash
+        docker exec -it api bash
 
 
     Once Bash is promoted hit following commands 

run_migrations.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+docker exec -it api bash -c "python manage.py makemigrations"
+docker exec -it api bash -c "python manage.py makemigrations login"
+docker exec -it api bash -c "python manage.py makemigrations core"
+docker exec -it api bash -c "python manage.py migrate"