Fixed race condition that caused service to be unhealthy and merge mi…

…grations (#2048)
ohcnetwork · Apr 3, 2024 · ef53bec · ef53bec
1 parent 7aa73b9
commit ef53bec
Show file tree

Hide file tree

Showing 13 changed files with 52 additions and 19 deletions.
diff --git a/care/facility/migrations/0425_merge_20240403_2055.py b/care/facility/migrations/0425_merge_20240403_2055.py
@@ -0,0 +1,15 @@
+# Generated by Django 4.2.10 on 2024-04-03 15:25
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("facility", "0422_alter_facilityinventorylog_quantity_and_more"),
+        (
+            "facility",
+            "0424_remove_patientregistration_age_and_add_patientregistration_death_datetime",
+        ),
+    ]
+
+    operations = []
diff --git a/docker-compose.local.yaml b/docker-compose.local.yaml
@@ -14,15 +14,21 @@ services:
     ports:
       - "9000:9000"
       - "9876:9876" #debugpy
+    restart: unless-stopped
     depends_on:
-      - db
-      - redis
+      db:
+        condition: service_started
+      redis:
+        condition: service_started
+      celery:
+        condition: service_healthy
 
   celery:
     image: care_local
     env_file:
       - ./docker/.local.env
     entrypoint: [ "bash", "scripts/celery-dev.sh" ]
+    restart: unless-stopped
     depends_on:
       - db
       - redis

diff --git a/docker-compose.pre-built.yaml b/docker-compose.pre-built.yaml
@@ -6,9 +6,14 @@ services:
     env_file:
       - ./docker/.prebuilt.env
     entrypoint: [ "bash", "start-ecs.sh" ]
+    restart: unless-stopped
     depends_on:
-      - db
-      - redis
+      db:
+        condition: service_started
+      redis:
+        condition: service_started
+      celery-beat:
+        condition: service_healthy
     ports:
       - "9000:9000"
 
@@ -17,15 +22,21 @@ services:
     env_file:
       - ./docker/.prebuilt.env
     entrypoint: [ "bash", "celery_worker-ecs.sh" ]
+    restart: unless-stopped
     depends_on:
-      - db
-      - redis
+      db:
+        condition: service_started
+      redis:
+        condition: service_started
+      celery-beat:
+        condition: service_healthy
 
   celery-beat:
     image: "ghcr.io/coronasafe/care:latest"
     env_file:
       - ./docker/.prebuilt.env
     entrypoint: [ "bash", "celery_beat-ecs.sh" ]
+    restart: unless-stopped
     depends_on:
       - db
       - redis
diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -7,20 +7,21 @@ networks:
 services:
   db:
     image: postgres:alpine
-    restart: always
+    restart: unless-stopped
     env_file:
       - ./docker/.prebuilt.env
     volumes:
       - postgres-data:/var/lib/postgresql/data
 
   redis:
     image: redis/redis-stack-server:6.2.6-v10
-    restart: always
+    restart: unless-stopped
     volumes:
       - redis-data:/data
 
   localstack:
     image: localstack/localstack:latest
+    restart: unless-stopped
     environment:
       - AWS_DEFAULT_REGION=ap-south-1
       - EDGE_PORT=4566
@@ -34,8 +35,8 @@ services:
       - "4566:4566"
 
   fidelius:
-    image: khavinshankar/fidelius:v1.0
-    restart: always
+    image: khavinshankar/fidelius:latest
+    restart: unless-stopped
 
 volumes:
   postgres-data:

diff --git a/docker/dev.Dockerfile b/docker/dev.Dockerfile
@@ -25,7 +25,7 @@ HEALTHCHECK \
   --interval=10s \
   --timeout=5s \
   --start-period=10s \
-  --retries=12 \
+  --retries=24 \
   CMD ["/app/scripts/healthcheck.sh"]
 
 WORKDIR /app
diff --git a/scripts/celery-dev.sh b/scripts/celery-dev.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-printf "celery" >> /tmp/container-role
+printf "celery" > /tmp/container-role
 
 if [ -z "${DATABASE_URL}" ]; then
     export DATABASE_URL="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}"

diff --git a/scripts/celery_beat-ecs.sh b/scripts/celery_beat-ecs.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-printf "celery-beat" >> /tmp/container-role
+printf "celery-beat" > /tmp/container-role
 
 if [ -z "${DATABASE_URL}" ]; then
     export DATABASE_URL="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}"

diff --git a/scripts/celery_beat.sh b/scripts/celery_beat.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-printf "celery-beat" >> /tmp/container-role
+printf "celery-beat" > /tmp/container-role
 
 if [ -z "${DATABASE_URL}" ]; then
     export DATABASE_URL="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}"

diff --git a/scripts/celery_worker-ecs.sh b/scripts/celery_worker-ecs.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-printf "celery-worker" >> /tmp/container-role
+printf "celery-worker" > /tmp/container-role
 
 if [ -z "${DATABASE_URL}" ]; then
     export DATABASE_URL="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}"

diff --git a/scripts/celery_worker.sh b/scripts/celery_worker.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-printf "celery-worker" >> /tmp/container-role
+printf "celery-worker" > /tmp/container-role
 
 if [ -z "${DATABASE_URL}" ]; then
     export DATABASE_URL="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}"

diff --git a/scripts/start-dev.sh b/scripts/start-dev.sh
@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-printf "api" >> /tmp/container-role
+printf "api" > /tmp/container-role
 
 cd /app
 

diff --git a/scripts/start-ecs.sh b/scripts/start-ecs.sh
@@ -4,7 +4,7 @@ set -o errexit
 set -o pipefail
 set -o nounset
 
-printf "api" >> /tmp/container-role
+printf "api" > /tmp/container-role
 
 if [ -z "${DATABASE_URL}" ]; then
     export DATABASE_URL="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}"

diff --git a/scripts/start.sh b/scripts/start.sh
@@ -4,7 +4,7 @@ set -o errexit
 set -o pipefail
 set -o nounset
 
-printf "api" >> /tmp/container-role
+printf "api" > /tmp/container-role
 
 if [ -z "${DATABASE_URL}" ]; then
     export DATABASE_URL="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}"