Merge FEGA related components from pipeline into SDA

.github/integration/rabbitmq-federation.yml

@@ -78,7 +78,7 @@ services:
       - BROKER_CLIENTKEY=/certs/client.key
       - BROKER_CACERT=/certs/ca.crt
       - LOG_LEVEL=debug
-    image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-pipeline
+    image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}
     restart: always
     volumes:
       - certs:/certs

.github/integration/scripts/charts/deploy_charts.sh

@@ -14,7 +14,7 @@ fi
 if [ "$1" == "sda-db" ]; then
     ROOTPASS=$(yq e '.global.db.password' .github/integration/scripts/charts/values.yaml)
     helm install postgres charts/sda-db \
-        --set image.tag="PR$2-postgres" \
+        --set image.tag="PR$2" \
         --set image.pullPolicy=IfNotPresent \
         --set global.postgresAdminPassword="$ROOTPASS" \
         --set global.tls.clusterIssuer=cert-issuer \
@@ -27,7 +27,7 @@ fi
 if [ "$1" == "sda-mq" ]; then
     ADMINPASS=$(yq e '.global.broker.password' .github/integration/scripts/charts/values.yaml)
     helm install broker charts/sda-mq \
-        --set image.tag="PR$2-rabbitmq" \
+        --set image.tag="PR$2" \
         --set image.pullPolicy=IfNotPresent \
         --set global.adminPassword="$ADMINPASS" \
         --set global.adminUser=admin \

.github/integration/scripts/charts/values.yaml

@@ -67,9 +67,6 @@ global:
 auth:
   replicaCount: 1
   resources: null
-backup:
-  deploy: true
-  resources: null
 doa:
   deploy: false
 download:

.github/integration/scripts/make_sda_credentials.sh

@@ -8,15 +8,17 @@ for n in download finalize inbox ingest mapper sync verify; do
     echo "creating credentials for: $n"
 
     if [ "$n" = inbox ]; then
-        psql -U postgres -h postgres -d sda -c "DROP ROLE IF EXISTS inbox;"
-        psql -U postgres -h postgres -d sda -c "CREATE ROLE inbox;"
-        psql -U postgres -h postgres -d sda -c "GRANT base, ingest TO inbox;"
+        psql -U postgres -h postgres -d sda -c "DROP ROLE IF EXISTS $n;"
+        psql -U postgres -h postgres -d sda -c "CREATE ROLE $n;"
+        psql -U postgres -h postgres -d sda -c "GRANT ingest TO $n;"
     fi
 
     if [ "$n" = ingest ]; then
         psql -U postgres -h postgres -d sda -c "GRANT UPDATE ON local_ega.main TO ingest;"
     fi
 
+    psql -U postgres -h postgres -d sda -c "GRANT base TO $n;"
+
     psql -U postgres -h postgres -d sda -c "ALTER ROLE $n LOGIN PASSWORD '$n';"
 
     ## password and permissions for MQ

.github/integration/sda-integration.yml

@@ -24,9 +24,9 @@ services:
       - POSTGRES_PASSWORD=rootpasswd
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U postgres"]
-      interval: 5s
-      timeout: 20s
-      retries: 3
+      interval: 10s
+      timeout: 2s
+      retries: 6
     image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-postgres
     ports:
       - "15432:5432"
@@ -46,9 +46,9 @@ services:
           "-c",
           "rabbitmq-diagnostics -q check_running && rabbitmq-diagnostics -q check_local_alarms",
         ]
-      interval: 5s
-      timeout: 20s
-      retries: 3
+      interval: 10s
+      timeout: 5s
+      retries: 6
     image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-rabbitmq
     ports:
       - "15672:15672"
@@ -66,9 +66,9 @@ services:
       - MINIO_SERVER_URL=http://127.0.0.1:9000
     healthcheck:
       test: ["CMD", "curl", "-fkq", "http://localhost:9000/minio/health/live"]
-      interval: 5s
-      timeout: 20s
-      retries: 3
+      interval: 10s
+      timeout: 2s
+      retries: 6
     ports:
       - "19000:9000"
       - "19001:9001"
@@ -98,6 +98,7 @@ services:
     environment:
       - BROKER_PASSWORD=inbox
       - BROKER_USER=inbox
+      - BROKER_ROUTINGKEY=inbox
       - DB_PASSWORD=inbox
       - DB_USER=inbox
     restart: always
@@ -108,6 +109,105 @@ services:
       - "18000:8000"
       - "18001:8001"
 
+  ingest:
+    image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}
+    command: [ sda-ingest ]
+    container_name: ingest
+    depends_on:
+      credentials:
+        condition: service_completed_successfully
+      minio:
+        condition: service_healthy
+      postgres:
+        condition: service_healthy
+      rabbitmq:
+        condition: service_healthy
+    environment:
+      - BROKER_PASSWORD=ingest
+      - BROKER_USER=ingest
+      - BROKER_QUEUE=ingest
+      - BROKER_ROUTINGKEY=archived
+      - DB_PASSWORD=ingest
+      - DB_USER=ingest
+    restart: always
+    volumes:
+      - ./sda/config.yaml:/config.yaml
+      - shared:/shared
+
+  verify:
+    image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}
+    command: [ sda-verify ]
+    container_name: verify
+    depends_on:
+      credentials:
+        condition: service_completed_successfully
+      minio:
+        condition: service_healthy
+      postgres:
+        condition: service_healthy
+      rabbitmq:
+        condition: service_healthy
+    environment:
+      - BROKER_PASSWORD=verify
+      - BROKER_USER=verify
+      - BROKER_QUEUE=archived
+      - BROKER_ROUTINGKEY=verified
+      - DB_PASSWORD=verify
+      - DB_USER=verify
+    restart: always
+    volumes:
+      - ./sda/config.yaml:/config.yaml
+      - shared:/shared
+
+  finalize:
+    image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}
+    command: [ sda-finalize ]
+    container_name: finalize
+    depends_on:
+      credentials:
+        condition: service_completed_successfully
+      minio:
+        condition: service_healthy
+      postgres:
+        condition: service_healthy
+      rabbitmq:
+        condition: service_healthy
+    environment:
+      - BROKER_PASSWORD=finalize
+      - BROKER_USER=finalize
+      - BROKER_QUEUE=accession
+      - BROKER_ROUTINGKEY=completed
+      - DB_PASSWORD=finalize
+      - DB_USER=finalize
+    restart: always
+    volumes:
+      - ./sda/config.yaml:/config.yaml
+      - shared:/shared
+
+  mapper:
+    image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}
+    command: [ sda-mapper ]
+    container_name: mapper
+    depends_on:
+      credentials:
+        condition: service_completed_successfully
+      minio:
+        condition: service_healthy
+      postgres:
+        condition: service_healthy
+      rabbitmq:
+        condition: service_healthy
+    environment:
+      - BROKER_PASSWORD=mapper
+      - BROKER_USER=mapper
+      - BROKER_QUEUE=mappings
+      - DB_PASSWORD=mapper
+      - DB_USER=mapper
+    restart: always
+    volumes:
+      - ./sda/config.yaml:/config.yaml
+      - shared:/shared
+
   oidc:
     container_name: oidc
     command:
@@ -122,9 +222,9 @@ services:
         condition: service_completed_successfully
     healthcheck:
       test: ["CMD", "python3", "-c", 'import requests; print(requests.get(url = "http://localhost:8080/jwk").text)']
-      interval: 5s
-      timeout: 20s
-      retries: 3
+      interval: 10s
+      timeout: 2s
+      retries: 6
     image: python:3.10-slim
     ports:
       - "8080:8080"
@@ -142,8 +242,16 @@ services:
     depends_on:
       credentials:
         condition: service_completed_successfully
+      finalize:
+        condition: service_started
+      ingest:
+        condition: service_started
+      mapper:
+        condition: service_started
       s3inbox:
         condition: service_started
+      verify:
+        condition: service_started
     environment:
       - PGPASSWORD=rootpasswd
     image: python:3.10-slim-bullseye

.github/integration/sda/config.yaml

@@ -1,7 +1,28 @@
 log:
   format: "json"
-aws:
-  url: "http://s3:9000"
+  level: "debug"
+archive:
+  type: s3
+  url: "http://s3"
+  port: 9000
+  readypath: "/minio/health/ready"
+  accessKey: "access"
+  secretKey: "secretKey"
+  bucket: "archive"
+  region: "us-east-1"
+backup:
+  type: s3
+  url: "http://s3"
+  port: 9000
+  readypath: "/minio/health/ready"
+  accessKey: "access"
+  secretKey: "secretKey"
+  bucket: "backup"
+  region: "us-east-1"
+inbox:
+  type: s3
+  url: "http://s3"
+  port: 9000
   readypath: "/minio/health/ready"
   accessKey: "access"
   secretKey: "secretKey"
@@ -15,7 +36,7 @@ broker:
   password: ""
   vhost: "/sda"
   exchange: "sda"
-  routingKey: "inbox"
+  routingKey: ""
   ssl: "false"
 
 db:
@@ -26,6 +47,9 @@ db:
   database: "sda"
   sslmode: "disable"
 
+c4gh:
+  filePath: /shared/c4gh.sec.pem
+  passphrase: "c4ghpass"
 
 server:
   cert: ""

.github/integration/tests/run_scripts.sh

@@ -4,7 +4,7 @@ set -e
 apt-get -o DPkg::Lock::Timeout=60 update > /dev/null
 apt-get -o DPkg::Lock::Timeout=60 install -y postgresql-client > /dev/null
 
-find "$1"/*.sh 2>/dev/null |  sort -t/ -k3 -n | while read -r runscript; do
+for runscript in "$1"/*.sh; do
     echo "Executing test script $runscript"
     bash -x "$runscript"
 done

.github/integration/tests/sda/10_upload_test.sh

@@ -14,14 +14,15 @@ for t in curl jq postgresql-client; do
     fi
 done
 
+pip -q install --upgrade pip
 pip -q install s3cmd
 
 cd shared || true
 
 for file in NA12878.bam NA12878_20k_b37.bam; do
     curl -s -L -o /shared/$file "https://github.com/ga4gh/htsget-refserver/raw/main/data/gcp/gatk-test-data/wgs_bam/$file"
     if [ ! -f "$file.c4gh" ]; then
-        /shared/crypt4gh encrypt -p c4gh.pub.pem -f "$file"
+        yes | /shared/crypt4gh encrypt -p c4gh.pub.pem -f "$file"
     fi
     s3cmd -c s3cfg put "$file.c4gh" s3://test_dummy.org/
 done

.github/integration/tests/sda/20_ingest-verify_test.sh

@@ -0,0 +1,70 @@
+#!/bin/sh
+set -e
+
+cd shared || true
+
+for file in NA12878.bam NA12878_20k_b37.bam; do
+    ENC_SHA=$(sha256sum "$file.c4gh" | cut -d' ' -f 1)
+    ENC_MD5=$(md5sum "$file.c4gh" | cut -d' ' -f 1)
+
+    ## get correlation id from upload message
+    CORRID=$(
+        curl -s -X POST \
+            -H "content-type:application/json" \
+            -u guest:guest http://rabbitmq:15672/api/queues/sda/inbox/get \
+            -d '{"count":1,"encoding":"auto","ackmode":"ack_requeue_false"}' | jq -r .[0].properties.correlation_id
+    )
+
+    ## publish message to trigger ingestion
+    properties=$(
+        jq -c -n \
+            --argjson delivery_mode 2 \
+            --arg correlation_id "$CORRID" \
+            --arg content_encoding UTF-8 \
+            --arg content_type application/json \
+            '$ARGS.named'
+    )
+
+    encrypted_checksums=$(
+        jq -c -n \
+            --arg sha256 "$ENC_SHA" \
+            --arg md5 "$ENC_MD5" \
+            '$ARGS.named|to_entries|map(with_entries(select(.key=="key").key="type"))'
+    )
+
+    ingest_payload=$(
+        jq -r -c -n \
+            --arg type ingest \
+            --arg user [email protected] \
+            --arg filepath test_dummy.org/"$file.c4gh" \
+            --argjson encrypted_checksums "$encrypted_checksums" \
+            '$ARGS.named|@base64'
+    )
+
+    ingest_body=$(
+        jq -c -n \
+            --arg vhost sda \
+            --arg name sda \
+            --argjson properties "$properties" \
+            --arg routing_key "ingest" \
+            --arg payload_encoding base64 \
+            --arg payload "$ingest_payload" \
+            '$ARGS.named'
+    )
+
+    curl -s -u guest:guest "http://rabbitmq:15672/api/exchanges/sda/sda/publish" \
+        -H 'Content-Type: application/json;charset=UTF-8' \
+        -d "$ingest_body"
+done
+
+echo "waiting for verify to complete"
+RETRY_TIMES=0
+until [ "$(curl -su guest:guest http://rabbitmq:15672/api/queues/sda/verified/ | jq -r '.messages_ready')" -eq 2 ]; do
+    echo "waiting for verify to complete"
+    RETRY_TIMES=$((RETRY_TIMES + 1))
+    if [ "$RETRY_TIMES" -eq 30 ]; then
+        echo "::error::Time out while waiting for verify to complete"
+        exit 1
+    fi
+    sleep 2
+done