feat(executor): upgrade to Snakemake v7.32.4 (reanahub#81)

Amend the overridden executor to reflect the changes in the new version
of Snakemake, in particular with regard to the change of the
`_wait_for_jobs` method into a coroutine.

Closes reanahub#31
Closes reanahub/reana-client#655

Dockerfile

@@ -1,5 +1,5 @@
 # This file is part of REANA.
-# Copyright (C) 2021, 2022, 2023 CERN.
+# Copyright (C) 2021, 2022, 2023, 2024 CERN.
 #
 # REANA is free software; you can redistribute it and/or modify it
 # under the terms of the MIT License; see LICENSE file for more details.
@@ -82,8 +82,6 @@ RUN chmod +x /usr/local/bin/magick
 
 # Are we debugging?
 ARG DEBUG=0
-# hadolint ignore=DL3013
-RUN if [ "${DEBUG}" -gt 0 ]; then pip install --no-cache-dir -e ".[debug,xrootd]"; else pip install --no-cache-dir ".[xrootd]"; fi;
 
 # Are we building with locally-checked-out shared modules?
 # hadolint ignore=DL3013
@@ -95,13 +93,21 @@ RUN if test -e modules/reana-commons; then \
       fi \
     fi
 
+# hadolint ignore=DL3013
+RUN if [ "${DEBUG}" -gt 0 ]; then pip install --no-cache-dir -e ".[debug,xrootd]"; else pip install --no-cache-dir ".[xrootd]"; fi;
+
 # Check for any broken Python dependencies
 RUN pip check
 
 # Set useful environment variables
 ENV TERM=xterm \
     PYTHONPATH=/workdir
 
+# Create and switch to REANA user to be able to create snakemake-specific
+# directories in the home folder.
+RUN useradd reana --uid 1000 --create-home
+USER reana
+
 # Set image labels
 LABEL org.opencontainers.image.authors="[email protected]"
 LABEL org.opencontainers.image.created="2023-12-12"

reana_workflow_engine_snakemake/executor.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 # This file is part of REANA.
-# Copyright (C) 2021, 2022, 2023 CERN.
+# Copyright (C) 2021, 2022, 2023, 2024 CERN.
 #
 # REANA is free software; you can redistribute it and/or modify it
 # under the terms of the MIT License; see LICENSE file for more details.
@@ -10,14 +10,15 @@
 
 import os
 import logging
-import time
+import asyncio
 from collections import namedtuple
 from typing import Callable
 
 from bravado.exception import HTTPNotFound
 from reana_commons.config import REANA_DEFAULT_SNAKEMAKE_ENV_IMAGE
 from reana_commons.utils import build_progress_message
 from snakemake import snakemake
+from snakemake.common import async_lock
 from snakemake.executors import ClusterExecutor, GenericClusterExecutor
 from snakemake.jobs import Job
 from snakemake import scheduler  # for monkeypatch
@@ -187,13 +188,13 @@ def _get_job_status_from_controller(self, job_id: str) -> str:
             )
             return JobStatus.failed.name
 
-    def _wait_for_jobs(self):
+    async def _wait_for_jobs(self):
         """Override _wait_for_jobs method to poll job-controller for job statuses.
 
         Original GenericClusterExecutor._wait_for_jobs method checks success/failure via .jobfinished or .jobfailed files.
         """
         while True:
-            with self.lock:
+            async with async_lock(self.lock):
                 if not self.wait:
                     return
                 active_jobs = self.active_jobs
@@ -215,15 +216,15 @@ def _wait_for_jobs(self):
                 else:
                     still_running.append(active_job)
 
-            with self.lock:
+            async with async_lock(self.lock):
                 # Even though we have set active_jobs to a new empty list at the
                 # beginning of _wait_for_jobs, here that list might not be empty anymore
                 # as more jobs might have been added while we were fetching the job
                 # statuses from r-j-controller. For this reason we have to extend the
                 # list, instead of simply setting active_jobs to still_running.
                 self.active_jobs.extend(still_running)
 
-            time.sleep(POLL_JOBS_STATUS_SLEEP_IN_SECONDS)
+            await asyncio.sleep(POLL_JOBS_STATUS_SLEEP_IN_SECONDS)
 
 
 def submit_job(rjc_api_client, publisher, job_request_body):

setup.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 # This file is part of REANA.
-# Copyright (C) 2021, 2022, 2023 CERN.
+# Copyright (C) 2021, 2022, 2023, 2024 CERN.
 #
 # REANA is free software; you can redistribute it and/or modify it
 # under the terms of the MIT License; see LICENSE file for more details.
@@ -51,7 +51,7 @@
 ]
 
 install_requires = [
-    "reana-commons[snakemake_reports]>=0.9.4,<0.10.0",
+    "reana-commons[snakemake_reports]>=0.9.6,<0.10.0",
 ]
 
 packages = find_packages()