Merge pull request #17949 from davelopez/replace_default_celery_backend

Replace sample Celery result_backend in config

doc/source/admin/galaxy_options.rst

@@ -5154,7 +5154,9 @@
     only if you have setup a Celery worker for Galaxy and you have
     configured the `celery_conf` option below. Specifically, you need
     to set the `result_backend` option in the `celery_conf` option to
-    a valid Celery result backend URL. For details, see
+    a valid Celery result backend URL. By default, Galaxy uses an
+    SQLite database at '<data_dir>/results.sqlite' for storing task
+    results. For details, see
     https://docs.galaxyproject.org/en/master/admin/production.html#use-celery-for-asynchronous-tasks
 :Default: ``false``
 :Type: bool
@@ -5169,15 +5171,17 @@
     To refer to a task by name, use the template `galaxy.foo` where
     `foo` is the function name of the task defined in the
     galaxy.celery.tasks module.
-    The `broker_url` option, if unset, defaults to the value of
-    `amqp_internal_connection`. The `result_backend` option must be
-    set if the `enable_celery_tasks` option is set.
+    The `broker_url` option, if unset or null, defaults to the value
+    of `amqp_internal_connection`. The `result_backend` option, if
+    unset or null, defaults to an SQLite database at
+    '<data_dir>/results.sqlite' for storing task results. Please use a
+    more robust backend (e.g. Redis) for production setups.
     The galaxy.fetch_data task can be disabled by setting its route to
     "disabled": `galaxy.fetch_data: disabled`. (Other tasks cannot be
     disabled on a per-task basis at this time.)
     For details, see Celery documentation at
     https://docs.celeryq.dev/en/stable/userguide/configuration.html.
-:Default: ``{'result_backend': 'redis://127.0.0.1:6379/0', 'task_routes': {'galaxy.fetch_data': 'galaxy.external', 'galaxy.set_job_metadata': 'galaxy.external'}}``
+:Default: ``{'broker_url': None, 'result_backend': None, 'task_routes': {'galaxy.fetch_data': 'galaxy.external', 'galaxy.set_job_metadata': 'galaxy.external'}}``
 :Type: any
 
 

lib/galaxy/config/__init__.py

@@ -1082,6 +1082,9 @@ def _process_config(self, kwargs: Dict[str, Any]) -> None:
             self.amqp_internal_connection = (
                 f"sqlalchemy+sqlite:///{self._in_data_dir('control.sqlite')}?isolation_level=IMMEDIATE"
             )
+
+        self._process_celery_config()
+
         self.pretty_datetime_format = expand_pretty_datetime_format(self.pretty_datetime_format)
         try:
             with open(self.user_preferences_extra_conf_path) as stream:
@@ -1203,6 +1206,12 @@ def _load_theme(path: str, theme_dict: dict):
         else:
             _load_theme(self.themes_config_file, self.themes)
 
+    def _process_celery_config(self):
+        if self.celery_conf and self.celery_conf.get("result_backend") is None:
+            # If the result_backend is not set, use a SQLite database in the data directory
+            result_backend = f"db+sqlite:///{self._in_data_dir('results.sqlite')}?isolation_level=IMMEDIATE"
+            self.celery_conf["result_backend"] = result_backend
+
     def _check_database_connection_strings(self):
         """
         Verify connection URI strings in galaxy's configuration are parseable with urllib.

lib/galaxy/config/sample/galaxy.yml.sample

@@ -2761,24 +2761,29 @@ galaxy:
   # only if you have setup a Celery worker for Galaxy and you have
   # configured the `celery_conf` option below. Specifically, you need to
   # set the `result_backend` option in the `celery_conf` option to a
-  # valid Celery result backend URL. For details, see
+  # valid Celery result backend URL. By default, Galaxy uses an SQLite
+  # database at '<data_dir>/results.sqlite' for storing task results.
+  # For details, see
   # https://docs.galaxyproject.org/en/master/admin/production.html#use-celery-for-asynchronous-tasks
   #enable_celery_tasks: false
 
   # Configuration options passed to Celery.
   # To refer to a task by name, use the template `galaxy.foo` where
   # `foo` is the function name of the task defined in the
   # galaxy.celery.tasks module.
-  # The `broker_url` option, if unset, defaults to the value of
-  # `amqp_internal_connection`. The `result_backend` option must be set
-  # if the `enable_celery_tasks` option is set.
+  # The `broker_url` option, if unset or null, defaults to the value of
+  # `amqp_internal_connection`. The `result_backend` option, if unset or
+  # null, defaults to an SQLite database at '<data_dir>/results.sqlite'
+  # for storing task results. Please use a more robust backend (e.g.
+  # Redis) for production setups.
   # The galaxy.fetch_data task can be disabled by setting its route to
   # "disabled": `galaxy.fetch_data: disabled`. (Other tasks cannot be
   # disabled on a per-task basis at this time.)
   # For details, see Celery documentation at
   # https://docs.celeryq.dev/en/stable/userguide/configuration.html.
   #celery_conf:
-  #  result_backend: redis://127.0.0.1:6379/0
+  #  broker_url: null
+  #  result_backend: null
   #  task_routes:
   #    galaxy.fetch_data: galaxy.external
   #    galaxy.set_job_metadata: galaxy.external

lib/galaxy/config/schemas/config_schema.yml

@@ -3764,14 +3764,15 @@ mapping:
           Activate this only if you have setup a Celery worker for Galaxy and you have
           configured the `celery_conf` option below. Specifically, you need to set the
           `result_backend` option in the `celery_conf` option to a valid Celery result
-          backend URL.
+          backend URL. By default, Galaxy uses an SQLite database at '<data_dir>/results.sqlite' for storing task results.
           For details, see https://docs.galaxyproject.org/en/master/admin/production.html#use-celery-for-asynchronous-tasks
           
       celery_conf:
         type: any
         required: false
         default:
-          result_backend: redis://127.0.0.1:6379/0
+          broker_url: null
+          result_backend: null
           task_routes:
             'galaxy.fetch_data': 'galaxy.external'
             'galaxy.set_job_metadata': 'galaxy.external'
@@ -3781,8 +3782,9 @@ mapping:
           To refer to a task by name, use the template `galaxy.foo` where `foo` is the function name
           of the task defined in the galaxy.celery.tasks module.
 
-          The `broker_url` option, if unset, defaults to the value of `amqp_internal_connection`.
-          The `result_backend` option must be set if the `enable_celery_tasks` option is set.
+          The `broker_url` option, if unset or null, defaults to the value of `amqp_internal_connection`.
+          The `result_backend` option, if unset or null, defaults to an SQLite database at '<data_dir>/results.sqlite'
+          for storing task results. Please use a more robust backend (e.g. Redis) for production setups.
 
           The galaxy.fetch_data task can be disabled by setting its route to "disabled": `galaxy.fetch_data: disabled`.
           (Other tasks cannot be disabled on a per-task basis at this time.)