Merge pull request #18481 from kysrpex/interactivetoolssqlalchemy

Support high-availability setups for the interactive tools proxy

doc/source/admin/galaxy_options.rst

@@ -2112,13 +2112,34 @@
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
 :Description:
-    Map for interactivetool proxy.
+    Map for the interactivetool proxy. Mappings are stored in a SQLite
+    database file located on this path. As an alternative, you may
+    also store them in any other RDBMS supported by SQLAlchemy using
+    the option ``interactivetoolsproxy_map``, which overrides this
+    one.
     The value of this option will be resolved with respect to
     <data_dir>.
 :Default: ``interactivetools_map.sqlite``
 :Type: str
 
 
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+``interactivetoolsproxy_map``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:Description:
+    Use a database supported by SQLAlchemy as map for the
+    interactivetool proxy. When this option is set, the value of
+    ``interactivetools_map`` is ignored. The value of this option must
+    be a `SQLAlchemy database URL
+    <https://docs.sqlalchemy.org/en/20/core/engines.html#database-urls>`_.
+    Mappings are written to the table "gxitproxy" within the database.
+    This value cannot match ``database_connection`` nor
+    ``install_database_connection``.
+:Default: ``None``
+:Type: str
+
+
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 ``interactivetools_prefix``
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~

doc/source/admin/special_topics/interactivetools.rst

@@ -102,6 +102,19 @@ The ``gx-it-proxy`` config relates to an important service in the InteractiveToo
 proxy. ``gx-it-proxy`` runs as a separate process listening at port 4002 (by default). HTTP requests are decoded based on
 the URL and headers, then somewhat massaged, and finally forwarded to the correct entry point port of the target InteractiveTool.
 
+.. note::
+
+    Entry point mappings used by the proxy are stored on a SQLite database file located at ``interactivetools_map``. In
+    `some situations <https://www.sqlite.org/whentouse.html#situations_where_a_client_server_rdbms_may_work_better>`_,
+    SQLite may not be the best choice. A common case is a high-availability production setup, meaning that multiple
+    copies of Galaxy are running on different servers behind a load balancer.
+
+    For these situations, there exists an optional |configuration option interactivetoolsproxy_map|_ that allows using
+    any database supported by SQLAlchemy (it overrides ``interactivetools_map``).
+
+.. |configuration option interactivetoolsproxy_map| replace:: configuration option ``interactivetoolsproxy_map``
+.. _configuration option interactivetoolsproxy_map: ../config.html#interactivetoolsproxy-map
+
 .. note::
 
     A previous config option ``interactivetools_shorten_url`` was removed in commit `#73100de <https://github.com/galaxyproject/galaxy/pull/16795/commits/73100de17149ca3486c83b8c6ded74987c68a836>`_

lib/galaxy/config/__init__.py

@@ -1103,10 +1103,46 @@ def _process_config(self, kwargs: Dict[str, Any]) -> None:
         self.proxy_session_map = self.dynamic_proxy_session_map
         self.manage_dynamic_proxy = self.dynamic_proxy_manage  # Set to false if being launched externally
 
-        # InteractiveTools propagator mapping file
-        self.interactivetools_map = self._in_root_dir(
-            kwargs.get("interactivetools_map", self._in_data_dir("interactivetools_map.sqlite"))
-        )
+        # Interactive tools proxy mapping
+        if self.interactivetoolsproxy_map is None:
+            self.interactivetools_map = "sqlite:///" + self._in_root_dir(
+                kwargs.get("interactivetools_map", self._in_data_dir("interactivetools_map.sqlite"))
+            )
+        else:
+            self.interactivetools_map = None  # overridden by `self.interactivetoolsproxy_map`
+
+            # ensure the database URL for the SQLAlchemy map does not match that of a Galaxy DB
+            urls = {
+                setting: urlparse(value)
+                for setting, value in (
+                    ("interactivetoolsproxy_map", self.interactivetoolsproxy_map),
+                    ("database_connection", self.database_connection),
+                    ("install_database_connection", self.install_database_connection),
+                )
+                if value is not None
+            }
+
+            def is_in_conflict(url1, url2):
+                return all(
+                    (
+                        url1.scheme == url2.scheme,
+                        url1.hostname == url2.hostname,
+                        url1.port == url2.port,
+                        url1.path == url2.path,
+                    )
+                )
+
+            conflicting_settings = {
+                setting
+                for setting, url in tuple(urls.items())[1:]  # exclude "interactivetoolsproxy_map"
+                if is_in_conflict(url, list(urls.values())[0])  # compare with "interactivetoolsproxy_map"
+            }
+
+            if conflicting_settings:
+                raise ConfigurationError(
+                    f"Option `{tuple(urls)[0]}` cannot take the same value as: %s"
+                    % ", ".join(f"`{setting}`" for setting in conflicting_settings)
+                )
 
         # Compliance/Policy variables
         self.redact_username_during_deletion = False
@@ -1227,6 +1263,8 @@ def try_parsing(value, name):
 
         try_parsing(self.database_connection, "database_connection")
         try_parsing(self.install_database_connection, "install_database_connection")
+        if self.interactivetoolsproxy_map is not None:
+            try_parsing(self.interactivetoolsproxy_map, "interactivetoolsproxy_map")
         try_parsing(self.amqp_internal_connection, "amqp_internal_connection")
 
     def _configure_dataset_storage(self):

lib/galaxy/config/sample/galaxy.yml.sample

@@ -191,7 +191,7 @@ gravity:
 
     # Routes file to monitor.
     # Should be set to the same path as ``interactivetools_map`` in the ``galaxy:`` section. This is ignored if
-    # ``interactivetools_map is set``.
+    # ``interactivetools_map is set.
     # sessions: database/interactivetools_map.sqlite
 
     # Include verbose messages in gx-it-proxy
@@ -1348,11 +1348,24 @@ galaxy:
   # subdomain. Defaults to "/".
   #interactivetools_base_path: /
 
-  # Map for interactivetool proxy.
+  # Map for the interactivetool proxy. Mappings are stored in a SQLite
+  # database file located on this path. As an alternative, you may also
+  # store them in any other RDBMS supported by SQLAlchemy using the
+  # option ``interactivetoolsproxy_map``, which overrides this one.
   # The value of this option will be resolved with respect to
   # <data_dir>.
   #interactivetools_map: interactivetools_map.sqlite
 
+  # Use a database supported by SQLAlchemy as map for the
+  # interactivetool proxy. When this option is set, the value of
+  # ``interactivetools_map`` is ignored. The value of this option must
+  # be a `SQLAlchemy database URL
+  # <https://docs.sqlalchemy.org/en/20/core/engines.html#database-urls>`_.
+  # Mappings are written to the table "gxitproxy" within the database.
+  # This value cannot match ``database_connection`` nor
+  # ``install_database_connection``.
+  #interactivetoolsproxy_map: null
+
   # Prefix to use in the formation of the subdomain or path for
   # interactive tools
   #interactivetools_prefix: interactivetool

lib/galaxy/config/schemas/config_schema.yml

@@ -1520,7 +1520,21 @@ mapping:
         default: interactivetools_map.sqlite
         path_resolves_to: data_dir
         desc: |
-          Map for interactivetool proxy.
+          Map for the interactivetool proxy. Mappings are stored in a SQLite database file
+          located on this path. As an alternative, you may also store them in any other RDBMS
+          supported by SQLAlchemy using the option ``interactivetoolsproxy_map``, which
+          overrides this one.
+
+      interactivetoolsproxy_map:
+        type: str
+        required: false
+        desc: |
+          Use a database supported by SQLAlchemy as map for the interactivetool proxy.
+          When this option is set, the value of ``interactivetools_map`` is ignored. The
+          value of this option must be a
+          `SQLAlchemy database URL <https://docs.sqlalchemy.org/en/20/core/engines.html#database-urls>`_.
+          Mappings are written to the table "gxitproxy" within the database. This value cannot match
+          ``database_connection`` nor ``install_database_connection``.
 
       interactivetools_prefix:
         type: str