FastAPI route for remote files

lib/galaxy/files/_schema.py

@@ -0,0 +1,85 @@
+from enum import Enum
+from typing import List
+
+from pydantic import (
+    BaseModel,
+    Extra,
+    Field,
+)
+
+
+class RemoteFilesTarget(str, Enum):
+    ftpdir = "ftpdir"
+    userdir = "userdir"
+    importdir = "importdir"
+
+
+class RemoteFilesFormat(str, Enum):
+    flat = "flat"
+    jstree = "jstree"
+    uri = "uri"
+
+
+class RemoteFilesDisableMode(str, Enum):
+    folders = "folders"
+    files = "files"
+
+
+class FilesSourcePlugin(BaseModel):
+    id: str = Field(
+        ...,  # This field is required
+        title="ID",
+        description="The `FilesSource` plugin identifier",
+        example="_import",
+    )
+    type: str = Field(
+        ...,  # This field is required
+        title="Type",
+        description="The type of the plugin.",
+        example="gximport",
+    )
+    uri_root: str = Field(
+        ...,  # This field is required
+        title="URI root",
+        description="The URI root used by this type of plugin.",
+        example="gximport://",
+    )
+    label: str = Field(
+        ...,  # This field is required
+        title="Label",
+        description="The display label for this plugin.",
+        example="Library Import Directory",
+    )
+    doc: str = Field(
+        ...,  # This field is required
+        title="Documentation",
+        description="Documentation or extended description for this plugin.",
+        example="Galaxy's library import directory",
+    )
+    writable: bool = Field(
+        ...,  # This field is required
+        title="Writeable",
+        description="Whether this files source plugin allows write access.",
+        example=False,
+    )
+
+    class Config:
+        # This allows additional fields (that are not validated)
+        # to be serialized/deserealized. This allows to have
+        # different fields depending on the plugin type
+        extra = Extra.allow
+
+
+class FilesSourcePluginList(BaseModel):
+    __root__: List[FilesSourcePlugin] = Field(
+        default=[],
+        title='List of files source plugins',
+        example=[{
+            "id": "_import",
+            "type": "gximport",
+            "uri_root": "gximport://",
+            "label": "Library Import Directory",
+            "doc": "Galaxy's library import directory",
+            "writable": False
+        }]
+    )

lib/galaxy/managers/remote_files.py

@@ -0,0 +1,114 @@
+
+import hashlib
+from operator import itemgetter
+from typing import (
+    Any,
+    Dict,
+    List,
+    Optional,
+)
+
+from pydantic.tools import parse_obj_as
+
+from galaxy import exceptions
+from galaxy.app import StructuredApp
+from galaxy.files import (
+    ConfiguredFileSources,
+    ProvidesUserFileSourcesUserContext,
+)
+from galaxy.files._schema import (
+    FilesSourcePluginList,
+    RemoteFilesDisableMode,
+    RemoteFilesFormat,
+    RemoteFilesTarget,
+)
+from galaxy.managers.context import ProvidesUserContext
+from galaxy.util import (
+    jstree,
+    smart_str,
+)
+
+
+class RemoteFilesManager:
+    """
+    Interface/service object for interacting with remote files.
+    """
+
+    def __init__(self, app: StructuredApp):
+        self._app = app
+
+    def index(
+        self,
+        user_ctx: ProvidesUserContext,
+        target: str,
+        format: Optional[RemoteFilesFormat],
+        recursive: Optional[bool],
+        disable: Optional[RemoteFilesDisableMode],
+    ) -> List[Dict[str, Any]]:
+        """Returns a list of remote files available to the user."""
+
+        user_file_source_context = ProvidesUserFileSourcesUserContext(user_ctx)
+        default_recursive = False
+        default_format = RemoteFilesFormat.uri
+
+        if "://" in target:
+            uri = target
+        elif target == RemoteFilesTarget.userdir:
+            uri = "gxuserimport://"
+            default_format = RemoteFilesFormat.flat
+            default_recursive = True
+        elif target == RemoteFilesTarget.importdir:
+            uri = 'gximport://'
+            default_format = RemoteFilesFormat.flat
+            default_recursive = True
+        elif target in [RemoteFilesTarget.ftpdir, 'ftp']:  # legacy, allow both
+            uri = 'gxftp://'
+            default_format = RemoteFilesFormat.flat
+            default_recursive = True
+        else:
+            raise exceptions.RequestParameterInvalidException(f"Invalid target parameter supplied [{target}]")
+
+        if format is None:
+            format = default_format
+
+        if recursive is None:
+            recursive = default_recursive
+
+        self._file_sources.validate_uri_root(uri, user_context=user_file_source_context)
+
+        file_source_path = self._file_sources.get_file_source_path(uri)
+        file_source = file_source_path.file_source
+        index = file_source.list(file_source_path.path, recursive=recursive, user_context=user_file_source_context)
+        if format == RemoteFilesFormat.flat:
+            # rip out directories, ensure sorted by path
+            index = [i for i in index if i["class"] == "File"]
+            index = sorted(index, key=itemgetter("path"))
+        if format == RemoteFilesFormat.jstree:
+            if disable is None:
+                disable = RemoteFilesDisableMode.folders
+
+            jstree_paths = []
+            for ent in index:
+                path = ent["path"]
+                path_hash = hashlib.sha1(smart_str(path)).hexdigest()
+                if ent["class"] == "Directory":
+                    path_type = 'folder'
+                    disabled = True if disable == RemoteFilesDisableMode.folders else False
+                else:
+                    path_type = 'file'
+                    disabled = True if disable == RemoteFilesDisableMode.files else False
+
+                jstree_paths.append(jstree.Path(path, path_hash, {'type': path_type, 'state': {'disabled': disabled}, 'li_attr': {'full_path': path}}))
+            userdir_jstree = jstree.JSTree(jstree_paths)
+            index = userdir_jstree.jsonData()
+
+        return index
+
+    def get_files_source_plugins(self) -> FilesSourcePluginList:
+        """Display plugin information for each of the gxfiles:// URI targets available."""
+        plugins = self._file_sources.plugins_to_dict()
+        return parse_obj_as(FilesSourcePluginList, plugins)
+
+    @property
+    def _file_sources(self) -> ConfiguredFileSources:
+        return self._app.file_sources

lib/galaxy/webapps/galaxy/api/remote_files.py

@@ -1,25 +1,115 @@
 """
 API operations on remote files.
 """
-import hashlib
 import logging
-from operator import itemgetter
-
-from galaxy import exceptions
-from galaxy.files import ProvidesUserFileSourcesUserContext
-from galaxy.managers.context import ProvidesUserContext
-from galaxy.util import (
-    jstree,
-    smart_str,
+from typing import Any, Dict, List, Optional
+
+from fastapi import Depends
+from fastapi.param_functions import Query
+from fastapi_utils.cbv import cbv
+from fastapi_utils.inferring_router import InferringRouter as APIRouter
+
+from galaxy.files._schema import (
+    FilesSourcePluginList,
+    RemoteFilesDisableMode,
+    RemoteFilesFormat,
+    RemoteFilesTarget,
 )
+from galaxy.managers.context import ProvidesUserContext
+from galaxy.managers.remote_files import RemoteFilesManager
+from galaxy.structured_app import StructuredApp
 from galaxy.web import expose_api
 from galaxy.webapps.base.controller import BaseAPIController
+from . import (
+    get_app,
+    get_trans,
+)
 
 log = logging.getLogger(__name__)
 
+router = APIRouter(tags=['remote files'])
+
+TargetQueryParam: str = Query(
+    default=RemoteFilesTarget.ftpdir,
+    title="Target source",
+    description=(
+        "The source to load datasets from."
+        " Possible values: ftpdir, userdir, importdir"
+    ),
+)
+
+FormatQueryParam: Optional[RemoteFilesFormat] = Query(
+    default=RemoteFilesFormat.flat,
+    title="Response format",
+    description=(
+        "The requested format of returned data. Either `flat` to simply list all the files"
+        " or `jstree` to get a tree representation of the files."
+    ),
+)
+
+RecursiveQueryParam: Optional[bool] = Query(
+    default=None,
+    title="Recursive",
+    description=(
+        "Wether to recursively lists all sub-directories."
+        " This will be `True` by default depending on the `target`."
+    ),
+)
+
+DisableModeQueryParam: Optional[RemoteFilesDisableMode] = Query(
+    default=None,
+    title="Disable mode",
+    description=(
+        "(This only applies when `format` is `jstree`)"
+        " The value can be either `folders` or `files` and it will disable the"
+        " corresponding nodes of the tree."
+    ),
+)
+
+
+def get_remote_files_manager(app: StructuredApp = Depends(get_app)) -> RemoteFilesManager:
+    return RemoteFilesManager(app)  # TODO: remove/refactor after merging #11180
+
+
+@cbv(router)
+class FastAPIRemoteFiles:
+    manager: RemoteFilesManager = Depends(get_remote_files_manager)
+
+    @router.get(
+        '/api/remote_files',
+        summary="Displays remote files available to the user.",
+        response_description="A list with details about the remote files available to the user.",
+    )
+    async def index(
+        self,
+        user_ctx: ProvidesUserContext = Depends(get_trans),
+        target: str = TargetQueryParam,
+        format: Optional[RemoteFilesFormat] = FormatQueryParam,
+        recursive: Optional[bool] = RecursiveQueryParam,
+        disable: Optional[RemoteFilesDisableMode] = DisableModeQueryParam
+    ) -> List[Dict[str, Any]]:
+        """Lists all remote files available to the user from different sources."""
+        return self.manager.index(user_ctx, target, format, recursive, disable)
+
+    @router.get(
+        '/api/remote_files/plugins',
+        summary="Display plugin information for each of the gxfiles:// URI targets available.",
+        response_description="A list with details about each plugin.",
+    )
+    async def plugins(
+        self,
+        user_ctx: ProvidesUserContext = Depends(get_trans),
+    ) -> FilesSourcePluginList:
+        """Display plugin information for each of the gxfiles:// URI targets available."""
+        return self.manager.get_files_source_plugins()
+
 
 class RemoteFilesAPIController(BaseAPIController):
 
+    def __init__(self, app):
+        super().__init__(app)
+        self.manager = RemoteFilesManager(app)
+
     @expose_api
     def index(self, trans: ProvidesUserContext, **kwd):
         """
@@ -39,60 +129,11 @@ def index(self, trans: ProvidesUserContext, **kwd):
         """
         # If set, target must be one of 'ftpdir' (default), 'userdir', 'importdir'
         target = kwd.get('target', 'ftpdir')
+        format = kwd.get('format', None)
+        recursive = kwd.get('recursive', None)
+        disable = kwd.get('disable', None)
 
-        user_context = ProvidesUserFileSourcesUserContext(trans)
-        default_recursive = False
-        default_format = "uri"
-
-        if "://" in target:
-            uri = target
-        elif target == 'userdir':
-            uri = "gxuserimport://"
-            default_format = "flat"
-            default_recursive = True
-        elif target == 'importdir':
-            uri = 'gximport://'
-            default_format = "flat"
-            default_recursive = True
-        elif target in ['ftpdir', 'ftp']:  # legacy, allow both
-            uri = 'gxftp://'
-            default_format = "flat"
-            default_recursive = True
-        else:
-            raise exceptions.RequestParameterInvalidException("Invalid target parameter supplied [%s]" % target)
-
-        format = kwd.get('format', default_format)
-        recursive = kwd.get('recursive', default_recursive)
-
-        file_sources = self.app.file_sources
-        file_sources.validate_uri_root(uri, user_context=user_context)
-
-        file_source_path = file_sources.get_file_source_path(uri)
-        file_source = file_source_path.file_source
-        index = file_source.list(file_source_path.path, recursive=recursive, user_context=user_context)
-        if format == "flat":
-            # rip out directories, ensure sorted by path
-            index = [i for i in index if i["class"] == "File"]
-            index = sorted(index, key=itemgetter("path"))
-        if format == "jstree":
-            disable = kwd.get('disable', 'folders')
-
-            jstree_paths = []
-            for ent in index:
-                path = ent["path"]
-                path_hash = hashlib.sha1(smart_str(path)).hexdigest()
-                if ent["class"] == "Directory":
-                    path_type = 'folder'
-                    disabled = True if disable == 'folders' else False
-                else:
-                    path_type = 'file'
-                    disabled = True if disable == 'files' else False
-
-                jstree_paths.append(jstree.Path(path, path_hash, {'type': path_type, 'state': {'disabled': disabled}, 'li_attr': {'full_path': path}}))
-            userdir_jstree = jstree.JSTree(jstree_paths)
-            index = userdir_jstree.jsonData()
-
-        return index
+        return self.manager.index(trans, target, format, recursive, disable)
 
     @expose_api
     def plugins(self, trans: ProvidesUserContext, **kwd):
@@ -104,4 +145,4 @@ def plugins(self, trans: ProvidesUserContext, **kwd):
         :returns:   list of configured plugins
         :rtype:     list
         """
-        return self.app.file_sources.plugins_to_dict()
+        return self.manager.get_files_source_plugins()