diff --git a/lib/galaxy/files/_schema.py b/lib/galaxy/files/_schema.py new file mode 100644 index 000000000000..3c49d2fa096a --- /dev/null +++ b/lib/galaxy/files/_schema.py @@ -0,0 +1,85 @@ +from enum import Enum +from typing import List + +from pydantic import ( + BaseModel, + Extra, + Field, +) + + +class RemoteFilesTarget(str, Enum): + ftpdir = "ftpdir" + userdir = "userdir" + importdir = "importdir" + + +class RemoteFilesFormat(str, Enum): + flat = "flat" + jstree = "jstree" + uri = "uri" + + +class RemoteFilesDisableMode(str, Enum): + folders = "folders" + files = "files" + + +class FilesSourcePlugin(BaseModel): + id: str = Field( + ..., # This field is required + title="ID", + description="The `FilesSource` plugin identifier", + example="_import", + ) + type: str = Field( + ..., # This field is required + title="Type", + description="The type of the plugin.", + example="gximport", + ) + uri_root: str = Field( + ..., # This field is required + title="URI root", + description="The URI root used by this type of plugin.", + example="gximport://", + ) + label: str = Field( + ..., # This field is required + title="Label", + description="The display label for this plugin.", + example="Library Import Directory", + ) + doc: str = Field( + ..., # This field is required + title="Documentation", + description="Documentation or extended description for this plugin.", + example="Galaxy's library import directory", + ) + writable: bool = Field( + ..., # This field is required + title="Writeable", + description="Whether this files source plugin allows write access.", + example=False, + ) + + class Config: + # This allows additional fields (that are not validated) + # to be serialized/deserealized. This allows to have + # different fields depending on the plugin type + extra = Extra.allow + + +class FilesSourcePluginList(BaseModel): + __root__: List[FilesSourcePlugin] = Field( + default=[], + title='List of files source plugins', + example=[{ + "id": "_import", + "type": "gximport", + "uri_root": "gximport://", + "label": "Library Import Directory", + "doc": "Galaxy's library import directory", + "writable": False + }] + ) diff --git a/lib/galaxy/managers/remote_files.py b/lib/galaxy/managers/remote_files.py new file mode 100644 index 000000000000..593f7c435601 --- /dev/null +++ b/lib/galaxy/managers/remote_files.py @@ -0,0 +1,114 @@ + +import hashlib +from operator import itemgetter +from typing import ( + Any, + Dict, + List, + Optional, +) + +from pydantic.tools import parse_obj_as + +from galaxy import exceptions +from galaxy.app import StructuredApp +from galaxy.files import ( + ConfiguredFileSources, + ProvidesUserFileSourcesUserContext, +) +from galaxy.files._schema import ( + FilesSourcePluginList, + RemoteFilesDisableMode, + RemoteFilesFormat, + RemoteFilesTarget, +) +from galaxy.managers.context import ProvidesUserContext +from galaxy.util import ( + jstree, + smart_str, +) + + +class RemoteFilesManager: + """ + Interface/service object for interacting with remote files. + """ + + def __init__(self, app: StructuredApp): + self._app = app + + def index( + self, + user_ctx: ProvidesUserContext, + target: str, + format: Optional[RemoteFilesFormat], + recursive: Optional[bool], + disable: Optional[RemoteFilesDisableMode], + ) -> List[Dict[str, Any]]: + """Returns a list of remote files available to the user.""" + + user_file_source_context = ProvidesUserFileSourcesUserContext(user_ctx) + default_recursive = False + default_format = RemoteFilesFormat.uri + + if "://" in target: + uri = target + elif target == RemoteFilesTarget.userdir: + uri = "gxuserimport://" + default_format = RemoteFilesFormat.flat + default_recursive = True + elif target == RemoteFilesTarget.importdir: + uri = 'gximport://' + default_format = RemoteFilesFormat.flat + default_recursive = True + elif target in [RemoteFilesTarget.ftpdir, 'ftp']: # legacy, allow both + uri = 'gxftp://' + default_format = RemoteFilesFormat.flat + default_recursive = True + else: + raise exceptions.RequestParameterInvalidException(f"Invalid target parameter supplied [{target}]") + + if format is None: + format = default_format + + if recursive is None: + recursive = default_recursive + + self._file_sources.validate_uri_root(uri, user_context=user_file_source_context) + + file_source_path = self._file_sources.get_file_source_path(uri) + file_source = file_source_path.file_source + index = file_source.list(file_source_path.path, recursive=recursive, user_context=user_file_source_context) + if format == RemoteFilesFormat.flat: + # rip out directories, ensure sorted by path + index = [i for i in index if i["class"] == "File"] + index = sorted(index, key=itemgetter("path")) + if format == RemoteFilesFormat.jstree: + if disable is None: + disable = RemoteFilesDisableMode.folders + + jstree_paths = [] + for ent in index: + path = ent["path"] + path_hash = hashlib.sha1(smart_str(path)).hexdigest() + if ent["class"] == "Directory": + path_type = 'folder' + disabled = True if disable == RemoteFilesDisableMode.folders else False + else: + path_type = 'file' + disabled = True if disable == RemoteFilesDisableMode.files else False + + jstree_paths.append(jstree.Path(path, path_hash, {'type': path_type, 'state': {'disabled': disabled}, 'li_attr': {'full_path': path}})) + userdir_jstree = jstree.JSTree(jstree_paths) + index = userdir_jstree.jsonData() + + return index + + def get_files_source_plugins(self) -> FilesSourcePluginList: + """Display plugin information for each of the gxfiles:// URI targets available.""" + plugins = self._file_sources.plugins_to_dict() + return parse_obj_as(FilesSourcePluginList, plugins) + + @property + def _file_sources(self) -> ConfiguredFileSources: + return self._app.file_sources diff --git a/lib/galaxy/webapps/galaxy/api/remote_files.py b/lib/galaxy/webapps/galaxy/api/remote_files.py index d7a7c0746017..fedb777e3381 100644 --- a/lib/galaxy/webapps/galaxy/api/remote_files.py +++ b/lib/galaxy/webapps/galaxy/api/remote_files.py @@ -1,25 +1,115 @@ """ API operations on remote files. """ -import hashlib import logging -from operator import itemgetter - -from galaxy import exceptions -from galaxy.files import ProvidesUserFileSourcesUserContext -from galaxy.managers.context import ProvidesUserContext -from galaxy.util import ( - jstree, - smart_str, +from typing import Any, Dict, List, Optional + +from fastapi import Depends +from fastapi.param_functions import Query +from fastapi_utils.cbv import cbv +from fastapi_utils.inferring_router import InferringRouter as APIRouter + +from galaxy.files._schema import ( + FilesSourcePluginList, + RemoteFilesDisableMode, + RemoteFilesFormat, + RemoteFilesTarget, ) +from galaxy.managers.context import ProvidesUserContext +from galaxy.managers.remote_files import RemoteFilesManager +from galaxy.structured_app import StructuredApp from galaxy.web import expose_api from galaxy.webapps.base.controller import BaseAPIController +from . import ( + get_app, + get_trans, +) log = logging.getLogger(__name__) +router = APIRouter(tags=['remote files']) + +TargetQueryParam: str = Query( + default=RemoteFilesTarget.ftpdir, + title="Target source", + description=( + "The source to load datasets from." + " Possible values: ftpdir, userdir, importdir" + ), +) + +FormatQueryParam: Optional[RemoteFilesFormat] = Query( + default=RemoteFilesFormat.flat, + title="Response format", + description=( + "The requested format of returned data. Either `flat` to simply list all the files" + " or `jstree` to get a tree representation of the files." + ), +) + +RecursiveQueryParam: Optional[bool] = Query( + default=None, + title="Recursive", + description=( + "Wether to recursively lists all sub-directories." + " This will be `True` by default depending on the `target`." + ), +) + +DisableModeQueryParam: Optional[RemoteFilesDisableMode] = Query( + default=None, + title="Disable mode", + description=( + "(This only applies when `format` is `jstree`)" + " The value can be either `folders` or `files` and it will disable the" + " corresponding nodes of the tree." + ), +) + + +def get_remote_files_manager(app: StructuredApp = Depends(get_app)) -> RemoteFilesManager: + return RemoteFilesManager(app) # TODO: remove/refactor after merging #11180 + + +@cbv(router) +class FastAPIRemoteFiles: + manager: RemoteFilesManager = Depends(get_remote_files_manager) + + @router.get( + '/api/remote_files', + summary="Displays remote files available to the user.", + response_description="A list with details about the remote files available to the user.", + ) + async def index( + self, + user_ctx: ProvidesUserContext = Depends(get_trans), + target: str = TargetQueryParam, + format: Optional[RemoteFilesFormat] = FormatQueryParam, + recursive: Optional[bool] = RecursiveQueryParam, + disable: Optional[RemoteFilesDisableMode] = DisableModeQueryParam + ) -> List[Dict[str, Any]]: + """Lists all remote files available to the user from different sources.""" + return self.manager.index(user_ctx, target, format, recursive, disable) + + @router.get( + '/api/remote_files/plugins', + summary="Display plugin information for each of the gxfiles:// URI targets available.", + response_description="A list with details about each plugin.", + ) + async def plugins( + self, + user_ctx: ProvidesUserContext = Depends(get_trans), + ) -> FilesSourcePluginList: + """Display plugin information for each of the gxfiles:// URI targets available.""" + return self.manager.get_files_source_plugins() + class RemoteFilesAPIController(BaseAPIController): + def __init__(self, app): + super().__init__(app) + self.manager = RemoteFilesManager(app) + @expose_api def index(self, trans: ProvidesUserContext, **kwd): """ @@ -39,60 +129,11 @@ def index(self, trans: ProvidesUserContext, **kwd): """ # If set, target must be one of 'ftpdir' (default), 'userdir', 'importdir' target = kwd.get('target', 'ftpdir') + format = kwd.get('format', None) + recursive = kwd.get('recursive', None) + disable = kwd.get('disable', None) - user_context = ProvidesUserFileSourcesUserContext(trans) - default_recursive = False - default_format = "uri" - - if "://" in target: - uri = target - elif target == 'userdir': - uri = "gxuserimport://" - default_format = "flat" - default_recursive = True - elif target == 'importdir': - uri = 'gximport://' - default_format = "flat" - default_recursive = True - elif target in ['ftpdir', 'ftp']: # legacy, allow both - uri = 'gxftp://' - default_format = "flat" - default_recursive = True - else: - raise exceptions.RequestParameterInvalidException("Invalid target parameter supplied [%s]" % target) - - format = kwd.get('format', default_format) - recursive = kwd.get('recursive', default_recursive) - - file_sources = self.app.file_sources - file_sources.validate_uri_root(uri, user_context=user_context) - - file_source_path = file_sources.get_file_source_path(uri) - file_source = file_source_path.file_source - index = file_source.list(file_source_path.path, recursive=recursive, user_context=user_context) - if format == "flat": - # rip out directories, ensure sorted by path - index = [i for i in index if i["class"] == "File"] - index = sorted(index, key=itemgetter("path")) - if format == "jstree": - disable = kwd.get('disable', 'folders') - - jstree_paths = [] - for ent in index: - path = ent["path"] - path_hash = hashlib.sha1(smart_str(path)).hexdigest() - if ent["class"] == "Directory": - path_type = 'folder' - disabled = True if disable == 'folders' else False - else: - path_type = 'file' - disabled = True if disable == 'files' else False - - jstree_paths.append(jstree.Path(path, path_hash, {'type': path_type, 'state': {'disabled': disabled}, 'li_attr': {'full_path': path}})) - userdir_jstree = jstree.JSTree(jstree_paths) - index = userdir_jstree.jsonData() - - return index + return self.manager.index(trans, target, format, recursive, disable) @expose_api def plugins(self, trans: ProvidesUserContext, **kwd): @@ -104,4 +145,4 @@ def plugins(self, trans: ProvidesUserContext, **kwd): :returns: list of configured plugins :rtype: list """ - return self.app.file_sources.plugins_to_dict() + return self.manager.get_files_source_plugins() diff --git a/lib/galaxy/webapps/galaxy/fast_app.py b/lib/galaxy/webapps/galaxy/fast_app.py index c4d7234529b7..f1e038d8e47c 100644 --- a/lib/galaxy/webapps/galaxy/fast_app.py +++ b/lib/galaxy/webapps/galaxy/fast_app.py @@ -29,6 +29,10 @@ "name": "tours", "description": "Operations with interactive tours.", }, + { + "name": "remote files", + "description": "Operations with remote dataset sources.", + }, ]