Merge pull request #17 from NLeSC-GO-common-infrastructure/fixes

General fixes

dcachefs/__init__.py

@@ -4,11 +4,14 @@
 
 from .__version__ import __version__
 from .dcachefs import dCacheFileSystem
-from .register_implementation import register_implementation
 
 logging.getLogger(__name__).addHandler(logging.NullHandler())
 
 __author__ = "Francesco Nattino"
 __email__ = '[email protected]'
 
-fsspec.register_implementation("dcache", dCacheFileSystem, clobber=True)
+fsspec.register_implementation(
+    "dcache",
+    "dcachefs.dCacheFileSystem",
+    clobber=True
+)

dcachefs/dcachefs.py

@@ -2,6 +2,7 @@
 import asyncio
 import logging
 import weakref
+import yarl
 
 from datetime import datetime
 from fsspec.asyn import sync_wrapper, sync, AsyncFileSystem
@@ -78,6 +79,7 @@ class dCacheFileSystem(AsyncFileSystem):
     :param loop: (optional) if asynchronous, event loop where to run coroutines
     :param batch_size: (int, optional) if asynchronous, number of coroutines to
         submit/wait on simultaneously
+    :param encoded: use encoded strings when formatting URLs
     :param storage_options: (dict, optional) keyword arguments passed on to the
         super-class
     """
@@ -95,6 +97,7 @@ def __init__(
         asynchronous=False,
         loop=None,
         batch_size=None,
+        encoded=True,
         **storage_options
     ):
         super().__init__(
@@ -108,6 +111,7 @@ def __init__(
         self.webdav_url = webdav_url
         self.client_kwargs = {} if client_kwargs is None else client_kwargs
         self.request_kwargs = {} if request_kwargs is None else request_kwargs
+        self.encoded = encoded
         if (username is None) ^ (password is None):
             raise ValueError('Username or password not provided')
         if (username is not None) and (password is not None):
@@ -139,6 +143,9 @@ def close_session(loop, session):
             # close after loop is dead
             connector._close()
 
+    def encode_url(self, url):
+        return yarl.URL(url, encoded=self.encoded)
+
     async def set_session(self):
         if self._session is None:
             self._session = await get_client(
@@ -186,7 +193,7 @@ def _strip_protocol(cls, path):
         if isinstance(path, list):
             return [cls._strip_protocol(p) for p in path]
         url = URL(path)
-        return url.path if "http" in url.scheme else path.split(":/")[-1]
+        return url.path if "http" in url.scheme else path.split("://")[-1]
 
     @classmethod
     def _get_kwargs_from_urls(cls, path):

environment.yml

@@ -0,0 +1,23 @@
+name: dcachefs
+channels:
+  - conda-forge
+dependencies:
+  - python=3.10
+  # dcachefs dependencies
+  - aiohttp
+  - fsspec
+  - urlpath
+  - yarl
+  # tutorial/optional 
+  - jupyterlab
+  - dask
+  - numpy 
+  - pandas
+  - xarray
+  - zarr
+  - panel
+  # test dependencies
+  - pytest
+  - pytest-cov
+  - pycodestyle
+  - webdavclient3 

notebooks/user_manual.ipynb

@@ -108,8 +108,8 @@
        "[{'name': '/test/testdir_1/file_1.txt',\n",
        "  'size': 12,\n",
        "  'type': 'file',\n",
-       "  'created': datetime.datetime(2022, 9, 14, 9, 28, 55, 841000),\n",
-       "  'modified': datetime.datetime(2022, 9, 14, 9, 28, 55, 881000)}]"
+       "  'created': datetime.datetime(2023, 5, 4, 22, 47, 35, 736000),\n",
+       "  'modified': datetime.datetime(2023, 5, 4, 22, 47, 35, 775000)}]"
       ]
      },
      "execution_count": 3,
@@ -152,8 +152,8 @@
        "{'name': '/test/testdir_1/',\n",
        " 'size': 512,\n",
        " 'type': 'directory',\n",
-       " 'created': datetime.datetime(2022, 9, 14, 9, 28, 55, 660000),\n",
-       " 'modified': datetime.datetime(2022, 9, 14, 9, 28, 56, 62000)}"
+       " 'created': datetime.datetime(2023, 5, 4, 22, 47, 35, 622000),\n",
+       " 'modified': datetime.datetime(2023, 5, 4, 22, 47, 35, 896000)}"
       ]
      },
      "execution_count": 5,
@@ -295,7 +295,7 @@
     {
      "data": {
       "text/plain": [
-       "datetime.datetime(2022, 9, 14, 9, 28, 55, 841000)"
+       "datetime.datetime(2023, 5, 4, 22, 47, 35, 736000)"
       ]
      },
      "execution_count": 12,
@@ -315,7 +315,7 @@
     {
      "data": {
       "text/plain": [
-       "datetime.datetime(2022, 9, 14, 9, 28, 55, 881000)"
+       "datetime.datetime(2023, 5, 4, 22, 47, 35, 775000)"
       ]
      },
      "execution_count": 13,
@@ -379,9 +379,9 @@
      "output_type": "stream",
      "text": [
       "/test\n",
-      "/test/testdir_2\n",
+      "/test/testdir_1\n",
       "/test/empty_testdir\n",
-      "/test/testdir_1\n"
+      "/test/testdir_2\n"
      ]
     }
    ],
@@ -440,7 +440,7 @@
     {
      "data": {
       "text/plain": [
-       "124669634392728841921114151018813876011"
+       "47296560241525359883320580310787008204"
       ]
      },
      "execution_count": 19,
@@ -624,42 +624,6 @@
     "It can be handy, however, to save these input parameters in a `fsspec` configuration file, by default in the directory file `${HOME}/.config/fsspec/`, so that they are always passed as input arguments when the \"dcache\" protocol is encountered (see the [section of the fsspec documentation on configuration](https://filesystem-spec.readthedocs.io/en/latest/features.html#configuration))."
    ]
   },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "One can also temporarily register the `dCacheFileSystem` class as the default implementation for handling URL-paths that include a different protocol. This is useful, for instance, when dealing with file paths that include the WebDAV door URL. One can thus use  the `register_implementation` context manager:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 26,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "b'Hello world!'"
-      ]
-     },
-     "execution_count": 26,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "from dcachefs import register_implementation\n",
-    "\n",
-    "# URI including the WeDAV door\n",
-    "uri = 'https://webdav.grid.surfsara.nl:2880/test/testdir_1/file_1.txt'\n",
-    "\n",
-    "with register_implementation(protocol='https'):\n",
-    "    # no need to specify here the webdav_url - it's already part of the URI\n",
-    "    with fsspec.open(uri, token=token, api_url=api_url) as f:\n",
-    "        content = f.read()\n",
-    "content"
-   ]
-  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -676,7 +640,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 27,
+   "execution_count": 26,
    "metadata": {},
    "outputs": [
     {
@@ -685,7 +649,7 @@
        "('Hello world!',)"
       ]
      },
-     "execution_count": 27,
+     "execution_count": 26,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -721,7 +685,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 28,
+   "execution_count": 27,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -747,7 +711,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 29,
+   "execution_count": 28,
    "metadata": {},
    "outputs": [
     {
@@ -844,7 +808,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.13"
+   "version": "3.10.10"
   }
  },
  "nbformat": 4,

requirements.txt

@@ -1,3 +1,4 @@
 aiohttp
 fsspec
 urlpath
+yarl

tests/test_dcachefs.py

@@ -35,6 +35,7 @@ def test_fs():
     api_url = os.environ['DCACHE_API_URL']
     webdav_url = os.environ['DCACHE_WEBDAV_URL']
     token = os.environ['DCACHE_TOKEN']
+    print(token)
     _setup_test_dir(webdav_url, token)
     return dCacheFileSystem(api_url=api_url,
                             token=token,

tests/test_init.py

@@ -0,0 +1,8 @@
+import fsspec
+
+from dcachefs import dCacheFileSystem
+
+
+def test_dcachefs_is_registered_as_dcache_protocol():
+    assert fsspec.get_filesystem_class('dcache') == dCacheFileSystem
+    assert isinstance(fsspec.filesystem('dcache'), dCacheFileSystem)