feat(Files): Show/hide hidden files & directories

hexa/files/api.py

@@ -132,38 +132,76 @@ def _prefix_to_dict(bucket_name, name: str):
     }
 
 
-def list_bucket_objects(bucket_name, prefix=None, page: int = 1, per_page=30):
+def list_bucket_objects(
+    bucket_name, prefix=None, page: int = 1, per_page=30, ignore_hidden_files=True
+):
+    """Returns the list of objects in a bucket with pagination support.
+    Objects starting with a dot can be ignored using `ignore_hidden_files`.
+
+    Args:
+        bucket_name (str): Bucket name
+        prefix (str, optional): The prefix the keys of the objects must have to be returned. Defaults to None.
+        page (int, optional): Page to return. Defaults to 1.
+        per_page (int, optional): Items per page. Defaults to 30.
+        ignore_hidden_files (bool, optional): Returns the hidden files and directories if `False`. Defaults to True.
+
+    """
     client = get_storage_client()
 
     request = client.list_blobs(
         bucket_name,
         prefix=prefix,
-        page_size=per_page,
+        # We take twice the number of items to be sure to have enough
+        page_size=per_page * 2,
         delimiter="/",
         include_trailing_delimiter=True,
     )
+    pages = request.pages
+
+    max_items = (page * per_page) + 1
+    start_offset = (page - 1) * per_page
+    end_offset = page * per_page
+
     objects = []
-    next_page = None
-    page_number = 0
-    for req_page in request.pages:
-        if request.page_number == page:
-            if page == 1:
-                # Add the prefix to the response if the user requests the first page
-                for prefix in request.prefixes:
-                    objects.append(_prefix_to_dict(bucket_name, prefix))
-
-            page_number = request.page_number
-            objects += [_blob_to_dict(obj) for obj in req_page if _is_dir(obj) is False]
-        elif request.page_number > page:
-            next_page = req_page
-            break
-
-    return ObjectsPage(
-        items=objects,
-        page_number=page_number,
-        has_previous_page=page_number > 1,
-        has_next_page=bool(next_page),
-    )
+    try:
+        current_page = next(pages)
+
+        # Start by adding all the prefixes
+        # Prefixes are virtual directories based on the delimiter specified in the request
+        # The API returns a list of keys that have the delimiter as a suffix (meaning they have objects in them)
+        for prefix in request.prefixes:
+            res = _prefix_to_dict(bucket_name, prefix)
+            if not ignore_hidden_files or not res["name"].startswith("."):
+                objects.append(res)
+        while len(objects) <= max_items:
+            for obj in current_page:
+                if _is_dir(obj):
+                    # We ignore objects that are directories (object with a size = 0 and ending with a /)
+                    # because they are already listed in the prefixes
+                    continue
+
+                res = _blob_to_dict(obj)
+                if not ignore_hidden_files or not res["name"].startswith("."):
+                    objects.append(res)
+
+            current_page = next(pages)
+
+        return ObjectsPage(
+            items=objects[start_offset:end_offset],
+            page_number=page,
+            has_previous_page=page > 1,
+            has_next_page=len(objects) > page * per_page,
+        )
+
+    except StopIteration:
+        # We reached the end of the list of pages. Let's return what we have and set the
+        # has_next_page to false
+        return ObjectsPage(
+            items=objects[start_offset:end_offset],
+            page_number=page,
+            has_previous_page=page > 1,
+            has_next_page=False,
+        )
 
 
 def ensure_is_folder(object_key: str):

hexa/files/graphql/schema.graphql

@@ -26,7 +26,7 @@ type BucketObjectPage {
 
 type Bucket {
   name: String!
-  objects(prefix: String, page: Int = 1, perPage: Int = 15): BucketObjectPage!
+  objects(prefix: String, page: Int = 1, perPage: Int = 15, ignoreHiddenFiles: Boolean = true): BucketObjectPage!
   object(key: String!): BucketObject
 }
 

hexa/files/schema/types.py

@@ -41,12 +41,23 @@ def resolve_bucket_name(workspace, info, **kwargs):
 
 @bucket_object.field("objects")
 @convert_kwargs_to_snake_case
-def resolve_bucket_objects(workspace, info, prefix=None, page=1, per_page=15, **kwargs):
+def resolve_bucket_objects(
+    workspace,
+    info,
+    prefix=None,
+    page=1,
+    per_page=15,
+    ignore_hidden_files=True,
+    **kwargs
+):
     if workspace.bucket_name is None:
         raise ImproperlyConfigured("Workspace does not have a bucket")
-
     page = list_bucket_objects(
-        workspace.bucket_name, prefix=prefix, page=page, per_page=per_page
+        workspace.bucket_name,
+        prefix=prefix,
+        page=page,
+        per_page=per_page,
+        ignore_hidden_files=ignore_hidden_files,
     )
 
     return page

hexa/files/tests/mocks/backend.py

@@ -9,11 +9,9 @@ def __init__(self, project=None):
             project = "test-project-" + str(uuid.uuid1())
         self.project = project
         self.buckets = {}
-        self.blobs = {}
 
     def reset(self):
         self.buckets = {}
-        self.blobs = {}
 
     def create_bucket(self, bucket_name, *args, **kwargs):
         pass
@@ -29,8 +27,6 @@ def create_mock_client(*args, **kwargs):
             return client
 
         def wrapper(*args, **kwargs):
-            self.reset()
-
             with patch("hexa.files.api.get_storage_client", create_mock_client):
                 return func(*args, **kwargs)
 

hexa/files/tests/mocks/blob.py

@@ -6,24 +6,21 @@ def __init__(
         self,
         name,
         bucket,
-        chunk_size=None,
-        encryption_key=None,
-        kms_key_name=None,
-        generation=None,
+        size=None,
+        content_type=None,
     ):
         self.name = _bytes_to_unicode(name)
-        self.chunk_size = chunk_size  # Check that setter accepts value.
-        self._bucket = bucket
-        # self._acl = ObjectACL(self)
-        if encryption_key is not None and kms_key_name is not None:
-            raise ValueError(
-                "Pass at most one of 'encryption_key' " "and 'kms_key_name'"
-            )
+        self.size = size
+        self._content_type = content_type
+        self.bucket = bucket
 
-        self._encryption_key = encryption_key
+    @property
+    def content_type(self):
+        return self._content_type
 
-        if kms_key_name is not None:
-            self._properties["kmsKeyName"] = kms_key_name
+    @property
+    def updated(self):
+        return None
 
-        if generation is not None:
-            self._properties["generation"] = generation
+    def __repr__(self) -> str:
+        return f"<MockBlob: {self.name}>"

hexa/files/tests/mocks/bucket.py

@@ -1,12 +1,6 @@
 from google.cloud.storage._helpers import _validate_name
 
-
-class MockBlob:
-    def __init__(self):
-        pass
-
-    def upload_from_filename(self, *args, **kwargs):
-        pass
+from .blob import MockBlob
 
 
 class MockBucket:
@@ -34,7 +28,9 @@ def list_blobs(self, *args, **kwargs):
         return self.client.list_blobs(self, *args, **kwargs)
 
     def blob(self, *args, **kwargs):
-        return MockBlob()
+        b = MockBlob(*args, bucket=self, **kwargs)
+        self._blobs.append(b)
+        return b
 
     def patch(self):
         pass