From c59a17ed31988c585c5e89ee9cf028c685e416f0 Mon Sep 17 00:00:00 2001 From: Mithun Shivashankar Date: Thu, 28 Nov 2024 09:45:21 +0100 Subject: [PATCH 1/7] Update item to hold target --- asab/library/item.py | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/asab/library/item.py b/asab/library/item.py index 15bc3ea94..18d00a2b5 100644 --- a/asab/library/item.py +++ b/asab/library/item.py @@ -3,21 +3,22 @@ @dataclasses.dataclass class LibraryItem: - """ - The data class that contains the info about a specific item in the library. + """ + The data class that contains the info about a specific item in the library. - Attributes: - name (str): The absolute path of the Item. It can be directly fed into `LibraryService.read(...)`. - type (str): Can be either `dir` if the Item is a directory or `item` if Item is of any other type. - layer (int): The number of highest layer in which this Item is found. The higher the number, the lower the layer is. - providers (list): List of `LibraryProvider` objects containing this Item. - disabled (bool): `True` if the Item is disabled, `False` otherwise. If the Item is disabled, `LibraryService.read(...)` will return `None`. - override (int): If `True`, this item is marked as an override for the providers with the same Item name. - """ - - name: str - type: str - layer: int - providers: list - disabled: bool = False - override: int = 0 # Default value for override is False + Attributes: + name (str): The absolute path of the Item. It can be directly fed into `LibraryService.read(...)`. + type (str): Can be either `dir` if the Item is a directory or `item` if Item is of any other type. + layer (int): The number of highest layer in which this Item is found. The higher the number, the lower the layer is. + providers (list): List of `LibraryProvider` objects containing this Item. + disabled (bool): `True` if the Item is disabled, `False` otherwise. If the Item is disabled, `LibraryService.read(...)` will return `None`. + override (int): If `True`, this item is marked as an override for the providers with the same Item name. + target (str): Specifies the target context, e.g., "tenant" or "global". Defaults to "global". + """ + name: str + type: str + layer: int + providers: list + disabled: bool = False + override: int = 0 + target: str = "global" # Default to "global" if not tenant-specific From 8472b102d24af83ac7077a2a22f88122ac3e4b7d Mon Sep 17 00:00:00 2001 From: Mithun Shivashankar Date: Thu, 28 Nov 2024 09:46:31 +0100 Subject: [PATCH 2/7] Add method get metadata --- asab/library/service.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/asab/library/service.py b/asab/library/service.py index b9cbc07b1..0a2f88834 100644 --- a/asab/library/service.py +++ b/asab/library/service.py @@ -453,6 +453,44 @@ def check_disabled(self, path: str) -> bool: return False + async def get_item_metadata(self, path: str) -> typing.Optional[dict]: + """ + Retrieve metadata for a specific file in the library, including its `target`. + + Args: + path (str): The path of the file to retrieve metadata for. + + Returns: + dict: Metadata for the specified file, including `target`, or None if not found. + """ + _validate_path_item(path) + + directory, filename = os.path.split(path) + + try: + # Fetch all items in the directory + items = await self.list(directory) + except Exception as e: + L.warning("Failed to list directory '{}' for path '{}': {}".format(directory, path, e)) + return None + + # Search for the specific file in the directory's items + for item in items: + if item.name == path and item.type == "item": + # Match found; return the item's metadata including `target` + return { + "name": item.name, + "type": item.type, + "layer": item.layer, + "providers": item.providers, + "disabled": item.disabled, + "override": item.override, + "target": item.target # Include the target in the metadata + } + + # If not found, return None + return None + async def export(self, path: str = "/", remove_path: bool = False) -> typing.IO: """ Return a file-like stream containing a gzipped tar archive of the library contents of the path. From aaf35495a908673ae31b93eaf7b1142008942f6c Mon Sep 17 00:00:00 2001 From: Mithun Shivashankar Date: Fri, 29 Nov 2024 08:39:19 +0100 Subject: [PATCH 3/7] Target to the list --- asab/library/providers/zookeeper.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/asab/library/providers/zookeeper.py b/asab/library/providers/zookeeper.py index d699a4a6d..fa2e90da2 100644 --- a/asab/library/providers/zookeeper.py +++ b/asab/library/providers/zookeeper.py @@ -286,7 +286,7 @@ async def list(self, path: str) -> list: tenant_node_path = self.build_path(path, tenant_specific=True) if tenant_node_path != global_node_path: tenant_nodes = await self.Zookeeper.get_children(tenant_node_path) or [] - tenant_items = await self.process_nodes(tenant_nodes, path) + tenant_items = await self.process_nodes(tenant_nodes, path, target="tenant") else: tenant_items = [] # Combine items, with tenant items taking precedence over global ones @@ -295,20 +295,27 @@ async def list(self, path: str) -> list: return list(combined_items.values()) - async def process_nodes(self, nodes, base_path): + async def process_nodes(self, nodes, base_path, target="global"): + """ + Processes a list of nodes and creates corresponding LibraryItem objects. + + Args: + nodes (list): List of node names to process. + base_path (str): The base path for the nodes. + target (str): Specifies the target context, e.g., "tenant" or "global". + + Returns: + list: A list of LibraryItem objects. + """ items = [] for node in nodes: # Remove any component that starts with '.' startswithdot = functools.reduce(lambda x, y: x or y.startswith('.'), node.split(os.path.sep), False) if startswithdot: continue - # Extract the last 5 characters of the node name - last_five_chars = node[-5:] - # Check if there is a period in the last five characters, - # We detect files in Zookeeper by the presence of a dot in the filename, - # but exclude filenames ending with '.io' or '.d' (e.g., 'logman.io', server_https.d) - # from being considered as files. + # Determine if this is a file or directory + last_five_chars = node[-5:] if '.' in last_five_chars and not node.endswith(('.io', '.d')): fname = base_path + node ftype = "item" @@ -316,11 +323,13 @@ async def process_nodes(self, nodes, base_path): fname = base_path + node + '/' ftype = "dir" + # Add the item with the specified target items.append(LibraryItem( name=fname, type=ftype, layer=self.Layer, providers=[self], + target=target )) return items From 7c2fd8facffd08d468f67747cabdd9788517a62f Mon Sep 17 00:00:00 2001 From: Mithun Shivashankar Date: Fri, 6 Dec 2024 16:41:17 +0100 Subject: [PATCH 4/7] Slight refactor --- asab/library/service.py | 46 ++++++++++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/asab/library/service.py b/asab/library/service.py index 0a2f88834..7ce100222 100644 --- a/asab/library/service.py +++ b/asab/library/service.py @@ -458,39 +458,51 @@ async def get_item_metadata(self, path: str) -> typing.Optional[dict]: Retrieve metadata for a specific file in the library, including its `target`. Args: - path (str): The path of the file to retrieve metadata for. + path (str): The absolute path of the file to retrieve metadata for. + Must start with '/' and include a filename with an extension. Returns: dict: Metadata for the specified file, including `target`, or None if not found. """ + # Validate the path format _validate_path_item(path) + # Split into directory and filename directory, filename = os.path.split(path) + if not directory or not filename: + L.warning("Invalid path '{}': missing directory or filename.".format(path)) + return None + try: # Fetch all items in the directory items = await self.list(directory) except Exception as e: - L.warning("Failed to list directory '{}' for path '{}': {}".format(directory, path, e)) + L.warning("Failed to list items in directory '{}': {}".format(directory, e)) return None - # Search for the specific file in the directory's items - for item in items: - if item.name == path and item.type == "item": - # Match found; return the item's metadata including `target` - return { - "name": item.name, - "type": item.type, - "layer": item.layer, - "providers": item.providers, - "disabled": item.disabled, - "override": item.override, - "target": item.target # Include the target in the metadata - } - - # If not found, return None + # Use dictionary for faster lookup + items_dict = {item.name: item for item in items} + + # Retrieve the item by path + item = items_dict.get(path) + if item and item.type == "item": + # Match found; return metadata including `target` + return { + "name": item.name, + "type": item.type, + "layer": item.layer, + "providers": item.providers, + "disabled": item.disabled, + "override": item.override, + "target": item.target, # Include the target in the metadata + } + + # Item not found + L.info("Item '{}' not found in directory '{}'.".format(filename, directory)) return None + async def export(self, path: str = "/", remove_path: bool = False) -> typing.IO: """ Return a file-like stream containing a gzipped tar archive of the library contents of the path. From 1f8ee52a110f98e3b295fdb82db03afdb871f200 Mon Sep 17 00:00:00 2001 From: Mithun Shivashankar Date: Wed, 11 Dec 2024 08:45:27 +0100 Subject: [PATCH 5/7] Add full list --- asab/library/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/asab/library/service.py b/asab/library/service.py index 7ce100222..5d595d9b0 100644 --- a/asab/library/service.py +++ b/asab/library/service.py @@ -476,7 +476,7 @@ async def get_item_metadata(self, path: str) -> typing.Optional[dict]: try: # Fetch all items in the directory - items = await self.list(directory) + items = await self.list("/") except Exception as e: L.warning("Failed to list items in directory '{}': {}".format(directory, e)) return None From ed9b64a38d43604cca38a368fc7f84e3e16834db Mon Sep 17 00:00:00 2001 From: Mithun Shivashankar Date: Wed, 11 Dec 2024 09:11:48 +0100 Subject: [PATCH 6/7] Make a fix --- asab/library/service.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/asab/library/service.py b/asab/library/service.py index 5d595d9b0..7e7d96961 100644 --- a/asab/library/service.py +++ b/asab/library/service.py @@ -473,10 +473,14 @@ async def get_item_metadata(self, path: str) -> typing.Optional[dict]: if not directory or not filename: L.warning("Invalid path '{}': missing directory or filename.".format(path)) return None + # Ensure directory ends with '/' + if not directory.endswith('/'): + directory += '/' try: # Fetch all items in the directory - items = await self.list("/") + items = await self.list(directory) + print(items) except Exception as e: L.warning("Failed to list items in directory '{}': {}".format(directory, e)) return None From 212ae092720a04caf3bdc2f58694506557a1a4f0 Mon Sep 17 00:00:00 2001 From: Mithun Shivashankar Date: Wed, 11 Dec 2024 09:35:46 +0100 Subject: [PATCH 7/7] Remove print --- asab/library/service.py | 1 - 1 file changed, 1 deletion(-) diff --git a/asab/library/service.py b/asab/library/service.py index 7e7d96961..deb431d9c 100644 --- a/asab/library/service.py +++ b/asab/library/service.py @@ -480,7 +480,6 @@ async def get_item_metadata(self, path: str) -> typing.Optional[dict]: try: # Fetch all items in the directory items = await self.list(directory) - print(items) except Exception as e: L.warning("Failed to list items in directory '{}': {}".format(directory, e)) return None