From a88a361b4154515912a6ca51e33ec285dde2d5f0 Mon Sep 17 00:00:00 2001 From: noidname01 <55401762+noidname01@users.noreply.github.com> Date: Fri, 10 May 2024 18:11:14 +0800 Subject: [PATCH] [#3206]improvement(client-python): Add Black for client-python (#3254) ### What changes were proposed in this pull request? * Add Black as code formatter, it enforce the coding style with 1. trailing commas and whitespaces 2. unified quotation marks(") 3. new lines 4. max line length 5. indents * Aligned with Pylint Rules * Add Black into Gradle and form a code formatting pipeline (pipInstall -> Black -> Pylint), this pipeline will run implicitly in `build` and `test` gradle tasks. > Note that Black still can't format long entire string exceeding `max line length` without enabling unstable features, please handle long strings with caution and make Pylint to ignore them if they are really necessary. ### Why are the changes needed? Fix: #3206, #3203 ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? No --------- Co-authored-by: TimWang --- clients/client-python/build.gradle.kts | 17 ++- clients/client-python/gravitino/api/audit.py | 1 + .../client-python/gravitino/api/auditable.py | 1 + .../client-python/gravitino/api/catalog.py | 18 ++- .../gravitino/api/catalog_change.py | 1 + .../client-python/gravitino/api/fileset.py | 2 + .../gravitino/api/fileset_change.py | 11 +- .../client-python/gravitino/api/metalake.py | 1 + .../gravitino/api/metalake_change.py | 21 +-- clients/client-python/gravitino/api/schema.py | 1 + .../gravitino/api/schema_change.py | 7 +- .../gravitino/api/supports_schemas.py | 8 +- .../gravitino/catalog/base_schema_catalog.py | 68 +++++++-- .../gravitino/catalog/fileset_catalog.py | 61 +++++--- .../client/gravitino_admin_client.py | 29 +++- .../gravitino/client/gravitino_client.py | 15 +- .../gravitino/client/gravitino_client_base.py | 11 +- .../gravitino/client/gravitino_metalake.py | 53 ++++--- .../gravitino/client/gravitino_version.py | 6 +- .../client-python/gravitino/dto/audit_dto.py | 17 ++- .../gravitino/dto/catalog_dto.py | 27 ++-- .../gravitino/dto/dto_converters.py | 41 ++++-- .../gravitino/dto/fileset_dto.py | 15 +- .../gravitino/dto/metalake_dto.py | 19 ++- .../dto/requests/catalog_create_request.py | 34 +++-- .../dto/requests/catalog_update_request.py | 27 ++-- .../dto/requests/catalog_updates_request.py | 6 +- .../dto/requests/fileset_create_request.py | 28 ++-- .../dto/requests/fileset_update_request.py | 14 +- .../dto/requests/fileset_updates_request.py | 8 +- .../dto/requests/metalake_create_request.py | 17 ++- .../dto/requests/metalake_update_request.py | 13 +- .../dto/requests/metalake_updates_request.py | 3 +- .../dto/requests/schema_create_request.py | 15 +- .../dto/requests/schema_update_request.py | 9 +- .../dto/requests/schema_updates_request.py | 8 +- .../gravitino/dto/responses/base_response.py | 3 +- .../dto/responses/catalog_list_response.py | 4 +- .../dto/responses/catalog_response.py | 8 +- .../gravitino/dto/responses/drop_response.py | 3 +- .../dto/responses/entity_list_response.py | 6 +- .../dto/responses/fileset_response.py | 12 +- .../dto/responses/metalake_list_response.py | 3 +- .../dto/responses/metalake_response.py | 11 +- .../dto/responses/schema_response.py | 12 +- .../client-python/gravitino/dto/schema_dto.py | 11 +- .../gravitino/dto/version_dto.py | 1 + .../exceptions/no_such_metalake_exception.py | 1 + .../exceptions/not_found_exception.py | 1 + .../gravitino/name_identifier.py | 60 +++++--- clients/client-python/gravitino/namespace.py | 76 ++++++---- .../gravitino/rest/rest_message.py | 2 + .../gravitino/utils/http_client.py | 6 +- clients/client-python/pylintrc | 17 ++- clients/client-python/requirements-dev.txt | 3 +- .../tests/integration/integration_test_env.py | 37 +++-- .../tests/integration/test_fileset_catalog.py | 133 ++++++++++++------ .../tests/integration/test_metalake.py | 88 ++++++++---- .../tests/integration/test_schema.py | 99 +++++++++---- 59 files changed, 859 insertions(+), 371 deletions(-) diff --git a/clients/client-python/build.gradle.kts b/clients/client-python/build.gradle.kts index 16e1a832d8a..f17bdfdcdea 100644 --- a/clients/client-python/build.gradle.kts +++ b/clients/client-python/build.gradle.kts @@ -38,7 +38,15 @@ tasks { args = listOf("install", "-e", ".[dev]") } + val black by registering(VenvTask::class) { + dependsOn(pipInstall) + venvExec = "black" + args = listOf("./gravitino", "./tests") + } + val pylint by registering(VenvTask::class) { + dependsOn(pipInstall) + mustRunAfter(black) venvExec = "pylint" args = listOf("./gravitino", "./tests") } @@ -50,7 +58,6 @@ tasks { gravitinoServer("start") } - dependsOn(pipInstall, pylint) venvExec = "python" args = listOf("-m", "unittest") workingDir = projectDir.resolve(".") @@ -67,7 +74,6 @@ tasks { } val build by registering(VenvTask::class) { - dependsOn(pipInstall, pylint) } val clean by registering(Delete::class) { @@ -79,4 +85,11 @@ tasks { deleteCacheDir("__pycache__") } } + + matching { + it.name.endsWith("envSetup") + }.all { + // add install package and code formatting before any tasks + finalizedBy(pipInstall, black, pylint) + } } diff --git a/clients/client-python/gravitino/api/audit.py b/clients/client-python/gravitino/api/audit.py index 08dc1407568..9483a99d7c4 100644 --- a/clients/client-python/gravitino/api/audit.py +++ b/clients/client-python/gravitino/api/audit.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + from abc import ABC, abstractmethod from datetime import datetime diff --git a/clients/client-python/gravitino/api/auditable.py b/clients/client-python/gravitino/api/auditable.py index fd7c6e8ad67..2c7e51f247f 100644 --- a/clients/client-python/gravitino/api/auditable.py +++ b/clients/client-python/gravitino/api/auditable.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + from abc import ABC, abstractmethod from gravitino.api.audit import Audit diff --git a/clients/client-python/gravitino/api/catalog.py b/clients/client-python/gravitino/api/catalog.py index fef0de70735..95134f0c631 100644 --- a/clients/client-python/gravitino/api/catalog.py +++ b/clients/client-python/gravitino/api/catalog.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + from abc import abstractmethod from enum import Enum from typing import Dict, Optional @@ -14,6 +15,7 @@ class Catalog(Auditable): """The interface of a catalog. The catalog is the second level entity in the gravitino system, containing a set of tables. """ + class Type(Enum): """The type of the catalog.""" @@ -92,9 +94,11 @@ def as_schemas(self) -> SupportsSchemas: Returns: The {@link SupportsSchemas} if the catalog supports schema operations. """ - raise UnsupportedOperationException("Catalog does not support schema operations") + raise UnsupportedOperationException( + "Catalog does not support schema operations" + ) - def as_table_catalog(self) -> 'TableCatalog': + def as_table_catalog(self) -> "TableCatalog": """ Raises: UnsupportedOperationException if the catalog does not support table operations. @@ -104,7 +108,7 @@ def as_table_catalog(self) -> 'TableCatalog': """ raise UnsupportedOperationException("Catalog does not support table operations") - def as_fileset_catalog(self) -> 'FilesetCatalog': + def as_fileset_catalog(self) -> "FilesetCatalog": """ Raises: UnsupportedOperationException if the catalog does not support fileset operations. @@ -112,9 +116,11 @@ def as_fileset_catalog(self) -> 'FilesetCatalog': Returns: the FilesetCatalog if the catalog supports fileset operations. """ - raise UnsupportedOperationException("Catalog does not support fileset operations") + raise UnsupportedOperationException( + "Catalog does not support fileset operations" + ) - def as_topic_catalog(self) -> 'TopicCatalog': + def as_topic_catalog(self) -> "TopicCatalog": """ Returns: the {@link TopicCatalog} if the catalog supports topic operations. @@ -126,4 +132,4 @@ def as_topic_catalog(self) -> 'TopicCatalog': class UnsupportedOperationException(Exception): - pass \ No newline at end of file + pass diff --git a/clients/client-python/gravitino/api/catalog_change.py b/clients/client-python/gravitino/api/catalog_change.py index 71ebd04f5a6..2e60e7d8a26 100644 --- a/clients/client-python/gravitino/api/catalog_change.py +++ b/clients/client-python/gravitino/api/catalog_change.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + from abc import ABC diff --git a/clients/client-python/gravitino/api/fileset.py b/clients/client-python/gravitino/api/fileset.py index c187d3680e0..a007782b47f 100644 --- a/clients/client-python/gravitino/api/fileset.py +++ b/clients/client-python/gravitino/api/fileset.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + from abc import abstractmethod from enum import Enum from typing import Optional, Dict @@ -20,6 +21,7 @@ class Fileset(Auditable): Fileset defines the basic properties of a fileset object. A catalog implementation with FilesetCatalog should implement this interface. """ + class Type(Enum): """An enum representing the type of the fileset object.""" diff --git a/clients/client-python/gravitino/api/fileset_change.py b/clients/client-python/gravitino/api/fileset_change.py index af4c6c02201..086f944d60a 100644 --- a/clients/client-python/gravitino/api/fileset_change.py +++ b/clients/client-python/gravitino/api/fileset_change.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + from abc import ABC from dataclasses import field @@ -65,7 +66,7 @@ def remove_property(property): class RenameFileset: """A fileset change to rename the fileset.""" - _new_name: str = field(metadata=config(field_name='new_name')) + _new_name: str = field(metadata=config(field_name="new_name")) def __init__(self, new_name): self._new_name = new_name @@ -113,7 +114,7 @@ def __str__(self): class UpdateFilesetComment: """A fileset change to update the fileset comment.""" - _new_comment: str = field(metadata=config(field_name='new_comment')) + _new_comment: str = field(metadata=config(field_name="new_comment")) def __init__(self, new_comment): self._new_comment = new_comment @@ -161,8 +162,8 @@ def __str__(self): class SetProperty: """A fileset change to set the property and value for the fileset.""" - _property: str = field(metadata=config(field_name='property')) - _value: str = field(metadata=config(field_name='value')) + _property: str = field(metadata=config(field_name="property")) + _value: str = field(metadata=config(field_name="value")) def __init__(self, property: str, value: str): self._property = property @@ -219,7 +220,7 @@ def __str__(self): class RemoveProperty: """A fileset change to remove a property from the fileset.""" - _property: str = field(metadata=config(field_name='property')) + _property: str = field(metadata=config(field_name="property")) def __init__(self, property: str): self._property = property diff --git a/clients/client-python/gravitino/api/metalake.py b/clients/client-python/gravitino/api/metalake.py index 63ec93e136d..a1e996f6d8b 100644 --- a/clients/client-python/gravitino/api/metalake.py +++ b/clients/client-python/gravitino/api/metalake.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + from abc import abstractmethod from typing import Optional, Dict diff --git a/clients/client-python/gravitino/api/metalake_change.py b/clients/client-python/gravitino/api/metalake_change.py index 19cb090a86f..1a7469d5da6 100644 --- a/clients/client-python/gravitino/api/metalake_change.py +++ b/clients/client-python/gravitino/api/metalake_change.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + from dataclasses import dataclass, field from dataclasses_json import config @@ -14,7 +15,7 @@ class MetalakeChange: """ @staticmethod - def rename(new_name: str) -> 'MetalakeChange.RenameMetalake': + def rename(new_name: str) -> "MetalakeChange.RenameMetalake": """Creates a new metalake change to rename the metalake. Args: @@ -26,7 +27,7 @@ def rename(new_name: str) -> 'MetalakeChange.RenameMetalake': return MetalakeChange.RenameMetalake(new_name) @staticmethod - def update_comment(new_comment: str) -> 'MetalakeChange.UpdateMetalakeComment': + def update_comment(new_comment: str) -> "MetalakeChange.UpdateMetalakeComment": """Creates a new metalake change to update the metalake comment. Args: @@ -38,7 +39,7 @@ def update_comment(new_comment: str) -> 'MetalakeChange.UpdateMetalakeComment': return MetalakeChange.UpdateMetalakeComment(new_comment) @staticmethod - def set_property(property: str, value: str) -> 'SetProperty': + def set_property(property: str, value: str) -> "SetProperty": """Creates a new metalake change to set a property and value pair for the metalake. Args: @@ -51,7 +52,7 @@ def set_property(property: str, value: str) -> 'SetProperty': return MetalakeChange.SetProperty(property, value) @staticmethod - def remove_property(property: str) -> 'RemoveProperty': + def remove_property(property: str) -> "RemoveProperty": """Creates a new metalake change to remove a property from the metalake. Args: @@ -65,7 +66,8 @@ def remove_property(property: str) -> 'RemoveProperty': @dataclass(frozen=True) class RenameMetalake: """A metalake change to rename the metalake.""" - _new_name: str = field(metadata=config(field_name='new_name')) + + _new_name: str = field(metadata=config(field_name="new_name")) def new_name(self) -> str: return self._new_name @@ -76,7 +78,8 @@ def __str__(self): @dataclass(frozen=True) class UpdateMetalakeComment: """A metalake change to update the metalake comment""" - _new_comment: str = field(metadata=config(field_name='new_comment')) + + _new_comment: str = field(metadata=config(field_name="new_comment")) def new_comment(self) -> str: return self._new_comment @@ -87,8 +90,9 @@ def __str__(self): @dataclass(frozen=True) class SetProperty: """A metalake change to set a property and value pair for the metalake""" - _property: str = field(metadata=config(field_name='property')) - _value: str = field(metadata=config(field_name='value')) + + _property: str = field(metadata=config(field_name="property")) + _value: str = field(metadata=config(field_name="value")) def property(self) -> str: return self._property @@ -102,6 +106,7 @@ def __str__(self): @dataclass(frozen=True) class RemoveProperty: """A metalake change to remove a property from the metalake""" + _property: str def property(self) -> str: diff --git a/clients/client-python/gravitino/api/schema.py b/clients/client-python/gravitino/api/schema.py index 92cf8cc72f2..204abf17d01 100644 --- a/clients/client-python/gravitino/api/schema.py +++ b/clients/client-python/gravitino/api/schema.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + from abc import abstractmethod from typing import Optional, Dict diff --git a/clients/client-python/gravitino/api/schema_change.py b/clients/client-python/gravitino/api/schema_change.py index 99138839420..c9ed2d83603 100644 --- a/clients/client-python/gravitino/api/schema_change.py +++ b/clients/client-python/gravitino/api/schema_change.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + from abc import ABC from dataclasses import field @@ -39,8 +40,8 @@ def remove_property(property: str): class SetProperty: """SchemaChange class to set the property and value pairs for the schema.""" - _property: str = field(metadata=config(field_name='property')) - _value: str = field(metadata=config(field_name='value')) + _property: str = field(metadata=config(field_name="property")) + _value: str = field(metadata=config(field_name="value")) def __init__(self, property: str, value: str): self._property = property @@ -97,7 +98,7 @@ def __str__(self): class RemoveProperty: """SchemaChange class to remove a property from the schema.""" - _property: str = field(metadata=config(field_name='property')) + _property: str = field(metadata=config(field_name="property")) def __init__(self, property: str): self._property = property diff --git a/clients/client-python/gravitino/api/supports_schemas.py b/clients/client-python/gravitino/api/supports_schemas.py index d237b03b8ce..12b9cf2a1d2 100644 --- a/clients/client-python/gravitino/api/supports_schemas.py +++ b/clients/client-python/gravitino/api/supports_schemas.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + from abc import ABC, abstractmethod from typing import List, Dict @@ -13,6 +14,7 @@ class NoSuchSchemaException(Exception): """Exception raised if the schema does not exist.""" + pass @@ -60,7 +62,9 @@ def schema_exists(self, ident: NameIdentifier) -> bool: return False @abstractmethod - def create_schema(self, ident: NameIdentifier, comment: str, properties: Dict[str, str]) -> Schema: + def create_schema( + self, ident: NameIdentifier, comment: str, properties: Dict[str, str] + ) -> Schema: """Create a schema in the catalog. Args: @@ -110,7 +114,7 @@ def alter_schema(self, ident: NameIdentifier, *changes: SchemaChange) -> Schema: @abstractmethod def drop_schema(self, ident: NameIdentifier, cascade: bool) -> bool: - """Drop a schema from the catalog. If cascade option is true, recursively + """Drop a schema from the catalog. If cascade option is true, recursively drop all objects within the schema. Args: diff --git a/clients/client-python/gravitino/catalog/base_schema_catalog.py b/clients/client-python/gravitino/catalog/base_schema_catalog.py index cd0b5c5e0a7..c0a6c1d9b10 100644 --- a/clients/client-python/gravitino/catalog/base_schema_catalog.py +++ b/clients/client-python/gravitino/catalog/base_schema_catalog.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + import logging from typing import Dict @@ -34,11 +35,24 @@ class BaseSchemaCatalog(CatalogDTO, SupportsSchemas): rest_client: HTTPClient """The REST client to send the requests.""" - def __init__(self, name: str = None, type: Catalog.Type = Catalog.Type.UNSUPPORTED, provider: str = None, - comment: str = None, properties: Dict[str, str] = None, audit: AuditDTO = None, - rest_client: HTTPClient = None): - super().__init__(_name=name, _type=type, _provider=provider, _comment=comment, _properties=properties, - _audit=audit) + def __init__( + self, + name: str = None, + type: Catalog.Type = Catalog.Type.UNSUPPORTED, + provider: str = None, + comment: str = None, + properties: Dict[str, str] = None, + audit: AuditDTO = None, + rest_client: HTTPClient = None, + ): + super().__init__( + _name=name, + _type=type, + _provider=provider, + _comment=comment, + _properties=properties, + _audit=audit, + ) self.rest_client = rest_client def as_schemas(self): @@ -57,12 +71,21 @@ def list_schemas(self, namespace: Namespace) -> [NameIdentifier]: A list of {@link NameIdentifier} of the schemas under the given catalog namespace. """ Namespace.check_schema(namespace) - resp = self.rest_client.get(BaseSchemaCatalog.format_schema_request_path(namespace)) - entity_list_response = EntityListResponse.from_json(resp.body, infer_missing=True) + resp = self.rest_client.get( + BaseSchemaCatalog.format_schema_request_path(namespace) + ) + entity_list_response = EntityListResponse.from_json( + resp.body, infer_missing=True + ) entity_list_response.validate() return entity_list_response.identifiers() - def create_schema(self, ident: NameIdentifier = None, comment: str = None, properties: Dict[str, str] = None) -> Schema: + def create_schema( + self, + ident: NameIdentifier = None, + comment: str = None, + properties: Dict[str, str] = None, + ) -> Schema: """Create a new schema with specified identifier, comment and metadata. Args: @@ -81,7 +104,9 @@ def create_schema(self, ident: NameIdentifier = None, comment: str = None, prope req = SchemaCreateRequest(ident.name(), comment, properties) req.validate() - resp = self.rest_client.post(BaseSchemaCatalog.format_schema_request_path(ident.namespace()), json=req) + resp = self.rest_client.post( + BaseSchemaCatalog.format_schema_request_path(ident.namespace()), json=req + ) schema_response = SchemaResponse.from_json(resp.body, infer_missing=True) schema_response.validate() @@ -101,7 +126,10 @@ def load_schema(self, ident: NameIdentifier) -> Schema: """ NameIdentifier.check_schema(ident) resp = self.rest_client.get( - BaseSchemaCatalog.format_schema_request_path(ident.namespace()) + "/" + ident.name()) + BaseSchemaCatalog.format_schema_request_path(ident.namespace()) + + "/" + + ident.name() + ) schema_response = SchemaResponse.from_json(resp.body, infer_missing=True) schema_response.validate() @@ -121,11 +149,17 @@ def alter_schema(self, ident: NameIdentifier, *changes: SchemaChange) -> Schema: The altered Schema. """ NameIdentifier.check_schema(ident) - reqs = [BaseSchemaCatalog.to_schema_update_request(change) for change in changes] + reqs = [ + BaseSchemaCatalog.to_schema_update_request(change) for change in changes + ] updatesRequest = SchemaUpdatesRequest(reqs) updatesRequest.validate() resp = self.rest_client.put( - BaseSchemaCatalog.format_schema_request_path(ident.namespace()) + "/" + ident.name(), updatesRequest) + BaseSchemaCatalog.format_schema_request_path(ident.namespace()) + + "/" + + ident.name(), + updatesRequest, + ) schema_response = SchemaResponse.from_json(resp.body, infer_missing=True) schema_response.validate() return schema_response.schema() @@ -147,7 +181,11 @@ def drop_schema(self, ident: NameIdentifier, cascade: bool) -> bool: try: params = {"cascade": str(cascade)} resp = self.rest_client.delete( - BaseSchemaCatalog.format_schema_request_path(ident.namespace()) + "/" + ident.name(), params=params) + BaseSchemaCatalog.format_schema_request_path(ident.namespace()) + + "/" + + ident.name(), + params=params, + ) drop_resp = DropResponse.from_json(resp.body, infer_missing=True) drop_resp.validate() return drop_resp.dropped() @@ -162,7 +200,9 @@ def format_schema_request_path(ns: Namespace): @staticmethod def to_schema_update_request(change: SchemaChange): if isinstance(change, SchemaChange.SetProperty): - return SchemaUpdateRequest.SetSchemaPropertyRequest(change.property(), change.value()) + return SchemaUpdateRequest.SetSchemaPropertyRequest( + change.property(), change.value() + ) elif isinstance(change, SchemaChange.RemoveProperty): return SchemaUpdateRequest.RemoveSchemaPropertyRequest(change.property()) else: diff --git a/clients/client-python/gravitino/catalog/fileset_catalog.py b/clients/client-python/gravitino/catalog/fileset_catalog.py index 60f4d3e843d..a2708441673 100644 --- a/clients/client-python/gravitino/catalog/fileset_catalog.py +++ b/clients/client-python/gravitino/catalog/fileset_catalog.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + import logging from typing import List, Dict @@ -29,9 +30,16 @@ class FilesetCatalog(BaseSchemaCatalog): example, schemas and filesets list, creation, update and deletion. A Fileset catalog is under the metalake. """ - def __init__(self, name: str = None, type: Catalog.Type = Catalog.Type.UNSUPPORTED, - provider: str = None, comment: str = None, properties: Dict[str, str] = None, - audit: AuditDTO = None, rest_client: HTTPClient = None): + def __init__( + self, + name: str = None, + type: Catalog.Type = Catalog.Type.UNSUPPORTED, + provider: str = None, + comment: str = None, + properties: Dict[str, str] = None, + audit: AuditDTO = None, + rest_client: HTTPClient = None, + ): super().__init__(name, type, provider, comment, properties, audit, rest_client) @@ -52,9 +60,7 @@ def list_filesets(self, namespace: Namespace) -> List[NameIdentifier]: """ Namespace.check_fileset(namespace) - resp = self.rest_client.get( - self.format_fileset_request_path(namespace) - ) + resp = self.rest_client.get(self.format_fileset_request_path(namespace)) entity_list_resp = EntityListResponse.from_json(resp.body, infer_missing=True) entity_list_resp.validate() @@ -74,14 +80,22 @@ def load_fileset(self, ident) -> Fileset: """ NameIdentifier.check_fileset(ident) - resp = self.rest_client.get(f"{self.format_fileset_request_path(ident.namespace())}/{ident.name()}") + resp = self.rest_client.get( + f"{self.format_fileset_request_path(ident.namespace())}/{ident.name()}" + ) fileset_resp = FilesetResponse.from_json(resp.body, infer_missing=True) fileset_resp.validate() return fileset_resp.fileset() - def create_fileset(self, ident: NameIdentifier, comment: str, type: Catalog.Type, - storage_location: str, properties: Dict[str, str]) -> Fileset: + def create_fileset( + self, + ident: NameIdentifier, + comment: str, + type: Catalog.Type, + storage_location: str, + properties: Dict[str, str], + ) -> Fileset: """Create a fileset metadata in the catalog. If the type of the fileset object is "MANAGED", the underlying storageLocation can be null, @@ -105,10 +119,17 @@ def create_fileset(self, ident: NameIdentifier, comment: str, type: Catalog.Type """ NameIdentifier.check_fileset(ident) - req = FilesetCreateRequest(name=ident.name(), comment=comment, type=type, - storage_location=storage_location, properties=properties) + req = FilesetCreateRequest( + name=ident.name(), + comment=comment, + type=type, + storage_location=storage_location, + properties=properties, + ) - resp = self.rest_client.post(self.format_fileset_request_path(ident.namespace()), req) + resp = self.rest_client.post( + self.format_fileset_request_path(ident.namespace()), req + ) fileset_resp = FilesetResponse.from_json(resp.body, infer_missing=True) fileset_resp.validate() @@ -130,11 +151,15 @@ def alter_fileset(self, ident, *changes) -> Fileset: """ NameIdentifier.check_fileset(ident) - updates = [FilesetCatalog.to_fileset_update_request(change) for change in changes] + updates = [ + FilesetCatalog.to_fileset_update_request(change) for change in changes + ] req = FilesetUpdatesRequest(updates) req.validate() - resp = self.rest_client.put(f"{self.format_fileset_request_path(ident.namespace())}/{ident.name()}", req) + resp = self.rest_client.put( + f"{self.format_fileset_request_path(ident.namespace())}/{ident.name()}", req + ) fileset_resp = FilesetResponse.from_json(resp.body, infer_missing=True) fileset_resp.validate() @@ -176,9 +201,13 @@ def to_fileset_update_request(change: FilesetChange): if isinstance(change, FilesetChange.RenameFileset): return FilesetUpdateRequest.RenameFilesetRequest(change.new_name()) elif isinstance(change, FilesetChange.UpdateFilesetComment): - return FilesetUpdateRequest.UpdateFilesetCommentRequest(change.new_comment()) + return FilesetUpdateRequest.UpdateFilesetCommentRequest( + change.new_comment() + ) elif isinstance(change, FilesetChange.SetProperty): - return FilesetUpdateRequest.SetFilesetPropertyRequest(change.property(), change.value()) + return FilesetUpdateRequest.SetFilesetPropertyRequest( + change.property(), change.value() + ) elif isinstance(change, FilesetChange.RemoveProperty): return FilesetUpdateRequest.RemoveFilesetPropertyRequest(change.property()) else: diff --git a/clients/client-python/gravitino/client/gravitino_admin_client.py b/clients/client-python/gravitino/client/gravitino_admin_client.py index f3ea30681fb..944a40a268a 100644 --- a/clients/client-python/gravitino/client/gravitino_admin_client.py +++ b/clients/client-python/gravitino/client/gravitino_admin_client.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + import logging from typing import List, Dict @@ -21,7 +22,8 @@ class GravitinoAdminClient(GravitinoClientBase): """ - Gravitino Client for the administrator to interact with the Gravitino API, allowing the client to list, load, create, and alter Metalakes. + Gravitino Client for the administrator to interact with the Gravitino API. + It allows the client to list, load, create, and alter Metalakes. Normal users should use {@link GravitinoClient} to connect with the Gravitino server. """ @@ -34,12 +36,19 @@ def list_metalakes(self) -> List[GravitinoMetalake]: An array of GravitinoMetalake objects representing the Metalakes. """ resp = self._rest_client.get(self.API_METALAKES_LIST_PATH) - metalake_list_resp = MetalakeListResponse.from_json(resp.body, infer_missing=True) + metalake_list_resp = MetalakeListResponse.from_json( + resp.body, infer_missing=True + ) metalake_list_resp.validate() - return [GravitinoMetalake(o, self._rest_client) for o in metalake_list_resp.metalakes()] + return [ + GravitinoMetalake(o, self._rest_client) + for o in metalake_list_resp.metalakes() + ] - def create_metalake(self, ident: NameIdentifier, comment: str, properties: Dict[str, str]) -> GravitinoMetalake: + def create_metalake( + self, ident: NameIdentifier, comment: str, properties: Dict[str, str] + ) -> GravitinoMetalake: """Creates a new Metalake using the Gravitino API. Args: @@ -63,7 +72,9 @@ def create_metalake(self, ident: NameIdentifier, comment: str, properties: Dict[ return GravitinoMetalake(metalake, self._rest_client) - def alter_metalake(self, ident: NameIdentifier, *changes: MetalakeChange) -> GravitinoMetalake: + def alter_metalake( + self, ident: NameIdentifier, *changes: MetalakeChange + ) -> GravitinoMetalake: """Alters a specific Metalake using the Gravitino API. Args: @@ -81,7 +92,9 @@ def alter_metalake(self, ident: NameIdentifier, *changes: MetalakeChange) -> Gra updates_request = MetalakeUpdatesRequest(reqs) updates_request.validate() - resp = self._rest_client.put(self.API_METALAKES_IDENTIFIER_PATH + ident.name(), updates_request) + resp = self._rest_client.put( + self.API_METALAKES_IDENTIFIER_PATH + ident.name(), updates_request + ) metalake_response = MetalakeResponse.from_json(resp.body, infer_missing=True) metalake_response.validate() metalake = metalake_response.metalake() @@ -100,7 +113,9 @@ def drop_metalake(self, ident: NameIdentifier) -> bool: NameIdentifier.check_metalake(ident) try: - resp = self._rest_client.delete(self.API_METALAKES_IDENTIFIER_PATH + ident.name()) + resp = self._rest_client.delete( + self.API_METALAKES_IDENTIFIER_PATH + ident.name() + ) dropResponse = DropResponse.from_json(resp.body, infer_missing=True) return dropResponse.dropped() diff --git a/clients/client-python/gravitino/client/gravitino_client.py b/clients/client-python/gravitino/client/gravitino_client.py index b022a1da974..3498358299b 100644 --- a/clients/client-python/gravitino/client/gravitino_client.py +++ b/clients/client-python/gravitino/client/gravitino_client.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + from typing import List, Dict from gravitino.api.catalog import Catalog @@ -30,6 +31,7 @@ class GravitinoClient(GravitinoClientBase): It uses an underlying {@link RESTClient} to send HTTP requests and receive responses from the API. """ + _metalake: GravitinoMetalake def __init__(self, uri: str, metalake_name: str): @@ -66,8 +68,17 @@ def list_catalogs_info(self, namespace: Namespace) -> List[Catalog]: def load_catalog(self, ident: NameIdentifier) -> Catalog: return self.get_metalake().load_catalog(ident) - def create_catalog(self, ident: NameIdentifier, type: Catalog.Type, provider: str, comment: str, properties: Dict[str, str]) -> Catalog: - return self.get_metalake().create_catalog(ident, type, provider, comment, properties) + def create_catalog( + self, + ident: NameIdentifier, + type: Catalog.Type, + provider: str, + comment: str, + properties: Dict[str, str], + ) -> Catalog: + return self.get_metalake().create_catalog( + ident, type, provider, comment, properties + ) def alter_catalog(self, ident: NameIdentifier, *changes: CatalogChange): return self.get_metalake().alter_catalog(ident, *changes) diff --git a/clients/client-python/gravitino/client/gravitino_client_base.py b/clients/client-python/gravitino/client/gravitino_client_base.py index affeec08ee7..cf17d1e6c39 100644 --- a/clients/client-python/gravitino/client/gravitino_client_base.py +++ b/clients/client-python/gravitino/client/gravitino_client_base.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + import logging from gravitino.client.gravitino_metalake import GravitinoMetalake @@ -18,13 +19,13 @@ class GravitinoClientBase: Base class for Gravitino Java client; It uses an underlying {@link RESTClient} to send HTTP requests and receive responses from the API. """ + _rest_client: HTTPClient """The REST client to communicate with the REST server""" API_METALAKES_LIST_PATH = "api/metalakes" """The REST API path for listing metalakes""" - API_METALAKES_IDENTIFIER_PATH = f"{API_METALAKES_LIST_PATH}/" """The REST API path prefix for load a specific metalake""" @@ -46,8 +47,12 @@ def load_metalake(self, ident: NameIdentifier) -> GravitinoMetalake: NameIdentifier.check_metalake(ident) - response = self._rest_client.get(GravitinoClientBase.API_METALAKES_IDENTIFIER_PATH + ident.name()) - metalake_response = MetalakeResponse.from_json(response.body, infer_missing=True) + response = self._rest_client.get( + GravitinoClientBase.API_METALAKES_IDENTIFIER_PATH + ident.name() + ) + metalake_response = MetalakeResponse.from_json( + response.body, infer_missing=True + ) metalake_response.validate() return GravitinoMetalake(metalake_response.metalake(), self._rest_client) diff --git a/clients/client-python/gravitino/client/gravitino_metalake.py b/clients/client-python/gravitino/client/gravitino_metalake.py index 95fdf41713e..b0ea4c8b79c 100644 --- a/clients/client-python/gravitino/client/gravitino_metalake.py +++ b/clients/client-python/gravitino/client/gravitino_metalake.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + import logging from typing import List, Dict @@ -47,8 +48,12 @@ class GravitinoMetalake(MetalakeDTO): API_METALAKES_CATALOGS_PATH = "api/metalakes/{}/catalogs/{}" def __init__(self, metalake: MetalakeDTO = None, client: HTTPClient = None): - super().__init__(_name=metalake.name(), _comment=metalake.comment(), _properties=metalake.properties(), - _audit=metalake.audit_info()) + super().__init__( + _name=metalake.name(), + _comment=metalake.comment(), + _properties=metalake.properties(), + _audit=metalake.audit_info(), + ) self.rest_client = client def list_catalogs(self, namespace: Namespace) -> List[NameIdentifier]: @@ -88,7 +93,10 @@ def list_catalogs_info(self, namespace: Namespace) -> List[Catalog]: response = self.rest_client.get(url, params=params) catalog_list = CatalogListResponse.from_json(response.body, infer_missing=True) - return [DTOConverters.to_catalog(catalog, self.rest_client) for catalog in catalog_list.catalogs()] + return [ + DTOConverters.to_catalog(catalog, self.rest_client) + for catalog in catalog_list.catalogs() + ] def load_catalog(self, ident: NameIdentifier) -> Catalog: """Load the catalog with specified identifier. @@ -103,17 +111,22 @@ def load_catalog(self, ident: NameIdentifier) -> Catalog: The Catalog with specified identifier. """ NameIdentifier.check_catalog(ident) - url = self.API_METALAKES_CATALOGS_PATH.format(ident.namespace().level(0), ident.name()) + url = self.API_METALAKES_CATALOGS_PATH.format( + ident.namespace().level(0), ident.name() + ) response = self.rest_client.get(url) catalog_resp = CatalogResponse.from_json(response.body, infer_missing=True) return DTOConverters.to_catalog(catalog_resp.catalog(), self.rest_client) - def create_catalog(self, ident: NameIdentifier, - type: Catalog.Type, - provider: str, - comment: str, - properties: Dict[str, str]) -> Catalog: + def create_catalog( + self, + ident: NameIdentifier, + type: Catalog.Type, + provider: str, + comment: str, + properties: Dict[str, str], + ) -> Catalog: """Create a new catalog with specified identifier, type, comment and properties. Args: @@ -132,11 +145,13 @@ def create_catalog(self, ident: NameIdentifier, """ NameIdentifier.check_catalog(ident) - catalog_create_request = CatalogCreateRequest(name=ident.name(), - type=type, - provider=provider, - comment=comment, - properties=properties) + catalog_create_request = CatalogCreateRequest( + name=ident.name(), + type=type, + provider=provider, + comment=comment, + properties=properties, + ) catalog_create_request.validate() url = f"api/metalakes/{ident.namespace().level(0)}/catalogs" @@ -165,7 +180,9 @@ def alter_catalog(self, ident: NameIdentifier, *changes: CatalogChange) -> Catal updates_request = CatalogUpdatesRequest(reqs) updates_request.validate() - url = self.API_METALAKES_CATALOGS_PATH.format(ident.namespace().level(0), ident.name()) + url = self.API_METALAKES_CATALOGS_PATH.format( + ident.namespace().level(0), ident.name() + ) response = self.rest_client.put(url, json=updates_request) catalog_response = CatalogResponse.from_json(response.body, infer_missing=True) catalog_response.validate() @@ -177,12 +194,14 @@ def drop_catalog(self, ident: NameIdentifier) -> bool: Args: ident the identifier of the catalog. - + Returns: true if the catalog is dropped successfully, false otherwise. """ try: - url = self.API_METALAKES_CATALOGS_PATH.format(ident.namespace().level(0), ident.name()) + url = self.API_METALAKES_CATALOGS_PATH.format( + ident.namespace().level(0), ident.name() + ) response = self.rest_client.delete(url) drop_response = DropResponse.from_json(response.body, infer_missing=True) diff --git a/clients/client-python/gravitino/client/gravitino_version.py b/clients/client-python/gravitino/client/gravitino_version.py index 9ae39c3e708..f3f004bbf60 100644 --- a/clients/client-python/gravitino/client/gravitino_version.py +++ b/clients/client-python/gravitino/client/gravitino_version.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + from dataclasses import dataclass from gravitino.dto.version_dto import VersionDTO @@ -10,5 +11,8 @@ @dataclass class GravitinoVersion(VersionDTO): """Gravitino version information.""" + def __init__(self, versionDTO): - super().__init__(versionDTO.version, versionDTO.compile_date, versionDTO.git_commit) + super().__init__( + versionDTO.version, versionDTO.compile_date, versionDTO.git_commit + ) diff --git a/clients/client-python/gravitino/dto/audit_dto.py b/clients/client-python/gravitino/dto/audit_dto.py index 0a05903f0fd..6ce56ca892e 100644 --- a/clients/client-python/gravitino/dto/audit_dto.py +++ b/clients/client-python/gravitino/dto/audit_dto.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + from dataclasses import dataclass, field from typing import Optional @@ -14,18 +15,22 @@ class AuditDTO(Audit, DataClassJsonMixin): """Data transfer object representing audit information.""" - _creator: Optional[str] = field(default=None, metadata=config(field_name='creator')) + _creator: Optional[str] = field(default=None, metadata=config(field_name="creator")) """The creator of the audit.""" - _create_time: Optional[str] = field(default=None, metadata=config( - field_name='createTime')) # TODO: Can't deserialized datetime from JSON + _create_time: Optional[str] = field( + default=None, metadata=config(field_name="createTime") + ) # TODO: Can't deserialized datetime from JSON """The create time of the audit.""" - _last_modifier: Optional[str] = field(default=None, metadata=config(field_name='lastModifier')) + _last_modifier: Optional[str] = field( + default=None, metadata=config(field_name="lastModifier") + ) """The last modifier of the audit.""" - _last_modified_time: Optional[str] = field(default=None, metadata=config( - field_name='lastModifiedTime')) # TODO: Can't deserialized datetime from JSON + _last_modified_time: Optional[str] = field( + default=None, metadata=config(field_name="lastModifiedTime") + ) # TODO: Can't deserialized datetime from JSON """The last modified time of the audit.""" def creator(self) -> str: diff --git a/clients/client-python/gravitino/dto/catalog_dto.py b/clients/client-python/gravitino/dto/catalog_dto.py index 012c224dfae..144d16400df 100644 --- a/clients/client-python/gravitino/dto/catalog_dto.py +++ b/clients/client-python/gravitino/dto/catalog_dto.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + from typing import Dict from dataclasses import dataclass, field @@ -15,16 +16,22 @@ class CatalogDTO(Catalog): """Data transfer object representing catalog information.""" - _name: str = field(metadata=config(field_name='name')) - _type: Catalog.Type = field(metadata=config(field_name='type')) - _provider: str = field(metadata=config(field_name='provider')) - _comment: str = field(metadata=config(field_name='comment')) - _properties: Dict[str, str] = field(metadata=config(field_name='properties')) - _audit: AuditDTO = field(default=None, metadata=config(field_name='audit')) - - def builder(self, name: str = None, type: Catalog.Type = Catalog.Type.UNSUPPORTED, - provider: str = None, comment: str = None, properties: Dict[str, str] = None, - audit: AuditDTO = None): + _name: str = field(metadata=config(field_name="name")) + _type: Catalog.Type = field(metadata=config(field_name="type")) + _provider: str = field(metadata=config(field_name="provider")) + _comment: str = field(metadata=config(field_name="comment")) + _properties: Dict[str, str] = field(metadata=config(field_name="properties")) + _audit: AuditDTO = field(default=None, metadata=config(field_name="audit")) + + def builder( + self, + name: str = None, + type: Catalog.Type = Catalog.Type.UNSUPPORTED, + provider: str = None, + comment: str = None, + properties: Dict[str, str] = None, + audit: AuditDTO = None, + ): self._name = name self._type = type self._provider = provider diff --git a/clients/client-python/gravitino/dto/dto_converters.py b/clients/client-python/gravitino/dto/dto_converters.py index 801f9b07d5d..82d14ab334d 100644 --- a/clients/client-python/gravitino/dto/dto_converters.py +++ b/clients/client-python/gravitino/dto/dto_converters.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + from gravitino.api.catalog import Catalog from gravitino.api.catalog_change import CatalogChange from gravitino.catalog.fileset_catalog import FilesetCatalog @@ -21,25 +22,33 @@ def to_metalake_update_request(change: MetalakeChange) -> object: if isinstance(change, MetalakeChange.RenameMetalake): return MetalakeUpdateRequest.RenameMetalakeRequest(change.new_name()) if isinstance(change, MetalakeChange.UpdateMetalakeComment): - return MetalakeUpdateRequest.UpdateMetalakeCommentRequest(change.new_comment()) + return MetalakeUpdateRequest.UpdateMetalakeCommentRequest( + change.new_comment() + ) if isinstance(change, MetalakeChange.SetProperty): - return MetalakeUpdateRequest.SetMetalakePropertyRequest(change.property(), change.value()) + return MetalakeUpdateRequest.SetMetalakePropertyRequest( + change.property(), change.value() + ) if isinstance(change, MetalakeChange.RemoveProperty): - return MetalakeUpdateRequest.RemoveMetalakePropertyRequest(change.property()) - + return MetalakeUpdateRequest.RemoveMetalakePropertyRequest( + change.property() + ) + raise ValueError(f"Unknown change type: {type(change).__name__}") @staticmethod def to_catalog(catalog: CatalogDTO, client: HTTPClient): if catalog.type() == Catalog.Type.FILESET: - return FilesetCatalog(name=catalog.name(), - type=catalog.type(), - provider=catalog.provider(), - comment=catalog.comment(), - properties=catalog.properties(), - audit=catalog.audit_info(), - rest_client=client) - + return FilesetCatalog( + name=catalog.name(), + type=catalog.type(), + provider=catalog.provider(), + comment=catalog.comment(), + properties=catalog.properties(), + audit=catalog.audit_info(), + rest_client=client, + ) + raise NotImplementedError("Unsupported catalog type: " + str(catalog.type())) @staticmethod @@ -50,9 +59,11 @@ def to_catalog_update_request(change: CatalogChange): return CatalogUpdateRequest.UpdateCatalogCommentRequest(change.new_comment) if isinstance(change, CatalogChange.SetProperty): # TODO - # pylint: disable=too-many-function-args - return CatalogUpdateRequest.SetCatalogPropertyRequest(change.property(), change.value()) + # pylint: disable=too-many-function-args + return CatalogUpdateRequest.SetCatalogPropertyRequest( + change.property(), change.value() + ) if isinstance(change, CatalogChange.RemoveProperty): return CatalogUpdateRequest.RemoveCatalogPropertyRequest(change._property) - + raise ValueError(f"Unknown change type: {type(change).__name__}") diff --git a/clients/client-python/gravitino/dto/fileset_dto.py b/clients/client-python/gravitino/dto/fileset_dto.py index 109443e63bb..2ea41495221 100644 --- a/clients/client-python/gravitino/dto/fileset_dto.py +++ b/clients/client-python/gravitino/dto/fileset_dto.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + from dataclasses import dataclass, field from typing import Optional, Dict @@ -15,12 +16,14 @@ class FilesetDTO(Fileset, DataClassJsonMixin): """Represents a Fileset DTO (Data Transfer Object).""" - _name: str = field(metadata=config(field_name='name')) - _comment: Optional[str] = field(metadata=config(field_name='comment')) - _type: Fileset.Type = field(metadata=config(field_name='type')) - _properties: Dict[str, str] = field(metadata=config(field_name='properties')) - _storage_location: str = field(default=None, metadata=config(field_name='storageLocation')) - _audit: AuditDTO = field(default=None, metadata=config(field_name='audit')) + _name: str = field(metadata=config(field_name="name")) + _comment: Optional[str] = field(metadata=config(field_name="comment")) + _type: Fileset.Type = field(metadata=config(field_name="type")) + _properties: Dict[str, str] = field(metadata=config(field_name="properties")) + _storage_location: str = field( + default=None, metadata=config(field_name="storageLocation") + ) + _audit: AuditDTO = field(default=None, metadata=config(field_name="audit")) def name(self) -> str: return self._name diff --git a/clients/client-python/gravitino/dto/metalake_dto.py b/clients/client-python/gravitino/dto/metalake_dto.py index 8fc08116d20..dfdb8573ae2 100644 --- a/clients/client-python/gravitino/dto/metalake_dto.py +++ b/clients/client-python/gravitino/dto/metalake_dto.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + from dataclasses import dataclass, field from typing import Optional, Dict @@ -16,16 +17,18 @@ class MetalakeDTO(Metalake, DataClassJsonMixin): """Represents a Metalake Data Transfer Object (DTO) that implements the Metalake interface.""" - _name: str = field(metadata=config(field_name='name')) + _name: str = field(metadata=config(field_name="name")) """The name of the Metalake DTO.""" - _comment: Optional[str] = field(metadata=config(field_name='comment')) + _comment: Optional[str] = field(metadata=config(field_name="comment")) """The comment of the Metalake DTO.""" - _properties: Optional[Dict[str, str]] = field(metadata=config(field_name='properties')) + _properties: Optional[Dict[str, str]] = field( + metadata=config(field_name="properties") + ) """The properties of the Metalake DTO.""" - _audit: Optional[AuditDTO] = field(metadata=config(field_name='audit')) + _audit: Optional[AuditDTO] = field(metadata=config(field_name="audit")) """The audit information of the Metalake DTO.""" def name(self) -> str: @@ -45,8 +48,12 @@ def equals(self, other): return True if not isinstance(other, MetalakeDTO): return False - return self._name == other._name and self._comment == other._comment and \ - self.property_equal(self._properties, other._properties) and self._audit == other._audit + return ( + self._name == other._name + and self._comment == other._comment + and self.property_equal(self._properties, other._properties) + and self._audit == other._audit + ) def property_equal(self, p1, p2): if p1 is None and p2 is None: diff --git a/clients/client-python/gravitino/dto/requests/catalog_create_request.py b/clients/client-python/gravitino/dto/requests/catalog_create_request.py index ef9a35fd4b5..0141a493690 100644 --- a/clients/client-python/gravitino/dto/requests/catalog_create_request.py +++ b/clients/client-python/gravitino/dto/requests/catalog_create_request.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + from dataclasses import dataclass, field from typing import Optional, Dict @@ -14,14 +15,23 @@ @dataclass class CatalogCreateRequest(RESTRequest): """Represents a request to create a catalog.""" - _name: str = field(metadata=config(field_name='name')) - _type: Catalog.Type = field(metadata=config(field_name='type')) - _provider: str = field(metadata=config(field_name='provider')) - _comment: Optional[str] = field(metadata=config(field_name='comment')) - _properties: Optional[Dict[str, str]] = field(metadata=config(field_name='properties')) - - def __init__(self, name: str = None, type: Catalog.Type = Catalog.Type.UNSUPPORTED, provider: str = None, - comment: str = None, properties: Dict[str, str] = None): + + _name: str = field(metadata=config(field_name="name")) + _type: Catalog.Type = field(metadata=config(field_name="type")) + _provider: str = field(metadata=config(field_name="provider")) + _comment: Optional[str] = field(metadata=config(field_name="comment")) + _properties: Optional[Dict[str, str]] = field( + metadata=config(field_name="properties") + ) + + def __init__( + self, + name: str = None, + type: Catalog.Type = Catalog.Type.UNSUPPORTED, + provider: str = None, + comment: str = None, + properties: Dict[str, str] = None, + ): self._name = name self._type = type self._provider = provider @@ -34,6 +44,8 @@ def validate(self): Raises: IllegalArgumentException if name or type are not set. """ - assert self._name is not None, "\"name\" field is required and cannot be empty" - assert self._type is not None, "\"type\" field is required and cannot be empty" - assert self._provider is not None, "\"provider\" field is required and cannot be empty" + assert self._name is not None, '"name" field is required and cannot be empty' + assert self._type is not None, '"type" field is required and cannot be empty' + assert ( + self._provider is not None + ), '"provider" field is required and cannot be empty' diff --git a/clients/client-python/gravitino/dto/requests/catalog_update_request.py b/clients/client-python/gravitino/dto/requests/catalog_update_request.py index e0edb652081..405d58cc578 100644 --- a/clients/client-python/gravitino/dto/requests/catalog_update_request.py +++ b/clients/client-python/gravitino/dto/requests/catalog_update_request.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + from abc import abstractmethod from dataclasses import field, dataclass from typing import Optional @@ -14,7 +15,7 @@ @dataclass class CatalogUpdateRequestBase(RESTRequest): - _type: str = field(metadata=config(field_name='@type')) + _type: str = field(metadata=config(field_name="@type")) def __init__(self, type: str): self._type = type @@ -28,7 +29,7 @@ class CatalogUpdateRequest: """Represents an interface for catalog update requests.""" class RenameCatalogRequest(CatalogUpdateRequestBase): - new_name: Optional[str] = field(metadata=config(field_name='newName')) + new_name: Optional[str] = field(metadata=config(field_name="newName")) def catalog_change(self): return CatalogChange.rename(self.new_name) @@ -39,21 +40,26 @@ def validate(self): Raises: IllegalArgumentException if the new name is not set. """ - assert self.new_name is None, '"newName" field is required and cannot be empty' + assert ( + self.new_name is None + ), '"newName" field is required and cannot be empty' class UpdateCatalogCommentRequest(CatalogUpdateRequestBase): """Request to update the comment of a catalog.""" - new_comment: Optional[str] = field(metadata=config(field_name='newComment')) + new_comment: Optional[str] = field(metadata=config(field_name="newComment")) def catalog_change(self): return CatalogChange.update_comment(self.new_comment) def validate(self): - assert self.new_comment is None, '"newComment" field is required and cannot be empty' + assert ( + self.new_comment is None + ), '"newComment" field is required and cannot be empty' class SetCatalogPropertyRequest(CatalogUpdateRequestBase): """Request to set a property on a catalog.""" + property: Optional[str] = None value: Optional[str] = None @@ -61,15 +67,20 @@ def catalog_change(self): return CatalogChange.set_property(self.property, self.value) def validate(self): - assert self.property is None, "\"property\" field is required and cannot be empty" - assert self.value is None, "\"value\" field is required and cannot be empty" + assert ( + self.property is None + ), '"property" field is required and cannot be empty' + assert self.value is None, '"value" field is required and cannot be empty' class RemoveCatalogPropertyRequest(CatalogUpdateRequestBase): """Request to remove a property from a catalog.""" + property: Optional[str] = None def catalog_change(self): return CatalogChange.remove_property(self.property) def validate(self): - assert self.property is None, "\"property\" field is required and cannot be empty" + assert ( + self.property is None + ), '"property" field is required and cannot be empty' diff --git a/clients/client-python/gravitino/dto/requests/catalog_updates_request.py b/clients/client-python/gravitino/dto/requests/catalog_updates_request.py index bbaab25352d..8437aa070a2 100644 --- a/clients/client-python/gravitino/dto/requests/catalog_updates_request.py +++ b/clients/client-python/gravitino/dto/requests/catalog_updates_request.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + from dataclasses import dataclass, field from typing import Optional, List @@ -14,7 +15,10 @@ @dataclass class CatalogUpdatesRequest(RESTRequest): """Represents a request containing multiple catalog updates.""" - _updates: Optional[List[CatalogUpdateRequest]] = field(metadata=config(field_name='updates'), default_factory=list) + + _updates: Optional[List[CatalogUpdateRequest]] = field( + metadata=config(field_name="updates"), default_factory=list + ) def __init__(self, updates: List[CatalogUpdateRequest] = None): self._updates = updates diff --git a/clients/client-python/gravitino/dto/requests/fileset_create_request.py b/clients/client-python/gravitino/dto/requests/fileset_create_request.py index 0111f87b2e5..5208308b649 100644 --- a/clients/client-python/gravitino/dto/requests/fileset_create_request.py +++ b/clients/client-python/gravitino/dto/requests/fileset_create_request.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + from dataclasses import dataclass, field from typing import Optional, Dict @@ -14,14 +15,25 @@ @dataclass class FilesetCreateRequest(RESTRequest): """Represents a request to create a fileset.""" - _name: str = field(metadata=config(field_name='name')) - _comment: Optional[str] = field(metadata=config(field_name='comment')) - _type: Optional[Fileset.Type] = field(metadata=config(field_name='type')) - _storage_location: Optional[str] = field(metadata=config(field_name='storageLocation')) - _properties: Optional[Dict[str, str]] = field(metadata=config(field_name='properties')) - - def __init__(self, name: str, comment: Optional[str] = None, type: Fileset.Type = None, - storage_location: Optional[str] = None, properties: Optional[Dict[str, str]] = None): + + _name: str = field(metadata=config(field_name="name")) + _comment: Optional[str] = field(metadata=config(field_name="comment")) + _type: Optional[Fileset.Type] = field(metadata=config(field_name="type")) + _storage_location: Optional[str] = field( + metadata=config(field_name="storageLocation") + ) + _properties: Optional[Dict[str, str]] = field( + metadata=config(field_name="properties") + ) + + def __init__( + self, + name: str, + comment: Optional[str] = None, + type: Fileset.Type = None, + storage_location: Optional[str] = None, + properties: Optional[Dict[str, str]] = None, + ): self._name = name self._comment = comment self._type = type diff --git a/clients/client-python/gravitino/dto/requests/fileset_update_request.py b/clients/client-python/gravitino/dto/requests/fileset_update_request.py index ed4ed40a3b2..1651c736280 100644 --- a/clients/client-python/gravitino/dto/requests/fileset_update_request.py +++ b/clients/client-python/gravitino/dto/requests/fileset_update_request.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + from abc import abstractmethod from dataclasses import dataclass, field @@ -13,7 +14,7 @@ @dataclass class FilesetUpdateRequestBase(RESTRequest): - _type: str = field(metadata=config(field_name='@type')) + _type: str = field(metadata=config(field_name="@type")) def __init__(self, type: str): self._type = type @@ -30,10 +31,9 @@ class FilesetUpdateRequest: class RenameFilesetRequest(FilesetUpdateRequestBase): """The fileset update request for renaming a fileset.""" - _new_name: str = field(metadata=config(field_name='newName')) + _new_name: str = field(metadata=config(field_name="newName")) """The new name for the Fileset.""" - def __init__(self, new_name: str): super().__init__("rename") self._new_name = new_name @@ -59,7 +59,7 @@ def fileset_change(self): class UpdateFilesetCommentRequest(FilesetUpdateRequestBase): """Represents a request to update the comment on a Fileset.""" - _new_comment: str = field(metadata=config(field_name='newComment')) + _new_comment: str = field(metadata=config(field_name="newComment")) """The new comment for the Fileset.""" def __init__(self, new_comment: str): @@ -83,10 +83,10 @@ def fileset_change(self): class SetFilesetPropertyRequest(FilesetUpdateRequestBase): """Represents a request to set a property on a Fileset.""" - _property: str = field(metadata=config(field_name='property')) + _property: str = field(metadata=config(field_name="property")) """The property to set.""" - _value: str = field(metadata=config(field_name='value')) + _value: str = field(metadata=config(field_name="value")) """The value of the property.""" def __init__(self, property: str, value: str): @@ -112,7 +112,7 @@ def fileset_change(self): class RemoveFilesetPropertyRequest(FilesetUpdateRequestBase): """Represents a request to remove a property from a Fileset.""" - _property: str = field(metadata=config(field_name='property')) + _property: str = field(metadata=config(field_name="property")) """The property to remove.""" def __init__(self, property: str): diff --git a/clients/client-python/gravitino/dto/requests/fileset_updates_request.py b/clients/client-python/gravitino/dto/requests/fileset_updates_request.py index 14c1766dc51..4f3c02b86b7 100644 --- a/clients/client-python/gravitino/dto/requests/fileset_updates_request.py +++ b/clients/client-python/gravitino/dto/requests/fileset_updates_request.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + from dataclasses import dataclass, field from typing import List @@ -14,10 +15,13 @@ @dataclass class FilesetUpdatesRequest(RESTRequest): """Request to represent updates to a fileset.""" - _updates: List[FilesetUpdateRequest] = field(metadata=config(field_name='updates'), default_factory=list) + + _updates: List[FilesetUpdateRequest] = field( + metadata=config(field_name="updates"), default_factory=list + ) def validate(self): if not self._updates: raise ValueError("Updates cannot be empty") for update_request in self._updates: - update_request.validate() \ No newline at end of file + update_request.validate() diff --git a/clients/client-python/gravitino/dto/requests/metalake_create_request.py b/clients/client-python/gravitino/dto/requests/metalake_create_request.py index 4d89c57b7d5..e1d49e3b8c1 100644 --- a/clients/client-python/gravitino/dto/requests/metalake_create_request.py +++ b/clients/client-python/gravitino/dto/requests/metalake_create_request.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + from typing import Optional, Dict from dataclasses import dataclass, field @@ -13,13 +14,17 @@ @dataclass class MetalakeCreateRequest(RESTRequest): - """"Represents a request to create a Metalake.""" + """Represents a request to create a Metalake.""" - _name: str = field(metadata=config(field_name='name')) - _comment: Optional[str] = field(metadata=config(field_name='comment')) - _properties: Optional[Dict[str, str]] = field(metadata=config(field_name='properties')) + _name: str = field(metadata=config(field_name="name")) + _comment: Optional[str] = field(metadata=config(field_name="comment")) + _properties: Optional[Dict[str, str]] = field( + metadata=config(field_name="properties") + ) - def __init__(self, name: str = None, comment: str = None, properties: Dict[str, str] = None): + def __init__( + self, name: str = None, comment: str = None, properties: Dict[str, str] = None + ): super().__init__() self._name = name.strip() if name else None @@ -31,4 +36,4 @@ def name(self) -> str: def validate(self): if not self._name: - raise ValueError("\"name\" field is required and cannot be empty") + raise ValueError('"name" field is required and cannot be empty') diff --git a/clients/client-python/gravitino/dto/requests/metalake_update_request.py b/clients/client-python/gravitino/dto/requests/metalake_update_request.py index 1261b988c7e..b207e6b21ff 100644 --- a/clients/client-python/gravitino/dto/requests/metalake_update_request.py +++ b/clients/client-python/gravitino/dto/requests/metalake_update_request.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + from abc import abstractmethod from dataclasses import dataclass, field @@ -13,7 +14,7 @@ @dataclass class MetalakeUpdateRequestBase(RESTRequest): - _type: str = field(metadata=config(field_name='@type')) + _type: str = field(metadata=config(field_name="@type")) def __init__(self, type: str): self._type = type @@ -30,7 +31,7 @@ class MetalakeUpdateRequest: class RenameMetalakeRequest(MetalakeUpdateRequestBase): """Represents a request to rename a Metalake.""" - _new_name: str = field(metadata=config(field_name='newName')) + _new_name: str = field(metadata=config(field_name="newName")) """The new name for the Metalake.""" def __init__(self, new_name: str): @@ -53,7 +54,7 @@ def metalake_change(self): class UpdateMetalakeCommentRequest(MetalakeUpdateRequestBase): """Represents a request to update the comment on a Metalake.""" - _new_comment: str = field(metadata=config(field_name='newComment')) + _new_comment: str = field(metadata=config(field_name="newComment")) """The new comment for the Metalake.""" def __init__(self, new_comment: str): @@ -76,10 +77,10 @@ def metalake_change(self): class SetMetalakePropertyRequest(MetalakeUpdateRequestBase): """Represents a request to set a property on a Metalake.""" - _property: str = field(metadata=config(field_name='property')) + _property: str = field(metadata=config(field_name="property")) """The property to set.""" - _value: str = field(metadata=config(field_name='value')) + _value: str = field(metadata=config(field_name="value")) """The value of the property.""" def __init__(self, property: str, value: str): @@ -105,7 +106,7 @@ def metalake_change(self): class RemoveMetalakePropertyRequest(MetalakeUpdateRequestBase): """Represents a request to remove a property from a Metalake.""" - _property: str = field(metadata=config(field_name='property')) + _property: str = field(metadata=config(field_name="property")) """The property to remove.""" def __init__(self, property: str): diff --git a/clients/client-python/gravitino/dto/requests/metalake_updates_request.py b/clients/client-python/gravitino/dto/requests/metalake_updates_request.py index 3e30ce30150..016e1eeaf66 100644 --- a/clients/client-python/gravitino/dto/requests/metalake_updates_request.py +++ b/clients/client-python/gravitino/dto/requests/metalake_updates_request.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + from dataclasses import dataclass, field from typing import List @@ -15,7 +16,7 @@ class MetalakeUpdatesRequest(RESTRequest): """Represents a request containing multiple Metalake updates.""" - _updates: List[MetalakeUpdateRequest] = field(metadata=config(field_name='updates')) + _updates: List[MetalakeUpdateRequest] = field(metadata=config(field_name="updates")) def __init__(self, updates: List[MetalakeUpdateRequest]): """Constructor for MetalakeUpdatesRequest. diff --git a/clients/client-python/gravitino/dto/requests/schema_create_request.py b/clients/client-python/gravitino/dto/requests/schema_create_request.py index abaf78db6ab..0d3154d2975 100644 --- a/clients/client-python/gravitino/dto/requests/schema_create_request.py +++ b/clients/client-python/gravitino/dto/requests/schema_create_request.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + from dataclasses import dataclass, field from typing import Optional, Dict @@ -14,14 +15,18 @@ class SchemaCreateRequest(RESTRequest): """Represents a request to create a schema.""" - _name: str = field(metadata=config(field_name='name')) - _comment: Optional[str] = field(metadata=config(field_name='comment')) - _properties: Optional[Dict[str, str]] = field(metadata=config(field_name='properties')) + _name: str = field(metadata=config(field_name="name")) + _comment: Optional[str] = field(metadata=config(field_name="comment")) + _properties: Optional[Dict[str, str]] = field( + metadata=config(field_name="properties") + ) - def __init__(self, name: str, comment: Optional[str], properties: Optional[Dict[str, str]]): + def __init__( + self, name: str, comment: Optional[str], properties: Optional[Dict[str, str]] + ): self._name = name self._comment = comment self._properties = properties def validate(self): - assert self._name is not None, "\"name\" field is required and cannot be empty" + assert self._name is not None, '"name" field is required and cannot be empty' diff --git a/clients/client-python/gravitino/dto/requests/schema_update_request.py b/clients/client-python/gravitino/dto/requests/schema_update_request.py index 83e4152b9fb..46c9b0cf90e 100644 --- a/clients/client-python/gravitino/dto/requests/schema_update_request.py +++ b/clients/client-python/gravitino/dto/requests/schema_update_request.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + from abc import abstractmethod from dataclasses import dataclass, field @@ -13,7 +14,7 @@ @dataclass class SchemaUpdateRequestBase(RESTRequest): - _type: str = field(metadata=config(field_name='@type')) + _type: str = field(metadata=config(field_name="@type")) def __init__(self, type: str): self._type = type @@ -31,10 +32,10 @@ class SchemaUpdateRequest: class SetSchemaPropertyRequest(SchemaUpdateRequestBase): """Represents a request to set a property on a Schema.""" - _property: str = field(metadata=config(field_name='property')) + _property: str = field(metadata=config(field_name="property")) """The property to set.""" - _value: str = field(metadata=config(field_name='value')) + _value: str = field(metadata=config(field_name="value")) """The value of the property.""" def __init__(self, property: str, value: str): @@ -60,7 +61,7 @@ def schema_change(self): class RemoveSchemaPropertyRequest(SchemaUpdateRequestBase): """Represents a request to remove a property from a Schema.""" - _property: str = field(metadata=config(field_name='property')) + _property: str = field(metadata=config(field_name="property")) """The property to remove.""" def __init__(self, property: str): diff --git a/clients/client-python/gravitino/dto/requests/schema_updates_request.py b/clients/client-python/gravitino/dto/requests/schema_updates_request.py index 27a8e5ff550..05d59c2d4fe 100644 --- a/clients/client-python/gravitino/dto/requests/schema_updates_request.py +++ b/clients/client-python/gravitino/dto/requests/schema_updates_request.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + from dataclasses import dataclass, field from typing import List @@ -14,7 +15,10 @@ @dataclass class SchemaUpdatesRequest(RESTRequest): """Represents a request to update a schema.""" - _updates: List[SchemaUpdateRequest] = field(metadata=config(field_name='updates'), default_factory=list) + + _updates: List[SchemaUpdateRequest] = field( + metadata=config(field_name="updates"), default_factory=list + ) def __init__(self, updates: List[SchemaUpdateRequest]): self._updates = updates @@ -28,4 +32,4 @@ def validate(self): if not self._updates: raise ValueError("Updates cannot be empty") for update_request in self._updates: - update_request.validate() \ No newline at end of file + update_request.validate() diff --git a/clients/client-python/gravitino/dto/responses/base_response.py b/clients/client-python/gravitino/dto/responses/base_response.py index 7c6644d267c..e272c95928e 100644 --- a/clients/client-python/gravitino/dto/responses/base_response.py +++ b/clients/client-python/gravitino/dto/responses/base_response.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + from dataclasses import dataclass, field from dataclasses_json import config @@ -13,7 +14,7 @@ class BaseResponse(RESTResponse): """Represents a base response for REST API calls.""" - _code: int = field(metadata=config(field_name='code')) + _code: int = field(metadata=config(field_name="code")) def code(self) -> int: return self._code diff --git a/clients/client-python/gravitino/dto/responses/catalog_list_response.py b/clients/client-python/gravitino/dto/responses/catalog_list_response.py index c38cc04892d..cb8b2f2580f 100644 --- a/clients/client-python/gravitino/dto/responses/catalog_list_response.py +++ b/clients/client-python/gravitino/dto/responses/catalog_list_response.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + from dataclasses import dataclass, field from typing import List @@ -14,7 +15,8 @@ @dataclass class CatalogListResponse(BaseResponse): """Represents a response for a list of catalogs with their information.""" - _catalogs: List[CatalogDTO] = field(metadata=config(field_name='catalogs')) + + _catalogs: List[CatalogDTO] = field(metadata=config(field_name="catalogs")) def catalogs(self) -> List[CatalogDTO]: return self._catalogs diff --git a/clients/client-python/gravitino/dto/responses/catalog_response.py b/clients/client-python/gravitino/dto/responses/catalog_response.py index 79c286747c3..6aba8d06bc2 100644 --- a/clients/client-python/gravitino/dto/responses/catalog_response.py +++ b/clients/client-python/gravitino/dto/responses/catalog_response.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + from dataclasses import dataclass, field from dataclasses_json import config @@ -13,7 +14,8 @@ @dataclass class CatalogResponse(BaseResponse): """Represents a response containing catalog information.""" - _catalog: CatalogDTO = field(metadata=config(field_name='catalog')) + + _catalog: CatalogDTO = field(metadata=config(field_name="catalog")) def validate(self): """Validates the response data. @@ -24,7 +26,9 @@ def validate(self): super().validate() assert self.catalog is not None, "catalog must not be null" - assert self.catalog.name() is not None, "catalog 'name' must not be null and empty" + assert ( + self.catalog.name() is not None + ), "catalog 'name' must not be null and empty" assert self.catalog.type() is not None, "catalog 'type' must not be null" assert self.catalog.audit_info() is not None, "catalog 'audit' must not be null" diff --git a/clients/client-python/gravitino/dto/responses/drop_response.py b/clients/client-python/gravitino/dto/responses/drop_response.py index fb2c548b2f0..8515135a600 100644 --- a/clients/client-python/gravitino/dto/responses/drop_response.py +++ b/clients/client-python/gravitino/dto/responses/drop_response.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + from dataclasses import dataclass, field from dataclasses_json import config @@ -13,7 +14,7 @@ class DropResponse(BaseResponse): """Represents a response for a drop operation.""" - _dropped: bool = field(metadata=config(field_name='dropped')) + _dropped: bool = field(metadata=config(field_name="dropped")) def dropped(self) -> bool: return self._dropped diff --git a/clients/client-python/gravitino/dto/responses/entity_list_response.py b/clients/client-python/gravitino/dto/responses/entity_list_response.py index 657df59e0a9..dc3cd71486e 100644 --- a/clients/client-python/gravitino/dto/responses/entity_list_response.py +++ b/clients/client-python/gravitino/dto/responses/entity_list_response.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + from dataclasses import dataclass, field from typing import List @@ -14,7 +15,8 @@ @dataclass class EntityListResponse(BaseResponse): """Represents a response containing a list of catalogs.""" - _idents: List[NameIdentifier] = field(metadata=config(field_name='identifiers')) + + _idents: List[NameIdentifier] = field(metadata=config(field_name="identifiers")) def identifiers(self) -> List[NameIdentifier]: return self._idents @@ -27,4 +29,4 @@ def validate(self): """ super().validate() - assert self._idents is not None, "identifiers must not be null" \ No newline at end of file + assert self._idents is not None, "identifiers must not be null" diff --git a/clients/client-python/gravitino/dto/responses/fileset_response.py b/clients/client-python/gravitino/dto/responses/fileset_response.py index 502d9f24852..56a5fbec62f 100644 --- a/clients/client-python/gravitino/dto/responses/fileset_response.py +++ b/clients/client-python/gravitino/dto/responses/fileset_response.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + from dataclasses import dataclass, field from dataclasses_json import config @@ -13,7 +14,8 @@ @dataclass class FilesetResponse(BaseResponse): """Response for fileset creation.""" - _fileset: FilesetDTO = field(metadata=config(field_name='fileset')) + + _fileset: FilesetDTO = field(metadata=config(field_name="fileset")) def fileset(self) -> FilesetDTO: return self._fileset @@ -27,5 +29,9 @@ def validate(self): super().validate() assert self._fileset is not None, "fileset must not be null" assert self._fileset.name, "fileset 'name' must not be null and empty" - assert self._fileset.storage_location, "fileset 'storageLocation' must not be null and empty" - assert self._fileset.type is not None, "fileset 'type' must not be null and empty" + assert ( + self._fileset.storage_location + ), "fileset 'storageLocation' must not be null and empty" + assert ( + self._fileset.type is not None + ), "fileset 'type' must not be null and empty" diff --git a/clients/client-python/gravitino/dto/responses/metalake_list_response.py b/clients/client-python/gravitino/dto/responses/metalake_list_response.py index b8679a3d521..182b2d96d4b 100644 --- a/clients/client-python/gravitino/dto/responses/metalake_list_response.py +++ b/clients/client-python/gravitino/dto/responses/metalake_list_response.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + from dataclasses import dataclass, field from typing import List @@ -15,7 +16,7 @@ class MetalakeListResponse(BaseResponse): """Represents a response containing a list of metalakes.""" - _metalakes: List[MetalakeDTO] = field(metadata=config(field_name='metalakes')) + _metalakes: List[MetalakeDTO] = field(metadata=config(field_name="metalakes")) def metalakes(self) -> List[MetalakeDTO]: return self._metalakes diff --git a/clients/client-python/gravitino/dto/responses/metalake_response.py b/clients/client-python/gravitino/dto/responses/metalake_response.py index c019029c9a3..e7d36231596 100644 --- a/clients/client-python/gravitino/dto/responses/metalake_response.py +++ b/clients/client-python/gravitino/dto/responses/metalake_response.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + from dataclasses import dataclass, field from typing import Optional @@ -15,7 +16,7 @@ class MetalakeResponse(BaseResponse): """Represents a response containing metalake information.""" - _metalake: Optional[MetalakeDTO] = field(metadata=config(field_name='metalake')) + _metalake: Optional[MetalakeDTO] = field(metadata=config(field_name="metalake")) def metalake(self) -> MetalakeDTO: return self._metalake @@ -27,5 +28,9 @@ def validate(self): super().validate() assert self._metalake is not None, "metalake must not be null" - assert self._metalake.name() is not None, "metalake 'name' must not be null and empty" - assert self._metalake.audit_info() is not None, "metalake 'audit' must not be null" + assert ( + self._metalake.name() is not None + ), "metalake 'name' must not be null and empty" + assert ( + self._metalake.audit_info() is not None + ), "metalake 'audit' must not be null" diff --git a/clients/client-python/gravitino/dto/responses/schema_response.py b/clients/client-python/gravitino/dto/responses/schema_response.py index 501b34e2390..1a630eea6e4 100644 --- a/clients/client-python/gravitino/dto/responses/schema_response.py +++ b/clients/client-python/gravitino/dto/responses/schema_response.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + from dataclasses import dataclass, field from dataclasses_json import DataClassJsonMixin, config @@ -13,10 +14,11 @@ @dataclass class SchemaResponse(BaseResponse, DataClassJsonMixin): """Represents a response for a schema.""" - _schema: SchemaDTO = field(metadata=config(field_name='schema')) - + + _schema: SchemaDTO = field(metadata=config(field_name="schema")) + # TODO - # pylint: disable=arguments-differ + # pylint: disable=arguments-differ def schema(self) -> SchemaDTO: return self._schema @@ -29,5 +31,7 @@ def validate(self): super().validate() assert self._schema is not None, "schema must be non-null" - assert self._schema.name() is not None, "schema 'name' must not be null and empty" + assert ( + self._schema.name() is not None + ), "schema 'name' must not be null and empty" assert self._schema.audit_info() is not None, "schema 'audit' must not be null" diff --git a/clients/client-python/gravitino/dto/schema_dto.py b/clients/client-python/gravitino/dto/schema_dto.py index 04efe536a8a..12739a60183 100644 --- a/clients/client-python/gravitino/dto/schema_dto.py +++ b/clients/client-python/gravitino/dto/schema_dto.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + from dataclasses import dataclass, field from typing import Optional, Dict @@ -16,16 +17,18 @@ class SchemaDTO(Schema): """Represents a Schema DTO (Data Transfer Object).""" - _name: str = field(metadata=config(field_name='name')) + _name: str = field(metadata=config(field_name="name")) """The name of the Metalake DTO.""" - _comment: Optional[str] = field(metadata=config(field_name='comment')) + _comment: Optional[str] = field(metadata=config(field_name="comment")) """The comment of the Metalake DTO.""" - _properties: Optional[Dict[str, str]] = field(metadata=config(field_name='properties')) + _properties: Optional[Dict[str, str]] = field( + metadata=config(field_name="properties") + ) """The properties of the Metalake DTO.""" - _audit: AuditDTO = field(metadata=config(field_name='audit')) + _audit: AuditDTO = field(metadata=config(field_name="audit")) """The audit information of the Metalake DTO.""" def name(self) -> str: diff --git a/clients/client-python/gravitino/dto/version_dto.py b/clients/client-python/gravitino/dto/version_dto.py index 881d1563f25..aa47e48283c 100644 --- a/clients/client-python/gravitino/dto/version_dto.py +++ b/clients/client-python/gravitino/dto/version_dto.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + from dataclasses import dataclass diff --git a/clients/client-python/gravitino/exceptions/no_such_metalake_exception.py b/clients/client-python/gravitino/exceptions/no_such_metalake_exception.py index 7e540168374..942e81c99a5 100644 --- a/clients/client-python/gravitino/exceptions/no_such_metalake_exception.py +++ b/clients/client-python/gravitino/exceptions/no_such_metalake_exception.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + from gravitino.exceptions.not_found_exception import NotFoundException diff --git a/clients/client-python/gravitino/exceptions/not_found_exception.py b/clients/client-python/gravitino/exceptions/not_found_exception.py index 3a51576571b..7c8ca0ca51b 100644 --- a/clients/client-python/gravitino/exceptions/not_found_exception.py +++ b/clients/client-python/gravitino/exceptions/not_found_exception.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + from gravitino.exceptions.gravitino_runtime_exception import GravitinoRuntimeException diff --git a/clients/client-python/gravitino/name_identifier.py b/clients/client-python/gravitino/name_identifier.py index 987f5b1ac68..a5042cade62 100644 --- a/clients/client-python/gravitino/name_identifier.py +++ b/clients/client-python/gravitino/name_identifier.py @@ -2,11 +2,14 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + from dataclasses import dataclass, field from dataclasses_json import DataClassJsonMixin, config -from gravitino.exceptions.illegal_name_identifier_exception import IllegalNameIdentifierException +from gravitino.exceptions.illegal_name_identifier_exception import ( + IllegalNameIdentifierException, +) from gravitino.namespace import Namespace @@ -18,10 +21,10 @@ class NameIdentifier(DataClassJsonMixin): schema. """ - _name: str = field(metadata=config(field_name='name')) - _namespace: Namespace = field(metadata=config(field_name='namespace')) + _name: str = field(metadata=config(field_name="name")) + _namespace: Namespace = field(metadata=config(field_name="namespace")) - DOT: str = '.' + DOT: str = "." @classmethod def builder(cls, namespace: Namespace, name: str): @@ -34,7 +37,7 @@ def name(self): return self._name @staticmethod - def of(*names: str) -> 'NameIdentifier': + def of(*names: str) -> "NameIdentifier": """Create the NameIdentifier with the given levels of names. Args: @@ -44,13 +47,17 @@ def of(*names: str) -> 'NameIdentifier': The created NameIdentifier """ - NameIdentifier.check(names is not None, "Cannot create a NameIdentifier with null names") - NameIdentifier.check(len(names) > 0, "Cannot create a NameIdentifier with no names") + NameIdentifier.check( + names is not None, "Cannot create a NameIdentifier with null names" + ) + NameIdentifier.check( + len(names) > 0, "Cannot create a NameIdentifier with no names" + ) return NameIdentifier.builder(Namespace.of(*names[:-1]), names[-1]) @staticmethod - def of_namespace(namespace: Namespace, name: str) -> 'NameIdentifier': + def of_namespace(namespace: Namespace, name: str) -> "NameIdentifier": """Create the NameIdentifier with the given Namespace and name. Args: @@ -63,7 +70,7 @@ def of_namespace(namespace: Namespace, name: str) -> 'NameIdentifier': return NameIdentifier.builder(namespace, name) @staticmethod - def of_metalake(metalake: str) -> 'NameIdentifier': + def of_metalake(metalake: str) -> "NameIdentifier": """Create the metalake NameIdentifier with the given name. Args: @@ -75,7 +82,7 @@ def of_metalake(metalake: str) -> 'NameIdentifier': return NameIdentifier.of(metalake) @staticmethod - def of_catalog(metalake: str, catalog: str) -> 'NameIdentifier': + def of_catalog(metalake: str, catalog: str) -> "NameIdentifier": """Create the catalog NameIdentifier with the given metalake and catalog name. Args: @@ -88,7 +95,7 @@ def of_catalog(metalake: str, catalog: str) -> 'NameIdentifier': return NameIdentifier.of(metalake, catalog) @staticmethod - def of_schema(metalake: str, catalog: str, schema: str) -> 'NameIdentifier': + def of_schema(metalake: str, catalog: str, schema: str) -> "NameIdentifier": """Create the schema NameIdentifier with the given metalake, catalog and schema name. Args: @@ -102,7 +109,9 @@ def of_schema(metalake: str, catalog: str, schema: str) -> 'NameIdentifier': return NameIdentifier.of(metalake, catalog, schema) @staticmethod - def of_table(metalake: str, catalog: str, schema: str, table: str) -> 'NameIdentifier': + def of_table( + metalake: str, catalog: str, schema: str, table: str + ) -> "NameIdentifier": """Create the table NameIdentifier with the given metalake, catalog, schema and table name. Args: @@ -117,7 +126,9 @@ def of_table(metalake: str, catalog: str, schema: str, table: str) -> 'NameIdent return NameIdentifier.of(metalake, catalog, schema, table) @staticmethod - def of_fileset(metalake: str, catalog: str, schema: str, fileset: str) -> 'NameIdentifier': + def of_fileset( + metalake: str, catalog: str, schema: str, fileset: str + ) -> "NameIdentifier": """Create the fileset NameIdentifier with the given metalake, catalog, schema and fileset name. Args: @@ -132,7 +143,9 @@ def of_fileset(metalake: str, catalog: str, schema: str, fileset: str) -> 'NameI return NameIdentifier.of(metalake, catalog, schema, fileset) @staticmethod - def of_topic(metalake: str, catalog: str, schema: str, topic: str) -> 'NameIdentifier': + def of_topic( + metalake: str, catalog: str, schema: str, topic: str + ) -> "NameIdentifier": """Create the topic NameIdentifier with the given metalake, catalog, schema and topic name. @@ -148,7 +161,7 @@ def of_topic(metalake: str, catalog: str, schema: str, topic: str) -> 'NameIdent return NameIdentifier.of(metalake, catalog, schema, topic) @staticmethod - def check_metalake(ident: 'NameIdentifier') -> None: + def check_metalake(ident: "NameIdentifier") -> None: """Check the given NameIdentifier is a metalake identifier. Throw an {@link IllegalNameIdentifierException} if it's not. @@ -159,7 +172,7 @@ def check_metalake(ident: 'NameIdentifier') -> None: Namespace.check_metalake(ident.namespace()) @staticmethod - def check_catalog(ident: 'NameIdentifier') -> None: + def check_catalog(ident: "NameIdentifier") -> None: """Check the given NameIdentifier is a catalog identifier. Throw an {@link IllegalNameIdentifierException} if it's not. @@ -170,7 +183,7 @@ def check_catalog(ident: 'NameIdentifier') -> None: Namespace.check_catalog(ident.namespace()) @staticmethod - def check_schema(ident: 'NameIdentifier') -> None: + def check_schema(ident: "NameIdentifier") -> None: """Check the given NameIdentifier is a schema identifier. Throw an {@link IllegalNameIdentifierException} if it's not. @@ -181,7 +194,7 @@ def check_schema(ident: 'NameIdentifier') -> None: Namespace.check_schema(ident.namespace()) @staticmethod - def check_table(ident: 'NameIdentifier') -> None: + def check_table(ident: "NameIdentifier") -> None: """Check the given NameIdentifier is a table identifier. Throw an {@link IllegalNameIdentifierException} if it's not. @@ -192,7 +205,7 @@ def check_table(ident: 'NameIdentifier') -> None: Namespace.check_table(ident.namespace()) @staticmethod - def check_fileset(ident: 'NameIdentifier') -> None: + def check_fileset(ident: "NameIdentifier") -> None: """Check the given NameIdentifier is a fileset identifier. Throw an {@link IllegalNameIdentifierException} if it's not. @@ -203,7 +216,7 @@ def check_fileset(ident: 'NameIdentifier') -> None: Namespace.check_fileset(ident.namespace()) @staticmethod - def check_topic(ident: 'NameIdentifier') -> None: + def check_topic(ident: "NameIdentifier") -> None: """Check the given NameIdentifier is a topic identifier. Throw an {@link IllegalNameIdentifierException} if it's not. @@ -214,7 +227,7 @@ def check_topic(ident: 'NameIdentifier') -> None: Namespace.check_topic(ident.namespace()) @staticmethod - def parse(identifier: str) -> 'NameIdentifier': + def parse(identifier: str) -> "NameIdentifier": """Create a NameIdentifier from the given identifier string. Args: @@ -223,7 +236,10 @@ def parse(identifier: str) -> 'NameIdentifier': Returns: The created NameIdentifier """ - NameIdentifier.check(identifier is not None and identifier != '', "Cannot parse a null or empty identifier") + NameIdentifier.check( + identifier is not None and identifier != "", + "Cannot parse a null or empty identifier", + ) parts = identifier.split(NameIdentifier.DOT) return NameIdentifier.of(*parts) diff --git a/clients/client-python/gravitino/namespace.py b/clients/client-python/gravitino/namespace.py index f4df5fbe31b..8cda8ae156e 100644 --- a/clients/client-python/gravitino/namespace.py +++ b/clients/client-python/gravitino/namespace.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + from typing import List @@ -19,7 +20,7 @@ def __init__(self, levels: List[str]): self._levels = levels @staticmethod - def empty() -> 'Namespace': + def empty() -> "Namespace": """Get an empty namespace. Returns: @@ -28,7 +29,7 @@ def empty() -> 'Namespace': return Namespace([]) @staticmethod - def of(*levels: str) -> 'Namespace': + def of(*levels: str) -> "Namespace": """Create a namespace with the given levels. Args: @@ -37,17 +38,22 @@ def of(*levels: str) -> 'Namespace': Returns: A namespace with the given levels """ - Namespace.check(levels is not None, "Cannot create a namespace with null levels") + Namespace.check( + levels is not None, "Cannot create a namespace with null levels" + ) if len(levels) == 0: return Namespace.empty() for level in levels: - Namespace.check(level is not None and level != "", "Cannot create a namespace with null or empty level") + Namespace.check( + level is not None and level != "", + "Cannot create a namespace with null or empty level", + ) return Namespace(list(levels)) @staticmethod - def of_metalake() -> 'Namespace': + def of_metalake() -> "Namespace": """Create a namespace for metalake. Returns: @@ -56,7 +62,7 @@ def of_metalake() -> 'Namespace': return Namespace.empty() @staticmethod - def of_catalog(metalake: str) -> 'Namespace': + def of_catalog(metalake: str) -> "Namespace": """Create a namespace for catalog. Args: @@ -68,7 +74,7 @@ def of_catalog(metalake: str) -> 'Namespace': return Namespace.of(metalake) @staticmethod - def of_schema(metalake: str, catalog: str) -> 'Namespace': + def of_schema(metalake: str, catalog: str) -> "Namespace": """Create a namespace for schema. Args: @@ -81,7 +87,7 @@ def of_schema(metalake: str, catalog: str) -> 'Namespace': return Namespace.of(metalake, catalog) @staticmethod - def of_table(metalake: str, catalog: str, schema: str) -> 'Namespace': + def of_table(metalake: str, catalog: str, schema: str) -> "Namespace": """Create a namespace for table. Args: @@ -95,7 +101,7 @@ def of_table(metalake: str, catalog: str, schema: str) -> 'Namespace': return Namespace.of(metalake, catalog, schema) @staticmethod - def of_fileset(metalake: str, catalog: str, schema: str) -> 'Namespace': + def of_fileset(metalake: str, catalog: str, schema: str) -> "Namespace": """Create a namespace for fileset. Args: @@ -109,7 +115,7 @@ def of_fileset(metalake: str, catalog: str, schema: str) -> 'Namespace': return Namespace.of(metalake, catalog, schema) @staticmethod - def of_topic(metalake: str, catalog: str, schema: str) -> 'Namespace': + def of_topic(metalake: str, catalog: str, schema: str) -> "Namespace": """Create a namespace for topic. Args: @@ -123,70 +129,82 @@ def of_topic(metalake: str, catalog: str, schema: str) -> 'Namespace': return Namespace.of(metalake, catalog, schema) @staticmethod - def check_metalake(namespace: 'Namespace') -> None: + def check_metalake(namespace: "Namespace") -> None: """Check if the given metalake namespace is legal, throw an IllegalNamespaceException if it's illegal. Args: namespace: The metalake namespace """ - Namespace.check(namespace is not None and namespace.is_empty(), - f"Metalake namespace must be non-null and empty, the input namespace is {namespace}") + Namespace.check( + namespace is not None and namespace.is_empty(), + f"Metalake namespace must be non-null and empty, the input namespace is {namespace}", + ) @staticmethod - def check_catalog(namespace: 'Namespace') -> None: + def check_catalog(namespace: "Namespace") -> None: """Check if the given catalog namespace is legal, throw an IllegalNamespaceException if it's illegal. Args: namespace: The catalog namespace """ - Namespace.check(namespace is not None and namespace.length() == 1, - f"Catalog namespace must be non-null and have 1 level, the input namespace is {namespace}") + Namespace.check( + namespace is not None and namespace.length() == 1, + f"Catalog namespace must be non-null and have 1 level, the input namespace is {namespace}", + ) @staticmethod - def check_schema(namespace: 'Namespace') -> None: + def check_schema(namespace: "Namespace") -> None: """Check if the given schema namespace is legal, throw an IllegalNamespaceException if it's illegal. Args: namespace: The schema namespace """ - Namespace.check(namespace is not None and namespace.length() == 2, - f"Schema namespace must be non-null and have 2 levels, the input namespace is {namespace}") + Namespace.check( + namespace is not None and namespace.length() == 2, + f"Schema namespace must be non-null and have 2 levels, the input namespace is {namespace}", + ) @staticmethod - def check_table(namespace: 'Namespace') -> None: + def check_table(namespace: "Namespace") -> None: """Check if the given table namespace is legal, throw an IllegalNamespaceException if it's illegal. Args: namespace: The table namespace """ - Namespace.check(namespace is not None and namespace.length() == 3, - f"Table namespace must be non-null and have 3 levels, the input namespace is {namespace}") + Namespace.check( + namespace is not None and namespace.length() == 3, + f"Table namespace must be non-null and have 3 levels, the input namespace is {namespace}", + ) @staticmethod - def check_fileset(namespace: 'Namespace') -> None: + def check_fileset(namespace: "Namespace") -> None: """Check if the given fileset namespace is legal, throw an IllegalNamespaceException if it's illegal. Args: namespace: The fileset namespace """ - Namespace.check(namespace is not None and namespace.length() == 3, - f"Fileset namespace must be non-null and have 3 levels, the input namespace is {namespace}") + Namespace.check( + namespace is not None and namespace.length() == 3, + f"Fileset namespace must be non-null and have 3 levels, the input namespace is {namespace}", + ) @staticmethod - def check_topic(namespace: 'Namespace') -> None: + def check_topic(namespace: "Namespace") -> None: """Check if the given topic namespace is legal, throw an IllegalNamespaceException if it's illegal. Args: namespace: The topic namespace """ - Namespace.check(namespace is not None and namespace.length() == 3, - f"Topic namespace must be non-null and have 3 levels, the input namespace is {namespace}") + Namespace.check( + namespace is not None and namespace.length() == 3, + f"Topic namespace must be non-null and have 3 levels, the input namespace is {namespace}", + ) def levels(self) -> List[str]: """Get the levels of the namespace. @@ -225,7 +243,7 @@ def is_empty(self) -> bool: """ return len(self._levels) == 0 - def __eq__(self, other: 'Namespace') -> bool: + def __eq__(self, other: "Namespace") -> bool: if not isinstance(other, Namespace): return False return self._levels == other._levels diff --git a/clients/client-python/gravitino/rest/rest_message.py b/clients/client-python/gravitino/rest/rest_message.py index 409b3802cff..92b30eac18d 100644 --- a/clients/client-python/gravitino/rest/rest_message.py +++ b/clients/client-python/gravitino/rest/rest_message.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + from abc import ABC, abstractmethod from dataclasses_json import DataClassJsonMixin @@ -31,6 +32,7 @@ def validate(self): class IllegalArgumentException(Exception): """Exception raised if a REST message is not valid according to the REST spec.""" + pass diff --git a/clients/client-python/gravitino/utils/http_client.py b/clients/client-python/gravitino/utils/http_client.py index 588c72ccfff..6de0ced6b4e 100644 --- a/clients/client-python/gravitino/utils/http_client.py +++ b/clients/client-python/gravitino/utils/http_client.py @@ -34,6 +34,7 @@ logger = logging.getLogger(__name__) + class Response: def __init__(self, response): self._status_code = response.getcode() @@ -126,7 +127,10 @@ def _request( if headers: self._update_headers(headers) else: - headers = {'Content-Type': 'application/json', 'Accept': 'application/vnd.gravitino.v1+json'} + headers = { + "Content-Type": "application/json", + "Accept": "application/vnd.gravitino.v1+json", + } self._update_headers(headers) if json: diff --git a/clients/client-python/pylintrc b/clients/client-python/pylintrc index ec5ace82c43..8f37656a5ba 100644 --- a/clients/client-python/pylintrc +++ b/clients/client-python/pylintrc @@ -32,10 +32,7 @@ disable=missing-class-docstring, protected-access, #TODO-fix too-many-arguments, #TODO-fix inconsistent-return-statements, #TODO-fix - line-too-long, #TODO-config invalid-name, #TODO-config - trailing-whitespace, #TODO-config - missing-final-newline #TODO-config [LOGGING] @@ -53,3 +50,17 @@ logging-modules=logging # system, and so shouldn't trigger E1101 when accessed. Python regular # expressions are accepted. generated-members=gravitino.utils.http_client.Response + +[FORMAT] + +# Maximum number of characters on a single line. +max-line-length=120 + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. +indent-string=' ' \ No newline at end of file diff --git a/clients/client-python/requirements-dev.txt b/clients/client-python/requirements-dev.txt index b92555189c1..0918af70f18 100644 --- a/clients/client-python/requirements-dev.txt +++ b/clients/client-python/requirements-dev.txt @@ -2,4 +2,5 @@ # This software is licensed under the Apache License version 2. requests dataclasses-json -pylint \ No newline at end of file +pylint +black \ No newline at end of file diff --git a/clients/client-python/tests/integration/integration_test_env.py b/clients/client-python/tests/integration/integration_test_env.py index 5bdb50a7c26..5362f7c1bce 100644 --- a/clients/client-python/tests/integration/integration_test_env.py +++ b/clients/client-python/tests/integration/integration_test_env.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + import logging import os import unittest @@ -41,32 +42,43 @@ def check_gravitino_server_status() -> bool: class IntegrationTestEnv(unittest.TestCase): """Provide real test environment for the Gravitino Server""" + gravitino_startup_script = None @classmethod def setUpClass(cls): - if os.environ.get('START_EXTERNAL_GRAVITINO') is not None: + if os.environ.get("START_EXTERNAL_GRAVITINO") is not None: # Maybe Gravitino server already startup by Gradle test command or developer manual startup. if not check_gravitino_server_status(): logger.error("ERROR: Can't find online Gravitino server!") return - GravitinoHome = os.environ.get('GRAVITINO_HOME') + GravitinoHome = os.environ.get("GRAVITINO_HOME") if GravitinoHome is None: - logger.error('Gravitino Python client integration test must configure `GRAVITINO_HOME`') + logger.error( + "Gravitino Python client integration test must configure `GRAVITINO_HOME`" + ) sys.exit(0) - cls.gravitino_startup_script = os.path.join(GravitinoHome, 'bin/gravitino.sh') + cls.gravitino_startup_script = os.path.join(GravitinoHome, "bin/gravitino.sh") if not os.path.exists(cls.gravitino_startup_script): - logger.error("Can't find Gravitino startup script: %s, " - "Please execute `./gradlew compileDistribution -x test` in the Gravitino project root " - "directory.", cls.gravitino_startup_script) + logger.error( + "Can't find Gravitino startup script: %s, " + "Please execute `./gradlew compileDistribution -x test` in the Gravitino project root " + "directory.", + cls.gravitino_startup_script, + ) sys.exit(0) logger.info("Starting integration test environment...") # Start Gravitino Server - result = subprocess.run([cls.gravitino_startup_script, 'start'], capture_output=True, text=True, check=False) + result = subprocess.run( + [cls.gravitino_startup_script, "start"], + capture_output=True, + text=True, + check=False, + ) if result.stdout: logger.info("stdout: %s", result.stdout) if result.stderr: @@ -78,11 +90,16 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): - if os.environ.get('START_EXTERNAL_GRAVITINO') is not None: + if os.environ.get("START_EXTERNAL_GRAVITINO") is not None: return logger.info("Stop integration test environment...") - result = subprocess.run([cls.gravitino_startup_script, 'stop'], capture_output=True, text=True, check=False) + result = subprocess.run( + [cls.gravitino_startup_script, "stop"], + capture_output=True, + text=True, + check=False, + ) if result.stdout: logger.info("stdout: %s", result.stdout) if result.stderr: diff --git a/clients/client-python/tests/integration/test_fileset_catalog.py b/clients/client-python/tests/integration/test_fileset_catalog.py index 394b951cb54..9eafd351f8e 100644 --- a/clients/client-python/tests/integration/test_fileset_catalog.py +++ b/clients/client-python/tests/integration/test_fileset_catalog.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + import logging from random import randint from typing import Dict, List @@ -34,18 +35,29 @@ class TestFilesetCatalog(IntegrationTestEnv): fileset_properties_value1: str = "fileset_properties_value1" fileset_properties_key2: str = "fileset_properties_key2" fileset_properties_value2: str = "fileset_properties_value2" - fileset_properties: Dict[str, str] = {fileset_properties_key1: fileset_properties_value1, - fileset_properties_key2: fileset_properties_value2} + fileset_properties: Dict[str, str] = { + fileset_properties_key1: fileset_properties_value1, + fileset_properties_key2: fileset_properties_value2, + } fileset_new_name = fileset_name + "_new" metalake_ident: NameIdentifier = NameIdentifier.of(metalake_name) - catalog_ident: NameIdentifier = NameIdentifier.of_catalog(metalake_name, catalog_name) - schema_ident: NameIdentifier = NameIdentifier.of_schema(metalake_name, catalog_name, schema_name) - fileset_ident: NameIdentifier = NameIdentifier.of_fileset(metalake_name, catalog_name, schema_name, fileset_name) - fileset_new_ident: NameIdentifier = NameIdentifier.of_fileset(metalake_name, catalog_name, schema_name, - fileset_new_name) - - gravitino_admin_client: GravitinoAdminClient = GravitinoAdminClient(uri="http://localhost:8090") + catalog_ident: NameIdentifier = NameIdentifier.of_catalog( + metalake_name, catalog_name + ) + schema_ident: NameIdentifier = NameIdentifier.of_schema( + metalake_name, catalog_name, schema_name + ) + fileset_ident: NameIdentifier = NameIdentifier.of_fileset( + metalake_name, catalog_name, schema_name, fileset_name + ) + fileset_new_ident: NameIdentifier = NameIdentifier.of_fileset( + metalake_name, catalog_name, schema_name, fileset_new_name + ) + + gravitino_admin_client: GravitinoAdminClient = GravitinoAdminClient( + uri="http://localhost:8090" + ) gravitino_client: GravitinoClient = None def setUp(self): @@ -56,37 +68,65 @@ def tearDown(self): def clean_test_data(self): try: - self.gravitino_client = GravitinoClient(uri="http://localhost:8090", metalake_name=self.metalake_name) + self.gravitino_client = GravitinoClient( + uri="http://localhost:8090", metalake_name=self.metalake_name + ) catalog = self.gravitino_client.load_catalog(ident=self.catalog_ident) - logger.info("Drop fileset %s[%s]", self.fileset_ident, - catalog.as_fileset_catalog().drop_fileset(ident=self.fileset_ident)) - logger.info("Drop fileset %s[%s]", self.fileset_new_ident, - catalog.as_fileset_catalog().drop_fileset(ident=self.fileset_new_ident)) - logger.info("Drop schema %s[%s]", self.schema_ident, - catalog.as_schemas().drop_schema(ident=self.schema_ident, cascade=True)) - logger.info("Drop catalog %s[%s]", self.catalog_ident, - self.gravitino_client.drop_catalog(ident=self.catalog_ident)) - logger.info("Drop metalake %s[%s]", self.metalake_ident, - self.gravitino_admin_client.drop_metalake(self.metalake_ident)) + logger.info( + "Drop fileset %s[%s]", + self.fileset_ident, + catalog.as_fileset_catalog().drop_fileset(ident=self.fileset_ident), + ) + logger.info( + "Drop fileset %s[%s]", + self.fileset_new_ident, + catalog.as_fileset_catalog().drop_fileset(ident=self.fileset_new_ident), + ) + logger.info( + "Drop schema %s[%s]", + self.schema_ident, + catalog.as_schemas().drop_schema(ident=self.schema_ident, cascade=True), + ) + logger.info( + "Drop catalog %s[%s]", + self.catalog_ident, + self.gravitino_client.drop_catalog(ident=self.catalog_ident), + ) + logger.info( + "Drop metalake %s[%s]", + self.metalake_ident, + self.gravitino_admin_client.drop_metalake(self.metalake_ident), + ) except Exception as e: logger.error("Clean test data failed: %s", e) def init_test_env(self): - self.gravitino_admin_client.create_metalake(ident=self.metalake_ident, comment="", properties={}) - self.gravitino_client = GravitinoClient(uri="http://localhost:8090", metalake_name=self.metalake_name) - catalog = self.gravitino_client.create_catalog(ident=self.catalog_ident, - type=CatalogDTO.Type.FILESET, - provider=self.catalog_provider, comment="", - properties={self.catalog_location_pcatarop: "/tmp/test1"}) - catalog.as_schemas().create_schema(ident=self.schema_ident, comment="", properties={}) + self.gravitino_admin_client.create_metalake( + ident=self.metalake_ident, comment="", properties={} + ) + self.gravitino_client = GravitinoClient( + uri="http://localhost:8090", metalake_name=self.metalake_name + ) + catalog = self.gravitino_client.create_catalog( + ident=self.catalog_ident, + type=CatalogDTO.Type.FILESET, + provider=self.catalog_provider, + comment="", + properties={self.catalog_location_pcatarop: "/tmp/test1"}, + ) + catalog.as_schemas().create_schema( + ident=self.schema_ident, comment="", properties={} + ) def create_fileset(self) -> Fileset: catalog = self.gravitino_client.load_catalog(ident=self.catalog_ident) - return (catalog.as_fileset_catalog().create_fileset(ident=self.fileset_ident, - type=Fileset.Type.MANAGED, - comment=self.fileset_comment, - storage_location=self.fileset_location, - properties=self.fileset_properties)) + return catalog.as_fileset_catalog().create_fileset( + ident=self.fileset_ident, + type=Fileset.Type.MANAGED, + comment=self.fileset_comment, + storage_location=self.fileset_location, + properties=self.fileset_properties, + ) def test_create_fileset(self): fileset = self.create_fileset() @@ -98,19 +138,25 @@ def test_create_fileset(self): def test_drop_fileset(self): self.create_fileset() catalog = self.gravitino_client.load_catalog(ident=self.catalog_ident) - self.assertTrue(catalog.as_fileset_catalog().drop_fileset(ident=self.fileset_ident)) + self.assertTrue( + catalog.as_fileset_catalog().drop_fileset(ident=self.fileset_ident) + ) def test_list_fileset(self): self.create_fileset() catalog = self.gravitino_client.load_catalog(ident=self.catalog_ident) - fileset_list: List[NameIdentifier] = (catalog.as_fileset_catalog() - .list_filesets(namespace=self.fileset_ident.namespace())) + fileset_list: List[NameIdentifier] = catalog.as_fileset_catalog().list_filesets( + namespace=self.fileset_ident.namespace() + ) self.assertTrue(any(item.name() == self.fileset_name for item in fileset_list)) def test_load_fileset(self): self.create_fileset() - fileset = (self.gravitino_client.load_catalog(ident=self.catalog_ident) - .as_fileset_catalog().load_fileset(ident=self.fileset_ident)) + fileset = ( + self.gravitino_client.load_catalog(ident=self.catalog_ident) + .as_fileset_catalog() + .load_fileset(ident=self.fileset_ident) + ) self.assertIsNotNone(fileset) self.assertEqual(fileset.name(), self.fileset_name) self.assertEqual(fileset.comment(), self.fileset_comment) @@ -123,9 +169,16 @@ def test_alter_fileset(self): changes = ( FilesetChange.remove_property(self.fileset_properties_key1), - FilesetChange.set_property(self.fileset_properties_key2, fileset_propertie_new_value), + FilesetChange.set_property( + self.fileset_properties_key2, fileset_propertie_new_value + ), ) catalog = self.gravitino_client.load_catalog(ident=self.catalog_ident) - fileset_new = catalog.as_fileset_catalog().alter_fileset(self.fileset_ident, *changes) - self.assertEqual(fileset_new.properties().get(self.fileset_properties_key2), fileset_propertie_new_value) + fileset_new = catalog.as_fileset_catalog().alter_fileset( + self.fileset_ident, *changes + ) + self.assertEqual( + fileset_new.properties().get(self.fileset_properties_key2), + fileset_propertie_new_value, + ) self.assertTrue(self.fileset_properties_key1 not in fileset_new.properties()) diff --git a/clients/client-python/tests/integration/test_metalake.py b/clients/client-python/tests/integration/test_metalake.py index c7a80d068d4..5efa1b99b3d 100644 --- a/clients/client-python/tests/integration/test_metalake.py +++ b/clients/client-python/tests/integration/test_metalake.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + import logging from typing import Dict, List @@ -26,17 +27,29 @@ class TestMetalake(IntegrationTestEnv): metalake_properties_value1: str = "metalake_properties_value1" metalake_properties_key2: str = "metalake_properties_key2" metalake_properties_value2: str = "metalake_properties_value2" - metalake_properties: Dict[str, str] = {metalake_properties_key1: metalake_properties_value1, - metalake_properties_key2: metalake_properties_value2} + metalake_properties: Dict[str, str] = { + metalake_properties_key1: metalake_properties_value1, + metalake_properties_key2: metalake_properties_value2, + } - gravitino_admin_client: GravitinoAdminClient = GravitinoAdminClient(uri="http://localhost:8090") + gravitino_admin_client: GravitinoAdminClient = GravitinoAdminClient( + uri="http://localhost:8090" + ) def tearDown(self): self.clean_test_data() def clean_test_data(self): - logger.info("Drop metalake %s[%s]", self.metalake_name, self.drop_metalake(self.metalake_name)) - logger.info("Drop metalake %s[%s]", self.metalake_new_name, self.drop_metalake(self.metalake_new_name)) + logger.info( + "Drop metalake %s[%s]", + self.metalake_name, + self.drop_metalake(self.metalake_name), + ) + logger.info( + "Drop metalake %s[%s]", + self.metalake_new_name, + self.drop_metalake(self.metalake_new_name), + ) def test_create_metalake(self): metalake = self.create_metalake(self.metalake_name) @@ -46,9 +59,11 @@ def test_create_metalake(self): self.assertEqual(metalake.audit_info().creator(), "anonymous") def create_metalake(self, metalake_name) -> GravitinoMetalake: - return self.gravitino_admin_client.create_metalake(NameIdentifier.of(metalake_name), - self.metalake_comment, - self.metalake_properties) + return self.gravitino_admin_client.create_metalake( + NameIdentifier.of(metalake_name), + self.metalake_comment, + self.metalake_properties, + ) def test_alter_metalake(self): self.create_metalake(self.metalake_name) @@ -61,13 +76,20 @@ def test_alter_metalake(self): MetalakeChange.rename(metalake_new_name), MetalakeChange.update_comment(metalake_new_comment), MetalakeChange.remove_property(self.metalake_properties_key1), - MetalakeChange.set_property(self.metalake_properties_key2, metalake_propertie_new_value), + MetalakeChange.set_property( + self.metalake_properties_key2, metalake_propertie_new_value + ), ) - metalake = self.gravitino_admin_client.alter_metalake(NameIdentifier.of(self.metalake_name), *changes) + metalake = self.gravitino_admin_client.alter_metalake( + NameIdentifier.of(self.metalake_name), *changes + ) self.assertEqual(metalake.name(), metalake_new_name) self.assertEqual(metalake.comment(), metalake_new_comment) - self.assertEqual(metalake.properties().get(self.metalake_properties_key2), metalake_propertie_new_value) + self.assertEqual( + metalake.properties().get(self.metalake_properties_key2), + metalake_propertie_new_value, + ) self.assertTrue(self.metalake_properties_key1 not in metalake.properties()) def drop_metalake(self, metalake_name: str) -> bool: @@ -85,39 +107,55 @@ def test_metalake_update_request_to_json(self): ) reqs = [DTOConverters.to_metalake_update_request(change) for change in changes] updates_request = MetalakeUpdatesRequest(reqs) - valid_json = ('{"updates": [{"@type": "rename", "newName": "my_metalake_new"}, ' - '{"@type": "updateComment", "newComment": "new metalake comment"}]}') + valid_json = ( + '{"updates": [{"@type": "rename", "newName": "my_metalake_new"}, ' + '{"@type": "updateComment", "newComment": "new metalake comment"}]}' + ) self.assertEqual(updates_request.to_json(), valid_json) def test_from_json_metalake_response(self): - str_json = (b'{"code":0,"metalake":{"name":"example_name18","comment":"This is a sample comment",' - b'"properties":{"key1":"value1","key2":"value2"},' - b'"audit":{"creator":"anonymous","createTime":"2024-04-05T10:10:35.218Z"}}}') + str_json = ( + b'{"code":0,"metalake":{"name":"example_name18","comment":"This is a sample comment",' + b'"properties":{"key1":"value1","key2":"value2"},' + b'"audit":{"creator":"anonymous","createTime":"2024-04-05T10:10:35.218Z"}}}' + ) metalake_response = MetalakeResponse.from_json(str_json, infer_missing=True) self.assertEqual(metalake_response.code(), 0) self.assertIsNotNone(metalake_response._metalake) self.assertEqual(metalake_response._metalake.name(), "example_name18") - self.assertEqual(metalake_response._metalake.audit_info().creator(), "anonymous") + self.assertEqual( + metalake_response._metalake.audit_info().creator(), "anonymous" + ) def test_from_error_json_metalake_response(self): - str_json = (b'{"code":0, "undefine-key1":"undefine-value1", ' - b'"metalake":{"undefine-key2":1, "name":"example_name18","comment":"This is a sample comment",' - b'"properties":{"key1":"value1","key2":"value2"},' - b'"audit":{"creator":"anonymous","createTime":"2024-04-05T10:10:35.218Z"}}}') + str_json = ( + b'{"code":0, "undefine-key1":"undefine-value1", ' + b'"metalake":{"undefine-key2":1, "name":"example_name18","comment":"This is a sample comment",' + b'"properties":{"key1":"value1","key2":"value2"},' + b'"audit":{"creator":"anonymous","createTime":"2024-04-05T10:10:35.218Z"}}}' + ) metalake_response = MetalakeResponse.from_json(str_json, infer_missing=True) self.assertEqual(metalake_response.code(), 0) self.assertIsNotNone(metalake_response._metalake) self.assertEqual(metalake_response._metalake.name(), "example_name18") - self.assertEqual(metalake_response._metalake.audit_info().creator(), "anonymous") + self.assertEqual( + metalake_response._metalake.audit_info().creator(), "anonymous" + ) def test_list_metalakes(self): self.create_metalake(self.metalake_name) - metalake_list: List[GravitinoMetalake] = self.gravitino_admin_client.list_metalakes() - self.assertTrue(any(item.name() == self.metalake_name for item in metalake_list)) + metalake_list: List[GravitinoMetalake] = ( + self.gravitino_admin_client.list_metalakes() + ) + self.assertTrue( + any(item.name() == self.metalake_name for item in metalake_list) + ) def test_load_metalakes(self): self.create_metalake(self.metalake_name) - metalake = self.gravitino_admin_client.load_metalake(NameIdentifier.of(self.metalake_name)) + metalake = self.gravitino_admin_client.load_metalake( + NameIdentifier.of(self.metalake_name) + ) self.assertIsNotNone(metalake) self.assertEqual(metalake.name(), self.metalake_name) self.assertEqual(metalake.comment(), self.metalake_comment) diff --git a/clients/client-python/tests/integration/test_schema.py b/clients/client-python/tests/integration/test_schema.py index a0fe41972dd..244017f6982 100644 --- a/clients/client-python/tests/integration/test_schema.py +++ b/clients/client-python/tests/integration/test_schema.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + import logging from random import randint from typing import Dict, List @@ -32,15 +33,25 @@ class TestSchema(IntegrationTestEnv): schema_properties_value1: str = "schema_properties_value1" schema_properties_key2: str = "schema_properties_key2" schema_properties_value2: str = "schema_properties_value2" - schema_properties: Dict[str, str] = {schema_properties_key1: schema_properties_value1, - schema_properties_key2: schema_properties_value2} + schema_properties: Dict[str, str] = { + schema_properties_key1: schema_properties_value1, + schema_properties_key2: schema_properties_value2, + } metalake_ident: NameIdentifier = NameIdentifier.of(metalake_name) - catalog_ident: NameIdentifier = NameIdentifier.of_catalog(metalake_name, catalog_name) - schema_ident: NameIdentifier = NameIdentifier.of_schema(metalake_name, catalog_name, schema_name) - schema_new_ident: NameIdentifier = NameIdentifier.of_schema(metalake_name, catalog_name, schema_new_name) - - gravitino_admin_client: GravitinoAdminClient = GravitinoAdminClient(uri="http://localhost:8090") + catalog_ident: NameIdentifier = NameIdentifier.of_catalog( + metalake_name, catalog_name + ) + schema_ident: NameIdentifier = NameIdentifier.of_schema( + metalake_name, catalog_name, schema_name + ) + schema_new_ident: NameIdentifier = NameIdentifier.of_schema( + metalake_name, catalog_name, schema_new_name + ) + + gravitino_admin_client: GravitinoAdminClient = GravitinoAdminClient( + uri="http://localhost:8090" + ) gravitino_client: GravitinoClient = None def setUp(self): @@ -50,32 +61,56 @@ def tearDown(self): self.clean_test_data() def init_test_env(self): - self.gravitino_admin_client.create_metalake(ident=self.metalake_ident, comment="", properties={}) - self.gravitino_client = GravitinoClient(uri="http://localhost:8090", metalake_name=self.metalake_name) - self.gravitino_client.create_catalog(ident=self.catalog_ident, type=Catalog.Type.FILESET, - provider=self.catalog_provider, comment="", - properties={self.catalog_location_prop: "/tmp/test_schema"}) + self.gravitino_admin_client.create_metalake( + ident=self.metalake_ident, comment="", properties={} + ) + self.gravitino_client = GravitinoClient( + uri="http://localhost:8090", metalake_name=self.metalake_name + ) + self.gravitino_client.create_catalog( + ident=self.catalog_ident, + type=Catalog.Type.FILESET, + provider=self.catalog_provider, + comment="", + properties={self.catalog_location_prop: "/tmp/test_schema"}, + ) def clean_test_data(self): try: - self.gravitino_client = GravitinoClient(uri="http://localhost:8090", metalake_name=self.metalake_name) + self.gravitino_client = GravitinoClient( + uri="http://localhost:8090", metalake_name=self.metalake_name + ) catalog = self.gravitino_client.load_catalog(ident=self.catalog_ident) - logger.info("Drop schema %s[%s]", self.schema_ident, - catalog.as_schemas().drop_schema(self.schema_ident, cascade=True)) - logger.info("Drop schema %s[%s]", self.schema_new_ident, - catalog.as_schemas().drop_schema(self.schema_new_ident, cascade=True)) - logger.info("Drop catalog %s[%s]", self.catalog_ident, - self.gravitino_client.drop_catalog(ident=self.catalog_ident)) - logger.info("Drop metalake %s[%s]", self.metalake_ident, - self.gravitino_admin_client.drop_metalake(self.metalake_ident)) + logger.info( + "Drop schema %s[%s]", + self.schema_ident, + catalog.as_schemas().drop_schema(self.schema_ident, cascade=True), + ) + logger.info( + "Drop schema %s[%s]", + self.schema_new_ident, + catalog.as_schemas().drop_schema(self.schema_new_ident, cascade=True), + ) + logger.info( + "Drop catalog %s[%s]", + self.catalog_ident, + self.gravitino_client.drop_catalog(ident=self.catalog_ident), + ) + logger.info( + "Drop metalake %s[%s]", + self.metalake_ident, + self.gravitino_admin_client.drop_metalake(self.metalake_ident), + ) except Exception as e: logger.error("Clean test data failed: %s", e) def create_schema(self) -> Schema: catalog = self.gravitino_client.load_catalog(ident=self.catalog_ident) - return catalog.as_schemas().create_schema(ident=self.schema_ident, - comment=self.schema_comment, - properties=self.schema_properties) + return catalog.as_schemas().create_schema( + ident=self.schema_ident, + comment=self.schema_comment, + properties=self.schema_properties, + ) def test_create_schema(self): schema = self.create_schema() @@ -87,13 +122,16 @@ def test_create_schema(self): def test_drop_schema(self): self.create_schema() catalog = self.gravitino_client.load_catalog(ident=self.catalog_ident) - self.assertTrue(catalog.as_schemas().drop_schema(ident=self.schema_ident, cascade=True)) + self.assertTrue( + catalog.as_schemas().drop_schema(ident=self.schema_ident, cascade=True) + ) def test_list_schema(self): self.create_schema() catalog = self.gravitino_client.load_catalog(ident=self.catalog_ident) schema_list: List[NameIdentifier] = catalog.as_schemas().list_schemas( - namespace=self.schema_ident.namespace()) + namespace=self.schema_ident.namespace() + ) self.assertTrue(any(item.name() == self.schema_name for item in schema_list)) def test_load_schema(self): @@ -112,9 +150,14 @@ def test_alter_schema(self): changes = ( SchemaChange.remove_property(self.schema_properties_key1), - SchemaChange.set_property(self.schema_properties_key2, schema_propertie_new_value), + SchemaChange.set_property( + self.schema_properties_key2, schema_propertie_new_value + ), ) catalog = self.gravitino_client.load_catalog(ident=self.catalog_ident) schema_new = catalog.as_schemas().alter_schema(self.schema_ident, *changes) - self.assertEqual(schema_new.properties().get(self.schema_properties_key2), schema_propertie_new_value) + self.assertEqual( + schema_new.properties().get(self.schema_properties_key2), + schema_propertie_new_value, + ) self.assertTrue(self.schema_properties_key1 not in schema_new.properties())