-
Notifications
You must be signed in to change notification settings - Fork 383
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Browse files
Browse the repository at this point in the history
…n client (#2898) ### What changes were proposed in this pull request? Support Fileset Catalog in Python client, support below functions: 1. list_filesets() 2. load_fileset() 3. create_fileset() 4. alter_fileset() 5. drop_fileset() ### Why are the changes needed? Fix: #2470 #2252 #2253 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? CI Passed
- Loading branch information
Showing
49 changed files
with
2,842 additions
and
190 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,4 @@ | ||
""" | ||
Copyright 2024 Datastrato Pvt Ltd. | ||
This software is licensed under the Apache License version 2. | ||
""" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,44 @@ | ||
""" | ||
Copyright 2024 Datastrato Pvt Ltd. | ||
This software is licensed under the Apache License version 2. | ||
""" | ||
from abc import ABC, abstractmethod | ||
from datetime import datetime | ||
|
||
|
||
class Audit(ABC): | ||
"""Represents the audit information of an entity.""" | ||
|
||
@abstractmethod | ||
def creator(self) -> str: | ||
"""The creator of the entity. | ||
Returns: | ||
the creator of the entity. | ||
""" | ||
pass | ||
|
||
@abstractmethod | ||
def create_time(self) -> datetime: | ||
"""The creation time of the entity. | ||
Returns: | ||
The creation time of the entity. | ||
""" | ||
pass | ||
|
||
@abstractmethod | ||
def last_modifier(self) -> str: | ||
""" | ||
Returns: | ||
The last modifier of the entity. | ||
""" | ||
pass | ||
|
||
@abstractmethod | ||
def last_modified_time(self) -> datetime: | ||
""" | ||
Returns: | ||
The last modified time of the entity. | ||
""" | ||
pass |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,18 @@ | ||
""" | ||
Copyright 2024 Datastrato Pvt Ltd. | ||
This software is licensed under the Apache License version 2. | ||
""" | ||
from abc import ABC, abstractmethod | ||
|
||
from gravitino.api.audit import Audit | ||
|
||
|
||
class Auditable(ABC): | ||
""" | ||
An auditable entity is an entity that has audit information associated with it. This audit | ||
information is used to track changes to the entity. | ||
""" | ||
|
||
@abstractmethod | ||
def audit_info(self) -> Audit: | ||
pass |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,129 @@ | ||
""" | ||
Copyright 2024 Datastrato Pvt Ltd. | ||
This software is licensed under the Apache License version 2. | ||
""" | ||
from abc import abstractmethod | ||
from enum import Enum | ||
from typing import Dict, Optional | ||
|
||
from gravitino.api.auditable import Auditable | ||
from gravitino.api.supports_schemas import SupportsSchemas | ||
|
||
|
||
class Catalog(Auditable): | ||
"""The interface of a catalog. The catalog is the second level entity in the gravitino system, | ||
containing a set of tables. | ||
""" | ||
class Type(Enum): | ||
"""The type of the catalog.""" | ||
|
||
RELATIONAL = "relational" | ||
""""Catalog Type for Relational Data Structure, like db.table, catalog.db.table.""" | ||
|
||
FILESET = "fileset" | ||
"""Catalog Type for Fileset System (including HDFS, S3, etc.), like path/to/file""" | ||
|
||
MESSAGING = "messaging" | ||
"""Catalog Type for Message Queue, like kafka://topic""" | ||
|
||
UNSUPPORTED = "unsupported" | ||
"""Catalog Type for test only.""" | ||
|
||
PROPERTY_PACKAGE = "package" | ||
"""A reserved property to specify the package location of the catalog. The "package" is a string | ||
of path to the folder where all the catalog related dependencies is located. The dependencies | ||
under the "package" will be loaded by Gravitino to create the catalog. | ||
The property "package" is not needed if the catalog is a built-in one, Gravitino will search | ||
the proper location using "provider" to load the dependencies. Only when the folder is in | ||
different location, the "package" property is needed. | ||
""" | ||
|
||
@abstractmethod | ||
def name(self) -> str: | ||
""" | ||
Returns: | ||
The name of the catalog. | ||
""" | ||
pass | ||
|
||
@abstractmethod | ||
def type(self) -> Type: | ||
""" | ||
Returns: | ||
The type of the catalog. | ||
""" | ||
pass | ||
|
||
@abstractmethod | ||
def provider(self) -> str: | ||
""" | ||
Returns: | ||
The provider of the catalog. | ||
""" | ||
pass | ||
|
||
@abstractmethod | ||
def comment(self) -> Optional[str]: | ||
"""The comment of the catalog. Note. this method will return null if the comment is not set for | ||
this catalog. | ||
Returns: | ||
The provider of the catalog. | ||
""" | ||
pass | ||
|
||
@abstractmethod | ||
def properties(self) -> Optional[Dict[str, str]]: | ||
""" | ||
The properties of the catalog. Note, this method will return null if the properties are not set. | ||
Returns: | ||
The properties of the catalog. | ||
""" | ||
pass | ||
|
||
def as_schemas(self) -> SupportsSchemas: | ||
"""Return the {@link SupportsSchemas} if the catalog supports schema operations. | ||
Raises: | ||
UnsupportedOperationException if the catalog does not support schema operations. | ||
Returns: | ||
The {@link SupportsSchemas} if the catalog supports schema operations. | ||
""" | ||
raise UnsupportedOperationException("Catalog does not support schema operations") | ||
|
||
def as_table_catalog(self) -> 'TableCatalog': | ||
""" | ||
Raises: | ||
UnsupportedOperationException if the catalog does not support table operations. | ||
Returns: | ||
the {@link TableCatalog} if the catalog supports table operations. | ||
""" | ||
raise UnsupportedOperationException("Catalog does not support table operations") | ||
|
||
def as_fileset_catalog(self) -> 'FilesetCatalog': | ||
""" | ||
Raises: | ||
UnsupportedOperationException if the catalog does not support fileset operations. | ||
Returns: | ||
the FilesetCatalog if the catalog supports fileset operations. | ||
""" | ||
raise UnsupportedOperationException("Catalog does not support fileset operations") | ||
|
||
def as_topic_catalog(self) -> 'TopicCatalog': | ||
""" | ||
Returns: | ||
the {@link TopicCatalog} if the catalog supports topic operations. | ||
Raises: | ||
UnsupportedOperationException if the catalog does not support topic operations. | ||
""" | ||
raise UnsupportedOperationException("Catalog does not support topic operations") | ||
|
||
|
||
class UnsupportedOperationException(Exception): | ||
pass |
Oops, something went wrong.