Skip to content

Commit

Permalink
[#2252][#2253][#2470] (PyClient): Add Fileset APIs in Gravitino Pytho…
Browse files Browse the repository at this point in the history
…n client (#2898)

### What changes were proposed in this pull request?

Support Fileset Catalog in Python client, support below functions:

1. list_filesets()
2. load_fileset()
3. create_fileset()
4. alter_fileset()
5. drop_fileset()

### Why are the changes needed?

Fix: #2470 #2252 #2253

### Does this PR introduce _any_ user-facing change?

N/A

### How was this patch tested?

CI Passed
  • Loading branch information
xunliu authored Apr 13, 2024
1 parent 282d082 commit 139a40a
Show file tree
Hide file tree
Showing 49 changed files with 2,842 additions and 190 deletions.
4 changes: 4 additions & 0 deletions clients/client-python/gravitino/api/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
"""
Copyright 2024 Datastrato Pvt Ltd.
This software is licensed under the Apache License version 2.
"""
44 changes: 44 additions & 0 deletions clients/client-python/gravitino/api/audit.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
"""
Copyright 2024 Datastrato Pvt Ltd.
This software is licensed under the Apache License version 2.
"""
from abc import ABC, abstractmethod
from datetime import datetime


class Audit(ABC):
"""Represents the audit information of an entity."""

@abstractmethod
def creator(self) -> str:
"""The creator of the entity.
Returns:
the creator of the entity.
"""
pass

@abstractmethod
def create_time(self) -> datetime:
"""The creation time of the entity.
Returns:
The creation time of the entity.
"""
pass

@abstractmethod
def last_modifier(self) -> str:
"""
Returns:
The last modifier of the entity.
"""
pass

@abstractmethod
def last_modified_time(self) -> datetime:
"""
Returns:
The last modified time of the entity.
"""
pass
18 changes: 18 additions & 0 deletions clients/client-python/gravitino/api/auditable.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
"""
Copyright 2024 Datastrato Pvt Ltd.
This software is licensed under the Apache License version 2.
"""
from abc import ABC, abstractmethod

from gravitino.api.audit import Audit


class Auditable(ABC):
"""
An auditable entity is an entity that has audit information associated with it. This audit
information is used to track changes to the entity.
"""

@abstractmethod
def audit_info(self) -> Audit:
pass
129 changes: 129 additions & 0 deletions clients/client-python/gravitino/api/catalog.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
"""
Copyright 2024 Datastrato Pvt Ltd.
This software is licensed under the Apache License version 2.
"""
from abc import abstractmethod
from enum import Enum
from typing import Dict, Optional

from gravitino.api.auditable import Auditable
from gravitino.api.supports_schemas import SupportsSchemas


class Catalog(Auditable):
"""The interface of a catalog. The catalog is the second level entity in the gravitino system,
containing a set of tables.
"""
class Type(Enum):
"""The type of the catalog."""

RELATIONAL = "relational"
""""Catalog Type for Relational Data Structure, like db.table, catalog.db.table."""

FILESET = "fileset"
"""Catalog Type for Fileset System (including HDFS, S3, etc.), like path/to/file"""

MESSAGING = "messaging"
"""Catalog Type for Message Queue, like kafka://topic"""

UNSUPPORTED = "unsupported"
"""Catalog Type for test only."""

PROPERTY_PACKAGE = "package"
"""A reserved property to specify the package location of the catalog. The "package" is a string
of path to the folder where all the catalog related dependencies is located. The dependencies
under the "package" will be loaded by Gravitino to create the catalog.
The property "package" is not needed if the catalog is a built-in one, Gravitino will search
the proper location using "provider" to load the dependencies. Only when the folder is in
different location, the "package" property is needed.
"""

@abstractmethod
def name(self) -> str:
"""
Returns:
The name of the catalog.
"""
pass

@abstractmethod
def type(self) -> Type:
"""
Returns:
The type of the catalog.
"""
pass

@abstractmethod
def provider(self) -> str:
"""
Returns:
The provider of the catalog.
"""
pass

@abstractmethod
def comment(self) -> Optional[str]:
"""The comment of the catalog. Note. this method will return null if the comment is not set for
this catalog.
Returns:
The provider of the catalog.
"""
pass

@abstractmethod
def properties(self) -> Optional[Dict[str, str]]:
"""
The properties of the catalog. Note, this method will return null if the properties are not set.
Returns:
The properties of the catalog.
"""
pass

def as_schemas(self) -> SupportsSchemas:
"""Return the {@link SupportsSchemas} if the catalog supports schema operations.
Raises:
UnsupportedOperationException if the catalog does not support schema operations.
Returns:
The {@link SupportsSchemas} if the catalog supports schema operations.
"""
raise UnsupportedOperationException("Catalog does not support schema operations")

def as_table_catalog(self) -> 'TableCatalog':
"""
Raises:
UnsupportedOperationException if the catalog does not support table operations.
Returns:
the {@link TableCatalog} if the catalog supports table operations.
"""
raise UnsupportedOperationException("Catalog does not support table operations")

def as_fileset_catalog(self) -> 'FilesetCatalog':
"""
Raises:
UnsupportedOperationException if the catalog does not support fileset operations.
Returns:
the FilesetCatalog if the catalog supports fileset operations.
"""
raise UnsupportedOperationException("Catalog does not support fileset operations")

def as_topic_catalog(self) -> 'TopicCatalog':
"""
Returns:
the {@link TopicCatalog} if the catalog supports topic operations.
Raises:
UnsupportedOperationException if the catalog does not support topic operations.
"""
raise UnsupportedOperationException("Catalog does not support topic operations")


class UnsupportedOperationException(Exception):
pass
Loading

0 comments on commit 139a40a

Please sign in to comment.