Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Add support for Postgresql dialect #741

Merged
merged 27 commits into from
Jun 17, 2022
Merged
Show file tree
Hide file tree
Changes from 19 commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
ce791cf
chore: regen (via synth) : fix conflicts
larkee Aug 16, 2021
e580775
feat: add NUMERIC support: conflicts resolved
larkee Aug 16, 2021
f7f0c3a
feat: add dialect support: fix conflicts
larkee Aug 16, 2021
95dd609
fix: update table queries to support PG dialect
larkee Aug 18, 2021
8e4349b
feat: add database dialect support for database factory
larkee Sep 3, 2021
f4a256f
test: add dialect support to system tests: resolve conflict, correct …
larkee Sep 10, 2021
9bc87d9
feat: postgres dialect - review fixes
ansh0l Jun 9, 2022
69713de
feat: postgres dialect - review fixes
ansh0l Jun 11, 2022
e112a1d
feat: postgres dialect - review fixes
ansh0l Jun 12, 2022
da92986
feat: postgres dialect - review fixes
ansh0l Jun 13, 2022
be00413
feat: postgres dialect - review fixes
ansh0l Jun 13, 2022
dd2a0ab
feat: postgres dialect - review fixes
ansh0l Jun 13, 2022
a0a8d8b
feat: postgres dialect - review fixes
ansh0l Jun 13, 2022
fc3db16
feat: postgres dialect - review fixes
ansh0l Jun 13, 2022
1bf0440
feat: postgres dialect - docstring fixes
ansh0l Jun 13, 2022
9a540fc
feat: fix linting
ansh0l Jun 14, 2022
0fb0e45
Merge branch 'main' into postgresql_dialect
ansh0l Jun 14, 2022
666285b
feat: add opentelemetry version in noxfile to remove failures
ansh0l Jun 15, 2022
8525bf5
feat: add opentelemetry version and constraints.txt
ansh0l Jun 15, 2022
f264eca
Revert "feat: add opentelemetry version and constraints.txt"
ansh0l Jun 15, 2022
2ae7b98
Revert "feat: add opentelemetry version in noxfile to remove failures"
ansh0l Jun 15, 2022
9d4ef52
feat: removing duplicate imports
ansh0l Jun 15, 2022
032a2b8
feat: correcting imports
ansh0l Jun 15, 2022
d879842
feat: correcting imports
ansh0l Jun 15, 2022
f9c274a
feat: skip backup tests
ansh0l Jun 17, 2022
41a03e0
feat: correct the import
ansh0l Jun 17, 2022
ba0bf4d
feat: fix linting
ansh0l Jun 17, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -294,7 +294,7 @@ class CreateDatabaseRequest(proto.Message):
Cloud Spanner will encrypt/decrypt all data at
rest using Google default encryption.
database_dialect (google.cloud.spanner_admin_database_v1.types.DatabaseDialect):
Optional. The dialect of the Cloud Spanner
Output only. The dialect of the Cloud Spanner
Database.
"""

Expand Down
2 changes: 2 additions & 0 deletions google/cloud/spanner_v1/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@
from .types.transaction import TransactionSelector
from .types.type import StructType
from .types.type import Type
from .types.type import TypeAnnotationCode
from .types.type import TypeCode
from .data_types import JsonObject

Expand Down Expand Up @@ -132,6 +133,7 @@
"TransactionOptions",
"TransactionSelector",
"Type",
"TypeAnnotationCode",
"TypeCode",
# Custom spanner related data types
"JsonObject",
Expand Down
10 changes: 9 additions & 1 deletion google/cloud/spanner_v1/backup.py
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,7 @@ def __init__(
self._encryption_info = None
self._max_expire_time = None
self._referencing_backups = None
self._database_dialect = None
if type(encryption_config) == dict:
if source_backup:
self._encryption_config = CopyBackupEncryptionConfig(
Expand Down Expand Up @@ -193,7 +194,7 @@ def referencing_databases(self):
@property
def encryption_info(self):
"""Encryption info for this backup.
:rtype: :class:`~google.clod.spanner_admin_database_v1.types.EncryptionInfo`
:rtype: :class:`~google.cloud.spanner_admin_database_v1.types.EncryptionInfo`
:returns: a class representing the encryption info
"""
return self._encryption_info
Expand All @@ -216,6 +217,13 @@ def referencing_backups(self):
"""
return self._referencing_backups

def database_dialect(self):
"""Database Dialect for this backup.
:rtype: :class:`~google.cloud.spanner_admin_database_v1.types.DatabaseDialect`
:returns: a class representing the dialect of this backup's database
"""
return self._database_dialect

@classmethod
def from_pb(cls, backup_pb, instance):
"""Create an instance of this class from a protobuf message.
Expand Down
45 changes: 42 additions & 3 deletions google/cloud/spanner_v1/database.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,17 @@
from google.cloud.spanner_v1.services.spanner.transports.grpc import (
SpannerGrpcTransport,
)
from google.cloud.spanner_admin_database_v1 import CreateDatabaseRequest
from google.cloud.spanner_admin_database_v1 import DatabaseDialect
from google.cloud.spanner_admin_database_v1 import EncryptionConfig
from google.cloud.spanner_admin_database_v1 import RestoreDatabaseEncryptionConfig
from google.cloud.spanner_admin_database_v1 import RestoreDatabaseRequest
from google.cloud.spanner_admin_database_v1 import UpdateDatabaseDdlRequest
from google.cloud.spanner_v1 import (
ExecuteSqlRequest,
TransactionSelector,
TransactionOptions,
)
from google.cloud.spanner_v1.table import Table


Expand All @@ -68,7 +79,7 @@

_LIST_TABLES_QUERY = """SELECT TABLE_NAME
FROM INFORMATION_SCHEMA.TABLES
WHERE SPANNER_STATE = 'COMMITTED'
{}
"""

DEFAULT_RETRY_BACKOFF = Retry(initial=0.02, maximum=32, multiplier=1.3)
Expand Down Expand Up @@ -114,6 +125,11 @@ class Database(object):
If a dict is provided, it must be of the same form as either of the protobuf
messages :class:`~google.cloud.spanner_admin_database_v1.types.EncryptionConfig`
or :class:`~google.cloud.spanner_admin_database_v1.types.RestoreDatabaseEncryptionConfig`
:type database_dialect:
:class:`~google.cloud.spanner_admin_database_v1.types.DatabaseDialect`
:param database_dialect:
(Optional) database dialect for the database

"""

_spanner_api = None
Expand All @@ -126,6 +142,7 @@ def __init__(
pool=None,
logger=None,
encryption_config=None,
database_dialect=DatabaseDialect.DATABASE_DIALECT_UNSPECIFIED,
):
self.database_id = database_id
self._instance = instance
Expand All @@ -141,6 +158,7 @@ def __init__(
self.log_commit_stats = False
self._logger = logger
self._encryption_config = encryption_config
self._database_dialect = database_dialect

if pool is None:
pool = BurstyPool()
Expand Down Expand Up @@ -294,6 +312,18 @@ def ddl_statements(self):
"""
return self._ddl_statements

@property
def database_dialect(self):
"""DDL Statements used to define database schema.

See
cloud.google.com/spanner/docs/data-definition-language

:rtype: :class:`google.cloud.spanner_admin_database_v1.types.DatabaseDialect`
:returns: the dialect of the database
"""
return self._database_dialect

@property
def logger(self):
"""Logger used by the database.
Expand Down Expand Up @@ -364,7 +394,10 @@ def create(self):
metadata = _metadata_with_prefix(self.name)
db_name = self.database_id
if "-" in db_name:
db_name = "`%s`" % (db_name,)
if self._database_dialect == DatabaseDialect.POSTGRESQL:
db_name = f'"{db_name}"'
else:
db_name = f"`{db_name}`"
if type(self._encryption_config) == dict:
self._encryption_config = EncryptionConfig(**self._encryption_config)

Expand All @@ -373,6 +406,7 @@ def create(self):
create_statement="CREATE DATABASE %s" % (db_name,),
extra_statements=list(self._ddl_statements),
encryption_config=self._encryption_config,
database_dialect=self._database_dialect,
)
future = api.create_database(request=request, metadata=metadata)
return future
Expand Down Expand Up @@ -418,6 +452,7 @@ def reload(self):
self._encryption_config = response.encryption_config
self._encryption_info = response.encryption_info
self._default_leader = response.default_leader
self._database_dialect = response.database_dialect

def update_ddl(self, ddl_statements, operation_id=""):
"""Update DDL for this database.
Expand Down Expand Up @@ -778,7 +813,11 @@ def list_tables(self):
resources within the current database.
"""
with self.snapshot() as snapshot:
results = snapshot.execute_sql(_LIST_TABLES_QUERY)
if self._database_dialect == DatabaseDialect.POSTGRESQL:
where_clause = "WHERE TABLE_SCHEMA = 'public'"
else:
where_clause = "WHERE SPANNER_STATE = 'COMMITTED'"
results = snapshot.execute_sql(_LIST_TABLES_QUERY.format(where_clause))
for row in results:
yield self.table(row[0])

Expand Down
8 changes: 8 additions & 0 deletions google/cloud/spanner_v1/instance.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
from google.cloud.spanner_admin_instance_v1 import Instance as InstancePB
from google.cloud.spanner_admin_database_v1.types import backup
from google.cloud.spanner_admin_database_v1.types import spanner_database_admin
from google.cloud.spanner_admin_database_v1 import DatabaseDialect
from google.cloud.spanner_admin_database_v1 import ListBackupsRequest
from google.cloud.spanner_admin_database_v1 import ListBackupOperationsRequest
from google.cloud.spanner_admin_database_v1 import ListDatabasesRequest
Expand Down Expand Up @@ -428,6 +429,7 @@ def database(
pool=None,
logger=None,
encryption_config=None,
database_dialect=DatabaseDialect.DATABASE_DIALECT_UNSPECIFIED,
):
"""Factory to create a database within this instance.

Expand Down Expand Up @@ -458,6 +460,11 @@ def database(
messages :class:`~google.cloud.spanner_admin_database_v1.types.EncryptionConfig`
or :class:`~google.cloud.spanner_admin_database_v1.types.RestoreDatabaseEncryptionConfig`

:type database_dialect:
:class:`~google.cloud.spanner_admin_database_v1.types.DatabaseDialect`
:param database_dialect:
(Optional) database dialect for the database

:rtype: :class:`~google.cloud.spanner_v1.database.Database`
:returns: a database owned by this instance.
"""
Expand All @@ -468,6 +475,7 @@ def database(
pool=pool,
logger=logger,
encryption_config=encryption_config,
database_dialect=database_dialect,
)

def list_databases(self, page_size=None):
Expand Down
2 changes: 2 additions & 0 deletions google/cloud/spanner_v1/param_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
"""Types exported from this package."""

from google.cloud.spanner_v1 import Type
from google.cloud.spanner_v1 import TypeAnnotationCode
from google.cloud.spanner_v1 import TypeCode
from google.cloud.spanner_v1 import StructType

Expand All @@ -29,6 +30,7 @@
TIMESTAMP = Type(code=TypeCode.TIMESTAMP)
NUMERIC = Type(code=TypeCode.NUMERIC)
JSON = Type(code=TypeCode.JSON)
PG_NUMERIC = Type(code=TypeCode.NUMERIC, type_annotation=TypeAnnotationCode.PG_NUMERIC)


def Array(element_type):
Expand Down
20 changes: 14 additions & 6 deletions google/cloud/spanner_v1/table.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@

from google.cloud.exceptions import NotFound

from google.cloud.spanner_admin_database_v1 import DatabaseDialect
from google.cloud.spanner_v1.types import (
Type,
TypeCode,
Expand All @@ -26,7 +27,7 @@
SELECT EXISTS(
SELECT TABLE_NAME
FROM INFORMATION_SCHEMA.TABLES
WHERE TABLE_NAME = @table_id
{}
)
"""
_GET_SCHEMA_TEMPLATE = "SELECT * FROM {} LIMIT 0"
Expand Down Expand Up @@ -76,11 +77,18 @@ def _exists(self, snapshot):
:rtype: bool
:returns: True if the table exists, else false.
"""
results = snapshot.execute_sql(
_EXISTS_TEMPLATE,
params={"table_id": self.table_id},
param_types={"table_id": Type(code=TypeCode.STRING)},
)
if self._database.database_dialect == DatabaseDialect.POSTGRESQL:
results = snapshot.execute_sql(
_EXISTS_TEMPLATE.format("WHERE TABLE_NAME = $1"),
params={"p1": self.table_id},
param_types={"p1": Type(code=TypeCode.STRING)},
)
else:
results = snapshot.execute_sql(
_EXISTS_TEMPLATE.format("WHERE TABLE_NAME = @table_id"),
params={"table_id": self.table_id},
param_types={"table_id": Type(code=TypeCode.STRING)},
)
return next(iter(results))[0]

@property
Expand Down
Loading