From 95461d6339f25bb1325d5f175fd4ba32fdbe0c60 Mon Sep 17 00:00:00 2001 From: Graham Hoyes Date: Mon, 8 Jul 2024 20:02:24 -0400 Subject: [PATCH] Spanner Postgres interface support (#555) Adds support for the [GCP Spanner Postgres Interface](https://cloud.google.com/spanner/docs/postgresql-interface), which only requires a slight modification to the postgres driver. This will not work with a GoogleSQL-flavored Spanner database; support for that will require a dedicated driver. The new functionality is accessed with a custom `spanner-postgres://` scheme. This is rather un-standard and something I just made up, so the exact scheme is open for debate. We shouldn't use just `spanner://`, since that may cause confusion down the line when a driver for Spanner with the GoogleSQL dialect is added. Related discussion: https://github.com/amacneil/dbmate/discussions/369 --- README.md | 26 +++++++++++++++ docker-compose.yml | 5 +++ pkg/driver/postgres/postgres.go | 12 +++---- pkg/driver/postgres/postgres_test.go | 47 ++++++++++++++++++++++++++++ 4 files changed, 84 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 0b657af7..d3bbcfb6 100644 --- a/README.md +++ b/README.md @@ -312,6 +312,32 @@ bigquery://host:port/projectid/location/dataset?disable_auth=true `disable_auth` (optional) - Pass `true` to skip Authentication, use only for testing and connecting to emulator. +#### Spanner (PostgreSQL Interface) + +Spanner support is currently limited to databases using the [PostgreSQL Dialect](https://cloud.google.com/spanner/docs/postgresql-interface), which must be chosen during database creation. For future Spanner with GoogleSQL support, see [this discussion](https://github.com/amacneil/dbmate/discussions/369). + +Spanner with the Postgres interface requires that the [PGAdapter](https://cloud.google.com/spanner/docs/pgadapter) is running. Use the following format for `DATABASE_URL`, with the host and port set to where the PGAdapter is running: + +```shell +DATABASE_URL="spanner-postgres://127.0.0.1:5432/database_name?sslmode=disable" +``` + +Note that specifying a username and password is not necessary, as authentication is handled by the PGAdapter (they will be ignored by the PGAdapter if specified). + +Other options of the [postgres driver](#postgresql) are supported. + +Spanner also doesn't allow DDL to be executed inside explicit transactions. You must therefore specify `transaction:false` on migrations that include DDL: + +```sql +-- migrate:up transaction:false +CREATE TABLE ... + +-- migrate:down transaction:false +DROP TABLE ... +``` + +Schema dumps are not currently supported, as `pg_dump` uses functions that are not provided by Spanner. + ### Creating Migrations To create a new migration, run `dbmate new create_users_table`. You can name the migration anything you like. This will create a file `db/migrations/20151127184807_create_users_table.sql` in the current directory: diff --git a/docker-compose.yml b/docker-compose.yml index ecc07570..217ac70c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,6 +13,7 @@ services: - clickhouse-cluster-01 - clickhouse-cluster-02 - bigquery + - spanner-emulator environment: CLICKHOUSE_TEST_URL: clickhouse://clickhouse:9000/dbmate_test CLICKHOUSE_CLUSTER_01_TEST_URL: clickhouse://ch-cluster-01:9000/dbmate_test @@ -20,6 +21,7 @@ services: MYSQL_TEST_URL: mysql://root:root@mysql/dbmate_test POSTGRES_TEST_URL: postgres://postgres:postgres@postgres/dbmate_test?sslmode=disable BIGQUERY_TEST_URL: bigquery://test/us-east5/dbmate_test?disable_auth=true&endpoint=http%3A%2F%2Fbigquery%3A9050 + SPANNER_POSTGRES_TEST_URL: spanner-postgres://spanner-emulator/dbmate_test?sslmode=disable dbmate: build: @@ -69,3 +71,6 @@ services: image: ghcr.io/goccy/bigquery-emulator:0.4.4 command: | --project=test --dataset=dbmate_test + + spanner-emulator: + image: gcr.io/cloud-spanner-pg-adapter/pgadapter-emulator diff --git a/pkg/driver/postgres/postgres.go b/pkg/driver/postgres/postgres.go index 8f91a408..99f3fac7 100644 --- a/pkg/driver/postgres/postgres.go +++ b/pkg/driver/postgres/postgres.go @@ -20,6 +20,7 @@ func init() { dbmate.RegisterDriver(NewDriver, "postgres") dbmate.RegisterDriver(NewDriver, "postgresql") dbmate.RegisterDriver(NewDriver, "redshift") + dbmate.RegisterDriver(NewDriver, "spanner-postgres") } // Driver provides top level database functions @@ -73,17 +74,16 @@ func connectionString(u *url.URL) string { } if port == "" { switch u.Scheme { - case "postgresql": - fallthrough - case "postgres": - port = "5432" case "redshift": port = "5439" + default: + port = "5432" } } // generate output URL out, _ := url.Parse(u.String()) + // force scheme back to postgres if there was another postgres-compatible scheme out.Scheme = "postgres" out.Host = fmt.Sprintf("%s:%s", hostname, port) out.RawQuery = query.Encode() @@ -439,8 +439,8 @@ func (drv *Driver) quotedMigrationsTableNameParts(db dbutil.Transaction) (string return "", "", err } - // Quote identifiers for Redshift - if drv.databaseURL.Scheme == "redshift" { + // Quote identifiers for Redshift and Spanner + if drv.databaseURL.Scheme == "redshift" || drv.databaseURL.Scheme == "spanner-postgres" { return pq.QuoteIdentifier(schema), pq.QuoteIdentifier(strings.Join(tableNameParts, ".")), nil } diff --git a/pkg/driver/postgres/postgres_test.go b/pkg/driver/postgres/postgres_test.go index 9b15e65f..d2dc4709 100644 --- a/pkg/driver/postgres/postgres_test.go +++ b/pkg/driver/postgres/postgres_test.go @@ -30,6 +30,15 @@ func testRedshiftDriver(t *testing.T) *Driver { return drv.(*Driver) } +func testSpannerPostgresDriver(t *testing.T) *Driver { + // URL to the spanner pgadapter, or a locally-running spanner emulator with the pgadapter + u := dbtest.GetenvURLOrSkip(t, "SPANNER_POSTGRES_TEST_URL") + drv, err := dbmate.New(u).Driver() + require.NoError(t, err) + + return drv.(*Driver) +} + func prepTestPostgresDB(t *testing.T) *sql.DB { drv := testPostgresDriver(t) @@ -64,6 +73,21 @@ func prepRedshiftTestDB(t *testing.T, drv *Driver) *sql.DB { return db } +func prepTestSpannerPostgresDB(t *testing.T, drv *Driver) *sql.DB { + // Spanner doesn't allow running `drop database`, so we just drop the migrations + // table instead + db, err := sql.Open("postgres", connectionString(drv.databaseURL)) + require.NoError(t, err) + + _, migrationsTable, err := drv.quotedMigrationsTableNameParts(db) + require.NoError(t, err) + + _, err = db.Exec(fmt.Sprintf("drop table if exists %s", migrationsTable)) + require.NoError(t, err) + + return db +} + func TestGetDriver(t *testing.T) { db := dbmate.New(dbtest.MustParseURL(t, "postgres://")) drvInterface, err := db.Driver() @@ -107,6 +131,7 @@ func TestConnectionString(t *testing.T) { {"postgres://bob:secret@/foo?host=/var/run/postgresql", "postgres://bob:secret@:5432/foo?host=%2Fvar%2Frun%2Fpostgresql"}, // redshift default port is 5439, not 5432 {"redshift://myhost/foo", "postgres://myhost:5439/foo"}, + {"spanner-postgres://myhost/foo", "postgres://myhost:5432/foo"}, } for _, c := range cases { @@ -420,6 +445,28 @@ func TestRedshiftCreateMigrationsTable(t *testing.T) { }) } +func TestSpannerPostgresCreateMigrationsTable(t *testing.T) { + t.Run("default schema", func(t *testing.T) { + drv := testSpannerPostgresDriver(t) + db := prepTestSpannerPostgresDB(t, drv) + defer dbutil.MustClose(db) + + // migrations table should not exist + count := 0 + err := db.QueryRow("select count(*) from public.schema_migrations").Scan(&count) + require.Error(t, err, "migrations table exists when it shouldn't") + require.Contains(t, err.Error(), "pq: relation \"public.schema_migrations\" does not exist") + + // create table + err = drv.CreateMigrationsTable(db) + require.NoError(t, err) + + // migrations table should exist + err = db.QueryRow("select count(*) from public.schema_migrations").Scan(&count) + require.NoError(t, err) + }) +} + func TestPostgresSelectMigrations(t *testing.T) { drv := testPostgresDriver(t) drv.migrationsTableName = "test_migrations"