From 2dddccdd86849aaa987f64d476f15e1156745f67 Mon Sep 17 00:00:00 2001 From: Utkarsh Munjal Date: Wed, 18 Sep 2024 14:48:30 +0530 Subject: [PATCH] [BACKPORT 2024.2][#23312] YSQL: Fix backup/restore for colocated tables without tablespace information. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Summary: Backport Description: No merge conflicts Original Description: Original commit: 0dbe7d6dc54f8b60f12bce2c67405c2279a81a1a / D36859 Previously, backup and restore operations for colocated tables failed when multiple tablespaces were present, and the `--use_tablespaces` option was not used. The issue arose because each tablespace could only contain one tablegroup, leading to conflicts during the restore process. Without tablespace information, we could only restore tables to a single tablegroup (corresponding to the default tablespace). However, since a tablegroup represents a (parent) colocated tablet, restoring multiple colocated tablets required the same number of tablegroups. The revision resolves this issue by enabling multiple tablegroups in the default tablespace during restoration. This is achieved through the following steps: * Originally, the tablegroup’s name is `colocation_`, but since the concept of tablespaces doesn’t apply after restoration, we rename the tablegroup as `colocation_restore_`. The default tablegroup's name remains unchanged after the restore process. * To retain the original name of the default tablegroup, we track which tables are associated with it during the backup. This allows us to correctly restore the tablegroup's properties. **Implementation** This update enhances the backup and restore functionality for colocated tables with tablespaces by addressing the issue as follows: - **Backup Phase**: The backup process now tags each table as either belonging to the default tablegroup or not, along with the tablegroup ID in YSQLDump. The tablegroup name, originally formatted as `colocation_`, will be renamed during restoration as described above. - **Restore Phase**: During restoration, the tablegroup name and its association with the default tablegroup are retrieved from YSQLDump. If the corresponding tablegroup doesn’t already exist, it is created in the default tablespace, allowing multiple tablegroups to coexist without conflicts. - **ALTER Table**: To move a tablegroup to another tablespace after restoration, the syntax: ```ALTER TABLE ALL IN TABLESPACE pg_default COLOCATED WITH SET TABLESPACE CASCADE;``` should be used, ensuring proper management of tablegroups and tablespaces. DB-12237 Test Plan: == Automated == ./yb_build.sh --cxx-test yb-backup-cross-feature-test --gtest_filter YBBackupTestColocatedTablesWithTablespaces.TestBackupColocatedTablesWithTablespaces ./yb_build.sh --cxx-test yb-backup-cross-feature-test --gtest_filter YBBackupTestColocatedTablesWithTablespaces.TestBackupColocatedTablesWithoutTablespaces ./yb_build.sh --java-test org.yb.pgsql.TestYsqlDump#ysqlDumpColocatedTablesWithTablespaces 1. **YBBackupTestColocatedTablesWithTablespaces.TestBackupColocatedTablesWithoutTablespaces**: This test case increases the number of tablespaces (and consequently the number of tablegroups) to replicate the scenario where multiple tablespaces exist. It validates that backup and restore work correctly when performed without the `--use_tablespaces` flag. This is the primary case that this revision addresses. 2. **YBBackupTestColocatedTablesWithTablespaces.TestBackupColocatedTablesWithTablespaces**: To ensure consistency, this test also increases the number of tablespaces but verifies that backup and restore function correctly when using the `--use_tablespaces` flag. 3. **org.yb.pgsql.TestYsqlDump#ysqlDumpColocatedTablesWithTablespaces**: This test checks that the YsqlDump output matches the expected format. Specifically, in the case of the fix, the dump file should include the line: ```SELECT pg_catalog.binary_upgrade_set_next_tablegroup_default(true);``` Reviewers: skumar, aagrawal, yguan Reviewed By: skumar Subscribers: ybase, yql, mlillibridge, svc_phabricator Differential Revision: https://phorge.dev.yugabyte.com/D38205 --- .../test/java/org/yb/pgsql/TestYsqlDump.java | 59 ++- src/postgres/src/backend/commands/indexcmds.c | 31 +- src/postgres/src/backend/commands/tablecmds.c | 31 +- .../src/backend/commands/tablegroup.c | 15 + src/postgres/src/backend/commands/ybccmds.c | 25 +- .../backend/utils/adt/pg_upgrade_support.c | 11 + .../src/backend/utils/adt/ruleutils.c | 20 - src/postgres/src/bin/pg_dump/pg_dump.c | 53 ++- .../src/include/catalog/binary_upgrade.h | 2 +- src/postgres/src/include/catalog/catalog.h | 2 +- src/postgres/src/include/catalog/pg_proc.dat | 6 + .../src/include/catalog/pg_yb_migration.dat | 4 +- .../src/include/commands/tablegroup.h | 1 + ...colocated_tables_with_tablespaces.data.sql | 424 ++++++++++++++++++ ...ribe_colocated_tables_with_tablespaces.out | 39 ++ ...dump_colocated_tables_with_tablespaces.sql | 28 ++ ...ribe_colocated_tables_with_tablespaces.sql | 6 + .../yb-backup/yb-backup-cross-feature-test.cc | 91 +++- src/yb/yql/pggate/ybc_pg_typedefs.h | 1 + ...ry_upgrade_set_next_tablegroup_default.sql | 25 ++ 20 files changed, 817 insertions(+), 57 deletions(-) create mode 100644 src/postgres/src/test/regress/data/yb_ysql_dump_colocated_tables_with_tablespaces.data.sql create mode 100644 src/postgres/src/test/regress/expected/yb_ysql_dump_describe_colocated_tables_with_tablespaces.out create mode 100644 src/postgres/src/test/regress/sql/yb_ysql_dump_colocated_tables_with_tablespaces.sql create mode 100644 src/postgres/src/test/regress/sql/yb_ysql_dump_describe_colocated_tables_with_tablespaces.sql create mode 100644 src/yb/yql/pgwrapper/ysql_migrations/V57__23312__binary_upgrade_set_next_tablegroup_default.sql diff --git a/java/yb-pgsql/src/test/java/org/yb/pgsql/TestYsqlDump.java b/java/yb-pgsql/src/test/java/org/yb/pgsql/TestYsqlDump.java index c025056177ca..399ea2465e72 100644 --- a/java/yb-pgsql/src/test/java/org/yb/pgsql/TestYsqlDump.java +++ b/java/yb-pgsql/src/test/java/org/yb/pgsql/TestYsqlDump.java @@ -45,6 +45,7 @@ public class TestYsqlDump extends BasePgSQLTest { private static final Logger LOG = LoggerFactory.getLogger(TestYsqlDump.class); private static enum IncludeYbMetadata { ON, OFF } + private static enum NoTableSpaces { ON, OFF } @Override public int getTestMethodTimeoutSec() { @@ -96,7 +97,8 @@ public void ysqlDumpWithYbMetadata() throws Exception { "expected/yb_ysql_dump_describe.out" /* expectedDescribeFileRelativePath */, "results/yb_ysql_dump.out" /* outputFileRelativePath */, "results/yb_ysql_dump_describe.out" /* outputDescribeFileRelativePath */, - IncludeYbMetadata.ON); + IncludeYbMetadata.ON, + NoTableSpaces.OFF); } @Test @@ -111,7 +113,8 @@ public void ysqlDumpAllWithYbMetadata() throws Exception { "expected/yb_ysql_dumpall_describe.out" /* expectedDescribeFileRelativePath */, "results/yb_ysql_dumpall.out" /* outputFileRelativePath */, "results/yb_ysql_dumpall_describe.out" /* outputDescribeFileRelativePath */, - IncludeYbMetadata.ON); + IncludeYbMetadata.ON, + NoTableSpaces.OFF); } @Test @@ -127,7 +130,8 @@ public void ysqlDumpWithoutYbMetadata() throws Exception { "results/yb_ysql_dump_without_ybmetadata.out" /* outputFileRelativePath */, "results/yb_ysql_dump_without_ybmetadata_describe.out" /* outputDescribeFileRelativePath */, - IncludeYbMetadata.OFF); + IncludeYbMetadata.OFF, + NoTableSpaces.OFF); } @Test @@ -143,7 +147,8 @@ public void ysqlDumpAllWithoutYbMetadata() throws Exception { "results/yb_ysql_dumpall_without_ybmetadata.out" /* outputFileRelativePath */, "results/yb_ysql_dumpall_without_ybmetadata_describe.out" /* outputDescribeFileRelativePath */, - IncludeYbMetadata.OFF); + IncludeYbMetadata.OFF, + NoTableSpaces.OFF); } @Test @@ -158,7 +163,42 @@ public void ysqlDumpColocatedDB() throws Exception { /* expectedDescribeFileRelativePath */, "results/yb_ysql_dump_colocated_database.out" /* outputFileRelativePath */, "results/yb_ysql_dump_describe_colocated_database.out" /* outputDescribeFileRelativePath */, - IncludeYbMetadata.ON); + IncludeYbMetadata.ON, + NoTableSpaces.OFF); + } + + @Test + public void ysqlDumpColocatedTablesWithTablespaces() throws Exception { + markClusterNeedsRecreation(); + restartClusterWithClusterBuilder(cb -> { + cb.addCommonFlag( + "allowed_preview_flags_csv", "ysql_enable_colocated_tables_with_tablespaces"); + cb.addCommonFlag("ysql_enable_colocated_tables_with_tablespaces", "true"); + cb.addCommonTServerFlag("placement_cloud", "testCloud"); + cb.addCommonTServerFlag("placement_region", "testRegion"); + cb.perTServerFlags(Arrays.asList( + Collections.singletonMap("placement_zone", "testZone1"), + Collections.singletonMap("placement_zone", "testZone2"), + Collections.singletonMap("placement_zone", "testZone3"))); + }); + LOG.info("created mini cluster"); + ysqlDumpTester( + "ysql_dump" /* binaryName */, + "colo_tables" /* dumpedDatabaseName */, + "sql/yb_ysql_dump_colocated_tables_with_tablespaces.sql" + /* inputFileRelativePath */, + "sql/yb_ysql_dump_describe_colocated_tables_with_tablespaces.sql" + /* inputDescribeFileRelativePath */, + "data/yb_ysql_dump_colocated_tables_with_tablespaces.data.sql" + /* expectedDumpRelativePath */, + "expected/yb_ysql_dump_describe_colocated_tables_with_tablespaces.out" + /* expectedDescribeFileRelativePath */, + "results/yb_ysql_dump_colocated_tables_with_tablespaces.out" + /* outputFileRelativePath */, + "results/yb_ysql_dump_describe_colocated_tables_with_tablespaces.out" + /* outputDescribeFileRelativePath */, + IncludeYbMetadata.ON, + NoTableSpaces.ON); } @Test @@ -180,7 +220,8 @@ public void ysqlDumpLegacyColocatedDB() throws Exception { "results/yb_ysql_dump_legacy_colocated_database.out" /* outputFileRelativePath */, "results/yb_ysql_dump_describe_legacy_colocated_database.out" /* outputDescribeFileRelativePath */, - IncludeYbMetadata.ON); + IncludeYbMetadata.ON, + NoTableSpaces.OFF); } void ysqlDumpTester(final String binaryName, @@ -191,7 +232,8 @@ void ysqlDumpTester(final String binaryName, final String expectedDescribeFileRelativePath, final String outputFileRelativePath, final String outputDescribeFileRelativePath, - final IncludeYbMetadata includeYbMetadata) throws Exception { + final IncludeYbMetadata includeYbMetadata, + final NoTableSpaces noTableSpaces) throws Exception { // Location of Postgres regression tests File pgRegressDir = PgRegressBuilder.PG_REGRESS_DIR; @@ -225,6 +267,9 @@ void ysqlDumpTester(final String binaryName, if (includeYbMetadata == IncludeYbMetadata.ON) { args.add("--include-yb-metadata"); } + if (noTableSpaces == NoTableSpaces.ON) { + args.add("--no-tablespaces"); + } if (!dumpedDatabaseName.isEmpty()) { Collections.addAll(args, "-d", dumpedDatabaseName); } diff --git a/src/postgres/src/backend/commands/indexcmds.c b/src/postgres/src/backend/commands/indexcmds.c index d558748867b9..4563fb5f42d4 100644 --- a/src/postgres/src/backend/commands/indexcmds.c +++ b/src/postgres/src/backend/commands/indexcmds.c @@ -68,6 +68,7 @@ #include "utils/tqual.h" /* YB includes. */ +#include "catalog/binary_upgrade.h" #include "catalog/pg_database.h" #include "commands/progress.h" #include "commands/tablegroup.h" @@ -764,7 +765,7 @@ DefineIndex(Oid relationId, char *tablegroup_name = NULL; if (OidIsValid(tablespaceId)) - { + { /* * We look in pg_shdepend rather than directly use the derived name, * as later we might need to associate an existing implicit tablegroup to a tablespace @@ -778,9 +779,31 @@ DefineIndex(Oid relationId, tablegroup_name = OidIsValid(tablegroupId) ? get_tablegroup_name(tablegroupId) : get_implicit_tablegroup_name(tablespaceId); - } - else - { + } + else if (yb_binary_restore && OidIsValid(binary_upgrade_next_tablegroup_oid)) + { + /* + * In yb_binary_restore if tablespaceId is not valid but + * binary_upgrade_next_tablegroup_oid is valid, that implies we are + * restoring without tablespace information. + * In this case all tables are restored to default tablespace, + * while maintaining the colocation properties, and tablegroup's name + * will be colocation_restore_tablegroupId, while default tablegroup's + * name would still be default. + */ + tablegroup_name = binary_upgrade_next_tablegroup_default ? + DEFAULT_TABLEGROUP_NAME : + get_restore_tablegroup_name( + binary_upgrade_next_tablegroup_oid); + binary_upgrade_next_tablegroup_default = false; + tablegroupId = get_tablegroup_oid(tablegroup_name, true); + } + else if (yb_binary_restore && OidIsValid(tablegroupId)) + { + tablegroup_name = get_tablegroup_name(tablegroupId); + } + else + { tablegroup_name = DEFAULT_TABLEGROUP_NAME; tablegroupId = get_tablegroup_oid(tablegroup_name, true); } diff --git a/src/postgres/src/backend/commands/tablecmds.c b/src/postgres/src/backend/commands/tablecmds.c index ef621d9d442f..8d6d34e7db6a 100644 --- a/src/postgres/src/backend/commands/tablecmds.c +++ b/src/postgres/src/backend/commands/tablecmds.c @@ -12702,6 +12702,21 @@ AlterTableMoveAll(AlterTableMoveAllStmt *stmt) yb_new_tablegroup_name = get_implicit_tablegroup_name(new_tablespaceoid); yb_orig_tablegroup_oid = get_tablegroup_oid(yb_orig_tablegroup_name, true); yb_new_tablegroup_oid = get_tablegroup_oid(yb_new_tablegroup_name, true); + /* + * If a relation name is passed with the ALTER TABLE ALL ... COLOCATED WITH + * ... SET TABLESPACE ... CASCADE command then we get the relation being + * passed. + */ + if (stmt->yb_relation != NULL) + { + yb_table_oid = RangeVarGetRelid(stmt->yb_relation, NoLock, false); + yb_table_rel = RelationIdGetRelation(yb_table_oid); + yb_colocated_with_tablegroup_oid = + YbGetTableProperties(yb_table_rel)->tablegroup_oid; + RelationClose(yb_table_rel); + yb_orig_tablegroup_oid = yb_colocated_with_tablegroup_oid; + yb_orig_tablegroup_name = get_tablegroup_name(yb_orig_tablegroup_oid); + } /* * The new tablespace must not have any colocated relations present in @@ -12727,20 +12742,6 @@ AlterTableMoveAll(AlterTableMoveAllStmt *stmt) " tablespace %s", stmt->orig_tablespacename), errhint("Use ALTER ... CASCADE to move colcated relations."))); - /* - * If a relation name is passed with the ALTER TABLE ALL ... COLOCATED WITH - * ... SET TABLESPACE ... CASCADE command then we get the relation being - * passed. - */ - if (stmt->yb_relation != NULL) - { - yb_table_oid = RangeVarGetRelid(stmt->yb_relation, NoLock, false); - yb_table_rel = RelationIdGetRelation(yb_table_oid); - yb_colocated_with_tablegroup_oid = - YbGetTableProperties(yb_table_rel)->tablegroup_oid; - RelationClose(yb_table_rel); - } - if (OidIsValid(yb_table_oid) && !MyDatabaseColocated) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), @@ -12964,7 +12965,7 @@ AlterTableMoveAll(AlterTableMoveAllStmt *stmt) * database. */ if (yb_cascade && OidIsValid(new_tablespaceoid) && - OidIsValid(orig_tablespaceoid) && MyDatabaseColocated && + MyDatabaseColocated && !OidIsValid(yb_new_tablegroup_oid) && OidIsValid(yb_orig_tablegroup_oid)) { diff --git a/src/postgres/src/backend/commands/tablegroup.c b/src/postgres/src/backend/commands/tablegroup.c index 10ad1b1f235d..4322417846de 100644 --- a/src/postgres/src/backend/commands/tablegroup.c +++ b/src/postgres/src/backend/commands/tablegroup.c @@ -83,6 +83,7 @@ #include "pg_yb_utils.h" Oid binary_upgrade_next_tablegroup_oid = InvalidOid; +bool binary_upgrade_next_tablegroup_default = false; /* * Create a table group. @@ -430,6 +431,20 @@ get_implicit_tablegroup_name(Oid oidSuffix) return tablegroup_name_from_tablespace; } +char* +get_restore_tablegroup_name(Oid oidSuffix) +{ + char *restore_tablegroup_name = (char*) palloc( + ( + 19 /* strlen("colocation_restore_") */ + 10 /* Max digits in OID */ + + 1 /* Null Terminator */ + ) * sizeof(char) + ); + + sprintf(restore_tablegroup_name, "colocation_restore_%u", oidSuffix); + return restore_tablegroup_name; +} + /* * Rename tablegroup */ diff --git a/src/postgres/src/backend/commands/ybccmds.c b/src/postgres/src/backend/commands/ybccmds.c index 858e11ba87b7..31eef158a8c0 100644 --- a/src/postgres/src/backend/commands/ybccmds.c +++ b/src/postgres/src/backend/commands/ybccmds.c @@ -73,6 +73,7 @@ #include "parser/parse_utilcmd.h" /* Yugabyte includes */ +#include "catalog/binary_upgrade.h" #include "catalog/pg_yb_tablegroup.h" #include "optimizer/clauses.h" @@ -539,7 +540,7 @@ CreateTableHandleSplitOptions(YBCPgStatement handle, TupleDesc desc, } } -/* +/* * The fields pgTableId and oldRelfileNodeId are used during table rewrite. * During table rewrite, pgTableId is used to relay to DocDB the pg table OID, * so that DocDB has a mapping from the table to the correct pg table OID. @@ -552,7 +553,7 @@ CreateTableHandleSplitOptions(YBCPgStatement handle, TupleDesc desc, * However, during table rewrites, stmt->relation points to the new transient * PG relation (named pg_temp_xxxx), but we want to create a DocDB table with * the same name as the original PG relation. - */ + */ void YBCCreateTable(CreateStmt *stmt, char *tableName, char relkind, TupleDesc desc, Oid relationId, Oid namespaceId, Oid tablegroupId, @@ -744,6 +745,26 @@ YBCCreateTable(CreateStmt *stmt, char *tableName, char relkind, TupleDesc desc, get_tablegroup_name(tablegroupId) : get_implicit_tablegroup_name(tablespaceId); } + else if (yb_binary_restore && + is_colocated_tables_with_tablespace_enabled && + OidIsValid(binary_upgrade_next_tablegroup_oid)) + { + /* + * In yb_binary_restore if tablespaceId is not valid but + * binary_upgrade_next_tablegroup_oid is valid, that implies we are + * restoring without tablespace information. + * In this case all tables are restored to default tablespace, + * while maintaining the colocation properties, and tablegroup's name + * will be colocation_restore_tablegroupId, while default tablegroup's + * name would still be default. + */ + tablegroup_name = binary_upgrade_next_tablegroup_default ? + DEFAULT_TABLEGROUP_NAME : + get_restore_tablegroup_name( + binary_upgrade_next_tablegroup_oid); + binary_upgrade_next_tablegroup_default = false; + tablegroupId = get_tablegroup_oid(tablegroup_name, true); + } else { tablegroup_name = DEFAULT_TABLEGROUP_NAME; diff --git a/src/postgres/src/backend/utils/adt/pg_upgrade_support.c b/src/postgres/src/backend/utils/adt/pg_upgrade_support.c index 54af5bf3d611..ff1cdf9eddb1 100644 --- a/src/postgres/src/backend/utils/adt/pg_upgrade_support.c +++ b/src/postgres/src/backend/utils/adt/pg_upgrade_support.c @@ -219,3 +219,14 @@ binary_upgrade_set_next_tablegroup_oid(PG_FUNCTION_ARGS) PG_RETURN_VOID(); } + +Datum +binary_upgrade_set_next_tablegroup_default(PG_FUNCTION_ARGS) +{ + bool next_tablegroup_default = PG_GETARG_BOOL(0); + + CHECK_IS_BINARY_UPGRADE; + binary_upgrade_next_tablegroup_default = next_tablegroup_default; + + PG_RETURN_VOID(); +} diff --git a/src/postgres/src/backend/utils/adt/ruleutils.c b/src/postgres/src/backend/utils/adt/ruleutils.c index 5314a06cd69c..3c8e281dc6a8 100644 --- a/src/postgres/src/backend/utils/adt/ruleutils.c +++ b/src/postgres/src/backend/utils/adt/ruleutils.c @@ -1561,26 +1561,6 @@ pg_get_indexdef_worker(Oid indexrelid, int colno, appendStringInfo(&buf, " %s", range_split_clause); } } - - Relation indrel = heap_open(indrelid, AccessShareLock); - - /* - * If the indexed table's tablegroup mismatches that of an - * index table, this is a leftover from beta days of tablegroup - * feature. We cannot replicate this via DDL statement anymore. - */ - if (YbGetTableProperties(indexrel)->tablegroup_oid != - YbGetTableProperties(indrel)->tablegroup_oid) - { - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("tablegroup of an index %s does not match its " - "indexed table, this is no longer supported", - NameStr(idxrelrec->relname)), - errhint("Please drop and re-create the index."))); - } - - heap_close(indrel, AccessShareLock); } /* diff --git a/src/postgres/src/bin/pg_dump/pg_dump.c b/src/postgres/src/bin/pg_dump/pg_dump.c index b860d7028e22..7eac3e4e0998 100644 --- a/src/postgres/src/bin/pg_dump/pg_dump.c +++ b/src/postgres/src/bin/pg_dump/pg_dump.c @@ -16059,7 +16059,8 @@ dumpTableSchema(Archive *fout, TableInfo *tbinfo) && (tbinfo->relkind == RELKIND_RELATION || tbinfo->relkind == RELKIND_MATVIEW || tbinfo->relkind == RELKIND_PARTITIONED_TABLE) && yb_properties && yb_properties->is_colocated - && !simple_string_list_member(&colocated_database_tablespaces, tbinfo->reltablespace)) + && (!simple_string_list_member(&colocated_database_tablespaces, tbinfo->reltablespace) + || dopt->outputNoTablespaces)) { simple_string_list_append(&colocated_database_tablespaces, tbinfo->reltablespace); /* @@ -16072,6 +16073,16 @@ dumpTableSchema(Archive *fout, TableInfo *tbinfo) appendPQExpBuffer(q, "SELECT pg_catalog.binary_upgrade_set_next_tablegroup_oid('%u'::pg_catalog.oid);\n", yb_properties->tablegroup_oid); + if (dopt->outputNoTablespaces) + { + if(strcmp(yb_properties->tablegroup_name, "default") == 0) + { + appendPQExpBufferStr(q, + "\n-- For YB colocation backup without tablespace information, must preserve default tablegroup tables\n"); + appendPQExpBuffer(q, + "SELECT pg_catalog.binary_upgrade_set_next_tablegroup_default(true);\n"); + } + } } appendPQExpBuffer(q, "CREATE %s%s %s", @@ -16919,6 +16930,30 @@ dumpIndex(Archive *fout, IndxInfo *indxinfo) binary_upgrade_set_pg_class_oids(fout, q, indxinfo->dobj.catId.oid, true); + if (dopt->outputNoTablespaces && is_colocated_database && !is_legacy_colocated_database) + { + YbTableProperties yb_properties; + yb_properties = (YbTableProperties) pg_malloc(sizeof(YbTablePropertiesData)); + PQExpBuffer yb_reloptions = createPQExpBuffer(); + getYbTablePropertiesAndReloptions(fout, yb_properties, yb_reloptions, + indxinfo->dobj.catId.oid, indxinfo->dobj.name, tbinfo->relkind); + + if(yb_properties && yb_properties->is_colocated){ + appendPQExpBufferStr(q, + "\n-- For YB colocation backup, must preserve implicit tablegroup pg_yb_tablegroup oid\n"); + appendPQExpBuffer(q, + "SELECT pg_catalog.binary_upgrade_set_next_tablegroup_oid('%u'::pg_catalog.oid);\n", + yb_properties->tablegroup_oid); + if(strcmp(yb_properties->tablegroup_name, "default") == 0) + { + appendPQExpBufferStr(q, + "\n-- For YB colocation backup without tablespace information, must preserve default tablegroup tables\n"); + appendPQExpBuffer(q, + "SELECT pg_catalog.binary_upgrade_set_next_tablegroup_default(true);\n"); + } + } + destroyPQExpBuffer(yb_reloptions); + } /* Plain secondary index */ appendPQExpBuffer(q, "%s", indxinfo->indexdef); appendPQExpBuffer(q, ";\n"); @@ -19154,6 +19189,22 @@ getYbTablePropertiesAndReloptions(Archive *fout, YbTableProperties properties, if (properties->is_colocated && !OidIsValid(properties->colocation_id)) exit_horribly(NULL, "colocation ID is not defined for a colocated table \"%s\"\n", relname); + + if (is_colocated_database && !is_legacy_colocated_database && properties->is_colocated) + { + query = createPQExpBuffer(); + /* Get name of the tablegroup.*/ + appendPQExpBuffer(query, + "SELECT * FROM pg_yb_tablegroup WHERE oid=%u", + properties->tablegroup_oid); + res = ExecuteSqlQueryForSingleRow(fout, query->data); + int i_grpname = PQfnumber(res, "grpname"); + properties->tablegroup_name = + PQgetisnull(res, 0, i_grpname) ? "" : PQgetvalue(res, 0, i_grpname); + + PQclear(res); + destroyPQExpBuffer(query); + } } diff --git a/src/postgres/src/include/catalog/binary_upgrade.h b/src/postgres/src/include/catalog/binary_upgrade.h index 0b71ae3328ce..278d1f982eb2 100644 --- a/src/postgres/src/include/catalog/binary_upgrade.h +++ b/src/postgres/src/include/catalog/binary_upgrade.h @@ -28,5 +28,5 @@ extern PGDLLIMPORT Oid binary_upgrade_next_pg_authid_oid; extern PGDLLIMPORT bool binary_upgrade_record_init_privs; extern PGDLLIMPORT Oid binary_upgrade_next_tablegroup_oid; - +extern PGDLLIMPORT bool binary_upgrade_next_tablegroup_default; #endif /* BINARY_UPGRADE_H */ diff --git a/src/postgres/src/include/catalog/catalog.h b/src/postgres/src/include/catalog/catalog.h index 8f134c00e1a8..0a91f1361174 100644 --- a/src/postgres/src/include/catalog/catalog.h +++ b/src/postgres/src/include/catalog/catalog.h @@ -29,7 +29,7 @@ * If you increment it, make sure you didn't forget to add a new SQL migration * (see pg_yb_migration.dat and src/yb/yql/pgwrapper/ysql_migrations/README.md) */ -#define YB_LAST_USED_OID 8070 +#define YB_LAST_USED_OID 8071 extern bool IsSystemRelation(Relation relation); extern bool IsToastRelation(Relation relation); diff --git a/src/postgres/src/include/catalog/pg_proc.dat b/src/postgres/src/include/catalog/pg_proc.dat index e3727603878f..cfe96933f646 100644 --- a/src/postgres/src/include/catalog/pg_proc.dat +++ b/src/postgres/src/include/catalog/pg_proc.dat @@ -10530,4 +10530,10 @@ proargmodes => '{o,o,o,o,o,o,o,o}', proargnames => '{status,description,query_id,start_time,diagnostics_interval_sec,bind_var_query_min_duration_ms,explain_params,path}', prosrc => 'yb_get_query_diagnostics_status'}, + +{ oid => '8071', descr => 'function to preserve tablegroup name', + proname => 'binary_upgrade_set_next_tablegroup_default', provolatile => 'v', + proparallel => 'r', prorettype => 'void', proargtypes => 'bool', + prosrc => 'binary_upgrade_set_next_tablegroup_default' }, + ] diff --git a/src/postgres/src/include/catalog/pg_yb_migration.dat b/src/postgres/src/include/catalog/pg_yb_migration.dat index 42b29c11cfb3..a16cbbd096bf 100644 --- a/src/postgres/src/include/catalog/pg_yb_migration.dat +++ b/src/postgres/src/include/catalog/pg_yb_migration.dat @@ -12,7 +12,7 @@ [ # For better version control conflict detection, list latest migration filename -# here: V56__22325__yb_query_diagnostics_status.sql -{ major => '56', minor => '0', name => '', time_applied => '_null_' } +# here: V57__23312__binary_upgrade_set_next_tablegroup_default.sql +{ major => '57', minor => '0', name => '', time_applied => '_null_' } ] diff --git a/src/postgres/src/include/commands/tablegroup.h b/src/postgres/src/include/commands/tablegroup.h index 19e343682158..485bfa9c5251 100644 --- a/src/postgres/src/include/commands/tablegroup.h +++ b/src/postgres/src/include/commands/tablegroup.h @@ -49,6 +49,7 @@ extern char *get_tablegroup_name(Oid grp_oid); extern void RemoveTablegroupById(Oid grp_oid, bool remove_implicit); extern char *get_implicit_tablegroup_name(Oid oidSuffix); +extern char *get_restore_tablegroup_name(Oid oidSuffix); extern ObjectAddress RenameTablegroup(const char *oldname, const char *newname); extern ObjectAddress AlterTablegroupOwner(const char *grpname, Oid newOwnerId); extern void ybAlterTablespaceForTablegroup(const char *grpname, diff --git a/src/postgres/src/test/regress/data/yb_ysql_dump_colocated_tables_with_tablespaces.data.sql b/src/postgres/src/test/regress/data/yb_ysql_dump_colocated_tables_with_tablespaces.data.sql new file mode 100644 index 000000000000..a9652bb7ee13 --- /dev/null +++ b/src/postgres/src/test/regress/data/yb_ysql_dump_colocated_tables_with_tablespaces.data.sql @@ -0,0 +1,424 @@ +-- +-- YSQL database dump +-- + +-- Dumped from database version 11.2-YB-2.23.1.0-b0 +-- Dumped by ysql_dump version 11.2-YB-2.23.1.0-b0 + +SET yb_binary_restore = true; +SET yb_ignore_pg_class_oids = false; +SET yb_non_ddl_txn_for_sys_tables_allowed = true; +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SELECT pg_catalog.set_config('search_path', '', false); +SET check_function_bodies = false; +SET client_min_messages = warning; +SET row_security = off; + +-- Set variable use_tablespaces (if not already set) +\if :{?use_tablespaces} +\else +\set use_tablespaces true +\endif + +-- Set variable use_roles (if not already set) +\if :{?use_roles} +\else +\set use_roles true +\endif + +SET default_with_oids = false; + +-- +-- Name: t2; Type: TABLE; Schema: public; Owner: yugabyte_test +-- + + +-- For binary upgrade, must preserve pg_type oid +SELECT pg_catalog.binary_upgrade_set_next_pg_type_oid('16390'::pg_catalog.oid); + + +-- For binary upgrade, must preserve pg_type array oid +SELECT pg_catalog.binary_upgrade_set_next_array_pg_type_oid('16389'::pg_catalog.oid); + + +-- For binary upgrade, must preserve pg_class oids +SELECT pg_catalog.binary_upgrade_set_next_heap_pg_class_oid('16388'::pg_catalog.oid); + + +-- For YB colocation backup, must preserve implicit tablegroup pg_yb_tablegroup oid +SELECT pg_catalog.binary_upgrade_set_next_tablegroup_oid('16391'::pg_catalog.oid); +CREATE TABLE public.t2 ( + col integer +) +WITH (colocation_id='20001'); + + +\if :use_roles + ALTER TABLE public.t2 OWNER TO yugabyte_test; +\endif + +-- +-- Name: mv1; Type: MATERIALIZED VIEW; Schema: public; Owner: yugabyte_test +-- + + +-- For binary upgrade, must preserve pg_type oid +SELECT pg_catalog.binary_upgrade_set_next_pg_type_oid('16417'::pg_catalog.oid); + + +-- For binary upgrade, must preserve pg_type array oid +SELECT pg_catalog.binary_upgrade_set_next_array_pg_type_oid('16416'::pg_catalog.oid); + + +-- For binary upgrade, must preserve pg_class oids +SELECT pg_catalog.binary_upgrade_set_next_heap_pg_class_oid('16415'::pg_catalog.oid); + + +-- For YB colocation backup, must preserve implicit tablegroup pg_yb_tablegroup oid +SELECT pg_catalog.binary_upgrade_set_next_tablegroup_oid('16402'::pg_catalog.oid); +CREATE MATERIALIZED VIEW public.mv1 +WITH (colocation_id='20003') AS + SELECT t2.col + FROM public.t2; + + +\if :use_roles + ALTER TABLE public.mv1 OWNER TO yugabyte_test; +\endif + +-- +-- Name: t1; Type: TABLE; Schema: public; Owner: yugabyte_test +-- + + +-- For binary upgrade, must preserve pg_type oid +SELECT pg_catalog.binary_upgrade_set_next_pg_type_oid('16386'::pg_catalog.oid); + + +-- For binary upgrade, must preserve pg_type array oid +SELECT pg_catalog.binary_upgrade_set_next_array_pg_type_oid('16385'::pg_catalog.oid); + + +-- For binary upgrade, must preserve pg_class oids +SELECT pg_catalog.binary_upgrade_set_next_heap_pg_class_oid('16384'::pg_catalog.oid); + + +-- For YB colocation backup, must preserve implicit tablegroup pg_yb_tablegroup oid +SELECT pg_catalog.binary_upgrade_set_next_tablegroup_oid('16387'::pg_catalog.oid); +CREATE TABLE public.t1 ( + col integer +) +WITH (colocation_id='20001'); + + +\if :use_roles + ALTER TABLE public.t1 OWNER TO yugabyte_test; +\endif + +-- +-- Name: t3; Type: TABLE; Schema: public; Owner: yugabyte_test +-- + + +-- For binary upgrade, must preserve pg_type oid +SELECT pg_catalog.binary_upgrade_set_next_pg_type_oid('16394'::pg_catalog.oid); + + +-- For binary upgrade, must preserve pg_type array oid +SELECT pg_catalog.binary_upgrade_set_next_array_pg_type_oid('16393'::pg_catalog.oid); + + +-- For binary upgrade, must preserve pg_class oids +SELECT pg_catalog.binary_upgrade_set_next_heap_pg_class_oid('16392'::pg_catalog.oid); + + +-- For YB colocation backup, must preserve implicit tablegroup pg_yb_tablegroup oid +SELECT pg_catalog.binary_upgrade_set_next_tablegroup_oid('16387'::pg_catalog.oid); +CREATE TABLE public.t3 ( + col integer +) +WITH (colocation_id='20002'); + + +\if :use_roles + ALTER TABLE public.t3 OWNER TO yugabyte_test; +\endif + +-- +-- Name: t4; Type: TABLE; Schema: public; Owner: yugabyte_test +-- + + +-- For binary upgrade, must preserve pg_type oid +SELECT pg_catalog.binary_upgrade_set_next_pg_type_oid('16397'::pg_catalog.oid); + + +-- For binary upgrade, must preserve pg_type array oid +SELECT pg_catalog.binary_upgrade_set_next_array_pg_type_oid('16396'::pg_catalog.oid); + + +-- For binary upgrade, must preserve pg_class oids +SELECT pg_catalog.binary_upgrade_set_next_heap_pg_class_oid('16395'::pg_catalog.oid); + + +-- For YB colocation backup, must preserve implicit tablegroup pg_yb_tablegroup oid +SELECT pg_catalog.binary_upgrade_set_next_tablegroup_oid('16398'::pg_catalog.oid); + +-- For YB colocation backup without tablespace information, must preserve default tablegroup tables +SELECT pg_catalog.binary_upgrade_set_next_tablegroup_default(true); +CREATE TABLE public.t4 ( + col integer +) +WITH (colocation_id='20001'); + + +\if :use_roles + ALTER TABLE public.t4 OWNER TO yugabyte_test; +\endif + +-- +-- Name: t5; Type: TABLE; Schema: public; Owner: yugabyte_test +-- + + +-- For binary upgrade, must preserve pg_type oid +SELECT pg_catalog.binary_upgrade_set_next_pg_type_oid('16401'::pg_catalog.oid); + + +-- For binary upgrade, must preserve pg_type array oid +SELECT pg_catalog.binary_upgrade_set_next_array_pg_type_oid('16400'::pg_catalog.oid); + + +-- For binary upgrade, must preserve pg_class oids +SELECT pg_catalog.binary_upgrade_set_next_heap_pg_class_oid('16399'::pg_catalog.oid); + + +-- For YB colocation backup, must preserve implicit tablegroup pg_yb_tablegroup oid +SELECT pg_catalog.binary_upgrade_set_next_tablegroup_oid('16402'::pg_catalog.oid); +CREATE TABLE public.t5 ( + col integer +) +WITH (colocation_id='20001'); + + +\if :use_roles + ALTER TABLE public.t5 OWNER TO yugabyte_test; +\endif + +-- +-- Name: t6; Type: TABLE; Schema: public; Owner: yugabyte_test +-- + + +-- For binary upgrade, must preserve pg_type oid +SELECT pg_catalog.binary_upgrade_set_next_pg_type_oid('16405'::pg_catalog.oid); + + +-- For binary upgrade, must preserve pg_type array oid +SELECT pg_catalog.binary_upgrade_set_next_array_pg_type_oid('16404'::pg_catalog.oid); + + +-- For binary upgrade, must preserve pg_class oids +SELECT pg_catalog.binary_upgrade_set_next_heap_pg_class_oid('16403'::pg_catalog.oid); + + +-- For YB colocation backup, must preserve implicit tablegroup pg_yb_tablegroup oid +SELECT pg_catalog.binary_upgrade_set_next_tablegroup_oid('16391'::pg_catalog.oid); +CREATE TABLE public.t6 ( + col integer +) +WITH (colocation_id='20002'); + + +\if :use_roles + ALTER TABLE public.t6 OWNER TO yugabyte_test; +\endif + +-- +-- Name: t7; Type: TABLE; Schema: public; Owner: yugabyte_test +-- + + +-- For binary upgrade, must preserve pg_type oid +SELECT pg_catalog.binary_upgrade_set_next_pg_type_oid('16408'::pg_catalog.oid); + + +-- For binary upgrade, must preserve pg_type array oid +SELECT pg_catalog.binary_upgrade_set_next_array_pg_type_oid('16407'::pg_catalog.oid); + + +-- For binary upgrade, must preserve pg_class oids +SELECT pg_catalog.binary_upgrade_set_next_heap_pg_class_oid('16406'::pg_catalog.oid); + + +-- For binary upgrade, must preserve pg_class oids +SELECT pg_catalog.binary_upgrade_set_next_index_pg_class_oid('16409'::pg_catalog.oid); + + +-- For YB colocation backup, must preserve implicit tablegroup pg_yb_tablegroup oid +SELECT pg_catalog.binary_upgrade_set_next_tablegroup_oid('16387'::pg_catalog.oid); +CREATE TABLE public.t7 ( + a integer, + b integer, + c integer, + d integer NOT NULL, + CONSTRAINT t7_pkey PRIMARY KEY(d ASC) +) +WITH (colocation_id='20003'); + + +\if :use_roles + ALTER TABLE public.t7 OWNER TO yugabyte_test; +\endif + +-- +-- Data for Name: t1; Type: TABLE DATA; Schema: public; Owner: yugabyte_test +-- + +COPY public.t1 (col) FROM stdin; +\. + + +-- +-- Data for Name: t2; Type: TABLE DATA; Schema: public; Owner: yugabyte_test +-- + +COPY public.t2 (col) FROM stdin; +\. + + +-- +-- Data for Name: t3; Type: TABLE DATA; Schema: public; Owner: yugabyte_test +-- + +COPY public.t3 (col) FROM stdin; +\. + + +-- +-- Data for Name: t4; Type: TABLE DATA; Schema: public; Owner: yugabyte_test +-- + +COPY public.t4 (col) FROM stdin; +\. + + +-- +-- Data for Name: t5; Type: TABLE DATA; Schema: public; Owner: yugabyte_test +-- + +COPY public.t5 (col) FROM stdin; +\. + + +-- +-- Data for Name: t6; Type: TABLE DATA; Schema: public; Owner: yugabyte_test +-- + +COPY public.t6 (col) FROM stdin; +\. + + +-- +-- Data for Name: t7; Type: TABLE DATA; Schema: public; Owner: yugabyte_test +-- + +COPY public.t7 (a, b, c, d) FROM stdin; +\. + + +-- +-- Name: i1; Type: INDEX; Schema: public; Owner: yugabyte_test +-- + + +-- For binary upgrade, must preserve pg_class oids +SELECT pg_catalog.binary_upgrade_set_next_index_pg_class_oid('16411'::pg_catalog.oid); + + +-- For YB colocation backup, must preserve implicit tablegroup pg_yb_tablegroup oid +SELECT pg_catalog.binary_upgrade_set_next_tablegroup_oid('16398'::pg_catalog.oid); + +-- For YB colocation backup without tablespace information, must preserve default tablegroup tables +SELECT pg_catalog.binary_upgrade_set_next_tablegroup_default(true); +CREATE INDEX NONCONCURRENTLY i1 ON public.t7 USING lsm (a ASC) WITH (colocation_id=20002); + + +-- +-- Name: i2; Type: INDEX; Schema: public; Owner: yugabyte_test +-- + + +-- For binary upgrade, must preserve pg_class oids +SELECT pg_catalog.binary_upgrade_set_next_index_pg_class_oid('16412'::pg_catalog.oid); + + +-- For YB colocation backup, must preserve implicit tablegroup pg_yb_tablegroup oid +SELECT pg_catalog.binary_upgrade_set_next_tablegroup_oid('16391'::pg_catalog.oid); +CREATE INDEX NONCONCURRENTLY i2 ON public.t7 USING lsm (b ASC) WITH (colocation_id=20003); + + +-- +-- Name: i21; Type: INDEX; Schema: public; Owner: yugabyte_test +-- + + +-- For binary upgrade, must preserve pg_class oids +SELECT pg_catalog.binary_upgrade_set_next_index_pg_class_oid('16414'::pg_catalog.oid); + + +-- For YB colocation backup, must preserve implicit tablegroup pg_yb_tablegroup oid +SELECT pg_catalog.binary_upgrade_set_next_tablegroup_oid('16402'::pg_catalog.oid); +CREATE INDEX NONCONCURRENTLY i21 ON public.t2 USING lsm (col ASC) WITH (colocation_id=20002); + + +-- +-- Name: i3; Type: INDEX; Schema: public; Owner: yugabyte_test +-- + + +-- For binary upgrade, must preserve pg_class oids +SELECT pg_catalog.binary_upgrade_set_next_index_pg_class_oid('16413'::pg_catalog.oid); + + +-- For YB colocation backup, must preserve implicit tablegroup pg_yb_tablegroup oid +SELECT pg_catalog.binary_upgrade_set_next_tablegroup_oid('16391'::pg_catalog.oid); +CREATE INDEX NONCONCURRENTLY i3 ON public.t7 USING lsm (c ASC) WITH (colocation_id=20004); + + +-- +-- Name: FUNCTION pg_stat_statements_reset(); Type: ACL; Schema: pg_catalog; Owner: postgres +-- + +\if :use_roles +SELECT pg_catalog.binary_upgrade_set_record_init_privs(true); +REVOKE ALL ON FUNCTION pg_catalog.pg_stat_statements_reset() FROM PUBLIC; +SELECT pg_catalog.binary_upgrade_set_record_init_privs(false); +\endif + + +-- +-- Name: TABLE pg_stat_statements; Type: ACL; Schema: pg_catalog; Owner: postgres +-- + +\if :use_roles +SELECT pg_catalog.binary_upgrade_set_record_init_privs(true); +GRANT SELECT ON TABLE pg_catalog.pg_stat_statements TO PUBLIC; +SELECT pg_catalog.binary_upgrade_set_record_init_privs(false); +\endif + + +-- +-- Name: mv1; Type: MATERIALIZED VIEW DATA; Schema: public; Owner: yugabyte_test +-- + +REFRESH MATERIALIZED VIEW public.mv1; + + +-- +-- YSQL database dump complete +-- diff --git a/src/postgres/src/test/regress/expected/yb_ysql_dump_describe_colocated_tables_with_tablespaces.out b/src/postgres/src/test/regress/expected/yb_ysql_dump_describe_colocated_tables_with_tablespaces.out new file mode 100644 index 000000000000..5a8c2dec35d0 --- /dev/null +++ b/src/postgres/src/test/regress/expected/yb_ysql_dump_describe_colocated_tables_with_tablespaces.out @@ -0,0 +1,39 @@ + List of relations + Schema | Name | Type | Owner +--------+------+-------------------+--------------- + public | mv1 | materialized view | yugabyte_test + public | t1 | table | yugabyte_test + public | t2 | table | yugabyte_test + public | t3 | table | yugabyte_test + public | t4 | table | yugabyte_test + public | t5 | table | yugabyte_test + public | t6 | table | yugabyte_test + public | t7 | table | yugabyte_test +(8 rows) + + List of tablegroups + Name | Owner | Access privileges | Description | Tablespace | Options +------------------+----------+-------------------+-------------+------------+--------- + colocation_16385 | postgres | | | tsp1 | + colocation_16386 | postgres | | | tsp2 | + colocation_16387 | postgres | | | tsp3 | + default | postgres | | | | +(4 rows) + + List of tablegroup tables + Group Name | Group Owner | Access privileges | Group Description | Group Tablespace | Group Options | Name | Type | Owner | Rel Description | Size +------------------+-------------+-------------------+-------------------+------------------+---------------+------+-------------------+---------------+-----------------+------ + colocation_16385 | postgres | | | tsp1 | | t1 | table | yugabyte_test | | + colocation_16385 | postgres | | | tsp1 | | t3 | table | yugabyte_test | | + colocation_16385 | postgres | | | tsp1 | | t7 | table | yugabyte_test | | + colocation_16386 | postgres | | | tsp2 | | t6 | table | yugabyte_test | | + colocation_16386 | postgres | | | tsp2 | | i2 | index | yugabyte_test | | + colocation_16386 | postgres | | | tsp2 | | i3 | index | yugabyte_test | | + colocation_16386 | postgres | | | tsp2 | | t2 | table | yugabyte_test | | + colocation_16387 | postgres | | | tsp3 | | mv1 | materialized view | yugabyte_test | | + colocation_16387 | postgres | | | tsp3 | | i21 | index | yugabyte_test | | + colocation_16387 | postgres | | | tsp3 | | t5 | table | yugabyte_test | | + default | postgres | | | | | i1 | index | yugabyte_test | | + default | postgres | | | | | t4 | table | yugabyte_test | | +(12 rows) + diff --git a/src/postgres/src/test/regress/sql/yb_ysql_dump_colocated_tables_with_tablespaces.sql b/src/postgres/src/test/regress/sql/yb_ysql_dump_colocated_tables_with_tablespaces.sql new file mode 100644 index 000000000000..9a741e81bd5b --- /dev/null +++ b/src/postgres/src/test/regress/sql/yb_ysql_dump_colocated_tables_with_tablespaces.sql @@ -0,0 +1,28 @@ +CREATE TABLESPACE tsp1 + WITH (replica_placement='{"num_replicas": 1, "placement_blocks": [ + {"cloud":"testCloud","region":"testRegion","zone":"testZone1","min_num_replicas":1}]}'); + +CREATE TABLESPACE tsp2 + WITH (replica_placement='{"num_replicas": 1, "placement_blocks": [ + {"cloud":"testCloud","region":"testRegion","zone":"testZone2","min_num_replicas":1}]}'); + +CREATE TABLESPACE tsp3 + WITH (replica_placement='{"num_replicas": 1, "placement_blocks": [ + {"cloud":"testCloud","region":"testRegion","zone":"testZone3","min_num_replicas":1}]}'); + +CREATE DATABASE colo_tables WITH COLOCATION=true; +\c colo_tables; + +CREATE TABLE t1 (col int) TABLESPACE tsp1; +CREATE TABLE t2 (col int) TABLESPACE tsp2; +CREATE TABLE t3 (col int) TABLESPACE tsp1; +CREATE TABLE t4 (col int) ; +CREATE TABLE t5 (col int) TABLESPACE tsp3; +CREATE TABLE t6 (col int) TABLESPACE tsp2; + +CREATE TABLE t7 (a int, b int, c int, d int PRIMARY KEY) TABLESPACE tsp1; +CREATE INDEX i1 ON t7(a); +CREATE INDEX i2 ON t7(b) TABLESPACE tsp2; +CREATE INDEX i3 ON t7(c) TABLESPACE tsp2; +CREATE INDEX i21 on t2(col) TABLESPACE tsp3; +CREATE MATERIALIZED VIEW mv1 TABLESPACE tsp3 AS SELECT * FROM t2; diff --git a/src/postgres/src/test/regress/sql/yb_ysql_dump_describe_colocated_tables_with_tablespaces.sql b/src/postgres/src/test/regress/sql/yb_ysql_dump_describe_colocated_tables_with_tablespaces.sql new file mode 100644 index 000000000000..9599c8aaae42 --- /dev/null +++ b/src/postgres/src/test/regress/sql/yb_ysql_dump_describe_colocated_tables_with_tablespaces.sql @@ -0,0 +1,6 @@ +-- tables +\d +-- tablegroups +\dgr+ +-- tablegroups & tables +\dgrt+ diff --git a/src/yb/tools/yb-backup/yb-backup-cross-feature-test.cc b/src/yb/tools/yb-backup/yb-backup-cross-feature-test.cc index a5780cc16674..558fe25a6d6c 100644 --- a/src/yb/tools/yb-backup/yb-backup-cross-feature-test.cc +++ b/src/yb/tools/yb-backup/yb-backup-cross-feature-test.cc @@ -70,6 +70,7 @@ class YBBackupTestColocatedTablesWithTablespaces : public YBBackupTest { public: void UpdateMiniClusterOptions(ExternalMiniClusterOptions* options) override { YBBackupTest::UpdateMiniClusterOptions(options); + options->replication_factor = 3; options->extra_master_flags.emplace_back( "--allowed_preview_flags_csv=ysql_enable_colocated_tables_with_tablespaces"); options->extra_master_flags.emplace_back( @@ -1036,7 +1037,7 @@ TEST_F( SetDbName(backup_db_name); ASSERT_NO_FATALS(CreateTable("CREATE TABLE t1 (a INT PRIMARY KEY) TABLESPACE tsp1")); - ASSERT_NO_FATALS(CreateTable("CREATE TABLE t2 (a INT PRIMARY KEY) TABLESPACE tsp1")); + ASSERT_NO_FATALS(CreateTable("CREATE TABLE t2 (a INT PRIMARY KEY)")); for (int i = 0; i < 3; ++i) { ASSERT_NO_FATALS(InsertOneRow(Format("INSERT INTO t1 VALUES ($0)", i))); @@ -1080,8 +1081,7 @@ TEST_F( --------+---------+-----------+----------+--------- a | integer | | not null | Indexes: - "t2_pkey" PRIMARY KEY, lsm (a ASC), tablespace "tsp1", colocation: true - Tablespace: "tsp1" + "t2_pkey" PRIMARY KEY, lsm (a ASC), colocation: true Colocation: true )#"); } @@ -1109,17 +1109,58 @@ TEST_F( ] }' )#"; + ASSERT_OK(cluster_->AddTabletServer(ExternalMiniClusterOptions::kDefaultStartCqlProxy, + {"--placement_cloud=cloud1", "--placement_region=datacenter1", "--placement_zone=rack2"})); + const std::string placement_info_2 = R"#( + '{ + "num_replicas" : 1, + "placement_blocks": [ + { + "cloud" : "cloud1", + "region" : "datacenter1", + "zone" : "rack2", + "min_num_replicas" : 1 + } + ] + }' + )#"; + ASSERT_OK(cluster_->AddTabletServer(ExternalMiniClusterOptions::kDefaultStartCqlProxy, + {"--placement_cloud=cloud1", "--placement_region=datacenter1", "--placement_zone=rack3"})); + const std::string placement_info_3 = R"#( + '{ + "num_replicas" : 1, + "placement_blocks": [ + { + "cloud" : "cloud1", + "region" : "datacenter1", + "zone" : "rack3", + "min_num_replicas" : 1 + } + ] + }' + )#"; ASSERT_NO_FATALS(RunPsqlCommand( "CREATE TABLESPACE tsp1 WITH (replica_placement=" + placement_info_1 + ")", "CREATE TABLESPACE")); + ASSERT_NO_FATALS(RunPsqlCommand( + "CREATE TABLESPACE tsp2 WITH (replica_placement=" + placement_info_2 + ")", + "CREATE TABLESPACE")); + + ASSERT_NO_FATALS(RunPsqlCommand( + "CREATE TABLESPACE tsp3 WITH (replica_placement=" + placement_info_3 + ")", + "CREATE TABLESPACE")); + ASSERT_NO_FATALS(RunPsqlCommand( Format("CREATE DATABASE $0 WITH COLOCATION=TRUE", backup_db_name), "CREATE DATABASE")); SetDbName(backup_db_name); ASSERT_NO_FATALS(CreateTable("CREATE TABLE t1 (a INT PRIMARY KEY) TABLESPACE tsp1")); - ASSERT_NO_FATALS(CreateTable("CREATE TABLE t2 (a INT PRIMARY KEY) TABLESPACE tsp1")); + ASSERT_NO_FATALS(CreateTable("CREATE TABLE t2 (a INT PRIMARY KEY) TABLESPACE tsp2")); + ASSERT_NO_FATALS(CreateTable("CREATE TABLE t3 (a INT PRIMARY KEY) TABLESPACE tsp3")); + ASSERT_NO_FATALS(CreateTable("CREATE TABLE t4 (a INT PRIMARY KEY)")); + ASSERT_NO_FATALS(CreateTable("CREATE TABLE t5 (a INT PRIMARY KEY) TABLESPACE tsp1")); for (int i = 0; i < 3; ++i) { ASSERT_NO_FATALS(InsertOneRow(Format("INSERT INTO t1 VALUES ($0)", i))); @@ -1165,6 +1206,48 @@ TEST_F( "t2_pkey" PRIMARY KEY, lsm (a ASC), colocation: true Colocation: true )#"); + RunPsqlCommand(" \\d t3", + R"#( + Table "public.t3" + Column | Type | Collation | Nullable | Default + --------+---------+-----------+----------+--------- + a | integer | | not null | + Indexes: + "t3_pkey" PRIMARY KEY, lsm (a ASC), colocation: true + Colocation: true + )#"); + RunPsqlCommand(" \\d t4", + R"#( + Table "public.t4" + Column | Type | Collation | Nullable | Default + --------+---------+-----------+----------+--------- + a | integer | | not null | + Indexes: + "t4_pkey" PRIMARY KEY, lsm (a ASC), colocation: true + Colocation: true + )#"); + RunPsqlCommand(" \\d t5", + R"#( + Table "public.t5" + Column | Type | Collation | Nullable | Default + --------+---------+-----------+----------+--------- + a | integer | | not null | + Indexes: + "t5_pkey" PRIMARY KEY, lsm (a ASC), colocation: true + Colocation: true + )#"); + RunPsqlCommand(" \\dgrt", + R"#( + List of tablegroup tables + Group Name | Group Owner | Name | Type | Owner + --------------------------+-------------+------+-------+---------- + colocation_restore_16387 | postgres | t1 | table | yugabyte + colocation_restore_16387 | postgres | t5 | table | yugabyte + colocation_restore_16393 | postgres | t2 | table | yugabyte + colocation_restore_16399 | postgres | t3 | table | yugabyte + default | postgres | t4 | table | yugabyte + (5 rows) + )#"); } class YBBackupPartitioningVersionTest : public YBBackupTest { diff --git a/src/yb/yql/pggate/ybc_pg_typedefs.h b/src/yb/yql/pggate/ybc_pg_typedefs.h index 68ba8b1abd35..526d91441b12 100644 --- a/src/yb/yql/pggate/ybc_pg_typedefs.h +++ b/src/yb/yql/pggate/ybc_pg_typedefs.h @@ -404,6 +404,7 @@ typedef struct YbTablePropertiesData { YBCPgOid tablegroup_oid; /* InvalidOid if none */ YBCPgOid colocation_id; /* 0 if not colocated */ size_t num_range_key_columns; + char *tablegroup_name; } YbTablePropertiesData; typedef struct YbTablePropertiesData* YbTableProperties; diff --git a/src/yb/yql/pgwrapper/ysql_migrations/V57__23312__binary_upgrade_set_next_tablegroup_default.sql b/src/yb/yql/pgwrapper/ysql_migrations/V57__23312__binary_upgrade_set_next_tablegroup_default.sql new file mode 100644 index 000000000000..54b4cf0b11ea --- /dev/null +++ b/src/yb/yql/pgwrapper/ysql_migrations/V57__23312__binary_upgrade_set_next_tablegroup_default.sql @@ -0,0 +1,25 @@ +SET LOCAL yb_non_ddl_txn_for_sys_tables_allowed TO true; +INSERT INTO pg_catalog.pg_proc ( + oid, proname, pronamespace, proowner, prolang, procost, prorows, provariadic, protransform, + prokind, prosecdef, proleakproof, proisstrict, proretset, provolatile, proparallel, pronargs, + pronargdefaults, prorettype, proargtypes, proallargtypes, proargmodes, proargnames, + proargdefaults, protrftypes, prosrc, probin, proconfig, proacl +) VALUES + (8071, 'binary_upgrade_set_next_tablegroup_default', 11, 10, 12, 1, 0, 0, '-', 'f', false, false, + true, false, 'v', 'r', 1, 0, 2278, '16', NULL, NULL, NULL, + NULL, NULL, 'binary_upgrade_set_next_tablegroup_default', NULL, NULL, NULL) +ON CONFLICT DO NOTHING; +-- Create dependency records for everything we (possibly) created. +-- Since pg_depend has no OID or unique constraint, using PL/pgSQL instead. +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT FROM pg_catalog.pg_depend + WHERE refclassid = 1255 AND refobjid = 8071 + ) THEN + INSERT INTO pg_catalog.pg_depend ( + classid, objid, objsubid, refclassid, refobjid, refobjsubid, deptype + ) VALUES + (0, 0, 0, 1255, 8071, 0, 'p'); + END IF; +END $$;