diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/EditXClusterConfig.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/EditXClusterConfig.java index 386e40a705e9..dedac43f1d76 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/EditXClusterConfig.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/EditXClusterConfig.java @@ -25,6 +25,7 @@ import javax.inject.Inject; import lombok.extern.slf4j.Slf4j; import org.springframework.util.CollectionUtils; +import org.yb.CommonTypes; import org.yb.master.MasterDdlOuterClass; import org.yb.master.MasterDdlOuterClass.ListTablesResponsePB.TableInfo; @@ -235,13 +236,31 @@ protected void addSubtasksToAddTablesToXClusterConfig( // Add the subtasks to set up replication for tables that need bootstrapping if any. if (!dbToTablesInfoMapNeedBootstrap.isEmpty()) { - // YSQL tables replication with bootstrapping can only be set up with DB granularity. The - // following subtasks remove tables in replication, so the replication can be set up again - // for all the tables in the DB including the new tables. Set tableIdsDeleteReplication = new HashSet<>(); dbToTablesInfoMapNeedBootstrap.forEach( (namespaceName, tablesInfo) -> { Set tableIdsNeedBootstrap = getTableIds(tablesInfo); + if (taskParams().getBootstrapParams().allowBootstrap && !tablesInfo.isEmpty()) { + if (tablesInfo.get(0).getTableType() == CommonTypes.TableType.PGSQL_TABLE_TYPE) { + List sourceTablesInfo = + getTableInfoListByNamespaceName( + ybService, + Universe.getOrBadRequest(xClusterConfig.getSourceUniverseUUID()), + CommonTypes.TableType.PGSQL_TABLE_TYPE, + namespaceName); + tableIdsNeedBootstrap.addAll(getTableIds(sourceTablesInfo)); + tableIdsNeedBootstrap.addAll(getTableIds(tablesInfo)); + } else { + groupByNamespaceName(requestedTableInfoList).get(namespaceName).stream() + .map(tableInfo -> getTableId(tableInfo)) + .forEach(tableIdsNeedBootstrap::add); + } + } + // YSQL tables replication with bootstrapping can only be set up with DB granularity. + // The + // following subtasks remove tables in replication, so the replication can be set up + // again + // for all the tables in the DB including the new tables. Set tableIdsNeedBootstrapInReplication = xClusterConfig.getTableIdsWithReplicationSetup( tableIdsNeedBootstrap, true /* done */); diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/XClusterConfigTaskBase.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/XClusterConfigTaskBase.java index 0effcde50e02..3b2b0c1e9718 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/XClusterConfigTaskBase.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/XClusterConfigTaskBase.java @@ -959,7 +959,8 @@ protected boolean syncReplicationSetUpStateForTables( if (bootstrapParams != null && bootstrapParams.tables != null) { // Ensure tables in bootstrapParams is a subset of requestedTableIds. - if (!requestedTableIds.containsAll(bootstrapParams.tables)) { + if (!bootstrapParams.allowBootstrap + && !requestedTableIds.containsAll(bootstrapParams.tables)) { throw new IllegalArgumentException( String.format( "The set of tables in bootstrapParams (%s) is not a subset of " @@ -977,13 +978,16 @@ protected boolean syncReplicationSetUpStateForTables( HashMap::new, (map, entry) -> map.put(entry.getKey(), entry.getValue()), HashMap::putAll); - bootstrapParams.tables = - getTableIdsWithoutTablesOnTargetInReplication( - ybService, - requestedTableInfoList, - sourceTableIdTargetTableIdWithBootstrapMap, - targetUniverse, - currentReplicationGroupName); + + if (!bootstrapParams.allowBootstrap) { + bootstrapParams.tables = + getTableIdsWithoutTablesOnTargetInReplication( + ybService, + requestedTableInfoList, + sourceTableIdTargetTableIdWithBootstrapMap, + targetUniverse, + currentReplicationGroupName); + } // If table type is YSQL and bootstrap is requested, all tables in that keyspace are selected. if (tableType == CommonTypes.TableType.PGSQL_TABLE_TYPE) { @@ -1009,8 +1013,9 @@ protected boolean syncReplicationSetUpStateForTables( .equals(namespaceId)) .map(tableInfo -> tableInfo.getId().toStringUtf8()) .collect(Collectors.toSet()); - if (tableIdsInNamespace.size() - != selectedTableIdsInNamespaceToBootstrap.size()) { + if (!bootstrapParams.allowBootstrap + && tableIdsInNamespace.size() + != selectedTableIdsInNamespaceToBootstrap.size()) { throw new IllegalArgumentException( String.format( "For YSQL tables, all the tables in a keyspace must be selected: " @@ -1385,6 +1390,44 @@ public static List getTableI .collect(Collectors.toList()); } + /** + * This method returns all the tablesInfo list present in the namespace on a universe. + * + * @param ybService The service to get a YB client from + * @param universe The universe to get the table schema information from + * @param tableType The table type to filter the tables + * @param namespaceName The namespace name to get the table schema information from + * @return A list of {@link MasterDdlOuterClass.ListTablesResponsePB.TableInfo} containing table + * info of the tables in the namespace + */ + public static List + getTableInfoListByNamespaceName( + YBClientService ybService, Universe universe, TableType tableType, String namespaceName) { + return getTableInfoList(ybService, universe).stream() + .filter(tableInfo -> tableInfo.getNamespace().getName().equals(namespaceName)) + .filter(tableInfo -> tableInfo.getTableType().equals(tableType)) + .collect(Collectors.toList()); + } + + /** + * This method returns all the tablesInfo list present in the namespace on a universe. + * + * @param ybService The service to get a YB client from + * @param universe The universe to get the table schema information from + * @param tableType The table type to filter the tables + * @param namespaceId The namespace Id to get the table schema information from + * @return A list of {@link MasterDdlOuterClass.ListTablesResponsePB.TableInfo} containing table + * info of the tables in the namespace + */ + public static List + getTableInfoListByNamespaceId( + YBClientService ybService, Universe universe, TableType tableType, String namespaceId) { + return getTableInfoList(ybService, universe).stream() + .filter(tableInfo -> tableInfo.getNamespace().getId().toStringUtf8().equals(namespaceId)) + .filter(tableInfo -> tableInfo.getTableType().equals(tableType)) + .collect(Collectors.toList()); + } + public static List getTableInfoList( YBClientService ybService, Universe universe, Collection tableIds) { return getTableInfoList(ybService, universe).stream() diff --git a/managed/src/main/java/com/yugabyte/yw/controllers/XClusterConfigController.java b/managed/src/main/java/com/yugabyte/yw/controllers/XClusterConfigController.java index 91ab1ec72583..bb39a7b0c2cc 100644 --- a/managed/src/main/java/com/yugabyte/yw/controllers/XClusterConfigController.java +++ b/managed/src/main/java/com/yugabyte/yw/controllers/XClusterConfigController.java @@ -76,6 +76,7 @@ import java.util.stream.Stream; import javax.annotation.Nullable; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; import org.yb.CommonTypes; import org.yb.CommonTypes.TableType; import org.yb.master.MasterDdlOuterClass; @@ -218,6 +219,14 @@ public Result create(UUID customerUUID, Http.Request request) { XClusterConfigTaskBase.getSourceTableIdTargetTableIdMap( requestedTableInfoList, targetTableInfoList); + if (createFormData.bootstrapParams != null + && createFormData.bootstrapParams.allowBootstrap + && !requestedTableInfoList.isEmpty()) { + createFormData.bootstrapParams.tables = + getAllBootstrapRequiredTableForXClusterRequestedTable( + ybService, requestedTableInfoList, createFormData.tables, sourceUniverse); + } + xClusterBootstrappingPreChecks( requestedTableInfoList, sourceTableInfoList, @@ -604,6 +613,18 @@ static XClusterConfigTaskParams getSetTablesTaskParams( XClusterConfigTaskBase.getSourceTableIdTargetTableIdMap( requestedTableInfoList, targetTableInfoList); + Set tablesInReplication = + new HashSet<>(CollectionUtils.union(tableIds, tableIdsToAdd)); + List allTablesInReplicationInfoList = + XClusterConfigTaskBase.getTableInfoList(ybService, sourceUniverse, tablesInReplication); + if (bootstrapParams != null + && !allTablesInReplicationInfoList.isEmpty() + && bootstrapParams.allowBootstrap) { + bootstrapParams.tables = + getAllBootstrapRequiredTableForXClusterRequestedTable( + ybService, allTablesInReplicationInfoList, tablesInReplication, sourceUniverse); + } + // We send null as sourceTableIdTargetTableIdMap because add table does not create tables // on the target universe through bootstrapping, and the user is responsible to create the // same table on the target universe. @@ -1514,7 +1535,8 @@ public static void xClusterBootstrappingPreChecks( Set requestedTableIds = XClusterConfigTaskBase.getTableIds(requestedTableInfoList); if (bootstrapParams != null && bootstrapParams.tables != null) { // Ensure tables in bootstrapParams is a subset of requestedTableIds. - if (!requestedTableIds.containsAll(bootstrapParams.tables)) { + if (!bootstrapParams.allowBootstrap + && !requestedTableIds.containsAll(bootstrapParams.tables)) { throw new IllegalArgumentException( String.format( "The set of tables in bootstrapParams (%s) is not a subset of " @@ -1587,8 +1609,9 @@ public static void xClusterBootstrappingPreChecks( .equals(namespaceId)) .map(tableInfo -> tableInfo.getId().toStringUtf8()) .collect(Collectors.toSet()); - if (tableIdsInNamespace.size() - != selectedTableIdsInNamespaceToBootstrap.size()) { + if (!bootstrapParams.allowBootstrap + && tableIdsInNamespace.size() + != selectedTableIdsInNamespaceToBootstrap.size()) { throw new IllegalArgumentException( String.format( "For YSQL tables, all the tables in a keyspace must be selected: " @@ -1600,4 +1623,40 @@ public static void xClusterBootstrappingPreChecks( } } } + + /** + * This method retrieves all the tables required for bootstrapping in a cross-cluster setup. It + * first checks the type of the tables in the replication info list. If the table type is + * PGSQL_TABLE_TYPE, it groups all tables by their namespace ID and adds all table IDs in each + * namespace to the bootstrapping list. For YCQL tables, we add all tables in the replication info + * list to the bootstrapping list. + * + * @param ybService The YB client service. + * @param tablesInReplicationInfoList A list of all tables in the replication info list. + * @param tablesInReplication A set of table IDs that are already in replication. + * @param sourceUniverse The source universe of the cross-cluster setup. + * @return A set of table IDs that are required for bootstrapping. + */ + public static Set getAllBootstrapRequiredTableForXClusterRequestedTable( + YBClientService ybService, + List tablesInReplicationInfoList, + Set tablesInReplication, + Universe sourceUniverse) { + + Set tableIdsForBootstrap = new HashSet<>(tablesInReplication); + CommonTypes.TableType tableType = tablesInReplicationInfoList.get(0).getTableType(); + if (tableType == CommonTypes.TableType.PGSQL_TABLE_TYPE) { + XClusterConfigTaskBase.groupByNamespaceId(tablesInReplicationInfoList) + .forEach( + (namespaceId, tablesInfoList) -> { + List namespaceTables = + XClusterConfigTaskBase.getTableInfoListByNamespaceId( + ybService, sourceUniverse, tableType, namespaceId); + namespaceTables.stream() + .map(tableInfo -> XClusterConfigTaskBase.getTableId(tableInfo)) + .forEach(tableIdsForBootstrap::add); + }); + } + return tableIdsForBootstrap; + } } diff --git a/managed/src/main/java/com/yugabyte/yw/forms/XClusterConfigCreateFormData.java b/managed/src/main/java/com/yugabyte/yw/forms/XClusterConfigCreateFormData.java index de65584bf7d0..3ea6f934c6f6 100644 --- a/managed/src/main/java/com/yugabyte/yw/forms/XClusterConfigCreateFormData.java +++ b/managed/src/main/java/com/yugabyte/yw/forms/XClusterConfigCreateFormData.java @@ -2,6 +2,7 @@ import com.yugabyte.yw.models.XClusterConfig; import com.yugabyte.yw.models.XClusterConfig.ConfigType; +import com.yugabyte.yw.models.common.YbaApi; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import java.util.Set; @@ -68,6 +69,14 @@ public static class BootstrapParams { @ApiModelProperty(value = "Parameters used to do Backup/restore", required = true) public BootstarpBackupParams backupRequestParams; + @ApiModelProperty( + value = + "WARNING: This is a preview API that could change. Allow backup on whole database when" + + " only set of tables require bootstrap", + required = false) + @YbaApi(visibility = YbaApi.YbaApiVisibility.PREVIEW, sinceYBAVersion = "2.23.0.0") + public boolean allowBootstrap = false; + @ApiModel(description = "Backup parameters for bootstrapping") @ToString public static class BootstarpBackupParams { diff --git a/managed/src/main/resources/swagger-strict.json b/managed/src/main/resources/swagger-strict.json index 31ecc1ff8597..f557f8350f40 100644 --- a/managed/src/main/resources/swagger-strict.json +++ b/managed/src/main/resources/swagger-strict.json @@ -2830,6 +2830,10 @@ "BootstrapParams" : { "description" : "Bootstrap parameters", "properties" : { + "allowBootstrap" : { + "description" : "WARNING: This is a preview API that could change. Allow backup on whole database when only set of tables require bootstrap", + "type" : "boolean" + }, "backupRequestParams" : { "$ref" : "#/definitions/BootstarpBackupParams", "description" : "Parameters used to do Backup/restore" diff --git a/managed/src/main/resources/swagger.json b/managed/src/main/resources/swagger.json index 93eb63cd8db8..f9d4a1ff5a21 100644 --- a/managed/src/main/resources/swagger.json +++ b/managed/src/main/resources/swagger.json @@ -2846,6 +2846,10 @@ "BootstrapParams" : { "description" : "Bootstrap parameters", "properties" : { + "allowBootstrap" : { + "description" : "WARNING: This is a preview API that could change. Allow backup on whole database when only set of tables require bootstrap", + "type" : "boolean" + }, "backupRequestParams" : { "$ref" : "#/definitions/BootstarpBackupParams", "description" : "Parameters used to do Backup/restore" diff --git a/managed/src/test/java/com/yugabyte/yw/controllers/XClusterConfigControllerTest.java b/managed/src/test/java/com/yugabyte/yw/controllers/XClusterConfigControllerTest.java index 95ed07e18a05..f3bee5fe8196 100644 --- a/managed/src/test/java/com/yugabyte/yw/controllers/XClusterConfigControllerTest.java +++ b/managed/src/test/java/com/yugabyte/yw/controllers/XClusterConfigControllerTest.java @@ -8,12 +8,14 @@ import static com.yugabyte.yw.common.ModelFactory.createUniverse; import static com.yugabyte.yw.common.ModelFactory.testCustomer; import static org.hamcrest.CoreMatchers.allOf; +import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyList; @@ -33,6 +35,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.ImmutableSet; import com.google.protobuf.ByteString; import com.yugabyte.yw.common.FakeDBApplication; import com.yugabyte.yw.common.ModelFactory; @@ -41,6 +44,7 @@ import com.yugabyte.yw.common.rbac.PermissionInfo.Action; import com.yugabyte.yw.common.rbac.PermissionInfo.ResourceType; import com.yugabyte.yw.forms.XClusterConfigCreateFormData; +import com.yugabyte.yw.forms.XClusterConfigEditFormData; import com.yugabyte.yw.metrics.MetricQueryResponse; import com.yugabyte.yw.models.Customer; import com.yugabyte.yw.models.CustomerTask; @@ -184,6 +188,20 @@ public void setUp() { when(mockService.getClient(targetUniverseMasterAddresses, targetUniverseCertificate)) .thenReturn(mockClient); + mockTableSchemaResponse(CommonTypes.TableType.YQL_TABLE_TYPE); + + apiEndpoint = "/api/customers/" + customer.getUuid() + "/xcluster_configs"; + + createFormData = new XClusterConfigCreateFormData(); + createFormData.name = configName; + createFormData.sourceUniverseUUID = sourceUniverseUUID; + createFormData.targetUniverseUUID = targetUniverseUUID; + createFormData.tables = exampleTables; + + setupMetricValues(); + } + + private void mockTableSchemaResponse(CommonTypes.TableType tableType) { GetTableSchemaResponse mockTableSchemaResponseTable1 = new GetTableSchemaResponse( 0, @@ -194,7 +212,7 @@ public void setUp() { exampleTableID1, null, true, - CommonTypes.TableType.YQL_TABLE_TYPE, + tableType, Collections.emptyList(), false); GetTableSchemaResponse mockTableSchemaResponseTable2 = @@ -207,7 +225,7 @@ public void setUp() { exampleTableID2, null, true, - CommonTypes.TableType.YQL_TABLE_TYPE, + tableType, Collections.emptyList(), false); try { @@ -219,16 +237,6 @@ public void setUp() { .thenReturn(mockTableSchemaResponseTable2); } catch (Exception ignored) { } - - apiEndpoint = "/api/customers/" + customer.getUuid() + "/xcluster_configs"; - - createFormData = new XClusterConfigCreateFormData(); - createFormData.name = configName; - createFormData.sourceUniverseUUID = sourceUniverseUUID; - createFormData.targetUniverseUUID = targetUniverseUUID; - createFormData.tables = exampleTables; - - setupMetricValues(); } private void setupMockClusterConfigWithXCluster(XClusterConfig xClusterConfig) { @@ -287,12 +295,16 @@ public void setupMetricValues() { } public void initClientGetTablesList() { + initClientGetTablesList(CommonTypes.TableType.YQL_TABLE_TYPE); + } + + public void initClientGetTablesList(CommonTypes.TableType tableType) { ListTablesResponse mockListTablesResponse = mock(ListTablesResponse.class); List tableInfoList = new ArrayList<>(); // Adding table 1. MasterDdlOuterClass.ListTablesResponsePB.TableInfo.Builder table1TableInfoBuilder = MasterDdlOuterClass.ListTablesResponsePB.TableInfo.newBuilder(); - table1TableInfoBuilder.setTableType(CommonTypes.TableType.YQL_TABLE_TYPE); + table1TableInfoBuilder.setTableType(tableType); table1TableInfoBuilder.setId(ByteString.copyFromUtf8(exampleTableID1)); table1TableInfoBuilder.setName(exampleTable1Name); table1TableInfoBuilder.setNamespace( @@ -304,7 +316,7 @@ public void initClientGetTablesList() { // Adding table 2. MasterDdlOuterClass.ListTablesResponsePB.TableInfo.Builder table2TableInfoBuilder = MasterDdlOuterClass.ListTablesResponsePB.TableInfo.newBuilder(); - table2TableInfoBuilder.setTableType(CommonTypes.TableType.YQL_TABLE_TYPE); + table2TableInfoBuilder.setTableType(tableType); table2TableInfoBuilder.setId(ByteString.copyFromUtf8(exampleTableID2)); table2TableInfoBuilder.setName(exampleTable2Name); table2TableInfoBuilder.setNamespace( @@ -1294,4 +1306,72 @@ public void testSyncWithReplGroupNameInvalidTargetUniverseUUID() { assertNoTasksCreated(); assertAuditEntry(0, customer.getUuid()); } + + @Test + public void testCreateXClusterConfigWithBootstrapRequiredOnPartialTables() { + + mockTableSchemaResponse(CommonTypes.TableType.PGSQL_TABLE_TYPE); + initClientGetTablesList(CommonTypes.TableType.PGSQL_TABLE_TYPE); + + createFormData.tables = ImmutableSet.of(exampleTableID1); + XClusterConfig xClusterConfig = + XClusterConfig.create(createFormData, XClusterConfigStatusType.Running); + + XClusterConfigCreateFormData.BootstrapParams params = + new XClusterConfigCreateFormData.BootstrapParams(); + params.allowBootstrap = false; + params.tables = Collections.singleton(exampleTableID2); + params.backupRequestParams = + new XClusterConfigCreateFormData.BootstrapParams.BootstarpBackupParams(); + params.backupRequestParams.storageConfigUUID = + ModelFactory.createS3StorageConfig(customer, "s3-config").getConfigUUID(); + XClusterConfigEditFormData editFormData = new XClusterConfigEditFormData(); + editFormData.tables = ImmutableSet.of(exampleTableID1, exampleTableID2); + editFormData.bootstrapParams = params; + + GetMasterClusterConfigResponse fakeClusterConfigResponse = + new GetMasterClusterConfigResponse( + 0, "", CatalogEntityInfo.SysClusterConfigEntryPB.getDefaultInstance(), null); + try { + when(mockClient.getMasterClusterConfig()).thenReturn(fakeClusterConfigResponse); + } catch (Exception ignore) { + } + + String editAPIEndpoint = apiEndpoint + "/" + xClusterConfig.getUuid(); + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> + doRequestWithAuthTokenAndBody( + "PUT", editAPIEndpoint, user.createAuthToken(), Json.toJson(editFormData))); + assertThat( + exception.getMessage(), + containsString("For YSQL tables, all the tables in a keyspace must be selected")); + + params.allowBootstrap = true; + Result result = + doRequestWithAuthTokenAndBody( + "PUT", editAPIEndpoint, user.createAuthToken(), Json.toJson(editFormData)); + assertOk(result); + + JsonNode resultJson = Json.parse(contentAsString(result)); + assertValue(resultJson, "taskUUID", taskUUID.toString()); + + CustomerTask customerTask = + CustomerTask.find.query().where().eq("task_uuid", taskUUID).findOne(); + assertNotNull(customerTask); + assertThat(customerTask.getCustomerUUID(), allOf(notNullValue(), equalTo(customer.getUuid()))); + assertThat( + customerTask.getTargetUUID(), + allOf(notNullValue(), equalTo(xClusterConfig.getSourceUniverseUUID()))); + assertThat(customerTask.getTaskUUID(), allOf(notNullValue(), equalTo(taskUUID))); + assertThat( + customerTask.getTargetType(), allOf(notNullValue(), equalTo(TargetType.XClusterConfig))); + assertThat(customerTask.getType(), allOf(notNullValue(), equalTo(CustomerTask.TaskType.Edit))); + assertThat(customerTask.getTargetName(), allOf(notNullValue(), equalTo(configName))); + + assertAuditEntry(1, customer.getUuid()); + + xClusterConfig.delete(); + } }