diff --git a/services/configurator-app/src/main/java/com/google/cloud/pso/bq_snapshot_manager/configurator/ConfiguratorController.java b/services/configurator-app/src/main/java/com/google/cloud/pso/bq_snapshot_manager/configurator/ConfiguratorController.java index e63ea60..05f8d7a 100644 --- a/services/configurator-app/src/main/java/com/google/cloud/pso/bq_snapshot_manager/configurator/ConfiguratorController.java +++ b/services/configurator-app/src/main/java/com/google/cloud/pso/bq_snapshot_manager/configurator/ConfiguratorController.java @@ -31,6 +31,7 @@ import com.google.cloud.pso.bq_snapshot_manager.services.backup_policy.BackupPolicyService; import com.google.cloud.pso.bq_snapshot_manager.services.backup_policy.BackupPolicyServiceFireStoreImpl; import com.google.cloud.pso.bq_snapshot_manager.services.pubsub.PubSubServiceImpl; +import com.google.cloud.pso.bq_snapshot_manager.services.scan.ResourceScannerImpl; import com.google.cloud.pso.bq_snapshot_manager.services.set.GCSPersistentSetImpl; import com.google.gson.Gson; import org.springframework.boot.SpringApplication; @@ -126,6 +127,7 @@ public ResponseEntity receiveMessage(@RequestBody PubSubEvent requestBody) { new BigQueryServiceImpl(configuratorRequest.getTargetTable().getProject()), backupPolicyService, new PubSubServiceImpl(), + new ResourceScannerImpl(), new GCSPersistentSetImpl(environment.getGcsFlagsBucket()), fallbackBackupPolicy, "configurator-flags", diff --git a/services/library/src/main/java/com/google/cloud/pso/bq_snapshot_manager/functions/f02_configurator/Configurator.java b/services/library/src/main/java/com/google/cloud/pso/bq_snapshot_manager/functions/f02_configurator/Configurator.java index 4b31f96..bca3973 100644 --- a/services/library/src/main/java/com/google/cloud/pso/bq_snapshot_manager/functions/f02_configurator/Configurator.java +++ b/services/library/src/main/java/com/google/cloud/pso/bq_snapshot_manager/functions/f02_configurator/Configurator.java @@ -16,6 +16,7 @@ import com.google.cloud.pso.bq_snapshot_manager.services.backup_policy.BackupPolicyService; import com.google.cloud.pso.bq_snapshot_manager.services.pubsub.PubSubPublishResults; import com.google.cloud.pso.bq_snapshot_manager.services.pubsub.PubSubService; +import com.google.cloud.pso.bq_snapshot_manager.services.scan.ResourceScanner; import com.google.cloud.pso.bq_snapshot_manager.services.set.PersistentSet; import org.springframework.scheduling.support.CronExpression; @@ -35,6 +36,8 @@ public class Configurator { private final BackupPolicyService backupPolicyService; private final PubSubService pubSubService; + + private final ResourceScanner resourceScanner; private final PersistentSet persistentSet; private final FallbackBackupPolicy fallbackBackupPolicy; private final String persistentSetObjectPrefix; @@ -44,6 +47,7 @@ public Configurator(ConfiguratorConfig config, BigQueryService bqService, BackupPolicyService backupPolicyService, PubSubService pubSubService, + ResourceScanner resourceScanner, PersistentSet persistentSet, FallbackBackupPolicy fallbackBackupPolicy, String persistentSetObjectPrefix, @@ -52,6 +56,7 @@ public Configurator(ConfiguratorConfig config, this.bqService = bqService; this.backupPolicyService = backupPolicyService; this.pubSubService = pubSubService; + this.resourceScanner = resourceScanner; this.persistentSet = persistentSet; this.fallbackBackupPolicy = fallbackBackupPolicy; this.persistentSetObjectPrefix = persistentSetObjectPrefix; @@ -241,8 +246,16 @@ public BackupPolicy getBackupPolicy(ConfiguratorRequest request) throws IOExcept ); // find the most granular fallback policy table > dataset > project - BackupPolicy fallbackPolicy = findFallbackBackupPolicy(fallbackBackupPolicy, - request.getTargetTable()).y(); + Tuple fallbackBackupPolicyTuple = findFallbackBackupPolicy( + fallbackBackupPolicy, + request.getTargetTable()); + BackupPolicy fallbackPolicy = fallbackBackupPolicyTuple.y(); + + logger.logInfoWithTracker(request.isDryRun(), + request.getTrackingId(), + request.getTargetTable(), + String.format("Will use a %s-level fallback policy", fallbackBackupPolicyTuple.x() ) + ); // if there is a system attached policy, then only use the last_xyz fields from it and use the latest fallback policy // the last_backup_at needs to be checked to determine if we should take a backup in this run @@ -391,9 +404,9 @@ public Tuple prepareSnapshotRequests(Backu return Tuple.of(bqSnapshotRequest, gcsSnapshotRequest); } - public static Tuple findFallbackBackupPolicy(FallbackBackupPolicy + public Tuple findFallbackBackupPolicy(FallbackBackupPolicy fallbackBackupPolicy, - TableSpec tableSpec) { + TableSpec tableSpec) throws IOException { BackupPolicy tableLevel = fallbackBackupPolicy.getTableOverrides().get(tableSpec.toSqlString()); if (tableLevel != null) { @@ -414,9 +427,16 @@ public static Tuple findFallbackBackupPolicy(FallbackBacku return Tuple.of("project", projectLevel); } - //TODO: check for folder level by getting the folder id of a project from the API + // API CALL + String folderId = resourceScanner.getParentFolderId(tableSpec.getProject()); + if(folderId != null){ + BackupPolicy folderLevel = fallbackBackupPolicy.getFolderOverrides().get(folderId); + if(folderLevel != null){ + return Tuple.of("folder", folderLevel); + } + } // else return the global default policy - return Tuple.of("global", fallbackBackupPolicy.getDefaultPolicy()); + return Tuple.of("default", fallbackBackupPolicy.getDefaultPolicy()); } } diff --git a/services/library/src/main/java/com/google/cloud/pso/bq_snapshot_manager/services/scan/ResourceScanner.java b/services/library/src/main/java/com/google/cloud/pso/bq_snapshot_manager/services/scan/ResourceScanner.java index be664e9..3cd96b5 100644 --- a/services/library/src/main/java/com/google/cloud/pso/bq_snapshot_manager/services/scan/ResourceScanner.java +++ b/services/library/src/main/java/com/google/cloud/pso/bq_snapshot_manager/services/scan/ResourceScanner.java @@ -33,4 +33,6 @@ public interface ResourceScanner { // list tables under a project/dataset in the format "project.dataset.table" List listTables(String project, String dataset) throws InterruptedException, NonRetryableApplicationException; + + String getParentFolderId(String project) throws IOException; } \ No newline at end of file diff --git a/services/library/src/main/java/com/google/cloud/pso/bq_snapshot_manager/services/scan/ResourceScannerImpl.java b/services/library/src/main/java/com/google/cloud/pso/bq_snapshot_manager/services/scan/ResourceScannerImpl.java index 5bef2f2..75e0720 100644 --- a/services/library/src/main/java/com/google/cloud/pso/bq_snapshot_manager/services/scan/ResourceScannerImpl.java +++ b/services/library/src/main/java/com/google/cloud/pso/bq_snapshot_manager/services/scan/ResourceScannerImpl.java @@ -33,6 +33,7 @@ import java.util.List; import java.util.stream.Collectors; import java.util.stream.StreamSupport; + import com.google.api.services.cloudresourcemanager.v3.CloudResourceManager; import com.google.api.services.iam.v1.IamScopes; @@ -50,30 +51,27 @@ public ResourceScannerImpl() throws IOException, GeneralSecurityException { @Override public List listTables(String projectId, String datasetId) { return StreamSupport.stream(bqService.listTables(DatasetId.of(projectId, datasetId)).iterateAll().spliterator(), - false) + false) .filter(t -> t.getDefinition().getType().equals(TableDefinition.Type.TABLE)) .map(t -> String.format("%s.%s.%s", projectId, datasetId, t.getTableId().getTable())) .collect(Collectors.toCollection(ArrayList::new)); } - - @Override public List listDatasets(String projectId) { return StreamSupport.stream(bqService.listDatasets(projectId) - .iterateAll() - .spliterator(), - false) + .iterateAll() + .spliterator(), + false) .map(d -> String.format("%s.%s", projectId, d.getDatasetId().getDataset())) .collect(Collectors.toCollection(ArrayList::new)); } - @Override public List listProjects(Long folderId) throws IOException { List projects = cloudResourceManager.projects().list() - .setParent("folders/"+folderId) + .setParent("folders/" + folderId) .execute() .getProjects(); @@ -84,6 +82,30 @@ public List listProjects(Long folderId) throws IOException { } + /** + * Returns the folder id of the direct parent of a project. + * If the project doesn't have a folder, it returns null. + * If the project doesn't exsist it will throw an exception + */ + @Override + public String getParentFolderId(String project) throws IOException { + String parentFolder = cloudResourceManager + .projects() + .get(String.format("projects/%s", project)) + .execute() + .getParent(); + + if (parentFolder == null) { + return null; + } + if (parentFolder.startsWith("folders/")){ + // API returns "folders/folder_name" and we just return folder_name + return parentFolder.substring(8); + } + // in all other cases, like the parent being organizations/xyz, return null + return null; + } + public static CloudResourceManager createCloudResourceManagerService() throws IOException, GeneralSecurityException { // Use the Application Default Credentials strategy for authentication. For more info, see: diff --git a/services/library/src/test/java/com/google/cloud/pso/bq_snapshot_manager/entities/FallbackBackupPolicyTest.java b/services/library/src/test/java/com/google/cloud/pso/bq_snapshot_manager/entities/FallbackBackupPolicyTest.java index 5c81af4..c50da7e 100644 --- a/services/library/src/test/java/com/google/cloud/pso/bq_snapshot_manager/entities/FallbackBackupPolicyTest.java +++ b/services/library/src/test/java/com/google/cloud/pso/bq_snapshot_manager/entities/FallbackBackupPolicyTest.java @@ -23,8 +23,8 @@ public void testParsing() throws JsonProcessingException { " \"backup_method\": \"BigQuery Snapshot\",\n" + " \"backup_time_travel_offset_days\": \"0\",\n" + " \"bq_snapshot_expiration_days\": \"15\",\n" + - " \"backup_storage_project\": \"project\",\n" + - " \"backup_operation_project\": \"project\",\n" + + " \"backup_storage_project\": \"storage_project\",\n" + + " \"backup_operation_project\": \"operation_project\",\n" + " \"bq_snapshot_storage_dataset\": \"dataset\",\n" + " \"gcs_snapshot_storage_location\": \"gs://bla/\",\n" + " \"config_source\": \"SYSTEM\"\n" + @@ -35,8 +35,8 @@ public void testParsing() throws JsonProcessingException { " \"backup_method\": \"BigQuery Snapshot\",\n" + " \"backup_time_travel_offset_days\": \"0\",\n" + " \"bq_snapshot_expiration_days\": \"15\",\n" + - " \"backup_storage_project\": \"project\",\n" + - " \"backup_operation_project\": \"project\",\n" + + " \"backup_storage_project\": \"storage_project\",\n" + + " \"backup_operation_project\": \"operation_project\",\n" + " \"bq_snapshot_storage_dataset\": \"dataset\",\n" + " \"gcs_snapshot_storage_location\": \"gs://bla/\",\n" + " \"config_source\": \"SYSTEM\"\n" + @@ -46,8 +46,8 @@ public void testParsing() throws JsonProcessingException { " \"backup_method\": \"BigQuery Snapshot\",\n" + " \"backup_time_travel_offset_days\": \"0\",\n" + " \"bq_snapshot_expiration_days\": \"15\",\n" + - " \"backup_storage_project\": \"project\",\n" + - " \"backup_operation_project\": \"project\",\n" + + " \"backup_storage_project\": \"storage_project\",\n" + + " \"backup_operation_project\": \"operation_project\",\n" + " \"bq_snapshot_storage_dataset\": \"dataset\",\n" + " \"gcs_snapshot_storage_location\": \"gs://bla/\",\n" + " \"config_source\": \"SYSTEM\"\n" + @@ -59,8 +59,8 @@ public void testParsing() throws JsonProcessingException { " \"backup_method\": \"BigQuery Snapshot\",\n" + " \"backup_time_travel_offset_days\": \"0\",\n" + " \"bq_snapshot_expiration_days\": \"15\",\n" + - " \"backup_storage_project\": \"project\",\n" + - " \"backup_operation_project\": \"project\",\n" + + " \"backup_storage_project\": \"storage_project\",\n" + + " \"backup_operation_project\": \"operation_project\",\n" + " \"bq_snapshot_storage_dataset\": \"dataset\",\n" + " \"gcs_snapshot_storage_location\": \"gs://bla/\",\n" + " \"config_source\": \"SYSTEM\"\n" + @@ -70,8 +70,8 @@ public void testParsing() throws JsonProcessingException { " \"backup_method\": \"BigQuery Snapshot\",\n" + " \"backup_time_travel_offset_days\": \"0\",\n" + " \"bq_snapshot_expiration_days\": \"15\",\n" + - " \"backup_storage_project\": \"project\",\n" + - " \"backup_operation_project\": \"project\",\n" + + " \"backup_storage_project\": \"storage_project\",\n" + + " \"backup_operation_project\": \"operation_project\",\n" + " \"bq_snapshot_storage_dataset\": \"dataset\",\n" + " \"gcs_snapshot_storage_location\": \"gs://bla/\",\n" + " \"config_source\": \"SYSTEM\"\n" + @@ -83,8 +83,8 @@ public void testParsing() throws JsonProcessingException { " \"backup_method\": \"BigQuery Snapshot\",\n" + " \"backup_time_travel_offset_days\": \"0\",\n" + " \"bq_snapshot_expiration_days\": \"15\",\n" + - " \"backup_storage_project\": \"project\",\n" + - " \"backup_operation_project\": \"project\",\n" + + " \"backup_storage_project\": \"storage_project\",\n" + + " \"backup_operation_project\": \"operation_project\",\n" + " \"bq_snapshot_storage_dataset\": \"dataset\",\n" + " \"gcs_snapshot_storage_location\": \"gs://bla/\",\n" + " \"config_source\": \"SYSTEM\"\n" + @@ -94,8 +94,8 @@ public void testParsing() throws JsonProcessingException { " \"backup_method\": \"BigQuery Snapshot\",\n" + " \"backup_time_travel_offset_days\": \"0\",\n" + " \"bq_snapshot_expiration_days\": \"15\",\n" + - " \"backup_storage_project\": \"project\",\n" + - " \"backup_operation_project\": \"project\",\n" + + " \"backup_storage_project\": \"storage_project\",\n" + + " \"backup_operation_project\": \"operation_project\",\n" + " \"bq_snapshot_storage_dataset\": \"dataset\",\n" + " \"gcs_snapshot_storage_location\": \"gs://bla/\",\n" + " \"config_source\": \"SYSTEM\"\n" + @@ -107,8 +107,8 @@ public void testParsing() throws JsonProcessingException { " \"backup_method\": \"BigQuery Snapshot\",\n" + " \"backup_time_travel_offset_days\": \"0\",\n" + " \"bq_snapshot_expiration_days\": \"15\",\n" + - " \"backup_storage_project\": \"project\",\n" + - " \"backup_operation_project\": \"project\",\n" + + " \"backup_storage_project\": \"storage_project\",\n" + + " \"backup_operation_project\": \"operation_project\",\n" + " \"bq_snapshot_storage_dataset\": \"dataset\",\n" + " \"gcs_snapshot_storage_location\": \"gs://bla/\",\n" + " \"config_source\": \"SYSTEM\"\n" + @@ -118,8 +118,8 @@ public void testParsing() throws JsonProcessingException { " \"backup_method\": \"BigQuery Snapshot\",\n" + " \"backup_time_travel_offset_days\": \"0\",\n" + " \"bq_snapshot_expiration_days\": \"15\",\n" + - " \"backup_storage_project\": \"project\",\n" + - " \"backup_operation_project\": \"project\",\n" + + " \"backup_storage_project\": \"storage_project\",\n" + + " \"backup_operation_project\": \"operation_project\",\n" + " \"bq_snapshot_storage_dataset\": \"dataset\",\n" + " \"gcs_snapshot_storage_location\": \"gs://bla/\",\n" + " \"config_source\": \"SYSTEM\"\n" + @@ -131,9 +131,9 @@ public void testParsing() throws JsonProcessingException { BackupMethod.BIGQUERY_SNAPSHOT, TimeTravelOffsetDays.DAYS_0, BackupConfigSource.SYSTEM, - "project" + "storage_project" ) - .setBackupOperationProject("project") + .setBackupOperationProject("operation_project") .setBigQuerySnapshotExpirationDays(15.0) .setBigQuerySnapshotStorageDataset("dataset") .setGcsSnapshotStorageLocation("gs://bla/") diff --git a/services/library/src/test/java/com/google/cloud/pso/bq_snapshot_manager/functions/f02_configurator/ConfiguratorTest.java b/services/library/src/test/java/com/google/cloud/pso/bq_snapshot_manager/functions/f02_configurator/ConfiguratorTest.java index 4dce9ce..e760cbb 100644 --- a/services/library/src/test/java/com/google/cloud/pso/bq_snapshot_manager/functions/f02_configurator/ConfiguratorTest.java +++ b/services/library/src/test/java/com/google/cloud/pso/bq_snapshot_manager/functions/f02_configurator/ConfiguratorTest.java @@ -12,6 +12,7 @@ import com.google.cloud.pso.bq_snapshot_manager.helpers.Utils; import com.google.cloud.pso.bq_snapshot_manager.services.PersistentSetTestImpl; import com.google.cloud.pso.bq_snapshot_manager.services.PubSubServiceTestImpl; +import com.google.cloud.pso.bq_snapshot_manager.services.ResourceScannerTestImpl; import com.google.cloud.pso.bq_snapshot_manager.services.bq.BigQueryService; import com.google.cloud.pso.bq_snapshot_manager.services.backup_policy.BackupPolicyService; import com.google.cloud.pso.bq_snapshot_manager.services.pubsub.PubSubPublishResults; @@ -37,112 +38,12 @@ public class ConfiguratorTest { "bq_backup_manager" ); - String jsonPolicyStr = "{\n" + - " \"default_policy\": {\n" + - " \"backup_cron\": \"* * * * * *\",\n" + - " \"backup_method\": \"BigQuery Snapshot\",\n" + - " \"backup_time_travel_offset_days\": \"0\",\n" + - " \"bq_snapshot_expiration_days\": \"15\",\n" + - " \"backup_storage_project\": \"project\",\n" + - " \"backup_operation_project\": \"project\",\n" + - " \"bq_snapshot_storage_dataset\": \"dataset\"\n" + - " },\n" + - " \"folder_overrides\": {\n" + - " \"folder1\": {\n" + - " \"backup_cron\": \"*****\",\n" + - " \"backup_method\": \"BigQuery Snapshot\",\n" + - " \"backup_time_travel_offset_days\": \"0\",\n" + - " \"bq_snapshot_expiration_days\": \"15\",\n" + - " \"backup_storage_project\": \"project\",\n" + - " \"backup_operation_project\": \"project\",\n" + - " \"bq_snapshot_storage_dataset\": \"dataset\",\n" + - " \"gcs_snapshot_storage_location\": \"gs://bla/\"\n" + - " },\n" + - " \"folder2\": {\n" + - " \"backup_cron\": \"*****\",\n" + - " \"backup_method\": \"BigQuery Snapshot\",\n" + - " \"backup_time_travel_offset_days\": \"0\",\n" + - " \"bq_snapshot_expiration_days\": \"15\",\n" + - " \"backup_storage_project\": \"project\",\n" + - " \"backup_operation_project\": \"project\",\n" + - " \"bq_snapshot_storage_dataset\": \"dataset\",\n" + - " \"gcs_snapshot_storage_location\": \"gs://bla/\"\n" + - " }\n" + - " },\n" + - " \"project_overrides\": {\n" + - " \"project1\": {\n" + - " \"backup_cron\": \"*****\",\n" + - " \"backup_method\": \"BigQuery Snapshot\",\n" + - " \"backup_time_travel_offset_days\": \"0\",\n" + - " \"bq_snapshot_expiration_days\": \"15\",\n" + - " \"backup_storage_project\": \"project\",\n" + - " \"backup_operation_project\": \"project\",\n" + - " \"bq_snapshot_storage_dataset\": \"dataset\",\n" + - " \"gcs_snapshot_storage_location\": \"gs://bla/\"\n" + - " },\n" + - " \"project2\": {\n" + - " \"backup_cron\": \"*****\",\n" + - " \"backup_method\": \"BigQuery Snapshot\",\n" + - " \"backup_time_travel_offset_days\": \"0\",\n" + - " \"bq_snapshot_expiration_days\": \"15\",\n" + - " \"backup_storage_project\": \"project\",\n" + - " \"backup_operation_project\": \"project\",\n" + - " \"bq_snapshot_storage_dataset\": \"dataset\",\n" + - " \"gcs_snapshot_storage_location\": \"gs://bla/\"\n" + - " }\n" + - " },\n" + - " \"dataset_overrides\": {\n" + - " \"dataset1\": {\n" + - " \"backup_cron\": \"*****\",\n" + - " \"backup_method\": \"BigQuery Snapshot\",\n" + - " \"backup_time_travel_offset_days\": \"0\",\n" + - " \"bq_snapshot_expiration_days\": \"15\",\n" + - " \"backup_storage_project\": \"project\",\n" + - " \"backup_operation_project\": \"project\",\n" + - " \"bq_snapshot_storage_dataset\": \"dataset\",\n" + - " \"gcs_snapshot_storage_location\": \"gs://bla/\"\n" + - " },\n" + - " \"dataset2\": {\n" + - " \"backup_cron\": \"*****\",\n" + - " \"backup_method\": \"BigQuery Snapshot\",\n" + - " \"backup_time_travel_offset_days\": \"0\",\n" + - " \"bq_snapshot_expiration_days\": \"15\",\n" + - " \"backup_storage_project\": \"project\",\n" + - " \"backup_operation_project\": \"project\",\n" + - " \"bq_snapshot_storage_dataset\": \"dataset\",\n" + - " \"gcs_snapshot_storage_location\": \"gs://bla/\"\n" + - " }\n" + - " },\n" + - " \"table_overrides\": {\n" + - " \"table1\": {\n" + - " \"backup_cron\": \"*****\",\n" + - " \"backup_method\": \"BigQuery Snapshot\",\n" + - " \"backup_time_travel_offset_days\": \"0\",\n" + - " \"bq_snapshot_expiration_days\": \"15\",\n" + - " \"backup_storage_project\": \"project\",\n" + - " \"backup_operation_project\": \"project\",\n" + - " \"bq_snapshot_storage_dataset\": \"dataset\",\n" + - " \"gcs_snapshot_storage_location\": \"gs://bla/\"\n" + - " },\n" + - " \"table2\": {\n" + - " \"backup_cron\": \"*****\",\n" + - " \"backup_method\": \"BigQuery Snapshot\",\n" + - " \"backup_time_travel_offset_days\": \"0\",\n" + - " \"bq_snapshot_expiration_days\": \"15\",\n" + - " \"backup_storage_project\": \"project\",\n" + - " \"backup_operation_project\": \"project\",\n" + - " \"bq_snapshot_storage_dataset\": \"dataset\",\n" + - " \"gcs_snapshot_storage_location\": \"gs://bla/\"\n" + - " }\n" + - " }\n" + - "}"; - - BackupPolicy testPolicy = new BackupPolicy.BackupPolicyBuilder("*****", + BackupPolicy testPolicy = new BackupPolicy.BackupPolicyBuilder("* * * * * *", BackupMethod.BIGQUERY_SNAPSHOT, TimeTravelOffsetDays.DAYS_0, BackupConfigSource.SYSTEM, - "project") - .setBackupOperationProject("project") + "storage_project") + .setBackupOperationProject("operation_project") .setBigQuerySnapshotExpirationDays(15.0) .setBigQuerySnapshotStorageDataset("dataset") .setGcsSnapshotStorageLocation("gs://bla") @@ -154,7 +55,7 @@ public class ConfiguratorTest { testPolicy, // folder level Stream.of( - new AbstractMap.SimpleEntry<>("folder1", testPolicy)) + new AbstractMap.SimpleEntry<>("700", testPolicy)) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)), // project level Stream.of( @@ -170,10 +71,61 @@ public class ConfiguratorTest { ); @Test - public void testFindFallbackBackupPolicy() { + public void testFindFallbackBackupPolicy() throws IOException { + + ConfiguratorConfig config = new ConfiguratorConfig( + "test-project", + "test-bqSnapshoterTopic", + "test-gcsSnapshoterTopic", + "test-templateId", + "bq_backup_manager" + ); + + Configurator configurator = new Configurator( + config, + new BigQueryService() { + @Override + public void createSnapshot(String jobId, TableSpec sourceTable, TableSpec destinationId, Timestamp snapshotExpirationTs, String trackingId) throws InterruptedException { + + } + + @Override + public void exportToGCS(String jobId, TableSpec sourceTable, String gcsDestinationUri, GCSSnapshotFormat exportFormat, @Nullable String csvFieldDelimiter, @Nullable Boolean csvPrintHeader, @Nullable Boolean useAvroLogicalTypes, String trackingId, Map jobLabels) throws InterruptedException { + + } + + @Override + public Long getTableCreationTime(TableSpec table) { + return Timestamp.MIN_VALUE.getSeconds(); + } + }, + new BackupPolicyService() { + + @Override + public void createOrUpdateBackupPolicyForTable(TableSpec tableSpec, BackupPolicy backupPolicy) { + + } + + @Override + public @Nullable BackupPolicy getBackupPolicyForTable(TableSpec tableSpec) throws IOException, IllegalArgumentException { + return testPolicy; + } + + @Override + public void shutdown() { + + } + }, + new PubSubServiceTestImpl(), + new ResourceScannerTestImpl(), + new PersistentSetTestImpl(), + fallbackBackupPolicy, + "test-prefix", + 2 + ); // test table level - Tuple tableLevel = Configurator.findFallbackBackupPolicy( + Tuple tableLevel = configurator.findFallbackBackupPolicy( fallbackBackupPolicy, TableSpec.fromSqlString("p1.d1.t1") ); @@ -182,7 +134,7 @@ public void testFindFallbackBackupPolicy() { assertEquals(testPolicy, tableLevel.y()); // test dataset level - Tuple datasetLevel = Configurator.findFallbackBackupPolicy( + Tuple datasetLevel = configurator.findFallbackBackupPolicy( fallbackBackupPolicy, TableSpec.fromSqlString("p1.d2.t1") ); @@ -191,7 +143,7 @@ public void testFindFallbackBackupPolicy() { assertEquals(testPolicy, datasetLevel.y()); // test project level - Tuple projectLevel = Configurator.findFallbackBackupPolicy( + Tuple projectLevel = configurator.findFallbackBackupPolicy( fallbackBackupPolicy, TableSpec.fromSqlString("p2.d1.t1") ); @@ -199,13 +151,22 @@ public void testFindFallbackBackupPolicy() { assertEquals("project", projectLevel.x()); assertEquals(testPolicy, projectLevel.y()); + // test folder level + Tuple folderLevel = configurator.findFallbackBackupPolicy( + fallbackBackupPolicy, + TableSpec.fromSqlString("p3.d1.t1") + ); + + assertEquals("folder", folderLevel.x()); + assertEquals(testPolicy, folderLevel.y()); + // test default level - Tuple defaultLevel = Configurator.findFallbackBackupPolicy( + Tuple defaultLevel = configurator.findFallbackBackupPolicy( fallbackBackupPolicy, TableSpec.fromSqlString("p9.d1.t1") ); - assertEquals("global", defaultLevel.x()); + assertEquals("default", defaultLevel.x()); assertEquals(testPolicy, defaultLevel.y()); } @@ -373,8 +334,9 @@ public void shutdown() { } }, new PubSubServiceTestImpl(), + new ResourceScannerTestImpl(), new PersistentSetTestImpl(), - FallbackBackupPolicy.fromJson(jsonPolicyStr), // has no effect since we return a static BackUpPolicy from DC stub + fallbackBackupPolicy, "test-prefix", 2 ); @@ -610,24 +572,13 @@ public void testConfiguratorWithSystemBqSnapshots() throws IOException, NonRetry "1665734583289-T", "1665734583289-T-xyz", false, - new BackupPolicy.BackupPolicyBuilder( - "* * * * * *", - BackupMethod.BIGQUERY_SNAPSHOT, - TimeTravelOffsetDays.DAYS_0, - BackupConfigSource.SYSTEM, - "project" - ) - .setBackupOperationProject("project") - .setBigQuerySnapshotExpirationDays(15.0) - .setBigQuerySnapshotStorageDataset("dataset") - .setLastBackupAt(Timestamp.MIN_VALUE) - .build() + testPolicy // forming the fallback policy ); - assertEquals(expectedSnapshoterRequest, actualSnapshoterRequest); } + @Test public void testConfiguratorWithNewlyCreatedTable() throws IOException, NonRetryableApplicationException, InterruptedException { BackupPolicy backupPolicy = new BackupPolicy.BackupPolicyBuilder("* * * * * *", @@ -646,8 +597,8 @@ public void testConfiguratorWithNewlyCreatedTable() throws IOException, NonRetry TableSpec targetTable = TableSpec.fromSqlString("testProject.testDataset.testTable"); // ref point - 3 days time travel < table creation time --> table is not ready for backup - Timestamp refPoint = Timestamp.parseTimestamp("2023-01-07 00:00:00"); - Timestamp tableCreationTs = Timestamp.parseTimestamp("2023-01-06 00:00:00"); + Timestamp refPoint = Timestamp.parseTimestamp("2023-01-07T00:00:00Z"); + Timestamp tableCreationTs = Timestamp.parseTimestamp("2023-01-06T00:00:00Z"); ConfiguratorResponse configuratorResponse = executeConfigurator( targetTable, diff --git a/services/library/src/test/java/com/google/cloud/pso/bq_snapshot_manager/services/ResourceScannerTestImpl.java b/services/library/src/test/java/com/google/cloud/pso/bq_snapshot_manager/services/ResourceScannerTestImpl.java index f49c469..2da1692 100644 --- a/services/library/src/test/java/com/google/cloud/pso/bq_snapshot_manager/services/ResourceScannerTestImpl.java +++ b/services/library/src/test/java/com/google/cloud/pso/bq_snapshot_manager/services/ResourceScannerTestImpl.java @@ -1,7 +1,9 @@ package com.google.cloud.pso.bq_snapshot_manager.services; +import com.google.api.gax.rpc.UnimplementedException; import com.google.cloud.pso.bq_snapshot_manager.entities.NonRetryableApplicationException; import com.google.cloud.pso.bq_snapshot_manager.services.scan.ResourceScanner; +import com.google.cloud.pso.bq_snapshot_manager.services.scan.ResourceScannerImpl; import com.google.common.collect.Lists; import java.io.IOException; @@ -48,4 +50,16 @@ public List listTables(String project, String dataset) throws Interrupte default: return new ArrayList<>(); } } + + @Override + public String getParentFolderId(String project) throws IOException { + switch(project){ + case "p1": return "500"; + case "p2": return "600"; + case "p3": + case "p4": + return "700"; + default: return null; + } + } }