diff --git a/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderAppTest.java b/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderAppTest.java index b5b500cedae2..76ca4bdf0de5 100644 --- a/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderAppTest.java +++ b/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderAppTest.java @@ -76,7 +76,7 @@ void testBootloaderAppBlankDb() throws Exception { mockedConfigs.getConfigDatabaseUrl()) .getAndInitialize(); val configsMigrator = new ConfigsDatabaseMigrator(configDatabase, this.getClass().getName()); - assertEquals("0.30.22.001", configsMigrator.getLatestMigration().getVersion().getVersion()); + assertEquals("0.32.8.001", configsMigrator.getLatestMigration().getVersion().getVersion()); val jobsPersistence = new DefaultJobPersistence(jobDatabase); assertEquals(version, jobsPersistence.getVersion().get()); diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java b/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java index 4d7dce757fb6..6afcda2f7488 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java +++ b/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java @@ -49,6 +49,10 @@ public static T deserialize(final String jsonString, final Class klass) { } } + public static T convertValue(final Object object, final Class klass) { + return OBJECT_MAPPER.convertValue(object, klass); + } + public static JsonNode deserialize(final String jsonString) { try { return OBJECT_MAPPER.readTree(jsonString); diff --git a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/DatabaseConfigPersistence.java b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/DatabaseConfigPersistence.java index 57d2dd400f79..28baf348311b 100644 --- a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/DatabaseConfigPersistence.java +++ b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/DatabaseConfigPersistence.java @@ -4,339 +4,1417 @@ package io.airbyte.config.persistence; -import static io.airbyte.db.instance.configs.jooq.Tables.AIRBYTE_CONFIGS; +import static io.airbyte.db.instance.configs.jooq.Tables.ACTOR; +import static io.airbyte.db.instance.configs.jooq.Tables.ACTOR_DEFINITION; +import static io.airbyte.db.instance.configs.jooq.Tables.ACTOR_OAUTH_PARAMETER; +import static io.airbyte.db.instance.configs.jooq.Tables.CONNECTION; +import static io.airbyte.db.instance.configs.jooq.Tables.CONNECTION_OPERATION; +import static io.airbyte.db.instance.configs.jooq.Tables.OPERATION; +import static io.airbyte.db.instance.configs.jooq.Tables.STATE; +import static io.airbyte.db.instance.configs.jooq.Tables.WORKSPACE; import static org.jooq.impl.DSL.asterisk; -import static org.jooq.impl.DSL.field; import static org.jooq.impl.DSL.select; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Sets; +import io.airbyte.commons.enums.Enums; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.util.MoreIterators; import io.airbyte.commons.version.AirbyteVersion; import io.airbyte.config.AirbyteConfig; import io.airbyte.config.ConfigSchema; -import io.airbyte.config.ConfigSchemaMigrationSupport; import io.airbyte.config.ConfigWithMetadata; import io.airbyte.config.Configs; +import io.airbyte.config.DestinationConnection; +import io.airbyte.config.DestinationOAuthParameter; +import io.airbyte.config.JobSyncConfig.NamespaceDefinitionType; +import io.airbyte.config.Notification; +import io.airbyte.config.OperatorDbt; +import io.airbyte.config.OperatorNormalization; +import io.airbyte.config.ResourceRequirements; +import io.airbyte.config.Schedule; +import io.airbyte.config.SourceConnection; +import io.airbyte.config.SourceOAuthParameter; import io.airbyte.config.StandardDestinationDefinition; import io.airbyte.config.StandardSourceDefinition; +import io.airbyte.config.StandardSourceDefinition.SourceType; +import io.airbyte.config.StandardSync; +import io.airbyte.config.StandardSync.Status; +import io.airbyte.config.StandardSyncOperation; +import io.airbyte.config.StandardSyncOperation.OperatorType; +import io.airbyte.config.StandardSyncState; +import io.airbyte.config.StandardWorkspace; +import io.airbyte.config.State; import io.airbyte.db.Database; import io.airbyte.db.ExceptionWrappingDatabase; +import io.airbyte.db.instance.configs.jooq.enums.ActorType; +import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.ConnectorSpecification; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; import java.sql.SQLException; import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Map.Entry; +import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; import java.util.stream.Stream; -import javax.annotation.Nullable; import org.jooq.DSLContext; -import org.jooq.Field; import org.jooq.JSONB; import org.jooq.Record; import org.jooq.Record1; import org.jooq.Result; -import org.jooq.impl.SQLDataType; +import org.jooq.SelectJoinStep; +import org.jooq.TableField; +import org.jooq.impl.TableImpl; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class DatabaseConfigPersistence implements ConfigPersistence { - private static final Logger LOGGER = LoggerFactory.getLogger(DatabaseConfigPersistence.class); - private final ExceptionWrappingDatabase database; + private static final Logger LOGGER = LoggerFactory.getLogger(DatabaseConfigPersistence.class); public DatabaseConfigPersistence(final Database database) { this.database = new ExceptionWrappingDatabase(database); } - /** - * If this is a migration deployment from an old version that relies on file system config - * persistence, copy the existing configs from local files. - */ - public DatabaseConfigPersistence migrateFileConfigs(final Configs serverConfigs) throws IOException { - database.transaction(ctx -> { - final boolean isInitialized = ctx.fetchExists(AIRBYTE_CONFIGS); - if (isInitialized) { - return null; - } + public ValidatingConfigPersistence withValidation() { + return new ValidatingConfigPersistence(this); + } - final boolean hasExistingFileConfigs = FileSystemConfigPersistence.hasExistingConfigs(serverConfigs.getConfigRoot()); - if (hasExistingFileConfigs) { - LOGGER.info("Load existing local config directory into configs database"); - final ConfigPersistence fileSystemPersistence = new FileSystemConfigPersistence(serverConfigs.getConfigRoot()); - copyConfigsFromSeed(ctx, fileSystemPersistence); - } + @Override + public T getConfig(final AirbyteConfig configType, final String configId, final Class clazz) + throws ConfigNotFoundException, JsonValidationException, IOException { + if (configType == ConfigSchema.STANDARD_WORKSPACE) { + return (T) getStandardWorkspace(configId); + } else if (configType == ConfigSchema.STANDARD_SOURCE_DEFINITION) { + return (T) getStandardSourceDefinition(configId); + } else if (configType == ConfigSchema.STANDARD_DESTINATION_DEFINITION) { + return (T) getStandardDestinationDefinition(configId); + } else if (configType == ConfigSchema.SOURCE_CONNECTION) { + return (T) getSourceConnection(configId); + } else if (configType == ConfigSchema.DESTINATION_CONNECTION) { + return (T) getDestinationConnection(configId); + } else if (configType == ConfigSchema.SOURCE_OAUTH_PARAM) { + return (T) getSourceOauthParam(configId); + } else if (configType == ConfigSchema.DESTINATION_OAUTH_PARAM) { + return (T) getDestinationOauthParam(configId); + } else if (configType == ConfigSchema.STANDARD_SYNC_OPERATION) { + return (T) getStandardSyncOperation(configId); + } else if (configType == ConfigSchema.STANDARD_SYNC) { + return (T) getStandardSync(configId); + } else if (configType == ConfigSchema.STANDARD_SYNC_STATE) { + return (T) getStandardSyncState(configId); + } else { + throw new IllegalArgumentException("Unknown Config Type " + configType); + } + } - return null; - }); + private StandardWorkspace getStandardWorkspace(final String configId) throws IOException, ConfigNotFoundException { + final List> result = listStandardWorkspaceWithMetadata(Optional.of(UUID.fromString(configId))); + validate(configId, result, ConfigSchema.STANDARD_WORKSPACE); + return result.get(0).getConfig(); + } - return this; + private StandardSourceDefinition getStandardSourceDefinition(final String configId) throws IOException, ConfigNotFoundException { + final List> result = + listStandardSourceDefinitionWithMetadata(Optional.of(UUID.fromString(configId))); + validate(configId, result, ConfigSchema.STANDARD_SOURCE_DEFINITION); + return result.get(0).getConfig(); } - @Override - public void loadData(final ConfigPersistence seedConfigPersistence) throws IOException { - database.transaction(ctx -> { - updateConfigsFromSeed(ctx, seedConfigPersistence); - return null; - }); + private StandardDestinationDefinition getStandardDestinationDefinition(final String configId) throws IOException, ConfigNotFoundException { + final List> result = + listStandardDestinationDefinitionWithMetadata(Optional.of(UUID.fromString(configId))); + validate(configId, result, ConfigSchema.STANDARD_DESTINATION_DEFINITION); + return result.get(0).getConfig(); } - public Set getInUseConnectorDockerImageNames() throws IOException { - return database.transaction(this::getConnectorRepositoriesInUse); + private SourceConnection getSourceConnection(String configId) throws IOException, ConfigNotFoundException { + final List> result = listSourceConnectionWithMetadata(Optional.of(UUID.fromString(configId))); + validate(configId, result, ConfigSchema.SOURCE_CONNECTION); + return result.get(0).getConfig(); } - public ValidatingConfigPersistence withValidation() { - return new ValidatingConfigPersistence(this); + private DestinationConnection getDestinationConnection(final String configId) throws IOException, ConfigNotFoundException { + final List> result = listDestinationConnectionWithMetadata(Optional.of(UUID.fromString(configId))); + validate(configId, result, ConfigSchema.DESTINATION_CONNECTION); + return result.get(0).getConfig(); } - @Override - public T getConfig(final AirbyteConfig configType, final String configId, final Class clazz) - throws ConfigNotFoundException, JsonValidationException, IOException { + private SourceOAuthParameter getSourceOauthParam(final String configId) throws IOException, ConfigNotFoundException { + final List> result = listSourceOauthParamWithMetadata(Optional.of(UUID.fromString(configId))); + validate(configId, result, ConfigSchema.SOURCE_OAUTH_PARAM); + return result.get(0).getConfig(); + } + + private DestinationOAuthParameter getDestinationOauthParam(final String configId) throws IOException, ConfigNotFoundException { + final List> result = listDestinationOauthParamWithMetadata(Optional.of(UUID.fromString(configId))); + validate(configId, result, ConfigSchema.DESTINATION_OAUTH_PARAM); + return result.get(0).getConfig(); + } + + private StandardSyncOperation getStandardSyncOperation(final String configId) throws IOException, ConfigNotFoundException { + final List> result = listStandardSyncOperationWithMetadata(Optional.of(UUID.fromString(configId))); + validate(configId, result, ConfigSchema.STANDARD_SYNC_OPERATION); + return result.get(0).getConfig(); + } + + private StandardSync getStandardSync(final String configId) throws IOException, ConfigNotFoundException { + final List> result = listStandardSyncWithMetadata(Optional.of(UUID.fromString(configId))); + validate(configId, result, ConfigSchema.STANDARD_SYNC); + return result.get(0).getConfig(); + } + + private StandardSyncState getStandardSyncState(final String configId) throws IOException, ConfigNotFoundException { + final List> result = listStandardSyncStateWithMetadata(Optional.of(UUID.fromString(configId))); + validate(configId, result, ConfigSchema.STANDARD_SYNC_STATE); + return result.get(0).getConfig(); + } + + private List connectionOperationIds(final UUID connectionId) throws IOException { final Result result = database.query(ctx -> ctx.select(asterisk()) - .from(AIRBYTE_CONFIGS) - .where(AIRBYTE_CONFIGS.CONFIG_TYPE.eq(configType.name()), AIRBYTE_CONFIGS.CONFIG_ID.eq(configId)) + .from(CONNECTION_OPERATION) + .where(CONNECTION_OPERATION.CONNECTION_ID.eq(connectionId)) .fetch()); + final List ids = new ArrayList<>(); + for (Record record : result) { + ids.add(record.get(CONNECTION_OPERATION.OPERATION_ID)); + } + + return ids; + } + + private void validate(final String configId, final List> result, final AirbyteConfig airbyteConfig) + throws ConfigNotFoundException { if (result.isEmpty()) { - throw new ConfigNotFoundException(configType, configId); + throw new ConfigNotFoundException(airbyteConfig, configId); } else if (result.size() > 1) { - throw new IllegalStateException(String.format("Multiple %s configs found for ID %s: %s", configType, configId, result)); + throw new IllegalStateException(String.format("Multiple %s configs found for ID %s: %s", airbyteConfig, configId, result)); } - - return Jsons.deserialize(result.get(0).get(AIRBYTE_CONFIGS.CONFIG_BLOB).data(), clazz); } @Override - public List listConfigs(final AirbyteConfig configType, final Class clazz) throws IOException { - final Result results = database.query(ctx -> ctx.select(asterisk()) - .from(AIRBYTE_CONFIGS) - .where(AIRBYTE_CONFIGS.CONFIG_TYPE.eq(configType.name())) - .orderBy(AIRBYTE_CONFIGS.CONFIG_TYPE, AIRBYTE_CONFIGS.CONFIG_ID) - .fetch()); - return results.stream() - .map(record -> Jsons.deserialize(record.get(AIRBYTE_CONFIGS.CONFIG_BLOB).data(), clazz)) - .collect(Collectors.toList()); + public List listConfigs(final AirbyteConfig configType, final Class clazz) throws JsonValidationException, IOException { + final List config = new ArrayList<>(); + listConfigsWithMetadata(configType, clazz).forEach(c -> config.add(c.getConfig())); + return config; } @Override public List> listConfigsWithMetadata(final AirbyteConfig configType, final Class clazz) throws IOException { - final Result results = database.query(ctx -> ctx.select(asterisk()) - .from(AIRBYTE_CONFIGS) - .where(AIRBYTE_CONFIGS.CONFIG_TYPE.eq(configType.name())) - .orderBy(AIRBYTE_CONFIGS.CONFIG_TYPE, AIRBYTE_CONFIGS.CONFIG_ID) - .fetch()); - return results.stream() - .map(record -> new ConfigWithMetadata<>( - record.get(AIRBYTE_CONFIGS.CONFIG_ID), - record.get(AIRBYTE_CONFIGS.CONFIG_TYPE), - record.get(AIRBYTE_CONFIGS.CREATED_AT).toInstant(), - record.get(AIRBYTE_CONFIGS.UPDATED_AT).toInstant(), - Jsons.deserialize(record.get(AIRBYTE_CONFIGS.CONFIG_BLOB).data(), clazz))) - .collect(Collectors.toList()); + final List> configWithMetadata = new ArrayList<>(); + if (configType == ConfigSchema.STANDARD_WORKSPACE) { + listStandardWorkspaceWithMetadata().forEach(c -> configWithMetadata.add((ConfigWithMetadata) c)); + } else if (configType == ConfigSchema.STANDARD_SOURCE_DEFINITION) { + listStandardSourceDefinitionWithMetadata().forEach(c -> configWithMetadata.add((ConfigWithMetadata) c)); + } else if (configType == ConfigSchema.STANDARD_DESTINATION_DEFINITION) { + listStandardDestinationDefinitionWithMetadata().forEach(c -> configWithMetadata.add((ConfigWithMetadata) c)); + } else if (configType == ConfigSchema.SOURCE_CONNECTION) { + listSourceConnectionWithMetadata().forEach(c -> configWithMetadata.add((ConfigWithMetadata) c)); + } else if (configType == ConfigSchema.DESTINATION_CONNECTION) { + listDestinationConnectionWithMetadata().forEach(c -> configWithMetadata.add((ConfigWithMetadata) c)); + } else if (configType == ConfigSchema.SOURCE_OAUTH_PARAM) { + listSourceOauthParamWithMetadata().forEach(c -> configWithMetadata.add((ConfigWithMetadata) c)); + } else if (configType == ConfigSchema.DESTINATION_OAUTH_PARAM) { + listDestinationOauthParamWithMetadata().forEach(c -> configWithMetadata.add((ConfigWithMetadata) c)); + } else if (configType == ConfigSchema.STANDARD_SYNC_OPERATION) { + listStandardSyncOperationWithMetadata().forEach(c -> configWithMetadata.add((ConfigWithMetadata) c)); + } else if (configType == ConfigSchema.STANDARD_SYNC) { + listStandardSyncWithMetadata().forEach(c -> configWithMetadata.add((ConfigWithMetadata) c)); + } else if (configType == ConfigSchema.STANDARD_SYNC_STATE) { + listStandardSyncStateWithMetadata().forEach(c -> configWithMetadata.add((ConfigWithMetadata) c)); + } else { + throw new IllegalArgumentException("Unknown Config Type " + configType); + } + + return configWithMetadata; } - @Override - public void writeConfig(final AirbyteConfig configType, final String configId, final T config) throws IOException { - final Map configIdToConfig = new HashMap<>() { + private List> listStandardWorkspaceWithMetadata() throws IOException { + return listStandardWorkspaceWithMetadata(Optional.empty()); + } + + private List> listStandardWorkspaceWithMetadata(final Optional configId) throws IOException { + final Result result = database.query(ctx -> { + final SelectJoinStep query = ctx.select(asterisk()).from(WORKSPACE); + if (configId.isPresent()) { + return query.where(WORKSPACE.ID.eq(configId.get())).fetch(); + } + return query.fetch(); + }); + + final List> standardWorkspaces = new ArrayList<>(); + for (final Record record : result) { + final List notificationList = new ArrayList<>(); + final List fetchedNotifications = Jsons.deserialize(record.get(WORKSPACE.NOTIFICATIONS).data(), List.class); + for (Object notification : fetchedNotifications) { + notificationList.add(Jsons.convertValue(notification, Notification.class)); + } + final StandardWorkspace workspace = buildStandardWorkspace(record, notificationList); + standardWorkspaces.add(new ConfigWithMetadata<>( + record.get(WORKSPACE.ID).toString(), + ConfigSchema.STANDARD_WORKSPACE.name(), + record.get(WORKSPACE.CREATED_AT).toInstant(), + record.get(WORKSPACE.UPDATED_AT).toInstant(), + workspace)); + } + return standardWorkspaces; + } + + private StandardWorkspace buildStandardWorkspace(final Record record, final List notificationList) { + return new StandardWorkspace() + .withWorkspaceId(record.get(WORKSPACE.ID)) + .withName(record.get(WORKSPACE.NAME)) + .withSlug(record.get(WORKSPACE.SLUG)) + .withInitialSetupComplete(record.get(WORKSPACE.INITIAL_SETUP_COMPLETE)) + .withCustomerId(record.get(WORKSPACE.CUSTOMER_ID)) + .withEmail(record.get(WORKSPACE.EMAIL)) + .withAnonymousDataCollection(record.get(WORKSPACE.ANONYMOUS_DATA_COLLECTION)) + .withNews(record.get(WORKSPACE.SEND_NEWSLETTER)) + .withSecurityUpdates(record.get(WORKSPACE.SEND_SECURITY_UPDATES)) + .withDisplaySetupWizard(record.get(WORKSPACE.DISPLAY_SETUP_WIZARD)) + .withTombstone(record.get(WORKSPACE.TOMBSTONE)) + .withNotifications(notificationList) + .withFirstCompletedSync(record.get(WORKSPACE.FIRST_SYNC_COMPLETE)) + .withFeedbackDone(record.get(WORKSPACE.FEEDBACK_COMPLETE)); + } + + private List> listStandardSourceDefinitionWithMetadata() throws IOException { + return listStandardSourceDefinitionWithMetadata(Optional.empty()); + } + + private List> listStandardSourceDefinitionWithMetadata(final Optional configId) + throws IOException { + final Result result = database.query(ctx -> { + final SelectJoinStep query = ctx.select(asterisk()).from(ACTOR_DEFINITION); + if (configId.isPresent()) { + return query.where(ACTOR_DEFINITION.ACTOR_TYPE.eq(ActorType.source), ACTOR_DEFINITION.ID.eq(configId.get())).fetch(); + } + return query.where(ACTOR_DEFINITION.ACTOR_TYPE.eq(ActorType.source)).fetch(); + }); + + final List> standardSourceDefinitions = new ArrayList<>(); + + for (final Record record : result) { + final StandardSourceDefinition standardSourceDefinition = buildStandardSourceDefinition(record); + standardSourceDefinitions.add(new ConfigWithMetadata<>( + record.get(ACTOR_DEFINITION.ID).toString(), + ConfigSchema.STANDARD_SOURCE_DEFINITION.name(), + record.get(ACTOR_DEFINITION.CREATED_AT).toInstant(), + record.get(ACTOR_DEFINITION.UPDATED_AT).toInstant(), + standardSourceDefinition)); + } + return standardSourceDefinitions; + } + + private StandardSourceDefinition buildStandardSourceDefinition(final Record record) { + return new StandardSourceDefinition() + .withSourceDefinitionId(record.get(ACTOR_DEFINITION.ID)) + .withDockerImageTag(record.get(ACTOR_DEFINITION.DOCKER_IMAGE_TAG)) + .withIcon(record.get(ACTOR_DEFINITION.ICON)) + .withDockerRepository(record.get(ACTOR_DEFINITION.DOCKER_REPOSITORY)) + .withDocumentationUrl(record.get(ACTOR_DEFINITION.DOCUMENTATION_URL)) + .withName(record.get(ACTOR_DEFINITION.NAME)) + .withSourceType(record.get(ACTOR_DEFINITION.SOURCE_TYPE) == null ? null + : Enums.toEnum(record.get(ACTOR_DEFINITION.SOURCE_TYPE, String.class), SourceType.class).orElseThrow()) + .withSpec(Jsons.deserialize(record.get(ACTOR_DEFINITION.SPEC).data(), ConnectorSpecification.class)); + } + + private List> listStandardDestinationDefinitionWithMetadata() throws IOException { + return listStandardDestinationDefinitionWithMetadata(Optional.empty()); + } + + private List> listStandardDestinationDefinitionWithMetadata(final Optional configId) + throws IOException { + final Result result = database.query(ctx -> { + final SelectJoinStep query = ctx.select(asterisk()).from(ACTOR_DEFINITION); + if (configId.isPresent()) { + return query.where(ACTOR_DEFINITION.ACTOR_TYPE.eq(ActorType.destination), ACTOR_DEFINITION.ID.eq(configId.get())).fetch(); + } + return query.where(ACTOR_DEFINITION.ACTOR_TYPE.eq(ActorType.destination)).fetch(); + }); + + final List> standardDestinationDefinitions = new ArrayList<>(); + + for (final Record record : result) { + final StandardDestinationDefinition standardDestinationDefinition = buildStandardDestinationDefinition(record); + standardDestinationDefinitions.add(new ConfigWithMetadata<>( + record.get(ACTOR_DEFINITION.ID).toString(), + ConfigSchema.STANDARD_DESTINATION_DEFINITION.name(), + record.get(ACTOR_DEFINITION.CREATED_AT).toInstant(), + record.get(ACTOR_DEFINITION.UPDATED_AT).toInstant(), + standardDestinationDefinition)); + } + return standardDestinationDefinitions; + } - { - put(configId, config); + private StandardDestinationDefinition buildStandardDestinationDefinition(final Record record) { + return new StandardDestinationDefinition() + .withDestinationDefinitionId(record.get(ACTOR_DEFINITION.ID)) + .withDockerImageTag(record.get(ACTOR_DEFINITION.DOCKER_IMAGE_TAG)) + .withIcon(record.get(ACTOR_DEFINITION.ICON)) + .withDockerRepository(record.get(ACTOR_DEFINITION.DOCKER_REPOSITORY)) + .withDocumentationUrl(record.get(ACTOR_DEFINITION.DOCUMENTATION_URL)) + .withName(record.get(ACTOR_DEFINITION.NAME)) + .withSpec(Jsons.deserialize(record.get(ACTOR_DEFINITION.SPEC).data(), ConnectorSpecification.class)); + } + + private List> listSourceConnectionWithMetadata() throws IOException { + return listSourceConnectionWithMetadata(Optional.empty()); + } + + private List> listSourceConnectionWithMetadata(final Optional configId) throws IOException { + final Result result = database.query(ctx -> { + final SelectJoinStep query = ctx.select(asterisk()).from(ACTOR); + if (configId.isPresent()) { + return query.where(ACTOR.ACTOR_TYPE.eq(ActorType.source), ACTOR.ID.eq(configId.get())).fetch(); + } + return query.where(ACTOR.ACTOR_TYPE.eq(ActorType.source)).fetch(); + }); + + final List> sourceConnections = new ArrayList<>(); + for (final Record record : result) { + final SourceConnection sourceConnection = buildSourceConnection(record); + sourceConnections.add(new ConfigWithMetadata<>( + record.get(ACTOR.ID).toString(), + ConfigSchema.SOURCE_CONNECTION.name(), + record.get(ACTOR.CREATED_AT).toInstant(), + record.get(ACTOR.UPDATED_AT).toInstant(), + sourceConnection)); + } + return sourceConnections; + } + + private SourceConnection buildSourceConnection(final Record record) { + return new SourceConnection() + .withSourceId(record.get(ACTOR.ID)) + .withConfiguration(Jsons.deserialize(record.get(ACTOR.CONFIGURATION).data())) + .withWorkspaceId(record.get(ACTOR.WORKSPACE_ID)) + .withSourceDefinitionId(record.get(ACTOR.ACTOR_DEFINITION_ID)) + .withTombstone(record.get(ACTOR.TOMBSTONE)) + .withName(record.get(ACTOR.NAME)); + } + + private List> listDestinationConnectionWithMetadata() throws IOException { + return listDestinationConnectionWithMetadata(Optional.empty()); + } + + private List> listDestinationConnectionWithMetadata(final Optional configId) throws IOException { + final Result result = database.query(ctx -> { + final SelectJoinStep query = ctx.select(asterisk()).from(ACTOR); + if (configId.isPresent()) { + return query.where(ACTOR.ACTOR_TYPE.eq(ActorType.destination), ACTOR.ID.eq(configId.get())).fetch(); } + return query.where(ACTOR.ACTOR_TYPE.eq(ActorType.destination)).fetch(); + }); + + final List> destinationConnections = new ArrayList<>(); + for (final Record record : result) { + final DestinationConnection destinationConnection = buildDestinationConnection(record); + destinationConnections.add(new ConfigWithMetadata<>( + record.get(ACTOR.ID).toString(), + ConfigSchema.DESTINATION_CONNECTION.name(), + record.get(ACTOR.CREATED_AT).toInstant(), + record.get(ACTOR.UPDATED_AT).toInstant(), + destinationConnection)); + } + return destinationConnections; + } - }; - writeConfigs(configType, configIdToConfig); + private DestinationConnection buildDestinationConnection(final Record record) { + return new DestinationConnection() + .withDestinationId(record.get(ACTOR.ID)) + .withConfiguration(Jsons.deserialize(record.get(ACTOR.CONFIGURATION).data())) + .withWorkspaceId(record.get(ACTOR.WORKSPACE_ID)) + .withDestinationDefinitionId(record.get(ACTOR.ACTOR_DEFINITION_ID)) + .withTombstone(record.get(ACTOR.TOMBSTONE)) + .withName(record.get(ACTOR.NAME)); + } + + private List> listSourceOauthParamWithMetadata() throws IOException { + return listSourceOauthParamWithMetadata(Optional.empty()); + } + + private List> listSourceOauthParamWithMetadata(final Optional configId) throws IOException { + final Result result = database.query(ctx -> { + final SelectJoinStep query = ctx.select(asterisk()).from(ACTOR_OAUTH_PARAMETER); + if (configId.isPresent()) { + return query.where(ACTOR_OAUTH_PARAMETER.ACTOR_TYPE.eq(ActorType.source), ACTOR_OAUTH_PARAMETER.ID.eq(configId.get())).fetch(); + } + return query.where(ACTOR_OAUTH_PARAMETER.ACTOR_TYPE.eq(ActorType.source)).fetch(); + }); + + final List> sourceOAuthParameters = new ArrayList<>(); + for (final Record record : result) { + final SourceOAuthParameter sourceOAuthParameter = buildSourceOAuthParameter(record); + sourceOAuthParameters.add(new ConfigWithMetadata<>( + record.get(ACTOR_OAUTH_PARAMETER.ID).toString(), + ConfigSchema.SOURCE_OAUTH_PARAM.name(), + record.get(ACTOR_OAUTH_PARAMETER.CREATED_AT).toInstant(), + record.get(ACTOR_OAUTH_PARAMETER.UPDATED_AT).toInstant(), + sourceOAuthParameter)); + } + return sourceOAuthParameters; + } + + private SourceOAuthParameter buildSourceOAuthParameter(final Record record) { + return new SourceOAuthParameter() + .withOauthParameterId(record.get(ACTOR_OAUTH_PARAMETER.ID)) + .withConfiguration(Jsons.deserialize(record.get(ACTOR_OAUTH_PARAMETER.CONFIGURATION).data())) + .withWorkspaceId(record.get(ACTOR_OAUTH_PARAMETER.WORKSPACE_ID)) + .withSourceDefinitionId(record.get(ACTOR_OAUTH_PARAMETER.ACTOR_DEFINITION_ID)); + } + + private List> listDestinationOauthParamWithMetadata() throws IOException { + return listDestinationOauthParamWithMetadata(Optional.empty()); + } + + private List> listDestinationOauthParamWithMetadata(final Optional configId) + throws IOException { + final Result result = database.query(ctx -> { + final SelectJoinStep query = ctx.select(asterisk()).from(ACTOR_OAUTH_PARAMETER); + if (configId.isPresent()) { + return query.where(ACTOR_OAUTH_PARAMETER.ACTOR_TYPE.eq(ActorType.destination), ACTOR_OAUTH_PARAMETER.ID.eq(configId.get())).fetch(); + } + return query.where(ACTOR_OAUTH_PARAMETER.ACTOR_TYPE.eq(ActorType.destination)).fetch(); + }); + + final List> destinationOAuthParameters = new ArrayList<>(); + for (final Record record : result) { + final DestinationOAuthParameter destinationOAuthParameter = buildDestinationOAuthParameter(record); + destinationOAuthParameters.add(new ConfigWithMetadata<>( + record.get(ACTOR_OAUTH_PARAMETER.ID).toString(), + ConfigSchema.DESTINATION_OAUTH_PARAM.name(), + record.get(ACTOR_OAUTH_PARAMETER.CREATED_AT).toInstant(), + record.get(ACTOR_OAUTH_PARAMETER.UPDATED_AT).toInstant(), + destinationOAuthParameter)); + } + return destinationOAuthParameters; + } + + private DestinationOAuthParameter buildDestinationOAuthParameter(final Record record) { + return new DestinationOAuthParameter() + .withOauthParameterId(record.get(ACTOR_OAUTH_PARAMETER.ID)) + .withConfiguration(Jsons.deserialize(record.get(ACTOR_OAUTH_PARAMETER.CONFIGURATION).data())) + .withWorkspaceId(record.get(ACTOR_OAUTH_PARAMETER.WORKSPACE_ID)) + .withDestinationDefinitionId(record.get(ACTOR_OAUTH_PARAMETER.ACTOR_DEFINITION_ID)); + } + + private List> listStandardSyncOperationWithMetadata() throws IOException { + return listStandardSyncOperationWithMetadata(Optional.empty()); + } + + private List> listStandardSyncOperationWithMetadata(final Optional configId) throws IOException { + final Result result = database.query(ctx -> { + final SelectJoinStep query = ctx.select(asterisk()).from(OPERATION); + if (configId.isPresent()) { + return query.where(OPERATION.ID.eq(configId.get())).fetch(); + } + return query.fetch(); + }); + + final List> standardSyncOperations = new ArrayList<>(); + for (final Record record : result) { + final StandardSyncOperation standardSyncOperation = buildStandardSyncOperation(record); + standardSyncOperations.add(new ConfigWithMetadata<>( + record.get(OPERATION.ID).toString(), + ConfigSchema.STANDARD_SYNC_OPERATION.name(), + record.get(OPERATION.CREATED_AT).toInstant(), + record.get(OPERATION.UPDATED_AT).toInstant(), + standardSyncOperation)); + } + return standardSyncOperations; + } + + private StandardSyncOperation buildStandardSyncOperation(final Record record) { + return new StandardSyncOperation() + .withOperationId(record.get(OPERATION.ID)) + .withName(record.get(OPERATION.NAME)) + .withWorkspaceId(record.get(OPERATION.WORKSPACE_ID)) + .withOperatorType(Enums.toEnum(record.get(OPERATION.OPERATOR_TYPE, String.class), OperatorType.class).orElseThrow()) + .withOperatorNormalization(Jsons.deserialize(record.get(OPERATION.OPERATOR_NORMALIZATION).data(), OperatorNormalization.class)) + .withOperatorDbt(Jsons.deserialize(record.get(OPERATION.OPERATOR_DBT).data(), OperatorDbt.class)) + .withTombstone(record.get(OPERATION.TOMBSTONE)); + } + + private List> listStandardSyncWithMetadata() throws IOException { + return listStandardSyncWithMetadata(Optional.empty()); + } + + private List> listStandardSyncWithMetadata(final Optional configId) throws IOException { + final Result result = database.query(ctx -> { + final SelectJoinStep query = ctx.select(asterisk()).from(CONNECTION); + if (configId.isPresent()) { + return query.where(CONNECTION.ID.eq(configId.get())).fetch(); + } + return query.fetch(); + }); + + final List> standardSyncs = new ArrayList<>(); + for (final Record record : result) { + final StandardSync standardSync = buildStandardSync(record); + standardSyncs.add(new ConfigWithMetadata<>( + record.get(CONNECTION.ID).toString(), + ConfigSchema.STANDARD_SYNC.name(), + record.get(CONNECTION.CREATED_AT).toInstant(), + record.get(CONNECTION.UPDATED_AT).toInstant(), + standardSync)); + } + return standardSyncs; + } + + private StandardSync buildStandardSync(final Record record) throws IOException { + return new StandardSync() + .withConnectionId(record.get(CONNECTION.ID)) + .withNamespaceDefinition( + Enums.toEnum(record.get(CONNECTION.NAMESPACE_DEFINITION, String.class), NamespaceDefinitionType.class) + .orElseThrow()) + .withNamespaceFormat(record.get(CONNECTION.NAMESPACE_FORMAT)) + .withPrefix(record.get(CONNECTION.PREFIX)) + .withSourceId(record.get(CONNECTION.SOURCE_ID)) + .withDestinationId(record.get(CONNECTION.DESTINATION_ID)) + .withName(record.get(CONNECTION.NAME)) + .withCatalog(Jsons.deserialize(record.get(CONNECTION.CATALOG).data(), ConfiguredAirbyteCatalog.class)) + .withStatus( + record.get(CONNECTION.STATUS) == null ? null : Enums.toEnum(record.get(CONNECTION.STATUS, String.class), Status.class).orElseThrow()) + .withSchedule(Jsons.deserialize(record.get(CONNECTION.SCHEDULE).data(), Schedule.class)) + .withManual(record.get(CONNECTION.MANUAL)) + .withOperationIds(connectionOperationIds(record.get(CONNECTION.ID))) + .withResourceRequirements(Jsons.deserialize(record.get(CONNECTION.RESOURCE_REQUIREMENTS).data(), ResourceRequirements.class)); + } + + private List> listStandardSyncStateWithMetadata() throws IOException { + return listStandardSyncStateWithMetadata(Optional.empty()); + } + + private List> listStandardSyncStateWithMetadata(final Optional configId) throws IOException { + final Result result = database.query(ctx -> { + final SelectJoinStep query = ctx.select(asterisk()).from(STATE); + if (configId.isPresent()) { + return query.where(STATE.CONNECTION_ID.eq(configId.get())).fetch(); + } + return query.fetch(); + }); + final List> standardSyncStates = new ArrayList<>(); + for (final Record record : result) { + final StandardSyncState standardSyncState = buildStandardSyncState(record); + standardSyncStates.add(new ConfigWithMetadata<>( + record.get(STATE.CONNECTION_ID).toString(), + ConfigSchema.STANDARD_SYNC_STATE.name(), + record.get(STATE.CREATED_AT).toInstant(), + record.get(STATE.UPDATED_AT).toInstant(), + standardSyncState)); + } + return standardSyncStates; + } + + private StandardSyncState buildStandardSyncState(final Record record) { + return new StandardSyncState() + .withConnectionId(record.get(STATE.CONNECTION_ID)) + .withState(Jsons.deserialize(record.get(STATE.STATE_).data(), State.class)); } @Override - public void writeConfigs(final AirbyteConfig configType, final Map configs) throws IOException { + public void writeConfig(final AirbyteConfig configType, final String configId, final T config) throws JsonValidationException, IOException { + if (configType == ConfigSchema.STANDARD_WORKSPACE) { + writeStandardWorkspace(Collections.singletonList((StandardWorkspace) config)); + } else if (configType == ConfigSchema.STANDARD_SOURCE_DEFINITION) { + writeStandardSourceDefinition(Collections.singletonList((StandardSourceDefinition) config)); + } else if (configType == ConfigSchema.STANDARD_DESTINATION_DEFINITION) { + writeStandardDestinationDefinition(Collections.singletonList((StandardDestinationDefinition) config)); + } else if (configType == ConfigSchema.SOURCE_CONNECTION) { + writeSourceConnection(Collections.singletonList((SourceConnection) config)); + } else if (configType == ConfigSchema.DESTINATION_CONNECTION) { + writeDestinationConnection(Collections.singletonList((DestinationConnection) config)); + } else if (configType == ConfigSchema.SOURCE_OAUTH_PARAM) { + writeSourceOauthParameter(Collections.singletonList((SourceOAuthParameter) config)); + } else if (configType == ConfigSchema.DESTINATION_OAUTH_PARAM) { + writeDestinationOauthParameter(Collections.singletonList((DestinationOAuthParameter) config)); + } else if (configType == ConfigSchema.STANDARD_SYNC_OPERATION) { + writeStandardSyncOperation(Collections.singletonList((StandardSyncOperation) config)); + } else if (configType == ConfigSchema.STANDARD_SYNC) { + writeStandardSync(Collections.singletonList((StandardSync) config)); + } else if (configType == ConfigSchema.STANDARD_SYNC_STATE) { + writeStandardSyncState(Collections.singletonList((StandardSyncState) config)); + } else { + throw new IllegalArgumentException("Unknown Config Type " + configType); + } + } + + private void writeStandardWorkspace(final List configs) throws IOException { database.transaction(ctx -> { - final OffsetDateTime timestamp = OffsetDateTime.now(); - configs.forEach((configId, config) -> { - final boolean isExistingConfig = ctx.fetchExists(select() - .from(AIRBYTE_CONFIGS) - .where(AIRBYTE_CONFIGS.CONFIG_TYPE.eq(configType.name()), AIRBYTE_CONFIGS.CONFIG_ID.eq(configId))); + writeStandardWorkspace(configs, ctx); + return null; + }); + } - if (isExistingConfig) { - updateConfigRecord(ctx, timestamp, configType.name(), Jsons.jsonNode(config), configId); - } else { - insertConfigRecord(ctx, timestamp, configType.name(), Jsons.jsonNode(config), - configType.getIdFieldName()); + private void writeStandardWorkspace(final List configs, final DSLContext ctx) { + final OffsetDateTime timestamp = OffsetDateTime.now(); + configs.forEach((standardWorkspace) -> { + final boolean isExistingConfig = ctx.fetchExists(select() + .from(WORKSPACE) + .where(WORKSPACE.ID.eq(standardWorkspace.getWorkspaceId()))); + + if (isExistingConfig) { + ctx.update(WORKSPACE) + .set(WORKSPACE.ID, standardWorkspace.getWorkspaceId()) + .set(WORKSPACE.CUSTOMER_ID, standardWorkspace.getCustomerId()) + .set(WORKSPACE.NAME, standardWorkspace.getName()) + .set(WORKSPACE.SLUG, standardWorkspace.getSlug()) + .set(WORKSPACE.EMAIL, standardWorkspace.getEmail()) + .set(WORKSPACE.INITIAL_SETUP_COMPLETE, standardWorkspace.getInitialSetupComplete()) + .set(WORKSPACE.ANONYMOUS_DATA_COLLECTION, standardWorkspace.getAnonymousDataCollection()) + .set(WORKSPACE.SEND_NEWSLETTER, standardWorkspace.getNews()) + .set(WORKSPACE.SEND_SECURITY_UPDATES, standardWorkspace.getSecurityUpdates()) + .set(WORKSPACE.DISPLAY_SETUP_WIZARD, standardWorkspace.getDisplaySetupWizard()) + .set(WORKSPACE.TOMBSTONE, standardWorkspace.getTombstone() != null && standardWorkspace.getTombstone()) + .set(WORKSPACE.NOTIFICATIONS, JSONB.valueOf(Jsons.serialize(standardWorkspace.getNotifications()))) + .set(WORKSPACE.FIRST_SYNC_COMPLETE, standardWorkspace.getFirstCompletedSync()) + .set(WORKSPACE.FEEDBACK_COMPLETE, standardWorkspace.getFeedbackDone()) + .set(WORKSPACE.UPDATED_AT, timestamp) + .where(WORKSPACE.ID.eq(standardWorkspace.getWorkspaceId())) + .execute(); + } else { + ctx.insertInto(WORKSPACE) + .set(WORKSPACE.ID, standardWorkspace.getWorkspaceId()) + .set(WORKSPACE.CUSTOMER_ID, standardWorkspace.getCustomerId()) + .set(WORKSPACE.NAME, standardWorkspace.getName()) + .set(WORKSPACE.SLUG, standardWorkspace.getSlug()) + .set(WORKSPACE.EMAIL, standardWorkspace.getEmail()) + .set(WORKSPACE.INITIAL_SETUP_COMPLETE, standardWorkspace.getInitialSetupComplete()) + .set(WORKSPACE.ANONYMOUS_DATA_COLLECTION, standardWorkspace.getAnonymousDataCollection()) + .set(WORKSPACE.SEND_NEWSLETTER, standardWorkspace.getNews()) + .set(WORKSPACE.SEND_SECURITY_UPDATES, standardWorkspace.getSecurityUpdates()) + .set(WORKSPACE.DISPLAY_SETUP_WIZARD, standardWorkspace.getDisplaySetupWizard()) + .set(WORKSPACE.TOMBSTONE, standardWorkspace.getTombstone() != null && standardWorkspace.getTombstone()) + .set(WORKSPACE.NOTIFICATIONS, JSONB.valueOf(Jsons.serialize(standardWorkspace.getNotifications()))) + .set(WORKSPACE.FIRST_SYNC_COMPLETE, standardWorkspace.getFirstCompletedSync()) + .set(WORKSPACE.FEEDBACK_COMPLETE, standardWorkspace.getFeedbackDone()) + .set(WORKSPACE.CREATED_AT, timestamp) + .set(WORKSPACE.UPDATED_AT, timestamp) + .execute(); + } + }); + } + + private void writeStandardSourceDefinition(final List configs) throws IOException { + database.transaction(ctx -> { + writeStandardSourceDefinition(configs, ctx); + return null; + }); + } + + private void writeStandardSourceDefinition(final List configs, final DSLContext ctx) { + final OffsetDateTime timestamp = OffsetDateTime.now(); + configs.forEach((standardSourceDefinition) -> { + final boolean isExistingConfig = ctx.fetchExists(select() + .from(ACTOR_DEFINITION) + .where(ACTOR_DEFINITION.ID.eq(standardSourceDefinition.getSourceDefinitionId()))); + + if (isExistingConfig) { + ctx.update(ACTOR_DEFINITION) + .set(ACTOR_DEFINITION.ID, standardSourceDefinition.getSourceDefinitionId()) + .set(ACTOR_DEFINITION.NAME, standardSourceDefinition.getName()) + .set(ACTOR_DEFINITION.DOCKER_REPOSITORY, standardSourceDefinition.getDockerRepository()) + .set(ACTOR_DEFINITION.DOCKER_IMAGE_TAG, standardSourceDefinition.getDockerImageTag()) + .set(ACTOR_DEFINITION.DOCUMENTATION_URL, standardSourceDefinition.getDocumentationUrl()) + .set(ACTOR_DEFINITION.ICON, standardSourceDefinition.getIcon()) + .set(ACTOR_DEFINITION.ACTOR_TYPE, ActorType.source) + .set(ACTOR_DEFINITION.SOURCE_TYPE, + standardSourceDefinition.getSourceType() == null ? null + : Enums.toEnum(standardSourceDefinition.getSourceType().value(), + io.airbyte.db.instance.configs.jooq.enums.SourceType.class).orElseThrow()) + .set(ACTOR_DEFINITION.SPEC, JSONB.valueOf(Jsons.serialize(standardSourceDefinition.getSpec()))) + .set(ACTOR_DEFINITION.UPDATED_AT, timestamp) + .where(ACTOR_DEFINITION.ID.eq(standardSourceDefinition.getSourceDefinitionId())) + .execute(); + + } else { + ctx.insertInto(ACTOR_DEFINITION) + .set(ACTOR_DEFINITION.ID, standardSourceDefinition.getSourceDefinitionId()) + .set(ACTOR_DEFINITION.NAME, standardSourceDefinition.getName()) + .set(ACTOR_DEFINITION.DOCKER_REPOSITORY, standardSourceDefinition.getDockerRepository()) + .set(ACTOR_DEFINITION.DOCKER_IMAGE_TAG, standardSourceDefinition.getDockerImageTag()) + .set(ACTOR_DEFINITION.DOCUMENTATION_URL, standardSourceDefinition.getDocumentationUrl()) + .set(ACTOR_DEFINITION.ICON, standardSourceDefinition.getIcon()) + .set(ACTOR_DEFINITION.ACTOR_TYPE, ActorType.source) + .set(ACTOR_DEFINITION.SOURCE_TYPE, + standardSourceDefinition.getSourceType() == null ? null + : Enums.toEnum(standardSourceDefinition.getSourceType().value(), + io.airbyte.db.instance.configs.jooq.enums.SourceType.class).orElseThrow()) + .set(ACTOR_DEFINITION.SPEC, JSONB.valueOf(Jsons.serialize(standardSourceDefinition.getSpec()))) + .set(ACTOR_DEFINITION.CREATED_AT, timestamp) + .set(ACTOR_DEFINITION.UPDATED_AT, timestamp) + .execute(); + } + }); + } + + private void writeStandardDestinationDefinition(final List configs) throws IOException { + database.transaction(ctx -> { + writeStandardDestinationDefinition(configs, ctx); + return null; + }); + } + + private void writeStandardDestinationDefinition(final List configs, final DSLContext ctx) { + final OffsetDateTime timestamp = OffsetDateTime.now(); + configs.forEach((standardDestinationDefinition) -> { + final boolean isExistingConfig = ctx.fetchExists(select() + .from(ACTOR_DEFINITION) + .where(ACTOR_DEFINITION.ID.eq(standardDestinationDefinition.getDestinationDefinitionId()))); + + if (isExistingConfig) { + ctx.update(ACTOR_DEFINITION) + .set(ACTOR_DEFINITION.ID, standardDestinationDefinition.getDestinationDefinitionId()) + .set(ACTOR_DEFINITION.NAME, standardDestinationDefinition.getName()) + .set(ACTOR_DEFINITION.DOCKER_REPOSITORY, standardDestinationDefinition.getDockerRepository()) + .set(ACTOR_DEFINITION.DOCKER_IMAGE_TAG, standardDestinationDefinition.getDockerImageTag()) + .set(ACTOR_DEFINITION.DOCUMENTATION_URL, standardDestinationDefinition.getDocumentationUrl()) + .set(ACTOR_DEFINITION.ICON, standardDestinationDefinition.getIcon()) + .set(ACTOR_DEFINITION.ACTOR_TYPE, ActorType.destination) + .set(ACTOR_DEFINITION.SPEC, JSONB.valueOf(Jsons.serialize(standardDestinationDefinition.getSpec()))) + .set(ACTOR_DEFINITION.UPDATED_AT, timestamp) + .where(ACTOR_DEFINITION.ID.eq(standardDestinationDefinition.getDestinationDefinitionId())) + .execute(); + + } else { + ctx.insertInto(ACTOR_DEFINITION) + .set(ACTOR_DEFINITION.ID, standardDestinationDefinition.getDestinationDefinitionId()) + .set(ACTOR_DEFINITION.NAME, standardDestinationDefinition.getName()) + .set(ACTOR_DEFINITION.DOCKER_REPOSITORY, standardDestinationDefinition.getDockerRepository()) + .set(ACTOR_DEFINITION.DOCKER_IMAGE_TAG, standardDestinationDefinition.getDockerImageTag()) + .set(ACTOR_DEFINITION.DOCUMENTATION_URL, standardDestinationDefinition.getDocumentationUrl()) + .set(ACTOR_DEFINITION.ICON, standardDestinationDefinition.getIcon()) + .set(ACTOR_DEFINITION.ACTOR_TYPE, ActorType.destination) + .set(ACTOR_DEFINITION.SPEC, JSONB.valueOf(Jsons.serialize(standardDestinationDefinition.getSpec()))) + .set(ACTOR_DEFINITION.CREATED_AT, timestamp) + .set(ACTOR_DEFINITION.UPDATED_AT, timestamp) + .execute(); + } + }); + } + + private void writeSourceConnection(final List configs) throws IOException { + database.transaction(ctx -> { + writeSourceConnection(configs, ctx); + return null; + }); + } + + private void writeSourceConnection(final List configs, final DSLContext ctx) { + final OffsetDateTime timestamp = OffsetDateTime.now(); + configs.forEach((sourceConnection) -> { + final boolean isExistingConfig = ctx.fetchExists(select() + .from(ACTOR) + .where(ACTOR.ID.eq(sourceConnection.getSourceId()))); + + if (isExistingConfig) { + ctx.update(ACTOR) + .set(ACTOR.ID, sourceConnection.getSourceId()) + .set(ACTOR.WORKSPACE_ID, sourceConnection.getWorkspaceId()) + .set(ACTOR.ACTOR_DEFINITION_ID, sourceConnection.getSourceDefinitionId()) + .set(ACTOR.NAME, sourceConnection.getName()) + .set(ACTOR.CONFIGURATION, JSONB.valueOf(Jsons.serialize(sourceConnection.getConfiguration()))) + .set(ACTOR.ACTOR_TYPE, ActorType.source) + .set(ACTOR.TOMBSTONE, sourceConnection.getTombstone() != null && sourceConnection.getTombstone()) + .set(ACTOR.UPDATED_AT, timestamp) + .where(ACTOR.ID.eq(sourceConnection.getSourceId())) + .execute(); + } else { + ctx.insertInto(ACTOR) + .set(ACTOR.ID, sourceConnection.getSourceId()) + .set(ACTOR.WORKSPACE_ID, sourceConnection.getWorkspaceId()) + .set(ACTOR.ACTOR_DEFINITION_ID, sourceConnection.getSourceDefinitionId()) + .set(ACTOR.NAME, sourceConnection.getName()) + .set(ACTOR.CONFIGURATION, JSONB.valueOf(Jsons.serialize(sourceConnection.getConfiguration()))) + .set(ACTOR.ACTOR_TYPE, ActorType.source) + .set(ACTOR.TOMBSTONE, sourceConnection.getTombstone() != null && sourceConnection.getTombstone()) + .set(ACTOR.CREATED_AT, timestamp) + .set(ACTOR.UPDATED_AT, timestamp) + .execute(); + } + }); + } + + private void writeDestinationConnection(final List configs) throws IOException { + database.transaction(ctx -> { + writeDestinationConnection(configs, ctx); + return null; + }); + } + + private void writeDestinationConnection(final List configs, final DSLContext ctx) { + final OffsetDateTime timestamp = OffsetDateTime.now(); + configs.forEach((destinationConnection) -> { + final boolean isExistingConfig = ctx.fetchExists(select() + .from(ACTOR) + .where(ACTOR.ID.eq(destinationConnection.getDestinationId()))); + + if (isExistingConfig) { + ctx.update(ACTOR) + .set(ACTOR.ID, destinationConnection.getDestinationId()) + .set(ACTOR.WORKSPACE_ID, destinationConnection.getWorkspaceId()) + .set(ACTOR.ACTOR_DEFINITION_ID, destinationConnection.getDestinationDefinitionId()) + .set(ACTOR.NAME, destinationConnection.getName()) + .set(ACTOR.CONFIGURATION, JSONB.valueOf(Jsons.serialize(destinationConnection.getConfiguration()))) + .set(ACTOR.ACTOR_TYPE, ActorType.destination) + .set(ACTOR.TOMBSTONE, destinationConnection.getTombstone() != null && destinationConnection.getTombstone()) + .set(ACTOR.UPDATED_AT, timestamp) + .where(ACTOR.ID.eq(destinationConnection.getDestinationId())) + .execute(); + + } else { + ctx.insertInto(ACTOR) + .set(ACTOR.ID, destinationConnection.getDestinationId()) + .set(ACTOR.WORKSPACE_ID, destinationConnection.getWorkspaceId()) + .set(ACTOR.ACTOR_DEFINITION_ID, destinationConnection.getDestinationDefinitionId()) + .set(ACTOR.NAME, destinationConnection.getName()) + .set(ACTOR.CONFIGURATION, JSONB.valueOf(Jsons.serialize(destinationConnection.getConfiguration()))) + .set(ACTOR.ACTOR_TYPE, ActorType.destination) + .set(ACTOR.TOMBSTONE, destinationConnection.getTombstone() != null && destinationConnection.getTombstone()) + .set(ACTOR.CREATED_AT, timestamp) + .set(ACTOR.UPDATED_AT, timestamp) + .execute(); + } + }); + } + + private void writeSourceOauthParameter(final List configs) throws IOException { + database.transaction(ctx -> { + writeSourceOauthParameter(configs, ctx); + return null; + }); + } + + private void writeSourceOauthParameter(final List configs, final DSLContext ctx) { + final OffsetDateTime timestamp = OffsetDateTime.now(); + configs.forEach((sourceOAuthParameter) -> { + final boolean isExistingConfig = ctx.fetchExists(select() + .from(ACTOR_OAUTH_PARAMETER) + .where(ACTOR_OAUTH_PARAMETER.ID.eq(sourceOAuthParameter.getOauthParameterId()))); + + if (isExistingConfig) { + ctx.update(ACTOR_OAUTH_PARAMETER) + .set(ACTOR_OAUTH_PARAMETER.ID, sourceOAuthParameter.getOauthParameterId()) + .set(ACTOR_OAUTH_PARAMETER.WORKSPACE_ID, sourceOAuthParameter.getWorkspaceId()) + .set(ACTOR_OAUTH_PARAMETER.ACTOR_DEFINITION_ID, sourceOAuthParameter.getSourceDefinitionId()) + .set(ACTOR_OAUTH_PARAMETER.CONFIGURATION, JSONB.valueOf(Jsons.serialize(sourceOAuthParameter.getConfiguration()))) + .set(ACTOR_OAUTH_PARAMETER.ACTOR_TYPE, ActorType.source) + .set(ACTOR_OAUTH_PARAMETER.UPDATED_AT, timestamp) + .where(ACTOR_OAUTH_PARAMETER.ID.eq(sourceOAuthParameter.getOauthParameterId())) + .execute(); + } else { + ctx.insertInto(ACTOR_OAUTH_PARAMETER) + .set(ACTOR_OAUTH_PARAMETER.ID, sourceOAuthParameter.getOauthParameterId()) + .set(ACTOR_OAUTH_PARAMETER.WORKSPACE_ID, sourceOAuthParameter.getWorkspaceId()) + .set(ACTOR_OAUTH_PARAMETER.ACTOR_DEFINITION_ID, sourceOAuthParameter.getSourceDefinitionId()) + .set(ACTOR_OAUTH_PARAMETER.CONFIGURATION, JSONB.valueOf(Jsons.serialize(sourceOAuthParameter.getConfiguration()))) + .set(ACTOR_OAUTH_PARAMETER.ACTOR_TYPE, ActorType.source) + .set(ACTOR_OAUTH_PARAMETER.CREATED_AT, timestamp) + .set(ACTOR_OAUTH_PARAMETER.UPDATED_AT, timestamp) + .execute(); + } + }); + } + + private void writeDestinationOauthParameter(final List configs) throws IOException { + database.transaction(ctx -> { + writeDestinationOauthParameter(configs, ctx); + return null; + }); + } + + private void writeDestinationOauthParameter(final List configs, final DSLContext ctx) { + final OffsetDateTime timestamp = OffsetDateTime.now(); + configs.forEach((destinationOAuthParameter) -> { + final boolean isExistingConfig = ctx.fetchExists(select() + .from(ACTOR_OAUTH_PARAMETER) + .where(ACTOR_OAUTH_PARAMETER.ID.eq(destinationOAuthParameter.getOauthParameterId()))); + + if (isExistingConfig) { + ctx.update(ACTOR_OAUTH_PARAMETER) + .set(ACTOR_OAUTH_PARAMETER.ID, destinationOAuthParameter.getOauthParameterId()) + .set(ACTOR_OAUTH_PARAMETER.WORKSPACE_ID, destinationOAuthParameter.getWorkspaceId()) + .set(ACTOR_OAUTH_PARAMETER.ACTOR_DEFINITION_ID, destinationOAuthParameter.getDestinationDefinitionId()) + .set(ACTOR_OAUTH_PARAMETER.CONFIGURATION, JSONB.valueOf(Jsons.serialize(destinationOAuthParameter.getConfiguration()))) + .set(ACTOR_OAUTH_PARAMETER.ACTOR_TYPE, ActorType.destination) + .set(ACTOR_OAUTH_PARAMETER.UPDATED_AT, timestamp) + .where(ACTOR_OAUTH_PARAMETER.ID.eq(destinationOAuthParameter.getOauthParameterId())) + .execute(); + + } else { + ctx.insertInto(ACTOR_OAUTH_PARAMETER) + .set(ACTOR_OAUTH_PARAMETER.ID, destinationOAuthParameter.getOauthParameterId()) + .set(ACTOR_OAUTH_PARAMETER.WORKSPACE_ID, destinationOAuthParameter.getWorkspaceId()) + .set(ACTOR_OAUTH_PARAMETER.ACTOR_DEFINITION_ID, destinationOAuthParameter.getDestinationDefinitionId()) + .set(ACTOR_OAUTH_PARAMETER.CONFIGURATION, JSONB.valueOf(Jsons.serialize(destinationOAuthParameter.getConfiguration()))) + .set(ACTOR_OAUTH_PARAMETER.ACTOR_TYPE, ActorType.destination) + .set(ACTOR_OAUTH_PARAMETER.CREATED_AT, timestamp) + .set(ACTOR_OAUTH_PARAMETER.UPDATED_AT, timestamp) + .execute(); + } + }); + + } + + private void writeStandardSyncOperation(final List configs) throws IOException { + database.transaction(ctx -> { + writeStandardSyncOperation(configs, ctx); + return null; + }); + } + + private void writeStandardSyncOperation(final List configs, final DSLContext ctx) { + final OffsetDateTime timestamp = OffsetDateTime.now(); + configs.forEach((standardSyncOperation) -> { + final boolean isExistingConfig = ctx.fetchExists(select() + .from(OPERATION) + .where(OPERATION.ID.eq(standardSyncOperation.getOperationId()))); + + if (isExistingConfig) { + ctx.update(OPERATION) + .set(OPERATION.ID, standardSyncOperation.getOperationId()) + .set(OPERATION.WORKSPACE_ID, standardSyncOperation.getWorkspaceId()) + .set(OPERATION.NAME, standardSyncOperation.getName()) + .set(OPERATION.OPERATOR_TYPE, Enums.toEnum(standardSyncOperation.getOperatorType().value(), + io.airbyte.db.instance.configs.jooq.enums.OperatorType.class).orElseThrow()) + .set(OPERATION.OPERATOR_NORMALIZATION, JSONB.valueOf(Jsons.serialize(standardSyncOperation.getOperatorNormalization()))) + .set(OPERATION.OPERATOR_DBT, JSONB.valueOf(Jsons.serialize(standardSyncOperation.getOperatorDbt()))) + .set(OPERATION.TOMBSTONE, standardSyncOperation.getTombstone() != null && standardSyncOperation.getTombstone()) + .set(OPERATION.UPDATED_AT, timestamp) + .where(OPERATION.ID.eq(standardSyncOperation.getOperationId())) + .execute(); + + } else { + ctx.insertInto(OPERATION) + .set(OPERATION.ID, standardSyncOperation.getOperationId()) + .set(OPERATION.WORKSPACE_ID, standardSyncOperation.getWorkspaceId()) + .set(OPERATION.NAME, standardSyncOperation.getName()) + .set(OPERATION.OPERATOR_TYPE, Enums.toEnum(standardSyncOperation.getOperatorType().value(), + io.airbyte.db.instance.configs.jooq.enums.OperatorType.class).orElseThrow()) + .set(OPERATION.OPERATOR_NORMALIZATION, JSONB.valueOf(Jsons.serialize(standardSyncOperation.getOperatorNormalization()))) + .set(OPERATION.OPERATOR_DBT, JSONB.valueOf(Jsons.serialize(standardSyncOperation.getOperatorDbt()))) + .set(OPERATION.TOMBSTONE, standardSyncOperation.getTombstone() != null && standardSyncOperation.getTombstone()) + .set(OPERATION.CREATED_AT, timestamp) + .set(OPERATION.UPDATED_AT, timestamp) + .execute(); + } + }); + } + + private void writeStandardSync(final List configs) throws IOException { + database.transaction(ctx -> { + writeStandardSync(configs, ctx); + return null; + }); + } + + private void writeStandardSync(final List configs, final DSLContext ctx) { + final OffsetDateTime timestamp = OffsetDateTime.now(); + configs.forEach((standardSync) -> { + final boolean isExistingConfig = ctx.fetchExists(select() + .from(CONNECTION) + .where(CONNECTION.ID.eq(standardSync.getConnectionId()))); + + if (isExistingConfig) { + ctx.update(CONNECTION) + .set(CONNECTION.ID, standardSync.getConnectionId()) + .set(CONNECTION.NAMESPACE_DEFINITION, Enums.toEnum(standardSync.getNamespaceDefinition().value(), + io.airbyte.db.instance.configs.jooq.enums.NamespaceDefinitionType.class).orElseThrow()) + .set(CONNECTION.NAMESPACE_FORMAT, standardSync.getNamespaceFormat()) + .set(CONNECTION.PREFIX, standardSync.getPrefix()) + .set(CONNECTION.SOURCE_ID, standardSync.getSourceId()) + .set(CONNECTION.DESTINATION_ID, standardSync.getDestinationId()) + .set(CONNECTION.NAME, standardSync.getName()) + .set(CONNECTION.CATALOG, JSONB.valueOf(Jsons.serialize(standardSync.getCatalog()))) + .set(CONNECTION.STATUS, standardSync.getStatus() == null ? null + : Enums.toEnum(standardSync.getStatus().value(), + io.airbyte.db.instance.configs.jooq.enums.StatusType.class).orElseThrow()) + .set(CONNECTION.SCHEDULE, JSONB.valueOf(Jsons.serialize(standardSync.getSchedule()))) + .set(CONNECTION.MANUAL, standardSync.getManual()) + .set(CONNECTION.RESOURCE_REQUIREMENTS, JSONB.valueOf(Jsons.serialize(standardSync.getResourceRequirements()))) + .set(CONNECTION.UPDATED_AT, timestamp) + .where(CONNECTION.ID.eq(standardSync.getConnectionId())) + .execute(); + + ctx.deleteFrom(CONNECTION_OPERATION) + .where(CONNECTION_OPERATION.CONNECTION_ID.eq(standardSync.getConnectionId())) + .execute(); + for (final UUID operationIdFromStandardSync : standardSync.getOperationIds()) { + ctx.insertInto(CONNECTION_OPERATION) + .set(CONNECTION_OPERATION.ID, UUID.randomUUID()) + .set(CONNECTION_OPERATION.CONNECTION_ID, standardSync.getConnectionId()) + .set(CONNECTION_OPERATION.OPERATION_ID, operationIdFromStandardSync) + .set(CONNECTION_OPERATION.CREATED_AT, timestamp) + .set(CONNECTION_OPERATION.UPDATED_AT, timestamp) + .execute(); + } + } else { + ctx.insertInto(CONNECTION) + .set(CONNECTION.ID, standardSync.getConnectionId()) + .set(CONNECTION.NAMESPACE_DEFINITION, Enums.toEnum(standardSync.getNamespaceDefinition().value(), + io.airbyte.db.instance.configs.jooq.enums.NamespaceDefinitionType.class).orElseThrow()) + .set(CONNECTION.NAMESPACE_FORMAT, standardSync.getNamespaceFormat()) + .set(CONNECTION.PREFIX, standardSync.getPrefix()) + .set(CONNECTION.SOURCE_ID, standardSync.getSourceId()) + .set(CONNECTION.DESTINATION_ID, standardSync.getDestinationId()) + .set(CONNECTION.NAME, standardSync.getName()) + .set(CONNECTION.CATALOG, JSONB.valueOf(Jsons.serialize(standardSync.getCatalog()))) + .set(CONNECTION.STATUS, standardSync.getStatus() == null ? null + : Enums.toEnum(standardSync.getStatus().value(), + io.airbyte.db.instance.configs.jooq.enums.StatusType.class).orElseThrow()) + .set(CONNECTION.SCHEDULE, JSONB.valueOf(Jsons.serialize(standardSync.getSchedule()))) + .set(CONNECTION.MANUAL, standardSync.getManual()) + .set(CONNECTION.RESOURCE_REQUIREMENTS, JSONB.valueOf(Jsons.serialize(standardSync.getResourceRequirements()))) + .set(CONNECTION.CREATED_AT, timestamp) + .set(CONNECTION.UPDATED_AT, timestamp) + .execute(); + for (final UUID operationIdFromStandardSync : standardSync.getOperationIds()) { + ctx.insertInto(CONNECTION_OPERATION) + .set(CONNECTION_OPERATION.ID, UUID.randomUUID()) + .set(CONNECTION_OPERATION.CONNECTION_ID, standardSync.getConnectionId()) + .set(CONNECTION_OPERATION.OPERATION_ID, operationIdFromStandardSync) + .set(CONNECTION_OPERATION.CREATED_AT, timestamp) + .set(CONNECTION_OPERATION.UPDATED_AT, timestamp) + .execute(); } - }); + } + }); + } + private void writeStandardSyncState(final List configs) throws IOException { + database.transaction(ctx -> { + writeStandardSyncState(configs, ctx); return null; }); } + private void writeStandardSyncState(final List configs, final DSLContext ctx) { + final OffsetDateTime timestamp = OffsetDateTime.now(); + configs.forEach((standardSyncState) -> { + final boolean isExistingConfig = ctx.fetchExists(select() + .from(STATE) + .where(STATE.CONNECTION_ID.eq(standardSyncState.getConnectionId()))); + + if (isExistingConfig) { + ctx.update(STATE) + .set(STATE.CONNECTION_ID, standardSyncState.getConnectionId()) + .set(STATE.STATE_, JSONB.valueOf(Jsons.serialize(standardSyncState.getState()))) + .set(STATE.UPDATED_AT, timestamp) + .where(STATE.CONNECTION_ID.eq(standardSyncState.getConnectionId())) + .execute(); + } else { + ctx.insertInto(STATE) + .set(STATE.ID, UUID.randomUUID()) + .set(STATE.CONNECTION_ID, standardSyncState.getConnectionId()) + .set(STATE.STATE_, JSONB.valueOf(Jsons.serialize(standardSyncState.getState()))) + .set(STATE.CREATED_AT, timestamp) + .set(STATE.UPDATED_AT, timestamp) + .execute(); + } + }); + } + + @Override + public void writeConfigs(final AirbyteConfig configType, final Map configs) throws IOException, JsonValidationException { + if (configType == ConfigSchema.STANDARD_WORKSPACE) { + writeStandardWorkspace(configs.values().stream().map(c -> (StandardWorkspace) c).collect(Collectors.toList())); + } else if (configType == ConfigSchema.STANDARD_SOURCE_DEFINITION) { + writeStandardSourceDefinition(configs.values().stream().map(c -> (StandardSourceDefinition) c).collect(Collectors.toList())); + } else if (configType == ConfigSchema.STANDARD_DESTINATION_DEFINITION) { + writeStandardDestinationDefinition(configs.values().stream().map(c -> (StandardDestinationDefinition) c).collect(Collectors.toList())); + } else if (configType == ConfigSchema.SOURCE_CONNECTION) { + writeSourceConnection(configs.values().stream().map(c -> (SourceConnection) c).collect(Collectors.toList())); + } else if (configType == ConfigSchema.DESTINATION_CONNECTION) { + writeDestinationConnection(configs.values().stream().map(c -> (DestinationConnection) c).collect(Collectors.toList())); + } else if (configType == ConfigSchema.SOURCE_OAUTH_PARAM) { + writeSourceOauthParameter(configs.values().stream().map(c -> (SourceOAuthParameter) c).collect(Collectors.toList())); + } else if (configType == ConfigSchema.DESTINATION_OAUTH_PARAM) { + writeDestinationOauthParameter(configs.values().stream().map(c -> (DestinationOAuthParameter) c).collect(Collectors.toList())); + } else if (configType == ConfigSchema.STANDARD_SYNC_OPERATION) { + writeStandardSyncOperation(configs.values().stream().map(c -> (StandardSyncOperation) c).collect(Collectors.toList())); + } else if (configType == ConfigSchema.STANDARD_SYNC) { + writeStandardSync(configs.values().stream().map(c -> (StandardSync) c).collect(Collectors.toList())); + } else if (configType == ConfigSchema.STANDARD_SYNC_STATE) { + writeStandardSyncState(configs.values().stream().map(c -> (StandardSyncState) c).collect(Collectors.toList())); + } else { + throw new IllegalArgumentException("Unknown Config Type " + configType); + } + } + @Override - public void deleteConfig(final AirbyteConfig configType, final String configId) throws IOException { + public void deleteConfig(final AirbyteConfig configType, final String configId) throws ConfigNotFoundException, IOException { + if (configType == ConfigSchema.STANDARD_WORKSPACE) { + deleteConfig(WORKSPACE, WORKSPACE.ID, UUID.fromString(configId)); + } else if (configType == ConfigSchema.STANDARD_SOURCE_DEFINITION) { + deleteConfig(ACTOR_DEFINITION, ACTOR_DEFINITION.ID, UUID.fromString(configId)); + } else if (configType == ConfigSchema.STANDARD_DESTINATION_DEFINITION) { + deleteConfig(ACTOR_DEFINITION, ACTOR_DEFINITION.ID, UUID.fromString(configId)); + } else if (configType == ConfigSchema.SOURCE_CONNECTION) { + deleteConfig(ACTOR, ACTOR.ID, UUID.fromString(configId)); + } else if (configType == ConfigSchema.DESTINATION_CONNECTION) { + deleteConfig(ACTOR, ACTOR.ID, UUID.fromString(configId)); + } else if (configType == ConfigSchema.SOURCE_OAUTH_PARAM) { + deleteConfig(ACTOR_OAUTH_PARAMETER, ACTOR_OAUTH_PARAMETER.ID, UUID.fromString(configId)); + } else if (configType == ConfigSchema.DESTINATION_OAUTH_PARAM) { + deleteConfig(ACTOR_OAUTH_PARAMETER, ACTOR_OAUTH_PARAMETER.ID, UUID.fromString(configId)); + } else if (configType == ConfigSchema.STANDARD_SYNC_OPERATION) { + deleteConfig(OPERATION, OPERATION.ID, UUID.fromString(configId)); + } else if (configType == ConfigSchema.STANDARD_SYNC) { + deleteStandardSync(configId); + } else if (configType == ConfigSchema.STANDARD_SYNC_STATE) { + deleteConfig(STATE, STATE.CONNECTION_ID, UUID.fromString(configId)); + } else { + throw new IllegalArgumentException("Unknown Config Type " + configType); + } + } + + private void deleteConfig(final TableImpl table, final TableField keyColumn, final UUID configId) + throws IOException { database.transaction(ctx -> { - final boolean isExistingConfig = ctx.fetchExists(select() - .from(AIRBYTE_CONFIGS) - .where(AIRBYTE_CONFIGS.CONFIG_TYPE.eq(configType.name()), AIRBYTE_CONFIGS.CONFIG_ID.eq(configId))); + deleteConfig(table, keyColumn, configId, ctx); + return null; + }); + } + + private void deleteConfig(final TableImpl table, + final TableField keyColumn, + final UUID configId, + final DSLContext ctx) { + final boolean isExistingConfig = ctx.fetchExists(select() + .from(table) + .where(keyColumn.eq(configId))); + + if (isExistingConfig) { + ctx.deleteFrom(table) + .where(keyColumn.eq(configId)) + .execute(); + } + } + + private void deleteStandardSync(final String configId) throws IOException { + database.transaction(ctx -> { + final UUID connectionId = UUID.fromString(configId); + deleteConfig(CONNECTION_OPERATION, CONNECTION_OPERATION.CONNECTION_ID, connectionId, ctx); + deleteConfig(STATE, STATE.CONNECTION_ID, connectionId, ctx); + deleteConfig(CONNECTION, CONNECTION.ID, connectionId, ctx); + return null; + }); + } + + @Override + public void replaceAllConfigs(final Map> configs, final boolean dryRun) throws IOException { + if (dryRun) { + return; + } + + LOGGER.info("Replacing all configs"); + final Set originalConfigs = new HashSet<>(configs.keySet()); + database.transaction(ctx -> { + ctx.truncate(WORKSPACE).restartIdentity().cascade().execute(); + ctx.truncate(ACTOR_DEFINITION).restartIdentity().cascade().execute(); + ctx.truncate(ACTOR).restartIdentity().cascade().execute(); + ctx.truncate(ACTOR_OAUTH_PARAMETER).restartIdentity().cascade().execute(); + ctx.truncate(OPERATION).restartIdentity().cascade().execute(); + ctx.truncate(CONNECTION).restartIdentity().cascade().execute(); + ctx.truncate(CONNECTION_OPERATION).restartIdentity().cascade().execute(); + ctx.truncate(STATE).restartIdentity().cascade().execute(); + + if (configs.containsKey(ConfigSchema.STANDARD_WORKSPACE)) { + configs.get(ConfigSchema.STANDARD_WORKSPACE).map(c -> (StandardWorkspace) c) + .forEach(c -> writeStandardWorkspace(Collections.singletonList(c), ctx)); + originalConfigs.remove(ConfigSchema.STANDARD_WORKSPACE); + } else { + LOGGER.warn(ConfigSchema.STANDARD_WORKSPACE + " not found"); + } + if (configs.containsKey(ConfigSchema.STANDARD_SOURCE_DEFINITION)) { + configs.get(ConfigSchema.STANDARD_SOURCE_DEFINITION).map(c -> (StandardSourceDefinition) c) + .forEach(c -> writeStandardSourceDefinition(Collections.singletonList(c), ctx)); + originalConfigs.remove(ConfigSchema.STANDARD_SOURCE_DEFINITION); + } else { + LOGGER.warn(ConfigSchema.STANDARD_SOURCE_DEFINITION + " not found"); + } + + if (configs.containsKey(ConfigSchema.STANDARD_DESTINATION_DEFINITION)) { + configs.get(ConfigSchema.STANDARD_DESTINATION_DEFINITION).map(c -> (StandardDestinationDefinition) c) + .forEach(c -> writeStandardDestinationDefinition(Collections.singletonList(c), ctx)); + originalConfigs.remove(ConfigSchema.STANDARD_DESTINATION_DEFINITION); + } else { + LOGGER.warn(ConfigSchema.STANDARD_DESTINATION_DEFINITION + " not found"); + } + + if (configs.containsKey(ConfigSchema.SOURCE_CONNECTION)) { + configs.get(ConfigSchema.SOURCE_CONNECTION).map(c -> (SourceConnection) c) + .forEach(c -> writeSourceConnection(Collections.singletonList(c), ctx)); + originalConfigs.remove(ConfigSchema.SOURCE_CONNECTION); + } else { + LOGGER.warn(ConfigSchema.SOURCE_CONNECTION + " not found"); + } + + if (configs.containsKey(ConfigSchema.DESTINATION_CONNECTION)) { + configs.get(ConfigSchema.DESTINATION_CONNECTION).map(c -> (DestinationConnection) c) + .forEach(c -> writeDestinationConnection(Collections.singletonList(c), ctx)); + originalConfigs.remove(ConfigSchema.DESTINATION_CONNECTION); + } else { + LOGGER.warn(ConfigSchema.DESTINATION_CONNECTION + " not found"); + } + + if (configs.containsKey(ConfigSchema.SOURCE_OAUTH_PARAM)) { + configs.get(ConfigSchema.SOURCE_OAUTH_PARAM).map(c -> (SourceOAuthParameter) c) + .forEach(c -> writeSourceOauthParameter(Collections.singletonList(c), ctx)); + originalConfigs.remove(ConfigSchema.SOURCE_OAUTH_PARAM); + } else { + LOGGER.warn(ConfigSchema.SOURCE_OAUTH_PARAM + " not found"); + } - if (isExistingConfig) { - ctx.deleteFrom(AIRBYTE_CONFIGS) - .where(AIRBYTE_CONFIGS.CONFIG_TYPE.eq(configType.name()), AIRBYTE_CONFIGS.CONFIG_ID.eq(configId)) - .execute(); - return null; + if (configs.containsKey(ConfigSchema.DESTINATION_OAUTH_PARAM)) { + configs.get(ConfigSchema.DESTINATION_OAUTH_PARAM).map(c -> (DestinationOAuthParameter) c) + .forEach(c -> writeDestinationOauthParameter(Collections.singletonList(c), ctx)); + originalConfigs.remove(ConfigSchema.DESTINATION_OAUTH_PARAM); + } else { + LOGGER.warn(ConfigSchema.DESTINATION_OAUTH_PARAM + " not found"); } - return null; - }); - } - @Override - public void replaceAllConfigs(final Map> configs, final boolean dryRun) throws IOException { - if (dryRun) { - return; - } + if (configs.containsKey(ConfigSchema.STANDARD_SYNC_OPERATION)) { + configs.get(ConfigSchema.STANDARD_SYNC_OPERATION).map(c -> (StandardSyncOperation) c) + .forEach(c -> writeStandardSyncOperation(Collections.singletonList(c), ctx)); + originalConfigs.remove(ConfigSchema.STANDARD_SYNC_OPERATION); + } else { + LOGGER.warn(ConfigSchema.STANDARD_SYNC_OPERATION + " not found"); + } - LOGGER.info("Replacing all configs"); + if (configs.containsKey(ConfigSchema.STANDARD_SYNC)) { + configs.get(ConfigSchema.STANDARD_SYNC).map(c -> (StandardSync) c).forEach(c -> writeStandardSync(Collections.singletonList(c), ctx)); + originalConfigs.remove(ConfigSchema.STANDARD_SYNC); + } else { + LOGGER.warn(ConfigSchema.STANDARD_SYNC + " not found"); + } - final OffsetDateTime timestamp = OffsetDateTime.now(); - final int insertionCount = database.transaction(ctx -> { - ctx.truncate(AIRBYTE_CONFIGS).restartIdentity().execute(); + if (configs.containsKey(ConfigSchema.STANDARD_SYNC_STATE)) { + configs.get(ConfigSchema.STANDARD_SYNC_STATE).map(c -> (StandardSyncState) c) + .forEach(c -> writeStandardSyncState(Collections.singletonList(c), ctx)); + originalConfigs.remove(ConfigSchema.STANDARD_SYNC_STATE); + } else { + LOGGER.warn(ConfigSchema.STANDARD_SYNC_STATE + " not found"); + } + + if (!originalConfigs.isEmpty()) { + originalConfigs.forEach(c -> LOGGER.warn("Unknown Config " + c + " ignored")); + } - return configs.entrySet().stream().map(entry -> { - final AirbyteConfig configType = entry.getKey(); - return entry.getValue() - .map(configObject -> insertConfigRecord(ctx, timestamp, configType.name(), Jsons.jsonNode(configObject), configType.getIdFieldName())) - .reduce(0, Integer::sum); - }).reduce(0, Integer::sum); + return null; }); - LOGGER.info("Config database is reset with {} records", insertionCount); + LOGGER.info("Config database is reset"); } @Override public Map> dumpConfigs() throws IOException { LOGGER.info("Exporting all configs..."); - final Map> results = database.query(ctx -> ctx.select(asterisk()) - .from(AIRBYTE_CONFIGS) - .orderBy(AIRBYTE_CONFIGS.CONFIG_TYPE, AIRBYTE_CONFIGS.CONFIG_ID) - .fetchGroups(AIRBYTE_CONFIGS.CONFIG_TYPE)); - return results.entrySet().stream().collect(Collectors.toMap( - Entry::getKey, - e -> e.getValue().stream().map(r -> Jsons.deserialize(r.get(AIRBYTE_CONFIGS.CONFIG_BLOB).data())))); - } - - /** - * @return the number of inserted records for convenience, which is always 1. - */ - @VisibleForTesting - int insertConfigRecord( - final DSLContext ctx, - final OffsetDateTime timestamp, - final String configType, - final JsonNode configJson, - @Nullable final String idFieldName) { - final String configId = idFieldName == null - ? UUID.randomUUID().toString() - : configJson.get(idFieldName).asText(); - LOGGER.info("Inserting {} record {}", configType, configId); - - final int insertionCount = ctx.insertInto(AIRBYTE_CONFIGS) - .set(AIRBYTE_CONFIGS.CONFIG_ID, configId) - .set(AIRBYTE_CONFIGS.CONFIG_TYPE, configType) - .set(AIRBYTE_CONFIGS.CONFIG_BLOB, JSONB.valueOf(Jsons.serialize(configJson))) - .set(AIRBYTE_CONFIGS.CREATED_AT, timestamp) - .set(AIRBYTE_CONFIGS.UPDATED_AT, timestamp) - .onConflict(AIRBYTE_CONFIGS.CONFIG_TYPE, AIRBYTE_CONFIGS.CONFIG_ID) - .doNothing() - .execute(); - if (insertionCount != 1) { - LOGGER.warn("{} config {} already exists (insertion record count: {})", configType, configId, insertionCount); - } - return insertionCount; - } - - /** - * @return the number of updated records. - */ - @VisibleForTesting - int updateConfigRecord(final DSLContext ctx, - final OffsetDateTime timestamp, - final String configType, - final JsonNode configJson, - final String configId) { - LOGGER.info("Updating {} record {}", configType, configId); - - final int updateCount = ctx.update(AIRBYTE_CONFIGS) - .set(AIRBYTE_CONFIGS.CONFIG_BLOB, JSONB.valueOf(Jsons.serialize(configJson))) - .set(AIRBYTE_CONFIGS.UPDATED_AT, timestamp) - .where(AIRBYTE_CONFIGS.CONFIG_TYPE.eq(configType), AIRBYTE_CONFIGS.CONFIG_ID.eq(configId)) - .execute(); - if (updateCount != 1) { - LOGGER.warn("{} config {} is not updated (updated record count: {})", configType, configId, updateCount); + final Map> result = new HashMap<>(); + final List> standardWorkspaceWithMetadata = listStandardWorkspaceWithMetadata(); + if (!standardWorkspaceWithMetadata.isEmpty()) { + result.put(ConfigSchema.STANDARD_WORKSPACE.name(), + standardWorkspaceWithMetadata + .stream() + .map(ConfigWithMetadata::getConfig) + .map(Jsons::jsonNode)); } - return updateCount; - } - - @VisibleForTesting - void copyConfigsFromSeed(final DSLContext ctx, final ConfigPersistence seedConfigPersistence) throws SQLException { - LOGGER.info("Loading seed data to config database..."); - - final Map> seedConfigs; - try { - seedConfigs = seedConfigPersistence.dumpConfigs(); - } catch (final IOException e) { - throw new SQLException(e); + final List> standardSourceDefinitionWithMetadata = listStandardSourceDefinitionWithMetadata(); + if (!standardSourceDefinitionWithMetadata.isEmpty()) { + result.put(ConfigSchema.STANDARD_SOURCE_DEFINITION.name(), + standardSourceDefinitionWithMetadata + .stream() + .map(ConfigWithMetadata::getConfig) + .map(Jsons::jsonNode)); } - - final OffsetDateTime timestamp = OffsetDateTime.now(); - final int insertionCount = seedConfigs.entrySet().stream().map(entry -> { - final String configType = entry.getKey(); - return entry.getValue().map(configJson -> { - final String idFieldName = ConfigSchemaMigrationSupport.CONFIG_SCHEMA_ID_FIELD_NAMES.get(configType); - return insertConfigRecord(ctx, timestamp, configType, configJson, idFieldName); - }).reduce(0, Integer::sum); - }).reduce(0, Integer::sum); - - LOGGER.info("Config database data loading completed with {} records", insertionCount); - } - - static class ConnectorInfo { - - final String definitionId; - final JsonNode definition; - final String dockerRepository; - final String dockerImageTag; - - ConnectorInfo(final String definitionId, final JsonNode definition) { - this.definitionId = definitionId; - this.definition = definition; - this.dockerRepository = definition.get("dockerRepository").asText(); - this.dockerImageTag = definition.get("dockerImageTag").asText(); + final List> standardDestinationDefinitionWithMetadata = + listStandardDestinationDefinitionWithMetadata(); + if (!standardDestinationDefinitionWithMetadata.isEmpty()) { + result.put(ConfigSchema.STANDARD_DESTINATION_DEFINITION.name(), + standardDestinationDefinitionWithMetadata + .stream() + .map(ConfigWithMetadata::getConfig) + .map(Jsons::jsonNode)); } - - @Override - public String toString() { - return String.format("%s: %s (%s)", dockerRepository, dockerImageTag, definitionId); + final List> sourceConnectionWithMetadata = listSourceConnectionWithMetadata(); + if (!sourceConnectionWithMetadata.isEmpty()) { + result.put(ConfigSchema.SOURCE_CONNECTION.name(), + sourceConnectionWithMetadata + .stream() + .map(ConfigWithMetadata::getConfig) + .map(Jsons::jsonNode)); } - - } - - private static class ConnectorCounter { - - private final int newCount; - private final int updateCount; - - private ConnectorCounter(final int newCount, final int updateCount) { - this.newCount = newCount; - this.updateCount = updateCount; + final List> destinationConnectionWithMetadata = listDestinationConnectionWithMetadata(); + if (!destinationConnectionWithMetadata.isEmpty()) { + result.put(ConfigSchema.DESTINATION_CONNECTION.name(), + destinationConnectionWithMetadata + .stream() + .map(ConfigWithMetadata::getConfig) + .map(Jsons::jsonNode)); + } + final List> sourceOauthParamWithMetadata = listSourceOauthParamWithMetadata(); + if (!sourceOauthParamWithMetadata.isEmpty()) { + result.put(ConfigSchema.SOURCE_OAUTH_PARAM.name(), + sourceOauthParamWithMetadata + .stream() + .map(ConfigWithMetadata::getConfig) + .map(Jsons::jsonNode)); + } + final List> destinationOauthParamWithMetadata = listDestinationOauthParamWithMetadata(); + if (!destinationOauthParamWithMetadata.isEmpty()) { + result.put(ConfigSchema.DESTINATION_OAUTH_PARAM.name(), + destinationOauthParamWithMetadata + .stream() + .map(ConfigWithMetadata::getConfig) + .map(Jsons::jsonNode)); } + final List> standardSyncOperationWithMetadata = listStandardSyncOperationWithMetadata(); + if (!standardSyncOperationWithMetadata.isEmpty()) { + result.put(ConfigSchema.STANDARD_SYNC_OPERATION.name(), + standardSyncOperationWithMetadata + .stream() + .map(ConfigWithMetadata::getConfig) + .map(Jsons::jsonNode)); + } + final List> standardSyncWithMetadata = listStandardSyncWithMetadata(); + if (!standardSyncWithMetadata.isEmpty()) { + result.put(ConfigSchema.STANDARD_SYNC.name(), + standardSyncWithMetadata + .stream() + .map(ConfigWithMetadata::getConfig) + .map(Jsons::jsonNode)); + } + final List> standardSyncStateWithMetadata = listStandardSyncStateWithMetadata(); + if (!standardSyncStateWithMetadata.isEmpty()) { + result.put(ConfigSchema.STANDARD_SYNC_STATE.name(), + standardSyncStateWithMetadata + .stream() + .map(ConfigWithMetadata::getConfig) + .map(Jsons::jsonNode)); + } + return result; + } + @Override + public void loadData(final ConfigPersistence seedConfigPersistence) throws IOException { + database.transaction(ctx -> { + updateConfigsFromSeed(ctx, seedConfigPersistence); + return null; + }); } @VisibleForTesting @@ -356,14 +1434,14 @@ void updateConfigsFromSeed(final DSLContext ctx, final ConfigPersistence seedCon final List latestSources = seedConfigPersistence.listConfigs( ConfigSchema.STANDARD_SOURCE_DEFINITION, StandardSourceDefinition.class); - final ConnectorCounter sourceConnectorCounter = updateConnectorDefinitions(ctx, timestamp, ConfigSchema.STANDARD_SOURCE_DEFINITION, + final ConnectorCounter sourceConnectorCounter = updateConnectorDefinitions(ctx, ConfigSchema.STANDARD_SOURCE_DEFINITION, latestSources, connectorRepositoriesInUse, connectorRepositoryToInfoMap); newConnectorCount += sourceConnectorCounter.newCount; updatedConnectorCount += sourceConnectorCounter.updateCount; final List latestDestinations = seedConfigPersistence.listConfigs( ConfigSchema.STANDARD_DESTINATION_DEFINITION, StandardDestinationDefinition.class); - final ConnectorCounter destinationConnectorCounter = updateConnectorDefinitions(ctx, timestamp, ConfigSchema.STANDARD_DESTINATION_DEFINITION, + final ConnectorCounter destinationConnectorCounter = updateConnectorDefinitions(ctx, ConfigSchema.STANDARD_DESTINATION_DEFINITION, latestDestinations, connectorRepositoriesInUse, connectorRepositoryToInfoMap); newConnectorCount += destinationConnectorCounter.newCount; updatedConnectorCount += destinationConnectorCounter.updateCount; @@ -374,6 +1452,84 @@ void updateConfigsFromSeed(final DSLContext ctx, final ConfigPersistence seedCon } } + /** + * @return A set of connectors (both source and destination) that are already used in standard + * syncs. We identify connectors by its repository name instead of definition id because + * connectors can be added manually by users, and their config ids are not always the same + * as those in the seed. + */ + private Set getConnectorRepositoriesInUse(final DSLContext ctx) { + final Set usedConnectorDefinitionIds = ctx + .select(ACTOR.ACTOR_DEFINITION_ID) + .from(ACTOR) + .fetch() + .stream() + .flatMap(row -> Stream.of(row.value1())) + .collect(Collectors.toSet()); + + return ctx.select(ACTOR_DEFINITION.DOCKER_REPOSITORY) + .from(ACTOR_DEFINITION) + .where(ACTOR_DEFINITION.ID.in(usedConnectorDefinitionIds)) + .fetch().stream() + .map(Record1::value1) + .collect(Collectors.toSet()); + } + + /** + * @return A map about current connectors (both source and destination). It maps from connector + * repository to its definition id and docker image tag. We identify a connector by its + * repository name instead of definition id because connectors can be added manually by + * users, and are not always the same as those in the seed. + */ + @VisibleForTesting + Map getConnectorRepositoryToInfoMap(final DSLContext ctx) { + return ctx.select(asterisk()) + .from(ACTOR_DEFINITION) + .fetch() + .stream() + .collect(Collectors.toMap( + row -> row.getValue(ACTOR_DEFINITION.DOCKER_REPOSITORY), + row -> { + final JsonNode jsonNode; + if (row.get(ACTOR_DEFINITION.ACTOR_TYPE) == ActorType.source) { + jsonNode = Jsons.jsonNode(new StandardSourceDefinition() + .withSourceDefinitionId(row.get(ACTOR_DEFINITION.ID)) + .withDockerImageTag(row.get(ACTOR_DEFINITION.DOCKER_IMAGE_TAG)) + .withIcon(row.get(ACTOR_DEFINITION.ICON)) + .withDockerRepository(row.get(ACTOR_DEFINITION.DOCKER_REPOSITORY)) + .withDocumentationUrl(row.get(ACTOR_DEFINITION.DOCUMENTATION_URL)) + .withName(row.get(ACTOR_DEFINITION.NAME)) + .withSourceType(row.get(ACTOR_DEFINITION.SOURCE_TYPE) == null ? null + : Enums.toEnum(row.get(ACTOR_DEFINITION.SOURCE_TYPE, String.class), SourceType.class).orElseThrow()) + .withSpec(Jsons.deserialize(row.get(ACTOR_DEFINITION.SPEC).data(), ConnectorSpecification.class))); + } else if (row.get(ACTOR_DEFINITION.ACTOR_TYPE) == ActorType.destination) { + jsonNode = Jsons.jsonNode(new StandardDestinationDefinition() + .withDestinationDefinitionId(row.get(ACTOR_DEFINITION.ID)) + .withDockerImageTag(row.get(ACTOR_DEFINITION.DOCKER_IMAGE_TAG)) + .withIcon(row.get(ACTOR_DEFINITION.ICON)) + .withDockerRepository(row.get(ACTOR_DEFINITION.DOCKER_REPOSITORY)) + .withDocumentationUrl(row.get(ACTOR_DEFINITION.DOCUMENTATION_URL)) + .withName(row.get(ACTOR_DEFINITION.NAME)) + .withSpec(Jsons.deserialize(row.get(ACTOR_DEFINITION.SPEC).data(), ConnectorSpecification.class))); + } else { + throw new RuntimeException("Unknown Actor Type " + row.get(ACTOR_DEFINITION.ACTOR_TYPE)); + } + return new ConnectorInfo(row.getValue(ACTOR_DEFINITION.ID).toString(), jsonNode); + }, + (c1, c2) -> { + final AirbyteVersion v1 = new AirbyteVersion(c1.dockerImageTag); + final AirbyteVersion v2 = new AirbyteVersion(c2.dockerImageTag); + LOGGER.warn("Duplicated connector version found for {}: {} ({}) vs {} ({})", + c1.dockerRepository, c1.dockerImageTag, c1.definitionId, c2.dockerImageTag, c2.definitionId); + final int comparison = v1.patchVersionCompareTo(v2); + if (comparison >= 0) { + return c1; + } else { + return c2; + } + })); + } + /** * @param connectorRepositoriesInUse when a connector is used in any standard sync, its definition * will not be updated. This is necessary because the new connector version may not be @@ -381,7 +1537,6 @@ void updateConfigsFromSeed(final DSLContext ctx, final ConfigPersistence seedCon */ @VisibleForTesting ConnectorCounter updateConnectorDefinitions(final DSLContext ctx, - final OffsetDateTime timestamp, final AirbyteConfig configType, final List latestDefinitions, final Set connectorRepositoriesInUse, @@ -397,7 +1552,14 @@ ConnectorCounter updateConnectorDefinitions(final DSLContext ctx, // Add new connector if (!connectorRepositoryToIdVersionMap.containsKey(repository)) { LOGGER.info("Adding new connector {}: {}", repository, latestDefinition); - newCount += insertConfigRecord(ctx, timestamp, configType.name(), latestDefinition, configType.getIdFieldName()); + if (configType == ConfigSchema.STANDARD_SOURCE_DEFINITION) { + writeStandardSourceDefinition(Collections.singletonList(Jsons.object(latestDefinition, StandardSourceDefinition.class)), ctx); + } else if (configType == ConfigSchema.STANDARD_DESTINATION_DEFINITION) { + writeStandardDestinationDefinition(Collections.singletonList(Jsons.object(latestDefinition, StandardDestinationDefinition.class)), ctx); + } else { + throw new RuntimeException("Unknown config type " + configType); + } + newCount++; continue; } @@ -420,7 +1582,14 @@ ConnectorCounter updateConnectorDefinitions(final DSLContext ctx, // Add new fields to the connector definition final JsonNode definitionToUpdate = getDefinitionWithNewFields(currentDefinition, latestDefinition, newFields); LOGGER.info("Connector {} has new fields: {}", repository, String.join(", ", newFields)); - updatedCount += updateConfigRecord(ctx, timestamp, configType.name(), definitionToUpdate, connectorInfo.definitionId); + if (configType == ConfigSchema.STANDARD_SOURCE_DEFINITION) { + writeStandardSourceDefinition(Collections.singletonList(Jsons.object(definitionToUpdate, StandardSourceDefinition.class)), ctx); + } else if (configType == ConfigSchema.STANDARD_DESTINATION_DEFINITION) { + writeStandardDestinationDefinition(Collections.singletonList(Jsons.object(definitionToUpdate, StandardDestinationDefinition.class)), ctx); + } else { + throw new RuntimeException("Unknown config type " + configType); + } + updatedCount++; } continue; } @@ -430,12 +1599,26 @@ ConnectorCounter updateConnectorDefinitions(final DSLContext ctx, if (hasNewVersion(connectorInfo.dockerImageTag, latestImageTag)) { // Update connector to the latest version LOGGER.info("Connector {} needs update: {} vs {}", repository, connectorInfo.dockerImageTag, latestImageTag); - updatedCount += updateConfigRecord(ctx, timestamp, configType.name(), latestDefinition, connectorInfo.definitionId); + if (configType == ConfigSchema.STANDARD_SOURCE_DEFINITION) { + writeStandardSourceDefinition(Collections.singletonList(Jsons.object(latestDefinition, StandardSourceDefinition.class)), ctx); + } else if (configType == ConfigSchema.STANDARD_DESTINATION_DEFINITION) { + writeStandardDestinationDefinition(Collections.singletonList(Jsons.object(latestDefinition, StandardDestinationDefinition.class)), ctx); + } else { + throw new RuntimeException("Unknown config type " + configType); + } + updatedCount++; } else if (newFields.size() > 0) { // Add new fields to the connector definition final JsonNode definitionToUpdate = getDefinitionWithNewFields(currentDefinition, latestDefinition, newFields); LOGGER.info("Connector {} has new fields: {}", repository, String.join(", ", newFields)); - updatedCount += updateConfigRecord(ctx, timestamp, configType.name(), definitionToUpdate, connectorInfo.definitionId); + if (configType == ConfigSchema.STANDARD_SOURCE_DEFINITION) { + writeStandardSourceDefinition(Collections.singletonList(Jsons.object(definitionToUpdate, StandardSourceDefinition.class)), ctx); + } else if (configType == ConfigSchema.STANDARD_DESTINATION_DEFINITION) { + writeStandardDestinationDefinition(Collections.singletonList(Jsons.object(definitionToUpdate, StandardDestinationDefinition.class)), ctx); + } else { + throw new RuntimeException("Unknown config type " + configType); + } + updatedCount++; } else { LOGGER.info("Connector {} does not need update: {}", repository, connectorInfo.dockerImageTag); } @@ -444,18 +1627,7 @@ ConnectorCounter updateConnectorDefinitions(final DSLContext ctx, return new ConnectorCounter(newCount, updatedCount); } - static boolean hasNewVersion(final String currentVersion, final String latestVersion) { - try { - return new AirbyteVersion(latestVersion).patchVersionCompareTo(new AirbyteVersion(currentVersion)) > 0; - } catch (final Exception e) { - LOGGER.error("Failed to check version: {} vs {}", currentVersion, latestVersion); - return false; - } - } - - /** - * @return new fields from the latest definition - */ + @VisibleForTesting static Set getNewFields(final JsonNode currentDefinition, final JsonNode latestDefinition) { final Set currentFields = MoreIterators.toSet(currentDefinition.fieldNames()); final Set latestFields = MoreIterators.toSet(latestDefinition.fieldNames()); @@ -465,75 +1637,118 @@ static Set getNewFields(final JsonNode currentDefinition, final JsonNode /** * @return a clone of the current definition with the new fields from the latest definition. */ + @VisibleForTesting static JsonNode getDefinitionWithNewFields(final JsonNode currentDefinition, final JsonNode latestDefinition, final Set newFields) { final ObjectNode currentClone = (ObjectNode) Jsons.clone(currentDefinition); newFields.forEach(field -> currentClone.set(field, latestDefinition.get(field))); return currentClone; } - /** - * @return A map about current connectors (both source and destination). It maps from connector - * repository to its definition id and docker image tag. We identify a connector by its - * repository name instead of definition id because connectors can be added manually by - * users, and are not always the same as those in the seed. - */ @VisibleForTesting - Map getConnectorRepositoryToInfoMap(final DSLContext ctx) { - final Field configField = field("config_blob", SQLDataType.JSONB).as("definition"); - final Field repoField = field("config_blob ->> 'dockerRepository'", SQLDataType.VARCHAR).as("repository"); - return ctx.select(AIRBYTE_CONFIGS.CONFIG_ID, repoField, configField) - .from(AIRBYTE_CONFIGS) - .where(AIRBYTE_CONFIGS.CONFIG_TYPE.in(ConfigSchema.STANDARD_SOURCE_DEFINITION.name(), ConfigSchema.STANDARD_DESTINATION_DEFINITION.name())) - .fetch().stream() - .collect(Collectors.toMap( - row -> row.getValue(repoField), - row -> new ConnectorInfo(row.getValue(AIRBYTE_CONFIGS.CONFIG_ID), Jsons.deserialize(row.getValue(configField).data())), - // when there are duplicated connector definitions, return the latest one - (c1, c2) -> { - final AirbyteVersion v1 = new AirbyteVersion(c1.dockerImageTag); - final AirbyteVersion v2 = new AirbyteVersion(c2.dockerImageTag); - LOGGER.warn("Duplicated connector version found for {}: {} ({}) vs {} ({})", - c1.dockerRepository, c1.dockerImageTag, c1.definitionId, c2.dockerImageTag, c2.definitionId); - final int comparison = v1.patchVersionCompareTo(v2); - if (comparison >= 0) { - return c1; - } else { - return c2; - } - })); + static boolean hasNewVersion(final String currentVersion, final String latestVersion) { + try { + return new AirbyteVersion(latestVersion).patchVersionCompareTo(new AirbyteVersion(currentVersion)) > 0; + } catch (final Exception e) { + LOGGER.error("Failed to check version: {} vs {}", currentVersion, latestVersion); + return false; + } } /** - * @return A set of connectors (both source and destination) that are already used in standard - * syncs. We identify connectors by its repository name instead of definition id because - * connectors can be added manually by users, and their config ids are not always the same - * as those in the seed. + * If this is a migration deployment from an old version that relies on file system config + * persistence, copy the existing configs from local files. */ - private Set getConnectorRepositoriesInUse(final DSLContext ctx) { - final Set usedConnectorDefinitionIds = new HashSet<>(); - // query for used source definitions - usedConnectorDefinitionIds.addAll(ctx - .select(field("config_blob ->> 'sourceDefinitionId'", SQLDataType.VARCHAR)) - .from(AIRBYTE_CONFIGS) - .where(AIRBYTE_CONFIGS.CONFIG_TYPE.eq(ConfigSchema.SOURCE_CONNECTION.name())) - .fetch().stream() - .flatMap(row -> Stream.of(row.value1())) - .collect(Collectors.toSet())); - // query for used destination definitions - usedConnectorDefinitionIds.addAll(ctx - .select(field("config_blob ->> 'destinationDefinitionId'", SQLDataType.VARCHAR)) - .from(AIRBYTE_CONFIGS) - .where(AIRBYTE_CONFIGS.CONFIG_TYPE.eq(ConfigSchema.DESTINATION_CONNECTION.name())) - .fetch().stream() - .flatMap(row -> Stream.of(row.value1())) - .collect(Collectors.toSet())); + public DatabaseConfigPersistence migrateFileConfigs(final Configs serverConfigs) throws IOException { + database.transaction(ctx -> { + final boolean isInitialized = ctx.fetchExists(ACTOR_DEFINITION); + if (isInitialized) { + return null; + } + + final boolean hasExistingFileConfigs = FileSystemConfigPersistence.hasExistingConfigs(serverConfigs.getConfigRoot()); + if (hasExistingFileConfigs) { + LOGGER.info("Load existing local config directory into configs database"); + final ConfigPersistence fileSystemPersistence = new FileSystemConfigPersistence(serverConfigs.getConfigRoot()); + copyConfigsFromSeed(ctx, fileSystemPersistence); + } + + return null; + }); + + return this; + } + + @VisibleForTesting + void copyConfigsFromSeed(final DSLContext ctx, final ConfigPersistence seedConfigPersistence) throws SQLException { + LOGGER.info("Loading seed data to config database..."); + + final Map> seedConfigs; + try { + seedConfigs = seedConfigPersistence.dumpConfigs(); + } catch (final IOException e) { + throw new SQLException(e); + } + + seedConfigs.forEach((configType, value) -> value.forEach(configJson -> { + if (configType.equals(ConfigSchema.STANDARD_WORKSPACE.name())) { + writeStandardWorkspace(Collections.singletonList(Jsons.object(configJson, StandardWorkspace.class)), ctx); + } else if (configType.equals(ConfigSchema.STANDARD_SOURCE_DEFINITION.name())) { + writeStandardSourceDefinition(Collections.singletonList(Jsons.object(configJson, StandardSourceDefinition.class)), ctx); + } else if (configType.equals(ConfigSchema.STANDARD_DESTINATION_DEFINITION.name())) { + writeStandardDestinationDefinition(Collections.singletonList(Jsons.object(configJson, StandardDestinationDefinition.class)), ctx); + } else if (configType.equals(ConfigSchema.SOURCE_CONNECTION.name())) { + writeSourceConnection(Collections.singletonList(Jsons.object(configJson, SourceConnection.class)), ctx); + } else if (configType.equals(ConfigSchema.DESTINATION_CONNECTION.name())) { + writeDestinationConnection(Collections.singletonList(Jsons.object(configJson, DestinationConnection.class)), ctx); + } else if (configType.equals(ConfigSchema.SOURCE_OAUTH_PARAM.name())) { + writeSourceOauthParameter(Collections.singletonList(Jsons.object(configJson, SourceOAuthParameter.class)), ctx); + } else if (configType.equals(ConfigSchema.DESTINATION_OAUTH_PARAM.name())) { + writeDestinationOauthParameter(Collections.singletonList(Jsons.object(configJson, DestinationOAuthParameter.class)), ctx); + } else if (configType.equals(ConfigSchema.STANDARD_SYNC_OPERATION.name())) { + writeStandardSyncOperation(Collections.singletonList(Jsons.object(configJson, StandardSyncOperation.class)), ctx); + } else if (configType.equals(ConfigSchema.STANDARD_SYNC.name())) { + writeStandardSync(Collections.singletonList(Jsons.object(configJson, StandardSync.class)), ctx); + } else if (configType.equals(ConfigSchema.STANDARD_SYNC_STATE.name())) { + writeStandardSyncState(Collections.singletonList(Jsons.object(configJson, StandardSyncState.class)), ctx); + } else { + throw new IllegalArgumentException("Unknown Config Type " + configType); + } + })); + + LOGGER.info("Config database data loading completed"); + } + + static class ConnectorInfo { + + final String definitionId; + final JsonNode definition; + final String dockerRepository; + final String dockerImageTag; + + ConnectorInfo(final String definitionId, final JsonNode definition) { + this.definitionId = definitionId; + this.definition = definition; + this.dockerRepository = definition.get("dockerRepository").asText(); + this.dockerImageTag = definition.get("dockerImageTag").asText(); + } + + @Override + public String toString() { + return String.format("%s: %s (%s)", dockerRepository, dockerImageTag, definitionId); + } + + } + + private static class ConnectorCounter { + + private final int newCount; + private final int updateCount; + + private ConnectorCounter(final int newCount, final int updateCount) { + this.newCount = newCount; + this.updateCount = updateCount; + } - return ctx.select(field("config_blob ->> 'dockerRepository'", SQLDataType.VARCHAR)) - .from(AIRBYTE_CONFIGS) - .where(AIRBYTE_CONFIGS.CONFIG_ID.in(usedConnectorDefinitionIds)) - .fetch().stream() - .map(Record1::value1) - .collect(Collectors.toSet()); } } diff --git a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/DeprecatedDatabaseConfigPersistence.java b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/DeprecatedDatabaseConfigPersistence.java new file mode 100644 index 000000000000..138fc1812925 --- /dev/null +++ b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/DeprecatedDatabaseConfigPersistence.java @@ -0,0 +1,543 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.config.persistence; + +import static io.airbyte.db.instance.configs.jooq.Tables.AIRBYTE_CONFIGS; +import static org.jooq.impl.DSL.asterisk; +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.select; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Sets; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.util.MoreIterators; +import io.airbyte.commons.version.AirbyteVersion; +import io.airbyte.config.AirbyteConfig; +import io.airbyte.config.ConfigSchema; +import io.airbyte.config.ConfigSchemaMigrationSupport; +import io.airbyte.config.ConfigWithMetadata; +import io.airbyte.config.Configs; +import io.airbyte.config.StandardDestinationDefinition; +import io.airbyte.config.StandardSourceDefinition; +import io.airbyte.db.Database; +import io.airbyte.db.ExceptionWrappingDatabase; +import io.airbyte.validation.json.JsonValidationException; +import java.io.IOException; +import java.sql.SQLException; +import java.time.OffsetDateTime; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import javax.annotation.Nullable; +import org.jooq.DSLContext; +import org.jooq.Field; +import org.jooq.JSONB; +import org.jooq.Record; +import org.jooq.Record1; +import org.jooq.Result; +import org.jooq.impl.SQLDataType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Use {@link DatabaseConfigPersistence} + */ +@Deprecated +public class DeprecatedDatabaseConfigPersistence implements ConfigPersistence { + + private static final Logger LOGGER = LoggerFactory.getLogger(DeprecatedDatabaseConfigPersistence.class); + + private final ExceptionWrappingDatabase database; + + public DeprecatedDatabaseConfigPersistence(final Database database) { + this.database = new ExceptionWrappingDatabase(database); + } + + /** + * If this is a migration deployment from an old version that relies on file system config + * persistence, copy the existing configs from local files. + */ + public DeprecatedDatabaseConfigPersistence migrateFileConfigs(final Configs serverConfigs) throws IOException { + database.transaction(ctx -> { + final boolean isInitialized = ctx.fetchExists(AIRBYTE_CONFIGS); + if (isInitialized) { + return null; + } + + final boolean hasExistingFileConfigs = FileSystemConfigPersistence.hasExistingConfigs(serverConfigs.getConfigRoot()); + if (hasExistingFileConfigs) { + LOGGER.info("Load existing local config directory into configs database"); + final ConfigPersistence fileSystemPersistence = new FileSystemConfigPersistence(serverConfigs.getConfigRoot()); + copyConfigsFromSeed(ctx, fileSystemPersistence); + } + + return null; + }); + + return this; + } + + @Override + public void loadData(final ConfigPersistence seedConfigPersistence) throws IOException { + database.transaction(ctx -> { + updateConfigsFromSeed(ctx, seedConfigPersistence); + return null; + }); + } + + public Set getInUseConnectorDockerImageNames() throws IOException { + return database.transaction(this::getConnectorRepositoriesInUse); + } + + public ValidatingConfigPersistence withValidation() { + return new ValidatingConfigPersistence(this); + } + + @Override + public T getConfig(final AirbyteConfig configType, final String configId, final Class clazz) + throws ConfigNotFoundException, JsonValidationException, IOException { + final Result result = database.query(ctx -> ctx.select(asterisk()) + .from(AIRBYTE_CONFIGS) + .where(AIRBYTE_CONFIGS.CONFIG_TYPE.eq(configType.name()), AIRBYTE_CONFIGS.CONFIG_ID.eq(configId)) + .fetch()); + + if (result.isEmpty()) { + throw new ConfigNotFoundException(configType, configId); + } else if (result.size() > 1) { + throw new IllegalStateException(String.format("Multiple %s configs found for ID %s: %s", configType, configId, result)); + } + + return Jsons.deserialize(result.get(0).get(AIRBYTE_CONFIGS.CONFIG_BLOB).data(), clazz); + } + + @Override + public List listConfigs(final AirbyteConfig configType, final Class clazz) throws IOException { + final Result results = database.query(ctx -> ctx.select(asterisk()) + .from(AIRBYTE_CONFIGS) + .where(AIRBYTE_CONFIGS.CONFIG_TYPE.eq(configType.name())) + .orderBy(AIRBYTE_CONFIGS.CONFIG_TYPE, AIRBYTE_CONFIGS.CONFIG_ID) + .fetch()); + return results.stream() + .map(record -> Jsons.deserialize(record.get(AIRBYTE_CONFIGS.CONFIG_BLOB).data(), clazz)) + .collect(Collectors.toList()); + } + + @Override + public List> listConfigsWithMetadata(final AirbyteConfig configType, final Class clazz) throws IOException { + final Result results = database.query(ctx -> ctx.select(asterisk()) + .from(AIRBYTE_CONFIGS) + .where(AIRBYTE_CONFIGS.CONFIG_TYPE.eq(configType.name())) + .orderBy(AIRBYTE_CONFIGS.CONFIG_TYPE, AIRBYTE_CONFIGS.CONFIG_ID) + .fetch()); + return results.stream() + .map(record -> new ConfigWithMetadata<>( + record.get(AIRBYTE_CONFIGS.CONFIG_ID), + record.get(AIRBYTE_CONFIGS.CONFIG_TYPE), + record.get(AIRBYTE_CONFIGS.CREATED_AT).toInstant(), + record.get(AIRBYTE_CONFIGS.UPDATED_AT).toInstant(), + Jsons.deserialize(record.get(AIRBYTE_CONFIGS.CONFIG_BLOB).data(), clazz))) + .collect(Collectors.toList()); + } + + @Override + public void writeConfig(final AirbyteConfig configType, final String configId, final T config) throws IOException { + final Map configIdToConfig = new HashMap<>() { + + { + put(configId, config); + } + + }; + writeConfigs(configType, configIdToConfig); + } + + @Override + public void writeConfigs(final AirbyteConfig configType, final Map configs) throws IOException { + database.transaction(ctx -> { + final OffsetDateTime timestamp = OffsetDateTime.now(); + configs.forEach((configId, config) -> { + final boolean isExistingConfig = ctx.fetchExists(select() + .from(AIRBYTE_CONFIGS) + .where(AIRBYTE_CONFIGS.CONFIG_TYPE.eq(configType.name()), AIRBYTE_CONFIGS.CONFIG_ID.eq(configId))); + + if (isExistingConfig) { + updateConfigRecord(ctx, timestamp, configType.name(), Jsons.jsonNode(config), configId); + } else { + insertConfigRecord(ctx, timestamp, configType.name(), Jsons.jsonNode(config), + configType.getIdFieldName()); + } + }); + + return null; + }); + } + + @Override + public void deleteConfig(final AirbyteConfig configType, final String configId) throws IOException { + database.transaction(ctx -> { + final boolean isExistingConfig = ctx.fetchExists(select() + .from(AIRBYTE_CONFIGS) + .where(AIRBYTE_CONFIGS.CONFIG_TYPE.eq(configType.name()), AIRBYTE_CONFIGS.CONFIG_ID.eq(configId))); + + if (isExistingConfig) { + ctx.deleteFrom(AIRBYTE_CONFIGS) + .where(AIRBYTE_CONFIGS.CONFIG_TYPE.eq(configType.name()), AIRBYTE_CONFIGS.CONFIG_ID.eq(configId)) + .execute(); + return null; + } + return null; + }); + } + + @Override + public void replaceAllConfigs(final Map> configs, final boolean dryRun) throws IOException { + if (dryRun) { + return; + } + + LOGGER.info("Replacing all configs"); + + final OffsetDateTime timestamp = OffsetDateTime.now(); + final int insertionCount = database.transaction(ctx -> { + ctx.truncate(AIRBYTE_CONFIGS).restartIdentity().execute(); + + return configs.entrySet().stream().map(entry -> { + final AirbyteConfig configType = entry.getKey(); + return entry.getValue() + .map(configObject -> insertConfigRecord(ctx, timestamp, configType.name(), Jsons.jsonNode(configObject), configType.getIdFieldName())) + .reduce(0, Integer::sum); + }).reduce(0, Integer::sum); + }); + + LOGGER.info("Config database is reset with {} records", insertionCount); + } + + @Override + public Map> dumpConfigs() throws IOException { + LOGGER.info("Exporting all configs..."); + + final Map> results = database.query(ctx -> ctx.select(asterisk()) + .from(AIRBYTE_CONFIGS) + .orderBy(AIRBYTE_CONFIGS.CONFIG_TYPE, AIRBYTE_CONFIGS.CONFIG_ID) + .fetchGroups(AIRBYTE_CONFIGS.CONFIG_TYPE)); + return results.entrySet().stream().collect(Collectors.toMap( + Entry::getKey, + e -> e.getValue().stream().map(r -> Jsons.deserialize(r.get(AIRBYTE_CONFIGS.CONFIG_BLOB).data())))); + } + + /** + * @return the number of inserted records for convenience, which is always 1. + */ + @VisibleForTesting + int insertConfigRecord( + final DSLContext ctx, + final OffsetDateTime timestamp, + final String configType, + final JsonNode configJson, + @Nullable final String idFieldName) { + final String configId = idFieldName == null + ? UUID.randomUUID().toString() + : configJson.get(idFieldName).asText(); + LOGGER.info("Inserting {} record {}", configType, configId); + + final int insertionCount = ctx.insertInto(AIRBYTE_CONFIGS) + .set(AIRBYTE_CONFIGS.CONFIG_ID, configId) + .set(AIRBYTE_CONFIGS.CONFIG_TYPE, configType) + .set(AIRBYTE_CONFIGS.CONFIG_BLOB, JSONB.valueOf(Jsons.serialize(configJson))) + .set(AIRBYTE_CONFIGS.CREATED_AT, timestamp) + .set(AIRBYTE_CONFIGS.UPDATED_AT, timestamp) + .onConflict(AIRBYTE_CONFIGS.CONFIG_TYPE, AIRBYTE_CONFIGS.CONFIG_ID) + .doNothing() + .execute(); + if (insertionCount != 1) { + LOGGER.warn("{} config {} already exists (insertion record count: {})", configType, configId, insertionCount); + } + return insertionCount; + } + + /** + * @return the number of updated records. + */ + @VisibleForTesting + int updateConfigRecord(final DSLContext ctx, + final OffsetDateTime timestamp, + final String configType, + final JsonNode configJson, + final String configId) { + LOGGER.info("Updating {} record {}", configType, configId); + + final int updateCount = ctx.update(AIRBYTE_CONFIGS) + .set(AIRBYTE_CONFIGS.CONFIG_BLOB, JSONB.valueOf(Jsons.serialize(configJson))) + .set(AIRBYTE_CONFIGS.UPDATED_AT, timestamp) + .where(AIRBYTE_CONFIGS.CONFIG_TYPE.eq(configType), AIRBYTE_CONFIGS.CONFIG_ID.eq(configId)) + .execute(); + if (updateCount != 1) { + LOGGER.warn("{} config {} is not updated (updated record count: {})", configType, configId, updateCount); + } + return updateCount; + } + + @VisibleForTesting + void copyConfigsFromSeed(final DSLContext ctx, final ConfigPersistence seedConfigPersistence) throws SQLException { + LOGGER.info("Loading seed data to config database..."); + + final Map> seedConfigs; + try { + seedConfigs = seedConfigPersistence.dumpConfigs(); + } catch (final IOException e) { + throw new SQLException(e); + } + + final OffsetDateTime timestamp = OffsetDateTime.now(); + final int insertionCount = seedConfigs.entrySet().stream().map(entry -> { + final String configType = entry.getKey(); + return entry.getValue().map(configJson -> { + final String idFieldName = ConfigSchemaMigrationSupport.CONFIG_SCHEMA_ID_FIELD_NAMES.get(configType); + return insertConfigRecord(ctx, timestamp, configType, configJson, idFieldName); + }).reduce(0, Integer::sum); + }).reduce(0, Integer::sum); + + LOGGER.info("Config database data loading completed with {} records", insertionCount); + } + + static class ConnectorInfo { + + final String definitionId; + final JsonNode definition; + final String dockerRepository; + final String dockerImageTag; + + ConnectorInfo(final String definitionId, final JsonNode definition) { + this.definitionId = definitionId; + this.definition = definition; + this.dockerRepository = definition.get("dockerRepository").asText(); + this.dockerImageTag = definition.get("dockerImageTag").asText(); + } + + @Override + public String toString() { + return String.format("%s: %s (%s)", dockerRepository, dockerImageTag, definitionId); + } + + } + + private static class ConnectorCounter { + + private final int newCount; + private final int updateCount; + + private ConnectorCounter(final int newCount, final int updateCount) { + this.newCount = newCount; + this.updateCount = updateCount; + } + + } + + @VisibleForTesting + void updateConfigsFromSeed(final DSLContext ctx, final ConfigPersistence seedConfigPersistence) throws SQLException { + LOGGER.info("Updating connector definitions from the seed if necessary..."); + + try { + final Set connectorRepositoriesInUse = getConnectorRepositoriesInUse(ctx); + LOGGER.info("Connectors in use: {}", connectorRepositoriesInUse); + + final Map connectorRepositoryToInfoMap = getConnectorRepositoryToInfoMap(ctx); + LOGGER.info("Current connector versions: {}", connectorRepositoryToInfoMap.values()); + + final OffsetDateTime timestamp = OffsetDateTime.now(); + int newConnectorCount = 0; + int updatedConnectorCount = 0; + + final List latestSources = seedConfigPersistence.listConfigs( + ConfigSchema.STANDARD_SOURCE_DEFINITION, StandardSourceDefinition.class); + final ConnectorCounter sourceConnectorCounter = updateConnectorDefinitions(ctx, timestamp, ConfigSchema.STANDARD_SOURCE_DEFINITION, + latestSources, connectorRepositoriesInUse, connectorRepositoryToInfoMap); + newConnectorCount += sourceConnectorCounter.newCount; + updatedConnectorCount += sourceConnectorCounter.updateCount; + + final List latestDestinations = seedConfigPersistence.listConfigs( + ConfigSchema.STANDARD_DESTINATION_DEFINITION, StandardDestinationDefinition.class); + final ConnectorCounter destinationConnectorCounter = updateConnectorDefinitions(ctx, timestamp, ConfigSchema.STANDARD_DESTINATION_DEFINITION, + latestDestinations, connectorRepositoriesInUse, connectorRepositoryToInfoMap); + newConnectorCount += destinationConnectorCounter.newCount; + updatedConnectorCount += destinationConnectorCounter.updateCount; + + LOGGER.info("Connector definitions have been updated ({} new connectors, and {} updates)", newConnectorCount, updatedConnectorCount); + } catch (final IOException | JsonValidationException e) { + throw new SQLException(e); + } + } + + /** + * @param connectorRepositoriesInUse when a connector is used in any standard sync, its definition + * will not be updated. This is necessary because the new connector version may not be + * backward compatible. + */ + @VisibleForTesting + ConnectorCounter updateConnectorDefinitions(final DSLContext ctx, + final OffsetDateTime timestamp, + final AirbyteConfig configType, + final List latestDefinitions, + final Set connectorRepositoriesInUse, + final Map connectorRepositoryToIdVersionMap) + throws IOException { + int newCount = 0; + int updatedCount = 0; + + for (final T definition : latestDefinitions) { + final JsonNode latestDefinition = Jsons.jsonNode(definition); + final String repository = latestDefinition.get("dockerRepository").asText(); + + // Add new connector + if (!connectorRepositoryToIdVersionMap.containsKey(repository)) { + LOGGER.info("Adding new connector {}: {}", repository, latestDefinition); + newCount += insertConfigRecord(ctx, timestamp, configType.name(), latestDefinition, configType.getIdFieldName()); + continue; + } + + final ConnectorInfo connectorInfo = connectorRepositoryToIdVersionMap.get(repository); + final JsonNode currentDefinition = connectorInfo.definition; + + // todo (lmossman) - this logic to remove the "spec" field is temporary; it is necessary to avoid + // breaking users who are actively using an old connector version, otherwise specs from the most + // recent connector versions may be inserted into the db which could be incompatible with the + // version they are actually using. + // Once the faux major version bump has been merged, this "new field" logic will be removed + // entirely. + final Set newFields = Sets.difference(getNewFields(currentDefinition, latestDefinition), Set.of("spec")); + + // Process connector in use + if (connectorRepositoriesInUse.contains(repository)) { + if (newFields.size() == 0) { + LOGGER.info("Connector {} is in use and has all fields; skip updating", repository); + } else { + // Add new fields to the connector definition + final JsonNode definitionToUpdate = getDefinitionWithNewFields(currentDefinition, latestDefinition, newFields); + LOGGER.info("Connector {} has new fields: {}", repository, String.join(", ", newFields)); + updatedCount += updateConfigRecord(ctx, timestamp, configType.name(), definitionToUpdate, connectorInfo.definitionId); + } + continue; + } + + // Process unused connector + final String latestImageTag = latestDefinition.get("dockerImageTag").asText(); + if (hasNewVersion(connectorInfo.dockerImageTag, latestImageTag)) { + // Update connector to the latest version + LOGGER.info("Connector {} needs update: {} vs {}", repository, connectorInfo.dockerImageTag, latestImageTag); + updatedCount += updateConfigRecord(ctx, timestamp, configType.name(), latestDefinition, connectorInfo.definitionId); + } else if (newFields.size() > 0) { + // Add new fields to the connector definition + final JsonNode definitionToUpdate = getDefinitionWithNewFields(currentDefinition, latestDefinition, newFields); + LOGGER.info("Connector {} has new fields: {}", repository, String.join(", ", newFields)); + updatedCount += updateConfigRecord(ctx, timestamp, configType.name(), definitionToUpdate, connectorInfo.definitionId); + } else { + LOGGER.info("Connector {} does not need update: {}", repository, connectorInfo.dockerImageTag); + } + } + + return new ConnectorCounter(newCount, updatedCount); + } + + static boolean hasNewVersion(final String currentVersion, final String latestVersion) { + try { + return new AirbyteVersion(latestVersion).patchVersionCompareTo(new AirbyteVersion(currentVersion)) > 0; + } catch (final Exception e) { + LOGGER.error("Failed to check version: {} vs {}", currentVersion, latestVersion); + return false; + } + } + + /** + * @return new fields from the latest definition + */ + static Set getNewFields(final JsonNode currentDefinition, final JsonNode latestDefinition) { + final Set currentFields = MoreIterators.toSet(currentDefinition.fieldNames()); + final Set latestFields = MoreIterators.toSet(latestDefinition.fieldNames()); + return Sets.difference(latestFields, currentFields); + } + + /** + * @return a clone of the current definition with the new fields from the latest definition. + */ + static JsonNode getDefinitionWithNewFields(final JsonNode currentDefinition, final JsonNode latestDefinition, final Set newFields) { + final ObjectNode currentClone = (ObjectNode) Jsons.clone(currentDefinition); + newFields.forEach(field -> currentClone.set(field, latestDefinition.get(field))); + return currentClone; + } + + /** + * @return A map about current connectors (both source and destination). It maps from connector + * repository to its definition id and docker image tag. We identify a connector by its + * repository name instead of definition id because connectors can be added manually by + * users, and are not always the same as those in the seed. + */ + @VisibleForTesting + Map getConnectorRepositoryToInfoMap(final DSLContext ctx) { + final Field configField = field("config_blob", SQLDataType.JSONB).as("definition"); + final Field repoField = field("config_blob ->> 'dockerRepository'", SQLDataType.VARCHAR).as("repository"); + return ctx.select(AIRBYTE_CONFIGS.CONFIG_ID, repoField, configField) + .from(AIRBYTE_CONFIGS) + .where(AIRBYTE_CONFIGS.CONFIG_TYPE.in(ConfigSchema.STANDARD_SOURCE_DEFINITION.name(), ConfigSchema.STANDARD_DESTINATION_DEFINITION.name())) + .fetch().stream() + .collect(Collectors.toMap( + row -> row.getValue(repoField), + row -> new ConnectorInfo(row.getValue(AIRBYTE_CONFIGS.CONFIG_ID), Jsons.deserialize(row.getValue(configField).data())), + // when there are duplicated connector definitions, return the latest one + (c1, c2) -> { + final AirbyteVersion v1 = new AirbyteVersion(c1.dockerImageTag); + final AirbyteVersion v2 = new AirbyteVersion(c2.dockerImageTag); + LOGGER.warn("Duplicated connector version found for {}: {} ({}) vs {} ({})", + c1.dockerRepository, c1.dockerImageTag, c1.definitionId, c2.dockerImageTag, c2.definitionId); + final int comparison = v1.patchVersionCompareTo(v2); + if (comparison >= 0) { + return c1; + } else { + return c2; + } + })); + } + + /** + * @return A set of connectors (both source and destination) that are already used in standard + * syncs. We identify connectors by its repository name instead of definition id because + * connectors can be added manually by users, and their config ids are not always the same + * as those in the seed. + */ + private Set getConnectorRepositoriesInUse(final DSLContext ctx) { + final Set usedConnectorDefinitionIds = new HashSet<>(); + // query for used source definitions + usedConnectorDefinitionIds.addAll(ctx + .select(field("config_blob ->> 'sourceDefinitionId'", SQLDataType.VARCHAR)) + .from(AIRBYTE_CONFIGS) + .where(AIRBYTE_CONFIGS.CONFIG_TYPE.eq(ConfigSchema.SOURCE_CONNECTION.name())) + .fetch().stream() + .flatMap(row -> Stream.of(row.value1())) + .collect(Collectors.toSet())); + // query for used destination definitions + usedConnectorDefinitionIds.addAll(ctx + .select(field("config_blob ->> 'destinationDefinitionId'", SQLDataType.VARCHAR)) + .from(AIRBYTE_CONFIGS) + .where(AIRBYTE_CONFIGS.CONFIG_TYPE.eq(ConfigSchema.DESTINATION_CONNECTION.name())) + .fetch().stream() + .flatMap(row -> Stream.of(row.value1())) + .collect(Collectors.toSet())); + + return ctx.select(field("config_blob ->> 'dockerRepository'", SQLDataType.VARCHAR)) + .from(AIRBYTE_CONFIGS) + .where(AIRBYTE_CONFIGS.CONFIG_ID.in(usedConnectorDefinitionIds)) + .fetch().stream() + .map(Record1::value1) + .collect(Collectors.toSet()); + } + +} diff --git a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/BaseDatabaseConfigPersistenceTest.java b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/BaseDatabaseConfigPersistenceTest.java index bb0053f52c18..de6dea695b43 100644 --- a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/BaseDatabaseConfigPersistenceTest.java +++ b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/BaseDatabaseConfigPersistenceTest.java @@ -25,6 +25,7 @@ import java.util.stream.Stream; import org.jooq.Record1; import org.jooq.Result; +import org.jooq.Table; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.testcontainers.containers.PostgreSQLContainer; @@ -114,8 +115,8 @@ protected void assertSameConfigDump(final Map> expected assertEquals(getMapWithSet(expected), getMapWithSet(actual)); } - protected void assertRecordCount(final int expectedCount) throws Exception { - final Result> recordCount = database.query(ctx -> ctx.select(count(asterisk())).from(table("airbyte_configs")).fetch()); + protected void assertRecordCount(final int expectedCount, final Table table) throws Exception { + final Result> recordCount = database.query(ctx -> ctx.select(count(asterisk())).from(table).fetch()); assertEquals(expectedCount, recordCount.get(0).value1()); } diff --git a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/BaseDeprecatedDatabaseConfigPersistenceTest.java b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/BaseDeprecatedDatabaseConfigPersistenceTest.java new file mode 100644 index 000000000000..d08f8f1f5fa2 --- /dev/null +++ b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/BaseDeprecatedDatabaseConfigPersistenceTest.java @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.config.persistence; + +import static org.jooq.impl.DSL.asterisk; +import static org.jooq.impl.DSL.count; +import static org.jooq.impl.DSL.table; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.config.ConfigSchema; +import io.airbyte.config.StandardDestinationDefinition; +import io.airbyte.config.StandardSourceDefinition; +import io.airbyte.config.StandardSourceDefinition.SourceType; +import io.airbyte.db.Database; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.UUID; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.jooq.Record1; +import org.jooq.Result; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.testcontainers.containers.PostgreSQLContainer; + +/** + * This class provides downstream tests with constants and helpers. + */ +public abstract class BaseDeprecatedDatabaseConfigPersistenceTest { + + protected static PostgreSQLContainer container; + protected static Database database; + protected static DeprecatedDatabaseConfigPersistence configPersistence; + + @BeforeAll + public static void dbSetup() { + container = new PostgreSQLContainer<>("postgres:13-alpine") + .withDatabaseName("airbyte") + .withUsername("docker") + .withPassword("docker"); + container.start(); + } + + @AfterAll + public static void dbDown() { + container.close(); + } + + protected static final StandardSourceDefinition SOURCE_GITHUB = new StandardSourceDefinition() + .withName("GitHub") + .withSourceDefinitionId(UUID.fromString("ef69ef6e-aa7f-4af1-a01d-ef775033524e")) + .withDockerRepository("airbyte/source-github") + .withDockerImageTag("0.2.3") + .withDocumentationUrl("https://docs.airbyte.io/integrations/sources/github") + .withIcon("github.svg") + .withSourceType(SourceType.API); + protected static final StandardSourceDefinition SOURCE_POSTGRES = new StandardSourceDefinition() + .withName("Postgres") + .withSourceDefinitionId(UUID.fromString("decd338e-5647-4c0b-adf4-da0e75f5a750")) + .withDockerRepository("airbyte/source-postgres") + .withDockerImageTag("0.3.11") + .withDocumentationUrl("https://docs.airbyte.io/integrations/sources/postgres") + .withIcon("postgresql.svg") + .withSourceType(SourceType.DATABASE); + protected static final StandardDestinationDefinition DESTINATION_SNOWFLAKE = new StandardDestinationDefinition() + .withName("Snowflake") + .withDestinationDefinitionId(UUID.fromString("424892c4-daac-4491-b35d-c6688ba547ba")) + .withDockerRepository("airbyte/destination-snowflake") + .withDockerImageTag("0.3.16") + .withDocumentationUrl("https://docs.airbyte.io/integrations/destinations/snowflake"); + protected static final StandardDestinationDefinition DESTINATION_S3 = new StandardDestinationDefinition() + .withName("S3") + .withDestinationDefinitionId(UUID.fromString("4816b78f-1489-44c1-9060-4b19d5fa9362")) + .withDockerRepository("airbyte/destination-s3") + .withDockerImageTag("0.1.12") + .withDocumentationUrl("https://docs.airbyte.io/integrations/destinations/s3"); + + protected static void writeSource(final ConfigPersistence configPersistence, final StandardSourceDefinition source) throws Exception { + configPersistence.writeConfig(ConfigSchema.STANDARD_SOURCE_DEFINITION, source.getSourceDefinitionId().toString(), source); + } + + protected static void writeDestination(final ConfigPersistence configPersistence, final StandardDestinationDefinition destination) + throws Exception { + configPersistence.writeConfig(ConfigSchema.STANDARD_DESTINATION_DEFINITION, destination.getDestinationDefinitionId().toString(), destination); + } + + protected static void writeDestinations(final ConfigPersistence configPersistence, final List destinations) + throws Exception { + final Map destinationsByID = destinations.stream() + .collect(Collectors.toMap(destinationDefinition -> destinationDefinition.getDestinationDefinitionId().toString(), Function.identity())); + configPersistence.writeConfigs(ConfigSchema.STANDARD_DESTINATION_DEFINITION, destinationsByID); + } + + protected static void deleteDestination(final ConfigPersistence configPersistence, final StandardDestinationDefinition destination) + throws Exception { + configPersistence.deleteConfig(ConfigSchema.STANDARD_DESTINATION_DEFINITION, destination.getDestinationDefinitionId().toString()); + } + + protected Map> getMapWithSet(final Map> input) { + return input.entrySet().stream().collect(Collectors.toMap( + Entry::getKey, + e -> e.getValue().collect(Collectors.toSet()))); + } + + // assertEquals cannot correctly check the equality of two maps with stream values, + // so streams are converted to sets before being compared. + protected void assertSameConfigDump(final Map> expected, final Map> actual) { + assertEquals(getMapWithSet(expected), getMapWithSet(actual)); + } + + protected void assertRecordCount(final int expectedCount) throws Exception { + final Result> recordCount = database.query(ctx -> ctx.select(count(asterisk())).from(table("airbyte_configs")).fetch()); + assertEquals(expectedCount, recordCount.get(0).value1()); + } + + protected void assertHasSource(final StandardSourceDefinition source) throws Exception { + assertEquals(source, configPersistence + .getConfig(ConfigSchema.STANDARD_SOURCE_DEFINITION, source.getSourceDefinitionId().toString(), + StandardSourceDefinition.class)); + } + + protected void assertHasDestination(final StandardDestinationDefinition destination) throws Exception { + assertEquals(destination, configPersistence + .getConfig(ConfigSchema.STANDARD_DESTINATION_DEFINITION, destination.getDestinationDefinitionId().toString(), + StandardDestinationDefinition.class)); + } + +} diff --git a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceE2EReadWriteTest.java b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceE2EReadWriteTest.java new file mode 100644 index 000000000000..4466de916c0c --- /dev/null +++ b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceE2EReadWriteTest.java @@ -0,0 +1,252 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.config.persistence; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.spy; + +import io.airbyte.config.ConfigSchema; +import io.airbyte.config.DestinationConnection; +import io.airbyte.config.DestinationOAuthParameter; +import io.airbyte.config.SourceConnection; +import io.airbyte.config.SourceOAuthParameter; +import io.airbyte.config.StandardDestinationDefinition; +import io.airbyte.config.StandardSourceDefinition; +import io.airbyte.config.StandardSync; +import io.airbyte.config.StandardSyncOperation; +import io.airbyte.config.StandardSyncState; +import io.airbyte.config.StandardWorkspace; +import io.airbyte.db.instance.configs.ConfigsDatabaseInstance; +import io.airbyte.db.instance.configs.ConfigsDatabaseMigrator; +import io.airbyte.db.instance.development.DevDatabaseMigrator; +import io.airbyte.db.instance.development.MigrationDevHelper; +import io.airbyte.validation.json.JsonValidationException; +import java.io.IOException; +import java.util.List; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class DatabaseConfigPersistenceE2EReadWriteTest extends BaseDatabaseConfigPersistenceTest { + + @BeforeEach + public void setup() throws Exception { + database = new ConfigsDatabaseInstance(container.getUsername(), container.getPassword(), container.getJdbcUrl()).getAndInitialize(); + configPersistence = spy(new DatabaseConfigPersistence(database)); + final ConfigsDatabaseMigrator configsDatabaseMigrator = + new ConfigsDatabaseMigrator(database, DatabaseConfigPersistenceLoadDataTest.class.getName()); + final DevDatabaseMigrator devDatabaseMigrator = new DevDatabaseMigrator(configsDatabaseMigrator); + MigrationDevHelper.runLastMigration(devDatabaseMigrator); + database.query(ctx -> ctx + .execute("TRUNCATE TABLE state, connection_operation, connection, operation, actor_oauth_parameter, actor, actor_definition, workspace")); + } + + @AfterEach + void tearDown() throws Exception { + database.close(); + } + + @Test + public void test() throws JsonValidationException, IOException, ConfigNotFoundException { + standardWorkspace(); + standardSourceDefinition(); + standardDestinationDefinition(); + sourceConnection(); + destinationConnection(); + sourceOauthParam(); + destinationOauthParam(); + standardSyncOperation(); + standardSync(); + standardSyncState(); + deletion(); + } + + private void deletion() throws ConfigNotFoundException, IOException, JsonValidationException { + // Deleting the workspace should delete everything except for definitions + configPersistence.deleteConfig(ConfigSchema.STANDARD_WORKSPACE, MockData.standardWorkspace().getWorkspaceId().toString()); + Assertions.assertTrue(configPersistence.listConfigs(ConfigSchema.STANDARD_SYNC_STATE, StandardSyncState.class).isEmpty()); + Assertions.assertTrue(configPersistence.listConfigs(ConfigSchema.STANDARD_SYNC, StandardSync.class).isEmpty()); + Assertions.assertTrue(configPersistence.listConfigs(ConfigSchema.STANDARD_SYNC_OPERATION, StandardSyncOperation.class).isEmpty()); + Assertions.assertTrue(configPersistence.listConfigs(ConfigSchema.DESTINATION_OAUTH_PARAM, DestinationOAuthParameter.class).isEmpty()); + Assertions.assertTrue(configPersistence.listConfigs(ConfigSchema.SOURCE_OAUTH_PARAM, SourceOAuthParameter.class).isEmpty()); + Assertions.assertTrue(configPersistence.listConfigs(ConfigSchema.DESTINATION_CONNECTION, SourceConnection.class).isEmpty()); + Assertions.assertTrue(configPersistence.listConfigs(ConfigSchema.STANDARD_WORKSPACE, StandardWorkspace.class).isEmpty()); + + Assertions.assertFalse(configPersistence.listConfigs(ConfigSchema.STANDARD_SOURCE_DEFINITION, StandardSourceDefinition.class).isEmpty()); + Assertions + .assertFalse(configPersistence.listConfigs(ConfigSchema.STANDARD_DESTINATION_DEFINITION, StandardDestinationDefinition.class).isEmpty()); + + for (final StandardSourceDefinition standardSourceDefinition : MockData.standardSourceDefinitions()) { + configPersistence.deleteConfig(ConfigSchema.STANDARD_SOURCE_DEFINITION, standardSourceDefinition.getSourceDefinitionId().toString()); + } + Assertions.assertTrue(configPersistence.listConfigs(ConfigSchema.STANDARD_SOURCE_DEFINITION, StandardSourceDefinition.class).isEmpty()); + + for (final StandardDestinationDefinition standardDestinationDefinition : MockData.standardDestinationDefinitions()) { + configPersistence + .deleteConfig(ConfigSchema.STANDARD_DESTINATION_DEFINITION, standardDestinationDefinition.getDestinationDefinitionId().toString()); + } + Assertions.assertTrue(configPersistence.listConfigs(ConfigSchema.STANDARD_DESTINATION_DEFINITION, StandardDestinationDefinition.class).isEmpty()); + } + + private void standardSyncState() throws JsonValidationException, IOException, ConfigNotFoundException { + for (final StandardSyncState standardSyncState : MockData.standardSyncStates()) { + configPersistence.writeConfig(ConfigSchema.STANDARD_SYNC_STATE, + standardSyncState.getConnectionId().toString(), + standardSyncState); + final StandardSyncState standardSyncStateFromDB = configPersistence.getConfig(ConfigSchema.STANDARD_SYNC_STATE, + standardSyncState.getConnectionId().toString(), + StandardSyncState.class); + Assertions.assertEquals(standardSyncState, standardSyncStateFromDB); + } + final List standardSyncStates = configPersistence + .listConfigs(ConfigSchema.STANDARD_SYNC_STATE, StandardSyncState.class); + Assertions.assertEquals(MockData.standardSyncStates().size(), standardSyncStates.size()); + assertThat(MockData.standardSyncStates()).hasSameElementsAs(standardSyncStates); + } + + private void standardSync() throws JsonValidationException, IOException, ConfigNotFoundException { + for (final StandardSync standardSync : MockData.standardSyncs()) { + configPersistence.writeConfig(ConfigSchema.STANDARD_SYNC, + standardSync.getConnectionId().toString(), + standardSync); + final StandardSync standardSyncFromDB = configPersistence.getConfig(ConfigSchema.STANDARD_SYNC, + standardSync.getConnectionId().toString(), + StandardSync.class); + Assertions.assertEquals(standardSync, standardSyncFromDB); + } + final List standardSyncs = configPersistence + .listConfigs(ConfigSchema.STANDARD_SYNC, StandardSync.class); + Assertions.assertEquals(MockData.standardSyncs().size(), standardSyncs.size()); + assertThat(MockData.standardSyncs()).hasSameElementsAs(standardSyncs); + } + + private void standardSyncOperation() throws JsonValidationException, IOException, ConfigNotFoundException { + for (final StandardSyncOperation standardSyncOperation : MockData.standardSyncOperations()) { + configPersistence.writeConfig(ConfigSchema.STANDARD_SYNC_OPERATION, + standardSyncOperation.getOperationId().toString(), + standardSyncOperation); + final StandardSyncOperation standardSyncOperationFromDB = configPersistence.getConfig(ConfigSchema.STANDARD_SYNC_OPERATION, + standardSyncOperation.getOperationId().toString(), + StandardSyncOperation.class); + Assertions.assertEquals(standardSyncOperation, standardSyncOperationFromDB); + } + final List standardSyncOperations = configPersistence + .listConfigs(ConfigSchema.STANDARD_SYNC_OPERATION, StandardSyncOperation.class); + Assertions.assertEquals(MockData.standardSyncOperations().size(), standardSyncOperations.size()); + assertThat(MockData.standardSyncOperations()).hasSameElementsAs(standardSyncOperations); + } + + private void destinationOauthParam() throws JsonValidationException, IOException, ConfigNotFoundException { + for (final DestinationOAuthParameter destinationOAuthParameter : MockData.destinationOauthParameters()) { + configPersistence.writeConfig(ConfigSchema.DESTINATION_OAUTH_PARAM, + destinationOAuthParameter.getOauthParameterId().toString(), + destinationOAuthParameter); + final DestinationOAuthParameter destinationOAuthParameterFromDB = configPersistence.getConfig(ConfigSchema.DESTINATION_OAUTH_PARAM, + destinationOAuthParameter.getOauthParameterId().toString(), + DestinationOAuthParameter.class); + Assertions.assertEquals(destinationOAuthParameter, destinationOAuthParameterFromDB); + } + final List destinationOAuthParameters = configPersistence + .listConfigs(ConfigSchema.DESTINATION_OAUTH_PARAM, DestinationOAuthParameter.class); + Assertions.assertEquals(MockData.destinationOauthParameters().size(), destinationOAuthParameters.size()); + assertThat(MockData.destinationOauthParameters()).hasSameElementsAs(destinationOAuthParameters); + } + + private void sourceOauthParam() throws JsonValidationException, IOException, ConfigNotFoundException { + for (final SourceOAuthParameter sourceOAuthParameter : MockData.sourceOauthParameters()) { + configPersistence.writeConfig(ConfigSchema.SOURCE_OAUTH_PARAM, + sourceOAuthParameter.getOauthParameterId().toString(), + sourceOAuthParameter); + final SourceOAuthParameter sourceOAuthParameterFromDB = configPersistence.getConfig(ConfigSchema.SOURCE_OAUTH_PARAM, + sourceOAuthParameter.getOauthParameterId().toString(), + SourceOAuthParameter.class); + Assertions.assertEquals(sourceOAuthParameter, sourceOAuthParameterFromDB); + } + final List sourceOAuthParameters = configPersistence + .listConfigs(ConfigSchema.SOURCE_OAUTH_PARAM, SourceOAuthParameter.class); + Assertions.assertEquals(MockData.sourceOauthParameters().size(), sourceOAuthParameters.size()); + assertThat(MockData.sourceOauthParameters()).hasSameElementsAs(sourceOAuthParameters); + } + + private void destinationConnection() throws JsonValidationException, IOException, ConfigNotFoundException { + for (final DestinationConnection destinationConnection : MockData.destinationConnections()) { + configPersistence.writeConfig(ConfigSchema.DESTINATION_CONNECTION, + destinationConnection.getDestinationId().toString(), + destinationConnection); + final DestinationConnection destinationConnectionFromDB = configPersistence.getConfig(ConfigSchema.DESTINATION_CONNECTION, + destinationConnection.getDestinationId().toString(), + DestinationConnection.class); + Assertions.assertEquals(destinationConnection, destinationConnectionFromDB); + } + final List destinationConnections = configPersistence + .listConfigs(ConfigSchema.DESTINATION_CONNECTION, DestinationConnection.class); + Assertions.assertEquals(MockData.destinationConnections().size(), destinationConnections.size()); + assertThat(MockData.destinationConnections()).hasSameElementsAs(destinationConnections); + } + + private void sourceConnection() throws JsonValidationException, IOException, ConfigNotFoundException { + for (final SourceConnection sourceConnection : MockData.sourceConnections()) { + configPersistence.writeConfig(ConfigSchema.SOURCE_CONNECTION, + sourceConnection.getSourceId().toString(), + sourceConnection); + final SourceConnection sourceConnectionFromDB = configPersistence.getConfig(ConfigSchema.SOURCE_CONNECTION, + sourceConnection.getSourceId().toString(), + SourceConnection.class); + Assertions.assertEquals(sourceConnection, sourceConnectionFromDB); + } + final List sourceConnections = configPersistence + .listConfigs(ConfigSchema.SOURCE_CONNECTION, SourceConnection.class); + Assertions.assertEquals(MockData.sourceConnections().size(), sourceConnections.size()); + assertThat(MockData.sourceConnections()).hasSameElementsAs(sourceConnections); + } + + private void standardDestinationDefinition() throws JsonValidationException, IOException, ConfigNotFoundException { + for (final StandardDestinationDefinition standardDestinationDefinition : MockData.standardDestinationDefinitions()) { + configPersistence.writeConfig(ConfigSchema.STANDARD_DESTINATION_DEFINITION, + standardDestinationDefinition.getDestinationDefinitionId().toString(), + standardDestinationDefinition); + final StandardDestinationDefinition standardDestinationDefinitionFromDB = configPersistence + .getConfig(ConfigSchema.STANDARD_DESTINATION_DEFINITION, + standardDestinationDefinition.getDestinationDefinitionId().toString(), + StandardDestinationDefinition.class); + Assertions.assertEquals(standardDestinationDefinition, standardDestinationDefinitionFromDB); + } + final List standardDestinationDefinitions = configPersistence + .listConfigs(ConfigSchema.STANDARD_DESTINATION_DEFINITION, StandardDestinationDefinition.class); + Assertions.assertEquals(MockData.standardDestinationDefinitions().size(), standardDestinationDefinitions.size()); + assertThat(MockData.standardDestinationDefinitions()).hasSameElementsAs(standardDestinationDefinitions); + } + + private void standardSourceDefinition() throws JsonValidationException, IOException, ConfigNotFoundException { + for (final StandardSourceDefinition standardSourceDefinition : MockData.standardSourceDefinitions()) { + configPersistence.writeConfig(ConfigSchema.STANDARD_SOURCE_DEFINITION, + standardSourceDefinition.getSourceDefinitionId().toString(), + standardSourceDefinition); + final StandardSourceDefinition standardSourceDefinitionFromDB = configPersistence.getConfig(ConfigSchema.STANDARD_SOURCE_DEFINITION, + standardSourceDefinition.getSourceDefinitionId().toString(), + StandardSourceDefinition.class); + Assertions.assertEquals(standardSourceDefinition, standardSourceDefinitionFromDB); + } + final List standardSourceDefinitions = configPersistence + .listConfigs(ConfigSchema.STANDARD_SOURCE_DEFINITION, StandardSourceDefinition.class); + Assertions.assertEquals(MockData.standardSourceDefinitions().size(), standardSourceDefinitions.size()); + assertThat(MockData.standardSourceDefinitions()).hasSameElementsAs(standardSourceDefinitions); + } + + private void standardWorkspace() throws JsonValidationException, IOException, ConfigNotFoundException { + configPersistence.writeConfig(ConfigSchema.STANDARD_WORKSPACE, + MockData.standardWorkspace().getWorkspaceId().toString(), + MockData.standardWorkspace()); + final StandardWorkspace standardWorkspace = configPersistence.getConfig(ConfigSchema.STANDARD_WORKSPACE, + MockData.standardWorkspace().getWorkspaceId().toString(), + StandardWorkspace.class); + final List standardWorkspaces = configPersistence.listConfigs(ConfigSchema.STANDARD_WORKSPACE, StandardWorkspace.class); + Assertions.assertEquals(MockData.standardWorkspace(), standardWorkspace); + Assertions.assertEquals(1, standardWorkspaces.size()); + Assertions.assertTrue(standardWorkspaces.contains(MockData.standardWorkspace())); + } + +} diff --git a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceLoadDataTest.java b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceLoadDataTest.java index 9177115f4886..cd875741fbca 100644 --- a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceLoadDataTest.java +++ b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceLoadDataTest.java @@ -4,6 +4,7 @@ package io.airbyte.config.persistence; +import static io.airbyte.db.instance.configs.jooq.Tables.ACTOR_DEFINITION; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.reset; @@ -19,7 +20,11 @@ import io.airbyte.config.SourceConnection; import io.airbyte.config.StandardDestinationDefinition; import io.airbyte.config.StandardSourceDefinition; +import io.airbyte.config.StandardWorkspace; import io.airbyte.db.instance.configs.ConfigsDatabaseInstance; +import io.airbyte.db.instance.configs.ConfigsDatabaseMigrator; +import io.airbyte.db.instance.development.DevDatabaseMigrator; +import io.airbyte.db.instance.development.MigrationDevHelper; import java.util.Collections; import java.util.UUID; import org.jooq.DSLContext; @@ -44,7 +49,12 @@ public class DatabaseConfigPersistenceLoadDataTest extends BaseDatabaseConfigPer public static void setup() throws Exception { database = new ConfigsDatabaseInstance(container.getUsername(), container.getPassword(), container.getJdbcUrl()).getAndInitialize(); configPersistence = spy(new DatabaseConfigPersistence(database)); - database.query(ctx -> ctx.execute("TRUNCATE TABLE airbyte_configs")); + final ConfigsDatabaseMigrator configsDatabaseMigrator = + new ConfigsDatabaseMigrator(database, DatabaseConfigPersistenceLoadDataTest.class.getName()); + final DevDatabaseMigrator devDatabaseMigrator = new DevDatabaseMigrator(configsDatabaseMigrator); + MigrationDevHelper.runLastMigration(devDatabaseMigrator); + database.query(ctx -> ctx + .execute("TRUNCATE TABLE state, connection_operation, connection, operation, actor_oauth_parameter, actor, actor_definition, workspace")); } @AfterAll @@ -70,7 +80,7 @@ public void testUpdateConfigsInNonEmptyDatabase() throws Exception { configPersistence.loadData(seedPersistence); // the new destination is added - assertRecordCount(3); + assertRecordCount(3, ACTOR_DEFINITION); assertHasDestination(DESTINATION_SNOWFLAKE); verify(configPersistence, times(1)).updateConfigsFromSeed(any(DSLContext.class), any(ConfigPersistence.class)); @@ -91,10 +101,20 @@ public void testNoUpdateForUsedConnector() throws Exception { // create connections to mark the source and destination as in use final DestinationConnection s3Connection = new DestinationConnection() .withDestinationId(UUID.randomUUID()) + .withWorkspaceId(UUID.randomUUID()) + .withName("s3Connection") .withDestinationDefinitionId(destinationS3V2.getDestinationDefinitionId()); + final StandardWorkspace standardWorkspace = new StandardWorkspace() + .withWorkspaceId(s3Connection.getWorkspaceId()) + .withName("workspace") + .withSlug("slug") + .withInitialSetupComplete(true); + configPersistence.writeConfig(ConfigSchema.STANDARD_WORKSPACE, standardWorkspace.getWorkspaceId().toString(), standardWorkspace); configPersistence.writeConfig(ConfigSchema.DESTINATION_CONNECTION, s3Connection.getDestinationId().toString(), s3Connection); final SourceConnection githubConnection = new SourceConnection() .withSourceId(UUID.randomUUID()) + .withWorkspaceId(standardWorkspace.getWorkspaceId()) + .withName("githubConnection") .withSourceDefinitionId(sourceGithubV2.getSourceDefinitionId()); configPersistence.writeConfig(ConfigSchema.SOURCE_CONNECTION, githubConnection.getSourceId().toString(), githubConnection); diff --git a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceMigrateFileConfigsTest.java b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceMigrateFileConfigsTest.java index df031215aba2..497446072850 100644 --- a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceMigrateFileConfigsTest.java +++ b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceMigrateFileConfigsTest.java @@ -4,6 +4,7 @@ package io.airbyte.config.persistence; +import static io.airbyte.db.instance.configs.jooq.Tables.ACTOR_DEFINITION; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; @@ -15,6 +16,9 @@ import io.airbyte.config.Configs; import io.airbyte.db.instance.configs.ConfigsDatabaseInstance; +import io.airbyte.db.instance.configs.ConfigsDatabaseMigrator; +import io.airbyte.db.instance.development.DevDatabaseMigrator; +import io.airbyte.db.instance.development.MigrationDevHelper; import java.nio.file.Files; import java.nio.file.Path; import java.util.UUID; @@ -37,6 +41,10 @@ public class DatabaseConfigPersistenceMigrateFileConfigsTest extends BaseDatabas public static void setup() throws Exception { database = new ConfigsDatabaseInstance(container.getUsername(), container.getPassword(), container.getJdbcUrl()).getAndInitialize(); configPersistence = spy(new DatabaseConfigPersistence(database)); + final ConfigsDatabaseMigrator configsDatabaseMigrator = + new ConfigsDatabaseMigrator(database, DatabaseConfigPersistenceLoadDataTest.class.getName()); + final DevDatabaseMigrator devDatabaseMigrator = new DevDatabaseMigrator(configsDatabaseMigrator); + MigrationDevHelper.runLastMigration(devDatabaseMigrator); } @AfterAll @@ -53,7 +61,8 @@ public void resetPersistence() throws Exception { reset(configs); when(configs.getConfigRoot()).thenReturn(ROOT_PATH); - database.query(ctx -> ctx.truncateTable("airbyte_configs").execute()); + database.query(ctx -> ctx + .execute("TRUNCATE TABLE state, connection_operation, connection, operation, actor_oauth_parameter, actor, actor_definition, workspace")); reset(configPersistence); } @@ -63,7 +72,7 @@ public void resetPersistence() throws Exception { public void testNewDeployment() throws Exception { configPersistence.migrateFileConfigs(configs); - assertRecordCount(0); + assertRecordCount(0, ACTOR_DEFINITION); verify(configPersistence, never()).copyConfigsFromSeed(any(DSLContext.class), any(ConfigPersistence.class)); verify(configPersistence, never()).updateConfigsFromSeed(any(DSLContext.class), any(ConfigPersistence.class)); @@ -76,7 +85,7 @@ public void testMigrationDeployment() throws Exception { configPersistence.migrateFileConfigs(configs); - assertRecordCount(2); + assertRecordCount(2, ACTOR_DEFINITION); assertHasSource(SOURCE_GITHUB); assertHasDestination(DESTINATION_S3); @@ -91,7 +100,7 @@ public void testUpdateDeployment() throws Exception { configPersistence.migrateFileConfigs(configs); - assertRecordCount(1); + assertRecordCount(1, ACTOR_DEFINITION); assertHasSource(SOURCE_GITHUB); verify(configPersistence, never()).copyConfigsFromSeed(any(DSLContext.class), any(ConfigPersistence.class)); diff --git a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceTest.java b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceTest.java index 853173ce43a9..f2a8042426fc 100644 --- a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceTest.java +++ b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceTest.java @@ -4,7 +4,9 @@ package io.airbyte.config.persistence; -import static io.airbyte.config.ConfigSchema.STANDARD_DESTINATION_DEFINITION; +import static io.airbyte.config.ConfigSchema.*; +import static io.airbyte.db.instance.configs.jooq.Tables.ACTOR_DEFINITION; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -22,9 +24,11 @@ import io.airbyte.config.StandardSourceDefinition; import io.airbyte.config.persistence.DatabaseConfigPersistence.ConnectorInfo; import io.airbyte.db.instance.configs.ConfigsDatabaseInstance; +import io.airbyte.db.instance.configs.ConfigsDatabaseMigrator; +import io.airbyte.db.instance.development.DevDatabaseMigrator; +import io.airbyte.db.instance.development.MigrationDevHelper; import java.time.Duration; import java.time.Instant; -import java.time.OffsetDateTime; import java.util.Collections; import java.util.List; import java.util.Map; @@ -46,7 +50,12 @@ public class DatabaseConfigPersistenceTest extends BaseDatabaseConfigPersistence public void setup() throws Exception { database = new ConfigsDatabaseInstance(container.getUsername(), container.getPassword(), container.getJdbcUrl()).getAndInitialize(); configPersistence = spy(new DatabaseConfigPersistence(database)); - database.query(ctx -> ctx.execute("TRUNCATE TABLE airbyte_configs")); + final ConfigsDatabaseMigrator configsDatabaseMigrator = + new ConfigsDatabaseMigrator(database, DatabaseConfigPersistenceLoadDataTest.class.getName()); + final DevDatabaseMigrator devDatabaseMigrator = new DevDatabaseMigrator(configsDatabaseMigrator); + MigrationDevHelper.runLastMigration(devDatabaseMigrator); + database.query(ctx -> ctx + .execute("TRUNCATE TABLE state, connection_operation, connection, operation, actor_oauth_parameter, actor, actor_definition, workspace")); } @AfterEach @@ -57,24 +66,24 @@ void tearDown() throws Exception { @Test public void testMultiWriteAndGetConfig() throws Exception { writeDestinations(configPersistence, Lists.newArrayList(DESTINATION_S3, DESTINATION_SNOWFLAKE)); - assertRecordCount(2); + assertRecordCount(2, ACTOR_DEFINITION); assertHasDestination(DESTINATION_S3); assertHasDestination(DESTINATION_SNOWFLAKE); - assertEquals( - List.of(DESTINATION_SNOWFLAKE, DESTINATION_S3), - configPersistence.listConfigs(STANDARD_DESTINATION_DEFINITION, StandardDestinationDefinition.class)); + assertThat(configPersistence + .listConfigs(STANDARD_DESTINATION_DEFINITION, StandardDestinationDefinition.class)) + .hasSameElementsAs(List.of(DESTINATION_SNOWFLAKE, DESTINATION_S3)); } @Test public void testWriteAndGetConfig() throws Exception { writeDestination(configPersistence, DESTINATION_S3); writeDestination(configPersistence, DESTINATION_SNOWFLAKE); - assertRecordCount(2); + assertRecordCount(2, ACTOR_DEFINITION); assertHasDestination(DESTINATION_S3); assertHasDestination(DESTINATION_SNOWFLAKE); - assertEquals( - List.of(DESTINATION_SNOWFLAKE, DESTINATION_S3), - configPersistence.listConfigs(STANDARD_DESTINATION_DEFINITION, StandardDestinationDefinition.class)); + assertThat(configPersistence + .listConfigs(STANDARD_DESTINATION_DEFINITION, StandardDestinationDefinition.class)) + .hasSameElementsAs(List.of(DESTINATION_SNOWFLAKE, DESTINATION_S3)); } @Test @@ -92,26 +101,25 @@ public void testListConfigWithMetadata() throws Exception { assertTrue(configWithMetadata.get(1).getCreatedAt().isAfter(now)); assertNotNull(configWithMetadata.get(0).getConfigId()); assertNotNull(configWithMetadata.get(1).getConfigId()); - assertEquals( - List.of(DESTINATION_SNOWFLAKE, DESTINATION_S3), - List.of(configWithMetadata.get(0).getConfig(), configWithMetadata.get(1).getConfig())); + assertThat(List.of(configWithMetadata.get(0).getConfig(), configWithMetadata.get(1).getConfig())) + .hasSameElementsAs(List.of(DESTINATION_SNOWFLAKE, DESTINATION_S3)); } @Test public void testDeleteConfig() throws Exception { writeDestination(configPersistence, DESTINATION_S3); writeDestination(configPersistence, DESTINATION_SNOWFLAKE); - assertRecordCount(2); + assertRecordCount(2, ACTOR_DEFINITION); assertHasDestination(DESTINATION_S3); assertHasDestination(DESTINATION_SNOWFLAKE); - assertEquals( - List.of(DESTINATION_SNOWFLAKE, DESTINATION_S3), - configPersistence.listConfigs(STANDARD_DESTINATION_DEFINITION, StandardDestinationDefinition.class)); + assertThat(configPersistence + .listConfigs(STANDARD_DESTINATION_DEFINITION, StandardDestinationDefinition.class)) + .hasSameElementsAs(List.of(DESTINATION_SNOWFLAKE, DESTINATION_S3)); deleteDestination(configPersistence, DESTINATION_S3); assertThrows(ConfigNotFoundException.class, () -> assertHasDestination(DESTINATION_S3)); - assertEquals( - List.of(DESTINATION_SNOWFLAKE), - configPersistence.listConfigs(STANDARD_DESTINATION_DEFINITION, StandardDestinationDefinition.class)); + assertThat(configPersistence + .listConfigs(STANDARD_DESTINATION_DEFINITION, StandardDestinationDefinition.class)) + .hasSameElementsAs(List.of(DESTINATION_SNOWFLAKE)); } @Test @@ -124,12 +132,12 @@ public void testReplaceAllConfigs() throws Exception { configPersistence.replaceAllConfigs(newConfigs, true); // dry run does not change anything - assertRecordCount(2); + assertRecordCount(2, ACTOR_DEFINITION); assertHasDestination(DESTINATION_S3); assertHasDestination(DESTINATION_SNOWFLAKE); configPersistence.replaceAllConfigs(newConfigs, false); - assertRecordCount(2); + assertRecordCount(2, ACTOR_DEFINITION); assertHasSource(SOURCE_GITHUB); assertHasSource(SOURCE_POSTGRES); } @@ -141,7 +149,7 @@ public void testDumpConfigs() throws Exception { writeDestination(configPersistence, DESTINATION_S3); final Map> actual = configPersistence.dumpConfigs(); final Map> expected = Map.of( - ConfigSchema.STANDARD_SOURCE_DEFINITION.name(), Stream.of(Jsons.jsonNode(SOURCE_GITHUB), Jsons.jsonNode(SOURCE_POSTGRES)), + STANDARD_SOURCE_DEFINITION.name(), Stream.of(Jsons.jsonNode(SOURCE_GITHUB), Jsons.jsonNode(SOURCE_POSTGRES)), STANDARD_DESTINATION_DEFINITION.name(), Stream.of(Jsons.jsonNode(DESTINATION_S3))); assertSameConfigDump(expected, actual); } @@ -153,10 +161,12 @@ public void testGetConnectorRepositoryToInfoMap() throws Exception { final String newVersion = "0.2.0"; final StandardSourceDefinition source1 = new StandardSourceDefinition() .withSourceDefinitionId(UUID.randomUUID()) + .withName("source-1") .withDockerRepository(connectorRepository) .withDockerImageTag(oldVersion); final StandardSourceDefinition source2 = new StandardSourceDefinition() .withSourceDefinitionId(UUID.randomUUID()) + .withName("source-2") .withDockerRepository(connectorRepository) .withDockerImageTag(newVersion); writeSource(configPersistence, source1); @@ -169,7 +179,6 @@ public void testGetConnectorRepositoryToInfoMap() throws Exception { @Test public void testInsertConfigRecord() throws Exception { - final OffsetDateTime timestamp = OffsetDateTime.now(); final UUID definitionId = UUID.randomUUID(); final String connectorRepository = "airbyte/test-connector"; @@ -177,66 +186,23 @@ public void testInsertConfigRecord() throws Exception { final StandardSourceDefinition source1 = new StandardSourceDefinition() .withSourceDefinitionId(definitionId) .withDockerRepository(connectorRepository) - .withDockerImageTag("0.1.2"); - int insertionCount = database.query(ctx -> configPersistence.insertConfigRecord( - ctx, - timestamp, - ConfigSchema.STANDARD_SOURCE_DEFINITION.name(), - Jsons.jsonNode(source1), - ConfigSchema.STANDARD_SOURCE_DEFINITION.getIdFieldName())); - assertEquals(1, insertionCount); + .withDockerImageTag("0.1.2") + .withName("random-name"); + writeSource(configPersistence, source1); // write an irrelevant source to make sure that it is not changed writeSource(configPersistence, SOURCE_GITHUB); - assertRecordCount(2); + assertRecordCount(2, ACTOR_DEFINITION); assertHasSource(source1); assertHasSource(SOURCE_GITHUB); - - // when the record already exists, it is ignored + // when the record already exists, it is updated final StandardSourceDefinition source2 = new StandardSourceDefinition() .withSourceDefinitionId(definitionId) .withDockerRepository(connectorRepository) - .withDockerImageTag("0.1.5"); - insertionCount = database.query(ctx -> configPersistence.insertConfigRecord( - ctx, - timestamp, - ConfigSchema.STANDARD_SOURCE_DEFINITION.name(), - Jsons.jsonNode(source2), - ConfigSchema.STANDARD_SOURCE_DEFINITION.getIdFieldName())); - assertEquals(0, insertionCount); - assertRecordCount(2); - assertHasSource(source1); - assertHasSource(SOURCE_GITHUB); - } - - @Test - public void testUpdateConfigRecord() throws Exception { - final OffsetDateTime timestamp = OffsetDateTime.now(); - final UUID definitionId = UUID.randomUUID(); - final String connectorRepository = "airbyte/test-connector"; - - final StandardSourceDefinition oldSource = new StandardSourceDefinition() - .withSourceDefinitionId(definitionId) - .withDockerRepository(connectorRepository) - .withDockerImageTag("0.3.5"); - writeSource(configPersistence, oldSource); - // write an irrelevant source to make sure that it is not changed - writeSource(configPersistence, SOURCE_GITHUB); - assertRecordCount(2); - assertHasSource(oldSource); - assertHasSource(SOURCE_GITHUB); - - final StandardSourceDefinition newSource = new StandardSourceDefinition() - .withSourceDefinitionId(definitionId) - .withDockerRepository(connectorRepository) - .withDockerImageTag("0.3.5"); - database.query(ctx -> configPersistence.updateConfigRecord( - ctx, - timestamp, - ConfigSchema.STANDARD_SOURCE_DEFINITION.name(), - Jsons.jsonNode(newSource), - definitionId.toString())); - assertRecordCount(2); - assertHasSource(newSource); + .withDockerImageTag("0.1.5") + .withName("random-name-2"); + writeSource(configPersistence, source2); + assertRecordCount(2, ACTOR_DEFINITION); + assertHasSource(source2); assertHasSource(SOURCE_GITHUB); } diff --git a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceUpdateConnectorDefinitionsTest.java b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceUpdateConnectorDefinitionsTest.java index c3923e48f1ff..52fe88d3a7ec 100644 --- a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceUpdateConnectorDefinitionsTest.java +++ b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceUpdateConnectorDefinitionsTest.java @@ -4,6 +4,7 @@ package io.airbyte.config.persistence; +import static io.airbyte.db.instance.configs.jooq.Tables.ACTOR_DEFINITION; import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.JsonNode; @@ -12,9 +13,11 @@ import io.airbyte.config.StandardSourceDefinition; import io.airbyte.config.persistence.DatabaseConfigPersistence.ConnectorInfo; import io.airbyte.db.instance.configs.ConfigsDatabaseInstance; +import io.airbyte.db.instance.configs.ConfigsDatabaseMigrator; +import io.airbyte.db.instance.development.DevDatabaseMigrator; +import io.airbyte.db.instance.development.MigrationDevHelper; import java.io.IOException; import java.sql.SQLException; -import java.time.OffsetDateTime; import java.util.Collections; import java.util.List; import java.util.Map; @@ -32,12 +35,15 @@ public class DatabaseConfigPersistenceUpdateConnectorDefinitionsTest extends BaseDatabaseConfigPersistenceTest { private static final JsonNode SOURCE_GITHUB_JSON = Jsons.jsonNode(SOURCE_GITHUB); - private static final OffsetDateTime TIMESTAMP = OffsetDateTime.now(); @BeforeAll public static void setup() throws Exception { database = new ConfigsDatabaseInstance(container.getUsername(), container.getPassword(), container.getJdbcUrl()).getAndInitialize(); configPersistence = new DatabaseConfigPersistence(database); + final ConfigsDatabaseMigrator configsDatabaseMigrator = + new ConfigsDatabaseMigrator(database, DatabaseConfigPersistenceLoadDataTest.class.getName()); + final DevDatabaseMigrator devDatabaseMigrator = new DevDatabaseMigrator(configsDatabaseMigrator); + MigrationDevHelper.runLastMigration(devDatabaseMigrator); } @AfterAll @@ -47,7 +53,8 @@ public static void tearDown() throws Exception { @BeforeEach public void resetDatabase() throws SQLException { - database.transaction(ctx -> ctx.truncateTable("airbyte_configs").execute()); + database.query(ctx -> ctx + .execute("TRUNCATE TABLE state, connection_operation, connection, operation, actor_oauth_parameter, actor, actor_definition, workspace")); } @Test @@ -150,7 +157,6 @@ private void assertUpdateConnectorDefinition(final List ctx.execute("TRUNCATE TABLE airbyte_configs")); + } + + @AfterAll + public static void tearDown() throws Exception { + database.close(); + } + + @BeforeEach + public void resetPersistence() { + reset(seedPersistence); + reset(configPersistence); + } + + @Test + @Order(1) + @DisplayName("When database is empty, configs should be inserted") + public void testUpdateConfigsInNonEmptyDatabase() throws Exception { + when(seedPersistence.listConfigs(ConfigSchema.STANDARD_SOURCE_DEFINITION, StandardSourceDefinition.class)) + .thenReturn(Lists.newArrayList(SOURCE_GITHUB)); + when(seedPersistence.listConfigs(ConfigSchema.STANDARD_DESTINATION_DEFINITION, StandardDestinationDefinition.class)) + .thenReturn(Lists.newArrayList(DESTINATION_S3, DESTINATION_SNOWFLAKE)); + + configPersistence.loadData(seedPersistence); + + // the new destination is added + assertRecordCount(3); + assertHasDestination(DESTINATION_SNOWFLAKE); + + verify(configPersistence, times(1)).updateConfigsFromSeed(any(DSLContext.class), any(ConfigPersistence.class)); + } + + @Test + @Order(2) + @DisplayName("When a connector is in use, its definition should not be updated") + public void testNoUpdateForUsedConnector() throws Exception { + // the seed has a newer version of s3 destination and github source + final StandardDestinationDefinition destinationS3V2 = Jsons.clone(DESTINATION_S3).withDockerImageTag("10000.1.0"); + when(seedPersistence.listConfigs(ConfigSchema.STANDARD_DESTINATION_DEFINITION, StandardDestinationDefinition.class)) + .thenReturn(Collections.singletonList(destinationS3V2)); + final StandardSourceDefinition sourceGithubV2 = Jsons.clone(SOURCE_GITHUB).withDockerImageTag("10000.15.3"); + when(seedPersistence.listConfigs(ConfigSchema.STANDARD_SOURCE_DEFINITION, StandardSourceDefinition.class)) + .thenReturn(Collections.singletonList(sourceGithubV2)); + + // create connections to mark the source and destination as in use + final DestinationConnection s3Connection = new DestinationConnection() + .withDestinationId(UUID.randomUUID()) + .withDestinationDefinitionId(destinationS3V2.getDestinationDefinitionId()); + configPersistence.writeConfig(ConfigSchema.DESTINATION_CONNECTION, s3Connection.getDestinationId().toString(), s3Connection); + final SourceConnection githubConnection = new SourceConnection() + .withSourceId(UUID.randomUUID()) + .withSourceDefinitionId(sourceGithubV2.getSourceDefinitionId()); + configPersistence.writeConfig(ConfigSchema.SOURCE_CONNECTION, githubConnection.getSourceId().toString(), githubConnection); + + configPersistence.loadData(seedPersistence); + // s3 destination is not updated + assertHasDestination(DESTINATION_S3); + assertHasSource(SOURCE_GITHUB); + } + + @Test + @Order(3) + @DisplayName("When a connector is not in use, its definition should be updated") + public void testUpdateForUnusedConnector() throws Exception { + // the seed has a newer version of snowflake destination + final StandardDestinationDefinition snowflakeV2 = Jsons.clone(DESTINATION_SNOWFLAKE).withDockerImageTag("10000.2.0"); + when(seedPersistence.listConfigs(ConfigSchema.STANDARD_DESTINATION_DEFINITION, StandardDestinationDefinition.class)) + .thenReturn(Collections.singletonList(snowflakeV2)); + + configPersistence.loadData(seedPersistence); + assertHasDestination(snowflakeV2); + } + +} diff --git a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DeprecatedDatabaseConfigPersistenceMigrateFileConfigsTest.java b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DeprecatedDatabaseConfigPersistenceMigrateFileConfigsTest.java new file mode 100644 index 000000000000..32eb348d68fb --- /dev/null +++ b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DeprecatedDatabaseConfigPersistenceMigrateFileConfigsTest.java @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.config.persistence; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import io.airbyte.config.Configs; +import io.airbyte.db.instance.configs.DeprecatedConfigsDatabaseInstance; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.UUID; +import org.jooq.DSLContext; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** + * Unit test for the {@link DeprecatedDatabaseConfigPersistence#migrateFileConfigs} method. + */ +public class DeprecatedDatabaseConfigPersistenceMigrateFileConfigsTest extends BaseDeprecatedDatabaseConfigPersistenceTest { + + private static Path ROOT_PATH; + private final Configs configs = mock(Configs.class); + + @BeforeAll + public static void setup() throws Exception { + database = new DeprecatedConfigsDatabaseInstance(container.getUsername(), container.getPassword(), container.getJdbcUrl()).getAndInitialize(); + configPersistence = spy(new DeprecatedDatabaseConfigPersistence(database)); + } + + @AfterAll + public static void tearDown() throws Exception { + database.close(); + } + + @BeforeEach + public void resetPersistence() throws Exception { + ROOT_PATH = Files.createTempDirectory( + Files.createDirectories(Path.of("/tmp/airbyte_tests")), + DeprecatedDatabaseConfigPersistenceMigrateFileConfigsTest.class.getSimpleName() + UUID.randomUUID()); + + reset(configs); + when(configs.getConfigRoot()).thenReturn(ROOT_PATH); + + database.query(ctx -> ctx.truncateTable("airbyte_configs").execute()); + + reset(configPersistence); + } + + @Test + @DisplayName("When database is not initialized, and there is no local config dir, do nothing") + public void testNewDeployment() throws Exception { + configPersistence.migrateFileConfigs(configs); + + assertRecordCount(0); + + verify(configPersistence, never()).copyConfigsFromSeed(any(DSLContext.class), any(ConfigPersistence.class)); + verify(configPersistence, never()).updateConfigsFromSeed(any(DSLContext.class), any(ConfigPersistence.class)); + } + + @Test + @DisplayName("When database is not initialized, and there is local config dir, copy from local dir") + public void testMigrationDeployment() throws Exception { + prepareLocalFilePersistence(); + + configPersistence.migrateFileConfigs(configs); + + assertRecordCount(2); + assertHasSource(SOURCE_GITHUB); + assertHasDestination(DESTINATION_S3); + + verify(configPersistence, times(1)).copyConfigsFromSeed(any(DSLContext.class), any(ConfigPersistence.class)); + } + + @Test + @DisplayName("When database has been initialized, do nothing") + public void testUpdateDeployment() throws Exception { + prepareLocalFilePersistence(); + writeSource(configPersistence, SOURCE_GITHUB); + + configPersistence.migrateFileConfigs(configs); + + assertRecordCount(1); + assertHasSource(SOURCE_GITHUB); + + verify(configPersistence, never()).copyConfigsFromSeed(any(DSLContext.class), any(ConfigPersistence.class)); + verify(configPersistence, never()).updateConfigsFromSeed(any(DSLContext.class), any(ConfigPersistence.class)); + } + + private void prepareLocalFilePersistence() throws Exception { + Files.createDirectories(ROOT_PATH.resolve(FileSystemConfigPersistence.CONFIG_DIR)); + final ConfigPersistence filePersistence = new FileSystemConfigPersistence(ROOT_PATH); + writeSource(filePersistence, SOURCE_GITHUB); + writeDestination(filePersistence, DESTINATION_S3); + } + +} diff --git a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DeprecatedDatabaseConfigPersistenceTest.java b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DeprecatedDatabaseConfigPersistenceTest.java new file mode 100644 index 000000000000..58456ec213b4 --- /dev/null +++ b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DeprecatedDatabaseConfigPersistenceTest.java @@ -0,0 +1,271 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.config.persistence; + +import static io.airbyte.config.ConfigSchema.STANDARD_DESTINATION_DEFINITION; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.spy; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.Lists; +import io.airbyte.commons.json.Jsons; +import io.airbyte.config.AirbyteConfig; +import io.airbyte.config.ConfigSchema; +import io.airbyte.config.ConfigWithMetadata; +import io.airbyte.config.StandardDestinationDefinition; +import io.airbyte.config.StandardSourceDefinition; +import io.airbyte.config.persistence.DeprecatedDatabaseConfigPersistence.ConnectorInfo; +import io.airbyte.db.instance.configs.DeprecatedConfigsDatabaseInstance; +import java.time.Duration; +import java.time.Instant; +import java.time.OffsetDateTime; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Stream; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * See {@link DeprecatedDatabaseConfigPersistenceLoadDataTest}, + * {@link DeprecatedDatabaseConfigPersistenceMigrateFileConfigsTest}, and + * {@link DeprecatedDatabaseConfigPersistenceUpdateConnectorDefinitionsTest} for testing of specific + * methods. + */ +public class DeprecatedDatabaseConfigPersistenceTest extends BaseDeprecatedDatabaseConfigPersistenceTest { + + @BeforeEach + public void setup() throws Exception { + database = new DeprecatedConfigsDatabaseInstance(container.getUsername(), container.getPassword(), container.getJdbcUrl()).getAndInitialize(); + configPersistence = spy(new DeprecatedDatabaseConfigPersistence(database)); + database.query(ctx -> ctx.execute("TRUNCATE TABLE airbyte_configs")); + } + + @AfterEach + void tearDown() throws Exception { + database.close(); + } + + @Test + public void testMultiWriteAndGetConfig() throws Exception { + writeDestinations(configPersistence, Lists.newArrayList(DESTINATION_S3, DESTINATION_SNOWFLAKE)); + assertRecordCount(2); + assertHasDestination(DESTINATION_S3); + assertHasDestination(DESTINATION_SNOWFLAKE); + assertEquals( + List.of(DESTINATION_SNOWFLAKE, DESTINATION_S3), + configPersistence.listConfigs(STANDARD_DESTINATION_DEFINITION, StandardDestinationDefinition.class)); + } + + @Test + public void testWriteAndGetConfig() throws Exception { + writeDestination(configPersistence, DESTINATION_S3); + writeDestination(configPersistence, DESTINATION_SNOWFLAKE); + assertRecordCount(2); + assertHasDestination(DESTINATION_S3); + assertHasDestination(DESTINATION_SNOWFLAKE); + assertEquals( + List.of(DESTINATION_SNOWFLAKE, DESTINATION_S3), + configPersistence.listConfigs(STANDARD_DESTINATION_DEFINITION, StandardDestinationDefinition.class)); + } + + @Test + public void testListConfigWithMetadata() throws Exception { + final Instant now = Instant.now().minus(Duration.ofSeconds(1)); + writeDestination(configPersistence, DESTINATION_S3); + writeDestination(configPersistence, DESTINATION_SNOWFLAKE); + final List> configWithMetadata = configPersistence + .listConfigsWithMetadata(STANDARD_DESTINATION_DEFINITION, StandardDestinationDefinition.class); + assertEquals(2, configWithMetadata.size()); + assertEquals("STANDARD_DESTINATION_DEFINITION", configWithMetadata.get(0).getConfigType()); + assertEquals("STANDARD_DESTINATION_DEFINITION", configWithMetadata.get(1).getConfigType()); + assertTrue(configWithMetadata.get(0).getCreatedAt().isAfter(now)); + assertTrue(configWithMetadata.get(0).getUpdatedAt().isAfter(now)); + assertTrue(configWithMetadata.get(1).getCreatedAt().isAfter(now)); + assertNotNull(configWithMetadata.get(0).getConfigId()); + assertNotNull(configWithMetadata.get(1).getConfigId()); + assertEquals( + List.of(DESTINATION_SNOWFLAKE, DESTINATION_S3), + List.of(configWithMetadata.get(0).getConfig(), configWithMetadata.get(1).getConfig())); + } + + @Test + public void testDeleteConfig() throws Exception { + writeDestination(configPersistence, DESTINATION_S3); + writeDestination(configPersistence, DESTINATION_SNOWFLAKE); + assertRecordCount(2); + assertHasDestination(DESTINATION_S3); + assertHasDestination(DESTINATION_SNOWFLAKE); + assertEquals( + List.of(DESTINATION_SNOWFLAKE, DESTINATION_S3), + configPersistence.listConfigs(STANDARD_DESTINATION_DEFINITION, StandardDestinationDefinition.class)); + deleteDestination(configPersistence, DESTINATION_S3); + assertThrows(ConfigNotFoundException.class, () -> assertHasDestination(DESTINATION_S3)); + assertEquals( + List.of(DESTINATION_SNOWFLAKE), + configPersistence.listConfigs(STANDARD_DESTINATION_DEFINITION, StandardDestinationDefinition.class)); + } + + @Test + public void testReplaceAllConfigs() throws Exception { + writeDestination(configPersistence, DESTINATION_S3); + writeDestination(configPersistence, DESTINATION_SNOWFLAKE); + + final Map> newConfigs = Map.of(ConfigSchema.STANDARD_SOURCE_DEFINITION, Stream.of(SOURCE_GITHUB, SOURCE_POSTGRES)); + + configPersistence.replaceAllConfigs(newConfigs, true); + + // dry run does not change anything + assertRecordCount(2); + assertHasDestination(DESTINATION_S3); + assertHasDestination(DESTINATION_SNOWFLAKE); + + configPersistence.replaceAllConfigs(newConfigs, false); + assertRecordCount(2); + assertHasSource(SOURCE_GITHUB); + assertHasSource(SOURCE_POSTGRES); + } + + @Test + public void testDumpConfigs() throws Exception { + writeSource(configPersistence, SOURCE_GITHUB); + writeSource(configPersistence, SOURCE_POSTGRES); + writeDestination(configPersistence, DESTINATION_S3); + final Map> actual = configPersistence.dumpConfigs(); + final Map> expected = Map.of( + ConfigSchema.STANDARD_SOURCE_DEFINITION.name(), Stream.of(Jsons.jsonNode(SOURCE_GITHUB), Jsons.jsonNode(SOURCE_POSTGRES)), + STANDARD_DESTINATION_DEFINITION.name(), Stream.of(Jsons.jsonNode(DESTINATION_S3))); + assertSameConfigDump(expected, actual); + } + + @Test + public void testGetConnectorRepositoryToInfoMap() throws Exception { + final String connectorRepository = "airbyte/duplicated-connector"; + final String oldVersion = "0.1.10"; + final String newVersion = "0.2.0"; + final StandardSourceDefinition source1 = new StandardSourceDefinition() + .withSourceDefinitionId(UUID.randomUUID()) + .withDockerRepository(connectorRepository) + .withDockerImageTag(oldVersion); + final StandardSourceDefinition source2 = new StandardSourceDefinition() + .withSourceDefinitionId(UUID.randomUUID()) + .withDockerRepository(connectorRepository) + .withDockerImageTag(newVersion); + writeSource(configPersistence, source1); + writeSource(configPersistence, source2); + final Map result = database.query(ctx -> configPersistence.getConnectorRepositoryToInfoMap(ctx)); + // when there are duplicated connector definitions, the one with the latest version should be + // retrieved + assertEquals(newVersion, result.get(connectorRepository).dockerImageTag); + } + + @Test + public void testInsertConfigRecord() throws Exception { + final OffsetDateTime timestamp = OffsetDateTime.now(); + final UUID definitionId = UUID.randomUUID(); + final String connectorRepository = "airbyte/test-connector"; + + // when the record does not exist, it is inserted + final StandardSourceDefinition source1 = new StandardSourceDefinition() + .withSourceDefinitionId(definitionId) + .withDockerRepository(connectorRepository) + .withDockerImageTag("0.1.2"); + int insertionCount = database.query(ctx -> configPersistence.insertConfigRecord( + ctx, + timestamp, + ConfigSchema.STANDARD_SOURCE_DEFINITION.name(), + Jsons.jsonNode(source1), + ConfigSchema.STANDARD_SOURCE_DEFINITION.getIdFieldName())); + assertEquals(1, insertionCount); + // write an irrelevant source to make sure that it is not changed + writeSource(configPersistence, SOURCE_GITHUB); + assertRecordCount(2); + assertHasSource(source1); + assertHasSource(SOURCE_GITHUB); + + // when the record already exists, it is ignored + final StandardSourceDefinition source2 = new StandardSourceDefinition() + .withSourceDefinitionId(definitionId) + .withDockerRepository(connectorRepository) + .withDockerImageTag("0.1.5"); + insertionCount = database.query(ctx -> configPersistence.insertConfigRecord( + ctx, + timestamp, + ConfigSchema.STANDARD_SOURCE_DEFINITION.name(), + Jsons.jsonNode(source2), + ConfigSchema.STANDARD_SOURCE_DEFINITION.getIdFieldName())); + assertEquals(0, insertionCount); + assertRecordCount(2); + assertHasSource(source1); + assertHasSource(SOURCE_GITHUB); + } + + @Test + public void testUpdateConfigRecord() throws Exception { + final OffsetDateTime timestamp = OffsetDateTime.now(); + final UUID definitionId = UUID.randomUUID(); + final String connectorRepository = "airbyte/test-connector"; + + final StandardSourceDefinition oldSource = new StandardSourceDefinition() + .withSourceDefinitionId(definitionId) + .withDockerRepository(connectorRepository) + .withDockerImageTag("0.3.5"); + writeSource(configPersistence, oldSource); + // write an irrelevant source to make sure that it is not changed + writeSource(configPersistence, SOURCE_GITHUB); + assertRecordCount(2); + assertHasSource(oldSource); + assertHasSource(SOURCE_GITHUB); + + final StandardSourceDefinition newSource = new StandardSourceDefinition() + .withSourceDefinitionId(definitionId) + .withDockerRepository(connectorRepository) + .withDockerImageTag("0.3.5"); + database.query(ctx -> configPersistence.updateConfigRecord( + ctx, + timestamp, + ConfigSchema.STANDARD_SOURCE_DEFINITION.name(), + Jsons.jsonNode(newSource), + definitionId.toString())); + assertRecordCount(2); + assertHasSource(newSource); + assertHasSource(SOURCE_GITHUB); + } + + @Test + public void testHasNewVersion() { + assertTrue(DeprecatedDatabaseConfigPersistence.hasNewVersion("0.1.99", "0.2.0")); + assertFalse(DeprecatedDatabaseConfigPersistence.hasNewVersion("invalid_version", "0.2.0")); + } + + @Test + public void testGetNewFields() { + final JsonNode o1 = Jsons.deserialize("{ \"field1\": 1, \"field2\": 2 }"); + final JsonNode o2 = Jsons.deserialize("{ \"field1\": 1, \"field3\": 3 }"); + assertEquals(Collections.emptySet(), DeprecatedDatabaseConfigPersistence.getNewFields(o1, o1)); + assertEquals(Collections.singleton("field3"), DeprecatedDatabaseConfigPersistence.getNewFields(o1, o2)); + assertEquals(Collections.singleton("field2"), DeprecatedDatabaseConfigPersistence.getNewFields(o2, o1)); + } + + @Test + public void testGetDefinitionWithNewFields() { + final JsonNode current = Jsons.deserialize("{ \"field1\": 1, \"field2\": 2 }"); + final JsonNode latest = Jsons.deserialize("{ \"field1\": 1, \"field3\": 3, \"field4\": 4 }"); + final Set newFields = Set.of("field3"); + + assertEquals(current, DeprecatedDatabaseConfigPersistence.getDefinitionWithNewFields(current, latest, Collections.emptySet())); + + final JsonNode currentWithNewFields = Jsons.deserialize("{ \"field1\": 1, \"field2\": 2, \"field3\": 3 }"); + assertEquals(currentWithNewFields, DeprecatedDatabaseConfigPersistence.getDefinitionWithNewFields(current, latest, newFields)); + } + +} diff --git a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DeprecatedDatabaseConfigPersistenceUpdateConnectorDefinitionsTest.java b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DeprecatedDatabaseConfigPersistenceUpdateConnectorDefinitionsTest.java new file mode 100644 index 000000000000..c7c4a29780fd --- /dev/null +++ b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DeprecatedDatabaseConfigPersistenceUpdateConnectorDefinitionsTest.java @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.config.persistence; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.json.Jsons; +import io.airbyte.config.ConfigSchema; +import io.airbyte.config.StandardSourceDefinition; +import io.airbyte.config.persistence.DeprecatedDatabaseConfigPersistence.ConnectorInfo; +import io.airbyte.db.instance.configs.DeprecatedConfigsDatabaseInstance; +import java.io.IOException; +import java.sql.SQLException; +import java.time.OffsetDateTime; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** + * Unit test for the {@link DeprecatedDatabaseConfigPersistence#updateConnectorDefinitions} method. + */ +public class DeprecatedDatabaseConfigPersistenceUpdateConnectorDefinitionsTest extends BaseDeprecatedDatabaseConfigPersistenceTest { + + private static final JsonNode SOURCE_GITHUB_JSON = Jsons.jsonNode(SOURCE_GITHUB); + private static final OffsetDateTime TIMESTAMP = OffsetDateTime.now(); + + @BeforeAll + public static void setup() throws Exception { + database = new DeprecatedConfigsDatabaseInstance(container.getUsername(), container.getPassword(), container.getJdbcUrl()).getAndInitialize(); + configPersistence = new DeprecatedDatabaseConfigPersistence(database); + } + + @AfterAll + public static void tearDown() throws Exception { + database.close(); + } + + @BeforeEach + public void resetDatabase() throws SQLException { + database.transaction(ctx -> ctx.truncateTable("airbyte_configs").execute()); + } + + @Test + @DisplayName("When a connector does not exist, add it") + public void testNewConnector() throws Exception { + assertUpdateConnectorDefinition( + Collections.emptyList(), + Collections.emptyList(), + List.of(SOURCE_GITHUB), + Collections.singletonList(SOURCE_GITHUB)); + } + + @Test + @DisplayName("When an old connector is in use, if it has all fields, do not update it") + public void testOldConnectorInUseWithAllFields() throws Exception { + final StandardSourceDefinition currentSource = getSource().withDockerImageTag("0.0.0"); + final StandardSourceDefinition latestSource = getSource().withDockerImageTag("0.1000.0"); + + assertUpdateConnectorDefinition( + Collections.singletonList(currentSource), + Collections.singletonList(currentSource), + Collections.singletonList(latestSource), + Collections.singletonList(currentSource)); + } + + @Test + @DisplayName("When a old connector is in use, add missing fields, do not update its version") + public void testOldConnectorInUseWithMissingFields() throws Exception { + final StandardSourceDefinition currentSource = getSource().withDockerImageTag("0.0.0").withDocumentationUrl(null).withSourceType(null); + final StandardSourceDefinition latestSource = getSource().withDockerImageTag("0.1000.0"); + final StandardSourceDefinition currentSourceWithNewFields = getSource().withDockerImageTag("0.0.0"); + + assertUpdateConnectorDefinition( + Collections.singletonList(currentSource), + Collections.singletonList(currentSource), + Collections.singletonList(latestSource), + Collections.singletonList(currentSourceWithNewFields)); + } + + @Test + @DisplayName("When an unused connector has a new version, update it") + public void testUnusedConnectorWithOldVersion() throws Exception { + final StandardSourceDefinition currentSource = getSource().withDockerImageTag("0.0.0"); + final StandardSourceDefinition latestSource = getSource().withDockerImageTag("0.1000.0"); + + assertUpdateConnectorDefinition( + Collections.singletonList(currentSource), + Collections.emptyList(), + Collections.singletonList(latestSource), + Collections.singletonList(latestSource)); + } + + @Test + @DisplayName("When an unused connector has missing fields, add the missing fields, do not update its version") + public void testUnusedConnectorWithMissingFields() throws Exception { + final StandardSourceDefinition currentSource = getSource().withDockerImageTag("0.1000.0").withDocumentationUrl(null).withSourceType(null); + final StandardSourceDefinition latestSource = getSource().withDockerImageTag("0.99.0"); + final StandardSourceDefinition currentSourceWithNewFields = getSource().withDockerImageTag("0.1000.0"); + + assertUpdateConnectorDefinition( + Collections.singletonList(currentSource), + Collections.emptyList(), + Collections.singletonList(latestSource), + Collections.singletonList(currentSourceWithNewFields)); + } + + /** + * Clone a source for modification and testing. + */ + private StandardSourceDefinition getSource() { + return Jsons.object(Jsons.clone(SOURCE_GITHUB_JSON), StandardSourceDefinition.class); + } + + /** + * @param currentSources all sources currently exist in the database + * @param currentSourcesInUse a subset of currentSources; sources currently used in data syncing + */ + private void assertUpdateConnectorDefinition(final List currentSources, + final List currentSourcesInUse, + final List latestSources, + final List expectedUpdatedSources) + throws Exception { + for (final StandardSourceDefinition source : currentSources) { + writeSource(configPersistence, source); + } + + for (final StandardSourceDefinition source : currentSourcesInUse) { + assertTrue(currentSources.contains(source), "currentSourcesInUse must exist in currentSources"); + } + + final Set sourceRepositoriesInUse = currentSourcesInUse.stream() + .map(StandardSourceDefinition::getDockerRepository) + .collect(Collectors.toSet()); + final Map currentSourceRepositoryToInfo = currentSources.stream() + .collect(Collectors.toMap( + StandardSourceDefinition::getDockerRepository, + s -> new ConnectorInfo(s.getSourceDefinitionId().toString(), Jsons.jsonNode(s)))); + + database.transaction(ctx -> { + try { + configPersistence.updateConnectorDefinitions( + ctx, + TIMESTAMP, + ConfigSchema.STANDARD_SOURCE_DEFINITION, + latestSources, + sourceRepositoriesInUse, + currentSourceRepositoryToInfo); + } catch (final IOException e) { + throw new SQLException(e); + } + return null; + }); + + assertRecordCount(expectedUpdatedSources.size()); + for (final StandardSourceDefinition source : expectedUpdatedSources) { + assertHasSource(source); + } + } + +} diff --git a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/MockData.java b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/MockData.java new file mode 100644 index 000000000000..7a1e9f7dc7a3 --- /dev/null +++ b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/MockData.java @@ -0,0 +1,344 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.config.persistence; + +import com.google.common.collect.Lists; +import io.airbyte.commons.json.Jsons; +import io.airbyte.config.DestinationConnection; +import io.airbyte.config.DestinationOAuthParameter; +import io.airbyte.config.JobSyncConfig.NamespaceDefinitionType; +import io.airbyte.config.Notification; +import io.airbyte.config.Notification.NotificationType; +import io.airbyte.config.OperatorDbt; +import io.airbyte.config.OperatorNormalization; +import io.airbyte.config.OperatorNormalization.Option; +import io.airbyte.config.ResourceRequirements; +import io.airbyte.config.Schedule; +import io.airbyte.config.Schedule.TimeUnit; +import io.airbyte.config.SlackNotificationConfiguration; +import io.airbyte.config.SourceConnection; +import io.airbyte.config.SourceOAuthParameter; +import io.airbyte.config.StandardDestinationDefinition; +import io.airbyte.config.StandardSourceDefinition; +import io.airbyte.config.StandardSourceDefinition.SourceType; +import io.airbyte.config.StandardSync; +import io.airbyte.config.StandardSync.Status; +import io.airbyte.config.StandardSyncOperation; +import io.airbyte.config.StandardSyncOperation.OperatorType; +import io.airbyte.config.StandardSyncState; +import io.airbyte.config.StandardWorkspace; +import io.airbyte.config.State; +import io.airbyte.protocol.models.AirbyteCatalog; +import io.airbyte.protocol.models.AuthSpecification; +import io.airbyte.protocol.models.AuthSpecification.AuthType; +import io.airbyte.protocol.models.CatalogHelpers; +import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.ConnectorSpecification; +import io.airbyte.protocol.models.DestinationSyncMode; +import io.airbyte.protocol.models.JsonSchemaPrimitive; +import io.airbyte.protocol.models.SyncMode; +import java.net.URI; +import java.time.Instant; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.UUID; + +public class MockData { + + private static final UUID WORKSPACE_ID = UUID.randomUUID(); + private static final UUID WORKSPACE_CUSTOMER_ID = UUID.randomUUID(); + private static final UUID SOURCE_DEFINITION_ID_1 = UUID.randomUUID(); + private static final UUID SOURCE_DEFINITION_ID_2 = UUID.randomUUID(); + private static final UUID DESTINATION_DEFINITION_ID_1 = UUID.randomUUID(); + private static final UUID DESTINATION_DEFINITION_ID_2 = UUID.randomUUID(); + private static final UUID SOURCE_ID_1 = UUID.randomUUID(); + private static final UUID SOURCE_ID_2 = UUID.randomUUID(); + private static final UUID DESTINATION_ID_1 = UUID.randomUUID(); + private static final UUID DESTINATION_ID_2 = UUID.randomUUID(); + private static final UUID OPERATION_ID_1 = UUID.randomUUID(); + private static final UUID OPERATION_ID_2 = UUID.randomUUID(); + private static final UUID CONNECTION_ID_1 = UUID.randomUUID(); + private static final UUID CONNECTION_ID_2 = UUID.randomUUID(); + private static final UUID CONNECTION_ID_3 = UUID.randomUUID(); + private static final UUID CONNECTION_ID_4 = UUID.randomUUID(); + private static final UUID SOURCE_OAUTH_PARAMETER_ID_1 = UUID.randomUUID(); + private static final UUID SOURCE_OAUTH_PARAMETER_ID_2 = UUID.randomUUID(); + private static final UUID DESTINATION_OAUTH_PARAMETER_ID_1 = UUID.randomUUID(); + private static final UUID DESTINATION_OAUTH_PARAMETER_ID_2 = UUID.randomUUID(); + private static final Instant NOW = Instant.parse("2021-12-15T20:30:40.00Z"); + + public static StandardWorkspace standardWorkspace() { + final Notification notification = new Notification() + .withNotificationType(NotificationType.SLACK) + .withSendOnFailure(true) + .withSendOnSuccess(true) + .withSlackConfiguration(new SlackNotificationConfiguration().withWebhook("webhook-url")); + return new StandardWorkspace() + .withWorkspaceId(WORKSPACE_ID) + .withCustomerId(WORKSPACE_CUSTOMER_ID) + .withName("test-workspace") + .withSlug("random-string") + .withEmail("abc@xyz.com") + .withInitialSetupComplete(true) + .withAnonymousDataCollection(true) + .withNews(true) + .withSecurityUpdates(true) + .withDisplaySetupWizard(true) + .withTombstone(false) + .withNotifications(Collections.singletonList(notification)) + .withFirstCompletedSync(true) + .withFeedbackDone(true); + } + + public static List standardSourceDefinitions() { + final ConnectorSpecification connectorSpecification = connectorSpecification(); + final StandardSourceDefinition standardSourceDefinition1 = new StandardSourceDefinition() + .withSourceDefinitionId(SOURCE_DEFINITION_ID_1) + .withSourceType(SourceType.API) + .withName("random-source-1") + .withDockerImageTag("tag-1") + .withDockerRepository("repository-1") + .withDocumentationUrl("documentation-url-1") + .withIcon("icon-1") + .withSpec(connectorSpecification); + final StandardSourceDefinition standardSourceDefinition2 = new StandardSourceDefinition() + .withSourceDefinitionId(SOURCE_DEFINITION_ID_2) + .withSourceType(SourceType.DATABASE) + .withName("random-source-2") + .withDockerImageTag("tag-2") + .withDockerRepository("repository-2") + .withDocumentationUrl("documentation-url-2") + .withIcon("icon-2"); + return Arrays.asList(standardSourceDefinition1, standardSourceDefinition2); + } + + private static ConnectorSpecification connectorSpecification() { + return new ConnectorSpecification() + .withAuthSpecification(new AuthSpecification().withAuthType(AuthType.OAUTH_2_0)) + .withConnectionSpecification(Jsons.jsonNode("'{\"name\":\"John\", \"age\":30, \"car\":null}'")) + .withDocumentationUrl(URI.create("whatever")) + .withAdvancedAuth(null) + .withChangelogUrl(URI.create("whatever")) + .withSupportedDestinationSyncModes(Arrays.asList(DestinationSyncMode.APPEND, DestinationSyncMode.OVERWRITE, DestinationSyncMode.APPEND_DEDUP)) + .withSupportsDBT(true) + .withSupportsIncremental(true) + .withSupportsNormalization(true); + } + + public static List standardDestinationDefinitions() { + final ConnectorSpecification connectorSpecification = connectorSpecification(); + final StandardDestinationDefinition standardDestinationDefinition1 = new StandardDestinationDefinition() + .withDestinationDefinitionId(DESTINATION_DEFINITION_ID_1) + .withName("random-destination-1") + .withDockerImageTag("tag-3") + .withDockerRepository("repository-3") + .withDocumentationUrl("documentation-url-3") + .withIcon("icon-3") + .withSpec(connectorSpecification); + final StandardDestinationDefinition standardDestinationDefinition2 = new StandardDestinationDefinition() + .withDestinationDefinitionId(DESTINATION_DEFINITION_ID_2) + .withName("random-destination-2") + .withDockerImageTag("tag-4") + .withDockerRepository("repository-4") + .withDocumentationUrl("documentation-url-4") + .withIcon("icon-4") + .withSpec(connectorSpecification); + return Arrays.asList(standardDestinationDefinition1, standardDestinationDefinition2); + } + + public static List sourceConnections() { + final SourceConnection sourceConnection1 = new SourceConnection() + .withName("source-1") + .withTombstone(false) + .withSourceDefinitionId(SOURCE_DEFINITION_ID_1) + .withWorkspaceId(WORKSPACE_ID) + .withConfiguration(Jsons.jsonNode("'{\"name\":\"John\", \"age\":30, \"car\":null}'")) + .withSourceId(SOURCE_ID_1); + final SourceConnection sourceConnection2 = new SourceConnection() + .withName("source-2") + .withTombstone(false) + .withSourceDefinitionId(SOURCE_DEFINITION_ID_2) + .withWorkspaceId(WORKSPACE_ID) + .withConfiguration(Jsons.jsonNode("'{\"name\":\"John\", \"age\":30, \"car\":null}'")) + .withSourceId(SOURCE_ID_2); + return Arrays.asList(sourceConnection1, sourceConnection2); + } + + public static List destinationConnections() { + final DestinationConnection destinationConnection1 = new DestinationConnection() + .withName("destination-1") + .withTombstone(false) + .withDestinationDefinitionId(DESTINATION_DEFINITION_ID_1) + .withWorkspaceId(WORKSPACE_ID) + .withConfiguration(Jsons.jsonNode("'{\"name\":\"John\", \"age\":30, \"car\":null}'")) + .withDestinationId(DESTINATION_ID_1); + final DestinationConnection destinationConnection2 = new DestinationConnection() + .withName("destination-2") + .withTombstone(false) + .withDestinationDefinitionId(DESTINATION_DEFINITION_ID_2) + .withWorkspaceId(WORKSPACE_ID) + .withConfiguration(Jsons.jsonNode("'{\"name\":\"John\", \"age\":30, \"car\":null}'")) + .withDestinationId(DESTINATION_ID_2); + return Arrays.asList(destinationConnection1, destinationConnection2); + } + + public static List sourceOauthParameters() { + final SourceOAuthParameter sourceOAuthParameter1 = new SourceOAuthParameter() + .withConfiguration(Jsons.jsonNode("'{\"name\":\"John\", \"age\":30, \"car\":null}'")) + .withWorkspaceId(WORKSPACE_ID) + .withSourceDefinitionId(SOURCE_DEFINITION_ID_1) + .withOauthParameterId(SOURCE_OAUTH_PARAMETER_ID_1); + final SourceOAuthParameter sourceOAuthParameter2 = new SourceOAuthParameter() + .withConfiguration(Jsons.jsonNode("'{\"name\":\"John\", \"age\":30, \"car\":null}'")) + .withWorkspaceId(WORKSPACE_ID) + .withSourceDefinitionId(SOURCE_DEFINITION_ID_2) + .withOauthParameterId(SOURCE_OAUTH_PARAMETER_ID_2); + return Arrays.asList(sourceOAuthParameter1, sourceOAuthParameter2); + } + + public static List destinationOauthParameters() { + final DestinationOAuthParameter destinationOAuthParameter1 = new DestinationOAuthParameter() + .withConfiguration(Jsons.jsonNode("'{\"name\":\"John\", \"age\":30, \"car\":null}'")) + .withWorkspaceId(WORKSPACE_ID) + .withDestinationDefinitionId(DESTINATION_DEFINITION_ID_1) + .withOauthParameterId(DESTINATION_OAUTH_PARAMETER_ID_1); + final DestinationOAuthParameter destinationOAuthParameter2 = new DestinationOAuthParameter() + .withConfiguration(Jsons.jsonNode("'{\"name\":\"John\", \"age\":30, \"car\":null}'")) + .withWorkspaceId(WORKSPACE_ID) + .withDestinationDefinitionId(DESTINATION_DEFINITION_ID_2) + .withOauthParameterId(DESTINATION_OAUTH_PARAMETER_ID_2); + return Arrays.asList(destinationOAuthParameter1, destinationOAuthParameter2); + } + + public static List standardSyncOperations() { + final OperatorDbt operatorDbt = new OperatorDbt() + .withDbtArguments("dbt-arguments") + .withDockerImage("image-tag") + .withGitRepoBranch("git-repo-branch") + .withGitRepoUrl("git-repo-url"); + final StandardSyncOperation standardSyncOperation1 = new StandardSyncOperation() + .withName("operation-1") + .withTombstone(false) + .withOperationId(OPERATION_ID_1) + .withWorkspaceId(WORKSPACE_ID) + .withOperatorDbt(operatorDbt) + .withOperatorNormalization(null) + .withOperatorType(OperatorType.DBT); + final StandardSyncOperation standardSyncOperation2 = new StandardSyncOperation() + .withName("operation-1") + .withTombstone(false) + .withOperationId(OPERATION_ID_2) + .withWorkspaceId(WORKSPACE_ID) + .withOperatorDbt(null) + .withOperatorNormalization(new OperatorNormalization().withOption(Option.BASIC)) + .withOperatorType(OperatorType.NORMALIZATION); + return Arrays.asList(standardSyncOperation1, standardSyncOperation2); + } + + public static List standardSyncs() { + final ResourceRequirements resourceRequirements = new ResourceRequirements() + .withCpuRequest("1") + .withCpuLimit("1") + .withMemoryRequest("1") + .withMemoryLimit("1"); + final Schedule schedule = new Schedule().withTimeUnit(TimeUnit.DAYS).withUnits(1L); + final StandardSync standardSync1 = new StandardSync() + .withOperationIds(Arrays.asList(OPERATION_ID_1, OPERATION_ID_2)) + .withConnectionId(CONNECTION_ID_1) + .withSourceId(SOURCE_ID_1) + .withDestinationId(DESTINATION_ID_1) + .withCatalog(getConfiguredCatalog()) + .withName("standard-sync-1") + .withManual(true) + .withNamespaceDefinition(NamespaceDefinitionType.CUSTOMFORMAT) + .withNamespaceFormat("") + .withPrefix("") + .withResourceRequirements(resourceRequirements) + .withStatus(Status.ACTIVE) + .withSchedule(schedule); + + final StandardSync standardSync2 = new StandardSync() + .withOperationIds(Arrays.asList(OPERATION_ID_1, OPERATION_ID_2)) + .withConnectionId(CONNECTION_ID_2) + .withSourceId(SOURCE_ID_1) + .withDestinationId(DESTINATION_ID_2) + .withCatalog(getConfiguredCatalog()) + .withName("standard-sync-2") + .withManual(true) + .withNamespaceDefinition(NamespaceDefinitionType.SOURCE) + .withNamespaceFormat("") + .withPrefix("") + .withResourceRequirements(resourceRequirements) + .withStatus(Status.ACTIVE) + .withSchedule(schedule); + + final StandardSync standardSync3 = new StandardSync() + .withOperationIds(Arrays.asList(OPERATION_ID_1, OPERATION_ID_2)) + .withConnectionId(CONNECTION_ID_3) + .withSourceId(SOURCE_ID_2) + .withDestinationId(DESTINATION_ID_1) + .withCatalog(getConfiguredCatalog()) + .withName("standard-sync-3") + .withManual(true) + .withNamespaceDefinition(NamespaceDefinitionType.DESTINATION) + .withNamespaceFormat("") + .withPrefix("") + .withResourceRequirements(resourceRequirements) + .withStatus(Status.ACTIVE) + .withSchedule(schedule); + + final StandardSync standardSync4 = new StandardSync() + .withOperationIds(Arrays.asList(OPERATION_ID_1, OPERATION_ID_2)) + .withConnectionId(CONNECTION_ID_4) + .withSourceId(SOURCE_ID_2) + .withDestinationId(DESTINATION_ID_2) + .withCatalog(getConfiguredCatalog()) + .withName("standard-sync-4") + .withManual(true) + .withNamespaceDefinition(NamespaceDefinitionType.CUSTOMFORMAT) + .withNamespaceFormat("") + .withPrefix("") + .withResourceRequirements(resourceRequirements) + .withStatus(Status.INACTIVE) + .withSchedule(schedule); + + return Arrays.asList(standardSync1, standardSync2, standardSync3, standardSync4); + } + + private static ConfiguredAirbyteCatalog getConfiguredCatalog() { + final AirbyteCatalog catalog = new AirbyteCatalog().withStreams(List.of( + CatalogHelpers.createAirbyteStream( + "models", + "models_schema", + io.airbyte.protocol.models.Field.of("id", JsonSchemaPrimitive.NUMBER), + io.airbyte.protocol.models.Field.of("make_id", JsonSchemaPrimitive.NUMBER), + io.airbyte.protocol.models.Field.of("model", JsonSchemaPrimitive.STRING)) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) + .withSourceDefinedPrimaryKey(List.of(List.of("id"))))); + return CatalogHelpers.toDefaultConfiguredCatalog(catalog); + } + + public static List standardSyncStates() { + final StandardSyncState standardSyncState1 = new StandardSyncState() + .withConnectionId(CONNECTION_ID_1) + .withState(new State().withState(Jsons.jsonNode("'{\"name\":\"John\", \"age\":30, \"car\":null}'"))); + final StandardSyncState standardSyncState2 = new StandardSyncState() + .withConnectionId(CONNECTION_ID_2) + .withState(new State().withState(Jsons.jsonNode("'{\"name\":\"John\", \"age\":30, \"car\":null}'"))); + final StandardSyncState standardSyncState3 = new StandardSyncState() + .withConnectionId(CONNECTION_ID_3) + .withState(new State().withState(Jsons.jsonNode("'{\"name\":\"John\", \"age\":30, \"car\":null}'"))); + final StandardSyncState standardSyncState4 = new StandardSyncState() + .withConnectionId(CONNECTION_ID_4) + .withState(new State().withState(Jsons.jsonNode("'{\"name\":\"John\", \"age\":30, \"car\":null}'"))); + return Arrays.asList(standardSyncState1, standardSyncState2, standardSyncState3, standardSyncState4); + } + + public static Instant now() { + return NOW; + } + +} diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/FlywayDatabaseMigrator.java b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/FlywayDatabaseMigrator.java index d53539da812e..3ee251f2bd4c 100644 --- a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/FlywayDatabaseMigrator.java +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/FlywayDatabaseMigrator.java @@ -52,10 +52,6 @@ public FlywayDatabaseMigrator(final Database database, final Flyway flyway) { this.flyway = flyway; } - private static String getDefaultMigrationFileLocation(final String dbIdentifier) { - return String.format("classpath:io/airbyte/db/instance/%s/migrations", dbIdentifier); - } - private static FluentConfiguration getConfiguration(final Database database, final String dbIdentifier, final String migrationRunner, @@ -98,12 +94,20 @@ public BaselineResult createBaseline() { @Override public String dumpSchema() throws IOException { - return new ExceptionWrappingDatabase(database).query(ctx -> ctx.meta().ddl().queryStream() + return getDisclaimer() + new ExceptionWrappingDatabase(database).query(ctx -> ctx.meta().ddl().queryStream() .map(query -> query.toString() + ";") .filter(statement -> !statement.startsWith("create schema")) .collect(Collectors.joining("\n"))); } + protected String getDisclaimer() { + return """ + // The content of the file is just to have a basic idea of the current state of the database and is not fully accurate.\040 + // It is also not used by any piece of code to generate anything.\040 + // It doesn't contain the enums created in the database and the default values might also be buggy.\s + """ + '\n'; + } + @VisibleForTesting public Database getDatabase() { return database; diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/FlywayMigrationDatabase.java b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/FlywayMigrationDatabase.java index c627b7e82d65..445d9fcbb14e 100644 --- a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/FlywayMigrationDatabase.java +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/FlywayMigrationDatabase.java @@ -32,12 +32,6 @@ public abstract class FlywayMigrationDatabase extends PostgresDatabase { private Connection connection; - private final String schemaDumpFile; - - protected FlywayMigrationDatabase(final String schemaDumpFile) { - this.schemaDumpFile = schemaDumpFile; - } - protected abstract Database getAndInitializeDatabase(String username, String password, String connectionString) throws IOException; protected abstract DatabaseMigrator getDatabaseMigrator(Database database); diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/MinimumFlywayMigrationVersionCheck.java b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/MinimumFlywayMigrationVersionCheck.java index e211e5ecad13..90ec91bb301e 100644 --- a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/MinimumFlywayMigrationVersionCheck.java +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/MinimumFlywayMigrationVersionCheck.java @@ -57,7 +57,8 @@ public static void assertDatabase(DatabaseInstance db, long timeoutMs) { public static void assertMigrations(DatabaseMigrator migrator, String minimumFlywayVersion, long timeoutMs) throws InterruptedException { var currWaitingTime = 0; var currDatabaseMigrationVersion = migrator.getLatestMigration().getVersion().getVersion(); - + LOGGER.info("Current database migration version " + currDatabaseMigrationVersion); + LOGGER.info("Minimum Flyway version required " + minimumFlywayVersion); while (currDatabaseMigrationVersion.compareTo(minimumFlywayVersion) < 0) { if (currWaitingTime >= timeoutMs) { throw new RuntimeException("Timeout while waiting for database to fulfill minimum flyway migration version.."); diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/configs/ConfigsDatabaseInstance.java b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/configs/ConfigsDatabaseInstance.java index 9cba5ed4ebb9..3213682b7c71 100644 --- a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/configs/ConfigsDatabaseInstance.java +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/configs/ConfigsDatabaseInstance.java @@ -7,6 +7,8 @@ import com.google.common.annotations.VisibleForTesting; import io.airbyte.commons.resources.MoreResources; import io.airbyte.db.Database; +import io.airbyte.db.Databases; +import io.airbyte.db.ExceptionWrappingDatabase; import io.airbyte.db.instance.BaseDatabaseInstance; import io.airbyte.db.instance.DatabaseInstance; import java.io.IOException; @@ -23,7 +25,11 @@ public class ConfigsDatabaseInstance extends BaseDatabaseInstance implements Dat private static final Function IS_CONFIGS_DATABASE_READY = database -> { try { LOGGER.info("Testing if airbyte_configs has been created and seeded..."); - return database.query(ctx -> hasData(ctx, "airbyte_configs")); + if (database.query(ctx -> hasTable(ctx, "airbyte_configs")) && database.query(ctx -> hasData(ctx, "airbyte_configs"))) { + return true; + } + + return database.query(ctx -> hasTable(ctx, "state")); } catch (Exception e) { return false; } @@ -31,11 +37,56 @@ public class ConfigsDatabaseInstance extends BaseDatabaseInstance implements Dat @VisibleForTesting public ConfigsDatabaseInstance(final String username, final String password, final String connectionString, final String schema) { - super(username, password, connectionString, schema, DATABASE_LOGGING_NAME, ConfigsDatabaseSchema.getTableNames(), IS_CONFIGS_DATABASE_READY); + super(username, password, connectionString, schema, DATABASE_LOGGING_NAME, ConfigsDatabaseTables.getTableNames(), IS_CONFIGS_DATABASE_READY); } public ConfigsDatabaseInstance(final String username, final String password, final String connectionString) throws IOException { this(username, password, connectionString, MoreResources.readResource(SCHEMA_PATH)); } + @Override + public boolean isInitialized() throws IOException { + final Database database = Databases.createPostgresDatabaseWithRetry( + username, + password, + connectionString, + isDatabaseConnected(databaseName)); + return new ExceptionWrappingDatabase(database).transaction(ctx -> { + // when we start fresh airbyte instance, we start with airbyte_configs configs table and then flyway + // breaks the table into individual table. + // state is the last table created by flyway migration. + // This is why we check if either of the two tables are present or not + if (hasTable(ctx, "state") || hasTable(ctx, "airbyte_configs")) { + LOGGER.info("The {} database is initialized", databaseName); + return true; + } + LOGGER.info("The {} database is not initialized; initializing it with schema", databaseName); + return false; + }); + } + + @Override + public Database getAndInitialize() throws IOException { + // When we need to setup the database, it means the database will be initialized after + // we connect to the database. So the database itself is considered ready as long as + // the connection is alive. + final Database database = Databases.createPostgresDatabaseWithRetry( + username, + password, + connectionString, + isDatabaseConnected(databaseName)); + + new ExceptionWrappingDatabase(database).transaction(ctx -> { + if (hasTable(ctx, "state") || hasTable(ctx, "airbyte_configs")) { + LOGGER.info("The {} database has been initialized", databaseName); + return null; + } + LOGGER.info("The {} database has not been initialized; initializing it with schema: {}", databaseName, initialSchema); + ctx.execute(initialSchema); + return null; + }); + + return database; + } + } diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/configs/ConfigsDatabaseSchema.java b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/configs/ConfigsDatabaseSchema.java deleted file mode 100644 index b62969310f2c..000000000000 --- a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/configs/ConfigsDatabaseSchema.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2021 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.db.instance.configs; - -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.commons.json.JsonSchemas; -import io.airbyte.db.instance.TableSchema; -import io.airbyte.validation.json.JsonSchemaValidator; -import java.io.File; -import java.nio.file.Path; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -public enum ConfigsDatabaseSchema implements TableSchema { - - AIRBYTE_CONFIGS("AirbyteConfigs.yaml"); - - static final Path SCHEMAS_ROOT = JsonSchemas.prepareSchemas("configs_database", ConfigsDatabaseSchema.class); - - private final String schemaFilename; - - ConfigsDatabaseSchema(final String schemaFilename) { - this.schemaFilename = schemaFilename; - } - - @Override - public String getTableName() { - return name().toLowerCase(); - } - - @Override - public JsonNode getTableDefinition() { - final File schemaFile = SCHEMAS_ROOT.resolve(schemaFilename).toFile(); - return JsonSchemaValidator.getSchema(schemaFile); - } - - /** - * @return table names in lower case - */ - public static Set getTableNames() { - return Stream.of(ConfigsDatabaseSchema.values()).map(ConfigsDatabaseSchema::getTableName).collect(Collectors.toSet()); - } - -} diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/configs/ConfigsDatabaseTables.java b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/configs/ConfigsDatabaseTables.java new file mode 100644 index 000000000000..2fecb97e5b27 --- /dev/null +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/configs/ConfigsDatabaseTables.java @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.db.instance.configs; + +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public enum ConfigsDatabaseTables { + + ACTOR, + ACTOR_DEFINITION, + ACTOR_OAUTH_PARAMETER, + CONNECTION, + CONNECTION_OPERATION, + OPERATION, + STATE, + WORKSPACE; + + public String getTableName() { + return name().toLowerCase(); + } + + /** + * @return table names in lower case + */ + public static Set getTableNames() { + return Stream.of(ConfigsDatabaseTables.values()).map(ConfigsDatabaseTables::getTableName).collect(Collectors.toSet()); + } + +} diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/configs/ConfigsFlywayMigrationDatabase.java b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/configs/ConfigsFlywayMigrationDatabase.java index f7b65a05015c..e9662e75748c 100644 --- a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/configs/ConfigsFlywayMigrationDatabase.java +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/configs/ConfigsFlywayMigrationDatabase.java @@ -14,10 +14,6 @@ */ public class ConfigsFlywayMigrationDatabase extends FlywayMigrationDatabase { - public ConfigsFlywayMigrationDatabase() { - super("src/main/resources/configs_database/schema_dump.txt"); - } - @Override protected Database getAndInitializeDatabase(final String username, final String password, final String connectionString) throws IOException { return new ConfigsDatabaseInstance(username, password, connectionString).getAndInitialize(); diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/configs/DeprecatedConfigsDatabaseInstance.java b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/configs/DeprecatedConfigsDatabaseInstance.java new file mode 100644 index 000000000000..2518278e42e5 --- /dev/null +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/configs/DeprecatedConfigsDatabaseInstance.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.db.instance.configs; + +import com.google.common.annotations.VisibleForTesting; +import io.airbyte.commons.resources.MoreResources; +import io.airbyte.db.Database; +import io.airbyte.db.instance.BaseDatabaseInstance; +import io.airbyte.db.instance.DatabaseInstance; +import java.io.IOException; +import java.util.function.Function; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Use {@link ConfigsDatabaseInstance} + */ +@Deprecated +public class DeprecatedConfigsDatabaseInstance extends BaseDatabaseInstance implements DatabaseInstance { + + private static final Logger LOGGER = LoggerFactory.getLogger(DeprecatedConfigsDatabaseInstance.class); + + private static final String DATABASE_LOGGING_NAME = "airbyte configs"; + private static final String SCHEMA_PATH = "configs_database/schema.sql"; + private static final Function IS_CONFIGS_DATABASE_READY = database -> { + try { + LOGGER.info("Testing if airbyte_configs has been created and seeded..."); + return database.query(ctx -> hasData(ctx, "airbyte_configs")); + } catch (Exception e) { + return false; + } + }; + + @VisibleForTesting + public DeprecatedConfigsDatabaseInstance(final String username, final String password, final String connectionString, final String schema) { + super(username, password, connectionString, schema, DATABASE_LOGGING_NAME, DeprecatedConfigsDatabaseTables.getTableNames(), + IS_CONFIGS_DATABASE_READY); + } + + public DeprecatedConfigsDatabaseInstance(final String username, final String password, final String connectionString) throws IOException { + this(username, password, connectionString, MoreResources.readResource(SCHEMA_PATH)); + } + +} diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/configs/DeprecatedConfigsDatabaseTables.java b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/configs/DeprecatedConfigsDatabaseTables.java new file mode 100644 index 000000000000..97675610dcff --- /dev/null +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/configs/DeprecatedConfigsDatabaseTables.java @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.db.instance.configs; + +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@Deprecated +public enum DeprecatedConfigsDatabaseTables { + + AIRBYTE_CONFIGS; + + public String getTableName() { + return name().toLowerCase(); + } + + /** + * @return table names in lower case + */ + public static Set getTableNames() { + return Stream.of(DeprecatedConfigsDatabaseTables.values()).map(DeprecatedConfigsDatabaseTables::getTableName).collect(Collectors.toSet()); + } + +} diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_32_8_001__AirbyteConfigDatabaseDenormalization.java b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_32_8_001__AirbyteConfigDatabaseDenormalization.java new file mode 100644 index 000000000000..f2b28d61c856 --- /dev/null +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_32_8_001__AirbyteConfigDatabaseDenormalization.java @@ -0,0 +1,778 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.db.instance.configs.migrations; + +import static org.jooq.impl.DSL.asterisk; +import static org.jooq.impl.DSL.currentOffsetDateTime; +import static org.jooq.impl.DSL.foreignKey; +import static org.jooq.impl.DSL.primaryKey; + +import com.google.common.annotations.VisibleForTesting; +import io.airbyte.commons.enums.Enums; +import io.airbyte.commons.json.Jsons; +import io.airbyte.config.AirbyteConfig; +import io.airbyte.config.ConfigSchema; +import io.airbyte.config.ConfigWithMetadata; +import io.airbyte.config.DestinationConnection; +import io.airbyte.config.DestinationOAuthParameter; +import io.airbyte.config.SourceConnection; +import io.airbyte.config.SourceOAuthParameter; +import io.airbyte.config.StandardDestinationDefinition; +import io.airbyte.config.StandardSourceDefinition; +import io.airbyte.config.StandardSync; +import io.airbyte.config.StandardSyncOperation; +import io.airbyte.config.StandardSyncState; +import io.airbyte.config.StandardWorkspace; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.util.List; +import java.util.UUID; +import java.util.stream.Collectors; +import org.flywaydb.core.api.migration.BaseJavaMigration; +import org.flywaydb.core.api.migration.Context; +import org.jooq.Catalog; +import org.jooq.DSLContext; +import org.jooq.EnumType; +import org.jooq.Field; +import org.jooq.JSONB; +import org.jooq.Record; +import org.jooq.Result; +import org.jooq.Schema; +import org.jooq.impl.DSL; +import org.jooq.impl.SQLDataType; +import org.jooq.impl.SchemaImpl; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class V0_32_8_001__AirbyteConfigDatabaseDenormalization extends BaseJavaMigration { + + private static final Logger LOGGER = LoggerFactory.getLogger(V0_32_8_001__AirbyteConfigDatabaseDenormalization.class); + + @Override + public void migrate(final Context context) throws Exception { + + // Warning: please do not use any jOOQ generated code to write a migration. + // As database schema changes, the generated jOOQ code can be deprecated. So + // old migration may not compile if there is any generated code. + final DSLContext ctx = DSL.using(context.getConnection()); + migrate(ctx); + } + + @VisibleForTesting + public static void migrate(final DSLContext ctx) { + createEnums(ctx); + createAndPopulateWorkspace(ctx); + createAndPopulateActorDefinition(ctx); + createAndPopulateActor(ctx); + crateAndPopulateActorOauthParameter(ctx); + createAndPopulateOperation(ctx); + createAndPopulateConnection(ctx); + createAndPopulateState(ctx); + } + + private static void createEnums(final DSLContext ctx) { + ctx.createType("source_type").asEnum("api", "file", "database", "custom").execute(); + LOGGER.info("source_type enum created"); + ctx.createType("actor_type").asEnum("source", "destination").execute(); + LOGGER.info("actor_type enum created"); + ctx.createType("operator_type").asEnum("normalization", "dbt").execute(); + LOGGER.info("operator_type enum created"); + ctx.createType("namespace_definition_type").asEnum("source", "destination", "customformat").execute(); + LOGGER.info("namespace_definition_type enum created"); + ctx.createType("status_type").asEnum("active", "inactive", "deprecated").execute(); + LOGGER.info("status_type enum created"); + } + + private static void createAndPopulateWorkspace(final DSLContext ctx) { + final Field id = DSL.field("id", SQLDataType.UUID.nullable(false)); + final Field name = DSL.field("name", SQLDataType.VARCHAR(256).nullable(false)); + final Field slug = DSL.field("slug", SQLDataType.VARCHAR(256).nullable(false)); + final Field initialSetupComplete = DSL.field("initial_setup_complete", SQLDataType.BOOLEAN.nullable(false)); + final Field customerId = DSL.field("customer_id", SQLDataType.UUID.nullable(true)); + final Field email = DSL.field("email", SQLDataType.VARCHAR(256).nullable(true)); + final Field anonymousDataCollection = DSL.field("anonymous_data_collection", SQLDataType.BOOLEAN.nullable(true)); + final Field sendNewsletter = DSL.field("send_newsletter", SQLDataType.BOOLEAN.nullable(true)); + final Field sendSecurityUpdates = DSL.field("send_security_updates", SQLDataType.BOOLEAN.nullable(true)); + final Field displaySetupWizard = DSL.field("display_setup_wizard", SQLDataType.BOOLEAN.nullable(true)); + final Field tombstone = DSL.field("tombstone", SQLDataType.BOOLEAN.nullable(false).defaultValue(false)); + final Field notifications = DSL.field("notifications", SQLDataType.JSONB.nullable(true)); + final Field firstSyncComplete = DSL.field("first_sync_complete", SQLDataType.BOOLEAN.nullable(true)); + final Field feedbackComplete = DSL.field("feedback_complete", SQLDataType.BOOLEAN.nullable(true)); + final Field createdAt = + DSL.field("created_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false).defaultValue(currentOffsetDateTime())); + final Field updatedAt = + DSL.field("updated_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false).defaultValue(currentOffsetDateTime())); + + ctx.createTableIfNotExists("workspace") + .columns(id, + customerId, + name, + slug, + email, + initialSetupComplete, + anonymousDataCollection, + sendNewsletter, + sendSecurityUpdates, + displaySetupWizard, + tombstone, + notifications, + firstSyncComplete, + feedbackComplete, + createdAt, + updatedAt) + .constraints(primaryKey(id)) + .execute(); + LOGGER.info("workspace table created"); + final List> configsWithMetadata = listConfigsWithMetadata(ConfigSchema.STANDARD_WORKSPACE, + StandardWorkspace.class, + ctx); + + for (final ConfigWithMetadata configWithMetadata : configsWithMetadata) { + final StandardWorkspace standardWorkspace = configWithMetadata.getConfig(); + ctx.insertInto(DSL.table("workspace")) + .set(id, standardWorkspace.getWorkspaceId()) + .set(customerId, standardWorkspace.getCustomerId()) + .set(name, standardWorkspace.getName()) + .set(slug, standardWorkspace.getSlug()) + .set(email, standardWorkspace.getEmail()) + .set(initialSetupComplete, standardWorkspace.getInitialSetupComplete()) + .set(anonymousDataCollection, standardWorkspace.getAnonymousDataCollection()) + .set(sendNewsletter, standardWorkspace.getNews()) + .set(sendSecurityUpdates, standardWorkspace.getSecurityUpdates()) + .set(displaySetupWizard, standardWorkspace.getDisplaySetupWizard()) + .set(tombstone, standardWorkspace.getTombstone() != null && standardWorkspace.getTombstone()) + .set(notifications, JSONB.valueOf(Jsons.serialize(standardWorkspace.getNotifications()))) + .set(firstSyncComplete, standardWorkspace.getFirstCompletedSync()) + .set(feedbackComplete, standardWorkspace.getFeedbackDone()) + .set(createdAt, OffsetDateTime.ofInstant(configWithMetadata.getCreatedAt(), ZoneOffset.UTC)) + .set(updatedAt, OffsetDateTime.ofInstant(configWithMetadata.getUpdatedAt(), ZoneOffset.UTC)) + .execute(); + } + LOGGER.info("workspace table populated with " + configsWithMetadata.size() + " records"); + } + + private static void createAndPopulateActorDefinition(DSLContext ctx) { + final Field id = DSL.field("id", SQLDataType.UUID.nullable(false)); + final Field name = DSL.field("name", SQLDataType.VARCHAR(256).nullable(false)); + final Field dockerRepository = DSL.field("docker_repository", SQLDataType.VARCHAR(256).nullable(false)); + final Field dockerImageTag = DSL.field("docker_image_tag", SQLDataType.VARCHAR(256).nullable(false)); + final Field documentationUrl = DSL.field("documentation_url", SQLDataType.VARCHAR(256).nullable(true)); + final Field spec = DSL.field("spec", SQLDataType.JSONB.nullable(false)); + final Field icon = DSL.field("icon", SQLDataType.VARCHAR(256).nullable(true)); + final Field actorType = DSL.field("actor_type", SQLDataType.VARCHAR.asEnumDataType(ActorType.class).nullable(false)); + final Field sourceType = DSL.field("source_type", SQLDataType.VARCHAR.asEnumDataType(SourceType.class).nullable(true)); + final Field createdAt = + DSL.field("created_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false).defaultValue(currentOffsetDateTime())); + final Field updatedAt = + DSL.field("updated_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false).defaultValue(currentOffsetDateTime())); + + ctx.createTableIfNotExists("actor_definition") + .columns(id, + name, + dockerRepository, + dockerImageTag, + documentationUrl, + icon, + actorType, + sourceType, + spec, + createdAt, + updatedAt) + .constraints(primaryKey(id)) + .execute(); + + LOGGER.info("actor_definition table created"); + + final List> sourceDefinitionsWithMetadata = listConfigsWithMetadata( + ConfigSchema.STANDARD_SOURCE_DEFINITION, + StandardSourceDefinition.class, + ctx); + + for (final ConfigWithMetadata configWithMetadata : sourceDefinitionsWithMetadata) { + final StandardSourceDefinition standardSourceDefinition = configWithMetadata.getConfig(); + ctx.insertInto(DSL.table("actor_definition")) + .set(id, standardSourceDefinition.getSourceDefinitionId()) + .set(name, standardSourceDefinition.getName()) + .set(dockerRepository, standardSourceDefinition.getDockerRepository()) + .set(dockerImageTag, standardSourceDefinition.getDockerImageTag()) + .set(documentationUrl, standardSourceDefinition.getDocumentationUrl()) + .set(icon, standardSourceDefinition.getIcon()) + .set(actorType, ActorType.source) + .set(sourceType, standardSourceDefinition.getSourceType() == null ? null + : Enums.toEnum(standardSourceDefinition.getSourceType().value(), SourceType.class).orElseThrow()) + .set(spec, JSONB.valueOf(Jsons.serialize(standardSourceDefinition.getSpec()))) + .set(createdAt, OffsetDateTime.ofInstant(configWithMetadata.getCreatedAt(), ZoneOffset.UTC)) + .set(updatedAt, OffsetDateTime.ofInstant(configWithMetadata.getUpdatedAt(), ZoneOffset.UTC)) + .execute(); + } + LOGGER.info("actor_definition table populated with " + sourceDefinitionsWithMetadata.size() + " source definition records"); + + final List> destinationDefinitionsWithMetadata = listConfigsWithMetadata( + ConfigSchema.STANDARD_DESTINATION_DEFINITION, + StandardDestinationDefinition.class, + ctx); + + for (final ConfigWithMetadata configWithMetadata : destinationDefinitionsWithMetadata) { + final StandardDestinationDefinition standardDestinationDefinition = configWithMetadata.getConfig(); + ctx.insertInto(DSL.table("actor_definition")) + .set(id, standardDestinationDefinition.getDestinationDefinitionId()) + .set(name, standardDestinationDefinition.getName()) + .set(dockerRepository, standardDestinationDefinition.getDockerRepository()) + .set(dockerImageTag, standardDestinationDefinition.getDockerImageTag()) + .set(documentationUrl, standardDestinationDefinition.getDocumentationUrl()) + .set(icon, standardDestinationDefinition.getIcon()) + .set(actorType, ActorType.destination) + .set(spec, JSONB.valueOf(Jsons.serialize(standardDestinationDefinition.getSpec()))) + .set(createdAt, OffsetDateTime.ofInstant(configWithMetadata.getCreatedAt(), ZoneOffset.UTC)) + .set(updatedAt, OffsetDateTime.ofInstant(configWithMetadata.getUpdatedAt(), ZoneOffset.UTC)) + .execute(); + } + LOGGER.info("actor_definition table populated with " + destinationDefinitionsWithMetadata.size() + " destination definition records"); + } + + private static void createAndPopulateActor(DSLContext ctx) { + final Field id = DSL.field("id", SQLDataType.UUID.nullable(false)); + final Field name = DSL.field("name", SQLDataType.VARCHAR(256).nullable(false)); + final Field actorDefinitionId = DSL.field("actor_definition_id", SQLDataType.UUID.nullable(false)); + final Field workspaceId = DSL.field("workspace_id", SQLDataType.UUID.nullable(false)); + final Field configuration = DSL.field("configuration", SQLDataType.JSONB.nullable(false)); + final Field actorType = DSL.field("actor_type", SQLDataType.VARCHAR.asEnumDataType(ActorType.class).nullable(false)); + final Field tombstone = DSL.field("tombstone", SQLDataType.BOOLEAN.nullable(false).defaultValue(false)); + final Field createdAt = + DSL.field("created_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false).defaultValue(currentOffsetDateTime())); + final Field updatedAt = + DSL.field("updated_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false).defaultValue(currentOffsetDateTime())); + + ctx.createTableIfNotExists("actor") + .columns(id, + workspaceId, + actorDefinitionId, + name, + configuration, + actorType, + tombstone, + createdAt, + updatedAt) + .constraints(primaryKey(id), + foreignKey(workspaceId).references("workspace", "id").onDeleteCascade(), + foreignKey(actorDefinitionId).references("actor_definition", "id").onDeleteCascade()) + .execute(); + ctx.createIndex("actor_actor_definition_id_idx").on("actor", "actor_definition_id").execute(); + + LOGGER.info("actor table created"); + + final List> sourcesWithMetadata = listConfigsWithMetadata( + ConfigSchema.SOURCE_CONNECTION, + SourceConnection.class, + ctx); + + for (final ConfigWithMetadata configWithMetadata : sourcesWithMetadata) { + final SourceConnection sourceConnection = configWithMetadata.getConfig(); + ctx.insertInto(DSL.table("actor")) + .set(id, sourceConnection.getSourceId()) + .set(workspaceId, sourceConnection.getWorkspaceId()) + .set(actorDefinitionId, sourceConnection.getSourceDefinitionId()) + .set(name, sourceConnection.getName()) + .set(configuration, JSONB.valueOf(Jsons.serialize(sourceConnection.getConfiguration()))) + .set(actorType, ActorType.source) + .set(tombstone, sourceConnection.getTombstone() != null && sourceConnection.getTombstone()) + .set(createdAt, OffsetDateTime.ofInstant(configWithMetadata.getCreatedAt(), ZoneOffset.UTC)) + .set(updatedAt, OffsetDateTime.ofInstant(configWithMetadata.getUpdatedAt(), ZoneOffset.UTC)) + .execute(); + } + LOGGER.info("actor table populated with " + sourcesWithMetadata.size() + " source records"); + + final List> destinationsWithMetadata = listConfigsWithMetadata( + ConfigSchema.DESTINATION_CONNECTION, + DestinationConnection.class, + ctx); + + for (final ConfigWithMetadata configWithMetadata : destinationsWithMetadata) { + final DestinationConnection destinationConnection = configWithMetadata.getConfig(); + ctx.insertInto(DSL.table("actor")) + .set(id, destinationConnection.getDestinationId()) + .set(workspaceId, destinationConnection.getWorkspaceId()) + .set(actorDefinitionId, destinationConnection.getDestinationDefinitionId()) + .set(name, destinationConnection.getName()) + .set(configuration, JSONB.valueOf(Jsons.serialize(destinationConnection.getConfiguration()))) + .set(actorType, ActorType.destination) + .set(tombstone, destinationConnection.getTombstone() != null && destinationConnection.getTombstone()) + .set(createdAt, OffsetDateTime.ofInstant(configWithMetadata.getCreatedAt(), ZoneOffset.UTC)) + .set(updatedAt, OffsetDateTime.ofInstant(configWithMetadata.getUpdatedAt(), ZoneOffset.UTC)) + .execute(); + } + LOGGER.info("actor table populated with " + destinationsWithMetadata.size() + " destination records"); + } + + private static void crateAndPopulateActorOauthParameter(final DSLContext ctx) { + final Field id = DSL.field("id", SQLDataType.UUID.nullable(false)); + final Field actorDefinitionId = DSL.field("actor_definition_id", SQLDataType.UUID.nullable(false)); + final Field configuration = DSL.field("configuration", SQLDataType.JSONB.nullable(false)); + final Field workspaceId = DSL.field("workspace_id", SQLDataType.UUID.nullable(true)); + final Field actorType = DSL.field("actor_type", SQLDataType.VARCHAR.asEnumDataType(ActorType.class).nullable(false)); + final Field createdAt = + DSL.field("created_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false).defaultValue(currentOffsetDateTime())); + final Field updatedAt = + DSL.field("updated_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false).defaultValue(currentOffsetDateTime())); + + ctx.createTableIfNotExists("actor_oauth_parameter") + .columns(id, + workspaceId, + actorDefinitionId, + configuration, + actorType, + createdAt, + updatedAt) + .constraints(primaryKey(id), + foreignKey(workspaceId).references("workspace", "id").onDeleteCascade(), + foreignKey(actorDefinitionId).references("actor_definition", "id").onDeleteCascade()) + .execute(); + + LOGGER.info("actor_oauth_parameter table created"); + + final List> sourceOauthParamsWithMetadata = listConfigsWithMetadata( + ConfigSchema.SOURCE_OAUTH_PARAM, + SourceOAuthParameter.class, + ctx); + + for (final ConfigWithMetadata configWithMetadata : sourceOauthParamsWithMetadata) { + final SourceOAuthParameter sourceOAuthParameter = configWithMetadata.getConfig(); + ctx.insertInto(DSL.table("actor_oauth_parameter")) + .set(id, sourceOAuthParameter.getOauthParameterId()) + .set(workspaceId, sourceOAuthParameter.getWorkspaceId()) + .set(actorDefinitionId, sourceOAuthParameter.getSourceDefinitionId()) + .set(configuration, JSONB.valueOf(Jsons.serialize(sourceOAuthParameter.getConfiguration()))) + .set(actorType, ActorType.source) + .set(createdAt, OffsetDateTime.ofInstant(configWithMetadata.getCreatedAt(), ZoneOffset.UTC)) + .set(updatedAt, OffsetDateTime.ofInstant(configWithMetadata.getUpdatedAt(), ZoneOffset.UTC)) + .execute(); + } + + LOGGER.info("actor_oauth_parameter table populated with " + sourceOauthParamsWithMetadata.size() + " source oauth params records"); + + final List> destinationOauthParamsWithMetadata = listConfigsWithMetadata( + ConfigSchema.DESTINATION_OAUTH_PARAM, + DestinationOAuthParameter.class, + ctx); + + for (final ConfigWithMetadata configWithMetadata : destinationOauthParamsWithMetadata) { + final DestinationOAuthParameter destinationOAuthParameter = configWithMetadata.getConfig(); + ctx.insertInto(DSL.table("actor_oauth_parameter")) + .set(id, destinationOAuthParameter.getOauthParameterId()) + .set(workspaceId, destinationOAuthParameter.getWorkspaceId()) + .set(actorDefinitionId, destinationOAuthParameter.getDestinationDefinitionId()) + .set(configuration, JSONB.valueOf(Jsons.serialize(destinationOAuthParameter.getConfiguration()))) + .set(actorType, ActorType.destination) + .set(createdAt, OffsetDateTime.ofInstant(configWithMetadata.getCreatedAt(), ZoneOffset.UTC)) + .set(updatedAt, OffsetDateTime.ofInstant(configWithMetadata.getUpdatedAt(), ZoneOffset.UTC)) + .execute(); + } + + LOGGER.info("actor_oauth_parameter table populated with " + destinationOauthParamsWithMetadata.size() + " destination oauth params records"); + } + + private static void createAndPopulateOperation(final DSLContext ctx) { + final Field id = DSL.field("id", SQLDataType.UUID.nullable(false)); + final Field workspaceId = DSL.field("workspace_id", SQLDataType.UUID.nullable(false)); + final Field name = DSL.field("name", SQLDataType.VARCHAR(256).nullable(false)); + final Field operatorType = DSL.field("operator_type", SQLDataType.VARCHAR.asEnumDataType(OperatorType.class).nullable(false)); + final Field operatorNormalization = DSL.field("operator_normalization", SQLDataType.JSONB.nullable(true)); + final Field operatorDbt = DSL.field("operator_dbt", SQLDataType.JSONB.nullable(true)); + final Field tombstone = DSL.field("tombstone", SQLDataType.BOOLEAN.nullable(false).defaultValue(false)); + final Field createdAt = + DSL.field("created_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false).defaultValue(currentOffsetDateTime())); + final Field updatedAt = + DSL.field("updated_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false).defaultValue(currentOffsetDateTime())); + + ctx.createTableIfNotExists("operation") + .columns(id, + workspaceId, + name, + operatorType, + operatorNormalization, + operatorDbt, + tombstone, + createdAt, + updatedAt) + .constraints(primaryKey(id), + foreignKey(workspaceId).references("workspace", "id").onDeleteCascade()) + .execute(); + + LOGGER.info("operation table created"); + + final List> configsWithMetadata = listConfigsWithMetadata( + ConfigSchema.STANDARD_SYNC_OPERATION, + StandardSyncOperation.class, + ctx); + + for (final ConfigWithMetadata configWithMetadata : configsWithMetadata) { + final StandardSyncOperation standardSyncOperation = configWithMetadata.getConfig(); + ctx.insertInto(DSL.table("operation")) + .set(id, standardSyncOperation.getOperationId()) + .set(workspaceId, standardSyncOperation.getWorkspaceId()) + .set(name, standardSyncOperation.getName()) + .set(operatorType, standardSyncOperation.getOperatorType() == null ? null + : Enums.toEnum(standardSyncOperation.getOperatorType().value(), OperatorType.class).orElseThrow()) + .set(operatorNormalization, JSONB.valueOf(Jsons.serialize(standardSyncOperation.getOperatorNormalization()))) + .set(operatorDbt, JSONB.valueOf(Jsons.serialize(standardSyncOperation.getOperatorDbt()))) + .set(tombstone, standardSyncOperation.getTombstone() != null && standardSyncOperation.getTombstone()) + .set(createdAt, OffsetDateTime.ofInstant(configWithMetadata.getCreatedAt(), ZoneOffset.UTC)) + .set(updatedAt, OffsetDateTime.ofInstant(configWithMetadata.getUpdatedAt(), ZoneOffset.UTC)) + .execute(); + } + + LOGGER.info("operation table populated with " + configsWithMetadata.size() + " records"); + } + + private static void createConnectionOperation(final DSLContext ctx) { + final Field id = DSL.field("id", SQLDataType.UUID.nullable(false)); + final Field connectionId = DSL.field("connection_id", SQLDataType.UUID.nullable(false)); + final Field operationId = DSL.field("operation_id", SQLDataType.UUID.nullable(false)); + final Field createdAt = + DSL.field("created_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false).defaultValue(currentOffsetDateTime())); + final Field updatedAt = + DSL.field("updated_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false).defaultValue(currentOffsetDateTime())); + + ctx.createTableIfNotExists("connection_operation") + .columns(id, + connectionId, + operationId, + createdAt, + updatedAt) + .constraints(primaryKey(id, connectionId, operationId), + foreignKey(connectionId).references("connection", "id").onDeleteCascade(), + foreignKey(operationId).references("operation", "id").onDeleteCascade()) + .execute(); + LOGGER.info("connection_operation table created"); + } + + private static void createAndPopulateConnection(final DSLContext ctx) { + final Field id = DSL.field("id", SQLDataType.UUID.nullable(false)); + final Field namespaceDefinition = DSL + .field("namespace_definition", SQLDataType.VARCHAR.asEnumDataType(NamespaceDefinitionType.class).nullable(false)); + final Field namespaceFormat = DSL.field("namespace_format", SQLDataType.VARCHAR(256).nullable(true)); + final Field prefix = DSL.field("prefix", SQLDataType.VARCHAR(256).nullable(true)); + final Field sourceId = DSL.field("source_id", SQLDataType.UUID.nullable(false)); + final Field destinationId = DSL.field("destination_id", SQLDataType.UUID.nullable(false)); + final Field name = DSL.field("name", SQLDataType.VARCHAR(256).nullable(false)); + final Field catalog = DSL.field("catalog", SQLDataType.JSONB.nullable(false)); + final Field status = DSL.field("status", SQLDataType.VARCHAR.asEnumDataType(StatusType.class).nullable(true)); + final Field schedule = DSL.field("schedule", SQLDataType.JSONB.nullable(true)); + final Field manual = DSL.field("manual", SQLDataType.BOOLEAN.nullable(false)); + final Field resourceRequirements = DSL.field("resource_requirements", SQLDataType.JSONB.nullable(true)); + final Field createdAt = + DSL.field("created_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false).defaultValue(currentOffsetDateTime())); + final Field updatedAt = + DSL.field("updated_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false).defaultValue(currentOffsetDateTime())); + + ctx.createTableIfNotExists("connection") + .columns(id, + namespaceDefinition, + namespaceFormat, + prefix, + sourceId, + destinationId, + name, + catalog, + status, + schedule, + manual, + resourceRequirements, + createdAt, + updatedAt) + .constraints(primaryKey(id), + foreignKey(sourceId).references("actor", "id").onDeleteCascade(), + foreignKey(destinationId).references("actor", "id").onDeleteCascade()) + .execute(); + ctx.createIndex("connection_source_id_idx").on("connection", "source_id").execute(); + ctx.createIndex("connection_destination_id_idx").on("connection", "destination_id").execute(); + + LOGGER.info("connection table created"); + createConnectionOperation(ctx); + + final List> configsWithMetadata = listConfigsWithMetadata( + ConfigSchema.STANDARD_SYNC, + StandardSync.class, + ctx); + for (final ConfigWithMetadata configWithMetadata : configsWithMetadata) { + final StandardSync standardSync = configWithMetadata.getConfig(); + ctx.insertInto(DSL.table("connection")) + .set(id, standardSync.getConnectionId()) + .set(namespaceDefinition, standardSync.getNamespaceDefinition() == null ? null + : Enums.toEnum(standardSync.getNamespaceDefinition().value(), NamespaceDefinitionType.class).orElseThrow()) + .set(namespaceFormat, standardSync.getNamespaceFormat()) + .set(prefix, standardSync.getPrefix()) + .set(sourceId, standardSync.getSourceId()) + .set(destinationId, standardSync.getDestinationId()) + .set(name, standardSync.getName()) + .set(catalog, JSONB.valueOf(Jsons.serialize(standardSync.getCatalog()))) + .set(status, standardSync.getStatus() == null ? null : Enums.toEnum(standardSync.getStatus().value(), StatusType.class).orElseThrow()) + .set(schedule, JSONB.valueOf(Jsons.serialize(standardSync.getSchedule()))) + .set(manual, standardSync.getManual()) + .set(resourceRequirements, JSONB.valueOf(Jsons.serialize(standardSync.getResourceRequirements()))) + .set(createdAt, OffsetDateTime.ofInstant(configWithMetadata.getCreatedAt(), ZoneOffset.UTC)) + .set(updatedAt, OffsetDateTime.ofInstant(configWithMetadata.getUpdatedAt(), ZoneOffset.UTC)) + .execute(); + populateConnectionOperation(ctx, configWithMetadata); + } + + LOGGER.info("connection table populated with " + configsWithMetadata.size() + " records"); + } + + private static void createAndPopulateState(final DSLContext ctx) { + final Field id = DSL.field("id", SQLDataType.UUID.nullable(false)); + final Field connectionId = DSL.field("connection_id", SQLDataType.UUID.nullable(false)); + final Field state = DSL.field("state", SQLDataType.JSONB.nullable(true)); + final Field createdAt = + DSL.field("created_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false).defaultValue(currentOffsetDateTime())); + final Field updatedAt = + DSL.field("updated_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false).defaultValue(currentOffsetDateTime())); + + ctx.createTableIfNotExists("state") + .columns(id, + connectionId, + state, + createdAt, + updatedAt) + .constraints(primaryKey(id, connectionId), + foreignKey(connectionId).references("connection", "id").onDeleteCascade()) + .execute(); + + LOGGER.info("state table created"); + + final List> configsWithMetadata = listConfigsWithMetadata( + ConfigSchema.STANDARD_SYNC_STATE, + StandardSyncState.class, + ctx); + + for (final ConfigWithMetadata configWithMetadata : configsWithMetadata) { + final StandardSyncState standardSyncState = configWithMetadata.getConfig(); + ctx.insertInto(DSL.table("state")) + .set(id, UUID.randomUUID()) + .set(connectionId, standardSyncState.getConnectionId()) + .set(state, JSONB.valueOf(Jsons.serialize(standardSyncState.getState()))) + .set(createdAt, OffsetDateTime.ofInstant(configWithMetadata.getCreatedAt(), ZoneOffset.UTC)) + .set(updatedAt, OffsetDateTime.ofInstant(configWithMetadata.getUpdatedAt(), ZoneOffset.UTC)) + .execute(); + } + + LOGGER.info("state table populated with " + configsWithMetadata.size() + " records"); + } + + private static void populateConnectionOperation(final DSLContext ctx, + final ConfigWithMetadata standardSyncWithMetadata) { + final Field id = DSL.field("id", SQLDataType.UUID.nullable(false)); + final Field connectionId = DSL.field("connection_id", SQLDataType.UUID.nullable(false)); + final Field operationId = DSL.field("operation_id", SQLDataType.UUID.nullable(false)); + final Field createdAt = + DSL.field("created_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false).defaultValue(currentOffsetDateTime())); + final Field updatedAt = + DSL.field("updated_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false).defaultValue(currentOffsetDateTime())); + + final StandardSync standardSync = standardSyncWithMetadata.getConfig(); + for (final UUID operationIdFromStandardSync : standardSync.getOperationIds()) { + ctx.insertInto(DSL.table("connection_operation")) + .set(id, UUID.randomUUID()) + .set(connectionId, standardSync.getConnectionId()) + .set(operationId, operationIdFromStandardSync) + .set(createdAt, OffsetDateTime.ofInstant(standardSyncWithMetadata.getCreatedAt(), ZoneOffset.UTC)) + .set(updatedAt, OffsetDateTime.ofInstant(standardSyncWithMetadata.getUpdatedAt(), ZoneOffset.UTC)) + .execute(); + } + LOGGER.info("connection_operation table populated with " + standardSync.getOperationIds().size() + " records"); + } + + private static List> listConfigsWithMetadata(final AirbyteConfig airbyteConfigType, + final Class clazz, + final DSLContext ctx) { + final Field configId = DSL.field("config_id", SQLDataType.VARCHAR(36).nullable(false)); + final Field configType = DSL.field("config_type", SQLDataType.VARCHAR(60).nullable(false)); + final Field createdAt = + DSL.field("created_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false).defaultValue(currentOffsetDateTime())); + final Field updatedAt = + DSL.field("updated_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false).defaultValue(currentOffsetDateTime())); + final Field configBlob = DSL.field("config_blob", SQLDataType.JSONB.nullable(false)); + final Result results = ctx.select(asterisk()).from(DSL.table("airbyte_configs")).where(configType.eq(airbyteConfigType.name())).fetch(); + + return results.stream().map(record -> new ConfigWithMetadata<>( + record.get(configId), + record.get(configType), + record.get(createdAt).toInstant(), + record.get(updatedAt).toInstant(), + Jsons.deserialize(record.get(configBlob).data(), clazz))) + .collect(Collectors.toList()); + } + + public enum SourceType implements EnumType { + + api("api"), + file("file"), + database("database"), + custom("custom"); + + private final String literal; + + SourceType(String literal) { + this.literal = literal; + } + + @Override + public Catalog getCatalog() { + return getSchema() == null ? null : getSchema().getCatalog(); + } + + @Override + public Schema getSchema() { + return new SchemaImpl(DSL.name("public"), null); + + } + + @Override + public String getName() { + return "source_type"; + } + + @Override + public String getLiteral() { + return literal; + } + + } + + public enum NamespaceDefinitionType implements EnumType { + + source("source"), + destination("destination"), + customformat("customformat"); + + private final String literal; + + NamespaceDefinitionType(String literal) { + this.literal = literal; + } + + @Override + public Catalog getCatalog() { + return getSchema() == null ? null : getSchema().getCatalog(); + } + + @Override + public Schema getSchema() { + return new SchemaImpl(DSL.name("public"), null); + } + + @Override + public String getName() { + return "namespace_definition_type"; + } + + @Override + public String getLiteral() { + return literal; + } + + } + + public enum StatusType implements EnumType { + + active("active"), + inactive("inactive"), + deprecated("deprecated"); + + private final String literal; + + StatusType(String literal) { + this.literal = literal; + } + + @Override + public Catalog getCatalog() { + return getSchema() == null ? null : getSchema().getCatalog(); + } + + @Override + public Schema getSchema() { + return new SchemaImpl(DSL.name("public"), null); + } + + @Override + public String getName() { + return "status_type"; + } + + @Override + public String getLiteral() { + return literal; + } + + } + + public enum OperatorType implements EnumType { + + normalization("normalization"), + dbt("dbt"); + + private final String literal; + + OperatorType(String literal) { + this.literal = literal; + } + + @Override + public Catalog getCatalog() { + return getSchema() == null ? null : getSchema().getCatalog(); + } + + @Override + public Schema getSchema() { + return new SchemaImpl(DSL.name("public"), null); + } + + @Override + public String getName() { + return "operator_type"; + } + + @Override + public String getLiteral() { + return literal; + } + + } + + public enum ActorType implements EnumType { + + source("source"), + destination("destination"); + + private final String literal; + + ActorType(String literal) { + this.literal = literal; + } + + @Override + public Catalog getCatalog() { + return getSchema() == null ? null : getSchema().getCatalog(); + } + + @Override + public Schema getSchema() { + return new SchemaImpl(DSL.name("public"), null); + } + + @Override + public String getName() { + return "actor_type"; + } + + @Override + public String getLiteral() { + return literal; + } + + } + +} diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/jobs/JobsFlywayMigrationDatabase.java b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/jobs/JobsFlywayMigrationDatabase.java index bc6bb3a049da..6bcb85e0528b 100644 --- a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/jobs/JobsFlywayMigrationDatabase.java +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/jobs/JobsFlywayMigrationDatabase.java @@ -14,10 +14,6 @@ */ public class JobsFlywayMigrationDatabase extends FlywayMigrationDatabase { - public JobsFlywayMigrationDatabase() { - super("src/main/resources/jobs_database/schema_dump.txt"); - } - @Override protected Database getAndInitializeDatabase(final String username, final String password, final String connectionString) throws IOException { return new JobsDatabaseInstance(username, password, connectionString).getAndInitialize(); diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/test/TestDatabaseProviders.java b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/test/TestDatabaseProviders.java index 7bb34b7dc588..b15cc81db76a 100644 --- a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/test/TestDatabaseProviders.java +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/test/TestDatabaseProviders.java @@ -24,11 +24,6 @@ public class TestDatabaseProviders { private final Optional> container; private boolean runMigration = true; - public TestDatabaseProviders(final Configs configs) { - this.configs = Optional.of(configs); - this.container = Optional.empty(); - } - public TestDatabaseProviders(final PostgreSQLContainer container) { this.configs = Optional.empty(); this.container = Optional.of(container); diff --git a/airbyte-db/lib/src/main/resources/configs_database/normalized_tables_schema.txt b/airbyte-db/lib/src/main/resources/configs_database/normalized_tables_schema.txt new file mode 100644 index 000000000000..ec44b1595f96 --- /dev/null +++ b/airbyte-db/lib/src/main/resources/configs_database/normalized_tables_schema.txt @@ -0,0 +1,216 @@ +// The file represents the schema of each of the config table + + enum_schema | enum_name | enum_value +-------------+---------------------------+------------------------------ + public | job_status | pending + public | job_status | running + public | job_status | incomplete + public | job_status | failed + public | job_status | succeeded + public | job_status | cancelled + public | attempt_status | running + public | attempt_status | failed + public | attempt_status | succeeded + public | job_config_type | check_connection_source + public | job_config_type | check_connection_destination + public | job_config_type | discover_schema + public | job_config_type | get_spec + public | job_config_type | sync + public | job_config_type | reset_connection + public | source_type | api + public | source_type | file + public | source_type | database + public | source_type | custom + public | actor_type | source + public | actor_type | destination + public | operator_type | normalization + public | operator_type | dbt + public | namespace_definition_type | source + public | namespace_definition_type | destination + public | namespace_definition_type | customformat + public | status_type | active + public | status_type | inactive + public | status_type | deprecated + + + + + Table "public.workspace" + Column | Type | Collation | Nullable | Default +---------------------------+--------------------------+-----------+----------+------------------- + id | uuid | | not null | + customer_id | uuid | | | + name | character varying(256) | | not null | + slug | character varying(256) | | not null | + email | character varying(256) | | | + initial_setup_complete | boolean | | not null | + anonymous_data_collection | boolean | | | + send_newsletter | boolean | | | + send_security_updates | boolean | | | + display_setup_wizard | boolean | | | + tombstone | boolean | | not null | false + notifications | jsonb | | | + first_sync_complete | boolean | | | + feedback_complete | boolean | | | + created_at | timestamp with time zone | | not null | CURRENT_TIMESTAMP + updated_at | timestamp with time zone | | not null | CURRENT_TIMESTAMP +Indexes: + "workspace_pkey" PRIMARY KEY, btree (id) +Referenced by: + TABLE "actor_oauth_parameter" CONSTRAINT "actor_oauth_parameter_workspace_id_fkey" FOREIGN KEY (workspace_id) REFERENCES workspace(id) ON DELETE CASCADE + TABLE "actor" CONSTRAINT "actor_workspace_id_fkey" FOREIGN KEY (workspace_id) REFERENCES workspace(id) ON DELETE CASCADE + TABLE "operation" CONSTRAINT "operation_workspace_id_fkey" FOREIGN KEY (workspace_id) REFERENCES workspace(id) ON DELETE CASCADE + + + + + Table "public.actor_definition" + Column | Type | Collation | Nullable | Default +-------------------+--------------------------+-----------+----------+------------------- + id | uuid | | not null | + name | character varying(256) | | not null | + docker_repository | character varying(256) | | not null | + docker_image_tag | character varying(256) | | not null | + documentation_url | character varying(256) | | | + icon | character varying(256) | | | + actor_type | actor_type | | not null | + source_type | source_type | | | + spec | jsonb | | not null | + created_at | timestamp with time zone | | not null | CURRENT_TIMESTAMP + updated_at | timestamp with time zone | | not null | CURRENT_TIMESTAMP +Indexes: + "actor_definition_pkey" PRIMARY KEY, btree (id) +Referenced by: + TABLE "actor" CONSTRAINT "actor_actor_definition_id_fkey" FOREIGN KEY (actor_definition_id) REFERENCES actor_definition(id) ON DELETE CASCADE + TABLE "actor_oauth_parameter" CONSTRAINT "actor_oauth_parameter_actor_definition_id_fkey" FOREIGN KEY (actor_definition_id) REFERENCES actor_definition(id) ON DELETE CASCADE + + + + + Table "public.actor" + Column | Type | Collation | Nullable | Default +---------------------+--------------------------+-----------+----------+------------------- + id | uuid | | not null | + workspace_id | uuid | | not null | + actor_definition_id | uuid | | not null | + name | character varying(256) | | not null | + configuration | jsonb | | not null | + actor_type | actor_type | | not null | + tombstone | boolean | | not null | false + created_at | timestamp with time zone | | not null | CURRENT_TIMESTAMP + updated_at | timestamp with time zone | | not null | CURRENT_TIMESTAMP +Indexes: + "actor_pkey" PRIMARY KEY, btree (id) + "actor_actor_definition_id_idx" btree (actor_definition_id) +Foreign-key constraints: + "actor_actor_definition_id_fkey" FOREIGN KEY (actor_definition_id) REFERENCES actor_definition(id) ON DELETE CASCADE + "actor_workspace_id_fkey" FOREIGN KEY (workspace_id) REFERENCES workspace(id) ON DELETE CASCADE +Referenced by: + TABLE "connection" CONSTRAINT "connection_destination_id_fkey" FOREIGN KEY (destination_id) REFERENCES actor(id) ON DELETE CASCADE + TABLE "connection" CONSTRAINT "connection_source_id_fkey" FOREIGN KEY (source_id) REFERENCES actor(id) ON DELETE CASCADE + + + + + Table "public.actor_oauth_parameter" + Column | Type | Collation | Nullable | Default +---------------------+--------------------------+-----------+----------+------------------- + id | uuid | | not null | + workspace_id | uuid | | | + actor_definition_id | uuid | | not null | + configuration | jsonb | | not null | + actor_type | actor_type | | not null | + created_at | timestamp with time zone | | not null | CURRENT_TIMESTAMP + updated_at | timestamp with time zone | | not null | CURRENT_TIMESTAMP +Indexes: + "actor_oauth_parameter_pkey" PRIMARY KEY, btree (id) +Foreign-key constraints: + "actor_oauth_parameter_actor_definition_id_fkey" FOREIGN KEY (actor_definition_id) REFERENCES actor_definition(id) ON DELETE CASCADE + "actor_oauth_parameter_workspace_id_fkey" FOREIGN KEY (workspace_id) REFERENCES workspace(id) ON DELETE CASCADE + + + + + Table "public.operation" + Column | Type | Collation | Nullable | Default +------------------------+--------------------------+-----------+----------+------------------- + id | uuid | | not null | + workspace_id | uuid | | not null | + name | character varying(256) | | not null | + operator_type | operator_type | | not null | + operator_normalization | jsonb | | | + operator_dbt | jsonb | | | + tombstone | boolean | | not null | false + created_at | timestamp with time zone | | not null | CURRENT_TIMESTAMP + updated_at | timestamp with time zone | | not null | CURRENT_TIMESTAMP +Indexes: + "operation_pkey" PRIMARY KEY, btree (id) +Foreign-key constraints: + "operation_workspace_id_fkey" FOREIGN KEY (workspace_id) REFERENCES workspace(id) ON DELETE CASCADE +Referenced by: + TABLE "connection_operation" CONSTRAINT "connection_operation_operation_id_fkey" FOREIGN KEY (operation_id) REFERENCES operation(id) ON DELETE CASCADE + + + + + Table "public.connection" + Column | Type | Collation | Nullable | Default +-----------------------+---------------------------+-----------+----------+------------------- + id | uuid | | not null | + namespace_definition | namespace_definition_type | | not null | + namespace_format | character varying(256) | | | + prefix | character varying(256) | | | + source_id | uuid | | not null | + destination_id | uuid | | not null | + name | character varying(256) | | not null | + catalog | jsonb | | not null | + status | status_type | | | + schedule | jsonb | | | + manual | boolean | | not null | + resource_requirements | jsonb | | | + created_at | timestamp with time zone | | not null | CURRENT_TIMESTAMP + updated_at | timestamp with time zone | | not null | CURRENT_TIMESTAMP +Indexes: + "connection_pkey" PRIMARY KEY, btree (id) + "connection_destination_id_idx" btree (destination_id) + "connection_source_id_idx" btree (source_id) +Foreign-key constraints: + "connection_destination_id_fkey" FOREIGN KEY (destination_id) REFERENCES actor(id) ON DELETE CASCADE + "connection_source_id_fkey" FOREIGN KEY (source_id) REFERENCES actor(id) ON DELETE CASCADE +Referenced by: + TABLE "connection_operation" CONSTRAINT "connection_operation_connection_id_fkey" FOREIGN KEY (connection_id) REFERENCES connection(id) ON DELETE CASCADE + TABLE "state" CONSTRAINT "state_connection_id_fkey" FOREIGN KEY (connection_id) REFERENCES connection(id) ON DELETE CASCADE + + + + + Table "public.connection_operation" + Column | Type | Collation | Nullable | Default +---------------+--------------------------+-----------+----------+------------------- + id | uuid | | not null | + connection_id | uuid | | not null | + operation_id | uuid | | not null | + created_at | timestamp with time zone | | not null | CURRENT_TIMESTAMP + updated_at | timestamp with time zone | | not null | CURRENT_TIMESTAMP +Indexes: + "connection_operation_pkey" PRIMARY KEY, btree (id, connection_id, operation_id) +Foreign-key constraints: + "connection_operation_connection_id_fkey" FOREIGN KEY (connection_id) REFERENCES connection(id) ON DELETE CASCADE + "connection_operation_operation_id_fkey" FOREIGN KEY (operation_id) REFERENCES operation(id) ON DELETE CASCADE + + + + + Table "public.state" + Column | Type | Collation | Nullable | Default +---------------+--------------------------+-----------+----------+------------------- + id | uuid | | not null | + connection_id | uuid | | not null | + state | jsonb | | | + created_at | timestamp with time zone | | not null | CURRENT_TIMESTAMP + updated_at | timestamp with time zone | | not null | CURRENT_TIMESTAMP +Indexes: + "state_pkey" PRIMARY KEY, btree (id, connection_id) +Foreign-key constraints: + "state_connection_id_fkey" FOREIGN KEY (connection_id) REFERENCES connection(id) ON DELETE CASCADE + diff --git a/airbyte-db/lib/src/main/resources/configs_database/schema_dump.txt b/airbyte-db/lib/src/main/resources/configs_database/schema_dump.txt index 6c9e11d400e4..863d6c9654db 100644 --- a/airbyte-db/lib/src/main/resources/configs_database/schema_dump.txt +++ b/airbyte-db/lib/src/main/resources/configs_database/schema_dump.txt @@ -1,3 +1,46 @@ +// The content of the file is just to have a basic idea of the current state of the database and is not fully accurate. +// It is also not used by any piece of code to generate anything. +// It doesn't contain the enums created in the database and the default values might also be buggy. + +create table "public"."actor"( + "id" uuid not null, + "workspace_id" uuid not null, + "actor_definition_id" uuid not null, + "name" varchar(256) not null, + "configuration" jsonb not null, + "actor_type" actor_type not null, + "tombstone" bool not null default false, + "created_at" timestamptz(35) not null default null, + "updated_at" timestamptz(35) not null default null, + constraint "actor_pkey" + primary key ("id") +); +create table "public"."actor_definition"( + "id" uuid not null, + "name" varchar(256) not null, + "docker_repository" varchar(256) not null, + "docker_image_tag" varchar(256) not null, + "documentation_url" varchar(256) null, + "icon" varchar(256) null, + "actor_type" actor_type not null, + "source_type" source_type null, + "spec" jsonb not null, + "created_at" timestamptz(35) not null default null, + "updated_at" timestamptz(35) not null default null, + constraint "actor_definition_pkey" + primary key ("id") +); +create table "public"."actor_oauth_parameter"( + "id" uuid not null, + "workspace_id" uuid null, + "actor_definition_id" uuid not null, + "configuration" jsonb not null, + "actor_type" actor_type not null, + "created_at" timestamptz(35) not null default null, + "updated_at" timestamptz(35) not null default null, + constraint "actor_oauth_parameter_pkey" + primary key ("id") +); create table "public"."airbyte_configs"( "id" int8 generated by default as identity not null, "config_id" varchar(36) not null, @@ -22,6 +65,126 @@ create table "public"."airbyte_configs_migrations"( constraint "airbyte_configs_migrations_pk" primary key ("installed_rank") ); +create table "public"."connection"( + "id" uuid not null, + "namespace_definition" namespace_definition_type not null, + "namespace_format" varchar(256) null, + "prefix" varchar(256) null, + "source_id" uuid not null, + "destination_id" uuid not null, + "name" varchar(256) not null, + "catalog" jsonb not null, + "status" status_type null, + "schedule" jsonb null, + "manual" bool not null, + "resource_requirements" jsonb null, + "created_at" timestamptz(35) not null default null, + "updated_at" timestamptz(35) not null default null, + constraint "connection_pkey" + primary key ("id") +); +create table "public"."connection_operation"( + "id" uuid not null, + "connection_id" uuid not null, + "operation_id" uuid not null, + "created_at" timestamptz(35) not null default null, + "updated_at" timestamptz(35) not null default null, + constraint "connection_operation_pkey" + primary key ( + "id", + "connection_id", + "operation_id" + ) +); +create table "public"."operation"( + "id" uuid not null, + "workspace_id" uuid not null, + "name" varchar(256) not null, + "operator_type" operator_type not null, + "operator_normalization" jsonb null, + "operator_dbt" jsonb null, + "tombstone" bool not null default false, + "created_at" timestamptz(35) not null default null, + "updated_at" timestamptz(35) not null default null, + constraint "operation_pkey" + primary key ("id") +); +create table "public"."state"( + "id" uuid not null, + "connection_id" uuid not null, + "state" jsonb null, + "created_at" timestamptz(35) not null default null, + "updated_at" timestamptz(35) not null default null, + constraint "state_pkey" + primary key ( + "id", + "connection_id" + ) +); +create table "public"."workspace"( + "id" uuid not null, + "customer_id" uuid null, + "name" varchar(256) not null, + "slug" varchar(256) not null, + "email" varchar(256) null, + "initial_setup_complete" bool not null, + "anonymous_data_collection" bool null, + "send_newsletter" bool null, + "send_security_updates" bool null, + "display_setup_wizard" bool null, + "tombstone" bool not null default false, + "notifications" jsonb null, + "first_sync_complete" bool null, + "feedback_complete" bool null, + "created_at" timestamptz(35) not null default null, + "updated_at" timestamptz(35) not null default null, + constraint "workspace_pkey" + primary key ("id") +); +alter table "public"."actor" + add constraint "actor_actor_definition_id_fkey" + foreign key ("actor_definition_id") + references "public"."actor_definition" ("id"); +alter table "public"."actor" + add constraint "actor_workspace_id_fkey" + foreign key ("workspace_id") + references "public"."workspace" ("id"); +alter table "public"."actor_oauth_parameter" + add constraint "actor_oauth_parameter_actor_definition_id_fkey" + foreign key ("actor_definition_id") + references "public"."actor_definition" ("id"); +alter table "public"."actor_oauth_parameter" + add constraint "actor_oauth_parameter_workspace_id_fkey" + foreign key ("workspace_id") + references "public"."workspace" ("id"); +alter table "public"."connection" + add constraint "connection_destination_id_fkey" + foreign key ("destination_id") + references "public"."actor" ("id"); +alter table "public"."connection" + add constraint "connection_source_id_fkey" + foreign key ("source_id") + references "public"."actor" ("id"); +alter table "public"."connection_operation" + add constraint "connection_operation_connection_id_fkey" + foreign key ("connection_id") + references "public"."connection" ("id"); +alter table "public"."connection_operation" + add constraint "connection_operation_operation_id_fkey" + foreign key ("operation_id") + references "public"."operation" ("id"); +alter table "public"."operation" + add constraint "operation_workspace_id_fkey" + foreign key ("workspace_id") + references "public"."workspace" ("id"); +alter table "public"."state" + add constraint "state_connection_id_fkey" + foreign key ("connection_id") + references "public"."connection" ("id"); +create index "actor_actor_definition_id_idx" on "public"."actor"("actor_definition_id" asc); +create unique index "actor_pkey" on "public"."actor"("id" asc); +create unique index "actor_definition_pkey" on "public"."actor_definition"("id" asc); +create unique index "actor_oauth_parameter_pkey" on "public"."actor_oauth_parameter"("id" asc); create index "airbyte_configs_id_idx" on "public"."airbyte_configs"("config_id" asc); create unique index "airbyte_configs_pkey" on "public"."airbyte_configs"("id" asc); create unique index "airbyte_configs_type_id_idx" on "public"."airbyte_configs"( @@ -30,3 +193,17 @@ create unique index "airbyte_configs_type_id_idx" on "public"."airbyte_configs"( ); create unique index "airbyte_configs_migrations_pk" on "public"."airbyte_configs_migrations"("installed_rank" asc); create index "airbyte_configs_migrations_s_idx" on "public"."airbyte_configs_migrations"("success" asc); +create index "connection_destination_id_idx" on "public"."connection"("destination_id" asc); +create unique index "connection_pkey" on "public"."connection"("id" asc); +create index "connection_source_id_idx" on "public"."connection"("source_id" asc); +create unique index "connection_operation_pkey" on "public"."connection_operation"( + "id" asc, + "connection_id" asc, + "operation_id" asc +); +create unique index "operation_pkey" on "public"."operation"("id" asc); +create unique index "state_pkey" on "public"."state"( + "id" asc, + "connection_id" asc +); +create unique index "workspace_pkey" on "public"."workspace"("id" asc); diff --git a/airbyte-db/lib/src/main/resources/jobs_database/schema_dump.txt b/airbyte-db/lib/src/main/resources/jobs_database/schema_dump.txt index 98a400f66637..32657bdf7af1 100644 --- a/airbyte-db/lib/src/main/resources/jobs_database/schema_dump.txt +++ b/airbyte-db/lib/src/main/resources/jobs_database/schema_dump.txt @@ -1,3 +1,7 @@ +// The content of the file is just to have a basic idea of the current state of the database and is not fully accurate. +// It is also not used by any piece of code to generate anything. +// It doesn't contain the enums created in the database and the default values might also be buggy. + create table "public"."airbyte_jobs_migrations"( "installed_rank" int4 not null, "version" varchar(50) null, diff --git a/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/ConfigsDatabaseInstanceTest.java b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/DeprecatedConfigsDatabaseInstanceTest.java similarity index 82% rename from airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/ConfigsDatabaseInstanceTest.java rename to airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/DeprecatedConfigsDatabaseInstanceTest.java index f85d0b3f4437..36b8f088d414 100644 --- a/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/ConfigsDatabaseInstanceTest.java +++ b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/DeprecatedConfigsDatabaseInstanceTest.java @@ -12,13 +12,13 @@ import org.jooq.exception.DataAccessException; import org.junit.jupiter.api.Test; -class ConfigsDatabaseInstanceTest extends AbstractConfigsDatabaseTest { +class DeprecatedConfigsDatabaseInstanceTest extends AbstractConfigsDatabaseTest { @Test public void testGet() throws Exception { // when the database has been initialized and loaded with data (in setup method), the get method // should return the database - database = new ConfigsDatabaseInstance(container.getUsername(), container.getPassword(), container.getJdbcUrl()).getInitialized(); + database = new DeprecatedConfigsDatabaseInstance(container.getUsername(), container.getPassword(), container.getJdbcUrl()).getInitialized(); // check table database.query(ctx -> ctx.fetchExists(select().from(AIRBYTE_CONFIGS))); } @@ -39,7 +39,8 @@ public void testGetAndInitialize() throws Exception { // when the configs database has been initialized, calling getAndInitialize again will not change // anything final String testSchema = "CREATE TABLE IF NOT EXISTS airbyte_test_configs(id BIGINT PRIMARY KEY);"; - database = new ConfigsDatabaseInstance(container.getUsername(), container.getPassword(), container.getJdbcUrl(), testSchema).getAndInitialize(); + database = new DeprecatedConfigsDatabaseInstance(container.getUsername(), container.getPassword(), container.getJdbcUrl(), testSchema) + .getAndInitialize(); // the airbyte_test_configs table does not exist assertThrows(DataAccessException.class, () -> database.query(ctx -> ctx.fetchExists(select().from("airbyte_test_configs")))); } diff --git a/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/migrations/SetupForNormalizedTablesTest.java b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/migrations/SetupForNormalizedTablesTest.java new file mode 100644 index 000000000000..97f1bff30381 --- /dev/null +++ b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/migrations/SetupForNormalizedTablesTest.java @@ -0,0 +1,428 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.db.instance.configs.migrations; + +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.table; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.Lists; +import io.airbyte.commons.json.Jsons; +import io.airbyte.config.AirbyteConfig; +import io.airbyte.config.ConfigSchema; +import io.airbyte.config.DestinationConnection; +import io.airbyte.config.DestinationOAuthParameter; +import io.airbyte.config.JobSyncConfig.NamespaceDefinitionType; +import io.airbyte.config.Notification; +import io.airbyte.config.Notification.NotificationType; +import io.airbyte.config.OperatorDbt; +import io.airbyte.config.OperatorNormalization; +import io.airbyte.config.OperatorNormalization.Option; +import io.airbyte.config.ResourceRequirements; +import io.airbyte.config.Schedule; +import io.airbyte.config.Schedule.TimeUnit; +import io.airbyte.config.SlackNotificationConfiguration; +import io.airbyte.config.SourceConnection; +import io.airbyte.config.SourceOAuthParameter; +import io.airbyte.config.StandardDestinationDefinition; +import io.airbyte.config.StandardSourceDefinition; +import io.airbyte.config.StandardSourceDefinition.SourceType; +import io.airbyte.config.StandardSync; +import io.airbyte.config.StandardSync.Status; +import io.airbyte.config.StandardSyncOperation; +import io.airbyte.config.StandardSyncOperation.OperatorType; +import io.airbyte.config.StandardSyncState; +import io.airbyte.config.StandardWorkspace; +import io.airbyte.config.State; +import io.airbyte.protocol.models.AirbyteCatalog; +import io.airbyte.protocol.models.AuthSpecification; +import io.airbyte.protocol.models.AuthSpecification.AuthType; +import io.airbyte.protocol.models.CatalogHelpers; +import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.ConnectorSpecification; +import io.airbyte.protocol.models.DestinationSyncMode; +import io.airbyte.protocol.models.JsonSchemaPrimitive; +import io.airbyte.protocol.models.SyncMode; +import java.net.URI; +import java.time.Instant; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import javax.annotation.Nullable; +import org.jooq.DSLContext; +import org.jooq.Field; +import org.jooq.JSONB; +import org.jooq.Record; +import org.jooq.Table; + +public class SetupForNormalizedTablesTest { + + private static final UUID WORKSPACE_ID = UUID.randomUUID(); + private static final UUID WORKSPACE_CUSTOMER_ID = UUID.randomUUID(); + private static final UUID SOURCE_DEFINITION_ID_1 = UUID.randomUUID(); + private static final UUID SOURCE_DEFINITION_ID_2 = UUID.randomUUID(); + private static final UUID DESTINATION_DEFINITION_ID_1 = UUID.randomUUID(); + private static final UUID DESTINATION_DEFINITION_ID_2 = UUID.randomUUID(); + private static final UUID SOURCE_ID_1 = UUID.randomUUID(); + private static final UUID SOURCE_ID_2 = UUID.randomUUID(); + private static final UUID DESTINATION_ID_1 = UUID.randomUUID(); + private static final UUID DESTINATION_ID_2 = UUID.randomUUID(); + private static final UUID OPERATION_ID_1 = UUID.randomUUID(); + private static final UUID OPERATION_ID_2 = UUID.randomUUID(); + private static final UUID CONNECTION_ID_1 = UUID.randomUUID(); + private static final UUID CONNECTION_ID_2 = UUID.randomUUID(); + private static final UUID CONNECTION_ID_3 = UUID.randomUUID(); + private static final UUID CONNECTION_ID_4 = UUID.randomUUID(); + private static final UUID SOURCE_OAUTH_PARAMETER_ID_1 = UUID.randomUUID(); + private static final UUID SOURCE_OAUTH_PARAMETER_ID_2 = UUID.randomUUID(); + private static final UUID DESTINATION_OAUTH_PARAMETER_ID_1 = UUID.randomUUID(); + private static final UUID DESTINATION_OAUTH_PARAMETER_ID_2 = UUID.randomUUID(); + private static final Table AIRBYTE_CONFIGS = table("airbyte_configs"); + private static final Field CONFIG_ID = field("config_id", String.class); + private static final Field CONFIG_TYPE = field("config_type", String.class); + private static final Field CONFIG_BLOB = field("config_blob", JSONB.class); + private static final Field CREATED_AT = field("created_at", OffsetDateTime.class); + private static final Field UPDATED_AT = field("updated_at", OffsetDateTime.class); + private static final Instant NOW = Instant.parse("2021-12-15T20:30:40.00Z"); + + public static void setup(final DSLContext context) { + createConfigInOldTable(context, standardWorkspace(), ConfigSchema.STANDARD_WORKSPACE); + + for (final StandardSourceDefinition standardSourceDefinition : standardSourceDefinitions()) { + createConfigInOldTable(context, standardSourceDefinition, ConfigSchema.STANDARD_SOURCE_DEFINITION); + } + + for (final StandardDestinationDefinition standardDestinationDefinition : standardDestinationDefinitions()) { + createConfigInOldTable(context, standardDestinationDefinition, ConfigSchema.STANDARD_DESTINATION_DEFINITION); + } + + for (final SourceConnection sourceConnection : sourceConnections()) { + createConfigInOldTable(context, sourceConnection, ConfigSchema.SOURCE_CONNECTION); + } + + for (final DestinationConnection destinationConnection : destinationConnections()) { + createConfigInOldTable(context, destinationConnection, ConfigSchema.DESTINATION_CONNECTION); + } + + for (final SourceOAuthParameter sourceOAuthParameter : sourceOauthParameters()) { + createConfigInOldTable(context, sourceOAuthParameter, ConfigSchema.SOURCE_OAUTH_PARAM); + } + + for (final DestinationOAuthParameter destinationOAuthParameter : destinationOauthParameters()) { + createConfigInOldTable(context, destinationOAuthParameter, ConfigSchema.DESTINATION_OAUTH_PARAM); + } + + for (final StandardSyncOperation standardSyncOperation : standardSyncOperations()) { + createConfigInOldTable(context, standardSyncOperation, ConfigSchema.STANDARD_SYNC_OPERATION); + } + + for (final StandardSync standardSync : standardSyncs()) { + createConfigInOldTable(context, standardSync, ConfigSchema.STANDARD_SYNC); + } + for (final StandardSyncState standardSyncState : standardSyncStates()) { + createConfigInOldTable(context, standardSyncState, ConfigSchema.STANDARD_SYNC_STATE); + } + } + + private static void createConfigInOldTable(final DSLContext context, final T config, final AirbyteConfig configType) { + insertConfigRecord( + context, + configType.name(), + Jsons.jsonNode(config), + configType.getIdFieldName()); + } + + private static void insertConfigRecord( + final DSLContext context, + final String configType, + final JsonNode configJson, + @Nullable final String idFieldName) { + final String configId = idFieldName == null ? UUID.randomUUID().toString() : configJson.get(idFieldName).asText(); + context.insertInto(AIRBYTE_CONFIGS) + .set(CONFIG_ID, configId) + .set(CONFIG_TYPE, configType) + .set(CONFIG_BLOB, JSONB.valueOf(Jsons.serialize(configJson))) + .set(CREATED_AT, OffsetDateTime.ofInstant(NOW, ZoneOffset.UTC)) + .set(UPDATED_AT, OffsetDateTime.ofInstant(NOW, ZoneOffset.UTC)) + .onConflict(CONFIG_TYPE, CONFIG_ID) + .doNothing() + .execute(); + } + + public static StandardWorkspace standardWorkspace() { + final Notification notification = new Notification() + .withNotificationType(NotificationType.SLACK) + .withSendOnFailure(true) + .withSendOnSuccess(true) + .withSlackConfiguration(new SlackNotificationConfiguration().withWebhook("webhook-url")); + return new StandardWorkspace() + .withWorkspaceId(WORKSPACE_ID) + .withCustomerId(WORKSPACE_CUSTOMER_ID) + .withName("test-workspace") + .withSlug("random-string") + .withEmail("abc@xyz.com") + .withInitialSetupComplete(true) + .withAnonymousDataCollection(true) + .withNews(true) + .withSecurityUpdates(true) + .withDisplaySetupWizard(true) + .withTombstone(false) + .withNotifications(Collections.singletonList(notification)) + .withFirstCompletedSync(true) + .withFeedbackDone(true); + } + + public static List standardSourceDefinitions() { + final ConnectorSpecification connectorSpecification = connectorSpecification(); + final StandardSourceDefinition standardSourceDefinition1 = new StandardSourceDefinition() + .withSourceDefinitionId(SOURCE_DEFINITION_ID_1) + .withSourceType(SourceType.API) + .withName("random-source-1") + .withDockerImageTag("tag-1") + .withDockerRepository("repository-1") + .withDocumentationUrl("documentation-url-1") + .withIcon("icon-1") + .withSpec(connectorSpecification); + final StandardSourceDefinition standardSourceDefinition2 = new StandardSourceDefinition() + .withSourceDefinitionId(SOURCE_DEFINITION_ID_2) + .withSourceType(SourceType.DATABASE) + .withName("random-source-2") + .withDockerImageTag("tag-2") + .withDockerRepository("repository-2") + .withDocumentationUrl("documentation-url-2") + .withIcon("icon-2"); + return Arrays.asList(standardSourceDefinition1, standardSourceDefinition2); + } + + private static ConnectorSpecification connectorSpecification() { + return new ConnectorSpecification() + .withAuthSpecification(new AuthSpecification().withAuthType(AuthType.OAUTH_2_0)) + .withConnectionSpecification(Jsons.jsonNode("'{\"name\":\"John\", \"age\":30, \"car\":null}'")) + .withDocumentationUrl(URI.create("whatever")) + .withAdvancedAuth(null) + .withChangelogUrl(URI.create("whatever")) + .withSupportedDestinationSyncModes(Arrays.asList(DestinationSyncMode.APPEND, DestinationSyncMode.OVERWRITE, DestinationSyncMode.APPEND_DEDUP)) + .withSupportsDBT(true) + .withSupportsIncremental(true) + .withSupportsNormalization(true); + } + + public static List standardDestinationDefinitions() { + final ConnectorSpecification connectorSpecification = connectorSpecification(); + final StandardDestinationDefinition standardDestinationDefinition1 = new StandardDestinationDefinition() + .withDestinationDefinitionId(DESTINATION_DEFINITION_ID_1) + .withName("random-destination-1") + .withDockerImageTag("tag-3") + .withDockerRepository("repository-3") + .withDocumentationUrl("documentation-url-3") + .withIcon("icon-3") + .withSpec(connectorSpecification); + final StandardDestinationDefinition standardDestinationDefinition2 = new StandardDestinationDefinition() + .withDestinationDefinitionId(DESTINATION_DEFINITION_ID_2) + .withName("random-destination-2") + .withDockerImageTag("tag-4") + .withDockerRepository("repository-4") + .withDocumentationUrl("documentation-url-4") + .withIcon("icon-4") + .withSpec(connectorSpecification); + return Arrays.asList(standardDestinationDefinition1, standardDestinationDefinition2); + } + + public static List sourceConnections() { + final SourceConnection sourceConnection1 = new SourceConnection() + .withName("source-1") + .withTombstone(false) + .withSourceDefinitionId(SOURCE_DEFINITION_ID_1) + .withWorkspaceId(WORKSPACE_ID) + .withConfiguration(Jsons.jsonNode("'{\"name\":\"John\", \"age\":30, \"car\":null}'")) + .withSourceId(SOURCE_ID_1); + final SourceConnection sourceConnection2 = new SourceConnection() + .withName("source-2") + .withTombstone(false) + .withSourceDefinitionId(SOURCE_DEFINITION_ID_2) + .withWorkspaceId(WORKSPACE_ID) + .withConfiguration(Jsons.jsonNode("'{\"name\":\"John\", \"age\":30, \"car\":null}'")) + .withSourceId(SOURCE_ID_2); + return Arrays.asList(sourceConnection1, sourceConnection2); + } + + public static List destinationConnections() { + final DestinationConnection destinationConnection1 = new DestinationConnection() + .withName("destination-1") + .withTombstone(false) + .withDestinationDefinitionId(DESTINATION_DEFINITION_ID_1) + .withWorkspaceId(WORKSPACE_ID) + .withConfiguration(Jsons.jsonNode("'{\"name\":\"John\", \"age\":30, \"car\":null}'")) + .withDestinationId(DESTINATION_ID_1); + final DestinationConnection destinationConnection2 = new DestinationConnection() + .withName("destination-2") + .withTombstone(false) + .withDestinationDefinitionId(DESTINATION_DEFINITION_ID_2) + .withWorkspaceId(WORKSPACE_ID) + .withConfiguration(Jsons.jsonNode("'{\"name\":\"John\", \"age\":30, \"car\":null}'")) + .withDestinationId(DESTINATION_ID_2); + return Arrays.asList(destinationConnection1, destinationConnection2); + } + + public static List sourceOauthParameters() { + final SourceOAuthParameter sourceOAuthParameter1 = new SourceOAuthParameter() + .withConfiguration(Jsons.jsonNode("'{\"name\":\"John\", \"age\":30, \"car\":null}'")) + .withWorkspaceId(WORKSPACE_ID) + .withSourceDefinitionId(SOURCE_DEFINITION_ID_1) + .withOauthParameterId(SOURCE_OAUTH_PARAMETER_ID_1); + final SourceOAuthParameter sourceOAuthParameter2 = new SourceOAuthParameter() + .withConfiguration(Jsons.jsonNode("'{\"name\":\"John\", \"age\":30, \"car\":null}'")) + .withWorkspaceId(WORKSPACE_ID) + .withSourceDefinitionId(SOURCE_DEFINITION_ID_2) + .withOauthParameterId(SOURCE_OAUTH_PARAMETER_ID_2); + return Arrays.asList(sourceOAuthParameter1, sourceOAuthParameter2); + } + + public static List destinationOauthParameters() { + final DestinationOAuthParameter destinationOAuthParameter1 = new DestinationOAuthParameter() + .withConfiguration(Jsons.jsonNode("'{\"name\":\"John\", \"age\":30, \"car\":null}'")) + .withWorkspaceId(WORKSPACE_ID) + .withDestinationDefinitionId(DESTINATION_DEFINITION_ID_1) + .withOauthParameterId(DESTINATION_OAUTH_PARAMETER_ID_1); + final DestinationOAuthParameter destinationOAuthParameter2 = new DestinationOAuthParameter() + .withConfiguration(Jsons.jsonNode("'{\"name\":\"John\", \"age\":30, \"car\":null}'")) + .withWorkspaceId(WORKSPACE_ID) + .withDestinationDefinitionId(DESTINATION_DEFINITION_ID_2) + .withOauthParameterId(DESTINATION_OAUTH_PARAMETER_ID_2); + return Arrays.asList(destinationOAuthParameter1, destinationOAuthParameter2); + } + + public static List standardSyncOperations() { + final OperatorDbt operatorDbt = new OperatorDbt() + .withDbtArguments("dbt-arguments") + .withDockerImage("image-tag") + .withGitRepoBranch("git-repo-branch") + .withGitRepoUrl("git-repo-url"); + final StandardSyncOperation standardSyncOperation1 = new StandardSyncOperation() + .withName("operation-1") + .withTombstone(false) + .withOperationId(OPERATION_ID_1) + .withWorkspaceId(WORKSPACE_ID) + .withOperatorDbt(operatorDbt) + .withOperatorNormalization(null) + .withOperatorType(OperatorType.DBT); + final StandardSyncOperation standardSyncOperation2 = new StandardSyncOperation() + .withName("operation-1") + .withTombstone(false) + .withOperationId(OPERATION_ID_2) + .withWorkspaceId(WORKSPACE_ID) + .withOperatorDbt(null) + .withOperatorNormalization(new OperatorNormalization().withOption(Option.BASIC)) + .withOperatorType(OperatorType.NORMALIZATION); + return Arrays.asList(standardSyncOperation1, standardSyncOperation2); + } + + public static List standardSyncs() { + final ResourceRequirements resourceRequirements = new ResourceRequirements() + .withCpuRequest("1") + .withCpuLimit("1") + .withMemoryRequest("1") + .withMemoryLimit("1"); + final Schedule schedule = new Schedule().withTimeUnit(TimeUnit.DAYS).withUnits(1L); + final StandardSync standardSync1 = new StandardSync() + .withOperationIds(Arrays.asList(OPERATION_ID_1, OPERATION_ID_2)) + .withConnectionId(CONNECTION_ID_1) + .withSourceId(SOURCE_ID_1) + .withDestinationId(DESTINATION_ID_1) + .withCatalog(getConfiguredCatalog()) + .withName("standard-sync-1") + .withManual(true) + .withNamespaceDefinition(NamespaceDefinitionType.CUSTOMFORMAT) + .withNamespaceFormat("") + .withPrefix("") + .withResourceRequirements(resourceRequirements) + .withStatus(Status.ACTIVE) + .withSchedule(schedule); + + final StandardSync standardSync2 = new StandardSync() + .withOperationIds(Arrays.asList(OPERATION_ID_1, OPERATION_ID_2)) + .withConnectionId(CONNECTION_ID_2) + .withSourceId(SOURCE_ID_1) + .withDestinationId(DESTINATION_ID_2) + .withCatalog(getConfiguredCatalog()) + .withName("standard-sync-2") + .withManual(true) + .withNamespaceDefinition(NamespaceDefinitionType.SOURCE) + .withNamespaceFormat("") + .withPrefix("") + .withResourceRequirements(resourceRequirements) + .withStatus(Status.ACTIVE) + .withSchedule(schedule); + + final StandardSync standardSync3 = new StandardSync() + .withOperationIds(Arrays.asList(OPERATION_ID_1, OPERATION_ID_2)) + .withConnectionId(CONNECTION_ID_3) + .withSourceId(SOURCE_ID_2) + .withDestinationId(DESTINATION_ID_1) + .withCatalog(getConfiguredCatalog()) + .withName("standard-sync-3") + .withManual(true) + .withNamespaceDefinition(NamespaceDefinitionType.DESTINATION) + .withNamespaceFormat("") + .withPrefix("") + .withResourceRequirements(resourceRequirements) + .withStatus(Status.ACTIVE) + .withSchedule(schedule); + + final StandardSync standardSync4 = new StandardSync() + .withOperationIds(Arrays.asList(OPERATION_ID_1, OPERATION_ID_2)) + .withConnectionId(CONNECTION_ID_4) + .withSourceId(SOURCE_ID_2) + .withDestinationId(DESTINATION_ID_2) + .withCatalog(getConfiguredCatalog()) + .withName("standard-sync-4") + .withManual(true) + .withNamespaceDefinition(NamespaceDefinitionType.CUSTOMFORMAT) + .withNamespaceFormat("") + .withPrefix("") + .withResourceRequirements(resourceRequirements) + .withStatus(Status.INACTIVE) + .withSchedule(schedule); + + return Arrays.asList(standardSync1, standardSync2, standardSync3, standardSync4); + } + + private static ConfiguredAirbyteCatalog getConfiguredCatalog() { + final AirbyteCatalog catalog = new AirbyteCatalog().withStreams(List.of( + CatalogHelpers.createAirbyteStream( + "models", + "models_schema", + io.airbyte.protocol.models.Field.of("id", JsonSchemaPrimitive.NUMBER), + io.airbyte.protocol.models.Field.of("make_id", JsonSchemaPrimitive.NUMBER), + io.airbyte.protocol.models.Field.of("model", JsonSchemaPrimitive.STRING)) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) + .withSourceDefinedPrimaryKey(List.of(List.of("id"))))); + return CatalogHelpers.toDefaultConfiguredCatalog(catalog); + } + + public static List standardSyncStates() { + final StandardSyncState standardSyncState1 = new StandardSyncState() + .withConnectionId(CONNECTION_ID_1) + .withState(new State().withState(Jsons.jsonNode("'{\"name\":\"John\", \"age\":30, \"car\":null}'"))); + final StandardSyncState standardSyncState2 = new StandardSyncState() + .withConnectionId(CONNECTION_ID_2) + .withState(new State().withState(Jsons.jsonNode("'{\"name\":\"John\", \"age\":30, \"car\":null}'"))); + final StandardSyncState standardSyncState3 = new StandardSyncState() + .withConnectionId(CONNECTION_ID_3) + .withState(new State().withState(Jsons.jsonNode("'{\"name\":\"John\", \"age\":30, \"car\":null}'"))); + final StandardSyncState standardSyncState4 = new StandardSyncState() + .withConnectionId(CONNECTION_ID_4) + .withState(new State().withState(Jsons.jsonNode("'{\"name\":\"John\", \"age\":30, \"car\":null}'"))); + return Arrays.asList(standardSyncState1, standardSyncState2, standardSyncState3, standardSyncState4); + } + + public static Instant now() { + return NOW; + } + +} diff --git a/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_32_8_001__AirbyteConfigDatabaseDenormalization_Test.java b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_32_8_001__AirbyteConfigDatabaseDenormalization_Test.java new file mode 100644 index 000000000000..70bc8ccd1638 --- /dev/null +++ b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_32_8_001__AirbyteConfigDatabaseDenormalization_Test.java @@ -0,0 +1,454 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.db.instance.configs.migrations; + +import static io.airbyte.db.instance.configs.migrations.SetupForNormalizedTablesTest.*; +import static io.airbyte.db.instance.configs.migrations.SetupForNormalizedTablesTest.destinationOauthParameters; +import static io.airbyte.db.instance.configs.migrations.SetupForNormalizedTablesTest.standardSyncOperations; +import static io.airbyte.db.instance.configs.migrations.SetupForNormalizedTablesTest.standardSyncStates; +import static io.airbyte.db.instance.configs.migrations.SetupForNormalizedTablesTest.standardSyncs; +import static io.airbyte.db.instance.configs.migrations.SetupForNormalizedTablesTest.standardWorkspace; +import static org.jooq.impl.DSL.asterisk; +import static org.jooq.impl.DSL.table; + +import io.airbyte.commons.enums.Enums; +import io.airbyte.commons.json.Jsons; +import io.airbyte.config.DestinationConnection; +import io.airbyte.config.DestinationOAuthParameter; +import io.airbyte.config.Notification; +import io.airbyte.config.OperatorDbt; +import io.airbyte.config.OperatorNormalization; +import io.airbyte.config.ResourceRequirements; +import io.airbyte.config.Schedule; +import io.airbyte.config.SourceConnection; +import io.airbyte.config.SourceOAuthParameter; +import io.airbyte.config.StandardDestinationDefinition; +import io.airbyte.config.StandardSourceDefinition; +import io.airbyte.config.StandardSync; +import io.airbyte.config.StandardSyncOperation; +import io.airbyte.config.StandardSyncState; +import io.airbyte.config.StandardWorkspace; +import io.airbyte.config.State; +import io.airbyte.db.Database; +import io.airbyte.db.instance.configs.AbstractConfigsDatabaseTest; +import io.airbyte.db.instance.configs.migrations.V0_32_8_001__AirbyteConfigDatabaseDenormalization.ActorType; +import io.airbyte.db.instance.configs.migrations.V0_32_8_001__AirbyteConfigDatabaseDenormalization.NamespaceDefinitionType; +import io.airbyte.db.instance.configs.migrations.V0_32_8_001__AirbyteConfigDatabaseDenormalization.OperatorType; +import io.airbyte.db.instance.configs.migrations.V0_32_8_001__AirbyteConfigDatabaseDenormalization.SourceType; +import io.airbyte.db.instance.configs.migrations.V0_32_8_001__AirbyteConfigDatabaseDenormalization.StatusType; +import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.ConnectorSpecification; +import java.io.IOException; +import java.sql.SQLException; +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import org.jooq.DSLContext; +import org.jooq.Field; +import org.jooq.JSONB; +import org.jooq.Record; +import org.jooq.Result; +import org.jooq.impl.DSL; +import org.jooq.impl.SQLDataType; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class V0_32_8_001__AirbyteConfigDatabaseDenormalization_Test extends AbstractConfigsDatabaseTest { + + @Test + public void testCompleteMigration() throws IOException, SQLException { + final Database database = getDatabase(); + final DSLContext context = DSL.using(database.getDataSource().getConnection()); + SetupForNormalizedTablesTest.setup(context); + + V0_32_8_001__AirbyteConfigDatabaseDenormalization.migrate(context); + + assertDataForWorkspace(context); + assertDataForSourceDefinition(context); + assertDataForDestinationDefinition(context); + assertDataForSourceConnection(context); + assertDataForDestinationConnection(context); + assertDataForSourceOauthParams(context); + assertDataForDestinationOauthParams(context); + assertDataForOperations(context); + assertDataForConnections(context); + assertDataForStandardSyncStates(context); + } + + private void assertDataForWorkspace(final DSLContext context) { + Result workspaces = context.select(asterisk()) + .from(table("workspace")) + .fetch(); + Assertions.assertEquals(1, workspaces.size()); + + final Field id = DSL.field("id", SQLDataType.UUID.nullable(false)); + final Field name = DSL.field("name", SQLDataType.VARCHAR(256).nullable(false)); + final Field slug = DSL.field("slug", SQLDataType.VARCHAR(256).nullable(false)); + final Field initialSetupComplete = DSL.field("initial_setup_complete", SQLDataType.BOOLEAN.nullable(false)); + final Field customerId = DSL.field("customer_id", SQLDataType.UUID.nullable(true)); + final Field email = DSL.field("email", SQLDataType.VARCHAR(256).nullable(true)); + final Field anonymousDataCollection = DSL.field("anonymous_data_collection", SQLDataType.BOOLEAN.nullable(true)); + final Field sendNewsletter = DSL.field("send_newsletter", SQLDataType.BOOLEAN.nullable(true)); + final Field sendSecurityUpdates = DSL.field("send_security_updates", SQLDataType.BOOLEAN.nullable(true)); + final Field displaySetupWizard = DSL.field("display_setup_wizard", SQLDataType.BOOLEAN.nullable(true)); + final Field tombstone = DSL.field("tombstone", SQLDataType.BOOLEAN.nullable(true)); + final Field notifications = DSL.field("notifications", SQLDataType.JSONB.nullable(true)); + final Field firstSyncComplete = DSL.field("first_sync_complete", SQLDataType.BOOLEAN.nullable(true)); + final Field feedbackComplete = DSL.field("feedback_complete", SQLDataType.BOOLEAN.nullable(true)); + final Field createdAt = DSL.field("created_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false)); + final Field updatedAt = DSL.field("updated_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false)); + + final Record workspace = workspaces.get(0); + + final List notificationList = new ArrayList<>(); + final List fetchedNotifications = Jsons.deserialize(workspace.get(notifications).data(), List.class); + for (Object notification : fetchedNotifications) { + notificationList.add(Jsons.convertValue(notification, Notification.class)); + } + final StandardWorkspace workspaceFromNewTable = new StandardWorkspace() + .withWorkspaceId(workspace.get(id)) + .withName(workspace.get(name)) + .withSlug(workspace.get(slug)) + .withInitialSetupComplete(workspace.get(initialSetupComplete)) + .withCustomerId(workspace.get(customerId)) + .withEmail(workspace.get(email)) + .withAnonymousDataCollection(workspace.get(anonymousDataCollection)) + .withNews(workspace.get(sendNewsletter)) + .withSecurityUpdates(workspace.get(sendSecurityUpdates)) + .withDisplaySetupWizard(workspace.get(displaySetupWizard)) + .withTombstone(workspace.get(tombstone)) + .withNotifications(notificationList) + .withFirstCompletedSync(workspace.get(firstSyncComplete)) + .withFeedbackDone(workspace.get(feedbackComplete)); + Assertions.assertEquals(standardWorkspace(), workspaceFromNewTable); + Assertions.assertEquals(now(), workspace.get(createdAt).toInstant()); + Assertions.assertEquals(now(), workspace.get(updatedAt).toInstant()); + } + + private void assertDataForSourceDefinition(final DSLContext context) { + final Field id = DSL.field("id", SQLDataType.UUID.nullable(false)); + final Field name = DSL.field("name", SQLDataType.VARCHAR(256).nullable(false)); + final Field dockerRepository = DSL.field("docker_repository", SQLDataType.VARCHAR(256).nullable(false)); + final Field dockerImageTag = DSL.field("docker_image_tag", SQLDataType.VARCHAR(256).nullable(false)); + final Field documentationUrl = DSL.field("documentation_url", SQLDataType.VARCHAR(256).nullable(false)); + final Field spec = DSL.field("spec", SQLDataType.JSONB.nullable(false)); + final Field icon = DSL.field("icon", SQLDataType.VARCHAR(256).nullable(true)); + final Field actorType = DSL.field("actor_type", SQLDataType.VARCHAR.asEnumDataType(ActorType.class).nullable(false)); + final Field sourceType = DSL.field("source_type", SQLDataType.VARCHAR.asEnumDataType(SourceType.class).nullable(true)); + final Field createdAt = DSL.field("created_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false)); + final Field updatedAt = DSL.field("updated_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false)); + + final Result sourceDefinitions = context.select(asterisk()) + .from(table("actor_definition")) + .where(actorType.eq(ActorType.source)) + .fetch(); + final List expectedDefinitions = standardSourceDefinitions(); + Assertions.assertEquals(expectedDefinitions.size(), sourceDefinitions.size()); + + for (final Record sourceDefinition : sourceDefinitions) { + final StandardSourceDefinition standardSourceDefinition = new StandardSourceDefinition() + .withSourceDefinitionId(sourceDefinition.get(id)) + .withDockerImageTag(sourceDefinition.get(dockerImageTag)) + .withIcon(sourceDefinition.get(icon)) + .withDockerRepository(sourceDefinition.get(dockerRepository)) + .withDocumentationUrl(sourceDefinition.get(documentationUrl)) + .withName(sourceDefinition.get(name)) + .withSourceType(Enums.toEnum(sourceDefinition.get(sourceType, String.class), StandardSourceDefinition.SourceType.class).orElseThrow()) + .withSpec(Jsons.deserialize(sourceDefinition.get(spec).data(), ConnectorSpecification.class)); + Assertions.assertTrue(expectedDefinitions.contains(standardSourceDefinition)); + Assertions.assertEquals(now(), sourceDefinition.get(createdAt).toInstant()); + Assertions.assertEquals(now(), sourceDefinition.get(updatedAt).toInstant()); + } + } + + private void assertDataForDestinationDefinition(final DSLContext context) { + final Field id = DSL.field("id", SQLDataType.UUID.nullable(false)); + final Field name = DSL.field("name", SQLDataType.VARCHAR(256).nullable(false)); + final Field dockerRepository = DSL.field("docker_repository", SQLDataType.VARCHAR(256).nullable(false)); + final Field dockerImageTag = DSL.field("docker_image_tag", SQLDataType.VARCHAR(256).nullable(false)); + final Field documentationUrl = DSL.field("documentation_url", SQLDataType.VARCHAR(256).nullable(false)); + final Field spec = DSL.field("spec", SQLDataType.JSONB.nullable(false)); + final Field icon = DSL.field("icon", SQLDataType.VARCHAR(256).nullable(true)); + final Field actorType = DSL.field("actor_type", SQLDataType.VARCHAR.asEnumDataType(ActorType.class).nullable(false)); + final Field sourceType = DSL.field("source_type", SQLDataType.VARCHAR.asEnumDataType(SourceType.class).nullable(true)); + final Field createdAt = DSL.field("created_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false)); + final Field updatedAt = DSL.field("updated_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false)); + + final Result destinationDefinitions = context.select(asterisk()) + .from(table("actor_definition")) + .where(actorType.eq(ActorType.destination)) + .fetch(); + final List expectedDefinitions = standardDestinationDefinitions(); + Assertions.assertEquals(expectedDefinitions.size(), destinationDefinitions.size()); + + for (final Record record : destinationDefinitions) { + final StandardDestinationDefinition standardDestinationDefinition = new StandardDestinationDefinition() + .withDestinationDefinitionId(record.get(id)) + .withDockerImageTag(record.get(dockerImageTag)) + .withIcon(record.get(icon)) + .withDockerRepository(record.get(dockerRepository)) + .withDocumentationUrl(record.get(documentationUrl)) + .withName(record.get(name)) + .withSpec(Jsons.deserialize(record.get(spec).data(), ConnectorSpecification.class)); + Assertions.assertTrue(expectedDefinitions.contains(standardDestinationDefinition)); + Assertions.assertNull(record.get(sourceType)); + Assertions.assertEquals(now(), record.get(createdAt).toInstant()); + Assertions.assertEquals(now(), record.get(updatedAt).toInstant()); + } + } + + private void assertDataForSourceConnection(final DSLContext context) { + final Field id = DSL.field("id", SQLDataType.UUID.nullable(false)); + final Field name = DSL.field("name", SQLDataType.VARCHAR(256).nullable(false)); + final Field actorDefinitionId = DSL.field("actor_definition_id", SQLDataType.UUID.nullable(false)); + final Field workspaceId = DSL.field("workspace_id", SQLDataType.UUID.nullable(false)); + final Field configuration = DSL.field("configuration", SQLDataType.JSONB.nullable(false)); + final Field actorType = DSL.field("actor_type", SQLDataType.VARCHAR.asEnumDataType(ActorType.class).nullable(false)); + final Field tombstone = DSL.field("tombstone", SQLDataType.BOOLEAN.nullable(false)); + final Field createdAt = DSL.field("created_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false)); + final Field updatedAt = DSL.field("updated_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false)); + + final Result sourceConnections = context.select(asterisk()) + .from(table("actor")) + .where(actorType.eq(ActorType.source)) + .fetch(); + final List expectedDefinitions = sourceConnections(); + Assertions.assertEquals(expectedDefinitions.size(), sourceConnections.size()); + + for (final Record record : sourceConnections) { + final SourceConnection sourceConnection = new SourceConnection() + .withSourceId(record.get(id)) + .withConfiguration(Jsons.deserialize(record.get(configuration).data())) + .withWorkspaceId(record.get(workspaceId)) + .withSourceDefinitionId(record.get(actorDefinitionId)) + .withTombstone(record.get(tombstone)) + .withName(record.get(name)); + + Assertions.assertTrue(expectedDefinitions.contains(sourceConnection)); + Assertions.assertEquals(now(), record.get(createdAt).toInstant()); + Assertions.assertEquals(now(), record.get(updatedAt).toInstant()); + } + } + + private void assertDataForDestinationConnection(final DSLContext context) { + final Field id = DSL.field("id", SQLDataType.UUID.nullable(false)); + final Field name = DSL.field("name", SQLDataType.VARCHAR(256).nullable(false)); + final Field actorDefinitionId = DSL.field("actor_definition_id", SQLDataType.UUID.nullable(false)); + final Field workspaceId = DSL.field("workspace_id", SQLDataType.UUID.nullable(false)); + final Field configuration = DSL.field("configuration", SQLDataType.JSONB.nullable(false)); + final Field actorType = DSL.field("actor_type", SQLDataType.VARCHAR.asEnumDataType(ActorType.class).nullable(false)); + final Field tombstone = DSL.field("tombstone", SQLDataType.BOOLEAN.nullable(false)); + final Field createdAt = DSL.field("created_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false)); + final Field updatedAt = DSL.field("updated_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false)); + + final Result destinationConnections = context.select(asterisk()) + .from(table("actor")) + .where(actorType.eq(ActorType.destination)) + .fetch(); + final List expectedDefinitions = destinationConnections(); + Assertions.assertEquals(expectedDefinitions.size(), destinationConnections.size()); + + for (final Record record : destinationConnections) { + final DestinationConnection destinationConnection = new DestinationConnection() + .withDestinationId(record.get(id)) + .withConfiguration(Jsons.deserialize(record.get(configuration).data())) + .withWorkspaceId(record.get(workspaceId)) + .withDestinationDefinitionId(record.get(actorDefinitionId)) + .withTombstone(record.get(tombstone)) + .withName(record.get(name)); + + Assertions.assertTrue(expectedDefinitions.contains(destinationConnection)); + Assertions.assertEquals(now(), record.get(createdAt).toInstant()); + Assertions.assertEquals(now(), record.get(updatedAt).toInstant()); + } + } + + private void assertDataForSourceOauthParams(final DSLContext context) { + final Field id = DSL.field("id", SQLDataType.UUID.nullable(false)); + final Field actorDefinitionId = DSL.field("actor_definition_id", SQLDataType.UUID.nullable(false)); + final Field configuration = DSL.field("configuration", SQLDataType.JSONB.nullable(false)); + final Field workspaceId = DSL.field("workspace_id", SQLDataType.UUID.nullable(true)); + final Field actorType = DSL.field("actor_type", SQLDataType.VARCHAR.asEnumDataType(ActorType.class).nullable(false)); + final Field createdAt = DSL.field("created_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false)); + final Field updatedAt = DSL.field("updated_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false)); + + final Result sourceOauthParams = context.select(asterisk()) + .from(table("actor_oauth_parameter")) + .where(actorType.eq(ActorType.source)) + .fetch(); + final List expectedDefinitions = sourceOauthParameters(); + Assertions.assertEquals(expectedDefinitions.size(), sourceOauthParams.size()); + + for (final Record record : sourceOauthParams) { + final SourceOAuthParameter sourceOAuthParameter = new SourceOAuthParameter() + .withOauthParameterId(record.get(id)) + .withConfiguration(Jsons.deserialize(record.get(configuration).data())) + .withWorkspaceId(record.get(workspaceId)) + .withSourceDefinitionId(record.get(actorDefinitionId)); + Assertions.assertTrue(expectedDefinitions.contains(sourceOAuthParameter)); + Assertions.assertEquals(now(), record.get(createdAt).toInstant()); + Assertions.assertEquals(now(), record.get(updatedAt).toInstant()); + } + } + + private void assertDataForDestinationOauthParams(final DSLContext context) { + final Field id = DSL.field("id", SQLDataType.UUID.nullable(false)); + final Field actorDefinitionId = DSL.field("actor_definition_id", SQLDataType.UUID.nullable(false)); + final Field configuration = DSL.field("configuration", SQLDataType.JSONB.nullable(false)); + final Field workspaceId = DSL.field("workspace_id", SQLDataType.UUID.nullable(true)); + final Field actorType = DSL.field("actor_type", SQLDataType.VARCHAR.asEnumDataType(ActorType.class).nullable(false)); + final Field createdAt = DSL.field("created_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false)); + final Field updatedAt = DSL.field("updated_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false)); + + final Result destinationOauthParams = context.select(asterisk()) + .from(table("actor_oauth_parameter")) + .where(actorType.eq(ActorType.destination)) + .fetch(); + final List expectedDefinitions = destinationOauthParameters(); + Assertions.assertEquals(expectedDefinitions.size(), destinationOauthParams.size()); + + for (final Record record : destinationOauthParams) { + final DestinationOAuthParameter destinationOAuthParameter = new DestinationOAuthParameter() + .withOauthParameterId(record.get(id)) + .withConfiguration(Jsons.deserialize(record.get(configuration).data())) + .withWorkspaceId(record.get(workspaceId)) + .withDestinationDefinitionId(record.get(actorDefinitionId)); + Assertions.assertTrue(expectedDefinitions.contains(destinationOAuthParameter)); + Assertions.assertEquals(now(), record.get(createdAt).toInstant()); + Assertions.assertEquals(now(), record.get(updatedAt).toInstant()); + } + } + + private void assertDataForOperations(final DSLContext context) { + final Field id = DSL.field("id", SQLDataType.UUID.nullable(false)); + final Field workspaceId = DSL.field("workspace_id", SQLDataType.UUID.nullable(false)); + final Field name = DSL.field("name", SQLDataType.VARCHAR(256).nullable(false)); + final Field operatorType = DSL.field("operator_type", SQLDataType.VARCHAR.asEnumDataType(OperatorType.class).nullable(false)); + final Field operatorNormalization = DSL.field("operator_normalization", SQLDataType.JSONB.nullable(true)); + final Field operatorDbt = DSL.field("operator_dbt", SQLDataType.JSONB.nullable(true)); + final Field tombstone = DSL.field("tombstone", SQLDataType.BOOLEAN.nullable(true)); + final Field createdAt = DSL.field("created_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false)); + final Field updatedAt = DSL.field("updated_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false)); + + final Result standardSyncOperations = context.select(asterisk()) + .from(table("operation")) + .fetch(); + final List expectedDefinitions = standardSyncOperations(); + Assertions.assertEquals(expectedDefinitions.size(), standardSyncOperations.size()); + + for (final Record record : standardSyncOperations) { + final StandardSyncOperation standardSyncOperation = new StandardSyncOperation() + .withOperationId(record.get(id)) + .withName(record.get(name)) + .withWorkspaceId(record.get(workspaceId)) + .withOperatorType(Enums.toEnum(record.get(operatorType, String.class), StandardSyncOperation.OperatorType.class).orElseThrow()) + .withOperatorNormalization(Jsons.deserialize(record.get(operatorNormalization).data(), OperatorNormalization.class)) + .withOperatorDbt(Jsons.deserialize(record.get(operatorDbt).data(), OperatorDbt.class)) + .withTombstone(record.get(tombstone)); + + Assertions.assertTrue(expectedDefinitions.contains(standardSyncOperation)); + Assertions.assertEquals(now(), record.get(createdAt).toInstant()); + Assertions.assertEquals(now(), record.get(updatedAt).toInstant()); + } + } + + private void assertDataForConnections(final DSLContext context) { + final Field id = DSL.field("id", SQLDataType.UUID.nullable(false)); + final Field namespaceDefinition = DSL + .field("namespace_definition", SQLDataType.VARCHAR.asEnumDataType(NamespaceDefinitionType.class).nullable(false)); + final Field namespaceFormat = DSL.field("namespace_format", SQLDataType.VARCHAR(256).nullable(true)); + final Field prefix = DSL.field("prefix", SQLDataType.VARCHAR(256).nullable(true)); + final Field sourceId = DSL.field("source_id", SQLDataType.UUID.nullable(false)); + final Field destinationId = DSL.field("destination_id", SQLDataType.UUID.nullable(false)); + final Field name = DSL.field("name", SQLDataType.VARCHAR(256).nullable(false)); + final Field catalog = DSL.field("catalog", SQLDataType.JSONB.nullable(false)); + final Field status = DSL.field("status", SQLDataType.VARCHAR.asEnumDataType(StatusType.class).nullable(true)); + final Field schedule = DSL.field("schedule", SQLDataType.JSONB.nullable(true)); + final Field manual = DSL.field("manual", SQLDataType.BOOLEAN.nullable(false)); + final Field resourceRequirements = DSL.field("resource_requirements", SQLDataType.JSONB.nullable(true)); + final Field createdAt = DSL.field("created_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false)); + final Field updatedAt = DSL.field("updated_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false)); + + final Result standardSyncs = context.select(asterisk()) + .from(table("connection")) + .fetch(); + final List expectedStandardSyncs = standardSyncs(); + Assertions.assertEquals(expectedStandardSyncs.size(), standardSyncs.size()); + + for (final Record record : standardSyncs) { + final StandardSync standardSync = new StandardSync() + .withConnectionId(record.get(id)) + .withNamespaceDefinition( + Enums.toEnum(record.get(namespaceDefinition, String.class), io.airbyte.config.JobSyncConfig.NamespaceDefinitionType.class) + .orElseThrow()) + .withNamespaceFormat(record.get(namespaceFormat)) + .withPrefix(record.get(prefix)) + .withSourceId(record.get(sourceId)) + .withDestinationId(record.get(destinationId)) + .withName(record.get(name)) + .withCatalog(Jsons.deserialize(record.get(catalog).data(), ConfiguredAirbyteCatalog.class)) + .withStatus(Enums.toEnum(record.get(status, String.class), StandardSync.Status.class).orElseThrow()) + .withSchedule(Jsons.deserialize(record.get(schedule).data(), Schedule.class)) + .withManual(record.get(manual)) + .withOperationIds(connectionOperationIds(record.get(id), context)) + .withResourceRequirements(Jsons.deserialize(record.get(resourceRequirements).data(), ResourceRequirements.class)); + + Assertions.assertTrue(expectedStandardSyncs.contains(standardSync)); + Assertions.assertEquals(now(), record.get(createdAt).toInstant()); + Assertions.assertEquals(now(), record.get(updatedAt).toInstant()); + } + } + + private List connectionOperationIds(final UUID connectionIdTo, final DSLContext context) { + final Field id = DSL.field("id", SQLDataType.UUID.nullable(false)); + final Field connectionId = DSL.field("connection_id", SQLDataType.UUID.nullable(false)); + final Field operationId = DSL.field("operation_id", SQLDataType.UUID.nullable(false)); + final Field createdAt = DSL.field("created_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false)); + final Field updatedAt = DSL.field("updated_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false)); + + final Result connectionOperations = context.select(asterisk()) + .from(table("connection_operation")) + .where(connectionId.eq(connectionIdTo)) + .fetch(); + + final List ids = new ArrayList<>(); + + for (Record record : connectionOperations) { + ids.add(record.get(operationId)); + Assertions.assertNotNull(record.get(id)); + Assertions.assertEquals(now(), record.get(createdAt).toInstant()); + Assertions.assertEquals(now(), record.get(updatedAt).toInstant()); + } + + return ids; + } + + private void assertDataForStandardSyncStates(final DSLContext context) { + final Field id = DSL.field("id", SQLDataType.UUID.nullable(false)); + final Field connectionId = DSL.field("connection_id", SQLDataType.UUID.nullable(false)); + final Field state = DSL.field("state", SQLDataType.JSONB.nullable(true)); + final Field createdAt = DSL.field("created_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false)); + final Field updatedAt = DSL.field("updated_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false)); + + final Result standardSyncStates = context.select(asterisk()) + .from(table("state")) + .fetch(); + final List expectedStandardSyncsStates = standardSyncStates(); + Assertions.assertEquals(expectedStandardSyncsStates.size(), standardSyncStates.size()); + + for (final Record record : standardSyncStates) { + final StandardSyncState standardSyncState = new StandardSyncState() + .withConnectionId(record.get(connectionId)) + .withState(Jsons.deserialize(record.get(state).data(), State.class)); + + Assertions.assertTrue(expectedStandardSyncsStates.contains(standardSyncState)); + Assertions.assertNotNull(record.get(id)); + Assertions.assertEquals(now(), record.get(createdAt).toInstant()); + Assertions.assertEquals(now(), record.get(updatedAt).toInstant()); + } + } + +} diff --git a/airbyte-db/lib/src/test/java/io/airbyte/db/instance/toys/ToysDatabaseMigrator.java b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/toys/ToysDatabaseMigrator.java index 3e8e3be15460..a77eed3e809f 100644 --- a/airbyte-db/lib/src/test/java/io/airbyte/db/instance/toys/ToysDatabaseMigrator.java +++ b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/toys/ToysDatabaseMigrator.java @@ -19,4 +19,9 @@ public ToysDatabaseMigrator(final Database database, final String migrationRunne super(database, DB_IDENTIFIER, migrationRunner, MIGRATION_FILE_LOCATION); } + @Override + protected String getDisclaimer() { + return ""; + } + } diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/ArchiveHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/ArchiveHandlerTest.java index 5e3b4ca0d9c0..7c451633efcd 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/ArchiveHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/ArchiveHandlerTest.java @@ -22,8 +22,12 @@ import io.airbyte.commons.version.AirbyteVersion; import io.airbyte.config.ConfigSchema; import io.airbyte.config.DestinationConnection; +import io.airbyte.config.Notification; +import io.airbyte.config.Notification.NotificationType; +import io.airbyte.config.SlackNotificationConfiguration; import io.airbyte.config.SourceConnection; import io.airbyte.config.StandardSourceDefinition; +import io.airbyte.config.StandardSourceDefinition.SourceType; import io.airbyte.config.StandardWorkspace; import io.airbyte.config.init.YamlSeedConfigPersistence; import io.airbyte.config.persistence.ConfigPersistence; @@ -160,15 +164,36 @@ void testFullExportImportRoundTrip() throws Exception { StandardSourceDefinition.class) // This source definition is on an old version .withDockerImageTag(sourceS3DefinitionVersion); + final Notification notification = new Notification() + .withNotificationType(NotificationType.SLACK) + .withSendOnFailure(true) + .withSendOnSuccess(true) + .withSlackConfiguration(new SlackNotificationConfiguration().withWebhook("webhook-url")); + final StandardWorkspace standardWorkspace = new StandardWorkspace() + .withWorkspaceId(UUID.randomUUID()) + .withCustomerId(UUID.randomUUID()) + .withName("test-workspace") + .withSlug("random-string") + .withEmail("abc@xyz.com") + .withInitialSetupComplete(true) + .withAnonymousDataCollection(true) + .withNews(true) + .withSecurityUpdates(true) + .withDisplaySetupWizard(true) + .withTombstone(false) + .withNotifications(Collections.singletonList(notification)) + .withFirstCompletedSync(true) + .withFeedbackDone(true); final SourceConnection sourceConnection = new SourceConnection() .withSourceDefinitionId(sourceS3DefinitionId) .withSourceId(UUID.randomUUID()) - .withWorkspaceId(UUID.randomUUID()) + .withWorkspaceId(standardWorkspace.getWorkspaceId()) .withName("Test source") .withConfiguration(Jsons.deserialize("{}")) .withTombstone(false); // Write source connection and an old source definition. + configPersistence.writeConfig(ConfigSchema.STANDARD_WORKSPACE, standardWorkspace.getWorkspaceId().toString(), standardWorkspace); configPersistence.writeConfig(ConfigSchema.SOURCE_CONNECTION, sourceConnection.getSourceId().toString(), sourceConnection); configPersistence.writeConfig(ConfigSchema.STANDARD_SOURCE_DEFINITION, sourceS3DefinitionId.toString(), sourceS3Definition); @@ -244,17 +269,28 @@ void testLightWeightExportImportRoundTrip() throws Exception { .map(SourceConnection::getSourceId) .collect(Collectors.toList()).get(0); + final StandardSourceDefinition standardSourceDefinition = new StandardSourceDefinition() + .withSourceDefinitionId(UUID.randomUUID()) + .withSourceType(SourceType.API) + .withName("random-source-1") + .withDockerImageTag("tag-1") + .withDockerRepository("repository-1") + .withDocumentationUrl("documentation-url-1") + .withIcon("icon-1") + .withSpec(new ConnectorSpecification()); + final SourceConnection sourceConnection = new SourceConnection() .withWorkspaceId(secondWorkspaceId) .withSourceId(secondSourceId) .withName("Some new names") - .withSourceDefinitionId(UUID.randomUUID()) + .withSourceDefinitionId(standardSourceDefinition.getSourceDefinitionId()) .withTombstone(false) .withConfiguration(Jsons.emptyObject()); final ConnectorSpecification emptyConnectorSpec = mock(ConnectorSpecification.class); when(emptyConnectorSpec.getConnectionSpecification()).thenReturn(Jsons.emptyObject()); + configRepository.writeStandardSourceDefinition(standardSourceDefinition); configRepository.writeSourceConnection(sourceConnection, emptyConnectorSpec); // check that first workspace is unchanged even though modifications were made to second workspace @@ -272,10 +308,12 @@ void testLightWeightExportImportRoundTrip() throws Exception { assertSameConfigDump(secondWorkspaceDump, configRepository.dumpConfigs()); } - private void setupWorkspaceData(final UUID workspaceId) throws IOException { + private void setupWorkspaceData(final UUID workspaceId) throws IOException, JsonValidationException { configPersistence.writeConfig(ConfigSchema.STANDARD_WORKSPACE, workspaceId.toString(), new StandardWorkspace() .withWorkspaceId(workspaceId) .withName("test-workspace") + .withSlug(workspaceId.toString()) + .withInitialSetupComplete(false) .withTombstone(false)); }