From 79bc44342fc1afa1bc33668ed6d946ffda89a511 Mon Sep 17 00:00:00 2001 From: annie-mac Date: Mon, 25 Mar 2024 14:14:10 -0700 Subject: [PATCH] Revert "KafkaV2SinkConnector (#38973)" This reverts commit 6c983ab4f67008d8203fb12638f02f944ee358c5. --- .../checkstyle/checkstyle-suppressions.xml | 1 - eng/versioning/external_dependencies.txt | 1 - .../azure-cosmos-kafka-connect/CHANGELOG.md | 1 - .../doc/configuration-reference.md | 13 - sdk/cosmos/azure-cosmos-kafka-connect/pom.xml | 25 +- .../kafka/connect/CosmosDBSinkConnector.java | 63 -- .../connect/CosmosDBSourceConnector.java | 20 +- .../implementation/CosmosClientStore.java | 4 +- ...fkaCosmosConfig.java => CosmosConfig.java} | 53 +- ...mosConstants.java => CosmosConstants.java} | 2 +- ...elper.java => CosmosExceptionsHelper.java} | 36 +- .../implementation/KafkaCosmosSchedulers.java | 19 - .../sink/CosmosDBWriteException.java | 20 - .../implementation/sink/CosmosSinkConfig.java | 325 ---------- .../sink/CosmosSinkContainersConfig.java | 24 - .../implementation/sink/CosmosSinkTask.java | 95 --- .../sink/CosmosSinkTaskConfig.java | 13 - .../sink/CosmosSinkWriteConfig.java | 54 -- .../connect/implementation/sink/IWriter.java | 13 - .../implementation/sink/IdStrategies.java | 31 - .../sink/ItemWriteStrategy.java | 32 - .../sink/KafkaCosmosBulkWriter.java | 335 ---------- .../sink/KafkaCosmosPointWriter.java | 179 ------ .../sink/KafkaCosmosWriterBase.java | 121 ---- .../implementation/sink/SinkOperation.java | 64 -- .../sink/SinkRecordTransformer.java | 110 ---- .../implementation/sink/StructToJsonMap.java | 118 ---- .../sink/ToleranceOnErrorLevel.java | 28 - .../sink/idstrategy/AbstractIdStrategy.java | 27 - .../idstrategy/AbstractIdStrategyConfig.java | 19 - .../sink/idstrategy/FullKeyStrategy.java | 16 - .../sink/idstrategy/IdStrategy.java | 11 - .../idstrategy/KafkaMetadataStrategy.java | 22 - .../KafkaMetadataStrategyConfig.java | 53 -- .../sink/idstrategy/ProvidedInConfig.java | 53 -- .../idstrategy/ProvidedInKeyStrategy.java | 10 - .../sink/idstrategy/ProvidedInStrategy.java | 46 -- .../idstrategy/ProvidedInValueStrategy.java | 10 - .../sink/idstrategy/TemplateStrategy.java | 71 --- .../idstrategy/TemplateStrategyConfig.java | 57 -- .../source/CosmosChangeFeedModes.java | 2 +- .../CosmosChangeFeedStartFromModes.java | 2 +- .../source/CosmosSourceConfig.java | 190 +++--- .../source/CosmosSourceContainersConfig.java | 11 +- .../source/CosmosSourceTask.java | 10 +- .../source/MetadataMonitorThread.java | 4 +- .../src/main/java/module-info.java | 1 - .../connect/CosmosDBSinkConnectorTest.java | 154 ----- .../connect/CosmosDBSourceConnectorTest.java | 70 ++- .../connect/CosmosDbSinkConnectorITest.java | 99 --- .../kafka/connect/KafkaCosmosConfigEntry.java | 29 - .../connect/KafkaCosmosConnectContainer.java | 39 +- .../connect/KafkaCosmosReflectionUtils.java | 10 - .../connect/KafkaCosmosTestSuiteBase.java | 16 +- .../sink/CosmosSinkTaskTest.java | 571 ------------------ .../idStrategy/ProvidedInStrategyTest.java | 223 ------- .../source/MetadataMonitorThreadTest.java | 6 +- .../azure/cosmos/CosmosAsyncContainer.java | 5 - .../ImplementationBridgeHelpers.java | 2 - 59 files changed, 204 insertions(+), 3435 deletions(-) delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/CosmosDBSinkConnector.java rename sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/{KafkaCosmosConfig.java => CosmosConfig.java} (81%) rename sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/{KafkaCosmosConstants.java => CosmosConstants.java} (94%) rename sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/{KafkaCosmosExceptionsHelper.java => CosmosExceptionsHelper.java} (63%) delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/KafkaCosmosSchedulers.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosDBWriteException.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosSinkConfig.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosSinkContainersConfig.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosSinkTask.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosSinkTaskConfig.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosSinkWriteConfig.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/IWriter.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/IdStrategies.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/ItemWriteStrategy.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/KafkaCosmosBulkWriter.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/KafkaCosmosPointWriter.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/KafkaCosmosWriterBase.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/SinkOperation.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/SinkRecordTransformer.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/StructToJsonMap.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/ToleranceOnErrorLevel.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/AbstractIdStrategy.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/AbstractIdStrategyConfig.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/FullKeyStrategy.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/IdStrategy.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/KafkaMetadataStrategy.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/KafkaMetadataStrategyConfig.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/ProvidedInConfig.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/ProvidedInKeyStrategy.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/ProvidedInStrategy.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/ProvidedInValueStrategy.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/TemplateStrategy.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/TemplateStrategyConfig.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/CosmosDBSinkConnectorTest.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/CosmosDbSinkConnectorITest.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/KafkaCosmosConfigEntry.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosSinkTaskTest.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/implementation/sink/idStrategy/ProvidedInStrategyTest.java diff --git a/eng/code-quality-reports/src/main/resources/checkstyle/checkstyle-suppressions.xml b/eng/code-quality-reports/src/main/resources/checkstyle/checkstyle-suppressions.xml index b11579484fd0a..db3789c74a934 100755 --- a/eng/code-quality-reports/src/main/resources/checkstyle/checkstyle-suppressions.xml +++ b/eng/code-quality-reports/src/main/resources/checkstyle/checkstyle-suppressions.xml @@ -319,7 +319,6 @@ the main ServiceBusClientBuilder. --> - diff --git a/eng/versioning/external_dependencies.txt b/eng/versioning/external_dependencies.txt index e879fca40aafd..183fb2585631f 100644 --- a/eng/versioning/external_dependencies.txt +++ b/eng/versioning/external_dependencies.txt @@ -394,7 +394,6 @@ cosmos_org.scalastyle:scalastyle-maven-plugin;1.0.0 ## Cosmos Kafka connector under sdk\cosmos\azure-cosmos-kafka-connect\pom.xml # Cosmos Kafka connector runtime dependencies cosmos_org.apache.kafka:connect-api;3.6.0 -cosmos_com.jayway.jsonpath:json-path;2.9.0 # Cosmos Kafka connector tests only cosmos_org.apache.kafka:connect-runtime;3.6.0 cosmos_org.testcontainers:testcontainers;1.19.5 diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/CHANGELOG.md b/sdk/cosmos/azure-cosmos-kafka-connect/CHANGELOG.md index 840ffe29ca47f..4797133def4bc 100644 --- a/sdk/cosmos/azure-cosmos-kafka-connect/CHANGELOG.md +++ b/sdk/cosmos/azure-cosmos-kafka-connect/CHANGELOG.md @@ -4,7 +4,6 @@ #### Features Added * Added Source connector. See [PR 38748](https://github.com/Azure/azure-sdk-for-java/pull/38748) -* Added Sink connector. See [PR 38973](https://github.com/Azure/azure-sdk-for-java/pull/38973) #### Breaking Changes diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/doc/configuration-reference.md b/sdk/cosmos/azure-cosmos-kafka-connect/doc/configuration-reference.md index 064aaab81a76e..8512ba8d7af79 100644 --- a/sdk/cosmos/azure-cosmos-kafka-connect/doc/configuration-reference.md +++ b/sdk/cosmos/azure-cosmos-kafka-connect/doc/configuration-reference.md @@ -23,16 +23,3 @@ | `kafka.connect.cosmos.source.metadata.storage.topic` | `_cosmos.metadata.topic` | The name of the topic where the metadata are stored. The metadata topic will be created if it does not already exist, else it will use the pre-created topic. | | `kafka.connect.cosmos.source.messageKey.enabled` | `true` | Whether to set the kafka record message key. | | `kafka.connect.cosmos.source.messageKey.field` | `id` | The field to use as the message key. | - -## Sink Connector Configuration -| Config Property Name | Default | Description | -|:---------------------------------------------------------------|:--------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `kafka.connect.cosmos.sink.database.name` | None | Cosmos DB database name. | -| `kafka.connect.cosmos.sink.containers.topicMap` | None | A comma delimited list of Kafka topics mapped to Cosmos containers. For example: topic1#con1,topic2#con2. | -| `kafka.connect.cosmos.sink.errors.tolerance` | `None` | Error tolerance level after exhausting all retries. `None` for fail on error. `All` for log and continue | -| `kafka.connect.cosmos.sink.bulk.enabled` | `true` | Flag to indicate whether Cosmos DB bulk mode is enabled for Sink connector. By default it is true. | -| `kafka.connect.cosmos.sink.bulk.maxConcurrentCosmosPartitions` | `-1` | Cosmos DB Item Write Max Concurrent Cosmos Partitions. If not specified it will be determined based on the number of the container's physical partitions which would indicate every batch is expected to have data from all Cosmos physical partitions. If specified it indicates from at most how many Cosmos Physical Partitions each batch contains data. So this config can be used to make bulk processing more efficient when input data in each batch has been repartitioned to balance to how many Cosmos partitions each batch needs to write. This is mainly useful for very large containers (with hundreds of physical partitions. | -| `kafka.connect.cosmos.sink.bulk.initialBatchSize` | `1` | Cosmos DB initial bulk micro batch size - a micro batch will be flushed to the backend when the number of documents enqueued exceeds this size - or the target payload size is met. The micro batch size is getting automatically tuned based on the throttling rate. By default the initial micro batch size is 1. Reduce this when you want to avoid that the first few requests consume too many RUs. | -| `kafka.connect.cosmos.sink.write.strategy` | `ItemOverwrite` | Cosmos DB Item write Strategy: `ItemOverwrite` (using upsert), `ItemAppend` (using create, ignore pre-existing items i.e., Conflicts), `ItemDelete` (deletes based on id/pk of data frame), `ItemDeleteIfNotModified` (deletes based on id/pk of data frame if etag hasn't changed since collecting id/pk), `ItemOverwriteIfNotModified` (using create if etag is empty, update/replace with etag pre-condition otherwise, if document was updated the pre-condition failure is ignored) | -| `kafka.connect.cosmos.sink.maxRetryCount` | `10` | Cosmos DB max retry attempts on write failures for Sink connector. By default, the connector will retry on transient write errors for up to 10 times. | -| `kafka.connect.cosmos.sink.id.strategy` | `ProvidedInValueStrategy` | A strategy used to populate the document with an ``id``. Valid strategies are: ``TemplateStrategy``, ``FullKeyStrategy``, ``KafkaMetadataStrategy``, ``ProvidedInKeyStrategy``, ``ProvidedInValueStrategy``. Configuration properties prefixed with``id.strategy`` are passed through to the strategy. For example, when using ``id.strategy=TemplateStrategy`` , the property ``id.strategy.template`` is passed through to the template strategy and used to specify the template string to be used in constructing the ``id``. | diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/pom.xml b/sdk/cosmos/azure-cosmos-kafka-connect/pom.xml index e97f4ae4ebde5..440b5e7b0dbf3 100644 --- a/sdk/cosmos/azure-cosmos-kafka-connect/pom.xml +++ b/sdk/cosmos/azure-cosmos-kafka-connect/pom.xml @@ -47,14 +47,8 @@ Licensed under the MIT License. --add-opens com.azure.cosmos.kafka.connect/com.azure.cosmos.kafka.connect=ALL-UNNAMED --add-opens com.azure.cosmos.kafka.connect/com.azure.cosmos.kafka.connect.implementation=ALL-UNNAMED --add-opens com.azure.cosmos.kafka.connect/com.azure.cosmos.kafka.connect.implementation.source=com.fasterxml.jackson.databind,ALL-UNNAMED - --add-opens com.azure.cosmos.kafka.connect/com.azure.cosmos.kafka.connect.implementation.sink=ALL-UNNAMED - --add-opens com.azure.cosmos.kafka.connect/com.azure.cosmos.kafka.connect.implementation.sink.idStrategy=ALL-UNNAMED - --add-opens com.azure.cosmos/com.azure.cosmos.implementation=ALL-UNNAMED --add-opens com.azure.cosmos/com.azure.cosmos.implementation.routing=ALL-UNNAMED - --add-opens com.azure.cosmos/com.azure.cosmos.implementation.caches=ALL-UNNAMED - --add-opens com.azure.cosmos/com.azure.cosmos.implementation.faultinjection=ALL-UNNAMED --add-opens com.azure.cosmos/com.azure.cosmos.implementation.apachecommons.lang=ALL-UNNAMED - --add-opens com.azure.cosmos/com.azure.cosmos.implementation.guava25.base=ALL-UNNAMED @@ -86,13 +80,6 @@ Licensed under the MIT License. provided - - com.azure - azure-cosmos-test - 1.0.0-beta.7 - test - - org.apache.commons commons-collections4 @@ -106,11 +93,6 @@ Licensed under the MIT License. test 1.10.0 - - com.jayway.jsonpath - json-path - 2.9.0 - org.apache.kafka @@ -253,7 +235,6 @@ Licensed under the MIT License. com.azure:* org.apache.kafka:connect-api:[3.6.0] io.confluent:kafka-connect-maven-plugin:[0.12.0] - com.jayway.jsonpath:json-path:[2.9.0] org.sourcelab:kafka-connect-client:[4.0.4] @@ -338,10 +319,6 @@ Licensed under the MIT License. reactor ${shadingPrefix}.reactor - - com.jayway.jsonpath - ${shadingPrefix}.com.jayway.jsonpath - @@ -479,7 +456,7 @@ Licensed under the MIT License. - kafka + kafka-integration kafka diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/CosmosDBSinkConnector.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/CosmosDBSinkConnector.java deleted file mode 100644 index 91319959ba691..0000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/CosmosDBSinkConnector.java +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect; - -import com.azure.cosmos.kafka.connect.implementation.KafkaCosmosConstants; -import com.azure.cosmos.kafka.connect.implementation.sink.CosmosSinkConfig; -import com.azure.cosmos.kafka.connect.implementation.sink.CosmosSinkTask; -import org.apache.kafka.common.config.ConfigDef; -import org.apache.kafka.connect.connector.Task; -import org.apache.kafka.connect.sink.SinkConnector; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -/** - * A Sink connector that publishes topic messages to CosmosDB. - */ -public class CosmosDBSinkConnector extends SinkConnector { - private static final Logger LOGGER = LoggerFactory.getLogger(CosmosDBSinkConnector.class); - - private CosmosSinkConfig sinkConfig; - - @Override - public void start(Map props) { - LOGGER.info("Starting the kafka cosmos sink connector"); - this.sinkConfig = new CosmosSinkConfig(props); - } - - @Override - public Class taskClass() { - return CosmosSinkTask.class; - } - - @Override - public List> taskConfigs(int maxTasks) { - LOGGER.info("Setting task configurations with maxTasks {}", maxTasks); - List> configs = new ArrayList<>(); - for (int i = 0; i < maxTasks; i++) { - configs.add(this.sinkConfig.originalsStrings()); - } - - return configs; - } - - @Override - public void stop() { - LOGGER.debug("Kafka Cosmos sink connector {} is stopped."); - } - - @Override - public ConfigDef config() { - return CosmosSinkConfig.getConfigDef(); - } - - @Override - public String version() { - return KafkaCosmosConstants.CURRENT_VERSION; - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/CosmosDBSourceConnector.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/CosmosDBSourceConnector.java index 4bbb794a5906a..8545b65ed2029 100644 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/CosmosDBSourceConnector.java +++ b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/CosmosDBSourceConnector.java @@ -14,8 +14,8 @@ import com.azure.cosmos.implementation.query.CompositeContinuationToken; import com.azure.cosmos.implementation.routing.Range; import com.azure.cosmos.kafka.connect.implementation.CosmosClientStore; -import com.azure.cosmos.kafka.connect.implementation.KafkaCosmosConstants; -import com.azure.cosmos.kafka.connect.implementation.KafkaCosmosExceptionsHelper; +import com.azure.cosmos.kafka.connect.implementation.CosmosConstants; +import com.azure.cosmos.kafka.connect.implementation.CosmosExceptionsHelper; import com.azure.cosmos.kafka.connect.implementation.source.CosmosSourceConfig; import com.azure.cosmos.kafka.connect.implementation.source.CosmosSourceOffsetStorageReader; import com.azure.cosmos.kafka.connect.implementation.source.CosmosSourceTask; @@ -102,8 +102,8 @@ public ConfigDef config() { @Override public String version() { - return KafkaCosmosConstants.CURRENT_VERSION; - } + return CosmosConstants.CURRENT_VERSION; + } // TODO[public preview]: how this is being used private List> getTaskConfigs(int maxTasks) { Pair> taskUnits = this.getAllTaskUnits(); @@ -314,7 +314,7 @@ private List> getFeedRanges(CosmosContainerProperties containerPro .getContainer(containerProperties.getId()) .getFeedRanges() .onErrorMap(throwable -> - KafkaCosmosExceptionsHelper.convertToConnectException( + CosmosExceptionsHelper.convertToConnectException( throwable, "GetFeedRanges failed for container " + containerProperties.getId())) .block() @@ -324,7 +324,15 @@ private List> getFeedRanges(CosmosContainerProperties containerPro } private Map getContainersTopicMap(List allContainers) { - Map topicMapFromConfig = this.config.getContainersConfig().getContainerToTopicMap(); + Map topicMapFromConfig = + this.config.getContainersConfig().getContainersTopicMap() + .stream() + .map(containerTopicMapString -> containerTopicMapString.split("#")) + .collect( + Collectors.toMap( + containerTopicMapArray -> containerTopicMapArray[1], + containerTopicMapArray -> containerTopicMapArray[0])); + Map effectiveContainersTopicMap = new HashMap<>(); allContainers.forEach(containerProperties -> { // by default, we are using container id as the topic name as well unless customer override through containers.topicMap diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/CosmosClientStore.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/CosmosClientStore.java index 10589369d0e8d..40812a54500ca 100644 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/CosmosClientStore.java +++ b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/CosmosClientStore.java @@ -36,9 +36,9 @@ public static CosmosAsyncClient getCosmosClient(CosmosAccountConfig accountConfi private static String getUserAgentSuffix(CosmosAccountConfig accountConfig) { if (StringUtils.isNotEmpty(accountConfig.getApplicationName())) { - return KafkaCosmosConstants.USER_AGENT_SUFFIX + "|" + accountConfig.getApplicationName(); + return CosmosConstants.USER_AGENT_SUFFIX + "|" + accountConfig.getApplicationName(); } - return KafkaCosmosConstants.USER_AGENT_SUFFIX; + return CosmosConstants.USER_AGENT_SUFFIX; } } diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/KafkaCosmosConfig.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/CosmosConfig.java similarity index 81% rename from sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/KafkaCosmosConfig.java rename to sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/CosmosConfig.java index 5c75056b6f7be..4aecdff89c87a 100644 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/KafkaCosmosConfig.java +++ b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/CosmosConfig.java @@ -5,7 +5,6 @@ import com.azure.cosmos.implementation.Strings; import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; -import com.azure.cosmos.kafka.connect.implementation.source.CosmosSourceContainersConfig; import org.apache.kafka.common.config.AbstractConfig; import org.apache.kafka.common.config.ConfigDef; import org.apache.kafka.common.config.ConfigException; @@ -21,7 +20,7 @@ /** * Common Configuration for Cosmos DB Kafka source connector and sink connector. */ -public class KafkaCosmosConfig extends AbstractConfig { +public class CosmosConfig extends AbstractConfig { protected static final ConfigDef.Validator NON_EMPTY_STRING = new ConfigDef.NonEmptyString(); private static final String CONFIG_PREFIX = "kafka.connect.cosmos."; @@ -52,7 +51,7 @@ public class KafkaCosmosConfig extends AbstractConfig { private final CosmosAccountConfig accountConfig; - public KafkaCosmosConfig(ConfigDef config, Map parsedConfig) { + public CosmosConfig(ConfigDef config, Map parsedConfig) { super(config, parsedConfig); this.accountConfig = this.parseAccountConfig(); } @@ -152,18 +151,6 @@ public CosmosAccountConfig getAccountConfig() { return accountConfig; } - protected static List convertToList(String configValue) { - if (StringUtils.isNotEmpty(configValue)) { - if (configValue.startsWith("[") && configValue.endsWith("]")) { - configValue = configValue.substring(1, configValue.length() - 1); - } - - return Arrays.stream(configValue.split(",")).map(String::trim).collect(Collectors.toList()); - } - - return new ArrayList<>(); - } - public static class AccountEndpointValidator implements ConfigDef.Validator { @Override @SuppressWarnings("unchecked") @@ -186,39 +173,15 @@ public String toString() { } } - public static class ContainersTopicMapValidator implements ConfigDef.Validator { - private static final String INVALID_TOPIC_MAP_FORMAT = - "Invalid entry for topic-container map. The topic-container map should be a comma-delimited " - + "list of Kafka topic to Cosmos containers. Each mapping should be a pair of Kafka " - + "topic and Cosmos container separated by '#'. For example: topic1#con1,topic2#con2."; - - @Override - @SuppressWarnings("unchecked") - public void ensureValid(String name, Object o) { - String configValue = (String) o; - if (StringUtils.isEmpty(configValue)) { - return; + protected static List convertToList(String configValue) { + if (StringUtils.isNotEmpty(configValue)) { + if (configValue.startsWith("[") && configValue.endsWith("]")) { + configValue = configValue.substring(1, configValue.length() - 1); } - List containerTopicMapList = convertToList(configValue); - - // validate each item should be in topic#container format - boolean invalidFormatExists = - containerTopicMapList - .stream() - .anyMatch(containerTopicMap -> - containerTopicMap - .split(CosmosSourceContainersConfig.CONTAINER_TOPIC_MAP_SEPARATOR) - .length != 2); - - if (invalidFormatExists) { - throw new ConfigException(name, o, INVALID_TOPIC_MAP_FORMAT); - } + return Arrays.stream(configValue.split(",")).map(String::trim).collect(Collectors.toList()); } - @Override - public String toString() { - return "Containers topic map"; - } + return new ArrayList<>(); } } diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/KafkaCosmosConstants.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/CosmosConstants.java similarity index 94% rename from sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/KafkaCosmosConstants.java rename to sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/CosmosConstants.java index 01a89ef20f532..afeac46866b95 100644 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/KafkaCosmosConstants.java +++ b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/CosmosConstants.java @@ -5,7 +5,7 @@ import com.azure.core.util.CoreUtils; -public class KafkaCosmosConstants { +public class CosmosConstants { public static final String PROPERTIES_FILE_NAME = "azure-cosmos-kafka-connect.properties"; public static final String CURRENT_VERSION = CoreUtils.getProperties(PROPERTIES_FILE_NAME).get("version"); public static final String CURRENT_NAME = CoreUtils.getProperties(PROPERTIES_FILE_NAME).get("name"); diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/KafkaCosmosExceptionsHelper.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/CosmosExceptionsHelper.java similarity index 63% rename from sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/KafkaCosmosExceptionsHelper.java rename to sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/CosmosExceptionsHelper.java index 0d5a8bb8759e3..7424128828408 100644 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/KafkaCosmosExceptionsHelper.java +++ b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/CosmosExceptionsHelper.java @@ -8,7 +8,7 @@ import org.apache.kafka.connect.errors.ConnectException; import org.apache.kafka.connect.errors.RetriableException; -public class KafkaCosmosExceptionsHelper { +public class CosmosExceptionsHelper { public static boolean isTransientFailure(int statusCode, int substatusCode) { return statusCode == HttpConstants.StatusCodes.GONE || statusCode == HttpConstants.StatusCodes.SERVICE_UNAVAILABLE @@ -43,42 +43,10 @@ public static boolean isFeedRangeGoneException(int statusCode, int substatusCode } public static ConnectException convertToConnectException(Throwable throwable, String message) { - if (KafkaCosmosExceptionsHelper.isTransientFailure(throwable)) { + if (CosmosExceptionsHelper.isTransientFailure(throwable)) { return new RetriableException(message, throwable); } return new ConnectException(message, throwable); } - - public static boolean isResourceExistsException(Throwable throwable) { - if (throwable instanceof CosmosException) { - return ((CosmosException) throwable).getStatusCode() == HttpConstants.StatusCodes.CONFLICT; - } - - return false; - } - - public static boolean isNotFoundException(Throwable throwable) { - if (throwable instanceof CosmosException) { - return ((CosmosException) throwable).getStatusCode() == HttpConstants.StatusCodes.NOTFOUND; - } - - return false; - } - - public static boolean isPreconditionFailedException(Throwable throwable) { - if (throwable instanceof CosmosException) { - return ((CosmosException) throwable).getStatusCode() == HttpConstants.StatusCodes.PRECONDITION_FAILED; - } - - return false; - } - - public static boolean isTimeoutException(Throwable throwable) { - if (throwable instanceof CosmosException) { - return ((CosmosException) throwable).getStatusCode() == HttpConstants.StatusCodes.REQUEST_TIMEOUT; - } - - return false; - } } diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/KafkaCosmosSchedulers.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/KafkaCosmosSchedulers.java deleted file mode 100644 index 081d0baa30f2a..0000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/KafkaCosmosSchedulers.java +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation; - -import reactor.core.scheduler.Scheduler; -import reactor.core.scheduler.Schedulers; - -public class KafkaCosmosSchedulers { - private static final String SINK_BOUNDED_ELASTIC_THREAD_NAME = "sink-bounded-elastic"; - private static final int TTL_FOR_SCHEDULER_WORKER_IN_SECONDS = 60; // same as BoundedElasticScheduler.DEFAULT_TTL_SECONDS - public static final Scheduler SINK_BOUNDED_ELASTIC = Schedulers.newBoundedElastic( - Schedulers.DEFAULT_BOUNDED_ELASTIC_SIZE, - Schedulers.DEFAULT_BOUNDED_ELASTIC_QUEUESIZE, - SINK_BOUNDED_ELASTIC_THREAD_NAME, - TTL_FOR_SCHEDULER_WORKER_IN_SECONDS, - true - ); -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosDBWriteException.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosDBWriteException.java deleted file mode 100644 index 69f99f35f8baf..0000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosDBWriteException.java +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink; - -import org.apache.kafka.connect.errors.ConnectException; - -/** - * Generic CosmosDb sink write exceptions. - */ -public class CosmosDBWriteException extends ConnectException { - /** - * - */ - private static final long serialVersionUID = 1L; - - public CosmosDBWriteException(String message) { - super(message); - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosSinkConfig.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosSinkConfig.java deleted file mode 100644 index d029105846a56..0000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosSinkConfig.java +++ /dev/null @@ -1,325 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink; - -import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; -import com.azure.cosmos.kafka.connect.implementation.KafkaCosmosConfig; -import org.apache.kafka.common.config.ConfigDef; -import org.apache.kafka.common.config.ConfigException; - -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -/** - * Common Configuration for Cosmos DB Kafka sink connector. - */ -public class CosmosSinkConfig extends KafkaCosmosConfig { - private static final String SINK_CONFIG_PREFIX = "kafka.connect.cosmos.sink."; - - // error tolerance - public static final String TOLERANCE_ON_ERROR_CONFIG = SINK_CONFIG_PREFIX + "errors.tolerance"; - public static final String TOLERANCE_ON_ERROR_DOC = - "Error tolerance level after exhausting all retries. 'None' for fail on error. 'All' for log and continue"; - public static final String TOLERANCE_ON_ERROR_DISPLAY = "Error tolerance level."; - public static final String DEFAULT_TOLERANCE_ON_ERROR = ToleranceOnErrorLevel.NONE.getName(); - - // sink bulk config - public static final String BULK_ENABLED_CONF = SINK_CONFIG_PREFIX + "bulk.enabled"; - private static final String BULK_ENABLED_DOC = - "Flag to indicate whether Cosmos DB bulk mode is enabled for Sink connector. By default it is true."; - private static final String BULK_ENABLED_DISPLAY = "enable bulk mode."; - private static final boolean DEFAULT_BULK_ENABLED = true; - - // TODO[Public Preview]: Add other write config, for example patch, bulkUpdate - public static final String BULK_MAX_CONCURRENT_PARTITIONS_CONF = SINK_CONFIG_PREFIX + "bulk.maxConcurrentCosmosPartitions"; - private static final String BULK_MAX_CONCURRENT_PARTITIONS_DOC = - "Cosmos DB Item Write Max Concurrent Cosmos Partitions." - + " If not specified it will be determined based on the number of the container's physical partitions -" - + " which would indicate every batch is expected to have data from all Cosmos physical partitions." - + " If specified it indicates from at most how many Cosmos Physical Partitions each batch contains data." - + " So this config can be used to make bulk processing more efficient when input data in each batch has been" - + " repartitioned to balance to how many Cosmos partitions each batch needs to write. This is mainly" - + " useful for very large containers (with hundreds of physical partitions)."; - private static final String BULK_MAX_CONCURRENT_PARTITIONS_DISPLAY = "Cosmos DB Item Write Max Concurrent Cosmos Partitions."; - private static final int DEFAULT_BULK_MAX_CONCURRENT_PARTITIONS = -1; - - public static final String BULK_INITIAL_BATCH_SIZE_CONF = SINK_CONFIG_PREFIX + "bulk.initialBatchSize"; - private static final String BULK_INITIAL_BATCH_SIZE_DOC = - "Cosmos DB initial bulk micro batch size - a micro batch will be flushed to the backend " - + "when the number of documents enqueued exceeds this size - or the target payload size is met. The micro batch " - + "size is getting automatically tuned based on the throttling rate. By default the " - + "initial micro batch size is 1. Reduce this when you want to avoid that the first few requests consume " - + "too many RUs."; - private static final String BULK_INITIAL_BATCH_SIZE_DISPLAY = "Cosmos DB initial bulk micro batch size."; - private static final int DEFAULT_BULK_INITIAL_BATCH_SIZE = 1; // start with small value to avoid initial RU spike - - // write strategy - public static final String WRITE_STRATEGY_CONF = SINK_CONFIG_PREFIX + "write.strategy"; - private static final String WRITE_STRATEGY_DOC = "Cosmos DB Item write Strategy: `ItemOverwrite` (using upsert), `ItemAppend` (using create, " - + "ignore pre-existing items i.e., Conflicts), `ItemDelete` (deletes based on id/pk of data frame), " - + "`ItemDeleteIfNotModified` (deletes based on id/pk of data frame if etag hasn't changed since collecting " - + "id/pk), `ItemOverwriteIfNotModified` (using create if etag is empty, update/replace with etag pre-condition " - + "otherwise, if document was updated the pre-condition failure is ignored)"; - private static final String WRITE_STRATEGY_DISPLAY = "Cosmos DB Item write Strategy."; - private static final String DEFAULT_WRITE_STRATEGY = ItemWriteStrategy.ITEM_OVERWRITE.getName(); - - // max retry - public static final String MAX_RETRY_COUNT_CONF = SINK_CONFIG_PREFIX + "maxRetryCount"; - private static final String MAX_RETRY_COUNT_DOC = - "Cosmos DB max retry attempts on write failures for Sink connector. By default, the connector will retry on transient write errors for up to 10 times."; - private static final String MAX_RETRY_COUNT_DISPLAY = "Cosmos DB max retry attempts on write failures for Sink connector."; - private static final int DEFAULT_MAX_RETRY_COUNT = 10; - - // database name - private static final String DATABASE_NAME_CONF = SINK_CONFIG_PREFIX + "database.name"; - private static final String DATABASE_NAME_CONF_DOC = "Cosmos DB database name."; - private static final String DATABASE_NAME_CONF_DISPLAY = "Cosmos DB database name."; - - // container topic map - public static final String CONTAINERS_TOPIC_MAP_CONF = SINK_CONFIG_PREFIX + "containers.topicMap"; - private static final String CONTAINERS_TOPIC_MAP_DOC = - "A comma delimited list of Kafka topics mapped to Cosmos containers. For example: topic1#con1,topic2#con2."; - private static final String CONTAINERS_TOPIC_MAP_DISPLAY = "Topic-Container map"; - - // TODO[Public preview]: re-examine idStrategy implementation - // id.strategy - public static final String ID_STRATEGY_CONF = SINK_CONFIG_PREFIX + "id.strategy"; - public static final String ID_STRATEGY_DOC = - "A strategy used to populate the document with an ``id``. Valid strategies are: " - + "``TemplateStrategy``, ``FullKeyStrategy``, ``KafkaMetadataStrategy``, " - + "``ProvidedInKeyStrategy``, ``ProvidedInValueStrategy``. Configuration " - + "properties prefixed with``id.strategy`` are passed through to the strategy. For " - + "example, when using ``id.strategy=TemplateStrategy`` , " - + "the property ``id.strategy.template`` is passed through to the template strategy " - + "and used to specify the template string to be used in constructing the ``id``."; - public static final String ID_STRATEGY_DISPLAY = "ID Strategy"; - public static final String DEFAULT_ID_STRATEGY = IdStrategies.PROVIDED_IN_VALUE_STRATEGY.getName(); - - // TODO[Public Preview] Verify whether compression need to happen in connector - - private final CosmosSinkWriteConfig writeConfig; - private final CosmosSinkContainersConfig containersConfig; - private final IdStrategies idStrategy; - - public CosmosSinkConfig(Map parsedConfig) { - this(getConfigDef(), parsedConfig); - } - - public CosmosSinkConfig(ConfigDef config, Map parsedConfig) { - super(config, parsedConfig); - this.writeConfig = this.parseWriteConfig(); - this.containersConfig = this.parseContainersConfig(); - this.idStrategy = this.parseIdStrategy(); - } - - public static ConfigDef getConfigDef() { - ConfigDef configDef = KafkaCosmosConfig.getConfigDef(); - - defineWriteConfig(configDef); - defineContainersConfig(configDef); - defineIdStrategyConfig(configDef); - return configDef; - } - - private static void defineWriteConfig(ConfigDef configDef) { - final String writeConfigGroupName = "Write config"; - int writeConfigGroupOrder = 0; - configDef - .define( - BULK_ENABLED_CONF, - ConfigDef.Type.BOOLEAN, - DEFAULT_BULK_ENABLED, - ConfigDef.Importance.MEDIUM, - BULK_ENABLED_DOC, - writeConfigGroupName, - writeConfigGroupOrder++, - ConfigDef.Width.MEDIUM, - BULK_ENABLED_DISPLAY - ) - .define( - BULK_MAX_CONCURRENT_PARTITIONS_CONF, - ConfigDef.Type.INT, - DEFAULT_BULK_MAX_CONCURRENT_PARTITIONS, - ConfigDef.Importance.LOW, - BULK_MAX_CONCURRENT_PARTITIONS_DOC, - writeConfigGroupName, - writeConfigGroupOrder++, - ConfigDef.Width.MEDIUM, - BULK_MAX_CONCURRENT_PARTITIONS_DISPLAY - ) - .define( - BULK_INITIAL_BATCH_SIZE_CONF, - ConfigDef.Type.INT, - DEFAULT_BULK_INITIAL_BATCH_SIZE, - ConfigDef.Importance.MEDIUM, - BULK_INITIAL_BATCH_SIZE_DOC, - writeConfigGroupName, - writeConfigGroupOrder++, - ConfigDef.Width.MEDIUM, - BULK_INITIAL_BATCH_SIZE_DISPLAY - ) - .define( - WRITE_STRATEGY_CONF, - ConfigDef.Type.STRING, - DEFAULT_WRITE_STRATEGY, - new ItemWriteStrategyValidator(), - ConfigDef.Importance.HIGH, - WRITE_STRATEGY_DOC, - writeConfigGroupName, - writeConfigGroupOrder++, - ConfigDef.Width.LONG, - WRITE_STRATEGY_DISPLAY - ) - .define( - MAX_RETRY_COUNT_CONF, - ConfigDef.Type.INT, - DEFAULT_MAX_RETRY_COUNT, - ConfigDef.Importance.MEDIUM, - MAX_RETRY_COUNT_DOC, - writeConfigGroupName, - writeConfigGroupOrder++, - ConfigDef.Width.MEDIUM, - MAX_RETRY_COUNT_DISPLAY - ) - .define( - TOLERANCE_ON_ERROR_CONFIG, - ConfigDef.Type.STRING, - DEFAULT_TOLERANCE_ON_ERROR, - ConfigDef.Importance.HIGH, - TOLERANCE_ON_ERROR_DOC, - writeConfigGroupName, - writeConfigGroupOrder++, - ConfigDef.Width.MEDIUM, - TOLERANCE_ON_ERROR_DISPLAY - ); - } - - private static void defineContainersConfig(ConfigDef configDef) { - final String containersGroupName = "Containers"; - int containersGroupOrder = 0; - - configDef - .define( - DATABASE_NAME_CONF, - ConfigDef.Type.STRING, - ConfigDef.NO_DEFAULT_VALUE, - NON_EMPTY_STRING, - ConfigDef.Importance.HIGH, - DATABASE_NAME_CONF_DOC, - containersGroupName, - containersGroupOrder++, - ConfigDef.Width.LONG, - DATABASE_NAME_CONF_DISPLAY - ) - .define( - CONTAINERS_TOPIC_MAP_CONF, - ConfigDef.Type.STRING, - ConfigDef.NO_DEFAULT_VALUE, - new ContainersTopicMapValidator(), - ConfigDef.Importance.MEDIUM, - CONTAINERS_TOPIC_MAP_DOC, - containersGroupName, - containersGroupOrder++, - ConfigDef.Width.LONG, - CONTAINERS_TOPIC_MAP_DISPLAY - ); - } - - private static void defineIdStrategyConfig(ConfigDef configDef) { - final String idStrategyConfigGroupName = "ID Strategy"; - int idStrategyConfigGroupOrder = 0; - configDef - .define( - ID_STRATEGY_CONF, - ConfigDef.Type.STRING, - DEFAULT_ID_STRATEGY, - ConfigDef.Importance.HIGH, - ID_STRATEGY_DOC, - idStrategyConfigGroupName, - idStrategyConfigGroupOrder++, - ConfigDef.Width.MEDIUM, - ID_STRATEGY_DISPLAY); - } - - private CosmosSinkWriteConfig parseWriteConfig() { - boolean bulkEnabled = this.getBoolean(BULK_ENABLED_CONF); - int bulkMaxConcurrentCosmosPartitions = this.getInt(BULK_MAX_CONCURRENT_PARTITIONS_CONF); - int bulkInitialBatchSize = this.getInt(BULK_INITIAL_BATCH_SIZE_CONF); - ItemWriteStrategy writeStrategy = this.parseItemWriteStrategy(); - int maxRetryCount = this.getInt(MAX_RETRY_COUNT_CONF); - ToleranceOnErrorLevel toleranceOnErrorLevel = this.parseToleranceOnErrorLevel(); - - return new CosmosSinkWriteConfig( - bulkEnabled, - bulkMaxConcurrentCosmosPartitions, - bulkInitialBatchSize, - writeStrategy, - maxRetryCount, - toleranceOnErrorLevel); - } - - private CosmosSinkContainersConfig parseContainersConfig() { - String databaseName = this.getString(DATABASE_NAME_CONF); - Map topicToContainerMap = this.getTopicToContainerMap(); - - return new CosmosSinkContainersConfig(databaseName, topicToContainerMap); - } - - private Map getTopicToContainerMap() { - List containersTopicMapList = convertToList(this.getString(CONTAINERS_TOPIC_MAP_CONF)); - return containersTopicMapList - .stream() - .map(containerTopicMapString -> containerTopicMapString.split("#")) - .collect( - Collectors.toMap( - containerTopicMapArray -> containerTopicMapArray[0], - containerTopicMapArray -> containerTopicMapArray[1])); - } - - private ItemWriteStrategy parseItemWriteStrategy() { - return ItemWriteStrategy.fromName(this.getString(WRITE_STRATEGY_CONF)); - } - - private ToleranceOnErrorLevel parseToleranceOnErrorLevel() { - return ToleranceOnErrorLevel.fromName(this.getString(TOLERANCE_ON_ERROR_CONFIG)); - } - - private IdStrategies parseIdStrategy() { - return IdStrategies.fromName(this.getString(ID_STRATEGY_CONF)); - } - - public CosmosSinkWriteConfig getWriteConfig() { - return writeConfig; - } - - public CosmosSinkContainersConfig getContainersConfig() { - return containersConfig; - } - - public IdStrategies getIdStrategy() { - return idStrategy; - } - - public static class ItemWriteStrategyValidator implements ConfigDef.Validator { - @Override - @SuppressWarnings("unchecked") - public void ensureValid(String name, Object o) { - String itemWriteStrategyString = (String) o; - if (StringUtils.isEmpty(itemWriteStrategyString)) { - throw new ConfigException(name, o, "WriteStrategy can not be empty or null"); - } - - ItemWriteStrategy itemWriteStrategy = ItemWriteStrategy.fromName(itemWriteStrategyString); - if (itemWriteStrategy == null) { - throw new ConfigException(name, o, "Invalid ItemWriteStrategy. Allowed values " + ItemWriteStrategy.values()); - } - } - - @Override - public String toString() { - return "ItemWriteStrategy. Only allow " + ItemWriteStrategy.values(); - } - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosSinkContainersConfig.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosSinkContainersConfig.java deleted file mode 100644 index 9cb3273f8ebda..0000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosSinkContainersConfig.java +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink; - -import java.util.Map; - -public class CosmosSinkContainersConfig { - private final String databaseName; - private final Map topicToContainerMap; - - public CosmosSinkContainersConfig(String databaseName, Map topicToContainerMap) { - this.databaseName = databaseName; - this.topicToContainerMap = topicToContainerMap; - } - - public String getDatabaseName() { - return databaseName; - } - - public Map getTopicToContainerMap() { - return topicToContainerMap; - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosSinkTask.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosSinkTask.java deleted file mode 100644 index 79d881169b3da..0000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosSinkTask.java +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink; - -import com.azure.cosmos.CosmosAsyncClient; -import com.azure.cosmos.CosmosAsyncContainer; -import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; -import com.azure.cosmos.kafka.connect.implementation.CosmosClientStore; -import com.azure.cosmos.kafka.connect.implementation.KafkaCosmosConstants; -import org.apache.kafka.connect.sink.SinkRecord; -import org.apache.kafka.connect.sink.SinkTask; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -public class CosmosSinkTask extends SinkTask { - private static final Logger LOGGER = LoggerFactory.getLogger(CosmosSinkTask.class); - private CosmosSinkTaskConfig sinkTaskConfig; - private CosmosAsyncClient cosmosClient; - private SinkRecordTransformer sinkRecordTransformer; - private IWriter cosmosWriter; - - @Override - public String version() { - return KafkaCosmosConstants.CURRENT_VERSION; - } - - @Override - public void start(Map props) { - LOGGER.info("Starting the kafka cosmos sink task..."); - this.sinkTaskConfig = new CosmosSinkTaskConfig(props); - this.cosmosClient = CosmosClientStore.getCosmosClient(this.sinkTaskConfig.getAccountConfig()); - this.sinkRecordTransformer = new SinkRecordTransformer(this.sinkTaskConfig); - - if (this.sinkTaskConfig.getWriteConfig().isBulkEnabled()) { - this.cosmosWriter = - new KafkaCosmosBulkWriter(this.sinkTaskConfig.getWriteConfig(), this.context.errantRecordReporter()); - } else { - this.cosmosWriter = - new KafkaCosmosPointWriter(this.sinkTaskConfig.getWriteConfig(), context.errantRecordReporter()); - } - - // TODO[public preview]: in V1, it will create the database if does not exists, but why? - } - - @Override - public void put(Collection records) { - if (records == null || records.isEmpty()) { - LOGGER.debug("No records to be written"); - return; - } - - LOGGER.debug("Sending {} records to be written", records.size()); - - // group by container - Map> recordsByContainer = - records.stream().collect( - Collectors.groupingBy( - record -> this.sinkTaskConfig - .getContainersConfig() - .getTopicToContainerMap() - .getOrDefault(record.topic(), StringUtils.EMPTY))); - - if (recordsByContainer.containsKey(StringUtils.EMPTY)) { - throw new IllegalStateException("There is no container defined for topics " + recordsByContainer.get(StringUtils.EMPTY)); - } - - for (Map.Entry> entry : recordsByContainer.entrySet()) { - String containerName = entry.getKey(); - CosmosAsyncContainer container = - this.cosmosClient - .getDatabase(this.sinkTaskConfig.getContainersConfig().getDatabaseName()) - .getContainer(containerName); - - // transform sink records, for example populating id - List transformedRecords = sinkRecordTransformer.transform(containerName, entry.getValue()); - this.cosmosWriter.write(container, transformedRecords); - } - } - - @Override - public void stop() { - LOGGER.info("Stopping Kafka CosmosDB sink task..."); - if (this.cosmosClient != null) { - this.cosmosClient.close(); - } - - this.cosmosClient = null; - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosSinkTaskConfig.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosSinkTaskConfig.java deleted file mode 100644 index f438f3f620eb9..0000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosSinkTaskConfig.java +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink; - -import java.util.Map; - -// Currently the sink task config shares the same config as sink connector config -public class CosmosSinkTaskConfig extends CosmosSinkConfig { - public CosmosSinkTaskConfig(Map parsedConfig) { - super(parsedConfig); - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosSinkWriteConfig.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosSinkWriteConfig.java deleted file mode 100644 index 98d758deb6e3d..0000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosSinkWriteConfig.java +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink; - -public class CosmosSinkWriteConfig { - private final boolean bulkEnabled; - private final int bulkMaxConcurrentCosmosPartitions; - private final int bulkInitialBatchSize; - private final ItemWriteStrategy itemWriteStrategy; - private final int maxRetryCount; - - private final ToleranceOnErrorLevel toleranceOnErrorLevel; - - public CosmosSinkWriteConfig( - boolean bulkEnabled, - int bulkMaxConcurrentCosmosPartitions, - int bulkInitialBatchSize, - ItemWriteStrategy itemWriteStrategy, - int maxRetryCount, - ToleranceOnErrorLevel toleranceOnErrorLevel) { - - this.bulkEnabled = bulkEnabled; - this.bulkMaxConcurrentCosmosPartitions = bulkMaxConcurrentCosmosPartitions; - this.bulkInitialBatchSize = bulkInitialBatchSize; - this.itemWriteStrategy = itemWriteStrategy; - this.maxRetryCount = maxRetryCount; - this.toleranceOnErrorLevel = toleranceOnErrorLevel; - } - - public boolean isBulkEnabled() { - return bulkEnabled; - } - - public int getBulkMaxConcurrentCosmosPartitions() { - return bulkMaxConcurrentCosmosPartitions; - } - - public int getBulkInitialBatchSize() { - return bulkInitialBatchSize; - } - - public ItemWriteStrategy getItemWriteStrategy() { - return itemWriteStrategy; - } - - public int getMaxRetryCount() { - return maxRetryCount; - } - - public ToleranceOnErrorLevel getToleranceOnErrorLevel() { - return toleranceOnErrorLevel; - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/IWriter.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/IWriter.java deleted file mode 100644 index 18cee5577b7d6..0000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/IWriter.java +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink; - -import com.azure.cosmos.CosmosAsyncContainer; -import org.apache.kafka.connect.sink.SinkRecord; - -import java.util.List; - -public interface IWriter { - void write(CosmosAsyncContainer container, List sinkRecords); -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/IdStrategies.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/IdStrategies.java deleted file mode 100644 index dcc3568dc2fb1..0000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/IdStrategies.java +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink; - -public enum IdStrategies { - TEMPLATE_STRATEGY("TemplateStrategy"), - FULL_KEY_STRATEGY("FullKeyStrategy"), - KAFKA_METADATA_STRATEGY("KafkaMetadataStrategy"), - PROVIDED_IN_KEY_STRATEGY("ProvidedInKeyStrategy"), - PROVIDED_IN_VALUE_STRATEGY("ProvidedInValueStrategy"); - - private final String name; - - IdStrategies(String name) { - this.name = name; - } - - public String getName() { - return name; - } - - public static IdStrategies fromName(String name) { - for (IdStrategies mode : IdStrategies.values()) { - if (mode.getName().equalsIgnoreCase(name)) { - return mode; - } - } - return null; - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/ItemWriteStrategy.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/ItemWriteStrategy.java deleted file mode 100644 index 988f577b5a267..0000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/ItemWriteStrategy.java +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink; - -public enum ItemWriteStrategy { - ITEM_OVERWRITE("ItemOverwrite"), - ITEM_APPEND("ItemAppend"), - ITEM_DELETE("ItemDelete"), - ITEM_DELETE_IF_NOT_MODIFIED("ItemDeleteIfNotModified"), - ITEM_OVERWRITE_IF_NOT_MODIFIED("ItemOverwriteIfNotModified"); - - // TODO[Public Preview] Add ItemPatch, ItemBulkUpdate - private final String name; - - ItemWriteStrategy(String name) { - this.name = name; - } - - public String getName() { - return name; - } - - public static ItemWriteStrategy fromName(String name) { - for (ItemWriteStrategy mode : ItemWriteStrategy.values()) { - if (mode.getName().equalsIgnoreCase(name)) { - return mode; - } - } - return null; - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/KafkaCosmosBulkWriter.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/KafkaCosmosBulkWriter.java deleted file mode 100644 index 97515fa530feb..0000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/KafkaCosmosBulkWriter.java +++ /dev/null @@ -1,335 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink; - -import com.azure.cosmos.BridgeInternal; -import com.azure.cosmos.CosmosAsyncContainer; -import com.azure.cosmos.CosmosException; -import com.azure.cosmos.implementation.HttpConstants; -import com.azure.cosmos.implementation.ImplementationBridgeHelpers; -import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; -import com.azure.cosmos.kafka.connect.implementation.KafkaCosmosExceptionsHelper; -import com.azure.cosmos.kafka.connect.implementation.KafkaCosmosSchedulers; -import com.azure.cosmos.models.CosmosBulkExecutionOptions; -import com.azure.cosmos.models.CosmosBulkItemRequestOptions; -import com.azure.cosmos.models.CosmosBulkItemResponse; -import com.azure.cosmos.models.CosmosBulkOperationResponse; -import com.azure.cosmos.models.CosmosBulkOperations; -import com.azure.cosmos.models.CosmosItemOperation; -import com.azure.cosmos.models.PartitionKeyDefinition; -import org.apache.kafka.connect.sink.ErrantRecordReporter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import reactor.core.publisher.Flux; -import reactor.core.publisher.Mono; -import reactor.core.publisher.SignalType; -import reactor.core.publisher.Sinks; - -import java.time.Duration; -import java.util.List; -import java.util.Random; -import java.util.concurrent.atomic.AtomicInteger; - -import static com.azure.cosmos.implementation.guava25.base.Preconditions.checkNotNull; - -public class KafkaCosmosBulkWriter extends KafkaCosmosWriterBase { - private static final Logger LOGGER = LoggerFactory.getLogger(KafkaCosmosBulkWriter.class); - private static final int MAX_DELAY_ON_408_REQUEST_TIMEOUT_IN_MS = 10000; - private static final int MIN_DELAY_ON_408_REQUEST_TIMEOUT_IN_MS = 1000; - private static final Random RANDOM = new Random(); - - private final CosmosSinkWriteConfig writeConfig; - private final Sinks.EmitFailureHandler emitFailureHandler; - - public KafkaCosmosBulkWriter( - CosmosSinkWriteConfig writeConfig, - ErrantRecordReporter errantRecordReporter) { - super(errantRecordReporter); - checkNotNull(writeConfig, "Argument 'writeConfig' can not be null"); - - this.writeConfig = writeConfig; - this.emitFailureHandler = new KafkaCosmosEmitFailureHandler(); - } - - @Override - public void writeCore(CosmosAsyncContainer container, List sinkOperations) { - Sinks.Many bulkRetryEmitter = Sinks.many().unicast().onBackpressureBuffer(); - CosmosBulkExecutionOptions bulkExecutionOptions = this.getBulkExecutionOperations(); - AtomicInteger totalPendingRecords = new AtomicInteger(sinkOperations.size()); - Runnable onTaskCompleteCheck = () -> { - if (totalPendingRecords.decrementAndGet() <= 0) { - bulkRetryEmitter.emitComplete(emitFailureHandler); - } - }; - - Flux.fromIterable(sinkOperations) - .flatMap(sinkOperation -> this.getBulkOperation(container, sinkOperation)) - .collectList() - .flatMapMany(itemOperations -> { - - Flux> cosmosBulkOperationResponseFlux = - container - .executeBulkOperations( - Flux.fromIterable(itemOperations) - .mergeWith(bulkRetryEmitter.asFlux()) - .publishOn(KafkaCosmosSchedulers.SINK_BOUNDED_ELASTIC), - bulkExecutionOptions); - return cosmosBulkOperationResponseFlux; - }) - .flatMap(itemResponse -> { - SinkOperation sinkOperation = itemResponse.getOperation().getContext(); - checkNotNull(sinkOperation, "sinkOperation should not be null"); - - if (itemResponse.getResponse() != null && itemResponse.getResponse().isSuccessStatusCode()) { - // success - this.completeSinkOperation(sinkOperation, onTaskCompleteCheck); - } else { - BulkOperationFailedException exception = handleErrorStatusCode( - itemResponse.getResponse(), - itemResponse.getException(), - sinkOperation); - - if (shouldIgnore(exception)) { - this.completeSinkOperation(sinkOperation, onTaskCompleteCheck); - } else { - if (shouldRetry(exception, sinkOperation.getRetryCount(), this.writeConfig.getMaxRetryCount())) { - sinkOperation.setException(exception); - return this.scheduleRetry(container, itemResponse.getOperation().getContext(), bulkRetryEmitter, exception); - } else { - // operation failed after exhausting all retries - this.completeSinkOperationWithFailure(sinkOperation, exception, onTaskCompleteCheck); - if (this.writeConfig.getToleranceOnErrorLevel() == ToleranceOnErrorLevel.ALL) { - LOGGER.warn( - "Could not upload record {} to CosmosDB after exhausting all retries, " - + "but ToleranceOnErrorLevel is all, will only log the error message. ", - sinkOperation.getSinkRecord().key(), - sinkOperation.getException()); - return Mono.empty(); - } else { - return Mono.error(exception); - } - } - } - } - - return Mono.empty(); - }) - .subscribeOn(KafkaCosmosSchedulers.SINK_BOUNDED_ELASTIC) - .blockLast(); - } - - private CosmosBulkExecutionOptions getBulkExecutionOperations() { - CosmosBulkExecutionOptions bulkExecutionOptions = new CosmosBulkExecutionOptions(); - bulkExecutionOptions.setInitialMicroBatchSize(this.writeConfig.getBulkInitialBatchSize()); - if (this.writeConfig.getBulkMaxConcurrentCosmosPartitions() > 0) { - ImplementationBridgeHelpers - .CosmosBulkExecutionOptionsHelper - .getCosmosBulkExecutionOptionsAccessor() - .setMaxConcurrentCosmosPartitions(bulkExecutionOptions, this.writeConfig.getBulkMaxConcurrentCosmosPartitions()); - } - - return bulkExecutionOptions; - } - - private Mono getBulkOperation( - CosmosAsyncContainer container, - SinkOperation sinkOperation) { - - return this.getPartitionKeyDefinition(container) - .flatMap(partitionKeyDefinition -> { - CosmosItemOperation cosmosItemOperation; - - switch (this.writeConfig.getItemWriteStrategy()) { - case ITEM_OVERWRITE: - cosmosItemOperation = this.getUpsertItemOperation(sinkOperation, partitionKeyDefinition); - break; - case ITEM_OVERWRITE_IF_NOT_MODIFIED: - String etag = getEtag(sinkOperation.getSinkRecord().value()); - if (StringUtils.isEmpty(etag)) { - cosmosItemOperation = this.getCreateItemOperation(sinkOperation, partitionKeyDefinition); - } else { - cosmosItemOperation = this.getReplaceItemOperation(sinkOperation, partitionKeyDefinition, etag); - } - break; - case ITEM_APPEND: - cosmosItemOperation = this.getCreateItemOperation(sinkOperation, partitionKeyDefinition); - break; - case ITEM_DELETE: - cosmosItemOperation = this.getDeleteItemOperation(sinkOperation, partitionKeyDefinition, null); - break; - case ITEM_DELETE_IF_NOT_MODIFIED: - String itemDeleteEtag = getEtag(sinkOperation.getSinkRecord().value()); - cosmosItemOperation = this.getDeleteItemOperation(sinkOperation, partitionKeyDefinition, itemDeleteEtag); - break; - default: - return Mono.error(new IllegalArgumentException(this.writeConfig.getItemWriteStrategy() + " is not supported")); - } - - return Mono.just(cosmosItemOperation); - }); - } - - private CosmosItemOperation getUpsertItemOperation( - SinkOperation sinkOperation, - PartitionKeyDefinition partitionKeyDefinition) { - - return CosmosBulkOperations.getUpsertItemOperation( - sinkOperation.getSinkRecord().value(), - this.getPartitionKeyValue(sinkOperation.getSinkRecord().value(), partitionKeyDefinition), - sinkOperation); - } - - private CosmosItemOperation getCreateItemOperation( - SinkOperation sinkOperation, - PartitionKeyDefinition partitionKeyDefinition) { - return CosmosBulkOperations.getCreateItemOperation( - sinkOperation.getSinkRecord().value(), - this.getPartitionKeyValue(sinkOperation.getSinkRecord().value(), partitionKeyDefinition), - sinkOperation); - } - - private CosmosItemOperation getReplaceItemOperation( - SinkOperation sinkOperation, - PartitionKeyDefinition partitionKeyDefinition, - String etag) { - - CosmosBulkItemRequestOptions itemRequestOptions = new CosmosBulkItemRequestOptions(); - if (StringUtils.isNotEmpty(etag)) { - itemRequestOptions.setIfMatchETag(etag); - } - - return CosmosBulkOperations.getReplaceItemOperation( - getId(sinkOperation.getSinkRecord().value()), - sinkOperation.getSinkRecord().value(), - this.getPartitionKeyValue(sinkOperation.getSinkRecord().value(), partitionKeyDefinition), - new CosmosBulkItemRequestOptions().setIfMatchETag(etag), - sinkOperation); - } - - private CosmosItemOperation getDeleteItemOperation( - SinkOperation sinkOperation, - PartitionKeyDefinition partitionKeyDefinition, - String etag) { - - CosmosBulkItemRequestOptions itemRequestOptions = new CosmosBulkItemRequestOptions(); - if (StringUtils.isNotEmpty(etag)) { - itemRequestOptions.setIfMatchETag(etag); - } - - return CosmosBulkOperations.getDeleteItemOperation( - this.getId(sinkOperation.getSinkRecord().value()), - this.getPartitionKeyValue(sinkOperation.getSinkRecord().value(), partitionKeyDefinition), - itemRequestOptions, - sinkOperation); - } - - private Mono scheduleRetry( - CosmosAsyncContainer container, - SinkOperation sinkOperation, - Sinks.Many bulkRetryEmitter, - BulkOperationFailedException exception) { - - sinkOperation.retry(); - Mono retryMono = - getBulkOperation(container, sinkOperation) - .flatMap(itemOperation -> { - bulkRetryEmitter.emitNext(itemOperation, emitFailureHandler); - return Mono.empty(); - }); - - if (KafkaCosmosExceptionsHelper.isTimeoutException(exception)) { - Duration delayDuration = Duration.ofMillis( - MIN_DELAY_ON_408_REQUEST_TIMEOUT_IN_MS - + RANDOM.nextInt(MAX_DELAY_ON_408_REQUEST_TIMEOUT_IN_MS - MIN_DELAY_ON_408_REQUEST_TIMEOUT_IN_MS)); - - return retryMono.delaySubscription(delayDuration); - } - - return retryMono; - } - - BulkOperationFailedException handleErrorStatusCode( - CosmosBulkItemResponse itemResponse, - Exception exception, - SinkOperation sinkOperationContext) { - - int effectiveStatusCode = - itemResponse != null - ? itemResponse.getStatusCode() - : (exception != null && exception instanceof CosmosException ? ((CosmosException) exception).getStatusCode() : HttpConstants.StatusCodes.REQUEST_TIMEOUT); - int effectiveSubStatusCode = - itemResponse != null - ? itemResponse.getSubStatusCode() - : (exception != null && exception instanceof CosmosException ? ((CosmosException) exception).getSubStatusCode() : 0); - - String errorMessage = - String.format( - "Request failed with effectiveStatusCode: {%s}, effectiveSubStatusCode: {%s}, kafkaOffset: {%s}, kafkaPartition: {%s}, topic: {%s}", - effectiveStatusCode, - effectiveSubStatusCode, - sinkOperationContext.getKafkaOffset(), - sinkOperationContext.getKafkaPartition(), - sinkOperationContext.getTopic()); - - - return new BulkOperationFailedException(effectiveStatusCode, effectiveSubStatusCode, errorMessage, exception); - } - - private boolean shouldIgnore(BulkOperationFailedException failedException) { - switch (this.writeConfig.getItemWriteStrategy()) { - case ITEM_APPEND: - return KafkaCosmosExceptionsHelper.isResourceExistsException(failedException); - case ITEM_DELETE: - return KafkaCosmosExceptionsHelper.isNotFoundException(failedException); - case ITEM_DELETE_IF_NOT_MODIFIED: - return KafkaCosmosExceptionsHelper.isNotFoundException(failedException) - || KafkaCosmosExceptionsHelper.isPreconditionFailedException(failedException); - case ITEM_OVERWRITE_IF_NOT_MODIFIED: - return KafkaCosmosExceptionsHelper.isResourceExistsException(failedException) - || KafkaCosmosExceptionsHelper.isNotFoundException(failedException) - || KafkaCosmosExceptionsHelper.isPreconditionFailedException(failedException); - default: - return false; - } - } - - private void completeSinkOperation(SinkOperation sinkOperationContext, Runnable onCompleteRunnable) { - sinkOperationContext.complete(); - onCompleteRunnable.run(); - } - - public void completeSinkOperationWithFailure( - SinkOperation sinkOperationContext, - Exception exception, - Runnable onCompleteRunnable) { - - sinkOperationContext.setException(exception); - sinkOperationContext.complete(); - onCompleteRunnable.run(); - - this.sendToDlqIfConfigured(sinkOperationContext); - } - - private static class BulkOperationFailedException extends CosmosException { - protected BulkOperationFailedException(int statusCode, int subStatusCode, String message, Throwable cause) { - super(statusCode, message, null, cause); - BridgeInternal.setSubStatusCode(this, subStatusCode); - } - } - - private static class KafkaCosmosEmitFailureHandler implements Sinks.EmitFailureHandler { - - @Override - public boolean onEmitFailure(SignalType signalType, Sinks.EmitResult emitResult) { - if (emitResult.equals(Sinks.EmitResult.FAIL_NON_SERIALIZED)) { - LOGGER.debug("emitFailureHandler - Signal: {}, Result: {}", signalType, emitResult.toString()); - return true; - } else { - LOGGER.error("emitFailureHandler - Signal: {}, Result: {}", signalType, emitResult.toString()); - return false; - } - } - } -} - diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/KafkaCosmosPointWriter.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/KafkaCosmosPointWriter.java deleted file mode 100644 index a13c5d1a5266e..0000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/KafkaCosmosPointWriter.java +++ /dev/null @@ -1,179 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink; - -import com.azure.cosmos.CosmosAsyncContainer; -import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; -import com.azure.cosmos.implementation.guava25.base.Function; -import com.azure.cosmos.kafka.connect.implementation.KafkaCosmosExceptionsHelper; -import com.azure.cosmos.kafka.connect.implementation.KafkaCosmosSchedulers; -import com.azure.cosmos.models.CosmosItemRequestOptions; -import org.apache.kafka.connect.sink.ErrantRecordReporter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import reactor.core.publisher.Mono; - -import java.util.List; - -import static com.azure.cosmos.implementation.guava25.base.Preconditions.checkNotNull; - -public class KafkaCosmosPointWriter extends KafkaCosmosWriterBase { - private static final Logger LOGGER = LoggerFactory.getLogger(KafkaCosmosPointWriter.class); - - private final CosmosSinkWriteConfig writeConfig; - - public KafkaCosmosPointWriter( - CosmosSinkWriteConfig writeConfig, - ErrantRecordReporter errantRecordReporter) { - super(errantRecordReporter); - checkNotNull(writeConfig, "Argument 'writeConfig' can not be null"); - this.writeConfig = writeConfig; - } - - @Override - public void writeCore(CosmosAsyncContainer container, List sinkOperations) { - for (SinkOperation sinkOperation : sinkOperations) { - switch (this.writeConfig.getItemWriteStrategy()) { - case ITEM_OVERWRITE: - this.upsertWithRetry(container, sinkOperation); - break; - case ITEM_OVERWRITE_IF_NOT_MODIFIED: - String etag = this.getEtag(sinkOperation.getSinkRecord().value()); - if (StringUtils.isNotEmpty(etag)) { - this.replaceIfNotModifiedWithRetry(container, sinkOperation, etag); - } else { - this.createWithRetry(container, sinkOperation); - } - break; - case ITEM_APPEND: - this.createWithRetry(container, sinkOperation); - break; - case ITEM_DELETE: - this.deleteWithRetry(container, sinkOperation, false); - break; - case ITEM_DELETE_IF_NOT_MODIFIED: - this.deleteWithRetry(container, sinkOperation, true); - break; - default: - throw new IllegalArgumentException(this.writeConfig.getItemWriteStrategy() + " is not supported"); - } - } - } - - private void upsertWithRetry(CosmosAsyncContainer container, SinkOperation sinkOperation) { - executeWithRetry( - (operation) -> container.upsertItem(operation.getSinkRecord().value()).then(), - (throwable) -> false, // no exceptions should be ignored - sinkOperation - ); - } - - private void createWithRetry(CosmosAsyncContainer container, SinkOperation sinkOperation) { - executeWithRetry( - (operation) -> container.createItem(operation.getSinkRecord().value()).then(), - (throwable) -> KafkaCosmosExceptionsHelper.isResourceExistsException(throwable), - sinkOperation - ); - } - - private void replaceIfNotModifiedWithRetry(CosmosAsyncContainer container, SinkOperation sinkOperation, String etag) { - executeWithRetry( - (operation) -> { - CosmosItemRequestOptions itemRequestOptions = new CosmosItemRequestOptions(); - itemRequestOptions.setIfMatchETag(etag); - - return this.getPartitionKeyDefinition(container) - .flatMap(partitionKeyDefinition -> { - return container.replaceItem( - operation.getSinkRecord().value(), - getId(operation.getSinkRecord().value()), - getPartitionKeyValue(operation.getSinkRecord().value(), partitionKeyDefinition), - itemRequestOptions).then(); - }); - }, - (throwable) -> { - return KafkaCosmosExceptionsHelper.isNotFoundException(throwable) - || KafkaCosmosExceptionsHelper.isPreconditionFailedException(throwable); - }, - sinkOperation - ); - } - - private void deleteWithRetry(CosmosAsyncContainer container, SinkOperation sinkOperation, boolean onlyIfModified) { - executeWithRetry( - (operation) -> { - CosmosItemRequestOptions itemRequestOptions = new CosmosItemRequestOptions(); - if (onlyIfModified) { - String etag = this.getEtag(operation.getSinkRecord().value()); - if (StringUtils.isNotEmpty(etag)) { - itemRequestOptions.setIfMatchETag(etag); - } - } - - return this.getPartitionKeyDefinition(container) - .flatMap(partitionKeyDefinition -> { - return container.deleteItem( - getId(operation.getSinkRecord().value()), - getPartitionKeyValue(operation.getSinkRecord().value(), partitionKeyDefinition), - itemRequestOptions - ); - }).then(); - }, - (throwable) -> { - return KafkaCosmosExceptionsHelper.isNotFoundException(throwable) - || KafkaCosmosExceptionsHelper.isPreconditionFailedException(throwable); - }, - sinkOperation - ); - } - - private void executeWithRetry( - Function> execution, - Function shouldIgnoreFunc, - SinkOperation sinkOperation) { - - Mono.just(this) - .flatMap(data -> { - if (sinkOperation.getRetryCount() > 0) { - LOGGER.debug("Retry for sinkRecord {}", sinkOperation.getSinkRecord().key()); - } - return execution.apply(sinkOperation); - }) - .doOnSuccess(response -> sinkOperation.complete()) - .onErrorResume(throwable -> { - if (shouldIgnoreFunc.apply(throwable)) { - sinkOperation.complete(); - return Mono.empty(); - } - - if (shouldRetry(throwable, sinkOperation.getRetryCount(), this.writeConfig.getMaxRetryCount())) { - sinkOperation.setException(throwable); - sinkOperation.retry(); - - return Mono.empty(); - } else { - // request failed after exhausted all retries - this.sendToDlqIfConfigured(sinkOperation); - - sinkOperation.setException(throwable); - sinkOperation.complete(); - - if (this.writeConfig.getToleranceOnErrorLevel() == ToleranceOnErrorLevel.ALL) { - LOGGER.warn( - "Could not upload record {} to CosmosDB after exhausting all retries, but ToleranceOnErrorLevel is all, will only log the error message. ", - sinkOperation.getSinkRecord().key(), - sinkOperation.getException()); - return Mono.empty(); - } else { - return Mono.error(sinkOperation.getException()); - } - } - }) - .repeat(() -> !sinkOperation.isCompleted()) - .then() - .subscribeOn(KafkaCosmosSchedulers.SINK_BOUNDED_ELASTIC) - .block(); - } -} - diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/KafkaCosmosWriterBase.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/KafkaCosmosWriterBase.java deleted file mode 100644 index 108f3fefa7cac..0000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/KafkaCosmosWriterBase.java +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink; - -import com.azure.cosmos.CosmosAsyncContainer; -import com.azure.cosmos.CosmosBridgeInternal; -import com.azure.cosmos.implementation.DocumentCollection; -import com.azure.cosmos.implementation.ImplementationBridgeHelpers; -import com.azure.cosmos.implementation.Strings; -import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; -import com.azure.cosmos.implementation.routing.PartitionKeyInternal; -import com.azure.cosmos.kafka.connect.implementation.KafkaCosmosExceptionsHelper; -import com.azure.cosmos.models.PartitionKey; -import com.azure.cosmos.models.PartitionKeyDefinition; -import org.apache.kafka.connect.sink.ErrantRecordReporter; -import org.apache.kafka.connect.sink.SinkRecord; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import reactor.core.publisher.Mono; - -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import static com.azure.cosmos.implementation.guava25.base.Preconditions.checkArgument; - -public abstract class KafkaCosmosWriterBase implements IWriter { - private static final Logger LOGGER = LoggerFactory.getLogger(KafkaCosmosWriterBase.class); - private static final String ID = "id"; - private static final String ETAG = "_etag"; - private final ErrantRecordReporter errantRecordReporter; - - public KafkaCosmosWriterBase(ErrantRecordReporter errantRecordReporter) { - this.errantRecordReporter = errantRecordReporter; - } - - abstract void writeCore(CosmosAsyncContainer container, List sinkOperations); - - @Override - public void write(CosmosAsyncContainer container, List sinkRecords) { - if (sinkRecords == null || sinkRecords.isEmpty()) { - LOGGER.debug("No records to be written to container {}", container.getId()); - return; - } - LOGGER.debug("Write {} records to container {}", sinkRecords.size(), container.getId()); - - // For each sinkRecord, it has a 1:1 mapping SinkOperation which contains sinkRecord and related context: retryCount, succeeded or failure. - List sinkOperations = - sinkRecords - .stream() - .map(sinkRecord -> new SinkOperation(sinkRecord)) - .collect(Collectors.toList()); - - try { - writeCore(container, sinkOperations); - } catch (Exception e) { - LOGGER.error("Write failed. ", e); - throw new CosmosDBWriteException(e.getMessage()); - } - } - - @SuppressWarnings("unchecked") - protected String getId(Object recordValue) { - checkArgument(recordValue instanceof Map, "Argument 'recordValue' is not valid map format."); - return ((Map) recordValue).get(ID).toString(); - } - - @SuppressWarnings("unchecked") - protected String getEtag(Object recordValue) { - checkArgument(recordValue instanceof Map, "Argument 'recordValue' is not valid map format."); - return ((Map) recordValue).getOrDefault(ETAG, Strings.Emtpy).toString(); - } - - @SuppressWarnings("unchecked") - protected PartitionKey getPartitionKeyValue(Object recordValue, PartitionKeyDefinition partitionKeyDefinition) { - checkArgument(recordValue instanceof Map, "Argument 'recordValue' is not valid map format."); - - //TODO[Public Preview]: add support for sub-partition - String partitionKeyPath = StringUtils.join(partitionKeyDefinition.getPaths(), ""); - Map recordMap = (Map) recordValue; - Object partitionKeyValue = recordMap.get(partitionKeyPath.substring(1)); - PartitionKeyInternal partitionKeyInternal = PartitionKeyInternal.fromObjectArray(Collections.singletonList(partitionKeyValue), false); - - return ImplementationBridgeHelpers - .PartitionKeyHelper - .getPartitionKeyAccessor() - .toPartitionKey(partitionKeyInternal); - } - - protected boolean shouldRetry(Throwable exception, int attemptedCount, int maxRetryCount) { - if (attemptedCount >= maxRetryCount) { - return false; - } - - return KafkaCosmosExceptionsHelper.isTransientFailure(exception); - } - - protected Mono getPartitionKeyDefinition(CosmosAsyncContainer container) { - return Mono.just(CosmosBridgeInternal.getAsyncDocumentClient(container.getDatabase()).getCollectionCache()) - .flatMap(collectionCache -> { - return collectionCache - .resolveByNameAsync( - null, - ImplementationBridgeHelpers - .CosmosAsyncContainerHelper - .getCosmosAsyncContainerAccessor() - .getLinkWithoutTrailingSlash(container), - null, - new DocumentCollection()) - .map(documentCollection -> documentCollection.getPartitionKey()); - }); - } - - protected void sendToDlqIfConfigured(SinkOperation sinkOperationContext) { - if (this.errantRecordReporter != null) { - errantRecordReporter.report(sinkOperationContext.getSinkRecord(), sinkOperationContext.getException()); - } - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/SinkOperation.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/SinkOperation.java deleted file mode 100644 index 599a71ec99848..0000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/SinkOperation.java +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink; - -import org.apache.kafka.connect.sink.SinkRecord; - -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; - -public class SinkOperation { - private final SinkRecord sinkRecord; - private final AtomicInteger retryCount; - private final AtomicReference exception; - private final AtomicBoolean completed; - - public SinkOperation(SinkRecord sinkRecord) { - this.sinkRecord = sinkRecord; - this.retryCount = new AtomicInteger(0); - this.exception = new AtomicReference<>(null); - this.completed = new AtomicBoolean(false); - } - - public SinkRecord getSinkRecord() { - return this.sinkRecord; - } - - public long getKafkaOffset() { - return this.sinkRecord.kafkaOffset(); - } - - public Integer getKafkaPartition() { - return this.sinkRecord.kafkaPartition(); - } - - public String getTopic() { - return this.sinkRecord.topic(); - } - - public int getRetryCount() { - return this.retryCount.get(); - } - - public void retry() { - this.retryCount.incrementAndGet(); - } - - public Throwable getException() { - return this.exception.get(); - } - - public void setException(Throwable exception) { - this.exception.set(exception); - } - - public boolean isCompleted() { - return this.completed.get(); - } - - public void complete() { - this.completed.set(true); - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/SinkRecordTransformer.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/SinkRecordTransformer.java deleted file mode 100644 index 007d09bb793d7..0000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/SinkRecordTransformer.java +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink; - -import com.azure.cosmos.kafka.connect.implementation.sink.idstrategy.AbstractIdStrategyConfig; -import com.azure.cosmos.kafka.connect.implementation.sink.idstrategy.FullKeyStrategy; -import com.azure.cosmos.kafka.connect.implementation.sink.idstrategy.IdStrategy; -import com.azure.cosmos.kafka.connect.implementation.sink.idstrategy.KafkaMetadataStrategy; -import com.azure.cosmos.kafka.connect.implementation.sink.idstrategy.ProvidedInKeyStrategy; -import com.azure.cosmos.kafka.connect.implementation.sink.idstrategy.ProvidedInValueStrategy; -import com.azure.cosmos.kafka.connect.implementation.sink.idstrategy.TemplateStrategy; -import org.apache.kafka.connect.data.Struct; -import org.apache.kafka.connect.sink.SinkRecord; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.slf4j.MDC; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -public class SinkRecordTransformer { - private static final Logger LOGGER = LoggerFactory.getLogger(SinkRecordTransformer.class); - - private final IdStrategy idStrategy; - - public SinkRecordTransformer(CosmosSinkTaskConfig sinkTaskConfig) { - this.idStrategy = this.createIdStrategy(sinkTaskConfig); - } - - @SuppressWarnings("unchecked") - public List transform(String containerName, List sinkRecords) { - List toBeWrittenRecordList = new ArrayList<>(); - for (SinkRecord record : sinkRecords) { - if (record.key() != null) { - MDC.put(String.format("CosmosDbSink-%s", containerName), record.key().toString()); - } - - LOGGER.trace( - "Key Schema [{}], Key [{}], Value type [{}], Value schema [{}]", - record.keySchema(), - record.key(), - record.value() == null ? null : record.value().getClass().getName(), - record.value() == null ? null : record.valueSchema()); - - Object recordValue; - if (record.value() instanceof Struct) { - recordValue = StructToJsonMap.toJsonMap((Struct) record.value()); - } else if (record.value() instanceof Map) { - recordValue = StructToJsonMap.handleMap((Map) record.value()); - } else { - recordValue = record.value(); - } - - maybeInsertId(recordValue, record); - - // Create an updated record with from the current record and the updated record value - final SinkRecord updatedRecord = new SinkRecord(record.topic(), - record.kafkaPartition(), - record.keySchema(), - record.key(), - record.valueSchema(), - recordValue, - record.kafkaOffset(), - record.timestamp(), - record.timestampType(), - record.headers()); - - toBeWrittenRecordList.add(updatedRecord); - } - - return toBeWrittenRecordList; - } - - @SuppressWarnings("unchecked") - private void maybeInsertId(Object recordValue, SinkRecord sinkRecord) { - if (!(recordValue instanceof Map)) { - return; - } - Map recordMap = (Map) recordValue; - recordMap.put(AbstractIdStrategyConfig.ID, this.idStrategy.generateId(sinkRecord)); - } - - private IdStrategy createIdStrategy(CosmosSinkTaskConfig sinkTaskConfig) { - IdStrategy idStrategyClass; - switch (sinkTaskConfig.getIdStrategy()) { - case FULL_KEY_STRATEGY: - idStrategyClass = new FullKeyStrategy(); - break; - case TEMPLATE_STRATEGY: - idStrategyClass = new TemplateStrategy(); - break; - case KAFKA_METADATA_STRATEGY: - idStrategyClass = new KafkaMetadataStrategy(); - break; - case PROVIDED_IN_VALUE_STRATEGY: - idStrategyClass = new ProvidedInValueStrategy(); - break; - case PROVIDED_IN_KEY_STRATEGY: - idStrategyClass = new ProvidedInKeyStrategy(); - break; - default: - throw new IllegalArgumentException(sinkTaskConfig.getIdStrategy() + " is not supported"); - } - - idStrategyClass.configure(sinkTaskConfig.originalsWithPrefix(AbstractIdStrategyConfig.PREFIX)); - return idStrategyClass; - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/StructToJsonMap.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/StructToJsonMap.java deleted file mode 100644 index 388baa3778f8d..0000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/StructToJsonMap.java +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink; - -import org.apache.kafka.connect.data.Date; -import org.apache.kafka.connect.data.Field; -import org.apache.kafka.connect.data.Schema; -import org.apache.kafka.connect.data.Struct; -import org.apache.kafka.connect.data.Time; -import org.apache.kafka.connect.data.Timestamp; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -// TODO[Public Preview]: Double check logic here, copied over from V1 -public class StructToJsonMap { - - public static Map toJsonMap(Struct struct) { - if (struct == null) { - return null; - } - Map jsonMap = new HashMap(0); - List fields = struct.schema().fields(); - for (Field field : fields) { - String fieldName = field.name(); - Schema.Type fieldType = field.schema().type(); - String schemaName = field.schema().name(); - switch (fieldType) { - case STRING: - jsonMap.put(fieldName, struct.getString(fieldName)); - break; - case INT32: - if (Date.LOGICAL_NAME.equals(schemaName) || Time.LOGICAL_NAME.equals(schemaName)) { - jsonMap.put(fieldName, (java.util.Date) struct.get(fieldName)); - } else { - jsonMap.put(fieldName, struct.getInt32(fieldName)); - } - break; - case INT16: - jsonMap.put(fieldName, struct.getInt16(fieldName)); - break; - case INT64: - if (Timestamp.LOGICAL_NAME.equals(schemaName)) { - jsonMap.put(fieldName, (java.util.Date) struct.get(fieldName)); - } else { - jsonMap.put(fieldName, struct.getInt64(fieldName)); - } - break; - case FLOAT32: - jsonMap.put(fieldName, struct.getFloat32(fieldName)); - break; - case FLOAT64: - jsonMap.put(fieldName, struct.getFloat64(fieldName)); - break; - case BOOLEAN: - jsonMap.put(fieldName, struct.getBoolean(fieldName)); - break; - case ARRAY: - List fieldArray = struct.getArray(fieldName); - if (fieldArray != null && !fieldArray.isEmpty() && fieldArray.get(0) instanceof Struct) { - // If Array contains list of Structs - List jsonArray = new ArrayList<>(); - fieldArray.forEach(item -> { - jsonArray.add(toJsonMap((Struct) item)); - }); - jsonMap.put(fieldName, jsonArray); - } else { - jsonMap.put(fieldName, fieldArray); - } - break; - case STRUCT: - jsonMap.put(fieldName, toJsonMap(struct.getStruct(fieldName))); - break; - case MAP: - jsonMap.put(fieldName, handleMap(struct.getMap(fieldName))); - break; - default: - jsonMap.put(fieldName, struct.get(fieldName)); - break; - } - } - return jsonMap; - } - - @SuppressWarnings("unchecked") - public static Map handleMap(Map map) { - if (map == null) { - return null; - } - Map cacheMap = new HashMap<>(); - map.forEach((key, value) -> { - if (value instanceof Map) { - cacheMap.put(key, handleMap((Map) value)); - } else if (value instanceof Struct) { - cacheMap.put(key, toJsonMap((Struct) value)); - } else if (value instanceof List) { - List list = (List) value; - List jsonArray = new ArrayList<>(); - list.forEach(item -> { - if (item instanceof Struct) { - jsonArray.add(toJsonMap((Struct) item)); - } else if (item instanceof Map) { - jsonArray.add(handleMap((Map) item)); - } else { - jsonArray.add(item); - } - }); - cacheMap.put(key, jsonArray); - } else { - cacheMap.put(key, value); - } - }); - return cacheMap; - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/ToleranceOnErrorLevel.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/ToleranceOnErrorLevel.java deleted file mode 100644 index d169fd2484b0a..0000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/ToleranceOnErrorLevel.java +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink; - -public enum ToleranceOnErrorLevel { - NONE("None"), - ALL("All"); - - private final String name; - - ToleranceOnErrorLevel(String name) { - this.name = name; - } - - public String getName() { - return name; - } - - public static ToleranceOnErrorLevel fromName(String name) { - for (ToleranceOnErrorLevel mode : ToleranceOnErrorLevel.values()) { - if (mode.getName().equalsIgnoreCase(name)) { - return mode; - } - } - return null; - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/AbstractIdStrategy.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/AbstractIdStrategy.java deleted file mode 100644 index 41bc401b81360..0000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/AbstractIdStrategy.java +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink.idstrategy; - -import java.util.Map; -import java.util.regex.Pattern; - -public abstract class AbstractIdStrategy implements IdStrategy { - private static final String SANITIZED_CHAR = "_"; - private static final Pattern SANITIZE_ID_PATTERN = Pattern.compile("[/\\\\?#]"); - - protected Map configs; - - @Override - public void configure(Map configs) { - this.configs = configs; - } - - /** - * Replaces all characters that cannot be part of the ID with {@value SANITIZED_CHAR}. - *

The following characters are restricted and cannot be used in the Id property: '/', '\\', '?', '#' - */ - public static String sanitizeId(String unsanitized) { - return SANITIZE_ID_PATTERN.matcher(unsanitized).replaceAll(SANITIZED_CHAR); - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/AbstractIdStrategyConfig.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/AbstractIdStrategyConfig.java deleted file mode 100644 index 1713ecc79636e..0000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/AbstractIdStrategyConfig.java +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink.idstrategy; - -import org.apache.kafka.common.config.AbstractConfig; -import org.apache.kafka.common.config.ConfigDef; - -import java.util.Map; - -public class AbstractIdStrategyConfig extends AbstractConfig { - public static final String ID = "id"; - public static final String ID_STRATEGY = ID + ".strategy"; - public static final String PREFIX = ID_STRATEGY + "."; - - public AbstractIdStrategyConfig(ConfigDef definition, Map originals) { - super(definition, originals); - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/FullKeyStrategy.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/FullKeyStrategy.java deleted file mode 100644 index 62fc6f72a3406..0000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/FullKeyStrategy.java +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink.idstrategy; - -import java.util.HashMap; -import java.util.Map; - -public class FullKeyStrategy extends TemplateStrategy { - @Override - public void configure(Map configs) { - Map conf = new HashMap<>(configs); - conf.put(TemplateStrategyConfig.TEMPLATE_CONFIG, "${key}"); - super.configure(conf); - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/IdStrategy.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/IdStrategy.java deleted file mode 100644 index b4ce03d4f73e3..0000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/IdStrategy.java +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink.idstrategy; - -import org.apache.kafka.common.Configurable; -import org.apache.kafka.connect.sink.SinkRecord; - -public interface IdStrategy extends Configurable { - String generateId(SinkRecord record); -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/KafkaMetadataStrategy.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/KafkaMetadataStrategy.java deleted file mode 100644 index 99d705f062f4c..0000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/KafkaMetadataStrategy.java +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink.idstrategy; - -import java.util.HashMap; -import java.util.Map; - -public class KafkaMetadataStrategy extends TemplateStrategy { - private KafkaMetadataStrategyConfig config; - - @Override - public void configure(Map configs) { - config = new KafkaMetadataStrategyConfig(configs); - Map conf = new HashMap<>(configs); - conf.put(TemplateStrategyConfig.TEMPLATE_CONFIG, - "${topic}" + config.delimiter() - + "${partition}" + config.delimiter() + "${offset}"); - - super.configure(conf); - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/KafkaMetadataStrategyConfig.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/KafkaMetadataStrategyConfig.java deleted file mode 100644 index b29d59e4409ca..0000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/KafkaMetadataStrategyConfig.java +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink.idstrategy; - -import org.apache.kafka.common.config.ConfigDef; - -import java.util.Map; - -public class KafkaMetadataStrategyConfig extends AbstractIdStrategyConfig { - public static final String DELIMITER_CONFIG = "delimiter"; - public static final String DELIMITER_CONFIG_DEFAULT = "-"; - public static final String DELIMITER_CONFIG_DOC = "The delimiter between metadata components"; - public static final String DELIMITER_CONFIG_DISPLAY = "Kafka Metadata"; - - private String delimiter; - - public KafkaMetadataStrategyConfig(Map props) { - this(getConfig(), props); - } - - public KafkaMetadataStrategyConfig(ConfigDef definition, Map originals) { - super(definition, originals); - - this.delimiter = getString(DELIMITER_CONFIG); - } - - public static ConfigDef getConfig() { - ConfigDef result = new ConfigDef(); - - final String groupName = "Kafka Metadata Parameters"; - int groupOrder = 0; - - result.define( - DELIMITER_CONFIG, - ConfigDef.Type.STRING, - DELIMITER_CONFIG_DEFAULT, - ConfigDef.Importance.MEDIUM, - DELIMITER_CONFIG_DOC, - groupName, - groupOrder++, - ConfigDef.Width.MEDIUM, - DELIMITER_CONFIG_DISPLAY - ); - - return result; - } - - public String delimiter() { - return delimiter; - } -} - diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/ProvidedInConfig.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/ProvidedInConfig.java deleted file mode 100644 index de3892baa1d2e..0000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/ProvidedInConfig.java +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink.idstrategy; - -import org.apache.kafka.common.config.ConfigDef; - -import java.util.Map; - -public class ProvidedInConfig extends AbstractIdStrategyConfig { - public static final String JSON_PATH_CONFIG = "jsonPath"; - public static final String JSON_PATH_CONFIG_DEFAULT = "$.id"; - public static final String JSON_PATH_CONFIG_DOC = "A JsonPath expression to select the desired component to use as ``id``"; - public static final String JSON_PATH_CONFIG_DISPLAY = "JSON Path"; - private final String jsonPath; - - public ProvidedInConfig(Map props) { - this(getConfig(), props); - } - - public ProvidedInConfig(ConfigDef definition, Map originals) { - super(definition, originals); - - this.jsonPath = getString(JSON_PATH_CONFIG); - } - - - public static ConfigDef getConfig() { - ConfigDef result = new ConfigDef(); - - final String groupName = "JsonPath Parameters"; - int groupOrder = 0; - - result.define( - JSON_PATH_CONFIG, - ConfigDef.Type.STRING, - JSON_PATH_CONFIG_DEFAULT, - ConfigDef.Importance.MEDIUM, - JSON_PATH_CONFIG_DOC, - groupName, - groupOrder++, - ConfigDef.Width.MEDIUM, - JSON_PATH_CONFIG_DISPLAY - ); - - return result; - } - - public String jsonPath() { - return jsonPath; - } -} - diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/ProvidedInKeyStrategy.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/ProvidedInKeyStrategy.java deleted file mode 100644 index 930e2435351e8..0000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/ProvidedInKeyStrategy.java +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink.idstrategy; - -public class ProvidedInKeyStrategy extends ProvidedInStrategy { - public ProvidedInKeyStrategy() { - super(ProvidedIn.KEY); - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/ProvidedInStrategy.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/ProvidedInStrategy.java deleted file mode 100644 index 79b4ed19f655d..0000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/ProvidedInStrategy.java +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink.idstrategy; - -import org.apache.kafka.connect.data.Values; -import org.apache.kafka.connect.errors.ConnectException; -import org.apache.kafka.connect.sink.SinkRecord; -import com.jayway.jsonpath.JsonPath; - -import java.util.Map; - -class ProvidedInStrategy extends AbstractIdStrategy { - protected enum ProvidedIn { - KEY, - VALUE - } - - private final ProvidedIn where; - - private ProvidedInConfig config; - - ProvidedInStrategy(ProvidedIn where) { - this.where = where; - } - - @Override - public String generateId(SinkRecord record) { - String value = where == ProvidedIn.KEY - ? Values.convertToString(record.keySchema(), record.key()) - : Values.convertToString(record.valueSchema(), record.value()); - try { - Object object = JsonPath.parse(value).read(config.jsonPath()); - return sanitizeId(Values.convertToString(null, object)); - } catch (Exception e) { - throw new ConnectException("Could not evaluate JsonPath " + config.jsonPath(), e); - } - } - - @Override - public void configure(Map configs) { - config = new ProvidedInConfig(configs); - super.configure(configs); - } -} - diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/ProvidedInValueStrategy.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/ProvidedInValueStrategy.java deleted file mode 100644 index ca5b794fc8bd8..0000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/ProvidedInValueStrategy.java +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink.idstrategy; - -public class ProvidedInValueStrategy extends ProvidedInStrategy { - public ProvidedInValueStrategy() { - super(ProvidedIn.VALUE); - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/TemplateStrategy.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/TemplateStrategy.java deleted file mode 100644 index 1f40827050dcc..0000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/TemplateStrategy.java +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink.idstrategy; - -import com.azure.cosmos.implementation.guava25.collect.ImmutableMap; -import org.apache.kafka.connect.data.Values; -import org.apache.kafka.connect.sink.SinkRecord; - -import java.util.Map; -import java.util.function.Function; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -public class TemplateStrategy extends AbstractIdStrategy { - private static final String KEY = "key"; - private static final String TOPIC = "topic"; - private static final String PARTITION = "partition"; - private static final String OFFSET = "offset"; - - private static final String PATTERN_TEMPLATE = "\\$\\{(%s)\\}"; - private static final Pattern PATTERN; - - private TemplateStrategyConfig config; - - private static final Map> METHODS_BY_VARIABLE; - - static { - ImmutableMap.Builder> builder = ImmutableMap.builder(); - builder.put(KEY, (r) -> Values.convertToString(r.keySchema(), r.key())); - builder.put(TOPIC, SinkRecord::topic); - builder.put(PARTITION, (r) -> r.kafkaPartition().toString()); - builder.put(OFFSET, (r) -> Long.toString(r.kafkaOffset())); - METHODS_BY_VARIABLE = builder.build(); - - String pattern = String.format(PATTERN_TEMPLATE, - METHODS_BY_VARIABLE.keySet().stream().collect(Collectors.joining("|"))); - PATTERN = Pattern.compile(pattern); - } - - @Override - public String generateId(SinkRecord record) { - String template = config.template(); - return sanitizeId(resolveAll(template, record)); - } - - @Override - public void configure(Map configs) { - config = new TemplateStrategyConfig(configs); - - super.configure(configs); - } - - private String resolveAll(String template, SinkRecord record) { - int lastIndex = 0; - StringBuilder output = new StringBuilder(); - Matcher matcher = PATTERN.matcher(template); - while (matcher.find()) { - output.append(template, lastIndex, matcher.start()) - .append(METHODS_BY_VARIABLE.get(matcher.group(1)).apply(record)); - - lastIndex = matcher.end(); - } - if (lastIndex < template.length()) { - output.append(template, lastIndex, template.length()); - } - return output.toString(); - } -} - diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/TemplateStrategyConfig.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/TemplateStrategyConfig.java deleted file mode 100644 index c03c605438d9b..0000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/TemplateStrategyConfig.java +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink.idstrategy; - -import org.apache.kafka.common.config.ConfigDef; - -import java.util.Map; - -public class TemplateStrategyConfig extends AbstractIdStrategyConfig { - public static final String TEMPLATE_CONFIG = "template"; - public static final String TEMPLATE_CONFIG_DEFAULT = ""; - public static final String TEMPLATE_CONFIG_DOC = - "The template string to use for determining the ``id``. The template can contain the " - + "following variables that are bound to their values on the Kafka record:" - + "${topic}, ${partition}, ${offset}, ${key}. For example, the template " - + "``${topic}-${key}`` would use the topic name and the entire key in the ``id``, " - + "separated by '-'"; - public static final String TEMPLATE_CONFIG_DISPLAY = "Template"; - private final String template; - - public TemplateStrategyConfig(Map props) { - this(getConfig(), props); - } - - public TemplateStrategyConfig(ConfigDef definition, Map originals) { - super(definition, originals); - - this.template = getString(TEMPLATE_CONFIG); - } - - public static ConfigDef getConfig() { - ConfigDef result = new ConfigDef(); - - final String groupName = "Template Parameters"; - int groupOrder = 0; - - result.define( - TEMPLATE_CONFIG, - ConfigDef.Type.STRING, - TEMPLATE_CONFIG_DEFAULT, - ConfigDef.Importance.MEDIUM, - TEMPLATE_CONFIG_DOC, - groupName, - groupOrder++, - ConfigDef.Width.MEDIUM, - TEMPLATE_CONFIG_DISPLAY - ); - - return result; - } - - public String template() { - return template; - } -} - diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosChangeFeedModes.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosChangeFeedModes.java index 9917a19145c8c..3753001f3e44e 100644 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosChangeFeedModes.java +++ b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosChangeFeedModes.java @@ -18,7 +18,7 @@ public String getName() { public static CosmosChangeFeedModes fromName(String name) { for (CosmosChangeFeedModes mode : CosmosChangeFeedModes.values()) { - if (mode.getName().equalsIgnoreCase(name)) { + if (mode.getName().equals(name)) { return mode; } } diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosChangeFeedStartFromModes.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosChangeFeedStartFromModes.java index 15be8610bceed..439df9f826a5d 100644 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosChangeFeedStartFromModes.java +++ b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosChangeFeedStartFromModes.java @@ -19,7 +19,7 @@ public String getName() { public static CosmosChangeFeedStartFromModes fromName(String name) { for (CosmosChangeFeedStartFromModes startFromModes : CosmosChangeFeedStartFromModes.values()) { - if (startFromModes.getName().equalsIgnoreCase(name)) { + if (startFromModes.getName().equals(name)) { return startFromModes; } } diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosSourceConfig.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosSourceConfig.java index 1365d1e53575e..1ede731b5498c 100644 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosSourceConfig.java +++ b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosSourceConfig.java @@ -5,7 +5,7 @@ import com.azure.cosmos.implementation.Strings; import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; -import com.azure.cosmos.kafka.connect.implementation.KafkaCosmosConfig; +import com.azure.cosmos.kafka.connect.implementation.CosmosConfig; import org.apache.kafka.common.config.ConfigDef; import org.apache.kafka.common.config.ConfigException; @@ -14,83 +14,82 @@ import java.time.format.DateTimeParseException; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; /** * Common Configuration for Cosmos DB Kafka source connector. */ -public class CosmosSourceConfig extends KafkaCosmosConfig { +public class CosmosSourceConfig extends CosmosConfig { // configuration only targets to source connector - private static final String SOURCE_CONF_PREFIX = "kafka.connect.cosmos.source."; + private static final String SOURCE_CONFIG_PREFIX = "kafka.connect.cosmos.source."; // database name - private static final String DATABASE_NAME_CONF = SOURCE_CONF_PREFIX + "database.name"; + private static final String DATABASE_NAME_CONF = SOURCE_CONFIG_PREFIX + "database.name"; private static final String DATABASE_NAME_CONF_DOC = "Cosmos DB database name."; private static final String DATABASE_NAME_CONF_DISPLAY = "Cosmos DB database name."; // Source containers config - private static final String CONTAINERS_INCLUDE_ALL_CONF = SOURCE_CONF_PREFIX + "containers.includeAll"; - private static final String CONTAINERS_INCLUDE_ALL_CONF_DOC = "Flag to indicate whether reading from all containers."; - private static final String CONTAINERS_INCLUDE_ALL_CONF_DISPLAY = "Include all containers."; + private static final String CONTAINERS_INCLUDE_ALL_CONFIG = SOURCE_CONFIG_PREFIX + "containers.includeAll"; + private static final String CONTAINERS_INCLUDE_ALL_CONFIG_DOC = "Flag to indicate whether reading from all containers."; + private static final String CONTAINERS_INCLUDE_ALL_CONFIG_DISPLAY = "Include all containers."; private static final boolean DEFAULT_CONTAINERS_INCLUDE_ALL = false; - private static final String CONTAINERS_INCLUDED_LIST_CONF = SOURCE_CONF_PREFIX + "containers.includedList"; - private static final String CONTAINERS_INCLUDED_LIST_CONF_DOC = + private static final String CONTAINERS_INCLUDED_LIST_CONFIG = SOURCE_CONFIG_PREFIX + "containers.includedList"; + private static final String CONTAINERS_INCLUDED_LIST_CONFIG_DOC = "Containers included. This config will be ignored if kafka.connect.cosmos.source.includeAllContainers is true."; - private static final String CONTAINERS_INCLUDED_LIST_CONF_DISPLAY = "Containers included."; + private static final String CONTAINERS_INCLUDED_LIST_CONFIG_DISPLAY = "Containers included."; - private static final String CONTAINERS_TOPIC_MAP_CONF = SOURCE_CONF_PREFIX + "containers.topicMap"; - private static final String CONTAINERS_TOPIC_MAP_CONF_DOC = + private static final String CONTAINERS_TOPIC_MAP_CONFIG = SOURCE_CONFIG_PREFIX + "containers.topicMap"; + private static final String CONTAINERS_TOPIC_MAP_CONFIG_DOC = "A comma delimited list of Kafka topics mapped to Cosmos containers. For example: topic1#con1,topic2#con2. " + "By default, container name is used as the name of the kafka topic to publish data to, " + "can use this property to override the default config "; - private static final String CONTAINERS_TOPIC_MAP_CONF_DISPLAY = "Cosmos container topic map."; + private static final String CONTAINERS_TOPIC_MAP_CONFIG_DISPLAY = "Cosmos container topic map."; // changeFeed config - private static final String CHANGE_FEED_START_FROM_CONF = SOURCE_CONF_PREFIX + "changeFeed.startFrom"; - private static final String CHANGE_FEED_START_FROM_CONF_DOC = "ChangeFeed Start from settings (Now, Beginning " + private static final String CHANGE_FEED_START_FROM_CONFIG = SOURCE_CONFIG_PREFIX + "changeFeed.startFrom"; + private static final String CHANGE_FEED_START_FROM_CONFIG_DOC = "ChangeFeed Start from settings (Now, Beginning " + "or a certain point in time (UTC) for example 2020-02-10T14:15:03) - the default value is 'Beginning'. "; - private static final String CHANGE_FEED_START_FROM_CONF_DISPLAY = "Change feed start from."; + private static final String CHANGE_FEED_START_FROM_CONFIG_DISPLAY = "Change feed start from."; private static final String DEFAULT_CHANGE_FEED_START_FROM = CosmosChangeFeedStartFromModes.BEGINNING.getName(); - private static final String CHANGE_FEED_MODE_CONF = SOURCE_CONF_PREFIX + "changeFeed.mode"; - private static final String CHANGE_FEED_MODE_CONF_DOC = "ChangeFeed mode (LatestVersion or AllVersionsAndDeletes)"; - private static final String CHANGE_FEED_MODE_CONF_DISPLAY = "ChangeFeed mode (LatestVersion or AllVersionsAndDeletes)"; + private static final String CHANGE_FEED_MODE_CONFIG = SOURCE_CONFIG_PREFIX + "changeFeed.mode"; + private static final String CHANGE_FEED_MODE_CONFIG_DOC = "ChangeFeed mode (LatestVersion or AllVersionsAndDeletes)"; + private static final String CHANGE_FEED_MODE_CONFIG_DISPLAY = "ChangeFeed mode (LatestVersion or AllVersionsAndDeletes)"; private static final String DEFAULT_CHANGE_FEED_MODE = CosmosChangeFeedModes.LATEST_VERSION.getName(); - private static final String CHANGE_FEED_MAX_ITEM_COUNT_CONF = SOURCE_CONF_PREFIX + "changeFeed.maxItemCountHint"; - private static final String CHANGE_FEED_MAX_ITEM_COUNT_CONF_DOC = + private static final String CHANGE_FEED_MAX_ITEM_COUNT_CONFIG = SOURCE_CONFIG_PREFIX + "changeFeed.maxItemCountHint"; + private static final String CHANGE_FEED_MAX_ITEM_COUNT_CONFIG_DOC = "The maximum number of documents returned in a single change feed request." + " But the number of items received might be higher than the specified value if multiple items are changed by the same transaction." + " The default is 1000."; - private static final String CHANGE_FEED_MAX_ITEM_COUNT_CONF_DISPLAY = "The maximum number hint of documents returned in a single request. "; + private static final String CHANGE_FEED_MAX_ITEM_COUNT_CONFIG_DISPLAY = "The maximum number hint of documents returned in a single request. "; private static final int DEFAULT_CHANGE_FEED_MAX_ITEM_COUNT = 1000; // Metadata config - private static final String METADATA_POLL_DELAY_MS_CONF = SOURCE_CONF_PREFIX + "metadata.poll.delay.ms"; - private static final String METADATA_POLL_DELAY_MS_CONF_DOC = + private static final String METADATA_POLL_DELAY_MS_CONFIG = SOURCE_CONFIG_PREFIX + "metadata.poll.delay.ms"; + private static final String METADATA_POLL_DELAY_MS_CONFIG_DOC = "Indicates how often to check the metadata changes (including container split/merge, adding/removing/recreated containers). " + "When changes are detected, it will reconfigure the tasks. Default is 5 minutes."; - private static final String METADATA_POLL_DELAY_MS_CONF_DISPLAY = "Metadata polling delay in ms."; + private static final String METADATA_POLL_DELAY_MS_CONFIG_DISPLAY = "Metadata polling delay in ms."; private static final int DEFAULT_METADATA_POLL_DELAY_MS = 5 * 60 * 1000; // default is every 5 minutes - private static final String METADATA_STORAGE_TOPIC_CONF = SOURCE_CONF_PREFIX + "metadata.storage.topic"; - private static final String METADATA_STORAGE_TOPIC_CONF_DOC = "The name of the topic where the metadata are stored. " + private static final String METADATA_STORAGE_TOPIC_CONFIG = SOURCE_CONFIG_PREFIX + "metadata.storage.topic"; + private static final String METADATA_STORAGE_TOPIC_CONFIG_DOC = "The name of the topic where the metadata are stored. " + "The metadata topic will be created if it does not already exist, else it will use the pre-created topic."; - private static final String METADATA_STORAGE_TOPIC_CONF_DISPLAY = "Metadata storage topic."; + private static final String METADATA_STORAGE_TOPIC_CONFIG_DISPLAY = "Metadata storage topic."; private static final String DEFAULT_METADATA_STORAGE_TOPIC = "_cosmos.metadata.topic"; // messageKey - private static final String MESSAGE_KEY_ENABLED_CONF = SOURCE_CONF_PREFIX + "messageKey.enabled"; + private static final String MESSAGE_KEY_ENABLED_CONF = SOURCE_CONFIG_PREFIX + "messageKey.enabled"; private static final String MESSAGE_KEY_ENABLED_CONF_DOC = "Whether to set the kafka record message key."; private static final String MESSAGE_KEY_ENABLED_CONF_DISPLAY = "Kafka record message key enabled."; private static final boolean DEFAULT_MESSAGE_KEY_ENABLED = true; - private static final String MESSAGE_KEY_FIELD_CONF = SOURCE_CONF_PREFIX + "messageKey.field"; - private static final String MESSAGE_KEY_FIELD_CONF_DOC = "The field to use as the message key."; - private static final String MESSAGE_KEY_FIELD_CONF_DISPLAY = "Kafka message key field."; - private static final String DEFAULT_MESSAGE_KEY_FIELD = "id"; + private static final String MESSAGE_KEY_FIELD_CONFIG = SOURCE_CONFIG_PREFIX + "messageKey.field"; + private static final String MESSAGE_KEY_FIELD_CONFIG_DOC = "The field to use as the message key."; + private static final String MESSAGE_KEY_FIELD_CONFIG_DISPLAY = "Kafka message key field."; + private static final String DEFAULT_MESSAGE_KEY_FIELD = "id"; // TODO: should we use pk instead? private final CosmosSourceContainersConfig containersConfig; private final CosmosMetadataConfig metadataConfig; @@ -110,7 +109,7 @@ public CosmosSourceConfig(ConfigDef configDef, Map parsedConfigs) { } public static ConfigDef getConfigDef() { - ConfigDef configDef = KafkaCosmosConfig.getConfigDef(); + ConfigDef configDef = CosmosConfig.getConfigDef(); defineContainersConfig(configDef); defineMetadataConfig(configDef); @@ -138,38 +137,38 @@ private static void defineContainersConfig(ConfigDef result) { DATABASE_NAME_CONF_DISPLAY ) .define( - CONTAINERS_INCLUDE_ALL_CONF, + CONTAINERS_INCLUDE_ALL_CONFIG, ConfigDef.Type.BOOLEAN, DEFAULT_CONTAINERS_INCLUDE_ALL, ConfigDef.Importance.HIGH, - CONTAINERS_INCLUDE_ALL_CONF_DOC, + CONTAINERS_INCLUDE_ALL_CONFIG_DOC, containersGroupName, containersGroupOrder++, ConfigDef.Width.MEDIUM, - CONTAINERS_INCLUDE_ALL_CONF_DISPLAY + CONTAINERS_INCLUDE_ALL_CONFIG_DISPLAY ) .define( - CONTAINERS_INCLUDED_LIST_CONF, + CONTAINERS_INCLUDED_LIST_CONFIG, ConfigDef.Type.STRING, Strings.Emtpy, ConfigDef.Importance.MEDIUM, - CONTAINERS_INCLUDED_LIST_CONF_DOC, + CONTAINERS_INCLUDED_LIST_CONFIG_DOC, containersGroupName, containersGroupOrder++, ConfigDef.Width.LONG, - CONTAINERS_INCLUDED_LIST_CONF_DISPLAY + CONTAINERS_INCLUDED_LIST_CONFIG_DISPLAY ) .define( - CONTAINERS_TOPIC_MAP_CONF, + CONTAINERS_TOPIC_MAP_CONFIG, ConfigDef.Type.STRING, Strings.Emtpy, new ContainersTopicMapValidator(), ConfigDef.Importance.MEDIUM, - CONTAINERS_TOPIC_MAP_CONF_DOC, + CONTAINERS_TOPIC_MAP_CONFIG_DOC, containersGroupName, containersGroupOrder++, ConfigDef.Width.LONG, - CONTAINERS_TOPIC_MAP_CONF_DISPLAY + CONTAINERS_TOPIC_MAP_CONFIG_DISPLAY ); } @@ -179,28 +178,28 @@ private static void defineMetadataConfig(ConfigDef result) { result .define( - METADATA_POLL_DELAY_MS_CONF, + METADATA_POLL_DELAY_MS_CONFIG, ConfigDef.Type.INT, DEFAULT_METADATA_POLL_DELAY_MS, new PositiveValueValidator(), ConfigDef.Importance.MEDIUM, - METADATA_POLL_DELAY_MS_CONF_DOC, + METADATA_POLL_DELAY_MS_CONFIG_DOC, metadataGroupName, metadataGroupOrder++, ConfigDef.Width.MEDIUM, - METADATA_POLL_DELAY_MS_CONF_DISPLAY + METADATA_POLL_DELAY_MS_CONFIG_DISPLAY ) .define( - METADATA_STORAGE_TOPIC_CONF, + METADATA_STORAGE_TOPIC_CONFIG, ConfigDef.Type.STRING, DEFAULT_METADATA_STORAGE_TOPIC, NON_EMPTY_STRING, ConfigDef.Importance.HIGH, - METADATA_STORAGE_TOPIC_CONF_DOC, + METADATA_STORAGE_TOPIC_CONFIG_DOC, metadataGroupName, metadataGroupOrder++, ConfigDef.Width.LONG, - METADATA_STORAGE_TOPIC_CONF_DISPLAY + METADATA_STORAGE_TOPIC_CONFIG_DISPLAY ); } @@ -210,40 +209,40 @@ private static void defineChangeFeedConfig(ConfigDef result) { result .define( - CHANGE_FEED_MODE_CONF, + CHANGE_FEED_MODE_CONFIG, ConfigDef.Type.STRING, DEFAULT_CHANGE_FEED_MODE, new ChangeFeedModeValidator(), ConfigDef.Importance.HIGH, - CHANGE_FEED_MODE_CONF_DOC, + CHANGE_FEED_MODE_CONFIG_DOC, changeFeedGroupName, changeFeedGroupOrder++, ConfigDef.Width.MEDIUM, - CHANGE_FEED_MODE_CONF_DISPLAY + CHANGE_FEED_MODE_CONFIG_DISPLAY ) .define( - CHANGE_FEED_START_FROM_CONF, + CHANGE_FEED_START_FROM_CONFIG, ConfigDef.Type.STRING, DEFAULT_CHANGE_FEED_START_FROM, new ChangeFeedStartFromValidator(), ConfigDef.Importance.HIGH, - CHANGE_FEED_START_FROM_CONF_DOC, + CHANGE_FEED_START_FROM_CONFIG_DOC, changeFeedGroupName, changeFeedGroupOrder++, ConfigDef.Width.MEDIUM, - CHANGE_FEED_START_FROM_CONF_DISPLAY + CHANGE_FEED_START_FROM_CONFIG_DISPLAY ) .define( - CHANGE_FEED_MAX_ITEM_COUNT_CONF, + CHANGE_FEED_MAX_ITEM_COUNT_CONFIG, ConfigDef.Type.INT, DEFAULT_CHANGE_FEED_MAX_ITEM_COUNT, new PositiveValueValidator(), ConfigDef.Importance.MEDIUM, - CHANGE_FEED_MAX_ITEM_COUNT_CONF_DOC, + CHANGE_FEED_MAX_ITEM_COUNT_CONFIG_DOC, changeFeedGroupName, changeFeedGroupOrder++, ConfigDef.Width.MEDIUM, - CHANGE_FEED_MAX_ITEM_COUNT_CONF_DISPLAY + CHANGE_FEED_MAX_ITEM_COUNT_CONFIG_DISPLAY ); } @@ -264,23 +263,23 @@ private static void defineMessageKeyConfig(ConfigDef result) { MESSAGE_KEY_ENABLED_CONF_DISPLAY ) .define( - MESSAGE_KEY_FIELD_CONF, + MESSAGE_KEY_FIELD_CONFIG, ConfigDef.Type.STRING, DEFAULT_MESSAGE_KEY_FIELD, ConfigDef.Importance.HIGH, - MESSAGE_KEY_FIELD_CONF_DOC, + MESSAGE_KEY_FIELD_CONFIG_DOC, messageGroupName, messageGroupOrder++, ConfigDef.Width.MEDIUM, - MESSAGE_KEY_FIELD_CONF_DISPLAY + MESSAGE_KEY_FIELD_CONFIG_DISPLAY ); } private CosmosSourceContainersConfig parseContainersConfig() { String databaseName = this.getString(DATABASE_NAME_CONF); - boolean includeAllContainers = this.getBoolean(CONTAINERS_INCLUDE_ALL_CONF); + boolean includeAllContainers = this.getBoolean(CONTAINERS_INCLUDE_ALL_CONFIG); List containersIncludedList = this.getContainersIncludedList(); - Map containersTopicMap = this.getContainerToTopicMap(); + List containersTopicMap = this.getContainersTopicMap(); return new CosmosSourceContainersConfig( databaseName, @@ -291,23 +290,16 @@ private CosmosSourceContainersConfig parseContainersConfig() { } private List getContainersIncludedList() { - return convertToList(this.getString(CONTAINERS_INCLUDED_LIST_CONF)); + return convertToList(this.getString(CONTAINERS_INCLUDED_LIST_CONFIG)); } - private Map getContainerToTopicMap() { - List containerTopicMapList = convertToList(this.getString(CONTAINERS_TOPIC_MAP_CONF)); - return containerTopicMapList - .stream() - .map(containerTopicMapString -> containerTopicMapString.split("#")) - .collect( - Collectors.toMap( - containerTopicMapArray -> containerTopicMapArray[1], - containerTopicMapArray -> containerTopicMapArray[0])); + private List getContainersTopicMap() { + return convertToList(this.getString(CONTAINERS_TOPIC_MAP_CONFIG)); } private CosmosMetadataConfig parseMetadataConfig() { - int metadataPollDelayInMs = this.getInt(METADATA_POLL_DELAY_MS_CONF); - String metadataTopicName = this.getString(METADATA_STORAGE_TOPIC_CONF); + int metadataPollDelayInMs = this.getInt(METADATA_POLL_DELAY_MS_CONFIG); + String metadataTopicName = this.getString(METADATA_STORAGE_TOPIC_CONFIG); return new CosmosMetadataConfig(metadataPollDelayInMs, metadataTopicName); } @@ -316,7 +308,7 @@ private CosmosSourceChangeFeedConfig parseChangeFeedConfig() { CosmosChangeFeedModes changeFeedModes = this.parseChangeFeedMode(); CosmosChangeFeedStartFromModes changeFeedStartFromMode = this.parseChangeFeedStartFromMode(); Instant changeFeedStartFrom = this.parseChangeFeedStartFrom(changeFeedStartFromMode); - Integer changeFeedMaxItemCountHint = this.getInt(CHANGE_FEED_MAX_ITEM_COUNT_CONF); + Integer changeFeedMaxItemCountHint = this.getInt(CHANGE_FEED_MAX_ITEM_COUNT_CONFIG); return new CosmosSourceChangeFeedConfig( changeFeedModes, @@ -327,12 +319,12 @@ private CosmosSourceChangeFeedConfig parseChangeFeedConfig() { private CosmosSourceMessageKeyConfig parseMessageKeyConfig() { boolean messageKeyEnabled = this.getBoolean(MESSAGE_KEY_ENABLED_CONF); - String messageKeyField = this.getString(MESSAGE_KEY_FIELD_CONF); + String messageKeyField = this.getString(MESSAGE_KEY_FIELD_CONFIG); return new CosmosSourceMessageKeyConfig(messageKeyEnabled, messageKeyField); } private CosmosChangeFeedStartFromModes parseChangeFeedStartFromMode() { - String changeFeedStartFrom = this.getString(CHANGE_FEED_START_FROM_CONF); + String changeFeedStartFrom = this.getString(CHANGE_FEED_START_FROM_CONFIG); if (changeFeedStartFrom.equalsIgnoreCase(CosmosChangeFeedStartFromModes.BEGINNING.getName())) { return CosmosChangeFeedStartFromModes.BEGINNING; } @@ -346,7 +338,7 @@ private CosmosChangeFeedStartFromModes parseChangeFeedStartFromMode() { private Instant parseChangeFeedStartFrom(CosmosChangeFeedStartFromModes startFromMode) { if (startFromMode == CosmosChangeFeedStartFromModes.POINT_IN_TIME) { - String changeFeedStartFrom = this.getString(CHANGE_FEED_START_FROM_CONF); + String changeFeedStartFrom = this.getString(CHANGE_FEED_START_FROM_CONFIG); return Instant.from(DateTimeFormatter.ISO_INSTANT.parse(changeFeedStartFrom.trim())); } @@ -354,7 +346,7 @@ private Instant parseChangeFeedStartFrom(CosmosChangeFeedStartFromModes startFro } private CosmosChangeFeedModes parseChangeFeedMode() { - String changeFeedMode = this.getString(CHANGE_FEED_MODE_CONF); + String changeFeedMode = this.getString(CHANGE_FEED_MODE_CONFIG); return CosmosChangeFeedModes.fromName(changeFeedMode); } @@ -374,6 +366,42 @@ public CosmosSourceMessageKeyConfig getMessageKeyConfig() { return messageKeyConfig; } + public static class ContainersTopicMapValidator implements ConfigDef.Validator { + private static final String INVALID_TOPIC_MAP_FORMAT = + "Invalid entry for topic-container map. The topic-container map should be a comma-delimited " + + "list of Kafka topic to Cosmos containers. Each mapping should be a pair of Kafka " + + "topic and Cosmos container separated by '#'. For example: topic1#con1,topic2#con2."; + + @Override + @SuppressWarnings("unchecked") + public void ensureValid(String name, Object o) { + String configValue = (String) o; + if (StringUtils.isEmpty(configValue)) { + return; + } + + List containerTopicMapList = convertToList(configValue); + + // validate each item should be in topic#container format + boolean invalidFormatExists = + containerTopicMapList + .stream() + .anyMatch(containerTopicMap -> + containerTopicMap + .split(CosmosSourceContainersConfig.CONTAINER_TOPIC_MAP_SEPARATOR) + .length != 2); + + if (invalidFormatExists) { + throw new ConfigException(name, o, INVALID_TOPIC_MAP_FORMAT); + } + } + + @Override + public String toString() { + return "Containers topic map"; + } + } + public static class ChangeFeedModeValidator implements ConfigDef.Validator { @Override @SuppressWarnings("unchecked") diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosSourceContainersConfig.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosSourceContainersConfig.java index 7f9a2d4284e85..2d61b49fedb06 100644 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosSourceContainersConfig.java +++ b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosSourceContainersConfig.java @@ -6,7 +6,6 @@ import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; import java.util.List; -import java.util.Map; import static com.azure.cosmos.implementation.guava25.base.Preconditions.checkArgument; import static com.azure.cosmos.implementation.guava25.base.Preconditions.checkNotNull; @@ -17,13 +16,13 @@ public class CosmosSourceContainersConfig { private final String databaseName; private final boolean includeAllContainers; private final List includedContainers; - private final Map containerToTopicMap; + private final List containersTopicMap; public CosmosSourceContainersConfig( String databaseName, boolean includeAllContainers, List includedContainers, - Map containerToTopicMap) { + List containersTopicMap) { checkArgument(StringUtils.isNotEmpty(databaseName), "Argument 'databaseName' can not be null"); checkNotNull(includedContainers, "Argument 'includedContainers' can not be null"); @@ -31,7 +30,7 @@ public CosmosSourceContainersConfig( this.databaseName = databaseName; this.includeAllContainers = includeAllContainers; this.includedContainers = includedContainers; - this.containerToTopicMap = containerToTopicMap; + this.containersTopicMap = containersTopicMap; } public String getDatabaseName() { @@ -46,7 +45,7 @@ public List getIncludedContainers() { return includedContainers; } - public Map getContainerToTopicMap() { - return containerToTopicMap; + public List getContainersTopicMap() { + return containersTopicMap; } } diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosSourceTask.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosSourceTask.java index ec358965e765f..94593c06c6cb0 100644 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosSourceTask.java +++ b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosSourceTask.java @@ -14,8 +14,8 @@ import com.azure.cosmos.implementation.feedranges.FeedRangeEpkImpl; import com.azure.cosmos.implementation.guava25.base.Stopwatch; import com.azure.cosmos.kafka.connect.implementation.CosmosClientStore; -import com.azure.cosmos.kafka.connect.implementation.KafkaCosmosConstants; -import com.azure.cosmos.kafka.connect.implementation.KafkaCosmosExceptionsHelper; +import com.azure.cosmos.kafka.connect.implementation.CosmosConstants; +import com.azure.cosmos.kafka.connect.implementation.CosmosExceptionsHelper; import com.azure.cosmos.models.CosmosChangeFeedRequestOptions; import com.azure.cosmos.models.FeedRange; import com.azure.cosmos.models.FeedResponse; @@ -46,7 +46,7 @@ public class CosmosSourceTask extends SourceTask { @Override public String version() { - return KafkaCosmosConstants.CURRENT_VERSION; + return CosmosConstants.CURRENT_VERSION; } @Override @@ -112,7 +112,7 @@ public List poll() { this.taskUnitsQueue.add(taskUnit); // TODO[Public Preview]: add checking for max retries checking - throw KafkaCosmosExceptionsHelper.convertToConnectException(e, "PollTask failed"); + throw CosmosExceptionsHelper.convertToConnectException(e, "PollTask failed"); } } @@ -174,7 +174,7 @@ private Pair, Boolean> executeFeedRangeTask(FeedRangeTaskUnit return Pair.of(records, false); }) .onErrorResume(throwable -> { - if (KafkaCosmosExceptionsHelper.isFeedRangeGoneException(throwable)) { + if (CosmosExceptionsHelper.isFeedRangeGoneException(throwable)) { return this.handleFeedRangeGone(feedRangeTaskUnit) .map(shouldRemoveOriginalTaskUnit -> Pair.of(new ArrayList<>(), shouldRemoveOriginalTaskUnit)); } diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/MetadataMonitorThread.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/MetadataMonitorThread.java index 699b3ad9375d5..b9dc02ae85a37 100644 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/MetadataMonitorThread.java +++ b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/MetadataMonitorThread.java @@ -11,7 +11,7 @@ import com.azure.cosmos.implementation.feedranges.FeedRangeEpkImpl; import com.azure.cosmos.implementation.feedranges.FeedRangeInternal; import com.azure.cosmos.implementation.routing.Range; -import com.azure.cosmos.kafka.connect.implementation.KafkaCosmosExceptionsHelper; +import com.azure.cosmos.kafka.connect.implementation.CosmosExceptionsHelper; import com.azure.cosmos.models.CosmosContainerProperties; import com.azure.cosmos.models.SqlParameter; import com.azure.cosmos.models.SqlQuerySpec; @@ -140,7 +140,7 @@ public Mono> getAllContainers() { .byPage() .flatMapIterable(response -> response.getResults()) .collectList() - .onErrorMap(throwable -> KafkaCosmosExceptionsHelper.convertToConnectException(throwable, "getAllContainers failed.")); + .onErrorMap(throwable -> CosmosExceptionsHelper.convertToConnectException(throwable, "getAllContainers failed.")); } public List getContainerRidsFromOffset() { diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/module-info.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/module-info.java index cfea2ab8a3213..5d8cd88ccae6a 100644 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/module-info.java +++ b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/module-info.java @@ -7,7 +7,6 @@ requires transitive com.azure.cosmos; requires kafka.clients; requires connect.api; - requires json.path; // public API surface area diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/CosmosDBSinkConnectorTest.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/CosmosDBSinkConnectorTest.java deleted file mode 100644 index cc332096b5e16..0000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/CosmosDBSinkConnectorTest.java +++ /dev/null @@ -1,154 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect; - -import com.azure.cosmos.implementation.Strings; -import com.azure.cosmos.implementation.TestConfigurations; -import com.azure.cosmos.kafka.connect.implementation.sink.CosmosSinkTask; -import com.azure.cosmos.kafka.connect.implementation.sink.IdStrategies; -import com.azure.cosmos.kafka.connect.implementation.sink.ItemWriteStrategy; -import org.apache.kafka.common.config.Config; -import org.apache.kafka.common.config.ConfigDef; -import org.apache.kafka.common.config.ConfigValue; -import org.testng.annotations.Test; - -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.stream.Collectors; - -import static com.azure.cosmos.kafka.connect.CosmosDBSinkConnectorTest.SinkConfigs.ALL_VALID_CONFIGS; -import static org.assertj.core.api.AssertionsForClassTypes.assertThat; -import static org.testng.Assert.assertEquals; - -public class CosmosDBSinkConnectorTest extends KafkaCosmosTestSuiteBase { - @Test(groups = "unit") - public void taskClass() { - CosmosDBSinkConnector sinkConnector = new CosmosDBSinkConnector(); - assertEquals(sinkConnector.taskClass(), CosmosSinkTask.class); - } - - @Test(groups = "unit") - public void config() { - CosmosDBSinkConnector sinkConnector = new CosmosDBSinkConnector(); - ConfigDef configDef = sinkConnector.config(); - Map configs = configDef.configKeys(); - List> allValidConfigs = ALL_VALID_CONFIGS; - - for (KafkaCosmosConfigEntry sinkConfigEntry : allValidConfigs) { - assertThat(configs.containsKey(sinkConfigEntry.getName())).isTrue(); - - configs.containsKey(sinkConfigEntry.getName()); - if (sinkConfigEntry.isOptional()) { - assertThat(configs.get(sinkConfigEntry.getName()).defaultValue).isEqualTo(sinkConfigEntry.getDefaultValue()); - } else { - assertThat(configs.get(sinkConfigEntry.getName()).defaultValue).isEqualTo(ConfigDef.NO_DEFAULT_VALUE); - } - } - } - - @Test(groups = "unit") - public void requiredConfig() { - Config config = new CosmosDBSinkConnector().validate(Collections.emptyMap()); - Map> errorMessages = config.configValues().stream() - .collect(Collectors.toMap(ConfigValue::name, ConfigValue::errorMessages)); - assertThat(errorMessages.get("kafka.connect.cosmos.accountEndpoint").size()).isGreaterThan(0); - assertThat(errorMessages.get("kafka.connect.cosmos.accountKey").size()).isGreaterThan(0); - assertThat(errorMessages.get("kafka.connect.cosmos.sink.database.name").size()).isGreaterThan(0); - assertThat(errorMessages.get("kafka.connect.cosmos.sink.containers.topicMap").size()).isGreaterThan(0); - } - - @Test(groups = "unit") - public void taskConfigs() { - CosmosDBSinkConnector sinkConnector = new CosmosDBSinkConnector(); - - Map sinkConfigMap = new HashMap<>(); - sinkConfigMap.put("kafka.connect.cosmos.accountEndpoint", TestConfigurations.HOST); - sinkConfigMap.put("kafka.connect.cosmos.accountKey", TestConfigurations.MASTER_KEY); - sinkConfigMap.put("kafka.connect.cosmos.sink.database.name", databaseName); - sinkConfigMap.put("kafka.connect.cosmos.sink.containers.topicMap", singlePartitionContainerName + "#" + singlePartitionContainerName); - sinkConnector.start(sinkConfigMap); - - int maxTask = 2; - List> taskConfigs = sinkConnector.taskConfigs(maxTask); - assertThat(taskConfigs.size()).isEqualTo(maxTask); - - for (Map taskConfig : taskConfigs) { - assertThat(taskConfig.get("kafka.connect.cosmos.accountEndpoint")).isEqualTo(TestConfigurations.HOST); - assertThat(taskConfig.get("kafka.connect.cosmos.accountKey")).isEqualTo(TestConfigurations.MASTER_KEY); - assertThat(taskConfig.get("kafka.connect.cosmos.sink.database.name")).isEqualTo(databaseName); - assertThat(taskConfig.get("kafka.connect.cosmos.sink.containers.topicMap")) - .isEqualTo(singlePartitionContainerName + "#" + singlePartitionContainerName); - } - } - - @Test(groups = "unit") - public void misFormattedConfig() { - CosmosDBSinkConnector sinkConnector = new CosmosDBSinkConnector(); - Map sinkConfigMap = this.getValidSinkConfig(); - - String topicMapConfigName = "kafka.connect.cosmos.sink.containers.topicMap"; - sinkConfigMap.put(topicMapConfigName, UUID.randomUUID().toString()); - - Config validatedConfig = sinkConnector.validate(sinkConfigMap); - ConfigValue configValue = - validatedConfig - .configValues() - .stream() - .filter(config -> config.name().equalsIgnoreCase(topicMapConfigName)) - .findFirst() - .get(); - - assertThat(configValue.errorMessages()).isNotNull(); - assertThat( - configValue - .errorMessages() - .get(0) - .contains( - "The topic-container map should be a comma-delimited list of Kafka topic to Cosmos containers." + - " Each mapping should be a pair of Kafka topic and Cosmos container separated by '#'." + - " For example: topic1#con1,topic2#con2.")) - .isTrue(); - - // TODO[Public Preview]: add other config validations - } - - private Map getValidSinkConfig() { - Map sinkConfigMap = new HashMap<>(); - sinkConfigMap.put("kafka.connect.cosmos.accountEndpoint", TestConfigurations.HOST); - sinkConfigMap.put("kafka.connect.cosmos.accountKey", TestConfigurations.MASTER_KEY); - sinkConfigMap.put("kafka.connect.cosmos.sink.database.name", databaseName); - sinkConfigMap.put("kafka.connect.cosmos.sink.containers.topicMap", singlePartitionContainerName + "#" + singlePartitionContainerName); - - return sinkConfigMap; - } - - public static class SinkConfigs { - public static final List> ALL_VALID_CONFIGS = Arrays.asList( - new KafkaCosmosConfigEntry("kafka.connect.cosmos.accountEndpoint", null, false), - new KafkaCosmosConfigEntry("kafka.connect.cosmos.accountKey", null, false), - new KafkaCosmosConfigEntry("kafka.connect.cosmos.useGatewayMode", false, true), - new KafkaCosmosConfigEntry("kafka.connect.cosmos.preferredRegionsList", Strings.Emtpy, true), - new KafkaCosmosConfigEntry("kafka.connect.cosmos.applicationName", Strings.Emtpy, true), - new KafkaCosmosConfigEntry("kafka.connect.cosmos.sink.errors.tolerance", "None", true), - new KafkaCosmosConfigEntry("kafka.connect.cosmos.sink.bulk.enabled", true, true), - new KafkaCosmosConfigEntry("kafka.connect.cosmos.sink.bulk.maxConcurrentCosmosPartitions", -1, true), - new KafkaCosmosConfigEntry("kafka.connect.cosmos.sink.bulk.initialBatchSize", 1, true), - new KafkaCosmosConfigEntry( - "kafka.connect.cosmos.sink.write.strategy", - ItemWriteStrategy.ITEM_OVERWRITE.getName(), - true), - new KafkaCosmosConfigEntry("kafka.connect.cosmos.sink.maxRetryCount", 10, true), - new KafkaCosmosConfigEntry("kafka.connect.cosmos.sink.database.name", null, false), - new KafkaCosmosConfigEntry("kafka.connect.cosmos.sink.containers.topicMap", null, false), - new KafkaCosmosConfigEntry( - "kafka.connect.cosmos.sink.id.strategy", - IdStrategies.PROVIDED_IN_VALUE_STRATEGY.getName(), - true) - ); - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/CosmosDBSourceConnectorTest.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/CosmosDBSourceConnectorTest.java index a69da0cc6bf4b..9bddf4b229e19 100644 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/CosmosDBSourceConnectorTest.java +++ b/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/CosmosDBSourceConnectorTest.java @@ -66,9 +66,9 @@ public void config() { CosmosDBSourceConnector sourceConnector = new CosmosDBSourceConnector(); ConfigDef configDef = sourceConnector.config(); Map configs = configDef.configKeys(); - List> allValidConfigs = ALL_VALID_CONFIGS; + List> allValidConfigs = ALL_VALID_CONFIGS; - for (KafkaCosmosConfigEntry sourceConfigEntry : allValidConfigs) { + for (SourceConfigEntry sourceConfigEntry : allValidConfigs) { assertThat(configs.containsKey(sourceConfigEntry.getName())).isTrue(); configs.containsKey(sourceConfigEntry.getName()); @@ -374,15 +374,15 @@ public void getTaskConfigsAfterMerge() throws JsonProcessingException { @Test(groups = "unit") public void missingRequiredConfig() { - List> requiredConfigs = + List> requiredConfigs = ALL_VALID_CONFIGS .stream() - .filter(sourceConfigEntry -> !sourceConfigEntry.isOptional()) + .filter(sourceConfigEntry -> !sourceConfigEntry.isOptional) .collect(Collectors.toList()); assertThat(requiredConfigs.size()).isGreaterThan(1); CosmosDBSourceConnector sourceConnector = new CosmosDBSourceConnector(); - for (KafkaCosmosConfigEntry configEntry : requiredConfigs) { + for (SourceConfigEntry configEntry : requiredConfigs) { Map sourceConfigMap = this.getValidSourceConfig(); sourceConfigMap.remove(configEntry.getName()); @@ -391,7 +391,7 @@ public void missingRequiredConfig() { validatedConfig .configValues() .stream() - .filter(config -> config.name().equalsIgnoreCase(configEntry.getName())) + .filter(config -> config.name().equalsIgnoreCase(configEntry.name)) .findFirst() .get(); @@ -579,33 +579,57 @@ private void validateMetadataTask( assertThat(expectedMetadataTaskUnit).isEqualTo(metadataTaskUnitFromTaskConfig); } + public static class SourceConfigEntry { + private final String name; + private final T defaultValue; + private final boolean isOptional; + + public SourceConfigEntry(String name, T defaultValue, boolean isOptional) { + this.name = name; + this.defaultValue = defaultValue; + this.isOptional = isOptional; + } + + public String getName() { + return name; + } + + public T getDefaultValue() { + return defaultValue; + } + + public boolean isOptional() { + return isOptional; + } + } + public static class SourceConfigs { - public static final List> ALL_VALID_CONFIGS = Arrays.asList( - new KafkaCosmosConfigEntry("kafka.connect.cosmos.accountEndpoint", null, false), - new KafkaCosmosConfigEntry("kafka.connect.cosmos.accountKey", null, false), - new KafkaCosmosConfigEntry("kafka.connect.cosmos.useGatewayMode", false, true), - new KafkaCosmosConfigEntry("kafka.connect.cosmos.preferredRegionsList", Strings.Emtpy, true), - new KafkaCosmosConfigEntry("kafka.connect.cosmos.applicationName", Strings.Emtpy, true), - new KafkaCosmosConfigEntry("kafka.connect.cosmos.source.database.name", null, false), - new KafkaCosmosConfigEntry("kafka.connect.cosmos.source.containers.includeAll", false, true), - new KafkaCosmosConfigEntry("kafka.connect.cosmos.source.containers.includedList", Strings.Emtpy, true), - new KafkaCosmosConfigEntry("kafka.connect.cosmos.source.containers.topicMap", Strings.Emtpy, true), - new KafkaCosmosConfigEntry( + public static final List> ALL_VALID_CONFIGS = Arrays.asList( + new SourceConfigEntry("kafka.connect.cosmos.accountEndpoint", null, false), + new SourceConfigEntry("kafka.connect.cosmos.accountKey", null, false), + new SourceConfigEntry("kafka.connect.cosmos.useGatewayMode", false, true), + new SourceConfigEntry("kafka.connect.cosmos.preferredRegionsList", Strings.Emtpy, true), + new SourceConfigEntry("kafka.connect.cosmos.applicationName", Strings.Emtpy, true), + new SourceConfigEntry("kafka.connect.cosmos.source.database.name", null, false), + new SourceConfigEntry("kafka.connect.cosmos.source.containers.includeAll", false, true), + new SourceConfigEntry("kafka.connect.cosmos.source.containers.includedList", Strings.Emtpy, true), + new SourceConfigEntry("kafka.connect.cosmos.source.containers.topicMap", Strings.Emtpy, true), + new SourceConfigEntry( "kafka.connect.cosmos.source.changeFeed.startFrom", CosmosChangeFeedStartFromModes.BEGINNING.getName(), true), - new KafkaCosmosConfigEntry( + new SourceConfigEntry( "kafka.connect.cosmos.source.changeFeed.mode", CosmosChangeFeedModes.LATEST_VERSION.getName(), true), - new KafkaCosmosConfigEntry("kafka.connect.cosmos.source.changeFeed.maxItemCountHint", 1000, true), - new KafkaCosmosConfigEntry("kafka.connect.cosmos.source.metadata.poll.delay.ms", 5 * 60 * 1000, true), - new KafkaCosmosConfigEntry( + new SourceConfigEntry("kafka.connect.cosmos.source.changeFeed.maxItemCountHint", 1000, true), + new SourceConfigEntry("kafka.connect.cosmos.source.metadata.poll.delay.ms", 5 * 60 * 1000, true), + new SourceConfigEntry( "kafka.connect.cosmos.source.metadata.storage.topic", "_cosmos.metadata.topic", true), - new KafkaCosmosConfigEntry("kafka.connect.cosmos.source.messageKey.enabled", true, true), - new KafkaCosmosConfigEntry("kafka.connect.cosmos.source.messageKey.field", "id", true) + new SourceConfigEntry("kafka.connect.cosmos.source.messageKey.enabled", true, true), + new SourceConfigEntry("kafka.connect.cosmos.source.messageKey.field", "id", true) ); } } diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/CosmosDbSinkConnectorITest.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/CosmosDbSinkConnectorITest.java deleted file mode 100644 index 9a0714e9f5d03..0000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/CosmosDbSinkConnectorITest.java +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect; - -import com.azure.cosmos.CosmosAsyncClient; -import com.azure.cosmos.CosmosAsyncContainer; -import com.azure.cosmos.implementation.Utils; -import com.azure.cosmos.kafka.connect.implementation.CosmosClientStore; -import com.azure.cosmos.kafka.connect.implementation.sink.CosmosSinkConfig; -import com.fasterxml.jackson.databind.JsonNode; -import org.apache.kafka.clients.producer.KafkaProducer; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.connect.json.JsonConverter; -import org.apache.kafka.connect.storage.StringConverter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.testng.annotations.Test; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; - -import static org.assertj.core.api.AssertionsForClassTypes.assertThat; - -public class CosmosDbSinkConnectorITest extends KafkaCosmosIntegrationTestSuiteBase { - private static final Logger logger = LoggerFactory.getLogger(CosmosDbSinkConnectorITest.class); - - // TODO[public preview]: add more integration tests - @Test(groups = { "kafka-integration"}, timeOut = TIMEOUT) - public void sinkToSingleContainer() throws InterruptedException { - Map sinkConnectorConfig = new HashMap<>(); - - sinkConnectorConfig.put("topics", singlePartitionContainerName); - sinkConnectorConfig.put("value.converter", JsonConverter.class.getName()); - // TODO[Public Preview]: add tests for with schema - sinkConnectorConfig.put("value.converter.schemas.enable", "false"); - sinkConnectorConfig.put("key.converter", StringConverter.class.getName()); - sinkConnectorConfig.put("connector.class", "com.azure.cosmos.kafka.connect.CosmosDBSinkConnector"); - sinkConnectorConfig.put("kafka.connect.cosmos.accountEndpoint", KafkaCosmosTestConfigurations.HOST); - sinkConnectorConfig.put("kafka.connect.cosmos.accountKey", KafkaCosmosTestConfigurations.MASTER_KEY); - sinkConnectorConfig.put("kafka.connect.cosmos.applicationName", "Test"); - sinkConnectorConfig.put("kafka.connect.cosmos.sink.database.name", databaseName); - sinkConnectorConfig.put("kafka.connect.cosmos.sink.containers.topicMap", singlePartitionContainerName + "#" + singlePartitionContainerName); - - // Create topic ahead of time - kafkaCosmosConnectContainer.createTopic(singlePartitionContainerName, 1); - - CosmosSinkConfig sinkConfig = new CosmosSinkConfig(sinkConnectorConfig); - CosmosAsyncClient client = CosmosClientStore.getCosmosClient(sinkConfig.getAccountConfig()); - CosmosAsyncContainer container = client.getDatabase(databaseName).getContainer(singlePartitionContainerName); - - String connectorName = "simpleTest-" + UUID.randomUUID(); - try { - // register the sink connector - kafkaCosmosConnectContainer.registerConnector(connectorName, sinkConnectorConfig); - - KafkaProducer kafkaProducer = kafkaCosmosConnectContainer.getProducer(); - - // first create few records in the topic - logger.info("Creating sink records..."); - List recordValueIds = new ArrayList<>(); - for (int i = 0; i < 10; i++) { - TestItem testItem = TestItem.createNewItem(); - ProducerRecord record = - new ProducerRecord<>(singlePartitionContainerName, testItem.getId(), Utils.getSimpleObjectMapper().valueToTree(testItem)); - kafkaProducer.send(record); - recordValueIds.add(testItem.getId()); - } - - // Wait for some time for the sink connector to process all records - Thread.sleep(5000); - // read from the container and verify all the items are created - String query = "select * from c"; - List createdItemIds = container.queryItems(query, TestItem.class) - .byPage() - .flatMapIterable(response -> response.getResults()) - .map(TestItem::getId) - .collectList() - .block(); - assertThat(createdItemIds.size()).isEqualTo(recordValueIds.size()); - assertThat(createdItemIds.containsAll(recordValueIds)).isTrue(); - - } finally { - if (client != null) { - logger.info("cleaning container {}", singlePartitionContainerName); - cleanUpContainer(client, databaseName, singlePartitionContainerName); - client.close(); - } - - // IMPORTANT: remove the connector after use - if (kafkaCosmosConnectContainer != null) { - kafkaCosmosConnectContainer.deleteConnector(connectorName); - } - } - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/KafkaCosmosConfigEntry.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/KafkaCosmosConfigEntry.java deleted file mode 100644 index ec0ee09b3e65f..0000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/KafkaCosmosConfigEntry.java +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect; - -public class KafkaCosmosConfigEntry { - private final String name; - private final T defaultValue; - private final boolean isOptional; - - public KafkaCosmosConfigEntry(String name, T defaultValue, boolean isOptional) { - this.name = name; - this.defaultValue = defaultValue; - this.isOptional = isOptional; - } - - public String getName() { - return name; - } - - public T getDefaultValue() { - return defaultValue; - } - - public boolean isOptional() { - return isOptional; - } - -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/KafkaCosmosConnectContainer.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/KafkaCosmosConnectContainer.java index f0bef9d3d65cc..6587222aaee5e 100644 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/KafkaCosmosConnectContainer.java +++ b/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/KafkaCosmosConnectContainer.java @@ -8,12 +8,8 @@ import org.apache.kafka.clients.admin.AdminClient; import org.apache.kafka.clients.admin.NewTopic; import org.apache.kafka.clients.consumer.KafkaConsumer; -import org.apache.kafka.clients.producer.KafkaProducer; -import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.common.serialization.StringDeserializer; -import org.apache.kafka.common.serialization.StringSerializer; import org.apache.kafka.connect.json.JsonDeserializer; -import org.apache.kafka.connect.json.JsonSerializer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.sourcelab.kafka.connect.apiclient.Configuration; @@ -32,7 +28,6 @@ public class KafkaCosmosConnectContainer extends GenericContainer kafkaConsumer; - private KafkaProducer kafkaProducer; private AdminClient adminClient; private int replicationFactor = 1; @@ -61,7 +56,7 @@ private void defaultConfig() { private Properties defaultConsumerConfig() { Properties kafkaConsumerProperties = new Properties(); - kafkaConsumerProperties.put("group.id", "IntegrationTest-Consumer"); + kafkaConsumerProperties.put("group.id", "IntegrationTest"); kafkaConsumerProperties.put("value.deserializer", JsonDeserializer.class.getName()); kafkaConsumerProperties.put("key.deserializer", StringDeserializer.class.getName()); kafkaConsumerProperties.put("sasl.mechanism", "PLAIN"); @@ -70,21 +65,6 @@ private Properties defaultConsumerConfig() { return kafkaConsumerProperties; } - private Properties defaultProducerConfig() { - Properties kafkaProducerProperties = new Properties(); - - kafkaProducerProperties.put(ProducerConfig.CLIENT_ID_CONFIG, "IntegrationTest-producer"); - kafkaProducerProperties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); - kafkaProducerProperties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, JsonSerializer.class.getName()); - kafkaProducerProperties.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, 2000L); - kafkaProducerProperties.put(ProducerConfig.ACKS_CONFIG, "all"); - kafkaProducerProperties.put("sasl.mechanism", "PLAIN"); - kafkaProducerProperties.put("client.dns.lookup", "use_all_dns_ips"); - kafkaProducerProperties.put("session.timeout.ms", "45000"); - - return kafkaProducerProperties; - } - public KafkaCosmosConnectContainer withLocalKafkaContainer(final KafkaContainer kafkaContainer) { withNetwork(kafkaContainer.getNetwork()); @@ -112,11 +92,6 @@ public KafkaCosmosConnectContainer withLocalBootstrapServer(String localBootstra Properties consumerProperties = defaultConsumerConfig(); consumerProperties.put("bootstrap.servers", localBootstrapServer); this.kafkaConsumer = new KafkaConsumer<>(consumerProperties); - - Properties producerProperties = defaultProducerConfig(); - producerProperties.put("bootstrap.servers", localBootstrapServer); - this.kafkaProducer = new KafkaProducer<>(producerProperties); - this.adminClient = this.getAdminClient(localBootstrapServer); return self(); } @@ -129,14 +104,6 @@ public KafkaCosmosConnectContainer withCloudBootstrapServer() { consumerProperties.put("sasl.mechanism", "PLAIN"); this.kafkaConsumer = new KafkaConsumer<>(consumerProperties); - - Properties producerProperties = defaultProducerConfig(); - producerProperties.put("bootstrap.servers", KafkaCosmosTestConfigurations.BOOTSTRAP_SERVER); - producerProperties.put("sasl.jaas.config", KafkaCosmosTestConfigurations.SASL_JAAS); - producerProperties.put("security.protocol", "SASL_SSL"); - producerProperties.put("sasl.mechanism", "PLAIN"); - this.kafkaProducer = new KafkaProducer<>(producerProperties); - this.adminClient = this.getAdminClient(KafkaCosmosTestConfigurations.BOOTSTRAP_SERVER); this.replicationFactor = 3; return self(); @@ -175,10 +142,6 @@ public KafkaConsumer getConsumer() { return this.kafkaConsumer; } - public KafkaProducer getProducer() { - return this.kafkaProducer; - } - public String getTarget() { return "http://" + getContainerIpAddress() + ":" + getMappedPort(KAFKA_CONNECT_PORT); } diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/KafkaCosmosReflectionUtils.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/KafkaCosmosReflectionUtils.java index cce96e741deb3..e15f33693460c 100644 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/KafkaCosmosReflectionUtils.java +++ b/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/KafkaCosmosReflectionUtils.java @@ -4,12 +4,10 @@ package com.azure.cosmos.kafka.connect; import com.azure.cosmos.CosmosAsyncClient; -import com.azure.cosmos.kafka.connect.implementation.sink.CosmosSinkTask; import com.azure.cosmos.kafka.connect.implementation.source.CosmosSourceConfig; import com.azure.cosmos.kafka.connect.implementation.source.CosmosSourceOffsetStorageReader; import com.azure.cosmos.kafka.connect.implementation.source.MetadataMonitorThread; import org.apache.commons.lang3.reflect.FieldUtils; -import org.apache.kafka.connect.sink.SinkTaskContext; import org.apache.kafka.connect.storage.OffsetStorageReader; public class KafkaCosmosReflectionUtils { @@ -61,12 +59,4 @@ public static CosmosSourceOffsetStorageReader getSourceOffsetStorageReader(Cosmo public static OffsetStorageReader getOffsetStorageReader(CosmosSourceOffsetStorageReader sourceOffsetStorageReader) { return get(sourceOffsetStorageReader,"offsetStorageReader"); } - - public static void setSinkTaskContext(CosmosSinkTask sinkTask, SinkTaskContext sinkTaskContext) { - set(sinkTask, sinkTaskContext, "context"); - } - - public static CosmosAsyncClient getSinkTaskCosmosClient(CosmosSinkTask sinkTask) { - return get(sinkTask,"cosmosClient"); - } } diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/KafkaCosmosTestSuiteBase.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/KafkaCosmosTestSuiteBase.java index 8e601948a6c31..faaebde85a391 100644 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/KafkaCosmosTestSuiteBase.java +++ b/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/KafkaCosmosTestSuiteBase.java @@ -88,7 +88,7 @@ protected static CosmosContainerProperties getSinglePartitionContainer(CosmosAsy credential = new AzureKeyCredential(KafkaCosmosTestConfigurations.MASTER_KEY); } - @BeforeSuite(groups = { "kafka", "kafka-integration" }, timeOut = SUITE_SETUP_TIMEOUT) + @BeforeSuite(groups = { "kafka" }, timeOut = SUITE_SETUP_TIMEOUT) public static void beforeSuite() { logger.info("beforeSuite Started"); @@ -120,19 +120,7 @@ public static void beforeSuite() { } } - @BeforeSuite(groups = { "unit" }, timeOut = SUITE_SETUP_TIMEOUT) - public static void beforeSuiteUnit() { - logger.info("beforeSuite for unit tests started"); - - databaseName = - StringUtils.isEmpty(databaseName) ? "KafkaCosmosTest-" + UUID.randomUUID() : databaseName; - multiPartitionContainerName = - StringUtils.isEmpty(multiPartitionContainerName) ? UUID.randomUUID().toString() : multiPartitionContainerName; - singlePartitionContainerName = - StringUtils.isEmpty(singlePartitionContainerName) ? UUID.randomUUID().toString() : singlePartitionContainerName; - } - - @AfterSuite(groups = { "kafka", "kafka-integration" }, timeOut = SUITE_SHUTDOWN_TIMEOUT) + @AfterSuite(groups = { "kafka" }, timeOut = SUITE_SHUTDOWN_TIMEOUT) public static void afterSuite() { logger.info("afterSuite Started"); diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosSinkTaskTest.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosSinkTaskTest.java deleted file mode 100644 index 89cc8322b0506..0000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosSinkTaskTest.java +++ /dev/null @@ -1,571 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink; - -import com.azure.cosmos.CosmosAsyncClient; -import com.azure.cosmos.CosmosAsyncContainer; -import com.azure.cosmos.implementation.TestConfigurations; -import com.azure.cosmos.implementation.Utils; -import com.azure.cosmos.kafka.connect.KafkaCosmosReflectionUtils; -import com.azure.cosmos.kafka.connect.KafkaCosmosTestSuiteBase; -import com.azure.cosmos.kafka.connect.TestItem; -import com.azure.cosmos.kafka.connect.implementation.source.JsonToStruct; -import com.azure.cosmos.models.CosmosContainerProperties; -import com.azure.cosmos.models.PartitionKey; -import com.azure.cosmos.test.faultinjection.CosmosFaultInjectionHelper; -import com.azure.cosmos.test.faultinjection.FaultInjectionConditionBuilder; -import com.azure.cosmos.test.faultinjection.FaultInjectionResultBuilders; -import com.azure.cosmos.test.faultinjection.FaultInjectionRule; -import com.azure.cosmos.test.faultinjection.FaultInjectionRuleBuilder; -import com.azure.cosmos.test.faultinjection.FaultInjectionServerErrorType; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.node.ObjectNode; -import org.apache.kafka.connect.data.ConnectSchema; -import org.apache.kafka.connect.data.Schema; -import org.apache.kafka.connect.data.SchemaAndValue; -import org.apache.kafka.connect.sink.SinkRecord; -import org.apache.kafka.connect.sink.SinkTaskContext; -import org.mockito.Mockito; -import org.testng.annotations.DataProvider; -import org.testng.annotations.Test; -import reactor.core.publisher.Mono; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.stream.Collectors; - -import static org.assertj.core.api.AssertionsForClassTypes.assertThat; - - -public class CosmosSinkTaskTest extends KafkaCosmosTestSuiteBase { - @DataProvider(name = "sinkTaskParameterProvider") - public Object[][] sinkTaskParameterProvider() { - return new Object[][]{ - // flag to indicate whether bulk enabled or not, sink record value schema - { true, Schema.Type.MAP }, - { false, Schema.Type.MAP }, - { true, Schema.Type.STRUCT }, - { false, Schema.Type.STRUCT } - }; - } - - @DataProvider(name = "bulkEnableParameterProvider") - public Object[][] bulkEnableParameterProvider() { - return new Object[][]{ - // flag to indicate whether bulk enabled or not - { true }, - { false } - }; - } - - @Test(groups = { "kafka" }, dataProvider = "sinkTaskParameterProvider", timeOut = TIMEOUT) - public void sinkWithValidRecords(boolean bulkEnabled, Schema.Type valueSchemaType) { - String topicName = singlePartitionContainerName; - - Map sinkConfigMap = new HashMap<>(); - sinkConfigMap.put("kafka.connect.cosmos.accountEndpoint", TestConfigurations.HOST); - sinkConfigMap.put("kafka.connect.cosmos.accountKey", TestConfigurations.MASTER_KEY); - sinkConfigMap.put("kafka.connect.cosmos.sink.database.name", databaseName); - sinkConfigMap.put("kafka.connect.cosmos.sink.containers.topicMap", topicName + "#" + singlePartitionContainerName); - sinkConfigMap.put("kafka.connect.cosmos.sink.bulk.enabled", String.valueOf(bulkEnabled)); - - CosmosSinkTask sinkTask = new CosmosSinkTask(); - SinkTaskContext sinkTaskContext = Mockito.mock(SinkTaskContext.class); - Mockito.when(sinkTaskContext.errantRecordReporter()).thenReturn(null); - KafkaCosmosReflectionUtils.setSinkTaskContext(sinkTask, sinkTaskContext); - sinkTask.start(sinkConfigMap); - - CosmosAsyncClient cosmosClient = KafkaCosmosReflectionUtils.getSinkTaskCosmosClient(sinkTask); - CosmosContainerProperties singlePartitionContainerProperties = getSinglePartitionContainer(cosmosClient); - CosmosAsyncContainer container = cosmosClient.getDatabase(databaseName).getContainer(singlePartitionContainerProperties.getId()); - - try { - List sinkRecordList = new ArrayList<>(); - List toBeCreateItems = new ArrayList<>(); - this.createSinkRecords( - 10, - topicName, - valueSchemaType, - toBeCreateItems, - sinkRecordList); - - sinkTask.put(sinkRecordList); - - // get all the items - List writtenItemIds = new ArrayList<>(); - String query = "select * from c"; - container.queryItems(query, TestItem.class) - .byPage() - .flatMap(response -> { - writtenItemIds.addAll( - response.getResults().stream().map(TestItem::getId).collect(Collectors.toList())); - return Mono.empty(); - }) - .blockLast(); - - assertThat(writtenItemIds.size()).isEqualTo(toBeCreateItems.size()); - List toBeCreateItemIds = toBeCreateItems.stream().map(TestItem::getId).collect(Collectors.toList()); - assertThat(writtenItemIds.containsAll(toBeCreateItemIds)).isTrue(); - - } finally { - if (cosmosClient != null) { - cleanUpContainer(cosmosClient, databaseName, singlePartitionContainerProperties.getId()); - sinkTask.stop(); - } - } - } - - @Test(groups = { "kafka" }, dataProvider = "bulkEnableParameterProvider", timeOut = 10 * TIMEOUT) - public void retryOnServiceUnavailable(boolean bulkEnabled) { - String topicName = singlePartitionContainerName; - - Map sinkConfigMap = new HashMap<>(); - sinkConfigMap.put("kafka.connect.cosmos.accountEndpoint", TestConfigurations.HOST); - sinkConfigMap.put("kafka.connect.cosmos.accountKey", TestConfigurations.MASTER_KEY); - sinkConfigMap.put("kafka.connect.cosmos.sink.database.name", databaseName); - sinkConfigMap.put("kafka.connect.cosmos.sink.containers.topicMap", topicName + "#" + singlePartitionContainerName); - sinkConfigMap.put("kafka.connect.cosmos.sink.bulk.enabled", String.valueOf(bulkEnabled)); - - CosmosSinkTask sinkTask = new CosmosSinkTask(); - SinkTaskContext sinkTaskContext = Mockito.mock(SinkTaskContext.class); - Mockito.when(sinkTaskContext.errantRecordReporter()).thenReturn(null); - KafkaCosmosReflectionUtils.setSinkTaskContext(sinkTask, sinkTaskContext); - sinkTask.start(sinkConfigMap); - - CosmosAsyncClient cosmosClient = KafkaCosmosReflectionUtils.getSinkTaskCosmosClient(sinkTask); - CosmosContainerProperties singlePartitionContainerProperties = getSinglePartitionContainer(cosmosClient); - CosmosAsyncContainer container = cosmosClient.getDatabase(databaseName).getContainer(singlePartitionContainerProperties.getId()); - - // configure fault injection rule - FaultInjectionRule goneExceptionRule = - new FaultInjectionRuleBuilder("goneExceptionRule-" + UUID.randomUUID()) - .condition(new FaultInjectionConditionBuilder().build()) - .result( - FaultInjectionResultBuilders - .getResultBuilder(FaultInjectionServerErrorType.GONE) - .build()) - // high enough so the batch requests will fail with 503 in the first time but low enough so the second retry from kafka connector can succeed - .hitLimit(10) - .build(); - - try { - List sinkRecordList = new ArrayList<>(); - List toBeCreateItems = new ArrayList<>(); - this.createSinkRecords( - 10, - topicName, - Schema.Type.STRUCT, - toBeCreateItems, - sinkRecordList); - - CosmosFaultInjectionHelper.configureFaultInjectionRules(container, Arrays.asList(goneExceptionRule)).block(); - sinkTask.put(sinkRecordList); - - // get all the items - List writtenItemIds = new ArrayList<>(); - String query = "select * from c"; - container.queryItems(query, TestItem.class) - .byPage() - .flatMap(response -> { - writtenItemIds.addAll( - response.getResults().stream().map(TestItem::getId).collect(Collectors.toList())); - return Mono.empty(); - }) - .blockLast(); - - assertThat(writtenItemIds.size()).isEqualTo(toBeCreateItems.size()); - List toBeCreateItemIds = toBeCreateItems.stream().map(TestItem::getId).collect(Collectors.toList()); - assertThat(toBeCreateItemIds.containsAll(writtenItemIds)).isTrue(); - - } finally { - goneExceptionRule.disable(); - if (cosmosClient != null) { - cleanUpContainer(cosmosClient, databaseName, singlePartitionContainerProperties.getId()); - cosmosClient.close(); - } - } - } - - @Test(groups = { "kafka" }, dataProvider = "bulkEnableParameterProvider", timeOut = TIMEOUT) - public void sinkWithItemAppend(boolean bulkEnabled) { - String topicName = singlePartitionContainerName; - - Map sinkConfigMap = new HashMap<>(); - sinkConfigMap.put("kafka.connect.cosmos.accountEndpoint", TestConfigurations.HOST); - sinkConfigMap.put("kafka.connect.cosmos.accountKey", TestConfigurations.MASTER_KEY); - sinkConfigMap.put("kafka.connect.cosmos.sink.database.name", databaseName); - sinkConfigMap.put("kafka.connect.cosmos.sink.containers.topicMap", topicName + "#" + singlePartitionContainerName); - sinkConfigMap.put("kafka.connect.cosmos.sink.bulk.enabled", String.valueOf(bulkEnabled)); - sinkConfigMap.put("kafka.connect.cosmos.sink.write.strategy", ItemWriteStrategy.ITEM_APPEND.getName()); - - CosmosSinkTask sinkTask = new CosmosSinkTask(); - SinkTaskContext sinkTaskContext = Mockito.mock(SinkTaskContext.class); - Mockito.when(sinkTaskContext.errantRecordReporter()).thenReturn(null); - KafkaCosmosReflectionUtils.setSinkTaskContext(sinkTask, sinkTaskContext); - sinkTask.start(sinkConfigMap); - - CosmosAsyncClient cosmosClient = KafkaCosmosReflectionUtils.getSinkTaskCosmosClient(sinkTask); - CosmosContainerProperties singlePartitionContainerProperties = getSinglePartitionContainer(cosmosClient); - CosmosAsyncContainer container = cosmosClient.getDatabase(databaseName).getContainer(singlePartitionContainerProperties.getId()); - - try { - List sinkRecordList = new ArrayList<>(); - List toBeCreateItems = new ArrayList<>(); - this.createSinkRecords( - 10, - topicName, - Schema.Type.MAP, - toBeCreateItems, - sinkRecordList); - - sinkTask.put(sinkRecordList); - - // get all the items - List writtenItemIds = this.getAllItemIds(container); - - assertThat(toBeCreateItems.size()).isEqualTo(writtenItemIds.size()); - List toBeCreateItemIds = toBeCreateItems.stream().map(TestItem::getId).collect(Collectors.toList()); - assertThat(toBeCreateItemIds.containsAll(writtenItemIds)).isTrue(); - - // add the same batch sink records, 409 should be ignored - sinkTask.put(sinkRecordList); - } finally { - if (cosmosClient != null) { - cleanUpContainer(cosmosClient, databaseName, singlePartitionContainerProperties.getId()); - sinkTask.stop(); - } - } - } - - @Test(groups = { "kafka" }, dataProvider = "bulkEnableParameterProvider", timeOut = 3 * TIMEOUT) - public void sinkWithItemOverwriteIfNotModified(boolean bulkEnabled) { - String topicName = singlePartitionContainerName; - - Map sinkConfigMap = new HashMap<>(); - sinkConfigMap.put("kafka.connect.cosmos.accountEndpoint", TestConfigurations.HOST); - sinkConfigMap.put("kafka.connect.cosmos.accountKey", TestConfigurations.MASTER_KEY); - sinkConfigMap.put("kafka.connect.cosmos.sink.database.name", databaseName); - sinkConfigMap.put("kafka.connect.cosmos.sink.containers.topicMap", topicName + "#" + singlePartitionContainerName); - sinkConfigMap.put("kafka.connect.cosmos.sink.bulk.enabled", String.valueOf(bulkEnabled)); - sinkConfigMap.put("kafka.connect.cosmos.sink.write.strategy", ItemWriteStrategy.ITEM_OVERWRITE_IF_NOT_MODIFIED.getName()); - - CosmosSinkTask sinkTask = new CosmosSinkTask(); - SinkTaskContext sinkTaskContext = Mockito.mock(SinkTaskContext.class); - Mockito.when(sinkTaskContext.errantRecordReporter()).thenReturn(null); - KafkaCosmosReflectionUtils.setSinkTaskContext(sinkTask, sinkTaskContext); - sinkTask.start(sinkConfigMap); - - CosmosAsyncClient cosmosClient = KafkaCosmosReflectionUtils.getSinkTaskCosmosClient(sinkTask); - CosmosContainerProperties singlePartitionContainerProperties = getSinglePartitionContainer(cosmosClient); - CosmosAsyncContainer container = cosmosClient.getDatabase(databaseName).getContainer(singlePartitionContainerProperties.getId()); - - try { - List sinkRecordList = new ArrayList<>(); - List toBeCreateItems = new ArrayList<>(); - this.createSinkRecords( - 10, - topicName, - Schema.Type.MAP, - toBeCreateItems, - sinkRecordList); - - sinkTask.put(sinkRecordList); - - // get all the items - List writtenItemIds = this.getAllItemIds(container); - - assertThat(toBeCreateItems.size()).isEqualTo(writtenItemIds.size()); - List toBeCreateItemIds = toBeCreateItems.stream().map(TestItem::getId).collect(Collectors.toList()); - assertThat(toBeCreateItemIds.containsAll(writtenItemIds)).isTrue(); - - ObjectNode existedItem = - container - .readItem(toBeCreateItems.get(0).getId(), new PartitionKey(toBeCreateItems.get(0).getMypk()), ObjectNode.class) - .block() - .getItem(); - - // test precondition-failed exception will be ignored - logger.info( - "Testing precondition-failed exception will be ignored for ItemWriteStrategy " - + ItemWriteStrategy.ITEM_OVERWRITE_IF_NOT_MODIFIED.getName()); - - ObjectNode itemWithWrongEtag = Utils.getSimpleObjectMapper().createObjectNode(); - itemWithWrongEtag.setAll(existedItem); - itemWithWrongEtag.put("_etag", UUID.randomUUID().toString()); - SinkRecord sinkRecordWithWrongEtag = - this.getSinkRecord( - topicName, - itemWithWrongEtag, - new ConnectSchema(Schema.Type.STRING), - itemWithWrongEtag.get("id").asText(), - Schema.Type.MAP); - - sinkTask.put(Arrays.asList(sinkRecordWithWrongEtag)); - - // test with correct etag, the item can be modified - logger.info( - "Testing item can be modified with correct etag for ItemWriteStrategy " - + ItemWriteStrategy.ITEM_OVERWRITE_IF_NOT_MODIFIED.getName()); - ObjectNode modifiedItem = Utils.getSimpleObjectMapper().createObjectNode(); - modifiedItem.setAll(existedItem); - modifiedItem.put("prop", UUID.randomUUID().toString()); - SinkRecord sinkRecordWithModifiedItem = - this.getSinkRecord( - topicName, - modifiedItem, - new ConnectSchema(Schema.Type.STRING), - modifiedItem.get("id").asText(), - Schema.Type.MAP); - sinkTask.put(Arrays.asList(sinkRecordWithModifiedItem)); - - existedItem = - container - .readItem(toBeCreateItems.get(0).getId(), new PartitionKey(toBeCreateItems.get(0).getMypk()), ObjectNode.class) - .block() - .getItem(); - assertThat(existedItem.get("prop").asText()).isEqualTo(modifiedItem.get("prop").asText()); - - } finally { - if (cosmosClient != null) { - cleanUpContainer(cosmosClient, databaseName, singlePartitionContainerProperties.getId()); - sinkTask.stop(); - } - } - } - - @Test(groups = { "kafka" }, dataProvider = "bulkEnableParameterProvider", timeOut = 3 * TIMEOUT) - public void sinkWithItemDelete(boolean bulkEnabled) { - String topicName = singlePartitionContainerName; - - Map sinkConfigMap = new HashMap<>(); - sinkConfigMap.put("kafka.connect.cosmos.accountEndpoint", TestConfigurations.HOST); - sinkConfigMap.put("kafka.connect.cosmos.accountKey", TestConfigurations.MASTER_KEY); - sinkConfigMap.put("kafka.connect.cosmos.sink.database.name", databaseName); - sinkConfigMap.put("kafka.connect.cosmos.sink.containers.topicMap", topicName + "#" + singlePartitionContainerName); - sinkConfigMap.put("kafka.connect.cosmos.sink.bulk.enabled", String.valueOf(bulkEnabled)); - sinkConfigMap.put("kafka.connect.cosmos.sink.write.strategy", ItemWriteStrategy.ITEM_DELETE.getName()); - - CosmosSinkTask sinkTask = new CosmosSinkTask(); - SinkTaskContext sinkTaskContext = Mockito.mock(SinkTaskContext.class); - Mockito.when(sinkTaskContext.errantRecordReporter()).thenReturn(null); - KafkaCosmosReflectionUtils.setSinkTaskContext(sinkTask, sinkTaskContext); - sinkTask.start(sinkConfigMap); - - CosmosAsyncClient cosmosClient = KafkaCosmosReflectionUtils.getSinkTaskCosmosClient(sinkTask); - CosmosContainerProperties singlePartitionContainerProperties = getSinglePartitionContainer(cosmosClient); - CosmosAsyncContainer container = cosmosClient.getDatabase(databaseName).getContainer(singlePartitionContainerProperties.getId()); - - try { - List sinkRecordList = new ArrayList<>(); - List toBeCreateItems = new ArrayList<>(); - this.createSinkRecords( - 10, - topicName, - Schema.Type.MAP, - toBeCreateItems, - sinkRecordList); - - // first time delete, ignore 404 exceptions - sinkTask.put(sinkRecordList); - - // creating the items in the container - for (TestItem testItem : toBeCreateItems) { - container.createItem(testItem).block(); - } - - // get all the items - List createdItemIds = this.getAllItemIds(container); - - assertThat(toBeCreateItems.size()).isEqualTo(createdItemIds.size()); - List toBeCreateItemIds = toBeCreateItems.stream().map(TestItem::getId).collect(Collectors.toList()); - assertThat(toBeCreateItemIds.containsAll(createdItemIds)).isTrue(); - - // now using the connector to delete the items - sinkTask.put(sinkRecordList); - - // verify all the items have deleted - List existingItemIds = this.getAllItemIds(container); - - assertThat(existingItemIds.isEmpty()).isTrue(); - - } finally { - if (cosmosClient != null) { - cleanUpContainer(cosmosClient, databaseName, singlePartitionContainerProperties.getId()); - sinkTask.stop(); - } - } - } - - @Test(groups = { "kafka" }, dataProvider = "bulkEnableParameterProvider", timeOut = 3 * TIMEOUT) - public void sinkWithItemDeleteIfNotModified(boolean bulkEnabled) throws InterruptedException { - String topicName = singlePartitionContainerName; - - Map sinkConfigMap = new HashMap<>(); - sinkConfigMap.put("kafka.connect.cosmos.accountEndpoint", TestConfigurations.HOST); - sinkConfigMap.put("kafka.connect.cosmos.accountKey", TestConfigurations.MASTER_KEY); - sinkConfigMap.put("kafka.connect.cosmos.sink.database.name", databaseName); - sinkConfigMap.put("kafka.connect.cosmos.sink.containers.topicMap", topicName + "#" + singlePartitionContainerName); - sinkConfigMap.put("kafka.connect.cosmos.sink.bulk.enabled", String.valueOf(bulkEnabled)); - sinkConfigMap.put("kafka.connect.cosmos.sink.write.strategy", ItemWriteStrategy.ITEM_DELETE_IF_NOT_MODIFIED.getName()); - - CosmosSinkTask sinkTask = new CosmosSinkTask(); - SinkTaskContext sinkTaskContext = Mockito.mock(SinkTaskContext.class); - Mockito.when(sinkTaskContext.errantRecordReporter()).thenReturn(null); - KafkaCosmosReflectionUtils.setSinkTaskContext(sinkTask, sinkTaskContext); - sinkTask.start(sinkConfigMap); - - CosmosAsyncClient cosmosClient = KafkaCosmosReflectionUtils.getSinkTaskCosmosClient(sinkTask); - CosmosContainerProperties singlePartitionContainerProperties = getSinglePartitionContainer(cosmosClient); - CosmosAsyncContainer container = cosmosClient.getDatabase(databaseName).getContainer(singlePartitionContainerProperties.getId()); - - try { - List sinkRecordList = new ArrayList<>(); - List toBeCreateItems = new ArrayList<>(); - this.createSinkRecords( - 10, - topicName, - Schema.Type.MAP, - toBeCreateItems, - sinkRecordList); - - // first time delete, ignore 404 exceptions - sinkTask.put(sinkRecordList); - - // creating the items in the container - for (TestItem testItem : toBeCreateItems) { - container.createItem(testItem).block(); - } - - // get all the items - List createdItems = this.getAllItems(container); - List createdItemIds = - createdItems - .stream() - .map(objectNode -> objectNode.get("id").asText()) - .collect(Collectors.toList()); - List expectItemIds = toBeCreateItems.stream().map(TestItem::getId).collect(Collectors.toList()); - assertThat(toBeCreateItems.size()).isEqualTo(createdItemIds.size()); - assertThat(expectItemIds.containsAll(createdItemIds)).isTrue(); - - // using wrong etag to delete the items, verify no item will be deleted - List sinkRecordsWithWrongEtag = new ArrayList<>(); - for (ObjectNode createdItem : createdItems) { - ObjectNode testItemWithWrongEtag = Utils.getSimpleObjectMapper().createObjectNode(); - testItemWithWrongEtag.setAll(createdItem); - testItemWithWrongEtag.put("_etag", UUID.randomUUID().toString()); - sinkRecordsWithWrongEtag.add( - this.getSinkRecord( - topicName, - testItemWithWrongEtag, - new ConnectSchema(Schema.Type.STRING), - createdItem.get("id").asText(), - Schema.Type.STRUCT) - ); - } - sinkTask.put(sinkRecordsWithWrongEtag); - Thread.sleep(500); // delete happens in the background - List existingItemIds = this.getAllItemIds(container); - assertThat(existingItemIds.size()).isEqualTo(createdItemIds.size()); - assertThat(existingItemIds.containsAll(createdItemIds)).isTrue(); - - // verify all the items have deleted - List sinkRecordsWithCorrectEtag = new ArrayList<>(); - for (ObjectNode createdItem : createdItems) { - sinkRecordsWithCorrectEtag.add( - this.getSinkRecord( - topicName, - createdItem, - new ConnectSchema(Schema.Type.STRING), - createdItem.get("id").asText(), - Schema.Type.STRUCT) - ); - } - - sinkTask.put(sinkRecordsWithCorrectEtag); - Thread.sleep(500); // delete happens in the background - existingItemIds = this.getAllItemIds(container); - assertThat(existingItemIds.isEmpty()).isTrue(); - - } finally { - if (cosmosClient != null) { - cleanUpContainer(cosmosClient, databaseName, singlePartitionContainerProperties.getId()); - sinkTask.stop(); - } - } - } - - private SinkRecord getSinkRecord( - String topicName, - ObjectNode objectNode, - Schema keySchema, - String keyValue, - Schema.Type valueSchemaType) { - if (valueSchemaType == Schema.Type.STRUCT) { - SchemaAndValue schemaAndValue = - JsonToStruct.recordToSchemaAndValue(objectNode); - - return new SinkRecord( - topicName, - 1, - keySchema, - keyValue, - schemaAndValue.schema(), - schemaAndValue.value(), - 0L); - } else { - return new SinkRecord( - topicName, - 1, - keySchema, - keyValue, - new ConnectSchema(Schema.Type.MAP), - Utils.getSimpleObjectMapper().convertValue(objectNode, new TypeReference>() {}), - 0L); - } - } - - private void createSinkRecords( - int numberOfItems, - String topicName, - Schema.Type valueSchemaType, - List createdItems, - List sinkRecordList) { - - Schema keySchema = new ConnectSchema(Schema.Type.STRING); - - for (int i = 0; i < numberOfItems; i++) { - TestItem testItem = TestItem.createNewItem(); - createdItems.add(testItem); - - SinkRecord sinkRecord = - this.getSinkRecord( - topicName, - Utils.getSimpleObjectMapper().convertValue(testItem, ObjectNode.class), - keySchema, - testItem.getId(), - valueSchemaType); - sinkRecordList.add(sinkRecord); - } - } - - private List getAllItemIds(CosmosAsyncContainer container) { - return getAllItems(container) - .stream() - .map(objectNode -> objectNode.get("id").asText()) - .collect(Collectors.toList()); - } - - private List getAllItems(CosmosAsyncContainer container) { - String query = "select * from c"; - return container.queryItems(query, ObjectNode.class) - .byPage() - .flatMapIterable(response -> response.getResults()) - .collectList() - .block(); - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/implementation/sink/idStrategy/ProvidedInStrategyTest.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/implementation/sink/idStrategy/ProvidedInStrategyTest.java deleted file mode 100644 index 19c8fb92dd374..0000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/implementation/sink/idStrategy/ProvidedInStrategyTest.java +++ /dev/null @@ -1,223 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink.idStrategy; - -import com.azure.cosmos.kafka.connect.implementation.sink.idstrategy.IdStrategy; -import com.azure.cosmos.kafka.connect.implementation.sink.idstrategy.ProvidedInConfig; -import com.azure.cosmos.kafka.connect.implementation.sink.idstrategy.ProvidedInKeyStrategy; -import com.azure.cosmos.kafka.connect.implementation.sink.idstrategy.ProvidedInValueStrategy; -import org.apache.kafka.connect.data.Schema; -import org.apache.kafka.connect.data.SchemaBuilder; -import org.apache.kafka.connect.data.Struct; -import org.apache.kafka.connect.errors.ConnectException; -import org.apache.kafka.connect.sink.SinkRecord; -import org.mockito.Mockito; -import org.testng.annotations.DataProvider; -import org.testng.annotations.Test; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.Map; - -import static org.assertj.core.api.AssertionsForClassTypes.assertThat; -import static org.mockito.Mockito.when; - -public class ProvidedInStrategyTest { - protected static final int TIMEOUT = 60000; - - @DataProvider(name = "idStrategyParameterProvider") - public static Object[][] idStrategyParameterProvider() { - return new Object[][]{ - { new ProvidedInValueStrategy() }, - { new ProvidedInKeyStrategy() }, - }; - } - - private void returnOnKeyOrValue( - Schema schema, - Object ret, - IdStrategy idStrategy, - SinkRecord sinkRecord) { - if (idStrategy.getClass() == ProvidedInKeyStrategy.class) { - when(sinkRecord.keySchema()).thenReturn(schema); - when(sinkRecord.key()).thenReturn(ret); - } else { - when(sinkRecord.valueSchema()).thenReturn(schema); - when(sinkRecord.value()).thenReturn(ret); - } - } - - @Test(groups = { "unit" }, dataProvider = "idStrategyParameterProvider", expectedExceptions = ConnectException.class, timeOut = TIMEOUT) - public void valueNotStructOrMapShouldFail(IdStrategy idStrategy) { - idStrategy.configure(new HashMap<>()); - SinkRecord sinkRecord = Mockito.mock(SinkRecord.class); - returnOnKeyOrValue(Schema.STRING_SCHEMA, "a string", idStrategy, sinkRecord); - idStrategy.generateId(sinkRecord); - } - - @Test(groups = { "unit" }, dataProvider = "idStrategyParameterProvider", expectedExceptions = ConnectException.class, timeOut = TIMEOUT) - public void noIdInValueShouldFail(IdStrategy idStrategy) { - idStrategy.configure(new HashMap<>()); - SinkRecord sinkRecord = Mockito.mock(SinkRecord.class); - returnOnKeyOrValue(null, new HashMap<>(), idStrategy, sinkRecord); - idStrategy.generateId(sinkRecord); - } - - @Test(groups = { "unit" }, dataProvider = "idStrategyParameterProvider", timeOut = TIMEOUT) - public void stringIdOnMapShouldReturn(IdStrategy idStrategy) { - idStrategy.configure(new HashMap<>()); - SinkRecord sinkRecord = Mockito.mock(SinkRecord.class); - returnOnKeyOrValue( - null, - new HashMap(){{ put("id", "1234567"); }}, - idStrategy, - sinkRecord); - assertThat("1234567").isEqualTo(idStrategy.generateId(sinkRecord)); - } - - @Test(groups = { "unit" }, dataProvider = "idStrategyParameterProvider", timeOut = TIMEOUT) - public void nonStringIdOnMapShouldReturn(IdStrategy idStrategy) { - idStrategy.configure(new HashMap<>()); - SinkRecord sinkRecord = Mockito.mock(SinkRecord.class); - returnOnKeyOrValue( - null, - new HashMap(){{ put("id", 1234567); }}, - idStrategy, - sinkRecord); - assertThat("1234567").isEqualTo(idStrategy.generateId(sinkRecord)); - } - - @Test(groups = { "unit" }, dataProvider = "idStrategyParameterProvider", timeOut = TIMEOUT) - public void stringIdOnStructShouldReturn(IdStrategy idStrategy) { - idStrategy.configure(new HashMap<>()); - Schema schema = SchemaBuilder.struct() - .field("id", Schema.STRING_SCHEMA) - .build(); - - SinkRecord sinkRecord = Mockito.mock(SinkRecord.class); - Struct struct = new Struct(schema).put("id", "1234567"); - returnOnKeyOrValue(struct.schema(), struct, idStrategy, sinkRecord); - - assertThat("1234567").isEqualTo(idStrategy.generateId(sinkRecord)); - } - - @Test(groups = { "unit" }, dataProvider = "idStrategyParameterProvider", timeOut = TIMEOUT) - public void structIdOnStructShouldReturn(IdStrategy idStrategy) { - idStrategy.configure(new HashMap<>()); - Schema idSchema = SchemaBuilder.struct() - .field("name", Schema.STRING_SCHEMA) - .build(); - Schema schema = SchemaBuilder.struct() - .field("id", idSchema) - .build(); - Struct struct = new Struct(schema) - .put("id", new Struct(idSchema).put("name", "cosmos kramer")); - SinkRecord sinkRecord = Mockito.mock(SinkRecord.class); - returnOnKeyOrValue(struct.schema(), struct, idStrategy, sinkRecord); - - assertThat("{\"name\":\"cosmos kramer\"}").isEqualTo(idStrategy.generateId(sinkRecord)); - } - - @Test(groups = { "unit" }, dataProvider = "idStrategyParameterProvider", timeOut = TIMEOUT) - public void jsonPathOnStruct(IdStrategy idStrategy) { - idStrategy.configure(new HashMap(){{ put(ProvidedInConfig.JSON_PATH_CONFIG, "$.id.name"); }}); - - Schema idSchema = SchemaBuilder.struct() - .field("name", Schema.STRING_SCHEMA) - .build(); - Schema schema = SchemaBuilder.struct() - .field("id", idSchema) - .build(); - Struct struct = new Struct(schema) - .put("id", new Struct(idSchema).put("name", "franz kafka")); - - SinkRecord sinkRecord = Mockito.mock(SinkRecord.class); - returnOnKeyOrValue(struct.schema(), struct, idStrategy, sinkRecord); - assertThat("franz kafka").isEqualTo(idStrategy.generateId(sinkRecord)); - } - - @Test(groups = { "unit" }, dataProvider = "idStrategyParameterProvider", timeOut = TIMEOUT) - public void jsonPathOnMap(IdStrategy idStrategy) { - idStrategy.configure(new HashMap(){{ put(ProvidedInConfig.JSON_PATH_CONFIG, "$.id.name"); }}); - - SinkRecord sinkRecord = Mockito.mock(SinkRecord.class); - returnOnKeyOrValue( - null, - new HashMap(){{ put("id", new HashMap(){{ put("name", "franz kafka"); }}); }}, - idStrategy, - sinkRecord); - assertThat("franz kafka").isEqualTo(idStrategy.generateId(sinkRecord)); - } - - @Test(groups = { "unit" }, dataProvider = "idStrategyParameterProvider", expectedExceptions = ConnectException.class, timeOut = TIMEOUT) - public void invalidJsonPathThrows(IdStrategy idStrategy) { - idStrategy.configure(new HashMap(){{ put(ProvidedInConfig.JSON_PATH_CONFIG, "invalid.path"); }}); - SinkRecord sinkRecord = Mockito.mock(SinkRecord.class); - returnOnKeyOrValue( - null, - new HashMap(){{ put("id", new HashMap(){{ put("name", "franz kafka"); }}); }}, - idStrategy, - sinkRecord); - - idStrategy.generateId(sinkRecord); - } - - @Test(groups = { "unit" }, dataProvider = "idStrategyParameterProvider", expectedExceptions = ConnectException.class, timeOut = TIMEOUT) - public void jsonPathNotExistThrows(IdStrategy idStrategy) { - idStrategy.configure(new HashMap(){{ put(ProvidedInConfig.JSON_PATH_CONFIG, "$.id.not.exist"); }}); - SinkRecord sinkRecord = Mockito.mock(SinkRecord.class); - returnOnKeyOrValue( - null, - new HashMap(){{ put("id", new HashMap(){{ put("name", "franz kafka"); }}); }}, - idStrategy, - sinkRecord); - - idStrategy.generateId(sinkRecord); - } - - @Test(groups = { "unit" }, dataProvider = "idStrategyParameterProvider", timeOut = TIMEOUT) - public void complexJsonPath(IdStrategy idStrategy) { - Map map1 = new LinkedHashMap<>(); - map1.put("id", 0); - map1.put("name", "cosmos kramer"); - map1.put("occupation", "unknown"); - Map map2 = new LinkedHashMap<>(); - map2.put("id", 1); - map2.put("name", "franz kafka"); - map2.put("occupation", "writer"); - - SinkRecord sinkRecord = Mockito.mock(SinkRecord.class); - returnOnKeyOrValue( - null, - new HashMap() {{ put("id", Arrays.asList(map1, map2)); }}, - idStrategy, - sinkRecord); - - idStrategy.configure(new HashMap(){{ put(ProvidedInConfig.JSON_PATH_CONFIG, "$.id[0].name"); }}); - assertThat("cosmos kramer").isEqualTo(idStrategy.generateId(sinkRecord)); - - idStrategy.configure(new HashMap(){{ put(ProvidedInConfig.JSON_PATH_CONFIG, "$.id[1].name"); }}); - assertThat("franz kafka").isEqualTo(idStrategy.generateId(sinkRecord)); - - idStrategy.configure(new HashMap(){{ put(ProvidedInConfig.JSON_PATH_CONFIG, "$.id[*].id"); }}); - assertThat("[0,1]").isEqualTo(idStrategy.generateId(sinkRecord)); - - idStrategy.configure(new HashMap(){{ put(ProvidedInConfig.JSON_PATH_CONFIG, "$.id"); }}); - assertThat("[{\"id\":0,\"name\":\"cosmos kramer\",\"occupation\":\"unknown\"},{\"id\":1,\"name\":\"franz kafka\",\"occupation\":\"writer\"}]").isEqualTo(idStrategy.generateId(sinkRecord)); - } - - @Test(groups = { "unit" }, dataProvider = "idStrategyParameterProvider", timeOut = TIMEOUT) - public void generatedIdSanitized(IdStrategy idStrategy) { - idStrategy.configure(new HashMap<>()); - SinkRecord sinkRecord = Mockito.mock(SinkRecord.class); - returnOnKeyOrValue( - null, - new HashMap() {{put("id", "#my/special\\id?");}}, - idStrategy, - sinkRecord); - - assertThat("_my_special_id_").isEqualTo(idStrategy.generateId(sinkRecord)); - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/implementation/source/MetadataMonitorThreadTest.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/implementation/source/MetadataMonitorThreadTest.java index 76479de109810..e73dbdb628f69 100644 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/implementation/source/MetadataMonitorThreadTest.java +++ b/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/implementation/source/MetadataMonitorThreadTest.java @@ -56,7 +56,7 @@ public void requestTaskReconfigurationOnContainersChange() throws InterruptedExc databaseName, true, new ArrayList(), - new HashMap()); + new ArrayList()); CosmosMetadataConfig metadataConfig = new CosmosMetadataConfig(500, "_cosmos.metadata.topic"); SourceConnectorContext sourceConnectorContext = Mockito.mock(SourceConnectorContext.class); @@ -105,7 +105,7 @@ public void requestTaskReconfigurationOnSplit() throws InterruptedException { databaseName, false, Arrays.asList(multiPartitionContainerName), - new HashMap()); + new ArrayList()); CosmosMetadataConfig metadataConfig = new CosmosMetadataConfig(500, "_cosmos.metadata.topic"); SourceConnectorContext sourceConnectorContext = Mockito.mock(SourceConnectorContext.class); @@ -176,7 +176,7 @@ public void requestTaskReconfigurationOnMerge() throws InterruptedException { databaseName, false, Arrays.asList(singlePartitionContainerName), - new HashMap()); + new ArrayList()); CosmosMetadataConfig metadataConfig = new CosmosMetadataConfig(500, "_cosmos.metadata.topic"); SourceConnectorContext sourceConnectorContext = Mockito.mock(SourceConnectorContext.class); diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/CosmosAsyncContainer.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/CosmosAsyncContainer.java index bd7a295bc9aeb..c6aa862610c94 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/CosmosAsyncContainer.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/CosmosAsyncContainer.java @@ -2784,11 +2784,6 @@ public Mono> trySplitFeedRange( return cosmosAsyncContainer.trySplitFeedRange(feedRange, targetedCountAfterSplit); } - - @Override - public String getLinkWithoutTrailingSlash(CosmosAsyncContainer cosmosAsyncContainer) { - return cosmosAsyncContainer.getLinkWithoutTrailingSlash(); - } }); } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ImplementationBridgeHelpers.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ImplementationBridgeHelpers.java index 8b2c937e4915c..3f53869986d94 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ImplementationBridgeHelpers.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ImplementationBridgeHelpers.java @@ -946,8 +946,6 @@ Mono> trySplitFeedRange( CosmosAsyncContainer cosmosAsyncContainer, FeedRange feedRange, int targetedCountAfterSplit); - - String getLinkWithoutTrailingSlash(CosmosAsyncContainer cosmosAsyncContainer); } }