diff --git a/pom.xml b/pom.xml index 339fd94..59aeb5a 100644 --- a/pom.xml +++ b/pom.xml @@ -44,6 +44,7 @@ 2.0.0 2.27.2 2.7.0 + 3.3.2 ${project.basedir}/src/test/java/ UTF-8 UTF-8 @@ -61,7 +62,11 @@ - + + dev.failsafe + failsafe + ${failsafe.version} + io.cdap.cdap cdap-api @@ -365,12 +370,6 @@ 2.0.2 test - - com.github.rholder - guava-retrying - 2.0.0 - - diff --git a/src/main/java/io/cdap/plugin/successfactors/connector/SuccessFactorsConnector.java b/src/main/java/io/cdap/plugin/successfactors/connector/SuccessFactorsConnector.java index da10995..c377536 100644 --- a/src/main/java/io/cdap/plugin/successfactors/connector/SuccessFactorsConnector.java +++ b/src/main/java/io/cdap/plugin/successfactors/connector/SuccessFactorsConnector.java @@ -151,7 +151,7 @@ List listEntities() throws TransportException, IOException { URL dataURL = HttpUrl.parse(config.getBaseURL()).newBuilder().build().url(); SuccessFactorsTransporter successFactorsHttpClient = new SuccessFactorsTransporter(config); SuccessFactorsResponseContainer responseContainer = successFactorsHttpClient.callSuccessFactorsEntity - (dataURL, MediaType.APPLICATION_JSON, METADATA); + (dataURL, MediaType.APPLICATION_JSON); try (InputStream inputStream = responseContainer.getResponseStream()) { BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); String result = reader.lines().collect(Collectors.joining("")); @@ -225,7 +225,11 @@ private InputStream callEntityData(long top, String entityName) addQueryParameter(TOP_OPTION, String.valueOf(top)).addQueryParameter(SELECT_OPTION, selectFields.toString()) .build().url(); SuccessFactorsTransporter successFactorsHttpClient = new SuccessFactorsTransporter(config); - SuccessFactorsResponseContainer responseContainer = successFactorsHttpClient.callSuccessFactorsWithRetry(dataURL); + SuccessFactorsResponseContainer responseContainer = + successFactorsHttpClient.callSuccessFactorsWithRetry( + dataURL, MediaType.APPLICATION_JSON, SuccessFactorsPluginConfig.DEFAULT_INITIAL_RETRY_DURATION_SECONDS, + SuccessFactorsPluginConfig.DEFAULT_MAX_RETRY_DURATION_SECONDS, + SuccessFactorsPluginConfig.DEFAULT_RETRY_MULTIPLIER, SuccessFactorsPluginConfig.DEFAULT_MAX_RETRY_COUNT); ExceptionParser.checkAndThrowException("", responseContainer); return responseContainer.getResponseStream(); @@ -255,7 +259,7 @@ private InputStream getMetaDataStream(String entity) throws TransportException, .addPathSegment(METADATACALL).build().url(); SuccessFactorsTransporter successFactorsHttpClient = new SuccessFactorsTransporter(config); SuccessFactorsResponseContainer responseContainer = successFactorsHttpClient - .callSuccessFactorsEntity(metadataURL, MediaType.APPLICATION_XML, METADATA); + .callSuccessFactorsEntity(metadataURL, MediaType.APPLICATION_XML); return responseContainer.getResponseStream(); } } diff --git a/src/main/java/io/cdap/plugin/successfactors/connector/SuccessFactorsConnectorConfig.java b/src/main/java/io/cdap/plugin/successfactors/connector/SuccessFactorsConnectorConfig.java index 02bac6c..57377f3 100644 --- a/src/main/java/io/cdap/plugin/successfactors/connector/SuccessFactorsConnectorConfig.java +++ b/src/main/java/io/cdap/plugin/successfactors/connector/SuccessFactorsConnectorConfig.java @@ -20,7 +20,6 @@ import io.cdap.cdap.api.annotation.Name; import io.cdap.cdap.api.plugin.PluginConfig; import io.cdap.cdap.etl.api.FailureCollector; -import io.cdap.plugin.successfactors.common.exception.SuccessFactorsServiceException; import io.cdap.plugin.successfactors.common.exception.TransportException; import io.cdap.plugin.successfactors.common.util.ResourceConstants; import io.cdap.plugin.successfactors.common.util.SuccessFactorsUtil; @@ -153,7 +152,7 @@ public void validateConnection(FailureCollector collector) { SuccessFactorsResponseContainer responseContainer = null; try { responseContainer = - successFactorsHttpClient.callSuccessFactorsEntity(testerURL, MediaType.APPLICATION_JSON, TEST); + successFactorsHttpClient.callSuccessFactorsEntity(testerURL, MediaType.APPLICATION_JSON); } catch (TransportException e) { LOG.error("Unable to fetch the response", e); collector.addFailure("Unable to call SuccessFactorsEntity", diff --git a/src/main/java/io/cdap/plugin/successfactors/source/config/SuccessFactorsPluginConfig.java b/src/main/java/io/cdap/plugin/successfactors/source/config/SuccessFactorsPluginConfig.java index 62849fa..9aab7aa 100644 --- a/src/main/java/io/cdap/plugin/successfactors/source/config/SuccessFactorsPluginConfig.java +++ b/src/main/java/io/cdap/plugin/successfactors/source/config/SuccessFactorsPluginConfig.java @@ -54,6 +54,14 @@ public class SuccessFactorsPluginConfig extends PluginConfig { private static final String COMMON_ACTION = ResourceConstants.ERR_MISSING_PARAM_OR_MACRO_ACTION.getMsgForKey(); private static final Pattern PATTERN = Pattern.compile("\\(.*\\)"); private static final String SAP_SUCCESSFACTORS_ENTITY_NAME = "Entity Name"; + private static final String NAME_INITIAL_RETRY_DURATION = "initialRetryDuration"; + private static final String NAME_MAX_RETRY_DURATION = "maxRetryDuration"; + private static final String NAME_RETRY_MULTIPLIER = "retryMultiplier"; + private static final String NAME_MAX_RETRY_COUNT = "maxRetryCount"; + public static final int DEFAULT_INITIAL_RETRY_DURATION_SECONDS = 2; + public static final int DEFAULT_RETRY_MULTIPLIER = 2; + public static final int DEFAULT_MAX_RETRY_COUNT = 3; + public static final int DEFAULT_MAX_RETRY_DURATION_SECONDS = 10; @Macro @Name(ENTITY_NAME) @@ -128,6 +136,30 @@ public class SuccessFactorsPluginConfig extends PluginConfig { @Description("The existing connection to use.") private SuccessFactorsConnectorConfig connection; + @Name(NAME_INITIAL_RETRY_DURATION) + @Description("Time taken for the first retry. Default is 2 seconds.") + @Nullable + @Macro + private Integer initialRetryDuration; + + @Name(NAME_MAX_RETRY_DURATION) + @Description("Maximum time in seconds retries can take. Default is 300 seconds.") + @Nullable + @Macro + private Integer maxRetryDuration; + + @Name(NAME_MAX_RETRY_COUNT) + @Description("Maximum number of retries allowed. Default is 3.") + @Nullable + @Macro + private Integer maxRetryCount; + + @Name(NAME_RETRY_MULTIPLIER) + @Description("Multiplier for exponential backoff. Default is 2.") + @Nullable + @Macro + private Integer retryMultiplier; + @VisibleForTesting public SuccessFactorsPluginConfig(String referenceName, String baseURL, @@ -142,7 +174,11 @@ public SuccessFactorsPluginConfig(String referenceName, @Nullable String selectOption, @Nullable String expandOption, @Nullable String additionalQueryParameters, - String paginationType) { + String paginationType, + @Nullable Integer initialRetryDuration, + @Nullable Integer maxRetryDuration, + @Nullable Integer retryMultiplier, + @Nullable Integer maxRetryCount) { this.connection = new SuccessFactorsConnectorConfig(username, password, baseURL, proxyUrl, proxyPassword, proxyUsername); this.referenceName = referenceName; @@ -153,6 +189,10 @@ public SuccessFactorsPluginConfig(String referenceName, this.expandOption = expandOption; this.paginationType = paginationType; this.additionalQueryParameters = additionalQueryParameters; + this.initialRetryDuration = initialRetryDuration; + this.maxRetryDuration = maxRetryDuration; + this.retryMultiplier = retryMultiplier; + this.maxRetryCount = maxRetryCount; } @Nullable public SuccessFactorsConnectorConfig getConnection() { @@ -210,6 +250,22 @@ public String getAdditionalQueryParameters() { return this.additionalQueryParameters; } + public int getInitialRetryDuration() { + return initialRetryDuration == null ? DEFAULT_INITIAL_RETRY_DURATION_SECONDS : initialRetryDuration; + } + + public int getMaxRetryDuration() { + return maxRetryDuration == null ? DEFAULT_MAX_RETRY_DURATION_SECONDS : maxRetryDuration; + } + + public int getRetryMultiplier() { + return retryMultiplier == null ? DEFAULT_RETRY_MULTIPLIER : retryMultiplier; + } + + public int getMaxRetryCount() { + return maxRetryCount == null ? DEFAULT_MAX_RETRY_COUNT : maxRetryCount; + } + /** * Checks if the call to SuccessFactors service is required for metadata creation. * condition parameters: ['host' | 'serviceName' | 'entityName' | 'username' | 'password'] @@ -243,6 +299,7 @@ public void validatePluginParameters(FailureCollector failureCollector) { validateMandatoryParameters(failureCollector); validateBasicCredentials(failureCollector); validateEntityParameter(failureCollector); + validateRetryConfiguration(failureCollector); failureCollector.getOrThrowException(); } @@ -292,6 +349,43 @@ private void validateEntityParameter(FailureCollector failureCollector) { } } + /** + * Validates the retry configuration. + * + * @param failureCollector {@code FailureCollector} + */ + public void validateRetryConfiguration(FailureCollector failureCollector) { + if (containsMacro(NAME_INITIAL_RETRY_DURATION) || containsMacro(NAME_MAX_RETRY_DURATION) || + containsMacro(NAME_MAX_RETRY_COUNT) || containsMacro(NAME_RETRY_MULTIPLIER)) { + return; + } + if (initialRetryDuration != null && initialRetryDuration <= 0) { + failureCollector.addFailure("Initial retry duration must be greater than 0.", + "Please specify a valid initial retry duration.") + .withConfigProperty(NAME_INITIAL_RETRY_DURATION); + } + if (maxRetryDuration != null && maxRetryDuration <= 0) { + failureCollector.addFailure("Max retry duration must be greater than 0.", + "Please specify a valid max retry duration.") + .withConfigProperty(NAME_MAX_RETRY_DURATION); + } + if (maxRetryCount != null && maxRetryCount <= 0) { + failureCollector.addFailure("Max retry count must be greater than 0.", + "Please specify a valid max retry count.") + .withConfigProperty(NAME_MAX_RETRY_COUNT); + } + if (retryMultiplier != null && retryMultiplier <= 1) { + failureCollector.addFailure("Retry multiplier must be strictly greater than 1.", + "Please specify a valid retry multiplier.") + .withConfigProperty(NAME_RETRY_MULTIPLIER); + } + if (maxRetryDuration != null && initialRetryDuration != null && maxRetryDuration <= initialRetryDuration) { + failureCollector.addFailure("Max retry duration must be greater than initial retry duration.", + "Please specify a valid max retry duration.") + .withConfigProperty(NAME_MAX_RETRY_DURATION); + } + } + /** * Helper class to simplify {@link SuccessFactorsPluginConfig} class creation. */ @@ -310,6 +404,10 @@ public static class Builder { private String proxyUrl; private String proxyUsername; private String proxyPassword; + private Integer initialRetryDuration; + private Integer maxRetryDuration; + private Integer retryMultiplier; + private Integer maxRetryCount; public Builder referenceName(String referenceName) { this.referenceName = referenceName; @@ -379,11 +477,29 @@ public Builder additionalQueryParameters(@Nullable String additionalQueryParamet return this; } + public Builder setInitialRetryDuration(Integer initialRetryDuration) { + this.initialRetryDuration = initialRetryDuration; + return this; + } + public Builder setMaxRetryDuration(Integer maxRetryDuration) { + this.maxRetryDuration = maxRetryDuration; + return this; + } + public Builder setRetryMultiplier(Integer retryMultiplier) { + this.retryMultiplier = retryMultiplier; + return this; + } + public Builder setMaxRetryCount(Integer maxRetryCount) { + this.maxRetryCount = maxRetryCount; + return this; + } + public SuccessFactorsPluginConfig build() { return new SuccessFactorsPluginConfig(referenceName, baseURL, entityName, associateEntityName, username, password, proxyUrl, proxyUsername, proxyPassword, filterOption, selectOption, expandOption, additionalQueryParameters, - paginationType); + paginationType, initialRetryDuration, maxRetryDuration, + retryMultiplier, maxRetryCount); } } } diff --git a/src/main/java/io/cdap/plugin/successfactors/source/service/SuccessFactorsService.java b/src/main/java/io/cdap/plugin/successfactors/source/service/SuccessFactorsService.java index 0780dc6..0e21ed6 100644 --- a/src/main/java/io/cdap/plugin/successfactors/source/service/SuccessFactorsService.java +++ b/src/main/java/io/cdap/plugin/successfactors/source/service/SuccessFactorsService.java @@ -97,7 +97,7 @@ public SuccessFactorsService(SuccessFactorsPluginConfig pluginConfig, public void checkSuccessFactorsURL() throws TransportException, SuccessFactorsServiceException { SuccessFactorsResponseContainer responseContainer = - successFactorsHttpClient.callSuccessFactorsEntity(urlContainer.getTesterURL(), MediaType.APPLICATION_JSON, TEST); + successFactorsHttpClient.callSuccessFactorsEntity(urlContainer.getTesterURL(), MediaType.APPLICATION_JSON); ExceptionParser.checkAndThrowException(ResourceConstants.ERR_FAILED_ENTITY_VALIDATION.getMsgForKey(), responseContainer); @@ -163,7 +163,9 @@ private SuccessFactorsEntityProvider fetchServiceMetadata(InputStream metadataSt */ private InputStream callEntityMetadata() throws TransportException { SuccessFactorsResponseContainer responseContainer = successFactorsHttpClient - .callSuccessFactorsEntity(urlContainer.getMetadataURL(), MediaType.APPLICATION_XML, METADATA); + .callSuccessFactorsWithRetry(urlContainer.getMetadataURL(), MediaType.APPLICATION_XML, pluginConfig + .getInitialRetryDuration(), pluginConfig.getMaxRetryDuration(), pluginConfig.getRetryMultiplier(), + pluginConfig.getMaxRetryCount()); return responseContainer.getResponseStream(); } @@ -320,7 +322,10 @@ private InputStream callEntityData(@Nullable Long skip, @Nullable Long top) } else { dataURL = urlContainer.getDataFetchURL(skip, top); } - SuccessFactorsResponseContainer responseContainer = successFactorsHttpClient.callSuccessFactorsWithRetry(dataURL); + SuccessFactorsResponseContainer responseContainer = + successFactorsHttpClient.callSuccessFactorsWithRetry( + dataURL, MediaType.APPLICATION_JSON, pluginConfig.getInitialRetryDuration(), pluginConfig.getMaxRetryDuration(), + pluginConfig.getRetryMultiplier(), pluginConfig.getMaxRetryCount()); ExceptionParser.checkAndThrowException("", responseContainer); return responseContainer.getResponseStream(); @@ -337,7 +342,7 @@ public List getNonNavigationalProperties() throws TransportException, Su /** * Filter the data stream after removing the expanded entity data. - * + * * Data stream after conversion to JSON has the following format: * "d": { * "results": [ diff --git a/src/main/java/io/cdap/plugin/successfactors/source/transport/SuccessFactorsTransporter.java b/src/main/java/io/cdap/plugin/successfactors/source/transport/SuccessFactorsTransporter.java index 730a50a..9b85b0c 100644 --- a/src/main/java/io/cdap/plugin/successfactors/source/transport/SuccessFactorsTransporter.java +++ b/src/main/java/io/cdap/plugin/successfactors/source/transport/SuccessFactorsTransporter.java @@ -16,11 +16,9 @@ package io.cdap.plugin.successfactors.source.transport; -import com.github.rholder.retry.RetryException; -import com.github.rholder.retry.Retryer; -import com.github.rholder.retry.RetryerBuilder; -import com.github.rholder.retry.StopStrategies; -import com.github.rholder.retry.WaitStrategies; +import dev.failsafe.Failsafe; +import dev.failsafe.FailsafeException; +import dev.failsafe.RetryPolicy; import io.cdap.cdap.api.retry.RetryableException; import io.cdap.plugin.successfactors.common.exception.TransportException; import io.cdap.plugin.successfactors.common.util.ResourceConstants; @@ -43,13 +41,10 @@ import java.net.Proxy; import java.net.URL; import java.nio.charset.StandardCharsets; +import java.time.Duration; import java.util.Base64; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -import javax.ws.rs.core.MediaType; - /** * This {@code SuccessFactorsTransporter} class is used to * make a rest web service call to the SAP SuccessFactors exposed services. @@ -58,8 +53,6 @@ public class SuccessFactorsTransporter { static final String SERVICE_VERSION = "dataserviceversion"; private static final Logger LOG = LoggerFactory.getLogger(SuccessFactorsTransporter.class); private static final long CONNECTION_TIMEOUT = 300; - private static final long WAIT_TIME = 5; - private static final long MAX_NUMBER_OF_RETRY_ATTEMPTS = 5; private SuccessFactorsConnectorConfig config; private Response response; @@ -77,11 +70,10 @@ public SuccessFactorsTransporter(SuccessFactorsConnectorConfig pluginConfig) { * * @param endpoint type of URL * @param mediaType mediaType for Accept header property, supported types are 'application/json' & 'application/xml' - * @param fetchType type of call i.e. TEST / METADATA / COUNT, used for logging purpose. * @return {@code SuccessFactorsResponseContainer} * @throws TransportException any http client exceptions are wrapped under it */ - public SuccessFactorsResponseContainer callSuccessFactorsEntity(URL endpoint, String mediaType, String fetchType) + public SuccessFactorsResponseContainer callSuccessFactorsEntity(URL endpoint, String mediaType) throws TransportException { try { @@ -100,13 +92,27 @@ public SuccessFactorsResponseContainer callSuccessFactorsEntity(URL endpoint, St * * @param endpoint record fetch URL * @return {@code SuccessFactorsResponseContainer} - * @throws IOException any http client exceptions * @throws TransportException any error while preparing the {@code OkHttpClient} */ - public SuccessFactorsResponseContainer callSuccessFactorsWithRetry(URL endpoint) - throws IOException, TransportException { - - Response res = retrySapTransportCall(endpoint, MediaType.APPLICATION_JSON); + public SuccessFactorsResponseContainer callSuccessFactorsWithRetry(URL endpoint, String mediaType, + int initialRetryDuration, int maxRetryDuration, + int retryMultiplier, int maxRetryCount) + throws TransportException { + LOG.debug( + "Retrying the call to SuccessFactors with initialRetryDuration: {}, maxRetryDuration: {}, retryMultiplier: {}, " + + "maxRetryCount: {}", + initialRetryDuration, maxRetryDuration, retryMultiplier, maxRetryCount); + LOG.debug("Endpoint: {}, MediaType: {}", endpoint, mediaType); + Response res; + try { + res = Failsafe.with(getRetryPolicy(initialRetryDuration, maxRetryDuration, retryMultiplier, maxRetryCount)) + .get(() -> retrySapTransportCall(endpoint, mediaType)); + } catch (FailsafeException e) { + if (e.getCause() != null) { + throw new RuntimeException(e.getCause()); + } + throw e; + } try { return prepareResponseContainer(res); @@ -122,33 +128,39 @@ public SuccessFactorsResponseContainer callSuccessFactorsWithRetry(URL endpoint) * @param endpoint record fetch URL * @param mediaType mediaType for Accept header property * @return {@code Response} - * @throws IOException if all retries fail */ - public Response retrySapTransportCall(URL endpoint, String mediaType) throws IOException { - Callable fetchRecords = () -> { + public Response retrySapTransportCall(URL endpoint, String mediaType) { + try { response = transport(endpoint, mediaType); if (response != null && response.code() >= HttpURLConnection.HTTP_INTERNAL_ERROR) { throw new RetryableException(); } - return true; - }; - - Retryer retryer = RetryerBuilder.newBuilder() - .retryIfExceptionOfType(RetryableException.class) - .withWaitStrategy(WaitStrategies.exponentialWait(WAIT_TIME, TimeUnit.SECONDS)) - .withStopStrategy(StopStrategies.stopAfterAttempt((int) MAX_NUMBER_OF_RETRY_ATTEMPTS)) - .build(); - - try { - retryer.call(fetchRecords); - } catch (RetryException | ExecutionException e) { + } catch (Exception e) { LOG.error("Data Recovery failed for URL {}.", endpoint); - throw new IOException(); + if (e instanceof RetryableException) { + throw (RetryableException) e; + } else if (e instanceof IOException) { + throw new RetryableException("IOException occurred while calling SuccessFactors."); + } else { + throw new RuntimeException(e); + } } - return response; } + private RetryPolicy getRetryPolicy(int initialRetryDuration, int maxRetryDuration, int retryMultiplier, + int maxRetryCount) { + return RetryPolicy.builder() + .handle(RetryableException.class) + .withBackoff(Duration.ofSeconds(initialRetryDuration), + Duration.ofSeconds(maxRetryDuration), retryMultiplier) + .withMaxRetries(maxRetryCount) + .onRetry(event -> LOG.debug("Retrying SapTransportCall. Retry count: {}", event.getAttemptCount())) + .onSuccess(event -> LOG.debug("SapTransportCall executed successfully.")) + .onRetriesExceeded(event -> LOG.error("Retry limit reached for SapTransportCall.")) + .build(); + } + /** * Make an HTTP/S call to the given URL. * diff --git a/src/test/java/io/cdap/plugin/successfactors/connector/SuccessFactorsConnectorTest.java b/src/test/java/io/cdap/plugin/successfactors/connector/SuccessFactorsConnectorTest.java index d058b68..10d43db 100644 --- a/src/test/java/io/cdap/plugin/successfactors/connector/SuccessFactorsConnectorTest.java +++ b/src/test/java/io/cdap/plugin/successfactors/connector/SuccessFactorsConnectorTest.java @@ -89,7 +89,7 @@ public void testValidateSuccessfulConnection() throws TransportException, Succes new Expectations(SuccessFactorsUrlContainer.class, SuccessFactorsTransporter.class, SuccessFactorsSchemaGenerator.class) { { - successFactorsTransporter.callSuccessFactorsEntity(null, anyString, anyString); + successFactorsTransporter.callSuccessFactorsEntity(null, anyString); result = getSuccessfulResponseContainer(); minTimes = 1; } @@ -105,7 +105,7 @@ public void testValidateUnauthorisedConnection() throws TransportException, Succ new Expectations(SuccessFactorsUrlContainer.class, SuccessFactorsTransporter.class, SuccessFactorsSchemaGenerator.class) { { - successFactorsTransporter.callSuccessFactorsEntity(null, anyString, anyString); + successFactorsTransporter.callSuccessFactorsEntity(null, anyString); result = getUnauthorisedResponseContainer(); minTimes = 1; } @@ -120,7 +120,7 @@ public void testValidateNotFoundConnection() throws TransportException, SuccessF new Expectations(SuccessFactorsUrlContainer.class, SuccessFactorsTransporter.class, SuccessFactorsSchemaGenerator.class) { { - successFactorsTransporter.callSuccessFactorsEntity(null, anyString, anyString); + successFactorsTransporter.callSuccessFactorsEntity(null, anyString); result = getNotFoundResponseContainer(); minTimes = 1; } @@ -151,7 +151,7 @@ public void testGenerateSpec() throws TransportException, IOException { MockFailureCollector collector = new MockFailureCollector(); new Expectations(SuccessFactorsTransporter.class) { { - successFactorsTransporter.callSuccessFactorsEntity(null, anyString, anyString); + successFactorsTransporter.callSuccessFactorsEntity(null, anyString); result = getSuccessfulResponseContainer(); minTimes = 1; @@ -190,7 +190,7 @@ public void testGenerateSpecWithSchema() throws TransportException, IOException, result = getPluginSchema(); minTimes = 1; - successFactorsTransporter.callSuccessFactorsEntity(null, anyString, anyString); + successFactorsTransporter.callSuccessFactorsEntity(null, anyString); result = getResponseContainer(); minTimes = 1; } @@ -231,7 +231,7 @@ public void testBrowse() throws IOException, TransportException { result = entities; minTimes = 1; - successFactorsTransporter.callSuccessFactorsEntity(null, anyString, anyString); + successFactorsTransporter.callSuccessFactorsEntity(null, anyString); result = getResponseContainer(); minTimes = 1; } @@ -258,7 +258,7 @@ public void testSampleWithoutSampleData() throws IOException, TransportException ConnectorContext context = new MockConnectorContext(new MockConnectorConfigurer()); new Expectations(SuccessFactorsTransporter.class, SuccessFactorsTransporter.class, SuccessFactorsConnector.class) { { - successFactorsTransporter.callSuccessFactorsEntity(null, anyString, anyString); + successFactorsTransporter.callSuccessFactorsEntity(null, anyString); result = getResponseContainer(); minTimes = 1; } diff --git a/src/test/java/io/cdap/plugin/successfactors/source/SuccessFactorsSourceTest.java b/src/test/java/io/cdap/plugin/successfactors/source/SuccessFactorsSourceTest.java index d19f1ab..af53dc3 100644 --- a/src/test/java/io/cdap/plugin/successfactors/source/SuccessFactorsSourceTest.java +++ b/src/test/java/io/cdap/plugin/successfactors/source/SuccessFactorsSourceTest.java @@ -69,6 +69,7 @@ public class SuccessFactorsSourceTest { private EntityProvider entityProvider; private SuccessFactorsUrlContainer successFactorsUrlContainer; private SuccessFactorsSchemaGenerator successFactorsSchemaGenerator; + private SuccessFactorsTransporter successFactorsHttpClient; @Before @@ -163,6 +164,7 @@ public void testConfigurePipelineWSchemaNotNull() throws SuccessFactorsServiceEx successFactorsTransporter = new SuccessFactorsTransporter(pluginConfig.getConnection()); successFactorsUrlContainer = new SuccessFactorsUrlContainer(pluginConfig); successFactorsSchemaGenerator = new SuccessFactorsSchemaGenerator(new SuccessFactorsEntityProvider(edm)); + successFactorsHttpClient = new SuccessFactorsTransporter(pluginConfig.getConnection()); new Expectations(SuccessFactorsUrlContainer.class, SuccessFactorsTransporter.class, SuccessFactorsSchemaGenerator.class) { @@ -175,7 +177,11 @@ public void testConfigurePipelineWSchemaNotNull() throws SuccessFactorsServiceEx result = getUrl(); minTimes = 1; - successFactorsTransporter.callSuccessFactorsEntity(null, anyString, anyString); + successFactorsHttpClient.callSuccessFactorsWithRetry(null, anyString, anyInt, anyInt, anyInt, anyInt); + result = getResponseContainer(); + minTimes = 1; + + successFactorsTransporter.callSuccessFactorsEntity(null, anyString); result = getResponseContainer(); minTimes = 1; diff --git a/src/test/java/io/cdap/plugin/successfactors/source/config/SuccessFactorsPluginConfigTest.java b/src/test/java/io/cdap/plugin/successfactors/source/config/SuccessFactorsPluginConfigTest.java index b30c39f..4af5aaf 100644 --- a/src/test/java/io/cdap/plugin/successfactors/source/config/SuccessFactorsPluginConfigTest.java +++ b/src/test/java/io/cdap/plugin/successfactors/source/config/SuccessFactorsPluginConfigTest.java @@ -174,4 +174,102 @@ public void testRefactoredPluginPropertyValues() { Assert.assertEquals("Entity name not trimmed", "entity-name", pluginConfig.getEntityName()); Assert.assertEquals("Select option not trimmed", "col1,col2,parent/col1,col3", pluginConfig.getSelectOption()); } + + @Test + public void testValidateRetryConfigurationWithDefaultValues() { + SuccessFactorsPluginConfig pluginConfig = pluginConfigBuilder + .setInitialRetryDuration(SuccessFactorsPluginConfig.DEFAULT_INITIAL_RETRY_DURATION_SECONDS) + .setMaxRetryDuration(SuccessFactorsPluginConfig.DEFAULT_MAX_RETRY_DURATION_SECONDS) + .setMaxRetryCount(SuccessFactorsPluginConfig.DEFAULT_MAX_RETRY_COUNT) + .setRetryMultiplier(SuccessFactorsPluginConfig.DEFAULT_RETRY_MULTIPLIER) + .build(); + pluginConfig.validateRetryConfiguration(failureCollector); + Assert.assertEquals(0, failureCollector.getValidationFailures().size()); + } + + @Test + public void testValidateRetryConfigurationWithInvalidInitialRetryDuration() { + SuccessFactorsPluginConfig pluginConfig = pluginConfigBuilder + .setInitialRetryDuration(-1) + .setMaxRetryDuration(SuccessFactorsPluginConfig.DEFAULT_MAX_RETRY_DURATION_SECONDS) + .setMaxRetryCount(SuccessFactorsPluginConfig.DEFAULT_MAX_RETRY_COUNT) + .setRetryMultiplier(SuccessFactorsPluginConfig.DEFAULT_RETRY_MULTIPLIER) + .build(); + pluginConfig.validateRetryConfiguration(failureCollector); + Assert.assertEquals(1, failureCollector.getValidationFailures().size()); + Assert.assertEquals("Initial retry duration must be greater than 0.", + failureCollector.getValidationFailures().get(0).getMessage()); + } + + @Test + public void testValidateRetryConfigurationWithInvalidMaxRetryDuration() { + SuccessFactorsPluginConfig pluginConfig = pluginConfigBuilder + .setInitialRetryDuration(SuccessFactorsPluginConfig.DEFAULT_INITIAL_RETRY_DURATION_SECONDS) + .setMaxRetryDuration(-1) + .setMaxRetryCount(SuccessFactorsPluginConfig.DEFAULT_MAX_RETRY_COUNT) + .setRetryMultiplier(SuccessFactorsPluginConfig.DEFAULT_RETRY_MULTIPLIER) + .build(); + pluginConfig.validateRetryConfiguration(failureCollector); + Assert.assertEquals(2, failureCollector.getValidationFailures().size()); + Assert.assertEquals("Max retry duration must be greater than 0.", + failureCollector.getValidationFailures().get(0).getMessage()); + Assert.assertEquals("Max retry duration must be greater than initial retry duration.", + failureCollector.getValidationFailures().get(1).getMessage()); + } + + @Test + public void testValidateRetryConfigurationWithInvalidRetryMultiplier() { + SuccessFactorsPluginConfig pluginConfig = pluginConfigBuilder + .setInitialRetryDuration(SuccessFactorsPluginConfig.DEFAULT_INITIAL_RETRY_DURATION_SECONDS) + .setMaxRetryDuration(SuccessFactorsPluginConfig.DEFAULT_MAX_RETRY_DURATION_SECONDS) + .setMaxRetryCount(SuccessFactorsPluginConfig.DEFAULT_MAX_RETRY_COUNT) + .setRetryMultiplier(-1) + .build(); + pluginConfig.validateRetryConfiguration(failureCollector); + Assert.assertEquals(1, failureCollector.getValidationFailures().size()); + Assert.assertEquals("Retry multiplier must be strictly greater than 1.", + failureCollector.getValidationFailures().get(0).getMessage()); + } + + @Test + public void testValidateRetryConfigurationWithInvalidRetryMultiplierAndMaxRetryCount() { + SuccessFactorsPluginConfig pluginConfig = pluginConfigBuilder + .setInitialRetryDuration(SuccessFactorsPluginConfig.DEFAULT_INITIAL_RETRY_DURATION_SECONDS) + .setMaxRetryDuration(SuccessFactorsPluginConfig.DEFAULT_MAX_RETRY_DURATION_SECONDS) + .setMaxRetryCount(1) + .setRetryMultiplier(-1) + .build(); + pluginConfig.validateRetryConfiguration(failureCollector); + Assert.assertEquals(1, failureCollector.getValidationFailures().size()); + Assert.assertEquals("Retry multiplier must be strictly greater than 1.", + failureCollector.getValidationFailures().get(0).getMessage()); + } + + @Test + public void testValidateRetryConfigurationWithMultiplierOne() { + SuccessFactorsPluginConfig pluginConfig = pluginConfigBuilder + .setInitialRetryDuration(SuccessFactorsPluginConfig.DEFAULT_INITIAL_RETRY_DURATION_SECONDS) + .setMaxRetryDuration(SuccessFactorsPluginConfig.DEFAULT_MAX_RETRY_DURATION_SECONDS) + .setMaxRetryCount(SuccessFactorsPluginConfig.DEFAULT_MAX_RETRY_COUNT) + .setRetryMultiplier(1) + .build(); + pluginConfig.validateRetryConfiguration(failureCollector); + Assert.assertEquals(1, failureCollector.getValidationFailures().size()); + Assert.assertEquals("Retry multiplier must be strictly greater than 1.", + failureCollector.getValidationFailures().get(0).getMessage()); + } + + @Test + public void testValidateRetryConfigurationWithMaxRetryLessThanInitialRetry() { + SuccessFactorsPluginConfig pluginConfig = pluginConfigBuilder + .setInitialRetryDuration(20) + .setMaxRetryDuration(10) + .setMaxRetryCount(1) + .setRetryMultiplier(2) + .build(); + pluginConfig.validateRetryConfiguration(failureCollector); + Assert.assertEquals(1, failureCollector.getValidationFailures().size()); + Assert.assertEquals("Max retry duration must be greater than initial retry duration.", + failureCollector.getValidationFailures().get(0).getMessage()); + } } diff --git a/src/test/java/io/cdap/plugin/successfactors/source/input/SuccessFactorsInputFormatTest.java b/src/test/java/io/cdap/plugin/successfactors/source/input/SuccessFactorsInputFormatTest.java index fd63057..bd62534 100644 --- a/src/test/java/io/cdap/plugin/successfactors/source/input/SuccessFactorsInputFormatTest.java +++ b/src/test/java/io/cdap/plugin/successfactors/source/input/SuccessFactorsInputFormatTest.java @@ -38,7 +38,7 @@ import java.util.Base64; @RunWith(PowerMockRunner.class) -@PrepareForTest({SuccessFactorsTransporter.class, SuccessFactorsConnectorConfig.class, Gson.class}) +@PrepareForTest({SuccessFactorsConnectorConfig.class, Gson.class}) public class SuccessFactorsInputFormatTest { public static final String SUCCESSFACTORS_PLUGIN_PROPERTIES = "successFactorsPluginProperties"; public static final String ENCODED_ENTITY_METADATA_STRING = "encodedMetadataString"; @@ -57,7 +57,8 @@ public void initializeTests() { "selectOption", "expandOption", "additionalQueryParameters", - null)); + null, null, + null, null, null)); } @Test diff --git a/src/test/java/io/cdap/plugin/successfactors/source/metadata/SuccessFactorsSchemaGeneratorTest.java b/src/test/java/io/cdap/plugin/successfactors/source/metadata/SuccessFactorsSchemaGeneratorTest.java index c237099..afa344b 100644 --- a/src/test/java/io/cdap/plugin/successfactors/source/metadata/SuccessFactorsSchemaGeneratorTest.java +++ b/src/test/java/io/cdap/plugin/successfactors/source/metadata/SuccessFactorsSchemaGeneratorTest.java @@ -49,7 +49,8 @@ public void setup() throws EntityProviderException { pluginConfig = new SuccessFactorsPluginConfig("referenceName", "baseUR", "entityName", "associateEntityName", "username", "password", null, null, null, "filterOption", "selectOption", - "expandOption", "additionalQueryParameters", "paginationType"); + "expandOption", "additionalQueryParameters", "paginationType", + null, null, null, null); } @Test diff --git a/src/test/java/io/cdap/plugin/successfactors/source/transport/SuccessFactorsTransporterTest.java b/src/test/java/io/cdap/plugin/successfactors/source/transport/SuccessFactorsTransporterTest.java index 90c05df..538045b 100644 --- a/src/test/java/io/cdap/plugin/successfactors/source/transport/SuccessFactorsTransporterTest.java +++ b/src/test/java/io/cdap/plugin/successfactors/source/transport/SuccessFactorsTransporterTest.java @@ -18,6 +18,7 @@ import com.github.tomakehurst.wiremock.client.WireMock; import com.github.tomakehurst.wiremock.junit.WireMockRule; +import io.cdap.cdap.api.retry.RetryableException; import io.cdap.plugin.successfactors.common.exception.TransportException; import io.cdap.plugin.successfactors.common.util.ResourceConstants; import io.cdap.plugin.successfactors.source.config.SuccessFactorsPluginConfig; @@ -33,11 +34,13 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; +import org.mockito.Mockito; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.net.HttpURLConnection; import java.net.SocketTimeoutException; +import java.net.URL; import java.net.UnknownHostException; import java.security.KeyManagementException; import java.security.NoSuchAlgorithmException; @@ -52,6 +55,8 @@ import javax.ws.rs.core.MediaType; import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; public class SuccessFactorsTransporterTest { @@ -220,4 +225,17 @@ public void testConnectionTimeout() throws TransportException { transporter.callSuccessFactors(successFactorsURL.getTesterURL(), MediaType.APPLICATION_JSON, SuccessFactorsService.TEST); } + + @Test(expected = RetryableException.class) + public void testValidNumberOfRetry() throws Exception { + SuccessFactorsTransporter transporterSpy = Mockito.spy(transporter); + int retryCount = 2; + try { + transporterSpy.callSuccessFactorsWithRetry(new URL("http://127.0.0.1/"), "TEST", + 1, 3, 2, retryCount); + } finally { + verify(transporterSpy, times(retryCount + 1)) + .retrySapTransportCall(Mockito.any(URL.class), Mockito.anyString()); + } + } } diff --git a/src/test/java/io/cdap/plugin/successfactors/source/transport/SuccessFactorsUrlContainerTest.java b/src/test/java/io/cdap/plugin/successfactors/source/transport/SuccessFactorsUrlContainerTest.java index 540c3c9..9f64c2c 100644 --- a/src/test/java/io/cdap/plugin/successfactors/source/transport/SuccessFactorsUrlContainerTest.java +++ b/src/test/java/io/cdap/plugin/successfactors/source/transport/SuccessFactorsUrlContainerTest.java @@ -38,7 +38,8 @@ public void initializeTests() { "selectOption", "expandOption", "", - null)); + null, null, + null, null, null)); } @Test public void testGetTesterURL() { @@ -78,7 +79,8 @@ public void testGetURLWithAdditionalQueryParameters() { "", "", "startDate=2023-01-01&endDate=2023-02-02", - null)); + null, null, + null, null, null)); SuccessFactorsUrlContainer urlContainer = new SuccessFactorsUrlContainer(pluginConfig); String expectedUrl = "https://successfactors.com/EmpJob?startDate=2023-01-01&endDate=2023-02-02&%24top=1"; URL actualUrl = urlContainer.getTesterURL(); diff --git a/widgets/SuccessFactors-batchsource.json b/widgets/SuccessFactors-batchsource.json index 8cb6189..65e1f99 100644 --- a/widgets/SuccessFactors-batchsource.json +++ b/widgets/SuccessFactors-batchsource.json @@ -174,6 +174,42 @@ } ] } + }, + { + "widget-type": "hidden", + "label": "Initial Retry Duration (Seconds)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": "2", + "minimum": "1" + } + }, + { + "widget-type": "hidden", + "label": "Max Retry Duration (Seconds)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": "300", + "minimum": "1" + } + }, + { + "widget-type": "hidden", + "label": "Max Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": "3", + "minimum": "1" + } + }, + { + "widget-type": "hidden", + "label": "Retry Multiplier", + "name": "retryMultiplier", + "widget-attributes": { + "default": "2", + "placeholder": "The multiplier to use on retry attempts." + } } ] }