diff --git a/src/main/java/org/eclipse/xpanse/terraform/boot/TerraformBootApplication.java b/src/main/java/org/eclipse/xpanse/terraform/boot/TerraformBootApplication.java index 7147770..efcf35f 100644 --- a/src/main/java/org/eclipse/xpanse/terraform/boot/TerraformBootApplication.java +++ b/src/main/java/org/eclipse/xpanse/terraform/boot/TerraformBootApplication.java @@ -8,10 +8,12 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration; +import org.springframework.scheduling.annotation.EnableAsync; /** * Main entry class to terraform-boot. This class can be directly executed to start the server. */ +@EnableAsync @SpringBootApplication(exclude = {SecurityAutoConfiguration.class}) public class TerraformBootApplication { diff --git a/src/main/java/org/eclipse/xpanse/terraform/boot/api/TerraformApiController.java b/src/main/java/org/eclipse/xpanse/terraform/boot/api/TerraformApiController.java index de20a01..9bb4e6b 100644 --- a/src/main/java/org/eclipse/xpanse/terraform/boot/api/TerraformApiController.java +++ b/src/main/java/org/eclipse/xpanse/terraform/boot/api/TerraformApiController.java @@ -14,6 +14,8 @@ import org.eclipse.xpanse.terraform.boot.models.enums.HealthStatus; import org.eclipse.xpanse.terraform.boot.models.request.TerraformDeployRequest; import org.eclipse.xpanse.terraform.boot.models.request.TerraformDestroyRequest; +import org.eclipse.xpanse.terraform.boot.models.request.async.TerraformAsyncDeployRequest; +import org.eclipse.xpanse.terraform.boot.models.request.async.TerraformAsyncDestroyRequest; import org.eclipse.xpanse.terraform.boot.models.response.TerraformResult; import org.eclipse.xpanse.terraform.boot.models.validation.TerraformValidationResult; import org.eclipse.xpanse.terraform.boot.terraform.TerraformExecutor; @@ -100,7 +102,7 @@ public TerraformResult deploy( * @return Returns the status of the resources destroy. */ @Tag(name = "Terraform", description = "APIs for running Terraform commands") - @Operation(description = "Validate the Terraform modules") + @Operation(description = "Destroy the Terraform modules") @DeleteMapping(value = "/destroy/{module_directory}", produces = MediaType.APPLICATION_JSON_VALUE) @ResponseStatus(HttpStatus.OK) @@ -111,4 +113,38 @@ public TerraformResult destroy( @Valid @RequestBody TerraformDestroyRequest terraformDestroyRequest) { return this.terraformExecutor.destroy(terraformDestroyRequest, moduleDirectory); } + + /** + * Method to async deploy resources requested in a workspace. + * + */ + @Tag(name = "Terraform", description = "APIs for running Terraform commands") + @Operation(description = "async deploy resources via Terraform") + @PostMapping(value = "/deploy/async/{module_directory}", produces = + MediaType.APPLICATION_JSON_VALUE) + @ResponseStatus(HttpStatus.ACCEPTED) + public void asyncDeploy( + @Parameter(name = "module_directory", + description = "directory name where the Terraform module files exist.") + @PathVariable("module_directory") String moduleDirectory, + @Valid @RequestBody TerraformAsyncDeployRequest asyncDeployRequest) { + terraformExecutor.asyncDeploy(asyncDeployRequest, moduleDirectory); + } + + /** + * Method to async destroy resources requested in a workspace. + * + */ + @Tag(name = "Terraform", description = "APIs for running Terraform commands") + @Operation(description = "Async destroy the Terraform modules") + @DeleteMapping(value = "/destroy/async/{module_directory}", + produces = MediaType.APPLICATION_JSON_VALUE) + @ResponseStatus(HttpStatus.ACCEPTED) + public void asyncDestroy( + @Parameter(name = "module_directory", + description = "directory name where the Terraform module files exist.") + @PathVariable("module_directory") String moduleDirectory, + @Valid @RequestBody TerraformAsyncDestroyRequest asyncDestroyRequest) { + terraformExecutor.asyncDestroy(asyncDestroyRequest, moduleDirectory); + } } diff --git a/src/main/java/org/eclipse/xpanse/terraform/boot/async/ServiceThreadPoolTaskExecutor.java b/src/main/java/org/eclipse/xpanse/terraform/boot/async/ServiceThreadPoolTaskExecutor.java new file mode 100644 index 0000000..3782f2f --- /dev/null +++ b/src/main/java/org/eclipse/xpanse/terraform/boot/async/ServiceThreadPoolTaskExecutor.java @@ -0,0 +1,39 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * SPDX-FileCopyrightText: Huawei Inc. + * + */ + +package org.eclipse.xpanse.terraform.boot.async; + +import java.util.concurrent.Callable; +import java.util.concurrent.Future; +import org.slf4j.MDC; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; + +/** + * Overwrite the thread pool to solve the problem that the traceId will be lost in the process of + * printing the log. + */ +public class ServiceThreadPoolTaskExecutor extends ThreadPoolTaskExecutor { + + public ServiceThreadPoolTaskExecutor() { + super(); + } + + @Override + public void execute(Runnable task) { + super.execute(ThreadMdcUtil.wrap(task, MDC.getCopyOfContextMap())); + } + + + @Override + public Future submit(Callable task) { + return super.submit(ThreadMdcUtil.wrap(task, MDC.getCopyOfContextMap())); + } + + @Override + public Future submit(Runnable task) { + return super.submit(ThreadMdcUtil.wrap(task, MDC.getCopyOfContextMap())); + } +} diff --git a/src/main/java/org/eclipse/xpanse/terraform/boot/async/TaskConfiguration.java b/src/main/java/org/eclipse/xpanse/terraform/boot/async/TaskConfiguration.java new file mode 100644 index 0000000..65d4986 --- /dev/null +++ b/src/main/java/org/eclipse/xpanse/terraform/boot/async/TaskConfiguration.java @@ -0,0 +1,41 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * SPDX-FileCopyrightText: Huawei Inc. + * + */ + +package org.eclipse.xpanse.terraform.boot.async; + +import java.util.concurrent.Executor; +import java.util.concurrent.ThreadPoolExecutor; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * Customize the thread pool. Define ThreadPoolTaskExecutor named taskExecutor to replace @Async's + * default thread pool. + */ +@Configuration +public class TaskConfiguration { + + private static final int CPU_COUNT = Runtime.getRuntime().availableProcessors(); + + /** + * Define ThreadPoolTaskExecutor named taskExecutor. + * + * @return executor + */ + @Bean("taskExecutor") + public Executor taskExecutor() { + ServiceThreadPoolTaskExecutor + executor = new ServiceThreadPoolTaskExecutor(); + executor.setCorePoolSize(CPU_COUNT * 2); + executor.setMaxPoolSize(20); + executor.setQueueCapacity(200); + executor.setKeepAliveSeconds(300); + executor.setThreadNamePrefix("thread-pool-"); + executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy()); + executor.initialize(); + return executor; + } +} diff --git a/src/main/java/org/eclipse/xpanse/terraform/boot/async/ThreadMdcUtil.java b/src/main/java/org/eclipse/xpanse/terraform/boot/async/ThreadMdcUtil.java new file mode 100644 index 0000000..66dfa6b --- /dev/null +++ b/src/main/java/org/eclipse/xpanse/terraform/boot/async/ThreadMdcUtil.java @@ -0,0 +1,68 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * SPDX-FileCopyrightText: Huawei Inc. + * + */ + +package org.eclipse.xpanse.terraform.boot.async; + +import java.util.Map; +import java.util.concurrent.Callable; +import org.slf4j.MDC; + +/** + * Bean to get mdc logging info. + */ +public final class ThreadMdcUtil { + + /** + * When the parent thread submits a callable task to the thread pool, it copies the data in its + * own MDC to the child thread. + * + * @param callable callable task + * @param context context + * @param return object type + * @return T + */ + public static Callable wrap(final Callable callable, + final Map context) { + return new Callable<>() { + @Override + public T call() throws Exception { + if (context == null) { + MDC.clear(); + } else { + MDC.setContextMap(context); + } + try { + return callable.call(); + } finally { + MDC.clear(); + } + } + }; + } + + /** + * When the parent thread submits a runnable task to the thread pool, it copies the data in its + * own MDC to the child thread. + * + * @param runnable runnable task + * @param context context + * @return thread task + */ + public static Runnable wrap(final Runnable runnable, final Map context) { + return () -> { + if (context == null) { + MDC.clear(); + } else { + MDC.setContextMap(context); + } + try { + runnable.run(); + } finally { + MDC.clear(); + } + }; + } +} diff --git a/src/main/java/org/eclipse/xpanse/terraform/boot/config/RestTemplateConfig.java b/src/main/java/org/eclipse/xpanse/terraform/boot/config/RestTemplateConfig.java new file mode 100644 index 0000000..6be1240 --- /dev/null +++ b/src/main/java/org/eclipse/xpanse/terraform/boot/config/RestTemplateConfig.java @@ -0,0 +1,40 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * SPDX-FileCopyrightText: Huawei Inc. + */ + +package org.eclipse.xpanse.terraform.boot.config; + + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.http.client.ClientHttpRequestFactory; +import org.springframework.http.client.SimpleClientHttpRequestFactory; +import org.springframework.web.client.RestTemplate; + +/** + * Configuration class for RestTemplate. + */ +@Configuration +public class RestTemplateConfig { + + /** + * Create RestTemplate to IOC. + */ + @Bean + public RestTemplate restTemplate(ClientHttpRequestFactory factory) { + return new RestTemplate(factory); + } + + /** + * Create ClientHttpRequestFactory to IOC. + */ + @Bean + public ClientHttpRequestFactory simpleClientHttpRequestFactory() { + SimpleClientHttpRequestFactory factory = new SimpleClientHttpRequestFactory(); + factory.setConnectTimeout(15000); + factory.setReadTimeout(5000); + return factory; + } + +} diff --git a/src/main/java/org/eclipse/xpanse/terraform/boot/models/enums/AuthType.java b/src/main/java/org/eclipse/xpanse/terraform/boot/models/enums/AuthType.java new file mode 100644 index 0000000..2d1e8a8 --- /dev/null +++ b/src/main/java/org/eclipse/xpanse/terraform/boot/models/enums/AuthType.java @@ -0,0 +1,14 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * SPDX-FileCopyrightText: Huawei Inc. + */ + +package org.eclipse.xpanse.terraform.boot.models.enums; + +/** + * The permission type class when calling back. + */ +public enum AuthType { + NONE, + OAUTH2; +} diff --git a/src/main/java/org/eclipse/xpanse/terraform/boot/api/exceptions/TerraformApiExceptionHandler.java b/src/main/java/org/eclipse/xpanse/terraform/boot/models/exceptions/TerraformApiExceptionHandler.java similarity index 94% rename from src/main/java/org/eclipse/xpanse/terraform/boot/api/exceptions/TerraformApiExceptionHandler.java rename to src/main/java/org/eclipse/xpanse/terraform/boot/models/exceptions/TerraformApiExceptionHandler.java index 2f4e8a8..9406d5e 100644 --- a/src/main/java/org/eclipse/xpanse/terraform/boot/api/exceptions/TerraformApiExceptionHandler.java +++ b/src/main/java/org/eclipse/xpanse/terraform/boot/models/exceptions/TerraformApiExceptionHandler.java @@ -3,14 +3,12 @@ * SPDX-FileCopyrightText: Huawei Inc. */ -package org.eclipse.xpanse.terraform.boot.api.exceptions; +package org.eclipse.xpanse.terraform.boot.models.exceptions; import java.util.ArrayList; import java.util.Collections; import java.util.List; import lombok.extern.slf4j.Slf4j; -import org.eclipse.xpanse.terraform.boot.models.exceptions.TerraformExecutorException; -import org.eclipse.xpanse.terraform.boot.models.exceptions.UnsupportedEnumValueException; import org.eclipse.xpanse.terraform.boot.models.response.Response; import org.eclipse.xpanse.terraform.boot.models.response.ResultType; import org.springframework.http.HttpStatus; diff --git a/src/main/java/org/eclipse/xpanse/terraform/boot/models/request/async/TerraformAsyncDeployRequest.java b/src/main/java/org/eclipse/xpanse/terraform/boot/models/request/async/TerraformAsyncDeployRequest.java new file mode 100644 index 0000000..64623dd --- /dev/null +++ b/src/main/java/org/eclipse/xpanse/terraform/boot/models/request/async/TerraformAsyncDeployRequest.java @@ -0,0 +1,23 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * SPDX-FileCopyrightText: Huawei Inc. + */ + +package org.eclipse.xpanse.terraform.boot.models.request.async; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.constraints.NotNull; +import lombok.Data; +import org.eclipse.xpanse.terraform.boot.models.request.TerraformDeployRequest; + +/** + * Data model for the terraform async deploy requests. + */ +@Data +public class TerraformAsyncDeployRequest extends TerraformDeployRequest { + + @NotNull + @Schema(description = "Configuration information of webhook.") + private WebhookConfig webhookConfig; + +} diff --git a/src/main/java/org/eclipse/xpanse/terraform/boot/models/request/async/TerraformAsyncDestroyRequest.java b/src/main/java/org/eclipse/xpanse/terraform/boot/models/request/async/TerraformAsyncDestroyRequest.java new file mode 100644 index 0000000..4a8daf2 --- /dev/null +++ b/src/main/java/org/eclipse/xpanse/terraform/boot/models/request/async/TerraformAsyncDestroyRequest.java @@ -0,0 +1,23 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * SPDX-FileCopyrightText: Huawei Inc. + */ + +package org.eclipse.xpanse.terraform.boot.models.request.async; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.constraints.NotNull; +import lombok.Data; +import org.eclipse.xpanse.terraform.boot.models.request.TerraformDestroyRequest; + +/** + * Data model for the terraform async destroy requests. + */ +@Data +public class TerraformAsyncDestroyRequest extends TerraformDestroyRequest { + + @NotNull + @Schema(description = "Configuration information of webhook.") + private WebhookConfig webhookConfig; + +} diff --git a/src/main/java/org/eclipse/xpanse/terraform/boot/models/request/async/WebhookConfig.java b/src/main/java/org/eclipse/xpanse/terraform/boot/models/request/async/WebhookConfig.java new file mode 100644 index 0000000..241eb2d --- /dev/null +++ b/src/main/java/org/eclipse/xpanse/terraform/boot/models/request/async/WebhookConfig.java @@ -0,0 +1,26 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * SPDX-FileCopyrightText: Huawei Inc. + */ + +package org.eclipse.xpanse.terraform.boot.models.request.async; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.constraints.NotNull; +import lombok.Data; +import org.eclipse.xpanse.terraform.boot.models.enums.AuthType; + +/** + * Configuration information class of webhook. + */ +@Data +public class WebhookConfig { + + @NotNull + @Schema(description = "Callback address after deployment/destroy is completed.") + private String url; + + @NotNull + @Schema(description = "The permission type when calling back.") + private AuthType authType; +} diff --git a/src/main/java/org/eclipse/xpanse/terraform/boot/terraform/TerraformExecutor.java b/src/main/java/org/eclipse/xpanse/terraform/boot/terraform/TerraformExecutor.java index cc7c54d..7556865 100644 --- a/src/main/java/org/eclipse/xpanse/terraform/boot/terraform/TerraformExecutor.java +++ b/src/main/java/org/eclipse/xpanse/terraform/boot/terraform/TerraformExecutor.java @@ -21,13 +21,17 @@ import org.eclipse.xpanse.terraform.boot.models.exceptions.TerraformExecutorException; import org.eclipse.xpanse.terraform.boot.models.request.TerraformDeployRequest; import org.eclipse.xpanse.terraform.boot.models.request.TerraformDestroyRequest; +import org.eclipse.xpanse.terraform.boot.models.request.async.TerraformAsyncDeployRequest; +import org.eclipse.xpanse.terraform.boot.models.request.async.TerraformAsyncDestroyRequest; import org.eclipse.xpanse.terraform.boot.models.response.TerraformResult; import org.eclipse.xpanse.terraform.boot.models.validation.TerraformValidationResult; import org.eclipse.xpanse.terraform.boot.terraform.utils.SystemCmd; import org.eclipse.xpanse.terraform.boot.terraform.utils.SystemCmdResult; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Component; +import org.springframework.web.client.RestTemplate; /** * An executor for terraform. @@ -47,6 +51,8 @@ public class TerraformExecutor { private final SystemCmd systemCmd; + private final RestTemplate restTemplate; + private final boolean isStdoutStdErrLoggingEnabled; private final String customTerraformBinary; @@ -63,7 +69,7 @@ public class TerraformExecutor { * @param terraformLogLevel value of `terraform.log.level` property */ @Autowired - public TerraformExecutor(SystemCmd systemCmd, + public TerraformExecutor(SystemCmd systemCmd, RestTemplate restTemplate, @Value("${terraform.root.module.directory}") String moduleParentDirectoryPath, @Value("${log.terraform.stdout.stderr:true}") @@ -80,6 +86,7 @@ public TerraformExecutor(SystemCmd systemCmd, this.moduleParentDirectoryPath = moduleParentDirectoryPath; } this.systemCmd = systemCmd; + this.restTemplate = restTemplate; this.customTerraformBinary = customTerraformBinary; this.isStdoutStdErrLoggingEnabled = isStdoutStdErrLoggingEnabled; this.terraformLogLevel = terraformLogLevel; @@ -165,30 +172,25 @@ private SystemCmdResult execute(String cmd, String workspace, * Deploy a source by terraform. */ public TerraformResult deploy(TerraformDeployRequest terraformDeployRequest, String workspace) { + SystemCmdResult result; if (Boolean.TRUE.equals(terraformDeployRequest.getIsPlanOnly())) { - SystemCmdResult tfPlanResult = - tfPlan(terraformDeployRequest.getVariables(), - terraformDeployRequest.getEnvVariables(), workspace); - return TerraformResult.builder() - .commandStdOutput(tfPlanResult.getCommandStdOutput()) - .commandStdError(tfPlanResult.getCommandStdError()) - .isCommandSuccessful(tfPlanResult.isCommandSuccessful()) - .build(); + result = tfPlan(terraformDeployRequest.getVariables(), + terraformDeployRequest.getEnvVariables(), workspace); } else { - SystemCmdResult applyResult = tfApplyCommand(terraformDeployRequest.getVariables(), + result = tfApplyCommand(terraformDeployRequest.getVariables(), terraformDeployRequest.getEnvVariables(), getModuleFullPath(workspace)); - if (!applyResult.isCommandSuccessful()) { + if (!result.isCommandSuccessful()) { log.error("TFExecutor.tfApply failed."); throw new TerraformExecutorException("TFExecutor.tfApply failed.", - applyResult.getCommandStdError()); + result.getCommandStdError()); } - return TerraformResult.builder() - .commandStdOutput(applyResult.getCommandStdOutput()) - .commandStdError(applyResult.getCommandStdError()) - .isCommandSuccessful(applyResult.isCommandSuccessful()) - .build(); } + return TerraformResult.builder() + .commandStdOutput(result.getCommandStdOutput()) + .commandStdError(result.getCommandStdError()) + .isCommandSuccessful(result.isCommandSuccessful()) + .build(); } /** @@ -213,6 +215,27 @@ public TerraformResult destroy(TerraformDestroyRequest terraformDestroyRequest, .build(); } + /** + * Async deploy a source by terraform. + */ + @Async("taskExecutor") + public void asyncDeploy(TerraformAsyncDeployRequest asyncDeployRequest, + String workspace) { + TerraformResult result = deploy(asyncDeployRequest, workspace); + log.info("Deployment service complete, {}", result.getCommandStdOutput()); + restTemplate.postForLocation(asyncDeployRequest.getWebhookConfig().getUrl(), result); + } + + /** + * Async destroy resource of the service. + */ + @Async("taskExecutor") + public void asyncDestroy(TerraformAsyncDestroyRequest asyncDestroyRequest, String workspace) { + TerraformResult result = destroy(asyncDestroyRequest, workspace); + log.info("Destroy service complete, {}", result.getCommandStdOutput()); + restTemplate.postForLocation(asyncDestroyRequest.getWebhookConfig().getUrl(), result); + } + /** * Executes terraform validate command. *