diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6cb7047..4a71db9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,7 +15,11 @@ on: - Major Version - Minor Version - Patch Version - + DEFAULT_TERRAFORM_VERSION: + type: string + description: Input the default version of Terraform to install + required: true + default: '1.6.0' env: BOT_USER_NAME: eclipse-xpanse-bot BOT_EMAIL_ID: xpanse-bot@eclipse.org @@ -103,7 +107,8 @@ jobs: tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest,${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.new_version.outputs.next-version }} labels: ${{ steps.meta.outputs.labels }} provenance: false - + build-args: | + DEFAULT_TERRAFORM_VERSION=${{ github.event.inputs.DEFAULT_TERRAFORM_VERSION }} - name: Push POM updates with release version uses: EndBug/add-and-commit@v9 with: diff --git a/Dockerfile b/Dockerfile index b9197b2..f4893ba 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,10 +3,15 @@ FROM eclipse-temurin:21-jre-alpine RUN addgroup -S terraform-boot && adduser -S -G terraform-boot terraform-boot RUN apk update && \ apk add --no-cache unzip wget -ENV TERRAFORM_VERSION=1.4.4 -RUN wget https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip -RUN unzip terraform_${TERRAFORM_VERSION}_linux_amd64.zip -RUN mv terraform /usr/bin/terraform + +ENV TERRAFORM_INSTALL_PATH=/opt/terraform +ENV DEFAULT_TERRAFORM_VERSION=1.6.0 +ENV TERRAFORM_VERSIONS=1.6.0,1.7.0,1.8.0,1.9.0 +COPY install_terraform.sh /install_terraform.sh +RUN chmod +x /install_terraform.sh +RUN echo "Downloading and installing Terraform with multiple versions $TERRAFORM_VERSIONS into path $TERRAFORM_INSTALL_PATH"; \ + /install_terraform.sh "$TERRAFORM_INSTALL_PATH" "$DEFAULT_TERRAFORM_VERSION" "$TERRAFORM_VERSIONS" + COPY target/terraform-boot-*.jar terraform-boot.jar USER terraform-boot -ENTRYPOINT ["java","-jar","terraform-boot.jar"] \ No newline at end of file +ENTRYPOINT ["java", "-Dterraform.install.dir=${TERRAFORM_INSTALL_PATH}", "-Dterraform.versions=${TERRAFORM_VERSIONS}", "-jar", "terraform-boot.jar"] \ No newline at end of file diff --git a/install_terraform.sh b/install_terraform.sh new file mode 100644 index 0000000..82d3030 --- /dev/null +++ b/install_terraform.sh @@ -0,0 +1,42 @@ +#!/bin/sh +if [ -z "$1" ] || [ -z "$2" ]; then + echo "Error：TERRAFORM_INSTALL_PATH and DEFAULT_OPENTOFU_VERSION could not be empty." + exit 1 +fi +TERRAFORM_INSTALL_PATH="$1" +DEFAULT_TERRAFORM_VERSION="$2" +TERRAFORM_VERSIONS="$3" +mkdir -p "${TERRAFORM_INSTALL_PATH}" +# Install default version of Terraform in system path and custom path +echo "Start installing Terraform with default version ${DEFAULT_TERRAFORM_VERSION}" +wget -c "https://releases.hashicorp.com/terraform/${DEFAULT_TERRAFORM_VERSION}/terraform_${DEFAULT_TERRAFORM_VERSION}_linux_amd64.zip" +if [ -f "terraform_${DEFAULT_TERRAFORM_VERSION}_linux_amd64.zip" ]; then + unzip -o "terraform_${DEFAULT_TERRAFORM_VERSION}_linux_amd64.zip" + cp -f terraform /usr/bin/terraform + chmod +x /usr/bin/terraform + mv -f terraform "${TERRAFORM_INSTALL_PATH}/terraform-${DEFAULT_TERRAFORM_VERSION}" + chmod +x "${TERRAFORM_INSTALL_PATH}/terraform-${DEFAULT_TERRAFORM_VERSION}" + rm -f "terraform_${DEFAULT_TERRAFORM_VERSION}_linux_amd64.zip" + echo "Installed Terraform with default version ${DEFAULT_TERRAFORM_VERSION} into path ${TERRAFORM_INSTALL_PATH} successfully." +else + echo "Failed to download zip package of Terraform with default version terraform_${DEFAULT_TERRAFORM_VERSION}_linux_amd64.zip." +fi +if [ -z "$TERRAFORM_VERSIONS" ]; then + echo "No Terraform versions specified, skip installing Terraform versions." + exit 0 +fi +# Install versions of Terraform specified in TERRAFORM_VERSIONS into custom path +VERSIONS=$(echo "$TERRAFORM_VERSIONS" | tr ',' '\n' | tr -d ' ') +for version in $VERSIONS; do + echo "Start installing Terraform with version ${version} into path ${TERRAFORM_INSTALL_PATH}" + wget -c "https://releases.hashicorp.com/terraform/${version}/terraform_${version}_linux_amd64.zip" + if [ ! -f "terraform_${version}_linux_amd64.zip" ]; then + echo "Failed to download zip package of Terraform with version terraform_${version}_linux_amd64.zip." + continue + fi + unzip -o "terraform_${version}_linux_amd64.zip" + mv -f terraform "${TERRAFORM_INSTALL_PATH}/terraform-${version}" + chmod +x "${TERRAFORM_INSTALL_PATH}/terraform-${version}" + rm -f "terraform_${version}_linux_amd64.zip" + echo "Installed Terraform with version ${version} into path ${TERRAFORM_INSTALL_PATH} successfully." +done diff --git a/pom.xml b/pom.xml index 55169fc..6abe2ee 100644 --- a/pom.xml +++ b/pom.xml @@ -30,7 +30,7 @@ 1.1.0 7.0.0.202409031743-r 5.4.0 - 1.18.1 + 1.326 3.13.0 3.5.0 3.5.0 @@ -54,7 +54,14 @@ org.springframework.boot spring-boot-starter-web - + + org.springframework.boot + spring-boot-starter-cache + + + com.github.ben-manes.caffeine + caffeine + org.projectlombok lombok @@ -119,9 +126,9 @@ ${semver4j.version} - org.jsoup - jsoup - ${jsoup.version} + org.kohsuke + github-api + ${github-api.version} diff --git a/src/main/java/org/eclipse/xpanse/terraform/boot/TerraformBootApplication.java b/src/main/java/org/eclipse/xpanse/terraform/boot/TerraformBootApplication.java index 65d6349..afb2033 100644 --- a/src/main/java/org/eclipse/xpanse/terraform/boot/TerraformBootApplication.java +++ b/src/main/java/org/eclipse/xpanse/terraform/boot/TerraformBootApplication.java @@ -8,14 +8,18 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration; +import org.springframework.cache.annotation.EnableCaching; import org.springframework.retry.annotation.EnableRetry; import org.springframework.scheduling.annotation.EnableAsync; +import org.springframework.scheduling.annotation.EnableScheduling; /** * Main entry class to terraform-boot. This class can be directly executed to start the server. */ @EnableRetry @EnableAsync +@EnableCaching +@EnableScheduling @SpringBootApplication(exclude = {SecurityAutoConfiguration.class}) public class TerraformBootApplication { diff --git a/src/main/java/org/eclipse/xpanse/terraform/boot/api/controllers/TerraformBootFromDirectoryApi.java b/src/main/java/org/eclipse/xpanse/terraform/boot/api/controllers/TerraformBootFromDirectoryApi.java index 9a7075d..6fe2657 100644 --- a/src/main/java/org/eclipse/xpanse/terraform/boot/api/controllers/TerraformBootFromDirectoryApi.java +++ b/src/main/java/org/eclipse/xpanse/terraform/boot/api/controllers/TerraformBootFromDirectoryApi.java @@ -26,8 +26,8 @@ import org.eclipse.xpanse.terraform.boot.models.request.directory.TerraformModifyFromDirectoryRequest; import org.eclipse.xpanse.terraform.boot.models.response.TerraformResult; import org.eclipse.xpanse.terraform.boot.models.validation.TerraformValidationResult; -import org.eclipse.xpanse.terraform.boot.terraform.TerraformVersionHelper; import org.eclipse.xpanse.terraform.boot.terraform.service.TerraformDirectoryService; +import org.eclipse.xpanse.terraform.boot.terraform.tool.TerraformVersionsHelper; import org.slf4j.MDC; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -79,7 +79,7 @@ public TerraformValidationResult validateFromDirectory( @PathVariable("module_directory") String moduleDirectory, @Parameter(name = "terraform_version", description = "version of Terraform to execute the module files.") - @NotBlank @Pattern(regexp = TerraformVersionHelper.TERRAFORM_REQUIRED_VERSION_REGEX) + @NotBlank @Pattern(regexp = TerraformVersionsHelper.TERRAFORM_REQUIRED_VERSION_REGEX) @PathVariable("terraform_version") String terraformVersion) { UUID uuid = UUID.randomUUID(); MDC.put(REQUEST_ID, uuid.toString()); diff --git a/src/main/java/org/eclipse/xpanse/terraform/boot/cache/CaffeineCacheConfig.java b/src/main/java/org/eclipse/xpanse/terraform/boot/cache/CaffeineCacheConfig.java new file mode 100644 index 0000000..54f049e --- /dev/null +++ b/src/main/java/org/eclipse/xpanse/terraform/boot/cache/CaffeineCacheConfig.java @@ -0,0 +1,44 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * SPDX-FileCopyrightText: Huawei Inc. + * + */ + +package org.eclipse.xpanse.terraform.boot.cache; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import lombok.extern.slf4j.Slf4j; +import org.springframework.cache.CacheManager; +import org.springframework.cache.caffeine.CaffeineCacheManager; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * Caffeine cache configuration class. + */ +@Slf4j +@Configuration +public class CaffeineCacheConfig { + + public static final String TERRAFORM_VERSIONS_CACHE_NAME = "TERRAFORM_VERSIONS_CACHE"; + + /** + * Config cache manager with caffeine. + * + * @return caffeineCacheManager + */ + @Bean + public CacheManager caffeineCacheManager() { + CaffeineCacheManager cacheManager = new CaffeineCacheManager(); + cacheManager.registerCustomCache(TERRAFORM_VERSIONS_CACHE_NAME, + getTerraformVersionsCache()); + return cacheManager; + } + + + private Cache getTerraformVersionsCache() { + return Caffeine.newBuilder().build(); + } + +} diff --git a/src/main/java/org/eclipse/xpanse/terraform/boot/models/plan/TerraformPlanFromDirectoryRequest.java b/src/main/java/org/eclipse/xpanse/terraform/boot/models/plan/TerraformPlanFromDirectoryRequest.java index 5ab8864..86f6467 100644 --- a/src/main/java/org/eclipse/xpanse/terraform/boot/models/plan/TerraformPlanFromDirectoryRequest.java +++ b/src/main/java/org/eclipse/xpanse/terraform/boot/models/plan/TerraformPlanFromDirectoryRequest.java @@ -13,7 +13,7 @@ import java.util.Map; import java.util.UUID; import lombok.Data; -import org.eclipse.xpanse.terraform.boot.terraform.TerraformVersionHelper; +import org.eclipse.xpanse.terraform.boot.terraform.tool.TerraformVersionsHelper; /** * Data model for the generating terraform plan. @@ -26,7 +26,7 @@ public class TerraformPlanFromDirectoryRequest { @NotNull @NotBlank - @Pattern(regexp = TerraformVersionHelper.TERRAFORM_REQUIRED_VERSION_REGEX) + @Pattern(regexp = TerraformVersionsHelper.TERRAFORM_REQUIRED_VERSION_REGEX) @Schema(description = "The required version of the terraform which will execute the scripts.") String terraformVersion; diff --git a/src/main/java/org/eclipse/xpanse/terraform/boot/models/request/directory/TerraformDeployFromDirectoryRequest.java b/src/main/java/org/eclipse/xpanse/terraform/boot/models/request/directory/TerraformDeployFromDirectoryRequest.java index 3ff72dd..cee5c57 100644 --- a/src/main/java/org/eclipse/xpanse/terraform/boot/models/request/directory/TerraformDeployFromDirectoryRequest.java +++ b/src/main/java/org/eclipse/xpanse/terraform/boot/models/request/directory/TerraformDeployFromDirectoryRequest.java @@ -13,7 +13,7 @@ import java.util.Map; import java.util.UUID; import lombok.Data; -import org.eclipse.xpanse.terraform.boot.terraform.TerraformVersionHelper; +import org.eclipse.xpanse.terraform.boot.terraform.tool.TerraformVersionsHelper; /** * Data model for the terraform deploy requests. @@ -26,7 +26,7 @@ public class TerraformDeployFromDirectoryRequest { @NotNull @NotBlank - @Pattern(regexp = TerraformVersionHelper.TERRAFORM_REQUIRED_VERSION_REGEX) + @Pattern(regexp = TerraformVersionsHelper.TERRAFORM_REQUIRED_VERSION_REGEX) @Schema(description = "The required version of the terraform which will execute the scripts.") String terraformVersion; diff --git a/src/main/java/org/eclipse/xpanse/terraform/boot/models/request/directory/TerraformDestroyFromDirectoryRequest.java b/src/main/java/org/eclipse/xpanse/terraform/boot/models/request/directory/TerraformDestroyFromDirectoryRequest.java index 4542323..a3745b0 100644 --- a/src/main/java/org/eclipse/xpanse/terraform/boot/models/request/directory/TerraformDestroyFromDirectoryRequest.java +++ b/src/main/java/org/eclipse/xpanse/terraform/boot/models/request/directory/TerraformDestroyFromDirectoryRequest.java @@ -15,7 +15,7 @@ import java.util.Map; import java.util.UUID; import lombok.Data; -import org.eclipse.xpanse.terraform.boot.terraform.TerraformVersionHelper; +import org.eclipse.xpanse.terraform.boot.terraform.tool.TerraformVersionsHelper; /** * Data model for the terraform destroy requests. @@ -28,7 +28,7 @@ public class TerraformDestroyFromDirectoryRequest { @NotNull @NotBlank - @Pattern(regexp = TerraformVersionHelper.TERRAFORM_REQUIRED_VERSION_REGEX) + @Pattern(regexp = TerraformVersionsHelper.TERRAFORM_REQUIRED_VERSION_REGEX) @Schema(description = "The required version of the terraform which will execute the scripts.") String terraformVersion; diff --git a/src/main/java/org/eclipse/xpanse/terraform/boot/models/request/directory/TerraformModifyFromDirectoryRequest.java b/src/main/java/org/eclipse/xpanse/terraform/boot/models/request/directory/TerraformModifyFromDirectoryRequest.java index eab71d9..7a417b8 100644 --- a/src/main/java/org/eclipse/xpanse/terraform/boot/models/request/directory/TerraformModifyFromDirectoryRequest.java +++ b/src/main/java/org/eclipse/xpanse/terraform/boot/models/request/directory/TerraformModifyFromDirectoryRequest.java @@ -15,7 +15,7 @@ import java.util.Map; import java.util.UUID; import lombok.Data; -import org.eclipse.xpanse.terraform.boot.terraform.TerraformVersionHelper; +import org.eclipse.xpanse.terraform.boot.terraform.tool.TerraformVersionsHelper; /** * Data model for the terraform modify requests. @@ -28,7 +28,7 @@ public class TerraformModifyFromDirectoryRequest { @NotNull @NotBlank - @Pattern(regexp = TerraformVersionHelper.TERRAFORM_REQUIRED_VERSION_REGEX) + @Pattern(regexp = TerraformVersionsHelper.TERRAFORM_REQUIRED_VERSION_REGEX) @Schema(description = "The required version of the terraform which will execute the scripts.") String terraformVersion; diff --git a/src/main/java/org/eclipse/xpanse/terraform/boot/terraform/TerraformInstaller.java b/src/main/java/org/eclipse/xpanse/terraform/boot/terraform/TerraformInstaller.java deleted file mode 100644 index 5d99382..0000000 --- a/src/main/java/org/eclipse/xpanse/terraform/boot/terraform/TerraformInstaller.java +++ /dev/null @@ -1,202 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * SPDX-FileCopyrightText: Huawei Inc. - */ - -package org.eclipse.xpanse.terraform.boot.terraform; - -import jakarta.annotation.Resource; -import java.io.BufferedOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.net.URI; -import java.net.URL; -import java.nio.channels.Channels; -import java.nio.channels.ReadableByteChannel; -import java.nio.file.Files; -import java.nio.file.StandardCopyOption; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.eclipse.xpanse.terraform.boot.models.exceptions.InvalidTerraformToolException; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.retry.annotation.Backoff; -import org.springframework.retry.annotation.Retryable; -import org.springframework.stereotype.Component; - -/** - * Bean help to install terraform with specific version. - */ -@Slf4j -@Component -public class TerraformInstaller { - - @Value("${terraform.download.base.url:https://releases.hashicorp.com/terraform}") - private String terraformDownloadBaseUrl; - @Value("${terraform.install.dir:/opt/terraform}") - private String terraformInstallDir; - @Resource - private TerraformVersionHelper versionHelper; - - /** - * Find the executable binary path of the Terraform tool that matches the required version. - * If no matching executable binary is found, install the Terraform tool with the required - * version and then return the path. - * - * @param requiredVersion The required version of Terraform tool. - * @return The path of the executable binary. - */ - @Retryable(retryFor = InvalidTerraformToolException.class, - maxAttemptsExpression = "${spring.retry.max-attempts}", - backoff = @Backoff(delayExpression = "${spring.retry.delay-millions}")) - public String getExecutorPathThatMatchesRequiredVersion(String requiredVersion) { - if (StringUtils.isBlank(requiredVersion)) { - log.info("No required version of terraform is specified, use the default terraform."); - return "terraform"; - } - String[] operatorAndNumber = - this.versionHelper.getOperatorAndNumberFromRequiredVersion(requiredVersion); - String requiredOperator = operatorAndNumber[0]; - String requiredNumber = operatorAndNumber[1]; - // Get path of the executor matched required version in the environment. - String matchedVersionExecutorPath = - this.versionHelper.getExecutorPathMatchedRequiredVersion( - this.terraformInstallDir, requiredOperator, requiredNumber); - if (StringUtils.isBlank(matchedVersionExecutorPath)) { - log.info("Not found any terraform executor matched the required version {} from the " - + "terraform installation dir {}, start to download and install one.", - requiredVersion, this.terraformInstallDir); - return installTerraformByRequiredVersion(requiredOperator, requiredNumber); - } - return matchedVersionExecutorPath; - } - - private String installTerraformByRequiredVersion(String requiredOperator, - String requiredNumber) { - String matchedVersionNumber = - this.versionHelper.getBestAvailableVersionFromTerraformWebsite( - this.terraformDownloadBaseUrl, requiredOperator, requiredNumber); - if (StringUtils.isBlank(matchedVersionNumber)) { - String errorMsg = String.format("Not found any terraform executor matched the required " - + "version %s from the terraform download url %s.", - requiredOperator + requiredNumber, this.terraformDownloadBaseUrl); - log.error(errorMsg); - throw new InvalidTerraformToolException(errorMsg); - } - // Install the executor with specific version into the path. - String terraformExecutorName = - this.versionHelper.getTerraformExecutorName(matchedVersionNumber); - File terraformExecutorFile = new File(this.terraformInstallDir, terraformExecutorName); - File parentDir = terraformExecutorFile.getParentFile(); - try { - if (!parentDir.exists()) { - log.info("Created the installation dir {} {}.", parentDir.getAbsolutePath(), - parentDir.mkdirs() ? "successfully" : "failed"); - } - // download the binary zip file into the installation directory - File terraformZipFile = downloadTerraformBinaryZipFile(matchedVersionNumber); - String executorName = terraformExecutorFile.getName(); - // unzip the zip file and move the executable binary to the installation directory - unzipBinaryZipToGetExecutor(terraformZipFile, executorName); - } catch (IOException e) { - log.error(e.getMessage(), e); - throw new InvalidTerraformToolException(e.getMessage()); - } - // delete the non-executable files - deleteNonExecutorFiles(parentDir); - // check the executable binary - - if (this.versionHelper.checkIfExecutorVersionIsValid( - terraformExecutorFile, requiredOperator, requiredNumber)) { - log.info("Installed terraform version {} into the dir {} successfully.", - terraformExecutorFile.getAbsolutePath(), requiredOperator + requiredNumber); - return terraformExecutorFile.getAbsolutePath(); - } - String errorMsg = String.format("Installing terraform version %s into the dir %s failed. ", - requiredOperator + requiredNumber, this.terraformInstallDir); - log.error(errorMsg); - throw new InvalidTerraformToolException(errorMsg); - } - - - private File downloadTerraformBinaryZipFile(String versionNumber) throws IOException { - String binaryZipFileName = this.versionHelper.getTerraformBinaryFileName(versionNumber); - File binaryZipFile = new File(this.terraformInstallDir, binaryZipFileName); - String binaryDownloadUrl = this.versionHelper.getTerraformBinaryDownloadUrl( - this.terraformDownloadBaseUrl, versionNumber, binaryZipFileName); - URL url = URI.create(binaryDownloadUrl).toURL(); - try (ReadableByteChannel rbc = Channels.newChannel(url.openStream()); - FileOutputStream fos = new FileOutputStream(binaryZipFile, false)) { - log.info("Downloading terraform binary file from {} to {}", url, - binaryZipFile.getAbsolutePath()); - fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); - log.info("Downloaded terraform binary file from {} to {} successfully.", - url, binaryZipFile.getAbsolutePath()); - } - return binaryZipFile; - } - - - private void unzipBinaryZipToGetExecutor(File binaryZipFile, String executorName) - throws IOException { - if (!binaryZipFile.exists()) { - String errorMsg = String.format("Terraform binary zip file %s not found.", - binaryZipFile.getAbsolutePath()); - log.error(errorMsg); - throw new IOException(errorMsg); - } - try (ZipInputStream zis = new ZipInputStream(new FileInputStream(binaryZipFile))) { - log.info("Unzipping Terraform zip package {}", binaryZipFile.getAbsolutePath()); - ZipEntry entry; - while ((entry = zis.getNextEntry()) != null) { - if (!entry.isDirectory()) { - String entryName = entry.getName(); - File entryDestinationFile = new File(this.terraformInstallDir, entryName); - if (isExecutorFileInZipForTerraform(executorName)) { - extractFile(zis, entryDestinationFile); - File executorFile = new File(this.terraformInstallDir, executorName); - Files.move(entryDestinationFile.toPath(), executorFile.toPath(), - StandardCopyOption.REPLACE_EXISTING); - log.info("Unzipped Terraform binary file {} to get the executor {} " - + "successfully.", - binaryZipFile.getAbsolutePath(), executorFile.getAbsolutePath()); - } - } - } - } - } - - private boolean isExecutorFileInZipForTerraform(String entryName) { - return entryName.startsWith("terraform"); - } - - - private void extractFile(ZipInputStream zis, File destinationFile) throws IOException { - try (BufferedOutputStream bos = new BufferedOutputStream( - new FileOutputStream(destinationFile))) { - byte[] bytesIn = new byte[4096]; - int read; - while ((read = zis.read(bytesIn)) != -1) { - bos.write(bytesIn, 0, read); - } - } - } - - private void deleteNonExecutorFiles(File dir) { - File[] files = dir.listFiles(); - if (files != null) { - for (File file : files) { - if (file.isDirectory()) { - deleteNonExecutorFiles(file); - } else { - if (!file.getName().startsWith("terraform-") && !file.delete()) { - log.warn("Failed to delete file {}.", file.getAbsolutePath()); - } - } - } - } - } -} diff --git a/src/main/java/org/eclipse/xpanse/terraform/boot/terraform/service/TerraformDirectoryService.java b/src/main/java/org/eclipse/xpanse/terraform/boot/terraform/service/TerraformDirectoryService.java index ef8ee41..314c127 100644 --- a/src/main/java/org/eclipse/xpanse/terraform/boot/terraform/service/TerraformDirectoryService.java +++ b/src/main/java/org/eclipse/xpanse/terraform/boot/terraform/service/TerraformDirectoryService.java @@ -40,8 +40,8 @@ import org.eclipse.xpanse.terraform.boot.models.response.TerraformResult; import org.eclipse.xpanse.terraform.boot.models.validation.TerraformValidationResult; import org.eclipse.xpanse.terraform.boot.terraform.TerraformExecutor; -import org.eclipse.xpanse.terraform.boot.terraform.TerraformInstaller; -import org.eclipse.xpanse.terraform.boot.terraform.TerraformVersionHelper; +import org.eclipse.xpanse.terraform.boot.terraform.tool.TerraformInstaller; +import org.eclipse.xpanse.terraform.boot.terraform.tool.TerraformVersionsHelper; import org.eclipse.xpanse.terraform.boot.terraform.utils.SystemCmdResult; import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -73,7 +73,7 @@ public class TerraformDirectoryService { @Resource private TerraformInstaller installer; @Resource - private TerraformVersionHelper versionHelper; + private TerraformVersionsHelper versionHelper; @Value("${clean.workspace.after.deployment.enabled:true}") private Boolean cleanWorkspaceAfterDeployment; diff --git a/src/main/java/org/eclipse/xpanse/terraform/boot/terraform/tool/TerraformInstaller.java b/src/main/java/org/eclipse/xpanse/terraform/boot/terraform/tool/TerraformInstaller.java new file mode 100644 index 0000000..f7c0f19 --- /dev/null +++ b/src/main/java/org/eclipse/xpanse/terraform/boot/terraform/tool/TerraformInstaller.java @@ -0,0 +1,111 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * SPDX-FileCopyrightText: Huawei Inc. + */ + +package org.eclipse.xpanse.terraform.boot.terraform.tool; + +import jakarta.annotation.Resource; +import java.io.File; +import java.util.Set; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.eclipse.xpanse.terraform.boot.models.exceptions.InvalidTerraformToolException; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.retry.annotation.Backoff; +import org.springframework.retry.annotation.Retryable; +import org.springframework.stereotype.Component; + +/** + * Bean help to install terraform with specific version. + */ +@Slf4j +@Component +public class TerraformInstaller { + + @Value("${terraform.download.base.url:https://releases.hashicorp.com/terraform}") + private String terraformDownloadBaseUrl; + @Value("${terraform.install.dir:/opt/terraform}") + private String terraformInstallDir; + @Resource + private TerraformVersionsCache versionsCache; + @Resource + private TerraformVersionsHelper versionHelper; + + /** + * Find the executable binary path of the Terraform tool that matches the required version. + * If no matching executable binary is found, install the Terraform tool with the required + * version and then return the path. + * + * @param requiredVersion The required version of Terraform tool. + * @return The path of the executable binary. + */ + @Retryable(retryFor = InvalidTerraformToolException.class, + maxAttemptsExpression = "${spring.retry.max-attempts}", + backoff = @Backoff(delayExpression = "${spring.retry.delay-millions}")) + public String getExecutorPathThatMatchesRequiredVersion(String requiredVersion) { + if (StringUtils.isBlank(requiredVersion)) { + log.info("No required version of terraform is specified, use the default terraform."); + return "terraform"; + } + String[] operatorAndNumber = + this.versionHelper.getOperatorAndNumberFromRequiredVersion(requiredVersion); + String requiredOperator = operatorAndNumber[0]; + String requiredNumber = operatorAndNumber[1]; + // Get path of the executor matched required version in the environment. + String matchedVersionExecutorPath = + this.versionHelper.getExecutorPathMatchedRequiredVersion( + this.terraformInstallDir, requiredOperator, requiredNumber); + if (StringUtils.isBlank(matchedVersionExecutorPath)) { + log.info("Not found any terraform executor matched the required version {} from the " + + "terraform installation dir {}, start to download and install one.", + requiredVersion, this.terraformInstallDir); + return installTerraformByRequiredVersion(requiredOperator, requiredNumber); + } + return matchedVersionExecutorPath; + } + + private String installTerraformByRequiredVersion(String requiredOperator, + String requiredNumber) { + String bestVersionNumber = getBestAvailableVersionMatchingRequiredVersion( + requiredOperator, requiredNumber); + File installedExecutorFile = this.versionHelper.installTerraformWithVersion( + bestVersionNumber, this.terraformDownloadBaseUrl, this.terraformInstallDir); + if (this.versionHelper.checkIfExecutorVersionIsValid(installedExecutorFile, + requiredOperator, requiredNumber)) { + log.info("Terraform with version {} installed successfully.", installedExecutorFile); + return installedExecutorFile.getAbsolutePath(); + } + String errorMsg = String.format("Installing terraform with version %s into the dir %s " + + "failed. ", bestVersionNumber, this.terraformInstallDir); + log.error(errorMsg); + throw new InvalidTerraformToolException(errorMsg); + } + + + /** + * Get the best available version in download url. + * + * @param requiredOperator operator in required version + * @param requiredNumber number in required version + * @return the best available version existed in download url. + */ + private String getBestAvailableVersionMatchingRequiredVersion(String requiredOperator, + String requiredNumber) { + Set availableVersions = this.versionsCache.getAvailableVersions(); + String bestAvailableVersion = + this.versionHelper.findBestVersionFromAllAvailableVersions(availableVersions, + requiredOperator, requiredNumber); + if (StringUtils.isNotBlank(bestAvailableVersion)) { + log.info("Found the best available version {} for terraform by the required version " + + "{}.", bestAvailableVersion, requiredOperator + requiredNumber); + return bestAvailableVersion; + } + String errorMsg = String.format("Failed to find available versions for terraform by the " + + "required version %s.", requiredOperator + requiredNumber); + log.error(errorMsg); + throw new InvalidTerraformToolException(errorMsg); + } + + +} diff --git a/src/main/java/org/eclipse/xpanse/terraform/boot/terraform/tool/TerraformVersionsCache.java b/src/main/java/org/eclipse/xpanse/terraform/boot/terraform/tool/TerraformVersionsCache.java new file mode 100644 index 0000000..670cb20 --- /dev/null +++ b/src/main/java/org/eclipse/xpanse/terraform/boot/terraform/tool/TerraformVersionsCache.java @@ -0,0 +1,51 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * SPDX-FileCopyrightText: Huawei Inc. + */ + +package org.eclipse.xpanse.terraform.boot.terraform.tool; + + +import static org.eclipse.xpanse.terraform.boot.cache.CaffeineCacheConfig.TERRAFORM_VERSIONS_CACHE_NAME; + +import jakarta.annotation.Resource; +import java.util.Set; +import lombok.extern.slf4j.Slf4j; +import org.springframework.cache.annotation.Cacheable; +import org.springframework.stereotype.Component; + +/** + * Bean to update the cache of versions of OpenTofu. + */ +@Slf4j +@Component +public class TerraformVersionsCache { + + @Resource + private TerraformVersionsFetcher versionsFetcher; + + /** + * Get the available versions of OpenTofu. + * + * @return Set of available versions. + */ + @Cacheable(value = TERRAFORM_VERSIONS_CACHE_NAME, key = "'all'") + public Set getAvailableVersions() { + try { + return versionsFetcher.fetchAvailableVersionsFromTerraformWebsite(); + } catch (Exception e) { + return versionsFetcher.getDefaultVersionsFromConfig(); + } + } + + /** + * Update the cache of versions of OpenTofu. + * + * @param versions List of available versions. + */ + @Cacheable(value = TERRAFORM_VERSIONS_CACHE_NAME, key = "'all'") + public void updateCachedVersions(Set versions) { + log.info("Updated OpenTofu versions cache with versions:{}.", versions); + } + +} diff --git a/src/main/java/org/eclipse/xpanse/terraform/boot/terraform/tool/TerraformVersionsCacheManager.java b/src/main/java/org/eclipse/xpanse/terraform/boot/terraform/tool/TerraformVersionsCacheManager.java new file mode 100644 index 0000000..2778547 --- /dev/null +++ b/src/main/java/org/eclipse/xpanse/terraform/boot/terraform/tool/TerraformVersionsCacheManager.java @@ -0,0 +1,72 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * SPDX-FileCopyrightText: Huawei Inc. + */ + +package org.eclipse.xpanse.terraform.boot.terraform.tool; + + +import jakarta.annotation.PostConstruct; +import jakarta.annotation.Resource; +import java.util.Objects; +import java.util.Set; +import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; + +/** + * Bean to update the cache of versions of Terraform. + */ +@Slf4j +@Component +public class TerraformVersionsCacheManager { + + @Resource + private TerraformVersionsCache versionsCache; + + @Resource + private TerraformVersionsFetcher versionsFetcher; + + /** + * Initialize the cache of available versions of Terraform. + */ + @PostConstruct + public void initializeCache() { + Set versions = versionsCache.getAvailableVersions(); + log.info("Initialized Terraform versions cache with versions:{}.", versions); + } + + /** + * Update the cache with the versions fetched from the Terraform website. + * This method is scheduled run once a day. + */ + @Scheduled(cron = "0 0 1 * * ?") + public void fetchVersionsFromWebsiteAndLoadCache() { + try { + Set availableVersionsFromWebsite = + versionsFetcher.fetchAvailableVersionsFromTerraformWebsite(); + versionsCache.updateCachedVersions(availableVersionsFromWebsite); + log.info("Updated the cache with versions:{} fetched from Terraform website.", + availableVersionsFromWebsite); + } catch (Exception e) { + log.error("Failed to update the cache with versions fetched from Terraform website.", + e); + } + } + + /** + * Update the cache when the default versions cached. + * This method is scheduled to run every one hour. + * But does nothing if the cache already contains versions other than the default version list. + */ + @Scheduled(cron = "0 1 * * * ?") + public void fetchVersionsFromWebsiteAndLoadCacheIfCacheHasOnlyDefaultVersions() { + Set cachedVersions = versionsCache.getAvailableVersions(); + Set defaultVersions = versionsFetcher.getDefaultVersionsFromConfig(); + if (!CollectionUtils.isEmpty(defaultVersions) + && Objects.equals(cachedVersions, defaultVersions)) { + fetchVersionsFromWebsiteAndLoadCache(); + } + } +} diff --git a/src/main/java/org/eclipse/xpanse/terraform/boot/terraform/tool/TerraformVersionsFetcher.java b/src/main/java/org/eclipse/xpanse/terraform/boot/terraform/tool/TerraformVersionsFetcher.java new file mode 100644 index 0000000..5f0b509 --- /dev/null +++ b/src/main/java/org/eclipse/xpanse/terraform/boot/terraform/tool/TerraformVersionsFetcher.java @@ -0,0 +1,81 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * SPDX-FileCopyrightText: Huawei Inc. + */ + +package org.eclipse.xpanse.terraform.boot.terraform.tool; + +import java.util.HashSet; +import java.util.Set; +import java.util.regex.Pattern; +import lombok.extern.slf4j.Slf4j; +import org.eclipse.xpanse.terraform.boot.models.exceptions.InvalidTerraformToolException; +import org.kohsuke.github.GHRepository; +import org.kohsuke.github.GHTag; +import org.kohsuke.github.GitHub; +import org.kohsuke.github.GitHubBuilder; +import org.kohsuke.github.PagedIterable; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.retry.annotation.Backoff; +import org.springframework.retry.annotation.Retryable; +import org.springframework.stereotype.Component; + +/** + * Class to get available versions of Terraform. + */ +@Slf4j +@Component +public class TerraformVersionsFetcher { + + private static final Pattern OFFICIAL_VERSION_PATTERN = + Pattern.compile("^v(\\d{1,3}\\.\\d{1,3}\\.\\d{1,3})$"); + @Value("${terraform.github.api.endpoint:https://api.github.com}") + private String terraformGithubApiEndpoint; + @Value("${terraform.github.repository:hashicorp/terraform}") + private String terraformGithubRepository; + @Value("${terraform.versions}") + private String defaultVersionsString; + + + /** + * Fetch all available versions from Terraform website. + * + * @return all available versions from Terraform website. + */ + @Retryable(retryFor = Exception.class, + maxAttemptsExpression = "${spring.retry.max-attempts}", + backoff = @Backoff(delayExpression = "${spring.retry.delay-millions}")) + public Set fetchAvailableVersionsFromTerraformWebsite() throws Exception { + Set allVersions = new HashSet<>(); + GitHub gitHub = new GitHubBuilder().withEndpoint(terraformGithubApiEndpoint).build(); + GHRepository repository = gitHub.getRepository(terraformGithubRepository); + PagedIterable tags = repository.listTags(); + tags.forEach(tag -> { + String version = tag.getName(); + if (OFFICIAL_VERSION_PATTERN.matcher(version).matches()) { + // remove the prefix 'v' + allVersions.add(version.substring(1)); + } + }); + log.info("Get available versions: {} from Terraform website.", allVersions); + if (allVersions.isEmpty()) { + String errorMsg = "No available versions found from Terraform website"; + throw new InvalidTerraformToolException(errorMsg); + } + return allVersions; + } + + /** + * Get default versions from config. + * + * @return default versions. + */ + public Set getDefaultVersionsFromConfig() { + Set defaultVersions = + Set.of(defaultVersionsString.replaceAll("//s+", "").split(",")); + log.info("Get default versions: {} from Terraform versions config value: {}", + defaultVersions, defaultVersionsString); + return defaultVersions; + } + +} diff --git a/src/main/java/org/eclipse/xpanse/terraform/boot/terraform/TerraformVersionHelper.java b/src/main/java/org/eclipse/xpanse/terraform/boot/terraform/tool/TerraformVersionsHelper.java similarity index 53% rename from src/main/java/org/eclipse/xpanse/terraform/boot/terraform/TerraformVersionHelper.java rename to src/main/java/org/eclipse/xpanse/terraform/boot/terraform/tool/TerraformVersionsHelper.java index 87a408a..4070a85 100644 --- a/src/main/java/org/eclipse/xpanse/terraform/boot/terraform/TerraformVersionHelper.java +++ b/src/main/java/org/eclipse/xpanse/terraform/boot/terraform/tool/TerraformVersionsHelper.java @@ -3,28 +3,34 @@ * SPDX-FileCopyrightText: Huawei Inc. */ -package org.eclipse.xpanse.terraform.boot.terraform; +package org.eclipse.xpanse.terraform.boot.terraform.tool; import jakarta.annotation.Resource; +import java.io.BufferedOutputStream; import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; import java.io.IOException; -import java.util.ArrayList; +import java.net.URI; +import java.net.URL; +import java.nio.channels.Channels; +import java.nio.channels.ReadableByteChannel; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; import java.util.Arrays; import java.util.Comparator; import java.util.HashMap; -import java.util.List; import java.util.Map; +import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.eclipse.xpanse.terraform.boot.models.exceptions.InvalidTerraformToolException; import org.eclipse.xpanse.terraform.boot.terraform.utils.SystemCmd; import org.eclipse.xpanse.terraform.boot.terraform.utils.SystemCmdResult; -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; import org.semver4j.Semver; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; @@ -34,7 +40,7 @@ */ @Slf4j @Component -public class TerraformVersionHelper { +public class TerraformVersionsHelper { /** * Terraform version required version regex. @@ -45,10 +51,11 @@ public class TerraformVersionHelper { Pattern.compile(TERRAFORM_REQUIRED_VERSION_REGEX); private static final Pattern TERRAFORM_VERSION_OUTPUT_PATTERN = Pattern.compile("^Terraform\\s+v(\\d{1,3}\\.\\d{1,3}\\.\\d{1,3})\\b"); - private static final Pattern TERRAFORM_BINARY_HREF_PATTERN = - Pattern.compile("^/terraform/(\\d{1,3}\\.\\d{1,3}\\.\\d{1,3})/"); + private static final String TERRAFORM_BINARY_DOWNLOAD_URL_FORMAT = + "%s/%s/terraform_%s_%s_%s.zip"; private static final String OS_NAME = System.getProperty("os.name").toLowerCase(); private static final String OS_ARCH = System.getProperty("os.arch").toLowerCase(); + private static final String TERRAFORM_EXECUTOR_PREFIX = "terraform-"; @Resource private SystemCmd systemCmd; @@ -71,7 +78,8 @@ public String getExecutorPathMatchedRequiredVersion(String installationDir, } Map executorVersionFileMap = new HashMap<>(); Arrays.stream(installDir.listFiles()) - .filter(f -> f.isFile() && f.canExecute() && f.getName().startsWith("terraform-")) + .filter(f -> f.isFile() && f.canExecute() + && f.getName().startsWith(TERRAFORM_EXECUTOR_PREFIX)) .forEach(f -> { String versionNumber = getVersionFromExecutorPath(f.getAbsolutePath()); executorVersionFileMap.put(versionNumber, f); @@ -79,8 +87,8 @@ public String getExecutorPathMatchedRequiredVersion(String installationDir, if (CollectionUtils.isEmpty(executorVersionFileMap)) { return null; } - String findBestVersion = findBestVersion(executorVersionFileMap.keySet().stream().toList(), - requiredOperator, requiredNumber); + String findBestVersion = findBestVersionFromAllAvailableVersions( + executorVersionFileMap.keySet(), requiredOperator, requiredNumber); if (StringUtils.isNotBlank(findBestVersion)) { File executorFile = executorVersionFileMap.get(findBestVersion); if (checkIfExecutorVersionIsValid(executorFile, requiredOperator, requiredNumber)) { @@ -116,43 +124,16 @@ public String[] getOperatorAndNumberFromRequiredVersion(String requiredVersion) /** - * Get the best available version in download url. + * Find the best version from all available versions. * - * @param downloadBaseUrl the base url of download - * @param requiredOperator operator in required version - * @param requiredNumber number in required version - * @return the best available version existed in download url. + * @param allAvailableVersions all available versions + * @param requiredOperator operator in required version + * @param requiredNumber number in required version + * @return the best version */ - public String getBestAvailableVersionFromTerraformWebsite(String downloadBaseUrl, - String requiredOperator, - String requiredNumber) { - List allVersionsCanBeDownloaded = new ArrayList<>(); - try { - Document doc = Jsoup.connect(downloadBaseUrl).get(); - Elements elements = doc.select("a[href]"); - for (Element element : elements) { - String href = element.attr("href"); - Matcher matcher = TERRAFORM_BINARY_HREF_PATTERN.matcher(href); - if (matcher.find()) { - allVersionsCanBeDownloaded.add(matcher.group(1)); - } - } - } catch (IOException e) { - String errorMsg = String.format("Query all versions of Terraform from the download url" - + " %s error.", downloadBaseUrl); - log.error(errorMsg, e); - } - if (CollectionUtils.isEmpty(allVersionsCanBeDownloaded)) { - String errorMsg = String.format("Not found any versions of Terraform from the " - + "download url %s.", downloadBaseUrl); - log.error(errorMsg); - throw new InvalidTerraformToolException(errorMsg); - } - return findBestVersion(allVersionsCanBeDownloaded, requiredOperator, requiredNumber); - } - - private String findBestVersion(List allAvailableVersions, - String requiredOperator, String requiredNumber) { + public String findBestVersionFromAllAvailableVersions(Set allAvailableVersions, + String requiredOperator, + String requiredNumber) { if (CollectionUtils.isEmpty(allAvailableVersions) || StringUtils.isBlank(requiredOperator) || StringUtils.isBlank(requiredNumber)) { return null; @@ -180,7 +161,8 @@ private String findBestVersion(List allAvailableVersions, * @param requiredNumber number in required version * @return true if the version is valid, otherwise return false. */ - public boolean checkIfExecutorVersionIsValid(File executorFile, String requiredOperator, + public boolean checkIfExecutorVersionIsValid(File executorFile, + String requiredOperator, String requiredNumber) { if (!executorFile.exists() && !executorFile.isFile()) { return false; @@ -194,15 +176,11 @@ public boolean checkIfExecutorVersionIsValid(File executorFile, String requiredO return false; } } - SystemCmdResult versionCheckResult = systemCmd.execute( - executorFile.getAbsolutePath() + " -v", - 5, System.getProperty("java.io.tmpdir"), false, new HashMap<>()); - if (versionCheckResult.isCommandSuccessful()) { - String actualVersion = - getActualVersionFromCommandOutput(versionCheckResult.getCommandStdOutput()); - return isVersionSatisfied(actualVersion, requiredOperator, requiredNumber); + String actualVersion = getExactVersionOfExecutor(executorFile.getAbsolutePath()); + if (StringUtils.isBlank(actualVersion)) { + return false; } - return false; + return isVersionSatisfied(actualVersion, requiredOperator, requiredNumber); } @@ -216,22 +194,125 @@ public String getExactVersionOfExecutor(String executorPath) { if (StringUtils.isBlank(executorPath)) { return null; } - String exactVersion; + SystemCmdResult versionCheckResult = systemCmd.execute(executorPath + " -v", + 5, System.getProperty("java.io.tmpdir"), false, new HashMap<>()); + if (versionCheckResult.isCommandSuccessful()) { + return getActualVersionFromCommandOutput(versionCheckResult.getCommandStdOutput()); + } else { + log.error(versionCheckResult.getCommandStdError()); + return null; + } + } + + + /** + * Install terraform with specific version. + * + * @param versionNumber the version number + * @param downloadBaseUrl download base url + * @param installDir installation directory + * @return the path of the installed executor. + */ + public File installTerraformWithVersion(String versionNumber, String downloadBaseUrl, + String installDir) { + // Install the executor with specific version into the path. + String terraformExecutorName = getTerraformExecutorName(versionNumber); + File terraformExecutorFile = new File(installDir, terraformExecutorName); + File parentDir = terraformExecutorFile.getParentFile(); try { - SystemCmdResult versionCheckResult = systemCmd.execute(executorPath + " -v", - 5, System.getProperty("java.io.tmpdir"), false, new HashMap<>()); - if (versionCheckResult.isCommandSuccessful()) { - exactVersion = - getActualVersionFromCommandOutput(versionCheckResult.getCommandStdOutput()); - } else { - log.error(versionCheckResult.getCommandStdError()); - exactVersion = getVersionFromExecutorPath(executorPath); + if (!parentDir.exists()) { + log.info("Created the installation dir {} {}.", parentDir.getAbsolutePath(), + parentDir.mkdirs() ? "successfully" : "failed"); + } + // download the binary zip file into the installation directory + File terraformZipFile = downloadTerraformBinaryZipFile( + versionNumber, downloadBaseUrl, installDir); + // unzip the zip file and move the executable binary to the installation directory + unzipBinaryZipToGetExecutor(terraformZipFile, terraformExecutorFile); + } catch (IOException e) { + log.error(e.getMessage(), e); + throw new InvalidTerraformToolException(e.getMessage()); + } + // delete the non-executable files + deleteNonExecutorFiles(parentDir); + return terraformExecutorFile; + } + + private File downloadTerraformBinaryZipFile(String versionNumber, String downloadBaseUrl, + String installDir) throws IOException { + String binaryDownloadUrl = getTerraformBinaryDownloadUrl(downloadBaseUrl, versionNumber); + String binaryZipFileName = getTerraformBinaryZipFileName(binaryDownloadUrl); + File binaryZipFile = new File(installDir, binaryZipFileName); + URL url = URI.create(binaryDownloadUrl).toURL(); + try (ReadableByteChannel rbc = Channels.newChannel(url.openStream()); + FileOutputStream fos = new FileOutputStream(binaryZipFile, false)) { + log.info("Downloading terraform binary file from {} to {}", url, + binaryZipFile.getAbsolutePath()); + fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); + log.info("Downloaded terraform binary file from {} to {} successfully.", + url, binaryZipFile.getAbsolutePath()); + } + return binaryZipFile; + } + + + private void unzipBinaryZipToGetExecutor(File binaryZipFile, File executorFile) + throws IOException { + if (!binaryZipFile.exists()) { + String errorMsg = String.format("Terraform binary zip file %s not found.", + binaryZipFile.getAbsolutePath()); + log.error(errorMsg); + throw new IOException(errorMsg); + } + try (ZipInputStream zis = new ZipInputStream(new FileInputStream(binaryZipFile))) { + log.info("Unzipping terraform binary zip file {}", binaryZipFile.getAbsolutePath()); + ZipEntry entry; + while ((entry = zis.getNextEntry()) != null) { + if (!entry.isDirectory()) { + String entryName = entry.getName(); + File entryDestinationFile = new File(executorFile.getParentFile(), entryName); + if (isExecutorFileInZipForTerraform(entryName)) { + extractFile(zis, entryDestinationFile); + Files.move(entryDestinationFile.toPath(), executorFile.toPath(), + StandardCopyOption.REPLACE_EXISTING); + log.info("Unzipped terraform file {} and extract the executor {} " + + "successfully.", binaryZipFile.getAbsolutePath(), + executorFile.getAbsolutePath()); + } + } + } + } + } + + private boolean isExecutorFileInZipForTerraform(String entryName) { + return entryName.startsWith("terraform"); + } + + + private void extractFile(ZipInputStream zis, File destinationFile) throws IOException { + try (BufferedOutputStream bos = new BufferedOutputStream( + new FileOutputStream(destinationFile))) { + byte[] bytesIn = new byte[4096]; + int read; + while ((read = zis.read(bytesIn)) != -1) { + bos.write(bytesIn, 0, read); + } + } + } + + private void deleteNonExecutorFiles(File dir) { + File[] files = dir.listFiles(); + if (files != null) { + for (File file : files) { + if (file.isDirectory()) { + deleteNonExecutorFiles(file); + } else { + if (!file.getName().startsWith(TERRAFORM_EXECUTOR_PREFIX) && !file.delete()) { + log.warn("Failed to delete file {}.", file.getAbsolutePath()); + } + } } - } catch (Exception e) { - log.error("Get exact version of the executor {} failed.", executorPath, e); - exactVersion = executorPath; } - return exactVersion; } @@ -247,9 +328,7 @@ private String getActualVersionFromCommandOutput(String commandOutput) { if (matcher.find()) { return matcher.group(1); } - String errorMsg = String.format("Cannot find the version from command output：%s", - commandOutput); - throw new InvalidTerraformToolException(errorMsg); + return null; } private boolean isVersionSatisfied(String actualNumber, String requiredOperator, @@ -274,35 +353,24 @@ private boolean isVersionSatisfied(String actualNumber, String requiredOperator, * @return binary file name */ public String getTerraformExecutorName(String versionNumber) { - return String.format("terraform-%s", versionNumber); - } - - /** - * Get terraform binary file name. - * - * @param versionNumber version number - * @return binary file name - */ - public String getTerraformBinaryFileName(String versionNumber) { - return String.format("terraform_%s_%s_%s.zip", - versionNumber, getOperatingSystemCode(), OS_ARCH); + return TERRAFORM_EXECUTOR_PREFIX + versionNumber; } - /** - * Get terraform binary file download url. + * Get whole download url of the executor binary file. * * @param downloadBaseUrl download base url * @param versionNumber version number - * @param binaryFileName binary file name - * @return binary file name + * @return whole download url of the executor binary file */ - public String getTerraformBinaryDownloadUrl(String downloadBaseUrl, - String versionNumber, String binaryFileName) { - return String.format("%s/%s/%s", downloadBaseUrl, versionNumber, binaryFileName); - + private String getTerraformBinaryDownloadUrl(String downloadBaseUrl, String versionNumber) { + return String.format(TERRAFORM_BINARY_DOWNLOAD_URL_FORMAT, downloadBaseUrl, versionNumber, + versionNumber, getOperatingSystemCode(), OS_ARCH); } + private String getTerraformBinaryZipFileName(String executorBinaryDownloadUrl) { + return executorBinaryDownloadUrl.substring(executorBinaryDownloadUrl.lastIndexOf("/") + 1); + } private String getOperatingSystemCode() { if (OS_NAME.contains("windows")) { diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index dca1bcb..94a2b1c 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -17,4 +17,7 @@ clean.workspace.after.deployment.enabled=true spring.retry.max-attempts=3 spring.retry.delay-millions=1000 terraform.install.dir=/opt/terraform -terraform.download.base.url=https://releases.hashicorp.com/terraform \ No newline at end of file +terraform.download.base.url=https://releases.hashicorp.com/terraform +terraform.versions=1.6.0,1.7.0,1.8.0,1.9.0 +terraform.github.api.endpoint=https://api.github.com +terraform.github.repository=hashicorp/terraform \ No newline at end of file diff --git a/src/test/java/org/eclipse/xpanse/terraform/boot/terraform/TerraformInstallerTest.java b/src/test/java/org/eclipse/xpanse/terraform/boot/terraform/tool/TerraformInstallerTest.java similarity index 95% rename from src/test/java/org/eclipse/xpanse/terraform/boot/terraform/TerraformInstallerTest.java rename to src/test/java/org/eclipse/xpanse/terraform/boot/terraform/tool/TerraformInstallerTest.java index e1b7bf7..4398c9a 100644 --- a/src/test/java/org/eclipse/xpanse/terraform/boot/terraform/TerraformInstallerTest.java +++ b/src/test/java/org/eclipse/xpanse/terraform/boot/terraform/tool/TerraformInstallerTest.java @@ -1,4 +1,4 @@ -package org.eclipse.xpanse.terraform.boot.terraform; +package org.eclipse.xpanse.terraform.boot.terraform.tool; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -16,7 +16,7 @@ class TerraformInstallerTest { @Resource private TerraformInstaller installer; @Resource - private TerraformVersionHelper versionHelper; + private TerraformVersionsHelper versionHelper; @Test void testGetExecutableTerraformByVersion() { @@ -49,7 +49,5 @@ void testGetExecutableTerraformByVersion() { String requiredVersion4 = ">= 100.0.0"; assertThrows(InvalidTerraformToolException.class, () -> installer.getExecutorPathThatMatchesRequiredVersion(requiredVersion4)); - - } } \ No newline at end of file