From 2e30b497f76bdca1904ec2310ed293f0d25ec3f7 Mon Sep 17 00:00:00 2001 From: baixinsui Date: Mon, 30 Sep 2024 15:20:49 +0800 Subject: [PATCH] support multiple versions of terraform in docker --- .github/workflows/release.yml | 7 + Dockerfile | 15 +- install_terraform.sh | 42 +++ pom.xml | 15 +- .../boot/TerraformBootApplication.java | 2 + .../boot/cache/CaffeineCacheConfig.java | 52 +++ .../boot/terraform/TerraformInstaller.java | 162 +++------ .../terraform/TerraformVersionHelper.java | 309 +++++++++++++----- src/main/resources/application.properties | 4 +- .../terraform/TerraformInstallerTest.java | 2 - 10 files changed, 387 insertions(+), 223 deletions(-) create mode 100644 install_terraform.sh create mode 100644 src/main/java/org/eclipse/xpanse/terraform/boot/cache/CaffeineCacheConfig.java diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1d46f2b..aed5610 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -21,6 +21,9 @@ env: BOT_EMAIL_ID: xpanse-bot@eclipse.org REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} + TERRAFORM_INSTALL_PATH: /opt/terraform + DEFAULT_TERRAFORM_VERSION: 1.6.0 + TERRAFORM_VERSIONS: 1.6.0,1.6.1,1.7.0,1.8.0,1.9.0 jobs: release: @@ -103,6 +106,10 @@ jobs: tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest,${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.new_version.outputs.next-version }} labels: ${{ steps.meta.outputs.labels }} provenance: false + build-args: | + TERRAFORM_INSTALL_PATH=${{ env.TERRAFORM_INSTALL_PATH }} + DEFAULT_TERRAFORM_VERSION=${{ env.DEFAULT_TERRAFORM_VERSION }} + TERRAFORM_VERSIONS=${{ env.TERRAFORM_VERSIONS }} - name: Push POM updates with release version uses: EndBug/add-and-commit@v9 diff --git a/Dockerfile b/Dockerfile index b9197b2..de64db0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,10 +3,15 @@ FROM eclipse-temurin:21-jre-alpine RUN addgroup -S terraform-boot && adduser -S -G terraform-boot terraform-boot RUN apk update && \ apk add --no-cache unzip wget -ENV TERRAFORM_VERSION=1.4.4 -RUN wget https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip -RUN unzip terraform_${TERRAFORM_VERSION}_linux_amd64.zip -RUN mv terraform /usr/bin/terraform + +ENV TERRAFORM_INSTALL_PATH=/opt/terraform +ENV DEFAULT_TERRAFORM_VERSION=1.6.0 +ENV TERRAFORM_VERSIONS=1.6.0,1.7.0,1.8.0,1.9.0 +COPY install_terraform.sh /install_terraform.sh +RUN chmod +x /install_terraform.sh +RUN echo "Downloading and installing Terraform with multiple versions $TERRAFORM_VERSIONS into path $TERRAFORM_INSTALL_PATH"; \ + /install_terraform.sh "$TERRAFORM_INSTALL_PATH" "$DEFAULT_TERRAFORM_VERSION" "$TERRAFORM_VERSIONS" + COPY target/terraform-boot-*.jar terraform-boot.jar USER terraform-boot -ENTRYPOINT ["java","-jar","terraform-boot.jar"] \ No newline at end of file +ENTRYPOINT ["java","-Dterraform.install.dir=${TERRAFORM_INSTALL_PATH}", "-jar","terraform-boot.jar"] diff --git a/install_terraform.sh b/install_terraform.sh new file mode 100644 index 0000000..82d3030 --- /dev/null +++ b/install_terraform.sh @@ -0,0 +1,42 @@ +#!/bin/sh +if [ -z "$1" ] || [ -z "$2" ]; then + echo "Error：TERRAFORM_INSTALL_PATH and DEFAULT_OPENTOFU_VERSION could not be empty." + exit 1 +fi +TERRAFORM_INSTALL_PATH="$1" +DEFAULT_TERRAFORM_VERSION="$2" +TERRAFORM_VERSIONS="$3" +mkdir -p "${TERRAFORM_INSTALL_PATH}" +# Install default version of Terraform in system path and custom path +echo "Start installing Terraform with default version ${DEFAULT_TERRAFORM_VERSION}" +wget -c "https://releases.hashicorp.com/terraform/${DEFAULT_TERRAFORM_VERSION}/terraform_${DEFAULT_TERRAFORM_VERSION}_linux_amd64.zip" +if [ -f "terraform_${DEFAULT_TERRAFORM_VERSION}_linux_amd64.zip" ]; then + unzip -o "terraform_${DEFAULT_TERRAFORM_VERSION}_linux_amd64.zip" + cp -f terraform /usr/bin/terraform + chmod +x /usr/bin/terraform + mv -f terraform "${TERRAFORM_INSTALL_PATH}/terraform-${DEFAULT_TERRAFORM_VERSION}" + chmod +x "${TERRAFORM_INSTALL_PATH}/terraform-${DEFAULT_TERRAFORM_VERSION}" + rm -f "terraform_${DEFAULT_TERRAFORM_VERSION}_linux_amd64.zip" + echo "Installed Terraform with default version ${DEFAULT_TERRAFORM_VERSION} into path ${TERRAFORM_INSTALL_PATH} successfully." +else + echo "Failed to download zip package of Terraform with default version terraform_${DEFAULT_TERRAFORM_VERSION}_linux_amd64.zip." +fi +if [ -z "$TERRAFORM_VERSIONS" ]; then + echo "No Terraform versions specified, skip installing Terraform versions." + exit 0 +fi +# Install versions of Terraform specified in TERRAFORM_VERSIONS into custom path +VERSIONS=$(echo "$TERRAFORM_VERSIONS" | tr ',' '\n' | tr -d ' ') +for version in $VERSIONS; do + echo "Start installing Terraform with version ${version} into path ${TERRAFORM_INSTALL_PATH}" + wget -c "https://releases.hashicorp.com/terraform/${version}/terraform_${version}_linux_amd64.zip" + if [ ! -f "terraform_${version}_linux_amd64.zip" ]; then + echo "Failed to download zip package of Terraform with version terraform_${version}_linux_amd64.zip." + continue + fi + unzip -o "terraform_${version}_linux_amd64.zip" + mv -f terraform "${TERRAFORM_INSTALL_PATH}/terraform-${version}" + chmod +x "${TERRAFORM_INSTALL_PATH}/terraform-${version}" + rm -f "terraform_${version}_linux_amd64.zip" + echo "Installed Terraform with version ${version} into path ${TERRAFORM_INSTALL_PATH} successfully." +done diff --git a/pom.xml b/pom.xml index b128e87..5d2d349 100644 --- a/pom.xml +++ b/pom.xml @@ -30,7 +30,6 @@ 1.1.0 7.0.0.202409031743-r 5.4.0 - 1.18.1 3.13.0 3.5.0 3.5.0 @@ -54,7 +53,14 @@ org.springframework.boot spring-boot-starter-web - + + org.springframework.boot + spring-boot-starter-cache + + + com.github.ben-manes.caffeine + caffeine + org.projectlombok lombok @@ -118,11 +124,6 @@ semver4j ${semver4j.version} - - org.jsoup - jsoup - ${jsoup.version} - diff --git a/src/main/java/org/eclipse/xpanse/terraform/boot/TerraformBootApplication.java b/src/main/java/org/eclipse/xpanse/terraform/boot/TerraformBootApplication.java index 65d6349..5972166 100644 --- a/src/main/java/org/eclipse/xpanse/terraform/boot/TerraformBootApplication.java +++ b/src/main/java/org/eclipse/xpanse/terraform/boot/TerraformBootApplication.java @@ -8,6 +8,7 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration; +import org.springframework.cache.annotation.EnableCaching; import org.springframework.retry.annotation.EnableRetry; import org.springframework.scheduling.annotation.EnableAsync; @@ -16,6 +17,7 @@ */ @EnableRetry @EnableAsync +@EnableCaching @SpringBootApplication(exclude = {SecurityAutoConfiguration.class}) public class TerraformBootApplication { diff --git a/src/main/java/org/eclipse/xpanse/terraform/boot/cache/CaffeineCacheConfig.java b/src/main/java/org/eclipse/xpanse/terraform/boot/cache/CaffeineCacheConfig.java new file mode 100644 index 0000000..894d435 --- /dev/null +++ b/src/main/java/org/eclipse/xpanse/terraform/boot/cache/CaffeineCacheConfig.java @@ -0,0 +1,52 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * SPDX-FileCopyrightText: Huawei Inc. + * + */ + +package org.eclipse.xpanse.terraform.boot.cache; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import java.util.concurrent.TimeUnit; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.cache.CacheManager; +import org.springframework.cache.caffeine.CaffeineCacheManager; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * Caffeine cache configuration class. + */ +@Slf4j +@Configuration +public class CaffeineCacheConfig { + public static final String TERRAFORM_VERSIONS_CACHE_NAME = "TERRAFORM_VERSIONS_CACHE"; + + public static final int DEFAULT_CACHE_EXPIRE_TIME_IN_MINUTES = 60; + + @Value("${terraform.versions.cache.expire.time.in.minutes:60}") + private long terraformVersionCacheDuration; + + /** + * Config cache manager with caffeine. + * + * @return caffeineCacheManager + */ + @Bean + public CacheManager caffeineCacheManager() { + CaffeineCacheManager cacheManager = new CaffeineCacheManager(); + cacheManager.registerCustomCache(TERRAFORM_VERSIONS_CACHE_NAME, + getTerraformVersionsCache()); + return cacheManager; + } + + + private Cache getTerraformVersionsCache() { + long duration = terraformVersionCacheDuration > 0 ? terraformVersionCacheDuration + : DEFAULT_CACHE_EXPIRE_TIME_IN_MINUTES; + return Caffeine.newBuilder().expireAfterWrite(duration, TimeUnit.MINUTES).build(); + } + +} diff --git a/src/main/java/org/eclipse/xpanse/terraform/boot/terraform/TerraformInstaller.java b/src/main/java/org/eclipse/xpanse/terraform/boot/terraform/TerraformInstaller.java index 5d99382..805c4cd 100644 --- a/src/main/java/org/eclipse/xpanse/terraform/boot/terraform/TerraformInstaller.java +++ b/src/main/java/org/eclipse/xpanse/terraform/boot/terraform/TerraformInstaller.java @@ -6,19 +6,9 @@ package org.eclipse.xpanse.terraform.boot.terraform; import jakarta.annotation.Resource; -import java.io.BufferedOutputStream; import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; import java.io.IOException; -import java.net.URI; -import java.net.URL; -import java.nio.channels.Channels; -import java.nio.channels.ReadableByteChannel; -import java.nio.file.Files; -import java.nio.file.StandardCopyOption; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; +import java.util.List; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.eclipse.xpanse.terraform.boot.models.exceptions.InvalidTerraformToolException; @@ -76,127 +66,53 @@ public String getExecutorPathThatMatchesRequiredVersion(String requiredVersion) private String installTerraformByRequiredVersion(String requiredOperator, String requiredNumber) { - String matchedVersionNumber = - this.versionHelper.getBestAvailableVersionFromTerraformWebsite( - this.terraformDownloadBaseUrl, requiredOperator, requiredNumber); - if (StringUtils.isBlank(matchedVersionNumber)) { - String errorMsg = String.format("Not found any terraform executor matched the required " - + "version %s from the terraform download url %s.", - requiredOperator + requiredNumber, this.terraformDownloadBaseUrl); - log.error(errorMsg); - throw new InvalidTerraformToolException(errorMsg); - } - // Install the executor with specific version into the path. - String terraformExecutorName = - this.versionHelper.getTerraformExecutorName(matchedVersionNumber); - File terraformExecutorFile = new File(this.terraformInstallDir, terraformExecutorName); - File parentDir = terraformExecutorFile.getParentFile(); - try { - if (!parentDir.exists()) { - log.info("Created the installation dir {} {}.", parentDir.getAbsolutePath(), - parentDir.mkdirs() ? "successfully" : "failed"); - } - // download the binary zip file into the installation directory - File terraformZipFile = downloadTerraformBinaryZipFile(matchedVersionNumber); - String executorName = terraformExecutorFile.getName(); - // unzip the zip file and move the executable binary to the installation directory - unzipBinaryZipToGetExecutor(terraformZipFile, executorName); - } catch (IOException e) { - log.error(e.getMessage(), e); - throw new InvalidTerraformToolException(e.getMessage()); + String bestVersionNumber = getBestAvailableVersionMatchingRequiredVersion( + requiredOperator, requiredNumber); + File installedExecutorFile = this.versionHelper.installTerraformWithVersion( + bestVersionNumber, this.terraformDownloadBaseUrl, this.terraformInstallDir); + if (this.versionHelper.checkIfExecutorVersionIsValid(installedExecutorFile, + requiredOperator, requiredNumber)) { + log.info("Terraform with version {} installed successfully.", installedExecutorFile); + return installedExecutorFile.getAbsolutePath(); } - // delete the non-executable files - deleteNonExecutorFiles(parentDir); - // check the executable binary - - if (this.versionHelper.checkIfExecutorVersionIsValid( - terraformExecutorFile, requiredOperator, requiredNumber)) { - log.info("Installed terraform version {} into the dir {} successfully.", - terraformExecutorFile.getAbsolutePath(), requiredOperator + requiredNumber); - return terraformExecutorFile.getAbsolutePath(); - } - String errorMsg = String.format("Installing terraform version %s into the dir %s failed. ", - requiredOperator + requiredNumber, this.terraformInstallDir); + String errorMsg = String.format("Installing terraform with version %s into the dir %s " + + "failed. ", bestVersionNumber, this.terraformInstallDir); log.error(errorMsg); throw new InvalidTerraformToolException(errorMsg); } - private File downloadTerraformBinaryZipFile(String versionNumber) throws IOException { - String binaryZipFileName = this.versionHelper.getTerraformBinaryFileName(versionNumber); - File binaryZipFile = new File(this.terraformInstallDir, binaryZipFileName); - String binaryDownloadUrl = this.versionHelper.getTerraformBinaryDownloadUrl( - this.terraformDownloadBaseUrl, versionNumber, binaryZipFileName); - URL url = URI.create(binaryDownloadUrl).toURL(); - try (ReadableByteChannel rbc = Channels.newChannel(url.openStream()); - FileOutputStream fos = new FileOutputStream(binaryZipFile, false)) { - log.info("Downloading terraform binary file from {} to {}", url, - binaryZipFile.getAbsolutePath()); - fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); - log.info("Downloaded terraform binary file from {} to {} successfully.", - url, binaryZipFile.getAbsolutePath()); - } - return binaryZipFile; - } - - - private void unzipBinaryZipToGetExecutor(File binaryZipFile, String executorName) - throws IOException { - if (!binaryZipFile.exists()) { - String errorMsg = String.format("Terraform binary zip file %s not found.", - binaryZipFile.getAbsolutePath()); - log.error(errorMsg); - throw new IOException(errorMsg); + /** + * Get the best available version in download url. + * + * @param requiredOperator operator in required version + * @param requiredNumber number in required version + * @return the best available version existed in download url. + */ + private String getBestAvailableVersionMatchingRequiredVersion(String requiredOperator, + String requiredNumber) { + List availableVersions; + try { + availableVersions = this.versionHelper.fetchAvailableVersionsFromTerraformWebsite(); + } catch (IOException e) { + String errorMsg = "Failed to fetch available versions from terraform website."; + log.error(errorMsg, e); + throw new InvalidTerraformToolException(errorMsg); } - try (ZipInputStream zis = new ZipInputStream(new FileInputStream(binaryZipFile))) { - log.info("Unzipping Terraform zip package {}", binaryZipFile.getAbsolutePath()); - ZipEntry entry; - while ((entry = zis.getNextEntry()) != null) { - if (!entry.isDirectory()) { - String entryName = entry.getName(); - File entryDestinationFile = new File(this.terraformInstallDir, entryName); - if (isExecutorFileInZipForTerraform(executorName)) { - extractFile(zis, entryDestinationFile); - File executorFile = new File(this.terraformInstallDir, executorName); - Files.move(entryDestinationFile.toPath(), executorFile.toPath(), - StandardCopyOption.REPLACE_EXISTING); - log.info("Unzipped Terraform binary file {} to get the executor {} " - + "successfully.", - binaryZipFile.getAbsolutePath(), executorFile.getAbsolutePath()); - } - } - } + log.info("Fetched available versions from terraform website: {}", availableVersions); + String bestAvailableVersion = + this.versionHelper.findBestVersionFromAllAvailableVersions(availableVersions, + requiredOperator, requiredNumber); + if (StringUtils.isNotBlank(bestAvailableVersion)) { + log.info("Found the best available version {} for terraform by the required version " + + "{}.", bestAvailableVersion, requiredOperator + requiredNumber); + return bestAvailableVersion; } + String errorMsg = String.format("Failed to find available versions for terraform by the " + + "required version %s.", requiredOperator + requiredNumber); + log.error(errorMsg); + throw new InvalidTerraformToolException(errorMsg); } - private boolean isExecutorFileInZipForTerraform(String entryName) { - return entryName.startsWith("terraform"); - } - - - private void extractFile(ZipInputStream zis, File destinationFile) throws IOException { - try (BufferedOutputStream bos = new BufferedOutputStream( - new FileOutputStream(destinationFile))) { - byte[] bytesIn = new byte[4096]; - int read; - while ((read = zis.read(bytesIn)) != -1) { - bos.write(bytesIn, 0, read); - } - } - } - private void deleteNonExecutorFiles(File dir) { - File[] files = dir.listFiles(); - if (files != null) { - for (File file : files) { - if (file.isDirectory()) { - deleteNonExecutorFiles(file); - } else { - if (!file.getName().startsWith("terraform-") && !file.delete()) { - log.warn("Failed to delete file {}.", file.getAbsolutePath()); - } - } - } - } - } } diff --git a/src/main/java/org/eclipse/xpanse/terraform/boot/terraform/TerraformVersionHelper.java b/src/main/java/org/eclipse/xpanse/terraform/boot/terraform/TerraformVersionHelper.java index 87a408a..c10780a 100644 --- a/src/main/java/org/eclipse/xpanse/terraform/boot/terraform/TerraformVersionHelper.java +++ b/src/main/java/org/eclipse/xpanse/terraform/boot/terraform/TerraformVersionHelper.java @@ -5,9 +5,23 @@ package org.eclipse.xpanse.terraform.boot.terraform; +import static org.eclipse.xpanse.terraform.boot.cache.CaffeineCacheConfig.TERRAFORM_VERSIONS_CACHE_NAME; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import jakarta.annotation.Resource; +import java.io.BufferedOutputStream; import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; import java.io.IOException; +import java.net.URI; +import java.net.URL; +import java.nio.channels.Channels; +import java.nio.channels.ReadableByteChannel; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; @@ -16,18 +30,22 @@ import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.eclipse.xpanse.terraform.boot.models.exceptions.InvalidTerraformToolException; import org.eclipse.xpanse.terraform.boot.terraform.utils.SystemCmd; import org.eclipse.xpanse.terraform.boot.terraform.utils.SystemCmdResult; -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; import org.semver4j.Semver; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.cache.annotation.Cacheable; +import org.springframework.http.ResponseEntity; +import org.springframework.retry.annotation.Backoff; +import org.springframework.retry.annotation.Retryable; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; +import org.springframework.web.client.RestTemplate; /** * Defines methods for handling terraform with required version. @@ -45,11 +63,19 @@ public class TerraformVersionHelper { Pattern.compile(TERRAFORM_REQUIRED_VERSION_REGEX); private static final Pattern TERRAFORM_VERSION_OUTPUT_PATTERN = Pattern.compile("^Terraform\\s+v(\\d{1,3}\\.\\d{1,3}\\.\\d{1,3})\\b"); - private static final Pattern TERRAFORM_BINARY_HREF_PATTERN = - Pattern.compile("^/terraform/(\\d{1,3}\\.\\d{1,3}\\.\\d{1,3})/"); + private static final String TERRAFORM_BINARY_DOWNLOAD_URL_FORMAT = + "%s/%s/terraform_%s_%s_%s.zip"; + private static final Pattern OFFICIAL_VERSION_PATTERN = + Pattern.compile("^v(\\d{1,3}\\.\\d{1,3}\\.\\d{1,3})$"); private static final String OS_NAME = System.getProperty("os.name").toLowerCase(); private static final String OS_ARCH = System.getProperty("os.arch").toLowerCase(); + private static final String TERRAFORM_EXECUTOR_PREFIX = "terraform-"; + private final ObjectMapper objectMapper = new ObjectMapper(); + @Value("${terraform.versions.query.url:https://api.github.com/repos/hashicorp/terraform/tags}") + private String terraformVersionsQueryUrl; + @Resource + private RestTemplate restTemplate; @Resource private SystemCmd systemCmd; @@ -71,7 +97,8 @@ public String getExecutorPathMatchedRequiredVersion(String installationDir, } Map executorVersionFileMap = new HashMap<>(); Arrays.stream(installDir.listFiles()) - .filter(f -> f.isFile() && f.canExecute() && f.getName().startsWith("terraform-")) + .filter(f -> f.isFile() && f.canExecute() + && f.getName().startsWith(TERRAFORM_EXECUTOR_PREFIX)) .forEach(f -> { String versionNumber = getVersionFromExecutorPath(f.getAbsolutePath()); executorVersionFileMap.put(versionNumber, f); @@ -79,8 +106,9 @@ public String getExecutorPathMatchedRequiredVersion(String installationDir, if (CollectionUtils.isEmpty(executorVersionFileMap)) { return null; } - String findBestVersion = findBestVersion(executorVersionFileMap.keySet().stream().toList(), - requiredOperator, requiredNumber); + String findBestVersion = findBestVersionFromAllAvailableVersions( + executorVersionFileMap.keySet().stream().toList(), requiredOperator, + requiredNumber); if (StringUtils.isNotBlank(findBestVersion)) { File executorFile = executorVersionFileMap.get(findBestVersion); if (checkIfExecutorVersionIsValid(executorFile, requiredOperator, requiredNumber)) { @@ -114,45 +142,69 @@ public String[] getOperatorAndNumberFromRequiredVersion(String requiredVersion) throw new InvalidTerraformToolException(errorMsg); } - /** - * Get the best available version in download url. + * Fetch all available versions from terraform website. * - * @param downloadBaseUrl the base url of download - * @param requiredOperator operator in required version - * @param requiredNumber number in required version - * @return the best available version existed in download url. + * @return all available versions. */ - public String getBestAvailableVersionFromTerraformWebsite(String downloadBaseUrl, - String requiredOperator, - String requiredNumber) { - List allVersionsCanBeDownloaded = new ArrayList<>(); - try { - Document doc = Jsoup.connect(downloadBaseUrl).get(); - Elements elements = doc.select("a[href]"); - for (Element element : elements) { - String href = element.attr("href"); - Matcher matcher = TERRAFORM_BINARY_HREF_PATTERN.matcher(href); - if (matcher.find()) { - allVersionsCanBeDownloaded.add(matcher.group(1)); - } + @Cacheable(TERRAFORM_VERSIONS_CACHE_NAME) + @Retryable(retryFor = IOException.class, + maxAttemptsExpression = "${spring.retry.max-attempts}", + backoff = @Backoff(delayExpression = "${spring.retry.delay-millions}")) + public List fetchAvailableVersionsFromTerraformWebsite() throws IOException { + List allVersions = new ArrayList<>(); + int currentPage = 1; + int perPage = 100; + boolean hasNextPage = true; + while (hasNextPage) { + String queryVersionsWithPage = terraformVersionsQueryUrl + + "?page=" + currentPage + "&per_page=" + perPage; + List versionsInCurrentPage; + try { + versionsInCurrentPage = fetchVersions(queryVersionsWithPage); + } catch (IOException e) { + log.error("Failed to fetch terraform versions from {}", queryVersionsWithPage, e); + throw e; + } + if (CollectionUtils.isEmpty(versionsInCurrentPage)) { + hasNextPage = false; + } else { + allVersions.addAll(versionsInCurrentPage); + currentPage++; } - } catch (IOException e) { - String errorMsg = String.format("Query all versions of Terraform from the download url" - + " %s error.", downloadBaseUrl); - log.error(errorMsg, e); } - if (CollectionUtils.isEmpty(allVersionsCanBeDownloaded)) { - String errorMsg = String.format("Not found any versions of Terraform from the " - + "download url %s.", downloadBaseUrl); - log.error(errorMsg); - throw new InvalidTerraformToolException(errorMsg); + return allVersions; + } + + private List fetchVersions(String fetchVersionsApiUrl) throws JsonProcessingException { + List versions = new ArrayList<>(); + ResponseEntity responseEntity = + restTemplate.getForEntity(fetchVersionsApiUrl, String.class); + if (responseEntity.getStatusCode().is2xxSuccessful()) { + String responseBody = responseEntity.getBody(); + JsonNode jsonNode = objectMapper.readTree(responseBody); + for (JsonNode releaseNode : jsonNode) { + String version = releaseNode.get("name").asText(); + if (OFFICIAL_VERSION_PATTERN.matcher(version).matches()) { + // remove the prefix 'v' + versions.add(version.substring(1)); + } + } } - return findBestVersion(allVersionsCanBeDownloaded, requiredOperator, requiredNumber); + return versions; } - private String findBestVersion(List allAvailableVersions, - String requiredOperator, String requiredNumber) { + /** + * Find the best version from all available versions. + * + * @param allAvailableVersions all available versions + * @param requiredOperator operator in required version + * @param requiredNumber number in required version + * @return the best version + */ + public String findBestVersionFromAllAvailableVersions(List allAvailableVersions, + String requiredOperator, + String requiredNumber) { if (CollectionUtils.isEmpty(allAvailableVersions) || StringUtils.isBlank(requiredOperator) || StringUtils.isBlank(requiredNumber)) { return null; @@ -180,7 +232,8 @@ private String findBestVersion(List allAvailableVersions, * @param requiredNumber number in required version * @return true if the version is valid, otherwise return false. */ - public boolean checkIfExecutorVersionIsValid(File executorFile, String requiredOperator, + public boolean checkIfExecutorVersionIsValid(File executorFile, + String requiredOperator, String requiredNumber) { if (!executorFile.exists() && !executorFile.isFile()) { return false; @@ -194,15 +247,11 @@ public boolean checkIfExecutorVersionIsValid(File executorFile, String requiredO return false; } } - SystemCmdResult versionCheckResult = systemCmd.execute( - executorFile.getAbsolutePath() + " -v", - 5, System.getProperty("java.io.tmpdir"), false, new HashMap<>()); - if (versionCheckResult.isCommandSuccessful()) { - String actualVersion = - getActualVersionFromCommandOutput(versionCheckResult.getCommandStdOutput()); - return isVersionSatisfied(actualVersion, requiredOperator, requiredNumber); + String actualVersion = getExactVersionOfExecutor(executorFile.getAbsolutePath()); + if (StringUtils.isBlank(actualVersion)) { + return false; } - return false; + return isVersionSatisfied(actualVersion, requiredOperator, requiredNumber); } @@ -216,22 +265,125 @@ public String getExactVersionOfExecutor(String executorPath) { if (StringUtils.isBlank(executorPath)) { return null; } - String exactVersion; + SystemCmdResult versionCheckResult = systemCmd.execute(executorPath + " -v", + 5, System.getProperty("java.io.tmpdir"), false, new HashMap<>()); + if (versionCheckResult.isCommandSuccessful()) { + return getActualVersionFromCommandOutput(versionCheckResult.getCommandStdOutput()); + } else { + log.error(versionCheckResult.getCommandStdError()); + return null; + } + } + + + /** + * Install terraform with specific version. + * + * @param versionNumber the version number + * @param downloadBaseUrl download base url + * @param installDir installation directory + * @return the path of the installed executor. + */ + public File installTerraformWithVersion(String versionNumber, String downloadBaseUrl, + String installDir) { + // Install the executor with specific version into the path. + String terraformExecutorName = getTerraformExecutorName(versionNumber); + File terraformExecutorFile = new File(installDir, terraformExecutorName); + File parentDir = terraformExecutorFile.getParentFile(); try { - SystemCmdResult versionCheckResult = systemCmd.execute(executorPath + " -v", - 5, System.getProperty("java.io.tmpdir"), false, new HashMap<>()); - if (versionCheckResult.isCommandSuccessful()) { - exactVersion = - getActualVersionFromCommandOutput(versionCheckResult.getCommandStdOutput()); - } else { - log.error(versionCheckResult.getCommandStdError()); - exactVersion = getVersionFromExecutorPath(executorPath); + if (!parentDir.exists()) { + log.info("Created the installation dir {} {}.", parentDir.getAbsolutePath(), + parentDir.mkdirs() ? "successfully" : "failed"); + } + // download the binary zip file into the installation directory + File terraformZipFile = downloadTerraformBinaryZipFile( + versionNumber, downloadBaseUrl, installDir); + // unzip the zip file and move the executable binary to the installation directory + unzipBinaryZipToGetExecutor(terraformZipFile, terraformExecutorFile); + } catch (IOException e) { + log.error(e.getMessage(), e); + throw new InvalidTerraformToolException(e.getMessage()); + } + // delete the non-executable files + deleteNonExecutorFiles(parentDir); + return terraformExecutorFile; + } + + private File downloadTerraformBinaryZipFile(String versionNumber, String downloadBaseUrl, + String installDir) throws IOException { + String binaryDownloadUrl = getTerraformBinaryDownloadUrl(downloadBaseUrl, versionNumber); + String binaryZipFileName = getTerraformBinaryZipFileName(binaryDownloadUrl); + File binaryZipFile = new File(installDir, binaryZipFileName); + URL url = URI.create(binaryDownloadUrl).toURL(); + try (ReadableByteChannel rbc = Channels.newChannel(url.openStream()); + FileOutputStream fos = new FileOutputStream(binaryZipFile, false)) { + log.info("Downloading terraform binary file from {} to {}", url, + binaryZipFile.getAbsolutePath()); + fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); + log.info("Downloaded terraform binary file from {} to {} successfully.", + url, binaryZipFile.getAbsolutePath()); + } + return binaryZipFile; + } + + + private void unzipBinaryZipToGetExecutor(File binaryZipFile, File executorFile) + throws IOException { + if (!binaryZipFile.exists()) { + String errorMsg = String.format("Terraform binary zip file %s not found.", + binaryZipFile.getAbsolutePath()); + log.error(errorMsg); + throw new IOException(errorMsg); + } + try (ZipInputStream zis = new ZipInputStream(new FileInputStream(binaryZipFile))) { + log.info("Unzipping terraform binary zip file {}", binaryZipFile.getAbsolutePath()); + ZipEntry entry; + while ((entry = zis.getNextEntry()) != null) { + if (!entry.isDirectory()) { + String entryName = entry.getName(); + File entryDestinationFile = new File(executorFile.getParentFile(), entryName); + if (isExecutorFileInZipForTerraform(entryName)) { + extractFile(zis, entryDestinationFile); + Files.move(entryDestinationFile.toPath(), executorFile.toPath(), + StandardCopyOption.REPLACE_EXISTING); + log.info("Unzipped terraform file {} and extract the executor {} " + + "successfully.", binaryZipFile.getAbsolutePath(), + executorFile.getAbsolutePath()); + } + } + } + } + } + + private boolean isExecutorFileInZipForTerraform(String entryName) { + return entryName.startsWith("terraform"); + } + + + private void extractFile(ZipInputStream zis, File destinationFile) throws IOException { + try (BufferedOutputStream bos = new BufferedOutputStream( + new FileOutputStream(destinationFile))) { + byte[] bytesIn = new byte[4096]; + int read; + while ((read = zis.read(bytesIn)) != -1) { + bos.write(bytesIn, 0, read); + } + } + } + + private void deleteNonExecutorFiles(File dir) { + File[] files = dir.listFiles(); + if (files != null) { + for (File file : files) { + if (file.isDirectory()) { + deleteNonExecutorFiles(file); + } else { + if (!file.getName().startsWith(TERRAFORM_EXECUTOR_PREFIX) && !file.delete()) { + log.warn("Failed to delete file {}.", file.getAbsolutePath()); + } + } } - } catch (Exception e) { - log.error("Get exact version of the executor {} failed.", executorPath, e); - exactVersion = executorPath; } - return exactVersion; } @@ -247,9 +399,7 @@ private String getActualVersionFromCommandOutput(String commandOutput) { if (matcher.find()) { return matcher.group(1); } - String errorMsg = String.format("Cannot find the version from command output：%s", - commandOutput); - throw new InvalidTerraformToolException(errorMsg); + return null; } private boolean isVersionSatisfied(String actualNumber, String requiredOperator, @@ -274,35 +424,24 @@ private boolean isVersionSatisfied(String actualNumber, String requiredOperator, * @return binary file name */ public String getTerraformExecutorName(String versionNumber) { - return String.format("terraform-%s", versionNumber); - } - - /** - * Get terraform binary file name. - * - * @param versionNumber version number - * @return binary file name - */ - public String getTerraformBinaryFileName(String versionNumber) { - return String.format("terraform_%s_%s_%s.zip", - versionNumber, getOperatingSystemCode(), OS_ARCH); + return TERRAFORM_EXECUTOR_PREFIX + versionNumber; } - /** - * Get terraform binary file download url. + * Get whole download url of the executor binary file. * * @param downloadBaseUrl download base url * @param versionNumber version number - * @param binaryFileName binary file name - * @return binary file name + * @return whole download url of the executor binary file */ - public String getTerraformBinaryDownloadUrl(String downloadBaseUrl, - String versionNumber, String binaryFileName) { - return String.format("%s/%s/%s", downloadBaseUrl, versionNumber, binaryFileName); - + private String getTerraformBinaryDownloadUrl(String downloadBaseUrl, String versionNumber) { + return String.format(TERRAFORM_BINARY_DOWNLOAD_URL_FORMAT, downloadBaseUrl, versionNumber, + versionNumber, getOperatingSystemCode(), OS_ARCH); } + private String getTerraformBinaryZipFileName(String executorBinaryDownloadUrl) { + return executorBinaryDownloadUrl.substring(executorBinaryDownloadUrl.lastIndexOf("/") + 1); + } private String getOperatingSystemCode() { if (OS_NAME.contains("windows")) { diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index dca1bcb..6fa7fec 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -17,4 +17,6 @@ clean.workspace.after.deployment.enabled=true spring.retry.max-attempts=3 spring.retry.delay-millions=1000 terraform.install.dir=/opt/terraform -terraform.download.base.url=https://releases.hashicorp.com/terraform \ No newline at end of file +terraform.download.base.url=https://releases.hashicorp.com/terraform +terraform.versions.query.url=https://api.github.com/repos/hashicorp/terraform/tags +terraform.versions.cache.expire.time.in.minutes=60 \ No newline at end of file diff --git a/src/test/java/org/eclipse/xpanse/terraform/boot/terraform/TerraformInstallerTest.java b/src/test/java/org/eclipse/xpanse/terraform/boot/terraform/TerraformInstallerTest.java index e1b7bf7..c23d531 100644 --- a/src/test/java/org/eclipse/xpanse/terraform/boot/terraform/TerraformInstallerTest.java +++ b/src/test/java/org/eclipse/xpanse/terraform/boot/terraform/TerraformInstallerTest.java @@ -49,7 +49,5 @@ void testGetExecutableTerraformByVersion() { String requiredVersion4 = ">= 100.0.0"; assertThrows(InvalidTerraformToolException.class, () -> installer.getExecutorPathThatMatchesRequiredVersion(requiredVersion4)); - - } } \ No newline at end of file