Skip to content

Commit

Permalink
#687: prevent checksum recheck (#688)
Browse files Browse the repository at this point in the history
  • Loading branch information
@hohwille
hohwille authored Oct 11, 2024
1 parent a70b82c commit 51a16e1
Show file tree
Hide file tree
Showing 5 changed files with 164 additions and 71 deletions.
14 changes: 12 additions & 2 deletions cli/src/main/java/com/devonfw/tools/ide/url/model/file/json/UrlStatusState.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,17 @@ public final class UrlStatusState {
*/
public UrlStatusState() {

this.timestamp = Instant.now();
this(Instant.now());
}

/**
* The constructor.
*
* @param timestamp the {@link #getTimestamp() timestamp}.
*/
public UrlStatusState(Instant timestamp) {

this.timestamp = timestamp;
}

/**
Expand Down Expand Up @@ -78,4 +88,4 @@ public String toString() {
return getClass().getSimpleName() + "@" + this.timestamp
+ ((this.message == null || this.message.isEmpty()) ? "" : ":" + this.message);
}
}
}
33 changes: 19 additions & 14 deletions cli/src/main/java/com/devonfw/tools/ide/url/updater/AbstractUrlUpdater.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,7 @@ protected HttpResponse<InputStream> doGetResponseAsStream(String url) {
*
* @param urlVersion the {@link UrlVersion} with the {@link UrlVersion#getName() version-number} to process.
* @param downloadUrl the URL of the download for the tool.
* @return true if the version was successfully updated, false otherwise.
* @return {@code true} if the version was successfully added, {@code false} otherwise.
*/
protected boolean doAddVersion(UrlVersion urlVersion, String downloadUrl) {

Expand All @@ -186,7 +186,7 @@ protected boolean doAddVersion(UrlVersion urlVersion, String downloadUrl) {
* @param edition the edition of the tool.
* @param urlVersion the {@link UrlVersion} with the {@link UrlVersion#getName() version-number} to process.
* @param downloadUrl the URL of the download for the tool.
* @return true if the version was successfully updated, false otherwise.
* @return {@code true} if the version was successfully added, {@code false} otherwise.
*/
protected boolean doAddVersion(String edition, UrlVersion urlVersion, String downloadUrl) {

Expand All @@ -200,7 +200,7 @@ protected boolean doAddVersion(String edition, UrlVersion urlVersion, String dow
* @param urlVersion the {@link UrlVersion} with the {@link UrlVersion#getName() version-number} to process.
* @param downloadUrl the URL of the download for the tool.
* @param os the {@link OperatingSystem} for the tool (can be null).
* @return true if the version was successfully updated, false otherwise.
* @return {@code true} if the version was successfully added, {@code false} otherwise.
*/
protected boolean doAddVersion(UrlVersion urlVersion, String downloadUrl, OperatingSystem os) {

Expand All @@ -214,7 +214,7 @@ protected boolean doAddVersion(UrlVersion urlVersion, String downloadUrl, Operat
* @param urlVersion the {@link UrlVersion} with the {@link UrlVersion#getName() version-number} to process.
* @param downloadUrl the URL of the download for the tool.
* @param os the {@link OperatingSystem} for the tool (can be null).
* @return true if the version was successfully updated, false otherwise.
* @return {@code true} if the version was successfully added, {@code false} otherwise.
*/
protected boolean doAddVersion(String edition, UrlVersion urlVersion, String downloadUrl, OperatingSystem os) {

Expand All @@ -228,7 +228,7 @@ protected boolean doAddVersion(String edition, UrlVersion urlVersion, String dow
* @param downloadUrl the URL of the download for the tool.
* @param os the {@link OperatingSystem} for the tool (can be null).
* @param architecture the optional {@link SystemArchitecture}.
* @return true if the version was successfully updated, false otherwise.
* @return {@code true} if the version was successfully added, {@code false} otherwise.
*/
protected boolean doAddVersion(UrlVersion urlVersion, String downloadUrl, OperatingSystem os,
SystemArchitecture architecture) {
Expand All @@ -244,7 +244,7 @@ protected boolean doAddVersion(UrlVersion urlVersion, String downloadUrl, Operat
* @param downloadUrl the URL of the download for the tool.
* @param os the {@link OperatingSystem} for the tool (can be null).
* @param architecture the optional {@link SystemArchitecture}.
* @return true if the version was successfully updated, false otherwise.
* @return {@code true} if the version was successfully added, {@code false} otherwise.
*/
protected boolean doAddVersion(String edition, UrlVersion urlVersion, String downloadUrl, OperatingSystem os,
SystemArchitecture architecture) {
Expand All @@ -261,7 +261,7 @@ protected boolean doAddVersion(String edition, UrlVersion urlVersion, String dow
* @param os the optional {@link OperatingSystem}.
* @param architecture the optional {@link SystemArchitecture}.
* @param checksum the existing checksum (e.g. from JSON metadata) or the empty {@link String} if not available and computation needed.
* @return {@code true} if the version was successfully updated, {@code false} otherwise.
* @return {@code true} if the version was successfully added, {@code false} otherwise.
*/
protected boolean doAddVersion(UrlVersion urlVersion, String url, OperatingSystem os, SystemArchitecture architecture, String checksum) {
return doAddVersion(getEdition(), urlVersion, url, os, architecture, checksum);
Expand All @@ -276,7 +276,7 @@ protected boolean doAddVersion(UrlVersion urlVersion, String url, OperatingSyste
* @param os the optional {@link OperatingSystem}.
* @param architecture the optional {@link SystemArchitecture}.
* @param checksum the existing checksum (e.g. from JSON metadata) or the empty {@link String} if not available and computation needed.
* @return {@code true} if the version was successfully updated, {@code false} otherwise.
* @return {@code true} if the version was successfully added, {@code false} otherwise.
*/
protected boolean doAddVersion(String edition, UrlVersion urlVersion, String url, OperatingSystem os, SystemArchitecture architecture,
String checksum) {
Expand All @@ -297,8 +297,7 @@ protected boolean doAddVersion(String edition, UrlVersion urlVersion, String url
}
url = url.replace("${edition}", edition);

return checkDownloadUrl(edition, url, urlVersion, os, architecture, checksum);

return doAddVersionUrlIfNewAndValid(edition, url, urlVersion, os, architecture, checksum);
}

/**
Expand Down Expand Up @@ -388,9 +387,16 @@ protected boolean isValidContentType(String contentType) {
* @param checksum the existing checksum (e.g. from JSON metadata) or the empty {@link String} if not available and computation needed.
* @return {@code true} if the download was checked successfully, {@code false} otherwise.
*/
private boolean checkDownloadUrl(String edition, String url, UrlVersion urlVersion, OperatingSystem os,
private boolean doAddVersionUrlIfNewAndValid(String edition, String url, UrlVersion urlVersion, OperatingSystem os,
SystemArchitecture architecture, String checksum) {

UrlDownloadFile urlDownloadFile = urlVersion.getUrls(os, architecture);
if (urlDownloadFile != null) {
if (urlDownloadFile.getUrls().contains(url)) {
logger.debug("Skipping add of already existing URL {}", url);
return false;
}
}
HttpResponse<?> response = doCheckDownloadViaHeadRequest(url);
int statusCode = response.statusCode();
String toolWithEdition = getToolWithEdition(edition);
Expand All @@ -400,7 +406,6 @@ private boolean checkDownloadUrl(String edition, String url, UrlVersion urlVersi

boolean update = false;

UrlDownloadFile urlDownloadFile = urlVersion.getUrls(os, architecture);
if (success) {
UrlChecksum urlChecksum = null;
if (urlDownloadFile != null) {
Expand Down Expand Up @@ -661,7 +666,7 @@ protected Set<String> getUrlFilenames() {
/**
* Checks if we are dependent on OS URL file names, can be overridden to disable OS dependency
*
* @return true if we want to check for missing OS URL file names, false if not
* @return {@code true} if we want to check for missing OS URL file names, {@code false} if not.
*/
protected boolean isOsDependent() {

Expand All @@ -672,7 +677,7 @@ protected boolean isOsDependent() {
* Checks if an OS URL file name was missing in {@link UrlVersion}
*
* @param urlVersion the {@link UrlVersion} to check
* @return true if an OS type was missing, false if not
* @return {@code true} if an OS type was missing, {@code false} if not.
*/
public boolean isMissingOs(UrlVersion urlVersion) {

Expand Down
6 changes: 0 additions & 6 deletions cli/src/test/java/com/devonfw/tools/ide/tool/UrlUpdaterMock.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
import java.util.HashSet;
import java.util.Set;

import com.devonfw.tools.ide.url.model.folder.UrlRepository;
import com.devonfw.tools.ide.url.model.folder.UrlVersion;
import com.devonfw.tools.ide.url.updater.AbstractUrlUpdater;
import com.devonfw.tools.ide.url.updater.UrlUpdater;
Expand Down Expand Up @@ -35,11 +34,6 @@ protected String getTool() {
return "mocked";
}

@Override
public void update(UrlRepository urlRepository) {
super.update(urlRepository);
}

@Override
protected Set<String> getVersions() {
return versions;
Expand Down
178 changes: 131 additions & 47 deletions cli/src/test/java/com/devonfw/tools/ide/tool/UrlUpdaterTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,25 @@
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Duration;
import java.time.Instant;
import java.time.temporal.ChronoUnit;

import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;

import com.devonfw.tools.ide.os.OperatingSystem;
import com.devonfw.tools.ide.os.SystemArchitecture;
import com.devonfw.tools.ide.url.model.file.UrlChecksum;
import com.devonfw.tools.ide.url.model.file.UrlDownloadFile;
import com.devonfw.tools.ide.url.model.file.UrlStatusFile;
import com.devonfw.tools.ide.url.model.file.json.StatusJson;
import com.devonfw.tools.ide.url.model.file.json.UrlStatus;
import com.devonfw.tools.ide.url.model.file.json.UrlStatusState;
import com.devonfw.tools.ide.url.model.folder.UrlEdition;
import com.devonfw.tools.ide.url.model.folder.UrlRepository;
import com.devonfw.tools.ide.url.model.folder.UrlTool;
import com.devonfw.tools.ide.url.model.folder.UrlVersion;
import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo;
import com.github.tomakehurst.wiremock.junit5.WireMockTest;

Expand Down Expand Up @@ -93,80 +104,153 @@ public void testUrlUpdaterIsNotUpdatingWhenStatusManualIsTrue(@TempDir Path temp
}

/**
* Tests if the timestamps of the status.json get updated properly. Creates an initial status.json with a success timestamp. Updates the status.json with an
* error timestamp and compares it with the success timestamp. Updates the status.json with a final success timestamp and compares it with the error
* timestamp.
* Tests if the timestamps of the status.json get updated properly if a first error is detected.
* <p>
* See: <a href="https://github.com/devonfw/ide/issues/1343">#1343</a> for reference.
*
* @param tempDir Temporary directory
* @param wmRuntimeInfo wireMock server on a random port
*/
@Test
public void testUrlUpdaterStatusJsonRefreshBugStillExisting(@TempDir Path tempDir, WireMockRuntimeInfo wmRuntimeInfo) {
public void testStatusJsonUpdateOnFirstError(@TempDir Path tempDir, WireMockRuntimeInfo wmRuntimeInfo) {

// arrange
stubFor(any(urlMatching("/os/.*")).willReturn(aResponse().withStatus(200).withBody("aBody")));

UrlRepository urlRepository = UrlRepository.load(tempDir);
UrlUpdaterMockSingle updater = new UrlUpdaterMockSingle(wmRuntimeInfo);

String statusUrl = wmRuntimeInfo.getHttpBaseUrl() + "/os/windows_x64_url.tgz";
String toolName = "mocked";
String editionName = "mocked";
String versionName = "1.0";
String url = wmRuntimeInfo.getHttpBaseUrl() + "/os/windows_x64_url.tgz";
Instant now = Instant.now();
Instant lastMonth = now.minus(31, ChronoUnit.DAYS);
UrlRepository urlRepository = UrlRepository.load(tempDir);
UrlTool urlTool = urlRepository.getOrCreateChild(toolName);
UrlEdition urlEdition = urlTool.getOrCreateChild(editionName);
UrlVersion urlVersion = urlEdition.getOrCreateChild(versionName);
// we create the structure of our tool version and URL to simulate it was valid last moth
UrlStatusFile statusFile = urlVersion.getOrCreateStatus();
UrlStatus status = statusFile.getStatusJson().getOrCreateUrlStatus(url);
UrlStatusState successState = new UrlStatusState(lastMonth); // ensure that we trigger a recheck of the URL
status.setSuccess(successState);
UrlDownloadFile urlDownloadFile = urlVersion.getOrCreateUrls(OperatingSystem.WINDOWS, SystemArchitecture.X64);
urlDownloadFile.addUrl(url);
UrlChecksum urlChecksum = urlVersion.getOrCreateChecksum(urlDownloadFile.getName());
urlChecksum.setChecksum("1234567890");
urlVersion.save();
UrlUpdaterMockSingle updater = new UrlUpdaterMockSingle(wmRuntimeInfo);
// now we want to simulate that the url got broken (404) and the updater is properly handling this
stubFor(any(urlMatching("/os/.*")).willReturn(aResponse().withStatus(404)));

// act
updater.update(urlRepository);

Path versionsPath = tempDir.resolve(toolName).resolve(editionName).resolve(versionName);

// assert
assertThat(versionsPath.resolve("status.json")).exists();

StatusJson statusJson = retrieveStatusJson(urlRepository, toolName, editionName, versionName);
status = statusJson.getStatus(url);
successState = status.getSuccess();
assertThat(successState).isNotNull();
assertThat(successState.getTimestamp()).isEqualTo(lastMonth);
UrlStatusState errorState = status.getError();
assertThat(errorState).isNotNull();
assertThat(errorState.getCode()).isEqualTo(404);
assertThat(Duration.between(errorState.getTimestamp(), now)).isLessThan(Duration.ofSeconds(5));
}

UrlStatus urlStatus = statusJson.getOrCreateUrlStatus(statusUrl);

Instant successTimestamp = urlStatus.getSuccess().getTimestamp();

assertThat(successTimestamp).isNotNull();

stubFor(any(urlMatching("/os/.*")).willReturn(aResponse().withStatus(404)));

// re-initialize UrlRepository for error timestamp
UrlRepository urlRepositoryWithError = UrlRepository.load(tempDir);
updater.update(urlRepositoryWithError);

statusJson = retrieveStatusJson(urlRepositoryWithError, toolName, editionName, versionName);

urlStatus = statusJson.getOrCreateUrlStatus(statusUrl);
successTimestamp = urlStatus.getSuccess().getTimestamp();
Instant errorTimestamp = urlStatus.getError().getTimestamp();
Integer errorCode = urlStatus.getError().getCode();

assertThat(errorCode).isEqualTo(404);
assertThat(errorTimestamp).isAfter(successTimestamp);
/**
* Tests if the timestamps of the status.json get updated properly on success after an error.
* <p>
* See: <a href="https://github.com/devonfw/ide/issues/1343">#1343</a> for reference.
*
* @param tempDir Temporary directory
* @param wmRuntimeInfo wireMock server on a random port
*/
@Test
public void testSuccessStateUpdatedAfterError(@TempDir Path tempDir, WireMockRuntimeInfo wmRuntimeInfo) {

// arrange
String toolName = "mocked";
String editionName = "mocked";
String versionName = "1.0";
String url = wmRuntimeInfo.getHttpBaseUrl() + "/os/windows_x64_url.tgz";
Instant now = Instant.now();
Instant lastMonth = now.minus(31, ChronoUnit.DAYS);
Instant lastSuccess = lastMonth.minus(1, ChronoUnit.DAYS);
UrlRepository urlRepository = UrlRepository.load(tempDir);
UrlTool urlTool = urlRepository.getOrCreateChild(toolName);
UrlEdition urlEdition = urlTool.getOrCreateChild(editionName);
UrlVersion urlVersion = urlEdition.getOrCreateChild(versionName);
// we create the structure of our tool version and URL to simulate it was valid last moth
UrlStatusFile statusFile = urlVersion.getOrCreateStatus();
UrlStatus status = statusFile.getStatusJson().getOrCreateUrlStatus(url);
UrlStatusState successState = new UrlStatusState(lastSuccess);
status.setSuccess(successState);
UrlStatusState errorState = new UrlStatusState(lastMonth);
errorState.setCode(404);
status.setError(errorState);
UrlDownloadFile urlDownloadFile = urlVersion.getOrCreateUrls(OperatingSystem.WINDOWS, SystemArchitecture.X64);
urlDownloadFile.addUrl(url);
UrlChecksum urlChecksum = urlVersion.getOrCreateChecksum(urlDownloadFile.getName());
urlChecksum.setChecksum("1234567890");
urlVersion.save();
UrlUpdaterMockSingle updater = new UrlUpdaterMockSingle(wmRuntimeInfo);
// now we want to simulate that the broken url is working again
stubFor(any(urlMatching("/os/.*")).willReturn(aResponse().withStatus(200).withHeader("Content-Type", "text/plain")));

// re-initialize UrlRepository for error timestamp
UrlRepository urlRepositoryWithSuccess = UrlRepository.load(tempDir);
updater.update(urlRepositoryWithSuccess);

assertThat(versionsPath.resolve("status.json")).exists();
// act
updater.update(urlRepository);

statusJson = retrieveStatusJson(urlRepositoryWithSuccess, toolName, editionName, versionName);
// assert
status = retrieveStatusJson(urlRepository, toolName, editionName, versionName).getStatus(url);
successState = status.getSuccess();
assertThat(successState).isNotNull();
assertThat(Duration.between(successState.getTimestamp(), now)).isLessThan(Duration.ofSeconds(5));
errorState = status.getError();
assertThat(errorState).isNotNull();
assertThat(errorState.getCode()).isEqualTo(404);
assertThat(errorState.getTimestamp()).isEqualTo(lastMonth);
}

urlStatus = statusJson.getOrCreateUrlStatus(statusUrl);
/**
* Tests if the the tool version gets entirely removed if all versions are broken for a long time.
*
* @param tempDir Temporary directory
* @param wmRuntimeInfo wireMock server on a random port
*/
@Test
public void testVersionRemovedIfErrorPersists(@TempDir Path tempDir, WireMockRuntimeInfo wmRuntimeInfo) {

successTimestamp = urlStatus.getSuccess().getTimestamp();
errorTimestamp = urlStatus.getError().getTimestamp();
errorCode = urlStatus.getError().getCode();
// arrange
String toolName = "mocked";
String editionName = "mocked";
String versionName = "1.0";
String url = wmRuntimeInfo.getHttpBaseUrl() + "/os/windows_x64_url.tgz";
Instant now = Instant.now();
Instant lastMonth = now.minus(31, ChronoUnit.DAYS);
Instant lastSuccess = lastMonth.minus(1, ChronoUnit.DAYS);
UrlRepository urlRepository = UrlRepository.load(tempDir);
UrlTool urlTool = urlRepository.getOrCreateChild(toolName);
UrlEdition urlEdition = urlTool.getOrCreateChild(editionName);
UrlVersion urlVersion = urlEdition.getOrCreateChild(versionName);
// we create the structure of our tool version and URL to simulate it was valid last moth
UrlStatusFile statusFile = urlVersion.getOrCreateStatus();
UrlStatus status = statusFile.getStatusJson().getOrCreateUrlStatus(url);
UrlStatusState successState = new UrlStatusState(lastSuccess);
status.setSuccess(successState);
UrlStatusState errorState = new UrlStatusState(lastMonth);
errorState.setCode(404);
status.setError(errorState);
UrlDownloadFile urlDownloadFile = urlVersion.getOrCreateUrls(OperatingSystem.WINDOWS, SystemArchitecture.X64);
urlDownloadFile.addUrl(url);
UrlChecksum urlChecksum = urlVersion.getOrCreateChecksum(urlDownloadFile.getName());
urlChecksum.setChecksum("1234567890");
urlVersion.save();
UrlUpdaterMockSingle updater = new UrlUpdaterMockSingle(wmRuntimeInfo);
// now we want to simulate that the url got broken (404) and the updater is properly handling this
stubFor(any(urlMatching("/os/.*")).willReturn(aResponse().withStatus(404)));

assertThat(errorCode).isEqualTo(200);
assertThat(errorTimestamp).isAfter(successTimestamp);
// act
updater.update(urlRepository);

// assert
assertThat(urlVersion.getPath()).doesNotExist();
}

/**
Expand Down
Loading

0 comments on commit 51a16e1

Please sign in to comment.