diff --git a/server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java b/server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java index 20890aa60..8b5905332 100644 --- a/server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java +++ b/server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java @@ -616,9 +616,9 @@ public ExtensionJson toExtensionVersionJson( } json.files = Maps.newLinkedHashMapWithExpectedSize(6); - var versionUrl = UrlUtil.createApiUrl(versionBaseUrl, json.version); + var fileBaseUrl = UrlUtil.createApiUrl(versionBaseUrl, json.version, "file"); for (var resource : resources) { - var fileUrl = UrlUtil.createApiUrl(versionUrl, "file", resource.getName()); + var fileUrl = UrlUtil.createApiUrl(fileBaseUrl, resource.getName().split("/")); json.files.put(resource.getType(), fileUrl); } diff --git a/server/src/main/java/org/eclipse/openvsx/RegistryAPI.java b/server/src/main/java/org/eclipse/openvsx/RegistryAPI.java index 2f2485eb5..c3716b58c 100644 --- a/server/src/main/java/org/eclipse/openvsx/RegistryAPI.java +++ b/server/src/main/java/org/eclipse/openvsx/RegistryAPI.java @@ -47,9 +47,10 @@ import springfox.documentation.annotations.ApiIgnore; +import javax.servlet.http.HttpServletRequest; + @RestController public class RegistryAPI { - private final static int REVIEW_TITLE_SIZE = 255; private final static int REVIEW_COMMENT_SIZE = 2048; @@ -172,7 +173,7 @@ public ResponseEntity getExtension( return new ResponseEntity<>(json, HttpStatus.NOT_FOUND); } - @GetMapping("/api/{namespace}/{extension}/{version}/file/{fileName:.+}") + @GetMapping("/api/{namespace}/{extension}/{version}/file/**") @CrossOrigin @ApiOperation("Access a file packaged by an extension") @ApiResponses({ @@ -195,15 +196,15 @@ public ResponseEntity getExtension( ) }) public ResponseEntity getFile( + HttpServletRequest request, @PathVariable @ApiParam(value = "Extension namespace", example = "redhat") String namespace, @PathVariable @ApiParam(value = "Extension name", example = "java") String extension, @PathVariable @ApiParam(value = "Extension version", example = "0.65.0") - String version, - @PathVariable @ApiParam(value = "Name of the file to access", example = "LICENSE.txt") - String fileName + String version ) { + var fileName = UrlUtil.extractWildcardPath(request); for (var registry : getRegistries()) { try { return registry.getFile(namespace, extension, version, fileName); diff --git a/server/src/main/java/org/eclipse/openvsx/adapter/VSCodeAdapter.java b/server/src/main/java/org/eclipse/openvsx/adapter/VSCodeAdapter.java index f4e2990c6..7ab52a33b 100644 --- a/server/src/main/java/org/eclipse/openvsx/adapter/VSCodeAdapter.java +++ b/server/src/main/java/org/eclipse/openvsx/adapter/VSCodeAdapter.java @@ -10,6 +10,7 @@ package org.eclipse.openvsx.adapter; import com.google.common.collect.Lists; +import org.apache.commons.lang3.ArrayUtils; import org.eclipse.openvsx.dto.ExtensionDTO; import org.eclipse.openvsx.dto.ExtensionVersionDTO; import org.eclipse.openvsx.dto.FileResourceDTO; @@ -224,7 +225,8 @@ private String createFileUrl(List singleResource, String versio } private String createFileUrl(FileResourceDTO resource, String versionUrl) { - return resource != null ? UrlUtil.createApiUrl(versionUrl, "file", resource.getName()) : null; + var segments = ArrayUtils.addAll(new String[]{"file"}, resource.getName().split("/")); + return resource != null ? UrlUtil.createApiUrl(versionUrl, segments) : null; } private ExtensionQueryResult toQueryResult(List extensions, long totalCount) { diff --git a/server/src/main/java/org/eclipse/openvsx/storage/AzureBlobStorageService.java b/server/src/main/java/org/eclipse/openvsx/storage/AzureBlobStorageService.java index d6b64b984..06748b290 100644 --- a/server/src/main/java/org/eclipse/openvsx/storage/AzureBlobStorageService.java +++ b/server/src/main/java/org/eclipse/openvsx/storage/AzureBlobStorageService.java @@ -11,9 +11,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.net.URI; -import java.net.URLEncoder; import javax.transaction.Transactional; import javax.transaction.Transactional.TxType; @@ -22,11 +20,11 @@ import com.azure.storage.blob.BlobContainerClientBuilder; import com.azure.storage.blob.models.BlobHttpHeaders; import com.azure.storage.blob.models.BlobStorageException; -import com.google.common.base.Preconditions; import com.google.common.base.Strings; -import org.eclipse.openvsx.entities.ExtensionVersion; +import org.apache.commons.lang3.ArrayUtils; import org.eclipse.openvsx.entities.FileResource; +import org.eclipse.openvsx.util.UrlUtil; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.http.HttpStatus; @@ -68,7 +66,7 @@ protected BlobContainerClient getContainerClient() { @Override @Transactional(TxType.MANDATORY) public void uploadFile(FileResource resource) { - var blobName = getBlobName(resource.getName(), resource.getExtension()); + var blobName = getBlobName(resource); if (Strings.isNullOrEmpty(serviceEndpoint)) { throw new IllegalStateException("Cannot upload file " + blobName + ": missing Azure blob service endpoint"); @@ -98,7 +96,7 @@ protected void uploadFile(byte[] content, String fileName, String blobName) { @Override public void removeFile(FileResource resource) { - var blobName = getBlobName(resource.getName(), resource.getExtension()); + var blobName = getBlobName(resource); if (Strings.isNullOrEmpty(serviceEndpoint)) { throw new IllegalStateException("Cannot remove file " + blobName + ": missing Azure blob service endpoint"); @@ -117,7 +115,7 @@ public void removeFile(FileResource resource) { @Override public URI getLocation(FileResource resource) { - var blobName = getBlobName(resource.getName(), resource.getExtension()); + var blobName = getBlobName(resource); if (Strings.isNullOrEmpty(serviceEndpoint)) { throw new IllegalStateException("Cannot determine location of file " + blobName + ": missing Azure blob service endpoint"); @@ -128,18 +126,13 @@ public URI getLocation(FileResource resource) { return URI.create(serviceEndpoint + blobContainer + "/" + blobName); } - protected String getBlobName(String name, ExtensionVersion extVersion) { - Preconditions.checkNotNull(name); - try { - var extension = extVersion.getExtension(); - var namespace = extension.getNamespace(); - return namespace.getName() - + "/" + extension.getName() - + "/" + URLEncoder.encode(extVersion.getVersion(), "UTF-8") - + "/" + name; - } catch (UnsupportedEncodingException exc) { - throw new RuntimeException(exc); - } + protected String getBlobName(FileResource resource) { + var extVersion = resource.getExtension(); + var extension = extVersion.getExtension(); + var namespace = extension.getNamespace(); + var segments = new String[]{namespace.getName(), extension.getName(), extVersion.getVersion()}; + segments = ArrayUtils.addAll(segments, resource.getName().split("/")); + return UrlUtil.createApiUrl("", segments).substring(1); // remove first '/' } } \ No newline at end of file diff --git a/server/src/main/java/org/eclipse/openvsx/storage/AzureDownloadCountService.java b/server/src/main/java/org/eclipse/openvsx/storage/AzureDownloadCountService.java index 3bee59774..7c269aaa2 100644 --- a/server/src/main/java/org/eclipse/openvsx/storage/AzureDownloadCountService.java +++ b/server/src/main/java/org/eclipse/openvsx/storage/AzureDownloadCountService.java @@ -31,11 +31,13 @@ import org.springframework.transaction.TransactionException; import org.springframework.transaction.support.TransactionTemplate; import org.springframework.util.StopWatch; +import org.springframework.web.util.UriUtils; import javax.persistence.EntityManager; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.net.URI; +import java.nio.charset.StandardCharsets; import java.time.Duration; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; @@ -177,7 +179,7 @@ private void processBlobItem(String blobName) { matchesStorageBlobContainer = storageBlobContainer.equals(container); } if(matchesStorageBlobContainer) { - var fileName = pathParams[pathParams.length - 1].toUpperCase(); + var fileName = UriUtils.decode(pathParams[pathParams.length - 1], StandardCharsets.UTF_8).toUpperCase(); var timestamps = files.getOrDefault(fileName, new ArrayList<>()); timestamps.add(LocalDateTime.parse(node.get("time").asText(), DateTimeFormatter.ISO_ZONED_DATE_TIME)); files.put(fileName, timestamps); diff --git a/server/src/main/java/org/eclipse/openvsx/storage/GoogleCloudStorageService.java b/server/src/main/java/org/eclipse/openvsx/storage/GoogleCloudStorageService.java index a3bb9598d..60adcc958 100644 --- a/server/src/main/java/org/eclipse/openvsx/storage/GoogleCloudStorageService.java +++ b/server/src/main/java/org/eclipse/openvsx/storage/GoogleCloudStorageService.java @@ -21,6 +21,7 @@ import com.google.common.base.Preconditions; import com.google.common.base.Strings; +import org.apache.commons.lang3.ArrayUtils; import org.eclipse.openvsx.entities.ExtensionVersion; import org.eclipse.openvsx.entities.FileResource; import org.eclipse.openvsx.util.UrlUtil; @@ -67,7 +68,7 @@ protected Storage getStorage() { @Override @Transactional(TxType.MANDATORY) public void uploadFile(FileResource resource) { - var objectId = getObjectId(resource.getName(), resource.getExtension()); + var objectId = getObjectId(resource); if (Strings.isNullOrEmpty(bucketId)) { throw new IllegalStateException("Cannot upload file " + objectId + ": missing Google bucket id"); @@ -91,7 +92,7 @@ protected void uploadFile(byte[] content, String fileName, String objectId) { @Override public void removeFile(FileResource resource) { - var objectId = getObjectId(resource.getName(), resource.getExtension()); + var objectId = getObjectId(resource); if (Strings.isNullOrEmpty(bucketId)) { throw new IllegalStateException("Cannot remove file " + objectId + ": missing Google bucket id"); @@ -105,21 +106,16 @@ public URI getLocation(FileResource resource) { throw new IllegalStateException("Cannot determine location of file " + resource.getName() + ": missing Google bucket id"); } - var extVersion = resource.getExtension(); - var extension = extVersion.getExtension(); - var namespace = extension.getNamespace(); - return URI.create(UrlUtil.createApiUrl(BASE_URL, bucketId, namespace.getName(), - extension.getName(), extVersion.getVersion(), resource.getName())); + return URI.create(BASE_URL + bucketId + "/" + getObjectId(resource)); } - protected String getObjectId(String name, ExtensionVersion extVersion) { - Preconditions.checkNotNull(name); + protected String getObjectId(FileResource resource) { + var extVersion = resource.getExtension(); var extension = extVersion.getExtension(); var namespace = extension.getNamespace(); - return namespace.getName() - + "/" + extension.getName() - + "/" + extVersion.getVersion() - + "/" + name; + var segments = new String[]{namespace.getName(), extension.getName(), extVersion.getVersion()}; + segments = ArrayUtils.addAll(segments, resource.getName().split("/")); + return UrlUtil.createApiUrl("", segments).substring(1); // remove first '/' } } \ No newline at end of file diff --git a/server/src/main/java/org/eclipse/openvsx/storage/StorageUtilService.java b/server/src/main/java/org/eclipse/openvsx/storage/StorageUtilService.java index 7702ebcbe..276a88882 100644 --- a/server/src/main/java/org/eclipse/openvsx/storage/StorageUtilService.java +++ b/server/src/main/java/org/eclipse/openvsx/storage/StorageUtilService.java @@ -22,6 +22,7 @@ import com.google.common.base.Strings; +import org.apache.commons.lang3.ArrayUtils; import org.eclipse.openvsx.entities.Download; import org.eclipse.openvsx.entities.ExtensionVersion; import org.eclipse.openvsx.entities.FileResource; @@ -145,8 +146,9 @@ public URI getLocation(FileResource resource) { private String getFileUrl(String name, ExtensionVersion extVersion, String serverUrl) { var extension = extVersion.getExtension(); var namespace = extension.getNamespace(); - return UrlUtil.createApiUrl(serverUrl, "api", namespace.getName(), extension.getName(), extVersion.getVersion(), - "file", name); + var segments = new String[]{"api", namespace.getName(), extension.getName(), extVersion.getVersion(), "file"}; + segments = ArrayUtils.addAll(segments, name.split("/")); + return UrlUtil.createApiUrl(serverUrl, segments); } /** @@ -155,10 +157,10 @@ private String getFileUrl(String name, ExtensionVersion extVersion, String serve public void addFileUrls(ExtensionVersion extVersion, String serverUrl, Map type2Url, String... types) { var extension = extVersion.getExtension(); var namespace = extension.getNamespace(); - var versionUrl = UrlUtil.createApiUrl(serverUrl, "api", namespace.getName(), extension.getName(), extVersion.getVersion()); + var baseUrl = UrlUtil.createApiUrl(serverUrl, "api", namespace.getName(), extension.getName(), extVersion.getVersion(), "file"); var resources = repositories.findFilesByType(extVersion, Arrays.asList(types)); for (var resource : resources) { - var fileUrl = UrlUtil.createApiUrl(versionUrl, "file", resource.getName()); + var fileUrl = UrlUtil.createApiUrl(baseUrl, resource.getName().split("/")); type2Url.put(resource.getType(), fileUrl); } } @@ -167,11 +169,11 @@ public Map getWebResourceUrls(ExtensionVersion extVersion, Strin var name2Url = new HashMap(); var extension = extVersion.getExtension(); var namespace = extension.getNamespace(); - var versionUrl = UrlUtil.createApiUrl(serverUrl, "api", namespace.getName(), extension.getName(), extVersion.getVersion()); + var baseUrl = UrlUtil.createApiUrl(serverUrl, "api", namespace.getName(), extension.getName(), extVersion.getVersion(), "file"); var resources = repositories.findFilesByType(extVersion, Arrays.asList(WEB_RESOURCE)); if (resources != null) { for (var resource : resources) { - var fileUrl = UrlUtil.createApiUrl(versionUrl, "file", resource.getName()); + var fileUrl = UrlUtil.createApiUrl(baseUrl, resource.getName().split("/")); name2Url.put(resource.getName(), fileUrl); } } diff --git a/server/src/main/java/org/eclipse/openvsx/util/UrlUtil.java b/server/src/main/java/org/eclipse/openvsx/util/UrlUtil.java index 2f5171b8e..84ae06ed1 100644 --- a/server/src/main/java/org/eclipse/openvsx/util/UrlUtil.java +++ b/server/src/main/java/org/eclipse/openvsx/util/UrlUtil.java @@ -9,9 +9,8 @@ ********************************************************************************/ package org.eclipse.openvsx.util; -import java.io.UnsupportedEncodingException; -import java.net.URLEncoder; import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import javax.servlet.http.HttpServletRequest; @@ -19,6 +18,7 @@ import org.springframework.web.context.request.RequestContextHolder; import org.springframework.web.context.request.ServletRequestAttributes; import org.springframework.web.servlet.HandlerMapping; +import org.springframework.web.util.UriUtils; public final class UrlUtil { @@ -33,7 +33,6 @@ public static String createApiUrl(String baseUrl, String... segments) { for (var segment : segments) { initialCapacity += segment.length() + 1; } - var charset = Charset.forName("UTF-8"); var result = new StringBuilder(initialCapacity); result.append(baseUrl); for (var segment : segments) { @@ -43,7 +42,7 @@ public static String createApiUrl(String baseUrl, String... segments) { continue; if (result.length() == 0 || result.charAt(result.length() - 1) != '/') result.append('/'); - result.append(URLEncoder.encode(segment, charset)); + result.append(UriUtils.encodePathSegment(segment, StandardCharsets.UTF_8)); } return result.toString(); } @@ -55,27 +54,24 @@ public static String createApiUrl(String baseUrl, String... segments) { public static String addQuery(String url, String... parameters) { if (parameters.length % 2 != 0) throw new IllegalArgumentException("Expected an even number of parameters."); - try { - var result = new StringBuilder(url); - var printedParams = 0; - for (var i = 0; i < parameters.length; i += 2) { - var key = parameters[i]; - var value = parameters[i + 1]; - if (key == null) - throw new NullPointerException("Parameter key must not be null"); - if (value != null) { - if (printedParams == 0) - result.append('?'); - else - result.append('&'); - result.append(key).append('=').append(URLEncoder.encode(value, "UTF-8")); - printedParams++; - } + + var result = new StringBuilder(url); + var printedParams = 0; + for (var i = 0; i < parameters.length; i += 2) { + var key = parameters[i]; + var value = parameters[i + 1]; + if (key == null) + throw new NullPointerException("Parameter key must not be null"); + if (value != null) { + if (printedParams == 0) + result.append('?'); + else + result.append('&'); + result.append(key).append('=').append(UriUtils.encodeQueryParam(value, StandardCharsets.UTF_8)); + printedParams++; } - return result.toString(); - } catch (UnsupportedEncodingException exc) { - throw new RuntimeException(exc); } + return result.toString(); } /** diff --git a/server/src/test/java/org/eclipse/openvsx/storage/AzureBlobStorageServiceTest.java b/server/src/test/java/org/eclipse/openvsx/storage/AzureBlobStorageServiceTest.java new file mode 100644 index 000000000..12990488b --- /dev/null +++ b/server/src/test/java/org/eclipse/openvsx/storage/AzureBlobStorageServiceTest.java @@ -0,0 +1,68 @@ +/** ****************************************************************************** + * Copyright (c) 2022 Precies. Software and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + * ****************************************************************************** */ +package org.eclipse.openvsx.storage; + +import org.eclipse.openvsx.entities.Extension; +import org.eclipse.openvsx.entities.ExtensionVersion; +import org.eclipse.openvsx.entities.FileResource; +import org.eclipse.openvsx.entities.Namespace; +import org.eclipse.openvsx.search.DatabaseSearchService; +import org.eclipse.openvsx.search.RelevanceService; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.context.annotation.Bean; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +import java.net.URI; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +@ExtendWith(SpringExtension.class) +@MockBean({ StorageUtilService.class }) +public class AzureBlobStorageServiceTest { + + @Autowired + AzureBlobStorageService service; + + @Test + public void testGetLocation() { + service.serviceEndpoint = "http://azure.blob.storage/"; + service.blobContainer = "blob-container"; + var namespace = new Namespace(); + namespace.setName("abelfubu"); + + var extension = new Extension(); + extension.setName("abelfubu-dark"); + extension.setNamespace(namespace); + + var extVersion = new ExtensionVersion(); + extVersion.setVersion("1.3.4"); + extVersion.setExtension(extension); + + var resource = new FileResource(); + resource.setName("extension/themes/abelFubu Dark+-color-theme.json"); + resource.setExtension(extVersion); + + var uri = service.getLocation(resource); + var expected = URI.create("http://azure.blob.storage/blob-container/abelfubu/abelfubu-dark/1.3.4/extension/themes/abelFubu%20Dark+-color-theme.json"); + assertEquals(expected, uri); + } + + @TestConfiguration + static class TestConfig { + @Bean + AzureBlobStorageService azureBlobStorageService() { + return new AzureBlobStorageService(); + } + } +} diff --git a/server/src/test/java/org/eclipse/openvsx/util/UriUtilTest.java b/server/src/test/java/org/eclipse/openvsx/util/UriUtilTest.java index c67e913b9..d8d7125c7 100644 --- a/server/src/test/java/org/eclipse/openvsx/util/UriUtilTest.java +++ b/server/src/test/java/org/eclipse/openvsx/util/UriUtilTest.java @@ -48,7 +48,7 @@ public void testApiUrl() throws Exception { public void testQuery() throws Exception { var url = "http://localhost/api/foo"; assertThat(UrlUtil.addQuery(url, "a", "1", "b", null, "c", "b\ta/r")) - .isEqualTo("http://localhost/api/foo?a=1&c=b%09a%2Fr"); + .isEqualTo("http://localhost/api/foo?a=1&c=b%09a/r"); } // Check base URL is localhost:8080 if there is no XForwarded headers