diff --git a/server/src/main/java/org/eclipse/openvsx/ExtensionService.java b/server/src/main/java/org/eclipse/openvsx/ExtensionService.java index 1caa5d052..a9aed3364 100644 --- a/server/src/main/java/org/eclipse/openvsx/ExtensionService.java +++ b/server/src/main/java/org/eclipse/openvsx/ExtensionService.java @@ -64,6 +64,7 @@ public ExtensionVersion publishVersion(InputStream content, PersonalAccessToken var extensionFile = createExtensionFile(content); var download = doPublish(extensionFile, null, token, TimeUtil.getCurrentUTC(), true); publishHandler.publishAsync(download, extensionFile, this); + publishHandler.schedulePublicIdJob(download); return download.getExtension(); } diff --git a/server/src/main/java/org/eclipse/openvsx/UrlConfigService.java b/server/src/main/java/org/eclipse/openvsx/UrlConfigService.java index 060bf471d..58689dc83 100644 --- a/server/src/main/java/org/eclipse/openvsx/UrlConfigService.java +++ b/server/src/main/java/org/eclipse/openvsx/UrlConfigService.java @@ -31,7 +31,7 @@ public String getUpstreamUrl() { return upstreamUrl; } - public String getUpstreamGalleryUrl() { + public String getUpstreamGalleryUrl() { return upstreamGalleryUrl; } diff --git a/server/src/main/java/org/eclipse/openvsx/adapter/LocalVSCodeService.java b/server/src/main/java/org/eclipse/openvsx/adapter/LocalVSCodeService.java index f69905f4c..a53f8538e 100644 --- a/server/src/main/java/org/eclipse/openvsx/adapter/LocalVSCodeService.java +++ b/server/src/main/java/org/eclipse/openvsx/adapter/LocalVSCodeService.java @@ -45,8 +45,6 @@ @Component public class LocalVSCodeService implements IVSCodeService { - private static final String BUILT_IN_EXTENSION_NAMESPACE = "vscode"; - @Autowired RepositoryService repositories; @@ -106,12 +104,12 @@ public ExtensionQueryResult extensionQuery(ExtensionQueryParam param, int defaul Long totalCount = null; List extensionsList; if (!extensionIds.isEmpty()) { - extensionsList = repositories.findActiveExtensionsByPublicId(extensionIds, BUILT_IN_EXTENSION_NAMESPACE); + extensionsList = repositories.findActiveExtensionsByPublicId(extensionIds, BuiltInExtensionUtil.getBuiltInNamespace()); } else if (!extensionNames.isEmpty()) { extensionsList = extensionNames.stream() .map(name -> name.split("\\.")) .filter(split -> split.length == 2) - .filter(split -> !isBuiltInExtensionNamespace(split[0])) + .filter(split -> !BuiltInExtensionUtil.isBuiltIn(split[0])) .map(split -> { var name = split[1]; var namespaceName = split[0]; @@ -125,7 +123,7 @@ public ExtensionQueryResult extensionQuery(ExtensionQueryParam param, int defaul try { var pageOffset = pageNumber * pageSize; var searchOptions = new SearchUtilService.Options(queryString, category, targetPlatform, pageSize, - pageOffset, sortOrder, sortBy, false, BUILT_IN_EXTENSION_NAMESPACE); + pageOffset, sortOrder, sortBy, false, BuiltInExtensionUtil.getBuiltInNamespace()); var searchResult = search.search(searchOptions); totalCount = searchResult.getTotalHits(); @@ -280,7 +278,7 @@ public ResponseEntity getAsset( String namespace, String extensionName, String version, String assetType, String targetPlatform, String restOfTheUrl ) { - if(isBuiltInExtensionNamespace(namespace)) { + if(BuiltInExtensionUtil.isBuiltIn(namespace)) { return new ResponseEntity<>(("Built-in extension namespace '" + namespace + "' not allowed").getBytes(StandardCharsets.UTF_8), null, HttpStatus.BAD_REQUEST); } @@ -344,7 +342,7 @@ private FileResource getFileFromDB(ExtensionVersion extVersion, String assetType @Override public String getItemUrl(String namespaceName, String extensionName) { - if(isBuiltInExtensionNamespace(namespaceName)) { + if(BuiltInExtensionUtil.isBuiltIn(namespaceName)) { throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "Built-in extension namespace '" + namespaceName + "' not allowed"); } @@ -358,7 +356,7 @@ public String getItemUrl(String namespaceName, String extensionName) { @Override public String download(String namespace, String extension, String version, String targetPlatform) { - if(isBuiltInExtensionNamespace(namespace)) { + if(BuiltInExtensionUtil.isBuiltIn(namespace)) { throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "Built-in extension namespace '" + namespace + "' not allowed"); } @@ -387,7 +385,7 @@ public String download(String namespace, String extension, String version, Strin @Override public ResponseEntity browse(String namespaceName, String extensionName, String version, String path) { - if(isBuiltInExtensionNamespace(namespaceName)) { + if(BuiltInExtensionUtil.isBuiltIn(namespaceName)) { return new ResponseEntity<>(("Built-in extension namespace '" + namespaceName + "' not allowed").getBytes(StandardCharsets.UTF_8), null, HttpStatus.BAD_REQUEST); } @@ -619,8 +617,4 @@ private boolean isWebResource(FileResource resource) { private boolean test(int flags, int flag) { return (flags & flag) != 0; } - - private boolean isBuiltInExtensionNamespace(String namespaceName) { - return namespaceName.equals(BUILT_IN_EXTENSION_NAMESPACE); - } } diff --git a/server/src/main/java/org/eclipse/openvsx/adapter/VSCodeIdDailyUpdateJobRequestHandler.java b/server/src/main/java/org/eclipse/openvsx/adapter/VSCodeIdDailyUpdateJobRequestHandler.java new file mode 100644 index 000000000..00763ef0d --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/adapter/VSCodeIdDailyUpdateJobRequestHandler.java @@ -0,0 +1,29 @@ +/** ****************************************************************************** + * Copyright (c) 2023 Precies. Software Ltd and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + * ****************************************************************************** */ +package org.eclipse.openvsx.adapter; + +import org.jobrunr.jobs.lambdas.JobRequest; +import org.jobrunr.jobs.lambdas.JobRequestHandler; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +@Component +@ConditionalOnProperty(value = "ovsx.data.mirror.enabled", havingValue = "false", matchIfMissing = true) +public class VSCodeIdDailyUpdateJobRequestHandler implements JobRequestHandler { + + @Autowired + VSCodeIdUpdateService service; + + @Override + public void run(JobRequest request) throws Exception { + service.updateAll(); + } +} diff --git a/server/src/main/java/org/eclipse/openvsx/adapter/VSCodeIdNewExtensionJobRequest.java b/server/src/main/java/org/eclipse/openvsx/adapter/VSCodeIdNewExtensionJobRequest.java new file mode 100644 index 000000000..17ede22e8 --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/adapter/VSCodeIdNewExtensionJobRequest.java @@ -0,0 +1,47 @@ +/** ****************************************************************************** + * Copyright (c) 2023 Precies. Software Ltd and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + * ****************************************************************************** */ +package org.eclipse.openvsx.adapter; + +import org.jobrunr.jobs.lambdas.JobRequest; +import org.jobrunr.jobs.lambdas.JobRequestHandler; + +public class VSCodeIdNewExtensionJobRequest implements JobRequest { + + private String namespace; + private String extension; + + public VSCodeIdNewExtensionJobRequest() {} + + public VSCodeIdNewExtensionJobRequest(String namespace, String extension) { + this.namespace = namespace; + this.extension = extension; + } + + @Override + public Class getJobRequestHandler() { + return VSCodeIdNewExtensionJobRequestHandler.class; + } + + public String getNamespace() { + return namespace; + } + + public void setNamespace(String namespace) { + this.namespace = namespace; + } + + public String getExtension() { + return extension; + } + + public void setExtension(String extension) { + this.extension = extension; + } +} diff --git a/server/src/main/java/org/eclipse/openvsx/adapter/VSCodeIdNewExtensionJobRequestHandler.java b/server/src/main/java/org/eclipse/openvsx/adapter/VSCodeIdNewExtensionJobRequestHandler.java new file mode 100644 index 000000000..03487e31a --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/adapter/VSCodeIdNewExtensionJobRequestHandler.java @@ -0,0 +1,28 @@ +/** ****************************************************************************** + * Copyright (c) 2023 Precies. Software Ltd and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + * ****************************************************************************** */ +package org.eclipse.openvsx.adapter; + +import org.jobrunr.jobs.lambdas.JobRequestHandler; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Component +public class VSCodeIdNewExtensionJobRequestHandler implements JobRequestHandler { + + @Autowired + VSCodeIdUpdateService service; + + @Override + public void run(VSCodeIdNewExtensionJobRequest jobRequest) throws Exception { + var namespaceName = jobRequest.getNamespace(); + var extensionName = jobRequest.getExtension(); + service.update(namespaceName, extensionName); + } +} diff --git a/server/src/main/java/org/eclipse/openvsx/adapter/VSCodeIdService.java b/server/src/main/java/org/eclipse/openvsx/adapter/VSCodeIdService.java index e69e541da..1805cbd56 100644 --- a/server/src/main/java/org/eclipse/openvsx/adapter/VSCodeIdService.java +++ b/server/src/main/java/org/eclipse/openvsx/adapter/VSCodeIdService.java @@ -13,19 +13,24 @@ import org.apache.commons.lang3.StringUtils; import org.eclipse.openvsx.UrlConfigService; import org.eclipse.openvsx.entities.Extension; -import org.eclipse.openvsx.repositories.RepositoryService; +import org.eclipse.openvsx.migration.HandlerJobRequest; import org.eclipse.openvsx.util.NamingUtil; import org.eclipse.openvsx.util.UrlUtil; +import org.jobrunr.scheduling.JobRequestScheduler; +import org.jobrunr.scheduling.cron.Cron; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.event.ApplicationStartedEvent; +import org.springframework.context.event.EventListener; import org.springframework.http.HttpEntity; import org.springframework.http.HttpHeaders; import org.springframework.http.MediaType; import org.springframework.stereotype.Component; import org.springframework.web.client.RestTemplate; -import java.util.UUID; +import java.time.ZoneId; @Component public class VSCodeIdService { @@ -38,35 +43,41 @@ public class VSCodeIdService { RestTemplate vsCodeIdRestTemplate; @Autowired - RepositoryService repositories; + UrlConfigService urlConfigService; @Autowired - UrlConfigService urlConfigService; + JobRequestScheduler scheduler; + + @Value("${ovsx.data.mirror.enabled:false}") + boolean mirrorEnabled; + + @Value("${ovsx.vscode.upstream.update-on-start:false}") + boolean updateOnStart; + + @EventListener + public void applicationStarted(ApplicationStartedEvent event) { + if(mirrorEnabled) { + return; + } + if(updateOnStart) { + scheduler.enqueue(new HandlerJobRequest<>(VSCodeIdDailyUpdateJobRequestHandler.class)); + } - public boolean setPublicIds(Extension extension) { - var updateExistingPublicIds = false; + scheduler.scheduleRecurrently("VSCodeIdDailyUpdate", Cron.daily(3), ZoneId.of("UTC"), new HandlerJobRequest<>(VSCodeIdDailyUpdateJobRequestHandler.class)); + } + + public void getUpstreamPublicIds(Extension extension) { + extension.setPublicId(null); + extension.getNamespace().setPublicId(null); var upstream = getUpstreamExtension(extension); if (upstream != null) { if (upstream.extensionId != null) { extension.setPublicId(upstream.extensionId); - updateExistingPublicIds = true; } if (upstream.publisher != null && upstream.publisher.publisherId != null) { extension.getNamespace().setPublicId(upstream.publisher.publisherId); } } - if (extension.getPublicId() == null) { - extension.setPublicId(createRandomId()); - } - if (extension.getNamespace().getPublicId() == null) { - extension.getNamespace().setPublicId(createRandomId()); - } - - return updateExistingPublicIds; - } - - private String createRandomId() { - return UUID.randomUUID().toString(); } private ExtensionQueryResult.Extension getUpstreamExtension(Extension extension) { diff --git a/server/src/main/java/org/eclipse/openvsx/adapter/VSCodeIdUpdateService.java b/server/src/main/java/org/eclipse/openvsx/adapter/VSCodeIdUpdateService.java new file mode 100644 index 000000000..04433d16f --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/adapter/VSCodeIdUpdateService.java @@ -0,0 +1,209 @@ +/** ****************************************************************************** + * Copyright (c) 2023 Precies. Software Ltd and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + * ****************************************************************************** */ +package org.eclipse.openvsx.adapter; + +import org.apache.commons.lang3.StringUtils; +import org.eclipse.openvsx.entities.Extension; +import org.eclipse.openvsx.repositories.RepositoryService; +import org.eclipse.openvsx.util.BuiltInExtensionUtil; +import org.eclipse.openvsx.util.NamingUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.util.*; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +@Component +public class VSCodeIdUpdateService { + private static final Logger LOGGER = LoggerFactory.getLogger(VSCodeIdUpdateService.class); + private static final Semaphore LOCK = new Semaphore(1); + + @Autowired + RepositoryService repositories; + + @Autowired + VSCodeIdService service; + + public void update(String namespaceName, String extensionName) throws InterruptedException { + var acquired = LOCK.tryAcquire(15, TimeUnit.SECONDS); + if(!acquired) { + throw new RuntimeException("Failed to update public id for " + NamingUtil.toExtensionId(namespaceName, extensionName)); + } + if(BuiltInExtensionUtil.isBuiltIn(namespaceName)) { + LOGGER.debug("SKIP BUILT-IN EXTENSION {}", NamingUtil.toExtensionId(namespaceName, extensionName)); + return; + } + + var extension = repositories.findPublicId(namespaceName, extensionName); + var extensionUpdates = new HashMap(); + updateExtensionPublicId(extension, extensionUpdates); + if(!extensionUpdates.isEmpty()) { + repositories.updateExtensionPublicIds(extensionUpdates); + } + + var namespaceUpdates = new HashMap(); + updateNamespacePublicId(extension, namespaceUpdates); + if(!namespaceUpdates.isEmpty()) { + repositories.updateNamespacePublicIds(namespaceUpdates); + } + LOCK.release(); + } + + private void updateExtensionPublicId(Extension extension, Map updates) { + LOGGER.debug("updateExtensionPublicId: {}", NamingUtil.toExtensionId(extension)); + service.getUpstreamPublicIds(extension); + if(extension.getPublicId() == null) { + var publicId = ""; + do { + publicId = UUID.randomUUID().toString(); + LOGGER.debug("RANDOM EXTENSION PUBLIC ID: {}", publicId); + } while(updates.containsValue(publicId) || repositories.extensionPublicIdExists(publicId)); + LOGGER.debug("RANDOM PUT UPDATE: {} - {}", extension.getId(), publicId); + updates.put(extension.getId(), publicId); + } else { + LOGGER.debug("UPSTREAM PUT UPDATE: {} - {}", extension.getId(), extension.getPublicId()); + updates.put(extension.getId(), extension.getPublicId()); + var duplicatePublicId = repositories.findPublicId(extension.getPublicId()); + if(duplicatePublicId != null) { + updateExtensionPublicId(duplicatePublicId, updates); + } + } + } + + private void updateNamespacePublicId(Extension extension, Map updates) { + LOGGER.debug("updateNamespacePublicId: {}", extension.getNamespace().getName()); + service.getUpstreamPublicIds(extension); + var namespace = extension.getNamespace(); + if(namespace.getPublicId() == null) { + var publicId = ""; + do { + publicId = UUID.randomUUID().toString(); + LOGGER.debug("RANDOM NAMESPACE PUBLIC ID: {}", publicId); + } while(updates.containsValue(publicId) || repositories.namespacePublicIdExists(publicId)); + LOGGER.debug("RANDOM PUT UPDATE: {} - {}", namespace.getId(), publicId); + updates.put(namespace.getId(), publicId); + } else { + LOGGER.debug("UPSTREAM PUT UPDATE: {} - {}", namespace.getId(), namespace.getPublicId()); + updates.put(namespace.getId(), namespace.getPublicId()); + var duplicatePublicId = repositories.findNamespacePublicId(namespace.getPublicId()); + if(duplicatePublicId != null) { + updateNamespacePublicId(duplicatePublicId, updates); + } + } + } + + public void updateAll() throws InterruptedException { + LOCK.acquire(); + LOGGER.debug("DAILY UPDATE ALL"); + var extensions = repositories.findAllPublicIds(); + var extensionPublicIdsMap = extensions.stream() + .filter(e -> StringUtils.isNotEmpty(e.getPublicId())) + .collect(Collectors.toMap(e -> e.getId(), e -> e.getPublicId())); + var namespacePublicIdsMap = extensions.stream() + .map(e -> e.getNamespace()) + .filter(n -> StringUtils.isNotEmpty(n.getPublicId())) + .collect(Collectors.toMap(n -> n.getId(), n -> n.getPublicId(), (id1, id2) -> id1)); + + var upstreamExtensionPublicIds = new HashMap(); + var upstreamNamespacePublicIds = new HashMap(); + for(var extension : extensions) { + if(BuiltInExtensionUtil.isBuiltIn(extension)) { + LOGGER.trace("SKIP BUILT-IN EXTENSION {}", NamingUtil.toExtensionId(extension)); + continue; + } + + LOGGER.trace("GET UPSTREAM PUBLIC ID: {} | {}", extension.getId(), NamingUtil.toExtensionId(extension)); + service.getUpstreamPublicIds(extension); + if(upstreamExtensionPublicIds.get(extension.getId()) == null) { + LOGGER.trace("ADD EXTENSION PUBLIC ID: {} - {}", extension.getId(), extension.getPublicId()); + upstreamExtensionPublicIds.put(extension.getId(), extension.getPublicId()); + } + + var namespace = extension.getNamespace(); + if(upstreamNamespacePublicIds.get(namespace.getId()) == null) { + LOGGER.trace("ADD NAMESPACE PUBLIC ID: {} - {}", namespace.getId(), namespace.getPublicId()); + upstreamNamespacePublicIds.put(namespace.getId(), namespace.getPublicId()); + } + } + + var changedExtensionPublicIds = getChangedPublicIds(upstreamExtensionPublicIds, extensionPublicIdsMap); + LOGGER.debug("UPSTREAM EXTENSIONS: {}", upstreamExtensionPublicIds.size()); + LOGGER.debug("CHANGED EXTENSIONS: {}", changedExtensionPublicIds.size()); + if(!changedExtensionPublicIds.isEmpty()) { + LOGGER.debug("CHANGED EXTENSION PUBLIC IDS"); + for(var entry : changedExtensionPublicIds.entrySet()) { + LOGGER.debug("{}: {}", entry.getKey(), entry.getValue()); + } + + repositories.updateExtensionPublicIds(changedExtensionPublicIds); + } + + var changedNamespacePublicIds = getChangedPublicIds(upstreamNamespacePublicIds, namespacePublicIdsMap); + LOGGER.debug("UPSTREAM NAMESPACES: {}", upstreamNamespacePublicIds.size()); + LOGGER.debug("CHANGED NAMESPACES: {}", changedNamespacePublicIds.size()); + if(!changedNamespacePublicIds.isEmpty()) { + LOGGER.debug("CHANGED NAMESPACE PUBLIC IDS"); + for(var entry : changedNamespacePublicIds.entrySet()) { + LOGGER.debug("{}: {}", entry.getKey(), entry.getValue()); + } + + repositories.updateNamespacePublicIds(changedNamespacePublicIds); + } + + LOCK.release(); + } + + private Map getChangedPublicIds(Map upstreamPublicIds, Map currentPublicIds) { + var changedPublicIds = new HashMap(); + upstreamPublicIds.entrySet().stream() + .filter(e -> !Objects.equals(currentPublicIds.get(e.getKey()), e.getValue())) + .forEach(e -> changedPublicIds.put(e.getKey(), e.getValue())); + + if(!changedPublicIds.isEmpty()) { + var newPublicIds = new HashSet<>(upstreamPublicIds.values()); + updatePublicIdNulls(changedPublicIds, newPublicIds, currentPublicIds); + } + + return changedPublicIds; + } + + private void updatePublicIdNulls(Map changedPublicIds, Set newPublicIds, Map publicIdMap) { + // remove unchanged random public ids + changedPublicIds.entrySet().removeIf((e) -> { + var publicId = e.getValue() == null ? publicIdMap.get(e.getKey()) : null; + var remove = publicId != null && !newPublicIds.contains(publicId); + if(remove) { + newPublicIds.add(publicId); + } + + return remove; + }); + + // put random UUIDs where upstream public id is missing + for(var key : changedPublicIds.keySet()) { + if(changedPublicIds.get(key) != null) { + continue; + } + + String publicId = null; + while(newPublicIds.contains(publicId)) { + publicId = UUID.randomUUID().toString(); + LOGGER.debug("NEW PUBLIC ID - {}: '{}'", key, publicId); + } + + changedPublicIds.put(key, publicId); + newPublicIds.add(publicId); + } + } +} diff --git a/server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionHandler.java b/server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionHandler.java index 41c2a8dcc..5ad652fc2 100644 --- a/server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionHandler.java +++ b/server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionHandler.java @@ -10,16 +10,18 @@ package org.eclipse.openvsx.publish; import com.google.common.base.Joiner; +import org.apache.commons.lang3.StringUtils; import org.eclipse.openvsx.ExtensionProcessor; import org.eclipse.openvsx.ExtensionService; import org.eclipse.openvsx.ExtensionValidator; import org.eclipse.openvsx.UserService; -import org.eclipse.openvsx.adapter.VSCodeIdService; +import org.eclipse.openvsx.adapter.VSCodeIdNewExtensionJobRequest; import org.eclipse.openvsx.entities.*; import org.eclipse.openvsx.repositories.RepositoryService; import org.eclipse.openvsx.util.ErrorResultException; import org.eclipse.openvsx.util.NamingUtil; import org.eclipse.openvsx.util.TempFile; +import org.jobrunr.scheduling.JobRequestScheduler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -31,9 +33,6 @@ import jakarta.transaction.Transactional; import java.io.IOException; import java.time.LocalDateTime; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -55,7 +54,7 @@ public class PublishExtensionVersionHandler { RepositoryService repositories; @Autowired - VSCodeIdService vsCodeIdService; + JobRequestScheduler scheduler; @Autowired UserService users; @@ -125,11 +124,6 @@ private ExtensionVersion createExtensionVersion(ExtensionProcessor processor, Us extension.setNamespace(namespace); extension.setPublishedDate(extVersion.getTimestamp()); - var updateExistingPublicIds = vsCodeIdService.setPublicIds(extension); - if(updateExistingPublicIds) { - updateExistingPublicIds(extension).forEach(service::updateExtensionPublicId); - } - entityManager.persist(extension); } else { var existingVersion = repositories.findVersion(extVersion.getVersion(), extVersion.getTargetPlatform(), extension); @@ -180,27 +174,6 @@ private String checkBundledExtension(String bundledExtension) { return bundledExtension; } - private List updateExistingPublicIds(Extension extension) { - var updated = true; - var updatedExtensions = new ArrayList(); - var newExtension = extension; - while(updated) { - updated = false; - var oldExtension = repositories.findExtensionByPublicId(newExtension.getPublicId()); - if (oldExtension != null && !oldExtension.equals(newExtension)) { - entityManager.detach(oldExtension); - updated = vsCodeIdService.setPublicIds(oldExtension); - } - if(updated) { - updatedExtensions.add(oldExtension); - newExtension = oldExtension; - } - } - - Collections.reverse(updatedExtensions); - return updatedExtensions; - } - @Async @Retryable public void publishAsync(FileResource download, TempFile extensionFile, ExtensionService extensionService) { @@ -265,4 +238,12 @@ private FileResource getSignatureResource(String signatureName, ExtensionVersion resource.setType(FileResource.DOWNLOAD_SIG); return resource; } + + public void schedulePublicIdJob(FileResource download) { + var extension = download.getExtension().getExtension(); + if(StringUtils.isEmpty(extension.getPublicId())) { + var namespace = extension.getNamespace(); + scheduler.enqueue(new VSCodeIdNewExtensionJobRequest(namespace.getName(), extension.getName())); + } + } } diff --git a/server/src/main/java/org/eclipse/openvsx/repositories/ExtensionJooqRepository.java b/server/src/main/java/org/eclipse/openvsx/repositories/ExtensionJooqRepository.java index 80ac9283d..b7fb65d74 100644 --- a/server/src/main/java/org/eclipse/openvsx/repositories/ExtensionJooqRepository.java +++ b/server/src/main/java/org/eclipse/openvsx/repositories/ExtensionJooqRepository.java @@ -11,10 +11,8 @@ import org.eclipse.openvsx.entities.Extension; import org.eclipse.openvsx.entities.Namespace; -import org.jooq.Condition; -import org.jooq.DSLContext; +import org.jooq.*; import org.jooq.Record; -import org.jooq.SelectQuery; import org.jooq.impl.DSL; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -22,6 +20,8 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; import static org.eclipse.openvsx.jooq.Tables.*; @@ -60,7 +60,69 @@ var record = query.fetchOne(); return record != null ? toExtension(record) : null; } - private SelectQuery findAllActive() { + public List findAllPublicIds() { + return findPublicId().fetch().map(this::toPublicId); + } + + public Extension findPublicId(String namespace, String extension) { + var query = findPublicId(); + query.addConditions( + DSL.upper(EXTENSION.NAME).eq(DSL.upper(extension)), + DSL.upper(NAMESPACE.NAME).eq(DSL.upper(namespace)) + ); + + var record = query.fetchOne(); + return record != null ? toPublicId(record) : null; + } + + public Extension findPublicId(String publicId) { + var query = findPublicId(); + query.addConditions(EXTENSION.PUBLIC_ID.eq(publicId)); + + var record = query.fetchOne(); + return record != null ? toPublicId(record) : null; + } + + public Extension findNamespacePublicId(String publicId) { + var query = findPublicId(); + query.addConditions(NAMESPACE.PUBLIC_ID.eq(publicId)); + query.addLimit(1); + + var record = query.fetchOne(); + return record != null ? toPublicId(record) : null; + } + + private SelectQuery findPublicId() { + var query = dsl.selectQuery(); + query.addSelect( + EXTENSION.ID, + EXTENSION.PUBLIC_ID, + EXTENSION.NAME, + NAMESPACE.ID, + NAMESPACE.PUBLIC_ID, + NAMESPACE.NAME + ); + + query.addFrom(EXTENSION); + query.addJoin(NAMESPACE, NAMESPACE.ID.eq(EXTENSION.NAMESPACE_ID)); + return query; + } + + private Extension toPublicId(Record record) { + var namespace = new Namespace(); + namespace.setId(record.get(NAMESPACE.ID)); + namespace.setPublicId(record.get(NAMESPACE.PUBLIC_ID)); + namespace.setName(record.get(NAMESPACE.NAME)); + + var extension = new Extension(); + extension.setId(record.get(EXTENSION.ID)); + extension.setPublicId(record.get(EXTENSION.PUBLIC_ID)); + extension.setName(record.get(EXTENSION.NAME)); + extension.setNamespace(namespace); + return extension; + } + + private SelectQuery findAllActive() { var query = dsl.selectQuery(); query.addSelect( EXTENSION.ID, @@ -105,4 +167,37 @@ private Extension toExtension(Record record) { return extension; } + + public void updatePublicId(long id, String publicId) { + dsl.update(EXTENSION) + .set(EXTENSION.PUBLIC_ID, publicId) + .where(EXTENSION.ID.eq(id)) + .execute(); + } + + public void updatePublicIds(Map publicIds) { + if(publicIds.isEmpty()) { + return; + } + + var extension = EXTENSION.as("e"); + var rows = publicIds.entrySet().stream() + .map(e -> DSL.row(e.getKey(), e.getValue())) + .collect(Collectors.toList()); + + var updates = DSL.values(rows.toArray(Row2[]::new)).as("u", "id", "public_id"); + dsl.update(extension) + .set(extension.PUBLIC_ID, updates.field("public_id", String.class)) + .from(updates) + .where(updates.field("id", Long.class).eq(extension.ID)) + .execute(); + } + + public boolean publicIdExists(String publicId) { + return dsl.selectOne() + .from(EXTENSION) + .where(EXTENSION.PUBLIC_ID.eq(publicId)) + .fetch() + .isNotEmpty(); + } } diff --git a/server/src/main/java/org/eclipse/openvsx/repositories/NamespaceJooqRepository.java b/server/src/main/java/org/eclipse/openvsx/repositories/NamespaceJooqRepository.java new file mode 100644 index 000000000..4996c8895 --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/repositories/NamespaceJooqRepository.java @@ -0,0 +1,55 @@ +/** ****************************************************************************** + * Copyright (c) 2023 Precies. Software Ltd and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + * ****************************************************************************** */ +package org.eclipse.openvsx.repositories; + +import org.jooq.DSLContext; +import org.jooq.Row2; +import org.jooq.impl.DSL; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.util.Map; +import java.util.stream.Collectors; + +import static org.eclipse.openvsx.jooq.Tables.EXTENSION; +import static org.eclipse.openvsx.jooq.Tables.NAMESPACE; + +@Component +public class NamespaceJooqRepository { + + @Autowired + DSLContext dsl; + + public void updatePublicIds(Map publicIds) { + if(publicIds.isEmpty()) { + return; + } + + var namespace = NAMESPACE.as("n"); + var rows = publicIds.entrySet().stream() + .map(e -> DSL.row(e.getKey(), e.getValue())) + .collect(Collectors.toList()); + + var updates = DSL.values(rows.toArray(Row2[]::new)).as("u", "id", "public_id"); + dsl.update(namespace) + .set(namespace.PUBLIC_ID, updates.field("public_id", String.class)) + .from(updates) + .where(updates.field("id", Long.class).eq(namespace.ID)) + .execute(); + } + + public boolean publicIdExists(String publicId) { + return dsl.selectOne() + .from(NAMESPACE) + .where(NAMESPACE.PUBLIC_ID.eq(publicId)) + .fetch() + .isNotEmpty(); + } +} diff --git a/server/src/main/java/org/eclipse/openvsx/repositories/RepositoryService.java b/server/src/main/java/org/eclipse/openvsx/repositories/RepositoryService.java index a9332eaf5..c7c5550f8 100644 --- a/server/src/main/java/org/eclipse/openvsx/repositories/RepositoryService.java +++ b/server/src/main/java/org/eclipse/openvsx/repositories/RepositoryService.java @@ -38,6 +38,7 @@ public class RepositoryService { .and(Sort.by(Sort.Direction.DESC, "timestamp")); @Autowired NamespaceRepository namespaceRepo; + @Autowired NamespaceJooqRepository namespaceJooqRepo; @Autowired ExtensionRepository extensionRepo; @Autowired ExtensionVersionRepository extensionVersionRepo; @Autowired FileResourceRepository fileResourceRepo; @@ -484,4 +485,40 @@ public void deleteAllKeyPairs() { public SignatureKeyPair findKeyPair(String publicId) { return signatureKeyPairRepo.findByPublicId(publicId); } + + public List findAllPublicIds() { + return extensionJooqRepo.findAllPublicIds(); + } + + public Extension findPublicId(String namespace, String extension) { + return extensionJooqRepo.findPublicId(namespace, extension); + } + + public Extension findPublicId(String publicId) { + return extensionJooqRepo.findPublicId(publicId); + } + + public Extension findNamespacePublicId(String publicId) { + return extensionJooqRepo.findNamespacePublicId(publicId); + } + + public void updateExtensionPublicIds(Map publicIds) { + extensionJooqRepo.updatePublicIds(publicIds); + } + + public void updateExtensionPublicId(long id, String publicId) { + extensionJooqRepo.updatePublicId(id, publicId); + } + + public void updateNamespacePublicIds(Map publicIds) { + namespaceJooqRepo.updatePublicIds(publicIds); + } + + public boolean extensionPublicIdExists(String publicId) { + return extensionJooqRepo.publicIdExists(publicId); + } + + public boolean namespacePublicIdExists(String publicId) { + return namespaceJooqRepo.publicIdExists(publicId); + } } \ No newline at end of file diff --git a/server/src/main/java/org/eclipse/openvsx/search/ElasticSearchService.java b/server/src/main/java/org/eclipse/openvsx/search/ElasticSearchService.java index cf916671a..e91711bff 100644 --- a/server/src/main/java/org/eclipse/openvsx/search/ElasticSearchService.java +++ b/server/src/main/java/org/eclipse/openvsx/search/ElasticSearchService.java @@ -22,6 +22,7 @@ import org.eclipse.openvsx.util.ErrorResultException; import org.eclipse.openvsx.util.TargetPlatform; import org.jobrunr.scheduling.JobRequestScheduler; +import org.jobrunr.scheduling.cron.Cron; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -96,7 +97,7 @@ public boolean isEnabled() { @Retryable(DataAccessResourceFailureException.class) @CacheEvict(value = CACHE_AVERAGE_REVIEW_RATING, allEntries = true) public void initSearchIndex(ApplicationStartedEvent event) { - scheduler.scheduleRecurrently("ElasticSearchUpdateIndex", "0 0 4 * * *", ZoneId.of("UTC"), new HandlerJobRequest<>(ElasticSearchUpdateIndexJobRequestHandler.class)); + scheduler.scheduleRecurrently("ElasticSearchUpdateIndex", Cron.daily(4), ZoneId.of("UTC"), new HandlerJobRequest<>(ElasticSearchUpdateIndexJobRequestHandler.class)); if (!isEnabled() || !clearOnStart && searchOperations.indexOps(ExtensionSearch.class).exists()) { return; } diff --git a/server/src/main/java/org/eclipse/openvsx/util/BuiltInExtensionUtil.java b/server/src/main/java/org/eclipse/openvsx/util/BuiltInExtensionUtil.java new file mode 100644 index 000000000..4d46665ea --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/util/BuiltInExtensionUtil.java @@ -0,0 +1,30 @@ +/** ****************************************************************************** + * Copyright (c) 2023 Precies. Software Ltd and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + * ****************************************************************************** */ +package org.eclipse.openvsx.util; + +import org.eclipse.openvsx.entities.Extension; + +public class BuiltInExtensionUtil { + private static final String BUILT_IN_EXTENSION_NAMESPACE = "vscode"; + + private BuiltInExtensionUtil() {} + + public static String getBuiltInNamespace() { + return BUILT_IN_EXTENSION_NAMESPACE; + } + + public static boolean isBuiltIn(String namespace) { + return BUILT_IN_EXTENSION_NAMESPACE.equals(namespace); + } + + public static boolean isBuiltIn(Extension extension) { + return BUILT_IN_EXTENSION_NAMESPACE.equals(extension.getNamespace().getName()); + } +} diff --git a/server/src/main/resources/db/migration/V1_43__Defer_Public_Id_Unique_Constraint.sql b/server/src/main/resources/db/migration/V1_43__Defer_Public_Id_Unique_Constraint.sql new file mode 100644 index 000000000..4d84bf78d --- /dev/null +++ b/server/src/main/resources/db/migration/V1_43__Defer_Public_Id_Unique_Constraint.sql @@ -0,0 +1,9 @@ +-- make unique constraints deferred, so that the unique constraint is evaluated at transaction commit time +-- instead of validating immediately row-by-row for each insert or update statement. +ALTER TABLE public.extension + DROP CONSTRAINT unique_extension_public_id, + ADD CONSTRAINT unique_extension_public_id UNIQUE (public_id) DEFERRABLE INITIALLY DEFERRED; + +ALTER TABLE public.namespace + DROP CONSTRAINT unique_namespace_public_id, + ADD CONSTRAINT unique_namespace_public_id UNIQUE (public_id) DEFERRABLE INITIALLY DEFERRED; \ No newline at end of file diff --git a/server/src/test/java/org/eclipse/openvsx/RegistryAPITest.java b/server/src/test/java/org/eclipse/openvsx/RegistryAPITest.java index a08ca92bb..33d2553e7 100644 --- a/server/src/test/java/org/eclipse/openvsx/RegistryAPITest.java +++ b/server/src/test/java/org/eclipse/openvsx/RegistryAPITest.java @@ -36,6 +36,7 @@ import org.eclipse.openvsx.storage.StorageUtilService; import org.eclipse.openvsx.util.TargetPlatform; import org.eclipse.openvsx.util.VersionService; +import org.jobrunr.scheduling.JobRequestScheduler; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; @@ -84,7 +85,7 @@ @MockBean({ ClientRegistrationRepository.class, UpstreamRegistryService.class, GoogleCloudStorageService.class, AzureBlobStorageService.class, VSCodeIdService.class, AzureDownloadCountService.class, CacheService.class, - EclipseService.class, PublishExtensionVersionService.class, SimpleMeterRegistry.class + EclipseService.class, PublishExtensionVersionService.class, SimpleMeterRegistry.class, JobRequestScheduler.class }) public class RegistryAPITest { diff --git a/server/src/test/java/org/eclipse/openvsx/adapter/VSCodeIdUpdateServiceTest.java b/server/src/test/java/org/eclipse/openvsx/adapter/VSCodeIdUpdateServiceTest.java new file mode 100644 index 000000000..9b42b9293 --- /dev/null +++ b/server/src/test/java/org/eclipse/openvsx/adapter/VSCodeIdUpdateServiceTest.java @@ -0,0 +1,662 @@ +/** ****************************************************************************** + * Copyright (c) 2023 Precies. Software Ltd and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + * ****************************************************************************** */ +package org.eclipse.openvsx.adapter; + +import org.eclipse.openvsx.entities.Extension; +import org.eclipse.openvsx.entities.Namespace; +import org.eclipse.openvsx.repositories.RepositoryService; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.context.annotation.Bean; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executors; + +import static org.junit.jupiter.api.Assertions.assertThrows; + +@ExtendWith(SpringExtension.class) +public class VSCodeIdUpdateServiceTest { + + @MockBean + RepositoryService repositories; + + @MockBean + VSCodeIdService idService; + + @Autowired + VSCodeIdUpdateService updateService; + + @Test + public void testUpdateAllNoChanges() throws InterruptedException { + var namespaceName1 = "foo"; + var namespacePublicId1 = UUID.randomUUID().toString(); + var extensionName1 = "bar"; + var extensionPublicId1 = UUID.randomUUID().toString(); + + var namespace1 = new Namespace(); + namespace1.setId(1L); + namespace1.setName(namespaceName1); + namespace1.setPublicId(namespacePublicId1); + + var extension1 = new Extension(); + extension1.setId(2L); + extension1.setName(extensionName1); + extension1.setNamespace(namespace1); + extension1.setPublicId(extensionPublicId1); + + var namespaceName2 = "baz"; + var namespacePublicId2 = UUID.randomUUID().toString(); + var extensionName2 = "foobar"; + var extensionPublicId2 = UUID.randomUUID().toString(); + + var namespace2 = new Namespace(); + namespace2.setId(3L); + namespace2.setName(namespaceName2); + namespace2.setPublicId(namespacePublicId2); + + var extension2 = new Extension(); + extension2.setId(4L); + extension2.setName(extensionName2); + extension2.setPublicId(extensionPublicId2); + extension2.setNamespace(namespace2); + + var namespaceName3 = "baz2"; + var namespacePublicId3 = UUID.randomUUID().toString(); + var extensionName3 = "foobar2"; + var extensionPublicId3 = UUID.randomUUID().toString(); + + var namespace3 = new Namespace(); + namespace3.setId(5L); + namespace3.setName(namespaceName3); + namespace3.setPublicId(namespacePublicId3); + + var extension3 = new Extension(); + extension3.setId(6L); + extension3.setName(extensionName3); + extension3.setPublicId(extensionPublicId3); + extension3.setNamespace(namespace3); + + Mockito.when(repositories.findAllPublicIds()).thenReturn(List.of(extension1, extension2, extension3)); + updateService.updateAll(); + Mockito.verify(repositories, Mockito.never()).updateExtensionPublicIds(Mockito.anyMap()); + Mockito.verify(repositories, Mockito.never()).updateNamespacePublicIds(Mockito.anyMap()); + } + + @Test + public void testUpdateAllRandomNoChanges() throws InterruptedException { + var namespaceName1 = "foo"; + var namespacePublicId1 = UUID.randomUUID().toString(); + var extensionName1 = "bar"; + var extensionPublicId1 = UUID.randomUUID().toString(); + + var namespace1 = new Namespace(); + namespace1.setId(1L); + namespace1.setName(namespaceName1); + namespace1.setPublicId(namespacePublicId1); + + var extension1 = new Extension(); + extension1.setId(2L); + extension1.setName(extensionName1); + extension1.setNamespace(namespace1); + extension1.setPublicId(extensionPublicId1); + + var namespaceName2 = "baz"; + var namespacePublicId2 = UUID.randomUUID().toString(); + var extensionName2 = "foobar"; + var extensionPublicId2 = UUID.randomUUID().toString(); + + var namespace2 = new Namespace(); + namespace2.setId(3L); + namespace2.setName(namespaceName2); + namespace2.setPublicId(namespacePublicId2); + + var extension2 = new Extension(); + extension2.setId(4L); + extension2.setName(extensionName2); + extension2.setPublicId(extensionPublicId2); + extension2.setNamespace(namespace2); + + var namespaceName3 = "baz2"; + var namespacePublicId3 = UUID.randomUUID().toString(); + var extensionName3 = "foobar2"; + var extensionPublicId3 = UUID.randomUUID().toString(); + + var namespace3 = new Namespace(); + namespace3.setId(5L); + namespace3.setName(namespaceName3); + namespace3.setPublicId(namespacePublicId3); + + var extension3 = new Extension(); + extension3.setId(6L); + extension3.setName(extensionName3); + extension3.setPublicId(extensionPublicId3); + extension3.setNamespace(namespace3); + + Mockito.doAnswer(invocation -> { + var ext = invocation.getArgument(0, Extension.class); + ext.setPublicId(null); + ext.getNamespace().setPublicId(null); + return null; + }).when(idService).getUpstreamPublicIds(extension1); + Mockito.doAnswer(invocation -> { + var ext = invocation.getArgument(0, Extension.class); + ext.setPublicId(null); + ext.getNamespace().setPublicId(null); + return null; + }).when(idService).getUpstreamPublicIds(extension2); + Mockito.doAnswer(invocation -> { + var ext = invocation.getArgument(0, Extension.class); + ext.setPublicId(null); + ext.getNamespace().setPublicId(null); + return null; + }).when(idService).getUpstreamPublicIds(extension3); + Mockito.when(repositories.findAllPublicIds()).thenReturn(List.of(extension1, extension2, extension3)); + + updateService.updateAll(); + Mockito.verify(repositories, Mockito.never()).updateExtensionPublicIds(Mockito.anyMap()); + Mockito.verify(repositories, Mockito.never()).updateNamespacePublicIds(Mockito.anyMap()); + } + + @Test + public void testUpdateAllChange() throws InterruptedException { + var namespaceName1 = "foo"; + var namespacePublicId1 = UUID.randomUUID().toString(); + var extensionName1 = "bar"; + var extensionPublicId1 = UUID.randomUUID().toString(); + + var namespace1 = new Namespace(); + namespace1.setId(1L); + namespace1.setName(namespaceName1); + namespace1.setPublicId(namespacePublicId1); + + var extension1 = new Extension(); + extension1.setId(2L); + extension1.setName(extensionName1); + extension1.setNamespace(namespace1); + extension1.setPublicId(extensionPublicId1); + + var namespaceName2 = "baz"; + var namespacePublicId2 = UUID.randomUUID().toString(); + var extensionName2 = "foobar"; + var extensionPublicId2 = UUID.randomUUID().toString(); + + var namespace2 = new Namespace(); + namespace2.setId(3L); + namespace2.setName(namespaceName2); + namespace2.setPublicId(namespacePublicId2); + + var extension2 = new Extension(); + extension2.setId(4L); + extension2.setName(extensionName2); + extension2.setPublicId(extensionPublicId2); + extension2.setNamespace(namespace2); + + var namespaceName3 = "baz2"; + var namespacePublicId3 = UUID.randomUUID().toString(); + var extensionName3 = "foobar2"; + var extensionPublicId3 = UUID.randomUUID().toString(); + + var namespace3 = new Namespace(); + namespace3.setId(5L); + namespace3.setName(namespaceName3); + namespace3.setPublicId(namespacePublicId3); + + var extension3 = new Extension(); + extension3.setId(6L); + extension3.setName(extensionName3); + extension3.setPublicId(extensionPublicId3); + extension3.setNamespace(namespace3); + + Mockito.doAnswer(invocation -> { + var ext = invocation.getArgument(0, Extension.class); + ext.setPublicId(null); + ext.getNamespace().setPublicId(null); + return null; + }).when(idService).getUpstreamPublicIds(extension1); + Mockito.doAnswer(invocation -> { + var ext = invocation.getArgument(0, Extension.class); + ext.setPublicId(extensionPublicId3); + ext.getNamespace().setPublicId(namespacePublicId3); + return null; + }).when(idService).getUpstreamPublicIds(extension2); + Mockito.doAnswer(invocation -> { + var ext = invocation.getArgument(0, Extension.class); + ext.setPublicId(null); + ext.getNamespace().setPublicId(null); + return null; + }).when(idService).getUpstreamPublicIds(extension3); + Mockito.when(repositories.findAllPublicIds()).thenReturn(List.of(extension1, extension2, extension3)); + + var extensionUuid = UUID.randomUUID(); + var namespaceUuid = UUID.randomUUID(); + var uuidMock = Mockito.mockStatic(UUID.class); + uuidMock.when(UUID::randomUUID).thenReturn(extensionUuid, namespaceUuid); + + updateService.updateAll(); + Mockito.verify(repositories, Mockito.times(1)).updateExtensionPublicIds(Map.of( + extension2.getId(), extensionPublicId3, + extension3.getId(), extensionUuid.toString() + )); + Mockito.verify(repositories, Mockito.times(1)).updateNamespacePublicIds(Map.of( + namespace2.getId(), namespacePublicId3, + namespace3.getId(), namespaceUuid.toString() + )); + uuidMock.close(); + } + + @Test + public void testUpdateUpstream() throws InterruptedException { + var namespaceName = "foo"; + var namespacePublicId = "123-456-789"; + var extensionName = "bar"; + var extensionPublicId = "abc-def-ghi"; + + var namespace = new Namespace(); + namespace.setId(1L); + namespace.setName(namespaceName); + + var extension = new Extension(); + extension.setId(2L); + extension.setName(extensionName); + extension.setNamespace(namespace); + + Mockito.when(repositories.findPublicId(namespaceName, extensionName)).thenReturn(extension); + Mockito.doAnswer(invocation -> { + var ext = invocation.getArgument(0, Extension.class); + ext.setPublicId(extensionPublicId); + ext.getNamespace().setPublicId(namespacePublicId); + return null; + }).when(idService).getUpstreamPublicIds(extension); + + updateService.update(namespaceName, extensionName); + Mockito.verify(repositories).updateExtensionPublicIds(Mockito.argThat((Map map) -> { + return map.size() == 1 && map.get(extension.getId()).equals(extensionPublicId); + })); + Mockito.verify(repositories).updateNamespacePublicIds(Mockito.argThat((Map map) -> { + return map.size() == 1 && map.get(namespace.getId()).equals(namespacePublicId); + })); + } + + @Test + public void testUpdateRandom() throws InterruptedException { + var namespaceName = "foo"; + var namespaceUuid = UUID.randomUUID(); + var extensionName = "bar"; + var extensionUuid = UUID.randomUUID(); + + var namespace = new Namespace(); + namespace.setId(1L); + namespace.setName(namespaceName); + + var extension = new Extension(); + extension.setId(2L); + extension.setName(extensionName); + extension.setNamespace(namespace); + + Mockito.when(repositories.findPublicId(namespaceName, extensionName)).thenReturn(extension); + var uuidMock = Mockito.mockStatic(UUID.class); + uuidMock.when(UUID::randomUUID).thenReturn(extensionUuid, namespaceUuid); + + updateService.update(namespaceName, extensionName); + Mockito.verify(repositories).updateExtensionPublicIds(Mockito.argThat((Map map) -> { + return map.size() == 1 && map.get(extension.getId()).equals(extensionUuid.toString()); + })); + Mockito.verify(repositories).updateNamespacePublicIds(Mockito.argThat((Map map) -> { + return map.size() == 1 && map.get(namespace.getId()).equals(namespaceUuid.toString()); + })); + uuidMock.close(); + } + + @Test + public void testUpdateRandomExists() throws InterruptedException { + var namespaceName = "foo"; + var namespaceUuid1 = UUID.randomUUID(); + var namespaceUuid2 = UUID.randomUUID(); + var extensionName = "bar"; + var extensionUuid1 = UUID.randomUUID(); + var extensionUuid2 = UUID.randomUUID(); + + var namespace = new Namespace(); + namespace.setId(1L); + namespace.setName(namespaceName); + + var extension = new Extension(); + extension.setId(2L); + extension.setName(extensionName); + extension.setNamespace(namespace); + + Mockito.when(repositories.findPublicId(namespaceName, extensionName)).thenReturn(extension); + Mockito.when(repositories.extensionPublicIdExists(extensionUuid1.toString())).thenReturn(true); + Mockito.when(repositories.namespacePublicIdExists(namespaceUuid1.toString())).thenReturn(true); + var uuidMock = Mockito.mockStatic(UUID.class); + uuidMock.when(UUID::randomUUID) + .thenReturn(extensionUuid1, extensionUuid2, namespaceUuid1, namespaceUuid2); + + updateService.update(namespaceName, extensionName); + Mockito.verify(repositories).updateExtensionPublicIds(Mockito.argThat((Map map) -> { + return map.size() == 1 && map.get(extension.getId()).equals(extensionUuid2.toString()); + })); + Mockito.verify(repositories).updateNamespacePublicIds(Mockito.argThat((Map map) -> { + return map.size() == 1 && map.get(namespace.getId()).equals(namespaceUuid2.toString()); + })); + uuidMock.close(); + } + + @Test + public void testUpdateDuplicateUpstreamChanges() throws InterruptedException { + var namespaceName1 = "foo"; + var namespacePublicId1 = UUID.randomUUID().toString(); + var extensionName1 = "bar"; + var extensionPublicId1 = UUID.randomUUID().toString(); + + var namespace1 = new Namespace(); + namespace1.setId(1L); + namespace1.setName(namespaceName1); + + var extension1 = new Extension(); + extension1.setId(2L); + extension1.setName(extensionName1); + extension1.setNamespace(namespace1); + + var namespaceName2 = "baz"; + var namespaceUuid2 = UUID.randomUUID(); + var extensionName2 = "foobar"; + var extensionUuid2 = UUID.randomUUID(); + + var namespace2 = new Namespace(); + namespace2.setId(3L); + namespace2.setName(namespaceName2); + namespace2.setPublicId(namespacePublicId1); + + var extension2 = new Extension(); + extension2.setId(4L); + extension2.setName(extensionName2); + extension2.setPublicId(extensionPublicId1); + extension2.setNamespace(namespace2); + + var namespaceName3 = "baz2"; + var namespaceUuid3 = UUID.randomUUID(); + var extensionName3 = "foobar2"; + var extensionUuid3 = UUID.randomUUID(); + + var namespace3 = new Namespace(); + namespace3.setId(5L); + namespace3.setName(namespaceName3); + namespace3.setPublicId(namespaceUuid2.toString()); + + var extension3 = new Extension(); + extension3.setId(6L); + extension3.setName(extensionName3); + extension3.setPublicId(extensionUuid2.toString()); + extension3.setNamespace(namespace3); + + var namespaceName4 = "baz3"; + var namespaceUuid4 = UUID.randomUUID(); + var extensionName4 = "foobar3"; + var extensionUuid4 = UUID.randomUUID(); + + var namespace4 = new Namespace(); + namespace4.setId(7L); + namespace4.setName(namespaceName4); + namespace4.setPublicId(namespaceUuid3.toString()); + + var extension4 = new Extension(); + extension4.setId(8L); + extension4.setName(extensionName4); + extension4.setPublicId(extensionUuid3.toString()); + extension4.setNamespace(namespace4); + + Mockito.when(repositories.findPublicId(namespaceName1, extensionName1)).thenReturn(extension1); + Mockito.when(repositories.findPublicId(extensionPublicId1)).thenReturn(extension2); + Mockito.when(repositories.findNamespacePublicId(namespacePublicId1)).thenReturn(extension2); + Mockito.when(repositories.findPublicId(extensionUuid2.toString())).thenReturn(extension3); + Mockito.when(repositories.findNamespacePublicId(namespaceUuid2.toString())).thenReturn(extension3); + Mockito.when(repositories.findPublicId(extensionUuid3.toString())).thenReturn(extension4); + Mockito.when(repositories.findNamespacePublicId(namespaceUuid3.toString())).thenReturn(extension4); + + Mockito.doAnswer(invocation -> { + var ext = invocation.getArgument(0, Extension.class); + ext.setPublicId(extensionPublicId1); + ext.getNamespace().setPublicId(namespacePublicId1); + return null; + }).when(idService).getUpstreamPublicIds(extension1); + Mockito.doAnswer(invocation -> { + var ext = invocation.getArgument(0, Extension.class); + ext.setPublicId(extensionUuid2.toString()); + ext.getNamespace().setPublicId(namespaceUuid2.toString()); + return null; + }).when(idService).getUpstreamPublicIds(extension2); + Mockito.doAnswer(invocation -> { + var ext = invocation.getArgument(0, Extension.class); + ext.setPublicId(extensionUuid3.toString()); + ext.getNamespace().setPublicId(namespaceUuid3.toString()); + return null; + }).when(idService).getUpstreamPublicIds(extension3); + Mockito.doAnswer(invocation -> { + var ext = invocation.getArgument(0, Extension.class); + ext.setPublicId(extensionUuid4.toString()); + ext.getNamespace().setPublicId(namespaceUuid4.toString()); + return null; + }).when(idService).getUpstreamPublicIds(extension4); + + updateService.update(namespaceName1, extensionName1); + Mockito.verify(repositories).updateExtensionPublicIds(Mockito.argThat((Map map) -> { + return map.size() == 4 + && map.get(extension1.getId()).equals(extensionPublicId1) + && map.get(extension2.getId()).equals(extensionUuid2.toString()) + && map.get(extension3.getId()).equals(extensionUuid3.toString()) + && map.get(extension4.getId()).equals(extensionUuid4.toString()); + })); + Mockito.verify(repositories).updateNamespacePublicIds(Mockito.argThat((Map map) -> { + return map.size() == 4 + && map.get(namespace1.getId()).equals(namespacePublicId1) + && map.get(namespace2.getId()).equals(namespaceUuid2.toString()) + && map.get(namespace3.getId()).equals(namespaceUuid3.toString()) + && map.get(namespace4.getId()).equals(namespaceUuid4.toString()); + })); + } + + @Test + public void testUpdateDuplicateRandom() throws InterruptedException { + var namespaceName1 = "foo"; + var namespacePublicId1 = UUID.randomUUID().toString(); + var extensionName1 = "bar"; + var extensionPublicId1 = UUID.randomUUID().toString(); + + var namespace1 = new Namespace(); + namespace1.setId(1L); + namespace1.setName(namespaceName1); + + var extension1 = new Extension(); + extension1.setId(2L); + extension1.setName(extensionName1); + extension1.setNamespace(namespace1); + + var namespaceName2 = "baz"; + var namespaceUuid2 = UUID.randomUUID(); + var extensionName2 = "foobar"; + var extensionUuid2 = UUID.randomUUID(); + + var namespace2 = new Namespace(); + namespace2.setId(3L); + namespace2.setName(namespaceName2); + namespace2.setPublicId(namespacePublicId1); + + var extension2 = new Extension(); + extension2.setId(4L); + extension2.setName(extensionName2); + extension2.setPublicId(extensionPublicId1); + extension2.setNamespace(namespace2); + + Mockito.when(repositories.findPublicId(namespaceName1, extensionName1)).thenReturn(extension1); + Mockito.when(repositories.findPublicId(extensionPublicId1)).thenReturn(extension2); + Mockito.when(repositories.findNamespacePublicId(namespacePublicId1)).thenReturn(extension2); + + Mockito.doAnswer(invocation -> { + var ext = invocation.getArgument(0, Extension.class); + ext.setPublicId(extensionPublicId1); + ext.getNamespace().setPublicId(namespacePublicId1); + return null; + }).when(idService).getUpstreamPublicIds(extension1); + Mockito.doAnswer(invocation -> { + var ext = invocation.getArgument(0, Extension.class); + ext.setPublicId(null); + ext.getNamespace().setPublicId(null); + return null; + }).when(idService).getUpstreamPublicIds(extension2); + var uuidMock = Mockito.mockStatic(UUID.class); + uuidMock.when(UUID::randomUUID).thenReturn(extensionUuid2, namespaceUuid2); + + updateService.update(namespaceName1, extensionName1); + Mockito.verify(repositories).updateExtensionPublicIds(Mockito.argThat((Map map) -> { + return map.size() == 2 + && map.get(extension1.getId()).equals(extensionPublicId1) + && map.get(extension2.getId()).equals(extensionUuid2.toString()); + })); + Mockito.verify(repositories).updateNamespacePublicIds(Mockito.argThat((Map map) -> { + return map.size() == 2 + && map.get(namespace1.getId()).equals(namespacePublicId1) + && map.get(namespace2.getId()).equals(namespaceUuid2.toString()); + })); + uuidMock.close(); + } + + @Test + public void testUpdateWaitsUntilUpdateAllIsFinished() throws InterruptedException { + var namespaceName = "foo"; + var namespacePublicId = UUID.randomUUID().toString(); + var extensionName = "bar"; + var extensionPublicId = UUID.randomUUID().toString(); + + var namespace = new Namespace(); + namespace.setId(1L); + namespace.setName(namespaceName); + + var extension = new Extension(); + extension.setId(2L); + extension.setName(extensionName); + extension.setNamespace(namespace); + + Mockito.doAnswer(invocation -> { + var ext = invocation.getArgument(0, Extension.class); + ext.setPublicId(extensionPublicId); + ext.getNamespace().setPublicId(namespacePublicId); + return null; + }).when(idService).getUpstreamPublicIds(extension); + Mockito.when(repositories.findPublicId(namespaceName, extensionName)).thenReturn(extension); + Mockito.when(repositories.findAllPublicIds()).thenAnswer(invocation -> { + Thread.sleep(1000); + return List.of(extension); + }); + + var executor = Executors.newFixedThreadPool(2); + var future1 = CompletableFuture.runAsync(() -> { + try { + updateService.updateAll(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + }, executor); + var future2 = CompletableFuture.runAsync(() -> { + try { + updateService.update("foo", "bar"); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + }, executor); + + CompletableFuture.allOf(future1, future2).join(); + var order = Mockito.inOrder(repositories); + order.verify(repositories).findAllPublicIds(); + order.verify(repositories).updateExtensionPublicIds(Mockito.anyMap()); + order.verify(repositories).updateNamespacePublicIds(Mockito.anyMap()); + order.verify(repositories).findPublicId(namespaceName, extensionName); + order.verify(repositories).updateExtensionPublicIds(Mockito.anyMap()); + order.verify(repositories).updateNamespacePublicIds(Mockito.anyMap()); + } + + @Test + public void testUpdateTimeout() throws InterruptedException { + Mockito.when(repositories.findAllPublicIds()).thenAnswer(invocation -> { + Thread.sleep(20000); + return Collections.emptyList(); + }); + + var executor = Executors.newFixedThreadPool(2); + var future1 = CompletableFuture.runAsync(() -> { + try { + updateService.updateAll(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + }, executor); + var future2 = CompletableFuture.runAsync(() -> { + assertThrows(RuntimeException.class, () -> updateService.update("foo", "bar")); + }, executor); + CompletableFuture.allOf(future1, future2).join(); + } + + @Test + public void testUpdateAllWait() throws InterruptedException { + var namespaceName = "foo"; + var extensionName = "bar"; + Mockito.when(repositories.findPublicId(namespaceName, extensionName)).thenAnswer(invocation -> { + Thread.sleep(20000); + + var namespace = new Namespace(); + namespace.setId(1L); + namespace.setName(namespaceName); + + var extension = new Extension(); + extension.setId(2L); + extension.setName(extensionName); + extension.setNamespace(namespace); + return extension; + }); + + var executor = Executors.newFixedThreadPool(2); + var future1 = CompletableFuture.runAsync(() -> { + try { + updateService.update(namespaceName, extensionName); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + }, executor); + var future2 = CompletableFuture.runAsync(() -> { + try { + updateService.updateAll(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + }, executor); + CompletableFuture.allOf(future1, future2).join(); + + var order = Mockito.inOrder(repositories); + order.verify(repositories).findPublicId(namespaceName, extensionName); + order.verify(repositories).findAllPublicIds(); + } + + @TestConfiguration + static class TestConfig { + @Bean + VSCodeIdUpdateService vsCodeIdUpdateService() { + return new VSCodeIdUpdateService(); + } + } +} diff --git a/server/src/test/java/org/eclipse/openvsx/repositories/RepositoryServiceSmokeTest.java b/server/src/test/java/org/eclipse/openvsx/repositories/RepositoryServiceSmokeTest.java index 19b1b03d2..409d4d8bd 100644 --- a/server/src/test/java/org/eclipse/openvsx/repositories/RepositoryServiceSmokeTest.java +++ b/server/src/test/java/org/eclipse/openvsx/repositories/RepositoryServiceSmokeTest.java @@ -23,6 +23,7 @@ import jakarta.transaction.Transactional; import java.lang.reflect.Modifier; import java.time.LocalDateTime; +import java.util.Collections; import java.util.List; import java.util.stream.Stream; @@ -183,7 +184,16 @@ void testExecuteQueries() { () -> repositories.findVersionStringsSorted(extension, "targetPlatform", true), () -> repositories.findActiveVersions(queryRequest), () -> repositories.findActiveVersionStringsSorted(LONG_LIST,"targetPlatform"), - () -> repositories.findActiveVersionReferencesSorted(List.of(extension)) + () -> repositories.findActiveVersionReferencesSorted(List.of(extension)), + () -> repositories.findAllPublicIds(), + () -> repositories.findPublicId("namespaceName", "extensionName"), + () -> repositories.findPublicId("namespaceName.extensionName"), + () -> repositories.findNamespacePublicId("namespaceName.extensionName"), + () -> repositories.updateExtensionPublicIds(Collections.emptyMap()), + () -> repositories.updateExtensionPublicId(1L, "namespaceName.extensionName"), + () -> repositories.updateNamespacePublicIds(Collections.emptyMap()), + () -> repositories.extensionPublicIdExists("namespaceName.extensionName"), + () -> repositories.namespacePublicIdExists("namespaceName.extensionName") ); // check that we did not miss anything