Merge pull request #822 from amvanbaren/bugfix/issue-819

Fix "unique_extension_public_id" unique constraint violation
eclipse · Oct 27, 2023 · 32d7643 · 32d7643
2 parents fa979e2 + 5b15de0
commit 32d7643
Show file tree

Hide file tree

Showing 18 changed files with 1,271 additions and 71 deletions.
diff --git a/server/src/main/java/org/eclipse/openvsx/ExtensionService.java b/server/src/main/java/org/eclipse/openvsx/ExtensionService.java
@@ -64,6 +64,7 @@ public ExtensionVersion publishVersion(InputStream content, PersonalAccessToken
         var extensionFile = createExtensionFile(content);
         var download = doPublish(extensionFile, null, token, TimeUtil.getCurrentUTC(), true);
         publishHandler.publishAsync(download, extensionFile, this);
+        publishHandler.schedulePublicIdJob(download);
         return download.getExtension();
     }
 

diff --git a/server/src/main/java/org/eclipse/openvsx/UrlConfigService.java b/server/src/main/java/org/eclipse/openvsx/UrlConfigService.java
@@ -31,7 +31,7 @@ public String getUpstreamUrl() {
         return upstreamUrl;
     }
 
-    public String  getUpstreamGalleryUrl() {
+    public String getUpstreamGalleryUrl() {
         return upstreamGalleryUrl;
     }
 

diff --git a/server/src/main/java/org/eclipse/openvsx/adapter/LocalVSCodeService.java b/server/src/main/java/org/eclipse/openvsx/adapter/LocalVSCodeService.java
@@ -45,8 +45,6 @@
 @Component
 public class LocalVSCodeService implements IVSCodeService {
 
-    private static final String BUILT_IN_EXTENSION_NAMESPACE = "vscode";
-
     @Autowired
     RepositoryService repositories;
 
@@ -106,12 +104,12 @@ public ExtensionQueryResult extensionQuery(ExtensionQueryParam param, int defaul
         Long totalCount = null;
         List<Extension> extensionsList;
         if (!extensionIds.isEmpty()) {
-            extensionsList = repositories.findActiveExtensionsByPublicId(extensionIds, BUILT_IN_EXTENSION_NAMESPACE);
+            extensionsList = repositories.findActiveExtensionsByPublicId(extensionIds, BuiltInExtensionUtil.getBuiltInNamespace());
         } else if (!extensionNames.isEmpty()) {
             extensionsList = extensionNames.stream()
                     .map(name -> name.split("\\."))
                     .filter(split -> split.length == 2)
-                    .filter(split -> !isBuiltInExtensionNamespace(split[0]))
+                    .filter(split -> !BuiltInExtensionUtil.isBuiltIn(split[0]))
                     .map(split -> {
                         var name = split[1];
                         var namespaceName = split[0];
@@ -125,7 +123,7 @@ public ExtensionQueryResult extensionQuery(ExtensionQueryParam param, int defaul
             try {
                 var pageOffset = pageNumber * pageSize;
                 var searchOptions = new SearchUtilService.Options(queryString, category, targetPlatform, pageSize,
-                        pageOffset, sortOrder, sortBy, false, BUILT_IN_EXTENSION_NAMESPACE);
+                        pageOffset, sortOrder, sortBy, false, BuiltInExtensionUtil.getBuiltInNamespace());
 
                 var searchResult = search.search(searchOptions);
                 totalCount = searchResult.getTotalHits();
@@ -280,7 +278,7 @@ public ResponseEntity<byte[]> getAsset(
             String namespace, String extensionName, String version, String assetType, String targetPlatform,
             String restOfTheUrl
     ) {
-        if(isBuiltInExtensionNamespace(namespace)) {
+        if(BuiltInExtensionUtil.isBuiltIn(namespace)) {
             return new ResponseEntity<>(("Built-in extension namespace '" + namespace + "' not allowed").getBytes(StandardCharsets.UTF_8), null, HttpStatus.BAD_REQUEST);
         }
 
@@ -344,7 +342,7 @@ private FileResource getFileFromDB(ExtensionVersion extVersion, String assetType
 
     @Override
     public String getItemUrl(String namespaceName, String extensionName) {
-        if(isBuiltInExtensionNamespace(namespaceName)) {
+        if(BuiltInExtensionUtil.isBuiltIn(namespaceName)) {
             throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "Built-in extension namespace '" + namespaceName + "' not allowed");
         }
 
@@ -358,7 +356,7 @@ public String getItemUrl(String namespaceName, String extensionName) {
 
     @Override
     public String download(String namespace, String extension, String version, String targetPlatform) {
-        if(isBuiltInExtensionNamespace(namespace)) {
+        if(BuiltInExtensionUtil.isBuiltIn(namespace)) {
             throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "Built-in extension namespace '" + namespace + "' not allowed");
         }
 
@@ -387,7 +385,7 @@ public String download(String namespace, String extension, String version, Strin
 
     @Override
     public ResponseEntity<byte[]> browse(String namespaceName, String extensionName, String version, String path) {
-        if(isBuiltInExtensionNamespace(namespaceName)) {
+        if(BuiltInExtensionUtil.isBuiltIn(namespaceName)) {
             return new ResponseEntity<>(("Built-in extension namespace '" + namespaceName + "' not allowed").getBytes(StandardCharsets.UTF_8), null, HttpStatus.BAD_REQUEST);
         }
 
@@ -619,8 +617,4 @@ private boolean isWebResource(FileResource resource) {
     private boolean test(int flags, int flag) {
         return (flags & flag) != 0;
     }
-
-    private boolean isBuiltInExtensionNamespace(String namespaceName) {
-        return namespaceName.equals(BUILT_IN_EXTENSION_NAMESPACE);
-    }
 }
diff --git a/server/src/main/java/org/eclipse/openvsx/adapter/VSCodeIdDailyUpdateJobRequestHandler.java b/server/src/main/java/org/eclipse/openvsx/adapter/VSCodeIdDailyUpdateJobRequestHandler.java
@@ -0,0 +1,29 @@
+/** ******************************************************************************
+ * Copyright (c) 2023 Precies. Software Ltd and others
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v. 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ * ****************************************************************************** */
+package org.eclipse.openvsx.adapter;
+
+import org.jobrunr.jobs.lambdas.JobRequest;
+import org.jobrunr.jobs.lambdas.JobRequestHandler;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.stereotype.Component;
+
+@Component
+@ConditionalOnProperty(value = "ovsx.data.mirror.enabled", havingValue = "false", matchIfMissing = true)
+public class VSCodeIdDailyUpdateJobRequestHandler implements JobRequestHandler<JobRequest> {
+
+    @Autowired
+    VSCodeIdUpdateService service;
+
+    @Override
+    public void run(JobRequest request) throws Exception {
+        service.updateAll();
+    }
+}
diff --git a/server/src/main/java/org/eclipse/openvsx/adapter/VSCodeIdNewExtensionJobRequest.java b/server/src/main/java/org/eclipse/openvsx/adapter/VSCodeIdNewExtensionJobRequest.java
@@ -0,0 +1,47 @@
+/** ******************************************************************************
+ * Copyright (c) 2023 Precies. Software Ltd and others
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v. 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ * ****************************************************************************** */
+package org.eclipse.openvsx.adapter;
+
+import org.jobrunr.jobs.lambdas.JobRequest;
+import org.jobrunr.jobs.lambdas.JobRequestHandler;
+
+public class VSCodeIdNewExtensionJobRequest implements JobRequest {
+
+    private String namespace;
+    private String extension;
+
+    public VSCodeIdNewExtensionJobRequest() {}
+
+    public VSCodeIdNewExtensionJobRequest(String namespace, String extension) {
+        this.namespace = namespace;
+        this.extension = extension;
+    }
+
+    @Override
+    public Class<? extends JobRequestHandler> getJobRequestHandler() {
+        return VSCodeIdNewExtensionJobRequestHandler.class;
+    }
+
+    public String getNamespace() {
+        return namespace;
+    }
+
+    public void setNamespace(String namespace) {
+        this.namespace = namespace;
+    }
+
+    public String getExtension() {
+        return extension;
+    }
+
+    public void setExtension(String extension) {
+        this.extension = extension;
+    }
+}
diff --git a/server/src/main/java/org/eclipse/openvsx/adapter/VSCodeIdNewExtensionJobRequestHandler.java b/server/src/main/java/org/eclipse/openvsx/adapter/VSCodeIdNewExtensionJobRequestHandler.java
@@ -0,0 +1,28 @@
+/** ******************************************************************************
+ * Copyright (c) 2023 Precies. Software Ltd and others
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v. 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ * ****************************************************************************** */
+package org.eclipse.openvsx.adapter;
+
+import org.jobrunr.jobs.lambdas.JobRequestHandler;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+@Component
+public class VSCodeIdNewExtensionJobRequestHandler implements JobRequestHandler<VSCodeIdNewExtensionJobRequest> {
+
+    @Autowired
+    VSCodeIdUpdateService service;
+
+    @Override
+    public void run(VSCodeIdNewExtensionJobRequest jobRequest) throws Exception {
+        var namespaceName = jobRequest.getNamespace();
+        var extensionName = jobRequest.getExtension();
+        service.update(namespaceName, extensionName);
+    }
+}
diff --git a/server/src/main/java/org/eclipse/openvsx/adapter/VSCodeIdService.java b/server/src/main/java/org/eclipse/openvsx/adapter/VSCodeIdService.java
@@ -13,19 +13,24 @@
 import org.apache.commons.lang3.StringUtils;
 import org.eclipse.openvsx.UrlConfigService;
 import org.eclipse.openvsx.entities.Extension;
-import org.eclipse.openvsx.repositories.RepositoryService;
+import org.eclipse.openvsx.migration.HandlerJobRequest;
 import org.eclipse.openvsx.util.NamingUtil;
 import org.eclipse.openvsx.util.UrlUtil;
+import org.jobrunr.scheduling.JobRequestScheduler;
+import org.jobrunr.scheduling.cron.Cron;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.context.event.ApplicationStartedEvent;
+import org.springframework.context.event.EventListener;
 import org.springframework.http.HttpEntity;
 import org.springframework.http.HttpHeaders;
 import org.springframework.http.MediaType;
 import org.springframework.stereotype.Component;
 import org.springframework.web.client.RestTemplate;
 
-import java.util.UUID;
+import java.time.ZoneId;
 
 @Component
 public class VSCodeIdService {
@@ -38,35 +43,41 @@ public class VSCodeIdService {
     RestTemplate vsCodeIdRestTemplate;
 
     @Autowired
-    RepositoryService repositories;
+    UrlConfigService urlConfigService;
 
     @Autowired
-    UrlConfigService urlConfigService;
+    JobRequestScheduler scheduler;
+
+    @Value("${ovsx.data.mirror.enabled:false}")
+    boolean mirrorEnabled;
+
+    @Value("${ovsx.vscode.upstream.update-on-start:false}")
+    boolean updateOnStart;
+
+    @EventListener
+    public void applicationStarted(ApplicationStartedEvent event) {
+        if(mirrorEnabled) {
+            return;
+        }
+        if(updateOnStart) {
+            scheduler.enqueue(new HandlerJobRequest<>(VSCodeIdDailyUpdateJobRequestHandler.class));
+        }
 
-    public boolean setPublicIds(Extension extension) {
-        var updateExistingPublicIds = false;
+        scheduler.scheduleRecurrently("VSCodeIdDailyUpdate", Cron.daily(3), ZoneId.of("UTC"), new HandlerJobRequest<>(VSCodeIdDailyUpdateJobRequestHandler.class));
+    }
+
+    public void getUpstreamPublicIds(Extension extension) {
+        extension.setPublicId(null);
+        extension.getNamespace().setPublicId(null);
         var upstream = getUpstreamExtension(extension);
         if (upstream != null) {
             if (upstream.extensionId != null) {
                 extension.setPublicId(upstream.extensionId);
-                updateExistingPublicIds = true;
             }
             if (upstream.publisher != null && upstream.publisher.publisherId != null) {
                 extension.getNamespace().setPublicId(upstream.publisher.publisherId);
             }
         }
-        if (extension.getPublicId() == null) {
-            extension.setPublicId(createRandomId());
-        }
-        if (extension.getNamespace().getPublicId() == null) {
-            extension.getNamespace().setPublicId(createRandomId());
-        }
-
-        return updateExistingPublicIds;
-    }
-
-    private String createRandomId() {
-        return UUID.randomUUID().toString();
     }
 
     private ExtensionQueryResult.Extension getUpstreamExtension(Extension extension) {