update to NXRM 3.13, implement BlobStore#undelete (#11)

* feat: implement Blobstore#undelete Similar to other implementations, includes integration test that confirms expected function.
sonatype-nexus-community · Aug 23, 2018 · efbdf42 · efbdf42
1 parent 2089f61
commit efbdf42
Show file tree

Hide file tree

Showing 6 changed files with 114 additions and 6 deletions.
diff --git a/README.md b/README.md
@@ -18,7 +18,7 @@ Nexus Repository Google Cloud Storage Blobstore
 [![Build Status](https://travis-ci.org/sonatype-nexus-community/nexus-blobstore-google-cloud.svg?branch=master)](https://travis-ci.org/sonatype-nexus-community/nexus-blobstore-google-cloud) [![Join the chat at https://gitter.im/sonatype/nexus-developers](https://badges.gitter.im/sonatype/nexus-developers.svg)](https://gitter.im/sonatype/nexus-developers?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
 This project adds [Google Cloud Object Storage](https://cloud.google.com/storage/) backed blobstores to Sonatype Nexus 
-Repository 3.  It allows Nexus Repository to store the components and assets in Google Cloud instead of a
+Repository 3.13 and later.  It allows Nexus Repository to store the components and assets in Google Cloud instead of a
 local filesystem.
 
 Contribution Guidelines

diff --git a/pom.xml b/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.sonatype.nexus.plugins</groupId>
     <artifactId>nexus-plugins</artifactId>
-    <version>3.11.0-01</version>
+    <version>3.13.0-01</version>
   </parent>
 
   <artifactId>nexus-blobstore-google-cloud</artifactId>

diff --git a/src/main/java/org/sonatype/nexus/blobstore/gcloud/internal/GoogleCloudBlobAttributes.java b/src/main/java/org/sonatype/nexus/blobstore/gcloud/internal/GoogleCloudBlobAttributes.java
@@ -146,6 +146,10 @@ private Properties writeTo(final Properties properties) {
       properties.put(DELETED_ATTRIBUTE, Boolean.toString(deleted));
       properties.put(DELETED_REASON_ATTRIBUTE, getDeletedReason());
     }
+    else {
+      properties.remove(DELETED_ATTRIBUTE);
+      properties.remove(DELETED_REASON_ATTRIBUTE);
+    }
     return properties;
   }
 }
diff --git a/src/main/java/org/sonatype/nexus/blobstore/gcloud/internal/GoogleCloudBlobStore.java b/src/main/java/org/sonatype/nexus/blobstore/gcloud/internal/GoogleCloudBlobStore.java
@@ -17,6 +17,7 @@
 import java.nio.channels.Channels;
 import java.nio.file.Path;
 import java.util.Map;
+import java.util.Optional;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.locks.Lock;
 import java.util.stream.Stream;
@@ -38,6 +39,7 @@
 import org.sonatype.nexus.blobstore.api.BlobStoreException;
 import org.sonatype.nexus.blobstore.api.BlobStoreMetrics;
 import org.sonatype.nexus.blobstore.api.BlobStoreUsageChecker;
+import org.sonatype.nexus.common.log.DryRunPrefix;
 import org.sonatype.nexus.common.stateguard.Guarded;
 import org.sonatype.nexus.common.stateguard.StateGuardLifecycleSupport;
 import org.sonatype.nexus.logging.task.ProgressLogIntervalHelper;
@@ -63,6 +65,7 @@
 import static com.google.common.cache.CacheLoader.from;
 import static java.lang.String.format;
 import static org.sonatype.nexus.blobstore.DirectPathLocationStrategy.DIRECT_PATH_ROOT;
+import static org.sonatype.nexus.blobstore.api.BlobAttributesConstants.HEADER_PREFIX;
 import static org.sonatype.nexus.common.stateguard.StateGuardLifecycleSupport.State.FAILED;
 import static org.sonatype.nexus.common.stateguard.StateGuardLifecycleSupport.State.NEW;
 import static org.sonatype.nexus.common.stateguard.StateGuardLifecycleSupport.State.STARTED;
@@ -118,15 +121,19 @@ public class GoogleCloudBlobStore
 
   private LoadingCache<BlobId, GoogleCloudStorageBlob> liveBlobs;
 
+  private final DryRunPrefix dryRunPrefix;
+
   @Inject
   public GoogleCloudBlobStore(final GoogleCloudStorageFactory storageFactory,
                               final BlobIdLocationResolver blobIdLocationResolver,
                               final GoogleCloudBlobStoreMetricsStore metricsStore,
-                              final GoogleCloudDatastoreFactory datastoreFactory) {
+                              final GoogleCloudDatastoreFactory datastoreFactory,
+                              final DryRunPrefix dryRunPrefix) {
     this.storageFactory = checkNotNull(storageFactory);
     this.blobIdLocationResolver = checkNotNull(blobIdLocationResolver);
     this.metricsStore = metricsStore;
     this.datastoreFactory = datastoreFactory;
+    this.dryRunPrefix = dryRunPrefix;
   }
 
   @Override
@@ -421,7 +428,7 @@ String basename(final com.google.cloud.storage.Blob blob) {
   }
 
   /**
-   * @return the {@link BlobAttributes} for the blod, or null
+   * @return the {@link BlobAttributes} for the blob, or null
    * @throws BlobStoreException if an {@link IOException} occurs
    */
   @Override
@@ -461,6 +468,44 @@ public boolean exists(final BlobId blobId) {
     return getBlobAttributes(blobId) != null;
   }
 
+  @Override
+  public boolean undelete(@Nullable final BlobStoreUsageChecker blobStoreUsageChecker,
+                          final BlobId blobId,
+                          final BlobAttributes attributes,
+                          final boolean isDryRun)
+  {
+    checkNotNull(attributes);
+    String logPrefix = isDryRun ? dryRunPrefix.get() : "";
+    Optional<String> blobName = Optional.of(attributes)
+        .map(BlobAttributes::getProperties)
+        .map(p -> p.getProperty(HEADER_PREFIX + BLOB_NAME_HEADER));
+    if (!blobName.isPresent()) {
+      log.error("Property not present: {}, for blob id: {}, at path: {}", HEADER_PREFIX + BLOB_NAME_HEADER,
+          blobId, attributePath(blobId));
+      return false;
+    }
+    if (attributes.isDeleted() && blobStoreUsageChecker != null &&
+        blobStoreUsageChecker.test(this, blobId, blobName.get())) {
+      String deletedReason = attributes.getDeletedReason();
+      if (!isDryRun) {
+        attributes.setDeleted(false);
+        attributes.setDeletedReason(null);
+        try {
+          attributes.store();
+        }
+        catch (IOException e) {
+          log.error("Error while un-deleting blob id: {}, deleted reason: {}, blob store: {}, blob name: {}",
+              blobId, deletedReason, blobStoreConfiguration.getName(), blobName.get(), e);
+        }
+      }
+      log.warn(
+          "{}Soft-deleted blob still in use, un-deleting blob id: {}, deleted reason: {}, blob store: {}, blob name: {}",
+          logPrefix, blobId, deletedReason, blobStoreConfiguration.getName(), blobName.get());
+      return true;
+    }
+    return false;
+  }
+
   Blob createInternal(final Map<String, String> headers, BlobIngester ingester) {
     checkNotNull(headers);
 

diff --git a/src/test/java/org/sonatype/nexus/blobstore/gcloud/internal/GoogleCloudBlobStoreIT.groovy b/src/test/java/org/sonatype/nexus/blobstore/gcloud/internal/GoogleCloudBlobStoreIT.groovy
@@ -19,8 +19,13 @@ import org.sonatype.nexus.blobstore.BlobIdLocationResolver
 import org.sonatype.nexus.blobstore.DefaultBlobIdLocationResolver
 import org.sonatype.nexus.blobstore.PeriodicJobService
 import org.sonatype.nexus.blobstore.PeriodicJobService.PeriodicJob
+import org.sonatype.nexus.blobstore.api.Blob
+import org.sonatype.nexus.blobstore.api.BlobAttributes
 import org.sonatype.nexus.blobstore.api.BlobId
+import org.sonatype.nexus.blobstore.api.BlobStore
 import org.sonatype.nexus.blobstore.api.BlobStoreConfiguration
+import org.sonatype.nexus.blobstore.api.BlobStoreUsageChecker
+import org.sonatype.nexus.common.log.DryRunPrefix
 import org.sonatype.nexus.common.node.NodeAccess
 
 import com.google.cloud.storage.Blob.BlobSourceOption
@@ -64,6 +69,8 @@ class GoogleCloudBlobStoreIT
 
   GoogleCloudBlobStore blobStore
 
+  BlobStoreUsageChecker usageChecker = Mock()
+
   def setup() {
     config.attributes = [
         'google cloud storage': [
@@ -76,11 +83,14 @@ class GoogleCloudBlobStoreIT
 
     metricsStore = new GoogleCloudBlobStoreMetricsStore(periodicJobService, nodeAccess)
     // can't start metrics store until blobstore init is done (which creates the bucket)
-    blobStore = new GoogleCloudBlobStore(storageFactory, blobIdLocationResolver, metricsStore, datastoreFactory)
+    blobStore = new GoogleCloudBlobStore(storageFactory, blobIdLocationResolver, metricsStore, datastoreFactory,
+        new DryRunPrefix("TEST "))
     blobStore.init(config)
 
     blobStore.start()
     metricsStore.start()
+
+    usageChecker.test(_, _, _) >> true
   }
 
   def cleanup() {
@@ -128,6 +138,54 @@ class GoogleCloudBlobStoreIT
       results.contains(new BlobId("${DIRECT_PATH_PREFIX}health-check/repo1/details/bootstrap.min.css"))
   }
 
+  def "undelete successfully makes blob accessible"() {
+    given:
+      Blob blob = blobStore.create(new ByteArrayInputStream('hello'.getBytes()),
+          [ (BlobStore.BLOB_NAME_HEADER): 'foo',
+            (BlobStore.CREATED_BY_HEADER): 'someuser' ] )
+      assert blob != null
+      assert blobStore.delete(blob.id, 'testing')
+      BlobAttributes deletedAttributes = blobStore.getBlobAttributes(blob.id)
+      assert deletedAttributes.deleted
+      assert deletedAttributes.deletedReason == 'testing'
+
+    when:
+      !blobStore.undelete(usageChecker, blob.id, deletedAttributes, false)
+
+    then:
+      Blob after = blobStore.get(blob.id)
+      after != null
+      BlobAttributes attributesAfter = blobStore.getBlobAttributes(blob.id)
+      !attributesAfter.deleted
+  }
+
+  def "undelete does nothing when dry run is true"() {
+    given:
+      Blob blob = blobStore.create(new ByteArrayInputStream('hello'.getBytes()),
+          [ (BlobStore.BLOB_NAME_HEADER): 'foo',
+            (BlobStore.CREATED_BY_HEADER): 'someuser' ] )
+      assert blob != null
+      BlobAttributes attributes = blobStore.getBlobAttributes(blob.id)
+      assert blobStore.delete(blob.id, 'testing')
+      BlobAttributes deletedAttributes = blobStore.getBlobAttributes(blob.id)
+      assert deletedAttributes.deleted
+
+    when:
+      blobStore.undelete(usageChecker, blob.id, attributes, true)
+
+    then:
+      Blob after = blobStore.get(blob.id)
+      after == null
+      BlobAttributes attributesAfter = blobStore.getBlobAttributes(blob.id)
+      attributesAfter.deleted
+  }
+
+  def "undelete does nothing on non-existent blob"() {
+    expect:
+      BlobAttributes attributes = Mock()
+      !blobStore.undelete(usageChecker, new BlobId("nonexistent"), attributes, false)
+  }
+
   def createFile(Storage storage, String path) {
     storage.create(BlobInfo.newBuilder(bucketName, path).build(),
       "content".bytes)

diff --git a/src/test/java/org/sonatype/nexus/blobstore/gcloud/internal/GoogleCloudBlobStoreTest.groovy b/src/test/java/org/sonatype/nexus/blobstore/gcloud/internal/GoogleCloudBlobStoreTest.groovy
@@ -22,6 +22,7 @@ import org.sonatype.nexus.blobstore.api.BlobId
 import org.sonatype.nexus.blobstore.api.BlobStore
 import org.sonatype.nexus.blobstore.api.BlobStoreConfiguration
 import org.sonatype.nexus.blobstore.api.BlobStoreException
+import org.sonatype.nexus.common.log.DryRunPrefix
 
 import com.google.api.gax.paging.Page
 import com.google.cloud.datastore.Datastore
@@ -57,7 +58,7 @@ class GoogleCloudBlobStoreTest
       (BlobStore.CREATED_BY_HEADER): 'admin'
   ]
   GoogleCloudBlobStore blobStore = new GoogleCloudBlobStore(
-      storageFactory, blobIdLocationResolver, metricsStore, datastoreFactory)
+      storageFactory, blobIdLocationResolver, metricsStore, datastoreFactory, new DryRunPrefix("TEST "))
 
   def config = new BlobStoreConfiguration()