diff --git a/README.md b/README.md index fc08ebe..dc08b5e 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ Nexus Repository Google Cloud Storage Blobstore [![Build Status](https://travis-ci.org/sonatype-nexus-community/nexus-blobstore-google-cloud.svg?branch=master)](https://travis-ci.org/sonatype-nexus-community/nexus-blobstore-google-cloud) [![Join the chat at https://gitter.im/sonatype/nexus-developers](https://badges.gitter.im/sonatype/nexus-developers.svg)](https://gitter.im/sonatype/nexus-developers?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) This project adds [Google Cloud Object Storage](https://cloud.google.com/storage/) backed blobstores to Sonatype Nexus -Repository 3. It allows Nexus Repository to store the components and assets in Google Cloud instead of a +Repository 3.13 and later. It allows Nexus Repository to store the components and assets in Google Cloud instead of a local filesystem. Contribution Guidelines diff --git a/pom.xml b/pom.xml index 07e0b58..441cf37 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ org.sonatype.nexus.plugins nexus-plugins - 3.11.0-01 + 3.13.0-01 nexus-blobstore-google-cloud diff --git a/src/main/java/org/sonatype/nexus/blobstore/gcloud/internal/GoogleCloudBlobAttributes.java b/src/main/java/org/sonatype/nexus/blobstore/gcloud/internal/GoogleCloudBlobAttributes.java index 05764f5..1e5649e 100644 --- a/src/main/java/org/sonatype/nexus/blobstore/gcloud/internal/GoogleCloudBlobAttributes.java +++ b/src/main/java/org/sonatype/nexus/blobstore/gcloud/internal/GoogleCloudBlobAttributes.java @@ -146,6 +146,10 @@ private Properties writeTo(final Properties properties) { properties.put(DELETED_ATTRIBUTE, Boolean.toString(deleted)); properties.put(DELETED_REASON_ATTRIBUTE, getDeletedReason()); } + else { + properties.remove(DELETED_ATTRIBUTE); + properties.remove(DELETED_REASON_ATTRIBUTE); + } return properties; } } diff --git a/src/main/java/org/sonatype/nexus/blobstore/gcloud/internal/GoogleCloudBlobStore.java b/src/main/java/org/sonatype/nexus/blobstore/gcloud/internal/GoogleCloudBlobStore.java index c3e6184..9dbb352 100644 --- a/src/main/java/org/sonatype/nexus/blobstore/gcloud/internal/GoogleCloudBlobStore.java +++ b/src/main/java/org/sonatype/nexus/blobstore/gcloud/internal/GoogleCloudBlobStore.java @@ -17,6 +17,7 @@ import java.nio.channels.Channels; import java.nio.file.Path; import java.util.Map; +import java.util.Optional; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.Lock; import java.util.stream.Stream; @@ -38,6 +39,7 @@ import org.sonatype.nexus.blobstore.api.BlobStoreException; import org.sonatype.nexus.blobstore.api.BlobStoreMetrics; import org.sonatype.nexus.blobstore.api.BlobStoreUsageChecker; +import org.sonatype.nexus.common.log.DryRunPrefix; import org.sonatype.nexus.common.stateguard.Guarded; import org.sonatype.nexus.common.stateguard.StateGuardLifecycleSupport; import org.sonatype.nexus.logging.task.ProgressLogIntervalHelper; @@ -63,6 +65,7 @@ import static com.google.common.cache.CacheLoader.from; import static java.lang.String.format; import static org.sonatype.nexus.blobstore.DirectPathLocationStrategy.DIRECT_PATH_ROOT; +import static org.sonatype.nexus.blobstore.api.BlobAttributesConstants.HEADER_PREFIX; import static org.sonatype.nexus.common.stateguard.StateGuardLifecycleSupport.State.FAILED; import static org.sonatype.nexus.common.stateguard.StateGuardLifecycleSupport.State.NEW; import static org.sonatype.nexus.common.stateguard.StateGuardLifecycleSupport.State.STARTED; @@ -118,15 +121,19 @@ public class GoogleCloudBlobStore private LoadingCache liveBlobs; + private final DryRunPrefix dryRunPrefix; + @Inject public GoogleCloudBlobStore(final GoogleCloudStorageFactory storageFactory, final BlobIdLocationResolver blobIdLocationResolver, final GoogleCloudBlobStoreMetricsStore metricsStore, - final GoogleCloudDatastoreFactory datastoreFactory) { + final GoogleCloudDatastoreFactory datastoreFactory, + final DryRunPrefix dryRunPrefix) { this.storageFactory = checkNotNull(storageFactory); this.blobIdLocationResolver = checkNotNull(blobIdLocationResolver); this.metricsStore = metricsStore; this.datastoreFactory = datastoreFactory; + this.dryRunPrefix = dryRunPrefix; } @Override @@ -421,7 +428,7 @@ String basename(final com.google.cloud.storage.Blob blob) { } /** - * @return the {@link BlobAttributes} for the blod, or null + * @return the {@link BlobAttributes} for the blob, or null * @throws BlobStoreException if an {@link IOException} occurs */ @Override @@ -461,6 +468,44 @@ public boolean exists(final BlobId blobId) { return getBlobAttributes(blobId) != null; } + @Override + public boolean undelete(@Nullable final BlobStoreUsageChecker blobStoreUsageChecker, + final BlobId blobId, + final BlobAttributes attributes, + final boolean isDryRun) + { + checkNotNull(attributes); + String logPrefix = isDryRun ? dryRunPrefix.get() : ""; + Optional blobName = Optional.of(attributes) + .map(BlobAttributes::getProperties) + .map(p -> p.getProperty(HEADER_PREFIX + BLOB_NAME_HEADER)); + if (!blobName.isPresent()) { + log.error("Property not present: {}, for blob id: {}, at path: {}", HEADER_PREFIX + BLOB_NAME_HEADER, + blobId, attributePath(blobId)); + return false; + } + if (attributes.isDeleted() && blobStoreUsageChecker != null && + blobStoreUsageChecker.test(this, blobId, blobName.get())) { + String deletedReason = attributes.getDeletedReason(); + if (!isDryRun) { + attributes.setDeleted(false); + attributes.setDeletedReason(null); + try { + attributes.store(); + } + catch (IOException e) { + log.error("Error while un-deleting blob id: {}, deleted reason: {}, blob store: {}, blob name: {}", + blobId, deletedReason, blobStoreConfiguration.getName(), blobName.get(), e); + } + } + log.warn( + "{}Soft-deleted blob still in use, un-deleting blob id: {}, deleted reason: {}, blob store: {}, blob name: {}", + logPrefix, blobId, deletedReason, blobStoreConfiguration.getName(), blobName.get()); + return true; + } + return false; + } + Blob createInternal(final Map headers, BlobIngester ingester) { checkNotNull(headers); diff --git a/src/test/java/org/sonatype/nexus/blobstore/gcloud/internal/GoogleCloudBlobStoreIT.groovy b/src/test/java/org/sonatype/nexus/blobstore/gcloud/internal/GoogleCloudBlobStoreIT.groovy index c0d797d..0708eae 100644 --- a/src/test/java/org/sonatype/nexus/blobstore/gcloud/internal/GoogleCloudBlobStoreIT.groovy +++ b/src/test/java/org/sonatype/nexus/blobstore/gcloud/internal/GoogleCloudBlobStoreIT.groovy @@ -19,8 +19,13 @@ import org.sonatype.nexus.blobstore.BlobIdLocationResolver import org.sonatype.nexus.blobstore.DefaultBlobIdLocationResolver import org.sonatype.nexus.blobstore.PeriodicJobService import org.sonatype.nexus.blobstore.PeriodicJobService.PeriodicJob +import org.sonatype.nexus.blobstore.api.Blob +import org.sonatype.nexus.blobstore.api.BlobAttributes import org.sonatype.nexus.blobstore.api.BlobId +import org.sonatype.nexus.blobstore.api.BlobStore import org.sonatype.nexus.blobstore.api.BlobStoreConfiguration +import org.sonatype.nexus.blobstore.api.BlobStoreUsageChecker +import org.sonatype.nexus.common.log.DryRunPrefix import org.sonatype.nexus.common.node.NodeAccess import com.google.cloud.storage.Blob.BlobSourceOption @@ -64,6 +69,8 @@ class GoogleCloudBlobStoreIT GoogleCloudBlobStore blobStore + BlobStoreUsageChecker usageChecker = Mock() + def setup() { config.attributes = [ 'google cloud storage': [ @@ -76,11 +83,14 @@ class GoogleCloudBlobStoreIT metricsStore = new GoogleCloudBlobStoreMetricsStore(periodicJobService, nodeAccess) // can't start metrics store until blobstore init is done (which creates the bucket) - blobStore = new GoogleCloudBlobStore(storageFactory, blobIdLocationResolver, metricsStore, datastoreFactory) + blobStore = new GoogleCloudBlobStore(storageFactory, blobIdLocationResolver, metricsStore, datastoreFactory, + new DryRunPrefix("TEST ")) blobStore.init(config) blobStore.start() metricsStore.start() + + usageChecker.test(_, _, _) >> true } def cleanup() { @@ -128,6 +138,54 @@ class GoogleCloudBlobStoreIT results.contains(new BlobId("${DIRECT_PATH_PREFIX}health-check/repo1/details/bootstrap.min.css")) } + def "undelete successfully makes blob accessible"() { + given: + Blob blob = blobStore.create(new ByteArrayInputStream('hello'.getBytes()), + [ (BlobStore.BLOB_NAME_HEADER): 'foo', + (BlobStore.CREATED_BY_HEADER): 'someuser' ] ) + assert blob != null + assert blobStore.delete(blob.id, 'testing') + BlobAttributes deletedAttributes = blobStore.getBlobAttributes(blob.id) + assert deletedAttributes.deleted + assert deletedAttributes.deletedReason == 'testing' + + when: + !blobStore.undelete(usageChecker, blob.id, deletedAttributes, false) + + then: + Blob after = blobStore.get(blob.id) + after != null + BlobAttributes attributesAfter = blobStore.getBlobAttributes(blob.id) + !attributesAfter.deleted + } + + def "undelete does nothing when dry run is true"() { + given: + Blob blob = blobStore.create(new ByteArrayInputStream('hello'.getBytes()), + [ (BlobStore.BLOB_NAME_HEADER): 'foo', + (BlobStore.CREATED_BY_HEADER): 'someuser' ] ) + assert blob != null + BlobAttributes attributes = blobStore.getBlobAttributes(blob.id) + assert blobStore.delete(blob.id, 'testing') + BlobAttributes deletedAttributes = blobStore.getBlobAttributes(blob.id) + assert deletedAttributes.deleted + + when: + blobStore.undelete(usageChecker, blob.id, attributes, true) + + then: + Blob after = blobStore.get(blob.id) + after == null + BlobAttributes attributesAfter = blobStore.getBlobAttributes(blob.id) + attributesAfter.deleted + } + + def "undelete does nothing on non-existent blob"() { + expect: + BlobAttributes attributes = Mock() + !blobStore.undelete(usageChecker, new BlobId("nonexistent"), attributes, false) + } + def createFile(Storage storage, String path) { storage.create(BlobInfo.newBuilder(bucketName, path).build(), "content".bytes) diff --git a/src/test/java/org/sonatype/nexus/blobstore/gcloud/internal/GoogleCloudBlobStoreTest.groovy b/src/test/java/org/sonatype/nexus/blobstore/gcloud/internal/GoogleCloudBlobStoreTest.groovy index 17fc792..84e2e0e 100644 --- a/src/test/java/org/sonatype/nexus/blobstore/gcloud/internal/GoogleCloudBlobStoreTest.groovy +++ b/src/test/java/org/sonatype/nexus/blobstore/gcloud/internal/GoogleCloudBlobStoreTest.groovy @@ -22,6 +22,7 @@ import org.sonatype.nexus.blobstore.api.BlobId import org.sonatype.nexus.blobstore.api.BlobStore import org.sonatype.nexus.blobstore.api.BlobStoreConfiguration import org.sonatype.nexus.blobstore.api.BlobStoreException +import org.sonatype.nexus.common.log.DryRunPrefix import com.google.api.gax.paging.Page import com.google.cloud.datastore.Datastore @@ -57,7 +58,7 @@ class GoogleCloudBlobStoreTest (BlobStore.CREATED_BY_HEADER): 'admin' ] GoogleCloudBlobStore blobStore = new GoogleCloudBlobStore( - storageFactory, blobIdLocationResolver, metricsStore, datastoreFactory) + storageFactory, blobIdLocationResolver, metricsStore, datastoreFactory, new DryRunPrefix("TEST ")) def config = new BlobStoreConfiguration()