From 141dcbe718dc60513d4202a75503b09f8783bf9a Mon Sep 17 00:00:00 2001 From: Andrew Azores Date: Mon, 11 Mar 2024 10:37:35 -0400 Subject: [PATCH] chore(storage): refactor bucket check/create on startup (#321) --- .../java/io/cryostat/ExceptionMappers.java | 7 +++ src/main/java/io/cryostat/StorageBuckets.java | 56 +++++++++++++++++++ .../io/cryostat/events/S3TemplateService.java | 53 ++++-------------- .../io/cryostat/recordings/Recordings.java | 42 ++++---------- .../java/io/cryostat/reports/Reports.java | 33 ++--------- 5 files changed, 91 insertions(+), 100 deletions(-) create mode 100644 src/main/java/io/cryostat/StorageBuckets.java diff --git a/src/main/java/io/cryostat/ExceptionMappers.java b/src/main/java/io/cryostat/ExceptionMappers.java index cb3bb2cae..4ca58ab88 100644 --- a/src/main/java/io/cryostat/ExceptionMappers.java +++ b/src/main/java/io/cryostat/ExceptionMappers.java @@ -35,6 +35,7 @@ import org.jboss.resteasy.reactive.RestResponse.ResponseBuilder; import org.jboss.resteasy.reactive.server.ServerExceptionMapper; import org.projectnessie.cel.tools.ScriptException; +import software.amazon.awssdk.services.s3.model.NoSuchBucketException; import software.amazon.awssdk.services.s3.model.NoSuchKeyException; public class ExceptionMappers { @@ -51,6 +52,12 @@ public RestResponse mapNoSuchElementException(NoSuchElementException ex) { return RestResponse.notFound(); } + @ServerExceptionMapper + public RestResponse mapNoSuchBucketException(NoSuchBucketException ex) { + logger.error(ex); + return RestResponse.status(HttpResponseStatus.BAD_GATEWAY.code()); + } + @ServerExceptionMapper public RestResponse mapConstraintViolationException(ConstraintViolationException ex) { logger.warn(ex); diff --git a/src/main/java/io/cryostat/StorageBuckets.java b/src/main/java/io/cryostat/StorageBuckets.java new file mode 100644 index 000000000..4500a08da --- /dev/null +++ b/src/main/java/io/cryostat/StorageBuckets.java @@ -0,0 +1,56 @@ +/* + * Copyright The Cryostat Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cryostat; + +import io.cryostat.util.HttpStatusCodeIdentifier; + +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import org.jboss.logging.Logger; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.CreateBucketRequest; +import software.amazon.awssdk.services.s3.model.HeadBucketRequest; + +@ApplicationScoped +public class StorageBuckets { + + @Inject S3Client storage; + @Inject Logger logger; + + public void createIfNecessary(String bucket) { + boolean exists = false; + logger.infov("Checking if storage bucket \"{0}\" exists ...", bucket); + try { + exists = + HttpStatusCodeIdentifier.isSuccessCode( + storage.headBucket(HeadBucketRequest.builder().bucket(bucket).build()) + .sdkHttpResponse() + .statusCode()); + logger.infov("Storage bucket \"{0}\" exists? {1}", bucket, exists); + } catch (Exception e) { + logger.info(e); + } + if (!exists) { + logger.infov("Attempting to create storage bucket \"{0}\" ...", bucket); + try { + storage.createBucket(CreateBucketRequest.builder().bucket(bucket).build()); + logger.infov("Storage bucket \"{0}\" created", bucket); + } catch (Exception e) { + logger.error(e); + } + } + } +} diff --git a/src/main/java/io/cryostat/events/S3TemplateService.java b/src/main/java/io/cryostat/events/S3TemplateService.java index f71171bce..bdce3ac98 100644 --- a/src/main/java/io/cryostat/events/S3TemplateService.java +++ b/src/main/java/io/cryostat/events/S3TemplateService.java @@ -38,13 +38,13 @@ import io.cryostat.ConfigProperties; import io.cryostat.Producers; +import io.cryostat.StorageBuckets; import io.cryostat.core.FlightRecorderException; import io.cryostat.core.templates.MutableTemplateService; import io.cryostat.core.templates.MutableTemplateService.InvalidEventTemplateException; import io.cryostat.core.templates.MutableTemplateService.InvalidXmlException; import io.cryostat.core.templates.Template; import io.cryostat.core.templates.TemplateType; -import io.cryostat.util.HttpStatusCodeIdentifier; import io.cryostat.ws.MessagingServer; import io.cryostat.ws.Notification; @@ -65,11 +65,9 @@ import org.jsoup.parser.Parser; import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.model.CreateBucketRequest; import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; import software.amazon.awssdk.services.s3.model.GetObjectRequest; import software.amazon.awssdk.services.s3.model.GetObjectTaggingRequest; -import software.amazon.awssdk.services.s3.model.HeadBucketRequest; import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; import software.amazon.awssdk.services.s3.model.PutObjectRequest; import software.amazon.awssdk.services.s3.model.S3Object; @@ -82,7 +80,11 @@ public class S3TemplateService implements MutableTemplateService { static final String EVENT_TEMPLATE_CREATED = "TemplateUploaded"; static final String EVENT_TEMPLATE_DELETED = "TemplateDeleted"; + @ConfigProperty(name = ConfigProperties.AWS_BUCKET_NAME_EVENT_TEMPLATES) + String bucket; + @Inject S3Client storage; + @Inject StorageBuckets storageBuckets; @Inject EventBus bus; @@ -92,33 +94,8 @@ public class S3TemplateService implements MutableTemplateService { @Inject Logger logger; - @ConfigProperty(name = ConfigProperties.AWS_BUCKET_NAME_EVENT_TEMPLATES) - String eventTemplatesBucket; - void onStart(@Observes StartupEvent evt) { - // FIXME refactor this to a reusable utility method since this is done for custom event - // templates, archived recordings, and archived reports - boolean exists = false; - try { - exists = - HttpStatusCodeIdentifier.isSuccessCode( - storage.headBucket( - HeadBucketRequest.builder() - .bucket(eventTemplatesBucket) - .build()) - .sdkHttpResponse() - .statusCode()); - } catch (Exception e) { - logger.info(e); - } - if (!exists) { - try { - storage.createBucket( - CreateBucketRequest.builder().bucket(eventTemplatesBucket).build()); - } catch (Exception e) { - logger.error(e); - } - } + storageBuckets.createIfNecessary(bucket); } @Override @@ -170,17 +147,13 @@ public Optional getXml(String templateName, TemplateType unused) @Blocking private List getObjects() { - var builder = ListObjectsV2Request.builder().bucket(eventTemplatesBucket); + var builder = ListObjectsV2Request.builder().bucket(bucket); return storage.listObjectsV2(builder.build()).contents(); } @Blocking private Template convertObject(S3Object object) throws InvalidEventTemplateException { - var req = - GetObjectTaggingRequest.builder() - .bucket(eventTemplatesBucket) - .key(object.key()) - .build(); + var req = GetObjectTaggingRequest.builder().bucket(bucket).key(object.key()).build(); var tagging = storage.getObjectTagging(req); var list = tagging.tagSet(); if (!tagging.hasTagSet() || list.isEmpty()) { @@ -222,7 +195,7 @@ private Template convertObject(S3Object object) throws InvalidEventTemplateExcep @Blocking private InputStream getModel(String name) { - var req = GetObjectRequest.builder().bucket(eventTemplatesBucket).key(name).build(); + var req = GetObjectRequest.builder().bucket(bucket).key(name).build(); return storage.getObject(req); } @@ -273,7 +246,7 @@ public Template addTemplate(InputStream stream) String provider = getAttributeValue(root, "provider"); storage.putObject( PutObjectRequest.builder() - .bucket(eventTemplatesBucket) + .bucket(bucket) .key(templateName) .contentType(ContentType.APPLICATION_XML.getMimeType()) .tagging(createTemplateTagging(templateName, description, provider)) @@ -303,11 +276,7 @@ public void deleteTemplate(String templateName) { .filter(t -> t.getName().equals(templateName)) .findFirst() .orElseThrow(); - var req = - DeleteObjectRequest.builder() - .bucket(eventTemplatesBucket) - .key(templateName) - .build(); + var req = DeleteObjectRequest.builder().bucket(bucket).key(templateName).build(); if (storage.deleteObject(req).sdkHttpResponse().isSuccessful()) { bus.publish( MessagingServer.class.getName(), diff --git a/src/main/java/io/cryostat/recordings/Recordings.java b/src/main/java/io/cryostat/recordings/Recordings.java index 3bc71b72a..0bcea5e3c 100644 --- a/src/main/java/io/cryostat/recordings/Recordings.java +++ b/src/main/java/io/cryostat/recordings/Recordings.java @@ -43,6 +43,7 @@ import io.cryostat.ConfigProperties; import io.cryostat.Producers; +import io.cryostat.StorageBuckets; import io.cryostat.V2Response; import io.cryostat.core.EventOptionsBuilder; import io.cryostat.core.RecordingOptionsCustomizer; @@ -56,7 +57,6 @@ import io.cryostat.targets.Target; import io.cryostat.targets.TargetConnectionManager; import io.cryostat.util.HttpMimeType; -import io.cryostat.util.HttpStatusCodeIdentifier; import io.cryostat.ws.MessagingServer; import io.cryostat.ws.Notification; @@ -96,12 +96,10 @@ import org.jboss.resteasy.reactive.multipart.FileUpload; import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.model.CreateBucketRequest; import software.amazon.awssdk.services.s3.model.Delete; import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; import software.amazon.awssdk.services.s3.model.GetObjectRequest; -import software.amazon.awssdk.services.s3.model.HeadBucketRequest; import software.amazon.awssdk.services.s3.model.ObjectIdentifier; import software.amazon.awssdk.services.s3.model.PutObjectRequest; import software.amazon.awssdk.services.s3.model.S3Object; @@ -112,7 +110,6 @@ @Path("") public class Recordings { - @Inject Logger logger; @Inject TargetConnectionManager connectionManager; @Inject EventBus bus; @Inject RecordingOptionsBuilderFactory recordingOptionsBuilderFactory; @@ -120,18 +117,20 @@ public class Recordings { @Inject EventOptionsBuilder.Factory eventOptionsBuilderFactory; @Inject Clock clock; @Inject S3Client storage; + @Inject StorageBuckets storageBuckets; @Inject S3Presigner presigner; @Inject RemoteRecordingInputStreamFactory remoteRecordingStreamFactory; @Inject ScheduledExecutorService scheduler; @Inject ObjectMapper mapper; @Inject RecordingHelper recordingHelper; + @Inject Logger logger; @Inject @Named(Producers.BASE64_URL) Base64 base64Url; @ConfigProperty(name = ConfigProperties.AWS_BUCKET_NAME_ARCHIVES) - String archiveBucket; + String bucket; @ConfigProperty(name = ConfigProperties.GRAFANA_DATASOURCE_URL) Optional grafanaDatasourceURL; @@ -149,26 +148,7 @@ public class Recordings { Optional externalStorageUrl; void onStart(@Observes StartupEvent evt) { - boolean exists = false; - try { - exists = - HttpStatusCodeIdentifier.isSuccessCode( - storage.headBucket( - HeadBucketRequest.builder() - .bucket(archiveBucket) - .build()) - .sdkHttpResponse() - .statusCode()); - } catch (Exception e) { - logger.info(e); - } - if (!exists) { - try { - storage.createBucket(CreateBucketRequest.builder().bucket(archiveBucket).build()); - } catch (Exception e) { - logger.error(e); - } - } + storageBuckets.createIfNecessary(bucket); } @GET @@ -273,7 +253,7 @@ public void agentPush( new Notification(event.category().category(), event.payload())); storage.deleteObjects( DeleteObjectsRequest.builder() - .bucket(archiveBucket) + .bucket(bucket) .delete( Delete.builder() .objects( @@ -362,7 +342,7 @@ Map doUpload(FileUpload recording, Metadata metadata, String jvm String key = recordingHelper.archivedRecordingKey(jvmId, filename); storage.putObject( PutObjectRequest.builder() - .bucket(archiveBucket) + .bucket(bucket) .key(key) .contentType(RecordingHelper.JFR_MIME) .tagging(recordingHelper.createMetadataTagging(new Metadata(labels))) @@ -399,7 +379,7 @@ public void delete(@RestPath String filename) throws Exception { // TODO scan all prefixes for matching filename? This is an old v1 API problem. storage.deleteObject( DeleteObjectRequest.builder() - .bucket(archiveBucket) + .bucket(bucket) .key(String.format("%s/%s", "uploads", filename)) .build()); } @@ -775,11 +755,11 @@ public void deleteArchivedRecording(@RestPath String jvmId, @RestPath String fil connectUrl, metadata); logger.infov( "Sending S3 deletion request for {0} {1}", - archiveBucket, recordingHelper.archivedRecordingKey(jvmId, filename)); + bucket, recordingHelper.archivedRecordingKey(jvmId, filename)); var resp = storage.deleteObject( DeleteObjectRequest.builder() - .bucket(archiveBucket) + .bucket(bucket) .key(recordingHelper.archivedRecordingKey(jvmId, filename)) .build()); logger.infov( @@ -1048,7 +1028,7 @@ public Response handleStorageDownload(@RestPath String encodedKey, @RestQuery St logger.infov("Handling presigned download request for {0}", pair); GetObjectRequest getRequest = GetObjectRequest.builder() - .bucket(archiveBucket) + .bucket(bucket) .key(recordingHelper.archivedRecordingKey(pair)) .build(); GetObjectPresignRequest presignRequest = diff --git a/src/main/java/io/cryostat/reports/Reports.java b/src/main/java/io/cryostat/reports/Reports.java index daafcbdd9..0950fc82b 100644 --- a/src/main/java/io/cryostat/reports/Reports.java +++ b/src/main/java/io/cryostat/reports/Reports.java @@ -20,10 +20,10 @@ import java.util.Map; import io.cryostat.ConfigProperties; +import io.cryostat.StorageBuckets; import io.cryostat.core.reports.InterruptibleReportGenerator.AnalysisResult; import io.cryostat.recordings.RecordingHelper; import io.cryostat.targets.Target; -import io.cryostat.util.HttpStatusCodeIdentifier; import io.quarkus.runtime.StartupEvent; import io.smallrye.common.annotation.Blocking; @@ -42,47 +42,26 @@ import org.jboss.logging.Logger; import org.jboss.resteasy.reactive.RestPath; import org.jboss.resteasy.reactive.RestResponse; -import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.model.CreateBucketRequest; -import software.amazon.awssdk.services.s3.model.HeadBucketRequest; @Path("") public class Reports { - @Inject RecordingHelper helper; - @Inject ReportsService reportsService; - @Inject Logger logger; - @ConfigProperty(name = ConfigProperties.STORAGE_CACHE_ENABLED) boolean storageCacheEnabled; @ConfigProperty(name = ConfigProperties.ARCHIVED_REPORTS_STORAGE_CACHE_NAME) String bucket; - @Inject S3Client storage; + @Inject StorageBuckets storageBuckets; + @Inject RecordingHelper helper; + @Inject ReportsService reportsService; + @Inject Logger logger; // FIXME this observer cannot be declared on the StorageCachingReportsService decorator. // Refactor to put this somewhere more sensible void onStart(@Observes StartupEvent evt) { if (storageCacheEnabled) { - boolean exists = false; - try { - exists = - HttpStatusCodeIdentifier.isSuccessCode( - storage.headBucket( - HeadBucketRequest.builder().bucket(bucket).build()) - .sdkHttpResponse() - .statusCode()); - } catch (Exception e) { - logger.info(e); - } - if (!exists) { - try { - storage.createBucket(CreateBucketRequest.builder().bucket(bucket).build()); - } catch (Exception e) { - logger.error(e); - } - } + storageBuckets.createIfNecessary(bucket); } }