Skip to content

Commit

Permalink
chore(storage): refactor bucket check/create on startup (#321)
Browse files Browse the repository at this point in the history
  • Loading branch information
@andrewazores
andrewazores authored Mar 11, 2024
1 parent ec5fcd0 commit 141dcbe
Show file tree
Hide file tree
Showing 5 changed files with 91 additions and 100 deletions.
7 changes: 7 additions & 0 deletions src/main/java/io/cryostat/ExceptionMappers.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import org.jboss.resteasy.reactive.RestResponse.ResponseBuilder;
import org.jboss.resteasy.reactive.server.ServerExceptionMapper;
import org.projectnessie.cel.tools.ScriptException;
import software.amazon.awssdk.services.s3.model.NoSuchBucketException;
import software.amazon.awssdk.services.s3.model.NoSuchKeyException;

public class ExceptionMappers {
Expand All @@ -51,6 +52,12 @@ public RestResponse<Void> mapNoSuchElementException(NoSuchElementException ex) {
return RestResponse.notFound();
}

@ServerExceptionMapper
public RestResponse<Void> mapNoSuchBucketException(NoSuchBucketException ex) {
logger.error(ex);
return RestResponse.status(HttpResponseStatus.BAD_GATEWAY.code());
}

@ServerExceptionMapper
public RestResponse<Void> mapConstraintViolationException(ConstraintViolationException ex) {
logger.warn(ex);
Expand Down
56 changes: 56 additions & 0 deletions src/main/java/io/cryostat/StorageBuckets.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
/*
* Copyright The Cryostat Authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.cryostat;

import io.cryostat.util.HttpStatusCodeIdentifier;

import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import org.jboss.logging.Logger;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
import software.amazon.awssdk.services.s3.model.HeadBucketRequest;

@ApplicationScoped
public class StorageBuckets {

@Inject S3Client storage;
@Inject Logger logger;

public void createIfNecessary(String bucket) {
boolean exists = false;
logger.infov("Checking if storage bucket \"{0}\" exists ...", bucket);
try {
exists =
HttpStatusCodeIdentifier.isSuccessCode(
storage.headBucket(HeadBucketRequest.builder().bucket(bucket).build())
.sdkHttpResponse()
.statusCode());
logger.infov("Storage bucket \"{0}\" exists? {1}", bucket, exists);
} catch (Exception e) {
logger.info(e);
}
if (!exists) {
logger.infov("Attempting to create storage bucket \"{0}\" ...", bucket);
try {
storage.createBucket(CreateBucketRequest.builder().bucket(bucket).build());
logger.infov("Storage bucket \"{0}\" created", bucket);
} catch (Exception e) {
logger.error(e);
}
}
}
}
53 changes: 11 additions & 42 deletions src/main/java/io/cryostat/events/S3TemplateService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,13 +38,13 @@

import io.cryostat.ConfigProperties;
import io.cryostat.Producers;
import io.cryostat.StorageBuckets;
import io.cryostat.core.FlightRecorderException;
import io.cryostat.core.templates.MutableTemplateService;
import io.cryostat.core.templates.MutableTemplateService.InvalidEventTemplateException;
import io.cryostat.core.templates.MutableTemplateService.InvalidXmlException;
import io.cryostat.core.templates.Template;
import io.cryostat.core.templates.TemplateType;
import io.cryostat.util.HttpStatusCodeIdentifier;
import io.cryostat.ws.MessagingServer;
import io.cryostat.ws.Notification;

Expand All @@ -65,11 +65,9 @@
import org.jsoup.parser.Parser;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectTaggingRequest;
import software.amazon.awssdk.services.s3.model.HeadBucketRequest;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
import software.amazon.awssdk.services.s3.model.S3Object;
Expand All @@ -82,7 +80,11 @@ public class S3TemplateService implements MutableTemplateService {
static final String EVENT_TEMPLATE_CREATED = "TemplateUploaded";
static final String EVENT_TEMPLATE_DELETED = "TemplateDeleted";

@ConfigProperty(name = ConfigProperties.AWS_BUCKET_NAME_EVENT_TEMPLATES)
String bucket;

@Inject S3Client storage;
@Inject StorageBuckets storageBuckets;

@Inject EventBus bus;

Expand All @@ -92,33 +94,8 @@ public class S3TemplateService implements MutableTemplateService {

@Inject Logger logger;

@ConfigProperty(name = ConfigProperties.AWS_BUCKET_NAME_EVENT_TEMPLATES)
String eventTemplatesBucket;

void onStart(@Observes StartupEvent evt) {
// FIXME refactor this to a reusable utility method since this is done for custom event
// templates, archived recordings, and archived reports
boolean exists = false;
try {
exists =
HttpStatusCodeIdentifier.isSuccessCode(
storage.headBucket(
HeadBucketRequest.builder()
.bucket(eventTemplatesBucket)
.build())
.sdkHttpResponse()
.statusCode());
} catch (Exception e) {
logger.info(e);
}
if (!exists) {
try {
storage.createBucket(
CreateBucketRequest.builder().bucket(eventTemplatesBucket).build());
} catch (Exception e) {
logger.error(e);
}
}
storageBuckets.createIfNecessary(bucket);
}

@Override
Expand Down Expand Up @@ -170,17 +147,13 @@ public Optional<Document> getXml(String templateName, TemplateType unused)

@Blocking
private List<S3Object> getObjects() {
var builder = ListObjectsV2Request.builder().bucket(eventTemplatesBucket);
var builder = ListObjectsV2Request.builder().bucket(bucket);
return storage.listObjectsV2(builder.build()).contents();
}

@Blocking
private Template convertObject(S3Object object) throws InvalidEventTemplateException {
var req =
GetObjectTaggingRequest.builder()
.bucket(eventTemplatesBucket)
.key(object.key())
.build();
var req = GetObjectTaggingRequest.builder().bucket(bucket).key(object.key()).build();
var tagging = storage.getObjectTagging(req);
var list = tagging.tagSet();
if (!tagging.hasTagSet() || list.isEmpty()) {
Expand Down Expand Up @@ -222,7 +195,7 @@ private Template convertObject(S3Object object) throws InvalidEventTemplateExcep

@Blocking
private InputStream getModel(String name) {
var req = GetObjectRequest.builder().bucket(eventTemplatesBucket).key(name).build();
var req = GetObjectRequest.builder().bucket(bucket).key(name).build();
return storage.getObject(req);
}

Expand Down Expand Up @@ -273,7 +246,7 @@ public Template addTemplate(InputStream stream)
String provider = getAttributeValue(root, "provider");
storage.putObject(
PutObjectRequest.builder()
.bucket(eventTemplatesBucket)
.bucket(bucket)
.key(templateName)
.contentType(ContentType.APPLICATION_XML.getMimeType())
.tagging(createTemplateTagging(templateName, description, provider))
Expand Down Expand Up @@ -303,11 +276,7 @@ public void deleteTemplate(String templateName) {
.filter(t -> t.getName().equals(templateName))
.findFirst()
.orElseThrow();
var req =
DeleteObjectRequest.builder()
.bucket(eventTemplatesBucket)
.key(templateName)
.build();
var req = DeleteObjectRequest.builder().bucket(bucket).key(templateName).build();
if (storage.deleteObject(req).sdkHttpResponse().isSuccessful()) {
bus.publish(
MessagingServer.class.getName(),
Expand Down
42 changes: 11 additions & 31 deletions src/main/java/io/cryostat/recordings/Recordings.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@

import io.cryostat.ConfigProperties;
import io.cryostat.Producers;
import io.cryostat.StorageBuckets;
import io.cryostat.V2Response;
import io.cryostat.core.EventOptionsBuilder;
import io.cryostat.core.RecordingOptionsCustomizer;
Expand All @@ -56,7 +57,6 @@
import io.cryostat.targets.Target;
import io.cryostat.targets.TargetConnectionManager;
import io.cryostat.util.HttpMimeType;
import io.cryostat.util.HttpStatusCodeIdentifier;
import io.cryostat.ws.MessagingServer;
import io.cryostat.ws.Notification;

Expand Down Expand Up @@ -96,12 +96,10 @@
import org.jboss.resteasy.reactive.multipart.FileUpload;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
import software.amazon.awssdk.services.s3.model.Delete;
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.HeadBucketRequest;
import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
import software.amazon.awssdk.services.s3.model.S3Object;
Expand All @@ -112,26 +110,27 @@
@Path("")
public class Recordings {

@Inject Logger logger;
@Inject TargetConnectionManager connectionManager;
@Inject EventBus bus;
@Inject RecordingOptionsBuilderFactory recordingOptionsBuilderFactory;
@Inject RecordingOptionsCustomizer recordingOptionsCustomizer;
@Inject EventOptionsBuilder.Factory eventOptionsBuilderFactory;
@Inject Clock clock;
@Inject S3Client storage;
@Inject StorageBuckets storageBuckets;
@Inject S3Presigner presigner;
@Inject RemoteRecordingInputStreamFactory remoteRecordingStreamFactory;
@Inject ScheduledExecutorService scheduler;
@Inject ObjectMapper mapper;
@Inject RecordingHelper recordingHelper;
@Inject Logger logger;

@Inject
@Named(Producers.BASE64_URL)
Base64 base64Url;

@ConfigProperty(name = ConfigProperties.AWS_BUCKET_NAME_ARCHIVES)
String archiveBucket;
String bucket;

@ConfigProperty(name = ConfigProperties.GRAFANA_DATASOURCE_URL)
Optional<String> grafanaDatasourceURL;
Expand All @@ -149,26 +148,7 @@ public class Recordings {
Optional<String> externalStorageUrl;

void onStart(@Observes StartupEvent evt) {
boolean exists = false;
try {
exists =
HttpStatusCodeIdentifier.isSuccessCode(
storage.headBucket(
HeadBucketRequest.builder()
.bucket(archiveBucket)
.build())
.sdkHttpResponse()
.statusCode());
} catch (Exception e) {
logger.info(e);
}
if (!exists) {
try {
storage.createBucket(CreateBucketRequest.builder().bucket(archiveBucket).build());
} catch (Exception e) {
logger.error(e);
}
}
storageBuckets.createIfNecessary(bucket);
}

@GET
Expand Down Expand Up @@ -273,7 +253,7 @@ public void agentPush(
new Notification(event.category().category(), event.payload()));
storage.deleteObjects(
DeleteObjectsRequest.builder()
.bucket(archiveBucket)
.bucket(bucket)
.delete(
Delete.builder()
.objects(
Expand Down Expand Up @@ -362,7 +342,7 @@ Map<String, Object> doUpload(FileUpload recording, Metadata metadata, String jvm
String key = recordingHelper.archivedRecordingKey(jvmId, filename);
storage.putObject(
PutObjectRequest.builder()
.bucket(archiveBucket)
.bucket(bucket)
.key(key)
.contentType(RecordingHelper.JFR_MIME)
.tagging(recordingHelper.createMetadataTagging(new Metadata(labels)))
Expand Down Expand Up @@ -399,7 +379,7 @@ public void delete(@RestPath String filename) throws Exception {
// TODO scan all prefixes for matching filename? This is an old v1 API problem.
storage.deleteObject(
DeleteObjectRequest.builder()
.bucket(archiveBucket)
.bucket(bucket)
.key(String.format("%s/%s", "uploads", filename))
.build());
}
Expand Down Expand Up @@ -775,11 +755,11 @@ public void deleteArchivedRecording(@RestPath String jvmId, @RestPath String fil
connectUrl, metadata);
logger.infov(
"Sending S3 deletion request for {0} {1}",
archiveBucket, recordingHelper.archivedRecordingKey(jvmId, filename));
bucket, recordingHelper.archivedRecordingKey(jvmId, filename));
var resp =
storage.deleteObject(
DeleteObjectRequest.builder()
.bucket(archiveBucket)
.bucket(bucket)
.key(recordingHelper.archivedRecordingKey(jvmId, filename))
.build());
logger.infov(
Expand Down Expand Up @@ -1048,7 +1028,7 @@ public Response handleStorageDownload(@RestPath String encodedKey, @RestQuery St
logger.infov("Handling presigned download request for {0}", pair);
GetObjectRequest getRequest =
GetObjectRequest.builder()
.bucket(archiveBucket)
.bucket(bucket)
.key(recordingHelper.archivedRecordingKey(pair))
.build();
GetObjectPresignRequest presignRequest =
Expand Down
Loading

0 comments on commit 141dcbe

Please sign in to comment.