-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: implement ionos s3 storage off of main branch
- Loading branch information
1 parent
ab36f7a
commit 9c08907
Showing
8 changed files
with
379 additions
and
17 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
226 changes: 226 additions & 0 deletions
226
...cks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3BlobStorage.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,226 @@ | ||
| using Amazon.S3; | ||
| using Amazon.S3.Model; | ||
| using Amazon.S3.Transfer; | ||
| using Microsoft.Extensions.Logging; | ||
| using System.Net; | ||
| using Backbone.BuildingBlocks.Application.Abstractions.Exceptions; | ||
| using Backbone.BuildingBlocks.Application.Abstractions.Infrastructure.Persistence.BlobStorage; | ||
| using Microsoft.Extensions.Options; | ||
|
|
||
| namespace Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.Ionos; | ||
| public class IonosS3BlobStorage : IBlobStorage, IDisposable | ||
| { | ||
| private readonly IAmazonS3 _s3Client; | ||
| private readonly List<ChangedBlob> _changedBlobs; | ||
| private readonly IList<RemovedBlob> _removedBlobs; | ||
| private readonly string _bucketName; | ||
| private readonly IonosS3Options _config; | ||
| private readonly ILogger<IonosS3BlobStorage> _logger; | ||
|
|
||
| public IonosS3BlobStorage(IOptions<IonosS3Options> config, ILogger<IonosS3BlobStorage> logger) | ||
| { | ||
| var s3Config = new AmazonS3Config | ||
| { | ||
| ServiceURL = config.Value.ServiceUrl, | ||
| ForcePathStyle = true | ||
| }; | ||
|
|
||
| _s3Client = new AmazonS3Client(config.Value.AccessKey, config.Value.SecretKey, s3Config); | ||
| _changedBlobs = new List<ChangedBlob>(); | ||
| _removedBlobs = new List<RemovedBlob>(); | ||
| _bucketName = config.Value.BucketName; | ||
| _logger = logger; | ||
| } | ||
|
|
||
| public void Add(string folder, string id, byte[] content) | ||
| { | ||
| _changedBlobs.Add(new ChangedBlob(folder, id, content)); | ||
| } | ||
|
|
||
| public void Remove(string folder, string id) | ||
| { | ||
| _removedBlobs.Add(new RemovedBlob(folder, id)); | ||
| } | ||
|
|
||
| public void Dispose() | ||
| { | ||
| _changedBlobs.Clear(); | ||
| _removedBlobs.Clear(); | ||
| } | ||
|
|
||
| public async Task<byte[]> FindAsync(string folder, string id) | ||
| { | ||
| _logger.LogTrace("Reading blob with key '{blobId}'...", id); | ||
|
|
||
| try | ||
| { | ||
| var request = new GetObjectRequest | ||
| { | ||
| BucketName = _bucketName, | ||
| Key = $"{folder}/{id}" | ||
| }; | ||
|
|
||
| using var response = await _s3Client.GetObjectAsync(request); | ||
| using var memoryStream = new MemoryStream(); | ||
| await response.ResponseStream.CopyToAsync(memoryStream); | ||
|
|
||
| _logger.LogTrace("Found blob with key '{blobId}'.", id); | ||
| return memoryStream.ToArray(); | ||
| } | ||
| catch (AmazonS3Exception ex) when (ex.StatusCode == HttpStatusCode.NotFound) | ||
| { | ||
| _logger.LogError("A blob with key '{blobId}' was not found.", id); | ||
| throw new NotFoundException("Blob", ex); | ||
| } | ||
| catch (Exception ex) | ||
| { | ||
| _logger.LogError(ex, "Error downloading blob with key '{blobId}'.", id); | ||
| throw; | ||
| } | ||
| } | ||
|
|
||
| public Task<IAsyncEnumerable<string>> FindAllAsync(string folder, string? prefix = null) | ||
| { | ||
| return Task.FromResult(FindAllBlobsAsync(folder, prefix)); | ||
| } | ||
|
|
||
| private async IAsyncEnumerable<string> FindAllBlobsAsync(string folder, string? prefix) | ||
| { | ||
| _logger.LogTrace("Listing all blobs..."); | ||
|
|
||
| var request = new ListObjectsV2Request | ||
| { | ||
| BucketName = _bucketName, | ||
| Prefix = prefix != null ? $"{folder}/{prefix}" : folder | ||
| }; | ||
|
|
||
| ListObjectsV2Response response; | ||
| do | ||
| { | ||
| response = await _s3Client.ListObjectsV2Async(request); | ||
|
|
||
| foreach (var obj in response.S3Objects) | ||
| { | ||
| yield return obj.Key; | ||
| } | ||
|
|
||
| request.ContinuationToken = response.NextContinuationToken; | ||
| } while (response.IsTruncated); | ||
|
|
||
| _logger.LogTrace("Found all blobs."); | ||
| } | ||
|
|
||
| public async Task SaveAsync() | ||
| { | ||
| await UploadChangedBlobs(); | ||
| await DeleteRemovedBlobs(); | ||
| } | ||
|
|
||
| private async Task UploadChangedBlobs() | ||
| { | ||
| _logger.LogTrace("Uploading '{changedBlobsCount}' changed blobs...", _changedBlobs.Count); | ||
|
|
||
| var changedBlobs = new List<ChangedBlob>(_changedBlobs); | ||
|
|
||
| foreach (var blob in changedBlobs) | ||
| { | ||
| await EnsureKeyDoesNotExist(blob.Folder, blob.Name); | ||
|
|
||
| using var memoryStream = new MemoryStream(blob.Content); | ||
|
|
||
| try | ||
| { | ||
| _logger.LogTrace("Uploading blob with key '{blobName}'...", blob.Name); | ||
|
|
||
| var request = new TransferUtilityUploadRequest | ||
| { | ||
| InputStream = memoryStream, | ||
| Key = $"{blob.Folder}/{blob.Name}", | ||
| BucketName = _bucketName | ||
| }; | ||
|
|
||
| var transferUtility = new TransferUtility(_s3Client); | ||
| await transferUtility.UploadAsync(request); | ||
|
|
||
| _logger.LogTrace("Upload of blob with key '{blobName}' was successful.", blob.Name); | ||
| } | ||
| catch (Exception ex) | ||
| { | ||
| _logger.LogError(ex, "Error uploading blob with key '{blobName}'.", blob.Name); | ||
| throw; | ||
| } | ||
| finally | ||
| { | ||
| _changedBlobs.Remove(blob); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| private async Task EnsureKeyDoesNotExist(string folder, string key) | ||
| { | ||
| try | ||
| { | ||
| var request = new GetObjectRequest | ||
| { | ||
| BucketName = _bucketName, | ||
| Key = $"{folder}/{key}" | ||
| }; | ||
|
|
||
| await _s3Client.GetObjectAsync(request); | ||
|
|
||
| _logger.LogError("A blob with key '{blobName}' already exists.", key); | ||
| throw new BlobAlreadyExistsException(key); | ||
| } | ||
| catch (AmazonS3Exception ex) when (ex.StatusCode == HttpStatusCode.NotFound) | ||
| { | ||
| return; | ||
| } | ||
| } | ||
|
|
||
| private async Task DeleteRemovedBlobs() | ||
| { | ||
| _logger.LogTrace("Deleting '{removedBlobsCount}' blobs...", _removedBlobs.Count); | ||
|
|
||
| var blobsToDelete = new List<RemovedBlob>(_removedBlobs); | ||
|
|
||
| foreach (var blob in blobsToDelete) | ||
| { | ||
| try | ||
| { | ||
| var request = new DeleteObjectRequest | ||
| { | ||
| BucketName = _bucketName, | ||
| Key = $"{blob.Folder}/{blob.Name}" | ||
| }; | ||
|
|
||
| await _s3Client.DeleteObjectAsync(request); | ||
|
|
||
| _removedBlobs.Remove(blob); | ||
| } | ||
| catch (AmazonS3Exception ex) when (ex.StatusCode == HttpStatusCode.NotFound) | ||
| { | ||
| _logger.LogError("A blob with key '{blobId}' was not found.", blob.Name); | ||
| throw new NotFoundException($"Blob with key '{blob.Name}' was not found.", ex); | ||
| } | ||
| catch (Exception ex) | ||
| { | ||
| _logger.LogError(ex, "Error deleting blob with key '{blobName}'.", blob.Name); | ||
| throw; | ||
| } | ||
| } | ||
|
|
||
| _logger.LogTrace("Deletion successful."); | ||
| } | ||
|
|
||
| private record ChangedBlob(string Folder, string Name, byte[] Content); | ||
|
|
||
| private record RemovedBlob(string Folder, string Name); | ||
| } | ||
|
|
||
| public class IonosS3Config | ||
| { | ||
| public required string ServiceUrl { get; set; } | ||
| public required string AccessKey { get; set; } | ||
| public required string SecretKey { get; set; } | ||
| public required string BucketName { get; set; } | ||
| } |
24 changes: 24 additions & 0 deletions
24
...s/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3ClientFactory.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,24 @@ | ||
| using Amazon.S3; | ||
| using Microsoft.Extensions.Options; | ||
|
|
||
| namespace Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.Ionos; | ||
| public class IonosS3ClientFactory | ||
| { | ||
| private readonly IonosS3Options _options; | ||
|
|
||
| public IonosS3ClientFactory(IOptions<IonosS3Options> options) | ||
| { | ||
| _options = options.Value; | ||
| } | ||
|
|
||
| public IAmazonS3 CreateClient() | ||
| { | ||
| var config = new AmazonS3Config | ||
| { | ||
| ServiceURL = _options.ServiceUrl, | ||
| ForcePathStyle = true | ||
| }; | ||
|
|
||
| return new AmazonS3Client(_options.AccessKey, _options.SecretKey, config); | ||
| } | ||
| } |
37 changes: 37 additions & 0 deletions
37
...Blocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3ServiceCollectionExtensions.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,37 @@ | ||
| using Backbone.BuildingBlocks.Application.Abstractions.Infrastructure.Persistence.BlobStorage; | ||
| using Microsoft.Extensions.DependencyInjection; | ||
|
|
||
| namespace Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.Ionos; | ||
| public static class IonosS3ServiceCollectionExtensions | ||
| { | ||
| public static void AddIonosS3(this IServiceCollection services, | ||
| Action<IonosS3Options> setupOptions) | ||
| { | ||
| var options = new IonosS3Options(); | ||
| setupOptions.Invoke(options); | ||
|
|
||
| services.AddIonosS3(options); | ||
| } | ||
|
|
||
| public static void AddIonosS3(this IServiceCollection services, IonosS3Options options) | ||
| { | ||
| services.Configure<IonosS3Options>(opt => | ||
| { | ||
| opt.ServiceUrl = options.ServiceUrl; | ||
| opt.AccessKey = options.AccessKey; | ||
| opt.SecretKey = options.SecretKey; | ||
| opt.BucketName = options.BucketName; | ||
| }); | ||
|
|
||
| services.AddSingleton<IonosS3ClientFactory>(); | ||
| services.AddScoped<IBlobStorage, IonosS3BlobStorage>(); | ||
| } | ||
| } | ||
|
|
||
| public class IonosS3Options | ||
| { | ||
| public string ServiceUrl { get; set; } = string.Empty; | ||
| public string AccessKey { get; set; } = string.Empty; | ||
| public string SecretKey { get; set; } = string.Empty; | ||
| public string BucketName { get; set; } = string.Empty; | ||
| } |
Oops, something went wrong.