diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/BuildingBlocks.Infrastructure.csproj b/BuildingBlocks/src/BuildingBlocks.Infrastructure/BuildingBlocks.Infrastructure.csproj index ce022f0ff9..1dc048b99a 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/BuildingBlocks.Infrastructure.csproj +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/BuildingBlocks.Infrastructure.csproj @@ -2,20 +2,21 @@ + - - - - - - - + + + + + + + - + - - + + diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs index 1387b01adc..1f1f16c53c 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs @@ -1,5 +1,8 @@ +using System.ComponentModel.DataAnnotations; +using Backbone.BuildingBlocks.Application.Abstractions.Infrastructure.Persistence.BlobStorage; using Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.AzureStorageAccount; using Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.GoogleCloudStorage; +using Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.Ionos; using Backbone.Tooling.Extensions; using Microsoft.Extensions.DependencyInjection; @@ -9,6 +12,7 @@ public static class BlobStorageServiceCollectionExtensions { public const string AZURE_CLOUD_PROVIDER = "Azure"; public const string GOOGLE_CLOUD_PROVIDER = "GoogleCloud"; + public const string IONOS_CLOUD_PROVIDER = "Ionos"; public static void AddBlobStorage(this IServiceCollection services, Action setupOptions) { @@ -31,6 +35,19 @@ public static void AddBlobStorage(this IServiceCollection services, BlobStorageO googleCloudStorageOptions.GcpAuthJson = options.ConnectionInfo; googleCloudStorageOptions.BucketName = options.Container; }); + else if (options.CloudProvider == IONOS_CLOUD_PROVIDER) + { + services.Configure(opt => + { + opt.ServiceUrl = options.IonosS3Config.ServiceUrl; + opt.AccessKey = options.IonosS3Config.AccessKey; + opt.SecretKey = options.IonosS3Config.SecretKey; + opt.BucketName = options.IonosS3Config.BucketName; + }); + + services.AddSingleton(); + services.AddScoped(); + } else if (options.CloudProvider.IsNullOrEmpty()) throw new NotSupportedException("No cloud provider was specified."); else @@ -41,9 +58,29 @@ public static void AddBlobStorage(this IServiceCollection services, BlobStorageO public class BlobStorageOptions { - public string CloudProvider { get; set; } = null!; + [Required] + [MinLength(1)] + [RegularExpression("Azure|GoogleCloud|Ionos")] + public string CloudProvider { get; set; } = string.Empty; + + public string? ConnectionInfo { get; set; } = string.Empty; + public string? Container { get; set; } = string.Empty; + + public IonosS3Config? IonosS3Config { get; set; } = new IonosS3Config(); +} + +public class IonosS3Config +{ + [Required] + public string ServiceUrl { get; set; } = string.Empty; - public string Container { get; set; } = null!; + [Required] + public string AccessKey { get; set; } = string.Empty; - public string? ConnectionInfo { get; set; } = null; + [Required] + public string SecretKey { get; set; } = string.Empty; + + [Required] + public string BucketName { get; set; } = string.Empty; } + diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3BlobStorage.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3BlobStorage.cs new file mode 100644 index 0000000000..b703876f04 --- /dev/null +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3BlobStorage.cs @@ -0,0 +1,226 @@ +using Amazon.S3; +using Amazon.S3.Model; +using Amazon.S3.Transfer; +using Microsoft.Extensions.Logging; +using System.Net; +using Backbone.BuildingBlocks.Application.Abstractions.Exceptions; +using Backbone.BuildingBlocks.Application.Abstractions.Infrastructure.Persistence.BlobStorage; +using Microsoft.Extensions.Options; + +namespace Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.Ionos; +public class IonosS3BlobStorage : IBlobStorage, IDisposable +{ + private readonly IAmazonS3 _s3Client; + private readonly List _changedBlobs; + private readonly IList _removedBlobs; + private readonly string _bucketName; + private readonly IonosS3Options _config; + private readonly ILogger _logger; + + public IonosS3BlobStorage(IOptions config, ILogger logger) + { + var s3Config = new AmazonS3Config + { + ServiceURL = config.Value.ServiceUrl, + ForcePathStyle = true + }; + + _s3Client = new AmazonS3Client(config.Value.AccessKey, config.Value.SecretKey, s3Config); + _changedBlobs = new List(); + _removedBlobs = new List(); + _bucketName = config.Value.BucketName; + _logger = logger; + } + + public void Add(string folder, string id, byte[] content) + { + _changedBlobs.Add(new ChangedBlob(folder, id, content)); + } + + public void Remove(string folder, string id) + { + _removedBlobs.Add(new RemovedBlob(folder, id)); + } + + public void Dispose() + { + _changedBlobs.Clear(); + _removedBlobs.Clear(); + } + + public async Task FindAsync(string folder, string id) + { + _logger.LogTrace("Reading blob with key '{blobId}'...", id); + + try + { + var request = new GetObjectRequest + { + BucketName = _bucketName, + Key = $"{folder}/{id}" + }; + + using var response = await _s3Client.GetObjectAsync(request); + using var memoryStream = new MemoryStream(); + await response.ResponseStream.CopyToAsync(memoryStream); + + _logger.LogTrace("Found blob with key '{blobId}'.", id); + return memoryStream.ToArray(); + } + catch (AmazonS3Exception ex) when (ex.StatusCode == HttpStatusCode.NotFound) + { + _logger.LogError("A blob with key '{blobId}' was not found.", id); + throw new NotFoundException("Blob", ex); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error downloading blob with key '{blobId}'.", id); + throw; + } + } + + public Task> FindAllAsync(string folder, string? prefix = null) + { + return Task.FromResult(FindAllBlobsAsync(folder, prefix)); + } + + private async IAsyncEnumerable FindAllBlobsAsync(string folder, string? prefix) + { + _logger.LogTrace("Listing all blobs..."); + + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = prefix != null ? $"{folder}/{prefix}" : folder + }; + + ListObjectsV2Response response; + do + { + response = await _s3Client.ListObjectsV2Async(request); + + foreach (var obj in response.S3Objects) + { + yield return obj.Key; + } + + request.ContinuationToken = response.NextContinuationToken; + } while (response.IsTruncated); + + _logger.LogTrace("Found all blobs."); + } + + public async Task SaveAsync() + { + await UploadChangedBlobs(); + await DeleteRemovedBlobs(); + } + + private async Task UploadChangedBlobs() + { + _logger.LogTrace("Uploading '{changedBlobsCount}' changed blobs...", _changedBlobs.Count); + + var changedBlobs = new List(_changedBlobs); + + foreach (var blob in changedBlobs) + { + await EnsureKeyDoesNotExist(blob.Folder, blob.Name); + + using var memoryStream = new MemoryStream(blob.Content); + + try + { + _logger.LogTrace("Uploading blob with key '{blobName}'...", blob.Name); + + var request = new TransferUtilityUploadRequest + { + InputStream = memoryStream, + Key = $"{blob.Folder}/{blob.Name}", + BucketName = _bucketName + }; + + var transferUtility = new TransferUtility(_s3Client); + await transferUtility.UploadAsync(request); + + _logger.LogTrace("Upload of blob with key '{blobName}' was successful.", blob.Name); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error uploading blob with key '{blobName}'.", blob.Name); + throw; + } + finally + { + _changedBlobs.Remove(blob); + } + } + } + + private async Task EnsureKeyDoesNotExist(string folder, string key) + { + try + { + var request = new GetObjectRequest + { + BucketName = _bucketName, + Key = $"{folder}/{key}" + }; + + await _s3Client.GetObjectAsync(request); + + _logger.LogError("A blob with key '{blobName}' already exists.", key); + throw new BlobAlreadyExistsException(key); + } + catch (AmazonS3Exception ex) when (ex.StatusCode == HttpStatusCode.NotFound) + { + return; + } + } + + private async Task DeleteRemovedBlobs() + { + _logger.LogTrace("Deleting '{removedBlobsCount}' blobs...", _removedBlobs.Count); + + var blobsToDelete = new List(_removedBlobs); + + foreach (var blob in blobsToDelete) + { + try + { + var request = new DeleteObjectRequest + { + BucketName = _bucketName, + Key = $"{blob.Folder}/{blob.Name}" + }; + + await _s3Client.DeleteObjectAsync(request); + + _removedBlobs.Remove(blob); + } + catch (AmazonS3Exception ex) when (ex.StatusCode == HttpStatusCode.NotFound) + { + _logger.LogError("A blob with key '{blobId}' was not found.", blob.Name); + throw new NotFoundException($"Blob with key '{blob.Name}' was not found.", ex); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error deleting blob with key '{blobName}'.", blob.Name); + throw; + } + } + + _logger.LogTrace("Deletion successful."); + } + + private record ChangedBlob(string Folder, string Name, byte[] Content); + + private record RemovedBlob(string Folder, string Name); +} + +public class IonosS3Config +{ + public required string ServiceUrl { get; set; } + public required string AccessKey { get; set; } + public required string SecretKey { get; set; } + public required string BucketName { get; set; } +} diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3ClientFactory.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3ClientFactory.cs new file mode 100644 index 0000000000..5033100130 --- /dev/null +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3ClientFactory.cs @@ -0,0 +1,24 @@ +using Amazon.S3; +using Microsoft.Extensions.Options; + +namespace Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.Ionos; +public class IonosS3ClientFactory +{ + private readonly IonosS3Options _options; + + public IonosS3ClientFactory(IOptions options) + { + _options = options.Value; + } + + public IAmazonS3 CreateClient() + { + var config = new AmazonS3Config + { + ServiceURL = _options.ServiceUrl, + ForcePathStyle = true + }; + + return new AmazonS3Client(_options.AccessKey, _options.SecretKey, config); + } +} diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3ServiceCollectionExtensions.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3ServiceCollectionExtensions.cs new file mode 100644 index 0000000000..6497d9730a --- /dev/null +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3ServiceCollectionExtensions.cs @@ -0,0 +1,37 @@ +using Backbone.BuildingBlocks.Application.Abstractions.Infrastructure.Persistence.BlobStorage; +using Microsoft.Extensions.DependencyInjection; + +namespace Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.Ionos; +public static class IonosS3ServiceCollectionExtensions +{ + public static void AddIonosS3(this IServiceCollection services, + Action setupOptions) + { + var options = new IonosS3Options(); + setupOptions.Invoke(options); + + services.AddIonosS3(options); + } + + public static void AddIonosS3(this IServiceCollection services, IonosS3Options options) + { + services.Configure(opt => + { + opt.ServiceUrl = options.ServiceUrl; + opt.AccessKey = options.AccessKey; + opt.SecretKey = options.SecretKey; + opt.BucketName = options.BucketName; + }); + + services.AddSingleton(); + services.AddScoped(); + } +} + +public class IonosS3Options +{ + public string ServiceUrl { get; set; } = string.Empty; + public string AccessKey { get; set; } = string.Empty; + public string SecretKey { get; set; } = string.Empty; + public string BucketName { get; set; } = string.Empty; +} diff --git a/Modules/Files/src/Files.ConsumerApi/Configuration.cs b/Modules/Files/src/Files.ConsumerApi/Configuration.cs index 9a9feb333b..7f057fa8b3 100644 --- a/Modules/Files/src/Files.ConsumerApi/Configuration.cs +++ b/Modules/Files/src/Files.ConsumerApi/Configuration.cs @@ -23,12 +23,14 @@ public class BlobStorageConfiguration { [Required] [MinLength(1)] - [RegularExpression("Azure|GoogleCloud")] + [RegularExpression("Azure|GoogleCloud|Ionos")] public string CloudProvider { get; set; } = string.Empty; public string ConnectionInfo { get; set; } = string.Empty; public string ContainerName { get; set; } = string.Empty; + + public IonosS3Config IonosS3Config { get; set; } = new IonosS3Config(); } public class SqlDatabaseConfiguration @@ -42,5 +44,20 @@ public class SqlDatabaseConfiguration [MinLength(1)] public string ConnectionString { get; set; } = string.Empty; } + + public class IonosS3Config + { + [Required] + public string ServiceUrl { get; set; } = string.Empty; + + [Required] + public string AccessKey { get; set; } = string.Empty; + + [Required] + public string SecretKey { get; set; } = string.Empty; + + [Required] + public string BucketName { get; set; } = string.Empty; + } } } diff --git a/Modules/Files/src/Files.ConsumerApi/FilesModule.cs b/Modules/Files/src/Files.ConsumerApi/FilesModule.cs index 775e50d97b..1af6198108 100644 --- a/Modules/Files/src/Files.ConsumerApi/FilesModule.cs +++ b/Modules/Files/src/Files.ConsumerApi/FilesModule.cs @@ -35,6 +35,11 @@ public override void ConfigureServices(IServiceCollection services, IConfigurati parsedConfiguration.Infrastructure.BlobStorage.ContainerName.IsNullOrEmpty() ? "files" : parsedConfiguration.Infrastructure.BlobStorage.ContainerName; + + if (options.BlobStorageOptions.IonosS3Config != null) options.BlobStorageOptions.IonosS3Config.AccessKey = parsedConfiguration.Infrastructure.BlobStorage.IonosS3Config.AccessKey; + if (options.BlobStorageOptions.IonosS3Config != null) options.BlobStorageOptions.IonosS3Config.BucketName = parsedConfiguration.Infrastructure.BlobStorage.IonosS3Config.BucketName; + if (options.BlobStorageOptions.IonosS3Config != null) options.BlobStorageOptions.IonosS3Config.SecretKey = parsedConfiguration.Infrastructure.BlobStorage.IonosS3Config.SecretKey; + if (options.BlobStorageOptions.IonosS3Config != null) options.BlobStorageOptions.IonosS3Config.ServiceUrl = parsedConfiguration.Infrastructure.BlobStorage.IonosS3Config.ServiceUrl; }); services.AddSqlDatabaseHealthCheck(Name, parsedConfiguration.Infrastructure.SqlDatabase.Provider, parsedConfiguration.Infrastructure.SqlDatabase.ConnectionString); diff --git a/Modules/Files/src/Files.Jobs.SanityCheck/Extensions/IConfigurationExtensions.cs b/Modules/Files/src/Files.Jobs.SanityCheck/Extensions/IConfigurationExtensions.cs index 5f27947003..1f3b4c3a34 100644 --- a/Modules/Files/src/Files.Jobs.SanityCheck/Extensions/IConfigurationExtensions.cs +++ b/Modules/Files/src/Files.Jobs.SanityCheck/Extensions/IConfigurationExtensions.cs @@ -31,12 +31,27 @@ public class BlobStorageConfiguration { [Required] [MinLength(1)] - [RegularExpression("Azure|GoogleCloud")] + [RegularExpression("Azure|GoogleCloud|Ionos")] public string CloudProvider { get; set; } = string.Empty; - [Required] - [MinLength(1)] public string ConnectionInfo { get; set; } = string.Empty; public string ContainerName { get; set; } = string.Empty; + + public IonosS3Config IonosS3Config { get; set; } = new IonosS3Config(); +} + +public class IonosS3Config +{ + [Required] + public string ServiceUrl { get; set; } = string.Empty; + + [Required] + public string AccessKey { get; set; } = string.Empty; + + [Required] + public string SecretKey { get; set; } = string.Empty; + + [Required] + public string BucketName { get; set; } = string.Empty; }