From 429e2204db4e0a0afb663b42898a636a3b2dfa31 Mon Sep 17 00:00:00 2001 From: Ke Deng <39106214+mrkdeng@users.noreply.github.com> Date: Fri, 19 Jan 2024 09:58:26 -0800 Subject: [PATCH] feat: add regional datastore service (#2180) * add regional database service * update unit tests * update s3 bucket name for prod * update unit tests * reference to prod s3 bucket * remove unused constants * call http client if s3 client calls didn't succeed * add stage to s3 bucket name --- .../DependencyInjection.cs | 1 + .../Interface/IRegionalDatastoreService.cs | 11 +++ .../Utils/Constants.cs | 6 ++ .../Utils/RegionalDatastoreService.cs | 90 +++++++++++++++++++ .../Checkers/ExternalCompatibilityChecker.cs | 12 +-- .../Checkers/NugetCompatibilityChecker.cs | 4 +- ...PortabilityAnalyzerCompatibilityChecker.cs | 10 +-- .../Checkers/SdkCompatibilityChecker.cs | 4 +- ...ilityCheckerRecommendationActionHandler.cs | 8 +- .../PortingAssistantNugetHandlerTest.cs | 15 ++-- .../UnitTests/NugetHandlerTest.cs | 13 ++- 11 files changed, 146 insertions(+), 28 deletions(-) create mode 100644 src/PortingAssistant.Compatibility.Common/Interface/IRegionalDatastoreService.cs create mode 100644 src/PortingAssistant.Compatibility.Common/Utils/RegionalDatastoreService.cs diff --git a/src/PortingAssistant.Client.Client/DependencyInjection.cs b/src/PortingAssistant.Client.Client/DependencyInjection.cs index abfdca9f5..93df1ffce 100644 --- a/src/PortingAssistant.Client.Client/DependencyInjection.cs +++ b/src/PortingAssistant.Client.Client/DependencyInjection.cs @@ -38,6 +38,7 @@ public static void AddAssessment(this IServiceCollection serviceCollection, Port serviceCollection.AddTransient(); serviceCollection.AddTransient(); serviceCollection.AddTransient(); + serviceCollection.AddTransient(); } public static IAsyncPolicy GetRetryPolicy() diff --git a/src/PortingAssistant.Compatibility.Common/Interface/IRegionalDatastoreService.cs b/src/PortingAssistant.Compatibility.Common/Interface/IRegionalDatastoreService.cs new file mode 100644 index 000000000..cd73f4e71 --- /dev/null +++ b/src/PortingAssistant.Compatibility.Common/Interface/IRegionalDatastoreService.cs @@ -0,0 +1,11 @@ +namespace PortingAssistant.Compatibility.Common.Interface +{ + public interface IRegionalDatastoreService + { + public Task DownloadRegionalS3FileAsync(string fileToDownload, bool isRegionalCall = false); + + public Task> ListRegionalNamespacesObjectAsync(bool isRegionalCall = false); + + public Task DownloadGitHubFileAsync(string fileToDownload); + } +} diff --git a/src/PortingAssistant.Compatibility.Common/Utils/Constants.cs b/src/PortingAssistant.Compatibility.Common/Utils/Constants.cs index f3e0287d5..9a2861feb 100644 --- a/src/PortingAssistant.Compatibility.Common/Utils/Constants.cs +++ b/src/PortingAssistant.Compatibility.Common/Utils/Constants.cs @@ -11,5 +11,11 @@ public class Constants public const string DefaultAssessmentTargetFramework = "net6.0"; public const string DestinationKeySuffix = "compatibility-result.json"; + + public const string BetaStageName = "beta"; + + public const string GammaStageName = "gamma"; + + public const string ProdStageName = "prod"; } } diff --git a/src/PortingAssistant.Compatibility.Common/Utils/RegionalDatastoreService.cs b/src/PortingAssistant.Compatibility.Common/Utils/RegionalDatastoreService.cs new file mode 100644 index 000000000..4ae833aa2 --- /dev/null +++ b/src/PortingAssistant.Compatibility.Common/Utils/RegionalDatastoreService.cs @@ -0,0 +1,90 @@ +using Amazon; +using Amazon.S3; +using PortingAssistant.Compatibility.Common.Interface; +using Amazon.S3.Model; +using Microsoft.Extensions.Logging; +using System.Net; + +namespace PortingAssistant.Compatibility.Common.Utils +{ + public class RegionalDatastoreService : IRegionalDatastoreService + { + private readonly IHttpService _httpService; + private readonly AmazonS3Client _s3Client; + private readonly bool _isLambdaEnvSetup; + private readonly string _regionaS3BucketName; + private readonly ILogger _logger; + + public RegionalDatastoreService( + IHttpService httpService, + ILogger logger + ) + { + _httpService = httpService; + _logger = logger; + string region = Environment.GetEnvironmentVariable("AWS_REGION"); + string stage = Environment.GetEnvironmentVariable("stage"); + + if (!string.IsNullOrEmpty(region) && (stage == Constants.BetaStageName || stage == Constants.GammaStageName || stage == Constants.ProdStageName)) + { + _isLambdaEnvSetup = true; + _regionaS3BucketName = stage == Constants.ProdStageName ? + $"portingassistant-datastore-{region}" : $"portingassistant-datastore-{stage}-{region}"; + _logger.LogInformation($"Read stage, region from environment: {stage}, {region}, set S3 bucket name: {_regionaS3BucketName}"); + _s3Client = new AmazonS3Client(RegionEndpoint.GetBySystemName(region)); + } + } + + public async Task DownloadGitHubFileAsync(string fileToDownload) + { + return await _httpService.DownloadGitHubFileAsync(fileToDownload); + } + + public async Task DownloadRegionalS3FileAsync(string fileToDownload, bool isRegionalCall = false) + { + try + { + _logger.LogInformation($"Downloading {fileToDownload} from regional S3 " + _regionaS3BucketName); + if (isRegionalCall && _isLambdaEnvSetup) + { + GetObjectRequest request = new GetObjectRequest + { + BucketName = _regionaS3BucketName, + Key = fileToDownload + }; + using (GetObjectResponse response = await _s3Client.GetObjectAsync(request)) + { + if (response.HttpStatusCode == HttpStatusCode.OK) + { + _logger.LogInformation($"Downloaded {fileToDownload} from " + _regionaS3BucketName); + return response.ResponseStream; + } + else + { + _logger.LogWarning($"Issues during downloading through S3 client from " + _regionaS3BucketName); + _logger.LogInformation($"Downloading file through Http client..."); + return await _httpService.DownloadS3FileAsync(fileToDownload); + } + } + + } + else + { + return await _httpService.DownloadS3FileAsync(fileToDownload); + } + } + catch (Exception ex) + { + _logger.LogError($"fail to download {fileToDownload}. " + ex.Message); + return null; + } + + } + + // TODO: This method could be deprecated since sdk namespaces won't change after each feature release + public Task> ListRegionalNamespacesObjectAsync(bool isRegionalCall = false) + { + return _httpService.ListNamespacesObjectAsync(); + } + } +} diff --git a/src/PortingAssistant.Compatibility.Core/Checkers/ExternalCompatibilityChecker.cs b/src/PortingAssistant.Compatibility.Core/Checkers/ExternalCompatibilityChecker.cs index 6186e07fa..d93be9816 100644 --- a/src/PortingAssistant.Compatibility.Core/Checkers/ExternalCompatibilityChecker.cs +++ b/src/PortingAssistant.Compatibility.Core/Checkers/ExternalCompatibilityChecker.cs @@ -9,7 +9,7 @@ namespace PortingAssistant.Compatibility.Core.Checkers { public class ExternalCompatibilityChecker : ICompatibilityChecker { - private readonly IHttpService _httpService; + private readonly IRegionalDatastoreService _regionalDatastoreService; private static readonly int _maxProcessConcurrency = 3; private static readonly SemaphoreSlim _semaphore = new SemaphoreSlim(_maxProcessConcurrency); private ILogger _logger; @@ -17,10 +17,10 @@ public class ExternalCompatibilityChecker : ICompatibilityChecker public virtual PackageSourceType CompatibilityCheckerType => PackageSourceType.NUGET; public ExternalCompatibilityChecker( - IHttpService httpService, + IRegionalDatastoreService regionalDatastoreService, ILogger logger) { - _httpService = httpService; + _regionalDatastoreService = regionalDatastoreService; _logger = logger; } @@ -80,7 +80,7 @@ private async void ProcessCompatibility( IEnumerable package { HashSet? apis = null; // OriginalDefinition PackageDetails packageDetails = null; - packageDetails = await GetPackageDetailFromS3(fileToDownload, _httpService, apis); + packageDetails = await GetPackageDetailFromS3(fileToDownload, apis); if (packageDetails == null || packageDetails.Name == null || !string.Equals(packageDetails.Name.Trim().ToLower(), packageToDownload.Trim().ToLower(), StringComparison.OrdinalIgnoreCase)) @@ -190,9 +190,9 @@ public class PackageFromS3 } - public async Task GetPackageDetailFromS3(string fileToDownload, IHttpService httpService, HashSet apis = null) + public async Task GetPackageDetailFromS3(string fileToDownload, HashSet apis = null) { - using var stream = await httpService.DownloadS3FileAsync(fileToDownload); + using var stream = await _regionalDatastoreService.DownloadRegionalS3FileAsync(fileToDownload, isRegionalCall: true); if (stream == null) { return null; diff --git a/src/PortingAssistant.Compatibility.Core/Checkers/NugetCompatibilityChecker.cs b/src/PortingAssistant.Compatibility.Core/Checkers/NugetCompatibilityChecker.cs index 94bed688b..fcc38744c 100644 --- a/src/PortingAssistant.Compatibility.Core/Checkers/NugetCompatibilityChecker.cs +++ b/src/PortingAssistant.Compatibility.Core/Checkers/NugetCompatibilityChecker.cs @@ -9,10 +9,10 @@ public class NugetCompatibilityChecker : ExternalCompatibilityChecker public override PackageSourceType CompatibilityCheckerType => PackageSourceType.NUGET; public ILogger _logger; public NugetCompatibilityChecker( - IHttpService httpService, + IRegionalDatastoreService regionalDatastoreService, ILogger logger ) - : base(httpService, logger) + : base(regionalDatastoreService, logger) { } } diff --git a/src/PortingAssistant.Compatibility.Core/Checkers/PortabilityAnalyzerCompatibilityChecker.cs b/src/PortingAssistant.Compatibility.Core/Checkers/PortabilityAnalyzerCompatibilityChecker.cs index f206ddb38..b2d415d93 100644 --- a/src/PortingAssistant.Compatibility.Core/Checkers/PortabilityAnalyzerCompatibilityChecker.cs +++ b/src/PortingAssistant.Compatibility.Core/Checkers/PortabilityAnalyzerCompatibilityChecker.cs @@ -15,7 +15,7 @@ namespace PortingAssistant.Compatibility.Core.Checkers public class PortabilityAnalyzerCompatibilityChecker : ICompatibilityChecker { private const string NamespaceLookupFile = "microsoftlibs.namespace.lookup.json"; - private readonly IHttpService _httpService; + private readonly IRegionalDatastoreService _regionalDatastoreService; private Dictionary _manifest; private static readonly int _maxProcessConcurrency = 3; private static readonly SemaphoreSlim _semaphore = new SemaphoreSlim(_maxProcessConcurrency); @@ -27,11 +27,11 @@ public class PortabilityAnalyzerCompatibilityChecker : ICompatibilityChecker /// /// The transferUtility object to read data from S3 public PortabilityAnalyzerCompatibilityChecker( - IHttpService httpService, + IRegionalDatastoreService regionalDatastoreService, ILogger logger ) { - _httpService = httpService; + _regionalDatastoreService = regionalDatastoreService; _manifest = null; _logger = logger; } @@ -124,7 +124,7 @@ private async void ProcessCompatibility( IEnumerable package try { _logger.LogInformation($"Downloading {url.Key} from {CompatibilityCheckerType}"); - using var stream = await _httpService.DownloadS3FileAsync(url.Key); + using var stream = await _regionalDatastoreService.DownloadRegionalS3FileAsync(url.Key, isRegionalCall: true); using var gzipStream = new GZipStream(stream, CompressionMode.Decompress); using var streamReader = new StreamReader(gzipStream); var packageFromS3 = JsonConvert.DeserializeObject(streamReader.ReadToEnd()); @@ -185,7 +185,7 @@ private async void ProcessCompatibility( IEnumerable package private async Task> GetManifestAsync() { // Download the lookup file "microsoftlibs.namespace.lookup.json" from S3. - using var stream = await _httpService.DownloadS3FileAsync(NamespaceLookupFile); + using var stream = await _regionalDatastoreService.DownloadRegionalS3FileAsync(NamespaceLookupFile, isRegionalCall: true); using var streamReader = new StreamReader(stream); var result = streamReader.ReadToEnd(); return JsonConvert.DeserializeObject(result).ToObject>(); diff --git a/src/PortingAssistant.Compatibility.Core/Checkers/SdkCompatibilityChecker.cs b/src/PortingAssistant.Compatibility.Core/Checkers/SdkCompatibilityChecker.cs index 8034356f1..87d9b2935 100644 --- a/src/PortingAssistant.Compatibility.Core/Checkers/SdkCompatibilityChecker.cs +++ b/src/PortingAssistant.Compatibility.Core/Checkers/SdkCompatibilityChecker.cs @@ -14,9 +14,9 @@ public class SdkCompatibilityChecker : ExternalCompatibilityChecker public override PackageSourceType CompatibilityCheckerType => PackageSourceType.SDK; private ILogger _logger; public SdkCompatibilityChecker( - IHttpService httpService, + IRegionalDatastoreService regionalDatastoreService, ILogger logger) - : base(httpService, logger) + : base(regionalDatastoreService, logger) { } } diff --git a/src/PortingAssistant.Compatibility.Core/CompatibilityCheckerRecommendationActionHandler.cs b/src/PortingAssistant.Compatibility.Core/CompatibilityCheckerRecommendationActionHandler.cs index 7e02cc07c..96efb4bb1 100644 --- a/src/PortingAssistant.Compatibility.Core/CompatibilityCheckerRecommendationActionHandler.cs +++ b/src/PortingAssistant.Compatibility.Core/CompatibilityCheckerRecommendationActionHandler.cs @@ -9,18 +9,18 @@ namespace PortingAssistant.Compatibility.Core // The CompatibilityCheckerRecommendationActionHandler checks and gets recommendation action file details ("namespace.json") from the datastore, if any. public class CompatibilityCheckerRecommendationActionHandler : ICompatibilityCheckerRecommendationActionHandler { - private readonly IHttpService _httpService; + private readonly IRegionalDatastoreService _regionalDatastoreService; private const string _recommendationFileSuffix = ".json"; private ILogger _logger; public PackageSourceType CompatibilityCheckerType => PackageSourceType.RECOMMENDATION; public CompatibilityCheckerRecommendationActionHandler( - IHttpService httpService, + IRegionalDatastoreService regionalDatastoreService, ILogger logger ) { - _httpService = httpService; + _regionalDatastoreService = regionalDatastoreService; _logger = logger; } @@ -39,7 +39,7 @@ public async Task> GetRecomm Stream? stream = null; try { - stream = await _httpService.DownloadS3FileAsync(recommendationDownloadPath); + stream = await _regionalDatastoreService.DownloadRegionalS3FileAsync(recommendationDownloadPath, isRegionalCall: true); using var streamReader = new StreamReader(stream); var recommendationFromS3 = JsonConvert.DeserializeObject(await streamReader.ReadToEndAsync()); recommendationActionDetailsNamespaceDict.Add(namespaceName, recommendationFromS3); diff --git a/tests/PortingAssistant.Client.UnitTests/PortingAssistantNugetHandlerTest.cs b/tests/PortingAssistant.Client.UnitTests/PortingAssistantNugetHandlerTest.cs index bc9634d31..d4fddae00 100644 --- a/tests/PortingAssistant.Client.UnitTests/PortingAssistantNugetHandlerTest.cs +++ b/tests/PortingAssistant.Client.UnitTests/PortingAssistantNugetHandlerTest.cs @@ -6,6 +6,7 @@ using System.Security.Cryptography; using System.Threading; using System.Threading.Tasks; +using Amazon.S3; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Moq; @@ -16,6 +17,7 @@ using PortingAssistant.Client.Common.Utils; using PortingAssistant.Compatibility.Common.Interface; using PortingAssistant.Compatibility.Common.Model; +using PortingAssistant.Compatibility.Common.Utils; using PortingAssistant.Compatibility.Core; using PortingAssistant.Compatibility.Core.Checkers; using ILogger = NuGet.Common.ILogger; @@ -26,6 +28,7 @@ public class PortingAssistantNuGetHandlerTest { private Mock _httpService; private Mock _fileSystem; + private IRegionalDatastoreService _regionalDatastoreService; private ExternalCompatibilityChecker _externalPackagesCompatibilityChecker; private PortabilityAnalyzerCompatibilityChecker _portabilityAnalyzerCompatibilityChecker; private SdkCompatibilityChecker _sdkCompatibilityChecker; @@ -205,6 +208,8 @@ public void OneTimeSetup() { //httpMessageHandler = new Mock _httpService = new Mock(); + var datastoreServiceLoggerMock = new Mock>(); + _regionalDatastoreService = new RegionalDatastoreService(_httpService.Object, datastoreServiceLoggerMock.Object); _fileSystem = new Mock(); } @@ -248,22 +253,22 @@ public void Setup() _externalPackagesCompatibilityChecker = new ExternalCompatibilityChecker( - _httpService.Object, + _regionalDatastoreService, NullLogger.Instance ); _portabilityAnalyzerCompatibilityChecker = new PortabilityAnalyzerCompatibilityChecker( - _httpService.Object, + _regionalDatastoreService, NullLogger.Instance ); _sdkCompatibilityChecker = new SdkCompatibilityChecker( - _httpService.Object, + _regionalDatastoreService, NullLogger.Instance ); _portabilityAnalyzerCompatibilityChecker = new PortabilityAnalyzerCompatibilityChecker( - _httpService.Object, + _regionalDatastoreService, NullLogger.Instance ); @@ -367,7 +372,7 @@ private ICompatibilityCheckerNuGetHandler GetCheckerWithException() private ExternalCompatibilityChecker GetExternalPackagesCompatibilityChecker() { var externalChecker = new ExternalCompatibilityChecker( - _httpService.Object, + _regionalDatastoreService, NullLogger.Instance ); diff --git a/tests/PortingAssistant.Compatibility.Core.Tests/UnitTests/NugetHandlerTest.cs b/tests/PortingAssistant.Compatibility.Core.Tests/UnitTests/NugetHandlerTest.cs index dbebf12e5..a20d56d46 100644 --- a/tests/PortingAssistant.Compatibility.Core.Tests/UnitTests/NugetHandlerTest.cs +++ b/tests/PortingAssistant.Compatibility.Core.Tests/UnitTests/NugetHandlerTest.cs @@ -7,6 +7,8 @@ using PortingAssistant.Compatibility.Core.Checkers; using Assert = NUnit.Framework.Assert; using Microsoft.Extensions.Logging; +using PortingAssistant.Compatibility.Common.Utils; +using Amazon.S3; namespace PortingAssistant.Compatibility.Core.Tests.UnitTests { @@ -14,6 +16,7 @@ namespace PortingAssistant.Compatibility.Core.Tests.UnitTests public class NugetHandlerTest { private Mock _httpService; + private IRegionalDatastoreService _regionalDatastoreService; private Mock _compatibilityCheckerNuGetHandler; private NugetCompatibilityChecker _nugetCompatibilityChecker; private PortabilityAnalyzerCompatibilityChecker _portabilityAnalyzerCompatibilityChecker; @@ -67,6 +70,8 @@ public class NugetHandlerTest public void OneTimeSetup() { _httpService = new Mock(); + var datastoreServiceLoggerMock = new Mock>(); + _regionalDatastoreService = new RegionalDatastoreService(_httpService.Object, datastoreServiceLoggerMock.Object); } [SetUp] @@ -142,17 +147,17 @@ public void Setup() _logger = Mock.Of>(); _nugetCompatibilityChecker = new NugetCompatibilityChecker( - _httpService.Object, + _regionalDatastoreService, Mock.Of>() ); _portabilityAnalyzerCompatibilityChecker = new PortabilityAnalyzerCompatibilityChecker( - _httpService.Object, + _regionalDatastoreService, Mock.Of>() ); _sdkCompatibilityChecker = new SdkCompatibilityChecker( - _httpService.Object, + _regionalDatastoreService, Mock.Of>() ); @@ -207,7 +212,7 @@ private ICompatibilityCheckerNuGetHandler GetCheckerWithException() private NugetCompatibilityChecker GetExternalPackagesCompatibilityChecker() { var externalChecker = new NugetCompatibilityChecker( - _httpService.Object, Mock.Of>()); + _regionalDatastoreService, Mock.Of>()); return externalChecker; }