feat: add regional datastore service (#2180)

* add regional database service * update unit tests * update s3 bucket name for prod * update unit tests * reference to prod s3 bucket * remove unused constants * call http client if s3 client calls didn't succeed * add stage to s3 bucket name
aws · Jan 19, 2024 · 429e220 · 429e220
1 parent b3b911d
commit 429e220
Show file tree

Hide file tree

Showing 11 changed files with 146 additions and 28 deletions.
diff --git a/src/PortingAssistant.Client.Client/DependencyInjection.cs b/src/PortingAssistant.Client.Client/DependencyInjection.cs
@@ -38,6 +38,7 @@ public static void AddAssessment(this IServiceCollection serviceCollection, Port
             serviceCollection.AddTransient<ICompatibilityChecker, Compatibility.Core.Checkers.SdkCompatibilityChecker>();
             serviceCollection.AddTransient<ICompatibilityChecker, Compatibility.Core.Checkers.PortabilityAnalyzerCompatibilityChecker>();
             serviceCollection.AddTransient<IHttpService, Compatibility.Common.Utils.HttpService>();
+            serviceCollection.AddTransient<IRegionalDatastoreService, Compatibility.Common.Utils.RegionalDatastoreService>();
         }
 
         public static IAsyncPolicy<HttpResponseMessage> GetRetryPolicy()

diff --git a/src/PortingAssistant.Compatibility.Common/Interface/IRegionalDatastoreService.cs b/src/PortingAssistant.Compatibility.Common/Interface/IRegionalDatastoreService.cs
@@ -0,0 +1,11 @@
+namespace PortingAssistant.Compatibility.Common.Interface
+{
+    public interface IRegionalDatastoreService
+    {
+        public Task<Stream?> DownloadRegionalS3FileAsync(string fileToDownload, bool isRegionalCall = false);
+
+        public Task<HashSet<string>> ListRegionalNamespacesObjectAsync(bool isRegionalCall = false);
+
+        public Task<Stream> DownloadGitHubFileAsync(string fileToDownload);
+    }
+}
diff --git a/src/PortingAssistant.Compatibility.Common/Utils/Constants.cs b/src/PortingAssistant.Compatibility.Common/Utils/Constants.cs
@@ -11,5 +11,11 @@ public class Constants
         public const string DefaultAssessmentTargetFramework = "net6.0";
 
         public const string DestinationKeySuffix = "compatibility-result.json";
+
+        public const string BetaStageName = "beta";
+
+        public const string GammaStageName = "gamma";
+
+        public const string ProdStageName = "prod";
     }
 }
diff --git a/src/PortingAssistant.Compatibility.Common/Utils/RegionalDatastoreService.cs b/src/PortingAssistant.Compatibility.Common/Utils/RegionalDatastoreService.cs
@@ -0,0 +1,90 @@
+using Amazon;
+using Amazon.S3;
+using PortingAssistant.Compatibility.Common.Interface;
+using Amazon.S3.Model;
+using Microsoft.Extensions.Logging;
+using System.Net;
+
+namespace PortingAssistant.Compatibility.Common.Utils
+{
+    public class RegionalDatastoreService : IRegionalDatastoreService
+    {
+        private readonly IHttpService _httpService;
+        private readonly AmazonS3Client _s3Client;
+        private readonly bool _isLambdaEnvSetup;
+        private readonly string _regionaS3BucketName;
+        private readonly ILogger<RegionalDatastoreService> _logger;
+
+        public RegionalDatastoreService(
+            IHttpService httpService,
+            ILogger<RegionalDatastoreService> logger
+            )
+        {
+            _httpService = httpService;
+            _logger = logger;
+            string region = Environment.GetEnvironmentVariable("AWS_REGION");
+            string stage = Environment.GetEnvironmentVariable("stage");
+
+            if (!string.IsNullOrEmpty(region) && (stage == Constants.BetaStageName || stage == Constants.GammaStageName || stage == Constants.ProdStageName)) 
+            {
+                _isLambdaEnvSetup = true;
+                _regionaS3BucketName = stage == Constants.ProdStageName ?
+                    $"portingassistant-datastore-{region}" : $"portingassistant-datastore-{stage}-{region}";
+                _logger.LogInformation($"Read stage, region from environment: {stage}, {region}, set S3 bucket name: {_regionaS3BucketName}");
+                _s3Client = new AmazonS3Client(RegionEndpoint.GetBySystemName(region));
+            }
+        }
+
+        public async Task<Stream> DownloadGitHubFileAsync(string fileToDownload)
+        {
+            return await _httpService.DownloadGitHubFileAsync(fileToDownload);
+        }
+
+        public async Task<Stream?> DownloadRegionalS3FileAsync(string fileToDownload, bool isRegionalCall = false)
+        {
+            try 
+            {
+                _logger.LogInformation($"Downloading {fileToDownload} from regional S3 " + _regionaS3BucketName);
+                if (isRegionalCall && _isLambdaEnvSetup)
+                {
+                    GetObjectRequest request = new GetObjectRequest
+                    {
+                        BucketName = _regionaS3BucketName,
+                        Key = fileToDownload
+                    };
+                    using (GetObjectResponse response = await _s3Client.GetObjectAsync(request))
+                    {
+                        if (response.HttpStatusCode == HttpStatusCode.OK)
+                        {
+                            _logger.LogInformation($"Downloaded {fileToDownload} from " + _regionaS3BucketName);
+                            return response.ResponseStream;
+                        }
+                        else
+                        {
+                            _logger.LogWarning($"Issues during downloading through S3 client from " + _regionaS3BucketName);
+                            _logger.LogInformation($"Downloading file through Http client...");
+                            return await _httpService.DownloadS3FileAsync(fileToDownload);
+                        }   
+                    }
+
+                }
+                else
+                {
+                    return await _httpService.DownloadS3FileAsync(fileToDownload);
+                }
+            }
+            catch (Exception ex) 
+            {
+                _logger.LogError($"fail to download {fileToDownload}. " + ex.Message);
+                return null;
+            }
+
+        }
+
+        // TODO: This method could be deprecated since sdk namespaces won't change after each feature release
+        public Task<HashSet<string>> ListRegionalNamespacesObjectAsync(bool isRegionalCall = false)
+        {
+            return _httpService.ListNamespacesObjectAsync();
+        }
+    }
+}
diff --git a/src/PortingAssistant.Compatibility.Core/Checkers/ExternalCompatibilityChecker.cs b/src/PortingAssistant.Compatibility.Core/Checkers/ExternalCompatibilityChecker.cs
@@ -9,18 +9,18 @@ namespace PortingAssistant.Compatibility.Core.Checkers
 {
     public class ExternalCompatibilityChecker : ICompatibilityChecker
     {
-        private readonly IHttpService _httpService;
+        private readonly IRegionalDatastoreService _regionalDatastoreService;
         private static readonly int _maxProcessConcurrency = 3;
         private static readonly SemaphoreSlim _semaphore = new SemaphoreSlim(_maxProcessConcurrency);
         private ILogger _logger;
 
         public virtual PackageSourceType CompatibilityCheckerType => PackageSourceType.NUGET;
 
         public ExternalCompatibilityChecker(
-            IHttpService httpService,
+            IRegionalDatastoreService regionalDatastoreService,
             ILogger<ExternalCompatibilityChecker> logger)
         {
-            _httpService = httpService;
+            _regionalDatastoreService = regionalDatastoreService;
             _logger = logger;
         }
 
@@ -80,7 +80,7 @@ private async void ProcessCompatibility( IEnumerable<PackageVersionPair> package
                 {
                     HashSet<string>? apis = null; // OriginalDefinition
                     PackageDetails packageDetails = null;
-                    packageDetails = await GetPackageDetailFromS3(fileToDownload, _httpService, apis); 
+                    packageDetails = await GetPackageDetailFromS3(fileToDownload, apis); 
 
                     if (packageDetails == null || packageDetails.Name == null || !string.Equals(packageDetails.Name.Trim().ToLower(),
                         packageToDownload.Trim().ToLower(), StringComparison.OrdinalIgnoreCase))
@@ -190,9 +190,9 @@ public class PackageFromS3
         }
 
 
-        public async Task<PackageDetails> GetPackageDetailFromS3(string fileToDownload, IHttpService httpService, HashSet<string> apis = null)
+        public async Task<PackageDetails> GetPackageDetailFromS3(string fileToDownload, HashSet<string> apis = null)
         {
-            using var stream = await httpService.DownloadS3FileAsync(fileToDownload);
+            using var stream = await _regionalDatastoreService.DownloadRegionalS3FileAsync(fileToDownload, isRegionalCall: true);
             if (stream == null)
             {
                 return null;

diff --git a/src/PortingAssistant.Compatibility.Core/Checkers/NugetCompatibilityChecker.cs b/src/PortingAssistant.Compatibility.Core/Checkers/NugetCompatibilityChecker.cs
@@ -9,10 +9,10 @@ public class NugetCompatibilityChecker : ExternalCompatibilityChecker
         public override PackageSourceType CompatibilityCheckerType => PackageSourceType.NUGET;
         public ILogger _logger;
         public NugetCompatibilityChecker(
-            IHttpService httpService,
+            IRegionalDatastoreService regionalDatastoreService,
             ILogger<NugetCompatibilityChecker> logger
         )
-            : base(httpService, logger)
+            : base(regionalDatastoreService, logger)
         {
         }
     }

diff --git a/src/PortingAssistant.Compatibility.Core/Checkers/PortabilityAnalyzerCompatibilityChecker.cs b/src/PortingAssistant.Compatibility.Core/Checkers/PortabilityAnalyzerCompatibilityChecker.cs
@@ -15,7 +15,7 @@ namespace PortingAssistant.Compatibility.Core.Checkers
     public class PortabilityAnalyzerCompatibilityChecker : ICompatibilityChecker
     {
         private const string NamespaceLookupFile = "microsoftlibs.namespace.lookup.json";
-        private readonly IHttpService _httpService;
+        private readonly IRegionalDatastoreService _regionalDatastoreService;
         private Dictionary<string, string> _manifest;
         private static readonly int _maxProcessConcurrency = 3;
         private static readonly SemaphoreSlim _semaphore = new SemaphoreSlim(_maxProcessConcurrency);
@@ -27,11 +27,11 @@ public class PortabilityAnalyzerCompatibilityChecker : ICompatibilityChecker
         /// </summary>
         /// <param name="httpService">The transferUtility object to read data from S3</param>
         public PortabilityAnalyzerCompatibilityChecker(
-            IHttpService httpService,
+            IRegionalDatastoreService regionalDatastoreService,
             ILogger<PortabilityAnalyzerCompatibilityChecker> logger
             )
         {
-            _httpService = httpService;
+            _regionalDatastoreService = regionalDatastoreService;
             _manifest = null;
             _logger = logger;
         }
@@ -124,7 +124,7 @@ private async void ProcessCompatibility( IEnumerable<PackageVersionPair> package
                 try
                 {
                     _logger.LogInformation($"Downloading {url.Key} from {CompatibilityCheckerType}");
-                    using var stream = await _httpService.DownloadS3FileAsync(url.Key);
+                    using var stream = await _regionalDatastoreService.DownloadRegionalS3FileAsync(url.Key, isRegionalCall: true);
                     using var gzipStream = new GZipStream(stream, CompressionMode.Decompress);
                     using var streamReader = new StreamReader(gzipStream);
                     var packageFromS3 = JsonConvert.DeserializeObject<PackageFromS3>(streamReader.ReadToEnd());
@@ -185,7 +185,7 @@ private async void ProcessCompatibility( IEnumerable<PackageVersionPair> package
         private async Task<Dictionary<string, string>> GetManifestAsync()
         {
             // Download the lookup file "microsoftlibs.namespace.lookup.json" from S3.
-            using var stream = await _httpService.DownloadS3FileAsync(NamespaceLookupFile);
+            using var stream = await _regionalDatastoreService.DownloadRegionalS3FileAsync(NamespaceLookupFile, isRegionalCall: true);
             using var streamReader = new StreamReader(stream);
             var result = streamReader.ReadToEnd();
             return JsonConvert.DeserializeObject<JObject>(result).ToObject<Dictionary<string, string>>();

diff --git a/src/PortingAssistant.Compatibility.Core/Checkers/SdkCompatibilityChecker.cs b/src/PortingAssistant.Compatibility.Core/Checkers/SdkCompatibilityChecker.cs
@@ -14,9 +14,9 @@ public class SdkCompatibilityChecker : ExternalCompatibilityChecker
         public override PackageSourceType CompatibilityCheckerType => PackageSourceType.SDK;
         private ILogger _logger;
         public SdkCompatibilityChecker(
-            IHttpService httpService,
+            IRegionalDatastoreService regionalDatastoreService,
             ILogger<SdkCompatibilityChecker> logger)
-            : base(httpService, logger)
+            : base(regionalDatastoreService, logger)
         {
         }
     }

diff --git a/src/PortingAssistant.Compatibility.Core/CompatibilityCheckerRecommendationActionHandler.cs b/src/PortingAssistant.Compatibility.Core/CompatibilityCheckerRecommendationActionHandler.cs
@@ -9,18 +9,18 @@ namespace PortingAssistant.Compatibility.Core
     // The CompatibilityCheckerRecommendationActionHandler checks and gets recommendation action file details ("namespace.json") from the datastore, if any.
     public class CompatibilityCheckerRecommendationActionHandler : ICompatibilityCheckerRecommendationActionHandler
     {
-        private readonly IHttpService _httpService;
+        private readonly IRegionalDatastoreService _regionalDatastoreService;
         private const string _recommendationFileSuffix = ".json";
         private ILogger _logger;
         public PackageSourceType CompatibilityCheckerType => PackageSourceType.RECOMMENDATION;
 
 
         public CompatibilityCheckerRecommendationActionHandler(
-            IHttpService httpService,
+            IRegionalDatastoreService regionalDatastoreService,
             ILogger<CompatibilityCheckerRecommendationActionHandler> logger
             )
         {
-            _httpService = httpService;
+            _regionalDatastoreService = regionalDatastoreService;
             _logger = logger;
         }
 
@@ -39,7 +39,7 @@ public async Task<Dictionary<string, RecommendationActionFileDetails>> GetRecomm
                 Stream? stream = null;
                 try
                 {
-                    stream = await _httpService.DownloadS3FileAsync(recommendationDownloadPath);
+                    stream = await _regionalDatastoreService.DownloadRegionalS3FileAsync(recommendationDownloadPath, isRegionalCall: true);
                     using var streamReader = new StreamReader(stream);
                     var recommendationFromS3 = JsonConvert.DeserializeObject<RecommendationActionFileDetails>(await streamReader.ReadToEndAsync());
                     recommendationActionDetailsNamespaceDict.Add(namespaceName, recommendationFromS3);

diff --git a/tests/PortingAssistant.Client.UnitTests/PortingAssistantNugetHandlerTest.cs b/tests/PortingAssistant.Client.UnitTests/PortingAssistantNugetHandlerTest.cs
@@ -6,6 +6,7 @@
 using System.Security.Cryptography;
 using System.Threading;
 using System.Threading.Tasks;
+using Amazon.S3;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Logging.Abstractions;
 using Moq;
@@ -16,6 +17,7 @@
 using PortingAssistant.Client.Common.Utils;
 using PortingAssistant.Compatibility.Common.Interface;
 using PortingAssistant.Compatibility.Common.Model;
+using PortingAssistant.Compatibility.Common.Utils;
 using PortingAssistant.Compatibility.Core;
 using PortingAssistant.Compatibility.Core.Checkers;
 using ILogger = NuGet.Common.ILogger;
@@ -26,6 +28,7 @@ public class PortingAssistantNuGetHandlerTest
     {
         private Mock<IHttpService> _httpService;
         private Mock<IFileSystem> _fileSystem;
+        private IRegionalDatastoreService _regionalDatastoreService;
         private ExternalCompatibilityChecker _externalPackagesCompatibilityChecker;
         private PortabilityAnalyzerCompatibilityChecker _portabilityAnalyzerCompatibilityChecker;
         private SdkCompatibilityChecker _sdkCompatibilityChecker;
@@ -205,6 +208,8 @@ public void OneTimeSetup()
         {
             //httpMessageHandler = new Mock<HttpMessageHandler>
             _httpService = new Mock<IHttpService>();
+            var datastoreServiceLoggerMock = new Mock<ILogger<RegionalDatastoreService>>();
+            _regionalDatastoreService = new RegionalDatastoreService(_httpService.Object, datastoreServiceLoggerMock.Object);
             _fileSystem = new Mock<IFileSystem>();
         }
 
@@ -248,22 +253,22 @@ public void Setup()
 
 
             _externalPackagesCompatibilityChecker = new ExternalCompatibilityChecker(
-                _httpService.Object,
+                _regionalDatastoreService,
                 NullLogger<ExternalCompatibilityChecker>.Instance
                 );
 
             _portabilityAnalyzerCompatibilityChecker = new PortabilityAnalyzerCompatibilityChecker(
-                _httpService.Object,
+                _regionalDatastoreService,
                 NullLogger<PortabilityAnalyzerCompatibilityChecker>.Instance
                 );
 
             _sdkCompatibilityChecker = new SdkCompatibilityChecker(
-                _httpService.Object,
+                _regionalDatastoreService,
                 NullLogger<SdkCompatibilityChecker>.Instance
                 );
 
             _portabilityAnalyzerCompatibilityChecker = new PortabilityAnalyzerCompatibilityChecker(
-                _httpService.Object,
+                _regionalDatastoreService,
                 NullLogger<PortabilityAnalyzerCompatibilityChecker>.Instance
                 );
 
@@ -367,7 +372,7 @@ private ICompatibilityCheckerNuGetHandler GetCheckerWithException()
         private ExternalCompatibilityChecker GetExternalPackagesCompatibilityChecker()
         {
             var externalChecker = new ExternalCompatibilityChecker(
-                _httpService.Object,
+                _regionalDatastoreService,
                 NullLogger<ExternalCompatibilityChecker>.Instance
             );
 

diff --git a/tests/PortingAssistant.Compatibility.Core.Tests/UnitTests/NugetHandlerTest.cs b/tests/PortingAssistant.Compatibility.Core.Tests/UnitTests/NugetHandlerTest.cs
@@ -7,13 +7,16 @@
 using PortingAssistant.Compatibility.Core.Checkers;
 using Assert = NUnit.Framework.Assert;
 using Microsoft.Extensions.Logging;
+using PortingAssistant.Compatibility.Common.Utils;
+using Amazon.S3;
 
 namespace PortingAssistant.Compatibility.Core.Tests.UnitTests
 {
 
     public class NugetHandlerTest
     {
         private Mock<IHttpService> _httpService;
+        private IRegionalDatastoreService _regionalDatastoreService;
         private Mock<ICompatibilityCheckerNuGetHandler> _compatibilityCheckerNuGetHandler;
         private NugetCompatibilityChecker _nugetCompatibilityChecker;
         private PortabilityAnalyzerCompatibilityChecker _portabilityAnalyzerCompatibilityChecker;
@@ -67,6 +70,8 @@ public class NugetHandlerTest
         public void OneTimeSetup()
         {
             _httpService = new Mock<IHttpService>();
+            var datastoreServiceLoggerMock = new Mock<ILogger<RegionalDatastoreService>>();
+            _regionalDatastoreService = new RegionalDatastoreService(_httpService.Object, datastoreServiceLoggerMock.Object);
         }
 
         [SetUp]
@@ -142,17 +147,17 @@ public void Setup()
              _logger = Mock.Of<ILogger<ICompatibilityChecker>>();
 
             _nugetCompatibilityChecker = new NugetCompatibilityChecker(
-                _httpService.Object,
+                _regionalDatastoreService,
                 Mock.Of<ILogger<NugetCompatibilityChecker>>()
                 );
 
             _portabilityAnalyzerCompatibilityChecker = new PortabilityAnalyzerCompatibilityChecker(
-                _httpService.Object,
+                _regionalDatastoreService,
                 Mock.Of<ILogger<PortabilityAnalyzerCompatibilityChecker>>()
                 );
 
             _sdkCompatibilityChecker = new SdkCompatibilityChecker(
-                _httpService.Object,
+                _regionalDatastoreService,
                 Mock.Of<ILogger<SdkCompatibilityChecker>>()
                 );
 
@@ -207,7 +212,7 @@ private ICompatibilityCheckerNuGetHandler GetCheckerWithException()
         private NugetCompatibilityChecker GetExternalPackagesCompatibilityChecker()
         {
             var externalChecker = new NugetCompatibilityChecker(
-                _httpService.Object, Mock.Of<ILogger<NugetCompatibilityChecker>>());
+                _regionalDatastoreService, Mock.Of<ILogger<NugetCompatibilityChecker>>());
 
             return externalChecker;
         }