Convert Refactor for Missing External MI Case

src/Microsoft.Health.Fhir.Core/Features/Operations/ConvertData/ContainerRegistryTemplateProvider.cs

@@ -52,35 +52,25 @@ public ContainerRegistryTemplateProvider(
         }
 
         /// <summary>
-        /// Fetch template collection from container registry or built-in archive.
+        /// Fetch template collection from container registry following a custom template convert request.
         /// </summary>
         /// <param name="request">The convert data request which contains template reference.</param>
         /// <param name="cancellationToken">Cancellation token to cancel the fetch operation.</param>
         /// <returns>Template collection.</returns>
         public async Task<List<Dictionary<string, Template>>> GetTemplateCollectionAsync(ConvertDataRequest request, CancellationToken cancellationToken)
         {
-            // We have embedded a default template collection in the templatemanagement package.
-            // If the template collection is the default reference, we don't need to retrieve token.
-            var accessToken = string.Empty;
-            if (!request.IsDefaultTemplateReference)
-            {
-                _logger.LogInformation("Using a custom template collection for data conversion.");
-
-                async Task<string> TokenEntryFactory(ICacheEntry entry)
-                {
-                    var token = await _containerRegistryTokenProvider.GetTokenAsync(request.RegistryServer, cancellationToken);
-                    entry.Size = token.Length;
-                    entry.AbsoluteExpiration = GetTokenAbsoluteExpiration(token);
-                    return token;
-                }
+            _logger.LogInformation("Using a custom template collection for data conversion.");
 
-                accessToken = await _cache.GetOrCreateAsync(GetCacheKey(request.RegistryServer), TokenEntryFactory);
-            }
-            else
+            async Task<string> TokenEntryFactory(ICacheEntry entry)
             {
-                _logger.LogInformation("Using the default template collection for data conversion.");
+                var token = await _containerRegistryTokenProvider.GetTokenAsync(request.RegistryServer, cancellationToken);
+                entry.Size = token.Length;
+                entry.AbsoluteExpiration = GetTokenAbsoluteExpiration(token);
+                return token;
             }
 
+            var accessToken = await _cache.GetOrCreateAsync(GetCacheKey(request.RegistryServer), TokenEntryFactory);
+
             try
             {
                 var provider = _templateCollectionProviderFactory.CreateTemplateCollectionProvider(request.TemplateCollectionReference, accessToken);
@@ -119,6 +109,7 @@ async Task<string> TokenEntryFactory(ICacheEntry entry)
         private static DateTimeOffset GetTokenAbsoluteExpiration(string accessToken)
         {
             var defaultExpiration = DateTimeOffset.Now.AddMinutes(30);
+
             if (accessToken.StartsWith("bearer ", StringComparison.OrdinalIgnoreCase))
             {
                 var jwtTokenText = accessToken.Substring(7);

src/Microsoft.Health.Fhir.Core/Features/Operations/ConvertData/ConvertDataEngine.cs

@@ -23,22 +23,22 @@ namespace Microsoft.Health.Fhir.Core.Features.Operations.ConvertData
 {
     public class ConvertDataEngine : IConvertDataEngine
     {
-        private readonly IConvertDataTemplateProvider _convertDataTemplateProvider;
+        private readonly ITemplateProviderFactory _templateProviderFactory;
         private readonly ConvertDataConfiguration _convertDataConfiguration;
         private readonly ILogger<ConvertDataEngine> _logger;
 
         private readonly Dictionary<DataType, IFhirConverter> _converterMap = new Dictionary<DataType, IFhirConverter>();
 
         public ConvertDataEngine(
-            IConvertDataTemplateProvider convertDataTemplateProvider,
+            ITemplateProviderFactory templateProviderFactory,
             IOptions<ConvertDataConfiguration> convertDataConfiguration,
             ILogger<ConvertDataEngine> logger)
         {
-            EnsureArg.IsNotNull(convertDataTemplateProvider, nameof(convertDataTemplateProvider));
+            EnsureArg.IsNotNull(templateProviderFactory, nameof(templateProviderFactory));
             EnsureArg.IsNotNull(convertDataConfiguration, nameof(convertDataConfiguration));
             EnsureArg.IsNotNull(logger, nameof(logger));
 
-            _convertDataTemplateProvider = convertDataTemplateProvider;
+            _templateProviderFactory = templateProviderFactory;
             _convertDataConfiguration = convertDataConfiguration.Value;
             _logger = logger;
 
@@ -47,7 +47,18 @@ public ConvertDataEngine(
 
         public async Task<ConvertDataResponse> Process(ConvertDataRequest convertRequest, CancellationToken cancellationToken)
         {
-            var templateCollection = await _convertDataTemplateProvider.GetTemplateCollectionAsync(convertRequest, cancellationToken);
+            IConvertDataTemplateProvider convertDataTemplateProvider;
+
+            if (convertRequest.IsDefaultTemplateReference)
+            {
+                convertDataTemplateProvider = _templateProviderFactory.GetDefaultTemplateProvider();
+            }
+            else
+            {
+                convertDataTemplateProvider = _templateProviderFactory.GetContainerRegistryTemplateProvider();
+            }
+
+            List<Dictionary<string, DotLiquid.Template>> templateCollection = await convertDataTemplateProvider.GetTemplateCollectionAsync(convertRequest, cancellationToken);
 
             ITemplateProvider templateProvider = new TemplateProvider(templateCollection);
             if (templateProvider == null)

src/Microsoft.Health.Fhir.Core/Features/Operations/ConvertData/DefaultTemplateProvider.cs

@@ -0,0 +1,100 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+using DotLiquid;
+using EnsureThat;
+using Microsoft.Extensions.Caching.Memory;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using Microsoft.Health.Fhir.Core.Configs;
+using Microsoft.Health.Fhir.Core.Features.Operations.ConvertData.Models;
+using Microsoft.Health.Fhir.Core.Messages.ConvertData;
+using Microsoft.Health.Fhir.TemplateManagement;
+using Microsoft.Health.Fhir.TemplateManagement.Exceptions;
+
+namespace Microsoft.Health.Fhir.Core.Features.Operations.ConvertData
+{
+    public class DefaultTemplateProvider : IConvertDataTemplateProvider, IDisposable
+    {
+        private bool _disposed = false;
+        private readonly ILogger _logger;
+        private readonly MemoryCache _cache;
+        private readonly ITemplateCollectionProviderFactory _templateCollectionProviderFactory;
+        private readonly ConvertDataConfiguration _convertDataConfig;
+
+        public DefaultTemplateProvider(
+            IOptions<ConvertDataConfiguration> convertDataConfig,
+            ILogger<IConvertDataTemplateProvider> logger)
+        {
+            EnsureArg.IsNotNull(convertDataConfig, nameof(convertDataConfig));
+            EnsureArg.IsNotNull(logger, nameof(logger));
+
+            _convertDataConfig = convertDataConfig.Value;
+
+            _logger = logger;
+
+            // Initialize cache and template collection provider factory
+            _cache = new MemoryCache(new MemoryCacheOptions
+            {
+                SizeLimit = _convertDataConfig.CacheSizeLimit,
+            });
+            _templateCollectionProviderFactory = new TemplateCollectionProviderFactory(_cache, Options.Create(_convertDataConfig.TemplateCollectionOptions));
+        }
+
+        /// <summary>
+        /// Fetch template collection from built-in archive following a default template convert request.
+        /// </summary>
+        /// <param name="request">The convert data request which contains template reference.</param>
+        /// <param name="cancellationToken">Cancellation token to cancel the fetch operation.</param>
+        /// <returns>Template collection.</returns>
+        public async Task<List<Dictionary<string, Template>>> GetTemplateCollectionAsync(ConvertDataRequest request, CancellationToken cancellationToken)
+        {
+            var accessToken = string.Empty;
+
+            _logger.LogInformation("Using the default template collection for data conversion.");
+
+            try
+            {
+                var provider = _templateCollectionProviderFactory.CreateTemplateCollectionProvider(request.TemplateCollectionReference, accessToken);
+                return await provider.GetTemplateCollectionAsync(cancellationToken);
+            }
+            catch (TemplateManagementException templateEx)
+            {
+                _logger.LogWarning(templateEx, "Template collection is invalid.");
+                throw new TemplateCollectionErrorException(string.Format(Core.Resources.FetchTemplateCollectionFailed, templateEx.Message), templateEx);
+            }
+            catch (Exception unhandledEx)
+            {
+                _logger.LogError(unhandledEx, "Unhandled exception: failed to get template collection.");
+                throw new FetchTemplateCollectionFailedException(string.Format(Core.Resources.FetchTemplateCollectionFailed, unhandledEx.Message), unhandledEx);
+            }
+        }
+
+        public void Dispose()
+        {
+            Dispose(true);
+            GC.SuppressFinalize(this);
+        }
+
+        protected virtual void Dispose(bool disposing)
+        {
+            if (_disposed)
+            {
+                return;
+            }
+
+            if (disposing)
+            {
+                _cache?.Dispose();
+            }
+
+            _disposed = true;
+        }
+    }
+}

src/Microsoft.Health.Fhir.Core/Features/Operations/ConvertData/FhirServerBuilderConvertDataRegistrationExtensions.cs

@@ -16,7 +16,7 @@ public static IFhirServerBuilder AddConvertData(this IFhirServerBuilder fhirServ
         {
             EnsureArg.IsNotNull(fhirServerBuilder, nameof(fhirServerBuilder));
 
-            fhirServerBuilder.AddConvertDataTemplateProvider()
+            fhirServerBuilder.AddConvertDataTemplateProviders()
                 .AddConvertDataEngine();
 
             return fhirServerBuilder;
@@ -29,9 +29,11 @@ private static IFhirServerBuilder AddConvertDataEngine(this IFhirServerBuilder f
             return fhirServerBuilder;
         }
 
-        private static IFhirServerBuilder AddConvertDataTemplateProvider(this IFhirServerBuilder fhirServerBuilder)
+        private static IFhirServerBuilder AddConvertDataTemplateProviders(this IFhirServerBuilder fhirServerBuilder)
         {
-            fhirServerBuilder.Services.AddSingleton<IConvertDataTemplateProvider, ContainerRegistryTemplateProvider>();
+            fhirServerBuilder.Services.AddSingleton<DefaultTemplateProvider>();
+            fhirServerBuilder.Services.AddSingleton<ContainerRegistryTemplateProvider>();
+            fhirServerBuilder.Services.AddSingleton<ITemplateProviderFactory, TemplateProviderFactory>();
 
             return fhirServerBuilder;
         }

src/Microsoft.Health.Fhir.Core/Features/Operations/ConvertData/ITemplateProviderFactory.cs

@@ -0,0 +1,14 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+namespace Microsoft.Health.Fhir.Core.Features.Operations.ConvertData
+{
+    public interface ITemplateProviderFactory
+    {
+        public IConvertDataTemplateProvider GetContainerRegistryTemplateProvider();
+
+        public IConvertDataTemplateProvider GetDefaultTemplateProvider();
+    }
+}

src/Microsoft.Health.Fhir.Core/Features/Operations/ConvertData/TemplateProviderFactory.cs

@@ -0,0 +1,31 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+using System;
+using EnsureThat;
+using Microsoft.Extensions.DependencyInjection;
+
+namespace Microsoft.Health.Fhir.Core.Features.Operations.ConvertData
+{
+    public class TemplateProviderFactory : ITemplateProviderFactory
+    {
+        private readonly IServiceProvider _sp;
+
+        public TemplateProviderFactory(IServiceProvider sp)
+        {
+            _sp = EnsureArg.IsNotNull(sp, nameof(sp));
+        }
+
+        public IConvertDataTemplateProvider GetContainerRegistryTemplateProvider()
+        {
+            return _sp.GetRequiredService<ContainerRegistryTemplateProvider>();
+        }
+
+        public IConvertDataTemplateProvider GetDefaultTemplateProvider()
+        {
+            return _sp.GetRequiredService<DefaultTemplateProvider>();
+        }
+    }
+}

src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Operations/ConvertData/ContainerRegistryTemplateProviderTests.cs

@@ -38,19 +38,23 @@ public async Task GivenDefaultTemplateReference_WhenFetchingTemplates_DefaultTem
             }
         }
 
-        [Fact]
-        public async Task GivenAnInvalidToken_WhenFetchingCustomTemplates_ExceptionShouldBeThrown()
+        private IConvertDataTemplateProvider GetDefaultTemplateProvider()
         {
-            var containerRegistryTemplateProvider = GetDefaultTemplateProvider();
-            var templateReference = "test.azurecr.io/templates:latest";
+            var convertDataConfig = new ConvertDataConfiguration
+            {
+                Enabled = true,
+                OperationTimeout = TimeSpan.FromSeconds(1),
+            };
+            convertDataConfig.ContainerRegistryServers.Add("test.azurecr.io");
 
-            await Assert.ThrowsAsync<ContainerRegistryNotAuthorizedException>(() => containerRegistryTemplateProvider.GetTemplateCollectionAsync(GetRequestWithTemplateReference(templateReference), CancellationToken.None));
+            var config = Options.Create(convertDataConfig);
+            return new DefaultTemplateProvider(config, new NullLogger<DefaultTemplateProvider>());
         }
 
-        private IConvertDataTemplateProvider GetDefaultTemplateProvider()
+        private IConvertDataTemplateProvider GetCustomTemplateProvider()
         {
             IContainerRegistryTokenProvider tokenProvider = Substitute.For<IContainerRegistryTokenProvider>();
-            tokenProvider.GetTokenAsync(default, default).ReturnsForAnyArgs("Bearer faketoken");
+            tokenProvider.GetTokenAsync(default, default).ReturnsForAnyArgs("Faketoken");
 
             var convertDataConfig = new ConvertDataConfiguration
             {