diff --git a/src/Microsoft.Health.Fhir.Core/Features/Operations/ConvertData/ContainerRegistryTemplateProvider.cs b/src/Microsoft.Health.Fhir.Core/Features/Operations/ConvertData/ContainerRegistryTemplateProvider.cs index 79dd962518..2bfa1f3e10 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Operations/ConvertData/ContainerRegistryTemplateProvider.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Operations/ConvertData/ContainerRegistryTemplateProvider.cs @@ -52,35 +52,25 @@ public ContainerRegistryTemplateProvider( } /// - /// Fetch template collection from container registry or built-in archive. + /// Fetch template collection from container registry following a custom template convert request. /// /// The convert data request which contains template reference. /// Cancellation token to cancel the fetch operation. /// Template collection. public async Task>> GetTemplateCollectionAsync(ConvertDataRequest request, CancellationToken cancellationToken) { - // We have embedded a default template collection in the templatemanagement package. - // If the template collection is the default reference, we don't need to retrieve token. - var accessToken = string.Empty; - if (!request.IsDefaultTemplateReference) - { - _logger.LogInformation("Using a custom template collection for data conversion."); - - async Task TokenEntryFactory(ICacheEntry entry) - { - var token = await _containerRegistryTokenProvider.GetTokenAsync(request.RegistryServer, cancellationToken); - entry.Size = token.Length; - entry.AbsoluteExpiration = GetTokenAbsoluteExpiration(token); - return token; - } + _logger.LogInformation("Using a custom template collection for data conversion."); - accessToken = await _cache.GetOrCreateAsync(GetCacheKey(request.RegistryServer), TokenEntryFactory); - } - else + async Task TokenEntryFactory(ICacheEntry entry) { - _logger.LogInformation("Using the default template collection for data conversion."); + var token = await _containerRegistryTokenProvider.GetTokenAsync(request.RegistryServer, cancellationToken); + entry.Size = token.Length; + entry.AbsoluteExpiration = GetTokenAbsoluteExpiration(token); + return token; } + var accessToken = await _cache.GetOrCreateAsync(GetCacheKey(request.RegistryServer), TokenEntryFactory); + try { var provider = _templateCollectionProviderFactory.CreateTemplateCollectionProvider(request.TemplateCollectionReference, accessToken); @@ -119,6 +109,7 @@ async Task TokenEntryFactory(ICacheEntry entry) private static DateTimeOffset GetTokenAbsoluteExpiration(string accessToken) { var defaultExpiration = DateTimeOffset.Now.AddMinutes(30); + if (accessToken.StartsWith("bearer ", StringComparison.OrdinalIgnoreCase)) { var jwtTokenText = accessToken.Substring(7); diff --git a/src/Microsoft.Health.Fhir.Core/Features/Operations/ConvertData/ConvertDataEngine.cs b/src/Microsoft.Health.Fhir.Core/Features/Operations/ConvertData/ConvertDataEngine.cs index d15a482986..71755a5496 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Operations/ConvertData/ConvertDataEngine.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Operations/ConvertData/ConvertDataEngine.cs @@ -23,22 +23,22 @@ namespace Microsoft.Health.Fhir.Core.Features.Operations.ConvertData { public class ConvertDataEngine : IConvertDataEngine { - private readonly IConvertDataTemplateProvider _convertDataTemplateProvider; + private readonly ITemplateProviderFactory _templateProviderFactory; private readonly ConvertDataConfiguration _convertDataConfiguration; private readonly ILogger _logger; private readonly Dictionary _converterMap = new Dictionary(); public ConvertDataEngine( - IConvertDataTemplateProvider convertDataTemplateProvider, + ITemplateProviderFactory templateProviderFactory, IOptions convertDataConfiguration, ILogger logger) { - EnsureArg.IsNotNull(convertDataTemplateProvider, nameof(convertDataTemplateProvider)); + EnsureArg.IsNotNull(templateProviderFactory, nameof(templateProviderFactory)); EnsureArg.IsNotNull(convertDataConfiguration, nameof(convertDataConfiguration)); EnsureArg.IsNotNull(logger, nameof(logger)); - _convertDataTemplateProvider = convertDataTemplateProvider; + _templateProviderFactory = templateProviderFactory; _convertDataConfiguration = convertDataConfiguration.Value; _logger = logger; @@ -47,7 +47,18 @@ public ConvertDataEngine( public async Task Process(ConvertDataRequest convertRequest, CancellationToken cancellationToken) { - var templateCollection = await _convertDataTemplateProvider.GetTemplateCollectionAsync(convertRequest, cancellationToken); + IConvertDataTemplateProvider convertDataTemplateProvider; + + if (convertRequest.IsDefaultTemplateReference) + { + convertDataTemplateProvider = _templateProviderFactory.GetDefaultTemplateProvider(); + } + else + { + convertDataTemplateProvider = _templateProviderFactory.GetContainerRegistryTemplateProvider(); + } + + List> templateCollection = await convertDataTemplateProvider.GetTemplateCollectionAsync(convertRequest, cancellationToken); ITemplateProvider templateProvider = new TemplateProvider(templateCollection); if (templateProvider == null) diff --git a/src/Microsoft.Health.Fhir.Core/Features/Operations/ConvertData/DefaultTemplateProvider.cs b/src/Microsoft.Health.Fhir.Core/Features/Operations/ConvertData/DefaultTemplateProvider.cs new file mode 100644 index 0000000000..05b4a8e55f --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Operations/ConvertData/DefaultTemplateProvider.cs @@ -0,0 +1,100 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using DotLiquid; +using EnsureThat; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.Health.Fhir.Core.Configs; +using Microsoft.Health.Fhir.Core.Features.Operations.ConvertData.Models; +using Microsoft.Health.Fhir.Core.Messages.ConvertData; +using Microsoft.Health.Fhir.TemplateManagement; +using Microsoft.Health.Fhir.TemplateManagement.Exceptions; + +namespace Microsoft.Health.Fhir.Core.Features.Operations.ConvertData +{ + public class DefaultTemplateProvider : IConvertDataTemplateProvider, IDisposable + { + private bool _disposed = false; + private readonly ILogger _logger; + private readonly MemoryCache _cache; + private readonly ITemplateCollectionProviderFactory _templateCollectionProviderFactory; + private readonly ConvertDataConfiguration _convertDataConfig; + + public DefaultTemplateProvider( + IOptions convertDataConfig, + ILogger logger) + { + EnsureArg.IsNotNull(convertDataConfig, nameof(convertDataConfig)); + EnsureArg.IsNotNull(logger, nameof(logger)); + + _convertDataConfig = convertDataConfig.Value; + + _logger = logger; + + // Initialize cache and template collection provider factory + _cache = new MemoryCache(new MemoryCacheOptions + { + SizeLimit = _convertDataConfig.CacheSizeLimit, + }); + _templateCollectionProviderFactory = new TemplateCollectionProviderFactory(_cache, Options.Create(_convertDataConfig.TemplateCollectionOptions)); + } + + /// + /// Fetch template collection from built-in archive following a default template convert request. + /// + /// The convert data request which contains template reference. + /// Cancellation token to cancel the fetch operation. + /// Template collection. + public async Task>> GetTemplateCollectionAsync(ConvertDataRequest request, CancellationToken cancellationToken) + { + var accessToken = string.Empty; + + _logger.LogInformation("Using the default template collection for data conversion."); + + try + { + var provider = _templateCollectionProviderFactory.CreateTemplateCollectionProvider(request.TemplateCollectionReference, accessToken); + return await provider.GetTemplateCollectionAsync(cancellationToken); + } + catch (TemplateManagementException templateEx) + { + _logger.LogWarning(templateEx, "Template collection is invalid."); + throw new TemplateCollectionErrorException(string.Format(Core.Resources.FetchTemplateCollectionFailed, templateEx.Message), templateEx); + } + catch (Exception unhandledEx) + { + _logger.LogError(unhandledEx, "Unhandled exception: failed to get template collection."); + throw new FetchTemplateCollectionFailedException(string.Format(Core.Resources.FetchTemplateCollectionFailed, unhandledEx.Message), unhandledEx); + } + } + + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + protected virtual void Dispose(bool disposing) + { + if (_disposed) + { + return; + } + + if (disposing) + { + _cache?.Dispose(); + } + + _disposed = true; + } + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Operations/ConvertData/FhirServerBuilderConvertDataRegistrationExtensions.cs b/src/Microsoft.Health.Fhir.Core/Features/Operations/ConvertData/FhirServerBuilderConvertDataRegistrationExtensions.cs index 327eb362f9..ea16e812b0 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Operations/ConvertData/FhirServerBuilderConvertDataRegistrationExtensions.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Operations/ConvertData/FhirServerBuilderConvertDataRegistrationExtensions.cs @@ -16,7 +16,7 @@ public static IFhirServerBuilder AddConvertData(this IFhirServerBuilder fhirServ { EnsureArg.IsNotNull(fhirServerBuilder, nameof(fhirServerBuilder)); - fhirServerBuilder.AddConvertDataTemplateProvider() + fhirServerBuilder.AddConvertDataTemplateProviders() .AddConvertDataEngine(); return fhirServerBuilder; @@ -29,9 +29,11 @@ private static IFhirServerBuilder AddConvertDataEngine(this IFhirServerBuilder f return fhirServerBuilder; } - private static IFhirServerBuilder AddConvertDataTemplateProvider(this IFhirServerBuilder fhirServerBuilder) + private static IFhirServerBuilder AddConvertDataTemplateProviders(this IFhirServerBuilder fhirServerBuilder) { - fhirServerBuilder.Services.AddSingleton(); + fhirServerBuilder.Services.AddSingleton(); + fhirServerBuilder.Services.AddSingleton(); + fhirServerBuilder.Services.AddSingleton(); return fhirServerBuilder; } diff --git a/src/Microsoft.Health.Fhir.Core/Features/Operations/ConvertData/ITemplateProviderFactory.cs b/src/Microsoft.Health.Fhir.Core/Features/Operations/ConvertData/ITemplateProviderFactory.cs new file mode 100644 index 0000000000..6a9e88582d --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Operations/ConvertData/ITemplateProviderFactory.cs @@ -0,0 +1,14 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +namespace Microsoft.Health.Fhir.Core.Features.Operations.ConvertData +{ + public interface ITemplateProviderFactory + { + public IConvertDataTemplateProvider GetContainerRegistryTemplateProvider(); + + public IConvertDataTemplateProvider GetDefaultTemplateProvider(); + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Operations/ConvertData/TemplateProviderFactory.cs b/src/Microsoft.Health.Fhir.Core/Features/Operations/ConvertData/TemplateProviderFactory.cs new file mode 100644 index 0000000000..6cb887f9bf --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Operations/ConvertData/TemplateProviderFactory.cs @@ -0,0 +1,31 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using EnsureThat; +using Microsoft.Extensions.DependencyInjection; + +namespace Microsoft.Health.Fhir.Core.Features.Operations.ConvertData +{ + public class TemplateProviderFactory : ITemplateProviderFactory + { + private readonly IServiceProvider _sp; + + public TemplateProviderFactory(IServiceProvider sp) + { + _sp = EnsureArg.IsNotNull(sp, nameof(sp)); + } + + public IConvertDataTemplateProvider GetContainerRegistryTemplateProvider() + { + return _sp.GetRequiredService(); + } + + public IConvertDataTemplateProvider GetDefaultTemplateProvider() + { + return _sp.GetRequiredService(); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Operations/ConvertData/ContainerRegistryTemplateProviderTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Operations/ConvertData/ContainerRegistryTemplateProviderTests.cs index 84f2989b3f..ed786e019c 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Operations/ConvertData/ContainerRegistryTemplateProviderTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Operations/ConvertData/ContainerRegistryTemplateProviderTests.cs @@ -38,19 +38,23 @@ public async Task GivenDefaultTemplateReference_WhenFetchingTemplates_DefaultTem } } - [Fact] - public async Task GivenAnInvalidToken_WhenFetchingCustomTemplates_ExceptionShouldBeThrown() + private IConvertDataTemplateProvider GetDefaultTemplateProvider() { - var containerRegistryTemplateProvider = GetDefaultTemplateProvider(); - var templateReference = "test.azurecr.io/templates:latest"; + var convertDataConfig = new ConvertDataConfiguration + { + Enabled = true, + OperationTimeout = TimeSpan.FromSeconds(1), + }; + convertDataConfig.ContainerRegistryServers.Add("test.azurecr.io"); - await Assert.ThrowsAsync(() => containerRegistryTemplateProvider.GetTemplateCollectionAsync(GetRequestWithTemplateReference(templateReference), CancellationToken.None)); + var config = Options.Create(convertDataConfig); + return new DefaultTemplateProvider(config, new NullLogger()); } - private IConvertDataTemplateProvider GetDefaultTemplateProvider() + private IConvertDataTemplateProvider GetCustomTemplateProvider() { IContainerRegistryTokenProvider tokenProvider = Substitute.For(); - tokenProvider.GetTokenAsync(default, default).ReturnsForAnyArgs("Bearer faketoken"); + tokenProvider.GetTokenAsync(default, default).ReturnsForAnyArgs("Faketoken"); var convertDataConfig = new ConvertDataConfiguration { diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Operations/ConvertData/ConvertDataEngineTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Operations/ConvertData/ConvertDataEngineTests.cs index 000bcc1a6c..4423264597 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Operations/ConvertData/ConvertDataEngineTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Operations/ConvertData/ConvertDataEngineTests.cs @@ -132,7 +132,7 @@ public async Task GivenFhirConvertDataRequest_WithADefaultTemplates_CorrectResul [InlineData("¶Š™œãý£¾.com/template:default")] public async Task GivenConvertDataRequest_WithUnconfiguredRegistry_ContainerRegistryNotConfiguredExceptionShouldBeThrown(string templateReference) { - var convertDataEngine = GetDefaultEngine(); + var convertDataEngine = GetCustomEngine(); var hl7v2Request = GetHl7V2RequestWithTemplateReference(templateReference); await Assert.ThrowsAsync(() => convertDataEngine.Process(hl7v2Request, CancellationToken.None)); @@ -256,7 +256,7 @@ public async Task GivenHl7V2TemplateNotInJsonFormat_WhenConvert_ExceptionShouldB { "ADT_A01", Template.Parse(@"""a"":""b""") }, }, }; - var convertDataEngine = GetCustomEngineWithTemplates(wrongTemplateCollection); + var convertDataEngine = GetDefaultEngineWithTemplates(wrongTemplateCollection); var request = GetHl7V2RequestWithDefaultTemplates(); var exception = await Assert.ThrowsAsync(() => convertDataEngine.Process(request, CancellationToken.None)); Assert.True(exception.InnerException is PostprocessException); @@ -272,7 +272,7 @@ public async Task GivenCcdaTemplateNotInJsonFormat_WhenConvert_ExceptionShouldBe { "CCD", Template.Parse(@"""a"":""b""") }, }, }; - var convertDataEngine = GetCustomEngineWithTemplates(wrongTemplateCollection); + var convertDataEngine = GetDefaultEngineWithTemplates(wrongTemplateCollection); var request = GetCcdaRequestWithDefaultTemplates(); var exception = await Assert.ThrowsAsync(() => convertDataEngine.Process(request, CancellationToken.None)); Assert.True(exception.InnerException is PostprocessException); @@ -288,7 +288,7 @@ public async Task GivenJsonTemplateNotInJsonFormat_WhenConvert_ExceptionShouldBe { "ExamplePatient", Template.Parse(@"""a"":""b""") }, }, }; - var convertDataEngine = GetCustomEngineWithTemplates(wrongTemplateCollection); + var convertDataEngine = GetDefaultEngineWithTemplates(wrongTemplateCollection); var request = GetJsonRequestWithDefaultTemplates(); var exception = await Assert.ThrowsAsync(() => convertDataEngine.Process(request, CancellationToken.None)); Assert.True(exception.InnerException is PostprocessException); @@ -304,7 +304,7 @@ public async Task GivenFhirTemplateNotInJsonFormat_WhenConvert_ExceptionShouldBe { "Patient", Template.Parse(@"""a"":""b""") }, }, }; - var convertDataEngine = GetCustomEngineWithTemplates(wrongTemplateCollection); + var convertDataEngine = GetDefaultEngineWithTemplates(wrongTemplateCollection); var request = GetFhirRequestWithDefaultTemplates(); var exception = await Assert.ThrowsAsync(() => convertDataEngine.Process(request, CancellationToken.None)); Assert.True(exception.InnerException is PostprocessException); @@ -393,22 +393,51 @@ private static ConvertDataRequest GetFhirRequestWithRootTemplate(string rootTemp private IConvertDataEngine GetDefaultEngine() { IOptions convertDataConfiguration = Options.Create(_config); - IContainerRegistryTokenProvider tokenProvider = Substitute.For(); - tokenProvider.GetTokenAsync(Arg.Any(), default).ReturnsForAnyArgs(x => GetToken(x[0].ToString(), _config)); - ContainerRegistryTemplateProvider templateProvider = new ContainerRegistryTemplateProvider(tokenProvider, convertDataConfiguration, new NullLogger()); + DefaultTemplateProvider templateProvider = new DefaultTemplateProvider(convertDataConfiguration, new NullLogger()); + ITemplateProviderFactory templateProviderFactory = Substitute.For(); + templateProviderFactory.GetDefaultTemplateProvider().ReturnsForAnyArgs(templateProvider); + + return new ConvertDataEngine( + templateProviderFactory, + convertDataConfiguration, + new NullLogger()); + } + + private IConvertDataEngine GetDefaultEngineWithTemplates(List> templateCollection) + { + var templateProviderFactory = Substitute.For(); + + IConvertDataTemplateProvider templateProvider = Substitute.For(); + + templateProvider.GetTemplateCollectionAsync(default, default).ReturnsForAnyArgs(templateCollection); + templateProviderFactory.GetDefaultTemplateProvider().ReturnsForAnyArgs(templateProvider); + return new ConvertDataEngine(templateProviderFactory, Options.Create(_config), new NullLogger()); + } + + private IConvertDataEngine GetCustomEngine() + { + IOptions convertDataConfiguration = Options.Create(_config); + IContainerRegistryTokenProvider containerRegistryTokenProvider = Substitute.For(); + containerRegistryTokenProvider.GetTokenAsync(Arg.Any(), default).ReturnsForAnyArgs(x => GetToken(x[0].ToString(), _config)); + + ContainerRegistryTemplateProvider templateProvider = new ContainerRegistryTemplateProvider(containerRegistryTokenProvider, convertDataConfiguration, new NullLogger()); + ITemplateProviderFactory templateProviderFactory = Substitute.For(); + templateProviderFactory.GetContainerRegistryTemplateProvider().ReturnsForAnyArgs(templateProvider); return new ConvertDataEngine( - templateProvider, + templateProviderFactory, convertDataConfiguration, new NullLogger()); } private IConvertDataEngine GetCustomEngineWithTemplates(List> templateCollection) { - var templateProvider = Substitute.For(); + var templateProviderFactory = Substitute.For(); + var templateProvider = Substitute.For(); templateProvider.GetTemplateCollectionAsync(default, default).ReturnsForAnyArgs(templateCollection); - return new ConvertDataEngine(templateProvider, Options.Create(_config), new NullLogger()); + templateProviderFactory.GetContainerRegistryTemplateProvider().ReturnsForAnyArgs(templateProvider); + return new ConvertDataEngine(templateProviderFactory, Options.Create(_config), new NullLogger()); } // For unit tests, we only use the built-in templates and here returns an empty token. diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Operations/ConvertData/ConvertDataRequestHandlerTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Operations/ConvertData/ConvertDataRequestHandlerTests.cs index 9494c86f4a..e36120f774 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Operations/ConvertData/ConvertDataRequestHandlerTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Operations/ConvertData/ConvertDataRequestHandlerTests.cs @@ -116,13 +116,12 @@ private ConvertDataRequestHandler GetRequestHandler() IOptions convertDataConfiguration = Options.Create(convertDataConfig); - IContainerRegistryTokenProvider tokenProvider = Substitute.For(); - tokenProvider.GetTokenAsync(Arg.Any(), default).ReturnsForAnyArgs(string.Empty); - - ContainerRegistryTemplateProvider templateProvider = new ContainerRegistryTemplateProvider(tokenProvider, convertDataConfiguration, new NullLogger()); + DefaultTemplateProvider templateProvider = new DefaultTemplateProvider(convertDataConfiguration, new NullLogger()); + ITemplateProviderFactory templateProviderFactory = Substitute.For(); + templateProviderFactory.GetDefaultTemplateProvider().Returns(templateProvider); var convertDataEngine = new ConvertDataEngine( - templateProvider, + templateProviderFactory, convertDataConfiguration, new NullLogger());