From 77776c718272e14c9980d094df1e6225252a1d44 Mon Sep 17 00:00:00 2001 From: Matt Serdar <99214729+v-mserdar@users.noreply.github.com> Date: Fri, 17 Mar 2023 10:32:18 -0700 Subject: [PATCH 1/9] new import tool --- Microsoft.Health.Fhir.sln | 9 +- tools/ImporterV2/App.config | 30 ++ tools/ImporterV2/ImportResponse.cs | 33 +++ tools/ImporterV2/ImporterV2.cs | 424 +++++++++++++++++++++++++++++ tools/ImporterV2/ImporterV2.csproj | 23 ++ tools/ImporterV2/Program.cs | 17 ++ tools/ImporterV2/readme.md | 4 + 7 files changed, 539 insertions(+), 1 deletion(-) create mode 100644 tools/ImporterV2/App.config create mode 100644 tools/ImporterV2/ImportResponse.cs create mode 100644 tools/ImporterV2/ImporterV2.cs create mode 100644 tools/ImporterV2/ImporterV2.csproj create mode 100644 tools/ImporterV2/Program.cs create mode 100644 tools/ImporterV2/readme.md diff --git a/Microsoft.Health.Fhir.sln b/Microsoft.Health.Fhir.sln index 27ad3a0af0..98a6bb1cf7 100644 --- a/Microsoft.Health.Fhir.sln +++ b/Microsoft.Health.Fhir.sln @@ -146,6 +146,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.TaskManage EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.Fhir.Store.Utils", "src\Microsoft.Health.Fhir.Store.Utils\Microsoft.Health.Fhir.Store.Utils.csproj", "{7A736E5F-DA6E-483F-AD5B-EE8F66828E36}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ImporterV2", "tools\ImporterV2\ImporterV2.csproj", "{E85BCB9A-5D6E-45AD-BE67-AEFA060FEBF1}" +EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Importer", "tools\Importer\Importer.csproj", "{F6B94905-B496-46AD-B2A7-2ABCB0B2A6B4}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Exporter", "tools\Exporter\Exporter.csproj", "{E468B6C6-9098-4293-AFD6-3B1675D67063}" @@ -408,6 +410,10 @@ Global {F4DE2945-80C5-48FE-B58A-4AD1264C9FEA}.Debug|Any CPU.Build.0 = Debug|Any CPU {F4DE2945-80C5-48FE-B58A-4AD1264C9FEA}.Release|Any CPU.ActiveCfg = Release|Any CPU {F4DE2945-80C5-48FE-B58A-4AD1264C9FEA}.Release|Any CPU.Build.0 = Release|Any CPU + {E85BCB9A-5D6E-45AD-BE67-AEFA060FEBF1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E85BCB9A-5D6E-45AD-BE67-AEFA060FEBF1}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E85BCB9A-5D6E-45AD-BE67-AEFA060FEBF1}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E85BCB9A-5D6E-45AD-BE67-AEFA060FEBF1}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -489,10 +495,11 @@ Global {5E456FD7-E5A5-41F4-A0D3-7215585AEB7A} = {FCD51BAF-BFE5-476F-B562-C9AB36AA9839} {A5DED132-32B1-4804-95F5-EBC6092EC8AE} = {85F39C13-BC62-49A0-9C06-3BBA724D35ED} {D6C90E8C-50AF-45D8-B2D3-1B9B07E65F3E} = {1295CCC3-73FB-4376-AE95-F6F31A37B152} + {E85BCB9A-5D6E-45AD-BE67-AEFA060FEBF1} = {B70945F4-01A6-4351-955B-C4A2943B5E3B} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution - SolutionGuid = {E370FB31-CF95-47D1-B1E1-863A77973FF8} RESX_SortFileContentOnSave = True + SolutionGuid = {E370FB31-CF95-47D1-B1E1-863A77973FF8} EndGlobalSection GlobalSection(SharedMSBuildProjectFiles) = preSolution test\Microsoft.Health.Fhir.Shared.Tests.E2E.Common\Microsoft.Health.Fhir.Shared.Tests.E2E.Common.projitems*{0478b687-7105-40f6-a2dc-81057890e944}*SharedItemsImports = 13 diff --git a/tools/ImporterV2/App.config b/tools/ImporterV2/App.config new file mode 100644 index 0000000000..d1ee84da94 --- /dev/null +++ b/tools/ImporterV2/App.config @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + + + + + + + diff --git a/tools/ImporterV2/ImportResponse.cs b/tools/ImporterV2/ImportResponse.cs new file mode 100644 index 0000000000..4ac3e9b551 --- /dev/null +++ b/tools/ImporterV2/ImportResponse.cs @@ -0,0 +1,33 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace Microsoft.Health.Fhir.ImporterV2 +{ +#pragma warning disable CA1812 // Avoid uninstantiated internal classes + internal sealed class ImportResponse + { + [JsonPropertyName("error")] + public List Error { get; set; } = new(); + + [JsonPropertyName("output")] + public List Output { get; set; } = new(); + + public sealed class Json + { + [JsonPropertyName("type")] + public string Type { get; set; } = string.Empty; + + [JsonPropertyName("count")] + public int Count { get; set; } + + [JsonPropertyName("inputUrl")] + public string InputUrl { get; set; } = string.Empty; + } + } +#pragma warning disable CA1812 // Avoid uninstantiated internal classes +} diff --git a/tools/ImporterV2/ImporterV2.cs b/tools/ImporterV2/ImporterV2.cs new file mode 100644 index 0000000000..d8cbb7416f --- /dev/null +++ b/tools/ImporterV2/ImporterV2.cs @@ -0,0 +1,424 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Configuration; +using System.Diagnostics; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; + +namespace Microsoft.Health.Fhir.ImporterV2 +{ + internal static class ImporterV2 + { + private static readonly string TokenEndpoint = ConfigurationManager.AppSettings["TokenEndpoint"] ?? string.Empty; + private static readonly string TokenGrantType = ConfigurationManager.AppSettings["grant_type"] ?? string.Empty; + private static readonly string TokenClientId = ConfigurationManager.AppSettings["client_id"] ?? string.Empty; + private static readonly string TokenClientSecret = ConfigurationManager.AppSettings["client_secret"] ?? string.Empty; + private static readonly string TokenResource = ConfigurationManager.AppSettings["resource"] ?? string.Empty; + private static readonly string ResourceType = ConfigurationManager.AppSettings["ResourceType"] ?? string.Empty; + private static readonly string ContainerName = ConfigurationManager.AppSettings["ContainerName"] ?? string.Empty; + private static readonly string ConnectionString = ConfigurationManager.AppSettings["ConnectionString"] ?? string.Empty; + private static readonly string FhirEndpoint = ConfigurationManager.AppSettings["FhirEndpoint"] ?? string.Empty; + private static readonly int NumberOfBlobsForImport = int.Parse(ConfigurationManager.AppSettings["NumberOfBlobsForImport"] ?? "1"); + private static readonly HttpClient HttpClient = new(); + private static BlobContainerClient s_blobContainerClientSource; + private static List s_blobItems; + private static HashSet s_importedBlobNames = new(); + private static readonly string OutputDirectory = Path.Combine(Directory.GetCurrentDirectory(), "importResults"); + private static readonly string OutputFileName = Path.Combine(OutputDirectory, "importer.txt"); + private static readonly string LocationUrlFileName = Path.Combine(OutputDirectory, "locationUrls.txt"); + + internal static async Task Run() + { + try + { + await Init(); + + // this may take too long and timeout so it may be worth commenting out if you have too many resources + // int currentResourceCount = await GetCurrentResourceCount(); + // Console.WriteLine($"{currentResourceCount:N0} {ResourceType} found"); + + // If it's ever needed to finish checking on a current import then add the full url here + // all attempted Urls for GetImportStatus should be saved to the LocationUrlFileName file + // await GetImportStatus("https://.fhir.azurehealthcareapis.com/_operations/import/32"); + + await RunImport(); + } + catch (Exception e) + { + Console.WriteLine(e); + } + finally + { + await WriteImportedBlobNames(); + } + } + + private static async Task Init() + { + if (!Directory.Exists(OutputDirectory)) + { + Directory.CreateDirectory(OutputDirectory); + } + + s_blobContainerClientSource = GetContainerClient(ContainerName); + await LoadImportedBlobItems(); + GetBlobItems(); + } + + private static async Task WriteLocationUrls(string url) + { + var names = new HashSet(); + if (File.Exists(LocationUrlFileName)) + { + var content = await File.ReadAllLinesAsync(LocationUrlFileName); + names = content.ToHashSet(); + } + + names.Add(url); + + await File.WriteAllTextAsync(LocationUrlFileName, string.Join(Environment.NewLine, names)); + } + + private static async Task LoadImportedBlobItems() + { + if (File.Exists(OutputFileName)) + { + var content = await File.ReadAllLinesAsync(OutputFileName); + s_importedBlobNames = content.ToHashSet(); + } + + Console.WriteLine($"Found {s_importedBlobNames.Count} blobs already processed."); + } + + private static async Task WriteImportedBlobNames() + { + if (File.Exists(OutputFileName)) + { + File.Delete(OutputFileName); + } + + if (s_importedBlobNames != null) + { + await File.WriteAllTextAsync(OutputFileName, string.Join(Environment.NewLine, s_importedBlobNames)); + Console.WriteLine($"Saved file: {OutputFileName}"); + } + } + + private static void GetBlobItems() + { + if (s_blobContainerClientSource != null) + { + s_blobItems = s_blobContainerClientSource.GetBlobs().Where(_ => _.Name.EndsWith($"{ResourceType}.ndjson", true, CultureInfo.CurrentCulture)).ToList(); + Console.WriteLine($"Total container BlobItems count = {s_blobItems.Count}"); + + foreach (string item in s_importedBlobNames) + { + var blobItem = s_blobItems.Where(e => e.Name.Contains(item, StringComparison.OrdinalIgnoreCase)).FirstOrDefault(); + if (blobItem != null) + { + s_blobItems.Remove(blobItem); + } + } + + s_blobItems = s_blobItems.Take(NumberOfBlobsForImport).ToList(); + + Console.WriteLine($"Working set of BlobItems count = {s_blobItems.Count}"); + } + } + + private static async Task RunImport() + { + if (s_blobContainerClientSource == null || s_blobItems == null || s_blobItems.Count == 0) + { +#pragma warning disable CA1303 // Do not pass literals as localized parameters + Console.WriteLine("No items to import."); +#pragma warning restore CA1303 // Do not pass literals as localized parameters + return; + } + + var sb = new StringBuilder(); + sb.Append("{\"resourceType\": \"Parameters\", \"parameter\": [{\"name\": \"inputFormat\",\"valueString\": \"application/fhir+ndjson\"},{\"name\": \"mode\",\"valueString\": \"InitialLoad\"},"); + + var size = 0L; + + foreach (var blob in s_blobItems) + { + if (blob.Properties.ContentLength != null) + { + size += blob.Properties.ContentLength.Value; + } + + var type = blob.Name.Split('/')[1].Split('.')[0]; + sb.AppendLine("{\"name\": \"input\",\"part\": [{\"name\": \"type\",\"valueString\": "); + sb.Append('"'); + sb.Append(type); + sb.AppendLine("\""); + sb.AppendLine(" },{\"name\": \"url\",\"valueUri\": "); + sb.Append('"'); + sb.Append($"{s_blobContainerClientSource.Uri}/{blob.Name}"); + sb.AppendLine("\""); + sb.AppendLine("}]},"); + } + + sb.Append("{\"name\": \"storageDetail\",\"part\": [{\"name\": \"type\",\"valueString\": \"azure-blob\"}]}]}"); + + Console.WriteLine($"TotalSize for blobs ={size:N0}"); + + var json = sb.ToString(); + var data = new StringContent(json, Encoding.UTF8, "application/fhir+json"); + var url = $"{FhirEndpoint}/$import"; + using var request = new HttpRequestMessage() + { + RequestUri = new Uri(url), + Method = HttpMethod.Post, + Content = data, + }; + + request.Headers.Accept.Add(new System.Net.Http.Headers.MediaTypeWithQualityHeaderValue("application/fhir+json")); + request.Headers.Add("Prefer", "respond-async"); + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", await GetToken()); + var response = await HttpClient.SendAsync(request); + + Console.WriteLine($"response.IsSuccessStatusCode = {response.IsSuccessStatusCode}"); + string content = await response.Content.ReadAsStringAsync(); + Console.WriteLine(content); + + if (response.Content.Headers.Contains("Content-Location")) + { + IEnumerable location = response.Content.Headers.GetValues("Content-Location"); + await GetImportStatus(location.First()); + } + } + + private static async Task GetImportStatus(string url) + { + await WriteLocationUrls(url); + + TimeSpan delay = TimeSpan.FromMinutes(5); + var swTotalTime = new Stopwatch(); + var swSingleTime = new Stopwatch(); + swTotalTime.Start(); + swSingleTime.Start(); + + while (true) + { + using var get = new HttpRequestMessage() + { + RequestUri = new Uri(url), + Method = HttpMethod.Get, + }; + + get.Headers.Authorization = new AuthenticationHeaderValue("Bearer", await GetToken()); + var response = await HttpClient.SendAsync(get, CancellationToken.None).ConfigureAwait(false); + string content = await response.Content.ReadAsStringAsync(); + + Console.WriteLine($"{Environment.NewLine}GET {get.RequestUri}"); + Console.WriteLine($"StatusCode = {response.StatusCode}"); + + ImportResponse importJson = TryParseJson(content); + PrintImportResponse(importJson); + bool addedUrl = SaveImportedUrl(importJson); + + Console.WriteLine($"{(addedUrl ? "Completed" : "Processing")} file elapsed time: {swSingleTime.Elapsed.Duration()}"); + if (addedUrl) + { + swSingleTime.Restart(); + } + + if (response.StatusCode == HttpStatusCode.OK) + { + swTotalTime.Stop(); + Console.WriteLine($"Completed: Total running time: {swTotalTime.Elapsed.Duration()}"); + break; + } + else if (response.StatusCode == HttpStatusCode.Accepted) + { + Console.WriteLine($"Total running time: {swTotalTime.Elapsed.Duration()} - awaiting {delay} before retry"); + await Task.Delay(delay); + continue; + } + else + { + Console.WriteLine($"Failed to get expected status for import: {url}"); + break; + } + } + + swSingleTime.Stop(); + swTotalTime.Stop(); + } + + private static bool SaveImportedUrl(ImportResponse response) + { + bool addedUrl = false; + if (response != null && s_blobContainerClientSource != null) + { + // in case there are only error conditions and no success in the output + foreach (ImportResponse.Json r in response.Error) + { + if (s_importedBlobNames.Add(r.InputUrl.Replace(s_blobContainerClientSource.Uri.ToString() + "/", string.Empty, StringComparison.OrdinalIgnoreCase))) + { + addedUrl = true; + } + } + + foreach (ImportResponse.Json r in response.Output) + { + if (s_importedBlobNames.Add(r.InputUrl.Replace(s_blobContainerClientSource.Uri.ToString() + "/", string.Empty, StringComparison.OrdinalIgnoreCase))) + { + addedUrl = true; + } + } + } + + return addedUrl; + } + + private static void PrintImportResponse(ImportResponse response) + { + if (response != null) + { + if (response.Output.Count == 0 && response.Error.Count == 0) + { +#pragma warning disable CA1303 // Do not pass literals as localized parameters + Console.WriteLine("No import results to report on"); +#pragma warning restore CA1303 // Do not pass literals as localized parameters + return; + } + + PrintResponse(response.Output); + if (response.Error.Count > 0) + { + Console.BackgroundColor = ConsoleColor.Black; + Console.ForegroundColor = ConsoleColor.Red; + PrintResponse(response.Error); + Console.ResetColor(); + } + } + } + + private static void PrintResponse(List response) + { + foreach (var item in response) + { + Console.WriteLine($"{string.Format("{0, 3}", response.IndexOf(item) + 1)} {string.Format("{0, 10}", item.Count.ToString("N0"))} {item.InputUrl}"); + } + } + + private static ImportResponse TryParseJson(string value) + { + ImportResponse parsedJson = default; + + if (string.IsNullOrWhiteSpace(value)) + { + return parsedJson; + } + else + { + value = value.Trim(); + if (value.StartsWith("{", StringComparison.Ordinal) && value.EndsWith("}", StringComparison.Ordinal)) + { + try + { + parsedJson = JsonConvert.DeserializeObject(value); + } + catch (JsonReaderException) + { + } + } + } + + return parsedJson; + } + + private static async Task GetCurrentResourceCount() + { + int total = 0; + try + { + HttpClient.Timeout = TimeSpan.FromMinutes(5); + using var get = new HttpRequestMessage(HttpMethod.Get, $"{FhirEndpoint}/{ResourceType}?_summary=count"); + get.Headers.Authorization = new AuthenticationHeaderValue("Bearer", await GetToken()); + using HttpResponseMessage response = await HttpClient.SendAsync(get); + + string content = await response.Content.ReadAsStringAsync(); + if (string.IsNullOrEmpty(content)) + { + var json = JObject.Parse(content); + _ = int.TryParse((string)json["total"], out total); + } + + if (!response.IsSuccessStatusCode) + { +#pragma warning disable CA1303 // Do not pass literals as localized parameters + Console.WriteLine($"Failed to get success from {nameof(GetCurrentResourceCount)}"); +#pragma warning restore CA1303 // Do not pass literals as localized parameters + } + } + catch (Exception e) + { + Console.WriteLine(e); + } + + return total; + } + + private static async Task GetToken() + { + string accessToken = string.Empty; + var parameters = new List>(); + parameters.Add(new KeyValuePair("grant_type", TokenGrantType)); + parameters.Add(new KeyValuePair("resource", TokenResource)); + parameters.Add(new KeyValuePair("client_id", TokenClientId)); + parameters.Add(new KeyValuePair("client_secret", TokenClientSecret)); + + using var request = new HttpRequestMessage(HttpMethod.Post, TokenEndpoint) + { + Content = new FormUrlEncodedContent(parameters), + }; + + using HttpResponseMessage accessTokenResponse = await HttpClient.SendAsync(request); + if (!accessTokenResponse.IsSuccessStatusCode) + { + Console.WriteLine($"Failed to get token. Status code: {accessTokenResponse.StatusCode}."); + } + else + { + string content = await accessTokenResponse.Content.ReadAsStringAsync(); + var json = JObject.Parse(content); + accessToken = (string)json["access_token"] ?? string.Empty; + } + + return accessToken; + } + + private static BlobContainerClient GetContainerClient(string containerName) + { + try + { + return new BlobServiceClient(ConnectionString).GetBlobContainerClient(containerName); + } + catch + { + Console.WriteLine($"Unable to parse storage reference or connect to storage account {containerName}."); + throw; + } + } + } +} diff --git a/tools/ImporterV2/ImporterV2.csproj b/tools/ImporterV2/ImporterV2.csproj new file mode 100644 index 0000000000..b0adca1ec5 --- /dev/null +++ b/tools/ImporterV2/ImporterV2.csproj @@ -0,0 +1,23 @@ + + + + Exe + Fhir.ImporterV2 + Microsoft.Health.Fhir.ImporterV2 + true + + + + + + + + + + + + Always + + + + diff --git a/tools/ImporterV2/Program.cs b/tools/ImporterV2/Program.cs new file mode 100644 index 0000000000..5bb689663c --- /dev/null +++ b/tools/ImporterV2/Program.cs @@ -0,0 +1,17 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Threading.Tasks; + +namespace Microsoft.Health.Fhir.ImporterV2 +{ + public static class Program + { + public static async Task Main() + { + await ImporterV2.Run(); + } + } +} diff --git a/tools/ImporterV2/readme.md b/tools/ImporterV2/readme.md new file mode 100644 index 0000000000..dd77c744b5 --- /dev/null +++ b/tools/ImporterV2/readme.md @@ -0,0 +1,4 @@ +Still considered a work in progress, this tool evolved so that it provided output logging for the both a POST to $import operation as well as GET for the $import status. status + +This import tool allows you to search for specific ndjson files from an azure container and POST an $import operation to your FHIR endpoint to be processed. +The app.config has descriptions for all of the necessary values that you will need to supply in order to run this tool. From c46774dec979ff0247d740b36d82a620ef3d4749 Mon Sep 17 00:00:00 2001 From: Matt Serdar <99214729+v-mserdar@users.noreply.github.com> Date: Fri, 17 Mar 2023 10:39:09 -0700 Subject: [PATCH 2/9] revised app.config --- tools/ImporterV2/App.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ImporterV2/App.config b/tools/ImporterV2/App.config index d1ee84da94..8adfd06869 100644 --- a/tools/ImporterV2/App.config +++ b/tools/ImporterV2/App.config @@ -21,7 +21,7 @@ - + From ab6f61a13294676a7c8249283608e016f3618cca Mon Sep 17 00:00:00 2001 From: Matt Serdar <99214729+v-mserdar@users.noreply.github.com> Date: Tue, 21 Mar 2023 12:33:55 -0700 Subject: [PATCH 3/9] Revised based on code review --- Microsoft.Health.Fhir.sln | 14 ++-- tools/ImporterV2/App.config | 42 +++++++----- tools/ImporterV2/ImportResponse.cs | 2 +- tools/ImporterV2/Program.cs | 4 +- ...orterV2.cs => RegisterAndMonitorImport.cs} | 64 +++++++++++++------ ...csproj => RegisterAndMonitorImport.csproj} | 4 +- 6 files changed, 80 insertions(+), 50 deletions(-) rename tools/ImporterV2/{ImporterV2.cs => RegisterAndMonitorImport.cs} (86%) rename tools/ImporterV2/{ImporterV2.csproj => RegisterAndMonitorImport.csproj} (79%) diff --git a/Microsoft.Health.Fhir.sln b/Microsoft.Health.Fhir.sln index 98a6bb1cf7..ff69d28e58 100644 --- a/Microsoft.Health.Fhir.sln +++ b/Microsoft.Health.Fhir.sln @@ -146,7 +146,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.TaskManage EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.Fhir.Store.Utils", "src\Microsoft.Health.Fhir.Store.Utils\Microsoft.Health.Fhir.Store.Utils.csproj", "{7A736E5F-DA6E-483F-AD5B-EE8F66828E36}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ImporterV2", "tools\ImporterV2\ImporterV2.csproj", "{E85BCB9A-5D6E-45AD-BE67-AEFA060FEBF1}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "RegisterAndMonitorImport", "tools\ImporterV2\RegisterAndMonitorImport.csproj", "{E85BCB9A-5D6E-45AD-BE67-AEFA060FEBF1}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Importer", "tools\Importer\Importer.csproj", "{F6B94905-B496-46AD-B2A7-2ABCB0B2A6B4}" EndProject @@ -346,6 +346,10 @@ Global {7A736E5F-DA6E-483F-AD5B-EE8F66828E36}.Debug|Any CPU.Build.0 = Debug|Any CPU {7A736E5F-DA6E-483F-AD5B-EE8F66828E36}.Release|Any CPU.ActiveCfg = Release|Any CPU {7A736E5F-DA6E-483F-AD5B-EE8F66828E36}.Release|Any CPU.Build.0 = Release|Any CPU + {E85BCB9A-5D6E-45AD-BE67-AEFA060FEBF1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E85BCB9A-5D6E-45AD-BE67-AEFA060FEBF1}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E85BCB9A-5D6E-45AD-BE67-AEFA060FEBF1}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E85BCB9A-5D6E-45AD-BE67-AEFA060FEBF1}.Release|Any CPU.Build.0 = Release|Any CPU {F6B94905-B496-46AD-B2A7-2ABCB0B2A6B4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {F6B94905-B496-46AD-B2A7-2ABCB0B2A6B4}.Debug|Any CPU.Build.0 = Debug|Any CPU {F6B94905-B496-46AD-B2A7-2ABCB0B2A6B4}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -410,10 +414,6 @@ Global {F4DE2945-80C5-48FE-B58A-4AD1264C9FEA}.Debug|Any CPU.Build.0 = Debug|Any CPU {F4DE2945-80C5-48FE-B58A-4AD1264C9FEA}.Release|Any CPU.ActiveCfg = Release|Any CPU {F4DE2945-80C5-48FE-B58A-4AD1264C9FEA}.Release|Any CPU.Build.0 = Release|Any CPU - {E85BCB9A-5D6E-45AD-BE67-AEFA060FEBF1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {E85BCB9A-5D6E-45AD-BE67-AEFA060FEBF1}.Debug|Any CPU.Build.0 = Debug|Any CPU - {E85BCB9A-5D6E-45AD-BE67-AEFA060FEBF1}.Release|Any CPU.ActiveCfg = Release|Any CPU - {E85BCB9A-5D6E-45AD-BE67-AEFA060FEBF1}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -480,6 +480,7 @@ Global {03AD4FC4-A56F-4E94-B295-B02A9E3E3CCD} = {7457B218-2651-49B5-BED8-22233889516A} {BB7512E7-E148-4AF8-9D34-AA47A6AE692D} = {7457B218-2651-49B5-BED8-22233889516A} {7A736E5F-DA6E-483F-AD5B-EE8F66828E36} = {B70945F4-01A6-4351-955B-C4A2943B5E3B} + {E85BCB9A-5D6E-45AD-BE67-AEFA060FEBF1} = {B70945F4-01A6-4351-955B-C4A2943B5E3B} {F6B94905-B496-46AD-B2A7-2ABCB0B2A6B4} = {B70945F4-01A6-4351-955B-C4A2943B5E3B} {E468B6C6-9098-4293-AFD6-3B1675D67063} = {B70945F4-01A6-4351-955B-C4A2943B5E3B} {A0320AE9-3F87-44A3-8263-5AF7E00085D4} = {38B3BA4A-3510-4615-BCC4-4C9B96A486C4} @@ -495,11 +496,10 @@ Global {5E456FD7-E5A5-41F4-A0D3-7215585AEB7A} = {FCD51BAF-BFE5-476F-B562-C9AB36AA9839} {A5DED132-32B1-4804-95F5-EBC6092EC8AE} = {85F39C13-BC62-49A0-9C06-3BBA724D35ED} {D6C90E8C-50AF-45D8-B2D3-1B9B07E65F3E} = {1295CCC3-73FB-4376-AE95-F6F31A37B152} - {E85BCB9A-5D6E-45AD-BE67-AEFA060FEBF1} = {B70945F4-01A6-4351-955B-C4A2943B5E3B} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution - RESX_SortFileContentOnSave = True SolutionGuid = {E370FB31-CF95-47D1-B1E1-863A77973FF8} + RESX_SortFileContentOnSave = True EndGlobalSection GlobalSection(SharedMSBuildProjectFiles) = preSolution test\Microsoft.Health.Fhir.Shared.Tests.E2E.Common\Microsoft.Health.Fhir.Shared.Tests.E2E.Common.projitems*{0478b687-7105-40f6-a2dc-81057890e944}*SharedItemsImports = 13 diff --git a/tools/ImporterV2/App.config b/tools/ImporterV2/App.config index 8adfd06869..bf263b2507 100644 --- a/tools/ImporterV2/App.config +++ b/tools/ImporterV2/App.config @@ -1,30 +1,38 @@  - - - - - - + + + + + + - + - - - - - - - - - - + + + + + + + + + + diff --git a/tools/ImporterV2/ImportResponse.cs b/tools/ImporterV2/ImportResponse.cs index 4ac3e9b551..84604202ff 100644 --- a/tools/ImporterV2/ImportResponse.cs +++ b/tools/ImporterV2/ImportResponse.cs @@ -6,7 +6,7 @@ using System.Collections.Generic; using System.Text.Json.Serialization; -namespace Microsoft.Health.Fhir.ImporterV2 +namespace Microsoft.Health.Fhir.RegisterAndMonitorImport { #pragma warning disable CA1812 // Avoid uninstantiated internal classes internal sealed class ImportResponse diff --git a/tools/ImporterV2/Program.cs b/tools/ImporterV2/Program.cs index 5bb689663c..d67c188c07 100644 --- a/tools/ImporterV2/Program.cs +++ b/tools/ImporterV2/Program.cs @@ -5,13 +5,13 @@ using System.Threading.Tasks; -namespace Microsoft.Health.Fhir.ImporterV2 +namespace Microsoft.Health.Fhir.RegisterAndMonitorImport { public static class Program { public static async Task Main() { - await ImporterV2.Run(); + await RegisterAndMonitorImport.Run(); } } } diff --git a/tools/ImporterV2/ImporterV2.cs b/tools/ImporterV2/RegisterAndMonitorImport.cs similarity index 86% rename from tools/ImporterV2/ImporterV2.cs rename to tools/ImporterV2/RegisterAndMonitorImport.cs index d8cbb7416f..0cd26317be 100644 --- a/tools/ImporterV2/ImporterV2.cs +++ b/tools/ImporterV2/RegisterAndMonitorImport.cs @@ -16,14 +16,15 @@ using System.Text; using System.Threading; using System.Threading.Tasks; +using Azure; using Azure.Storage.Blobs; using Azure.Storage.Blobs.Models; using Newtonsoft.Json; using Newtonsoft.Json.Linq; -namespace Microsoft.Health.Fhir.ImporterV2 +namespace Microsoft.Health.Fhir.RegisterAndMonitorImport { - internal static class ImporterV2 + internal static class RegisterAndMonitorImport { private static readonly string TokenEndpoint = ConfigurationManager.AppSettings["TokenEndpoint"] ?? string.Empty; private static readonly string TokenGrantType = ConfigurationManager.AppSettings["grant_type"] ?? string.Empty; @@ -42,22 +43,31 @@ internal static class ImporterV2 private static readonly string OutputDirectory = Path.Combine(Directory.GetCurrentDirectory(), "importResults"); private static readonly string OutputFileName = Path.Combine(OutputDirectory, "importer.txt"); private static readonly string LocationUrlFileName = Path.Combine(OutputDirectory, "locationUrls.txt"); + private static readonly string MonitorImportStatusEndpoint = ConfigurationManager.AppSettings["MonitorImportStatusEndpoint"] ?? string.Empty; + + private static bool IsMonitorImportStatusEndpoint => !string.IsNullOrWhiteSpace(MonitorImportStatusEndpoint); internal static async Task Run() { try { - await Init(); + if (IsMonitorImportStatusEndpoint) + { + Console.WriteLine($"Getting the import status for {MonitorImportStatusEndpoint}{Environment.NewLine}"); - // this may take too long and timeout so it may be worth commenting out if you have too many resources - // int currentResourceCount = await GetCurrentResourceCount(); - // Console.WriteLine($"{currentResourceCount:N0} {ResourceType} found"); + // all attempted Urls for GetImportStatus are appended to the LocationUrlFileName file + await GetImportStatus(MonitorImportStatusEndpoint); + } + else + { + await Init(); - // If it's ever needed to finish checking on a current import then add the full url here - // all attempted Urls for GetImportStatus should be saved to the LocationUrlFileName file - // await GetImportStatus("https://.fhir.azurehealthcareapis.com/_operations/import/32"); + // this may take too long and timeout so it may be worth commenting out if you have too many resources + // int currentResourceCount = await GetCurrentResourceCount(); + // Console.WriteLine($"{currentResourceCount:N0} {ResourceType} found"); - await RunImport(); + await RunImport(); + } } catch (Exception e) { @@ -113,7 +123,7 @@ private static async Task WriteImportedBlobNames() File.Delete(OutputFileName); } - if (s_importedBlobNames != null) + if (s_importedBlobNames != null && !IsMonitorImportStatusEndpoint) { await File.WriteAllTextAsync(OutputFileName, string.Join(Environment.NewLine, s_importedBlobNames)); Console.WriteLine($"Saved file: {OutputFileName}"); @@ -124,21 +134,33 @@ private static void GetBlobItems() { if (s_blobContainerClientSource != null) { - s_blobItems = s_blobContainerClientSource.GetBlobs().Where(_ => _.Name.EndsWith($"{ResourceType}.ndjson", true, CultureInfo.CurrentCulture)).ToList(); - Console.WriteLine($"Total container BlobItems count = {s_blobItems.Count}"); - - foreach (string item in s_importedBlobNames) + try { - var blobItem = s_blobItems.Where(e => e.Name.Contains(item, StringComparison.OrdinalIgnoreCase)).FirstOrDefault(); - if (blobItem != null) + s_blobItems = s_blobContainerClientSource.GetBlobs().Where(_ => _.Name.EndsWith($"{ResourceType}.ndjson", true, CultureInfo.CurrentCulture)).ToList(); + Console.WriteLine($"Total container BlobItems count = {s_blobItems.Count}"); + + foreach (string item in s_importedBlobNames) { - s_blobItems.Remove(blobItem); + var blobItem = s_blobItems.Where(e => e.Name.Contains(item, StringComparison.OrdinalIgnoreCase)).FirstOrDefault(); + if (blobItem != null) + { + s_blobItems.Remove(blobItem); + } } - } - s_blobItems = s_blobItems.Take(NumberOfBlobsForImport).ToList(); + s_blobItems = s_blobItems.Take(NumberOfBlobsForImport).ToList(); - Console.WriteLine($"Working set of BlobItems count = {s_blobItems.Count}"); + Console.WriteLine($"Working set of BlobItems count = {s_blobItems.Count}"); + } + catch (RequestFailedException e) + { + if (e.Status == (int)HttpStatusCode.Forbidden) + { + Console.WriteLine($"Your connection string {ConnectionString} is invalid. Please verify and try again.{Environment.NewLine}"); + } + + throw; + } } } diff --git a/tools/ImporterV2/ImporterV2.csproj b/tools/ImporterV2/RegisterAndMonitorImport.csproj similarity index 79% rename from tools/ImporterV2/ImporterV2.csproj rename to tools/ImporterV2/RegisterAndMonitorImport.csproj index b0adca1ec5..acdd0659db 100644 --- a/tools/ImporterV2/ImporterV2.csproj +++ b/tools/ImporterV2/RegisterAndMonitorImport.csproj @@ -2,8 +2,8 @@ Exe - Fhir.ImporterV2 - Microsoft.Health.Fhir.ImporterV2 + Fhir.RegisterAndMonitorImport + Microsoft.Health.Fhir.RegisterAndMonitorImport true From b95221194bec1da379cd67ddbf4209dd085d9355 Mon Sep 17 00:00:00 2001 From: Matt Serdar <99214729+v-mserdar@users.noreply.github.com> Date: Wed, 29 Mar 2023 16:09:26 -0700 Subject: [PATCH 4/9] revisions to handle oss; code cleanup --- tools/ImporterV2/App.config | 8 +++ tools/ImporterV2/RegisterAndMonitorImport.cs | 54 ++++++++++++-------- tools/ImporterV2/readme.md | 11 +++- 3 files changed, 49 insertions(+), 24 deletions(-) diff --git a/tools/ImporterV2/App.config b/tools/ImporterV2/App.config index bf263b2507..b8dcc317a0 100644 --- a/tools/ImporterV2/App.config +++ b/tools/ImporterV2/App.config @@ -21,6 +21,14 @@ do not add a file extension as ndjson is assumed and added to the ResourceType automatically --> + + + + + + + diff --git a/tools/ImporterV2/RegisterAndMonitorImport.cs b/tools/ImporterV2/RegisterAndMonitorImport.cs index fdf5c77184..124dc39efa 100644 --- a/tools/ImporterV2/RegisterAndMonitorImport.cs +++ b/tools/ImporterV2/RegisterAndMonitorImport.cs @@ -36,6 +36,7 @@ internal static class RegisterAndMonitorImport private static readonly string ConnectionString = ConfigurationManager.AppSettings["ConnectionString"] ?? string.Empty; private static readonly string FhirEndpoint = ConfigurationManager.AppSettings["FhirEndpoint"] ?? string.Empty; private static readonly int NumberOfBlobsForImport = int.Parse(ConfigurationManager.AppSettings["NumberOfBlobsForImport"] ?? "1"); + private static readonly TimeSpan ImportStatusDelay = TimeSpan.Parse(ConfigurationManager.AppSettings["ImportStatusDelay"]); private static readonly HttpClient HttpClient = new(); private static BlobContainerClient s_blobContainerClientSource; private static List s_blobItems; @@ -242,7 +243,6 @@ private static async Task GetImportStatus(string url) { await WriteLocationUrls(url); - var delay = TimeSpan.FromMinutes(5); var swTotalTime = new Stopwatch(); var swSingleTime = new Stopwatch(); swTotalTime.Start(); @@ -280,8 +280,8 @@ private static async Task GetImportStatus(string url) } else if (response.StatusCode == HttpStatusCode.Accepted) { - Console.WriteLine($"Total running time: {swTotalTime.Elapsed.Duration()} - awaiting {delay} before retry"); - await Task.Delay(delay); + Console.WriteLine($"Total running time: {swTotalTime.Elapsed.Duration()} - awaiting {ImportStatusDelay} before retry"); + await Task.Delay(ImportStatusDelay); continue; } else diff --git a/tools/ImporterV2/readme.md b/tools/ImporterV2/readme.md index d51a643a13..466c275fd5 100644 --- a/tools/ImporterV2/readme.md +++ b/tools/ImporterV2/readme.md @@ -9,3 +9,13 @@ This tool can perform the following operations. 3. Run an $import job by consuming ndjson blob files from the configured container and posting them to a Paas endpoint. The app.config has descriptions for all of the necessary values that you will need to supply in order to run this tool. + + +When running FHIR OSS be sure to have these configuration settings in the portal: +FhirServer__Operations__Import__Enabled = True +FhirServer__Operations__Import__InitialImportMode = True +FhirServer__Operations__Import__StorageAccountConnection = your_storageaccount_access_key_connection_string +FhirServer__Operations__IntegrationDataStore__StorageAccountConnection = your_storageaccount_access_key_connection_string + +**You may need to remove this setting to get import to work** +FhirServer__Operations__IntegrationDataStore__StorageAccountUri From a91df5c33680ea95a09514ba4809d713501009df Mon Sep 17 00:00:00 2001 From: Matt Serdar <99214729+v-mserdar@users.noreply.github.com> Date: Fri, 31 Mar 2023 15:12:22 -0700 Subject: [PATCH 6/9] revisions from code review --- Microsoft.Health.Fhir.sln | 2 +- .../App.config | 22 +++++++++++-------- .../ImportResponse.cs | 0 .../Program.cs | 0 .../RegisterAndMonitorImport.cs | 2 +- .../RegisterAndMonitorImport.csproj | 0 .../readme.md | 12 +++++----- 7 files changed, 22 insertions(+), 16 deletions(-) rename tools/{ImporterV2 => RegisterAndMonitorImport}/App.config (80%) rename tools/{ImporterV2 => RegisterAndMonitorImport}/ImportResponse.cs (100%) rename tools/{ImporterV2 => RegisterAndMonitorImport}/Program.cs (100%) rename tools/{ImporterV2 => RegisterAndMonitorImport}/RegisterAndMonitorImport.cs (99%) rename tools/{ImporterV2 => RegisterAndMonitorImport}/RegisterAndMonitorImport.csproj (100%) rename tools/{ImporterV2 => RegisterAndMonitorImport}/readme.md (69%) diff --git a/Microsoft.Health.Fhir.sln b/Microsoft.Health.Fhir.sln index ff69d28e58..3376251b1f 100644 --- a/Microsoft.Health.Fhir.sln +++ b/Microsoft.Health.Fhir.sln @@ -146,7 +146,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.TaskManage EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.Fhir.Store.Utils", "src\Microsoft.Health.Fhir.Store.Utils\Microsoft.Health.Fhir.Store.Utils.csproj", "{7A736E5F-DA6E-483F-AD5B-EE8F66828E36}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "RegisterAndMonitorImport", "tools\ImporterV2\RegisterAndMonitorImport.csproj", "{E85BCB9A-5D6E-45AD-BE67-AEFA060FEBF1}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "RegisterAndMonitorImport", "tools\RegisterAndMonitorImport\RegisterAndMonitorImport.csproj", "{E85BCB9A-5D6E-45AD-BE67-AEFA060FEBF1}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Importer", "tools\Importer\Importer.csproj", "{F6B94905-B496-46AD-B2A7-2ABCB0B2A6B4}" EndProject diff --git a/tools/ImporterV2/App.config b/tools/RegisterAndMonitorImport/App.config similarity index 80% rename from tools/ImporterV2/App.config rename to tools/RegisterAndMonitorImport/App.config index 16b01bacaa..9b0ec13396 100644 --- a/tools/ImporterV2/App.config +++ b/tools/RegisterAndMonitorImport/App.config @@ -11,13 +11,13 @@ https://{your fhir endpoint}/_operations/import/{import id} The expected endpoint would be similar to this: --> - - + + - + @@ -25,24 +25,28 @@ - + - - + + - - + - + diff --git a/tools/ImporterV2/ImportResponse.cs b/tools/RegisterAndMonitorImport/ImportResponse.cs similarity index 100% rename from tools/ImporterV2/ImportResponse.cs rename to tools/RegisterAndMonitorImport/ImportResponse.cs diff --git a/tools/ImporterV2/Program.cs b/tools/RegisterAndMonitorImport/Program.cs similarity index 100% rename from tools/ImporterV2/Program.cs rename to tools/RegisterAndMonitorImport/Program.cs diff --git a/tools/ImporterV2/RegisterAndMonitorImport.cs b/tools/RegisterAndMonitorImport/RegisterAndMonitorImport.cs similarity index 99% rename from tools/ImporterV2/RegisterAndMonitorImport.cs rename to tools/RegisterAndMonitorImport/RegisterAndMonitorImport.cs index 124dc39efa..5233b77da6 100644 --- a/tools/ImporterV2/RegisterAndMonitorImport.cs +++ b/tools/RegisterAndMonitorImport/RegisterAndMonitorImport.cs @@ -34,7 +34,7 @@ internal static class RegisterAndMonitorImport private static readonly string ResourceType = ConfigurationManager.AppSettings["ResourceType"] ?? string.Empty; private static readonly string ContainerName = ConfigurationManager.AppSettings["ContainerName"] ?? string.Empty; private static readonly string ConnectionString = ConfigurationManager.AppSettings["ConnectionString"] ?? string.Empty; - private static readonly string FhirEndpoint = ConfigurationManager.AppSettings["FhirEndpoint"] ?? string.Empty; + private static readonly string FhirEndpoint = TokenResource; private static readonly int NumberOfBlobsForImport = int.Parse(ConfigurationManager.AppSettings["NumberOfBlobsForImport"] ?? "1"); private static readonly TimeSpan ImportStatusDelay = TimeSpan.Parse(ConfigurationManager.AppSettings["ImportStatusDelay"]); private static readonly HttpClient HttpClient = new(); diff --git a/tools/ImporterV2/RegisterAndMonitorImport.csproj b/tools/RegisterAndMonitorImport/RegisterAndMonitorImport.csproj similarity index 100% rename from tools/ImporterV2/RegisterAndMonitorImport.csproj rename to tools/RegisterAndMonitorImport/RegisterAndMonitorImport.csproj diff --git a/tools/ImporterV2/readme.md b/tools/RegisterAndMonitorImport/readme.md similarity index 69% rename from tools/ImporterV2/readme.md rename to tools/RegisterAndMonitorImport/readme.md index 466c275fd5..b4d7919ec3 100644 --- a/tools/ImporterV2/readme.md +++ b/tools/RegisterAndMonitorImport/readme.md @@ -5,17 +5,19 @@ This tool can perform the following operations. 1. It can monitor a long running $import job by polling the status. Simply provide the MonitorImportStatusEndpoint and the token fields in the config file. Note that when this field is supplied then no other operation can be performed. Meaning you can't concurrently POST an $import. Consequently when this setting is empty then the tool looks to POST an $import job. -2. Run an $import job by consuming ndjson blob files from the configured container and posting them to an OSS endpoint. -3. Run an $import job by consuming ndjson blob files from the configured container and posting them to a Paas endpoint. +2. Register an $import job by consuming ndjson blob files from the configured container and posting them to an OSS endpoint. +3. Register an $import job by consuming ndjson blob files from the configured container and posting them to a Paas endpoint. -The app.config has descriptions for all of the necessary values that you will need to supply in order to run this tool. +If the tool stops for any reason then when you restart it will correctly skip any previously imported files by reading in the list of +files that were already imported and saved to disk. +The app.config has descriptions for all of the necessary values that you will need to supply in order to run this tool. When running FHIR OSS be sure to have these configuration settings in the portal: FhirServer__Operations__Import__Enabled = True FhirServer__Operations__Import__InitialImportMode = True -FhirServer__Operations__Import__StorageAccountConnection = your_storageaccount_access_key_connection_string FhirServer__Operations__IntegrationDataStore__StorageAccountConnection = your_storageaccount_access_key_connection_string +TaskHosting__MaxRunningTaskCount = some_value_greater_than_1 -**You may need to remove this setting to get import to work** +**You need to remove this setting to get import to work** FhirServer__Operations__IntegrationDataStore__StorageAccountUri From a4bc7ea0304822c53f84e826bc797e6244fde4c6 Mon Sep 17 00:00:00 2001 From: Matt Serdar <99214729+v-mserdar@users.noreply.github.com> Date: Fri, 31 Mar 2023 15:53:01 -0700 Subject: [PATCH 7/9] final code review revisions --- tools/RegisterAndMonitorImport/App.config | 7 +++---- tools/RegisterAndMonitorImport/RegisterAndMonitorImport.cs | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/tools/RegisterAndMonitorImport/App.config b/tools/RegisterAndMonitorImport/App.config index 9b0ec13396..f80fc73478 100644 --- a/tools/RegisterAndMonitorImport/App.config +++ b/tools/RegisterAndMonitorImport/App.config @@ -37,20 +37,19 @@ - + diff --git a/tools/RegisterAndMonitorImport/RegisterAndMonitorImport.cs b/tools/RegisterAndMonitorImport/RegisterAndMonitorImport.cs index 5233b77da6..dd80c8b900 100644 --- a/tools/RegisterAndMonitorImport/RegisterAndMonitorImport.cs +++ b/tools/RegisterAndMonitorImport/RegisterAndMonitorImport.cs @@ -30,7 +30,7 @@ internal static class RegisterAndMonitorImport private static readonly string TokenGrantType = ConfigurationManager.AppSettings["grant_type"] ?? string.Empty; private static readonly string TokenClientId = ConfigurationManager.AppSettings["client_id"] ?? string.Empty; private static readonly string TokenClientSecret = ConfigurationManager.AppSettings["client_secret"] ?? string.Empty; - private static readonly string TokenResource = ConfigurationManager.AppSettings["resource"] ?? string.Empty; + private static readonly string TokenResource = ConfigurationManager.AppSettings["FhirEndpoint"] ?? string.Empty; private static readonly string ResourceType = ConfigurationManager.AppSettings["ResourceType"] ?? string.Empty; private static readonly string ContainerName = ConfigurationManager.AppSettings["ContainerName"] ?? string.Empty; private static readonly string ConnectionString = ConfigurationManager.AppSettings["ConnectionString"] ?? string.Empty; From 22a7dbc3dc66509094d8e7d71ca5979bba61ee79 Mon Sep 17 00:00:00 2001 From: Matt Serdar <99214729+v-mserdar@users.noreply.github.com> Date: Tue, 11 Apr 2023 16:31:13 -0700 Subject: [PATCH 8/9] changes from code review --- tools/RegisterAndMonitorImport/App.config | 55 ----------- .../ImportResponse.cs | 2 +- tools/RegisterAndMonitorImport/Program.cs | 14 ++- .../RegisterAndMonitorConfiguration.cs | 77 ++++++++++++++++ .../RegisterAndMonitorImport.cs | 91 +++++++++---------- .../RegisterAndMonitorImport.csproj | 8 +- .../RegisterAndMonitorImport/appsettings.json | 16 ++++ 7 files changed, 155 insertions(+), 108 deletions(-) delete mode 100644 tools/RegisterAndMonitorImport/App.config create mode 100644 tools/RegisterAndMonitorImport/RegisterAndMonitorConfiguration.cs create mode 100644 tools/RegisterAndMonitorImport/appsettings.json diff --git a/tools/RegisterAndMonitorImport/App.config b/tools/RegisterAndMonitorImport/App.config deleted file mode 100644 index f80fc73478..0000000000 --- a/tools/RegisterAndMonitorImport/App.config +++ /dev/null @@ -1,55 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/tools/RegisterAndMonitorImport/ImportResponse.cs b/tools/RegisterAndMonitorImport/ImportResponse.cs index 84604202ff..7037661ec3 100644 --- a/tools/RegisterAndMonitorImport/ImportResponse.cs +++ b/tools/RegisterAndMonitorImport/ImportResponse.cs @@ -6,7 +6,7 @@ using System.Collections.Generic; using System.Text.Json.Serialization; -namespace Microsoft.Health.Fhir.RegisterAndMonitorImport +namespace Microsoft.Health.Internal.Fhir.RegisterAndMonitorImport { #pragma warning disable CA1812 // Avoid uninstantiated internal classes internal sealed class ImportResponse diff --git a/tools/RegisterAndMonitorImport/Program.cs b/tools/RegisterAndMonitorImport/Program.cs index d67c188c07..99a15cbd16 100644 --- a/tools/RegisterAndMonitorImport/Program.cs +++ b/tools/RegisterAndMonitorImport/Program.cs @@ -4,14 +4,24 @@ // ------------------------------------------------------------------------------------------------- using System.Threading.Tasks; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; -namespace Microsoft.Health.Fhir.RegisterAndMonitorImport +namespace Microsoft.Health.Internal.Fhir.RegisterAndMonitorImport { public static class Program { public static async Task Main() { - await RegisterAndMonitorImport.Run(); + IConfigurationRoot configuration = new ConfigurationBuilder() + .AddJsonFile("appsettings.json", optional: false, reloadOnChange: true) + .Build(); + + RegisterAndMonitorConfiguration registerAndMonitorConfiguration = new(); + configuration.GetSection(RegisterAndMonitorConfiguration.SectionName).Bind(registerAndMonitorConfiguration); + + var import = new RegisterAndMonitorImport(registerAndMonitorConfiguration); + await import.Run(); } } } diff --git a/tools/RegisterAndMonitorImport/RegisterAndMonitorConfiguration.cs b/tools/RegisterAndMonitorImport/RegisterAndMonitorConfiguration.cs new file mode 100644 index 0000000000..dbd6e2de21 --- /dev/null +++ b/tools/RegisterAndMonitorImport/RegisterAndMonitorConfiguration.cs @@ -0,0 +1,77 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; + +namespace Microsoft.Health.Internal.Fhir.RegisterAndMonitorImport +{ + internal sealed class RegisterAndMonitorConfiguration + { + public const string SectionName = "RegisterAndMonitor"; + + /// + /// If you just want to monitor an import then provide a url for this key/value + /// Note that if this is provided then it only performs this operation and will not do any import. + /// The endpoint would be something similar to this https://{your fhir endpoint}/_operations/import/{import id} + /// + public string MonitorImportStatusEndpoint { get; set; } = string.Empty; + + /// + /// the time delay between calls to get the status of the import + /// generally smaller ndjson files can be used with smaller timespans + /// and larger ndjson files should have larger timespans + /// + public TimeSpan ImportStatusDelay { get; set; } = TimeSpan.FromMinutes(2); + + /// + /// Azure blob storage connection string + /// + public string ConnectionString { get; set; } = string.Empty; + + /// + /// Azure blob storage container name - used with the ConnectionString + /// + public string ContainerName { get; set; } = string.Empty; + + /// + /// the type of file it'll search for in the container + /// do not add a file extension - It's assumed and added to the ResourceType automatically + /// + public string ResourceType { get; set; } = string.Empty; + + /// + /// a simple flag to indicate that when true we're going to use the token for/// our http POST/GET + /// this is useful when you have a Paas deployment if you set to false then you could use it against an oss fhir service + /// + public bool UseBearerToken { get; set; } + + /// + /// the number of blobs we want to import at a time + /// + public int NumberOfBlobsForImport { get; set; } = 1; + + /// + /// the url of your FHIR endpoint - also used as the value for the key "resource" for getting a token + /// + public string FhirEndpoint { get; set; } + + /// + /// Used for the value of the key "grant_type" for getting a token + /// + public string TokenGrantType { get; set; } = "Client_Credentials"; + + /// + /// Used for the value of the key "client_id" for getting a token + /// + public string TokenClientId { get; set; } + + /// + /// Used for the value of the key "client_secret" for getting a token + /// + public string TokenClientSecret { get; set; } + + public bool IsMonitorImportStatusEndpoint => !string.IsNullOrWhiteSpace(MonitorImportStatusEndpoint); + } +} diff --git a/tools/RegisterAndMonitorImport/RegisterAndMonitorImport.cs b/tools/RegisterAndMonitorImport/RegisterAndMonitorImport.cs index dd80c8b900..e940dd1360 100644 --- a/tools/RegisterAndMonitorImport/RegisterAndMonitorImport.cs +++ b/tools/RegisterAndMonitorImport/RegisterAndMonitorImport.cs @@ -5,7 +5,6 @@ using System; using System.Collections.Generic; -using System.Configuration; using System.Diagnostics; using System.Globalization; using System.IO; @@ -22,43 +21,39 @@ using Newtonsoft.Json; using Newtonsoft.Json.Linq; -namespace Microsoft.Health.Fhir.RegisterAndMonitorImport +namespace Microsoft.Health.Internal.Fhir.RegisterAndMonitorImport { - internal static class RegisterAndMonitorImport + internal sealed class RegisterAndMonitorImport { - private static readonly string TokenEndpoint = ConfigurationManager.AppSettings["TokenEndpoint"] ?? string.Empty; - private static readonly string TokenGrantType = ConfigurationManager.AppSettings["grant_type"] ?? string.Empty; - private static readonly string TokenClientId = ConfigurationManager.AppSettings["client_id"] ?? string.Empty; - private static readonly string TokenClientSecret = ConfigurationManager.AppSettings["client_secret"] ?? string.Empty; - private static readonly string TokenResource = ConfigurationManager.AppSettings["FhirEndpoint"] ?? string.Empty; - private static readonly string ResourceType = ConfigurationManager.AppSettings["ResourceType"] ?? string.Empty; - private static readonly string ContainerName = ConfigurationManager.AppSettings["ContainerName"] ?? string.Empty; - private static readonly string ConnectionString = ConfigurationManager.AppSettings["ConnectionString"] ?? string.Empty; - private static readonly string FhirEndpoint = TokenResource; - private static readonly int NumberOfBlobsForImport = int.Parse(ConfigurationManager.AppSettings["NumberOfBlobsForImport"] ?? "1"); - private static readonly TimeSpan ImportStatusDelay = TimeSpan.Parse(ConfigurationManager.AppSettings["ImportStatusDelay"]); + private readonly RegisterAndMonitorConfiguration monitorConfiguration; private static readonly HttpClient HttpClient = new(); private static BlobContainerClient s_blobContainerClientSource; private static List s_blobItems; private static HashSet s_importedBlobNames = new(); private static readonly string OutputDirectory = Path.Combine(Directory.GetCurrentDirectory(), "importResults"); + + // this is a list of all the ndjson file names that have been imported from your blob container + // thereby helping eliminate duplicate imports of the same blob private static readonly string OutputFileName = Path.Combine(OutputDirectory, "importer.txt"); + + // this is a list of all urls that were used to get the import status private static readonly string LocationUrlFileName = Path.Combine(OutputDirectory, "locationUrls.txt"); - private static readonly string MonitorImportStatusEndpoint = ConfigurationManager.AppSettings["MonitorImportStatusEndpoint"] ?? string.Empty; - private static readonly bool UseBearerToken = bool.Parse(ConfigurationManager.AppSettings["UseBearerToken"]); - private static bool IsMonitorImportStatusEndpoint => !string.IsNullOrWhiteSpace(MonitorImportStatusEndpoint); + public RegisterAndMonitorImport(RegisterAndMonitorConfiguration configuration) + { + monitorConfiguration = configuration; + } - internal static async Task Run() + internal async Task Run() { try { - if (IsMonitorImportStatusEndpoint) + if (monitorConfiguration.IsMonitorImportStatusEndpoint) { - Console.WriteLine($"Getting the import status for {MonitorImportStatusEndpoint}{Environment.NewLine}"); + Console.WriteLine($"Getting the import status for {monitorConfiguration.MonitorImportStatusEndpoint}{Environment.NewLine}"); // all attempted Urls for GetImportStatus are appended to the LocationUrlFileName file - await GetImportStatus(MonitorImportStatusEndpoint); + await GetImportStatus(monitorConfiguration.MonitorImportStatusEndpoint); } else { @@ -81,14 +76,14 @@ internal static async Task Run() } } - private static async Task Init() + private async Task Init() { if (!Directory.Exists(OutputDirectory)) { Directory.CreateDirectory(OutputDirectory); } - s_blobContainerClientSource = GetContainerClient(ContainerName); + s_blobContainerClientSource = GetContainerClient(monitorConfiguration.ContainerName); await LoadImportedBlobItems(); GetBlobItems(); } @@ -118,27 +113,27 @@ private static async Task LoadImportedBlobItems() Console.WriteLine($"Found {s_importedBlobNames.Count} blobs already processed."); } - private static async Task WriteImportedBlobNames() + private async Task WriteImportedBlobNames() { if (File.Exists(OutputFileName)) { File.Delete(OutputFileName); } - if (s_importedBlobNames != null && !IsMonitorImportStatusEndpoint) + if (s_importedBlobNames != null && !monitorConfiguration.IsMonitorImportStatusEndpoint) { await File.WriteAllTextAsync(OutputFileName, string.Join(Environment.NewLine, s_importedBlobNames)); Console.WriteLine($"Saved file: {OutputFileName}"); } } - private static void GetBlobItems() + private void GetBlobItems() { if (s_blobContainerClientSource != null) { try { - s_blobItems = s_blobContainerClientSource.GetBlobs().Where(_ => _.Name.EndsWith($"{ResourceType}.ndjson", true, CultureInfo.CurrentCulture)).ToList(); + s_blobItems = s_blobContainerClientSource.GetBlobs().Where(_ => _.Name.EndsWith($"{monitorConfiguration.ResourceType}.ndjson", true, CultureInfo.CurrentCulture)).ToList(); Console.WriteLine($"Total container BlobItems count = {s_blobItems.Count}"); foreach (string item in s_importedBlobNames) @@ -150,7 +145,7 @@ private static void GetBlobItems() } } - s_blobItems = s_blobItems.Take(NumberOfBlobsForImport).ToList(); + s_blobItems = s_blobItems.Take(monitorConfiguration.NumberOfBlobsForImport).ToList(); Console.WriteLine($"Working set of BlobItems count = {s_blobItems.Count}"); } @@ -158,7 +153,7 @@ private static void GetBlobItems() { if (e.Status == (int)HttpStatusCode.Forbidden) { - Console.WriteLine($"Your connection string {ConnectionString} is invalid. Please verify and try again.{Environment.NewLine}"); + Console.WriteLine($"Your connection string {monitorConfiguration.ConnectionString} is invalid. Please verify and try again.{Environment.NewLine}"); } throw; @@ -166,7 +161,7 @@ private static void GetBlobItems() } } - private static async Task RunImport() + private async Task RunImport() { if (s_blobContainerClientSource == null || s_blobItems == null || s_blobItems.Count == 0) { @@ -195,7 +190,7 @@ private static async Task RunImport() sb.AppendLine("\""); sb.AppendLine(" },{\"name\": \"url\",\"valueUri\": "); sb.Append('"'); - sb.Append($"{s_blobContainerClientSource.Uri}/{blob.Name}"); + sb.Append(CultureInfo.CurrentCulture, $"{s_blobContainerClientSource.Uri}/{blob.Name}"); sb.AppendLine("\""); sb.AppendLine("}]},"); } @@ -206,7 +201,7 @@ private static async Task RunImport() var json = sb.ToString(); var data = new StringContent(json, Encoding.UTF8, "application/fhir+json"); - var url = $"{FhirEndpoint}/$import"; + var url = $"{monitorConfiguration.FhirEndpoint}/$import"; using var requestMessage = new HttpRequestMessage() { RequestUri = new Uri(url), @@ -229,9 +224,9 @@ private static async Task RunImport() } } - private static async Task GetHttpResponseMessageAsync(HttpRequestMessage request) + private async Task GetHttpResponseMessageAsync(HttpRequestMessage request) { - if (UseBearerToken) + if (monitorConfiguration.UseBearerToken) { request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", await GetToken()); } @@ -239,7 +234,7 @@ private static async Task GetHttpResponseMessageAsync(HttpR return await HttpClient.SendAsync(request, CancellationToken.None).ConfigureAwait(false); } - private static async Task GetImportStatus(string url) + private async Task GetImportStatus(string url) { await WriteLocationUrls(url); @@ -280,8 +275,8 @@ private static async Task GetImportStatus(string url) } else if (response.StatusCode == HttpStatusCode.Accepted) { - Console.WriteLine($"Total running time: {swTotalTime.Elapsed.Duration()} - awaiting {ImportStatusDelay} before retry"); - await Task.Delay(ImportStatusDelay); + Console.WriteLine($"Total running time: {swTotalTime.Elapsed.Duration()} - awaiting {monitorConfiguration.ImportStatusDelay} before retry"); + await Task.Delay(monitorConfiguration.ImportStatusDelay); continue; } else @@ -348,7 +343,7 @@ private static void PrintResponse(List response) { foreach (ImportResponse.Json item in response) { - Console.WriteLine($"{string.Format("{0, 3}", response.IndexOf(item) + 1)} {string.Format("{0, 10}", item.Count.ToString("N0"))} {item.InputUrl}"); + Console.WriteLine($"{string.Format(CultureInfo.CurrentCulture, "{0, 3}", response.IndexOf(item) + 1)} {string.Format(CultureInfo.CurrentCulture, "{0, 10}", item.Count.ToString("N0", CultureInfo.CurrentCulture))} {item.InputUrl}"); } } @@ -378,13 +373,13 @@ private static ImportResponse TryParseJson(string value) return parsedJson; } - private static async Task GetCurrentResourceCount() + private async Task GetCurrentResourceCount() { int total = 0; try { HttpClient.Timeout = TimeSpan.FromMinutes(5); - using var requestMessage = new HttpRequestMessage(HttpMethod.Get, $"{FhirEndpoint}/{ResourceType}?_summary=count"); + using var requestMessage = new HttpRequestMessage(HttpMethod.Get, $"{monitorConfiguration.FhirEndpoint}/{monitorConfiguration.ResourceType}?_summary=count"); using HttpResponseMessage response = await GetHttpResponseMessageAsync(requestMessage); string content = await response.Content.ReadAsStringAsync(); @@ -409,18 +404,18 @@ private static async Task GetCurrentResourceCount() return total; } - private static async Task GetToken() + private async Task GetToken() { string accessToken = string.Empty; var parameters = new List> { - new KeyValuePair("grant_type", TokenGrantType), - new KeyValuePair("resource", TokenResource), - new KeyValuePair("client_id", TokenClientId), - new KeyValuePair("client_secret", TokenClientSecret), + new KeyValuePair("grant_type", monitorConfiguration.TokenGrantType), + new KeyValuePair("resource", monitorConfiguration.FhirEndpoint), + new KeyValuePair("client_id", monitorConfiguration.TokenClientId), + new KeyValuePair("client_secret", monitorConfiguration.TokenClientSecret), }; - using var request = new HttpRequestMessage(HttpMethod.Post, TokenEndpoint) + using var request = new HttpRequestMessage(HttpMethod.Post, monitorConfiguration.FhirEndpoint) { Content = new FormUrlEncodedContent(parameters), }; @@ -440,11 +435,11 @@ private static async Task GetToken() return accessToken; } - private static BlobContainerClient GetContainerClient(string containerName) + private BlobContainerClient GetContainerClient(string containerName) { try { - return new BlobServiceClient(ConnectionString).GetBlobContainerClient(containerName); + return new BlobServiceClient(monitorConfiguration.ConnectionString).GetBlobContainerClient(containerName); } catch { diff --git a/tools/RegisterAndMonitorImport/RegisterAndMonitorImport.csproj b/tools/RegisterAndMonitorImport/RegisterAndMonitorImport.csproj index acdd0659db..4f5167e556 100644 --- a/tools/RegisterAndMonitorImport/RegisterAndMonitorImport.csproj +++ b/tools/RegisterAndMonitorImport/RegisterAndMonitorImport.csproj @@ -1,21 +1,25 @@ + Exe Fhir.RegisterAndMonitorImport - Microsoft.Health.Fhir.RegisterAndMonitorImport + Microsoft.Health.Internal.Fhir.RegisterAndMonitorImport true + + + - + Always diff --git a/tools/RegisterAndMonitorImport/appsettings.json b/tools/RegisterAndMonitorImport/appsettings.json new file mode 100644 index 0000000000..defecc7682 --- /dev/null +++ b/tools/RegisterAndMonitorImport/appsettings.json @@ -0,0 +1,16 @@ +{ + "RegisterAndMonitor": { + "monitorImportStatusEndpoint": "", + "importStatusDelay": "00:00:30", + "connectionString": "", + "containerName": "", + "resourceType": "", + "useBearerToken": false, + "numberOfBlobsForImport": 2, + "fhirEndpoint": null, + "tokenEndpoint": "", + "grantType": "Client_Credentials", + "clientId": "", + "clientSecret": "" + } +} From d34646151696a0c0d54382b87fb1ed455f1ae18c Mon Sep 17 00:00:00 2001 From: Matt Serdar <99214729+v-mserdar@users.noreply.github.com> Date: Tue, 18 Apr 2023 14:06:46 -0700 Subject: [PATCH 9/9] Adding one additional comment for clarify --- .../RegisterAndMonitorImport/RegisterAndMonitorConfiguration.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/RegisterAndMonitorImport/RegisterAndMonitorConfiguration.cs b/tools/RegisterAndMonitorImport/RegisterAndMonitorConfiguration.cs index dbd6e2de21..3fdfcccd03 100644 --- a/tools/RegisterAndMonitorImport/RegisterAndMonitorConfiguration.cs +++ b/tools/RegisterAndMonitorImport/RegisterAndMonitorConfiguration.cs @@ -44,6 +44,7 @@ internal sealed class RegisterAndMonitorConfiguration /// /// a simple flag to indicate that when true we're going to use the token for/// our http POST/GET /// this is useful when you have a Paas deployment if you set to false then you could use it against an oss fhir service + /// and Token properties do not need to be set. /// public bool UseBearerToken { get; set; }