From d22f9f336e717d8737ab81d26e45254cf2241884 Mon Sep 17 00:00:00 2001 From: "David R. Williamson" Date: Wed, 15 Dec 2021 10:09:11 -0800 Subject: [PATCH] feat(hub-svc): add support and tests for configurations export (#2250) --- .../helpers/ImportExportDevicesHelpers.cs | 29 -- e2e/test/helpers/ImportExportHelpers.cs | 30 ++ .../RegistryManagerExportDevicesTests.cs | 308 ++++++++++++------ .../RegistryManagerImportDevicesTests.cs | 167 +++++++--- .../src/Configurations/Configuration.cs | 41 +-- .../Configurations/ConfigurationContent.cs | 6 +- .../Configurations/ConfigurationImportMode.cs | 21 ++ .../src/Configurations/ImportConfiguration.cs | 23 ++ iothub/service/src/ExportImportDevice.cs | 88 ++--- iothub/service/src/JobProperties.cs | 31 +- iothub/service/src/ManagedIdentity.cs | 17 +- iothub/service/src/RegistryManager.cs | 2 + 12 files changed, 511 insertions(+), 252 deletions(-) delete mode 100644 e2e/test/helpers/ImportExportDevicesHelpers.cs create mode 100644 e2e/test/helpers/ImportExportHelpers.cs create mode 100644 iothub/service/src/Configurations/ConfigurationImportMode.cs create mode 100644 iothub/service/src/Configurations/ImportConfiguration.cs diff --git a/e2e/test/helpers/ImportExportDevicesHelpers.cs b/e2e/test/helpers/ImportExportDevicesHelpers.cs deleted file mode 100644 index 7e753b4455..0000000000 --- a/e2e/test/helpers/ImportExportDevicesHelpers.cs +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. -using System.Collections.Generic; -using System.IO; -using System.Text; -using Newtonsoft.Json; - -namespace Microsoft.Azure.Devices.E2ETests.Helpers -{ - internal static class ImportExportDevicesHelpers - { - /// - /// Makes a stream compatible for writing to a storage blob of serialized, newline-delimited rows of the specified devices - /// - /// The devices to serialize - public static Stream BuildDevicesStream(IReadOnlyList devices) - { - var devicesFileSb = new StringBuilder(); - - foreach (ExportImportDevice device in devices) - { - devicesFileSb.AppendLine(JsonConvert.SerializeObject(device)); - } - - byte[] devicesFileInBytes = Encoding.Default.GetBytes(devicesFileSb.ToString()); - return new MemoryStream(devicesFileInBytes); - } - } -} diff --git a/e2e/test/helpers/ImportExportHelpers.cs b/e2e/test/helpers/ImportExportHelpers.cs new file mode 100644 index 0000000000..d762ede683 --- /dev/null +++ b/e2e/test/helpers/ImportExportHelpers.cs @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Collections.Generic; +using System.IO; +using System.Text; +using Newtonsoft.Json; + +namespace Microsoft.Azure.Devices.E2ETests.Helpers +{ + internal static class ImportExportHelpers + { + /// + /// Makes a stream compatible for writing to a storage blob of serialized, newline-delimited rows of the specified objects. + /// + /// The objects to serialize. + public static Stream BuildImportStream(IReadOnlyList items) + { + var itemsFileSb = new StringBuilder(); + + foreach (T item in items) + { + itemsFileSb.AppendLine(JsonConvert.SerializeObject(item)); + } + + byte[] itemsFileInBytes = Encoding.Default.GetBytes(itemsFileSb.ToString()); + return new MemoryStream(itemsFileInBytes); + } + } +} diff --git a/e2e/test/iothub/service/RegistryManagerExportDevicesTests.cs b/e2e/test/iothub/service/RegistryManagerExportDevicesTests.cs index d5f0b17a12..e2729e5fb1 100644 --- a/e2e/test/iothub/service/RegistryManagerExportDevicesTests.cs +++ b/e2e/test/iothub/service/RegistryManagerExportDevicesTests.cs @@ -1,8 +1,8 @@ // Copyright (c) Microsoft. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. + using System; using System.Collections.Generic; -using System.Diagnostics.Tracing; using System.Linq; using System.Threading.Tasks; using FluentAssertions; @@ -24,7 +24,7 @@ public class RegistryManagerExportDevicesTests : E2EMsTestBase // https://github.com/Azure/azure-sdk-for-net/issues/10476 private const string ExportFileNameDefault = "devices.txt"; - private const int MaxIterationWait = 30; + private const int MaxIterationWait = 60; private static readonly TimeSpan s_waitDuration = TimeSpan.FromSeconds(3); private static readonly char[] s_newlines = new char[] @@ -53,17 +53,23 @@ public async Task RegistryManager_ExportDevices(StorageAuthenticationType storag { // arrange - string edgeId1 = $"{nameof(RegistryManager_ExportDevices)}-Edge-{StorageContainer.GetRandomSuffix(4)}"; - string edgeId2 = $"{nameof(RegistryManager_ExportDevices)}-Edge-{StorageContainer.GetRandomSuffix(4)}"; - string deviceId = $"{nameof(RegistryManager_ExportDevices)}-{StorageContainer.GetRandomSuffix(4)}"; - string devicesFileName = $"{nameof(RegistryManager_ExportDevices)}-devicesexport-{StorageContainer.GetRandomSuffix(4)}.txt"; - using var registryManager = RegistryManager.CreateFromConnectionString(TestConfiguration.IoTHub.ConnectionString); + const string idPrefix = nameof(RegistryManager_ExportDevices); + + string edgeId1 = $"{idPrefix}-Edge-{StorageContainer.GetRandomSuffix(4)}"; + string edgeId2 = $"{idPrefix}-Edge-{StorageContainer.GetRandomSuffix(4)}"; + string deviceId = $"{idPrefix}-{StorageContainer.GetRandomSuffix(4)}"; + string configurationId = (idPrefix + Guid.NewGuid()).ToLower(); // Configuration Id characters must be all lower-case. + Logger.Trace($"Using Ids {deviceId}, {edgeId1}, {edgeId2}, and {configurationId}"); + + string devicesFileName = $"{idPrefix}-devicesexport-{StorageContainer.GetRandomSuffix(4)}.txt"; + string configsFileName = $"{idPrefix}-configsexport-{StorageContainer.GetRandomSuffix(4)}.txt"; + + using RegistryManager registryManager = RegistryManager.CreateFromConnectionString(TestConfiguration.IoTHub.ConnectionString); - Logger.Trace($"Using deviceId {deviceId}"); try { - string containerName = StorageContainer.BuildContainerName(nameof(RegistryManager_ExportDevices)); + string containerName = StorageContainer.BuildContainerName(idPrefix); using StorageContainer storageContainer = await StorageContainer .GetInstanceAsync(containerName) .ConfigureAwait(false); @@ -92,7 +98,7 @@ public async Task RegistryManager_ExportDevices(StorageAuthenticationType storag }) .ConfigureAwait(false); - await registryManager + Device device = await registryManager .AddDeviceAsync( new Device(deviceId) { @@ -101,123 +107,221 @@ await registryManager }) .ConfigureAwait(false); + Configuration configuration = await registryManager + .AddConfigurationAsync( + new Configuration(configurationId) + { + Priority = 2, + Labels = { { "labelName", "labelValue" } }, + TargetCondition = "*", + Content = + { + DeviceContent = { { "properties.desired.x", 4L } }, + }, + Metrics = + { + Queries = { { "successfullyConfigured", "select deviceId from devices where properties.reported.x = 4" } } + }, + }) + .ConfigureAwait(false); + // act - JobProperties exportJobResponse = null; - int tryCount = 0; - while (true) + JobProperties exportJobResponse = await CreateAndWaitForJobAsync( + storageAuthenticationType, + isUserAssignedMsi, + devicesFileName, + configsFileName, + registryManager, + containerUri) + .ConfigureAwait(false); + + // assert + await ValidateDevicesAsync( + devicesFileName, + storageContainer, + edge1, + edge2, + device) + .ConfigureAwait(false); + await ValidateConfigurationsAsync( + configsFileName, + storageContainer, + configuration) + .ConfigureAwait(false); + } + finally + { + await CleanUpDevicesAsync(edgeId1, edgeId2, deviceId, configurationId, registryManager).ConfigureAwait(false); + } + } + + private async Task CreateAndWaitForJobAsync( + StorageAuthenticationType storageAuthenticationType, + bool isUserAssignedMsi, + string devicesFileName, + string configsFileName, + RegistryManager registryManager, + Uri containerUri) + { + int tryCount = 0; + + ManagedIdentity identity = isUserAssignedMsi + ? new ManagedIdentity { - try - { - ManagedIdentity identity = null; - if (isUserAssignedMsi) - { - string userAssignedMsiResourceId = TestConfiguration.IoTHub.UserAssignedMsiResourceId; - identity = new ManagedIdentity - { - userAssignedIdentity = userAssignedMsiResourceId - }; - } - - var jobProperties = JobProperties.CreateForExportJob( - containerUri.ToString(), - true, - devicesFileName, - storageAuthenticationType, - identity); - exportJobResponse = await registryManager.ExportDevicesAsync(jobProperties).ConfigureAwait(false); - break; - } - // Concurrent jobs can be rejected, so implement a retry mechanism to handle conflicts with other tests - catch (JobQuotaExceededException) when (++tryCount < MaxIterationWait) - { - Logger.Trace($"JobQuotaExceededException... waiting."); - await Task.Delay(s_waitDuration).ConfigureAwait(false); - continue; - } + UserAssignedIdentity = TestConfiguration.IoTHub.UserAssignedMsiResourceId } + : null; + + JobProperties exportJobResponse = JobProperties.CreateForExportJob( + containerUri.ToString(), + true, + devicesFileName, + storageAuthenticationType, + identity); + exportJobResponse.IncludeConfigurations = true; + exportJobResponse.ConfigurationsBlobName = configsFileName; - // Wait for job to complete - for (int i = 0; i < MaxIterationWait; ++i) + while (tryCount < MaxIterationWait) + { + try { + exportJobResponse = await registryManager.ExportDevicesAsync(exportJobResponse).ConfigureAwait(false); + break; + } + // Concurrent jobs can be rejected, so implement a retry mechanism to handle conflicts with other tests + catch (JobQuotaExceededException) when (++tryCount < MaxIterationWait) + { + Logger.Trace($"JobQuotaExceededException... waiting."); await Task.Delay(s_waitDuration).ConfigureAwait(false); - exportJobResponse = await registryManager.GetJobAsync(exportJobResponse.JobId).ConfigureAwait(false); - Logger.Trace($"Job {exportJobResponse.JobId} is {exportJobResponse.Status} with progress {exportJobResponse.Progress}%"); - if (!s_incompleteJobs.Contains(exportJobResponse.Status)) - { - break; - } + continue; } + } - // assert + for (int i = 0; i < MaxIterationWait; ++i) + { + await Task.Delay(s_waitDuration).ConfigureAwait(false); + exportJobResponse = await registryManager.GetJobAsync(exportJobResponse.JobId).ConfigureAwait(false); + Logger.Trace($"Job {exportJobResponse.JobId} is {exportJobResponse.Status} with progress {exportJobResponse.Progress}%"); + if (!s_incompleteJobs.Contains(exportJobResponse.Status)) + { + break; + } + } - exportJobResponse.Status.Should().Be(JobStatus.Completed, "Otherwise import failed"); - exportJobResponse.FailureReason.Should().BeNullOrEmpty("Otherwise import failed"); + exportJobResponse.Status.Should().Be(JobStatus.Completed, "Otherwise import failed"); + exportJobResponse.FailureReason.Should().BeNullOrEmpty("Otherwise import failed"); - string devicesContent = await DownloadFileAsync(storageContainer, devicesFileName).ConfigureAwait(false); - string[] serializedDevices = devicesContent.Split(s_newlines, StringSplitOptions.RemoveEmptyEntries); + return exportJobResponse; + } + + private async Task ValidateDevicesAsync( + string devicesFileName, + StorageContainer storageContainer, + Device edge1, + Device edge2, + Device device) + { + string devicesContent = await DownloadFileAsync(storageContainer, devicesFileName).ConfigureAwait(false); + string[] serializedDevices = devicesContent.Split(s_newlines, StringSplitOptions.RemoveEmptyEntries); - bool foundDeviceInExport = false; - bool foundEdgeInExport = false; - foreach (string serializedDevice in serializedDevices) + bool foundEdge1InExport = false; + bool foundEdge2InExport = false; + bool foundDeviceInExport = false; + + foreach (string serializedDevice in serializedDevices) + { + // The first line may be a comment to the user, so skip any lines that don't start with a json object initial character: curly brace + if (serializedDevice[0] != '{') { - // The first line may be a comment to the user, so skip any lines that don't start with a json object initial character: curly brace - if (serializedDevice[0] != '{') - { - continue; - } - - if (foundEdgeInExport && foundDeviceInExport) - { - // we're done - break; - } - - ExportImportDevice exportedDevice = JsonConvert.DeserializeObject(serializedDevice); - if (StringComparer.Ordinal.Equals(exportedDevice.Id, edgeId2) && exportedDevice.Capabilities.IotEdge) - { - Logger.Trace($"Found edge2 in export as [{serializedDevice}]"); - foundEdgeInExport = true; - exportedDevice.DeviceScope.Should().Be(edge2.Scope, "Edges retain their own scope"); - - // This is broken. The export doesn't include the ParentScopes property. - // Disable this assert until it is fixed in the service. - //exportedDevice.ParentScopes.First().Should().Be(edge1.Scope); - continue; - } - - if (StringComparer.Ordinal.Equals(exportedDevice.Id, deviceId)) - { - Logger.Trace($"Found device in export as [{serializedDevice}]"); - foundDeviceInExport = true; - exportedDevice.DeviceScope.Should().Be(edge1.Scope); - continue; - } + continue; } - foundEdgeInExport.Should().BeTrue("Expected edge did not appear in the export"); - foundDeviceInExport.Should().BeTrue("Expected device did not appear in the export"); - } - finally - { - try + + if (foundEdge1InExport + && foundEdge2InExport + && foundDeviceInExport) + { + // we're done + break; + } + + ExportImportDevice exportedDevice = JsonConvert.DeserializeObject(serializedDevice); + + if (StringComparer.Ordinal.Equals(exportedDevice.Id, edge1.Id) && exportedDevice.Capabilities.IotEdge) + { + Logger.Trace($"Found edge1 in export as [{serializedDevice}]"); + foundEdge1InExport = true; + exportedDevice.DeviceScope.Should().Be(edge1.Scope, "Edges retain their own scope"); + continue; + } + + if (StringComparer.Ordinal.Equals(exportedDevice.Id, edge2.Id) && exportedDevice.Capabilities.IotEdge) { - await registryManager.RemoveDeviceAsync(deviceId).ConfigureAwait(false); - await registryManager.RemoveDeviceAsync(edgeId2).ConfigureAwait(false); - await registryManager.RemoveDeviceAsync(edgeId1).ConfigureAwait(false); + Logger.Trace($"Found edge2 in export as [{serializedDevice}]"); + foundEdge2InExport = true; + exportedDevice.DeviceScope.Should().Be(edge2.Scope, "Edges retain their own scope"); + continue; } - catch (Exception ex) + + if (StringComparer.Ordinal.Equals(exportedDevice.Id, device.Id)) { - Logger.Trace($"Failed to remove device during cleanup due to {ex}"); + Logger.Trace($"Found device in export as [{serializedDevice}]"); + foundDeviceInExport = true; + exportedDevice.DeviceScope.Should().Be(edge1.Scope); + continue; } } + foundEdge1InExport.Should().BeTrue("Expected edge did not appear in the export"); + foundEdge2InExport.Should().BeTrue("Expected edge did not appear in the export"); + foundDeviceInExport.Should().BeTrue("Expected device did not appear in the export"); + } + + private async Task ValidateConfigurationsAsync( + string configsFileName, + StorageContainer storageContainer, + Configuration configuration) + { + string configsContent = await DownloadFileAsync(storageContainer, configsFileName).ConfigureAwait(false); + string[] serializedConfigs = configsContent.Split(s_newlines, StringSplitOptions.RemoveEmptyEntries); + + bool foundConfig = false; + foreach (string serializedConfig in serializedConfigs) + { + Configuration exportedConfig = JsonConvert.DeserializeObject(serializedConfig); + if (StringComparer.Ordinal.Equals(exportedConfig.Id, configuration.Id)) + { + Logger.Trace($"Found config in export as [{serializedConfig}]"); + foundConfig = true; + } + } + + foundConfig.Should().BeTrue(); } private static async Task DownloadFileAsync(StorageContainer storageContainer, string fileName) { CloudBlockBlob exportFile = storageContainer.CloudBlobContainer.GetBlockBlobReference(fileName); - string fileContents = await exportFile.DownloadTextAsync().ConfigureAwait(false); + return await exportFile.DownloadTextAsync().ConfigureAwait(false); + } - return fileContents; + private async Task CleanUpDevicesAsync( + string edgeId1, + string edgeId2, + string deviceId, + string configurationId, + RegistryManager registryManager) + { + try + { + await registryManager.RemoveDeviceAsync(deviceId).ConfigureAwait(false); + await registryManager.RemoveDeviceAsync(edgeId2).ConfigureAwait(false); + await registryManager.RemoveDeviceAsync(edgeId1).ConfigureAwait(false); + await registryManager.RemoveConfigurationAsync(configurationId).ConfigureAwait(false); + } + catch (Exception ex) + { + Logger.Trace($"Failed to remove device/config during cleanup due to {ex}"); + } } } } diff --git a/e2e/test/iothub/service/RegistryManagerImportDevicesTests.cs b/e2e/test/iothub/service/RegistryManagerImportDevicesTests.cs index 3522d7e9e3..2ccc77d349 100644 --- a/e2e/test/iothub/service/RegistryManagerImportDevicesTests.cs +++ b/e2e/test/iothub/service/RegistryManagerImportDevicesTests.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. + using System; using System.Collections.Generic; using System.IO; @@ -46,23 +47,28 @@ public async Task RegistryManager_ImportDevices(StorageAuthenticationType storag { // arrange - string deviceId = $"{nameof(RegistryManager_ImportDevices)}-device-{StorageContainer.GetRandomSuffix(4)}"; - string devicesFileName = $"{nameof(RegistryManager_ImportDevices)}-{StorageContainer.GetRandomSuffix(4)}.txt"; - using var registryManager = RegistryManager.CreateFromConnectionString(TestConfiguration.IoTHub.ConnectionString); + const string idPrefix = nameof(RegistryManager_ImportDevices); + + string deviceId = $"{idPrefix}-device-{StorageContainer.GetRandomSuffix(4)}"; + string configId = $"{idPrefix}-config-{StorageContainer.GetRandomSuffix(4)}".ToLower(); // Configuration Id characters must be all lower-case. + Logger.Trace($"Using Ids {deviceId} and {configId}."); - Logger.Trace($"Using deviceId {deviceId}."); + string devicesFileName = $"{idPrefix}-devices-{StorageContainer.GetRandomSuffix(4)}.txt"; + string configsFileName = $"{idPrefix}-configs-{StorageContainer.GetRandomSuffix(4)}.txt"; + + using RegistryManager registryManager = RegistryManager.CreateFromConnectionString(TestConfiguration.IoTHub.ConnectionString); try { string containerName = StorageContainer.BuildContainerName(nameof(RegistryManager_ImportDevices)); using StorageContainer storageContainer = await StorageContainer.GetInstanceAsync(containerName).ConfigureAwait(false); - Logger.Trace($"Using container {storageContainer.Uri}"); + Logger.Trace($"Using devices container {storageContainer.Uri}"); Uri containerUri = storageAuthenticationType == StorageAuthenticationType.KeyBased ? storageContainer.SasUri : storageContainer.Uri; - using Stream devicesStream = ImportExportDevicesHelpers.BuildDevicesStream( + using Stream devicesStream = ImportExportHelpers.BuildImportStream( new List { new ExportImportDevice( @@ -74,54 +80,44 @@ public async Task RegistryManager_ImportDevices(StorageAuthenticationType storag }); await UploadFileAndConfirmAsync(storageContainer, devicesStream, devicesFileName).ConfigureAwait(false); - // act - - JobProperties importJobResponse = null; - int tryCount = 0; - while (true) - { - try + using Stream configsStream = ImportExportHelpers.BuildImportStream( + new List { - ManagedIdentity identity = null; - if (isUserAssignedMsi) + new ImportConfiguration(configId) { - string userAssignedMsiResourceId = TestConfiguration.IoTHub.UserAssignedMsiResourceId; - identity = new ManagedIdentity + ImportMode = ConfigurationImportMode.CreateOrUpdateIfMatchETag, + Priority = 3, + Labels = { { "labelName", "labelValue" } }, + TargetCondition = "*", + Content = { - userAssignedIdentity = userAssignedMsiResourceId - }; - } - - importJobResponse = await registryManager - .ImportDevicesAsync( - JobProperties.CreateForImportJob( - containerUri.ToString(), - containerUri.ToString(), - devicesFileName, - storageAuthenticationType)) - .ConfigureAwait(false); - break; - } - // Concurrent jobs can be rejected, so implement a retry mechanism to handle conflicts with other tests - catch (JobQuotaExceededException) when (++tryCount < MaxIterationWait) - { - Logger.Trace($"JobQuotaExceededException... waiting."); - await Task.Delay(s_waitDuration).ConfigureAwait(false); - continue; - } - } + DeviceContent = { { "properties.desired.x", 5L } }, + }, + Metrics = + { + Queries = { { "successfullyConfigured", "select deviceId from devices where properties.reported.x = 5" } } + }, + }, + }); + await UploadFileAndConfirmAsync(storageContainer, configsStream, configsFileName).ConfigureAwait(false); - // wait for job to complete - for (int i = 0; i < MaxIterationWait; ++i) - { - await Task.Delay(1000).ConfigureAwait(false); - importJobResponse = await registryManager.GetJobAsync(importJobResponse.JobId).ConfigureAwait(false); - Logger.Trace($"Job {importJobResponse.JobId} is {importJobResponse.Status} with progress {importJobResponse.Progress}%"); - if (!s_incompleteJobs.Contains(importJobResponse.Status)) + ManagedIdentity identity = isUserAssignedMsi + ? new ManagedIdentity { - break; + UserAssignedIdentity = TestConfiguration.IoTHub.UserAssignedMsiResourceId } - } + : null; + + // act + + JobProperties importJobResponse = await CreateAndWaitForJobAsync( + storageAuthenticationType, + devicesFileName, + configsFileName, + registryManager, + containerUri, + identity) + .ConfigureAwait(false); // assert @@ -130,38 +126,48 @@ public async Task RegistryManager_ImportDevices(StorageAuthenticationType storag // should not throw due to 404, but device may not immediately appear in registry Device device = null; + Configuration config = null; for (int i = 0; i < MaxIterationWait; ++i) { await Task.Delay(s_waitDuration).ConfigureAwait(false); try { device = await registryManager.GetDeviceAsync(deviceId).ConfigureAwait(false); + config = await registryManager.GetConfigurationAsync(configId).ConfigureAwait(false); break; } catch (Exception ex) { - Logger.Trace($"Could not find device on iteration {i} due to [{ex.Message}]"); + Logger.Trace($"Could not find device/config on iteration {i} due to [{ex.Message}]"); } } if (device == null) { Assert.Fail($"Device {deviceId} not found in registry manager"); } + if (config == null) + { + Assert.Fail($"Config {configId} not found in registry manager"); + } } finally { try { await registryManager.RemoveDeviceAsync(deviceId).ConfigureAwait(false); + await registryManager.RemoveConfigurationAsync(configId).ConfigureAwait(false); + } + catch (Exception ex) + { + Logger.Trace($"Failed to clean up device/config due to {ex}"); } - catch { } } } - private static async Task UploadFileAndConfirmAsync(StorageContainer storageContainer, Stream devicesFile, string fileName) + private static async Task UploadFileAndConfirmAsync(StorageContainer storageContainer, Stream fileContents, string fileName) { CloudBlockBlob cloudBlob = storageContainer.CloudBlobContainer.GetBlockBlobReference(fileName); - await cloudBlob.UploadFromStreamAsync(devicesFile).ConfigureAwait(false); + await cloudBlob.UploadFromStreamAsync(fileContents).ConfigureAwait(false); // wait for blob to be written bool foundBlob = false; @@ -174,7 +180,62 @@ private static async Task UploadFileAndConfirmAsync(StorageContainer storageCont break; } } - foundBlob.Should().BeTrue($"Failed to find {fileName} in storage container, required for test."); + foundBlob.Should().BeTrue($"Failed to find {fileName} in storage container - required for test."); + } + + private async Task CreateAndWaitForJobAsync( + StorageAuthenticationType storageAuthenticationType, + string devicesFileName, + string configsFileName, + RegistryManager registryManager, + Uri containerUri, + ManagedIdentity identity) + { + int tryCount = 0; + JobProperties importJobResponse = null; + + JobProperties jobProperties = JobProperties.CreateForImportJob( + containerUri.ToString(), + containerUri.ToString(), + devicesFileName, + storageAuthenticationType, + identity); + jobProperties.ConfigurationsBlobName = configsFileName; + jobProperties.IncludeConfigurations = true; + + while (tryCount < MaxIterationWait) + { + try + { + importJobResponse = await registryManager.ImportDevicesAsync(jobProperties).ConfigureAwait(false); + if (!string.IsNullOrWhiteSpace(importJobResponse.FailureReason)) + { + Logger.Trace($"Job failed due to {importJobResponse.FailureReason}"); + } + break; + } + // Concurrent jobs can be rejected, so implement a retry mechanism to handle conflicts with other tests + catch (JobQuotaExceededException) when (++tryCount < MaxIterationWait) + { + Logger.Trace($"JobQuotaExceededException... waiting."); + await Task.Delay(s_waitDuration).ConfigureAwait(false); + continue; + } + } + + // wait for job to complete + for (int i = 0; i < MaxIterationWait; ++i) + { + await Task.Delay(1000).ConfigureAwait(false); + importJobResponse = await registryManager.GetJobAsync(importJobResponse?.JobId).ConfigureAwait(false); + Logger.Trace($"Job {importJobResponse.JobId} is {importJobResponse.Status} with progress {importJobResponse.Progress}%"); + if (!s_incompleteJobs.Contains(importJobResponse.Status)) + { + break; + } + } + + return importJobResponse; } } } diff --git a/iothub/service/src/Configurations/Configuration.cs b/iothub/service/src/Configurations/Configuration.cs index 50b0c597fe..6eb6987e33 100644 --- a/iothub/service/src/Configurations/Configuration.cs +++ b/iothub/service/src/Configurations/Configuration.cs @@ -9,22 +9,22 @@ namespace Microsoft.Azure.Devices { /// - /// Device configurations provide the ability to perform IoT device configuration at scale. - /// You can define configurations and summarize compliance as the configuration is applied. + /// The configuration for IoT hub device and module twins. /// /// + /// Device configurations provide the ability to perform IoT device configuration at scale. + /// You can define configurations and summarize compliance as the configuration is applied. /// See for more details. /// - [System.Diagnostics.CodeAnalysis.SuppressMessage( - "Naming", - "CA1724:Type names should not match namespaces", - Justification = "Cannot change type names as it is considered a breaking change.")] public class Configuration : IETagHolder { /// /// Initializes a new instance of the class. /// - /// The configuration Id. Lowercase and the following special characters are allowed: [-+%_*!']. + /// + /// The configuration Id. + /// Lowercase and the following special characters are allowed: [-+%_*!']. + /// public Configuration(string configurationId) : this() { @@ -41,19 +41,19 @@ internal Configuration() } /// - /// Gets the identifier for the configuration. + /// The unique identifier of the configuration. /// [JsonProperty(PropertyName = "id", Required = Required.Always)] public string Id { get; internal set; } /// - /// Gets Schema version for the configuration + /// The schema version of the configuration. /// [JsonProperty(PropertyName = "schemaVersion", NullValueHandling = NullValueHandling.Ignore)] public string SchemaVersion { get; } /// - /// Gets or sets labels for the configuration + /// The key-value pairs used to describe the configuration. /// [JsonProperty(PropertyName = "labels", NullValueHandling = NullValueHandling.Ignore)] #pragma warning disable CA2227 // Collection properties should be read only @@ -62,7 +62,7 @@ internal Configuration() #pragma warning restore CA2227 // Collection properties should be read only /// - /// Gets or sets content for the configuration + /// The content of the configuration. /// [JsonProperty(PropertyName = "content", NullValueHandling = NullValueHandling.Ignore)] public ConfigurationContent Content { get; set; } = new ConfigurationContent(); @@ -74,45 +74,48 @@ internal Configuration() public string ContentType { get; } /// - /// Gets or sets target condition for the configuration + /// The query used to define the targeted devices or modules. /// + /// + /// The query is based on twin tags and/or reported properties. + /// [JsonProperty(PropertyName = "targetCondition")] public string TargetCondition { get; set; } /// - /// Gets creation time for the configuration + /// The creation date and time of the configuration. /// [JsonProperty(PropertyName = "createdTimeUtc")] public DateTime CreatedTimeUtc { get; internal set; } /// - /// Gets last update time for the configuration + /// The update date and time of the configuration. /// [JsonProperty(PropertyName = "lastUpdatedTimeUtc")] public DateTime LastUpdatedTimeUtc { get; internal set; } /// - /// Gets or sets priority for the configuration + /// The priority number assigned to the configuration. /// [JsonProperty(PropertyName = "priority")] public int Priority { get; set; } /// - /// System configuration metrics + /// The system metrics computed by the IoT Hub that cannot be customized. /// [JsonProperty(PropertyName = "systemMetrics", NullValueHandling = NullValueHandling.Ignore)] public ConfigurationMetrics SystemMetrics { get; internal set; } = new ConfigurationMetrics(); /// - /// Custom configuration metrics + /// The custom metrics specified by the developer as queries against twin reported properties. /// [JsonProperty(PropertyName = "metrics", NullValueHandling = NullValueHandling.Ignore)] public ConfigurationMetrics Metrics { get; set; } = new ConfigurationMetrics(); /// - /// Gets or sets configuration's ETag + /// The ETag of the configuration. /// - [JsonProperty(PropertyName = "etag")] + [JsonProperty(PropertyName = "etag", NullValueHandling = NullValueHandling.Ignore)] public string ETag { get; set; } } } diff --git a/iothub/service/src/Configurations/ConfigurationContent.cs b/iothub/service/src/Configurations/ConfigurationContent.cs index a203797d55..a8077d0a45 100644 --- a/iothub/service/src/Configurations/ConfigurationContent.cs +++ b/iothub/service/src/Configurations/ConfigurationContent.cs @@ -12,7 +12,7 @@ namespace Microsoft.Azure.Devices public class ConfigurationContent { /// - /// Gets or sets the configurations to be applied. + /// The modules configuration content. /// /// /// See @@ -25,7 +25,7 @@ public class ConfigurationContent #pragma warning restore CA2227 // Collection properties should be read only /// - /// Gets or sets the configurations to be applied on device modules + /// The device module configuration content. /// [JsonProperty(PropertyName = "moduleContent")] #pragma warning disable CA2227 // Collection properties should be read only @@ -34,7 +34,7 @@ public class ConfigurationContent #pragma warning restore CA2227 // Collection properties should be read only /// - /// Gets or sets the configurations to be applied on devices. + /// The device configuration content. /// [JsonProperty(PropertyName = "deviceContent")] #pragma warning disable CA2227 // Collection properties should be read only diff --git a/iothub/service/src/Configurations/ConfigurationImportMode.cs b/iothub/service/src/Configurations/ConfigurationImportMode.cs new file mode 100644 index 0000000000..137951cc67 --- /dev/null +++ b/iothub/service/src/Configurations/ConfigurationImportMode.cs @@ -0,0 +1,21 @@ +using System.Runtime.Serialization; +using Newtonsoft.Json; +using Newtonsoft.Json.Converters; + +namespace Microsoft.Azure.Devices +{ + /// + /// Identifies the behavior when merging a configuration to the registry during import actions. + /// + [JsonConverter(typeof(StringEnumConverter))] + public enum ConfigurationImportMode + { + /// + /// If a configuration does not exist with the specified Id, it is newly registered. + /// If the configuration already exists, existing information is overwritten with the provided input data only if there is an ETag match. + /// If there is an ETag mismatch, an error is written to the log file. + /// + [EnumMember(Value = "createOrUpdateIfMatchETag")] + CreateOrUpdateIfMatchETag = 4, + } +} diff --git a/iothub/service/src/Configurations/ImportConfiguration.cs b/iothub/service/src/Configurations/ImportConfiguration.cs new file mode 100644 index 0000000000..0a29895799 --- /dev/null +++ b/iothub/service/src/Configurations/ImportConfiguration.cs @@ -0,0 +1,23 @@ +using Newtonsoft.Json; + +namespace Microsoft.Azure.Devices +{ + /// + /// A class for creating and serializing a for a bulk import + /// job using . + /// + public class ImportConfiguration : Configuration + { + /// + public ImportConfiguration(string configurationId) + : base(configurationId) + { + } + + /// + /// The type of registry operation and E Tag preferences. + /// + [JsonProperty(PropertyName = "importMode", DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate)] + public ConfigurationImportMode ImportMode { get; set; } + } +} diff --git a/iothub/service/src/ExportImportDevice.cs b/iothub/service/src/ExportImportDevice.cs index f6ea5dd3c6..3ef0570bbc 100644 --- a/iothub/service/src/ExportImportDevice.cs +++ b/iothub/service/src/ExportImportDevice.cs @@ -1,12 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -// --------------------------------------------------------------- -// Copyright (c) Microsoft Corporation. All rights reserved. -// --------------------------------------------------------------- using System; -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; using Microsoft.Azure.Devices.Shared; using Newtonsoft.Json; @@ -21,19 +16,34 @@ public sealed class ExportImportDevice private string _twinETag; /// - /// Type definition for the property. + /// The desired and reported properties of the twin. /// + /// + /// Type definition for the property. + /// + /// The maximum depth of the object is 10. + /// public sealed class PropertyContainer { /// - /// Desired properties are requested updates by a service client. + /// The collection of desired property key-value pairs. /// + /// + /// The keys are UTF-8 encoded, case-sensitive and up-to 1KB in length. Allowed characters + /// exclude UNICODE control characters (segments C0 and C1), '.', '$' and space. The + /// desired porperty values are JSON objects, up-to 4KB in length. + /// [JsonProperty(PropertyName = "desired", NullValueHandling = NullValueHandling.Ignore)] public TwinCollection DesiredProperties { get; set; } /// - /// Reported properties are the latest value reported by the device. + /// The collection of reported property key-value pairs. /// + /// + /// The keys are UTF-8 encoded, case-sensitive and up-to 1KB in length. Allowed characters + /// exclude UNICODE control characters (segments C0 and C1), '.', '$' and space. The + /// reported property values are JSON objects, up-to 4KB in length. + /// [JsonProperty(PropertyName = "reported", NullValueHandling = NullValueHandling.Ignore)] public TwinCollection ReportedProperties { get; set; } } @@ -65,17 +75,16 @@ public ExportImportDevice(Device device, ImportMode importmode) Authentication = device.Authentication; Capabilities = device.Capabilities; DeviceScope = device.Scope; - ParentScopes = device.ParentScopes; } /// - /// Id of the device. + /// The unique identifier of the device. /// [JsonProperty(PropertyName = "id", Required = Required.Always)] public string Id { get; set; } /// - /// Module Id for the object. + /// The unique identifier of the module, if applicable. /// [JsonProperty(PropertyName = "moduleId", NullValueHandling = NullValueHandling.Ignore)] public string ModuleId { get; set; } @@ -83,6 +92,10 @@ public ExportImportDevice(Device device, ImportMode importmode) /// /// A string representing an ETag for the entity as per RFC7232. /// + /// + /// The value is only used if import mode is updateIfMatchETag, in that case the import operation is performed + /// only if this ETag matches the value maintained by the server. + /// [JsonProperty(PropertyName = "eTag", NullValueHandling = NullValueHandling.Ignore)] public string ETag { @@ -91,32 +104,46 @@ public string ETag } /// - /// Import mode of the device. + /// The type of registry operation and ETag preferences. /// [JsonProperty(PropertyName = "importMode", DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate)] public ImportMode ImportMode { get; set; } /// - /// Status of the device. + /// The status of the device or module. /// + /// + /// If disabled, it cannot connect to the service. + /// [JsonProperty(PropertyName = "status", Required = Required.Always)] public DeviceStatus Status { get; set; } /// - /// Status reason of the device. + /// The 128 character-long string that stores the reason for the device identity status. /// + /// + /// All UTF-8 characters are allowed. + /// [JsonProperty(PropertyName = "statusReason", NullValueHandling = NullValueHandling.Ignore)] public string StatusReason { get; set; } /// - /// Authentication mechanism of the device. + /// The authentication mechanism used by the module. /// + /// + /// This parameter is optional and defaults to SAS if not provided. In that case, primary/secondary + /// access keys are auto-generated. + /// [JsonProperty(PropertyName = "authentication")] public AuthenticationMechanism Authentication { get; set; } /// /// String representing a Twin ETag for the entity, as per RFC7232. /// + /// + /// The value is only used if import mode is updateIfMatchETag, in that case the import operation is + /// performed only if this ETag matches the value maintained by the server. + /// [JsonProperty(PropertyName = "twinETag", NullValueHandling = NullValueHandling.Ignore)] public string TwinETag { @@ -125,50 +152,37 @@ public string TwinETag } /// - /// Tags representing a collection of properties. + /// The JSON document read and written by the solution back end. The tags are not visible to device apps. /// [JsonProperty(PropertyName = "tags", NullValueHandling = NullValueHandling.Ignore)] public TwinCollection Tags { get; set; } /// - /// Desired and reported property bags + /// The desired and reported properties for the device or module. /// [JsonProperty(PropertyName = "properties", NullValueHandling = NullValueHandling.Ignore)] public PropertyContainer Properties { get; set; } /// - /// Status of capabilities enabled on the device + /// Status of capabilities enabled on the device or module. /// [JsonProperty(PropertyName = "capabilities", NullValueHandling = NullValueHandling.Ignore)] public DeviceCapabilities Capabilities { get; set; } /// - /// The scope of the device. For edge devices, this is auto-generated and immutable. For leaf devices, set this to create child/parent - /// relationship. + /// The scope of the device. For edge devices, this is auto-generated and immutable. For leaf + /// devices, set this to create child/parent relationship. /// /// - /// For leaf devices, the value to set a parent edge device can be retrieved from the parent edge device's device scope property. + /// For leaf devices, the value to set a parent edge device can be retrieved from the parent + /// edge device's device scope property. /// - /// For more information, see . + /// For more information, see + /// . /// [JsonProperty(PropertyName = "deviceScope", NullValueHandling = NullValueHandling.Include)] public string DeviceScope { get; set; } - /// - /// The scopes of the upper level edge devices if applicable. - /// - /// - /// For edge devices, the value to set a parent edge device can be retrieved from the parent edge device's property. - /// - /// For leaf devices, this could be set to the same value as or left for the service to copy over. - /// - /// For now, this list can only have 1 element in the collection. - /// - /// For more information, see . - /// - [JsonProperty(PropertyName = "parentScopes", NullValueHandling = NullValueHandling.Ignore)] - public IList ParentScopes { get; internal set; } = new List(); - private static string SanitizeETag(string eTag) { if (!string.IsNullOrWhiteSpace(eTag)) diff --git a/iothub/service/src/JobProperties.cs b/iothub/service/src/JobProperties.cs index fbab545b2b..b3cd816cb8 100644 --- a/iothub/service/src/JobProperties.cs +++ b/iothub/service/src/JobProperties.cs @@ -8,7 +8,8 @@ namespace Microsoft.Azure.Devices { /// /// Contains properties of a Job. - /// See online documentation for more infomration. + /// See online documentation + /// for more infomration. /// public class JobProperties { @@ -124,13 +125,31 @@ public JobProperties() [JsonProperty(PropertyName = "identity", NullValueHandling = NullValueHandling.Ignore)] public ManagedIdentity Identity { get; set; } + /// + /// Whether or not to include configurations in the import or export job. + /// + /// + /// The service assumes this is false, if not specified. If true, then configurations are included in the data export/import. + /// + [JsonProperty(PropertyName = "includeConfigurations", NullValueHandling = NullValueHandling.Ignore)] + public bool? IncludeConfigurations { get; set; } + + /// + /// Specifies the name of the blob to use when exporting/importing configurations. + /// + /// + /// The service assumes this is configurations.txt, if not specified. + /// + [JsonProperty(PropertyName = "configurationsBlobName", NullValueHandling = NullValueHandling.Ignore)] + public string ConfigurationsBlobName { get; set; } + #pragma warning disable CA1054 // Uri parameters should not be strings /// - /// Creates an instance of JobProperties with parameters ready to start an Import job + /// Creates an instance of JobProperties with parameters ready to start an import job. /// /// URI to a blob container that contains registry data to sync. Including a SAS token is dependent on the parameter. - /// URI to a blob container. This is used to output the status of the job and the results. Including a SAS token is dependent on the parameter. + /// URI to a blob container. This is used to output the status of the job and the results. Including a SAS token is dependent on the parameter. /// The blob name to be used when importing from the provided input blob container /// Specifies authentication type being used for connecting to storage account /// User assigned managed identity used to access storage account for import and export jobs. @@ -149,12 +168,12 @@ public static JobProperties CreateForImportJob( OutputBlobContainerUri = outputBlobContainerUri, InputBlobName = inputBlobName, StorageAuthenticationType = storageAuthenticationType, - Identity = identity + Identity = identity, }; } /// - /// Creates an instance of JobProperties with parameters ready to start an Import job + /// Creates an instance of JobProperties with parameters ready to start an export job. /// /// URI to a blob container. This is used to output the status of the job and the results. Including a SAS token is dependent on the parameter. /// Indicates if authorization keys are included in export output @@ -176,7 +195,7 @@ public static JobProperties CreateForExportJob( ExcludeKeysInExport = excludeKeysInExport, OutputBlobName = outputBlobName, StorageAuthenticationType = storageAuthenticationType, - Identity = identity + Identity = identity, }; } diff --git a/iothub/service/src/ManagedIdentity.cs b/iothub/service/src/ManagedIdentity.cs index 2fa2467be1..2c55ec1fa9 100644 --- a/iothub/service/src/ManagedIdentity.cs +++ b/iothub/service/src/ManagedIdentity.cs @@ -2,8 +2,7 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System; -using System.Collections.Generic; -using System.Text; +using System.ComponentModel; using Newtonsoft.Json; namespace Microsoft.Azure.Devices @@ -15,10 +14,22 @@ namespace Microsoft.Azure.Devices /// public class ManagedIdentity { + /// + /// The user identity resource Id used to access the storage account for import and export jobs. + /// + [Obsolete("Use UserAssignedIdentity instead")] + [EditorBrowsable(EditorBrowsableState.Never)] + [JsonIgnore] + public string userAssignedIdentity + { + get => UserAssignedIdentity; + set => UserAssignedIdentity = value; + } + /// /// The user identity resource Id used to access the storage account for import and export jobs. /// [JsonProperty(PropertyName = "userAssignedIdentity", NullValueHandling = NullValueHandling.Ignore)] - public string userAssignedIdentity { get; set; } + public string UserAssignedIdentity { get; set; } } } diff --git a/iothub/service/src/RegistryManager.cs b/iothub/service/src/RegistryManager.cs index 5044e19bfd..a92958bbcd 100644 --- a/iothub/service/src/RegistryManager.cs +++ b/iothub/service/src/RegistryManager.cs @@ -1410,6 +1410,7 @@ public virtual Task ExportDevicesAsync(string exportBlobContainer /// /// Parameters for the job. /// Task cancellation token. + /// Conditionally includes configurations, if specified. /// JobProperties of the newly created job. public virtual Task ExportDevicesAsync(JobProperties jobParameters, CancellationToken cancellationToken = default) { @@ -1507,6 +1508,7 @@ public virtual Task ImportDevicesAsync(string importBlobContainer /// /// Parameters for the job. /// Task cancellation token. + /// Conditionally includes configurations, if specified. /// JobProperties of the newly created job. public virtual Task ImportDevicesAsync(JobProperties jobParameters, CancellationToken cancellationToken = default) {