diff --git a/SDK v2 migration guide.md b/SDK v2 migration guide.md index 6163f8822a..96a3c30926 100644 --- a/SDK v2 migration guide.md +++ b/SDK v2 migration guide.md @@ -284,9 +284,9 @@ These span across all service clients. #### Notable breaking changes - Operations that offer concurrency protection using `ETag`s, now take a parameter `onlyIfUnchanged` that relies on the ETag property of the submitted entity. -- `IotHubServiceClient.Query.CreateAsync(...)` now returns an `AsyncPageable` of the queried results. - - Iterate on entries by using `await foreach (ClientTwin queriedTwin in client.Query.CreateAsync(queryText))`. - - Iterate on pages of entries by using `await foreach (Page queriedTwinPage in _client.Query.CreateAsync(queryText).AsPages())` +- `IotHubServiceClient.Query.Create(...)` now returns an `AsyncPageable` of the queried results. + - Iterate on entries by using `await foreach (ClientTwin queriedTwin in client.Query.Create(queryText))`. + - Iterate on pages of entries by using `await foreach (Page queriedTwinPage in _client.Query.Create(queryText).AsPages())`. - `JobProperties` properties that hold Azure Storage SAS URIs are now of type `System.Uri` instead of `string`. - `JobProperties` has been split into several classes with only the necessary properties for the specified operation. - See `ExportJobProperties`, `ImportJobProperties`, and `IotHubJobResponse`. @@ -326,7 +326,7 @@ These span across all service clients. | `DeviceConnectionState` | `ClientConnectionState` | See² | | `DeviceStatus` | `ClientStatus` | See² | | `DeviceCapabilities` | `ClientCapabilities` | See² | -| `RegistryManager.CreateQuery(...)` | `IotHubServiceClient.Query.CreateAsync(...)` | | +| `RegistryManager.CreateQuery(...)` | `IotHubServiceClient.Query.Create(...)` | | | `RegistryManager.AddConfigurationAsync(...)` | `IotHubServiceClient.Configurations.CreateAsync(...)` | | | `RegistryManager.GetConfigurationsAsync(int maxCount)`| `IotHubServiceClient.Configurations.GetAsync(int maxCount)` | | | `RegistryManager.RemoveConfigurationAsync(...)` | `IotHubServiceClient.Configurations.DeleteAsync(...)` | | @@ -385,7 +385,7 @@ These span across all service clients. |:---|:---|:---| | `JobsClient` | `IotHubServiceClient`, subclients `ScheduledJobs` | | | `JobClient.GetJobAsync(...)` | `IotHubServiceClient.ScheduledJobs.GetAsync(...)` | | -| `JobClient.CreateQuery()` | `IotHubServiceClient.ScheduledJobs.CreateQueryAsync()` | | +| `JobClient.CreateQuery()` | `IotHubServiceClient.ScheduledJobs.CreateQuery()` | | | `JobsClient.ScheduleTwinUpdateAsync(...)` | `IotHubServiceClient.ScheduledJobs.ScheduledTwinUpdateAsync(...)` | | | `JobType.ExportDevices` | `JobType.Export` | Matches the actual value expected by the service.¹ | | `JobType.ImportDevices` | `JobType.Import` | See¹ | @@ -449,7 +449,15 @@ These span across all service clients. #### Notable breaking changes -- Query methods (like for individual and group enrollments) now take a query string (and optionally a page size parameter), and the `Query` result no longer requires disposing. +- `ProvisioningServiceClient.DeviceRegistrationStates.CreateEnrollmentGroupQuery(...)` now returns an `AsyncPageable` of the queried results. + - Iterate on entries by using `await foreach (DeviceRegistrationState registrationState in client.DeviceRegistrationStates.CreateEnrollmentGroupQuery(queryText))`. + - Iterate on pages of entries by using `await foreach (Page registrationStatePage in client.DeviceRegistrationStates.CreateEnrollmentGroupQuery(queryText).AsPages())`. +- `ProvisioningServiceClient.IndividualEnrollments.CreateQuery(...)` now returns an `AsyncPageable` of the queried results. + - Iterate on entries by using `await foreach (IndividualEnrollment enrollment in client.IndividualEnrollments.CreateQuery(queryText))`. + - Iterate on pages of entries by using `await foreach (Page enrollmentsPage in client.IndividualEnrollments.CreateQuery(queryText).AsPages())`. +- `ProvisioningServiceClient.EnrollmentGroups.CreateQuery(...)` now returns an `AsyncPageable` of the queried results. + - Iterate on entries by using `await foreach (EnrollmentGroup enrollment in client.EnrollmentGroups.CreateQuery(queryText))`. + - Iterate on pages of entries by using `await foreach (Page enrollmentsPage in client.EnrollmentGroups.CreateQuery(queryText).AsPages())`. - ETag fields on the classes `IndividualEnrollment`, `EnrollmentGroup`, and `DeviceRegistrationState` are now taken as the `Azure.ETag` type instead of strings. - Twin.Tags is now of type `IDictionary`. - `CustomAllocationDefinition.WebhookUri` is now of type `System.Uri` instead of `System.String`. @@ -505,6 +513,10 @@ These span across all service clients. | `X509CertificateInfo.SHA256Thumbprint` | `X509CertificateInfo.Sha256Thumbprint` | See³ | | `ProvisioningServiceClientException` | `ProvisioningServiceException` | | | `ProvisioningClientCapabilities.IotEdge` | `InitialClientCapabilities.IsIotEdge` | Boolean properties should start with a verb, usually "Is". | +| `Query` | Class removed | `AsyncPageable` type replaces this type and is returned by all query functions now | +| `QueryResult` | Class removed | `AsyncPageable` type replaces this type and is returned by all query functions now | +| `QueryResultType` | Class removed | The `AsyncPageable` returned by each Query API has a hardcoded type now (`IndividualEnrollment`, `EnrollmentGroup`, or `DeviceRegistrationState`) | + ### Security provider client diff --git a/e2e/test/iothub/service/IoTHubCertificateValidationE2ETest.cs b/e2e/test/iothub/service/IoTHubCertificateValidationE2ETest.cs index c74953beef..115a726681 100644 --- a/e2e/test/iothub/service/IoTHubCertificateValidationE2ETest.cs +++ b/e2e/test/iothub/service/IoTHubCertificateValidationE2ETest.cs @@ -24,7 +24,7 @@ public async Task ServiceClient_QueryDevicesInvalidServiceCertificateHttp_Fails( using var sc = new IotHubServiceClient(TestConfiguration.IotHub.ConnectionStringInvalidServiceCertificate); // act - Func act = async () => await sc.Query.CreateAsync("select * from devices").GetAsyncEnumerator().MoveNextAsync().ConfigureAwait(false); + Func act = async () => await sc.Query.Create("select * from devices").GetAsyncEnumerator().MoveNextAsync().ConfigureAwait(false); // assert var error = await act.Should().ThrowAsync(); diff --git a/e2e/test/iothub/service/QueryClientE2ETests.cs b/e2e/test/iothub/service/QueryClientE2ETests.cs index b6cf2ae9cf..3d60ed00bc 100644 --- a/e2e/test/iothub/service/QueryClientE2ETests.cs +++ b/e2e/test/iothub/service/QueryClientE2ETests.cs @@ -45,7 +45,7 @@ public async Task TwinQuery_Works() await WaitForDevicesToBeQueryableAsync(serviceClient.Query, queryText, 2).ConfigureAwait(false); - AsyncPageable queryResponse = serviceClient.Query.CreateAsync(queryText); + AsyncPageable queryResponse = serviceClient.Query.Create(queryText); IAsyncEnumerator enumerator = queryResponse.GetAsyncEnumerator(); // assert @@ -77,8 +77,8 @@ public async Task TwinQuery_CustomPaginationWorks() await WaitForDevicesToBeQueryableAsync(serviceClient.Query, queryText, 3).ConfigureAwait(false); AsyncPageable queryResponse = serviceClient.Query. - CreateAsync(queryText); - IAsyncEnumerator> enumerator = queryResponse.AsPages(null, 1).GetAsyncEnumerator(); + Create(queryText); + await using IAsyncEnumerator> enumerator = queryResponse.AsPages(null, 1).GetAsyncEnumerator(); (await enumerator.MoveNextAsync().ConfigureAwait(false)).Should().BeTrue("Should have at least one page of jobs."); // assert @@ -89,13 +89,13 @@ public async Task TwinQuery_CustomPaginationWorks() // restart the query, but with a page size of 3 this time queryResponse = serviceClient.Query. - CreateAsync(queryText); - enumerator = queryResponse.AsPages(null, 3).GetAsyncEnumerator(); + Create(queryText); + await using IAsyncEnumerator> nextEnumerator = queryResponse.AsPages(null, 3).GetAsyncEnumerator(); // consume the first page of results so the next MoveNextAsync gets a new page - (await enumerator.MoveNextAsync().ConfigureAwait(false)).Should().BeTrue("Should have at least one page of jobs."); + (await nextEnumerator.MoveNextAsync().ConfigureAwait(false)).Should().BeTrue("Should have at least one page of jobs."); - currentPage = enumerator.Current; + currentPage = nextEnumerator.Current; currentPage.Values.Count.Should().Be(3); IEnumerator pageContentsEnumerator = currentPage.Values.GetEnumerator(); pageContentsEnumerator.MoveNext().Should().BeTrue(); @@ -111,7 +111,7 @@ public async Task TwinQuery_CustomPaginationWorks() ClientTwin thirdQueriedTwin = pageContentsEnumerator.Current; thirdQueriedTwin.DeviceId.Should().BeOneOf(testDevice1.Id, testDevice2.Id, testDevice3.Id); - (await enumerator.MoveNextAsync().ConfigureAwait(false)).Should().BeFalse("After 3 query results in one page, there should not be a second page"); + (await nextEnumerator.MoveNextAsync().ConfigureAwait(false)).Should().BeFalse("After 3 query results in one page, there should not be a second page"); } [TestMethod] @@ -131,14 +131,14 @@ public async Task TwinQuery_IterateByItemAcrossPages() await WaitForDevicesToBeQueryableAsync(serviceClient.Query, queryText, 3).ConfigureAwait(false); - AsyncPageable twinQuery = serviceClient.Query. - CreateAsync(queryText); + // For this test, we want the query logic to have to fetch multiple pages of results. To force + // that, set the page size to 1 when there are 3 total results to be queried. + IAsyncEnumerable> twinPages = serviceClient.Query. + Create(queryText) + .AsPages(null, 1); // assert - // For this test, we want the query logic to have to fetch multiple pages of results. To force - // that, set the page size to 1 when there are 3 total results to be queried. - IAsyncEnumerable> twinPages = twinQuery.AsPages(null, 1); var returnedTwinDeviceIds = new List(); await foreach (Page queriedTwinPage in twinPages) { @@ -146,6 +146,8 @@ public async Task TwinQuery_IterateByItemAcrossPages() { returnedTwinDeviceIds.Add(queriedTwin.DeviceId); } + + queriedTwinPage.GetRawResponse().Dispose(); } var expectedDeviceIds = new List() { testDevice1.Id, testDevice2.Id, testDevice3.Id }; @@ -172,7 +174,7 @@ public async Task TwinQuery_IterateByItemWorksWithinPage() await WaitForDevicesToBeQueryableAsync(serviceClient.Query, queryText, 3).ConfigureAwait(false); AsyncPageable twinQuery = serviceClient.Query - .CreateAsync(queryText); + .Create(queryText); // assert @@ -186,6 +188,8 @@ public async Task TwinQuery_IterateByItemWorksWithinPage() { returnedTwinDeviceIds.Add(queriedTwin.DeviceId); } + + queriedTwinPage.GetRawResponse().Dispose(); } var expectedDeviceIds = new List() { testDevice1.Id, testDevice2.Id, testDevice3.Id }; @@ -205,7 +209,7 @@ public async Task JobQuery_QueryWorks() string query = "SELECT * FROM devices.jobs"; await WaitForJobToBeQueryableAsync(serviceClient.Query, query, 1).ConfigureAwait(false); - AsyncPageable queryResponse = serviceClient.Query.CreateAsync(query); + AsyncPageable queryResponse = serviceClient.Query.Create(query); IAsyncEnumerator enumerator = queryResponse.GetAsyncEnumerator(); (await enumerator.MoveNextAsync().ConfigureAwait(false)).Should().BeTrue("Should have at least one page of jobs."); ScheduledJob queriedJob = enumerator.Current; @@ -234,7 +238,7 @@ public async Task JobQuery_QueryByTypeWorks() await ScheduleJobToBeQueriedAsync(serviceClient.ScheduledJobs, testDevice.Id).ConfigureAwait(false); await WaitForJobToBeQueryableAsync(serviceClient.Query, 1, null, null).ConfigureAwait(false); - AsyncPageable queryResponse = serviceClient.Query.CreateJobsQueryAsync(); + AsyncPageable queryResponse = serviceClient.Query.CreateJobsQuery(); IAsyncEnumerator enumerator = queryResponse.GetAsyncEnumerator(); (await enumerator.MoveNextAsync().ConfigureAwait(false)).Should().BeTrue("Should have at least one page of jobs."); ScheduledJob queriedJob = enumerator.Current; @@ -258,8 +262,8 @@ public async Task RawQuery_QueryWorks() string query = "SELECT COUNT() as TotalNumberOfDevices FROM devices"; - AsyncPageable queryResponse = serviceClient.Query.CreateAsync(query); - IAsyncEnumerator enumerator = queryResponse.GetAsyncEnumerator(); + AsyncPageable queryResponse = serviceClient.Query.Create(query); + await using IAsyncEnumerator enumerator = queryResponse.GetAsyncEnumerator(); (await enumerator.MoveNextAsync().ConfigureAwait(false)).Should().BeTrue("Should have at least one page of jobs."); RawQuerySerializationClass queriedJob = enumerator.Current; queriedJob.TotalNumberOfDevices.Should().BeGreaterOrEqualTo(0); @@ -271,15 +275,18 @@ private async Task WaitForDevicesToBeQueryableAsync(QueryClient queryClient, str // so keep executing the query until both devices are returned in the results or until a timeout. using var cancellationTokenSource = new CancellationTokenSource(_queryableDelayTimeout); CancellationToken cancellationToken = cancellationTokenSource.Token; - IAsyncEnumerator> enumerator = queryClient.CreateAsync(query).AsPages().GetAsyncEnumerator(); + IAsyncEnumerator> enumerator = queryClient.Create(query).AsPages().GetAsyncEnumerator(); await enumerator.MoveNextAsync(); while (enumerator.Current.Values.Count < expectedCount) { await Task.Delay(100).ConfigureAwait(false); - enumerator = queryClient.CreateAsync(query).AsPages().GetAsyncEnumerator(); + await enumerator.DisposeAsync().ConfigureAwait(false); + enumerator = queryClient.Create(query).AsPages().GetAsyncEnumerator(); await enumerator.MoveNextAsync().ConfigureAwait(false); cancellationToken.ThrowIfCancellationRequested(); // timed out waiting for the devices to become queryable } + + await enumerator.DisposeAsync().ConfigureAwait(false); } private async Task WaitForJobToBeQueryableAsync(QueryClient queryClient, string query, int expectedCount) @@ -287,14 +294,18 @@ private async Task WaitForJobToBeQueryableAsync(QueryClient queryClient, string // There is some latency between the creation of the test devices and when they are queryable, // so keep executing the query until both devices are returned in the results or until a timeout. using var cancellationTokenSource = new CancellationTokenSource(_queryableDelayTimeout); - IAsyncEnumerator> enumerator; - do + IAsyncEnumerator> enumerator = queryClient.Create(query).AsPages().GetAsyncEnumerator(); + await enumerator.MoveNextAsync().ConfigureAwait(false); + while (enumerator.Current.Values.Count < expectedCount) { await Task.Delay(100).ConfigureAwait(false); cancellationTokenSource.Token.IsCancellationRequested.Should().BeFalse("timed out waiting for the devices to become queryable"); - enumerator = queryClient.CreateAsync(query).AsPages().GetAsyncEnumerator(); + await enumerator.DisposeAsync().ConfigureAwait(false); + enumerator = queryClient.Create(query).AsPages().GetAsyncEnumerator(); await enumerator.MoveNextAsync().ConfigureAwait(false); - } while (enumerator.Current.Values.Count < expectedCount); + } + + await enumerator.DisposeAsync().ConfigureAwait(false); } private async Task WaitForJobToBeQueryableAsync(QueryClient queryClient, int expectedCount, JobType? jobType = null, JobStatus? status = null) @@ -308,15 +319,18 @@ private async Task WaitForJobToBeQueryableAsync(QueryClient queryClient, int exp JobType = jobType, JobStatus = status, }; - IAsyncEnumerator> enumerator = queryClient.CreateJobsQueryAsync(options).AsPages().GetAsyncEnumerator(); + IAsyncEnumerator> enumerator = queryClient.CreateJobsQuery(options).AsPages().GetAsyncEnumerator(); await enumerator.MoveNextAsync().ConfigureAwait(false); while (enumerator.Current.Values.Count < expectedCount) { cancellationTokenSource.Token.IsCancellationRequested.Should().BeFalse("timed out waiting for the devices to become queryable"); await Task.Delay(100).ConfigureAwait(false); - enumerator = queryClient.CreateJobsQueryAsync(options).AsPages().GetAsyncEnumerator(); + await enumerator.DisposeAsync().ConfigureAwait(false); + enumerator = queryClient.CreateJobsQuery(options).AsPages().GetAsyncEnumerator(); await enumerator.MoveNextAsync().ConfigureAwait(false); } + + await enumerator.DisposeAsync().ConfigureAwait(false); } private static async Task ScheduleJobToBeQueriedAsync(ScheduledJobsClient jobsClient, string deviceId) diff --git a/e2e/test/provisioning/ProvisioningCertificateValidationE2ETest.cs b/e2e/test/provisioning/ProvisioningCertificateValidationE2ETest.cs index 8cbe003904..d42f2d870b 100644 --- a/e2e/test/provisioning/ProvisioningCertificateValidationE2ETest.cs +++ b/e2e/test/provisioning/ProvisioningCertificateValidationE2ETest.cs @@ -8,6 +8,7 @@ using System.Security.Authentication; using System.Security.Cryptography.X509Certificates; using System.Threading.Tasks; +using Azure; using FluentAssertions; using Microsoft.Azure.Devices.E2ETests.Helpers; using Microsoft.Azure.Devices.Provisioning.Client; @@ -37,10 +38,10 @@ public async Task ProvisioningServiceClient_QueryInvalidServiceCertificateHttp_F { // arrange using var provisioningServiceClient = new ProvisioningServiceClient(TestConfiguration.Provisioning.ConnectionStringInvalidServiceCertificate); - Query q = provisioningServiceClient.EnrollmentGroups.CreateQuery("SELECT * FROM enrollmentGroups"); + AsyncPageable q = provisioningServiceClient.EnrollmentGroups.CreateQuery("SELECT * FROM enrollmentGroups"); // act - Func act = async () => await q.NextAsync(); + Func act = async () => await q.GetAsyncEnumerator().MoveNextAsync(); // assert var error = await act.Should().ThrowAsync().ConfigureAwait(false); diff --git a/e2e/test/provisioning/ProvisioningServiceDeviceRegistrationTests.cs b/e2e/test/provisioning/ProvisioningServiceDeviceRegistrationTests.cs new file mode 100644 index 0000000000..aaee7b0cf0 --- /dev/null +++ b/e2e/test/provisioning/ProvisioningServiceDeviceRegistrationTests.cs @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Net; +using System.Security.Cryptography.X509Certificates; +using System.Threading; +using System.Threading.Tasks; +using Azure; +using FluentAssertions; +using FluentAssertions.Specialized; +using Microsoft.Azure.Devices.E2ETests.Helpers; +using Microsoft.Azure.Devices.Provisioning.Service; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace Microsoft.Azure.Devices.E2ETests.Provisioning +{ + [TestClass] + [TestCategory("E2E")] + [TestCategory("DPS")] + public class ProvisioningServiceDeviceRegistrationTests : E2EMsTestBase + { + private static readonly string s_proxyServerAddress = TestConfiguration.IotHub.ProxyServerAddress; + private static readonly string s_devicePrefix = $"{nameof(ProvisioningServiceIndividualEnrollmentTests)}_"; + + private static readonly ProvisioningServiceExponentialBackoffRetryPolicy s_provisioningServiceRetryPolicy = new(20, TimeSpan.FromSeconds(3), true); + + [TestMethod] + [Timeout(TestTimeoutMilliseconds)] + public async Task ProvisioningServiceClient_DeviceRegistrationState_Query_Ok() + { + using var provisioningServiceClient = new ProvisioningServiceClient(TestConfiguration.Provisioning.ConnectionString); + + // Create an enrollment group so that the query is guaranteed to return at least one entry + string enrollmentGroupId = Guid.NewGuid().ToString(); + var enrollmentGroup = new EnrollmentGroup(enrollmentGroupId, new SymmetricKeyAttestation()); + await provisioningServiceClient.EnrollmentGroups.CreateOrUpdateAsync(enrollmentGroup).ConfigureAwait(false); + + try + { + string queryString = "SELECT * FROM enrollmentGroups"; + IAsyncEnumerable query = provisioningServiceClient.DeviceRegistrationStates.CreateEnrollmentGroupQuery(queryString, enrollmentGroupId); + await foreach (DeviceRegistrationState state in query) + { + state.LastUpdatedOnUtc.Should().NotBeNull(); + } + } + finally + { + try + { + await provisioningServiceClient.EnrollmentGroups.DeleteAsync(enrollmentGroupId).ConfigureAwait(false); + } + catch (Exception e) + { + // Failed to cleanup after the test, but don't fail the test because of this + VerboseTestLogger.WriteLine($"Failed to clean up enrollment group due to {e}"); + } + } + } + } +} \ No newline at end of file diff --git a/e2e/test/provisioning/ProvisioningServiceEnrollmentGroupTests.cs b/e2e/test/provisioning/ProvisioningServiceEnrollmentGroupTests.cs index 777ea1a0d2..dab55b758d 100644 --- a/e2e/test/provisioning/ProvisioningServiceEnrollmentGroupTests.cs +++ b/e2e/test/provisioning/ProvisioningServiceEnrollmentGroupTests.cs @@ -138,6 +138,52 @@ public async Task ProvisioningServiceClient_EnrollmentGroups_SymmetricKey_BulkOp deleteBulkEnrollmentResult.IsSuccessful.Should().BeTrue(); } + [TestMethod] + public async Task ProvisioningServiceClient_EnrollmentGroups_Query_Ok() + { + using var provisioningServiceClient = new ProvisioningServiceClient(TestConfiguration.Provisioning.ConnectionString); + + // Create an enrollment group so that the query is guaranteed to return at least one entry + string enrollmentGroupId = Guid.NewGuid().ToString(); + var enrollmentGroup = new EnrollmentGroup(enrollmentGroupId, new SymmetricKeyAttestation()); + await provisioningServiceClient.EnrollmentGroups + .CreateOrUpdateAsync(enrollmentGroup).ConfigureAwait(false); + + int maxCount = 5; + int currentCount = 0; + + try + { + string queryString = "SELECT * FROM enrollmentGroups"; + IAsyncEnumerable query = provisioningServiceClient.EnrollmentGroups.CreateQuery(queryString); + await foreach (EnrollmentGroup enrollment in query) + { + // Just checking that the returned type was, in fact, an enrollment group and that deserialization + // of the always-present field works. + enrollment.Id.Should().NotBeNull(); + + // Don't want to query all the enrollment groups. Just query a few. + if (++currentCount >= maxCount) + { + return; + } + } + } + finally + { + try + { + await provisioningServiceClient.EnrollmentGroups + .DeleteAsync(enrollmentGroupId).ConfigureAwait(false); + } + catch (Exception e) + { + // Failed to cleanup after the test, but don't fail the test because of this + VerboseTestLogger.WriteLine($"Failed to clean up enrollment group due to {e}"); + } + } + } + private static async Task ProvisioningServiceClient_GetEnrollmentGroupAttestation(AttestationMechanismType attestationType) { string groupId = AttestationTypeToString(attestationType) + "-" + Guid.NewGuid(); diff --git a/e2e/test/provisioning/ProvisioningServiceIndividualEnrollmentTests.cs b/e2e/test/provisioning/ProvisioningServiceIndividualEnrollmentTests.cs index 2ccac3e2cf..f22a4d63f2 100644 --- a/e2e/test/provisioning/ProvisioningServiceIndividualEnrollmentTests.cs +++ b/e2e/test/provisioning/ProvisioningServiceIndividualEnrollmentTests.cs @@ -225,12 +225,21 @@ private static async Task ProvisioningServiceClient_IndividualEnrollments_Query_ } using var provisioningServiceClient = new ProvisioningServiceClient(TestConfiguration.Provisioning.ConnectionString, options); + int maxCount = 5; + int currentCount = 0; string queryString = "SELECT * FROM enrollments"; - Query query = provisioningServiceClient.IndividualEnrollments.CreateQuery(queryString); - while (query.HasNext()) + IAsyncEnumerable query = provisioningServiceClient.IndividualEnrollments.CreateQuery(queryString); + await foreach (IndividualEnrollment enrollment in query) { - QueryResult queryResult = await query.NextAsync().ConfigureAwait(false); - queryResult.QueryType.Should().Be(QueryResultType.Enrollment); + // Just checking that the returned type was, in fact, an individual enrollment and that deserialization + // of the always-present fields works. + enrollment.RegistrationId.Should().NotBeNull(); + + // Don't want to query all the individual enrollments. Just query a few. + if (++currentCount >= maxCount) + { + return; + } } } diff --git a/iothub/service/samples/getting started/JobsSample/JobsSample.cs b/iothub/service/samples/getting started/JobsSample/JobsSample.cs index 663015e7fd..134dc4510b 100644 --- a/iothub/service/samples/getting started/JobsSample/JobsSample.cs +++ b/iothub/service/samples/getting started/JobsSample/JobsSample.cs @@ -68,7 +68,7 @@ public async Task RunSampleAsync() } // *************************************** Get all Jobs *************************************** - AsyncPageable queryResults = _jobClient.ScheduledJobs.CreateQueryAsync(); + AsyncPageable queryResults = _jobClient.ScheduledJobs.CreateQuery(); await foreach (ScheduledJob job in queryResults) { diff --git a/iothub/service/samples/how to guides/CleanupDevicesSample/CleanupDevicesSample.cs b/iothub/service/samples/how to guides/CleanupDevicesSample/CleanupDevicesSample.cs index 9f0d4654b0..4137104af5 100644 --- a/iothub/service/samples/how to guides/CleanupDevicesSample/CleanupDevicesSample.cs +++ b/iothub/service/samples/how to guides/CleanupDevicesSample/CleanupDevicesSample.cs @@ -174,7 +174,7 @@ private async Task PrintDeviceCountAsync() try { string countSqlQuery = "select count() AS numberOfDevices from devices"; - AsyncPageable> countQuery = _hubClient.Query.CreateAsync>(countSqlQuery); + AsyncPageable> countQuery = _hubClient.Query.Create>(countSqlQuery); IAsyncEnumerator> enumerator = countQuery.GetAsyncEnumerator(); if (!await enumerator.MoveNextAsync()) { @@ -226,13 +226,15 @@ private async Task> GetDeviceIdsToDeleteAsync( } string queryText = queryTextSb.ToString(); Console.WriteLine($"Using query: {queryText}"); - AsyncPageable devicesQuery = _hubClient.Query.CreateAsync(queryText); + AsyncPageable devicesQuery = _hubClient.Query.Create(queryText); await foreach (Page page in devicesQuery.AsPages(null, 1000)) { foreach (DeviceQueryResult queryResult in page.Values) { devicesToDelete.Add(new ExportImportDevice(new Device(queryResult.DeviceId), ImportMode.Delete)); } + + page.GetRawResponse().Dispose(); } return devicesToDelete; diff --git a/iothub/service/samples/how to guides/RegistryManagerSample/RegistryManagerSample.cs b/iothub/service/samples/how to guides/RegistryManagerSample/RegistryManagerSample.cs index fdcfdf5d75..93394f71d6 100644 --- a/iothub/service/samples/how to guides/RegistryManagerSample/RegistryManagerSample.cs +++ b/iothub/service/samples/how to guides/RegistryManagerSample/RegistryManagerSample.cs @@ -159,7 +159,7 @@ private async Task EnumerateTwinsAsync() string queryText = $"SELECT * FROM devices WHERE STARTSWITH(id, '{_parameters.DevicePrefix}')"; Console.WriteLine($"Using query text of: {queryText}"); - AsyncPageable query = _client.Query.CreateAsync(queryText); + AsyncPageable query = _client.Query.Create(queryText); await foreach (ClientTwin queriedTwin in query) { diff --git a/iothub/service/src/Jobs/ScheduledJobsClient.cs b/iothub/service/src/Jobs/ScheduledJobsClient.cs index 1ebf3f2a50..a531027539 100644 --- a/iothub/service/src/Jobs/ScheduledJobsClient.cs +++ b/iothub/service/src/Jobs/ScheduledJobsClient.cs @@ -111,20 +111,46 @@ await _internalRetryHandler } /// - /// Queries an iterable set of jobs for specified type and status. + /// Query all jobs or query jobs by type and/or status. /// - /// The optional parameters to run with the query. + /// The optional parameters to run the query with. /// Task cancellation token. - /// An iterable set of jobs for specified type and status. + /// An iterable set of the queried jobs. /// /// If IoT hub responded to the request with a non-successful status code. For example, if the provided /// request was throttled, with is thrown. /// For a complete list of possible error cases, see . /// - /// If the provided has requested cancellation. - public virtual AsyncPageable CreateQueryAsync(JobQueryOptions options = null, CancellationToken cancellationToken = default) + /// If the provided cancellation token has requested cancellation. + /// + /// Iterate over jobs: + /// + /// AsyncPageable<ScheduledJob> jobsQuery = iotHubServiceClient.Query.CreateJobsQuery(); + /// await foreach (ScheduledJob scheduledJob in jobsQuery) + /// { + /// Console.WriteLine(scheduledJob.JobId); + /// } + /// + /// Iterate over pages of twins: + /// + /// IAsyncEnumerable<Page<ScheduledJob>> jobsQuery = iotHubServiceClient.Query.CreateJobsQuery().AsPages(); + /// await foreach (Page<ScheduledJob> scheduledJobsPage in jobsQuery) + /// { + /// foreach (ScheduledJob scheduledJob in scheduledJobsPage.Values) + /// { + /// Console.WriteLine(scheduledJob.JobId); + /// } + /// + /// // Note that this is disposed for you while iterating item-by-item, but not when + /// // iterating page-by-page. That is why this sample has to manually call dispose + /// // on the response object here. + /// scheduledJobsPage.GetRawResponse().Dispose(); + /// } + /// + /// + public virtual AsyncPageable CreateQuery(JobQueryOptions options = null, CancellationToken cancellationToken = default) { - return _queryClient.CreateJobsQueryAsync(options, cancellationToken); + return _queryClient.CreateJobsQuery(options, cancellationToken); } /// diff --git a/iothub/service/src/Query/Models/QueriedPage.cs b/iothub/service/src/Query/Models/QueriedPage.cs index f5120f57e2..788a85e458 100644 --- a/iothub/service/src/Query/Models/QueriedPage.cs +++ b/iothub/service/src/Query/Models/QueriedPage.cs @@ -22,10 +22,8 @@ internal QueriedPage(HttpResponseMessage response, string payload) ContinuationToken = response.Headers.SafeGetValue(ContinuationTokenHeader); } - [JsonProperty("items")] internal IReadOnlyList Items { get; set; } - [JsonProperty("continuationToken")] internal string ContinuationToken { get; set; } } } diff --git a/iothub/service/src/Query/QueryClient.cs b/iothub/service/src/Query/QueryClient.cs index 117d49ba2b..66e2db030c 100644 --- a/iothub/service/src/Query/QueryClient.cs +++ b/iothub/service/src/Query/QueryClient.cs @@ -75,27 +75,43 @@ internal QueryClient( /// If the provided cancellation token has requested cancellation. /// /// - /// Iterate twins: + /// Iterate over twins: /// - /// AsyncPageable<Twin> twinQuery = iotHubServiceClient.Query.CreateAsync<Twin>("SELECT * FROM devices"); - /// await foreach (Twin queriedTwin in twinQuery) + /// AsyncPageable<Twin> twinsQuery = iotHubServiceClient.Query.Create<ClientTwin>("SELECT * FROM devices"); + /// await foreach (Twin queriedTwin in twinsQuery) /// { - /// Console.WriteLine(queriedTwin); + /// Console.WriteLine(queriedTwin.DeviceId); /// } /// /// Or scheduled jobs: /// - /// AsyncPageable<ScheduledJob> jobsQuery = await iotHubServiceClient.Query.CreateAsync<ScheduledJob>("SELECT * FROM devices.jobs"); + /// AsyncPageable<ScheduledJob> jobsQuery = iotHubServiceClient.Query.Create<ScheduledJob>("SELECT * FROM devices.jobs"); /// await foreach (ScheduledJob queriedJob in jobsQuery) /// { /// Console.WriteLine(queriedJob); /// } /// + /// Iterate over pages of twins: + /// + /// IAsyncEnumerable<Page<ClientTwin>> twinsQuery = iotHubServiceClient.Query.Create<ClientTwin>("SELECT * FROM devices").AsPages(); + /// await foreach (Page<ClientTwin> queriedTwinsPage in twinsQuery) + /// { + /// foreach (ClientTwin queriedTwin in queriedTwinsPage.Values) + /// { + /// Console.WriteLine(queriedTwin.DeviceId); + /// } + /// + /// // Note that this is disposed for you while iterating item-by-item, but not when + /// // iterating page-by-page. That is why this sample has to manually call dispose + /// // on the response object here. + /// queriedTwinsPage.GetRawResponse().Dispose(); + /// } + /// /// - public virtual AsyncPageable CreateAsync(string query, CancellationToken cancellationToken = default) + public virtual AsyncPageable Create(string query, CancellationToken cancellationToken = default) { if (Logging.IsEnabled) - Logging.Enter(this, "Creating query.", nameof(CreateAsync)); + Logging.Enter(this, "Creating query.", nameof(Create)); Argument.AssertNotNullOrWhiteSpace(query, nameof(query)); @@ -131,13 +147,13 @@ async Task> firstPageFunc(int? pageSizeHint) } catch (Exception ex) when (Logging.IsEnabled) { - Logging.Error(this, $"Creating query threw an exception: {ex}", nameof(CreateAsync)); + Logging.Error(this, $"Creating query threw an exception: {ex}", nameof(Create)); throw; } finally { if (Logging.IsEnabled) - Logging.Exit(this, "Creating query.", nameof(CreateAsync)); + Logging.Exit(this, "Creating query.", nameof(Create)); } } @@ -154,18 +170,35 @@ async Task> firstPageFunc(int? pageSizeHint) /// /// If the provided cancellation token has requested cancellation. /// + /// Iterate over jobs: /// - /// AsyncPageable<ScheduledJob> jobsQuery = await iotHubServiceClient.Query.CreateJobsQueryAsync(); + /// AsyncPageable<ScheduledJob> jobsQuery = iotHubServiceClient.Query.CreateJobsQuery(); /// await foreach (ScheduledJob scheduledJob in jobsQuery) /// { /// Console.WriteLine(scheduledJob.JobId); /// } /// + /// Iterate over pages of twins: + /// + /// IAsyncEnumerable<Page<ScheduledJob>> jobsQuery = iotHubServiceClient.Query.CreateJobsQuery().AsPages(); + /// await foreach (Page<ScheduledJob> scheduledJobsPage in jobsQuery) + /// { + /// foreach (ScheduledJob scheduledJob in scheduledJobsPage.Values) + /// { + /// Console.WriteLine(scheduledJob.JobId); + /// } + /// + /// // Note that this is disposed for you while iterating item-by-item, but not when + /// // iterating page-by-page. That is why this sample has to manually call dispose + /// // on the response object here. + /// scheduledJobsPage.GetRawResponse().Dispose(); + /// } + /// /// - public virtual AsyncPageable CreateJobsQueryAsync(JobQueryOptions options = default, CancellationToken cancellationToken = default) + public virtual AsyncPageable CreateJobsQuery(JobQueryOptions options = default, CancellationToken cancellationToken = default) { if (Logging.IsEnabled) - Logging.Enter(this, $"Creating query with jobType: {options?.JobType}, jobStatus: {options?.JobStatus}", nameof(CreateAsync)); + Logging.Enter(this, $"Creating query with jobType: {options?.JobType}, jobStatus: {options?.JobStatus}", nameof(Create)); cancellationToken.ThrowIfCancellationRequested(); @@ -209,13 +242,13 @@ async Task> firstPageFunc(int? pageSizeHint) } catch (Exception ex) when (Logging.IsEnabled) { - Logging.Error(this, $"Creating query with jobType: {options?.JobType}, jobStatus: {options?.JobStatus} threw an exception: {ex}", nameof(CreateAsync)); + Logging.Error(this, $"Creating query with jobType: {options?.JobType}, jobStatus: {options?.JobStatus} threw an exception: {ex}", nameof(Create)); throw; } finally { if (Logging.IsEnabled) - Logging.Exit(this, $"Creating query with jobType: {options?.JobType}, jobStatus: {options?.JobStatus}", nameof(CreateAsync)); + Logging.Exit(this, $"Creating query with jobType: {options?.JobType}, jobStatus: {options?.JobStatus}", nameof(Create)); } } diff --git a/iothub/service/src/Query/QueryResponse.cs b/iothub/service/src/Query/QueryResponse.cs index 5377f93bbb..8ccc21bc4f 100644 --- a/iothub/service/src/Query/QueryResponse.cs +++ b/iothub/service/src/Query/QueryResponse.cs @@ -19,13 +19,12 @@ namespace Microsoft.Azure.Devices internal class QueryResponse : Response { private HttpResponseMessage _httpResponse; - private Stream _bodyStream; private List _httpHeaders; internal QueryResponse(HttpResponseMessage httpResponse, Stream bodyStream) { _httpResponse = httpResponse; - _bodyStream = bodyStream; + ContentStream = bodyStream; _httpHeaders = new List(); foreach (var header in _httpResponse.Headers) @@ -38,11 +37,7 @@ internal QueryResponse(HttpResponseMessage httpResponse, Stream bodyStream) public override string ReasonPhrase => _httpResponse.ReasonPhrase; - public override Stream ContentStream - { - get => _bodyStream; - set => _bodyStream = value; - } + public override Stream ContentStream { get; set; } public override string ClientRequestId { @@ -53,7 +48,7 @@ public override string ClientRequestId public override void Dispose() { _httpResponse?.Dispose(); - _bodyStream?.Dispose(); + ContentStream?.Dispose(); } protected override bool ContainsHeader(string name) diff --git a/iothub/service/tests/QueryClientTests.cs b/iothub/service/tests/QueryClientTests.cs index 32418f018f..354763b7a2 100644 --- a/iothub/service/tests/QueryClientTests.cs +++ b/iothub/service/tests/QueryClientTests.cs @@ -64,7 +64,7 @@ public async Task QueryClient_CreateAsync() s_retryHandler); // act - AsyncPageable response = queryClient.CreateAsync(query); + AsyncPageable response = queryClient.Create(query); IAsyncEnumerator enumerator = response.GetAsyncEnumerator(); await enumerator.MoveNextAsync().ConfigureAwait(false); @@ -79,7 +79,7 @@ public async Task QueryClient_CreateAsync_NullParamterThrows() using var serviceClient = new IotHubServiceClient(s_connectionString); // act - Func act = async () => await serviceClient.Query.CreateAsync(null).GetAsyncEnumerator().MoveNextAsync(); + Func act = async () => await serviceClient.Query.Create(null).GetAsyncEnumerator().MoveNextAsync(); // assert await act.Should().ThrowAsync(); @@ -121,7 +121,7 @@ public async Task QueryClient_CreateAsync_IotHubNotFound_ThrowsIotHubServiceExce // act // query returns HttpStatusCode.NotFound - Func act = async () => await queryClient.CreateAsync("SELECT * FROM devices").GetAsyncEnumerator().MoveNextAsync(); + Func act = async () => await queryClient.Create("SELECT * FROM devices").GetAsyncEnumerator().MoveNextAsync(); // assert var error = await act.Should().ThrowAsync(); @@ -160,7 +160,7 @@ public async Task QueryClient_CreateJobsQueryAsync() s_retryHandler); // act - Func act = async () => await queryClient.CreateJobsQueryAsync().GetAsyncEnumerator().MoveNextAsync(); + Func act = async () => await queryClient.CreateJobsQuery().GetAsyncEnumerator().MoveNextAsync(); // assert await act.Should().NotThrowAsync(); @@ -201,7 +201,7 @@ public async Task QueryClient_CreateJobsQuery_IotHubNotFound_ThrowsIotHubService s_retryHandler); // act - Func act = async () => await queryClient.CreateJobsQueryAsync().GetAsyncEnumerator().MoveNextAsync(); + Func act = async () => await queryClient.CreateJobsQuery().GetAsyncEnumerator().MoveNextAsync(); // assert var error = await act.Should().ThrowAsync(); diff --git a/provisioning/service/samples/getting started/CleanupEnrollmentsSample/CleanupEnrollmentsSample.cs b/provisioning/service/samples/getting started/CleanupEnrollmentsSample/CleanupEnrollmentsSample.cs index 08826cdfba..0939625a4b 100644 --- a/provisioning/service/samples/getting started/CleanupEnrollmentsSample/CleanupEnrollmentsSample.cs +++ b/provisioning/service/samples/getting started/CleanupEnrollmentsSample/CleanupEnrollmentsSample.cs @@ -5,15 +5,13 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; +using Azure; using Newtonsoft.Json; namespace Microsoft.Azure.Devices.Provisioning.Service.Samples { internal class CleanupEnrollmentsSample { - // Maximum number of elements per query - DPS has a limit of 10. - private const int QueryPageSize = 10; - private readonly ProvisioningServiceClient _provisioningServiceClient; private static int s_individualEnrollmentsDeleted; private static int s_enrollmentGroupsDeleted; @@ -48,48 +46,39 @@ public async Task RunSampleAsync() private async Task QueryAndDeleteIndividualEnrollmentsAsync() { Console.WriteLine("Creating a query for enrollments..."); - Query query = _provisioningServiceClient.IndividualEnrollments.CreateQuery("SELECT * FROM enrollments", QueryPageSize); - while (query.HasNext()) + AsyncPageable query = _provisioningServiceClient.IndividualEnrollments.CreateQuery("SELECT * FROM enrollments"); + var individualEnrollments = new List(); + await foreach (IndividualEnrollment enrollment in query) { Console.WriteLine("Querying the next enrollments..."); - QueryResult queryResult = await query.NextAsync(); - IEnumerable items = queryResult.Items; - var individualEnrollments = new List(); - foreach (IndividualEnrollment enrollment in items.Cast()) - { - if (!_individualEnrollmentsToBeRetained.Contains(enrollment.RegistrationId, StringComparer.OrdinalIgnoreCase)) - { - individualEnrollments.Add(enrollment); - Console.WriteLine($"Individual enrollment to be deleted: {enrollment.RegistrationId}"); - s_individualEnrollmentsDeleted++; - } - } - if (individualEnrollments.Count > 0) + if (!_individualEnrollmentsToBeRetained.Contains(enrollment.RegistrationId, StringComparer.OrdinalIgnoreCase)) { - await DeleteBulkIndividualEnrollmentsAsync(individualEnrollments); + individualEnrollments.Add(enrollment); + Console.WriteLine($"Individual enrollment to be deleted: {enrollment.RegistrationId}"); + s_individualEnrollmentsDeleted++; } await Task.Delay(1000); } + + if (individualEnrollments.Count > 0) + { + await DeleteBulkIndividualEnrollmentsAsync(individualEnrollments); + } } private async Task QueryAndDeleteEnrollmentGroupsAsync() { Console.WriteLine("Creating a query for enrollment groups..."); - Query query = _provisioningServiceClient.EnrollmentGroups.CreateQuery("SELECT * FROM enrollmentGroups", QueryPageSize); - while (query.HasNext()) + AsyncPageable query = _provisioningServiceClient.EnrollmentGroups.CreateQuery("SELECT * FROM enrollmentGroups"); + await foreach (EnrollmentGroup enrollment in query) { Console.WriteLine("Querying the next enrollment groups..."); - QueryResult queryResult = await query.NextAsync(); - IEnumerable items = queryResult.Items; - foreach (EnrollmentGroup enrollment in items.Cast()) + if (!_groupEnrollmentsToBeRetained.Contains(enrollment.Id, StringComparer.OrdinalIgnoreCase)) { - if (!_groupEnrollmentsToBeRetained.Contains(enrollment.Id, StringComparer.OrdinalIgnoreCase)) - { - Console.WriteLine($"Enrollment group to be deleted: {enrollment.Id}"); - s_enrollmentGroupsDeleted++; - await _provisioningServiceClient.EnrollmentGroups.DeleteAsync(enrollment.Id); - } + Console.WriteLine($"Enrollment group to be deleted: {enrollment.Id}"); + s_enrollmentGroupsDeleted++; + await _provisioningServiceClient.EnrollmentGroups.DeleteAsync(enrollment.Id); } } } diff --git a/provisioning/service/samples/getting started/EnrollmentGroupSample/EnrollmentGroupSample.cs b/provisioning/service/samples/getting started/EnrollmentGroupSample/EnrollmentGroupSample.cs index 65fa8ee700..e36ef1fcb2 100644 --- a/provisioning/service/samples/getting started/EnrollmentGroupSample/EnrollmentGroupSample.cs +++ b/provisioning/service/samples/getting started/EnrollmentGroupSample/EnrollmentGroupSample.cs @@ -4,6 +4,7 @@ using System; using System.Linq; using System.Threading.Tasks; +using Azure; using Newtonsoft.Json; namespace Microsoft.Azure.Devices.Provisioning.Service.Samples @@ -47,36 +48,26 @@ public async Task QueryEnrollmentGroupAsync() { string queryText = "SELECT * FROM enrollmentGroups"; Console.WriteLine($"Running a query for enrollment groups: {queryText}"); - Query query = _provisioningServiceClient.EnrollmentGroups.CreateQuery(queryText); + AsyncPageable query = _provisioningServiceClient.EnrollmentGroups.CreateQuery(queryText); - while (query.HasNext()) + await foreach (EnrollmentGroup enrollmentGroup in query) { - QueryResult queryResult = await query.NextAsync(); - - foreach (EnrollmentGroup group in queryResult.Items.Cast()) - { - Console.WriteLine($"Found enrollment group {group.Id} is {group.ProvisioningStatus}."); - await EnumerateRegistrationsInGroupAsync(queryText, group); - } + Console.WriteLine($"Found enrollment group {enrollmentGroup.Id} is {enrollmentGroup.ProvisioningStatus}."); + await EnumerateRegistrationsInGroupAsync(queryText, enrollmentGroup); } } private async Task EnumerateRegistrationsInGroupAsync(string queryText, EnrollmentGroup group) { Console.WriteLine($"Registrations within group {group.Id}:"); - Query registrationQuery = _provisioningServiceClient.DeviceRegistrationStates.CreateEnrollmentGroupQuery(queryText, group.Id); + AsyncPageable registrationQuery = _provisioningServiceClient.DeviceRegistrationStates.CreateEnrollmentGroupQuery(queryText, group.Id); - while (registrationQuery.HasNext()) + await foreach (DeviceRegistrationState registration in registrationQuery) { - QueryResult registrationQueryResult = await registrationQuery.NextAsync(); - - foreach (DeviceRegistrationState registration in registrationQueryResult.Items.Cast()) + Console.WriteLine($"\t{registration.RegistrationId} for {registration.DeviceId} is {registration.Status}."); + if (registration.ErrorCode.HasValue) { - Console.WriteLine($"\t{registration.RegistrationId} for {registration.DeviceId} is {registration.Status}."); - if (registration.ErrorCode.HasValue) - { - Console.WriteLine($"\t\tWith error ({registration.ErrorCode.Value}): {registration.ErrorMessage}"); - } + Console.WriteLine($"\t\tWith error ({registration.ErrorCode.Value}): {registration.ErrorMessage}"); } } } diff --git a/provisioning/service/samples/getting started/IndividualEnrollmentSample/IndividualEnrollmentSample.cs b/provisioning/service/samples/getting started/IndividualEnrollmentSample/IndividualEnrollmentSample.cs index 75a4045ff0..7c4abfe960 100644 --- a/provisioning/service/samples/getting started/IndividualEnrollmentSample/IndividualEnrollmentSample.cs +++ b/provisioning/service/samples/getting started/IndividualEnrollmentSample/IndividualEnrollmentSample.cs @@ -4,6 +4,7 @@ using System; using System.Linq; using System.Threading.Tasks; +using Azure; namespace Microsoft.Azure.Devices.Provisioning.Service.Samples { @@ -84,16 +85,12 @@ public async Task GetIndividualEnrollmentInfoAsync() public async Task QueryIndividualEnrollmentsAsync() { - const string queryText = ""; - Query query = _provisioningServiceClient.IndividualEnrollments.CreateQuery(queryText); + const string queryText = "SELECT * FROM enrollments"; + AsyncPageable query = _provisioningServiceClient.IndividualEnrollments.CreateQuery(queryText); Console.WriteLine($"Querying for individual enrollments: {queryText}"); - while (query.HasNext()) + await foreach (IndividualEnrollment enrollment in query) { - QueryResult queryResult = await query.NextAsync(); - foreach (IndividualEnrollment enrollment in queryResult.Items.Cast()) - { - Console.WriteLine($"Individual enrollment '{enrollment.RegistrationId}'"); - } + Console.WriteLine($"Individual enrollment '{enrollment.RegistrationId}'"); } } diff --git a/provisioning/service/samples/how to guides/BulkOperationSample/BulkOperationSample.cs b/provisioning/service/samples/how to guides/BulkOperationSample/BulkOperationSample.cs index 16ece88a6a..784af5a446 100644 --- a/provisioning/service/samples/how to guides/BulkOperationSample/BulkOperationSample.cs +++ b/provisioning/service/samples/how to guides/BulkOperationSample/BulkOperationSample.cs @@ -14,9 +14,6 @@ public class BulkOperationSample private const string SampleRegistrationId1 = "myvalid-registratioid-csharp-1"; private const string SampleRegistrationId2 = "myvalid-registratioid-csharp-2"; - // Maximum number of elements per query. - private const int QueryPageSize = 100; - private static readonly List s_registrationIds = new() { SampleRegistrationId1, SampleRegistrationId2 }; public BulkOperationSample(ProvisioningServiceClient provisioningServiceClient) @@ -81,12 +78,11 @@ public async Task QueryIndividualEnrollmentsAsync() { Console.WriteLine("\nCreating a query for enrollments..."); - Query query = _provisioningServiceClient.IndividualEnrollments.CreateQuery("SELECT * FROM enrollments", QueryPageSize); - while (query.HasNext()) + IAsyncEnumerable query = _provisioningServiceClient.IndividualEnrollments.CreateQuery("SELECT * FROM enrollments"); + await foreach (IndividualEnrollment enrollment in query) { Console.WriteLine("\nQuerying the next enrollments..."); - QueryResult queryResult = await query.NextAsync(); - Console.WriteLine(queryResult); + Console.WriteLine(enrollment); } } } diff --git a/provisioning/service/src/DeviceRegistrationsClient.cs b/provisioning/service/src/DeviceRegistrationsClient.cs index 20d3421dfe..fa7ccc9008 100644 --- a/provisioning/service/src/DeviceRegistrationsClient.cs +++ b/provisioning/service/src/DeviceRegistrationsClient.cs @@ -18,10 +18,10 @@ namespace Microsoft.Azure.Devices.Provisioning.Service /// public class DeviceRegistrationStatesClient { - private const string ServiceName = "registrations"; - private const string DeviceRegistrationStatusUriFormat = "{0}/{1}"; + private const string DeviceRegistrationStatusUriFormat = "registrations/{0}"; + private const string DeviceRegistrationQueryUriFormat = "registrations/{0}/query"; - private readonly IContractApiHttp _contractApiHttp; + private readonly ContractApiHttp _contractApiHttp; private readonly RetryHandler _internalRetryHandler; /// @@ -31,7 +31,7 @@ protected DeviceRegistrationStatesClient() { } - internal DeviceRegistrationStatesClient(IContractApiHttp contractApiHttp, RetryHandler retryHandler) + internal DeviceRegistrationStatesClient(ContractApiHttp contractApiHttp, RetryHandler retryHandler) { _contractApiHttp = contractApiHttp; _internalRetryHandler = retryHandler; @@ -55,13 +55,13 @@ public async Task GetAsync(string registrationId, Cance cancellationToken.ThrowIfCancellationRequested(); - ContractApiResponse contractApiResponse = null; + HttpResponseMessage response = null; await _internalRetryHandler .RunWithRetryAsync( async () => { - contractApiResponse = await _contractApiHttp + response = await _contractApiHttp .RequestAsync( HttpMethod.Get, GetDeviceRegistrationStatusUri(registrationId), @@ -74,7 +74,8 @@ await _internalRetryHandler cancellationToken) .ConfigureAwait(false); - return JsonConvert.DeserializeObject(contractApiResponse.Body); + string payload = await response.Content.ReadAsStringAsync().ConfigureAwait(false); + return JsonConvert.DeserializeObject(payload); } /// @@ -134,24 +135,99 @@ await _contractApiHttp /// /// The SQL query. /// The enrollment group Id to query. - /// The int with the maximum number of items per iteration. It can be 0 for default, but not negative. /// The cancellation token. /// The iterable set of query results. /// If the provided is null. /// If the provided is empty or white space. - /// If the provided value is less than zero. /// If the provided has requested cancellation. - public Query CreateEnrollmentGroupQuery(string query, string enrollmentGroupId, int pageSize = 0, CancellationToken cancellationToken = default) + /// + /// Iterate over device registration states in an enrollment group: + /// + /// AsyncPageable<DeviceRegistrationState> deviceRegistrationStatesQuery = dpsServiceClient.DeviceRegistrationStates.CreateEnrollmentGroupQuery("SELECT * FROM enrollmentGroups", "myEnrollmentGroupId"); + /// await foreach (DeviceRegistrationState queriedState in deviceRegistrationStatesQuery) + /// { + /// Console.WriteLine(queriedState.RegistrationId); + /// } + /// + /// Iterate over pages of device registration states in an enrollment group: + /// + /// IAsyncEnumerable<Page<DeviceRegistrationState>> deviceRegistrationStatesQuery = dpsServiceClient.DeviceRegistrationState.CreateQuery("SELECT * FROM enrollmentGroups", "myEnrollmentGroupId").AsPages(); + /// await foreach (Page<DeviceRegistrationState> queriedStatePage in deviceRegistrationStatesQuery) + /// { + /// foreach (DeviceRegistrationState queriedState in queriedStatePage.Values) + /// { + /// Console.WriteLine(queriedState.RegistrationId); + /// } + /// + /// // Note that this is disposed for you while iterating item-by-item, but not when + /// // iterating page-by-page. That is why this sample has to manually call dispose + /// // on the response object here. + /// queriedStatePage.GetRawResponse().Dispose(); + /// } + /// + /// + public AsyncPageable CreateEnrollmentGroupQuery(string query, string enrollmentGroupId, CancellationToken cancellationToken = default) { + if (Logging.IsEnabled) + Logging.Enter(this, "Creating query.", nameof(CreateEnrollmentGroupQuery)); + Argument.AssertNotNullOrWhiteSpace(query, nameof(query)); - return new Query(GetDeviceRegistrationStatusUri(enrollmentGroupId).ToString(), query, _contractApiHttp, pageSize, _internalRetryHandler, cancellationToken); + + cancellationToken.ThrowIfCancellationRequested(); + + try + { + async Task> NextPageFunc(string continuationToken, int? pageSizeHint) + { + cancellationToken.ThrowIfCancellationRequested(); + return await QueryBuilder + .BuildAndSendRequestAsync( + _contractApiHttp, + _internalRetryHandler, + query, + GetDeviceRegistrationQueryUri(enrollmentGroupId), continuationToken, pageSizeHint, cancellationToken) + .ConfigureAwait(false); + } + + async Task> FirstPageFunc(int? pageSizeHint) + { + cancellationToken.ThrowIfCancellationRequested(); + return await QueryBuilder + .BuildAndSendRequestAsync( + _contractApiHttp, + _internalRetryHandler, + query, + GetDeviceRegistrationQueryUri(enrollmentGroupId), null, pageSizeHint, cancellationToken) + .ConfigureAwait(false); + } + + return PageableHelpers.CreateAsyncEnumerable(FirstPageFunc, NextPageFunc, null); + } + catch (Exception ex) when (Logging.IsEnabled) + { + Logging.Error(this, $"Creating query threw an exception: {ex}", nameof(CreateEnrollmentGroupQuery)); + throw; + } + finally + { + if (Logging.IsEnabled) + Logging.Exit(this, "Creating query.", nameof(CreateEnrollmentGroupQuery)); + } } private static Uri GetDeviceRegistrationStatusUri(string id) { id = WebUtility.UrlEncode(id); return new Uri( - string.Format(CultureInfo.InvariantCulture, DeviceRegistrationStatusUriFormat, ServiceName, id), + string.Format(CultureInfo.InvariantCulture, DeviceRegistrationStatusUriFormat, id), + UriKind.Relative); + } + + private static Uri GetDeviceRegistrationQueryUri(string id) + { + id = WebUtility.UrlEncode(id); + return new Uri( + string.Format(CultureInfo.InvariantCulture, DeviceRegistrationQueryUriFormat, id), UriKind.Relative); } } diff --git a/provisioning/service/src/EnrollmentGroupsClient.cs b/provisioning/service/src/EnrollmentGroupsClient.cs index ac6bfa5eb0..abd5917a13 100644 --- a/provisioning/service/src/EnrollmentGroupsClient.cs +++ b/provisioning/service/src/EnrollmentGroupsClient.cs @@ -21,12 +21,12 @@ namespace Microsoft.Azure.Devices.Provisioning.Service /// public class EnrollmentGroupsClient { - private const string ServiceName = "enrollmentGroups"; - private const string EnrollmentIdUriFormat = "{0}/{1}"; - private const string EnrollmentAttestationName = "attestationmechanism"; - private const string EnrollmentAttestationUriFormat = "{0}/{1}/{2}"; + private const string EnrollmentsUriFormat = "enrollmentGroups"; + private const string EnrollmentIdUriFormat = "enrollmentGroups/{0}"; + private const string EnrollmentAttestationUriFormat = "enrollmentGroups/{0}/attestationmechanism"; + private const string EnrollmentGroupQueryUriFormat = "enrollmentGroups/query"; - private readonly IContractApiHttp _contractApiHttp; + private readonly ContractApiHttp _contractApiHttp; private readonly RetryHandler _internalRetryHandler; /// @@ -36,7 +36,7 @@ protected EnrollmentGroupsClient() { } - internal EnrollmentGroupsClient(IContractApiHttp contractApiHttp, RetryHandler retryHandler) + internal EnrollmentGroupsClient(ContractApiHttp contractApiHttp, RetryHandler retryHandler) { _contractApiHttp = contractApiHttp; _internalRetryHandler = retryHandler; @@ -65,13 +65,13 @@ public async Task CreateOrUpdateAsync(EnrollmentGroup enrollmen cancellationToken.ThrowIfCancellationRequested(); - ContractApiResponse contractApiResponse = null; + HttpResponseMessage response = null; await _internalRetryHandler .RunWithRetryAsync( async () => { - contractApiResponse = await _contractApiHttp + response = await _contractApiHttp .RequestAsync( HttpMethod.Put, GetEnrollmentUri(enrollmentGroup.Id), @@ -84,7 +84,8 @@ await _internalRetryHandler cancellationToken) .ConfigureAwait(false); - return JsonConvert.DeserializeObject(contractApiResponse.Body); + string payload = await response.Content.ReadAsStringAsync().ConfigureAwait(false); + return JsonConvert.DeserializeObject(payload); } /// @@ -105,13 +106,13 @@ public async Task GetAsync(string enrollmentGroupId, Cancellati cancellationToken.ThrowIfCancellationRequested(); - ContractApiResponse contractApiResponse = null; + HttpResponseMessage response = null; await _internalRetryHandler .RunWithRetryAsync( async () => { - contractApiResponse = await _contractApiHttp + response = await _contractApiHttp .RequestAsync( HttpMethod.Get, GetEnrollmentUri(enrollmentGroupId), @@ -124,7 +125,8 @@ await _internalRetryHandler cancellationToken) .ConfigureAwait(false); - return JsonConvert.DeserializeObject(contractApiResponse.Body); + string payload = await response.Content.ReadAsStringAsync().ConfigureAwait(false); + return JsonConvert.DeserializeObject(payload); } /// @@ -228,13 +230,13 @@ public async Task RunBulkOperationAsync( Enrollments = enrollmentGroups.ToList(), }; - ContractApiResponse contractApiResponse = null; + HttpResponseMessage response = null; await _internalRetryHandler .RunWithRetryAsync( async () => { - contractApiResponse = await _contractApiHttp + response = await _contractApiHttp .RequestAsync( HttpMethod.Post, GetEnrollmentUri(), @@ -247,7 +249,8 @@ await _internalRetryHandler cancellationToken) .ConfigureAwait(false); - return JsonConvert.DeserializeObject(contractApiResponse.Body); + string payload = await response.Content.ReadAsStringAsync().ConfigureAwait(false); + return JsonConvert.DeserializeObject(payload); } /// @@ -256,23 +259,93 @@ await _internalRetryHandler /// /// The service expects a SQL-like query such as /// - /// "SELECT * FROM enrollments". - /// - /// For each iteration, the query will return a page of results. The maximum number of - /// items per page can be specified by the pageSize parameter. + /// "SELECT * FROM enrollmentGroups". /// /// The with the SQL query. It cannot be null. - /// The int with the maximum number of items per iteration. It can be 0 for default, but not negative. /// The cancellation token. /// The iterable set of query results. /// If the provided is null. /// If the provided is empty or white space. - /// If the provided value is less than zero. /// If the provided has requested cancellation. - public Query CreateQuery(string query, int pageSize = 0, CancellationToken cancellationToken = default) + /// + /// Iterate over enrollment groups: + /// + /// AsyncPageable<EnrollmentGroup> enrollmentGroupsQuery = dpsServiceClient.EnrollmentGroups.CreateQuery("SELECT * FROM enrollmentGroups"); + /// await foreach (EnrollmentGroup queriedEnrollment in enrollmentGroupsQuery) + /// { + /// Console.WriteLine(queriedEnrollment.Id); + /// } + /// + /// Iterate over pages of enrollment groups: + /// + /// IAsyncEnumerable<Page<EnrollmentGroup>> enrollmentGroupsQuery = dpsServiceClient.EnrollmentGroups.CreateQuery("SELECT * FROM enrollmentGroups").AsPages(); + /// await foreach (Page<EnrollmentGroup> queriedEnrollmentPage in enrollmentGroupsQuery) + /// { + /// foreach (EnrollmentGroup queriedEnrollment in queriedEnrollmentPage.Values) + /// { + /// Console.WriteLine(queriedEnrollment.Id); + /// } + /// + /// // Note that this is disposed for you while iterating item-by-item, but not when + /// // iterating page-by-page. That is why this sample has to manually call dispose + /// // on the response object here. + /// queriedEnrollmentPage.GetRawResponse().Dispose(); + /// } + /// + /// + public AsyncPageable CreateQuery(string query, CancellationToken cancellationToken = default) { + if (Logging.IsEnabled) + Logging.Enter(this, "Creating query.", nameof(CreateQuery)); + Argument.AssertNotNullOrWhiteSpace(query, nameof(query)); - return new Query(ServiceName, query, _contractApiHttp, pageSize, _internalRetryHandler, cancellationToken); + + cancellationToken.ThrowIfCancellationRequested(); + + try + { + async Task> NextPageFunc(string continuationToken, int? pageSizeHint) + { + cancellationToken.ThrowIfCancellationRequested(); + return await QueryBuilder + .BuildAndSendRequestAsync( + _contractApiHttp, + _internalRetryHandler, + query, + GetEnrollmentGroupQueryUri(), + continuationToken, + pageSizeHint, + cancellationToken) + .ConfigureAwait(false); + } + + async Task> FirstPageFunc(int? pageSizeHint) + { + cancellationToken.ThrowIfCancellationRequested(); + return await QueryBuilder + .BuildAndSendRequestAsync( + _contractApiHttp, + _internalRetryHandler, + query, + GetEnrollmentGroupQueryUri(), + null, + pageSizeHint, + cancellationToken) + .ConfigureAwait(false); + } + + return PageableHelpers.CreateAsyncEnumerable(FirstPageFunc, NextPageFunc, null); + } + catch (Exception ex) when (Logging.IsEnabled) + { + Logging.Error(this, $"Creating query threw an exception: {ex}", nameof(CreateQuery)); + throw; + } + finally + { + if (Logging.IsEnabled) + Logging.Exit(this, "Creating query.", nameof(CreateQuery)); + } } /// @@ -293,13 +366,13 @@ public async Task GetAttestationAsync(string enrollmentGro cancellationToken.ThrowIfCancellationRequested(); - ContractApiResponse contractApiResponse = null; + HttpResponseMessage response = null; await _internalRetryHandler .RunWithRetryAsync( async () => { - contractApiResponse = await _contractApiHttp + response = await _contractApiHttp .RequestAsync( HttpMethod.Post, GetEnrollmentAttestationUri(enrollmentGroupId), @@ -312,26 +385,32 @@ await _internalRetryHandler cancellationToken) .ConfigureAwait(false); - return JsonConvert.DeserializeObject(contractApiResponse.Body); + string payload = await response.Content.ReadAsStringAsync().ConfigureAwait(false); + return JsonConvert.DeserializeObject(payload); } private static Uri GetEnrollmentUri(string enrollmentGroupId = "") { if (string.IsNullOrWhiteSpace(enrollmentGroupId)) { - return new Uri(ServiceName, UriKind.Relative); + return new Uri(EnrollmentsUriFormat, UriKind.Relative); } enrollmentGroupId = WebUtility.UrlEncode(enrollmentGroupId); - return new Uri(string.Format(CultureInfo.InvariantCulture, EnrollmentIdUriFormat, ServiceName, enrollmentGroupId), UriKind.Relative); + return new Uri(string.Format(CultureInfo.InvariantCulture, EnrollmentIdUriFormat, enrollmentGroupId), UriKind.Relative); } private static Uri GetEnrollmentAttestationUri(string enrollmentGroupId) { enrollmentGroupId = WebUtility.UrlEncode(enrollmentGroupId); return new Uri( - string.Format(CultureInfo.InvariantCulture, EnrollmentAttestationUriFormat, ServiceName, enrollmentGroupId, EnrollmentAttestationName), + string.Format(CultureInfo.InvariantCulture, EnrollmentAttestationUriFormat, enrollmentGroupId), UriKind.Relative); } + + private static Uri GetEnrollmentGroupQueryUri() + { + return new Uri(EnrollmentGroupQueryUriFormat, UriKind.Relative); + } } } diff --git a/provisioning/service/src/Http/ContractApiHttp.cs b/provisioning/service/src/Http/ContractApiHttp.cs index 63b387fda7..8ce46ca20e 100644 --- a/provisioning/service/src/Http/ContractApiHttp.cs +++ b/provisioning/service/src/Http/ContractApiHttp.cs @@ -18,7 +18,7 @@ namespace Microsoft.Azure.Devices.Provisioning.Service { - internal sealed class ContractApiHttp : IContractApiHttp + internal sealed class ContractApiHttp : IDisposable { private const string MediaTypeForDeviceManagementApis = "application/json"; @@ -99,11 +99,11 @@ public ContractApiHttp( /// the string with the message body. It can be null or empty. /// the optional string with the match condition, normally an eTag. It can be null. /// the task cancellation Token. - /// The with the HTTP response. + /// The HTTP response. /// If the cancellation was requested. /// If there is an error in the HTTP communication /// between client and service or the service answers the request with error status. - public async Task RequestAsync( + public async Task RequestAsync( HttpMethod httpMethod, Uri requestUri, IDictionary customHeaders, @@ -111,11 +111,12 @@ public async Task RequestAsync( ETag eTag, CancellationToken cancellationToken) { - ContractApiResponse response; - using var msg = new HttpRequestMessage( httpMethod, new Uri($"{requestUri}?{SdkUtils.ApiVersionQueryString}", UriKind.Relative)); + + HttpResponseMessage httpResponse; + if (!string.IsNullOrEmpty(body)) { msg.Content = new StringContent(body, Encoding.UTF8, MediaTypeForDeviceManagementApis); @@ -145,18 +146,7 @@ public async Task RequestAsync( try { - using HttpResponseMessage httpResponse = await _httpClientObj.SendAsync(msg, cancellationToken).ConfigureAwait(false); - if (httpResponse == null) - { - throw new ProvisioningServiceException( - $"The response message was null when executing operation {httpMethod}.", isTransient: true); - } - - response = new ContractApiResponse( - await httpResponse.Content.ReadAsStringAsync().ConfigureAwait(false), - httpResponse.StatusCode, - httpResponse.Headers.ToDictionary(x => x.Key, x => x.Value.FirstOrDefault()), - httpResponse.ReasonPhrase); + httpResponse = await _httpClientObj.SendAsync(msg, cancellationToken).ConfigureAwait(false); } catch (AggregateException ex) { @@ -195,9 +185,9 @@ await httpResponse.Content.ReadAsStringAsync().ConfigureAwait(false), throw new ProvisioningServiceException($"The {httpMethod} operation timed out.", HttpStatusCode.RequestTimeout, ex); } - ValidateHttpResponse(response); + await ValidateHttpResponse(httpResponse).ConfigureAwait(false); - return response; + return httpResponse; } private static bool ContainsAuthenticationException(Exception ex) @@ -207,11 +197,11 @@ private static bool ContainsAuthenticationException(Exception ex) || ContainsAuthenticationException(ex.InnerException)); } - private static void ValidateHttpResponse(ContractApiResponse response) + private static async Task ValidateHttpResponse(HttpResponseMessage response) { - if (response.Body == null) + if (response.Content == null) { - throw new ProvisioningServiceException(response.ErrorMessage, response.StatusCode, response.Fields); + throw new ProvisioningServiceException(response.ReasonPhrase, response.StatusCode, response.Headers); } // Both 200 and 204 indicate a successful operation, so there is no reason to parse the body for an error code @@ -219,22 +209,23 @@ private static void ValidateHttpResponse(ContractApiResponse response) { try { - ResponseBody responseBody = JsonConvert.DeserializeObject(response.Body); + string payload = await response.Content.ReadAsStringAsync(); + ResponseBody responseBody = JsonConvert.DeserializeObject(payload); if (response.StatusCode >= HttpStatusCode.Ambiguous) { throw new ProvisioningServiceException( - $"{response.ErrorMessage}:{responseBody.Message}", + $"{response.ReasonPhrase}:{responseBody.Message}", response.StatusCode, responseBody.ErrorCode, responseBody.TrackingId, - response.Fields); + response.Headers); } } catch (JsonException jex) { throw new ProvisioningServiceException( - $"Fail to deserialize the received response body: {response.Body}", + $"Fail to deserialize the received response body: {response.Content}", jex, false); } diff --git a/provisioning/service/src/Http/ContractApiResponse.cs b/provisioning/service/src/Http/ContractApiResponse.cs deleted file mode 100644 index 8ecb86cd13..0000000000 --- a/provisioning/service/src/Http/ContractApiResponse.cs +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -using System.Collections.Generic; -using System.Net; - -namespace Microsoft.Azure.Devices.Provisioning.Service -{ - internal sealed class ContractApiResponse - { - internal ContractApiResponse( - string body, - HttpStatusCode statusCode, - IDictionary fields, - string errorMessage) - { - Body = body; - StatusCode = statusCode; - Fields = fields; - ErrorMessage = errorMessage; - } - - public HttpStatusCode StatusCode { get; private set; } - public string Body { get; private set; } - public IDictionary Fields { get; private set; } - public string ErrorMessage { get; private set; } - } -} diff --git a/provisioning/service/src/Http/IContractApiHttp.cs b/provisioning/service/src/Http/IContractApiHttp.cs deleted file mode 100644 index 783aa16f74..0000000000 --- a/provisioning/service/src/Http/IContractApiHttp.cs +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -using System; -using System.Collections.Generic; -using System.Net.Http; -using System.Threading; -using System.Threading.Tasks; -using Azure; - -namespace Microsoft.Azure.Devices.Provisioning.Service -{ - internal interface IContractApiHttp : IDisposable - { - Task RequestAsync( - HttpMethod httpMethod, - Uri requestUri, - IDictionary customHeaders, - string body, - ETag ifMatch, - CancellationToken cancellationToken); - } -} diff --git a/provisioning/service/src/IndividualEnrollmentsClient.cs b/provisioning/service/src/IndividualEnrollmentsClient.cs index b2943688ac..d2b559d078 100644 --- a/provisioning/service/src/IndividualEnrollmentsClient.cs +++ b/provisioning/service/src/IndividualEnrollmentsClient.cs @@ -21,13 +21,12 @@ namespace Microsoft.Azure.Devices.Provisioning.Service /// public class IndividualEnrollmentsClient { - private const string ServiceName = "enrollments"; - private const string EnrollmentIdUriFormat = "{0}/{1}"; - private const string EnrollmentAttestationName = "attestationmechanism"; - private const string EnrollmentUriFormat = "{0}"; - private const string EnrollmentAttestationUriFormat = "{0}/{1}/{2}"; + private const string EnrollmentIdUriFormat = "enrollments/{0}"; + private const string EnrollmentUriFormat = "enrollments"; + private const string EnrollmentAttestationUriFormat = "enrollments/{0}/attestationmechanism"; + private const string EnrollmentQueryUriFormat = "enrollments/query"; - private readonly IContractApiHttp _contractApiHttp; + private readonly ContractApiHttp _contractApiHttp; private readonly RetryHandler _internalRetryHandler; /// @@ -37,7 +36,7 @@ protected IndividualEnrollmentsClient() { } - internal IndividualEnrollmentsClient(IContractApiHttp contractApiHttp, RetryHandler retryHandler) + internal IndividualEnrollmentsClient(ContractApiHttp contractApiHttp, RetryHandler retryHandler) { _contractApiHttp = contractApiHttp; _internalRetryHandler = retryHandler; @@ -58,13 +57,13 @@ public async Task CreateOrUpdateAsync(IndividualEnrollment cancellationToken.ThrowIfCancellationRequested(); - ContractApiResponse contractApiResponse = null; + HttpResponseMessage response = null; await _internalRetryHandler .RunWithRetryAsync( async () => { - contractApiResponse = await _contractApiHttp + response = await _contractApiHttp .RequestAsync( HttpMethod.Put, GetEnrollmentUri(individualEnrollment.RegistrationId), @@ -77,7 +76,8 @@ await _internalRetryHandler cancellationToken) .ConfigureAwait(false); - return JsonConvert.DeserializeObject(contractApiResponse.Body); + string payload = await response.Content.ReadAsStringAsync().ConfigureAwait(false); + return JsonConvert.DeserializeObject(payload); } /// @@ -96,13 +96,13 @@ public async Task GetAsync(string registrationId, Cancella cancellationToken.ThrowIfCancellationRequested(); - ContractApiResponse contractApiResponse = null; + HttpResponseMessage response = null; await _internalRetryHandler .RunWithRetryAsync( async () => { - contractApiResponse = await _contractApiHttp + response = await _contractApiHttp .RequestAsync( HttpMethod.Get, GetEnrollmentUri(registrationId), @@ -115,7 +115,8 @@ await _internalRetryHandler cancellationToken) .ConfigureAwait(false); - return JsonConvert.DeserializeObject(contractApiResponse.Body); + string payload = await response.Content.ReadAsStringAsync().ConfigureAwait(false); + return JsonConvert.DeserializeObject(payload); } /// @@ -219,13 +220,13 @@ public async Task RunBulkOperationAsync( Enrollments = individualEnrollments.ToList(), }; - ContractApiResponse contractApiResponse = null; + HttpResponseMessage response = null; await _internalRetryHandler .RunWithRetryAsync( async () => { - contractApiResponse = await _contractApiHttp + response = await _contractApiHttp .RequestAsync( HttpMethod.Post, GetEnrollmentUri(), @@ -238,7 +239,8 @@ await _internalRetryHandler cancellationToken) .ConfigureAwait(false); - return JsonConvert.DeserializeObject(contractApiResponse.Body); + string payload = await response.Content.ReadAsStringAsync().ConfigureAwait(false); + return JsonConvert.DeserializeObject(payload); } /// @@ -248,23 +250,92 @@ await _internalRetryHandler /// The service expects a SQL-like query such as /// /// "SELECT * FROM enrollments". - /// - /// For each iteration, the query will return a page of results. The maximum number of - /// items per page can be specified by the pageSize parameter. /// /// The SQL query. It cannot be null. - /// The int with the maximum number of items per iteration. It can be 0 for default, but not negative. /// The cancellation token. /// The iterable set of query results. /// If the provided is null. /// If the provided is empty or white space. - /// If the provided is less than zero. /// If the provided has requested cancellation. - public Query CreateQuery(string query, int pageSize = 0, CancellationToken cancellationToken = default) + /// + /// Iterate over individual enrollments: + /// + /// AsyncPageable<IndividualEnrollment> individualEnrollmentsQuery = dpsServiceClient.IndividualEnrollments.CreateQuery("SELECT * FROM enrollments"); + /// await foreach (IndividualEnrollment queriedEnrollment in individualEnrollmentsQuery) + /// { + /// Console.WriteLine(queriedEnrollment.RegistrationId); + /// } + /// + /// Iterate over pages of individual enrollments: + /// + /// IAsyncEnumerable<Page<IndividualEnrollment>> individualEnrollmentsQuery = dpsServiceClient.IndividualEnrollments.CreateQuery("SELECT * FROM enrollments").AsPages(); + /// await foreach (Page<IndividualEnrollment> queriedEnrollmentPage in individualEnrollmentsQuery) + /// { + /// foreach (IndividualEnrollment queriedEnrollment in queriedEnrollmentPage.Values) + /// { + /// Console.WriteLine(queriedEnrollment.RegistrationId); + /// } + /// + /// // Note that this is disposed for you while iterating item-by-item, but not when + /// // iterating page-by-page. That is why this sample has to manually call dispose + /// // on the response object here. + /// queriedEnrollmentPage.GetRawResponse().Dispose(); + /// } + /// + /// + public AsyncPageable CreateQuery(string query, CancellationToken cancellationToken = default) { + if (Logging.IsEnabled) + Logging.Enter(this, "Creating query.", nameof(CreateQuery)); + Argument.AssertNotNullOrWhiteSpace(query, nameof(query)); - return new Query(ServiceName, query, _contractApiHttp, pageSize, _internalRetryHandler, cancellationToken); + cancellationToken.ThrowIfCancellationRequested(); + + try + { + async Task> NextPageFunc(string continuationToken, int? pageSizeHint) + { + cancellationToken.ThrowIfCancellationRequested(); + return await QueryBuilder + .BuildAndSendRequestAsync( + _contractApiHttp, + _internalRetryHandler, + query, + GetEnrollmentQueryUri(), + continuationToken, + pageSizeHint, + cancellationToken) + .ConfigureAwait(false); + } + + async Task> FirstPageFunc(int? pageSizeHint) + { + cancellationToken.ThrowIfCancellationRequested(); + return await QueryBuilder + .BuildAndSendRequestAsync( + _contractApiHttp, + _internalRetryHandler, + query, + GetEnrollmentQueryUri(), + null, + pageSizeHint, + cancellationToken) + .ConfigureAwait(false); + } + + return PageableHelpers.CreateAsyncEnumerable(FirstPageFunc, NextPageFunc, null); + } + catch (Exception ex) when (Logging.IsEnabled) + { + Logging.Error(this, $"Creating query threw an exception: {ex}", nameof(CreateQuery)); + throw; + } + finally + { + if (Logging.IsEnabled) + Logging.Exit(this, "Creating query.", nameof(CreateQuery)); + } } /// @@ -285,13 +356,13 @@ public async Task GetAttestationAsync(string registrationI cancellationToken.ThrowIfCancellationRequested(); - ContractApiResponse contractApiResponse = null; + HttpResponseMessage response = null; await _internalRetryHandler .RunWithRetryAsync( async () => { - contractApiResponse = await _contractApiHttp + response = await _contractApiHttp .RequestAsync( HttpMethod.Post, GetEnrollmentAttestationUri(registrationId), @@ -304,26 +375,32 @@ await _internalRetryHandler cancellationToken) .ConfigureAwait(false); - return JsonConvert.DeserializeObject(contractApiResponse.Body); + string payload = await response.Content.ReadAsStringAsync().ConfigureAwait(false); + return JsonConvert.DeserializeObject(payload); } private static Uri GetEnrollmentUri(string registrationId) { registrationId = WebUtility.UrlEncode(registrationId); - return new Uri(string.Format(CultureInfo.InvariantCulture, EnrollmentIdUriFormat, ServiceName, registrationId), UriKind.Relative); + return new Uri(string.Format(CultureInfo.InvariantCulture, EnrollmentIdUriFormat, registrationId), UriKind.Relative); } private static Uri GetEnrollmentUri() { - return new Uri(string.Format(CultureInfo.InvariantCulture, EnrollmentUriFormat, ServiceName), UriKind.Relative); + return new Uri(EnrollmentUriFormat, UriKind.Relative); } - private static Uri GetEnrollmentAttestationUri(string enrollmentGroupId) + private static Uri GetEnrollmentAttestationUri(string registrationId) { - enrollmentGroupId = WebUtility.UrlEncode(enrollmentGroupId); + registrationId = WebUtility.UrlEncode(registrationId); return new Uri( - string.Format(CultureInfo.InvariantCulture, EnrollmentAttestationUriFormat, ServiceName, enrollmentGroupId, EnrollmentAttestationName), + string.Format(CultureInfo.InvariantCulture, EnrollmentAttestationUriFormat, registrationId), UriKind.Relative); } + + private static Uri GetEnrollmentQueryUri() + { + return new Uri(EnrollmentQueryUriFormat, UriKind.Relative); + } } } diff --git a/provisioning/service/src/Models/QueriedPage.cs b/provisioning/service/src/Models/QueriedPage.cs new file mode 100644 index 0000000000..438614038b --- /dev/null +++ b/provisioning/service/src/Models/QueriedPage.cs @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Collections.Generic; +using System.Net.Http; +using Newtonsoft.Json; + +namespace Microsoft.Azure.Devices.Provisioning.Service +{ + /// + /// Contains the result of a twin, scheduled job or raw query. + /// + internal sealed class QueriedPage + { + private const string ContinuationTokenHeader = "x-ms-continuation"; + + // Payload is taken separately from http response because reading the payload should only be done + // in an async function. + internal QueriedPage(HttpResponseMessage response, string payload) + { + Items = JsonConvert.DeserializeObject>(payload); + ContinuationToken = response.Headers.SafeGetValue(ContinuationTokenHeader); + } + + internal IReadOnlyList Items { get; set; } + + internal string ContinuationToken { get; set; } + } +} diff --git a/provisioning/service/src/Models/Query.cs b/provisioning/service/src/Models/Query.cs deleted file mode 100644 index 5a22488f57..0000000000 --- a/provisioning/service/src/Models/Query.cs +++ /dev/null @@ -1,199 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -using System; -using System.Collections.Generic; -using System.Globalization; -using System.Net.Http; -using System.Threading; -using System.Threading.Tasks; -using Azure; -using Newtonsoft.Json; - -namespace Microsoft.Azure.Devices.Provisioning.Service -{ - /// - /// The query iterator. - /// - /// - /// The iterator is the result of the query factory for - /// - /// - /// - /// IndividualEnrollment - /// - /// - /// - /// - /// EnrollmentGroup - /// - /// - /// - /// - /// RegistrationStatus - /// - /// - /// - /// On all cases, the contains a SQL query that must follow the - /// Query Language for the Device Provisioning Service. - /// - /// Optionally, an Integer with the page size, can determine the maximum number of the items in the - /// returned by the . It must be any positive integer, and if it - /// contains 0, the Device Provisioning Service will ignore it and use a standard page size. - /// - /// You can use this Object as a standard iterator, just using the HasNext and NextAsync in a - /// while loop, up to the point where the HasNext contains false. But, keep - /// in mind that the can contain a empty list, even if the HasNext contained - /// true. For example, image that you have 10 IndividualEnrollment in the Device Provisioning Service - /// and you created new query with the PageSize equals 5. In the first iteration, HasNext - /// will contains true, and the first NextAsync will return a QueryResult with - /// 5 items. After, your code will check the HasNext, which will contains true again. Now, - /// before you get the next page, somebody deletes all the IndividualEnrollment. What happened, when you call the - /// NextAsync, it will return a valid QueryResult, but the - /// will contain an empty list. - /// - /// Besides the Items, the QueryResult contains the . - /// You can also store a query context (QuerySpecification + ContinuationToken) and restart it in the future, from - /// the point where you stopped. Just recreating the query with the same and calling - /// the passing the stored ContinuationToken. - /// - public class Query - { - private const string ContinuationTokenHeaderKey = "x-ms-continuation"; - private const string ItemTypeHeaderKey = "x-ms-item-type"; - private const string PageSizeHeaderKey = "x-ms-max-item-count"; - private const string QueryUriFormat = "{0}/query"; - - private readonly string _querySpecificationJson; - private readonly IContractApiHttp _contractApiHttp; - private readonly Uri _queryPath; - private readonly CancellationToken _cancellationToken; - private readonly RetryHandler _internalRetryHandler; - private bool _hasNext; - - internal Query( - string serviceName, - string query, - IContractApiHttp contractApiHttp, - int pageSize, - RetryHandler retryHandler, - CancellationToken cancellationToken) - { - if (pageSize < 0) - { - throw new ArgumentOutOfRangeException(nameof(pageSize), "Cannot be negative."); - } - - _contractApiHttp = contractApiHttp; - PageSize = pageSize; - _internalRetryHandler = retryHandler; - _cancellationToken = cancellationToken; - _querySpecificationJson = JsonConvert.SerializeObject(new QuerySpecification(query)); - _queryPath = GetQueryUri(serviceName); - ContinuationToken = null; - _hasNext = true; - } - - /// - /// Getter for has next. - /// - /// - /// Contains true if the query is not finished in the Device Provisioning Service, and another - /// iteration with may return more items. Call after - /// a true HasNext will result in a that can or - /// cannot contains elements. But call after a false HasNext - /// will result in a exception. - /// - public bool HasNext() - { - return _hasNext; - } - - /// - /// The number of items in the current page. - /// - public int PageSize { get; set; } - - /// - /// The token to retrieve the next page. - /// - public string ContinuationToken { get; set; } - - /// - /// Return the next page of result for the query using a new continuationToken. - /// - /// the string with the previous continuationToken. It cannot be null or empty. - /// The with the next page of items for the query. - /// If the query does no have more pages to return. - public async Task NextAsync(string continuationToken) - { - if (string.IsNullOrWhiteSpace(continuationToken)) - { - throw new InvalidOperationException($"There is no {nameof(continuationToken)} to get pending elements."); - } - - ContinuationToken = continuationToken; - _hasNext = true; - - return await NextAsync().ConfigureAwait(false); - } - - /// - /// Return the next page of result for the query. - /// - /// The with the next page of items for the query. - /// If the query does no have more pages to return. - public async Task NextAsync() - { - if (!_hasNext) - { - throw new InvalidOperationException("There are no more pending elements"); - } - - IDictionary headerParameters = new Dictionary(); - if (PageSize != 0) - { - headerParameters.Add(PageSizeHeaderKey, PageSize.ToString(CultureInfo.InvariantCulture)); - } - - if (!string.IsNullOrWhiteSpace(ContinuationToken)) - { - headerParameters.Add(ContinuationTokenHeaderKey, ContinuationToken); - } - - ContractApiResponse httpResponse = null; - - await _internalRetryHandler - .RunWithRetryAsync( - async () => - { - httpResponse = await _contractApiHttp - .RequestAsync( - HttpMethod.Post, - _queryPath, - headerParameters, - _querySpecificationJson, - new ETag(), - _cancellationToken) - .ConfigureAwait(false); - }, - _cancellationToken) - .ConfigureAwait(false); - - httpResponse.Fields.TryGetValue(ItemTypeHeaderKey, out string type); - httpResponse.Fields.TryGetValue(ContinuationTokenHeaderKey, out string continuationToken); - ContinuationToken = continuationToken; - - _hasNext = ContinuationToken != null; - - var result = new QueryResult(type, httpResponse.Body, ContinuationToken); - - return result; - } - - private static Uri GetQueryUri(string path) - { - return new Uri(string.Format(CultureInfo.InvariantCulture, QueryUriFormat, path), UriKind.Relative); - } - } -} diff --git a/provisioning/service/src/Models/QueryResponse.cs b/provisioning/service/src/Models/QueryResponse.cs new file mode 100644 index 0000000000..3f874482d3 --- /dev/null +++ b/provisioning/service/src/Models/QueryResponse.cs @@ -0,0 +1,78 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net.Http; +using Azure; +using Azure.Core; + +namespace Microsoft.Azure.Devices.Provisioning.Service +{ + /// + /// The local implementation of the Azure.Core Response type. Libraries in the azure-sdk-for-net repo have access to + /// helper functions to instantiate the abstract class Response, but this library is not in that repo yet. Because of that, + /// we need to implement the abstract class. + /// + internal class QueryResponse : Response + { + private HttpResponseMessage _httpResponse; + private List _httpHeaders; + + internal QueryResponse(HttpResponseMessage httpResponse, Stream bodyStream) + { + _httpResponse = httpResponse; + ContentStream = bodyStream; + + _httpHeaders = new List(); + foreach (var header in _httpResponse.Headers) + { + _httpHeaders.Add(new HttpHeader(header.Key, header.Value.First())); + } + } + + public override int Status => (int)_httpResponse.StatusCode; + + public override string ReasonPhrase => _httpResponse.ReasonPhrase; + + public override Stream ContentStream { get; set; } + + public override string ClientRequestId + { + get => throw new NotImplementedException("This SDK does not define this feature"); + set => throw new NotImplementedException("This SDK does not define this feature"); + } + + public override void Dispose() + { + _httpResponse?.Dispose(); + ContentStream?.Dispose(); + } + + protected override bool ContainsHeader(string name) + { + Argument.AssertNotNullOrWhiteSpace(name, nameof(name)); + return _httpResponse.Headers.Contains(name); + } + + protected override IEnumerable EnumerateHeaders() + { + return _httpHeaders; + } + + protected override bool TryGetHeader(string name, out string value) + { + Argument.AssertNotNullOrWhiteSpace(name, nameof(name)); + value = _httpResponse.Headers.SafeGetValue(name); + return string.IsNullOrWhiteSpace(value); + } + + protected override bool TryGetHeaderValues(string name, out IEnumerable values) + { + Argument.AssertNotNullOrWhiteSpace(name, nameof(name)); + return _httpResponse.Headers.TryGetValues(name, out values); + } + } +} diff --git a/provisioning/service/src/Models/QueryResult.cs b/provisioning/service/src/Models/QueryResult.cs deleted file mode 100644 index 9fe13ccd1b..0000000000 --- a/provisioning/service/src/Models/QueryResult.cs +++ /dev/null @@ -1,144 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -using System; -using System.Collections.Generic; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; - -namespace Microsoft.Azure.Devices.Provisioning.Service -{ - /// - /// Representation of a single Device Provisioning Service query response with a JSON deserializer. - /// - /// - /// It is the result of any query for the provisioning service. This class will parse the result and - /// return it in a best format possible. For the known formats in , you can - /// just cast the items. In case of unknown type, the items will contain a list of string - /// and you shall parse it by your own. - /// - /// The provisioning service query result is composed by 2 system properties and a body. This class exposes - /// it with 3 getters, , , and . - /// - /// The system properties are: - /// - /// - /// type: - /// Identify the type of the content in the body. You can use it to cast the objects - /// in the items list. See for the possible types and classes - /// to cast. - /// - /// - /// continuationToken: - /// Contains the token the uniquely identify the next page of information. The - /// service will return the next page of this query when you send a new query with - /// this token. - /// - /// - /// - public class QueryResult - { - /// - /// Creates an instance of this class. - /// - /// The string with type of the content in the body. - /// It cannot be null. - /// The string with the body in a JSON list format. - /// It cannot be null, or empty, if the type is different than `unknown`. - /// The string with the continuation token. - /// It can be null. - /// If is null. - /// If is empty or white space. - protected internal QueryResult(string typeString, string bodyString, string continuationToken) - { - QueryType = (QueryResultType)Enum.Parse(typeof(QueryResultType), typeString, true); - ContinuationToken = string.IsNullOrWhiteSpace(continuationToken) - ? null - : continuationToken; - - if (QueryType != QueryResultType.Unknown && string.IsNullOrWhiteSpace(bodyString)) - { - if (bodyString == null) - { - throw new ArgumentNullException(nameof(bodyString)); - } - - throw new ArgumentException("Invalid query body.", nameof(bodyString)); - } - - switch (QueryType) - { - case QueryResultType.Enrollment: - Items = JsonConvert.DeserializeObject>(bodyString); - break; - - case QueryResultType.EnrollmentGroup: - Items = JsonConvert.DeserializeObject>(bodyString); - break; - - case QueryResultType.DeviceRegistration: - Items = JsonConvert.DeserializeObject>(bodyString); - break; - - default: - if (bodyString == null) - { - Items = null; - } - else - { - try - { - Items = JsonConvert.DeserializeObject>(bodyString); - break; - } - catch (JsonException) - { } - - - try - { - Items = JsonConvert.DeserializeObject>(bodyString); - break; - } - catch (JsonException) - { } - - // If the result cannot be deserialized into a collection of objects - // then save off the result into a collection with the body as the only item. - Items = new string[] { bodyString }; - } - break; - } - } - - /// - /// The query result type. - /// - public QueryResultType QueryType { get; protected private set; } - - /// - /// The list of query result items. - /// - /// - /// Depending on the , these items can be cast to a corresponding type: - /// - /// - /// : - /// - /// - /// : - /// - /// - /// : - /// - /// - /// - public IEnumerable Items { get; protected private set; } - - /// - /// The query result continuation token. - /// - public string ContinuationToken { get; protected private set; } - } -} diff --git a/provisioning/service/src/Models/QueryResultType.cs b/provisioning/service/src/Models/QueryResultType.cs deleted file mode 100644 index 66e9da6516..0000000000 --- a/provisioning/service/src/Models/QueryResultType.cs +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -using Newtonsoft.Json; -using Newtonsoft.Json.Converters; - -namespace Microsoft.Azure.Devices.Provisioning.Service -{ - /// - /// The Device Provisioning Service query result type - /// - [JsonConverter(typeof(StringEnumConverter))] - public enum QueryResultType - { - /// - /// Unknown result type. - /// - /// - /// The provisioning service cannot parse the information in the body. - /// Cast the objects in the items using string and parser it depending on the query the sent. - /// - Unknown, - - /// - /// An individual enrollment. - /// - /// - /// The query result in a list of individual enrollments. Cast the objects in the items using . - /// - Enrollment, - - /// - /// An enrollment group. - /// - /// - /// The query result in a list of enrollment groups. Cast the objects in the items using . - /// - EnrollmentGroup, - - /// - /// A device registration. - /// - /// - /// The query result in a list of device registrations. Cast the objects in the items using . - /// - DeviceRegistration, - } -} diff --git a/provisioning/service/src/PageableHelpers.cs b/provisioning/service/src/PageableHelpers.cs new file mode 100644 index 0000000000..faa4b68452 --- /dev/null +++ b/provisioning/service/src/PageableHelpers.cs @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Azure; + +namespace Microsoft.Azure.Devices.Provisioning.Service +{ + /// + /// Copy of a subset of the helper functions defined in the Azure.Core class by the same name: + /// https://github.com/Azure/autorest.csharp/blob/main/src/assets/Generator.Shared/PageableHelpers.cs + /// + internal static class PageableHelpers + { + internal static AsyncPageable CreateAsyncEnumerable(Func>> firstPageFunc, Func>> nextPageFunc, int? pageSize = default) where T : notnull + { + AsyncPageFunc first = (continuationToken, pageSizeHint) => firstPageFunc(pageSizeHint); + AsyncPageFunc next = nextPageFunc != null ? new AsyncPageFunc(nextPageFunc) : null; + return new FuncAsyncPageable(first, next, pageSize); + } + + internal delegate Task> AsyncPageFunc(string continuationToken = default, int? pageSizeHint = default); + internal delegate Page PageFunc(string continuationToken = default, int? pageSizeHint = default); + + internal class FuncAsyncPageable : AsyncPageable where T : notnull + { + private readonly AsyncPageFunc _firstPageFunc; + private readonly AsyncPageFunc _nextPageFunc; + private readonly int? _defaultPageSize; + + internal FuncAsyncPageable(AsyncPageFunc firstPageFunc, AsyncPageFunc nextPageFunc, int? defaultPageSize = default) + { + _firstPageFunc = firstPageFunc; + _nextPageFunc = nextPageFunc; + _defaultPageSize = defaultPageSize; + } + + public override async IAsyncEnumerable> AsPages(string continuationToken = default, int? pageSizeHint = default) + { + AsyncPageFunc pageFunc = string.IsNullOrEmpty(continuationToken) ? _firstPageFunc : _nextPageFunc; + + if (pageFunc == null) + { + yield break; + } + + int? pageSize = pageSizeHint ?? _defaultPageSize; + do + { + Page pageResponse = await pageFunc(continuationToken, pageSize).ConfigureAwait(false); + yield return pageResponse; + continuationToken = pageResponse.ContinuationToken; + pageFunc = _nextPageFunc; + } while (!string.IsNullOrEmpty(continuationToken) && pageFunc != null); + } + } + } +} diff --git a/provisioning/service/src/ProvisioningServiceClient.cs b/provisioning/service/src/ProvisioningServiceClient.cs index 516b9e0474..5a6af9246d 100644 --- a/provisioning/service/src/ProvisioningServiceClient.cs +++ b/provisioning/service/src/ProvisioningServiceClient.cs @@ -17,7 +17,7 @@ namespace Microsoft.Azure.Devices.Provisioning.Service public class ProvisioningServiceClient : IDisposable { private readonly ServiceConnectionString _provisioningConnectionString; - private readonly IContractApiHttp _contractApiHttp; + private readonly ContractApiHttp _contractApiHttp; private readonly IProvisioningServiceRetryPolicy _retryPolicy; private readonly RetryHandler _retryHandler; diff --git a/provisioning/service/src/ProvisioningServiceException.cs b/provisioning/service/src/ProvisioningServiceException.cs index 39ddde04f4..900ee21663 100644 --- a/provisioning/service/src/ProvisioningServiceException.cs +++ b/provisioning/service/src/ProvisioningServiceException.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.Net; +using System.Net.Http.Headers; using System.Runtime.Serialization; namespace Microsoft.Azure.Devices.Provisioning.Service @@ -78,13 +79,13 @@ public ProvisioningServiceException(string message, HttpStatusCode statusCode, E /// /// The message. /// The 3-digit HTTP status code returned by Device Provisioning Service. - /// The HTTP headers. - public ProvisioningServiceException(string message, HttpStatusCode statusCode, IDictionary fields) + /// The HTTP headers. + public ProvisioningServiceException(string message, HttpStatusCode statusCode, HttpResponseHeaders headers) : base(message) { IsTransient = DetermineIfTransient(statusCode); StatusCode = statusCode; - Fields = fields; + Headers = headers; } /// @@ -94,15 +95,15 @@ public ProvisioningServiceException(string message, HttpStatusCode statusCode, I /// The 3-digit HTTP status code returned by Device Provisioning Service. /// The specific 6-digit error code in the DPS response, if available. /// Service reported tracking Id. Use this when reporting a service issue. - /// The HTTP headers. - public ProvisioningServiceException(string message, HttpStatusCode statusCode, int errorCode, string trackingId, IDictionary fields) + /// The HTTP headers. + public ProvisioningServiceException(string message, HttpStatusCode statusCode, int errorCode, string trackingId, HttpResponseHeaders headers) : base(message) { IsTransient = DetermineIfTransient(statusCode); StatusCode = statusCode; ErrorCode = errorCode; TrackingId = trackingId; - Fields = fields; + Headers = headers; } /// @@ -146,7 +147,7 @@ protected ProvisioningServiceException(SerializationInfo info, StreamingContext /// /// This is used by DPS E2E tests. /// - public IDictionary Fields { get; private set; } = new Dictionary(); + public HttpResponseHeaders Headers { get; private set; } private static bool DetermineIfTransient(HttpStatusCode statusCode) { diff --git a/provisioning/service/src/Utilities/Extensions/HttpHeadersExtensions.cs b/provisioning/service/src/Utilities/Extensions/HttpHeadersExtensions.cs new file mode 100644 index 0000000000..4a233fd9ae --- /dev/null +++ b/provisioning/service/src/Utilities/Extensions/HttpHeadersExtensions.cs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Collections.Generic; +using System.Linq; +using System.Net.Http.Headers; + +namespace Microsoft.Azure.Devices.Provisioning.Service +{ + /// + /// Extension helper methods for HttpHeaders. + /// + internal static class HttpHeadersExtensions + { + /// + /// Gets the first value associated with the supplied header name. + /// + /// The collection of HTTP headers and their values. + /// The header name whose value to get. + /// The first value corresponding to the supplied header name, if the name is found in the collection; otherwise, an empty string. + internal static string SafeGetValue(this HttpHeaders headers, string name) + { + return headers.TryGetValues(name, out IEnumerable values) + ? values.FirstOrDefault() + : null; + } + } +} diff --git a/provisioning/service/src/Utilities/QueryBuilder.cs b/provisioning/service/src/Utilities/QueryBuilder.cs new file mode 100644 index 0000000000..6e54fa381d --- /dev/null +++ b/provisioning/service/src/Utilities/QueryBuilder.cs @@ -0,0 +1,74 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.IO; +using Azure; +using System.Threading.Tasks; +using System.Threading; +using Newtonsoft.Json; +using System.Net.Http; + +namespace Microsoft.Azure.Devices.Provisioning.Service +{ + internal class QueryBuilder + { + private const string ContinuationTokenHeaderKey = "x-ms-continuation"; + private const string ItemTypeHeaderKey = "x-ms-item-type"; + private const string PageSizeHeaderKey = "x-ms-max-item-count"; + private const string QueryUriFormat = "{0}/query"; + + internal static async Task> BuildAndSendRequestAsync( + ContractApiHttp contractApiHttp, + RetryHandler retryHandler, + string query, + Uri path, + string continuationToken, + int? pageSizeHint, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + + var headers = new Dictionary(); + if (!string.IsNullOrWhiteSpace(continuationToken)) + { + headers.Add(ContinuationTokenHeaderKey, continuationToken); + } + + if (pageSizeHint != null) + { + headers.Add(PageSizeHeaderKey, pageSizeHint.ToString()); + } + + headers.Add("Content-Type", "application/json"); + + HttpResponseMessage response = null; + + await retryHandler + .RunWithRetryAsync( + async () => + { + response = await contractApiHttp + .RequestAsync( + HttpMethod.Post, + path, + null, + JsonConvert.SerializeObject(new QuerySpecification(query)), + new ETag(), + cancellationToken) + .ConfigureAwait(false); + }, + cancellationToken) + .ConfigureAwait(false); + + Stream responseStream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false); + string responsePayload = await response.Content.ReadAsStringAsync().ConfigureAwait(false); + QueriedPage page = new QueriedPage(response, responsePayload); +#pragma warning disable CA2000 // Dispose objects before losing scope + // The disposable QueryResponse object is the user's responsibility, not the SDK's + return Page.FromValues(page.Items, page.ContinuationToken, new QueryResponse(response, responseStream)); +#pragma warning restore CA2000 // Dispose objects before losing scope + } + } +} diff --git a/provisioning/service/tests/Config/QueryResultTests.cs b/provisioning/service/tests/Config/QueryResultTests.cs deleted file mode 100644 index 5affa7f9fd..0000000000 --- a/provisioning/service/tests/Config/QueryResultTests.cs +++ /dev/null @@ -1,320 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -using System; -using System.Collections.Generic; -using System.Linq; -using FluentAssertions; -using Microsoft.VisualStudio.TestTools.UnitTesting; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; - -namespace Microsoft.Azure.Devices.Provisioning.Service.Tests -{ - [TestClass] - [TestCategory("Unit")] - public class QueryResultTests - { - private const string SerializedNameUnknown = "unknown"; - private const string SerializedNameEnrollment = "enrollment"; - private const string SerializedNameEnrollmentGroup = "enrollmentGroup"; - private const string SerializedNameDeviceRegistration = "deviceRegistration"; - private const string SampleContinuationToken = "{\"token\":\"+RID:Defghij6KLMNOPQ==#RS:1#TRC:2#FPC:AUAAAAAAAAAJQABAAAAAAAk=\",\"range\":{\"min\":\"0123456789abcd\",\"max\":\"FF\"}}"; - private const string SampleListIntJson = "[1, 2, 3]"; - private const string SampleListJObjectJson = "[{\"a\":1}, {\"a\":2}, {\"a\":3}]"; - - private const string SampleEnrollmentJson1 = - " {\n" + - " \"registrationId\": \"registrationid-ae518a62-3480-4639-bce2-5b69a3bb35a3\",\n" + - " \"deviceId\": \"JavaDevice-c743c684-2190-4062-a5a8-efc416ad4dba\",\n" + - " \"attestation\": {\n" + - " \"type\": \"tpm\",\n" + - " \"tpm\": {\n" + - " \"endorsementKey\": \"randonendorsementkeyfortest==\"\n" + - " }\n" + - " },\n" + - " \"iotHubHostName\": \"ContosoIotHub.azure-devices.net\",\n" + - " \"provisioningStatus\": \"enabled\",\n" + - " \"createdDateTimeUtc\": \"2017-09-19T15:45:53.3981876Z\",\n" + - " \"lastUpdatedDateTimeUtc\": \"2017-09-19T15:45:53.3981876Z\",\n" + - " \"etag\": \"00000000-0000-0000-0000-00000000000\"\n" + - " }"; - private const string SampleEnrollmentJson2 = - " {\n" + - " \"registrationId\": \"registrationid-6bdaeb7c-51fc-4a67-b24e-64e42d3aa698\",\n" + - " \"deviceId\": \"JavaDevice-eb17e87a-11aa-4794-944f-bbbf1fb960a0\",\n" + - " \"attestation\": {\n" + - " \"type\": \"tpm\",\n" + - " \"tpm\": {\n" + - " \"endorsementKey\": \"randonendorsementkeyfortest==\"\n" + - " }\n" + - " },\n" + - " \"iotHubHostName\": \"ContosoIotHub.azure-devices.net\",\n" + - " \"provisioningStatus\": \"enabled\",\n" + - " \"createdDateTimeUtc\": \"2017-09-19T15:46:35.1533673Z\",\n" + - " \"lastUpdatedDateTimeUtc\": \"2017-09-19T15:46:35.1533673Z\",\n" + - " \"etag\": \"00000000-0000-0000-0000-00000000000\"\n" + - " }"; - private const string SampleEnrollmentsJson = - "[\n" + - SampleEnrollmentJson1 + ",\n" + - SampleEnrollmentJson2 + - "]"; - - private const string SampleEnrollmentGroupId = "valid-enrollment-group-id"; - private const string SampleCreateDateTimeUtcString = "2017-11-14T12:34:18.123Z"; - private const string SampleLastUpdatedDateTimeUtcString = "2017-11-14T12:34:18.321Z"; - private const string SampleEtag = "00000000-0000-0000-0000-00000000000"; - private const string SampleEnrollmentGroupJson1 = - "{\n" + - " \"enrollmentGroupId\":\"" + SampleEnrollmentGroupId + "\",\n" + - " \"attestation\":{\n" + - " \"type\":\"x509\",\n" + - " \"x509\":{\n" + - " \"signingCertificates\":{\n" + - " \"primary\":{\n" + - " \"info\": {\n" + - " \"subjectName\": \"CN=ROOT_00000000-0000-0000-0000-000000000000, OU=Azure IoT, O=MSFT, C=US\",\n" + - " \"sha1Thumbprint\": \"0000000000000000000000000000000000\",\n" + - " \"sha256Thumbprint\": \"" + SampleEnrollmentGroupId + "\",\n" + - " \"issuerName\": \"CN=ROOT_00000000-0000-0000-0000-000000000000, OU=Azure IoT, O=MSFT, C=US\",\n" + - " \"notBeforeUtc\": \"2017-11-14T12:34:18Z\",\n" + - " \"notAfterUtc\": \"2017-11-20T12:34:18Z\",\n" + - " \"serialNumber\": \"000000000000000000\",\n" + - " \"version\": 3\n" + - " }\n" + - " }\n" + - " }\n" + - " }\n" + - " },\n" + - " \"createdDateTimeUtc\": \"" + SampleCreateDateTimeUtcString + "\",\n" + - " \"lastUpdatedDateTimeUtc\": \"" + SampleLastUpdatedDateTimeUtcString + "\",\n" + - " \"etag\": \"" + SampleEtag + "\"\n" + - "}"; - private const string SampleEnrollmentGroupJson2 = - "{\n" + - " \"enrollmentGroupId\":\"" + SampleEnrollmentGroupId + "\",\n" + - " \"attestation\":{\n" + - " \"type\":\"x509\",\n" + - " \"x509\":{\n" + - " \"signingCertificates\":{\n" + - " \"primary\":{\n" + - " \"info\": {\n" + - " \"subjectName\": \"CN=ROOT_00000000-0000-0000-0000-000000000000, OU=Azure IoT, O=MSFT, C=US\",\n" + - " \"sha1Thumbprint\": \"0000000000000000000000000000000000\",\n" + - " \"sha256Thumbprint\": \"" + SampleEnrollmentGroupId + "\",\n" + - " \"issuerName\": \"CN=ROOT_00000000-0000-0000-0000-000000000000, OU=Azure IoT, O=MSFT, C=US\",\n" + - " \"notBeforeUtc\": \"2017-11-14T12:34:18Z\",\n" + - " \"notAfterUtc\": \"2017-11-20T12:34:18Z\",\n" + - " \"serialNumber\": \"000000000000000000\",\n" + - " \"version\": 3\n" + - " }\n" + - " }\n" + - " }\n" + - " }\n" + - " },\n" + - " \"createdDateTimeUtc\": \"" + SampleCreateDateTimeUtcString + "\",\n" + - " \"lastUpdatedDateTimeUtc\": \"" + SampleLastUpdatedDateTimeUtcString + "\",\n" + - " \"etag\": \"" + SampleEtag + "\"\n" + - "}"; - - private const string SampleEnrollmentGroupJson = - "[\n" + - SampleEnrollmentGroupJson1 + ",\n" + - SampleEnrollmentGroupJson2 + - "]"; - - private const string SampleRegistrationStatus1 = - "{\n" + - " \"registrationId\":\"registrationid-ae518a62-3480-4639-bce2-5b69a3bb35a3\",\n" + - " \"createdDateTimeUtc\": \"2017-09-19T15:46:35.1533673Z\",\n" + - " \"assignedHub\":\"ContosoIotHub.azure-devices.net\",\n" + - " \"deviceId\":\"JavaDevice-c743c684-2190-4062-a5a8-efc416ad4dba\",\n" + - " \"status\":\"assigned\",\n" + - " \"lastUpdatedDateTimeUtc\": \"2017-09-19T15:46:35.1533673Z\",\n" + - " \"errorCode\": 200,\n" + - " \"errorMessage\":\"Succeeded\",\n" + - " \"etag\": \"00000000-0000-0000-0000-00000000000\"\n" + - "}"; - private const string SampleRegistrationStatus2 = - "{\n" + - " \"registrationId\":\"registrationid-6bdaeb7c-51fc-4a67-b24e-64e42d3aa698\",\n" + - " \"createdDateTimeUtc\": \"2017-09-19T15:46:35.1533673Z\",\n" + - " \"assignedHub\":\"ContosoIotHub.azure-devices.net\",\n" + - " \"deviceId\":\"JavaDevice-c743c684-2190-4062-a5a8-efc416ad4dba\",\n" + - " \"status\":\"assigned\",\n" + - " \"lastUpdatedDateTimeUtc\": \"2017-09-19T15:46:35.1533673Z\",\n" + - " \"errorCode\": 200,\n" + - " \"errorMessage\":\"Succeeded\",\n" + - " \"etag\": \"00000000-0000-0000-0000-00000000000\"\n" + - "}"; - private const string SampleRegistrationStatusJson = - "[\n" + - SampleRegistrationStatus1 + ",\n" + - SampleRegistrationStatus2 + - "]"; - - [TestMethod] - public void QueryResultCtorThrowsOnNullTypeString() - { - Action act = () => _ = new QueryResult(null, SampleListIntJson, SampleContinuationToken); - act.Should().Throw(); - } - - [TestMethod] - public void QueryResultCtorThrowsOnEmptyTypeString() - { - Action act = () => _ = new QueryResult("", SampleListIntJson, SampleContinuationToken); - act.Should().Throw(); - } - - [TestMethod] - public void QueryResultCtorThrowsOnInvalidTypeString() - { - Action act = () => _ = new QueryResult("InvalidType", SampleListIntJson, SampleContinuationToken); - act.Should().Throw(); - } - - [TestMethod] - public void QueryResultCtorThrwosOnNullBodyString() - { - Action act = () => _ = new QueryResult(SerializedNameEnrollment, null, SampleContinuationToken); - act.Should().Throw(); - } - - [TestMethod] - public void QueryResultCtorThrwosOnEmptyBodyString() - { - Action act = () => _ = new QueryResult(SerializedNameEnrollment, "", SampleContinuationToken); - act.Should().Throw(); - } - - [TestMethod] - public void QueryResultCtorThrowsOnInvalidJson() - { - Action act = () => _ = new QueryResult(SerializedNameEnrollment, "[1, 2, ]", SampleContinuationToken); - act.Should().Throw(); - } - - [TestMethod] - public void QueryResultConstructorSucceedOnIndividualEnrollment() - { - // arrange - act - var queryResult = new QueryResult(SerializedNameEnrollment, SampleEnrollmentsJson, SampleContinuationToken); - - // assert - Assert.IsNotNull(queryResult); - Assert.AreEqual(QueryResultType.Enrollment, queryResult.QueryType); - IEnumerable items = queryResult.Items; - Assert.AreEqual(2, items.Count()); - Assert.IsTrue(items.FirstOrDefault() is IndividualEnrollment); - Assert.AreEqual(SampleContinuationToken, queryResult.ContinuationToken); - } - - [TestMethod] - public void QueryResultConstructorSucceedOnEnrollmentGroup() - { - // arrange - act - var queryResult = new QueryResult(SerializedNameEnrollmentGroup, SampleEnrollmentGroupJson, SampleContinuationToken); - - // assert - Assert.IsNotNull(queryResult); - Assert.AreEqual(QueryResultType.EnrollmentGroup, queryResult.QueryType); - IEnumerable items = queryResult.Items; - Assert.AreEqual(2, items.Count()); - Assert.IsTrue(items.First() is EnrollmentGroup); - Assert.AreEqual(SampleContinuationToken, queryResult.ContinuationToken); - } - - [TestMethod] - public void QueryResultConstructorSucceedOnDeviceRegistration() - { - // arrange - act - var queryResult = new QueryResult(SerializedNameDeviceRegistration, SampleRegistrationStatusJson, SampleContinuationToken); - - // assert - Assert.IsNotNull(queryResult); - Assert.AreEqual(QueryResultType.DeviceRegistration, queryResult.QueryType); - IEnumerable items = queryResult.Items; - Assert.AreEqual(2, items.Count()); - Assert.IsTrue(items.FirstOrDefault() is DeviceRegistrationState); - Assert.AreEqual(SampleContinuationToken, queryResult.ContinuationToken); - } - - [TestMethod] - public void QueryResultConstructorSucceedOnUnknownWithNullBody() - { - // arrange - act - var queryResult = new QueryResult(SerializedNameUnknown, null, SampleContinuationToken); - // assert - Assert.IsNotNull(queryResult); - Assert.AreEqual(QueryResultType.Unknown, queryResult.QueryType); - Assert.IsNull(queryResult.Items); - Assert.AreEqual(SampleContinuationToken, queryResult.ContinuationToken); - } - - [TestMethod] - public void QueryResultConstructorSucceedOnUnknownWithObjectListBody() - { - // arrange - act - var queryResult = new QueryResult(SerializedNameUnknown, SampleListJObjectJson, SampleContinuationToken); - - // assert - Assert.IsNotNull(queryResult); - Assert.AreEqual(QueryResultType.Unknown, queryResult.QueryType); - IEnumerable items = queryResult.Items; - Assert.AreEqual(3, items.Count()); - Assert.IsTrue(items.First() is JObject); - Assert.AreEqual(SampleContinuationToken, queryResult.ContinuationToken); - } - - [TestMethod] - public void QueryResultConstructorSucceedOnUnknownWithIntegerListBody() - { - // arrange - act - var queryResult = new QueryResult(SerializedNameUnknown, SampleListIntJson, SampleContinuationToken); - - // assert - Assert.IsNotNull(queryResult); - Assert.AreEqual(QueryResultType.Unknown, queryResult.QueryType); - IEnumerable items = queryResult.Items; - Assert.AreEqual(3, items.Count()); - Assert.IsTrue(items.FirstOrDefault() is long); - Assert.AreEqual(SampleContinuationToken, queryResult.ContinuationToken); - } - - [TestMethod] - public void QueryResultConstructorSucceedOnUnknownWithStringBody() - { - // arrange - string body = "This is a non deserializable body"; - - // act - var queryResult = new QueryResult(SerializedNameUnknown, body, SampleContinuationToken); - - // assert - Assert.IsNotNull(queryResult); - Assert.AreEqual(QueryResultType.Unknown, queryResult.QueryType); - IEnumerable items = queryResult.Items; - Assert.AreEqual(1, items.Count()); - Assert.AreEqual(body, items.First()); - Assert.AreEqual(SampleContinuationToken, queryResult.ContinuationToken); - } - - [TestMethod] - [DataRow(null)] - [DataRow("")] - public void QueryResultConstructorSucceedOnNonRealContinuationToken(string continuationToken) - { - // arrange - string body = "This is a non deserializable body"; - - // act - var queryResult = new QueryResult(SerializedNameUnknown, body, continuationToken); - - // assert - queryResult.Should().NotBeNull(); - queryResult.ContinuationToken.Should().BeNull(); - } - } -}