Skip to content
This repository has been archived by the owner on Nov 16, 2023. It is now read-only.

Do not delete e2e test resources on PR builds #227

Merged
merged 1 commit into from
Aug 24, 2021
Merged

Do not delete e2e test resources on PR builds #227

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 50 additions & 3 deletions iot-hub/Samples/service/CleanupDevicesSample/CleanupDevicesSample.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
using Azure.Storage.Blobs;
using Azure.Storage.Blobs.Models;
using Azure.Storage.Sas;
using Microsoft.Azure.Devices.Common.Exceptions;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.IO;
Expand All @@ -20,6 +22,8 @@ namespace Microsoft.Azure.Devices.Samples
public class CleanupDevicesSample
{
private const string ImportExportDevicesFileName = "devices.txt";
private const int MaxIterationWait = 60;
private static readonly TimeSpan WaitDuration = TimeSpan.FromSeconds(10);

private static readonly TimeSpan s_waitDuration = TimeSpan.FromSeconds(5);
private static readonly IReadOnlyList<JobStatus> s_completedJobs = new[]
Expand All @@ -37,7 +41,7 @@ public CleanupDevicesSample(RegistryManager rm, BlobContainerClient sc, List<str
{
_registryManager = rm ?? throw new ArgumentNullException(nameof(rm));
_blobContainerClient = sc ?? throw new ArgumentNullException(nameof(sc));

Console.WriteLine($"Delete devices with prefixes: {JsonConvert.SerializeObject(deleteDevicesWithPrefix)}");
vinagesh marked this conversation as resolved.
Show resolved Hide resolved
_deleteDevicesWithPrefix = deleteDevicesWithPrefix;
}

Expand Down Expand Up @@ -65,8 +69,29 @@ private async Task CleanupDevices()
excludeKeysInExport: true,
storageAuthenticationType: StorageAuthenticationType.KeyBased);

JobProperties exportAllDevicesJob = await _registryManager.ExportDevicesAsync(exportAllDevicesProperties);
JobProperties exportAllDevicesJob = null;
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Adding retries on jobs to protect against any currently running job. We could change the total wait time if necessary after observing for a while.


int tryCount = 0;
while (true)
{
try
{
exportAllDevicesJob = await _registryManager.ExportDevicesAsync(exportAllDevicesProperties);
vinagesh marked this conversation as resolved.
Show resolved Hide resolved
break;
}
// Wait for pending jobs to finish.
catch (JobQuotaExceededException) when (++tryCount < MaxIterationWait)
{
Console.WriteLine($"JobQuotaExceededException... waiting.");
await Task.Delay(WaitDuration);
}
}

if (exportAllDevicesJob == null)
{
throw new Exception("Export devices job failed.");
}

// Wait until the export job is finished.
while (true)
{
Expand Down Expand Up @@ -126,7 +151,29 @@ private async Task CleanupDevices()
inputBlobContainerUri: storageAccountSasUri,
outputBlobContainerUri: storageAccountSasUri,
storageAuthenticationType: StorageAuthenticationType.KeyBased);
JobProperties importDevicesToBeDeletedJob = await _registryManager.ImportDevicesAsync(importDevicesToBeDeletedProperties);

JobProperties importDevicesToBeDeletedJob = null;

tryCount = 0;
while (true)
{
try
{
importDevicesToBeDeletedJob = await _registryManager.ImportDevicesAsync(importDevicesToBeDeletedProperties);
drwill-ms marked this conversation as resolved.
Show resolved Hide resolved
break;
}
// Wait for pending jobs to finish.
catch (JobQuotaExceededException) when (++tryCount < MaxIterationWait)
{
Console.WriteLine($"JobQuotaExceededException... waiting.");
await Task.Delay(WaitDuration);
}
}

if (importDevicesToBeDeletedJob == null)
{
throw new Exception("Import devices job failed.");
}

// Wait until job is finished.
while (true)
Expand Down
3 changes: 2 additions & 1 deletion iot-hub/Samples/service/CleanupDevicesSample/csharp_devices_list.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,5 @@ digitaltwine2e-
tpm-registration-id-
ModuleNotFoundTestSasl_
RegistryManager_
EdgeDeploymentSample_
EdgeDeploymentSample_
RegistryManagerSample_
2 changes: 2 additions & 0 deletions iot-hub/Samples/service/CleanupDevicesSample/csharp_samples_devices_list.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
RegistryManagerSample_
EdgeDeploymentSample_
2 changes: 1 addition & 1 deletion iot-hub/Samples/service/RegistryManagerSample/Parameters.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ internal class Parameters
'd',
"DevicePrefix",
Required = false,
Default = "RegistryManagerSample-",
Copy link
Contributor Author

@vinagesh vinagesh Aug 23, 2021 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Changing to match prefix style used. This was not in the delete list earlier but the sample deletes the device so this is just to guard against leaked resources. Currently the hub does not have any devices with this prefix so we don't need to worry about deleting the ones with old prefix.

Default = "RegistryManagerSample_",
HelpText = "The prefix to use when creating devices.")]
public string DevicePrefix { get; set; }
}
Expand Down
8 changes: 8 additions & 0 deletions vsts/vsts.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,10 @@ jobs:
IOTHUB_PFX_X509_THUMBPRINT: $(IOTHUB-PFX-X509-THUMBPRINT)
DPS_IDSCOPE: $(DPS-IDSCOPE)
PROVISIONING_CONNECTION_STRING: $(PROVISIONING-CONNECTION-STRING)
# PATH-TO-DEVICE-PREFIX-FOR-DELETION-FILE is a variable configured on the pipeline. The possible values are: csharp_devices_list.csv, csharp_samples_devices_list.csv.
Copy link
Contributor Author

@vinagesh vinagesh Aug 23, 2021 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This variable is no longer a constant and no longer extracted from keyvault.
https://github.com/Azure/azure-iot-sdk-csharp/pull/2153/files

# The value of this variable determines whether to delete devices created by the sample or e2e tests.
# csharp_devices_list.csv contains the prefixes of devices created by e2e tests.
# csharp_samples_devices_list.csv contains the prefixes of devices created by samples.
PATH_TO_DEVICE_PREFIX_FOR_DELETION_FILE: $(PATH-TO-DEVICE-PREFIX-FOR-DELETION-FILE)
STORAGE_ACCOUNT_CONNECTION_STRING: $(STORAGE-ACCOUNT-CONNECTION-STRING)
FAR_AWAY_IOTHUB_CONNECTION_STRING: $(FAR-AWAY-IOTHUB-CONNECTION-STRING)
Expand Down Expand Up @@ -104,6 +108,10 @@ jobs:
IOTHUB_PFX_X509_THUMBPRINT: $(IOTHUB-PFX-X509-THUMBPRINT)
DPS_IDSCOPE: $(DPS-IDSCOPE)
PROVISIONING_CONNECTION_STRING: $(PROVISIONING-CONNECTION-STRING)
# PATH-TO-DEVICE-PREFIX-FOR-DELETION-FILE is a variable configured on the pipeline. The possible values are: csharp_devices_list.csv, csharp_samples_devices_list.csv.
# The value of this variable determines whether to delete devices created by the sample or e2e tests.
# csharp_devices_list.csv contains the prefixes of devices created by e2e tests.
# csharp_samples_devices_list.csv contains the prefixes of devices created by samples.
PATH_TO_DEVICE_PREFIX_FOR_DELETION_FILE: $(PATH-TO-DEVICE-PREFIX-FOR-DELETION-FILE)
STORAGE_ACCOUNT_CONNECTION_STRING: $(STORAGE-ACCOUNT-CONNECTION-STRING)
FAR_AWAY_IOTHUB_CONNECTION_STRING: $(FAR-AWAY-IOTHUB-CONNECTION-STRING)
Expand Down