Populate known data for regions and service tags for reference purpos…

…es (#168)
mburumaxwell · May 10, 2024 · 12a66cc · 12a66cc
1 parent 42e9e39
commit 12a66cc
Show file tree

Hide file tree

Showing 6 changed files with 313 additions and 14 deletions.
diff --git a/AzureIPNetworks/AzureIPNetworks.csproj b/AzureIPNetworks/AzureIPNetworks.csproj
@@ -11,6 +11,7 @@
 
   <ItemGroup>
     <InternalsVisibleTo Include="AzureIPNetworks.Tests" />
+    <InternalsVisibleTo Include="AzureIPNetworksDownloader" />
     <EmbeddedResource Include="Resources\**" />
   </ItemGroup>
 

diff --git a/AzureIPNetworks/AzureIPsProvider.cs b/AzureIPNetworks/AzureIPsProvider.cs
@@ -204,9 +204,7 @@ public async ValueTask<IEnumerable<ServiceTag>> GetServiceTagsAsync(AzureCloud c
 #endif
 }
 
-/// <summary>
-/// Implementation of <see cref="AzureIPsProvider"/> for locally cached data.
-/// </summary>
+/// <summary>Implementation of <see cref="AzureIPsProvider"/> for data in embedded files.</summary>
 internal class AzureIPsProviderLocal : AzureIPsProvider
 {
     /// <inheritdoc/>
@@ -222,10 +220,23 @@ protected override ValueTask<Stream> GetStreamAsync(AzureCloud cloud, Cancellati
     }
 }
 
-/// <summary>
-/// Implementation of <see cref="AzureIPsProvider"/> for downloading remote data once per run.
-/// </summary>
-/// <remarks>Creates an <see cref="AzureIPsProviderRemote"/> instance.</remarks>
+/// <summary>Implementation of <see cref="AzureIPsProvider"/> for data in embedded files.</summary>
+/// <param name="dir">Directory containing the files.</param>
+internal class AzureIPsProviderTemp(string dir) : AzureIPsProvider
+{
+    /// <inheritdoc/>
+    protected override ValueTask<Stream> GetStreamAsync(AzureCloud cloud, CancellationToken cancellationToken = default)
+    {
+        // read the JSON file from directory
+        var path = Path.Combine(dir, $"{cloud}.json");
+        var stream = File.OpenRead(path);
+
+        // deserialize the JSON file
+        return new ValueTask<Stream>(stream);
+    }
+}
+
+/// <summary>Implementation of <see cref="AzureIPsProvider"/> for downloading remote data once per application instance.</summary>
 /// <param name="downloader">The <see cref="AzureIPsDownloader"/> to use for downloading.</param>
 internal class AzureIPsProviderRemote(AzureIPsDownloader downloader) : AzureIPsProvider
 {

diff --git a/AzureIPNetworks/KnownData.cs b/AzureIPNetworks/KnownData.cs
@@ -0,0 +1,199 @@
+//------------------------------------------------------------------------------
+// <auto-generated>
+//     This code was generated by the StaticDataGenerator source generator
+//
+//     Changes to this file may cause incorrect behavior and will be lost if
+//     the code is regenerated.
+// </auto-generated>
+//------------------------------------------------------------------------------
+
+#nullable enable
+
+#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member
+
+using System.Collections;
+using System.Collections.Generic;
+
+namespace AzureIPNetworks;
+
+public static partial class KnownData
+{
+    // The regions
+    public static readonly IReadOnlyList<string> Regions = [
+        "",
+        "australiacentral",
+        "australiacentral2",
+        "australiaeast",
+        "australiasoutheast",
+        "brazilne",
+        "brazilse",
+        "brazilsouth",
+        "canadacentral",
+        "canadaeast",
+        "centralfrance",
+        "centralindia",
+        "centralus",
+        "centraluseuap",
+        "chinaeast",
+        "chinaeast2",
+        "chinaeast3",
+        "chinanorth",
+        "chinanorth2",
+        "chinanorth3",
+        "eastasia",
+        "eastus",
+        "eastus2",
+        "eastus2euap",
+        "germanycentral",
+        "germanyn",
+        "germanynortheast",
+        "germanywc",
+        "israelcentral",
+        "italynorth",
+        "japaneast",
+        "japanwest",
+        "jioindiacentral",
+        "jioindiawest",
+        "koreacentral",
+        "koreasouth",
+        "malaysiasouth",
+        "mexicocentral",
+        "northcentralus",
+        "northeurope",
+        "northeurope2",
+        "norwaye",
+        "norwayw",
+        "polandcentral",
+        "qatarcentral",
+        "southafricanorth",
+        "southafricawest",
+        "southcentralus",
+        "southeastasia",
+        "southfrance",
+        "southindia",
+        "spaincentral",
+        "swedencentral",
+        "swedensouth",
+        "switzerlandn",
+        "switzerlandw",
+        "taiwannorth",
+        "taiwannorthwest",
+        "uaecentral",
+        "uaenorth",
+        "uksouth",
+        "ukwest",
+        "usdodcentral",
+        "usdodeast",
+        "usgovarizona",
+        "usgoviowa",
+        "usgovtexas",
+        "usgovvirginia",
+        "usstagec",
+        "usstagee",
+        "westcentralus",
+        "westeurope",
+        "westindia",
+        "westus",
+        "westus2",
+        "westus3",
+    ];
+
+    // The services
+    public static readonly IReadOnlyList<string> Services = [
+        "",
+        "ActionGroup",
+        "ApplicationInsightsAvailability",
+        "AutonomousDevelopmentPlatform",
+        "AzureAD",
+        "AzureAdvancedThreatProtection",
+        "AzureAPIForFHIR",
+        "AzureApiManagement",
+        "AzureAppConfiguration",
+        "AzureAppService",
+        "AzureAppServiceManagement",
+        "AzureArcInfrastructure",
+        "AzureAttestation",
+        "AzureAutomation",
+        "AzureBackup",
+        "AzureBotService",
+        "AzureCognitiveSearch",
+        "AzureConnectors",
+        "AzureContainerRegistry",
+        "AzureCosmosDB",
+        "AzureDatabricks",
+        "AzureDataExplorerManagement",
+        "AzureDataLake",
+        "AzureDeviceUpdate",
+        "AzureDevOps",
+        "AzureDevSpaces",
+        "AzureDigitalTwins",
+        "AzureEventGrid",
+        "AzureEventHub",
+        "AzureFrontDoor",
+        "AzureIdentity",
+        "AzureInformationProtection",
+        "AzureIoTHub",
+        "AzureKeyVault",
+        "AzureLoadTestingInstanceManagement",
+        "AzureMachineLearning",
+        "AzureMachineLearningInference",
+        "AzureManagedGrafana",
+        "AzureMonitor",
+        "AzureMonitorForSAP",
+        "AzureOpenDatasets",
+        "AzurePortal",
+        "AzureResourceManager",
+        "AzureSecurityCenter",
+        "AzureSentinel",
+        "AzureServiceBus",
+        "AzureSignalR",
+        "AzureSiteRecovery",
+        "AzureSphereSecureService_Prod",
+        "AzureSpringCloud",
+        "AzureSQL",
+        "AzureStack",
+        "AzureStorage",
+        "AzureTrafficManager",
+        "AzureUpdateDelivery",
+        "AzureVideoAnalyzerForMedia",
+        "AzureWebPubSub",
+        "BatchNodeManagement",
+        "ChaosStudio",
+        "CognitiveServicesFrontend",
+        "CognitiveServicesManagement",
+        "DataFactory",
+        "Dynamics365BusinessCentral",
+        "Dynamics365ForMarketingEmail",
+        "ElasticAFD",
+        "EOPExtPublished",
+        "GatewayManager",
+        "Grafana",
+        "HDInsight",
+        "LogicApps",
+        "M365ManagementActivityApi",
+        "M365ManagementActivityApiWebhook",
+        "Marketplace",
+        "MicrosoftAzureFluidRelay",
+        "MicrosoftCloudAppSecurity",
+        "MicrosoftContainerRegistry",
+        "MicrosoftDefenderForEndpoint",
+        "MicrosoftPurviewPolicyDistribution",
+        "OneDsCollector",
+        "PowerBI",
+        "PowerPlatformInfra",
+        "PowerPlatformPlex",
+        "PowerQueryOnline",
+        "SCCservice",
+        "Scuba",
+        "SecurityCopilot",
+        "SerialConsole",
+        "ServiceFabric",
+        "SqlManagement",
+        "StorageMover",
+        "StorageSyncService",
+        "TridentKusto",
+        "WindowsAdminCenter",
+        "WindowsVirtualDesktop",
+        "WVDRelays",
+    ];
+}
diff --git a/AzureIPNetworks/ServiceTag.cs b/AzureIPNetworks/ServiceTag.cs
@@ -85,3 +85,8 @@ public enum AzureCloud
     ///
     AzureGermany,
 }
+
+/// <summary>
+/// Known data collected from the files and used just for reference
+/// </summary>
+public static partial class KnownData { }
diff --git a/AzureIPNetworksDownloader/AzureIPNetworksDownloader.csproj b/AzureIPNetworksDownloader/AzureIPNetworksDownloader.csproj
@@ -1,6 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk.Worker">
 
   <PropertyGroup>
+    <TargetFrameworks>net8.0</TargetFrameworks>
     <OutputType>Exe</OutputType>
     <IsPackable>false</IsPackable>
   </PropertyGroup>

diff --git a/AzureIPNetworksDownloader/Program.cs b/AzureIPNetworksDownloader/Program.cs
@@ -1,5 +1,7 @@
 using AzureIPNetworks;
 using System.Text.Json;
+using System.CodeDom.Compiler;
+using System.Text;
 using System.Text.Json.Serialization;
 
 var builder = Host.CreateApplicationBuilder(args);
@@ -9,37 +11,117 @@
 
 var app = builder.Build();
 
+var cancellationToken = CancellationToken.None;
 var logger = app.Services.GetRequiredService<ILogger<Program>>();
 var downloader = app.Services.GetRequiredService<AzureIPsDownloader>();
 
 var clouds = Enum.GetValues<AzureCloud>();
 var targetDirectory = Path.GetFullPath(Path.Combine(Directory.GetCurrentDirectory(), "../AzureIPNetworks/Resources"));
 var files = new Dictionary<AzureCloud, string>();
 
+// download file for each cloud
 foreach (var cloud in clouds)
 {
     var cloudName = cloud.ToString();
-    var (url, stream) = await downloader.DownloadAsync(cloud);
+    var (url, stream) = await downloader.DownloadAsync(cloud, cancellationToken);
     var path = $"{Path.Combine(targetDirectory, cloudName)}.json";
     if (File.Exists(path)) File.Delete(path);
-    var fs = new FileStream(path, FileMode.OpenOrCreate);
-    await stream.CopyToAsync(fs);
-    await fs.FlushAsync();
+    using var fs = new FileStream(path, FileMode.OpenOrCreate);
+    await stream.CopyToAsync(fs, cancellationToken);
+    await fs.FlushAsync(cancellationToken);
     logger.LogInformation("Completed writing service tag file for cloud {CloudName}", cloud);
     files[cloud] = Path.GetFileName(url);
 }
 
+// write list of file for reference
 if (files.Count == clouds.Length)
 {
     var path = $"{Path.Combine(targetDirectory, "files")}.json";
     if (File.Exists(path)) File.Delete(path);
-    var fs = new FileStream(path, FileMode.OpenOrCreate);
-    await JsonSerializer.SerializeAsync(fs, files, DownloaderJsonSerializerContext.Default.DictionaryAzureCloudString);
-    await fs.FlushAsync();
+    using var fs = new FileStream(path, FileMode.OpenOrCreate);
+    await JsonSerializer.SerializeAsync(fs, files, DownloaderJsonSerializerContext.Default.DictionaryAzureCloudString, cancellationToken);
+    await fs.FlushAsync(cancellationToken);
+    logger.LogInformation("Completed writing files.json");
+}
+
+{
+    var provider = new AzureIPsProviderTemp(targetDirectory);
+    var path = Path.Combine(targetDirectory, "../KnownData.cs");
+    await GenerateKnownDataAsync(clouds, provider, path, cancellationToken);
+    logger.LogInformation("Completed writing KnownData.cs");
 }
 
 logger.LogInformation("Finished!");
 
+static async Task GenerateKnownDataAsync(IEnumerable<AzureCloud> clouds, AzureIPsProvider provider, string path, CancellationToken cancellationToken = default)
+{
+    // find unique regions and services
+    var regions = new List<string>();
+    var services = new List<string>();
+    foreach (var cloud in clouds)
+    {
+        var tags = await provider.GetServiceTagsAsync(cloud, cancellationToken);
+        regions.AddRange(tags.Select(t => t.Properties.Region));
+        services.AddRange(tags.Select(t => t.Properties.SystemService));
+    }
+    regions = [.. regions.Distinct(StringComparer.OrdinalIgnoreCase).Order()];
+    services = [.. services.Distinct(StringComparer.OrdinalIgnoreCase).Order()];
+    const string Header = @"//------------------------------------------------------------------------------
+// <auto-generated>
+//     This code was generated by the StaticDataGenerator source generator
+//
+//     Changes to this file may cause incorrect behavior and will be lost if
+//     the code is regenerated.
+// </auto-generated>
+//------------------------------------------------------------------------------
+
+#nullable enable
+
+#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member
+";
+
+    var sb = new StringBuilder();
+    var writer = new IndentedTextWriter(new StringWriter(sb));
+
+    writer.WriteLine(Header);
+
+    writer.WriteLine("using System.Collections;");
+    writer.WriteLine("using System.Collections.Generic;");
+
+    // write the namespace
+    writer.WriteLine();
+    writer.WriteLine("namespace AzureIPNetworks;");
+    writer.WriteLine();
+
+    // begin the class
+    writer.WriteLine("public static partial class KnownData");
+    writer.WriteLine("{");
+    writer.Indent++;
+
+    writer.WriteLine("// The regions");
+    writer.WriteLine("public static readonly IReadOnlyList<string> Regions = [");
+    writer.Indent++;
+    foreach (var r in regions) writer.WriteLine($"\"{r}\",");
+    writer.Indent--;
+    writer.WriteLine("];");
+
+    writer.WriteLine("");
+    writer.WriteLine("// The services");
+    writer.WriteLine("public static readonly IReadOnlyList<string> Services = [");
+    writer.Indent++;
+    foreach (var s in services) writer.WriteLine($"\"{s}\",");
+    writer.Indent--;
+    writer.WriteLine("];");
+
+    // end the class
+    writer.Indent--;
+    writer.WriteLine("}");
+    await writer.FlushAsync(cancellationToken);
+
+    // output to file
+    await File.WriteAllTextAsync(path, sb.ToString(), Encoding.UTF8, cancellationToken);
+}
+
 [JsonSerializable(typeof(Dictionary<AzureCloud, string>))]
 [JsonSourceGenerationOptions(WriteIndented = true)]
 internal partial class DownloaderJsonSerializerContext : JsonSerializerContext { }