-
Notifications
You must be signed in to change notification settings - Fork 113
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[Egress Extensibility] Convert S3 to An Egress Extension (Pt 1) (#3702)
* Initial work on separating out S3 into an extension - untested * Initial work on separating out S3 into an extension - untested * Minor tweaks - extension can now start up, haven't been able to test that it actually works yet. * In progress. * Experimenting - having some issues. * Applied same EXPERIMENTAL changes I made in the main repo, and was able to successfully egress using the extension. * Some cleanup, getting ready for PR * Tweaks * Significant refactoring to share components between Azure and S3 * PR Feedback for Justin * Converted projects from Sdk.Web to Sdk * PR Feedback for Justin
- Loading branch information
1 parent
8d95a7c
commit 14b7525
Showing
39 changed files
with
767 additions
and
299 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
2 changes: 0 additions & 2 deletions
2
src/Extensions/AzureBlobStorage/AzureBlobEgressProviderOptions.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,155 +1,73 @@ | ||
| // Licensed to the .NET Foundation under one or more agreements. | ||
| // The .NET Foundation licenses this file to you under the MIT license. | ||
|
|
||
| using Microsoft.Diagnostics.Monitoring.Extension.Common; | ||
| using Microsoft.Extensions.Logging; | ||
| using System.CommandLine; | ||
| using System.CommandLine.Parsing; | ||
| using System.Text.Json; | ||
|
|
||
| namespace Microsoft.Diagnostics.Monitoring.AzureBlobStorage | ||
| { | ||
| internal sealed class Program | ||
| { | ||
| private static Stream StdInStream; | ||
| private static CancellationTokenSource CancelSource = new CancellationTokenSource(); | ||
|
|
||
| public static ILogger Logger { get; set; } | ||
|
|
||
| static async Task<int> Main(string[] args) | ||
| { | ||
| using var loggerFactory = LoggerFactory.Create(builder => | ||
| { | ||
| builder.AddConsole(); | ||
| }); | ||
|
|
||
| Logger = loggerFactory.CreateLogger<Program>(); | ||
|
|
||
| // Expected command line format is: dotnet-monitor-egress-azureblobstorage.exe Egress | ||
| RootCommand rootCommand = new RootCommand("Egresses an artifact to Azure Storage."); | ||
|
|
||
| Command egressCmd = new Command("Egress", "The class of extension being invoked; Egress is for egressing an artifact."); | ||
|
|
||
| egressCmd.SetHandler(Egress); | ||
|
|
||
| rootCommand.Add(egressCmd); | ||
|
|
||
| return await rootCommand.InvokeAsync(args); | ||
| } | ||
| ILogger logger = Utilities.CreateLogger(); | ||
|
|
||
| private static async Task<int> Egress() | ||
| { | ||
| EgressArtifactResult result = new(); | ||
| try | ||
| { | ||
| string jsonConfig = Console.ReadLine(); | ||
| ExtensionEgressPayload configPayload = JsonSerializer.Deserialize<ExtensionEgressPayload>(jsonConfig); | ||
| AzureBlobEgressProviderOptions options = BuildOptions(configPayload); | ||
|
|
||
| AzureBlobEgressProvider provider = new AzureBlobEgressProvider(Logger); | ||
|
|
||
| Console.CancelKeyPress += Console_CancelKeyPress; | ||
|
|
||
| result.ArtifactPath = await provider.EgressAsync(options, | ||
| GetStream, | ||
| configPayload.Settings, | ||
| CancelSource.Token); | ||
|
|
||
| result.Succeeded = true; | ||
| } | ||
| catch (Exception ex) | ||
| { | ||
| result.Succeeded = false; | ||
| result.FailureMessage = ex.Message; | ||
| } | ||
|
|
||
| string jsonBlob = JsonSerializer.Serialize<EgressArtifactResult>(result); | ||
| Console.Write(jsonBlob); | ||
|
|
||
| // return non-zero exit code when failed | ||
| return result.Succeeded ? 0 : 1; | ||
| } | ||
|
|
||
| private static AzureBlobEgressProviderOptions BuildOptions(ExtensionEgressPayload configPayload) | ||
| { | ||
| AzureBlobEgressProviderOptions options = GetOptions<AzureBlobEgressProviderOptions>(configPayload); | ||
| AzureBlobEgressProvider provider = new(logger); | ||
|
|
||
| // If account key was not provided but the name was provided, | ||
| // lookup the account key property value from EgressOptions.Properties | ||
| if (string.IsNullOrEmpty(options.AccountKey) && !string.IsNullOrEmpty(options.AccountKeyName)) | ||
| Action<ExtensionEgressPayload, AzureBlobEgressProviderOptions> configureOptions = (configPayload, options) => | ||
| { | ||
| if (configPayload.Properties.TryGetValue(options.AccountKeyName, out string accountKey)) | ||
| // If account key was not provided but the name was provided, | ||
| // lookup the account key property value from EgressOptions.Properties | ||
| if (string.IsNullOrEmpty(options.AccountKey) && !string.IsNullOrEmpty(options.AccountKeyName)) | ||
| { | ||
| options.AccountKey = accountKey; | ||
| if (configPayload.Properties.TryGetValue(options.AccountKeyName, out string accountKey)) | ||
| { | ||
| options.AccountKey = accountKey; | ||
| } | ||
| else | ||
| { | ||
| logger.EgressProviderUnableToFindPropertyKey(Constants.AzureBlobStorageProviderName, options.AccountKeyName); | ||
| } | ||
| } | ||
| else | ||
| { | ||
| Logger.EgressProviderUnableToFindPropertyKey(Constants.AzureBlobStorageProviderName, options.AccountKeyName); | ||
| } | ||
| } | ||
|
|
||
| // If shared access signature (SAS) was not provided but the name was provided, | ||
| // lookup the SAS property value from EgressOptions.Properties | ||
| if (string.IsNullOrEmpty(options.SharedAccessSignature) && !string.IsNullOrEmpty(options.SharedAccessSignatureName)) | ||
| { | ||
| if (configPayload.Properties.TryGetValue(options.SharedAccessSignatureName, out string sasSig)) | ||
| // If shared access signature (SAS) was not provided but the name was provided, | ||
| // lookup the SAS property value from EgressOptions.Properties | ||
| if (string.IsNullOrEmpty(options.SharedAccessSignature) && !string.IsNullOrEmpty(options.SharedAccessSignatureName)) | ||
| { | ||
| options.SharedAccessSignature = sasSig; | ||
| if (configPayload.Properties.TryGetValue(options.SharedAccessSignatureName, out string sasSig)) | ||
| { | ||
| options.SharedAccessSignature = sasSig; | ||
| } | ||
| else | ||
| { | ||
| logger.EgressProviderUnableToFindPropertyKey(Constants.AzureBlobStorageProviderName, options.SharedAccessSignatureName); | ||
| } | ||
| } | ||
| else | ||
| { | ||
| Logger.EgressProviderUnableToFindPropertyKey(Constants.AzureBlobStorageProviderName, options.SharedAccessSignatureName); | ||
| } | ||
| } | ||
|
|
||
| // If queue shared access signature (SAS) was not provided but the name was provided, | ||
| // lookup the SAS property value from EgressOptions.Properties | ||
| if (string.IsNullOrEmpty(options.QueueSharedAccessSignature) && !string.IsNullOrEmpty(options.QueueSharedAccessSignatureName)) | ||
| { | ||
| if (configPayload.Properties.TryGetValue(options.QueueSharedAccessSignatureName, out string signature)) | ||
| // If queue shared access signature (SAS) was not provided but the name was provided, | ||
| // lookup the SAS property value from EgressOptions.Properties | ||
| if (string.IsNullOrEmpty(options.QueueSharedAccessSignature) && !string.IsNullOrEmpty(options.QueueSharedAccessSignatureName)) | ||
| { | ||
| options.QueueSharedAccessSignature = signature; | ||
| if (configPayload.Properties.TryGetValue(options.QueueSharedAccessSignatureName, out string signature)) | ||
| { | ||
| options.QueueSharedAccessSignature = signature; | ||
| } | ||
| else | ||
| { | ||
| logger.EgressProviderUnableToFindPropertyKey(Constants.AzureBlobStorageProviderName, options.QueueSharedAccessSignatureName); | ||
| } | ||
| } | ||
| else | ||
| { | ||
| Logger.EgressProviderUnableToFindPropertyKey(Constants.AzureBlobStorageProviderName, options.QueueSharedAccessSignatureName); | ||
| } | ||
| } | ||
|
|
||
| return options; | ||
| } | ||
|
|
||
| private static T GetOptions<T>(ExtensionEgressPayload payload) where T : class, new() | ||
| { | ||
| IConfigurationBuilder builder = new ConfigurationBuilder(); | ||
|
|
||
| var configurationRoot = builder.AddInMemoryCollection(payload.Configuration).Build(); | ||
| }; | ||
|
|
||
| T options = new(); | ||
|
|
||
| configurationRoot.Bind(options); | ||
|
|
||
| return options; | ||
| } | ||
| // Expected command line format is: dotnet-monitor-egress-azureblobstorage.exe Egress | ||
| RootCommand rootCommand = new RootCommand("Egresses an artifact to Azure storage."); | ||
|
|
||
| private static void Console_CancelKeyPress(object sender, ConsoleCancelEventArgs e) | ||
| { | ||
| CancelSource.Cancel(); | ||
| StdInStream.Close(); | ||
| } | ||
| Command egressCmd = EgressHelper.CreateEgressCommand(provider, configureOptions); | ||
|
|
||
| private static async Task GetStream(Stream outputStream, CancellationToken cancellationToken) | ||
| { | ||
| const int DefaultBufferSize = 0x10000; | ||
| rootCommand.Add(egressCmd); | ||
|
|
||
| StdInStream = Console.OpenStandardInput(); | ||
| await StdInStream.CopyToAsync(outputStream, DefaultBufferSize, cancellationToken); | ||
| return await rootCommand.InvokeAsync(args); | ||
| } | ||
| } | ||
|
|
||
| internal sealed class ExtensionEgressPayload | ||
| { | ||
| public EgressArtifactSettings Settings { get; set; } | ||
| public Dictionary<string, string> Properties { get; set; } | ||
| public Dictionary<string, string> Configuration { get; set; } | ||
| public string ProviderName { get; set; } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
5 changes: 4 additions & 1 deletion
5
...zureBlobStorage/EgressArtifactSettings.cs → ...xtension.Common/EgressArtifactSettings.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
4 changes: 3 additions & 1 deletion
4
...sions/AzureBlobStorage/EgressException.cs → ...oring.Extension.Common/EgressException.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.