Skip to content

Commit

Permalink
Updates to feature/egressExtension Part 1 (#2759)
Browse files Browse the repository at this point in the history
  • Loading branch information
@kkeirstead
kkeirstead authored Oct 25, 2022
1 parent f83894c commit c804040
Show file tree
Hide file tree
Showing 44 changed files with 331 additions and 688 deletions.
121 changes: 0 additions & 121 deletions documentation/schema.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -965,17 +965,6 @@
"type": "object",
"additionalProperties": false,
"properties": {
"AzureBlobStorage": {
"type": [
"null",
"object"
],
"description": "Mapping of Azure blob storage egress provider names to their options.",
"default": {},
"additionalProperties": {
"$ref": "#/definitions/AzureBlobEgressProviderOptions"
}
},
"FileSystem": {
"type": [
"null",
Expand All @@ -1000,116 +989,6 @@
}
}
},
"AzureBlobEgressProviderOptions": {
"type": "object",
"additionalProperties": false,
"required": [
"AccountUri",
"ContainerName"
],
"properties": {
"AccountUri": {
"type": "string",
"description": "The URI of the Azure blob storage account.",
"format": "uri",
"minLength": 1
},
"AccountKey": {
"type": [
"null",
"string"
],
"description": "The account key used to access the Azure blob storage account."
},
"AccountKeyName": {
"type": [
"null",
"string"
],
"description": "The name of the account key used to look up the value from the Egress options Properties map."
},
"SharedAccessSignature": {
"type": [
"null",
"string"
],
"description": "The shared access signature (SAS) used to access the Azure blob and optionally queue storage accounts."
},
"SharedAccessSignatureName": {
"type": [
"null",
"string"
],
"description": "The name of the shared access signature (SAS) used to look up the value from the Egress options Properties map."
},
"ManagedIdentityClientId": {
"type": [
"null",
"string"
],
"description": "Client id of the Managed Identity used for authentication. The identity must have permissions to create containers and write to blob storage."
},
"ContainerName": {
"type": "string",
"description": "The name of the container to which the blob will be egressed. If egressing to the root container, use the \"$root\" sentinel value.",
"minLength": 1
},
"BlobPrefix": {
"type": [
"null",
"string"
],
"description": "The prefix to prepend to the blob name."
},
"CopyBufferSize": {
"type": [
"integer",
"null"
],
"description": "Buffer size used when copying data from an egress callback returning a stream to the egress callback that is provided a stream to which data is written.",
"format": "int32"
},
"QueueName": {
"type": [
"null",
"string"
],
"description": "The name of the queue to which a message will be dispatched upon writing to a blob."
},
"QueueAccountUri": {
"type": [
"null",
"string"
],
"description": "The URI of the Azure queue storage account.",
"format": "uri"
},
"QueueSharedAccessSignature": {
"type": [
"null",
"string"
],
"description": "The shared access signature (SAS) used to access the Azure queue storage account."
},
"QueueSharedAccessSignatureName": {
"type": [
"null",
"string"
],
"description": "The name of the queue shared access signature (SAS) used to look up the value from the Egress options Properties map."
},
"Metadata": {
"type": [
"null",
"object"
],
"description": "A mapping of metadata keys to environment variable names. The values of the environment variables will be added as metadata for egressed artifacts.",
"additionalProperties": {
"type": "string"
}
}
}
},
"FileSystemEgressProviderOptions": {
"type": "object",
"additionalProperties": false,
Expand Down
1 change: 1 addition & 0 deletions dotnet-monitor.sln
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Diagnostics.Monitoring.ExecuteActionApp", "src\Tests\Microsoft.Diagnostics.Monitoring.ExecuteActionApp\Microsoft.Diagnostics.Monitoring.ExecuteActionApp.csproj", "{A5A0CAAB-C200-44D2-BC93-8445C6E748AD}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AzureBlobStorage", "src\Extensions\AzureBlobStorage\AzureBlobStorage.csproj", "{5ED61A7B-F0AA-45F2-9E9A-8972FF7F7278}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Diagnostics.Monitoring.Profiler.UnitTests", "src\Tests\Microsoft.Diagnostics.Monitoring.Profiler.UnitTests\Microsoft.Diagnostics.Monitoring.Profiler.UnitTests.csproj", "{A25AC517-F7C6-43C6-B892-4A447914C42C}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Diagnostics.Monitoring.Profiler.UnitTestApp", "src\Tests\Microsoft.Diagnostics.Monitoring.Profiler.UnitTestApp\Microsoft.Diagnostics.Monitoring.Profiler.UnitTestApp.csproj", "{1CA2284B-A3A0-476A-9A93-A95E665E78BE}"
Expand Down
7 changes: 1 addition & 6 deletions ...zureBlobEgressProvider.AutoFlushStream.cs → ...zureBlobEgressProvider.AutoFlushStream.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,7 @@
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using System;
using System.IO;
using System.Threading;
using System.Threading.Tasks;

namespace Microsoft.Diagnostics.Tools.Monitor.Egress.AzureBlob
namespace Microsoft.Diagnostics.Monitoring.AzureBlobStorage
{
partial class AzureBlobEgressProvider
{
Expand Down
42 changes: 18 additions & 24 deletions ...ress/AzureBlob/AzureBlobEgressProvider.cs → ...rage/AzureBlob/AzureBlobEgressProvider.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,34 +9,28 @@
using Azure.Storage.Blobs.Models;
using Azure.Storage.Blobs.Specialized;
using Azure.Storage.Queues;
using Microsoft.Extensions.Logging;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Net;
using System.Threading;
using System.Threading.Tasks;

namespace Microsoft.Diagnostics.Tools.Monitor.Egress.AzureBlob
namespace Microsoft.Diagnostics.Monitoring.AzureBlobStorage
{
/// <summary>
/// Egress provider for egressing stream data to an Azure blob storage account.
/// </summary>
/// <remarks>
/// Blobs created through this provider will overwrite existing blobs if they have the same blob name.
/// </remarks>
internal partial class AzureBlobEgressProvider :
EgressProvider<AzureBlobEgressProviderOptions>
internal partial class AzureBlobEgressProvider
{
private int BlobStorageBufferSize = 4 * 1024 * 1024;
private readonly ILogger _logger;

public AzureBlobEgressProvider(ILogger<AzureBlobEgressProvider> logger)
: base(logger)
public AzureBlobEgressProvider(ILogger logger)
{
_logger = logger;
}

public override async Task<string> EgressAsync(
public async Task<string> EgressAsync(
string providerType,
string providerName,
AzureBlobEgressProviderOptions options,
Expand All @@ -54,7 +48,7 @@ public override async Task<string> EgressAsync(

BlobClient blobClient = containerClient.GetBlobClient(blobName);

Logger?.EgressProviderInvokeStreamAction(EgressProviderTypes.AzureBlobStorage);
_logger.EgressProviderInvokeStreamAction(Constants.AzureBlobStorageProviderName);
using var stream = await action(token);

// Write blob content, headers, and metadata
Expand All @@ -63,7 +57,7 @@ public override async Task<string> EgressAsync(
await SetBlobClientMetadata(blobClient, artifactSettings, token);

string blobUriString = GetBlobUri(blobClient);
Logger?.EgressProviderSavedStream(EgressProviderTypes.AzureBlobStorage, blobUriString);
_logger.EgressProviderSavedStream(Constants.AzureBlobStorageProviderName, blobUriString);

if (CheckQueueEgressOptions(options))
{
Expand All @@ -86,7 +80,7 @@ public override async Task<string> EgressAsync(
}
}

public override async Task<string> EgressAsync(
public async Task<string> EgressAsync(
string providerType,
string providerName,
AzureBlobEgressProviderOptions options,
Expand Down Expand Up @@ -119,7 +113,7 @@ public override async Task<string> EgressAsync(
//3. After 4Gi of data has been staged, the data will be commited. This can be forced earlier by flushing
//the stream.
// Since we want the data to be readily available, we automatically flush (and therefore commit) every time we fill up the buffer.
Logger?.EgressProviderInvokeStreamAction(EgressProviderTypes.AzureBlobStorage);
_logger.EgressProviderInvokeStreamAction(Constants.AzureBlobStorageProviderName);
await action(flushStream, token);

await flushStream.FlushAsync(token);
Expand All @@ -131,7 +125,7 @@ public override async Task<string> EgressAsync(
await SetBlobClientMetadata(blobClient, artifactSettings, token);

string blobUriString = GetBlobUri(blobClient);
Logger?.EgressProviderSavedStream(EgressProviderTypes.AzureBlobStorage, blobUriString);
_logger.EgressProviderSavedStream(Constants.AzureBlobStorageProviderName, blobUriString);

if (CheckQueueEgressOptions(options))
{
Expand Down Expand Up @@ -166,7 +160,7 @@ public async Task SetBlobClientMetadata(BlobBaseClient blobClient, EgressArtifac
}
else
{
Logger.DuplicateKeyInMetadata(metadataPair.Key);
_logger.DuplicateKeyInMetadata(metadataPair.Key);
}
}

Expand All @@ -177,7 +171,7 @@ public async Task SetBlobClientMetadata(BlobBaseClient blobClient, EgressArtifac
}
catch (Exception ex) when (ex is InvalidOperationException || ex is RequestFailedException)
{
Logger.InvalidMetadata(ex);
_logger.InvalidMetadata(ex);
await blobClient.SetMetadataAsync(artifactSettings.Metadata, cancellationToken: token);
}
}
Expand All @@ -186,7 +180,7 @@ public void AddConfiguredMetadataAsync(AzureBlobEgressProviderOptions options, E
{
if (artifactSettings.EnvBlock.Count == 0)
{
Logger.EnvironmentBlockNotSupported();
_logger.EnvironmentBlockNotSupported();
return;
}

Expand All @@ -198,7 +192,7 @@ public void AddConfiguredMetadataAsync(AzureBlobEgressProviderOptions options, E
}
else
{
Logger.EnvironmentVariableNotFound(metadataPair.Value);
_logger.EnvironmentVariableNotFound(metadataPair.Value);
}
}
}
Expand All @@ -210,7 +204,7 @@ private bool CheckQueueEgressOptions(AzureBlobEgressProviderOptions options)

if (queueNameSet ^ queueAccountUriSet)
{
Logger.QueueOptionsPartiallySet();
_logger.QueueOptionsPartiallySet();
}

return queueNameSet && queueAccountUriSet;
Expand Down Expand Up @@ -249,11 +243,11 @@ private async Task EgressMessageToQueue(string blobName, AzureBlobEgressProvider
}
catch (RequestFailedException ex) when (ex.Status == ((int)HttpStatusCode.NotFound))
{
Logger.QueueDoesNotExist(options.QueueName);
_logger.QueueDoesNotExist(options.QueueName);
}
catch (Exception ex)
{
Logger.WritingMessageToQueueFailed(options.QueueName, ex);
_logger.WritingMessageToQueueFailed(options.QueueName, ex);
}
}

Expand Down
6 changes: 2 additions & 4 deletions ...zureBlobEgressProviderOptions.Validate.cs → ...zureBlobEgressProviderOptions.Validate.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,9 @@
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using Microsoft.Diagnostics.Monitoring.WebApi;
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;

namespace Microsoft.Diagnostics.Tools.Monitor.Egress.AzureBlob
namespace Microsoft.Diagnostics.Monitoring.AzureBlobStorage
{
internal sealed partial class AzureBlobEgressProviderOptions :
IValidatableObject
Expand All @@ -25,7 +23,7 @@ IEnumerable<ValidationResult> IValidatableObject.Validate(ValidationContext vali
results.Add(
new ValidationResult(
string.Format(
OptionsDisplayStrings.ErrorMessage_CredentialsMissing,
Strings.ErrorMessage_CredentialsMissing,
nameof(AccountKey),
nameof(SharedAccessSignature),
nameof(ManagedIdentityClientId))));
Expand Down
47 changes: 47 additions & 0 deletions src/Extensions/AzureBlobStorage/AzureBlobEgressProviderOptions.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using System;
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;

namespace Microsoft.Diagnostics.Monitoring.AzureBlobStorage
{
/// <summary>
/// Egress provider options for Azure blob storage.
/// </summary>
internal sealed partial class AzureBlobEgressProviderOptions
{
[Required]
public Uri AccountUri { get; set; }

public string AccountKey { get; set; }

public string AccountKeyName { get; set; }

public string SharedAccessSignature { get; set; }

public string SharedAccessSignatureName { get; set; }

public string ManagedIdentityClientId { get; set; }

[Required]
public string ContainerName { get; set; }

public string BlobPrefix { get; set; }

public int? CopyBufferSize { get; set; }

public string QueueName { get; set; }

public Uri QueueAccountUri { get; set; }

public string QueueSharedAccessSignature { get; set; }

public string QueueSharedAccessSignatureName { get; set; }

public IDictionary<string, string> Metadata { get; set; }
= new Dictionary<string, string>(0);
}
}
Loading

0 comments on commit c804040

Please sign in to comment.