Add additional options for AWS credentials and rework client creation…

… according to provided gist

documentation/configuration.md

@@ -855,9 +855,11 @@ The Queue Message's payload will be the blob name (`<BlobPrefix>/<ArtifactName>`
 |---|---|---|---|
 | endpoint | string | true | An optional endpoint of S3 storage service. Can be left empty in case of using AWS. |
 | bucketName | string | true | The name of the s3 Bucket to which the blob will be egressed |
-| accountKeyName | string | true | The user credential for accessing the s3 service |
-| secretsFile | string | false | Path to a file on disk which holds the user's password for accessing the s3 storage. If not provided the `password` property must be set. |
-| accountKey | string | false | The user's password for accessing the s3 storage. If not provided the `SecretsFile` must be specified. |
+| accessKeyId | string | true | The AWS AccessKeyId for IAM user to login |
+| secretAccessKeyFile | string | false | Path to a file on disk which holds the user's password for accessing the s3 storage. If not provided the `password` property must be set. |
+| secretAccessKey | string | false | The AWS SecretAccessKey associated AccessKeyId for IAM user to login. |
+| awsProfileName | string | false | The AWS profile name to be used for login. |
+| awsProfilePath | string | false | The AWS profile path, if profile details not stored in default path. |
 | generatePresSignedUrl | bool | false | A boolean flag to control if either a pre-signed url is returned after successful upload or only the name of bucket and the artifacts S3 object key. |
 | preSignedUrlExpiryInMinutes | int | true | The number of minutes the generated pre-signed url should be accessible. |
 | copyBufferSize | int | false | The buffer size to use when copying data from the original artifact to the blob stream. There is a minimum size of 5 MB which is set when the given value is lower.|
@@ -874,8 +876,8 @@ The Queue Message's payload will be the blob name (`<BlobPrefix>/<ArtifactName>`
               "monitorS3Blob": {
                   "endpoint": "http://localhost:9000",
                   "bucketName": "myS3Bucket",
-                  "accountKeyName": "minioUser",
-                  "secretsFile": "C:\\Temp\\s3secret",
+                  "accessKeyId": "minioUser",
+                  "secretAccessKeyFile": "C:\\Temp\\s3secret",
                   "regionName": "us-east-1",
                   "generatePresSignedUrl" : true,
                   "preSignedUrlExpiryInMinutes" : 15,
@@ -897,8 +899,8 @@ The Queue Message's payload will be the blob name (`<BlobPrefix>/<ArtifactName>`
               "monitorS3Blob": {
                   "endpoint": "http://localhost:9000",
                   "bucketName": "myS3Bucket",
-                  "accountKeyName": "minioUser",
-                  "accountKey": "mySecretPassword",
+                  "accessKeyId": "minioUser",
+                  "secretAccessKey": "mySecretPassword",
                   "regionName": "us-east-1",
                   "generatePresSignedUrl" : true,
                   "preSignedUrlExpiryInMinutes" : 15,

src/Microsoft.Diagnostics.Monitoring.Options/OptionsDisplayStrings.resx

@@ -720,21 +720,29 @@
     <value>The endpoint of S3 to connect to. This is optional in case of using AWS storage.</value>
     <comment>The description provided for the Endpoint parameter on S3StorageEgressProviderOptions.</comment>
   </data>
-  <data name="DisplayAttributeDescription_S3StorageEgressProviderOptions_AccountKey" xml:space="preserve">
-    <value>The path to secrets file containing the password to login on S3</value>
-    <comment>The description provided for the AccountKey parameter on S3StorageEgressProviderOptions.</comment>
+  <data name="DisplayAttributeDescription_S3StorageEgressProviderOptions_AWSProfileName" xml:space="preserve">
+    <value>The AWS profile name to be used for login</value>
+    <comment>The description provided for the AWSProfileName parameter on S3StorageEgressProviderOptions.</comment>
+  </data>
+  <data name="DisplayAttributeDescription_S3StorageEgressProviderOptions_AWSProfilePath" xml:space="preserve">
+    <value>The AWS profile path, if profile details not stored in default path</value>
+    <comment>The description provided for the AWSProfilePath parameter on S3StorageEgressProviderOptions.</comment>
   </data>
   <data name="DisplayAttributeDescription_S3StorageEgressProviderOptions_RegionName" xml:space="preserve">
     <value>The name of the S3 region</value>
     <comment>The description provided for the RegionName parameter on S3StorageEgressProviderOptions.</comment>
   </data>
-  <data name="DisplayAttributeDescription_S3StorageEgressProviderOptions_SecretsFile" xml:space="preserve">
+  <data name="DisplayAttributeDescription_S3StorageEgressProviderOptions_SecretsAccessKeyFile" xml:space="preserve">
     <value>The path to secrets file to get the value for accountKey to connect to S3 storage</value>
-    <comment>The description provided for the SecretsFile parameter on S3StorageEgressProviderOptions.</comment>
+    <comment>The description provided for the SecretsAccessKeyFile parameter on S3StorageEgressProviderOptions.</comment>
+  </data>
+  <data name="DisplayAttributeDescription_S3StorageEgressProviderOptions_AccessKeyId" xml:space="preserve">
+    <value>The AWS AccessKeyId for IAM user to login</value>
+    <comment>The description provided for the AccessKeyId parameter on S3StorageEgressProviderOptions.</comment>
   </data>
-  <data name="DisplayAttributeDescription_S3StorageEgressProviderOptions_AccountKeyName" xml:space="preserve">
-    <value>The user-name used to login</value>
-    <comment>The description provided for the AccountKeyName parameter on S3StorageEgressProviderOptions.</comment>
+  <data name="DisplayAttributeDescription_S3StorageEgressProviderOptions_SecretAccessKey" xml:space="preserve">
+    <value>The AWS SecretAccessKey associated AccessKeyId for IAM user to login</value>
+    <comment>The description provided for the SecretAccessKey parameter on S3StorageEgressProviderOptions.</comment>
   </data>
   <data name="DisplayAttributeDescription_S3StorageEgressProviderOptions_GeneratePreSignedUrl" xml:space="preserve">
     <value>A boolean flag indicates if the return value of egress provider should be a pre-signed URL or only the bucket name and object id of uploaded entry.</value>

src/Microsoft.Diagnostics.Monitoring.Options/S3StorageEgressProviderOptions.cs

@@ -31,18 +31,27 @@ internal sealed class S3StorageEgressProviderOptions :
 
         [Display(
             ResourceType = typeof(OptionsDisplayStrings),
-            Description = nameof(OptionsDisplayStrings.DisplayAttributeDescription_S3StorageEgressProviderOptions_AccountKeyName))]
-        public string AccountKeyName { get; set; }
+            Description = nameof(OptionsDisplayStrings.DisplayAttributeDescription_S3StorageEgressProviderOptions_AccessKeyId))]
+        public string AccessKeyId { get; set; }
 
         [Display(
             ResourceType = typeof(OptionsDisplayStrings),
-            Description = nameof(OptionsDisplayStrings.DisplayAttributeDescription_S3StorageEgressProviderOptions_SecretsFile))]
-        public string SecretsFile { get; set; }
+            Description = nameof(OptionsDisplayStrings.DisplayAttributeDescription_S3StorageEgressProviderOptions_SecretsAccessKeyFile))]
+        public string SecretsAccessKeyFile { get; set; }
 
         [Display(
             ResourceType = typeof(OptionsDisplayStrings),
-            Description = nameof(OptionsDisplayStrings.DisplayAttributeDescription_S3StorageEgressProviderOptions_AccountKey))]
-        public string AccountKey { get; set; }
+            Description = nameof(OptionsDisplayStrings.DisplayAttributeDescription_S3StorageEgressProviderOptions_SecretAccessKey))]
+        public string SecretAccessKey { get; set; }
+
+        [Display(
+            ResourceType = typeof(OptionsDisplayStrings),
+            Description = nameof(OptionsDisplayStrings.DisplayAttributeDescription_S3StorageEgressProviderOptions_AWSProfileName))]
+        public string AwsProfileName { get; set; }
+        [Display(
+            ResourceType = typeof(OptionsDisplayStrings),
+            Description = nameof(OptionsDisplayStrings.DisplayAttributeDescription_S3StorageEgressProviderOptions_AWSProfilePath))]
+        public string AwsProfileFilePath { get; set; }
 
         [Display(
             ResourceType = typeof(OptionsDisplayStrings),
@@ -62,7 +71,7 @@ internal sealed class S3StorageEgressProviderOptions :
 
         public IEnumerable<ValidationResult> Validate(ValidationContext validationContext)
         {
-            if (string.IsNullOrEmpty(SecretsFile) && string.IsNullOrEmpty(AccountKey))
+            if (string.IsNullOrEmpty(SecretsAccessKeyFile) && string.IsNullOrEmpty(SecretAccessKey))
                 yield return new ValidationResult(OptionsDisplayStrings.ErrorMessage_EgressS3FailedMissingSecrets);
 
             if (string.IsNullOrEmpty(BucketName))

src/Tools/dotnet-monitor/Egress/S3/S3StorageEgressProvider.cs

@@ -8,6 +8,7 @@
 using System.Threading;
 using System.Threading.Tasks;
 using Amazon.Runtime;
+using Amazon.Runtime.CredentialManagement;
 using Amazon.S3;
 using Amazon.S3.Model;
 using Amazon.S3.Transfer;
@@ -130,23 +131,41 @@ private string GetResourceId(IAmazonS3 client, S3StorageEgressProviderOptions op
 
         private static async Task<IAmazonS3> CreateClientAsync(S3StorageEgressProviderOptions options, CancellationToken cancellationToken)
         {
-            string accountKey;
-            if (!string.IsNullOrEmpty(options.SecretsFile) && File.Exists(options.SecretsFile))
-                accountKey = await WrapException(async () => (await File.ReadAllTextAsync(options.SecretsFile, cancellationToken)).Trim());
-            else 
-                accountKey = options.AccountKey;
-
-            var credentials = new BasicAWSCredentials(options.AccountKeyName, accountKey);
-            var config = new AmazonS3Config
-            {
-                ServiceURL = options.Endpoint,
-                AuthenticationRegion = options.RegionName,
-                ForcePathStyle = true
-            };
-            var client = new AmazonS3Client(credentials, config);
+            AWSCredentials awsCredentials = null;
+            AmazonS3Config configuration = new();
+            // use the specified access key and the secrets taken from configuration or a local file
+            if (!string.IsNullOrEmpty(options.AccessKeyId) && (!string.IsNullOrEmpty(options.SecretAccessKey) || !string.IsNullOrEmpty(options.SecretsAccessKeyFile)))
+            {
+                string secretAccessKeyId;
+                if (!string.IsNullOrEmpty(options.SecretsAccessKeyFile) && File.Exists(options.SecretsAccessKeyFile))
+                    secretAccessKeyId = await WrapException(async () => (await File.ReadAllTextAsync(options.SecretsAccessKeyFile, cancellationToken)).Trim());
+                else
+                    secretAccessKeyId = options.SecretAccessKey;
+                awsCredentials = new BasicAWSCredentials(options.AccessKeyId, secretAccessKeyId);
+
+                configuration.ForcePathStyle = true;
+                configuration.ServiceURL = options.Endpoint;
+                configuration.AuthenticationRegion = options.RegionName;
+            }
+            // use configured AWS profile
+            else if (!string.IsNullOrEmpty(options.AwsProfileName))
+            {
+                CredentialProfileStoreChain chain = !string.IsNullOrEmpty(options.AwsProfileFilePath)
+                    ? new CredentialProfileStoreChain(options.AwsProfileFilePath)
+                    : new CredentialProfileStoreChain();
+
+                if (!chain.TryGetAWSCredentials(options.AwsProfileName, out awsCredentials))
+                    throw new AmazonClientException("AWS profile not found");
+            }
+
+            awsCredentials ??= FallbackCredentialsFactory.GetCredentials();
+
+            if (awsCredentials == null)
+                throw new AmazonClientException("Failed to find AWS Credentials for constructing AWS service client");
 
-            await VerifyBucketExistsAsync(client, options.BucketName, cancellationToken);
-            return client;
+            AmazonS3Client s3Client = new(awsCredentials, configuration);
+            await VerifyBucketExistsAsync(s3Client, options.BucketName, cancellationToken);
+            return s3Client;
         }
 
         private static async Task<string> InitMultiPartUploadAsync(IAmazonS3 client, string bucketName, EgressArtifactSettings artifactSettings, CancellationToken cancellationToken)