From d3541deb8bf9a5e6a60faa1e2284c31a6a786b5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gon=C3=A7alo=20Rodrigues?= Date: Tue, 16 Aug 2022 11:37:37 +0200 Subject: [PATCH] feat: drift detection for object buckets (#407) --- .../object_storage/aws_object_storage.go | 10 ++-- .../object_storage/azure_object_storage.go | 10 ++-- .../object_storage/gcp_storage_bucket.go | 8 ++-- resources/types/aws/object_storage.go | 32 +++++++++++-- resources/types/azure/object_storage.go | 47 +++++++++++++------ resources/types/gcp/object_storage.go | 20 ++++++-- .../object_storage/config.textproto | 6 +-- .../object_storage/object_storage/main.tf | 4 +- .../object_storage_object/config.textproto | 6 +-- .../object_storage_object/main.tf | 4 +- 10 files changed, 98 insertions(+), 49 deletions(-) diff --git a/resources/output/object_storage/aws_object_storage.go b/resources/output/object_storage/aws_object_storage.go index 8396cc1a..a75f4721 100644 --- a/resources/output/object_storage/aws_object_storage.go +++ b/resources/output/object_storage/aws_object_storage.go @@ -8,19 +8,19 @@ import ( // aws_s3_bucket type AwsS3Bucket struct { *common.AwsResource `hcl:",squash" default:"name=aws_s3_bucket"` - Bucket string `hcl:"bucket"` + Bucket string `hcl:"bucket" json:"bucket"` Arn string `json:"arn" hcle:"omitempty"` } type AwsS3BucketVersioning struct { - *common.AwsResource `hcl:",squash" default:"name=aws_s3_bucket_versioning"` - BucketId string `hcl:"bucket,expr"` - VersioningConfiguration VersioningConfiguration `hcl:"versioning_configuration"` + *common.AwsResource `hcl:",squash" default:"name=aws_s3_bucket_versioning" json:"*_common_._aws_resource"` + BucketId string `hcl:"bucket,expr" json:"bucket_id"` + VersioningConfiguration []VersioningConfiguration `hcl:"versioning_configuration,blocks" json:"versioning_configuration"` } type VersioningConfiguration struct { - Status string `hcl:"status"` + Status string `hcl:"status" json:"status"` } func (vpc *AwsS3Bucket) GetBucketId() string { diff --git a/resources/output/object_storage/azure_object_storage.go b/resources/output/object_storage/azure_object_storage.go index e4a393a2..1896ba0c 100644 --- a/resources/output/object_storage/azure_object_storage.go +++ b/resources/output/object_storage/azure_object_storage.go @@ -10,14 +10,14 @@ const AzureResourceName = "azurerm_storage_account" // azurerm_storage_account type AzureStorageAccount struct { *common.AzResource `hcl:",squash" default:"name=azurerm_storage_account"` - AccountTier string `hcl:"account_tier"` - AccountReplicationType string `hcl:"account_replication_type"` - AllowNestedItemsToBePublic bool `hcl:"allow_nested_items_to_be_public"` - BlobProperties BlobProperties `hcl:"blob_properties"` + AccountTier string `hcl:"account_tier" json:"account_tier"` + AccountReplicationType string `hcl:"account_replication_type" json:"account_replication_type"` + AllowNestedItemsToBePublic bool `hcl:"allow_nested_items_to_be_public" json:"allow_nested_items_to_be_public"` + BlobProperties []BlobProperties `hcl:"blob_properties,blocks" json:"blob_properties"` } type BlobProperties struct { - VersioningEnabled bool `hcl:"versioning_enabled"` + VersioningEnabled bool `hcl:"versioning_enabled" json:"versioning_enabled"` } func (r AzureStorageAccount) GetResourceName() string { diff --git a/resources/output/object_storage/gcp_storage_bucket.go b/resources/output/object_storage/gcp_storage_bucket.go index f889f1fd..ae1e0e92 100644 --- a/resources/output/object_storage/gcp_storage_bucket.go +++ b/resources/output/object_storage/gcp_storage_bucket.go @@ -4,11 +4,11 @@ import "github.com/multycloud/multy/resources/common" type GoogleStorageBucket struct { *common.GcpResource `hcl:",squash" default:"name=google_storage_bucket"` - UniformBucketLevelAccess bool `hcl:"uniform_bucket_level_access"` - Versioning []GoogleStorageBucketVersioning `hcl:"versioning,blocks" hcle:"omitempty"` - Location string `hcl:"location"` + UniformBucketLevelAccess bool `hcl:"uniform_bucket_level_access" json:"uniform_bucket_level_access"` + Versioning []GoogleStorageBucketVersioning `hcl:"versioning,blocks" hcle:"omitempty" json:"versioning"` + Location string `hcl:"location" json:"location"` } type GoogleStorageBucketVersioning struct { - Enabled bool `hcl:"enabled"` + Enabled bool `hcl:"enabled" json:"enabled"` } diff --git a/resources/types/aws/object_storage.go b/resources/types/aws/object_storage.go index 1386cee8..27e51de0 100644 --- a/resources/types/aws/object_storage.go +++ b/resources/types/aws/object_storage.go @@ -37,13 +37,35 @@ func (r AwsObjectStorage) FromState(state *output.TfState) (*resourcespb.ObjectS return out, nil } - stateResource, err := output.GetParsedById[object_storage.AwsS3Bucket](state, r.ResourceId) - if err != nil { - return nil, err + statuses := map[string]commonpb.ResourceStatus_Status{} + + if stateResource, exists, err := output.MaybeGetParsedById[object_storage.AwsS3Bucket](state, r.ResourceId); exists { + if err != nil { + return nil, err + } + + out.Name = stateResource.Bucket + out.AwsOutputs = &resourcespb.ObjectStorageAwsOutputs{S3BucketArn: stateResource.Arn} + } else { + statuses["aws_s3_bucket"] = commonpb.ResourceStatus_NEEDS_CREATE } - out.AwsOutputs = &resourcespb.ObjectStorageAwsOutputs{S3BucketArn: stateResource.Arn} + if stateResource, exists, err := output.MaybeGetParsedById[object_storage.AwsS3BucketVersioning](state, r.ResourceId); exists { + if err != nil { + return nil, err + } + out.Versioning = len(stateResource.VersioningConfiguration) > 0 && stateResource.VersioningConfiguration[0].Status == "Enabled" + } else { + out.Versioning = false + if r.Args.Versioning { + statuses["aws_s3_bucket_versioning"] = commonpb.ResourceStatus_NEEDS_CREATE + } + } + + if len(statuses) > 0 { + out.CommonParameters.ResourceStatus = &commonpb.ResourceStatus{Statuses: statuses} + } return out, nil } @@ -63,7 +85,7 @@ func (r AwsObjectStorage) Translate(resources.MultyContext) ([]output.TfBlock, e TerraformResource: output.TerraformResource{ResourceId: r.ResourceId}, }, BucketId: s3Bucket.GetBucketId(), - VersioningConfiguration: object_storage.VersioningConfiguration{Status: "Enabled"}, + VersioningConfiguration: []object_storage.VersioningConfiguration{{Status: "Enabled"}}, }) } return awsResources, nil diff --git a/resources/types/azure/object_storage.go b/resources/types/azure/object_storage.go index cd239ae3..b86b668a 100644 --- a/resources/types/azure/object_storage.go +++ b/resources/types/azure/object_storage.go @@ -39,24 +39,41 @@ func (r AzureObjectStorage) FromState(state *output.TfState) (*resourcespb.Objec return out, nil } - stateResource, err := output.GetParsedById[object_storage.AzureStorageAccount](state, r.ResourceId) - if err != nil { - return nil, err + statuses := map[string]commonpb.ResourceStatus_Status{} + out.AzureOutputs = &resourcespb.ObjectStorageAzureOutputs{} + + if stateResource, exists, err := output.MaybeGetParsedById[object_storage.AzureStorageAccount](state, r.ResourceId); exists { + if err != nil { + return nil, err + } + out.AzureOutputs.StorageAccountId = stateResource.ResourceId + out.Name = stateResource.Name + out.Versioning = len(stateResource.BlobProperties) > 0 && stateResource.BlobProperties[0].VersioningEnabled + } else { + statuses["azure_storage_account"] = commonpb.ResourceStatus_NEEDS_CREATE } - out.AzureOutputs = &resourcespb.ObjectStorageAzureOutputs{StorageAccountId: stateResource.ResourceId} - privContainer, err := output.GetParsedById[object_storage_object.AzureStorageContainer](state, r.getPrivateContainerId()) - if err != nil { - return nil, err + if privContainer, exists, err := output.MaybeGetParsedById[object_storage_object.AzureStorageContainer](state, r.getPrivateContainerId()); exists { + if err != nil { + return nil, err + } + out.AzureOutputs.PrivateStorageContainerId = privContainer.ResourceId + } else { + statuses["azure_private_storage_container"] = commonpb.ResourceStatus_NEEDS_CREATE } - out.AzureOutputs.PrivateStorageContainerId = privContainer.ResourceId - publicContainer, err := output.GetParsedById[object_storage_object.AzureStorageContainer](state, r.getPublicContainerId()) - if err != nil { - return nil, err + if publicContainer, exists, err := output.MaybeGetParsedById[object_storage_object.AzureStorageContainer](state, r.getPublicContainerId()); exists { + if err != nil { + return nil, err + } + out.AzureOutputs.PublicStorageContainerId = publicContainer.ResourceId + } else { + statuses["azure_public_storage_container"] = commonpb.ResourceStatus_NEEDS_CREATE } - out.AzureOutputs.PublicStorageContainerId = publicContainer.ResourceId + if len(statuses) > 0 { + out.CommonParameters.ResourceStatus = &commonpb.ResourceStatus{Statuses: statuses} + } return out, nil } @@ -65,15 +82,15 @@ func (r AzureObjectStorage) Translate(resources.MultyContext) ([]output.TfBlock, storageAccount := object_storage.AzureStorageAccount{ AzResource: common.NewAzResource( - r.ResourceId, common.RemoveSpecialChars(r.Args.Name), rgName, + r.ResourceId, r.Args.Name, rgName, r.GetCloudSpecificLocation(), ), AccountTier: "Standard", AccountReplicationType: "GZRS", AllowNestedItemsToBePublic: true, - BlobProperties: object_storage.BlobProperties{ + BlobProperties: []object_storage.BlobProperties{{ VersioningEnabled: r.Args.Versioning, - }, + }}, } return []output.TfBlock{ diff --git a/resources/types/gcp/object_storage.go b/resources/types/gcp/object_storage.go index c92864dc..1eb94d00 100644 --- a/resources/types/gcp/object_storage.go +++ b/resources/types/gcp/object_storage.go @@ -37,13 +37,23 @@ func (r GcpObjectStorage) FromState(state *output.TfState) (*resourcespb.ObjectS return out, nil } - stateResource, err := output.GetParsedById[object_storage.GoogleStorageBucket](state, r.ResourceId) - if err != nil { - return nil, err - } + statuses := map[string]commonpb.ResourceStatus_Status{} + + if stateResource, exists, err := output.MaybeGetParsedById[object_storage.GoogleStorageBucket](state, r.ResourceId); exists { + if err != nil { + return nil, err + } - out.GcpOutputs = &resourcespb.ObjectStorageGcpOutputs{StorageBucketId: stateResource.SelfLink} + out.GcpOutputs = &resourcespb.ObjectStorageGcpOutputs{StorageBucketId: stateResource.SelfLink} + out.Name = stateResource.Name + out.Versioning = len(stateResource.Versioning) > 0 && stateResource.Versioning[0].Enabled + } else { + statuses["gcp_storage_bucket"] = commonpb.ResourceStatus_NEEDS_CREATE + } + if len(statuses) > 0 { + out.CommonParameters.ResourceStatus = &commonpb.ResourceStatus{Statuses: statuses} + } return out, nil } diff --git a/test/_configs/object_storage/object_storage/config.textproto b/test/_configs/object_storage/object_storage/config.textproto index 4b0bd26f..998628d7 100755 --- a/test/_configs/object_storage/object_storage/config.textproto +++ b/test/_configs/object_storage/object_storage/config.textproto @@ -8,7 +8,7 @@ resources: { location: EU_WEST_1 cloud_provider: AWS } - name: "test-storage-12384761234" + name: "teststorage12384761234" versioning: true } } @@ -24,7 +24,7 @@ resources: { location: EU_WEST_1 cloud_provider: AZURE } - name: "test-storage-12384761234" + name: "teststorage12384761234" versioning: true } } @@ -40,7 +40,7 @@ resources: { location: EU_WEST_1 cloud_provider: GCP } - name: "test-storage-12384761234" + name: "teststorage12384761234" versioning: true gcp_override: { project: "multy-project" diff --git a/test/_configs/object_storage/object_storage/main.tf b/test/_configs/object_storage/object_storage/main.tf index 5f08217c..433688cd 100644 --- a/test/_configs/object_storage/object_storage/main.tf +++ b/test/_configs/object_storage/object_storage/main.tf @@ -1,6 +1,6 @@ resource "aws_s3_bucket" "obj_storage_aws" { provider = "aws.eu-west-1" - bucket = "test-storage-12384761234" + bucket = "teststorage12384761234" } resource "aws_s3_bucket_versioning" "obj_storage_aws" { provider = "aws.eu-west-1" @@ -45,7 +45,7 @@ provider "azurerm" { } } resource "google_storage_bucket" "object_storage_gcp" { - name = "test-storage-12384761234" + name = "teststorage12384761234" project = "multy-project" uniform_bucket_level_access = false versioning { diff --git a/test/_configs/object_storage_object/object_storage_object/config.textproto b/test/_configs/object_storage_object/object_storage_object/config.textproto index 2a0537ca..422d3cab 100755 --- a/test/_configs/object_storage_object/object_storage_object/config.textproto +++ b/test/_configs/object_storage_object/object_storage_object/config.textproto @@ -8,7 +8,7 @@ resources: { location: EU_WEST_1 cloud_provider: AZURE } - name: "test-storage-9999919" + name: "teststorage9999919" } } } @@ -23,7 +23,7 @@ resources: { location: EU_WEST_1 cloud_provider: AWS } - name: "test-storage-9999919" + name: "teststorage9999919" } } } @@ -106,7 +106,7 @@ resources: { location: EU_WEST_1 cloud_provider: GCP } - name: "test-storage-9999919" + name: "teststorage9999919" gcp_override: { project: "multy-project" } diff --git a/test/_configs/object_storage_object/object_storage_object/main.tf b/test/_configs/object_storage_object/object_storage_object/main.tf index 5a3ca3ba..fd714e16 100644 --- a/test/_configs/object_storage_object/object_storage_object/main.tf +++ b/test/_configs/object_storage_object/object_storage_object/main.tf @@ -26,7 +26,7 @@ resource "aws_s3_object" "file2_private_aws" { } resource "aws_s3_bucket" "obj_storage_aws" { provider = "aws.eu-west-1" - bucket = "test-storage-9999919" + bucket = "teststorage9999919" } resource "azurerm_storage_blob" "file1_public_azure" { name = "index.html" @@ -100,7 +100,7 @@ resource "google_storage_bucket_object" "file2_private_GCP" { provider = "google.europe-west1" } resource "google_storage_bucket" "obj_storage_GCP" { - name = "test-storage-9999919" + name = "teststorage9999919" project = "multy-project" uniform_bucket_level_access = false location = "europe-west1"