feat: drift detection for object buckets (#407)

multycloud · Aug 16, 2022 · d3541de · d3541de
1 parent 73a95fc
commit d3541de
Show file tree

Hide file tree

Showing 10 changed files with 98 additions and 49 deletions.
diff --git a/resources/output/object_storage/aws_object_storage.go b/resources/output/object_storage/aws_object_storage.go
@@ -8,19 +8,19 @@ import (
 // aws_s3_bucket
 type AwsS3Bucket struct {
 	*common.AwsResource `hcl:",squash" default:"name=aws_s3_bucket"`
-	Bucket              string `hcl:"bucket"`
+	Bucket              string `hcl:"bucket" json:"bucket"`
 
 	Arn string `json:"arn" hcle:"omitempty"`
 }
 
 type AwsS3BucketVersioning struct {
-	*common.AwsResource     `hcl:",squash" default:"name=aws_s3_bucket_versioning"`
-	BucketId                string                  `hcl:"bucket,expr"`
-	VersioningConfiguration VersioningConfiguration `hcl:"versioning_configuration"`
+	*common.AwsResource     `hcl:",squash" default:"name=aws_s3_bucket_versioning" json:"*_common_._aws_resource"`
+	BucketId                string                    `hcl:"bucket,expr" json:"bucket_id"`
+	VersioningConfiguration []VersioningConfiguration `hcl:"versioning_configuration,blocks" json:"versioning_configuration"`
 }
 
 type VersioningConfiguration struct {
-	Status string `hcl:"status"`
+	Status string `hcl:"status" json:"status"`
 }
 
 func (vpc *AwsS3Bucket) GetBucketId() string {

diff --git a/resources/output/object_storage/azure_object_storage.go b/resources/output/object_storage/azure_object_storage.go
@@ -10,14 +10,14 @@ const AzureResourceName = "azurerm_storage_account"
 // azurerm_storage_account
 type AzureStorageAccount struct {
 	*common.AzResource         `hcl:",squash" default:"name=azurerm_storage_account"`
-	AccountTier                string         `hcl:"account_tier"`
-	AccountReplicationType     string         `hcl:"account_replication_type"`
-	AllowNestedItemsToBePublic bool           `hcl:"allow_nested_items_to_be_public"`
-	BlobProperties             BlobProperties `hcl:"blob_properties"`
+	AccountTier                string           `hcl:"account_tier" json:"account_tier"`
+	AccountReplicationType     string           `hcl:"account_replication_type" json:"account_replication_type"`
+	AllowNestedItemsToBePublic bool             `hcl:"allow_nested_items_to_be_public" json:"allow_nested_items_to_be_public"`
+	BlobProperties             []BlobProperties `hcl:"blob_properties,blocks" json:"blob_properties"`
 }
 
 type BlobProperties struct {
-	VersioningEnabled bool `hcl:"versioning_enabled"`
+	VersioningEnabled bool `hcl:"versioning_enabled" json:"versioning_enabled"`
 }
 
 func (r AzureStorageAccount) GetResourceName() string {

diff --git a/resources/output/object_storage/gcp_storage_bucket.go b/resources/output/object_storage/gcp_storage_bucket.go
@@ -4,11 +4,11 @@ import "github.com/multycloud/multy/resources/common"
 
 type GoogleStorageBucket struct {
 	*common.GcpResource      `hcl:",squash"  default:"name=google_storage_bucket"`
-	UniformBucketLevelAccess bool                            `hcl:"uniform_bucket_level_access"`
-	Versioning               []GoogleStorageBucketVersioning `hcl:"versioning,blocks" hcle:"omitempty"`
-	Location                 string                          `hcl:"location"`
+	UniformBucketLevelAccess bool                            `hcl:"uniform_bucket_level_access" json:"uniform_bucket_level_access"`
+	Versioning               []GoogleStorageBucketVersioning `hcl:"versioning,blocks" hcle:"omitempty" json:"versioning"`
+	Location                 string                          `hcl:"location" json:"location"`
 }
 
 type GoogleStorageBucketVersioning struct {
-	Enabled bool `hcl:"enabled"`
+	Enabled bool `hcl:"enabled" json:"enabled"`
 }
diff --git a/resources/types/aws/object_storage.go b/resources/types/aws/object_storage.go
@@ -37,13 +37,35 @@ func (r AwsObjectStorage) FromState(state *output.TfState) (*resourcespb.ObjectS
 		return out, nil
 	}
 
-	stateResource, err := output.GetParsedById[object_storage.AwsS3Bucket](state, r.ResourceId)
-	if err != nil {
-		return nil, err
+	statuses := map[string]commonpb.ResourceStatus_Status{}
+
+	if stateResource, exists, err := output.MaybeGetParsedById[object_storage.AwsS3Bucket](state, r.ResourceId); exists {
+		if err != nil {
+			return nil, err
+		}
+
+		out.Name = stateResource.Bucket
+		out.AwsOutputs = &resourcespb.ObjectStorageAwsOutputs{S3BucketArn: stateResource.Arn}
+	} else {
+		statuses["aws_s3_bucket"] = commonpb.ResourceStatus_NEEDS_CREATE
 	}
 
-	out.AwsOutputs = &resourcespb.ObjectStorageAwsOutputs{S3BucketArn: stateResource.Arn}
+	if stateResource, exists, err := output.MaybeGetParsedById[object_storage.AwsS3BucketVersioning](state, r.ResourceId); exists {
+		if err != nil {
+			return nil, err
+		}
 
+		out.Versioning = len(stateResource.VersioningConfiguration) > 0 && stateResource.VersioningConfiguration[0].Status == "Enabled"
+	} else {
+		out.Versioning = false
+		if r.Args.Versioning {
+			statuses["aws_s3_bucket_versioning"] = commonpb.ResourceStatus_NEEDS_CREATE
+		}
+	}
+
+	if len(statuses) > 0 {
+		out.CommonParameters.ResourceStatus = &commonpb.ResourceStatus{Statuses: statuses}
+	}
 	return out, nil
 }
 
@@ -63,7 +85,7 @@ func (r AwsObjectStorage) Translate(resources.MultyContext) ([]output.TfBlock, e
 				TerraformResource: output.TerraformResource{ResourceId: r.ResourceId},
 			},
 			BucketId:                s3Bucket.GetBucketId(),
-			VersioningConfiguration: object_storage.VersioningConfiguration{Status: "Enabled"},
+			VersioningConfiguration: []object_storage.VersioningConfiguration{{Status: "Enabled"}},
 		})
 	}
 	return awsResources, nil

diff --git a/resources/types/azure/object_storage.go b/resources/types/azure/object_storage.go
@@ -39,24 +39,41 @@ func (r AzureObjectStorage) FromState(state *output.TfState) (*resourcespb.Objec
 		return out, nil
 	}
 
-	stateResource, err := output.GetParsedById[object_storage.AzureStorageAccount](state, r.ResourceId)
-	if err != nil {
-		return nil, err
+	statuses := map[string]commonpb.ResourceStatus_Status{}
+	out.AzureOutputs = &resourcespb.ObjectStorageAzureOutputs{}
+
+	if stateResource, exists, err := output.MaybeGetParsedById[object_storage.AzureStorageAccount](state, r.ResourceId); exists {
+		if err != nil {
+			return nil, err
+		}
+		out.AzureOutputs.StorageAccountId = stateResource.ResourceId
+		out.Name = stateResource.Name
+		out.Versioning = len(stateResource.BlobProperties) > 0 && stateResource.BlobProperties[0].VersioningEnabled
+	} else {
+		statuses["azure_storage_account"] = commonpb.ResourceStatus_NEEDS_CREATE
 	}
-	out.AzureOutputs = &resourcespb.ObjectStorageAzureOutputs{StorageAccountId: stateResource.ResourceId}
 
-	privContainer, err := output.GetParsedById[object_storage_object.AzureStorageContainer](state, r.getPrivateContainerId())
-	if err != nil {
-		return nil, err
+	if privContainer, exists, err := output.MaybeGetParsedById[object_storage_object.AzureStorageContainer](state, r.getPrivateContainerId()); exists {
+		if err != nil {
+			return nil, err
+		}
+		out.AzureOutputs.PrivateStorageContainerId = privContainer.ResourceId
+	} else {
+		statuses["azure_private_storage_container"] = commonpb.ResourceStatus_NEEDS_CREATE
 	}
-	out.AzureOutputs.PrivateStorageContainerId = privContainer.ResourceId
 
-	publicContainer, err := output.GetParsedById[object_storage_object.AzureStorageContainer](state, r.getPublicContainerId())
-	if err != nil {
-		return nil, err
+	if publicContainer, exists, err := output.MaybeGetParsedById[object_storage_object.AzureStorageContainer](state, r.getPublicContainerId()); exists {
+		if err != nil {
+			return nil, err
+		}
+		out.AzureOutputs.PublicStorageContainerId = publicContainer.ResourceId
+	} else {
+		statuses["azure_public_storage_container"] = commonpb.ResourceStatus_NEEDS_CREATE
 	}
-	out.AzureOutputs.PublicStorageContainerId = publicContainer.ResourceId
 
+	if len(statuses) > 0 {
+		out.CommonParameters.ResourceStatus = &commonpb.ResourceStatus{Statuses: statuses}
+	}
 	return out, nil
 }
 
@@ -65,15 +82,15 @@ func (r AzureObjectStorage) Translate(resources.MultyContext) ([]output.TfBlock,
 
 	storageAccount := object_storage.AzureStorageAccount{
 		AzResource: common.NewAzResource(
-			r.ResourceId, common.RemoveSpecialChars(r.Args.Name), rgName,
+			r.ResourceId, r.Args.Name, rgName,
 			r.GetCloudSpecificLocation(),
 		),
 		AccountTier:                "Standard",
 		AccountReplicationType:     "GZRS",
 		AllowNestedItemsToBePublic: true,
-		BlobProperties: object_storage.BlobProperties{
+		BlobProperties: []object_storage.BlobProperties{{
 			VersioningEnabled: r.Args.Versioning,
-		},
+		}},
 	}
 
 	return []output.TfBlock{

diff --git a/resources/types/gcp/object_storage.go b/resources/types/gcp/object_storage.go
@@ -37,13 +37,23 @@ func (r GcpObjectStorage) FromState(state *output.TfState) (*resourcespb.ObjectS
 		return out, nil
 	}
 
-	stateResource, err := output.GetParsedById[object_storage.GoogleStorageBucket](state, r.ResourceId)
-	if err != nil {
-		return nil, err
-	}
+	statuses := map[string]commonpb.ResourceStatus_Status{}
+
+	if stateResource, exists, err := output.MaybeGetParsedById[object_storage.GoogleStorageBucket](state, r.ResourceId); exists {
+		if err != nil {
+			return nil, err
+		}
 
-	out.GcpOutputs = &resourcespb.ObjectStorageGcpOutputs{StorageBucketId: stateResource.SelfLink}
+		out.GcpOutputs = &resourcespb.ObjectStorageGcpOutputs{StorageBucketId: stateResource.SelfLink}
+		out.Name = stateResource.Name
+		out.Versioning = len(stateResource.Versioning) > 0 && stateResource.Versioning[0].Enabled
+	} else {
+		statuses["gcp_storage_bucket"] = commonpb.ResourceStatus_NEEDS_CREATE
+	}
 
+	if len(statuses) > 0 {
+		out.CommonParameters.ResourceStatus = &commonpb.ResourceStatus{Statuses: statuses}
+	}
 	return out, nil
 }
 

diff --git a/test/_configs/object_storage/object_storage/config.textproto b/test/_configs/object_storage/object_storage/config.textproto
@@ -8,7 +8,7 @@ resources:  {
           location:  EU_WEST_1
           cloud_provider:  AWS
         }
-        name:  "test-storage-12384761234"
+        name:  "teststorage12384761234"
         versioning:  true
       }
     }
@@ -24,7 +24,7 @@ resources:  {
           location:  EU_WEST_1
           cloud_provider:  AZURE
         }
-        name:  "test-storage-12384761234"
+        name:  "teststorage12384761234"
         versioning:  true
       }
     }
@@ -40,7 +40,7 @@ resources:  {
           location:  EU_WEST_1
           cloud_provider:  GCP
         }
-        name:  "test-storage-12384761234"
+        name:  "teststorage12384761234"
         versioning:  true
         gcp_override: {
           project: "multy-project"

diff --git a/test/_configs/object_storage/object_storage/main.tf b/test/_configs/object_storage/object_storage/main.tf
@@ -1,6 +1,6 @@
 resource "aws_s3_bucket" "obj_storage_aws" {
   provider = "aws.eu-west-1"
-  bucket   = "test-storage-12384761234"
+  bucket   = "teststorage12384761234"
 }
 resource "aws_s3_bucket_versioning" "obj_storage_aws" {
   provider = "aws.eu-west-1"
@@ -45,7 +45,7 @@ provider "azurerm" {
   }
 }
 resource "google_storage_bucket" "object_storage_gcp" {
-  name                        = "test-storage-12384761234"
+  name                        = "teststorage12384761234"
   project                     = "multy-project"
   uniform_bucket_level_access = false
   versioning {

diff --git a/test/_configs/object_storage_object/object_storage_object/config.textproto b/test/_configs/object_storage_object/object_storage_object/config.textproto
@@ -8,7 +8,7 @@ resources:  {
           location:  EU_WEST_1
           cloud_provider:  AZURE
         }
-        name:  "test-storage-9999919"
+        name:  "teststorage9999919"
       }
     }
   }
@@ -23,7 +23,7 @@ resources:  {
           location:  EU_WEST_1
           cloud_provider:  AWS
         }
-        name:  "test-storage-9999919"
+        name:  "teststorage9999919"
       }
     }
   }
@@ -106,7 +106,7 @@ resources:  {
           location:  EU_WEST_1
           cloud_provider:  GCP
         }
-        name:  "test-storage-9999919"
+        name:  "teststorage9999919"
         gcp_override: {
           project: "multy-project"
         }

diff --git a/test/_configs/object_storage_object/object_storage_object/main.tf b/test/_configs/object_storage_object/object_storage_object/main.tf
@@ -26,7 +26,7 @@ resource "aws_s3_object" "file2_private_aws" {
 }
 resource "aws_s3_bucket" "obj_storage_aws" {
   provider = "aws.eu-west-1"
-  bucket   = "test-storage-9999919"
+  bucket   = "teststorage9999919"
 }
 resource "azurerm_storage_blob" "file1_public_azure" {
   name                   = "index.html"
@@ -100,7 +100,7 @@ resource "google_storage_bucket_object" "file2_private_GCP" {
   provider     = "google.europe-west1"
 }
 resource "google_storage_bucket" "obj_storage_GCP" {
-  name                        = "test-storage-9999919"
+  name                        = "teststorage9999919"
   project                     = "multy-project"
   uniform_bucket_level_access = false
   location                    = "europe-west1"