feat: drift detection for static objects

resources/output/object_storage/gcp_storage_bucket.go

@@ -7,6 +7,7 @@ type GoogleStorageBucket struct {
 	UniformBucketLevelAccess bool                            `hcl:"uniform_bucket_level_access" json:"uniform_bucket_level_access"`
 	Versioning               []GoogleStorageBucketVersioning `hcl:"versioning,blocks" hcle:"omitempty" json:"versioning"`
 	Location                 string                          `hcl:"location" json:"location"`
+	ForceDestroy             bool                            `hcl:"force_destroy"`
 }
 
 type GoogleStorageBucketVersioning struct {

resources/output/object_storage_object/aws_object_storage_object.go

@@ -6,10 +6,10 @@ import (
 
 type AwsS3BucketObject struct {
 	*common.AwsResource `hcl:",squash" default:"name=aws_s3_object"`
-	Bucket              string `hcl:"bucket,expr"`
-	Key                 string `hcl:"key"`
-	Acl                 string `hcl:"acl" hcle:"omitempty"`
-	ContentBase64       string `hcl:"content_base64"  hcle:"omitempty"`
-	ContentType         string `hcl:"content_type" hcle:"omitempty"`
-	Source              string `hcl:"source" hcle:"omitempty"`
+	Bucket              string `hcl:"bucket,expr" json:"bucket"`
+	Key                 string `hcl:"key" json:"key"`
+	Acl                 string `hcl:"acl" hcle:"omitempty" json:"acl"`
+	ContentBase64       string `hcl:"content_base64"  hcle:"omitempty" json:"content_base64"`
+	ContentType         string `hcl:"content_type" hcle:"omitempty" json:"content_type"`
+	Source              string `hcl:"source" hcle:"omitempty" json:"source"`
 }

resources/output/object_storage_object/azure_object_storage_object.go

@@ -8,12 +8,12 @@ import (
 // azurerm_storage_blob
 type AzureStorageAccountBlob struct {
 	*common.AzResource   `hcl:",squash" default:"name=azurerm_storage_blob"`
-	StorageAccountName   string `hcl:"storage_account_name,expr"`
-	StorageContainerName string `hcl:"storage_container_name,expr"`
-	Type                 string `hcl:"type"`
-	SourceContent        string `hcl:"source_content"  hcle:"omitempty"`
-	ContentType          string `hcl:"content_type" hcle:"omitempty"`
-	Source               string `hcl:"source,expr" hcle:"omitempty"`
+	StorageAccountName   string `hcl:"storage_account_name,expr" json:"storage_account_name"`
+	StorageContainerName string `hcl:"storage_container_name,expr" json:"storage_container_name"`
+	Type                 string `hcl:"type" json:"type"`
+	SourceContent        string `hcl:"source_content,expr"  hcle:"omitempty" json:"source_content"`
+	ContentType          string `hcl:"content_type" hcle:"omitempty" json:"content_type"`
+	Source               string `hcl:"source,expr" hcle:"omitempty" json:"source"`
 }
 
 // azurerm_storage_container

resources/output/object_storage_object/gcp_storage_object.go

@@ -7,15 +7,15 @@ import (
 
 type GoogleStorageBucketObject struct {
 	*common.GcpResource `hcl:",squash"  default:"name=google_storage_bucket_object"`
-	Bucket              string `hcl:"bucket,expr" hcl:"Bucket"`
-	Content             string `hcl:"content,expr"`
-	ContentType         string `hcl:"content_type"  hcle:"omitempty"`
+	Bucket              string `hcl:"bucket,expr" hcl:"Bucket" json:"bucket"`
+	Content             string `hcl:"content,expr" json:"content"`
+	ContentType         string `hcl:"content_type"  hcle:"omitempty" json:"content_type"`
 }
 
 type GoogleStorageObjectAccessControl struct {
 	*output.TerraformResource `hcl:",squash"  default:"name=google_storage_object_access_control"`
-	Object                    string `hcl:"object,expr"`
-	Bucket                    string `hcl:"bucket,expr"`
-	Role                      string `hcl:"role"`
-	Entity                    string `hcl:"entity"`
+	Object                    string `hcl:"object,expr" json:"object"`
+	Bucket                    string `hcl:"bucket,expr" json:"bucket"`
+	Role                      string `hcl:"role" json:"role"`
+	Entity                    string `hcl:"entity" json:"entity"`
 }

resources/types/aws/object_storage_object.go

@@ -8,7 +8,6 @@ import (
 	"github.com/multycloud/multy/resources"
 	"github.com/multycloud/multy/resources/common"
 	"github.com/multycloud/multy/resources/output"
-	"github.com/multycloud/multy/resources/output/object_storage"
 	"github.com/multycloud/multy/resources/output/object_storage_object"
 	"github.com/multycloud/multy/resources/types"
 )
@@ -22,35 +21,45 @@ func InitObjectStorageObject(vn *types.ObjectStorageObject) resources.ResourceTr
 }
 
 func (r AwsObjectStorageObject) FromState(state *output.TfState) (*resourcespb.ObjectStorageObjectResource, error) {
-	out := new(resourcespb.ObjectStorageObjectResource)
-	out.CommonParameters = &commonpb.CommonChildResourceParameters{
-		ResourceId:  r.ResourceId,
-		NeedsUpdate: false,
+	out := &resourcespb.ObjectStorageObjectResource{
+		CommonParameters: &commonpb.CommonChildResourceParameters{
+			ResourceId:  r.ResourceId,
+			NeedsUpdate: false,
+		},
+		Name:            r.Args.Name,
+		Acl:             r.Args.Acl,
+		ObjectStorageId: r.Args.ObjectStorageId,
+		ContentBase64:   r.Args.ContentBase64,
+		ContentType:     r.Args.ContentType,
+		Source:          r.Args.Source,
 	}
 
-	id, err := resources.GetMainOutputRef(r)
-	if err != nil {
-		return nil, err
+	if flags.DryRun {
+		return out, nil
 	}
 
-	out.Name = r.Args.Name
-	out.ContentBase64 = r.Args.ContentBase64
-	out.ContentType = r.Args.ContentType
-	out.ObjectStorageId = r.Args.ObjectStorageId
-	out.Acl = r.Args.Acl
-	out.Source = r.Args.Source
+	statuses := map[string]commonpb.ResourceStatus_Status{}
 
-	if !flags.DryRun {
-		stateResource, err := output.GetParsed[object_storage.AwsS3Bucket](state, id)
+	if stateResource, exists, err := output.MaybeGetParsedById[object_storage_object.AwsS3BucketObject](state, r.ResourceId); exists {
 		if err != nil {
 			return nil, err
 		}
-		out.Url = fmt.Sprintf("https://%s.s3.amazonaws.com/%s", stateResource.Bucket, r.Args.Name)
+		out.Name = stateResource.Key
+		out.Acl = parseObjectAcl(stateResource.Acl)
+		out.ContentBase64 = stateResource.ContentBase64
+		out.ContentType = stateResource.ContentType
+		out.Source = stateResource.Source
+		if r.Args.Acl == resourcespb.ObjectStorageObjectAcl_PUBLIC_READ {
+			out.Url = fmt.Sprintf("https://%s.s3.amazonaws.com/%s", stateResource.Bucket, stateResource.Key)
+		}
 		out.AwsOutputs = &resourcespb.ObjectStorageObjectAwsOutputs{S3BucketObjectId: stateResource.ResourceId}
 	} else {
-		out.Url = "dryrun"
+		statuses["aws_s3_bucket_object"] = commonpb.ResourceStatus_NEEDS_CREATE
 	}
 
+	if len(statuses) > 0 {
+		out.CommonParameters.ResourceStatus = &commonpb.ResourceStatus{Statuses: statuses}
+	}
 	return out, nil
 }
 
@@ -80,6 +89,15 @@ func (r AwsObjectStorageObject) Translate(resources.MultyContext) ([]output.TfBl
 	}}, nil
 }
 
+func parseObjectAcl(acl string) resourcespb.ObjectStorageObjectAcl {
+	switch acl {
+	case "public-read":
+		return resourcespb.ObjectStorageObjectAcl_PUBLIC_READ
+	default:
+		return resourcespb.ObjectStorageObjectAcl_PRIVATE
+	}
+}
+
 func (r AwsObjectStorageObject) GetMainResourceName() (string, error) {
 	return "aws_s3_object", nil
 }

resources/types/azure/object_storage_object.go

@@ -1,16 +1,15 @@
 package azure_resources
 
 import (
+	"encoding/base64"
 	"fmt"
 	"github.com/multycloud/multy/api/proto/commonpb"
 	"github.com/multycloud/multy/api/proto/resourcespb"
 	"github.com/multycloud/multy/flags"
 	"github.com/multycloud/multy/resources"
 	"github.com/multycloud/multy/resources/common"
 	"github.com/multycloud/multy/resources/output"
-	"github.com/multycloud/multy/resources/output/object_storage"
 	"github.com/multycloud/multy/resources/output/object_storage_object"
-	"github.com/multycloud/multy/resources/output/terraform"
 	"github.com/multycloud/multy/resources/types"
 )
 
@@ -23,35 +22,44 @@ func InitObjectStorageObject(vn *types.ObjectStorageObject) resources.ResourceTr
 }
 
 func (r AzureObjectStorageObject) FromState(state *output.TfState) (*resourcespb.ObjectStorageObjectResource, error) {
-	out := new(resourcespb.ObjectStorageObjectResource)
-	out.CommonParameters = &commonpb.CommonChildResourceParameters{
-		ResourceId:  r.ResourceId,
-		NeedsUpdate: false,
+	out := &resourcespb.ObjectStorageObjectResource{
+		CommonParameters: &commonpb.CommonChildResourceParameters{
+			ResourceId:  r.ResourceId,
+			NeedsUpdate: false,
+		},
+		Name:            r.Args.Name,
+		Acl:             r.Args.Acl,
+		ObjectStorageId: r.Args.ObjectStorageId,
+		ContentBase64:   r.Args.ContentBase64,
+		ContentType:     r.Args.ContentType,
+		Source:          r.Args.Source,
 	}
 
-	id, err := resources.GetMainOutputRef(AzureObjectStorage{r.Parent})
-	if err != nil {
-		return nil, err
+	if flags.DryRun {
+		return out, nil
 	}
 
-	out.Name = r.Args.Name
-	out.ContentBase64 = r.Args.ContentBase64
-	out.ContentType = r.Args.ContentType
-	out.ObjectStorageId = r.Args.ObjectStorageId
-	out.Acl = r.Args.Acl
-	out.Source = r.Args.Source
+	statuses := map[string]commonpb.ResourceStatus_Status{}
 
-	if !flags.DryRun {
-		stateResource, err := output.GetParsed[object_storage.AzureStorageAccount](state, id)
+	if stateResource, exists, err := output.MaybeGetParsedById[object_storage_object.AzureStorageAccountBlob](state, r.ResourceId); exists {
 		if err != nil {
 			return nil, err
 		}
-		out.Url = fmt.Sprintf("https://%s.blob.core.windows.net/public/%s", stateResource.AzResource.Name, r.Args.Name)
+		out.Name = stateResource.Name
+		out.ContentType = stateResource.ContentType
+		out.Source = stateResource.Source
+		out.ContentBase64 = base64.StdEncoding.EncodeToString([]byte(stateResource.SourceContent))
+		if r.Args.Acl == resourcespb.ObjectStorageObjectAcl_PUBLIC_READ {
+			out.Url = fmt.Sprintf("https://%s.blob.core.windows.net/public/%s", stateResource.StorageAccountName, r.Args.Name)
+		}
 		out.AzureOutputs = &resourcespb.ObjectStorageObjectAzureOutputs{StorageBlobId: stateResource.ResourceId}
 	} else {
-		out.Url = "dryrun"
+		statuses["azure_storage_account_blob"] = commonpb.ResourceStatus_NEEDS_CREATE
 	}
 
+	if len(statuses) > 0 {
+		out.CommonParameters.ResourceStatus = &commonpb.ResourceStatus{Statuses: statuses}
+	}
 	return out, nil
 }
 
@@ -62,9 +70,7 @@ func (r AzureObjectStorageObject) Translate(resources.MultyContext) ([]output.Tf
 	} else {
 		containerName = fmt.Sprintf("azurerm_storage_container.%s_private.name", r.ObjectStorage.ResourceId)
 	}
-	contentFile := terraform.NewLocalFile(r.ResourceId, r.Args.ContentBase64)
 	return []output.TfBlock{
-		contentFile,
 		object_storage_object.AzureStorageAccountBlob{
 			AzResource: &common.AzResource{
 				TerraformResource: output.TerraformResource{ResourceId: r.ResourceId},
@@ -73,8 +79,8 @@ func (r AzureObjectStorageObject) Translate(resources.MultyContext) ([]output.Tf
 			StorageAccountName:   fmt.Sprintf("azurerm_storage_account.%s.name", r.ObjectStorage.ResourceId),
 			StorageContainerName: containerName,
 			Type:                 "Block",
-			Source:               contentFile.GetFilename(),
 			ContentType:          r.Args.ContentType,
+			SourceContent:        fmt.Sprintf("base64decode(\"%s\")", r.Args.ContentBase64),
 		}}, nil
 }
 

resources/types/gcp/object_storage.go

@@ -62,6 +62,8 @@ func (r GcpObjectStorage) Translate(resources.MultyContext) ([]output.TfBlock, e
 		GcpResource:              common.NewGcpResource(r.ResourceId, r.Args.Name, r.Args.GetGcpOverride().GetProject()),
 		UniformBucketLevelAccess: false,
 		Location:                 r.GetCloudSpecificLocation(),
+		// this is needed, otherwise buckets with versioning can't be deleted normally
+		ForceDestroy: true,
 	}
 	if r.Args.Versioning {
 		o.Versioning = []object_storage.GoogleStorageBucketVersioning{{Enabled: true}}

resources/types/gcp/object_storage_object.go

@@ -1,6 +1,7 @@
 package gcp_resources
 
 import (
+	"encoding/base64"
 	"fmt"
 	"github.com/multycloud/multy/api/proto/commonpb"
 	"github.com/multycloud/multy/api/proto/resourcespb"
@@ -22,41 +23,50 @@ func InitObjectStorageObject(vn *types.ObjectStorageObject) resources.ResourceTr
 }
 
 func (r GcpObjectStorageObject) FromState(state *output.TfState) (*resourcespb.ObjectStorageObjectResource, error) {
-	out := new(resourcespb.ObjectStorageObjectResource)
-	out.CommonParameters = &commonpb.CommonChildResourceParameters{
-		ResourceId:  r.ResourceId,
-		NeedsUpdate: false,
+	out := &resourcespb.ObjectStorageObjectResource{
+		CommonParameters: &commonpb.CommonChildResourceParameters{
+			ResourceId:  r.ResourceId,
+			NeedsUpdate: false,
+		},
+		Name:            r.Args.Name,
+		Acl:             r.Args.Acl,
+		ObjectStorageId: r.Args.ObjectStorageId,
+		ContentBase64:   r.Args.ContentBase64,
+		ContentType:     r.Args.ContentType,
+		Source:          r.Args.Source,
 	}
 
-	id, err := resources.GetMainOutputRef(r)
-	if err != nil {
-		return nil, err
+	if flags.DryRun {
+		return out, nil
 	}
 
-	out.Name = r.Args.Name
-	out.ContentBase64 = r.Args.ContentBase64
-	out.ContentType = r.Args.ContentType
-	out.ObjectStorageId = r.Args.ObjectStorageId
-	out.Acl = r.Args.Acl
-	out.Source = r.Args.Source
+	statuses := map[string]commonpb.ResourceStatus_Status{}
 
-	if !flags.DryRun {
-		stateResource, err := output.GetParsed[object_storage_object.GoogleStorageBucketObject](state, id)
+	if stateResource, exists, err := output.MaybeGetParsedById[object_storage_object.GoogleStorageBucketObject](state, r.ResourceId); exists {
 		if err != nil {
 			return nil, err
 		}
+		out.Name = stateResource.Name
+		out.ContentType = stateResource.ContentType
+		out.ContentBase64 = base64.StdEncoding.EncodeToString([]byte(stateResource.Content))
 		out.Url = fmt.Sprintf("https://storage.googleapis.com/%s/%s", stateResource.Bucket, r.Args.Name)
 		out.GcpOutputs = &resourcespb.ObjectStorageObjectGcpOutputs{StorageBucketObjectId: stateResource.SelfLink}
-		if stateResource, exists, err := output.MaybeGetParsedById[object_storage_object.GoogleStorageObjectAccessControl](state, r.ResourceId); exists {
+		if aclResource, exists, err := output.MaybeGetParsedById[object_storage_object.GoogleStorageObjectAccessControl](state, r.ResourceId); exists {
 			if err != nil {
 				return nil, err
 			}
-			out.GcpOutputs.StorageObjectAccessControl = stateResource.ResourceId
+			out.GcpOutputs.StorageObjectAccessControl = aclResource.ResourceId
+			out.Acl = resourcespb.ObjectStorageObjectAcl_PUBLIC_READ
+		} else {
+			out.Acl = resourcespb.ObjectStorageObjectAcl_PRIVATE
 		}
 	} else {
-		out.Url = "dryrun"
+		statuses["azure_storage_account_blob"] = commonpb.ResourceStatus_NEEDS_CREATE
 	}
 
+	if len(statuses) > 0 {
+		out.CommonParameters.ResourceStatus = &commonpb.ResourceStatus{Statuses: statuses}
+	}
 	return out, nil
 }