[#3319] datalakestore gen2 filesystem resource

azurerm/config.go

@@ -194,6 +194,13 @@ func getArmClient(authConfig *authentication.Config, skipProviderRegistration bo
 	// Storage Endpoints
 	storageAuth := authConfig.BearerAuthorizerCallback(sender, oauthConfig)
 
+	// Filesystem Endpoints
+	filesystemEndpoint := env.ResourceIdentifiers.Storage
+	filesystemAuth, err := authConfig.GetAuthorizationToken(sender, oauthConfig, filesystemEndpoint)
+	if err != nil {
+		return nil, err
+	}
+
 	// Key Vault Endpoints
 	keyVaultAuth := authConfig.BearerAuthorizerCallback(sender, oauthConfig)
 
@@ -207,6 +214,7 @@ func getArmClient(authConfig *authentication.Config, skipProviderRegistration bo
 		ResourceManagerAuthorizer:   auth,
 		ResourceManagerEndpoint:     endpoint,
 		StorageAuthorizer:           storageAuth,
+		FilesystemAuthorizer:        filesystemAuth,
 		PollingDuration:             180 * time.Minute,
 		SkipProviderReg:             skipProviderRegistration,
 		DisableCorrelationRequestID: disableCorrelationRequestID,

azurerm/internal/common/client_options.go

@@ -25,6 +25,7 @@ type ClientOptions struct {
 	ResourceManagerAuthorizer autorest.Authorizer
 	ResourceManagerEndpoint   string
 	StorageAuthorizer         autorest.Authorizer
+	FilesystemAuthorizer      autorest.Authorizer
 
 	PollingDuration             time.Duration
 	SkipProviderReg             bool

azurerm/internal/services/storage/client.go

@@ -4,12 +4,15 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/Azure/go-autorest/autorest"
+
 	"github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2019-04-01/storage"
 	az "github.com/Azure/go-autorest/autorest/azure"
 	"github.com/terraform-providers/terraform-provider-azurerm/azurerm/internal/authorizers"
 	"github.com/terraform-providers/terraform-provider-azurerm/azurerm/internal/common"
 	"github.com/tombuildsstuff/giovanni/storage/2018-11-09/blob/blobs"
 	"github.com/tombuildsstuff/giovanni/storage/2018-11-09/blob/containers"
+	"github.com/tombuildsstuff/giovanni/storage/2018-11-09/datalakestore/filesystems"
 	"github.com/tombuildsstuff/giovanni/storage/2018-11-09/file/directories"
 	"github.com/tombuildsstuff/giovanni/storage/2018-11-09/file/shares"
 	"github.com/tombuildsstuff/giovanni/storage/2018-11-09/queue/queues"
@@ -18,7 +21,8 @@ import (
 )
 
 type Client struct {
-	AccountsClient storage.AccountsClient
+	AccountsClient       storage.AccountsClient
+	FilesystemAuthorizer autorest.Authorizer
 
 	environment az.Environment
 }
@@ -30,8 +34,9 @@ func BuildClient(options *common.ClientOptions) *Client {
 	// TODO: switch Storage Containers to using the storage.BlobContainersClient
 	// (which should fix #2977) when the storage clients have been moved in here
 	return &Client{
-		AccountsClient: accountsClient,
-		environment:    options.Environment,
+		AccountsClient:       accountsClient,
+		FilesystemAuthorizer: options.FilesystemAuthorizer,
+		environment:          options.Environment,
 	}
 }
 
@@ -59,6 +64,13 @@ func (client Client) ContainersClient(ctx context.Context, resourceGroup, accoun
 	return &containersClient, nil
 }
 
+func (client Client) FileSystemsClient(ctx context.Context, resourceGroup, accountName string) (*filesystems.Client, error) {
+	filesystemAuth := client.FilesystemAuthorizer
+	fileSystemsClient := filesystems.NewWithEnvironment(client.environment)
+	fileSystemsClient.Client.Authorizer = filesystemAuth
+	return &fileSystemsClient, nil
+}
+
 func (client Client) FileShareDirectoriesClient(ctx context.Context, resourceGroup, accountName string) (*directories.Client, error) {
 	accountKey, err := client.findAccountKey(ctx, resourceGroup, accountName)
 	if err != nil {

azurerm/provider.go

@@ -418,6 +418,7 @@ func Provider() terraform.ResourceProvider {
 		"azurerm_storage_share_directory":                                                resourceArmStorageShareDirectory(),
 		"azurerm_storage_table":                                                          resourceArmStorageTable(),
 		"azurerm_storage_table_entity":                                                   resourceArmStorageTableEntity(),
+		"azurerm_storage_filesystem":                                                     resourceArmStorageFilesystem(),
 		"azurerm_stream_analytics_job":                                                   resourceArmStreamAnalyticsJob(),
 		"azurerm_stream_analytics_function_javascript_udf":                               resourceArmStreamAnalyticsFunctionUDF(),
 		"azurerm_stream_analytics_output_blob":                                           resourceArmStreamAnalyticsOutputBlob(),

azurerm/resource_arm_storage_filesystem.go

@@ -0,0 +1,266 @@
+package azurerm
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"net/url"
+	"regexp"
+	"strings"
+	"time"
+
+	azauto "github.com/Azure/go-autorest/autorest/azure"
+	"github.com/hashicorp/terraform/helper/resource"
+	"github.com/hashicorp/terraform/helper/schema"
+	"github.com/terraform-providers/terraform-provider-azurerm/azurerm/helpers/azure"
+	"github.com/terraform-providers/terraform-provider-azurerm/azurerm/helpers/tf"
+	"github.com/terraform-providers/terraform-provider-azurerm/azurerm/internal/features"
+	"github.com/terraform-providers/terraform-provider-azurerm/azurerm/internal/services/storage"
+	"github.com/tombuildsstuff/giovanni/storage/2018-11-09/datalakestore/filesystems"
+)
+
+func resourceArmStorageFilesystem() *schema.Resource {
+	return &schema.Resource{
+		Create:        resourceArmStorageFilesystemCreate,
+		Read:          resourceArmStorageFilesystemRead,
+		Delete:        resourceArmStorageFilesystemDelete,
+		SchemaVersion: 1,
+
+		Importer: &schema.ResourceImporter{
+			State: schema.ImportStatePassthrough,
+		},
+
+		Schema: map[string]*schema.Schema{
+			"name": {
+				Type:         schema.TypeString,
+				Required:     true,
+				ForceNew:     true,
+				ValidateFunc: validateArmStorageFilesystemName,
+			},
+			"resource_group_name": azure.SchemaResourceGroupNameDeprecated(),
+			"storage_account_name": {
+				Type:     schema.TypeString,
+				Required: true,
+				ForceNew: true,
+			},
+
+			"properties": {
+				Type:     schema.TypeMap,
+				Computed: true,
+				Elem: &schema.Schema{
+					Type: schema.TypeString,
+				},
+			},
+		},
+	}
+}
+
+//Following the naming convention as laid out in the docs
+func validateArmStorageFilesystemName(v interface{}, k string) (warnings []string, errors []error) {
+	value := v.(string)
+	if !regexp.MustCompile(`^\$root$|^[0-9a-z-]+$`).MatchString(value) {
+		errors = append(errors, fmt.Errorf(
+			"only lowercase alphanumeric characters and hyphens allowed in %q: %q",
+			k, value))
+	}
+	if len(value) < 3 || len(value) > 63 {
+		errors = append(errors, fmt.Errorf(
+			"%q must be between 3 and 63 characters: %q", k, value))
+	}
+	if regexp.MustCompile(`^-`).MatchString(value) {
+		errors = append(errors, fmt.Errorf(
+			"%q cannot begin with a hyphen: %q", k, value))
+	}
+	return warnings, errors
+}
+
+func resourceArmStorageFilesystemCreate(d *schema.ResourceData, meta interface{}) error {
+	armClient := meta.(*ArmClient)
+	storageClient := armClient.Storage
+	ctx := armClient.StopContext
+
+	filesystemName := d.Get("name").(string)
+	accountName := d.Get("storage_account_name").(string)
+	propertiesRaw := d.Get("properties").(map[string]interface{})
+	properties := storage.ExpandMetaData(propertiesRaw)
+	resourceGroup, err := storageClient.FindResourceGroup(ctx, accountName)
+	if err != nil {
+		return fmt.Errorf("Error locating Resource Group for Storage Filesystem %q (Account %s): %s", filesystemName, accountName, err)
+	}
+	if resourceGroup == nil {
+		return fmt.Errorf("Unable to locate Resource Group for Storage Filesystem %q (Account %s)", filesystemName, accountName)
+	}
+
+	client, err := storageClient.FileSystemsClient(ctx, *resourceGroup, accountName)
+	if err != nil {
+		return fmt.Errorf("Error building Filesystems Client: %s", err)
+	}
+
+	resp, err := client.GetProperties(ctx, accountName, filesystemName)
+	id := fmt.Sprintf("https://%s.dfs.%s/%s", accountName, armClient.environment.StorageEndpointSuffix, filesystemName)
+	if features.ShouldResourcesBeImported() {
+		exists, err := (resp.StatusCode != http.StatusNotFound), err
+		if err != nil {
+			return fmt.Errorf("Error checking for existence of existing Filesystem %q (Account %q / Resource Group %q): %+v", filesystemName, accountName, *resourceGroup, err)
+		}
+
+		if exists {
+			return tf.ImportAsExistsError("azurerm_storage_filesystem", id)
+		}
+	}
+
+	log.Printf("[INFO] Creating filesystem %q in storage account %q.", filesystemName, accountName)
+	err = resource.Retry(120*time.Second, checkFilesystemIsCreated(ctx, client, accountName, filesystemName, properties))
+	if err != nil {
+		return fmt.Errorf("Error creating filesystem %q in storage account %q: %s", filesystemName, accountName, err)
+	}
+
+	d.SetId(id)
+	return resourceArmStorageFilesystemRead(d, meta)
+}
+
+// resourceAzureStorageFilesystemRead does all the necessary API calls to
+// read the status of the storage filesystem off Azure.
+func resourceArmStorageFilesystemRead(d *schema.ResourceData, meta interface{}) error {
+	armClient := meta.(*ArmClient)
+	storageClient := armClient.Storage
+	ctx := armClient.StopContext
+
+	id, err := parseStorageFilesystemID(d.Id(), armClient.environment)
+	if err != nil {
+		return err
+	}
+
+	resourceGroup, err := storageClient.FindResourceGroup(ctx, id.storageAccountName)
+	if err != nil {
+		return fmt.Errorf("Error locating Resource Group for Storage FileSystem %q (Account %s): %s", id.filesystemName, id.storageAccountName, err)
+	}
+	if resourceGroup == nil {
+		log.Printf("[DEBUG] Unable to locate Resource Group for Storage FileSystem %q (Account %s) - assuming removed & removing from state", id.filesystemName, id.storageAccountName)
+		d.SetId("")
+		return nil
+	}
+
+	client, err := storageClient.FileSystemsClient(ctx, *resourceGroup, id.storageAccountName)
+	if err != nil {
+		return err
+	}
+
+	resp, err := client.GetProperties(ctx, id.storageAccountName, id.filesystemName)
+	exists, err := (resp.StatusCode != http.StatusNotFound), err
+	if err != nil {
+		return fmt.Errorf("Error checking for existence of existing Filesystem %q (Account %q / Resource Group %q): %+v", id.filesystemName, id.storageAccountName, *resourceGroup, err)
+	}
+
+	if !exists {
+		log.Printf("[INFO] Storage filesystem %q does not exist in account %q, removing from state...", id.filesystemName, id.storageAccountName)
+		d.SetId("")
+		return nil
+	}
+
+	d.Set("name", id.filesystemName)
+	d.Set("storage_account_name", id.storageAccountName)
+	d.Set("resource_group_name", resourceGroup)
+
+	if err := d.Set("properties", resp.Properties); err != nil {
+		return fmt.Errorf("Error setting `properties`: %+v", err)
+	}
+
+	return nil
+}
+
+// resourceAzureStorageFilesystemDelete does all the necessary API calls to
+// delete a storage filesystem off Azure.
+func resourceArmStorageFilesystemDelete(d *schema.ResourceData, meta interface{}) error {
+	armClient := meta.(*ArmClient)
+	storageClient := armClient.Storage
+	ctx := armClient.StopContext
+
+	id, err := parseStorageFilesystemID(d.Id(), armClient.environment)
+	if err != nil {
+		return err
+	}
+
+	resourceGroup, err := storageClient.FindResourceGroup(ctx, id.storageAccountName)
+	if err != nil {
+		return fmt.Errorf("Error locating Resource Group for Storage FileSystem %q (Account %s): %s", id.filesystemName, id.storageAccountName, err)
+	}
+	if resourceGroup == nil {
+		return fmt.Errorf("Unable to locate Resource Group for Storage FileSystem %q (Account %s)", id.filesystemName, id.storageAccountName)
+	}
+
+	client, err := storageClient.FileSystemsClient(ctx, *resourceGroup, id.storageAccountName)
+	if err != nil {
+		return fmt.Errorf("Error building FileSystems Client: %s", err)
+	}
+
+	log.Printf("[INFO] Deleting storage filesystem %q in account %q", id.filesystemName, id.storageAccountName)
+
+	if _, err := client.Delete(ctx, id.storageAccountName, id.filesystemName); err != nil {
+		return fmt.Errorf("Error deleting storage filesystem %q from storage account %q: %s", id.filesystemName, id.storageAccountName, err)
+	}
+
+	return nil
+}
+
+func checkFilesystemIsCreated(ctx context.Context, filesystemClient *filesystems.Client, accountName string, name string, properties map[string]string) func() *resource.RetryError {
+	return func() *resource.RetryError {
+		if _, err := createIfNotExists(ctx, filesystemClient, accountName, name, properties); err != nil {
+			return resource.RetryableError(err)
+		}
+
+		return nil
+	}
+}
+
+// CreateIfNotExists creates a storage datalake gen2 filesystem if it does not exist. Returns
+// true if filesystem is newly created or false if filesystem already exists.
+func createIfNotExists(ctx context.Context, filesystemClient *filesystems.Client, accountName string, name string, properties map[string]string) (bool, error) {
+	resp, err := filesystemClient.Create(ctx, accountName, name, filesystems.CreateInput{
+		Properties: properties,
+	})
+	if resp.Response != nil {
+		defer drainRespBody(resp.Response)
+		if resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusCreated || resp.StatusCode == http.StatusConflict {
+			return resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusCreated, nil
+		}
+	}
+	return false, err
+}
+
+func drainRespBody(resp *http.Response) {
+	_, _ = io.Copy(ioutil.Discard, resp.Body)
+	resp.Body.Close()
+}
+
+type storageFilesystemId struct {
+	id                 string
+	storageAccountName string
+	filesystemName     string
+}
+
+func parseStorageFilesystemID(input string, environment azauto.Environment) (*storageFilesystemId, error) {
+	uri, err := url.Parse(input)
+	if err != nil {
+		return nil, fmt.Errorf("Error parsing %q as URI: %+v", input, err)
+	}
+
+	// remove the leading `/`
+	segments := strings.Split(strings.TrimPrefix(uri.Path, "/"), "/")
+	if len(segments) < 1 {
+		return nil, fmt.Errorf("Expected number of segments in the path to be < 1 but got %d", len(segments))
+	}
+
+	storageAccountName := strings.Replace(uri.Host, fmt.Sprintf(".dfs.%s", environment.StorageEndpointSuffix), "", 1)
+	filesystemName := segments[0]
+
+	id := storageFilesystemId{
+		id:                 input,
+		storageAccountName: storageAccountName,
+		filesystemName:     filesystemName,
+	}
+	return &id, nil
+}

go.mod

@@ -16,7 +16,7 @@ require (
 	github.com/satori/go.uuid v1.2.0
 	github.com/satori/uuid v0.0.0-20160927100844-b061729afc07
 	github.com/terraform-providers/terraform-provider-azuread v0.6.0
-	github.com/tombuildsstuff/giovanni v0.5.0
+	github.com/tombuildsstuff/giovanni v0.6.0
 	golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4
 	golang.org/x/net v0.0.0-20190502183928-7f726cade0ab
 	gopkg.in/yaml.v2 v2.2.2

go.sum

@@ -489,6 +489,8 @@ github.com/terraform-providers/terraform-provider-openstack v1.15.0/go.mod h1:2a
 github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tombuildsstuff/giovanni v0.5.0 h1:ih4qTvjOOAHubTdINDWtrTpHyHNzrqam4xcIWQY9A1A=
 github.com/tombuildsstuff/giovanni v0.5.0/go.mod h1:Xu/XU+DiRrKTDoCnJNGuh9ysD0eJyi/zU/naFh2aN9I=
+github.com/tombuildsstuff/giovanni v0.6.0 h1:8RwaDJJqbp8mMN/HOKEbGUvhW/yaGMzxN2XbkQ3lHuA=
+github.com/tombuildsstuff/giovanni v0.6.0/go.mod h1:Xu/XU+DiRrKTDoCnJNGuh9ysD0eJyi/zU/naFh2aN9I=
 github.com/ugorji/go v0.0.0-20180813092308-00b869d2f4a5 h1:cMjKdf4PxEBN9K5HaD9UMW8gkTbM0kMzkTa9SJe0WNQ=
 github.com/ugorji/go v0.0.0-20180813092308-00b869d2f4a5/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8 h1:3SVOIvH7Ae1KRYyQWRjXWJEA9sS/c/pjvH++55Gr648=