Azdatalake stg 78-82

sdk/storage/azdatalake/CHANGELOG.md

@@ -3,6 +3,10 @@
 ## 1.0.1 (Unreleased)
 
 ### Features Added
+* Encryption Scope For SAS
+* CPK for Datalake
+* Create SubDirectory Client
+* Service Version upgrade to 2021-06-08
 
 ### Breaking Changes
 
@@ -20,4 +24,4 @@
 
 ### Features Added
 
-* This is the initial preview release of the `azdatalake` library
+* This is the initial preview release of the `azdatalake` library

sdk/storage/azdatalake/README.md

@@ -1,6 +1,6 @@
 # ADLS Gen2 Storage SDK for Go
 
-> Service Version: 2020-10-02
+> Service Version: 2021-06-08
 
 Azure Data Lake Storage Gen2 (ADLS Gen2) is Microsoft's hierarchical object storage solution for the cloud with converged capabilities with Azure Blob Storage. 
 For example, Data Lake Storage Gen2 provides file system semantics, file-level security, and scale. 

sdk/storage/azdatalake/assets.json

@@ -2,5 +2,5 @@
   "AssetsRepo": "Azure/azure-sdk-assets",
   "AssetsRepoPrefixPath": "go",
   "TagPrefix": "go/storage/azdatalake",
-  "Tag": "go/storage/azdatalake_7c0fb050cb"
+  "Tag": "go/storage/azdatalake_ba5bfaa832"
 }

sdk/storage/azdatalake/directory/client.go

@@ -228,6 +228,29 @@ func (d *Client) NewFileClient(fileName string) (*file.Client, error) {
 	return (*file.Client)(base.NewPathClient(fileURL, newBlobURL, newBlobClient, d.generatedDirClientWithDFS().InternalClient().WithClientName(shared.FileClient), d.sharedKey(), d.identityCredential(), d.getClientOptions())), nil
 }
 
+// NewSubdirectoryClient creates a new directory.Client object by concatenating subdirectoryName to the end of this Client's URL.
+// The new directory.Client uses the same request policy pipeline as the Client.
+func (d *Client) NewSubdirectoryClient(subdirectoryName string) (*Client, error) {
+	subdirectoryName = url.PathEscape(strings.TrimRight(subdirectoryName, "/"))
+	subDirectoryURL := runtime.JoinPaths(d.DFSURL(), subdirectoryName)
+	newBlobURL, subDirectoryURL := shared.GetURLs(subDirectoryURL)
+	var newBlobClient *blockblob.Client
+	clientOptions := &blockblob.ClientOptions{ClientOptions: d.getClientOptions().ClientOptions}
+	var err error
+	if d.identityCredential() != nil {
+		newBlobClient, err = blockblob.NewClient(newBlobURL, *d.identityCredential(), clientOptions)
+	} else if d.sharedKey() != nil {
+		blobSharedKey, _ := exported.ConvertToBlobSharedKey(d.sharedKey())
+		newBlobClient, err = blockblob.NewClientWithSharedKeyCredential(newBlobURL, blobSharedKey, clientOptions)
+	} else {
+		newBlobClient, err = blockblob.NewClientWithNoCredential(newBlobURL, clientOptions)
+	}
+	if err != nil {
+		return nil, exported.ConvertToDFSError(err)
+	}
+	return (*Client)(base.NewPathClient(subDirectoryURL, newBlobURL, newBlobClient, d.generatedDirClientWithDFS().InternalClient().WithClientName(shared.DirectoryClient), d.sharedKey(), d.identityCredential(), d.getClientOptions())), nil
+}
+
 // Create creates a new directory.
 func (d *Client) Create(ctx context.Context, options *CreateOptions) (CreateResponse, error) {
 	lac, mac, httpHeaders, createOpts, cpkOpts := options.format()
@@ -295,7 +318,7 @@ func (d *Client) Rename(ctx context.Context, destinationPath string, options *Re
 		newPathURL = strings.Split(newPathURL, "?")[0] + "?" + newDestQuery
 	}
 	newBlobURL, _ := shared.GetURLs(newPathURL)
-	lac, mac, smac, createOpts := path.FormatRenameOptions(options, newSrcPath)
+	lac, mac, smac, createOpts, cpkOpts := path.FormatRenameOptions(options, newSrcPath)
 
 	if d.identityCredential() != nil {
 		newBlobClient, err = blockblob.NewClient(newBlobURL, *d.identityCredential(), nil)
@@ -310,7 +333,7 @@ func (d *Client) Rename(ctx context.Context, destinationPath string, options *Re
 		return RenameResponse{}, exported.ConvertToDFSError(err)
 	}
 	newDirClient := (*Client)(base.NewPathClient(newPathURL, newBlobURL, newBlobClient, d.generatedDirClientWithDFS().InternalClient().WithClientName(shared.DirectoryClient), d.sharedKey(), d.identityCredential(), d.getClientOptions()))
-	resp, err := newDirClient.generatedDirClientWithDFS().Create(ctx, createOpts, nil, lac, mac, smac, nil)
+	resp, err := newDirClient.generatedDirClientWithDFS().Create(ctx, createOpts, nil, lac, mac, smac, cpkOpts)
 	//return RenameResponse{
 	//	Response:           resp,
 	//	NewDirectoryClient: newDirClient,

sdk/storage/azdatalake/directory/client_test.go

@@ -72,7 +72,6 @@ type UnrecordedTestSuite struct {
 }
 
 func (s *UnrecordedTestSuite) TestCreateDirAndDeleteWithConnectionString() {
-
 	_require := require.New(s.T())
 	testName := s.T().Name()
 
@@ -147,6 +146,31 @@ func (s *RecordedTestSuite) TestCreateDirAndDelete() {
 	_require.NotNil(resp)
 }
 
+func (s *RecordedTestSuite) TestCreateDirUsingCPK() {
+	_require := require.New(s.T())
+	testName := s.T().Name()
+
+	filesystemName := testcommon.GenerateFileSystemName(testName)
+	fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil)
+	_require.NoError(err)
+	defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient)
+
+	_, err = fsClient.Create(context.Background(), nil)
+	_require.NoError(err)
+
+	dirName := testcommon.GenerateDirName(testName)
+	dirClient, err := testcommon.GetDirClient(filesystemName, dirName, s.T(), testcommon.TestAccountDatalake, nil)
+	_require.NoError(err)
+
+	dirOpts := &directory.CreateOptions{CPKInfo: &testcommon.TestCPKByValue}
+	resp, err := dirClient.Create(context.Background(), dirOpts)
+	_require.NoError(err)
+	_require.NotNil(resp)
+
+	_require.Equal(true, *(resp.IsServerEncrypted))
+	_require.Equal(testcommon.TestCPKByValue.EncryptionKeySHA256, resp.EncryptionKeySHA256)
+}
+
 func (s *RecordedTestSuite) TestGetAndCreateFileClient() {
 	_require := require.New(s.T())
 	testName := s.T().Name()
@@ -177,6 +201,70 @@ func (s *RecordedTestSuite) TestGetAndCreateFileClient() {
 	_require.NoError(err)
 }
 
+func (s *RecordedTestSuite) TestCreateNewSubdirectoryClient() {
+	_require := require.New(s.T())
+	testName := s.T().Name()
+
+	accountName, _ := testcommon.GetGenericAccountInfo(testcommon.TestAccountDatalake)
+	_require.Greater(len(accountName), 0)
+
+	svcClient, err := testcommon.GetServiceClient(s.T(), testcommon.TestAccountDatalake, nil)
+	_require.NoError(err)
+
+	fsName := testcommon.GenerateFileSystemName(testName)
+	fsClient := svcClient.NewFileSystemClient(fsName)
+
+	_, err = fsClient.Create(context.Background(), nil)
+	_require.NoError(err)
+
+	defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient)
+
+	dirName := testcommon.GenerateDirName(testName)
+	dirClient := fsClient.NewDirectoryClient(dirName)
+
+	_, err = dirClient.Create(context.Background(), nil)
+	_require.NoError(err)
+
+	subdirName := testcommon.GenerateSubDirName(testName)
+	subdirClient, err := dirClient.NewSubdirectoryClient(subdirName)
+	_require.NoError(err)
+
+	perm := "r-xr-x---"
+
+	_, err = subdirClient.Create(context.Background(), &directory.CreateOptions{
+		Permissions: &perm,
+		CPKInfo:     &testcommon.TestCPKByValue,
+	})
+	_require.NoError(err)
+
+	resp, err := subdirClient.GetProperties(context.Background(), &directory.GetPropertiesOptions{CPKInfo: &testcommon.TestCPKByValue})
+	_require.NoError(err)
+	_require.NotNil(resp.Permissions)
+	_require.Equal(perm, *resp.Permissions)
+	_require.Equal(*(resp.IsServerEncrypted), true)
+	_require.Equal(resp.EncryptionKeySHA256, testcommon.TestCPKByValue.EncryptionKeySHA256)
+
+	// Create a file under the new directory just to make sure we're not secretly targeting the parent
+	fileName := testcommon.GenerateFileName("newFile")
+	subdirFileClient, err := subdirClient.NewFileClient(fileName)
+	_require.NoError(err)
+
+	_, err = subdirFileClient.Create(context.Background(), nil)
+	_require.NoError(err)
+
+	fileContents := []byte(testcommon.DefaultData)
+	err = subdirFileClient.UploadBuffer(context.Background(), fileContents, nil)
+	_require.NoError(err)
+
+	// check for the file at the parent directory
+	dirFileClient, err := dirClient.NewFileClient(fileName)
+	_require.NoError(err)
+
+	_, err = dirFileClient.GetProperties(context.Background(), nil)
+	_require.Error(err) // we should get back a 404
+	_require.True(datalakeerror.HasCode(err, datalakeerror.PathNotFound))
+}
+
 func (s *RecordedTestSuite) TestCreateDirWithNilAccessConditions() {
 	_require := require.New(s.T())
 	testName := s.T().Name()
@@ -1912,6 +2000,65 @@ func (s *RecordedTestSuite) TestDirSetMetadataWithAccessConditions() {
 	_require.NoError(err)
 }
 
+func (s *RecordedTestSuite) TestDirSetMetadataWithCPK() {
+	_require := require.New(s.T())
+	testName := s.T().Name()
+
+	filesystemName := testcommon.GenerateFileSystemName(testName)
+	fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil)
+	_require.NoError(err)
+	defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient)
+
+	_, err = fsClient.Create(context.Background(), nil)
+	_require.NoError(err)
+
+	dirName := testcommon.GenerateDirName(testName)
+	dirClient, err := testcommon.GetDirClient(filesystemName, dirName, s.T(), testcommon.TestAccountDatalake, nil)
+	_require.NoError(err)
+
+	defer testcommon.DeleteDir(context.Background(), _require, dirClient)
+
+	resp, err := dirClient.Create(context.Background(), &directory.CreateOptions{CPKInfo: &testcommon.TestCPKByValue})
+	_require.NoError(err)
+	_require.NotNil(resp)
+
+	opts := &directory.SetMetadataOptions{
+		CPKInfo: &testcommon.TestCPKByValue,
+	}
+	_, err = dirClient.SetMetadata(context.Background(), testcommon.BasicMetadata, opts)
+	_require.NoError(err)
+}
+
+func (s *RecordedTestSuite) TestDirSetMetadataWithCPKNegative() {
+	_require := require.New(s.T())
+	testName := s.T().Name()
+
+	filesystemName := testcommon.GenerateFileSystemName(testName)
+	fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil)
+	_require.NoError(err)
+	defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient)
+
+	_, err = fsClient.Create(context.Background(), nil)
+	_require.NoError(err)
+
+	dirName := testcommon.GenerateDirName(testName)
+	dirClient, err := testcommon.GetDirClient(filesystemName, dirName, s.T(), testcommon.TestAccountDatalake, nil)
+	_require.NoError(err)
+
+	defer testcommon.DeleteDir(context.Background(), _require, dirClient)
+
+	resp, err := dirClient.Create(context.Background(), nil)
+	_require.NoError(err)
+	_require.NotNil(resp)
+
+	opts := &directory.SetMetadataOptions{
+		CPKInfo: &testcommon.TestCPKByValue,
+	}
+	_, err = dirClient.SetMetadata(context.Background(), testcommon.BasicMetadata, opts)
+	_require.Error(err)
+	_require.ErrorContains(err, "PathDoesNotUseCustomerSpecifiedEncryption")
+}
+
 func validatePropertiesSet(_require *require.Assertions, dirClient *directory.Client, disposition string) {
 	resp, err := dirClient.GetProperties(context.Background(), nil)
 	_require.NoError(err)
@@ -2243,6 +2390,38 @@ func (s *RecordedTestSuite) TestDirRenameNoOptions() {
 	//_require.Contains(resp1.NewDirectoryClient.DFSURL(), "newName")
 }
 
+func (s *RecordedTestSuite) TestDirRenameRequestWithCPK() {
+	_require := require.New(s.T())
+	testName := s.T().Name()
+
+	filesystemName := testcommon.GenerateFileSystemName(testName)
+	fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil)
+	_require.NoError(err)
+	defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient)
+
+	_, err = fsClient.Create(context.Background(), nil)
+	_require.NoError(err)
+
+	dirName := testcommon.GenerateDirName(testName)
+	dirClient, err := testcommon.GetDirClient(filesystemName, dirName, s.T(), testcommon.TestAccountDatalake, nil)
+	_require.NoError(err)
+
+	createOpts := &directory.CreateOptions{
+		CPKInfo: &testcommon.TestCPKByValue,
+	}
+
+	resp, err := dirClient.Create(context.Background(), createOpts)
+	_require.NoError(err)
+	_require.NotNil(resp)
+
+	renameFileOpts := &directory.RenameOptions{
+		CPKInfo: &testcommon.TestCPKByValue,
+	}
+
+	_, err = dirClient.Rename(context.Background(), "newName", renameFileOpts)
+	_require.NoError(err)
+}
+
 func (s *RecordedTestSuite) TestRenameDirWithNilAccessConditions() {
 	_require := require.New(s.T())
 	testName := s.T().Name()
@@ -2548,6 +2727,35 @@ func (s *RecordedTestSuite) TestDirGetPropertiesResponseCapture() {
 	_require.Equal("directory", respFromCtxService.Header.Get("x-ms-resource-type"))
 }
 
+func (s *RecordedTestSuite) TestDirGetPropertiesWithCPK() {
+	_require := require.New(s.T())
+	testName := s.T().Name()
+
+	filesystemName := testcommon.GenerateFileSystemName(testName)
+	fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil)
+	_require.NoError(err)
+	defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient)
+
+	_, err = fsClient.Create(context.Background(), nil)
+	_require.NoError(err)
+
+	dirName := testcommon.GenerateDirName(testName)
+	dirClient, err := testcommon.GetDirClient(filesystemName, dirName, s.T(), testcommon.TestAccountDatalake, nil)
+	_require.NoError(err)
+
+	dirOpts := &directory.CreateOptions{CPKInfo: &testcommon.TestCPKByValue}
+	resp, err := dirClient.Create(context.Background(), dirOpts)
+	_require.NoError(err)
+	_require.NotNil(resp)
+
+	getPropertiesOpts := &directory.GetPropertiesOptions{CPKInfo: &testcommon.TestCPKByValue}
+	response, err := dirClient.GetProperties(context.Background(), getPropertiesOpts)
+	_require.NoError(err)
+	_require.NotNil(response)
+	_require.Equal(*(resp.IsServerEncrypted), true)
+	_require.Equal(resp.EncryptionKeySHA256, testcommon.TestCPKByValue.EncryptionKeySHA256)
+}
+
 func (s *UnrecordedTestSuite) TestDirCreateDeleteUsingOAuth() {
 	_require := require.New(s.T())
 	testName := s.T().Name()

sdk/storage/azdatalake/file/client.go

@@ -278,7 +278,7 @@ func (f *Client) Rename(ctx context.Context, destinationPath string, options *Re
 		newPathURL = strings.Split(newPathURL, "?")[0] + "?" + newDestQuery
 	}
 	newBlobURL, _ := shared.GetURLs(newPathURL)
-	lac, mac, smac, createOpts := path.FormatRenameOptions(options, newSrcPath)
+	lac, mac, smac, createOpts, cpkOpts := path.FormatRenameOptions(options, newSrcPath)
 
 	if f.identityCredential() != nil {
 		newBlobClient, err = blockblob.NewClient(newBlobURL, *f.identityCredential(), nil)
@@ -293,7 +293,7 @@ func (f *Client) Rename(ctx context.Context, destinationPath string, options *Re
 		return RenameResponse{}, exported.ConvertToDFSError(err)
 	}
 	newFileClient := (*Client)(base.NewPathClient(newPathURL, newBlobURL, newBlobClient, f.generatedFileClientWithDFS().InternalClient().WithClientName(shared.FileClient), f.sharedKey(), f.identityCredential(), f.getClientOptions()))
-	resp, err := newFileClient.generatedFileClientWithDFS().Create(ctx, createOpts, nil, lac, mac, smac, nil)
+	resp, err := newFileClient.generatedFileClientWithDFS().Create(ctx, createOpts, nil, lac, mac, smac, cpkOpts)
 
 	//return RenameResponse{
 	//	Response:      resp,