Update notebook and spark job definition cmdlets to support folder pa…

…th (#16269)

-update change log

-add example for specifying multi-level folder path

-update FolderName to FolderPath as PowerShell Team suggested

-update foldername description in help doc

-resolve some comments in code review

-update change log

-add new parameter foldername

src/Synapse/Synapse/ChangeLog.md

@@ -19,6 +19,7 @@
 -->
 
 ## Upcoming Release
+* Updated `Set-AzSynapseNoteBook` and `Set-AzSynapseSparkJobDefinition` to support new parameter [-FolderPath]
 * Added support for Synapse data flow debug session
     - Added `Start-AzSynapseDataFlowDebugSession` cmdlet to start a Synapse Analytics data flow debug session.
     - Added `Add-AzSynapseDataFlowDebugSessionPackage` cmdlet

src/Synapse/Synapse/Commands/DataPlaneCommands/Artifact/Notebooks/SetAzureSynapseNotebook.cs

@@ -57,6 +57,10 @@ public class SetAzureSynapseNotebook : SynapseArtifactsCmdletBase
         [Alias("NotebookName")]
         public string Name { get; set; }
 
+        [Parameter(ValueFromPipelineByPropertyName = false, Mandatory = false, HelpMessage = HelpMessages.NoteBookFolderPath)]
+        [ValidateNotNullOrEmpty]
+        public string FolderPath { get; set; }
+
         [Parameter(ValueFromPipelineByPropertyName = false, ParameterSetName = SetByNameAndSparkPool,
             Mandatory = true, HelpMessage = HelpMessages.SparkPoolName)]
         [Parameter(ValueFromPipelineByPropertyName = false, ParameterSetName = SetByObjectAndSparkPool,
@@ -134,6 +138,13 @@ public override void ExecuteCmdlet()
                     notebookResource.Properties.SessionProperties = new NotebookSessionProperties(options["memory"] + "g", (int)options["cores"], options["memory"] + "g", (int)options["cores"], (int)options["nodeCount"]);
                 }
 
+                if (this.IsParameterBound(c => c.FolderPath))
+                {
+                    NotebookFolder folder = new NotebookFolder();
+                    folder.Name = this.FolderPath;
+                    notebookResource.Properties.Folder = folder;
+                }
+
                 WriteObject(new PSNotebookResource(SynapseAnalyticsClient.CreateOrUpdateNotebook(this.Name, notebookResource), this.WorkspaceName));
             }
         }

src/Synapse/Synapse/Commands/DataPlaneCommands/Artifact/SparkJobDefinitions/SetAzureSynapseSparkJobDefinition.cs

@@ -12,6 +12,7 @@
 // limitations under the License.
 // ----------------------------------------------------------------------------------
 
+using Azure.Analytics.Synapse.Artifacts.Models;
 using Microsoft.Azure.Commands.Common.Exceptions;
 using Microsoft.Azure.Commands.ResourceManager.Common.ArgumentCompleters;
 using Microsoft.Azure.Commands.Synapse.Common;
@@ -54,6 +55,10 @@ public class SetAzureSynapseSparkJobDefinition : SynapseArtifactsCmdletBase
         [Alias("File")]
         public string DefinitionFile { get; set; }
 
+        [Parameter(ValueFromPipelineByPropertyName = false, Mandatory = false, HelpMessage = HelpMessages.SparkConfigurationFolderPath)]
+        [ValidateNotNullOrEmpty]
+        public string FolderPath { get; set; }
+
         [Parameter(Mandatory = false, HelpMessage = HelpMessages.AsJob)]
         public SwitchParameter AsJob { get; set; }
 
@@ -67,7 +72,15 @@ public override void ExecuteCmdlet()
             if (this.ShouldProcess(this.WorkspaceName, String.Format(Resources.SettingSynapseSparkJobDefinition, this.Name, this.WorkspaceName)))
             {
                 string rawJsonContent = SynapseAnalyticsClient.ReadJsonFileContent(this.TryResolvePath(DefinitionFile));
-                WriteObject(new PSSparkJobDefinitionResource(SynapseAnalyticsClient.CreateOrUpdateSparkJobDefinition(this.Name, rawJsonContent)));
+                SparkJobDefinition sparkJobDefinition = JsonConvert.DeserializeObject<SparkJobDefinition>(rawJsonContent);
+                SparkJobDefinitionResource sparkJobDefinitionResource = new SparkJobDefinitionResource(sparkJobDefinition);
+                if (this.IsParameterBound(c => c.FolderPath))
+                {
+                    SparkJobDefinitionFolder folder = new SparkJobDefinitionFolder();
+                    folder.Name = FolderPath;
+                    sparkJobDefinitionResource.Properties.Folder = folder;
+                }
+                WriteObject(new PSSparkJobDefinitionResource(SynapseAnalyticsClient.CreateOrUpdateSparkJobDefinition(this.Name, sparkJobDefinitionResource)));
             }
         }
     }

src/Synapse/Synapse/Common/HelpMessages.cs

@@ -540,5 +540,9 @@ SELECT on dbo.myTable by public
         public const string DefaultResultLimit = "The default resultLimit is 5000";
 
         public const string SqlFilePath = "The SQL file path.";
+
+        public const string SparkConfigurationFolderPath = "The folder that this Spark job definition is in. If specify a multi-level path such as [rootFolder/subFolder], the Spark job definition will appear at the bottom level. If not specified, this Spark job definition will appear at the root level.";
+
+        public const string NoteBookFolderPath = "The folder that this notebook is in. If specify a multi-level path such as [rootFolder/subFolder], the notebook will appear at the bottom level. If not specified, this notebook will appear at the root level.";
     }
 }

src/Synapse/Synapse/Models/DataPlaneModels/Artifact/Notebooks/PSNotebook.cs

@@ -28,6 +28,7 @@ public PSNotebook(Notebook notebook)
             this.BigDataPool = new PSBigDataPoolReference(notebook?.BigDataPool);
             this.SessionProperties = new PSNotebookSessionProperties(notebook?.SessionProperties);
             this.Metadata = new PSNotebookMetadata(notebook?.Metadata);
+            this.Folder = new PSNotebookFolder(notebook?.Folder);
             this.NotebookFormat = notebook?.Nbformat;
             this.NotebookFormatMinor = notebook?.NbformatMinor;
             this.Cells = notebook?.Cells?.Select(element => new PSNotebookCell(element)).ToList();
@@ -46,6 +47,9 @@ public PSNotebook(Notebook notebook)
         [JsonProperty(PropertyName = "metadata")]
         public PSNotebookMetadata Metadata { get; set; }
 
+        [JsonIgnore]
+        public PSNotebookFolder Folder { get; set; }
+
         [DefaultValue(4)]
         [JsonProperty(DefaultValueHandling = DefaultValueHandling.Populate, PropertyName = "nbformat")]
         public int? NotebookFormat { get; set; }

src/Synapse/Synapse/Models/DataPlaneModels/Artifact/Notebooks/PSNotebookFolder.cs

@@ -0,0 +1,29 @@
+// ----------------------------------------------------------------------------------
+//
+// Copyright Microsoft Corporation
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// ----------------------------------------------------------------------------------
+
+using Azure.Analytics.Synapse.Artifacts.Models;
+using Newtonsoft.Json;
+
+namespace Microsoft.Azure.Commands.Synapse.Models
+{
+    public class PSNotebookFolder
+    {
+        public PSNotebookFolder(NotebookFolder folder)
+        {
+            this.Name = folder?.Name;
+        }
+
+        public string Name { get; set; }
+    }
+}

src/Synapse/Synapse/Models/DataPlaneModels/Artifact/SparkJobDefinitions/PSSparkJobDefinition.cs

@@ -24,6 +24,7 @@ public PSSparkJobDefinition(SparkJobDefinition properties)
             TargetBigDataPool = properties?.TargetBigDataPool != null ? new PSBigDataPoolReference(properties.TargetBigDataPool) : null;
             RequiredSparkVersion = properties?.RequiredSparkVersion;
             JobProperties = properties?.JobProperties != null ? new PSSparkJobProperties(properties.JobProperties) : null;
+            Folder = properties?.Folder != null ? new PSSparkJobDefinitionFolder(properties.Folder) : null;
         }
 
         /// <summary> The description of the Spark job definition. </summary>
@@ -40,5 +41,8 @@ public PSSparkJobDefinition(SparkJobDefinition properties)
 
         /// <summary> The properties of the Spark job. </summary>
         public PSSparkJobProperties JobProperties { get; set; }
+
+        /// <summary> The folder that this Spark job definition is in. If not specified, this Spark job definition will appear at the root level.</summary>
+        public PSSparkJobDefinitionFolder Folder { get; set; }
     }
 }

src/Synapse/Synapse/Models/DataPlaneModels/Artifact/SparkJobDefinitions/PSSparkJobDefinitionFolder.cs

@@ -0,0 +1,28 @@
+// ----------------------------------------------------------------------------------
+//
+// Copyright Microsoft Corporation
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// ----------------------------------------------------------------------------------
+
+using Azure.Analytics.Synapse.Artifacts.Models;
+
+namespace Microsoft.Azure.Commands.Synapse.Models
+{
+    public class PSSparkJobDefinitionFolder
+    {
+        public PSSparkJobDefinitionFolder(SparkJobDefinitionFolder folder)
+        {
+            this.Name = folder?.Name;
+        }
+
+        public string Name { get; set; }
+    }
+}

src/Synapse/Synapse/Models/DataPlaneModels/Artifact/SynapseAnalyticsArtifactsClient.cs

@@ -339,10 +339,10 @@ public BigDataPoolResourceInfo GetBigDataPool(string bigDataPoolName)
 
         #region Spark Job Definition
 
-        public SparkJobDefinitionResource CreateOrUpdateSparkJobDefinition(string SparkJobDefinitionName, string rawJsonContent)
+        public SparkJobDefinitionResource CreateOrUpdateSparkJobDefinition(string SparkJobDefinitionName, SparkJobDefinitionResource SparkJobDefinition)
         {
-            SparkJobDefinitionResource SparkJobDefinition = new SparkJobDefinitionResource(JsonConvert.DeserializeObject<SparkJobDefinition>(rawJsonContent));
-            return _sparkJobDefinitionClient.StartCreateOrUpdateSparkJobDefinition(SparkJobDefinitionName, SparkJobDefinition).Poll().Value;
+            var operation = _sparkJobDefinitionClient.StartCreateOrUpdateSparkJobDefinition(SparkJobDefinitionName, SparkJobDefinition);
+            return operation.Poll().Value;
         }
 
         public SparkJobDefinitionResource GetSparkJobDefinition(string SparkJobDefinitionName)

src/Synapse/Synapse/help/Set-AzSynapseNotebook.md

@@ -14,27 +14,28 @@ Creates or updates a notebook in a workspace.
 
 ### SetByName (Default)
 ```
-Set-AzSynapseNotebook -WorkspaceName <String> [-Name <String>] -DefinitionFile <String> [-AsJob]
- [-DefaultProfile <IAzureContextContainer>] [-WhatIf] [-Confirm] [<CommonParameters>]
+Set-AzSynapseNotebook -WorkspaceName <String> [-Name <String>] [-FolderPath <String>] -DefinitionFile <String>
+ [-AsJob] [-DefaultProfile <IAzureContextContainer>] [-WhatIf] [-Confirm] [<CommonParameters>]
 ```
 
 ### SetByNameAndSparkPool
 ```
-Set-AzSynapseNotebook -WorkspaceName <String> [-Name <String>] -SparkPoolName <String> [-ExecutorSize <String>]
- -ExecutorCount <Int32> -DefinitionFile <String> [-AsJob] [-DefaultProfile <IAzureContextContainer>] [-WhatIf]
- [-Confirm] [<CommonParameters>]
+Set-AzSynapseNotebook -WorkspaceName <String> [-Name <String>] [-FolderPath <String>] -SparkPoolName <String>
+ [-ExecutorSize <String>] -ExecutorCount <Int32> -DefinitionFile <String> [-AsJob]
+ [-DefaultProfile <IAzureContextContainer>] [-WhatIf] [-Confirm] [<CommonParameters>]
 ```
 
 ### SetByObject
 ```
-Set-AzSynapseNotebook -WorkspaceObject <PSSynapseWorkspace> [-Name <String>] -DefinitionFile <String> [-AsJob]
- [-DefaultProfile <IAzureContextContainer>] [-WhatIf] [-Confirm] [<CommonParameters>]
+Set-AzSynapseNotebook -WorkspaceObject <PSSynapseWorkspace> [-Name <String>] [-FolderPath <String>]
+ -DefinitionFile <String> [-AsJob] [-DefaultProfile <IAzureContextContainer>] [-WhatIf] [-Confirm]
+ [<CommonParameters>]
 ```
 
 ### SetByObjectAndSparkPool
 ```
-Set-AzSynapseNotebook -WorkspaceObject <PSSynapseWorkspace> [-Name <String>] -SparkPoolName <String>
- [-ExecutorSize <String>] -ExecutorCount <Int32> -DefinitionFile <String> [-AsJob]
+Set-AzSynapseNotebook -WorkspaceObject <PSSynapseWorkspace> [-Name <String>] [-FolderPath <String>]
+ -SparkPoolName <String> [-ExecutorSize <String>] -ExecutorCount <Int32> -DefinitionFile <String> [-AsJob]
  [-DefaultProfile <IAzureContextContainer>] [-WhatIf] [-Confirm] [<CommonParameters>]
 ```
 
@@ -56,13 +57,27 @@ This command creates or updates a notebook from notebook file notebook.ipynb in
 
 ### Example 2
 ```powershell
+PS C:\> Set-AzSynapseNotebook -WorkspaceName ContosoWorkspace -DefinitionFile "C:\\samples\\notebook.ipynb" -FolderPath ContosoFolder
+```
+
+This command creates or updates a notebook from notebook file notebook.ipynb and specify a folder path ContosoFolder where the notebook will be placed in the workspace named ContosoWorkspace.
+
+### Example 3
+```powershell
+PS C:\> Set-AzSynapseNotebook -WorkspaceName ContosoWorkspace -DefinitionFile "C:\\samples\\notebook.ipynb" -FolderPath ContosoFolder/SubFolder
+```
+
+This command creates or updates a notebook from notebook file notebook.ipynb and specify a multi-level folder path ContosoFolder/SubFolder where the notebook will be placed in the workspace named ContosoWorkspace.
+
+### Example 4
+```powershell
 PS C:\> $ws = Get-AzSynapseWorkspace -Name ContosoWorkspace
 PS C:\> $ws | Set-AzSynapseNotebook -DefinitionFile "C:\\samples\\notebook.ipynb"
 ```
 
 This command creates or updates a notebook from notebook file notebook.ipynb in the workspace named ContosoWorkspace through pipeline.
 
-### Example 3
+### Example 5
 ```powershell
 PS C:\> Set-AzSynapseNotebook -WorkspaceName ContosoWorkspace -DefinitionFile "C:\\samples\\notebook.ipynb" -SparkPoolName ContosoSparkPool -ExecutorCount 2
 ```
@@ -147,6 +162,21 @@ Accept pipeline input: False
 Accept wildcard characters: False
 ```
 
+### -FolderPath
+The folder that this notebook is in. If specify a multi-level path such as [rootFolder/subFolder], the notebook will appear at the bottom level. If not specified, this notebook will appear at the root level.
+
+```yaml
+Type: System.String
+Parameter Sets: (All)
+Aliases:
+
+Required: False
+Position: Named
+Default value: None
+Accept pipeline input: False
+Accept wildcard characters: False
+```
+
 ### -Name
 The notebook name.
 

src/Synapse/Synapse/help/Set-AzSynapseSparkJobDefinition.md

@@ -14,14 +14,16 @@ Creates a Spark job definition in workspace.
 
 ### SetByName (Default)
 ```
-Set-AzSynapseSparkJobDefinition -WorkspaceName <String> -Name <String> -DefinitionFile <String> [-AsJob]
- [-DefaultProfile <IAzureContextContainer>] [-WhatIf] [-Confirm] [<CommonParameters>]
+Set-AzSynapseSparkJobDefinition -WorkspaceName <String> -Name <String> -DefinitionFile <String>
+ [-FolderPath <String>] [-AsJob] [-DefaultProfile <IAzureContextContainer>] [-WhatIf] [-Confirm]
+ [<CommonParameters>]
 ```
 
 ### SetByObject
 ```
 Set-AzSynapseSparkJobDefinition -WorkspaceObject <PSSynapseWorkspace> -Name <String> -DefinitionFile <String>
- [-AsJob] [-DefaultProfile <IAzureContextContainer>] [-WhatIf] [-Confirm] [<CommonParameters>]
+ [-FolderPath <String>] [-AsJob] [-DefaultProfile <IAzureContextContainer>] [-WhatIf] [-Confirm]
+ [<CommonParameters>]
 ```
 
 ## DESCRIPTION
@@ -39,6 +41,22 @@ The command bases the Spark job definition on information in the sparkJobDefinit
 
 ### Example 2
 ```powershell
+PS C:\> Set-AzSynapseSparkJobDefinition -WorkspaceName ContosoWorkspace -Name ContosoSparkJobDefinition -DefinitionFile "C:\sparkJobDefinition.json" -FolderPath ContosoFolder
+```
+
+This command creates a Spark job definition named ContosoSparkJobDefinition and specify a folder path ContosoFolder where the spark job definition will be placed in the workspace named ContosoWorkspace.
+The command bases the Spark job definition on information in the sparkJobDefinition.json file.
+
+### Example 3
+```powershell
+PS C:\> Set-AzSynapseSparkJobDefinition -WorkspaceName ContosoWorkspace -Name ContosoSparkJobDefinition -DefinitionFile "C:\sparkJobDefinition.json" -FolderPath ContosoFolder/SubFolder
+```
+
+This command creates a Spark job definition named ContosoSparkJobDefinition and specify a multi-level folder path ContosoFolder/SubFolder where the spark job definition will be placed in the workspace named ContosoWorkspace.
+The command bases the Spark job definition on information in the sparkJobDefinition.json file.
+
+### Example 4
+```powershell
 PS C:\> $ws = Get-AzSynapseWorkspace -Name ContosoWorkspace
 PS C:\> $ws | Set-AzSynapseSparkJobDefinition -Name ContosoSparkJobDefinition -DefinitionFile "C:\sparkJobDefinition.json"
 ```
@@ -93,6 +111,21 @@ Accept pipeline input: False
 Accept wildcard characters: False
 ```
 
+### -FolderPath
+The folder that this Spark job definition is in. If specify a multi-level path such as [rootFolder/subFolder], the Spark job definition will appear at the bottom level. If not specified, this Spark job definition will appear at the root level.
+
+```yaml
+Type: System.String
+Parameter Sets: (All)
+Aliases:
+
+Required: False
+Position: Named
+Default value: None
+Accept pipeline input: False
+Accept wildcard characters: False
+```
+
 ### -Name
 The Spark job definition name.