Codesigning support for scripts in Tools

.github/workflows/create-prerelase.yml

@@ -8,6 +8,7 @@ on:
   push:
     paths: 
       - 'Scripts/**'
+      - 'Tools/**'
     branches: [ dev ]
 
 jobs:
@@ -16,22 +17,28 @@ jobs:
     if: "!contains(github.event.head_commit.message, '[no release]')"
     runs-on: windows-2019
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
-          fetch-depth: 0
+          fetch-depth: 1
       - name: Build scripts 
         id: build
         shell: powershell
         run: |
-          ./build.ps1 -Version prerelease
+          ./build.ps1 -Version dev
           $filename = "mslab_dev-$((Get-Date -Format "yyyyMMdd")).zip"
           mv ./Release.zip $filename
-          echo "::set-output name=filename::$filename"
-      - uses: "marvinpinto/action-automatic-releases@latest"
+          echo "filename=$filename" >> $env:GITHUB_OUTPUT
+      - name: Delete current dev prerelease
+        uses: cb80/delrel@latest
         with:
-          repo_token: "${{ secrets.GITHUB_TOKEN }}"
-          automatic_release_tag: "dev"
+          tag: dev
+      - name: Create new dev prerelease
+        uses: softprops/action-gh-release@v1
+        with:
+          tag_name: dev
+          name: dev branch preview
+          generate_release_notes: true
           prerelease: true
-          title: "dev branch preview"
           files: |
             ${{ steps.build.outputs.filename }}
+            Output/Tools/*.ps1

.github/workflows/create-release.yml

@@ -8,25 +8,28 @@ on:
   push:
     paths: 
       - 'Scripts/**'
+      - 'Tools/**'
     branches: [ master ]
 
 jobs:
   new-version:
+    environment: release
     name: Bump version
     if: "!contains(github.event.head_commit.message, '[no release]')"
     runs-on: windows-2019
     outputs:
       previous_tag: ${{ steps.bump.outputs.previous_tag }}
       new_tag: ${{ steps.bump.outputs.new_tag }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - id: bump
         name: Bump version
         run: |
           $today = Get-Date
           $newVersion = @($today.ToString("yy"), $today.ToString("MM"), "1")
           git fetch --tags
-          $hash = git rev-list --tags --topo-order --max-count=1
+          # Get the latest tag that matches our versioning schema (starts with letter v)
+          $hash = git rev-list --tags=v* --topo-order --max-count=1
           if($hash) {
             $currentTag = git describe --tags $hash
             $parts = $currentTag.Substring(1) -split '\.'
@@ -35,72 +38,67 @@ jobs:
 
           $newTag = "v" + ($newVersion -join ".")
           git tag $newTag
+          if(-not $?) {
+            throw "Tagging of new release version failed!"
+          }
+
+          git push origin $newTag
+
           "New version: $newTag"
-          echo "::set-output name=previous_tag::$currentTag"
-          echo "::set-output name=new_tag::$newTag"
-
-      - name: Push version tag
-        uses: ad-m/github-push-action@master
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          tags: true
+          echo "previous_tag=$currentTag" >> $env:GITHUB_OUTPUT
+          echo "new_tag=$newTag" >> $env:GITHUB_OUTPUT
 
   new-release:
     name: Create release
     if: "!contains(github.event.head_commit.message, '[no release]')"
-    runs-on: windows-2019
+    runs-on: self-hosted
     needs: new-version
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           fetch-depth: 0
-      - uses: azure/login@v1
-        with:
-          creds: ${{ secrets.AZURE_CREDENTIALS }}
-          enable-AzPSSession: true
-      - name: "Build scripts"
-        uses: azure/powershell@v1
+      - name: Build and sign release scripts
+        shell: pwsh
         env:
           SIGN_SCRIPT_URI: ${{ secrets.SIGN_SCRIPT_URI }}
-          CLIENT_ID: ${{ secrets.CLIENT_ID }} # just to ofusctate it in the output
-        with:
-          azPSVersion: "latest"
-          inlineScript: |
-            ./build.ps1 -Version ${{ needs.new-version.outputs.new_tag }} -SignScripts $true -SignScriptUri $env:SIGN_SCRIPT_URI -ClientId $env:CLIENT_ID
+          CLIENT_ID: ${{ secrets.CLIENT_ID }} # just to obfusctate it in the output
+        run: |
+          ./build.ps1 -Version ${{ needs.new-version.outputs.new_tag }} -SignScripts $true -SignScriptUri $env:SIGN_SCRIPT_URI -ClientId $env:CLIENT_ID
+          Move-Item ./Release.zip mslab_${{ needs.new-version.outputs.new_tag }}.zip
       - name: Create changelog
         id: changelog
         shell: powershell
         run: |
           if("${{ needs.new-version.outputs.previous_tag }}" -ne "") {
-            $changelog = (& { git log ${{ needs.new-version.outputs.previous_tag }}..HEAD --pretty=format:'- %s (%h)' --abbrev-commit -- Scripts }) -join '%0D%0A'
+            $changelog = (& { git log ${{ needs.new-version.outputs.previous_tag }}..HEAD --pretty=format:'- %s (%h)' --abbrev-commit -- Scripts Tools }) -join "`n"
             "Changes for ${{ needs.new-version.outputs.previous_tag }} are:"
             $changelog
           } else {
             $changelog = ""
           }
-          echo "::set-output name=changelog::$changelog"
-
-      - name: Create Github Release
-        id: create_release
-        uses: actions/create-release@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+          $changeLogContent = @"
+          :package: All MSLab scripts are in **mslab_${{ needs.new-version.outputs.new_tag }}.zip** file. 
+
+          :information_source: Remaining `.ps1` files in this release would be downloaded on-demand by MSLab scripts during deployment, only if needed.
+          "@
+
+          if($changelog -ne "") {
+            $changeLogContent += @"
+
+          :basket: Changes in this version:
+          $changelog
+          "@
+          }
+
+          Set-Content -Value $changeLogContent -Path .\changelog.md
+      - name: Create new release
+        uses: softprops/action-gh-release@v1
         with:
           tag_name: ${{ needs.new-version.outputs.new_tag }} # ${{ github.ref }}
-          release_name: Release ${{ needs.new-version.outputs.new_tag }} # ${{ github.ref }}
-          body: |
-            Changes in this version:
-            ${{ steps.changelog.outputs.changelog }}
-          draft: false
-          prerelease: false
-
-      - name: Upload ZIP to Release
-        id: upload-scripts
-        uses: actions/upload-release-asset@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          upload_url: ${{ steps.create_release.outputs.upload_url }}
-          asset_path: ./Release.zip
-          asset_name: mslab_${{ needs.new-version.outputs.new_tag }}.zip
-          asset_content_type: application/zip
+          name: Release ${{ needs.new-version.outputs.new_tag }} # ${{ github.ref }}
+          generate_release_notes: true
+          body_path: changelog.md
+          files: |
+            mslab_${{ needs.new-version.outputs.new_tag }}.zip
+            Output/Tools/*.ps1

Scripts/1_Prereq.ps1

@@ -83,37 +83,58 @@ function  Get-WindowsBuildNumber {
 #region Download Scripts
 
 #add scripts for VMM
-    $Filenames="1_SQL_Install","2_ADK_Install","3_SCVMM_Install"
-    foreach ($Filename in $filenames){
-        $Path="$PSScriptRoot\Temp\ToolsVHD\SCVMM\$Filename.ps1"
-        If (Test-Path -Path $Path){
+    $filenames = "1_SQL_Install", "2_ADK_Install", "3_SCVMM_Install"
+    foreach ($filename in $filenames) {
+        $Path = "$PSScriptRoot\Temp\ToolsVHD\SCVMM\$filename.ps1"
+        if (Test-Path -Path $Path) {
             WriteSuccess "`t $Filename is present, skipping download"
-        }else{
-            $FileContent=$null
-            $FileContent = (Invoke-WebRequest -UseBasicParsing -Uri "https://raw.githubusercontent.com/Microsoft/MSLab/master/Tools/$Filename.ps1").Content
-            if ($FileContent){
+        } else {
+            $FileContent = $null
+
+            try {
+                # try to download tagged version first
+                $FileContent = (Invoke-WebRequest -UseBasicParsing -Uri "https://raw.githubusercontent.com/microsoft/MSLab/$mslabVersion/Tools/$filename.ps1").Content
+            } catch {
+                WriteInfo "Download $filename failed with $($_.Exception.Message), trying master branch now"
+                # if that fails, try master branch
+                $FileContent = (Invoke-WebRequest -UseBasicParsing -Uri "https://raw.githubusercontent.com/microsoft/MSLab/master/Tools/$filename.ps1").Content
+            }
+
+            if ($FileContent) {
                 $script = New-Item $Path -type File -Force
                 $FileContent=$FileContent -replace "PasswordGoesHere",$LabConfig.AdminPassword #only applies to 1_SQL_Install and 3_SCVMM_Install.ps1
                 $FileContent=$FileContent -replace "DomainNameGoesHere",$LabConfig.DomainNetbiosName #only applies to 1_SQL_Install and 3_SCVMM_Install.ps1
                 Set-Content -path $script -value $FileContent
-            }else{
+            } else {
                 WriteErrorAndExit "Unable to download $Filename."
             }
         }
     }
 
 # add createparentdisks, DownloadLatestCU and PatchParentDisks scripts to Parent Disks folder
-    $FileNames = "CreateParentDisk", "DownloadLatestCUs", "PatchParentDisks", "CreateVMFleetDisk"
+    $fileNames = "CreateParentDisk", "DownloadLatestCUs", "PatchParentDisks", "CreateVMFleetDisk"
     if($LabConfig.Linux) {
-        $FileNames += "CreateLinuxParentDisk"
+        $fileNames += "CreateLinuxParentDisk"
     }
-    foreach ($filename in $filenames) {
+    foreach ($filename in $fileNames) {
         $Path="$PSScriptRoot\ParentDisks\$FileName.ps1"
         If (Test-Path -Path $Path) {
             WriteSuccess "`t $Filename is present, skipping download"
         } else {
             $FileContent = $null
-            $FileContent = (Invoke-WebRequest -UseBasicParsing -Uri "https://raw.githubusercontent.com/Microsoft/MSLab/master/Tools/$FileName.ps1").Content
+
+            try {
+                # try to download release version first
+                $file = (Invoke-WebRequest -UseBasicParsing -Uri "https://github.com/microsoft/MSLab/releases/download/$mslabVersion/$Filename.ps1")
+                if($file.Headers["Content-Type"] -eq "application/octet-stream") {
+                    $FileContent = [System.Text.Encoding]::UTF8.GetString($file.Content)
+                }
+            } catch {
+                WriteInfo "Download $filename failed with $($_.Exception.Message), trying master branch now"
+                # if that fails, try main branch
+                $FileContent = (Invoke-WebRequest -UseBasicParsing -Uri "https://raw.githubusercontent.com/microsoft/MSLab/master/Tools/$FileName.ps1").Content
+            }
+
             if ($FileContent) {
                 $script = New-Item "$PSScriptRoot\ParentDisks\$FileName.ps1" -type File -Force
                 Set-Content -path $script -value $FileContent
@@ -124,17 +145,23 @@ function  Get-WindowsBuildNumber {
     }
 
 # Download convert-windowsimage into Temp
-WriteInfoHighlighted "Testing Convert-windowsimage presence"
-If ( Test-Path -Path "$PSScriptRoot\Temp\Convert-WindowsImage.ps1" ) {
-    WriteSuccess "`t Convert-windowsimage.ps1 is present, skipping download"
-}else{ 
-    WriteInfo "`t Downloading Convert-WindowsImage"
-    try {
-        Invoke-WebRequest -UseBasicParsing -Uri "https://raw.githubusercontent.com/microsoft/MSLab/master/Tools/Convert-WindowsImage.ps1" -OutFile "$PSScriptRoot\Temp\Convert-WindowsImage.ps1"
-    } catch {
-        WriteError "`t Failed to download Convert-WindowsImage.ps1!"
+    WriteInfoHighlighted "Testing Convert-windowsimage presence"
+    $convertWindowsImagePath = "$PSScriptRoot\Temp\Convert-WindowsImage.ps1"
+    If (Test-Path -Path $convertWindowsImagePath) {
+        WriteSuccess "`t Convert-windowsimage.ps1 is present, skipping download"
+    } else {
+        WriteInfo "`t Downloading Convert-WindowsImage"
+        try {
+            Invoke-WebRequest -UseBasicParsing -Uri "https://github.com/microsoft/MSLab/releases/download/$mslabVersion/Convert-WindowsImage.ps1" -OutFile $convertWindowsImagePath
+        } catch {
+            try {
+                WriteInfo "Download Convert-windowsimage.ps1 failed with $($_.Exception.Message), trying master branch now"
+                Invoke-WebRequest -UseBasicParsing -Uri "https://raw.githubusercontent.com/microsoft/MSLab/master/Tools/Convert-WindowsImage.ps1" -OutFile $convertWindowsImagePath
+            } catch {
+                WriteError "`t Failed to download Convert-WindowsImage.ps1!"
+            }
+        }
     }
-}
 #endregion
 
 #region some tools to download

Tools/CreateParentDisk.ps1

@@ -40,13 +40,18 @@ If (-not $isAdmin) {
     #endregion
 
     #region download convert-windowsimage if needed and load it
-
-    if (!(Test-Path "$PSScriptRoot\Convert-WindowsImage.ps1")){
+    $convertWindowsImagePath = "$PSScriptRoot\Convert-WindowsImage.ps1"
+    if (-not (Test-Path -Path $convertWindowsImagePath)) {
         WriteInfo "`t Downloading Convert-WindowsImage"
         try {
-            Invoke-WebRequest -UseBasicParsing -Uri "https://raw.githubusercontent.com/microsoft/MSLab/master/Tools/Convert-WindowsImage.ps1" -OutFile "$PSScriptRoot\Convert-WindowsImage.ps1"
+            Invoke-WebRequest -UseBasicParsing -Uri "https://github.com/microsoft/MSLab/releases/download/$mslabVersion/Convert-WindowsImage.ps1" -OutFile $convertWindowsImagePath
         } catch {
-            WriteErrorAndExit "`t Failed to download Convert-WindowsImage.ps1!"
+            try {
+                WriteInfo "Download Convert-windowsimage.ps1 from releases ($mslabVersion) failed with $($_.Exception.Message), trying master branch now"
+                Invoke-WebRequest -UseBasicParsing -Uri "https://raw.githubusercontent.com/microsoft/MSLab/master/Tools/Convert-WindowsImage.ps1" -OutFile $convertWindowsImagePath
+            } catch {
+                WriteError "`t Failed to download Convert-WindowsImage.ps1!"
+            }
         }
     }