pull in feature/modelSerializer so ci proj ref legs are happy

eng/Packages.Data.props

@@ -89,7 +89,7 @@
     <PackageReference Update="Azure.Core.Expressions.DataFactory" Version="1.0.0-beta.4" />
     <PackageReference Update="Azure.Data.SchemaRegistry" Version="1.2.0" />
     <PackageReference Update="Azure.Data.Tables" Version="12.8.0" />
-    <PackageReference Update="Azure.Messaging.EventHubs" Version="5.9.2" />
+    <PackageReference Update="Azure.Messaging.EventHubs" Version="5.9.3" />
     <PackageReference Update="Azure.Messaging.EventGrid" Version="4.17.0" />
     <PackageReference Update="Azure.Messaging.ServiceBus" Version="7.16.0" />
     <PackageReference Update="Azure.Messaging.WebPubSub" Version="1.2.0" />
@@ -142,9 +142,9 @@
     <PackageReference Update="CloudNative.CloudEvents.SystemTextJson" Version="2.0.0" />
     <PackageReference Update="MessagePack" Version="1.9.11" />
     <PackageReference Update="Microsoft.AspNetCore.SignalR.Protocols.MessagePack" Version="1.1.5" />
-    <PackageReference Update="Microsoft.Azure.SignalR" Version="1.21.4" />
-    <PackageReference Update="Microsoft.Azure.SignalR.Management" Version="1.21.4" />
-    <PackageReference Update="Microsoft.Azure.SignalR.Protocols" Version="1.21.4" />
+    <PackageReference Update="Microsoft.Azure.SignalR" Version="1.21.6" />
+    <PackageReference Update="Microsoft.Azure.SignalR.Management" Version="1.21.6" />
+    <PackageReference Update="Microsoft.Azure.SignalR.Protocols" Version="1.21.6" />
     <PackageReference Update="Microsoft.Azure.SignalR.Serverless.Protocols" Version="1.9.0" />
     <PackageReference Update="Microsoft.Azure.WebJobs" Version="3.0.37" />
     <PackageReference Update="Microsoft.Azure.WebJobs.Sources" Version="3.0.37" />
@@ -174,7 +174,7 @@
     All should have PrivateAssets="All" set so they don't become package dependencies
   -->
   <ItemGroup>
-    <PackageReference Update="Microsoft.Azure.AutoRest.CSharp" Version="3.0.0-beta.20230907.2" PrivateAssets="All" />
+    <PackageReference Update="Microsoft.Azure.AutoRest.CSharp" Version="3.0.0-beta.20230912.3" PrivateAssets="All" />
     <PackageReference Update="Azure.ClientSdk.Analyzers" Version="0.1.1-dev.20230822.1" PrivateAssets="All" />
     <PackageReference Update="coverlet.collector" Version="3.2.0" PrivateAssets="All" />
     <PackageReference Update="Microsoft.CodeAnalysis.NetAnalyzers" Version="7.0.1" PrivateAssets="All" />

eng/common/scripts/Service-Level-Readme-Automation.ps1

@@ -1,12 +1,12 @@
 <#
 .SYNOPSIS
-The script is to generate service level readme if it is missing. 
+The script is to generate service level readme if it is missing.
 For exist ones, we do 2 things here:
 1. Generate the client but not import to the existing service level readme.
 2. Update the metadata of service level readme
 
 .DESCRIPTION
-Given a doc repo location, and the credential for fetching the ms.author. 
+Given a doc repo location, and the credential for fetching the ms.author.
 Generate missing service level readme and updating metadata of the existing ones.
 
 .PARAMETER DocRepoLocation
@@ -40,7 +40,10 @@ param(
   [string]$ClientSecret,
 
   [Parameter(Mandatory = $false)]
-  [string]$ReadmeFolderRoot = "docs-ref-services"
+  [string]$ReadmeFolderRoot = "docs-ref-services",
+
+  [Parameter(Mandatory = $false)]
+  [array]$Monikers = @('latest', 'preview', 'legacy')
 )
 . $PSScriptRoot/common.ps1
 . $PSScriptRoot/Helpers/Service-Level-Readme-Automation-Helpers.ps1
@@ -50,12 +53,11 @@ param(
 Set-StrictMode -Version 3
 
 $fullMetadata = Get-CSVMetadata
-$monikers = @("latest", "preview")
-foreach($moniker in $monikers) {
+foreach($moniker in $Monikers) {
   # The onboarded packages return is key-value pair, which key is the package index, and value is the package info from {metadata}.json
-  # E.g. 
+  # E.g.
   # Key as: @azure/storage-blob
-  # Value as: 
+  # Value as:
   # {
   #   "Name": "@azure/storage-blob",
   #   "Version": "12.10.0-beta.1",
@@ -73,22 +75,38 @@ foreach($moniker in $monikers) {
   $onboardedPackages = &$GetOnboardedDocsMsPackagesForMonikerFn `
     -DocRepoLocation $DocRepoLocation -moniker $moniker
   $csvMetadata = @()
+
   foreach($metadataEntry in $fullMetadata) {
     if ($metadataEntry.Package -and $metadataEntry.Hide -ne 'true') {
       $pkgKey = GetPackageKey $metadataEntry
-      if($onboardedPackages.ContainsKey($pkgKey)) {
-        if ($onboardedPackages[$pkgKey] -and $onboardedPackages[$pkgKey].DirectoryPath) {
-          if (!($metadataEntry.PSObject.Members.Name -contains "DirectoryPath")) {
-            Add-Member -InputObject $metadataEntry `
-              -MemberType NoteProperty `
-              -Name DirectoryPath `
-              -Value $onboardedPackages[$pkgKey].DirectoryPath
-          }
-        }
+
+      if (!$onboardedPackages.ContainsKey($pkgKey)) {
+        continue
+      }
+
+      $package = $onboardedPackages[$pkgKey]
+
+      if (!$package) {
         $csvMetadata += $metadataEntry
+        continue
       }
+
+      # If the metadata JSON entry has a DirectoryPath, but the CSV entry
+      # does not, add the DirectoryPath to the CSV entry
+      if (($package.PSObject.Members.Name -contains 'DirectoryPath') `
+        -and !($metadataEntry.PSObject.Members.Name -contains "DirectoryPath") ) {
+
+        Add-Member -InputObject $metadataEntry `
+          -MemberType NoteProperty `
+          -Name DirectoryPath `
+          -Value $package.DirectoryPath
+      }
+
+      $csvMetadata += $metadataEntry
+
     }
   }
+
   $packagesForService = @{}
   $allPackages = GetPackageLookup $csvMetadata
   foreach ($metadataKey in $allPackages.Keys) {

eng/common/scripts/Update-DocsMsPackageMonikers.ps1

@@ -0,0 +1,122 @@
+<#
+.SYNOPSIS
+Move metadata JSON and package-level overview markdown files for deprecated packages to the legacy folder.
+
+.DESCRIPTION
+Move onboarding information to the "legacy" moniker for whose support is "deprecated" in the Metadata CSV.
+Only one version of a package can be documented in the "legacy" moniker. If multiple versions are available,
+the "latest" version will be used and the "preview" version will be deleted.
+
+.PARAMETER DocRepoLocation
+The location of the target docs repository.
+#>
+
+param(
+    [Parameter(Mandatory = $true)]
+    [string] $DocRepoLocation
+)
+
+. (Join-Path $PSScriptRoot common.ps1)
+
+Set-StrictMode -Version 3
+
+function getPackageMetadata($moniker) { 
+    $jsonFiles = Get-ChildItem -Path (Join-Path $DocRepoLocation "metadata/$moniker") -Filter *.json
+    $metadata = @{}
+
+    foreach ($jsonFile in $jsonFiles) {
+        $packageMetadata = Get-Content $jsonFile -Raw | ConvertFrom-Json -AsHashtable
+        $packageIdentity = $packageMetadata.Name
+        if (Test-Path "Function:$GetPackageIdentity") {
+            $packageIdentity = &$GetPackageIdentity $packageMetadata
+        }
+
+        $metadata[$packageIdentity] = @{ File = $jsonFile; Metadata = $packageMetadata }
+    }
+
+    return $metadata
+}
+
+function getPackageInfoFromLookup($packageIdentity, $version, $lookupTable) { 
+    if ($lookupTable.ContainsKey($packageIdentity)) {
+        if ($lookupTable[$packageIdentity]['Metadata'].Version -eq $version) {
+            # Only return if the version matches
+            return $lookupTable[$packageIdentity]
+        }
+    }
+
+    return $null 
+}
+
+function moveToLegacy($packageInfo) { 
+    $docsMsMetadata = &$GetDocsMsMetadataForPackageFn -PackageInfo $packageInfo['Metadata']
+
+    Write-Host "Move to legacy: $($packageInfo['Metadata'].Name)"
+    $packageInfoPath = $packageInfo['File']
+    Move-Item "$($packageInfoPath.Directory)/$($packageInfoPath.BaseName).*" "$DocRepoLocation/metadata/legacy/" -Force
+
+    $readmePath = "$DocRepoLocation/$($docsMsMetadata.PreviewReadMeLocation)/$($docsMsMetadata.DocsMsReadMeName)-readme.md"
+    if (Test-Path $readmePath) {
+        Move-Item `
+            $readmePath `
+            "$DocRepoLocation/$($docsMsMetadata.LegacyReadMeLocation)/" `
+            -Force
+    }
+}
+
+function deletePackageInfo($packageInfo) { 
+    $docsMsMetadata = &$GetDocsMsMetadataForPackageFn -PackageInfo $packageInfo['Metadata']
+
+    Write-Host "Delete superseded package: $($packageInfo['Metadata'].Name)"
+    $packageInfoPath = $packageInfo['File']
+    Remove-Item "$($packageInfoPath.Directory)/$($packageInfoPath.BaseName).*" -Force
+
+    $readmePath = "$DocRepoLocation/$($docsMsMetadata.PreviewReadMeLocation)/$($docsMsMetadata.DocsMsReadMeName)-readme.md"
+    if (Test-Path $readmePath) {
+        Remove-Item $readmePath -Force
+    }
+}
+
+$metadataLookup = @{ 
+    'latest'  = getPackageMetadata 'latest'
+    'preview' = getPackageMetadata 'preview'
+}
+$deprecatedPackages = (Get-CSVMetadata).Where({ $_.Support -eq 'deprecated' })
+
+foreach ($package in $deprecatedPackages) {
+    $packageIdentity = $package.Package
+    if (Test-Path "Function:$GetPackageIdentityFromCsvMetadata") {
+        $packageIdentity = &$GetPackageIdentityFromCsvMetadata $package
+    }
+
+    $packageInfoPreview = $packageInfoLatest = $null
+    if ($package.VersionPreview) {
+        $packageInfoPreview = getPackageInfoFromLookup `
+            -packageIdentity $packageIdentity `
+            -version $package.VersionPreview `
+            -lookupTable $metadataLookup['preview']
+    }
+
+    if ($package.VersionGA) { 
+        $packageInfoLatest = getPackageInfoFromLookup `
+            -packageIdentity $packageIdentity `
+            -version $package.VersionGA `
+            -lookupTable $metadataLookup['latest']
+    }
+
+    if (!$packageInfoPreview -and !$packageInfoLatest) {
+        # Nothing to move or delete
+        continue
+    }
+
+    if ($packageInfoPreview -and $packageInfoLatest) {
+        # Delete metadata JSON and package-level overview markdown files for
+        # the preview version instead of moving both. This mitigates situations
+        # where the "latest" verison doesn't have a package-level overview 
+        # markdown file and the "preview" version does.
+        deletePackageInfo $packageInfoPreview
+        moveToLegacy $packageInfoLatest
+    } else {
+        moveToLegacy ($packageInfoPreview ?? $packageInfoLatest)
+    }
+}

eng/common/scripts/Verify-RequiredDocsJsonMembers.ps1

@@ -91,6 +91,16 @@ function Test-RequiredDocsJsonMembers($moniker) {
       $script:FoundError = $true
     }
 
+    if ($fileObject.PSObject.Members.Name -contains 'DirectoryPath') {
+      if ($null -eq $fileObject.DirectoryPath) { 
+        Write-Host "$path has a null DirectoryPath member. If the DirectoryPath is unknown please use the value `"`"."
+        $script:FoundError = $true
+      }
+    } else { 
+      Write-Host "$path is missing its DirectoryPath member. If the DirectoryPath is unknown please use the value `"`"."
+      $script:FoundError = $true
+    }
+
     if ($Language -eq "java") {
       if ($fileObject.PSObject.Members.Name -contains "Group")
       {

eng/common/scripts/common.ps1

@@ -67,3 +67,4 @@ $GetEmitterPackageLockPathFn = "Get-${Language}-EmitterPackageLockPath"
 $SetDocsPackageOnboarding = "Set-${Language}-DocsPackageOnboarding"
 $GetDocsPackagesAlreadyOnboarded = "Get-${Language}-DocsPackagesAlreadyOnboarded"
 $GetPackageIdentity = "Get-${Language}-PackageIdentity"
+$GetPackageIdentityFromCsvMetadata = "Get-${Language}-PackageIdentityFromCsvMetadata"

eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1

@@ -401,7 +401,7 @@ function CheckDependencies()
         $job | Remove-Job -Force
 
         if (($result -eq $null -and $job.State -ne "Completed") -or ($result | Select -Last 1) -ne 0) {
-            throw "Docker does not appear to be running. Start/restart docker."
+            throw "Docker does not appear to be running. Start/restart docker or re-run this script with -SkipPushImages"
         }
     }
 

eng/containers/UbuntuNetCoreKeyring/Dockerfile

@@ -1,4 +1,4 @@
-FROM mcr.microsoft.com/powershell:7.2.1-ubuntu-focal AS build
+FROM mcr.microsoft.com/azure-powershell:10.3.0-ubuntu-20.04 AS build
 
 ENV \
     NO_AT_BRIDGE=1 \
@@ -8,7 +8,7 @@ ENV \
     # Do not generate certificate
     DOTNET_GENERATE_ASPNET_CERTIFICATE=false \
     # SDK version
-    DOTNET_SDK_VERSION_5_0=5.0.404 \
+    DOTNET_SDK_VERSION_6_0=6.0.413 \
     DOTNET_SDK_VERSION_3_1=3.1.416 \
     # Enable correct mode for dotnet watch (only mode supported in a container)
     DOTNET_USE_POLLING_FILE_WATCHER=true \
@@ -18,7 +18,7 @@ ENV \
     POWERSHELL_DISTRIBUTION_CHANNEL=PSDocker-DotnetSDK-Ubuntu-20.04 \
     # Setup Dotnet envs
     DOTNET_ROOT=/usr/share/dotnet  \
-    PATH=$PATH:usr/share/dotnet    
+    PATH=$PATH:usr/share/dotnet
 
 # Install apt-add-repository
 RUN apt-get update && apt-get install -y software-properties-common
@@ -39,8 +39,8 @@ RUN apt-add-repository ppa:git-core/ppa \
 
 # Below adapated from https://hub.docker.com/_/microsoft-dotnet-sdk
 # https://github.com/dotnet/dotnet-docker/blob/b20c03e0644b42167d66a85fe6077ec2428a47fa/src/sdk/5.0/focal/amd64/Dockerfile
-RUN curl -fSL --output dotnet.tar.gz https://dotnetcli.azureedge.net/dotnet/Sdk/$DOTNET_SDK_VERSION_5_0/dotnet-sdk-$DOTNET_SDK_VERSION_5_0-linux-x64.tar.gz \
-    && dotnet_sha512='6f9b83b2b661ce3b033a04d4c50ff3a435efa288de1a48f58be1150e64c5dd9d6bd2a4bf40f697dcd7d64ffaac24f14cc4a874e738544c5d0e8113c474fd2ee0' \
+RUN curl -fSL --output dotnet.tar.gz https://dotnetcli.azureedge.net/dotnet/Sdk/$DOTNET_SDK_VERSION_6_0/dotnet-sdk-$DOTNET_SDK_VERSION_6_0-linux-x64.tar.gz \
+    && dotnet_sha512='ee0a77d54e6d4917be7310ff0abb3bad5525bfb4beb1db0c215e65f64eb46511f5f12d6c7ff465a1d4ab38577e6a1950fde479ee94839c50e627020328a702de' \
     && echo "$dotnet_sha512  dotnet.tar.gz" | sha512sum -c - \
     && mkdir -p /usr/share/dotnet \
     && tar -ozxf dotnet.tar.gz -C /usr/share/dotnet \

eng/emitter-package.json

@@ -1,6 +1,6 @@
 {
     "main": "dist/src/index.js",
     "dependencies": {
-      "@azure-tools/typespec-csharp": "0.2.0-beta.20230907.2"
+      "@azure-tools/typespec-csharp": "0.2.0-beta.20230912.3"
     }
 }

eng/pipelines/docindex.yml

@@ -38,12 +38,22 @@ jobs:
           Repositories:
             - Name: $(DocRepoOwner)/$(DocRepoName)
               WorkingDirectory: $(DocRepoLocation)
+
       # Install the tool for toc step to use
       - task: NuGetToolInstaller@1
         inputs:
           versionSpec: 6.3.1
-      # Call update docs ci script to onboard packages
+
       - template: /eng/common/pipelines/templates/steps/set-default-branch.yml
+
+      - task: Powershell@2
+        inputs:
+          pwsh: true
+          filePath: eng/common/scripts/Update-DocsMsPackageMonikers.ps1
+          arguments: -DocRepoLocation $(DocRepoLocation)
+        displayName: Move deprecated packages to legacy moniker
+        condition: and(succeeded(), or(eq(variables['Build.Reason'], 'Schedule'), eq(variables['Force.MainUpdate'], 'true')))
+
       - task: Powershell@2
         inputs:
           pwsh: true
@@ -109,6 +119,14 @@ jobs:
         displayName: Set package source variable
         workingDirectory: $(DocRepoLocation)
 
+      - task: Powershell@2
+        inputs:
+          pwsh: true
+          filePath: eng/common/scripts/Update-DocsMsPackageMonikers.ps1
+          arguments: -DocRepoLocation $(DocRepoLocation)
+        displayName: Move deprecated packages to legacy moniker
+        condition: and(succeeded(), or(eq(variables['Build.Reason'], 'Schedule'), eq(variables['Force.MainUpdate'], 'true')))
+
       - task: Powershell@2
         inputs:
           pwsh: true

eng/scripts/Language-Settings.ps1

@@ -298,6 +298,7 @@ function Get-dotnet-DocsMsMetadataForPackage($PackageInfo) {
     DocsMsReadMeName = $readmeName
     LatestReadMeLocation = 'api/overview/azure/latest'
     PreviewReadMeLocation = 'api/overview/azure/preview'
+    LegacyReadMeLocation = 'api/overview/azure/legacy'
     Suffix = ''
   }
 }
@@ -494,6 +495,25 @@ function UpdateDocsMsPackages($DocConfigFile, $Mode, $DocsMetadata) {
       continue
     }
 
+    if ($matchingPublishedPackage.Support -eq 'deprecated') { 
+      if ($Mode -eq 'legacy') { 
+
+        # Select the GA version, if none use the preview version
+        $updatedVersion = $matchingPublishedPackage.VersionGA.Trim()
+        if (!$updatedVersion) { 
+          $updatedVersion = $matchingPublishedPackage.VersionPreview.Trim()
+        }
+        $package.Versions = @($updatedVersion)
+
+        Write-Host "Add deprecated package to legacy moniker: $($package.Name)"
+        $outputPackages += $package
+      } else { 
+        Write-Host "Removing deprecated package: $($package.Name)"
+      }
+
+      continue
+    }
+
     $updatedVersion = $matchingPublishedPackage.VersionGA.Trim()
     if ($Mode -eq 'preview') {
       $updatedVersion = $matchingPublishedPackage.VersionPreview.Trim()