diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 8496ec92b6a..15ee387dd18 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,79 +1,79 @@ - + https://github.com/dotnet/corefx - 72da80db3fd14b84fefeef54ac0930c711f306f4 + ec138afba514ce9b53d931d66129e02e8be147a1 - + https://github.com/dotnet/core-setup - ee0c7ead1a46f06f98aff9102b785f532b71da9c + d31c9e8055992b61f4a1b8898b646e5fcb7b62b3 - + https://github.com/aspnet/Extensions - 531db4e313e577192cdbca3931c4298b6db1e611 + 57863aedad5bac2e109345a70fa4de2acfebf34e - + https://github.com/aspnet/Extensions - 531db4e313e577192cdbca3931c4298b6db1e611 + 57863aedad5bac2e109345a70fa4de2acfebf34e - + https://github.com/aspnet/Extensions - 531db4e313e577192cdbca3931c4298b6db1e611 + 57863aedad5bac2e109345a70fa4de2acfebf34e - + https://github.com/aspnet/Extensions - 531db4e313e577192cdbca3931c4298b6db1e611 + 57863aedad5bac2e109345a70fa4de2acfebf34e - + https://github.com/aspnet/Extensions - 531db4e313e577192cdbca3931c4298b6db1e611 + 57863aedad5bac2e109345a70fa4de2acfebf34e - + https://github.com/dotnet/core-setup - ee0c7ead1a46f06f98aff9102b785f532b71da9c + d31c9e8055992b61f4a1b8898b646e5fcb7b62b3 - + https://github.com/aspnet/Extensions - 531db4e313e577192cdbca3931c4298b6db1e611 + 57863aedad5bac2e109345a70fa4de2acfebf34e - + https://github.com/aspnet/Extensions - 531db4e313e577192cdbca3931c4298b6db1e611 + 57863aedad5bac2e109345a70fa4de2acfebf34e - + https://github.com/dotnet/corefx - 72da80db3fd14b84fefeef54ac0930c711f306f4 + ec138afba514ce9b53d931d66129e02e8be147a1 - + https://github.com/dotnet/core-setup - ee0c7ead1a46f06f98aff9102b785f532b71da9c + d31c9e8055992b61f4a1b8898b646e5fcb7b62b3 - + https://github.com/dotnet/core-setup - ee0c7ead1a46f06f98aff9102b785f532b71da9c + d31c9e8055992b61f4a1b8898b646e5fcb7b62b3 - + https://github.com/dotnet/core-setup - ee0c7ead1a46f06f98aff9102b785f532b71da9c + d31c9e8055992b61f4a1b8898b646e5fcb7b62b3 - + https://github.com/dotnet/corefx - 72da80db3fd14b84fefeef54ac0930c711f306f4 + ec138afba514ce9b53d931d66129e02e8be147a1 - + https://github.com/dotnet/corefx - 72da80db3fd14b84fefeef54ac0930c711f306f4 + ec138afba514ce9b53d931d66129e02e8be147a1 - + https://github.com/dotnet/corefx - 72da80db3fd14b84fefeef54ac0930c711f306f4 + ec138afba514ce9b53d931d66129e02e8be147a1 - + https://github.com/dotnet/arcade - 9946534da4f73e6242ca105f6798ab58119c9ab0 + 89fab80685c91024c8f9e21f1c37f62580f648f8 diff --git a/eng/Versions.props b/eng/Versions.props index e0e0280c024..26cf3ec9560 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -28,27 +28,27 @@ 1.0.19128.1-Preview - 3.0.0-preview7.19328.2 - 3.0.0-preview7.19328.2 - 3.0.0-preview7.19328.2 - 3.0.0-preview7.19328.2 - 3.0.0-preview7.19328.2 - 3.0.0-preview7.19328.2 - 3.0.0-preview7.19328.2 + 3.0.0-preview8.19352.3 + 3.0.0-preview8.19352.3 + 3.0.0-preview8.19352.3 + 3.0.0-preview8.19352.3 + 3.0.0-preview8.19352.3 + 3.0.0-preview8.19352.3 + 3.0.0-preview8.19352.3 - 4.6.0-preview7.19326.13 - 3.0.0-preview7.19326.13 - 1.6.0-preview7.19326.13 - 4.6.0-preview7.19326.13 - 4.6.0-preview7.19326.13 + 4.6.0-preview8.19352.1 + 3.0.0-preview8.19352.1 + 1.6.0-preview8.19352.1 + 4.6.0-preview8.19352.1 + 4.6.0-preview8.19352.1 - 3.0.0-preview7-27826-20 - 3.0.0-preview7-27826-20 - 3.0.0-preview7-27826-20 - 3.0.0-preview7-27826-20 - 2.1.0-preview7-27826-20 + 3.0.0-preview8-27902-02 + 3.0.0-preview8-27902-02 + 3.0.0-preview8-27902-02 + 3.0.0-preview8-27902-02 + 2.1.0-preview8-27902-02 - - - + diff --git a/eng/common/post-build/nuget-validation.ps1 b/eng/common/post-build/nuget-validation.ps1 new file mode 100644 index 00000000000..1bdced1e307 --- /dev/null +++ b/eng/common/post-build/nuget-validation.ps1 @@ -0,0 +1,28 @@ +# This script validates NuGet package metadata information using this +# tool: https://github.com/NuGet/NuGetGallery/tree/jver-verify/src/VerifyMicrosoftPackage + +param( + [Parameter(Mandatory=$true)][string] $PackagesPath, # Path to where the packages to be validated are + [Parameter(Mandatory=$true)][string] $ToolDestinationPath # Where the validation tool should be downloaded to +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 + +. $PSScriptRoot\..\tools.ps1 + +try { + $url = "https://raw.githubusercontent.com/NuGet/NuGetGallery/jver-verify/src/VerifyMicrosoftPackage/verify.ps1" + + New-Item -ItemType "directory" -Path ${ToolDestinationPath} -Force + + Invoke-WebRequest $url -OutFile ${ToolDestinationPath}\verify.ps1 + + & ${ToolDestinationPath}\verify.ps1 ${PackagesPath}\*.nupkg +} +catch { + Write-PipelineTaskError "NuGet package validation failed. Please check error logs." + Write-Host $_ + Write-Host $_.ScriptStackTrace + ExitWithExitCode 1 +} diff --git a/eng/common/post-build/promote-build.ps1 b/eng/common/post-build/promote-build.ps1 new file mode 100644 index 00000000000..84a608fa569 --- /dev/null +++ b/eng/common/post-build/promote-build.ps1 @@ -0,0 +1,53 @@ +param( + [Parameter(Mandatory=$true)][int] $BuildId, + [Parameter(Mandatory=$true)][int] $ChannelId, + [Parameter(Mandatory=$true)][string] $BarToken, + [string] $MaestroEndpoint = "https://maestro-prod.westus2.cloudapp.azure.com", + [string] $ApiVersion = "2019-01-16" +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 + +. $PSScriptRoot\..\tools.ps1 + +function Get-Headers([string]$accept, [string]$barToken) { + $headers = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' + $headers.Add('Accept',$accept) + $headers.Add('Authorization',"Bearer $barToken") + return $headers +} + +try { + $maestroHeaders = Get-Headers 'application/json' $BarToken + + # Get info about which channels the build has already been promoted to + $getBuildApiEndpoint = "$MaestroEndpoint/api/builds/${BuildId}?api-version=$ApiVersion" + $buildInfo = Invoke-WebRequest -Method Get -Uri $getBuildApiEndpoint -Headers $maestroHeaders | ConvertFrom-Json + + if (!$buildInfo) { + Write-Host "Build with BAR ID $BuildId was not found in BAR!" + ExitWithExitCode 1 + } + + # Find whether the build is already assigned to the channel or not + if ($buildInfo.channels) { + foreach ($channel in $buildInfo.channels) { + if ($channel.Id -eq $ChannelId) { + Write-Host "The build with BAR ID $BuildId is already on channel $ChannelId!" + ExitWithExitCode 0 + } + } + } + + Write-Host "Build not present in channel $ChannelId. Promoting build ... " + + $promoteBuildApiEndpoint = "$maestroEndpoint/api/channels/${ChannelId}/builds/${BuildId}?api-version=$ApiVersion" + Invoke-WebRequest -Method Post -Uri $promoteBuildApiEndpoint -Headers $maestroHeaders + Write-Host "done." +} +catch { + Write-Host "There was an error while trying to promote build '$BuildId' to channel '$ChannelId'" + Write-Host $_ + Write-Host $_.ScriptStackTrace +} diff --git a/eng/common/post-build/trigger-subscriptions.ps1 b/eng/common/post-build/trigger-subscriptions.ps1 new file mode 100644 index 00000000000..db8a839457a --- /dev/null +++ b/eng/common/post-build/trigger-subscriptions.ps1 @@ -0,0 +1,69 @@ +param( + [Parameter(Mandatory=$true)][string] $SourceRepo, + [Parameter(Mandatory=$true)][int] $ChannelId, + [string] $MaestroEndpoint = "https://maestro-prod.westus2.cloudapp.azure.com", + [string] $BarToken, + [string] $ApiVersion = "2019-01-16" +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 + +. $PSScriptRoot\..\tools.ps1 + +function Get-Headers([string]$accept, [string]$barToken) { + $headers = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' + $headers.Add('Accept',$accept) + $headers.Add('Authorization',"Bearer $barToken") + return $headers +} + +# Get all the $SourceRepo subscriptions +$normalizedSurceRepo = $SourceRepo.Replace('dnceng@', '') +$getSubscriptionsApiEndpoint = "$maestroEndpoint/api/subscriptions?sourceRepository=$normalizedSurceRepo&api-version=$apiVersion" +$headers = Get-Headers 'application/json' $barToken + +$subscriptions = Invoke-WebRequest -Uri $getSubscriptionsApiEndpoint -Headers $headers | ConvertFrom-Json + +if (!$subscriptions) { + Write-Host "No subscriptions found for source repo '$normalizedSurceRepo' in channel '$ChannelId'" + return +} + +$subscriptionsToTrigger = New-Object System.Collections.Generic.List[string] +$failedTriggeredSubscription = $false + +# Get all enabled subscriptions that need dependency flow on 'everyBuild' +foreach ($subscription in $subscriptions) { + if ($subscription.enabled -and $subscription.policy.updateFrequency -like 'everyBuild' -and $subscription.channel.id -eq $ChannelId) { + Write-Host "$subscription.id" + [void]$subscriptionsToTrigger.Add($subscription.id) + } +} + +foreach ($subscriptionToTrigger in $subscriptionsToTrigger) { + try { + $triggerSubscriptionApiEndpoint = "$maestroEndpoint/api/subscriptions/$subscriptionToTrigger/trigger?api-version=$apiVersion" + $headers = Get-Headers 'application/json' $BarToken + + Write-Host "Triggering subscription '$subscriptionToTrigger'..." + + Invoke-WebRequest -Uri $triggerSubscriptionApiEndpoint -Headers $headers -Method Post + + Write-Host "Subscription '$subscriptionToTrigger' triggered!" + } + catch + { + Write-Host "There was an error while triggering subscription '$subscriptionToTrigger'" + Write-Host $_ + Write-Host $_.ScriptStackTrace + $failedTriggeredSubscription = $true + } +} + +if ($failedTriggeredSubscription) { + Write-Host "At least one subscription failed to be triggered..." + ExitWithExitCode 1 +} + +Write-Host "All subscriptions were triggered successfully!" diff --git a/eng/common/sdl/extract-artifact-packages.ps1 b/eng/common/sdl/extract-artifact-packages.ps1 new file mode 100644 index 00000000000..1fdbb14329c --- /dev/null +++ b/eng/common/sdl/extract-artifact-packages.ps1 @@ -0,0 +1,70 @@ +param( + [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where artifact packages are stored + [Parameter(Mandatory=$true)][string] $ExtractPath # Full path to directory where the packages will be extracted +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 +$ExtractPackage = { + param( + [string] $PackagePath # Full path to a NuGet package + ) + + if (!(Test-Path $PackagePath)) { + Write-PipelineTaskError "Input file does not exist: $PackagePath" + ExitWithExitCode 1 + } + + $RelevantExtensions = @(".dll", ".exe", ".pdb") + Write-Host -NoNewLine "Extracting" ([System.IO.Path]::GetFileName($PackagePath)) "... " + + $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) + $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId + + Add-Type -AssemblyName System.IO.Compression.FileSystem + + [System.IO.Directory]::CreateDirectory($ExtractPath); + + try { + $zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath) + + $zip.Entries | + Where-Object {$RelevantExtensions -contains [System.IO.Path]::GetExtension($_.Name)} | + ForEach-Object { + $TargetFile = Join-Path -Path $ExtractPath -ChildPath $_.Name + + [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile, $true) + } + } + catch { + + } + finally { + $zip.Dispose() + } + } + function ExtractArtifacts { + if (!(Test-Path $InputPath)) { + Write-Host "Input Path does not exist: $InputPath" + ExitWithExitCode 0 + } + $Jobs = @() + Get-ChildItem "$InputPath\*.nupkg" | + ForEach-Object { + $Jobs += Start-Job -ScriptBlock $ExtractPackage -ArgumentList $_.FullName + } + + foreach ($Job in $Jobs) { + Wait-Job -Id $Job.Id | Receive-Job + } +} + +try { + Measure-Command { ExtractArtifacts } +} +catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + ExitWithExitCode 1 +} diff --git a/eng/common/sdl/packages.config b/eng/common/sdl/packages.config index b054737df13..fb9b7128637 100644 --- a/eng/common/sdl/packages.config +++ b/eng/common/sdl/packages.config @@ -1,4 +1,4 @@  - + diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml index acb4c55d735..5837f3d56d3 100644 --- a/eng/common/templates/job/execute-sdl.yml +++ b/eng/common/templates/job/execute-sdl.yml @@ -20,6 +20,16 @@ jobs: downloadType: specific files matchingPattern: "**" downloadPath: $(Build.SourcesDirectory)\artifacts + - powershell: eng/common/sdl/extract-artifact-packages.ps1 + -InputPath $(Build.SourcesDirectory)\artifacts\BlobArtifacts + -ExtractPath $(Build.SourcesDirectory)\artifacts\BlobArtifacts + displayName: Extract Blob Artifacts + continueOnError: ${{ parameters.continueOnError }} + - powershell: eng/common/sdl/extract-artifact-packages.ps1 + -InputPath $(Build.SourcesDirectory)\artifacts\PackageArtifacts + -ExtractPath $(Build.SourcesDirectory)\artifacts\PackageArtifacts + displayName: Extract Package Artifacts + continueOnError: ${{ parameters.continueOnError }} - task: NuGetToolInstaller@1 displayName: 'Install NuGet.exe' - task: NuGetCommand@2 @@ -36,7 +46,7 @@ jobs: continueOnError: ${{ parameters.continueOnError }} - ${{ if eq(parameters.overrideParameters, '') }}: - powershell: eng/common/sdl/execute-all-sdl-tools.ps1 - -GuardianPackageName Microsoft.Guardian.Cli.0.3.2 + -GuardianPackageName Microsoft.Guardian.Cli.0.6.0 -NugetPackageDirectory $(Build.SourcesDirectory)\.packages -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) ${{ parameters.additionalParameters }} diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index ff7346163f4..9e77ef1b54b 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -66,7 +66,6 @@ jobs: script: | Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(BARBuildId) Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value "$(DefaultChannels)" - Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(IsInternalBuild) Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(IsStableBuild) - task: PublishBuildArtifacts@1 displayName: Publish ReleaseConfigs Artifact diff --git a/eng/common/templates/post-build/channels/internal-servicing.yml b/eng/common/templates/post-build/channels/internal-servicing.yml new file mode 100644 index 00000000000..808d46b17f2 --- /dev/null +++ b/eng/common/templates/post-build/channels/internal-servicing.yml @@ -0,0 +1,170 @@ +parameters: + enableSymbolValidation: true + +stages: +- stage: IS_Publish + dependsOn: validate + variables: + - template: ../common-variables.yml + displayName: Internal Servicing + jobs: + - template: ../setup-maestro-vars.yml + + - job: + displayName: Symbol Publishing + dependsOn: setupMaestroVars + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.InternalServicing_30_Channel_Id) + variables: + - group: DotNet-Symbol-Server-Pats + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Artifacts + inputs: + downloadType: specific files + matchingPattern: "*Artifacts*" + + - task: PowerShell@2 + displayName: Publish + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet + /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) + /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) + /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' + /p:Configuration=Release + + - job: publish_assets + displayName: Publish Assets + dependsOn: setupMaestroVars + variables: + - group: DotNet-Blob-Feed + - group: Publish-Build-Assets + - group: AzureDevOps-Artifact-Feeds-Pats + - name: BARBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] + - name: IsStableBuild + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ] + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.InternalServicing_30_Channel_Id) + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: current + artifactName: PackageArtifacts + + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: current + artifactName: BlobArtifacts + + - task: DownloadBuildArtifacts@0 + displayName: Download Asset Manifests + inputs: + buildType: current + artifactName: AssetManifests + + - task: PowerShell@2 + displayName: Add Assets Location + env: + AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-unviersal-packages-rw) + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet + /p:ChannelId=$(InternalServicing_30_Channel_Id) + /p:IsStableBuild=$(IsStableBuild) + /p:IsInternalBuild=$(IsInternalBuild) + /p:RepositoryName=$(Build.Repository.Name) + /p:CommitSha=$(Build.SourceVersion) + /p:NugetPath=$(Agent.BuildDirectory)\Nuget\NuGet.exe + /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' + /p:TargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' + /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' + /p:BARBuildId=$(BARBuildId) + /p:MaestroApiEndpoint='https://maestro-prod.westus2.cloudapp.azure.com' + /p:BuildAssetRegistryToken='$(MaestroAccessToken)' + /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)\BlobArtifacts' + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)\PackageArtifacts' + /p:Configuration=Release + + - task: NuGetCommand@2 + displayName: Publish Packages to AzDO Feed + condition: contains(variables['TargetAzDOFeed'], 'pkgs.visualstudio.com') + inputs: + command: push + vstsFeed: $(AzDoFeedName) + packagesToPush: $(Build.ArtifactStagingDirectory)\PackageArtifacts\*.nupkg + publishVstsFeed: $(AzDoFeedName) + + - task: PowerShell@2 + displayName: Publish Blobs to AzDO Feed + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-blobs-to-azdo.ps1 + arguments: -FeedName $(AzDoFeedName) + -SourceFolderCollection $(Build.ArtifactStagingDirectory)/BlobArtifacts/ + -PersonalAccessToken $(dn-bot-dnceng-unviersal-packages-rw) + enabled: false + + - template: ../trigger-subscription.yml + parameters: + ChannelId: ${{ variables.InternalServicing_30_Channel_Id }} + +- stage: IS_PublishValidation + displayName: Publish Validation + variables: + - template: ../common-variables.yml + jobs: + - template: ../setup-maestro-vars.yml + + - ${{ if eq(parameters.enableSymbolValidation, 'true') }}: + - job: + displayName: Symbol Availability + dependsOn: setupMaestroVars + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.InternalServicing_30_Channel_Id) + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: current + artifactName: PackageArtifacts + + - task: PowerShell@2 + displayName: Check Symbol Availability + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/symbols-validation.ps1 + arguments: -InputPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ -ExtractPath $(Agent.BuildDirectory)/Temp/ -DotnetSymbolVersion $(SymbolToolVersion) + + - job: + displayName: Gather Drop + dependsOn: setupMaestroVars + variables: + BARBuildId: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.InternalServicing_30_Channel_Id) + pool: + vmImage: 'windows-2019' + steps: + - task: PowerShell@2 + displayName: Setup Darc CLI + inputs: + targetType: filePath + filePath: '$(Build.SourcesDirectory)/eng/common/darc-init.ps1' + + - task: PowerShell@2 + displayName: Run Darc gather-drop + inputs: + targetType: inline + script: | + darc gather-drop --non-shipping --continue-on-error --id $(BARBuildId) --output-dir $(Agent.BuildDirectory)/Temp/Drop/ --bar-uri https://maestro-prod.westus2.cloudapp.azure.com/ --password $(MaestroAccessToken) --latest-location + enabled: false + + - template: ../promote-build.yml + parameters: + ChannelId: ${{ variables.InternalServicing_30_Channel_Id }} \ No newline at end of file diff --git a/eng/common/templates/post-build/channels/public-dev-release.yml b/eng/common/templates/post-build/channels/public-dev-release.yml index c61eaa927d9..79c6822db72 100644 --- a/eng/common/templates/post-build/channels/public-dev-release.yml +++ b/eng/common/templates/post-build/channels/public-dev-release.yml @@ -20,17 +20,10 @@ stages: vmImage: 'windows-2019' steps: - task: DownloadBuildArtifacts@0 - displayName: Download PDB Artifacts + displayName: Download Artifacts inputs: - buildType: current - artifactName: PDBArtifacts - continueOnError: true - - - task: DownloadBuildArtifacts@0 - displayName: Download Blob Artifacts - inputs: - buildType: current - artifactName: BlobArtifacts + downloadType: specific files + matchingPattern: "*Artifacts*" - task: PowerShell@2 displayName: Publish @@ -44,13 +37,16 @@ stages: /p:Configuration=Release - job: - displayName: Publish to Static Feed + displayName: Publish Assets dependsOn: setupMaestroVars variables: - group: DotNet-Blob-Feed - group: Publish-Build-Assets + - group: AzureDevOps-Artifact-Feeds-Pats - name: BARBuildId value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] + - name: IsStableBuild + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ] condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicDevRelease_30_Channel_Id) pool: vmImage: 'windows-2019' @@ -74,22 +70,47 @@ stages: artifactName: AssetManifests - task: PowerShell@2 - displayName: Publish + displayName: Add Assets Location + env: + AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-unviersal-packages-rw) inputs: filePath: eng\common\sdk-task.ps1 - arguments: -task PublishToPackageFeed -restore -msbuildEngine dotnet - /p:AccountKeyToStaticFeed='$(dotnetfeed-storage-access-key-1)' + arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet + /p:ChannelId=$(PublicDevRelease_30_Channel_Id) + /p:IsStableBuild=$(IsStableBuild) + /p:IsInternalBuild=$(IsInternalBuild) + /p:RepositoryName=$(Build.Repository.Name) + /p:CommitSha=$(Build.SourceVersion) + /p:NugetPath=$(Agent.BuildDirectory)\Nuget\NuGet.exe + /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' + /p:TargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' + /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' /p:BARBuildId=$(BARBuildId) /p:MaestroApiEndpoint='https://maestro-prod.westus2.cloudapp.azure.com' /p:BuildAssetRegistryToken='$(MaestroAccessToken)' /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' - /p:ArtifactsCategory='$(_DotNetArtifactsCategory)' - /p:OverrideAssetsWithSameName=true - /p:PassIfExistingItemIdentical=true /p:Configuration=Release + - task: NuGetCommand@2 + displayName: Publish Packages to AzDO Feed + condition: contains(variables['TargetAzDOFeed'], 'pkgs.visualstudio.com') + inputs: + command: push + vstsFeed: $(AzDoFeedName) + packagesToPush: $(Build.ArtifactStagingDirectory)\PackageArtifacts\*.nupkg + publishVstsFeed: $(AzDoFeedName) + + - task: PowerShell@2 + displayName: Publish Blobs to AzDO Feed + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-blobs-to-azdo.ps1 + arguments: -FeedName $(AzDoFeedName) + -SourceFolderCollection $(Build.ArtifactStagingDirectory)/BlobArtifacts/ + -PersonalAccessToken $(dn-bot-dnceng-unviersal-packages-rw) + enabled: false + - stage: PublishValidation displayName: Publish Validation @@ -140,6 +161,6 @@ stages: script: | darc gather-drop --non-shipping --continue-on-error --id $(BARBuildId) --output-dir $(Agent.BuildDirectory)/Temp/Drop/ --bar-uri https://maestro-prod.westus2.cloudapp.azure.com/ --password $(MaestroAccessToken) --latest-location - - template: ../promote-build.yml - parameters: + - template: ../promote-build.yml + parameters: ChannelId: ${{ variables.PublicDevRelease_30_Channel_Id }} diff --git a/eng/common/templates/post-build/channels/public-release.yml b/eng/common/templates/post-build/channels/public-release.yml new file mode 100644 index 00000000000..25923020df8 --- /dev/null +++ b/eng/common/templates/post-build/channels/public-release.yml @@ -0,0 +1,170 @@ +parameters: + enableSymbolValidation: true + +stages: +- stage: PubRel_Publish + dependsOn: validate + variables: + - template: ../common-variables.yml + displayName: Public Release + jobs: + - template: ../setup-maestro-vars.yml + + - job: + displayName: Symbol Publishing + dependsOn: setupMaestroVars + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicRelease_30_Channel_Id) + variables: + - group: DotNet-Symbol-Server-Pats + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Artifacts + inputs: + downloadType: specific files + matchingPattern: "*Artifacts*" + + - task: PowerShell@2 + displayName: Publish + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet + /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) + /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) + /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' + /p:Configuration=Release + + - job: publish_assets + displayName: Publish Assets + dependsOn: setupMaestroVars + variables: + - group: DotNet-Blob-Feed + - group: Publish-Build-Assets + - group: AzureDevOps-Artifact-Feeds-Pats + - name: BARBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] + - name: IsStableBuild + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ] + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicRelease_30_Channel_Id) + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: current + artifactName: PackageArtifacts + + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: current + artifactName: BlobArtifacts + + - task: DownloadBuildArtifacts@0 + displayName: Download Asset Manifests + inputs: + buildType: current + artifactName: AssetManifests + + - task: PowerShell@2 + displayName: Publish + env: + AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-unviersal-packages-rw) + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet + /p:ChannelId=$(PublicRelease_30_Channel_Id) + /p:IsStableBuild=$(IsStableBuild) + /p:IsInternalBuild=$(IsInternalBuild) + /p:RepositoryName=$(Build.Repository.Name) + /p:CommitSha=$(Build.SourceVersion) + /p:NugetPath=$(Agent.BuildDirectory)/Nuget/NuGet.exe + /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' + /p:TargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' + /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' + /p:BARBuildId=$(BARBuildId) + /p:MaestroApiEndpoint='https://maestro-prod.westus2.cloudapp.azure.com' + /p:BuildAssetRegistryToken='$(MaestroAccessToken)' + /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)\BlobArtifacts' + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)\PackageArtifacts' + /p:Configuration=Release + + - task: NuGetCommand@2 + displayName: Publish Packages to AzDO Feed + condition: contains(variables['TargetAzDOFeed'], 'pkgs.visualstudio.com') + inputs: + command: push + vstsFeed: $(AzDoFeedName) + packagesToPush: $(Build.ArtifactStagingDirectory)\PackageArtifacts\*.nupkg + publishVstsFeed: $(AzDoFeedName) + + - task: PowerShell@2 + displayName: Publish Blobs to AzDO Feed + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-blobs-to-azdo.ps1 + arguments: -FeedName $(AzDoFeedName) + -SourceFolderCollection $(Build.ArtifactStagingDirectory)/BlobArtifacts/ + -PersonalAccessToken $(dn-bot-dnceng-unviersal-packages-rw) + enabled: false + + - template: ../trigger-subscription.yml + parameters: + ChannelId: ${{ variables.PublicRelease_30_Channel_Id }} + +- stage: PubRel_PublishValidation + displayName: Publish Validation + variables: + - template: ../common-variables.yml + jobs: + - template: ../setup-maestro-vars.yml + + - ${{ if eq(parameters.enableSymbolValidation, 'true') }}: + - job: + displayName: Symbol Availability + dependsOn: setupMaestroVars + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicRelease_30_Channel_Id) + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: current + artifactName: PackageArtifacts + + - task: PowerShell@2 + displayName: Check Symbol Availability + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/symbols-validation.ps1 + arguments: -InputPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ -ExtractPath $(Agent.BuildDirectory)/Temp/ -DotnetSymbolVersion $(SymbolToolVersion) + + - job: + displayName: Gather Drop + dependsOn: setupMaestroVars + variables: + BARBuildId: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicRelease_30_Channel_Id) + pool: + vmImage: 'windows-2019' + steps: + - task: PowerShell@2 + displayName: Setup Darc CLI + inputs: + targetType: filePath + filePath: '$(Build.SourcesDirectory)/eng/common/darc-init.ps1' + + - task: PowerShell@2 + displayName: Run Darc gather-drop + inputs: + targetType: inline + script: | + darc gather-drop --non-shipping --continue-on-error --id $(BARBuildId) --output-dir $(Agent.BuildDirectory)/Temp/Drop/ --bar-uri https://maestro-prod.westus2.cloudapp.azure.com/ --password $(MaestroAccessToken) --latest-location + enabled: false + + - template: ../promote-build.yml + parameters: + ChannelId: ${{ variables.PublicRelease_30_Channel_Id }} diff --git a/eng/common/templates/post-build/channels/public-validation-release.yml b/eng/common/templates/post-build/channels/public-validation-release.yml index 23725c6d620..114477d3adb 100644 --- a/eng/common/templates/post-build/channels/public-validation-release.yml +++ b/eng/common/templates/post-build/channels/public-validation-release.yml @@ -8,14 +8,17 @@ stages: - template: ../setup-maestro-vars.yml - job: - displayName: Publish to Static Feed + displayName: Publish Assets dependsOn: setupMaestroVars - condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicValidationRelease_30_Channel_Id) variables: - group: DotNet-Blob-Feed - group: Publish-Build-Assets + - group: AzureDevOps-Artifact-Feeds-Pats - name: BARBuildId value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] + - name: IsStableBuild + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ] + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicValidationRelease_30_Channel_Id) pool: vmImage: 'windows-2019' steps: @@ -38,21 +41,46 @@ stages: artifactName: AssetManifests - task: PowerShell@2 - displayName: Publish + displayName: Add Assets Location + env: + AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-unviersal-packages-rw) inputs: filePath: eng\common\sdk-task.ps1 - arguments: -task PublishToPackageFeed -restore -msbuildEngine dotnet - /p:AccountKeyToStaticFeed='$(dotnetfeed-storage-access-key-1)' + arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet + /p:ChannelId=$(PublicValidationRelease_30_Channel_Id) + /p:IsStableBuild=$(IsStableBuild) + /p:IsInternalBuild=$(IsInternalBuild) + /p:RepositoryName=$(Build.Repository.Name) + /p:CommitSha=$(Build.SourceVersion) + /p:NugetPath=$(Agent.BuildDirectory)\Nuget\NuGet.exe + /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' + /p:TargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' + /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' /p:BARBuildId=$(BARBuildId) /p:MaestroApiEndpoint='https://maestro-prod.westus2.cloudapp.azure.com' /p:BuildAssetRegistryToken='$(MaestroAccessToken)' /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' - /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' - /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' - /p:ArtifactsCategory='$(_DotNetArtifactsCategory)' - /p:OverrideAssetsWithSameName=true - /p:PassIfExistingItemIdentical=true + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)\BlobArtifacts' + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)\PackageArtifacts' /p:Configuration=Release + + - task: NuGetCommand@2 + displayName: Publish Packages to AzDO Feed + condition: contains(variables['TargetAzDOFeed'], 'pkgs.visualstudio.com') + inputs: + command: push + vstsFeed: $(AzDoFeedName) + packagesToPush: $(Build.ArtifactStagingDirectory)\PackageArtifacts\*.nupkg + publishVstsFeed: $(AzDoFeedName) + + - task: PowerShell@2 + displayName: Publish Blobs to AzDO Feed + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-blobs-to-azdo.ps1 + arguments: -FeedName $(AzDoFeedName) + -SourceFolderCollection $(Build.ArtifactStagingDirectory)/BlobArtifacts/ + -PersonalAccessToken $(dn-bot-dnceng-unviersal-packages-rw) + enabled: false - stage: PVR_PublishValidation @@ -86,6 +114,6 @@ stages: script: | darc gather-drop --non-shipping --continue-on-error --id $(BARBuildId) --output-dir $(Agent.BuildDirectory)/Temp/Drop/ --bar-uri https://maestro-prod.westus2.cloudapp.azure.com --password $(MaestroAccessToken) --latest-location - - template: ../promote-build.yml - parameters: + - template: ../promote-build.yml + parameters: ChannelId: ${{ variables.PublicValidationRelease_30_Channel_Id }} diff --git a/eng/common/templates/post-build/common-variables.yml b/eng/common/templates/post-build/common-variables.yml index 97b48d97fec..82834673520 100644 --- a/eng/common/templates/post-build/common-variables.yml +++ b/eng/common/templates/post-build/common-variables.yml @@ -5,5 +5,14 @@ variables: # .NET Tools - Validation PublicValidationRelease_30_Channel_Id: 9 + # .NET Core 3.0 Internal Servicing + InternalServicing_30_Channel_Id: 184 + + # .NET Core 3.0 Release + PublicRelease_30_Channel_Id: 19 + + # Whether the build is internal or not + IsInternalBuild: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }} + SourceLinkCLIVersion: 3.0.0 SymbolToolVersion: 1.0.1 diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index 2c411dd0098..daa799259c7 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -2,6 +2,7 @@ parameters: enableSourceLinkValidation: true enableSigningValidation: true enableSymbolValidation: true + enableNugetValidation: true SDLValidationParameters: enable: false params: '' @@ -11,6 +12,25 @@ stages: dependsOn: build displayName: Validate jobs: + - ${{ if eq(parameters.enableNugetValidation, 'true') }}: + - job: + displayName: NuGet Validation + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: current + artifactName: PackageArtifacts + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1 + arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ + -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ + - ${{ if eq(parameters.enableSigningValidation, 'true') }}: - job: displayName: Signing Validation @@ -65,3 +85,7 @@ stages: enableSymbolValidation: ${{ parameters.enableSymbolValidation }} - template: \eng\common\templates\post-build\channels\public-validation-release.yml + +- template: \eng\common\templates\post-build\channels\public-release.yml + +- template: \eng\common\templates\post-build\channels\internal-servicing.yml diff --git a/eng/common/templates/post-build/promote-build.yml b/eng/common/templates/post-build/promote-build.yml index d00317003b7..af48b0b339e 100644 --- a/eng/common/templates/post-build/promote-build.yml +++ b/eng/common/templates/post-build/promote-build.yml @@ -18,11 +18,7 @@ jobs: - task: PowerShell@2 displayName: Add Build to Channel inputs: - targetType: inline - script: | - $headers = @{ - "Accept" = "application/json" - "Authorization" = "Bearer $(MaestroAccessToken)" - } - Invoke-RestMethod -Method Post -Headers $headers -Uri https://maestro-prod.westus2.cloudapp.azure.com/api/channels/$(ChannelId)/builds/$(BARBuildId)?api-version=2019-01-16 - enabled: false + filePath: $(Build.SourcesDirectory)/eng/common/post-build/promote-build.ps1 + arguments: -BuildId $(BARBuildId) + -ChannelId $(ChannelId) + -BarToken $(MaestroAccessToken) diff --git a/eng/common/templates/post-build/setup-maestro-vars.yml b/eng/common/templates/post-build/setup-maestro-vars.yml index 0eddd6cd3b7..f6120dc1e1c 100644 --- a/eng/common/templates/post-build/setup-maestro-vars.yml +++ b/eng/common/templates/post-build/setup-maestro-vars.yml @@ -28,10 +28,8 @@ jobs: $Channels = "" $Content | Select -Index 1 | ForEach-Object { $Channels += "$_ ," } - $IsInternalBuild = $Content | Select -Index 2 - $IsStableBuild = $Content | Select -Index 3 + $IsStableBuild = $Content | Select -Index 2 Write-PipelineSetVariable -Name 'BARBuildId' -Value $BarId Write-PipelineSetVariable -Name 'InitialChannels' -Value "$Channels" - Write-PipelineSetVariable -Name 'IsInternalBuild' -Value $IsInternalBuild Write-PipelineSetVariable -Name 'IsStableBuild' -Value $IsStableBuild diff --git a/eng/common/templates/post-build/trigger-subscription.yml b/eng/common/templates/post-build/trigger-subscription.yml new file mode 100644 index 00000000000..65259d4e685 --- /dev/null +++ b/eng/common/templates/post-build/trigger-subscription.yml @@ -0,0 +1,11 @@ +parameters: + ChannelId: 0 + +steps: +- task: PowerShell@2 + displayName: Triggering subscriptions + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/trigger-subscriptions.ps1 + arguments: -SourceRepo $(Build.Repository.Uri) + -ChannelId ${{ parameters.ChannelId }} + -BarToken $(MaestroAccessTokenInt) \ No newline at end of file diff --git a/global.json b/global.json index 1914bc1183d..81da5d7c133 100644 --- a/global.json +++ b/global.json @@ -12,6 +12,6 @@ "version": "3.0.100-preview6-012264" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19323.4" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19330.1" } } diff --git a/src/EFCore/Metadata/Conventions/Infrastructure/ProviderConventionSetBuilder.cs b/src/EFCore/Metadata/Conventions/Infrastructure/ProviderConventionSetBuilder.cs index 5faed428dab..f4df70c8389 100644 --- a/src/EFCore/Metadata/Conventions/Infrastructure/ProviderConventionSetBuilder.cs +++ b/src/EFCore/Metadata/Conventions/Infrastructure/ProviderConventionSetBuilder.cs @@ -104,6 +104,7 @@ public virtual ConventionSet CreateConventionSet() var databaseGeneratedAttributeConvention = new DatabaseGeneratedAttributeConvention(Dependencies); var requiredPropertyAttributeConvention = new RequiredPropertyAttributeConvention(Dependencies); var nonNullableReferencePropertyConvention = new NonNullableReferencePropertyConvention(Dependencies); + var nonNullableNavigationConvention = new NonNullableNavigationConvention(Dependencies); var maxLengthAttributeConvention = new MaxLengthAttributeConvention(Dependencies); var stringLengthAttributeConvention = new StringLengthAttributeConvention(Dependencies); var timestampAttributeConvention = new TimestampAttributeConvention(Dependencies); @@ -171,11 +172,14 @@ public virtual ConventionSet CreateConventionSet() conventionSet.ModelFinalizedConventions.Add(foreignKeyIndexConvention); conventionSet.ModelFinalizedConventions.Add(foreignKeyPropertyDiscoveryConvention); conventionSet.ModelFinalizedConventions.Add(servicePropertyDiscoveryConvention); + conventionSet.ModelFinalizedConventions.Add(nonNullableReferencePropertyConvention); + conventionSet.ModelFinalizedConventions.Add(nonNullableNavigationConvention); conventionSet.ModelFinalizedConventions.Add(new ValidatingConvention(Dependencies)); + // Don't add any more conventions to ModelFinalizedConventions after ValidatingConvention conventionSet.NavigationAddedConventions.Add(backingFieldConvention); conventionSet.NavigationAddedConventions.Add(new RequiredNavigationAttributeConvention(Dependencies)); - conventionSet.NavigationAddedConventions.Add(new NonNullableNavigationConvention(Dependencies)); + conventionSet.NavigationAddedConventions.Add(nonNullableNavigationConvention); conventionSet.NavigationAddedConventions.Add(inversePropertyAttributeConvention); conventionSet.NavigationAddedConventions.Add(foreignKeyPropertyDiscoveryConvention); conventionSet.NavigationAddedConventions.Add(relationshipDiscoveryConvention); diff --git a/src/EFCore/Metadata/Conventions/NonNullableConventionBase.cs b/src/EFCore/Metadata/Conventions/NonNullableConventionBase.cs index 832aca36cc1..b8910fa42c8 100644 --- a/src/EFCore/Metadata/Conventions/NonNullableConventionBase.cs +++ b/src/EFCore/Metadata/Conventions/NonNullableConventionBase.cs @@ -2,9 +2,11 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; +using System.Collections.Generic; using System.Linq; using System.Reflection; using JetBrains.Annotations; +using Microsoft.EntityFrameworkCore.Metadata.Builders; using Microsoft.EntityFrameworkCore.Metadata.Conventions.Infrastructure; namespace Microsoft.EntityFrameworkCore.Metadata.Conventions @@ -13,11 +15,14 @@ namespace Microsoft.EntityFrameworkCore.Metadata.Conventions /// A base type for conventions that configure model aspects based on whether the member type /// is a non-nullable reference type. /// - public abstract class NonNullableConventionBase + public abstract class NonNullableConventionBase : IModelFinalizedConvention { + // For the interpretation of nullability metadata, see + // https://github.com/dotnet/roslyn/blob/master/docs/features/nullable-metadata.md + + private const string StateAnnotationName = "NonNullableConventionState"; private const string NullableAttributeFullName = "System.Runtime.CompilerServices.NullableAttribute"; - private Type _nullableAttrType; - private FieldInfo _nullableFlagsFieldInfo; + private const string NullableContextAttributeFullName = "System.Runtime.CompilerServices.NullableContextAttribute"; /// /// Creates a new instance of . @@ -33,40 +38,118 @@ protected NonNullableConventionBase([NotNull] ProviderConventionSetBuilderDepend /// protected virtual ProviderConventionSetBuilderDependencies Dependencies { get; } + private byte? GetNullabilityContextFlag(NonNullabilityConventionState state, Attribute[] attributes) + { + if (attributes.FirstOrDefault(a => a.GetType().FullName == NullableContextAttributeFullName) is Attribute attribute) + { + var attributeType = attribute.GetType(); + + if (attributeType != state.NullableContextAttrType) + { + state.NullableContextFlagFieldInfo = attributeType.GetField("Flag"); + state.NullableContextAttrType = attributeType; + } + + if (state.NullableContextFlagFieldInfo?.GetValue(attribute) is byte flag) + { + return flag; + } + } + + return null; + } + /// /// Returns a value indicating whether the member type is a non-nullable reference type. /// + /// The model builder used to build the model. /// The member info. /// true if the member type is a non-nullable reference type. - protected virtual bool IsNonNullable([NotNull] MemberInfo memberInfo) + protected virtual bool IsNonNullable( + [NotNull] IConventionModelBuilder modelBuilder, + [NotNull] MemberInfo memberInfo) { + var state = GetOrInitializeState(modelBuilder); + // For C# 8.0 nullable types, the C# currently synthesizes a NullableAttribute that expresses nullability into assemblies // it produces. If the model is spread across more than one assembly, there will be multiple versions of this attribute, // so look for it by name, caching to avoid reflection on every check. // Note that this may change - if https://github.com/dotnet/corefx/issues/36222 is done we can remove all of this. - if (!(Attribute.GetCustomAttributes(memberInfo, true) - .FirstOrDefault(a => a.GetType().FullName == NullableAttributeFullName) - is { } attribute)) + + // First look for NullableAttribute on the member itself + if (Attribute.GetCustomAttributes(memberInfo, true) + .FirstOrDefault(a => a.GetType().FullName == NullableAttributeFullName) is Attribute attribute) + { + var attributeType = attribute.GetType(); + + if (attributeType != state.NullableAttrType) + { + state.NullableFlagsFieldInfo = attributeType.GetField("NullableFlags"); + state.NullableAttrType = attributeType; + } + + if (state.NullableFlagsFieldInfo?.GetValue(attribute) is byte[] flags + && flags.FirstOrDefault() == 1) + { + return true; + } + } + + // No attribute on the member, try to find a NullableContextAttribute on the declaring type + var type = memberInfo.DeclaringType; + if (type != null) { - return false; + if (state.TypeNonNullabilityContextCache.TryGetValue(type, out var cachedTypeNonNullable)) + { + return cachedTypeNonNullable; + } + + var typeContextFlag = GetNullabilityContextFlag(state, Attribute.GetCustomAttributes(type)); + if (typeContextFlag.HasValue) + { + return state.TypeNonNullabilityContextCache[type] = typeContextFlag.Value == 1; + } } - var attributeType = attribute.GetType(); - if (attributeType != _nullableAttrType) + // Not found at the type level, try at the module level + var module = memberInfo.Module; + if (!state.ModuleNonNullabilityContextCache.TryGetValue(module, out var moduleNonNullable)) { - _nullableFlagsFieldInfo = attributeType.GetField("NullableFlags"); - _nullableAttrType = attributeType; + var moduleContextFlag = GetNullabilityContextFlag(state, Attribute.GetCustomAttributes(memberInfo.Module)); + moduleNonNullable = state.ModuleNonNullabilityContextCache[module] = + moduleContextFlag.HasValue && moduleContextFlag == 1; } - // For the interpretation of NullableFlags, see - // https://github.com/dotnet/roslyn/blob/master/docs/features/nullable-reference-types.md#annotations - if (_nullableFlagsFieldInfo?.GetValue(attribute) is byte[] flags - && flags.FirstOrDefault() == 1) + if (type != null) { - return true; + state.TypeNonNullabilityContextCache[type] = moduleNonNullable; } - return false; + return moduleNonNullable; + } + + private NonNullabilityConventionState GetOrInitializeState(IConventionModelBuilder modelBuilder) + => (NonNullabilityConventionState)( + modelBuilder.Metadata.FindAnnotation(StateAnnotationName) ?? + modelBuilder.Metadata.AddAnnotation(StateAnnotationName, new NonNullabilityConventionState()) + ).Value; + + /// + /// Called after a model is finalized. Removes the cached state annotation used by this convention. + /// + /// The builder for the model. + /// Additional information associated with convention execution. + public virtual void ProcessModelFinalized(IConventionModelBuilder modelBuilder, IConventionContext context) + => modelBuilder.Metadata.RemoveAnnotation(StateAnnotationName); + + private class NonNullabilityConventionState + { + public Type NullableAttrType; + public Type NullableContextAttrType; + public FieldInfo NullableFlagsFieldInfo; + public FieldInfo NullableContextFlagFieldInfo; + public Dictionary TypeNonNullabilityContextCache { get; } = new Dictionary(); + public Dictionary ModuleNonNullabilityContextCache { get; } = new Dictionary(); } } } diff --git a/src/EFCore/Metadata/Conventions/NonNullableNavigationConvention.cs b/src/EFCore/Metadata/Conventions/NonNullableNavigationConvention.cs index b3c90579715..7e104bbbb33 100644 --- a/src/EFCore/Metadata/Conventions/NonNullableNavigationConvention.cs +++ b/src/EFCore/Metadata/Conventions/NonNullableNavigationConvention.cs @@ -40,8 +40,9 @@ public virtual void ProcessNavigationAdded( Check.NotNull(relationshipBuilder, nameof(relationshipBuilder)); Check.NotNull(navigation, nameof(navigation)); - if (!IsNonNullable(navigation) - || navigation.IsCollection()) + var modelBuilder = relationshipBuilder.ModelBuilder; + + if (!IsNonNullable(modelBuilder, navigation) || navigation.IsCollection()) { return; } @@ -51,7 +52,7 @@ public virtual void ProcessNavigationAdded( var inverse = navigation.FindInverse(); if (inverse != null) { - if (IsNonNullable(inverse)) + if (IsNonNullable(modelBuilder, inverse)) { Dependencies.Logger.NonNullableReferenceOnBothNavigations(navigation, inverse); return; @@ -82,9 +83,9 @@ public virtual void ProcessNavigationAdded( context.StopProcessingIfChanged(relationshipBuilder.Metadata.DependentToPrincipal); } - private bool IsNonNullable(IConventionNavigation navigation) + private bool IsNonNullable(IConventionModelBuilder modelBuilder, IConventionNavigation navigation) => navigation.DeclaringEntityType.HasClrType() && navigation.DeclaringEntityType.GetRuntimeProperties().Find(navigation.Name) is PropertyInfo propertyInfo - && IsNonNullable(propertyInfo); + && IsNonNullable(modelBuilder, propertyInfo); } } diff --git a/src/EFCore/Metadata/Conventions/NonNullableReferencePropertyConvention.cs b/src/EFCore/Metadata/Conventions/NonNullableReferencePropertyConvention.cs index aa364b5d811..8fc712e8682 100644 --- a/src/EFCore/Metadata/Conventions/NonNullableReferencePropertyConvention.cs +++ b/src/EFCore/Metadata/Conventions/NonNullableReferencePropertyConvention.cs @@ -29,7 +29,7 @@ private void Process(IConventionPropertyBuilder propertyBuilder) // If the model is spread across multiple assemblies, it may contain different NullableAttribute types as // the compiler synthesizes them for each assembly. if (propertyBuilder.Metadata.GetIdentifyingMemberInfo() is MemberInfo memberInfo - && IsNonNullable(memberInfo)) + && IsNonNullable(propertyBuilder.ModelBuilder, memberInfo)) { propertyBuilder.IsRequired(true); }