From cea8860d5852d6fc3782eb4306db9a14a778d93a Mon Sep 17 00:00:00 2001 From: Ruben Guerrero Date: Wed, 1 Apr 2020 12:55:06 -0700 Subject: [PATCH] Add DevOps pipelines files (#1) --- .../publish-pipeline.yaml | 123 +++++++++++++ .../rebuild-pipeline.yaml | 167 ++++++++++++++++++ .../validation-pipeline.yaml | 107 +++++++++++ 3 files changed, 397 insertions(+) create mode 100644 DevOpsPipelineDefinitions/publish-pipeline.yaml create mode 100644 DevOpsPipelineDefinitions/rebuild-pipeline.yaml create mode 100644 DevOpsPipelineDefinitions/validation-pipeline.yaml diff --git a/DevOpsPipelineDefinitions/publish-pipeline.yaml b/DevOpsPipelineDefinitions/publish-pipeline.yaml new file mode 100644 index 0000000000000..ec10f1801019b --- /dev/null +++ b/DevOpsPipelineDefinitions/publish-pipeline.yaml @@ -0,0 +1,123 @@ +# Publish pipeline for Windows Package Manager. + +# Name of the run +name: '$(Build.DefinitionName)-$(Build.DefinitionVersion)-$(Date:yyyyMMdd)-$(Rev:r)' + +# Batch CI run. when a pipeline is running, the system waits until the run is completed, +# then starts another run with all changes that have not yet been built. +trigger: + batch: true + branches: + include: + - master + +pr: none + +jobs: + +# Agent phase. +- job: 'CommitProcessing' + displayName: 'Commit Processing' + pool: + vmImage: 'windows-latest' + variables: + skipComponentGovernanceDetection: ${{ true }} + runCodesignValidationInjection: ${{ false }} + steps: + + # Downloads all the setup files and its dependencies. + - task: AzureCLI@1 + displayName: 'Azure Setup' + inputs: + azureSubscription: '$(PackageManagerSubscription)' + scriptLocation: inlineScript + inlineScript: 'az storage blob download-batch -d . --pattern * -s servicewrapper --output none' + env: + AZURE_STORAGE_CONNECTION_STRING: $(ValidationStorageAccountConnectionString) + + # PackageManager setup + - script: 'pkgmgr_publish_setup.cmd' + name: 'pkgmgrsetup' + displayName: 'PackageManager Setup' + workingDirectory: scripts + env: + HOST_KEY: $(AzureFunctionHostKey) + SIGN_ENDPOINT: $(PackageManagerSignEndpoint) + + - task: CmdLine@2 + displayName: 'Validate Commits' + inputs: + script: 'PackageManagerServiceWrapper.exe validate-commits --operationId %BUILD_BUILDNUMBER%' + failOnStderr: true + condition: succeeded() + env: + AzureWebJobsStorage: $(ValidationStorageAccountConnectionString) + CacheConnectionString: $(ActiveCacheConnectionString) + PackageManagerEnvironment: $(PackageManagerEnvironment) + PackagePublisher: $(PackagePublisher) + DIApplicationInsightKey: $(DIApplicationInsightKey) + +# Agentless phase. Depends on previous job. +- job: 'SignPackage' + pool: server + timeoutInMinutes: 1500 + displayName: 'Sign package' + dependsOn: + - 'CommitProcessing' + variables: + HostKeySecret: $[ dependencies.CommitProcessing.outputs['pkgmgrsetup.hostkey']] + SignEndpointSecret: $[ dependencies.CommitProcessing.outputs['pkgmgrsetup.signEndpoint']] + steps: + + # Sign Package Manager package. + - task: AzureFunction@1 + displayName: 'Signing package' + inputs: + function: '$(SignEndpointSecret)' + key: '$(HostKeySecret)' + body: | + { + "operationId": "$(Build.BuildNumber)", + "pipelineType": "CommitPipeline", + "ProjectId": "$(system.TeamProjectId)", + "PlanId": "$(system.PlanId)", + "JobId": "$(system.JobId)", + "TimelineId": "$(system.TimelineId)", + "TaskInstanceId": "$(system.TaskInstanceId)", + "AuthToken": "$(system.AccessToken)" + } + waitForCompletion: "true" + +# Agent phase. Depends on previous job. +- job: 'Publish' + displayName: 'Publish' + pool: + vmImage: 'windows-latest' + variables: + skipComponentGovernanceDetection: ${{ true }} + runCodesignValidationInjection: ${{ false }} + dependsOn: + - 'SignPackage' + steps: + + # Downloads all the setup files and its dependencies. + - task: AzureCLI@1 + displayName: 'Azure Setup' + inputs: + azureSubscription: '$(PackageManagerSubscription)' + scriptLocation: inlineScript + inlineScript: 'az storage blob download-batch -d . --pattern * -s servicewrapper --output none' + env: + AZURE_STORAGE_CONNECTION_STRING: $(ValidationStorageAccountConnectionString) + + # Validates integrity of pull request. + - task: CmdLine@2 + displayName: 'Publish' + inputs: + script: 'PackageManagerServiceWrapper.exe publish --operationId %BUILD_BUILDNUMBER%' + failOnStderr: true + condition: succeeded() + env: + AzureWebJobsStorage: $(ValidationStorageAccountConnectionString) + CacheConnectionString: $(ActiveCacheConnectionString) + DIApplicationInsightKey: $(DIApplicationInsightKey) diff --git a/DevOpsPipelineDefinitions/rebuild-pipeline.yaml b/DevOpsPipelineDefinitions/rebuild-pipeline.yaml new file mode 100644 index 0000000000000..1f2793823c49e --- /dev/null +++ b/DevOpsPipelineDefinitions/rebuild-pipeline.yaml @@ -0,0 +1,167 @@ +# Rebuild pipeline for Windows Package Manager. + +# Name of the run +name: '$(Build.DefinitionName)-$(Build.DefinitionVersion)-$(Date:yyyyMMdd)-$(Rev:r)' + +trigger: none +pr: none + +jobs: + +# Agent phase. +- job: 'Rebuild' + displayName: 'Start Rebuild' + pool: + vmImage: 'windows-latest' + variables: + skipComponentGovernanceDetection: ${{ true }} + runCodesignValidationInjection: ${{ false }} + steps: + + # Allow scripts to access the system token. + - checkout: self + persistCredentials: true + clean: true + + # Downloads all the setup files and its dependencies. + - task: AzureCLI@1 + displayName: 'Azure Setup' + inputs: + azureSubscription: '$(PackageManagerSubscription)' + scriptLocation: inlineScript + inlineScript: 'az storage blob download-batch -d . --pattern * -s servicewrapper --output none' + env: + AZURE_STORAGE_CONNECTION_STRING: $(ValidationStorageAccountConnectionString) + + # PackageManager setup + - script: 'pkgmgr_publish_setup.cmd' + name: 'pkgmgrsetup' + displayName: 'PackageManager Setup' + workingDirectory: scripts + env: + HOST_KEY: $(AzureFunctionHostKey) + SIGN_ENDPOINT: $(PackageManagerSignEndpoint) + + - task: CmdLine@2 + displayName: 'Validate Manifests' + inputs: + script: 'PackageManagerServiceWrapper.exe rebuild --operationId %BUILD_BUILDNUMBER%' + failOnStderr: true + condition: succeeded() + env: + AzureWebJobsStorage: $(ValidationStorageAccountConnectionString) + PackageManagerEnvironment: $(PackageManagerEnvironment) + PackagePublisher: $(PackagePublisher) + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + DIApplicationInsightKey: $(DIApplicationInsightKey) + +# Agentless phase. Depends on previous job. +- job: 'SignPackage' + pool: server + timeoutInMinutes: 1500 + displayName: 'Sign package' + dependsOn: + - 'Rebuild' + variables: + HostKeySecret: $[ dependencies.Rebuild.outputs['pkgmgrsetup.hostkey']] + SignEndpointSecret: $[ dependencies.Rebuild.outputs['pkgmgrsetup.signEndpoint']] + steps: + + # Sign Package Manager package. + - task: AzureFunction@1 + displayName: 'Signing package' + inputs: + function: '$(SignEndpointSecret)' + key: '$(HostKeySecret)' + body: | + { + "operationId": "$(Build.BuildNumber)", + "pipelineType": "RebuildPipeline", + "ProjectId": "$(system.TeamProjectId)", + "PlanId": "$(system.PlanId)", + "JobId": "$(system.JobId)", + "TimelineId": "$(system.TimelineId)", + "TaskInstanceId": "$(system.TaskInstanceId)", + "AuthToken": "$(system.AccessToken)" + } + waitForCompletion: "true" + +# Agent phase. Depends on previous job. +- job: 'Publish' + displayName: 'Publish' + pool: + vmImage: 'windows-latest' + variables: + skipComponentGovernanceDetection: ${{ true }} + runCodesignValidationInjection: ${{ false }} + dependsOn: + - 'SignPackage' + steps: + + # Downloads all the setup files and its dependencies. + - task: AzureCLI@1 + displayName: 'Azure Setup' + inputs: + azureSubscription: '$(PackageManagerSubscription)' + scriptLocation: inlineScript + inlineScript: 'az storage blob download-batch -d . --pattern * -s servicewrapper --output none' + env: + AZURE_STORAGE_CONNECTION_STRING: $(ValidationStorageAccountConnectionString) + + # Publish rebuild. + - task: CmdLine@2 + displayName: 'Publish' + inputs: + script: 'PackageManagerServiceWrapper.exe rebuild-publish --operationId %BUILD_BUILDNUMBER%' + failOnStderr: true + condition: succeeded() + env: + AzureWebJobsStorage: $(ValidationStorageAccountConnectionString) + AzureServicesAuthConnectionString: $(AzureServicesAuthConnectionString) + CacheConnectionString: $(BackupCacheConnectionString) + CacheAStorageAccountConnectionString: $(CacheAStorageAccountConnectionString) + CacheAStorageAccountName: $(CacheAStorageAccountName) + CacheBStorageAccountConnectionString: $(CacheBStorageAccountConnectionString) + CacheBStorageAccountName: $(CacheBStorageAccountName) + PackageManagerSubscriptionId: $(PackageManagerSubscriptionId) + PackageManagerCdnProfile: $(PackageManagerCdnProfile) + PackageManagerConnectionStringKv: $(PackageManagerConnectionStringKv) + DIApplicationInsightKey: $(DIApplicationInsightKey) + +# Agent phase. Depends on previous job. +- job: 'Cleanup' + displayName: 'Cleanup' + pool: + vmImage: 'windows-latest' + dependsOn: + - 'Rebuild' + - 'Publish' + - 'SignPackage' + condition: succeededOrFailed() + variables: + skipComponentGovernanceDetection: ${{ true }} + runCodesignValidationInjection: ${{ false }} + steps: + + # Don't clone repo. + - checkout: none + + - task: AzureCLI@1 + displayName: 'Azure Setup' + inputs: + azureSubscription: '$(PackageManagerSubscription)' + scriptLocation: inlineScript + inlineScript: 'az storage blob download-batch -d . --pattern * -s servicewrapper --output none' + env: + AZURE_STORAGE_CONNECTION_STRING: $(ValidationStorageAccountConnectionString) + + - task: CmdLine@2 + displayName: 'Package Manager Cleanup' + inputs: + script: 'PackageManagerServiceWrapper.exe rebuild-cleanup --operationId %BUILD_BUILDNUMBER%' + failOnStderr: true + condition: succeeded() + env: + AzureWebJobsStorage: $(ValidationStorageAccountConnectionString) + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + DIApplicationInsightKey: $(DIApplicationInsightKey) diff --git a/DevOpsPipelineDefinitions/validation-pipeline.yaml b/DevOpsPipelineDefinitions/validation-pipeline.yaml new file mode 100644 index 0000000000000..0af38df98fe74 --- /dev/null +++ b/DevOpsPipelineDefinitions/validation-pipeline.yaml @@ -0,0 +1,107 @@ +# Validation pipeline for manifest on pull requests. + +# Name of the run +name: '$(Build.DefinitionName)-$(Build.DefinitionVersion)-$(System.PullRequest.PullRequestNumber)-$(Date:yyyyMMdd)-$(Rev:r)' + +trigger: none +pr: none + +jobs: + +# Agent phase. Process pull request changes and validate manifests. +- job: 'FileValidation' + displayName: 'Pull Request Validation' + pool: + vmImage: 'windows-latest' + steps: + + # Downloads all the setup files and its dependencies. + - task: AzureCLI@1 + displayName: 'Azure Setup' + inputs: + azureSubscription: '$(PackageManagerSubscription)' + scriptLocation: inlineScript + inlineScript: 'az storage blob download-batch -d . --pattern * -s servicewrapper --output none' + env: + AZURE_STORAGE_CONNECTION_STRING: $(ValidationStorageAccountConnectionString) + + # PackageManager setup + - script: 'pkgmgr_validation_setup.cmd' + name: 'pkgmgrsetup' + displayName: 'PackageManager Setup' + workingDirectory: scripts + env: + HOST_KEY: $(AzureFunctionHostKey) + SMART_SCREEN_ENDPOINT: $(PackageManagerSmartScreenEndpoint) + SCAN_ENDPOINT: $(PackageManagerScanEndpoint) + + # Validates integrity of pull request. + - task: CmdLine@2 + displayName: 'Validate Pull Request' + inputs: + script: 'PackageManagerServiceWrapper.exe process-pr --operationId %BUILD_BUILDNUMBER%' + failOnStderr: true + condition: succeeded() + env: + AzureWebJobsStorage: $(ValidationStorageAccountConnectionString) + DIApplicationInsightKey: $(DIApplicationInsightKey) + + # Validates manifest integrity. + - task: CmdLine@2 + displayName: 'Validate Manifest' + inputs: + script: 'PackageManagerServiceWrapper.exe validate-manifests --operationId %BUILD_BUILDNUMBER%' + failOnStderr: true + condition: succeeded() + env: + AzureWebJobsStorage: $(ValidationStorageAccountConnectionString) + DIApplicationInsightKey: $(DIApplicationInsightKey) + +# Agentless phase. Depends on previous job. +- job: 'ContentValidation' + pool: server + displayName: 'Manifest Content Validation' + timeoutInMinutes: 1500 + dependsOn: + - 'FileValidation' + variables: + HostKeySecret: $[ dependencies.FileValidation.outputs['pkgmgrsetup.hostkey']] + SmartScreenEndpointSecret: $[ dependencies.FileValidation.outputs['pkgmgrsetup.smartScreenEndpoint']] + ScanEndpointSecret: $[ dependencies.FileValidation.outputs['pkgmgrsetup.scanEndpoint']] + steps: + + # Scans all the urls from manifest contents. + - task: AzureFunction@1 + displayName: 'Validation URLs in manifest files' + inputs: + function: '$(SmartScreenEndpointSecret)' + key: '$(HostKeySecret)' + body: | + { + "operationId": "$(Build.BuildNumber)", + "ProjectId": "$(system.TeamProjectId)", + "PlanId": "$(system.PlanId)", + "JobId": "$(system.JobId)", + "TimelineId": "$(system.TimelineId)", + "TaskInstanceId": "$(system.TaskInstanceId)", + "AuthToken": "$(system.AccessToken)" + } + waitForCompletion: "true" + + # Scan installers in manifests. + - task: AzureFunction@1 + displayName: 'Installers Scan' + inputs: + function: '$(ScanEndpointSecret)' + key: '$(HostKeySecret)' + body: | + { + "operationId": "$(Build.BuildNumber)", + "ProjectId": "$(system.TeamProjectId)", + "PlanId": "$(system.PlanId)", + "JobId": "$(system.JobId)", + "TimelineId": "$(system.TimelineId)", + "TaskInstanceId": "$(system.TaskInstanceId)", + "AuthToken": "$(system.AccessToken)" + } + waitForCompletion: "true"