From 2afd48d10ad012d8fca9c2de83825bfdfb582c04 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Fri, 22 Nov 2024 17:06:26 +0000 Subject: [PATCH 01/12] EES-5446 - removing unhealthy check for Data Processor staging slot to reduce noise during deployments. Prior to this we would see lots of FileNotFoundExceptions in the staging slot only during deploys. --- .../public-api/publicApiDataProcessor.bicep | 38 ++----------------- 1 file changed, 3 insertions(+), 35 deletions(-) diff --git a/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep b/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep index 29a9a45063..59741bcaf4 100644 --- a/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep +++ b/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep @@ -6,6 +6,9 @@ param resourceNames ResourceNames @description('Specifies the location for all resources.') param location string +@description('Alert metric name prefix') +param metricsNamePrefix string + @description('The Application Insights key that is associated with this resource') param applicationInsightsKey string @@ -113,41 +116,6 @@ module dataProcessorFunctionAppModule '../../components/functionApp.bicep' = { } } -module functionAppHealthAlert '../../components/alerts/sites/healthAlert.bicep' = if (deployAlerts) { - name: '${resourceNames.publicApi.dataProcessor}HealthDeploy' - params: { - resourceNames: [resourceNames.publicApi.dataProcessor] - alertsGroupName: resourceNames.existingResources.alertsGroup - tagValues: tagValues - } -} - -module storageAccountAvailabilityAlerts '../../components/alerts/storageAccounts/availabilityAlert.bicep' = if (deployAlerts) { - name: '${resourceNames.publicApi.dataProcessor}StorageAvailabilityDeploy' - params: { - resourceNames: [ - dataProcessorFunctionAppModule.outputs.managementStorageAccountName - dataProcessorFunctionAppModule.outputs.slot1StorageAccountName - dataProcessorFunctionAppModule.outputs.slot2StorageAccountName - ] - alertsGroupName: resourceNames.existingResources.alertsGroup - tagValues: tagValues - } -} - -module fileServiceAvailabilityAlerts '../../components/alerts/fileServices/availabilityAlert.bicep' = if (deployAlerts) { - name: '${resourceNames.publicApi.dataProcessor}FsAvailabilityDeploy' - params: { - resourceNames: [ - dataProcessorFunctionAppModule.outputs.managementStorageAccountName - dataProcessorFunctionAppModule.outputs.slot1StorageAccountName - dataProcessorFunctionAppModule.outputs.slot2StorageAccountName - ] - alertsGroupName: resourceNames.existingResources.alertsGroup - tagValues: tagValues - } -} - output managedIdentityName string = dataProcessorFunctionAppManagedIdentity.name output managedIdentityClientId string = dataProcessorFunctionAppManagedIdentity.properties.clientId output publicApiDataFileShareMountPath string = publicApiDataFileShareMountPath From af64f987e09016f0392ac8c9bbb603f76577214a Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Mon, 25 Nov 2024 16:12:39 +0000 Subject: [PATCH 02/12] EES-5446 - added StatusCheck endpoint to provide count of active orchestrations via HTTP call. Added script to Data Processor deploy tasks to poll until active orchestrations are all complete prior to initiating slot swap --- .../public-api/publicApiDataProcessor.bicep | 1 + .../ci/jobs/deploy-data-processor.yml | 32 ++++++++++++++++++ .../public-api/components/functionApp.bicep | 2 +- .../templates/public-api/main.bicep | 5 +-- .../ProcessorFunctionsIntegrationTest.cs | 1 + .../Functions/StatusCheckFunction.cs | 33 +++++++++++++++++++ 6 files changed, 71 insertions(+), 3 deletions(-) create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/StatusCheckFunction.cs diff --git a/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep b/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep index 59741bcaf4..ef734a3027 100644 --- a/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep +++ b/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep @@ -119,3 +119,4 @@ module dataProcessorFunctionAppModule '../../components/functionApp.bicep' = { output managedIdentityName string = dataProcessorFunctionAppManagedIdentity.name output managedIdentityClientId string = dataProcessorFunctionAppManagedIdentity.properties.clientId output publicApiDataFileShareMountPath string = publicApiDataFileShareMountPath +output url string = dataProcessorFunctionAppModule.outputs.url diff --git a/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml b/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml index c55178694b..1e7a08daca 100644 --- a/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml +++ b/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml @@ -135,6 +135,38 @@ jobs: publicNetworkAccess=Disabled \ siteConfig.publicNetworkAccess=Disabled + - task: AzureCLI@2 + displayName: Wait for active orchestrations to complete + retryCountOnTaskFailure: 1 + inputs: + azureSubscription: ${{ parameters.serviceConnection }} + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + set -e + + pollingTimeSeconds=5 + maxAttempts=24 + attempts=1 + + while [ "$attempts" -le "$maxAttempts" ]; do + + echo "Attempt number $attempts to check if the Data Processor is ready for deployment" + + activeOrchestrations=`curl -s $dataProcessorFunctionAppUrl/api/StatusCheck | jq -r '.activeOrchestrations != 0'` + + if [[ "$activeOrchestrations" == "false2" ]]; then + echo "No active orchestrations running on the Data Processor - slot swapping can proceed" + exit 0 + fi + + attempts=$((attempts + 1)) + + done + + echo "Timed out waiting for active Data processor orchestrations to complete." + exit 1 + - task: AzureCLI@2 displayName: Swap slots retryCountOnTaskFailure: 1 diff --git a/infrastructure/templates/public-api/components/functionApp.bicep b/infrastructure/templates/public-api/components/functionApp.bicep index 4e274ad4cd..a2ccd99f2d 100644 --- a/infrastructure/templates/public-api/components/functionApp.bicep +++ b/infrastructure/templates/public-api/components/functionApp.bicep @@ -363,6 +363,6 @@ module privateEndpointModule 'privateEndpoint.bicep' = if (privateEndpointSubnet } output functionAppName string = functionApp.name -output managementStorageAccountName string = sharedStorageAccountName +output url string = 'https://${functionApp.properties.defaultHostName}'output managementStorageAccountName string = sharedStorageAccountName output slot1StorageAccountName string = slot1StorageAccountName output slot2StorageAccountName string = slot2StorageAccountName diff --git a/infrastructure/templates/public-api/main.bicep b/infrastructure/templates/public-api/main.bicep index 92c0cafd7e..9137450fb3 100644 --- a/infrastructure/templates/public-api/main.bicep +++ b/infrastructure/templates/public-api/main.bicep @@ -73,7 +73,7 @@ param deployContainerApp bool = true // TODO EES-5128 - Note that this has been added temporarily to avoid 10+ minute deploys where it appears that PSQL // will redeploy even if no changes exist in this deploy from the previous one. @description('Does the PostgreSQL Flexible Server require any updates? False by default to avoid unnecessarily lengthy deploys.') -param updatePsqlFlexibleServer bool = false +param deployPsqlFlexibleServer bool = false param deployAlerts bool = false @@ -208,7 +208,7 @@ module logAnalyticsWorkspaceModule 'application/shared/logAnalyticsWorkspace.bic } } -module postgreSqlServerModule 'application/shared/postgreSqlFlexibleServer.bicep' = if (updatePsqlFlexibleServer) { +module postgreSqlServerModule 'application/shared/postgreSqlFlexibleServer.bicep' = if (deployPsqlFlexibleServer) { name: 'postgreSqlFlexibleServerApplicationModuleDeploy' params: { location: location @@ -387,6 +387,7 @@ module dataProcessorModule 'application/public-api/publicApiDataProcessor.bicep' output dataProcessorContentDbConnectionStringSecretKey string = 'ees-publicapi-data-processor-connectionstring-contentdb' output dataProcessorPsqlConnectionStringSecretKey string = 'ees-publicapi-data-processor-connectionstring-publicdatadb' output dataProcessorFunctionAppManagedIdentityClientId string = dataProcessorModule.outputs.managedIdentityClientId +output dataProcessorFunctionAppUrl string = dataProcessorModule.outputs.url output coreStorageConnectionStringSecretKey string = coreStorage.outputs.coreStorageConnectionStringSecretKey output keyVaultName string = resourceNames.existingResources.keyVault diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/ProcessorFunctionsIntegrationTest.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/ProcessorFunctionsIntegrationTest.cs index b15821ee4c..b79e4c8267 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/ProcessorFunctionsIntegrationTest.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/ProcessorFunctionsIntegrationTest.cs @@ -301,6 +301,7 @@ protected override IEnumerable GetFunctionTypes() typeof(HandleProcessingFailureFunction), typeof(HealthCheckFunctions), typeof(BulkDeleteDataSetVersionsFunction), + typeof(StatusCheckFunction), ]; } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/StatusCheckFunction.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/StatusCheckFunction.cs new file mode 100644 index 0000000000..1878f9defd --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/StatusCheckFunction.cs @@ -0,0 +1,33 @@ +using Microsoft.AspNetCore.Mvc; +using Microsoft.Azure.Functions.Worker; +using Microsoft.DurableTask.Client; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; + +public class StatusCheckFunction +{ + private static readonly OrchestrationQuery ActiveOrchestrationsQuery = new() + { + Statuses = new List + { + OrchestrationRuntimeStatus.Pending, + OrchestrationRuntimeStatus.Running + } + }; + + [Function("StatusCheck")] + [Produces("application/json")] + public static async Task StatusCheck( + [HttpTrigger(AuthorizationLevel.Anonymous, "get")] +#pragma warning disable IDE0060 + HttpRequestMessage request, +#pragma warning restore IDE0060 + [DurableClient] DurableTaskClient client) + { + var activeOrchestrations = await client + .GetAllInstancesAsync(filter: ActiveOrchestrationsQuery) + .ToListAsync(); + + return new OkObjectResult(new { ActiveOrchestrations = activeOrchestrations.Count }); + } +} From 457d63a27cc79477266917fc18d458b77dc155f5 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Tue, 26 Nov 2024 10:19:24 +0000 Subject: [PATCH 03/12] EES-5446 - adding in support for allowing specific IP address ranges access to the Data Processor app. Temporarily allowing anonymous access and public internet access while trialling active orchestrations endpoint --- .../public-api/publicApiDataProcessor.bicep | 10 ++- .../public-api/ci/azure-pipelines.yml | 22 +++++-- .../ci/jobs/deploy-data-processor.yml | 66 +++++++++---------- .../templates/public-api/ci/stages/deploy.yml | 1 + .../public-api/components/functionApp.bicep | 23 +++++++ .../templates/public-api/main.bicep | 24 +++++-- .../templates/public-api/types.bicep | 1 + 7 files changed, 99 insertions(+), 48 deletions(-) diff --git a/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep b/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep index ef734a3027..efd3861fa4 100644 --- a/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep +++ b/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep @@ -18,8 +18,11 @@ param dataProcessorFunctionAppExists bool @description('Specifies the Application (Client) Id of a pre-existing App Registration used to represent the Data Processor Function App.') param dataProcessorAppRegistrationClientId string -@description('Public API Storage : Firewall rules.') -param storageFirewallRules FirewallRule[] = [] +@description('The IP address ranges that can access the Data Processor storage accounts.') +param storageFirewallRules FirewallRule[] + +@description('The IP address ranges that can access the Data Processor Function App endpoints.') +param functionAppFirewallRules FirewallRule[] = [] @description('Whether to create or update Azure Monitor alerts during this deploy') param deployAlerts bool @@ -75,7 +78,8 @@ module dataProcessorFunctionAppModule '../../components/functionApp.bicep' = { applicationInsightsKey: applicationInsightsKey subnetId: outboundVnetSubnet.id privateEndpointSubnetId: inboundVnetSubnet.id - publicNetworkAccessEnabled: false + publicNetworkAccessEnabled: true + functionAppEndpointFirewallRules: functionAppFirewallRules entraIdAuthentication: { appRegistrationClientId: dataProcessorAppRegistrationClientId allowedClientIds: [ diff --git a/infrastructure/templates/public-api/ci/azure-pipelines.yml b/infrastructure/templates/public-api/ci/azure-pipelines.yml index f3d3a98422..0942166943 100644 --- a/infrastructure/templates/public-api/ci/azure-pipelines.yml +++ b/infrastructure/templates/public-api/ci/azure-pipelines.yml @@ -1,12 +1,18 @@ trigger: none parameters: - - name: deployContainerApp - displayName: Can we deploy the Container App yet? This is dependent on the PostgreSQL Flexible Server being set up and having users manually added. - default: true - - name: updatePsqlFlexibleServer + - name: deploySharedPrivateDnsZones + displayName: Do the shared Private DNS Zones need creating or updating? + default: false + - name: deployPsqlFlexibleServer displayName: Does the PostgreSQL Flexible Server require any updates? False by default to avoid unnecessarily lengthy deploys. default: false + - name: deployContainerApp + displayName: Does the Public API Container App need creating or updating? This is dependent on the PostgreSQL Flexible Server being set up and having users manually added. + default: true + - name: deployDataProcessor + displayName: Does the Data Processor need creating or updating? + default: true - name: deployAlerts displayName: Whether to create or update Azure Monitor alerts during this deploy. default: false @@ -41,10 +47,14 @@ variables: value: $[eq(variables['Build.SourceBranch'], 'refs/heads/master')] - name: vmImageName value: ubuntu-latest + - name: deploySharedPrivateDnsZones + value: ${{ parameters.deploySharedPrivateDnsZones }} + - name: deployPsqlFlexibleServer + value: ${{ parameters.deployPsqlFlexibleServer }} - name: deployContainerApp value: ${{ parameters.deployContainerApp }} - - name: updatePsqlFlexibleServer - value: ${{ parameters.updatePsqlFlexibleServer }} + - name: deployDataProcessor + value: ${{ parameters.deployDataProcessor }} - name: deployAlerts value: ${{ parameters.deployAlerts }} diff --git a/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml b/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml index 1e7a08daca..d75023e53b 100644 --- a/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml +++ b/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml @@ -67,23 +67,23 @@ jobs: # Add Private Endpoint to Data Processor Function App into the VMSS VNet to allow # DevOps to deploy the Data Processor Function App without having to temporarily # make it publicly accessible. - - task: AzureCLI@2 - displayName: Temporarily enable public network access before deploy - retryCountOnTaskFailure: 1 - inputs: - azureSubscription: ${{ parameters.serviceConnection }} - scriptType: bash - scriptLocation: inlineScript - inlineScript: | - set -e + # - task: AzureCLI@2 + # displayName: Temporarily enable public network access before deploy + # retryCountOnTaskFailure: 1 + # inputs: + # azureSubscription: ${{ parameters.serviceConnection }} + # scriptType: bash + # scriptLocation: inlineScript + # inlineScript: | + # set -e - az functionapp update \ - --name $(dataProcessorFunctionAppName) \ - --resource-group $(resourceGroupName) \ - --slot staging \ - --set \ - publicNetworkAccess=Enabled \ - siteConfig.publicNetworkAccess=Enabled + # az functionapp update \ + # --name $(dataProcessorFunctionAppName) \ + # --resource-group $(resourceGroupName) \ + # --slot staging \ + # --set \ + # publicNetworkAccess=Enabled \ + # siteConfig.publicNetworkAccess=Enabled # TODO EES-5128 # Retry deploying the Function App in order to allow the staging slot the time to @@ -116,24 +116,24 @@ jobs: # Add Private Endpoint to Data Processor Function App into the VMSS VNet to allow # DevOps to deploy the Data Processor Function App without having to temporarily # make it publicly accessible. - - task: AzureCLI@2 - displayName: Disable public network access after deploy - retryCountOnTaskFailure: 1 - condition: always() - inputs: - azureSubscription: ${{ parameters.serviceConnection }} - scriptType: bash - scriptLocation: inlineScript - inlineScript: | - set -e + # - task: AzureCLI@2 + # displayName: Disable public network access after deploy + # retryCountOnTaskFailure: 1 + # condition: always() + # inputs: + # azureSubscription: ${{ parameters.serviceConnection }} + # scriptType: bash + # scriptLocation: inlineScript + # inlineScript: | + # set -e - az functionapp update \ - --name $(dataProcessorFunctionAppName) \ - --resource-group $(resourceGroupName) \ - --slot staging \ - --set \ - publicNetworkAccess=Disabled \ - siteConfig.publicNetworkAccess=Disabled + # az functionapp update \ + # --name $(dataProcessorFunctionAppName) \ + # --resource-group $(resourceGroupName) \ + # --slot staging \ + # --set \ + # publicNetworkAccess=Disabled \ + # siteConfig.publicNetworkAccess=Disabled - task: AzureCLI@2 displayName: Wait for active orchestrations to complete diff --git a/infrastructure/templates/public-api/ci/stages/deploy.yml b/infrastructure/templates/public-api/ci/stages/deploy.yml index 62adedfe7c..b3f2f4a7b9 100644 --- a/infrastructure/templates/public-api/ci/stages/deploy.yml +++ b/infrastructure/templates/public-api/ci/stages/deploy.yml @@ -60,4 +60,5 @@ stages: parameters: serviceConnection: ${{ parameters.serviceConnection }} environment: ${{ parameters.environment }} + condition: eq(variables.deployDataProcessor, true) dependsOn: DeployPublicApiInfrastructure diff --git a/infrastructure/templates/public-api/components/functionApp.bicep b/infrastructure/templates/public-api/components/functionApp.bicep index a2ccd99f2d..6363cff501 100644 --- a/infrastructure/templates/public-api/components/functionApp.bicep +++ b/infrastructure/templates/public-api/components/functionApp.bicep @@ -36,6 +36,9 @@ param privateEndpointSubnetId string? @description('Specifies whether this Function App is accessible from the public internet') param publicNetworkAccessEnabled bool = false +@description('IP address ranges that are allowed to access the Function App endpoints. Dependent on "publicNetworkAccessEnabled" being true.') +param functionAppEndpointFirewallRules FirewallRule[] = [] + @description('An existing Managed Identity\'s Resource Id with which to associate this Function App') param userAssignedManagedIdentityParams { id: string @@ -172,6 +175,15 @@ resource slot2FileShare 'Microsoft.Storage/storageAccounts/fileServices/shares@2 ] } +var firewallRules = [for firewallRule in functionAppEndpointFirewallRules: { + ipAddress: firewallRule.cidr + action: 'Allow' + tag: 'Default' + priority: 100 + name: firewallRule.name + description: firewallRule.description +}] + var commonSiteProperties = { enabled: true httpsOnly: true @@ -192,6 +204,17 @@ var commonSiteProperties = { } keyVaultReferenceIdentity: keyVaultReferenceIdentity publicNetworkAccess: publicNetworkAccessEnabled ? 'Enabled' : 'Disabled' + // ipSecurityRestrictions: firewallRules + ipSecurityRestrictionsDefaultAction: 'Deny' + scmIpSecurityRestrictions: [ + { + ipAddress: 'Any' + action: 'Allow' + priority: 2147483647 + name: 'Allow all' + description: 'Allow all access' + } + ] } // Create the main production deploy slot. diff --git a/infrastructure/templates/public-api/main.bicep b/infrastructure/templates/public-api/main.bicep index 9137450fb3..96a4820e5e 100644 --- a/infrastructure/templates/public-api/main.bicep +++ b/infrastructure/templates/public-api/main.bicep @@ -67,6 +67,9 @@ param dateProvisioned string = utcNow('u') @description('The tags of the Docker images to deploy.') param dockerImagesTag string = '' +@description('Do the shared Private DNS Zones need creating or updating?') +param deploySharedPrivateDnsZones bool = false + @description('Can we deploy the Container App yet? This is dependent on the PostgreSQL Flexible Server being set up and having users manually added.') param deployContainerApp bool = true @@ -170,7 +173,8 @@ module coreStorage 'application/shared/coreStorage.bicep' = { } } -module privateDnsZonesModule 'application/shared/privateDnsZones.bicep' = { +module privateDnsZonesModule 'application/shared/privateDnsZones.bicep' = + if (deploySharedPrivateDnsZones || deployPsqlFlexibleServer || deployDataProcessor) { name: 'privateDnsZonesApplicationModuleDeploy' params: { resourceNames: resourceNames @@ -366,11 +370,12 @@ module appGatewayModule 'application/shared/appGateway.bicep' = if (deployContai } } -module dataProcessorModule 'application/public-api/publicApiDataProcessor.bicep' = { +module dataProcessorModule 'application/public-api/publicApiDataProcessor.bicep' = if (deployDataProcessor) { name: 'publicApiDataProcessorApplicationModuleDeploy' params: { location: location resourceNames: resourceNames + metricsNamePrefix: '${subscription}PublicDataProcessor' applicationInsightsKey: appInsightsModule.outputs.appInsightsKey dataProcessorAppRegistrationClientId: dataProcessorAppRegistrationClientId storageFirewallRules: storageFirewallRules @@ -386,12 +391,19 @@ module dataProcessorModule 'application/public-api/publicApiDataProcessor.bicep' output dataProcessorContentDbConnectionStringSecretKey string = 'ees-publicapi-data-processor-connectionstring-contentdb' output dataProcessorPsqlConnectionStringSecretKey string = 'ees-publicapi-data-processor-connectionstring-publicdatadb' -output dataProcessorFunctionAppManagedIdentityClientId string = dataProcessorModule.outputs.managedIdentityClientId -output dataProcessorFunctionAppUrl string = dataProcessorModule.outputs.url + +output dataProcessorFunctionAppManagedIdentityClientId string = deployDataProcessor + ? dataProcessorModule.outputs.managedIdentityClientId + : '' +output dataProcessorFunctionAppUrl string = deployDataProcessor + ? dataProcessorModule.outputs.url + : '' + +output dataProcessorPublicApiDataFileShareMountPath string = deployDataProcessor + ? dataProcessorModule.outputs.publicApiDataFileShareMountPath + : '' output coreStorageConnectionStringSecretKey string = coreStorage.outputs.coreStorageConnectionStringSecretKey output keyVaultName string = resourceNames.existingResources.keyVault -output dataProcessorPublicApiDataFileShareMountPath string = dataProcessorModule.outputs.publicApiDataFileShareMountPath - output enableThemeDeletion bool = enableThemeDeletion diff --git a/infrastructure/templates/public-api/types.bicep b/infrastructure/templates/public-api/types.bicep index fb5199aafa..8dc60f8b9a 100644 --- a/infrastructure/templates/public-api/types.bicep +++ b/infrastructure/templates/public-api/types.bicep @@ -43,6 +43,7 @@ type ResourceNames = { @export() type FirewallRule = { name: string + description: string? cidr: string } From c4b07e2cc0f126c879d777145c6f06edb853d3a5 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Tue, 26 Nov 2024 11:08:18 +0000 Subject: [PATCH 04/12] EES-5446 - added dummy long-running orchestration with which we can test the deploy delay of the Data Processor. --- .../public-api/publicApiDataProcessor.bicep | 2 +- .../ci/jobs/deploy-data-processor.yml | 114 +++++++++--------- .../ci/jobs/deploy-infrastructure.yml | 6 +- .../templates/public-api/ci/stages/deploy.yml | 1 - .../public-api/ci/tasks/deploy-bicep.yml | 12 +- .../public-api/components/functionApp.bicep | 28 +++-- .../components/siteAzureAuthentication.bicep | 2 +- .../templates/public-api/main.bicep | 4 +- .../Extensions/HttpRequestExtensions.cs | 9 ++ .../Functions/LogRunningTriggerFunction.cs | 49 ++++++++ .../Functions/LongRunningOrchestration.cs | 55 +++++++++ .../Model/LongRunningOrchestrationContext.cs | 6 + 12 files changed, 212 insertions(+), 76 deletions(-) create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LogRunningTriggerFunction.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningOrchestration.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Model/LongRunningOrchestrationContext.cs diff --git a/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep b/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep index efd3861fa4..08fad8fa14 100644 --- a/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep +++ b/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep @@ -123,4 +123,4 @@ module dataProcessorFunctionAppModule '../../components/functionApp.bicep' = { output managedIdentityName string = dataProcessorFunctionAppManagedIdentity.name output managedIdentityClientId string = dataProcessorFunctionAppManagedIdentity.properties.clientId output publicApiDataFileShareMountPath string = publicApiDataFileShareMountPath -output url string = dataProcessorFunctionAppModule.outputs.url +output stagingUrl string = dataProcessorFunctionAppModule.outputs.stagingUrl diff --git a/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml b/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml index d75023e53b..40f856fd68 100644 --- a/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml +++ b/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml @@ -10,15 +10,16 @@ parameters: jobs: - deployment: DeployPublicDataProcessor displayName: Deploy Public Data Processor + condition: and(succeeded(), eq(variables.deployDataProcessor, true)) dependsOn: ${{ parameters.dependsOn }} environment: ${{ parameters.environment }} strategy: runOnce: deploy: steps: - - download: MainBuild - displayName: Download Public API Data Processor artifact - artifact: public-api-data-processor + # - download: MainBuild + # displayName: Download Public API Data Processor artifact + # artifact: public-api-data-processor - template: ../tasks/bicep-output-variables.yml parameters: @@ -27,41 +28,41 @@ jobs: # We do config updates out of Bicep template so we can implement slot swapping. # Changes are first deployed to the staging slot and combined with a fresh # code deploy prior to being swapped with the production slot. - - task: AzureCLI@2 - displayName: Update staging slot app settings - retryCountOnTaskFailure: 1 - inputs: - azureSubscription: ${{ parameters.serviceConnection }} - scriptType: bash - scriptLocation: inlineScript - inlineScript: | - set -e + # - task: AzureCLI@2 + # displayName: Update staging slot app settings + # retryCountOnTaskFailure: 1 + # inputs: + # azureSubscription: ${{ parameters.serviceConnection }} + # scriptType: bash + # scriptLocation: inlineScript + # inlineScript: | + # set -e - az functionapp config appsettings set \ - --name $(dataProcessorFunctionAppName) \ - --resource-group $(resourceGroupName) \ - --slot staging \ - --settings \ - "App__PrivateStorageConnectionString=@Microsoft.KeyVault(VaultName=$(keyVaultName); SecretName=$(coreStorageConnectionStringSecretKey))" \ - "App__EnableThemeDeletion=$(enableThemeDeletion)" \ - "AZURE_CLIENT_ID=$(dataProcessorFunctionAppManagedIdentityClientId)" \ - "DataFiles__BasePath=$(dataProcessorPublicApiDataFileShareMountPath)" + # az functionapp config appsettings set \ + # --name $(dataProcessorFunctionAppName) \ + # --resource-group $(resourceGroupName) \ + # --slot staging \ + # --settings \ + # "App__PrivateStorageConnectionString=@Microsoft.KeyVault(VaultName=$(keyVaultName); SecretName=$(coreStorageConnectionStringSecretKey))" \ + # "App__EnableThemeDeletion=$(enableThemeDeletion)" \ + # "AZURE_CLIENT_ID=$(dataProcessorFunctionAppManagedIdentityClientId)" \ + # "DataFiles__BasePath=$(dataProcessorPublicApiDataFileShareMountPath)" - az webapp config connection-string set \ - --name $(dataProcessorFunctionAppName) \ - --resource-group $(resourceGroupName) \ - --connection-string-type SQLAzure \ - --slot staging \ - --settings \ - "ContentDb=@Microsoft.KeyVault(VaultName=$(keyVaultName); SecretName=$(dataProcessorContentDbConnectionStringSecretKey))" + # az webapp config connection-string set \ + # --name $(dataProcessorFunctionAppName) \ + # --resource-group $(resourceGroupName) \ + # --connection-string-type SQLAzure \ + # --slot staging \ + # --settings \ + # "ContentDb=@Microsoft.KeyVault(VaultName=$(keyVaultName); SecretName=$(dataProcessorContentDbConnectionStringSecretKey))" - az webapp config connection-string set \ - --name $(dataProcessorFunctionAppName) \ - --resource-group $(resourceGroupName) \ - --connection-string-type PostgreSQL \ - --slot staging \ - --settings \ - "PublicDataDb=@Microsoft.KeyVault(VaultName=$(keyVaultName); SecretName=$(dataProcessorPsqlConnectionStringSecretKey))" + # az webapp config connection-string set \ + # --name $(dataProcessorFunctionAppName) \ + # --resource-group $(resourceGroupName) \ + # --connection-string-type PostgreSQL \ + # --slot staging \ + # --settings \ + # "PublicDataDb=@Microsoft.KeyVault(VaultName=$(keyVaultName); SecretName=$(dataProcessorPsqlConnectionStringSecretKey))" # TODO EES-5128 # Add Private Endpoint to Data Processor Function App into the VMSS VNet to allow @@ -97,20 +98,20 @@ jobs: # Client IDs / Identities that can access the Function App. The Service Principal # that is performing the deploy can be accessed by using the `addSpnToEnvironment` # config option in the task definition and using the $(servicePrincipalId) variable. - - task: AzureCLI@2 - displayName: Deploy to staging slot - retryCountOnTaskFailure: 10 - inputs: - azureSubscription: ${{ parameters.serviceConnection }} - scriptType: bash - scriptLocation: inlineScript - inlineScript: | - set -e - az functionapp deployment source config-zip \ - --src '$(Pipeline.Workspace)/MainBuild/public-api-data-processor/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.zip' \ - --name $(dataProcessorFunctionAppName) \ - --resource-group $(resourceGroupName) \ - --slot staging + # - task: AzureCLI@2 + # displayName: Deploy to staging slot + # retryCountOnTaskFailure: 10 + # inputs: + # azureSubscription: ${{ parameters.serviceConnection }} + # scriptType: bash + # scriptLocation: inlineScript + # inlineScript: | + # set -e + # az functionapp deployment source config-zip \ + # --src '$(Pipeline.Workspace)/MainBuild/public-api-data-processor/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.zip' \ + # --name $(dataProcessorFunctionAppName) \ + # --resource-group $(resourceGroupName) \ + # --slot staging # TODO EES-5128 # Add Private Endpoint to Data Processor Function App into the VMSS VNet to allow @@ -137,30 +138,35 @@ jobs: - task: AzureCLI@2 displayName: Wait for active orchestrations to complete - retryCountOnTaskFailure: 1 inputs: azureSubscription: ${{ parameters.serviceConnection }} scriptType: bash scriptLocation: inlineScript inlineScript: | set -e + set -x pollingTimeSeconds=5 - maxAttempts=24 + maxAttempts=50 attempts=1 while [ "$attempts" -le "$maxAttempts" ]; do echo "Attempt number $attempts to check if the Data Processor is ready for deployment" + echo "Calling $(dataProcessorFunctionAppStagingUrl)/api/StatusCheck to check for active orchestrations" - activeOrchestrations=`curl -s $dataProcessorFunctionAppUrl/api/StatusCheck | jq -r '.activeOrchestrations != 0'` - - if [[ "$activeOrchestrations" == "false2" ]]; then + activeOrchestrations=`curl -s $(dataProcessorFunctionAppStagingUrl)/api/StatusCheck | jq -r '.activeOrchestrations != 0'` + + echo "Orchestration results are $activeOrchestrations" + + if [[ "$activeOrchestrations" == "false" ]]; then echo "No active orchestrations running on the Data Processor - slot swapping can proceed" exit 0 fi attempts=$((attempts + 1)) + + sleep $pollingTimeSeconds done diff --git a/infrastructure/templates/public-api/ci/jobs/deploy-infrastructure.yml b/infrastructure/templates/public-api/ci/jobs/deploy-infrastructure.yml index 4deeb28a24..541651a229 100644 --- a/infrastructure/templates/public-api/ci/jobs/deploy-infrastructure.yml +++ b/infrastructure/templates/public-api/ci/jobs/deploy-infrastructure.yml @@ -35,8 +35,9 @@ jobs: action: validate serviceConnection: ${{ parameters.serviceConnection }} parameterFile: $(paramFile) + deploySharedPrivateDnsZones: false + deployPsqlFlexibleServer: false deployContainerApp: true - updatePsqlFlexibleServer: false deployAlerts: false dataProcessorExists: true @@ -62,8 +63,9 @@ jobs: action: create serviceConnection: ${{ parameters.serviceConnection }} parameterFile: $(paramFile) + deploySharedPrivateDnsZones: $(deploySharedPrivateDnsZones) + deployPsqlFlexibleServer: $(deployPsqlFlexibleServer) deployContainerApp: $(deployContainerApp) - updatePsqlFlexibleServer: $(updatePsqlFlexibleServer) deployAlerts: $(deployAlerts) dataProcessorExists: $(dataProcessorExists) diff --git a/infrastructure/templates/public-api/ci/stages/deploy.yml b/infrastructure/templates/public-api/ci/stages/deploy.yml index b3f2f4a7b9..62adedfe7c 100644 --- a/infrastructure/templates/public-api/ci/stages/deploy.yml +++ b/infrastructure/templates/public-api/ci/stages/deploy.yml @@ -60,5 +60,4 @@ stages: parameters: serviceConnection: ${{ parameters.serviceConnection }} environment: ${{ parameters.environment }} - condition: eq(variables.deployDataProcessor, true) dependsOn: DeployPublicApiInfrastructure diff --git a/infrastructure/templates/public-api/ci/tasks/deploy-bicep.yml b/infrastructure/templates/public-api/ci/tasks/deploy-bicep.yml index fa5ceeaa33..a4b5102cce 100644 --- a/infrastructure/templates/public-api/ci/tasks/deploy-bicep.yml +++ b/infrastructure/templates/public-api/ci/tasks/deploy-bicep.yml @@ -11,14 +11,20 @@ parameters: type: string - name: parameterFile type: string + - name: deploySharedPrivateDnsZones + type: string + - name: deployPsqlFlexibleServer + type: string - name: deployContainerApp type: string - - name: updatePsqlFlexibleServer + default: true + - name: deployDataProcessor type: string - name: deployAlerts type: string - name: dataProcessorExists type: string + default: true steps: - task: AzureCLI@2 @@ -45,8 +51,10 @@ steps: storageFirewallRules='$(maintenanceFirewallRules)' \ acrResourceGroupName='$(acrResourceGroupName)' \ dockerImagesTag='$(resources.pipeline.MainBuild.runName)' \ + deploySharedPrivateDnsZones=${{ parameters.deploySharedPrivateDnsZones }} \ + deployPsqlFlexibleServer=${{ parameters.deployPsqlFlexibleServer }} \ deployContainerApp=${{ parameters.deployContainerApp }} \ - updatePsqlFlexibleServer=${{ parameters.updatePsqlFlexibleServer }} \ + deployDataProcessor=${{ parameters.deployDataProcessor }} \ deployAlerts=${{ parameters.deployAlerts }} \ dataProcessorFunctionAppExists=${{ parameters.dataProcessorExists }} \ dataProcessorAppRegistrationClientId='$(dataProcessorAppRegistrationClientId)' \ diff --git a/infrastructure/templates/public-api/components/functionApp.bicep b/infrastructure/templates/public-api/components/functionApp.bicep index 6363cff501..af3cc7c910 100644 --- a/infrastructure/templates/public-api/components/functionApp.bicep +++ b/infrastructure/templates/public-api/components/functionApp.bicep @@ -201,20 +201,20 @@ var commonSiteProperties = { netFrameworkVersion: '8.0' linuxFxVersion: appServicePlanOS == 'Linux' ? 'DOTNET-ISOLATED|8.0' : null keyVaultReferenceIdentity: keyVaultReferenceIdentity + publicNetworkAccess: publicNetworkAccessEnabled ? 'Enabled' : 'Disabled' + // ipSecurityRestrictions: firewallRules + ipSecurityRestrictionsDefaultAction: 'Allow' // TODO Deny! + scmIpSecurityRestrictions: [ + { + ipAddress: 'Any' + action: 'Allow' + priority: 2147483647 + name: 'Allow all' + description: 'Allow all access' + } + ] } keyVaultReferenceIdentity: keyVaultReferenceIdentity - publicNetworkAccess: publicNetworkAccessEnabled ? 'Enabled' : 'Disabled' - // ipSecurityRestrictions: firewallRules - ipSecurityRestrictionsDefaultAction: 'Deny' - scmIpSecurityRestrictions: [ - { - ipAddress: 'Any' - action: 'Allow' - priority: 2147483647 - name: 'Allow all' - description: 'Allow all access' - } - ] } // Create the main production deploy slot. @@ -386,6 +386,8 @@ module privateEndpointModule 'privateEndpoint.bicep' = if (privateEndpointSubnet } output functionAppName string = functionApp.name -output url string = 'https://${functionApp.properties.defaultHostName}'output managementStorageAccountName string = sharedStorageAccountName +output url string = 'https://${functionApp.properties.defaultHostName}' +output stagingUrl string = 'https://${functionApp.name}-staging.azurewebsites.net' +output managementStorageAccountName string = sharedStorageAccountName output slot1StorageAccountName string = slot1StorageAccountName output slot2StorageAccountName string = slot2StorageAccountName diff --git a/infrastructure/templates/public-api/components/siteAzureAuthentication.bicep b/infrastructure/templates/public-api/components/siteAzureAuthentication.bicep index 0bbce6d3f9..6b1864737b 100644 --- a/infrastructure/templates/public-api/components/siteAzureAuthentication.bicep +++ b/infrastructure/templates/public-api/components/siteAzureAuthentication.bicep @@ -19,7 +19,7 @@ param requireAuthentication bool = true var properties = { globalValidation: { requireAuthentication: requireAuthentication - unauthenticatedClientAction: requireAuthentication ? 'Return401' : null + unauthenticatedClientAction: requireAuthentication ? 'Return401' : 'AllowAnonymous' } httpSettings: { requireHttps: true diff --git a/infrastructure/templates/public-api/main.bicep b/infrastructure/templates/public-api/main.bicep index 96a4820e5e..b65b17d44e 100644 --- a/infrastructure/templates/public-api/main.bicep +++ b/infrastructure/templates/public-api/main.bicep @@ -395,8 +395,8 @@ output dataProcessorPsqlConnectionStringSecretKey string = 'ees-publicapi-data-p output dataProcessorFunctionAppManagedIdentityClientId string = deployDataProcessor ? dataProcessorModule.outputs.managedIdentityClientId : '' -output dataProcessorFunctionAppUrl string = deployDataProcessor - ? dataProcessorModule.outputs.url +output dataProcessorFunctionAppStagingUrl string = deployDataProcessor + ? dataProcessorModule.outputs.stagingUrl : '' output dataProcessorPublicApiDataFileShareMountPath string = deployDataProcessor diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/HttpRequestExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/HttpRequestExtensions.cs index 97709252ed..3d95e32591 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/HttpRequestExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/HttpRequestExtensions.cs @@ -114,4 +114,13 @@ public static bool GetRequestParamBool( var paramValue = GetRequestParam(httpRequest, paramName, defaultValue + ""); return bool.Parse(paramValue); } + + public static int GetRequestParamInt( + this HttpRequest httpRequest, + string paramName, + int defaultValue) + { + var paramValue = GetRequestParam(httpRequest, paramName, defaultValue + ""); + return int.Parse(paramValue); + } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LogRunningTriggerFunction.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LogRunningTriggerFunction.cs new file mode 100644 index 0000000000..916f801a56 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LogRunningTriggerFunction.cs @@ -0,0 +1,49 @@ +using GovUk.Education.ExploreEducationStatistics.Common.Extensions; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Azure.Functions.Worker; +using Microsoft.DurableTask; +using Microsoft.DurableTask.Client; +using Microsoft.Extensions.Logging; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; + +public class LogRunningTriggerFunction( + ILogger logger) +{ + [Function(nameof(TriggerLongRunningOrchestration))] + public async Task TriggerLongRunningOrchestration( + [HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = nameof(TriggerLongRunningOrchestration))] + HttpRequest httpRequest, + [DurableClient] DurableTaskClient client, + CancellationToken cancellationToken) + { + var instanceId = Guid.NewGuid(); + + var durationSeconds = + httpRequest.GetRequestParamInt(paramName: "durationSeconds", 60); + + const string orchestratorName = + nameof(LogRunningOrchestration.ProcessLongRunningOrchestration); + + var options = new StartOrchestrationOptions { InstanceId = instanceId.ToString() }; + + logger.LogInformation( + "Scheduling '{OrchestratorName}' (InstanceId={InstanceId}, DurationSeconds={DurationSeconds}))", + orchestratorName, + instanceId, + durationSeconds); + + await client.ScheduleNewOrchestrationInstanceAsync( + orchestratorName, + new LongRunningOrchestrationContext + { + DurationSeconds = durationSeconds + }, + options, + cancellationToken); + + return new OkResult(); + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningOrchestration.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningOrchestration.cs new file mode 100644 index 0000000000..0e789ff8bf --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningOrchestration.cs @@ -0,0 +1,55 @@ +using System.Diagnostics; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Extensions; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; +using Microsoft.Azure.Functions.Worker; +using Microsoft.DurableTask; +using Microsoft.Extensions.Logging; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; + +public class LogRunningOrchestration(ILogger logger) +{ + [Function(nameof(ProcessLongRunningOrchestration))] + public static async Task ProcessLongRunningOrchestration( + [OrchestrationTrigger] TaskOrchestrationContext context, + LongRunningOrchestrationContext input) + { + var logger = context.CreateReplaySafeLogger(nameof(ProcessLongRunningOrchestration)); + + logger.LogInformation( + "Processing long-running orchestration (InstanceId={InstanceId}, DurationSeconds={DurationSeconds})", + context.InstanceId, + input.DurationSeconds); + + try + { + await context.CallActivity(nameof(LongRunningActivity), logger, context.InstanceId); + } + catch (Exception e) + { + logger.LogError(e, + "Activity failed with an exception (InstanceId={InstanceId}, DurationSeconds={DurationSeconds})", + context.InstanceId, + input.DurationSeconds); + + await context.CallActivity(ActivityNames.HandleProcessingFailure, logger, context.InstanceId); + } + } + + [Function(nameof(LongRunningActivity))] + public async Task LongRunningActivity( + [ActivityTrigger] Guid instanceId, + int durationSeconds, + CancellationToken cancellationToken) + { + var stopwatch = Stopwatch.StartNew(); + + while (stopwatch.Elapsed.Seconds < durationSeconds) + { + await Task.Delay(10000, cancellationToken); + + logger.LogInformation($"Long-running orchestration running for {stopwatch.Elapsed.Seconds} " + + $"out of {durationSeconds} seconds"); + } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Model/LongRunningOrchestrationContext.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Model/LongRunningOrchestrationContext.cs new file mode 100644 index 0000000000..40eb0ff82d --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Model/LongRunningOrchestrationContext.cs @@ -0,0 +1,6 @@ +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; + +public record LongRunningOrchestrationContext +{ + public required int DurationSeconds { get; init; } +} From d2c5cc8e85c79d7d63d8bc9043d0261a7262d2eb Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Tue, 26 Nov 2024 15:58:59 +0000 Subject: [PATCH 05/12] EES-5446 - grant Azure DevOps SPN access to Data Processor via EasyAuth and granting of an access token --- .../public-api/publicApiDataProcessor.bicep | 11 +- .../ci/jobs/deploy-data-processor.yml | 160 +++++++++++------- .../templates/public-api/ci/stages/deploy.yml | 10 +- .../public-api/ci/tasks/deploy-bicep.yml | 6 +- .../public-api/components/functionApp.bicep | 2 +- .../components/siteAzureAuthentication.bicep | 11 +- .../templates/public-api/main.bicep | 9 + 7 files changed, 130 insertions(+), 79 deletions(-) diff --git a/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep b/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep index 08fad8fa14..a7218c96c6 100644 --- a/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep +++ b/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep @@ -18,6 +18,10 @@ param dataProcessorFunctionAppExists bool @description('Specifies the Application (Client) Id of a pre-existing App Registration used to represent the Data Processor Function App.') param dataProcessorAppRegistrationClientId string +@description('Specifies the principal id of the Azure DevOps SPN.') +@secure() +param devopsServicePrincipalId string + @description('The IP address ranges that can access the Data Processor storage accounts.') param storageFirewallRules FirewallRule[] @@ -42,7 +46,6 @@ resource adminAppServiceIdentity 'Microsoft.ManagedIdentity/identities@2023-01-3 } var adminAppClientId = adminAppServiceIdentity.properties.clientId -var adminAppPrincipalId = adminAppServiceIdentity.properties.principalId resource publicApiStorageAccount 'Microsoft.Storage/storageAccounts@2023-05-01' existing = { name: resourceNames.publicApi.publicApiStorageAccount @@ -84,10 +87,9 @@ module dataProcessorFunctionAppModule '../../components/functionApp.bicep' = { appRegistrationClientId: dataProcessorAppRegistrationClientId allowedClientIds: [ adminAppClientId + devopsServicePrincipalId ] - allowedPrincipalIds: [ - adminAppPrincipalId - ] + allowedPrincipalIds: [] requireAuthentication: true } userAssignedManagedIdentityParams: { @@ -123,4 +125,5 @@ module dataProcessorFunctionAppModule '../../components/functionApp.bicep' = { output managedIdentityName string = dataProcessorFunctionAppManagedIdentity.name output managedIdentityClientId string = dataProcessorFunctionAppManagedIdentity.properties.clientId output publicApiDataFileShareMountPath string = publicApiDataFileShareMountPath +output url string = dataProcessorFunctionAppModule.outputs.url output stagingUrl string = dataProcessorFunctionAppModule.outputs.stagingUrl diff --git a/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml b/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml index 40f856fd68..6684191858 100644 --- a/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml +++ b/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml @@ -17,9 +17,9 @@ jobs: runOnce: deploy: steps: - # - download: MainBuild - # displayName: Download Public API Data Processor artifact - # artifact: public-api-data-processor + - download: MainBuild + displayName: Download Public API Data Processor artifact + artifact: public-api-data-processor - template: ../tasks/bicep-output-variables.yml parameters: @@ -28,41 +28,41 @@ jobs: # We do config updates out of Bicep template so we can implement slot swapping. # Changes are first deployed to the staging slot and combined with a fresh # code deploy prior to being swapped with the production slot. - # - task: AzureCLI@2 - # displayName: Update staging slot app settings - # retryCountOnTaskFailure: 1 - # inputs: - # azureSubscription: ${{ parameters.serviceConnection }} - # scriptType: bash - # scriptLocation: inlineScript - # inlineScript: | - # set -e + - task: AzureCLI@2 + displayName: Update staging slot app settings + retryCountOnTaskFailure: 1 + inputs: + azureSubscription: ${{ parameters.serviceConnection }} + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + set -e - # az functionapp config appsettings set \ - # --name $(dataProcessorFunctionAppName) \ - # --resource-group $(resourceGroupName) \ - # --slot staging \ - # --settings \ - # "App__PrivateStorageConnectionString=@Microsoft.KeyVault(VaultName=$(keyVaultName); SecretName=$(coreStorageConnectionStringSecretKey))" \ - # "App__EnableThemeDeletion=$(enableThemeDeletion)" \ - # "AZURE_CLIENT_ID=$(dataProcessorFunctionAppManagedIdentityClientId)" \ - # "DataFiles__BasePath=$(dataProcessorPublicApiDataFileShareMountPath)" + az functionapp config appsettings set \ + --name $(dataProcessorFunctionAppName) \ + --resource-group $(resourceGroupName) \ + --slot staging \ + --settings \ + "App__PrivateStorageConnectionString=@Microsoft.KeyVault(VaultName=$(keyVaultName); SecretName=$(coreStorageConnectionStringSecretKey))" \ + "App__EnableThemeDeletion=$(enableThemeDeletion)" \ + "AZURE_CLIENT_ID=$(dataProcessorFunctionAppManagedIdentityClientId)" \ + "DataFiles__BasePath=$(dataProcessorPublicApiDataFileShareMountPath)" - # az webapp config connection-string set \ - # --name $(dataProcessorFunctionAppName) \ - # --resource-group $(resourceGroupName) \ - # --connection-string-type SQLAzure \ - # --slot staging \ - # --settings \ - # "ContentDb=@Microsoft.KeyVault(VaultName=$(keyVaultName); SecretName=$(dataProcessorContentDbConnectionStringSecretKey))" + az webapp config connection-string set \ + --name $(dataProcessorFunctionAppName) \ + --resource-group $(resourceGroupName) \ + --connection-string-type SQLAzure \ + --slot staging \ + --settings \ + "ContentDb=@Microsoft.KeyVault(VaultName=$(keyVaultName); SecretName=$(dataProcessorContentDbConnectionStringSecretKey))" - # az webapp config connection-string set \ - # --name $(dataProcessorFunctionAppName) \ - # --resource-group $(resourceGroupName) \ - # --connection-string-type PostgreSQL \ - # --slot staging \ - # --settings \ - # "PublicDataDb=@Microsoft.KeyVault(VaultName=$(keyVaultName); SecretName=$(dataProcessorPsqlConnectionStringSecretKey))" + az webapp config connection-string set \ + --name $(dataProcessorFunctionAppName) \ + --resource-group $(resourceGroupName) \ + --connection-string-type PostgreSQL \ + --slot staging \ + --settings \ + "PublicDataDb=@Microsoft.KeyVault(VaultName=$(keyVaultName); SecretName=$(dataProcessorPsqlConnectionStringSecretKey))" # TODO EES-5128 # Add Private Endpoint to Data Processor Function App into the VMSS VNet to allow @@ -98,20 +98,20 @@ jobs: # Client IDs / Identities that can access the Function App. The Service Principal # that is performing the deploy can be accessed by using the `addSpnToEnvironment` # config option in the task definition and using the $(servicePrincipalId) variable. - # - task: AzureCLI@2 - # displayName: Deploy to staging slot - # retryCountOnTaskFailure: 10 - # inputs: - # azureSubscription: ${{ parameters.serviceConnection }} - # scriptType: bash - # scriptLocation: inlineScript - # inlineScript: | - # set -e - # az functionapp deployment source config-zip \ - # --src '$(Pipeline.Workspace)/MainBuild/public-api-data-processor/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.zip' \ - # --name $(dataProcessorFunctionAppName) \ - # --resource-group $(resourceGroupName) \ - # --slot staging + - task: AzureCLI@2 + displayName: Deploy to staging slot + retryCountOnTaskFailure: 10 + inputs: + azureSubscription: ${{ parameters.serviceConnection }} + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + set -e + az functionapp deployment source config-zip \ + --src '$(Pipeline.Workspace)/MainBuild/public-api-data-processor/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.zip' \ + --name $(dataProcessorFunctionAppName) \ + --resource-group $(resourceGroupName) \ + --slot staging # TODO EES-5128 # Add Private Endpoint to Data Processor Function App into the VMSS VNet to allow @@ -136,6 +136,43 @@ jobs: # publicNetworkAccess=Disabled \ # siteConfig.publicNetworkAccess=Disabled + - task: AzureCLI@2 + displayName: Wait for Data Processor to start up + inputs: + azureSubscription: ${{ parameters.serviceConnection }} + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + set -e + + accessToken=`az account get-access-token \ + --resource $(dataProcessorAppRegistrationClientId) \ + --query "accessToken" \ + -o tsv` + + pollingTimeSeconds=5 + maxAttempts=50 + attempt=1 + + while [ "$attempt" -le "$maxAttempts" ]; do + + httpStatusCode=`curl --write-out '%{http_code}' -H "Authorization: Bearer $accessToken" -s --output /dev/null $(dataProcessorFunctionAppStagingUrl)/api/HealthCheck` + + echo "Attempt number $attempt - calling $(dataProcessorFunctionAppStagingUrl)/api/HealthCheck to check for Data Processor staging slot to be healthy - got HTTP Status Code $httpStatusCode." + + if [[ "$httpStatusCode" == "200" ]]; then + echo "Data Processor staging slot has started successfully and is healthy - deployment can continue." + exit 0 + fi + + attempt=$((attempt + 1)) + sleep $pollingTimeSeconds + + done + + echo "Timed out waiting for Data processor to start up." + exit 1 + - task: AzureCLI@2 displayName: Wait for active orchestrations to complete inputs: @@ -144,33 +181,33 @@ jobs: scriptLocation: inlineScript inlineScript: | set -e - set -x + + accessToken=`az account get-access-token \ + --resource $(dataProcessorAppRegistrationClientId) \ + --query "accessToken" \ + -o tsv` pollingTimeSeconds=5 maxAttempts=50 - attempts=1 + attempt=1 - while [ "$attempts" -le "$maxAttempts" ]; do + while [ "$attempt" -le "$maxAttempts" ]; do - echo "Attempt number $attempts to check if the Data Processor is ready for deployment" - echo "Calling $(dataProcessorFunctionAppStagingUrl)/api/StatusCheck to check for active orchestrations" - - activeOrchestrations=`curl -s $(dataProcessorFunctionAppStagingUrl)/api/StatusCheck | jq -r '.activeOrchestrations != 0'` + activeOrchestrations=`curl -H "Authorization: Bearer $accessToken" -s $(dataProcessorFunctionAppUrl)/api/StatusCheck | jq -r '.activeOrchestrations != 0'` - echo "Orchestration results are $activeOrchestrations" + echo "Attempt number $attempt - called $(dataProcessorFunctionAppUrl)/api/StatusCheck to check for active orchestrations - currently $activeOrchestrations active orchestrations running." if [[ "$activeOrchestrations" == "false" ]]; then - echo "No active orchestrations running on the Data Processor - slot swapping can proceed" + echo "No active orchestrations running on the Data Processor - slot swapping can proceed." exit 0 fi - attempts=$((attempts + 1)) - + attempt=$((attempt + 1)) sleep $pollingTimeSeconds done - echo "Timed out waiting for active Data processor orchestrations to complete." + echo "Timed out waiting for active Data Processor orchestrations to complete." exit 1 - task: AzureCLI@2 @@ -182,6 +219,7 @@ jobs: scriptLocation: inlineScript inlineScript: | set -e + az functionapp deployment slot swap \ --name $(dataProcessorFunctionAppName) \ --resource-group $(resourceGroupName) \ diff --git a/infrastructure/templates/public-api/ci/stages/deploy.yml b/infrastructure/templates/public-api/ci/stages/deploy.yml index 62adedfe7c..aad36c1949 100644 --- a/infrastructure/templates/public-api/ci/stages/deploy.yml +++ b/infrastructure/templates/public-api/ci/stages/deploy.yml @@ -50,11 +50,11 @@ stages: environment: ${{ parameters.environment }} bicepParamFile: ${{ parameters.bicepParamFile }} - - template: ../jobs/deploy-api-docs.yml - parameters: - serviceConnection: ${{ parameters.serviceConnection }} - environment: ${{ parameters.environment }} - dependsOn: DeployPublicApiInfrastructure + # - template: ../jobs/deploy-api-docs.yml + # parameters: + # serviceConnection: ${{ parameters.serviceConnection }} + # environment: ${{ parameters.environment }} + # dependsOn: DeployPublicApiInfrastructure - template: ../jobs/deploy-data-processor.yml parameters: diff --git a/infrastructure/templates/public-api/ci/tasks/deploy-bicep.yml b/infrastructure/templates/public-api/ci/tasks/deploy-bicep.yml index a4b5102cce..17efac2dd5 100644 --- a/infrastructure/templates/public-api/ci/tasks/deploy-bicep.yml +++ b/infrastructure/templates/public-api/ci/tasks/deploy-bicep.yml @@ -33,9 +33,10 @@ steps: azureSubscription: ${{ parameters.serviceConnection }} scriptType: bash scriptLocation: inlineScript + addSpnToEnvironment: true inlineScript: | set -e - + az deployment group ${{ parameters.action }} \ --name $(infraDeployName) \ --resource-group $(resourceGroupName) \ @@ -58,4 +59,5 @@ steps: deployAlerts=${{ parameters.deployAlerts }} \ dataProcessorFunctionAppExists=${{ parameters.dataProcessorExists }} \ dataProcessorAppRegistrationClientId='$(dataProcessorAppRegistrationClientId)' \ - apiAppRegistrationClientId='$(apiAppRegistrationClientId)' + apiAppRegistrationClientId='$(apiAppRegistrationClientId)' \ + devopsServicePrincipalId="$servicePrincipalId" \ No newline at end of file diff --git a/infrastructure/templates/public-api/components/functionApp.bicep b/infrastructure/templates/public-api/components/functionApp.bicep index af3cc7c910..c212fc814c 100644 --- a/infrastructure/templates/public-api/components/functionApp.bicep +++ b/infrastructure/templates/public-api/components/functionApp.bicep @@ -386,7 +386,7 @@ module privateEndpointModule 'privateEndpoint.bicep' = if (privateEndpointSubnet } output functionAppName string = functionApp.name -output url string = 'https://${functionApp.properties.defaultHostName}' +output url string = 'https://${functionApp.name}.azurewebsites.net' output stagingUrl string = 'https://${functionApp.name}-staging.azurewebsites.net' output managementStorageAccountName string = sharedStorageAccountName output slot1StorageAccountName string = slot1StorageAccountName diff --git a/infrastructure/templates/public-api/components/siteAzureAuthentication.bicep b/infrastructure/templates/public-api/components/siteAzureAuthentication.bicep index 6b1864737b..e0f525dbcd 100644 --- a/infrastructure/templates/public-api/components/siteAzureAuthentication.bicep +++ b/infrastructure/templates/public-api/components/siteAzureAuthentication.bicep @@ -11,7 +11,7 @@ param stagingSlotName string = 'none' param allowedClientIds string[] = [] @description('Specifies an optional set of Principal Ids of Managed Identities that are allowed to access this resource') -param allowedPrincipalIds string[] = [] +param allowedPrincipalIds string[] @description('Specifies whether all calls to this resource should be authenticated or not. Defaults to true') param requireAuthentication bool = true @@ -35,15 +35,14 @@ var properties = { allowedAudiences: [ 'api://${clientId}' ] - defaultAuthorizationPolicy: { + defaultAuthorizationPolicy: union({ allowedApplications: union( [clientId], allowedClientIds ) - allowedPrincipals: { - identities: allowedPrincipalIds - } - } + }, length(allowedPrincipalIds) > 0 ? { + identities: allowedPrincipalIds + } : {}) } } } diff --git a/infrastructure/templates/public-api/main.bicep b/infrastructure/templates/public-api/main.bicep index b65b17d44e..30834a98c5 100644 --- a/infrastructure/templates/public-api/main.bicep +++ b/infrastructure/templates/public-api/main.bicep @@ -96,6 +96,10 @@ param dataProcessorAppRegistrationClientId string = '' @description('Specifies the Application (Client) Id of a pre-existing App Registration used to represent the API Container App.') param apiAppRegistrationClientId string = '' +@description('Specifies the principal id of the Azure DevOps SPN.') +@secure() +param devopsServicePrincipalId string = '' + @description('Specifies whether or not test Themes can be deleted in the environment.') param enableThemeDeletion bool = false @@ -378,6 +382,7 @@ module dataProcessorModule 'application/public-api/publicApiDataProcessor.bicep' metricsNamePrefix: '${subscription}PublicDataProcessor' applicationInsightsKey: appInsightsModule.outputs.appInsightsKey dataProcessorAppRegistrationClientId: dataProcessorAppRegistrationClientId + devopsServicePrincipalId: devopsServicePrincipalId storageFirewallRules: storageFirewallRules dataProcessorFunctionAppExists: dataProcessorFunctionAppExists deployAlerts: deployAlerts @@ -395,6 +400,10 @@ output dataProcessorPsqlConnectionStringSecretKey string = 'ees-publicapi-data-p output dataProcessorFunctionAppManagedIdentityClientId string = deployDataProcessor ? dataProcessorModule.outputs.managedIdentityClientId : '' + +output dataProcessorFunctionAppUrl string = deployDataProcessor + ? dataProcessorModule.outputs.url + : '' output dataProcessorFunctionAppStagingUrl string = deployDataProcessor ? dataProcessorModule.outputs.stagingUrl : '' From 94ac0901a90f79f1acd3d87695362c3e6e7dda92 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Wed, 27 Nov 2024 20:40:00 +0000 Subject: [PATCH 06/12] EES-5446 - adding network restrictions to lock down Data Processor down to Admin, Azure Pipeline runners (temporarily disabled in favour of whitelisting AzureCloud service tag) and general maintenance firewall rules. --- azure-pipelines-main.yml | 625 +++++++++--------- .../application/shared/virtualNetwork.bicep | 3 + .../ci/jobs/deploy-data-processor.yml | 51 +- .../public-api/ci/tasks/deploy-bicep.yml | 6 +- .../public-api/components/functionApp.bicep | 13 +- .../templates/public-api/main.bicep | 44 +- .../templates/public-api/types.bicep | 3 +- ...ction.cs => LongRunningTriggerFunction.cs} | 4 +- .../Functions/StatusCheckFunction.cs | 6 +- 9 files changed, 410 insertions(+), 345 deletions(-) rename src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/{LogRunningTriggerFunction.cs => LongRunningTriggerFunction.cs} (95%) diff --git a/azure-pipelines-main.yml b/azure-pipelines-main.yml index d5fe21618f..3ede71f80f 100644 --- a/azure-pipelines-main.yml +++ b/azure-pipelines-main.yml @@ -6,6 +6,7 @@ parameters: - master - dev - test + - EES-5446-prevent-function-app-slot-swap-until-orchestrations-complete variables: BuildConfiguration: Release @@ -59,64 +60,64 @@ jobs: # custom: format whitespace src/GovUk.Education.ExploreEducationStatistics.sln --verify-no-changes --severity error # arguments: --verify-no-changes --verbosity diagnostic - - task: DotNetCoreCLI@2 - displayName: Verify Formatting and Style - inputs: - command: custom - custom: format - ## TODO: Remove "--severity error" once style formatter has been run across project - arguments: style --verify-no-changes --verbosity diagnostic --severity error - projects: src/GovUk.Education.ExploreEducationStatistics.sln - - - task: DotNetCoreCLI@2 - displayName: Verify Formatting and Style - inputs: - command: custom - custom: format - ## TODO: Remove "--severity error" once work has been done to resolve build warnings (https://dfedigital.atlassian.net/browse/EES-4594). - arguments: analyzers --verify-no-changes --verbosity diagnostic --severity error - projects: src/GovUk.Education.ExploreEducationStatistics.sln +# - task: DotNetCoreCLI@2 +# displayName: Verify Formatting and Style +# inputs: +# command: custom +# custom: format +# ## TODO: Remove "--severity error" once style formatter has been run across project +# arguments: style --verify-no-changes --verbosity diagnostic --severity error +# projects: src/GovUk.Education.ExploreEducationStatistics.sln +# +# - task: DotNetCoreCLI@2 +# displayName: Verify Formatting and Style +# inputs: +# command: custom +# custom: format +# ## TODO: Remove "--severity error" once work has been done to resolve build warnings (https://dfedigital.atlassian.net/browse/EES-4594). +# arguments: analyzers --verify-no-changes --verbosity diagnostic --severity error +# projects: src/GovUk.Education.ExploreEducationStatistics.sln # TODO: Wrap these ^ three tasks up into a single `dotnet format` task once all 3 above TODOs are TO-DONE ;) - - task: DotNetCoreCLI@2 - displayName: Test - inputs: - command: test - projects: | - **/GovUk.*[Tt]ests/*.csproj - !**/GovUk.Education.ExploreEducationStatistics.Admin.Tests/*csproj - arguments: --configuration $(BuildConfiguration) - - - task: DotNetCoreCLI@2 - displayName: Package Data API - inputs: - command: publish - publishWebProjects: false - projects: '**/GovUk.Education.ExploreEducationStatistics.Data.Api.csproj' - arguments: --self-contained true -r win-x64 --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory)/data-api - zipAfterPublish: True - - - task: PublishPipelineArtifact@0 - displayName: Publish Data API artifact - inputs: - artifactName: data-api - targetPath: $(Build.ArtifactStagingDirectory)/data-api - - - task: DotNetCoreCLI@2 - displayName: Package Content API - inputs: - command: publish - publishWebProjects: false - projects: '**/GovUk.Education.ExploreEducationStatistics.Content.Api.csproj' - arguments: --self-contained true -r win-x64 --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory)/content-api - zipAfterPublish: True - - - task: PublishPipelineArtifact@0 - displayName: Publish Content API artifact - inputs: - artifactName: content-api - targetPath: $(Build.ArtifactStagingDirectory)/content-api +# - task: DotNetCoreCLI@2 +# displayName: Test +# inputs: +# command: test +# projects: | +# **/GovUk.*[Tt]ests/*.csproj +# !**/GovUk.Education.ExploreEducationStatistics.Admin.Tests/*csproj +# arguments: --configuration $(BuildConfiguration) +# +# - task: DotNetCoreCLI@2 +# displayName: Package Data API +# inputs: +# command: publish +# publishWebProjects: false +# projects: '**/GovUk.Education.ExploreEducationStatistics.Data.Api.csproj' +# arguments: --self-contained true -r win-x64 --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory)/data-api +# zipAfterPublish: True +# +# - task: PublishPipelineArtifact@0 +# displayName: Publish Data API artifact +# inputs: +# artifactName: data-api +# targetPath: $(Build.ArtifactStagingDirectory)/data-api +# +# - task: DotNetCoreCLI@2 +# displayName: Package Content API +# inputs: +# command: publish +# publishWebProjects: false +# projects: '**/GovUk.Education.ExploreEducationStatistics.Content.Api.csproj' +# arguments: --self-contained true -r win-x64 --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory)/content-api +# zipAfterPublish: True +# +# - task: PublishPipelineArtifact@0 +# displayName: Publish Content API artifact +# inputs: +# artifactName: content-api +# targetPath: $(Build.ArtifactStagingDirectory)/content-api - task: DotNetCoreCLI@2 displayName: Package Public API @@ -166,260 +167,260 @@ jobs: artifactName: public-api-data-processor targetPath: $(Build.ArtifactStagingDirectory)/public-api-data-processor - - task: DotNetCoreCLI@2 - displayName: Package Notifier Function - inputs: - command: publish - publishWebProjects: false - projects: '**/GovUk.Education.ExploreEducationStatistics.Notifier.csproj' - arguments: --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory)/notifier - zipAfterPublish: True - - - task: PublishPipelineArtifact@0 - displayName: Publish Notifier artifact - inputs: - artifactName: notifier - targetPath: $(Build.ArtifactStagingDirectory)/notifier - - - task: DotNetCoreCLI@2 - displayName: Package Publisher Function - inputs: - command: publish - publishWebProjects: false - projects: '**/GovUk.Education.ExploreEducationStatistics.Publisher.csproj' - arguments: --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory)/publisher - zipAfterPublish: True - - - task: PublishPipelineArtifact@0 - displayName: Publish Publisher artifact - inputs: - artifactName: publisher - targetPath: $(Build.ArtifactStagingDirectory)/publisher - - - task: DotNetCoreCLI@2 - displayName: Package Processor Function - inputs: - command: publish - publishWebProjects: false - projects: '**/GovUk.Education.ExploreEducationStatistics.Data.Processor.csproj' - arguments: --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory)/processor - zipAfterPublish: True - - - task: PublishPipelineArtifact@0 - displayName: Publish Processor artifact - inputs: - artifactName: processor - targetPath: $(Build.ArtifactStagingDirectory)/processor - - - job: Admin - pool: ees-ubuntu2204-xlarge - workspace: - clean: all - steps: - - task: UseNode@1 - displayName: Install Node.js $(NodeVersion) - inputs: - version: $(NodeVersion) - - - task: Bash@3 - displayName: corepack enable - inputs: - workingDir: . - targetType: inline - script: corepack enable - - - task: UseDotNet@2 - displayName: Install .NET 8.0 SDK - inputs: - version: 8.0.x - performMultiLevelLookup: true - - - task: DotNetCoreCLI@2 - displayName: Build - inputs: - projects: '**/GovUk.Education.ExploreEducationStatistics.Admin.csproj' - arguments: --configuration $(BuildConfiguration) - - - task: DotNetCoreCLI@2 - displayName: Test - inputs: - command: test - projects: '**/GovUk.Education.ExploreEducationStatistics.Admin.Tests.csproj' - arguments: --configuration $(BuildConfiguration) --collect "Code coverage" - - - task: Bash@3 - displayName: pnpm i - inputs: - targetType: inline - script: pnpm i - - - task: Bash@3 - displayName: pnpm run build - inputs: - targetType: inline - script: pnpm --filter=explore-education-statistics-admin run build - - - task: CopyFiles@2 - displayName: Copy files to wwwroot - inputs: - SourceFolder: src/explore-education-statistics-admin/build - TargetFolder: src/GovUk.Education.ExploreEducationStatistics.Admin/wwwroot - - - task: DotNetCoreCLI@2 - displayName: Package Admin app - inputs: - command: publish - publishWebProjects: false - projects: '**/GovUk.Education.ExploreEducationStatistics.Admin.csproj' - arguments: --self-contained true -r win-x64 --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory) - - - task: PublishPipelineArtifact@0 - displayName: Publish Admin artifact - inputs: - artifactName: admin - targetPath: $(Build.ArtifactStagingDirectory) - - - job: Frontend - pool: ees-ubuntu2204-xlarge - workspace: - clean: all - steps: - - task: UseNode@1 - displayName: Install Node.js $(NodeVersion) - inputs: - version: $(NodeVersion) - - - task: Bash@3 - displayName: corepack enable - inputs: - workingDir: . - targetType: inline - script: corepack enable - - - task: Bash@3 - displayName: pnpm i - inputs: - workingDir: . - targetType: inline - script: pnpm i - - - task: Bash@3 - displayName: pnpm tsc - inputs: - workingDir: . - targetType: inline - script: pnpm tsc - - - task: Bash@3 - displayName: pnpm lint - inputs: - workingDir: . - targetType: inline - script: pnpm lint - - - task: Bash@3 - displayName: pnpm format:check - inputs: - workingDir: . - targetType: inline - script: pnpm format:check - - - task: Bash@3 - displayName: pnpm test:ci - inputs: - workingDir: . - targetType: inline - script: pnpm test:ci - - - task: PublishTestResults@2 - displayName: Publish frontend test results - inputs: - testResultsFormat: JUnit - testResultsFiles: explore-education-statistics-*/junit-*.xml - searchFolder: ./src - testRunTitle: Release Jest tests - mergeTestResults: true - - - task: Bash@3 - displayName: pnpm run build - inputs: - targetType: inline - script: pnpm --filter=explore-education-statistics-frontend run build - - - task: Docker@2 - displayName: Build Public frontend Docker image - condition: and(succeeded(), eq(variables.IsBranchDeployable, true)) - inputs: - containerRegistry: $(AcrServiceConnection) - repository: ees-public-frontend - command: build - Dockerfile: docker/public-frontend/Dockerfile - buildContext: $(System.DefaultWorkingDirectory) - tags: $(Build.BuildNumber) - arguments: --build-arg BUILD_BUILDNUMBER=$(Build.BuildNumber) - env: - DOCKER_BUILDKIT: 1 - - - task: Docker@2 - displayName: Push Public frontend Docker image - condition: and(succeeded(), eq(variables.IsBranchDeployable, true)) - inputs: - containerRegistry: $(AcrServiceConnection) - repository: ees-public-frontend - command: push - tags: $(Build.BuildNumber) - - - job: ApiDocs - pool: - vmImage: ubuntu-22.04 - workspace: - clean: all - variables: - WorkingDirectory: src/explore-education-statistics-api-docs - steps: - - task: UseNode@1 - displayName: Install Node.js $(NodeVersion) - inputs: - version: $(NodeVersion) - - - task: UseRubyVersion@0 - displayName: Install Ruby $(RubyVersion) - inputs: - versionSpec: '>= $(RubyVersion)' - - - task: Bash@3 - displayName: Build - env: - TECH_DOCS_API_URL: https://dev.statistics.api.education.gov.uk - inputs: - workingDirectory: $(WorkingDirectory) - targetType: inline - script: | - bundle install - bundle exec middleman build - - - task: PublishPipelineArtifact@1 - displayName: Publish artifact - inputs: - artifactName: public-api-docs - targetPath: $(WorkingDirectory)/build - - - job: MiscellaneousArtifacts - pool: - vmImage: ubuntu-22.04 - workspace: - clean: all - steps: - - task: CopyFiles@2 - displayName: Copy Pipfiles to tests - inputs: - Contents: | - Pipfile - Pipfile.lock - TargetFolder: tests - - - task: PublishPipelineArtifact@0 - displayName: Publish test files - inputs: - artifactName: tests - targetPath: tests +# - task: DotNetCoreCLI@2 +# displayName: Package Notifier Function +# inputs: +# command: publish +# publishWebProjects: false +# projects: '**/GovUk.Education.ExploreEducationStatistics.Notifier.csproj' +# arguments: --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory)/notifier +# zipAfterPublish: True +# +# - task: PublishPipelineArtifact@0 +# displayName: Publish Notifier artifact +# inputs: +# artifactName: notifier +# targetPath: $(Build.ArtifactStagingDirectory)/notifier +# +# - task: DotNetCoreCLI@2 +# displayName: Package Publisher Function +# inputs: +# command: publish +# publishWebProjects: false +# projects: '**/GovUk.Education.ExploreEducationStatistics.Publisher.csproj' +# arguments: --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory)/publisher +# zipAfterPublish: True +# +# - task: PublishPipelineArtifact@0 +# displayName: Publish Publisher artifact +# inputs: +# artifactName: publisher +# targetPath: $(Build.ArtifactStagingDirectory)/publisher +# +# - task: DotNetCoreCLI@2 +# displayName: Package Processor Function +# inputs: +# command: publish +# publishWebProjects: false +# projects: '**/GovUk.Education.ExploreEducationStatistics.Data.Processor.csproj' +# arguments: --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory)/processor +# zipAfterPublish: True +# +# - task: PublishPipelineArtifact@0 +# displayName: Publish Processor artifact +# inputs: +# artifactName: processor +# targetPath: $(Build.ArtifactStagingDirectory)/processor + +# - job: Admin +# pool: ees-ubuntu2204-xlarge +# workspace: +# clean: all +# steps: +# - task: UseNode@1 +# displayName: Install Node.js $(NodeVersion) +# inputs: +# version: $(NodeVersion) +# +# - task: Bash@3 +# displayName: corepack enable +# inputs: +# workingDir: . +# targetType: inline +# script: corepack enable +# +# - task: UseDotNet@2 +# displayName: Install .NET 8.0 SDK +# inputs: +# version: 8.0.x +# performMultiLevelLookup: true +# +# - task: DotNetCoreCLI@2 +# displayName: Build +# inputs: +# projects: '**/GovUk.Education.ExploreEducationStatistics.Admin.csproj' +# arguments: --configuration $(BuildConfiguration) +# +# - task: DotNetCoreCLI@2 +# displayName: Test +# inputs: +# command: test +# projects: '**/GovUk.Education.ExploreEducationStatistics.Admin.Tests.csproj' +# arguments: --configuration $(BuildConfiguration) --collect "Code coverage" +# +# - task: Bash@3 +# displayName: pnpm i +# inputs: +# targetType: inline +# script: pnpm i +# +# - task: Bash@3 +# displayName: pnpm run build +# inputs: +# targetType: inline +# script: pnpm --filter=explore-education-statistics-admin run build +# +# - task: CopyFiles@2 +# displayName: Copy files to wwwroot +# inputs: +# SourceFolder: src/explore-education-statistics-admin/build +# TargetFolder: src/GovUk.Education.ExploreEducationStatistics.Admin/wwwroot +# +# - task: DotNetCoreCLI@2 +# displayName: Package Admin app +# inputs: +# command: publish +# publishWebProjects: false +# projects: '**/GovUk.Education.ExploreEducationStatistics.Admin.csproj' +# arguments: --self-contained true -r win-x64 --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory) +# +# - task: PublishPipelineArtifact@0 +# displayName: Publish Admin artifact +# inputs: +# artifactName: admin +# targetPath: $(Build.ArtifactStagingDirectory) +# +# - job: Frontend +# pool: ees-ubuntu2204-xlarge +# workspace: +# clean: all +# steps: +# - task: UseNode@1 +# displayName: Install Node.js $(NodeVersion) +# inputs: +# version: $(NodeVersion) +# +# - task: Bash@3 +# displayName: corepack enable +# inputs: +# workingDir: . +# targetType: inline +# script: corepack enable +# +# - task: Bash@3 +# displayName: pnpm i +# inputs: +# workingDir: . +# targetType: inline +# script: pnpm i +# +# - task: Bash@3 +# displayName: pnpm tsc +# inputs: +# workingDir: . +# targetType: inline +# script: pnpm tsc +# +# - task: Bash@3 +# displayName: pnpm lint +# inputs: +# workingDir: . +# targetType: inline +# script: pnpm lint +# +# - task: Bash@3 +# displayName: pnpm format:check +# inputs: +# workingDir: . +# targetType: inline +# script: pnpm format:check +# +# - task: Bash@3 +# displayName: pnpm test:ci +# inputs: +# workingDir: . +# targetType: inline +# script: pnpm test:ci +# +# - task: PublishTestResults@2 +# displayName: Publish frontend test results +# inputs: +# testResultsFormat: JUnit +# testResultsFiles: explore-education-statistics-*/junit-*.xml +# searchFolder: ./src +# testRunTitle: Release Jest tests +# mergeTestResults: true +# +# - task: Bash@3 +# displayName: pnpm run build +# inputs: +# targetType: inline +# script: pnpm --filter=explore-education-statistics-frontend run build +# +# - task: Docker@2 +# displayName: Build Public frontend Docker image +# condition: and(succeeded(), eq(variables.IsBranchDeployable, true)) +# inputs: +# containerRegistry: $(AcrServiceConnection) +# repository: ees-public-frontend +# command: build +# Dockerfile: docker/public-frontend/Dockerfile +# buildContext: $(System.DefaultWorkingDirectory) +# tags: $(Build.BuildNumber) +# arguments: --build-arg BUILD_BUILDNUMBER=$(Build.BuildNumber) +# env: +# DOCKER_BUILDKIT: 1 +# +# - task: Docker@2 +# displayName: Push Public frontend Docker image +# condition: and(succeeded(), eq(variables.IsBranchDeployable, true)) +# inputs: +# containerRegistry: $(AcrServiceConnection) +# repository: ees-public-frontend +# command: push +# tags: $(Build.BuildNumber) + +# - job: ApiDocs +# pool: +# vmImage: ubuntu-22.04 +# workspace: +# clean: all +# variables: +# WorkingDirectory: src/explore-education-statistics-api-docs +# steps: +# - task: UseNode@1 +# displayName: Install Node.js $(NodeVersion) +# inputs: +# version: $(NodeVersion) +# +# - task: UseRubyVersion@0 +# displayName: Install Ruby $(RubyVersion) +# inputs: +# versionSpec: '>= $(RubyVersion)' +# +# - task: Bash@3 +# displayName: Build +# env: +# TECH_DOCS_API_URL: https://dev.statistics.api.education.gov.uk +# inputs: +# workingDirectory: $(WorkingDirectory) +# targetType: inline +# script: | +# bundle install +# bundle exec middleman build +# +# - task: PublishPipelineArtifact@1 +# displayName: Publish artifact +# inputs: +# artifactName: public-api-docs +# targetPath: $(WorkingDirectory)/build +# +# - job: MiscellaneousArtifacts +# pool: +# vmImage: ubuntu-22.04 +# workspace: +# clean: all +# steps: +# - task: CopyFiles@2 +# displayName: Copy Pipfiles to tests +# inputs: +# Contents: | +# Pipfile +# Pipfile.lock +# TargetFolder: tests +# +# - task: PublishPipelineArtifact@0 +# displayName: Publish test files +# inputs: +# artifactName: tests +# targetPath: tests diff --git a/infrastructure/templates/public-api/application/shared/virtualNetwork.bicep b/infrastructure/templates/public-api/application/shared/virtualNetwork.bicep index ad5f11225b..04e86e6a38 100644 --- a/infrastructure/templates/public-api/application/shared/virtualNetwork.bicep +++ b/infrastructure/templates/public-api/application/shared/virtualNetwork.bicep @@ -83,6 +83,9 @@ output adminAppServiceSubnetStartIpAddress string = parseCidr(adminSubnet.proper @description('The last usable IP address for the Admin App Service Subnet.') output adminAppServiceSubnetEndIpAddress string = parseCidr(adminSubnet.properties.addressPrefix).lastUsable +@description('The IP address range for the Admin App Service Subnet.') +output adminAppServiceSubnetCidr string = adminSubnet.properties.addressPrefix + @description('The first usable IP address for the Publisher Function App Subnet.') output publisherFunctionAppSubnetStartIpAddress string = parseCidr(publisherSubnet.properties.addressPrefix).firstUsable diff --git a/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml b/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml index 6684191858..351f4a2935 100644 --- a/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml +++ b/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml @@ -137,14 +137,13 @@ jobs: # siteConfig.publicNetworkAccess=Disabled - task: AzureCLI@2 - displayName: Wait for Data Processor to start up + displayName: Wait for Data Processor staging slot to start up inputs: azureSubscription: ${{ parameters.serviceConnection }} scriptType: bash scriptLocation: inlineScript inlineScript: | - set -e - + accessToken=`az account get-access-token \ --resource $(dataProcessorAppRegistrationClientId) \ --query "accessToken" \ @@ -180,7 +179,6 @@ jobs: scriptType: bash scriptLocation: inlineScript inlineScript: | - set -e accessToken=`az account get-access-token \ --resource $(dataProcessorAppRegistrationClientId) \ @@ -193,9 +191,13 @@ jobs: while [ "$attempt" -le "$maxAttempts" ]; do - activeOrchestrations=`curl -H "Authorization: Bearer $accessToken" -s $(dataProcessorFunctionAppUrl)/api/StatusCheck | jq -r '.activeOrchestrations != 0'` + activeOrchestrationResults=`curl -H "Authorization: Bearer $accessToken" -s $(dataProcessorFunctionAppUrl)/api/StatusCheck` + activeOrchestrationCount=`echo $activeOrchestrationResults | jq -r '.activeOrchestrations'` + activeOrchestrations=`echo $activeOrchestrationResults | jq -r '.activeOrchestrations != 0'` - echo "Attempt number $attempt - called $(dataProcessorFunctionAppUrl)/api/StatusCheck to check for active orchestrations - currently $activeOrchestrations active orchestrations running." + echo $activeOrchestrationResults + + echo "Attempt number $attempt - called $(dataProcessorFunctionAppUrl)/api/StatusCheck to check for active orchestrations - currently $activeOrchestrationCount active orchestrations running." if [[ "$activeOrchestrations" == "false" ]]; then echo "No active orchestrations running on the Data Processor - slot swapping can proceed." @@ -225,3 +227,40 @@ jobs: --resource-group $(resourceGroupName) \ --slot staging \ --target-slot production + + - task: AzureCLI@2 + displayName: Check that Data Processor is ready to serve new requests + inputs: + azureSubscription: ${{ parameters.serviceConnection }} + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + set -x + + accessToken=`az account get-access-token \ + --resource $(dataProcessorAppRegistrationClientId) \ + --query "accessToken" \ + -o tsv` + + pollingTimeSeconds=5 + maxAttempts=50 + attempt=1 + + while [ "$attempt" -le "$maxAttempts" ]; do + + httpStatusCode=`curl --write-out '%{http_code}' -H "Authorization: Bearer $accessToken" -s --output /dev/null $(dataProcessorFunctionAppUrl)/api/HealthCheck` + + echo "Attempt number $attempt - calling $(dataProcessorFunctionAppUrl)/api/HealthCheck to check for Data Processor to be healthy - got HTTP Status Code $httpStatusCode." + + if [[ "$httpStatusCode" == "200" ]]; then + echo "Data Processor has started successfully and is healthy - ready to serve new requests." + exit 0 + fi + + attempt=$((attempt + 1)) + sleep $pollingTimeSeconds + + done + + echo "Timed out waiting for Data processor to start up." + exit 1 diff --git a/infrastructure/templates/public-api/ci/tasks/deploy-bicep.yml b/infrastructure/templates/public-api/ci/tasks/deploy-bicep.yml index 17efac2dd5..c18b37ca57 100644 --- a/infrastructure/templates/public-api/ci/tasks/deploy-bicep.yml +++ b/infrastructure/templates/public-api/ci/tasks/deploy-bicep.yml @@ -47,9 +47,8 @@ steps: resourceTags='$(resourceTags)' \ postgreSqlAdminName='$(postgreSqlAdminName)' \ postgreSqlAdminPassword='$(postgreSqlAdminPassword)' \ - postgreSqlFirewallRules='$(maintenanceFirewallRules)' \ postgreSqlEntraIdAdminPrincipals='$(postgreSqlEntraIdAdminPrincipals)' \ - storageFirewallRules='$(maintenanceFirewallRules)' \ + maintenanceFirewallRules='$(maintenanceFirewallRules)' \ acrResourceGroupName='$(acrResourceGroupName)' \ dockerImagesTag='$(resources.pipeline.MainBuild.runName)' \ deploySharedPrivateDnsZones=${{ parameters.deploySharedPrivateDnsZones }} \ @@ -60,4 +59,5 @@ steps: dataProcessorFunctionAppExists=${{ parameters.dataProcessorExists }} \ dataProcessorAppRegistrationClientId='$(dataProcessorAppRegistrationClientId)' \ apiAppRegistrationClientId='$(apiAppRegistrationClientId)' \ - devopsServicePrincipalId="$servicePrincipalId" \ No newline at end of file + devopsServicePrincipalId="$servicePrincipalId" \ + pipelineRunnerCidr="$(pipelineRunnerCidr)" \ No newline at end of file diff --git a/infrastructure/templates/public-api/components/functionApp.bicep b/infrastructure/templates/public-api/components/functionApp.bicep index c212fc814c..0126d4c050 100644 --- a/infrastructure/templates/public-api/components/functionApp.bicep +++ b/infrastructure/templates/public-api/components/functionApp.bicep @@ -175,13 +175,12 @@ resource slot2FileShare 'Microsoft.Storage/storageAccounts/fileServices/shares@2 ] } -var firewallRules = [for firewallRule in functionAppEndpointFirewallRules: { +var firewallRules = [for (firewallRule, index) in functionAppEndpointFirewallRules: { + name: firewallRule.name ipAddress: firewallRule.cidr action: 'Allow' - tag: 'Default' - priority: 100 - name: firewallRule.name - description: firewallRule.description + tag: firewallRule.tag ?? 'Default' + priority: firewallRule.priority ?? (100 + index) }] var commonSiteProperties = { @@ -202,8 +201,8 @@ var commonSiteProperties = { linuxFxVersion: appServicePlanOS == 'Linux' ? 'DOTNET-ISOLATED|8.0' : null keyVaultReferenceIdentity: keyVaultReferenceIdentity publicNetworkAccess: publicNetworkAccessEnabled ? 'Enabled' : 'Disabled' - // ipSecurityRestrictions: firewallRules - ipSecurityRestrictionsDefaultAction: 'Allow' // TODO Deny! + ipSecurityRestrictions: publicNetworkAccessEnabled && length(firewallRules) > 0 ? firewallRules : null + ipSecurityRestrictionsDefaultAction: publicNetworkAccessEnabled && length(firewallRules) > 0 ? 'Deny' : 'Allow' scmIpSecurityRestrictions: [ { ipAddress: 'Any' diff --git a/infrastructure/templates/public-api/main.bicep b/infrastructure/templates/public-api/main.bicep index 30834a98c5..b3824c9c90 100644 --- a/infrastructure/templates/public-api/main.bicep +++ b/infrastructure/templates/public-api/main.bicep @@ -10,8 +10,8 @@ param location string = resourceGroup().location @description('Public API Storage : Size of the file share in GB.') param publicApiDataFileShareQuota int = 1 -@description('Public API Storage : Firewall rules.') -param storageFirewallRules FirewallRule[] = [] +@description('Firewall rules for maintenance of the service by allowing key IP ranges access to resources.') +param maintenanceFirewallRules FirewallRule[] = [] @description('Database : administrator login name.') @minLength(0) @@ -36,9 +36,6 @@ param postgreSqlStorageSizeGB int = 32 @description('Database : Azure Database for PostgreSQL Autogrow setting.') param postgreSqlAutoGrowStatus string = 'Disabled' -@description('Database : Firewall rules.') -param postgreSqlFirewallRules FirewallRule[] = [] - @description('Database : Entra ID admin principal names for this resource') param postgreSqlEntraIdAdminPrincipals PrincipalNameAndId[] = [] @@ -70,14 +67,17 @@ param dockerImagesTag string = '' @description('Do the shared Private DNS Zones need creating or updating?') param deploySharedPrivateDnsZones bool = false -@description('Can we deploy the Container App yet? This is dependent on the PostgreSQL Flexible Server being set up and having users manually added.') -param deployContainerApp bool = true - // TODO EES-5128 - Note that this has been added temporarily to avoid 10+ minute deploys where it appears that PSQL // will redeploy even if no changes exist in this deploy from the previous one. @description('Does the PostgreSQL Flexible Server require any updates? False by default to avoid unnecessarily lengthy deploys.') param deployPsqlFlexibleServer bool = false +@description('Does the Public API Container App need creating or updating? This is dependent on the PostgreSQL Flexible Server being set up and having users manually added.') +param deployContainerApp bool = true + +@description('Does the Data Processor need creating or updating?') +param deployDataProcessor bool = true + param deployAlerts bool = false @description('Public URLs of other components in the service.') @@ -100,6 +100,9 @@ param apiAppRegistrationClientId string = '' @secure() param devopsServicePrincipalId string = '' +@description('Specifies the IP address range of the pipeline runners.') +param pipelineRunnerCidr string = '' + @description('Specifies whether or not test Themes can be deleted in the environment.') param enableThemeDeletion bool = false @@ -192,7 +195,7 @@ module publicApiStorageModule 'application/public-api/publicApiStorage.bicep' = location: location resourceNames: resourceNames publicApiDataFileShareQuota: publicApiDataFileShareQuota - storageFirewallRules: storageFirewallRules + storageFirewallRules: maintenanceFirewallRules deployAlerts: deployAlerts tagValues: tagValues } @@ -226,7 +229,7 @@ module postgreSqlServerModule 'application/shared/postgreSqlFlexibleServer.bicep entraIdAdminPrincipals: postgreSqlEntraIdAdminPrincipals privateEndpointSubnetId: vNetModule.outputs.psqlFlexibleServerSubnetRef autoGrowStatus: postgreSqlAutoGrowStatus - firewallRules: postgreSqlFirewallRules + firewallRules: maintenanceFirewallRules sku: postgreSqlSkuName storageSizeGB: postgreSqlStorageSizeGB deployAlerts: deployAlerts @@ -374,6 +377,11 @@ module appGatewayModule 'application/shared/appGateway.bicep' = if (deployContai } } +var adminSubnetFirewallRule = { + name: 'Admin App Service subnet range' + cidr: vNetModule.outputs.adminAppServiceSubnetCidr +} + module dataProcessorModule 'application/public-api/publicApiDataProcessor.bicep' = if (deployDataProcessor) { name: 'publicApiDataProcessorApplicationModuleDeploy' params: { @@ -383,7 +391,21 @@ module dataProcessorModule 'application/public-api/publicApiDataProcessor.bicep' applicationInsightsKey: appInsightsModule.outputs.appInsightsKey dataProcessorAppRegistrationClientId: dataProcessorAppRegistrationClientId devopsServicePrincipalId: devopsServicePrincipalId - storageFirewallRules: storageFirewallRules + storageFirewallRules: maintenanceFirewallRules + functionAppFirewallRules: union([ + adminSubnetFirewallRule + // TODO EES-5446 - add in when static IP range available for runner scale sets + // { + // name: 'Pipeline runner IP address range' + // cidr: pipelineRunnerCidr + // } + { + cidr: 'AzureCloud' + tag: 'ServiceTag' + priority: 100 + name: 'AzureCloud' + } + ], maintenanceFirewallRules) dataProcessorFunctionAppExists: dataProcessorFunctionAppExists deployAlerts: deployAlerts tagValues: tagValues diff --git a/infrastructure/templates/public-api/types.bicep b/infrastructure/templates/public-api/types.bicep index 8dc60f8b9a..eee0b62a22 100644 --- a/infrastructure/templates/public-api/types.bicep +++ b/infrastructure/templates/public-api/types.bicep @@ -43,8 +43,9 @@ type ResourceNames = { @export() type FirewallRule = { name: string - description: string? cidr: string + priority: int? + tag: string? } @export() diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LogRunningTriggerFunction.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningTriggerFunction.cs similarity index 95% rename from src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LogRunningTriggerFunction.cs rename to src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningTriggerFunction.cs index 916f801a56..c37e5a8c6d 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LogRunningTriggerFunction.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningTriggerFunction.cs @@ -9,8 +9,8 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; -public class LogRunningTriggerFunction( - ILogger logger) +public class LongRunningTriggerFunction( + ILogger logger) { [Function(nameof(TriggerLongRunningOrchestration))] public async Task TriggerLongRunningOrchestration( diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/StatusCheckFunction.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/StatusCheckFunction.cs index 1878f9defd..877e663232 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/StatusCheckFunction.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/StatusCheckFunction.cs @@ -1,3 +1,4 @@ +using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Microsoft.Azure.Functions.Worker; using Microsoft.DurableTask.Client; @@ -14,20 +15,19 @@ public class StatusCheckFunction OrchestrationRuntimeStatus.Running } }; - + [Function("StatusCheck")] [Produces("application/json")] public static async Task StatusCheck( [HttpTrigger(AuthorizationLevel.Anonymous, "get")] #pragma warning disable IDE0060 - HttpRequestMessage request, + HttpRequest request, #pragma warning restore IDE0060 [DurableClient] DurableTaskClient client) { var activeOrchestrations = await client .GetAllInstancesAsync(filter: ActiveOrchestrationsQuery) .ToListAsync(); - return new OkObjectResult(new { ActiveOrchestrations = activeOrchestrations.Count }); } } From e61c935c8f13b6ed3898697dc71d379971928d73 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Thu, 28 Nov 2024 15:23:55 +0000 Subject: [PATCH 07/12] EES-5446 - corrected long-running test orchestration. --- .../public-api/publicApiDataProcessor.bicep | 4 +-- .../public-api/publicApiStorage.bicep | 4 +-- .../shared/postgreSqlFlexibleServer.bicep | 4 +-- .../ci/jobs/deploy-data-processor.yml | 18 +++++++------ .../public-api/ci/tasks/deploy-bicep.yml | 2 +- .../public-api/components/functionApp.bicep | 8 +++--- .../components/postgresqlDatabase.bicep | 4 +-- .../components/storageAccount.bicep | 4 +-- .../templates/public-api/main.bicep | 21 +++++++++++----- .../templates/public-api/types.bicep | 9 +++++-- .../Functions/LongRunningFunctions.cs | 25 +++++++++++++++++++ .../Functions/LongRunningOrchestration.cs | 22 ++-------------- 12 files changed, 75 insertions(+), 50 deletions(-) create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningFunctions.cs diff --git a/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep b/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep index a7218c96c6..5891ab45d5 100644 --- a/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep +++ b/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep @@ -1,4 +1,4 @@ -import { ResourceNames, FirewallRule } from '../../types.bicep' +import { ResourceNames, FirewallRule, IpRange } from '../../types.bicep' @description('Specifies common resource naming variables.') param resourceNames ResourceNames @@ -23,7 +23,7 @@ param dataProcessorAppRegistrationClientId string param devopsServicePrincipalId string @description('The IP address ranges that can access the Data Processor storage accounts.') -param storageFirewallRules FirewallRule[] +param storageFirewallRules IpRange[] @description('The IP address ranges that can access the Data Processor Function App endpoints.') param functionAppFirewallRules FirewallRule[] = [] diff --git a/infrastructure/templates/public-api/application/public-api/publicApiStorage.bicep b/infrastructure/templates/public-api/application/public-api/publicApiStorage.bicep index f61c7f4efd..1207e1a971 100644 --- a/infrastructure/templates/public-api/application/public-api/publicApiStorage.bicep +++ b/infrastructure/templates/public-api/application/public-api/publicApiStorage.bicep @@ -1,4 +1,4 @@ -import { ResourceNames, FirewallRule } from '../../types.bicep' +import { ResourceNames, IpRange } from '../../types.bicep' param resourceNames ResourceNames @@ -9,7 +9,7 @@ param location string param publicApiDataFileShareQuota int @description('Public API Storage : Firewall rules.') -param storageFirewallRules FirewallRule[] +param storageFirewallRules IpRange[] @description('Specifies a set of tags with which to tag the resource in Azure.') param tagValues object diff --git a/infrastructure/templates/public-api/application/shared/postgreSqlFlexibleServer.bicep b/infrastructure/templates/public-api/application/shared/postgreSqlFlexibleServer.bicep index 2f9469be5d..a5a9660913 100644 --- a/infrastructure/templates/public-api/application/shared/postgreSqlFlexibleServer.bicep +++ b/infrastructure/templates/public-api/application/shared/postgreSqlFlexibleServer.bicep @@ -1,4 +1,4 @@ -import { ResourceNames, FirewallRule, PrincipalNameAndId } from '../../types.bicep' +import { ResourceNames, IpRange, PrincipalNameAndId } from '../../types.bicep' @description('Specifies common resource naming variables.') param resourceNames ResourceNames @@ -23,7 +23,7 @@ param storageSizeGB int = 32 param autoGrowStatus string = 'Disabled' @description('Firewall rules.') -param firewallRules FirewallRule[] = [] +param firewallRules IpRange[] = [] @description('Specifies the subnet id that the PostgreSQL private endpoint will be attached to.') param privateEndpointSubnetId string diff --git a/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml b/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml index 351f4a2935..794677c292 100644 --- a/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml +++ b/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml @@ -143,7 +143,7 @@ jobs: scriptType: bash scriptLocation: inlineScript inlineScript: | - + accessToken=`az account get-access-token \ --resource $(dataProcessorAppRegistrationClientId) \ --query "accessToken" \ @@ -155,9 +155,11 @@ jobs: while [ "$attempt" -le "$maxAttempts" ]; do + echo "Attempt number $attempt of $maxAttempts- calling $(dataProcessorFunctionAppStagingUrl)/api/HealthCheck to check for Data Processor staging being healthy." + httpStatusCode=`curl --write-out '%{http_code}' -H "Authorization: Bearer $accessToken" -s --output /dev/null $(dataProcessorFunctionAppStagingUrl)/api/HealthCheck` - echo "Attempt number $attempt - calling $(dataProcessorFunctionAppStagingUrl)/api/HealthCheck to check for Data Processor staging slot to be healthy - got HTTP Status Code $httpStatusCode." + echo "Got HTTP status code $httpStatusCode." if [[ "$httpStatusCode" == "200" ]]; then echo "Data Processor staging slot has started successfully and is healthy - deployment can continue." @@ -188,17 +190,17 @@ jobs: pollingTimeSeconds=5 maxAttempts=50 attempt=1 - + while [ "$attempt" -le "$maxAttempts" ]; do + echo "Attempt number $attempt of $maxAttempts - callinf $(dataProcessorFunctionAppUrl)/api/StatusCheck to check for active orchestrations." + activeOrchestrationResults=`curl -H "Authorization: Bearer $accessToken" -s $(dataProcessorFunctionAppUrl)/api/StatusCheck` activeOrchestrationCount=`echo $activeOrchestrationResults | jq -r '.activeOrchestrations'` activeOrchestrations=`echo $activeOrchestrationResults | jq -r '.activeOrchestrations != 0'` - echo $activeOrchestrationResults + echo "Currently $activeOrchestrationCount active orchestrations running." - echo "Attempt number $attempt - called $(dataProcessorFunctionAppUrl)/api/StatusCheck to check for active orchestrations - currently $activeOrchestrationCount active orchestrations running." - if [[ "$activeOrchestrations" == "false" ]]; then echo "No active orchestrations running on the Data Processor - slot swapping can proceed." exit 0 @@ -248,9 +250,11 @@ jobs: while [ "$attempt" -le "$maxAttempts" ]; do + echo "Attempt number $attempt of $maxAttempts - calling $(dataProcessorFunctionAppUrl)/api/HealthCheck to check for Data Processor being healthy." + httpStatusCode=`curl --write-out '%{http_code}' -H "Authorization: Bearer $accessToken" -s --output /dev/null $(dataProcessorFunctionAppUrl)/api/HealthCheck` - echo "Attempt number $attempt - calling $(dataProcessorFunctionAppUrl)/api/HealthCheck to check for Data Processor to be healthy - got HTTP Status Code $httpStatusCode." + echo "Got HTTP status code $httpStatusCode." if [[ "$httpStatusCode" == "200" ]]; then echo "Data Processor has started successfully and is healthy - ready to serve new requests." diff --git a/infrastructure/templates/public-api/ci/tasks/deploy-bicep.yml b/infrastructure/templates/public-api/ci/tasks/deploy-bicep.yml index c18b37ca57..1903174be7 100644 --- a/infrastructure/templates/public-api/ci/tasks/deploy-bicep.yml +++ b/infrastructure/templates/public-api/ci/tasks/deploy-bicep.yml @@ -48,7 +48,7 @@ steps: postgreSqlAdminName='$(postgreSqlAdminName)' \ postgreSqlAdminPassword='$(postgreSqlAdminPassword)' \ postgreSqlEntraIdAdminPrincipals='$(postgreSqlEntraIdAdminPrincipals)' \ - maintenanceFirewallRules='$(maintenanceFirewallRules)' \ + maintenanceIpRanges='$(maintenanceFirewallRules)' \ acrResourceGroupName='$(acrResourceGroupName)' \ dockerImagesTag='$(resources.pipeline.MainBuild.runName)' \ deploySharedPrivateDnsZones=${{ parameters.deploySharedPrivateDnsZones }} \ diff --git a/infrastructure/templates/public-api/components/functionApp.bicep b/infrastructure/templates/public-api/components/functionApp.bicep index 0126d4c050..f77f780882 100644 --- a/infrastructure/templates/public-api/components/functionApp.bicep +++ b/infrastructure/templates/public-api/components/functionApp.bicep @@ -1,4 +1,4 @@ -import { FirewallRule, AzureFileShareMount, EntraIdAuthentication } from '../types.bicep' +import { FirewallRule, IpRange, AzureFileshareMount, EntraIdAuthentication } from '../types.bicep' @description('Specifies the location for all resources.') param location string @@ -71,7 +71,7 @@ param healthCheckPath string? param azureFileShares AzureFileShareMount[] = [] @description('Specifies firewall rules for the various storage accounts in use by the Function App') -param storageFirewallRules FirewallRule[] = [] +param storageFirewallRules IpRange[] = [] var reserved = appServicePlanOS == 'Linux' @@ -179,8 +179,8 @@ var firewallRules = [for (firewallRule, index) in functionAppEndpointFirewallRul name: firewallRule.name ipAddress: firewallRule.cidr action: 'Allow' - tag: firewallRule.tag ?? 'Default' - priority: firewallRule.priority ?? (100 + index) + tag: firewallRule.tag != null ? firewallRule.tag : 'Default' + priority: firewallRule.priority != null ? firewallRule.priority : 100 + index }] var commonSiteProperties = { diff --git a/infrastructure/templates/public-api/components/postgresqlDatabase.bicep b/infrastructure/templates/public-api/components/postgresqlDatabase.bicep index 273e148d32..006c67627b 100644 --- a/infrastructure/templates/public-api/components/postgresqlDatabase.bicep +++ b/infrastructure/templates/public-api/components/postgresqlDatabase.bicep @@ -1,4 +1,4 @@ -import { FirewallRule, PrincipalNameAndId } from '../types.bicep' +import { IpRange, PrincipalNameAndId } from '../types.bicep' @description('Specifies the location for all resources.') param location string @@ -40,7 +40,7 @@ param geoRedundantBackup string = 'Disabled' param databaseNames string[] @description('An array of firewall rules containing IP address ranges') -param firewallRules FirewallRule[] = [] +param firewallRules IpRange[] = [] @description('An array of Entra ID admin principal names for this resource') param entraIdAdminPrincipals PrincipalNameAndId[] = [] diff --git a/infrastructure/templates/public-api/components/storageAccount.bicep b/infrastructure/templates/public-api/components/storageAccount.bicep index 9d5576d231..a3e865d0b1 100644 --- a/infrastructure/templates/public-api/components/storageAccount.bicep +++ b/infrastructure/templates/public-api/components/storageAccount.bicep @@ -1,4 +1,4 @@ -import { FirewallRule } from '../types.bicep' +import { IpRange } from '../types.bicep' @description('Specifies the location for all resources.') param location string @@ -10,7 +10,7 @@ param storageAccountName string param allowedSubnetIds string[] = [] @description('Storage Account Network Firewall Rules') -param firewallRules FirewallRule[] = [] +param firewallRules IpRange[] = [] @description('Storage Account SKU') param skuStorageResource 'Standard_LRS' | 'Standard_GRS' | 'Standard_RAGRS' | 'Standard_ZRS' | 'Premium_LRS' | 'Premium_ZRS' | 'Standard_GZRS' | 'Standard_RAGZRS' = 'Standard_LRS' diff --git a/infrastructure/templates/public-api/main.bicep b/infrastructure/templates/public-api/main.bicep index b3824c9c90..b985dd102f 100644 --- a/infrastructure/templates/public-api/main.bicep +++ b/infrastructure/templates/public-api/main.bicep @@ -1,5 +1,5 @@ import { abbreviations } from 'abbreviations.bicep' -import { FirewallRule, PrincipalNameAndId, StaticWebAppSku } from 'types.bicep' +import { IpRange, PrincipalNameAndId, StaticWebAppSku } from 'types.bicep' @description('Environment : Subscription name e.g. s101d01. Used as a prefix for created resources.') param subscription string = '' @@ -11,7 +11,7 @@ param location string = resourceGroup().location param publicApiDataFileShareQuota int = 1 @description('Firewall rules for maintenance of the service by allowing key IP ranges access to resources.') -param maintenanceFirewallRules FirewallRule[] = [] +param maintenanceIpRanges IpRange[] = [] @description('Database : administrator login name.') @minLength(0) @@ -166,6 +166,13 @@ var resourceNames = { } } +var maintenanceFirewallRules = [for maintenanceIpRange in maintenanceIpRanges: { + name: maintenanceIpRange.name + cidr: maintenanceIpRange.cidr + tag: 'Default' + priority: 100 +}] + module vNetModule 'application/shared/virtualNetwork.bicep' = { name: 'virtualNetworkApplicationModuleDeploy' params: { @@ -195,7 +202,7 @@ module publicApiStorageModule 'application/public-api/publicApiStorage.bicep' = location: location resourceNames: resourceNames publicApiDataFileShareQuota: publicApiDataFileShareQuota - storageFirewallRules: maintenanceFirewallRules + storageFirewallRules: maintenanceIpRanges deployAlerts: deployAlerts tagValues: tagValues } @@ -229,7 +236,7 @@ module postgreSqlServerModule 'application/shared/postgreSqlFlexibleServer.bicep entraIdAdminPrincipals: postgreSqlEntraIdAdminPrincipals privateEndpointSubnetId: vNetModule.outputs.psqlFlexibleServerSubnetRef autoGrowStatus: postgreSqlAutoGrowStatus - firewallRules: maintenanceFirewallRules + firewallRules: maintenanceIpRanges sku: postgreSqlSkuName storageSizeGB: postgreSqlStorageSizeGB deployAlerts: deployAlerts @@ -380,6 +387,8 @@ module appGatewayModule 'application/shared/appGateway.bicep' = if (deployContai var adminSubnetFirewallRule = { name: 'Admin App Service subnet range' cidr: vNetModule.outputs.adminAppServiceSubnetCidr + tag: 'Default' + priority: 100 } module dataProcessorModule 'application/public-api/publicApiDataProcessor.bicep' = if (deployDataProcessor) { @@ -391,7 +400,7 @@ module dataProcessorModule 'application/public-api/publicApiDataProcessor.bicep' applicationInsightsKey: appInsightsModule.outputs.appInsightsKey dataProcessorAppRegistrationClientId: dataProcessorAppRegistrationClientId devopsServicePrincipalId: devopsServicePrincipalId - storageFirewallRules: maintenanceFirewallRules + storageFirewallRules: maintenanceIpRanges functionAppFirewallRules: union([ adminSubnetFirewallRule // TODO EES-5446 - add in when static IP range available for runner scale sets @@ -402,7 +411,7 @@ module dataProcessorModule 'application/public-api/publicApiDataProcessor.bicep' { cidr: 'AzureCloud' tag: 'ServiceTag' - priority: 100 + priority: 101 name: 'AzureCloud' } ], maintenanceFirewallRules) diff --git a/infrastructure/templates/public-api/types.bicep b/infrastructure/templates/public-api/types.bicep index eee0b62a22..6d73131a9a 100644 --- a/infrastructure/templates/public-api/types.bicep +++ b/infrastructure/templates/public-api/types.bicep @@ -40,12 +40,17 @@ type ResourceNames = { } } +@export() +type IpRange = { + name: string + cidr: string +} @export() type FirewallRule = { name: string cidr: string - priority: int? - tag: string? + priority: int + tag: string } @export() diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningFunctions.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningFunctions.cs new file mode 100644 index 0000000000..f17bc7a812 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningFunctions.cs @@ -0,0 +1,25 @@ +using System.Diagnostics; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; +using Microsoft.Azure.Functions.Worker; +using Microsoft.Extensions.Logging; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; + +public class LongRunningFunctions(ILogger logger) +{ + [Function(nameof(LongRunningActivity))] + public async Task LongRunningActivity( + [ActivityTrigger] LongRunningOrchestrationContext input, + CancellationToken cancellationToken) + { + var stopwatch = Stopwatch.StartNew(); + + while (stopwatch.Elapsed.Seconds < input.DurationSeconds) + { + await Task.Delay(10000, cancellationToken); + + logger.LogInformation($"Long-running orchestration running for {stopwatch.Elapsed.Seconds} " + + $"out of {input.DurationSeconds} seconds"); + } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningOrchestration.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningOrchestration.cs index 0e789ff8bf..98e3330ec3 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningOrchestration.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningOrchestration.cs @@ -1,4 +1,3 @@ -using System.Diagnostics; using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Extensions; using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; using Microsoft.Azure.Functions.Worker; @@ -7,7 +6,7 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; -public class LogRunningOrchestration(ILogger logger) +public static class LogRunningOrchestration { [Function(nameof(ProcessLongRunningOrchestration))] public static async Task ProcessLongRunningOrchestration( @@ -23,7 +22,7 @@ public static async Task ProcessLongRunningOrchestration( try { - await context.CallActivity(nameof(LongRunningActivity), logger, context.InstanceId); + await context.CallActivity(nameof(LongRunningFunctions.LongRunningActivity), logger, input); } catch (Exception e) { @@ -35,21 +34,4 @@ public static async Task ProcessLongRunningOrchestration( await context.CallActivity(ActivityNames.HandleProcessingFailure, logger, context.InstanceId); } } - - [Function(nameof(LongRunningActivity))] - public async Task LongRunningActivity( - [ActivityTrigger] Guid instanceId, - int durationSeconds, - CancellationToken cancellationToken) - { - var stopwatch = Stopwatch.StartNew(); - - while (stopwatch.Elapsed.Seconds < durationSeconds) - { - await Task.Delay(10000, cancellationToken); - - logger.LogInformation($"Long-running orchestration running for {stopwatch.Elapsed.Seconds} " + - $"out of {durationSeconds} seconds"); - } - } } From 22af14968f6397c001b4325a3dbd7eef1243e263 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Fri, 29 Nov 2024 13:30:47 +0000 Subject: [PATCH 08/12] EES-5446 - breaking out deploy tasks into template files --- .../public-api/ci/azure-pipelines.yml | 5 + .../ci/jobs/deploy-data-processor.yml | 181 ++---------------- .../public-api/ci/tasks/deploy-bicep.yml | 3 +- .../ci/tasks/wait-for-endpoint-success.yml | 63 ++++++ .../wait-for-orchestrations-to-complete.yml | 74 +++++++ .../templates/public-api/main.bicep | 8 +- .../templates/public-api/types.bicep | 1 + 7 files changed, 169 insertions(+), 166 deletions(-) create mode 100644 infrastructure/templates/public-api/ci/tasks/wait-for-endpoint-success.yml create mode 100644 infrastructure/templates/public-api/ci/tasks/wait-for-orchestrations-to-complete.yml diff --git a/infrastructure/templates/public-api/ci/azure-pipelines.yml b/infrastructure/templates/public-api/ci/azure-pipelines.yml index 0942166943..5412ea9816 100644 --- a/infrastructure/templates/public-api/ci/azure-pipelines.yml +++ b/infrastructure/templates/public-api/ci/azure-pipelines.yml @@ -16,6 +16,9 @@ parameters: - name: deployAlerts displayName: Whether to create or update Azure Monitor alerts during this deploy. default: false + - name: awaitActiveOrchestrations + displayName: Should this deploy wait for active orchestrations in Function Apps to complete prior to deploying? + default: true - name: forceDeployToEnvironment displayName: Set to either dev or test to force a deploy to that environment from the chosen branch. type: string @@ -57,6 +60,8 @@ variables: value: ${{ parameters.deployDataProcessor }} - name: deployAlerts value: ${{ parameters.deployAlerts }} + - name: awaitActiveOrchestrations + value: ${{ parameters.awaitActiveOrchestrations }} pool: vmImage: $(vmImageName) diff --git a/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml b/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml index 794677c292..5124c1da6d 100644 --- a/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml +++ b/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml @@ -64,28 +64,6 @@ jobs: --settings \ "PublicDataDb=@Microsoft.KeyVault(VaultName=$(keyVaultName); SecretName=$(dataProcessorPsqlConnectionStringSecretKey))" - # TODO EES-5128 - # Add Private Endpoint to Data Processor Function App into the VMSS VNet to allow - # DevOps to deploy the Data Processor Function App without having to temporarily - # make it publicly accessible. - # - task: AzureCLI@2 - # displayName: Temporarily enable public network access before deploy - # retryCountOnTaskFailure: 1 - # inputs: - # azureSubscription: ${{ parameters.serviceConnection }} - # scriptType: bash - # scriptLocation: inlineScript - # inlineScript: | - # set -e - - # az functionapp update \ - # --name $(dataProcessorFunctionAppName) \ - # --resource-group $(resourceGroupName) \ - # --slot staging \ - # --set \ - # publicNetworkAccess=Enabled \ - # siteConfig.publicNetworkAccess=Enabled - # TODO EES-5128 # Retry deploying the Function App in order to allow the staging slot the time to # fully restart after config and network settings have been updated prior to deploy. @@ -113,107 +91,21 @@ jobs: --resource-group $(resourceGroupName) \ --slot staging - # TODO EES-5128 - # Add Private Endpoint to Data Processor Function App into the VMSS VNet to allow - # DevOps to deploy the Data Processor Function App without having to temporarily - # make it publicly accessible. - # - task: AzureCLI@2 - # displayName: Disable public network access after deploy - # retryCountOnTaskFailure: 1 - # condition: always() - # inputs: - # azureSubscription: ${{ parameters.serviceConnection }} - # scriptType: bash - # scriptLocation: inlineScript - # inlineScript: | - # set -e - - # az functionapp update \ - # --name $(dataProcessorFunctionAppName) \ - # --resource-group $(resourceGroupName) \ - # --slot staging \ - # --set \ - # publicNetworkAccess=Disabled \ - # siteConfig.publicNetworkAccess=Disabled - - - task: AzureCLI@2 - displayName: Wait for Data Processor staging slot to start up - inputs: - azureSubscription: ${{ parameters.serviceConnection }} - scriptType: bash - scriptLocation: inlineScript - inlineScript: | - - accessToken=`az account get-access-token \ - --resource $(dataProcessorAppRegistrationClientId) \ - --query "accessToken" \ - -o tsv` - - pollingTimeSeconds=5 - maxAttempts=50 - attempt=1 - - while [ "$attempt" -le "$maxAttempts" ]; do - - echo "Attempt number $attempt of $maxAttempts- calling $(dataProcessorFunctionAppStagingUrl)/api/HealthCheck to check for Data Processor staging being healthy." - - httpStatusCode=`curl --write-out '%{http_code}' -H "Authorization: Bearer $accessToken" -s --output /dev/null $(dataProcessorFunctionAppStagingUrl)/api/HealthCheck` - - echo "Got HTTP status code $httpStatusCode." - - if [[ "$httpStatusCode" == "200" ]]; then - echo "Data Processor staging slot has started successfully and is healthy - deployment can continue." - exit 0 - fi - - attempt=$((attempt + 1)) - sleep $pollingTimeSeconds - - done - - echo "Timed out waiting for Data processor to start up." - exit 1 - - - task: AzureCLI@2 - displayName: Wait for active orchestrations to complete - inputs: - azureSubscription: ${{ parameters.serviceConnection }} - scriptType: bash - scriptLocation: inlineScript - inlineScript: | - - accessToken=`az account get-access-token \ - --resource $(dataProcessorAppRegistrationClientId) \ - --query "accessToken" \ - -o tsv` - - pollingTimeSeconds=5 - maxAttempts=50 - attempt=1 - - while [ "$attempt" -le "$maxAttempts" ]; do - - echo "Attempt number $attempt of $maxAttempts - callinf $(dataProcessorFunctionAppUrl)/api/StatusCheck to check for active orchestrations." - - activeOrchestrationResults=`curl -H "Authorization: Bearer $accessToken" -s $(dataProcessorFunctionAppUrl)/api/StatusCheck` - activeOrchestrationCount=`echo $activeOrchestrationResults | jq -r '.activeOrchestrations'` - activeOrchestrations=`echo $activeOrchestrationResults | jq -r '.activeOrchestrations != 0'` - - echo "Currently $activeOrchestrationCount active orchestrations running." - - if [[ "$activeOrchestrations" == "false" ]]; then - echo "No active orchestrations running on the Data Processor - slot swapping can proceed." - exit 0 - fi - - attempt=$((attempt + 1)) - sleep $pollingTimeSeconds - - done - - echo "Timed out waiting for active Data Processor orchestrations to complete." - exit 1 + - template: ../tasks/wait-for-endpoint-success.yml + parameters: + serviceConnection: ${{ parameters.serviceConnection }} + displayName: Waiting for staging slot to start successfully + accessTokenScope: $(dataProcessorAppRegistrationClientId) + endpoint: $(dataProcessorFunctionAppStagingUrl)/api/HealthCheck + - template: ../tasks/wait-for-orchestrations-to-complete.yml + parameters: + serviceConnection: ${{ parameters.serviceConnection }} + displayName: Waiting for active orchestrations in the production slot to complete + accessTokenScope: $(dataProcessorAppRegistrationClientId) + endpoint: $(dataProcessorFunctionAppUrl)/api/StatusCheck + condition: eq(variables.awaitActiveOrchestrations, true) + - task: AzureCLI@2 displayName: Swap slots retryCountOnTaskFailure: 1 @@ -223,48 +115,15 @@ jobs: scriptLocation: inlineScript inlineScript: | set -e - az functionapp deployment slot swap \ --name $(dataProcessorFunctionAppName) \ --resource-group $(resourceGroupName) \ --slot staging \ --target-slot production - - task: AzureCLI@2 - displayName: Check that Data Processor is ready to serve new requests - inputs: - azureSubscription: ${{ parameters.serviceConnection }} - scriptType: bash - scriptLocation: inlineScript - inlineScript: | - set -x - - accessToken=`az account get-access-token \ - --resource $(dataProcessorAppRegistrationClientId) \ - --query "accessToken" \ - -o tsv` - - pollingTimeSeconds=5 - maxAttempts=50 - attempt=1 - - while [ "$attempt" -le "$maxAttempts" ]; do - - echo "Attempt number $attempt of $maxAttempts - calling $(dataProcessorFunctionAppUrl)/api/HealthCheck to check for Data Processor being healthy." - - httpStatusCode=`curl --write-out '%{http_code}' -H "Authorization: Bearer $accessToken" -s --output /dev/null $(dataProcessorFunctionAppUrl)/api/HealthCheck` - - echo "Got HTTP status code $httpStatusCode." - - if [[ "$httpStatusCode" == "200" ]]; then - echo "Data Processor has started successfully and is healthy - ready to serve new requests." - exit 0 - fi - - attempt=$((attempt + 1)) - sleep $pollingTimeSeconds - - done - - echo "Timed out waiting for Data processor to start up." - exit 1 + - template: ../tasks/wait-for-endpoint-success.yml + parameters: + serviceConnection: ${{ parameters.serviceConnection }} + displayName: Checking that production slot is healthy after slot swap + accessTokenScope: $(dataProcessorAppRegistrationClientId) + endpoint: $(dataProcessorFunctionAppUrl)/api/HealthCheck diff --git a/infrastructure/templates/public-api/ci/tasks/deploy-bicep.yml b/infrastructure/templates/public-api/ci/tasks/deploy-bicep.yml index 1903174be7..2e9d5ca351 100644 --- a/infrastructure/templates/public-api/ci/tasks/deploy-bicep.yml +++ b/infrastructure/templates/public-api/ci/tasks/deploy-bicep.yml @@ -59,5 +59,4 @@ steps: dataProcessorFunctionAppExists=${{ parameters.dataProcessorExists }} \ dataProcessorAppRegistrationClientId='$(dataProcessorAppRegistrationClientId)' \ apiAppRegistrationClientId='$(apiAppRegistrationClientId)' \ - devopsServicePrincipalId="$servicePrincipalId" \ - pipelineRunnerCidr="$(pipelineRunnerCidr)" \ No newline at end of file + devopsServicePrincipalId="$servicePrincipalId" \ No newline at end of file diff --git a/infrastructure/templates/public-api/ci/tasks/wait-for-endpoint-success.yml b/infrastructure/templates/public-api/ci/tasks/wait-for-endpoint-success.yml new file mode 100644 index 0000000000..77e9454e7d --- /dev/null +++ b/infrastructure/templates/public-api/ci/tasks/wait-for-endpoint-success.yml @@ -0,0 +1,63 @@ +parameters: + + - name: serviceConnection + type: string + + - name: displayName + type: string + default: Waiting for a successful response from endpoint + + - name: accessTokenScope + type: string + default: null + + - name: pollingDelaySeconds + type: number + default: 5 + + - name: maxAttempts + type: number + default: 50 + + - name: endpoint + type: string + +steps: + - task: AzureCLI@2 + displayName: ${{ parameters.displayName }} + inputs: + azureSubscription: ${{ parameters.serviceConnection }} + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + + if [ -n "${{ parameters.accessTokenScope }}" ]; then + accessToken=`az account get-access-token \ + --resource ${{ parameters.accessTokenScope }} \ + --query "accessToken" \ + -o tsv` + fi + + for attempt in $(seq 1 ${{ parameters.maxAttempts }}); + do + + echo "Attempt number $attempt of ${{ parameters.maxAttempts }} - calling ${{ parameters.endpoint }} to check for successful response." + + if [ -n "$accessToken" ]; then + httpStatusCode=`curl --write-out '%{http_code}' -H "Authorization: Bearer $accessToken" -s --output /dev/null ${{ parameters.endpoint }}` + else + httpStatusCode=`curl --write-out '%{http_code}' -s --output /dev/null ${{ parameters.endpoint }}` + fi + + if (( $httpStatusCode >= 200 && $httpStatusCode <= 204 )); then + echo "Received successful response with status code $httpStatusCode." + exit 0 + fi + + echo "Received response with status code $httpStatusCode. Retrying in ${{ parameters.pollingDelaySeconds }} seconds." + sleep ${{ parameters.pollingDelaySeconds }} + + done + + echo "Timed out waiting for successful response." + exit 1 \ No newline at end of file diff --git a/infrastructure/templates/public-api/ci/tasks/wait-for-orchestrations-to-complete.yml b/infrastructure/templates/public-api/ci/tasks/wait-for-orchestrations-to-complete.yml new file mode 100644 index 0000000000..cf957664c6 --- /dev/null +++ b/infrastructure/templates/public-api/ci/tasks/wait-for-orchestrations-to-complete.yml @@ -0,0 +1,74 @@ +parameters: + + - name: serviceConnection + type: string + + - name: displayName + type: string + default: Waiting for active orchestrations to complete + + - name: condition + type: string + + - name: accessTokenScope + type: string + default: null + + - name: pollingDelaySeconds + type: number + default: 5 + + - name: maxAttempts + type: number + default: 50 + + - name: endpoint + type: string + + - name: dependsOn + type: object + default: [] + +steps: + - task: AzureCLI@2 + displayName: ${{ parameters.displayName }} + condition: ${{ parameters.condition}} + inputs: + azureSubscription: ${{ parameters.serviceConnection }} + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + + if [ -n "${{ parameters.accessTokenScope }}" ]; then + accessToken=`az account get-access-token \ + --resource ${{ parameters.accessTokenScope }} \ + --query "accessToken" \ + -o tsv` + fi + + for attempt in $(seq 1 ${{ parameters.maxAttempts }}); + do + + echo "Attempt number $attempt of ${{ parameters.maxAttempts }} - calling ${{ parameters.endpoint }} to check for active orchestrations." + + if [ -n "$accessToken" ]; then + activeOrchestrationResults=`curl -H "Authorization: Bearer $accessToken" -s ${{ parameters.endpoint }}` + else + activeOrchestrationResults=`curl -s ${{ parameters.endpoint }}` + fi + + activeOrchestrationCount=`echo $activeOrchestrationResults | jq -r '.activeOrchestrations'` + activeOrchestrations=`echo $activeOrchestrationResults | jq -r '.activeOrchestrations != 0'` + + if [[ "$activeOrchestrations" == "false" ]]; then + echo "No active orchestrations are running." + exit 0 + fi + + echo "$activeOrchestrationCount active orchestrations are still running. Retrying in ${{ parameters.pollingDelaySeconds }} seconds." + sleep ${{ parameters.pollingDelaySeconds }} + + done + + echo "Timed out waiting for active orchestrations to complete." + exit 1 \ No newline at end of file diff --git a/infrastructure/templates/public-api/main.bicep b/infrastructure/templates/public-api/main.bicep index b985dd102f..4b7f611bce 100644 --- a/infrastructure/templates/public-api/main.bicep +++ b/infrastructure/templates/public-api/main.bicep @@ -100,8 +100,9 @@ param apiAppRegistrationClientId string = '' @secure() param devopsServicePrincipalId string = '' -@description('Specifies the IP address range of the pipeline runners.') -param pipelineRunnerCidr string = '' +// TODO EES-5446 - reinstate pipelineRunnerCidr when the DevOps runners have a static IP range available. +// @description('Specifies the IP address range of the pipeline runners.') +// param pipelineRunnerCidr string = '' @description('Specifies whether or not test Themes can be deleted in the environment.') param enableThemeDeletion bool = false @@ -403,11 +404,12 @@ module dataProcessorModule 'application/public-api/publicApiDataProcessor.bicep' storageFirewallRules: maintenanceIpRanges functionAppFirewallRules: union([ adminSubnetFirewallRule - // TODO EES-5446 - add in when static IP range available for runner scale sets + // TODO EES-5446 - reinstate when static IP range available for runner scale sets // { // name: 'Pipeline runner IP address range' // cidr: pipelineRunnerCidr // } + // TODO EES-5446 - remove service tag whitelisting when runner scale set IP range reinstated { cidr: 'AzureCloud' tag: 'ServiceTag' diff --git a/infrastructure/templates/public-api/types.bicep b/infrastructure/templates/public-api/types.bicep index 6d73131a9a..d21d724fad 100644 --- a/infrastructure/templates/public-api/types.bicep +++ b/infrastructure/templates/public-api/types.bicep @@ -45,6 +45,7 @@ type IpRange = { name: string cidr: string } + @export() type FirewallRule = { name: string From 1251ecbe510154f0949ddfaa50aa70d97cee077d Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Mon, 2 Dec 2024 09:16:45 +0000 Subject: [PATCH 09/12] EES-5446 - reverting temporary pipeline changes for speeding up test deploys --- azure-pipelines-main.yml | 625 +++++++++--------- .../templates/public-api/ci/stages/deploy.yml | 12 +- 2 files changed, 318 insertions(+), 319 deletions(-) diff --git a/azure-pipelines-main.yml b/azure-pipelines-main.yml index 3ede71f80f..d5fe21618f 100644 --- a/azure-pipelines-main.yml +++ b/azure-pipelines-main.yml @@ -6,7 +6,6 @@ parameters: - master - dev - test - - EES-5446-prevent-function-app-slot-swap-until-orchestrations-complete variables: BuildConfiguration: Release @@ -60,64 +59,64 @@ jobs: # custom: format whitespace src/GovUk.Education.ExploreEducationStatistics.sln --verify-no-changes --severity error # arguments: --verify-no-changes --verbosity diagnostic -# - task: DotNetCoreCLI@2 -# displayName: Verify Formatting and Style -# inputs: -# command: custom -# custom: format -# ## TODO: Remove "--severity error" once style formatter has been run across project -# arguments: style --verify-no-changes --verbosity diagnostic --severity error -# projects: src/GovUk.Education.ExploreEducationStatistics.sln -# -# - task: DotNetCoreCLI@2 -# displayName: Verify Formatting and Style -# inputs: -# command: custom -# custom: format -# ## TODO: Remove "--severity error" once work has been done to resolve build warnings (https://dfedigital.atlassian.net/browse/EES-4594). -# arguments: analyzers --verify-no-changes --verbosity diagnostic --severity error -# projects: src/GovUk.Education.ExploreEducationStatistics.sln + - task: DotNetCoreCLI@2 + displayName: Verify Formatting and Style + inputs: + command: custom + custom: format + ## TODO: Remove "--severity error" once style formatter has been run across project + arguments: style --verify-no-changes --verbosity diagnostic --severity error + projects: src/GovUk.Education.ExploreEducationStatistics.sln + + - task: DotNetCoreCLI@2 + displayName: Verify Formatting and Style + inputs: + command: custom + custom: format + ## TODO: Remove "--severity error" once work has been done to resolve build warnings (https://dfedigital.atlassian.net/browse/EES-4594). + arguments: analyzers --verify-no-changes --verbosity diagnostic --severity error + projects: src/GovUk.Education.ExploreEducationStatistics.sln # TODO: Wrap these ^ three tasks up into a single `dotnet format` task once all 3 above TODOs are TO-DONE ;) -# - task: DotNetCoreCLI@2 -# displayName: Test -# inputs: -# command: test -# projects: | -# **/GovUk.*[Tt]ests/*.csproj -# !**/GovUk.Education.ExploreEducationStatistics.Admin.Tests/*csproj -# arguments: --configuration $(BuildConfiguration) -# -# - task: DotNetCoreCLI@2 -# displayName: Package Data API -# inputs: -# command: publish -# publishWebProjects: false -# projects: '**/GovUk.Education.ExploreEducationStatistics.Data.Api.csproj' -# arguments: --self-contained true -r win-x64 --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory)/data-api -# zipAfterPublish: True -# -# - task: PublishPipelineArtifact@0 -# displayName: Publish Data API artifact -# inputs: -# artifactName: data-api -# targetPath: $(Build.ArtifactStagingDirectory)/data-api -# -# - task: DotNetCoreCLI@2 -# displayName: Package Content API -# inputs: -# command: publish -# publishWebProjects: false -# projects: '**/GovUk.Education.ExploreEducationStatistics.Content.Api.csproj' -# arguments: --self-contained true -r win-x64 --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory)/content-api -# zipAfterPublish: True -# -# - task: PublishPipelineArtifact@0 -# displayName: Publish Content API artifact -# inputs: -# artifactName: content-api -# targetPath: $(Build.ArtifactStagingDirectory)/content-api + - task: DotNetCoreCLI@2 + displayName: Test + inputs: + command: test + projects: | + **/GovUk.*[Tt]ests/*.csproj + !**/GovUk.Education.ExploreEducationStatistics.Admin.Tests/*csproj + arguments: --configuration $(BuildConfiguration) + + - task: DotNetCoreCLI@2 + displayName: Package Data API + inputs: + command: publish + publishWebProjects: false + projects: '**/GovUk.Education.ExploreEducationStatistics.Data.Api.csproj' + arguments: --self-contained true -r win-x64 --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory)/data-api + zipAfterPublish: True + + - task: PublishPipelineArtifact@0 + displayName: Publish Data API artifact + inputs: + artifactName: data-api + targetPath: $(Build.ArtifactStagingDirectory)/data-api + + - task: DotNetCoreCLI@2 + displayName: Package Content API + inputs: + command: publish + publishWebProjects: false + projects: '**/GovUk.Education.ExploreEducationStatistics.Content.Api.csproj' + arguments: --self-contained true -r win-x64 --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory)/content-api + zipAfterPublish: True + + - task: PublishPipelineArtifact@0 + displayName: Publish Content API artifact + inputs: + artifactName: content-api + targetPath: $(Build.ArtifactStagingDirectory)/content-api - task: DotNetCoreCLI@2 displayName: Package Public API @@ -167,260 +166,260 @@ jobs: artifactName: public-api-data-processor targetPath: $(Build.ArtifactStagingDirectory)/public-api-data-processor -# - task: DotNetCoreCLI@2 -# displayName: Package Notifier Function -# inputs: -# command: publish -# publishWebProjects: false -# projects: '**/GovUk.Education.ExploreEducationStatistics.Notifier.csproj' -# arguments: --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory)/notifier -# zipAfterPublish: True -# -# - task: PublishPipelineArtifact@0 -# displayName: Publish Notifier artifact -# inputs: -# artifactName: notifier -# targetPath: $(Build.ArtifactStagingDirectory)/notifier -# -# - task: DotNetCoreCLI@2 -# displayName: Package Publisher Function -# inputs: -# command: publish -# publishWebProjects: false -# projects: '**/GovUk.Education.ExploreEducationStatistics.Publisher.csproj' -# arguments: --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory)/publisher -# zipAfterPublish: True -# -# - task: PublishPipelineArtifact@0 -# displayName: Publish Publisher artifact -# inputs: -# artifactName: publisher -# targetPath: $(Build.ArtifactStagingDirectory)/publisher -# -# - task: DotNetCoreCLI@2 -# displayName: Package Processor Function -# inputs: -# command: publish -# publishWebProjects: false -# projects: '**/GovUk.Education.ExploreEducationStatistics.Data.Processor.csproj' -# arguments: --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory)/processor -# zipAfterPublish: True -# -# - task: PublishPipelineArtifact@0 -# displayName: Publish Processor artifact -# inputs: -# artifactName: processor -# targetPath: $(Build.ArtifactStagingDirectory)/processor - -# - job: Admin -# pool: ees-ubuntu2204-xlarge -# workspace: -# clean: all -# steps: -# - task: UseNode@1 -# displayName: Install Node.js $(NodeVersion) -# inputs: -# version: $(NodeVersion) -# -# - task: Bash@3 -# displayName: corepack enable -# inputs: -# workingDir: . -# targetType: inline -# script: corepack enable -# -# - task: UseDotNet@2 -# displayName: Install .NET 8.0 SDK -# inputs: -# version: 8.0.x -# performMultiLevelLookup: true -# -# - task: DotNetCoreCLI@2 -# displayName: Build -# inputs: -# projects: '**/GovUk.Education.ExploreEducationStatistics.Admin.csproj' -# arguments: --configuration $(BuildConfiguration) -# -# - task: DotNetCoreCLI@2 -# displayName: Test -# inputs: -# command: test -# projects: '**/GovUk.Education.ExploreEducationStatistics.Admin.Tests.csproj' -# arguments: --configuration $(BuildConfiguration) --collect "Code coverage" -# -# - task: Bash@3 -# displayName: pnpm i -# inputs: -# targetType: inline -# script: pnpm i -# -# - task: Bash@3 -# displayName: pnpm run build -# inputs: -# targetType: inline -# script: pnpm --filter=explore-education-statistics-admin run build -# -# - task: CopyFiles@2 -# displayName: Copy files to wwwroot -# inputs: -# SourceFolder: src/explore-education-statistics-admin/build -# TargetFolder: src/GovUk.Education.ExploreEducationStatistics.Admin/wwwroot -# -# - task: DotNetCoreCLI@2 -# displayName: Package Admin app -# inputs: -# command: publish -# publishWebProjects: false -# projects: '**/GovUk.Education.ExploreEducationStatistics.Admin.csproj' -# arguments: --self-contained true -r win-x64 --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory) -# -# - task: PublishPipelineArtifact@0 -# displayName: Publish Admin artifact -# inputs: -# artifactName: admin -# targetPath: $(Build.ArtifactStagingDirectory) -# -# - job: Frontend -# pool: ees-ubuntu2204-xlarge -# workspace: -# clean: all -# steps: -# - task: UseNode@1 -# displayName: Install Node.js $(NodeVersion) -# inputs: -# version: $(NodeVersion) -# -# - task: Bash@3 -# displayName: corepack enable -# inputs: -# workingDir: . -# targetType: inline -# script: corepack enable -# -# - task: Bash@3 -# displayName: pnpm i -# inputs: -# workingDir: . -# targetType: inline -# script: pnpm i -# -# - task: Bash@3 -# displayName: pnpm tsc -# inputs: -# workingDir: . -# targetType: inline -# script: pnpm tsc -# -# - task: Bash@3 -# displayName: pnpm lint -# inputs: -# workingDir: . -# targetType: inline -# script: pnpm lint -# -# - task: Bash@3 -# displayName: pnpm format:check -# inputs: -# workingDir: . -# targetType: inline -# script: pnpm format:check -# -# - task: Bash@3 -# displayName: pnpm test:ci -# inputs: -# workingDir: . -# targetType: inline -# script: pnpm test:ci -# -# - task: PublishTestResults@2 -# displayName: Publish frontend test results -# inputs: -# testResultsFormat: JUnit -# testResultsFiles: explore-education-statistics-*/junit-*.xml -# searchFolder: ./src -# testRunTitle: Release Jest tests -# mergeTestResults: true -# -# - task: Bash@3 -# displayName: pnpm run build -# inputs: -# targetType: inline -# script: pnpm --filter=explore-education-statistics-frontend run build -# -# - task: Docker@2 -# displayName: Build Public frontend Docker image -# condition: and(succeeded(), eq(variables.IsBranchDeployable, true)) -# inputs: -# containerRegistry: $(AcrServiceConnection) -# repository: ees-public-frontend -# command: build -# Dockerfile: docker/public-frontend/Dockerfile -# buildContext: $(System.DefaultWorkingDirectory) -# tags: $(Build.BuildNumber) -# arguments: --build-arg BUILD_BUILDNUMBER=$(Build.BuildNumber) -# env: -# DOCKER_BUILDKIT: 1 -# -# - task: Docker@2 -# displayName: Push Public frontend Docker image -# condition: and(succeeded(), eq(variables.IsBranchDeployable, true)) -# inputs: -# containerRegistry: $(AcrServiceConnection) -# repository: ees-public-frontend -# command: push -# tags: $(Build.BuildNumber) - -# - job: ApiDocs -# pool: -# vmImage: ubuntu-22.04 -# workspace: -# clean: all -# variables: -# WorkingDirectory: src/explore-education-statistics-api-docs -# steps: -# - task: UseNode@1 -# displayName: Install Node.js $(NodeVersion) -# inputs: -# version: $(NodeVersion) -# -# - task: UseRubyVersion@0 -# displayName: Install Ruby $(RubyVersion) -# inputs: -# versionSpec: '>= $(RubyVersion)' -# -# - task: Bash@3 -# displayName: Build -# env: -# TECH_DOCS_API_URL: https://dev.statistics.api.education.gov.uk -# inputs: -# workingDirectory: $(WorkingDirectory) -# targetType: inline -# script: | -# bundle install -# bundle exec middleman build -# -# - task: PublishPipelineArtifact@1 -# displayName: Publish artifact -# inputs: -# artifactName: public-api-docs -# targetPath: $(WorkingDirectory)/build -# -# - job: MiscellaneousArtifacts -# pool: -# vmImage: ubuntu-22.04 -# workspace: -# clean: all -# steps: -# - task: CopyFiles@2 -# displayName: Copy Pipfiles to tests -# inputs: -# Contents: | -# Pipfile -# Pipfile.lock -# TargetFolder: tests -# -# - task: PublishPipelineArtifact@0 -# displayName: Publish test files -# inputs: -# artifactName: tests -# targetPath: tests + - task: DotNetCoreCLI@2 + displayName: Package Notifier Function + inputs: + command: publish + publishWebProjects: false + projects: '**/GovUk.Education.ExploreEducationStatistics.Notifier.csproj' + arguments: --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory)/notifier + zipAfterPublish: True + + - task: PublishPipelineArtifact@0 + displayName: Publish Notifier artifact + inputs: + artifactName: notifier + targetPath: $(Build.ArtifactStagingDirectory)/notifier + + - task: DotNetCoreCLI@2 + displayName: Package Publisher Function + inputs: + command: publish + publishWebProjects: false + projects: '**/GovUk.Education.ExploreEducationStatistics.Publisher.csproj' + arguments: --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory)/publisher + zipAfterPublish: True + + - task: PublishPipelineArtifact@0 + displayName: Publish Publisher artifact + inputs: + artifactName: publisher + targetPath: $(Build.ArtifactStagingDirectory)/publisher + + - task: DotNetCoreCLI@2 + displayName: Package Processor Function + inputs: + command: publish + publishWebProjects: false + projects: '**/GovUk.Education.ExploreEducationStatistics.Data.Processor.csproj' + arguments: --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory)/processor + zipAfterPublish: True + + - task: PublishPipelineArtifact@0 + displayName: Publish Processor artifact + inputs: + artifactName: processor + targetPath: $(Build.ArtifactStagingDirectory)/processor + + - job: Admin + pool: ees-ubuntu2204-xlarge + workspace: + clean: all + steps: + - task: UseNode@1 + displayName: Install Node.js $(NodeVersion) + inputs: + version: $(NodeVersion) + + - task: Bash@3 + displayName: corepack enable + inputs: + workingDir: . + targetType: inline + script: corepack enable + + - task: UseDotNet@2 + displayName: Install .NET 8.0 SDK + inputs: + version: 8.0.x + performMultiLevelLookup: true + + - task: DotNetCoreCLI@2 + displayName: Build + inputs: + projects: '**/GovUk.Education.ExploreEducationStatistics.Admin.csproj' + arguments: --configuration $(BuildConfiguration) + + - task: DotNetCoreCLI@2 + displayName: Test + inputs: + command: test + projects: '**/GovUk.Education.ExploreEducationStatistics.Admin.Tests.csproj' + arguments: --configuration $(BuildConfiguration) --collect "Code coverage" + + - task: Bash@3 + displayName: pnpm i + inputs: + targetType: inline + script: pnpm i + + - task: Bash@3 + displayName: pnpm run build + inputs: + targetType: inline + script: pnpm --filter=explore-education-statistics-admin run build + + - task: CopyFiles@2 + displayName: Copy files to wwwroot + inputs: + SourceFolder: src/explore-education-statistics-admin/build + TargetFolder: src/GovUk.Education.ExploreEducationStatistics.Admin/wwwroot + + - task: DotNetCoreCLI@2 + displayName: Package Admin app + inputs: + command: publish + publishWebProjects: false + projects: '**/GovUk.Education.ExploreEducationStatistics.Admin.csproj' + arguments: --self-contained true -r win-x64 --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory) + + - task: PublishPipelineArtifact@0 + displayName: Publish Admin artifact + inputs: + artifactName: admin + targetPath: $(Build.ArtifactStagingDirectory) + + - job: Frontend + pool: ees-ubuntu2204-xlarge + workspace: + clean: all + steps: + - task: UseNode@1 + displayName: Install Node.js $(NodeVersion) + inputs: + version: $(NodeVersion) + + - task: Bash@3 + displayName: corepack enable + inputs: + workingDir: . + targetType: inline + script: corepack enable + + - task: Bash@3 + displayName: pnpm i + inputs: + workingDir: . + targetType: inline + script: pnpm i + + - task: Bash@3 + displayName: pnpm tsc + inputs: + workingDir: . + targetType: inline + script: pnpm tsc + + - task: Bash@3 + displayName: pnpm lint + inputs: + workingDir: . + targetType: inline + script: pnpm lint + + - task: Bash@3 + displayName: pnpm format:check + inputs: + workingDir: . + targetType: inline + script: pnpm format:check + + - task: Bash@3 + displayName: pnpm test:ci + inputs: + workingDir: . + targetType: inline + script: pnpm test:ci + + - task: PublishTestResults@2 + displayName: Publish frontend test results + inputs: + testResultsFormat: JUnit + testResultsFiles: explore-education-statistics-*/junit-*.xml + searchFolder: ./src + testRunTitle: Release Jest tests + mergeTestResults: true + + - task: Bash@3 + displayName: pnpm run build + inputs: + targetType: inline + script: pnpm --filter=explore-education-statistics-frontend run build + + - task: Docker@2 + displayName: Build Public frontend Docker image + condition: and(succeeded(), eq(variables.IsBranchDeployable, true)) + inputs: + containerRegistry: $(AcrServiceConnection) + repository: ees-public-frontend + command: build + Dockerfile: docker/public-frontend/Dockerfile + buildContext: $(System.DefaultWorkingDirectory) + tags: $(Build.BuildNumber) + arguments: --build-arg BUILD_BUILDNUMBER=$(Build.BuildNumber) + env: + DOCKER_BUILDKIT: 1 + + - task: Docker@2 + displayName: Push Public frontend Docker image + condition: and(succeeded(), eq(variables.IsBranchDeployable, true)) + inputs: + containerRegistry: $(AcrServiceConnection) + repository: ees-public-frontend + command: push + tags: $(Build.BuildNumber) + + - job: ApiDocs + pool: + vmImage: ubuntu-22.04 + workspace: + clean: all + variables: + WorkingDirectory: src/explore-education-statistics-api-docs + steps: + - task: UseNode@1 + displayName: Install Node.js $(NodeVersion) + inputs: + version: $(NodeVersion) + + - task: UseRubyVersion@0 + displayName: Install Ruby $(RubyVersion) + inputs: + versionSpec: '>= $(RubyVersion)' + + - task: Bash@3 + displayName: Build + env: + TECH_DOCS_API_URL: https://dev.statistics.api.education.gov.uk + inputs: + workingDirectory: $(WorkingDirectory) + targetType: inline + script: | + bundle install + bundle exec middleman build + + - task: PublishPipelineArtifact@1 + displayName: Publish artifact + inputs: + artifactName: public-api-docs + targetPath: $(WorkingDirectory)/build + + - job: MiscellaneousArtifacts + pool: + vmImage: ubuntu-22.04 + workspace: + clean: all + steps: + - task: CopyFiles@2 + displayName: Copy Pipfiles to tests + inputs: + Contents: | + Pipfile + Pipfile.lock + TargetFolder: tests + + - task: PublishPipelineArtifact@0 + displayName: Publish test files + inputs: + artifactName: tests + targetPath: tests diff --git a/infrastructure/templates/public-api/ci/stages/deploy.yml b/infrastructure/templates/public-api/ci/stages/deploy.yml index aad36c1949..956272d6cc 100644 --- a/infrastructure/templates/public-api/ci/stages/deploy.yml +++ b/infrastructure/templates/public-api/ci/stages/deploy.yml @@ -16,7 +16,7 @@ parameters: type: string - name: dependsOn type: object - default: [] + default: [ ] - name: trigger type: string default: automatic @@ -50,11 +50,11 @@ stages: environment: ${{ parameters.environment }} bicepParamFile: ${{ parameters.bicepParamFile }} - # - template: ../jobs/deploy-api-docs.yml - # parameters: - # serviceConnection: ${{ parameters.serviceConnection }} - # environment: ${{ parameters.environment }} - # dependsOn: DeployPublicApiInfrastructure + - template: ../jobs/deploy-api-docs.yml + parameters: + serviceConnection: ${{ parameters.serviceConnection }} + environment: ${{ parameters.environment }} + dependsOn: DeployPublicApiInfrastructure - template: ../jobs/deploy-data-processor.yml parameters: From 782043fa7e734484ce68762ac6120908b5a49985 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Mon, 2 Dec 2024 11:26:40 +0000 Subject: [PATCH 10/12] EES-5446 - making data processor and PSQL deploys independent of private DNS zone deployment to save lots of time during a standard deploy --- .../templates/public-api/ci/stages/deploy.yml | 2 +- .../public-api/ci/tasks/wait-for-endpoint-success.yml | 8 +------- .../ci/tasks/wait-for-orchestrations-to-complete.yml | 10 +--------- infrastructure/templates/public-api/main.bicep | 4 ++-- .../Functions/LongRunningOrchestration.cs | 2 +- .../Functions/LongRunningTriggerFunction.cs | 2 +- 6 files changed, 7 insertions(+), 21 deletions(-) diff --git a/infrastructure/templates/public-api/ci/stages/deploy.yml b/infrastructure/templates/public-api/ci/stages/deploy.yml index 956272d6cc..62adedfe7c 100644 --- a/infrastructure/templates/public-api/ci/stages/deploy.yml +++ b/infrastructure/templates/public-api/ci/stages/deploy.yml @@ -16,7 +16,7 @@ parameters: type: string - name: dependsOn type: object - default: [ ] + default: [] - name: trigger type: string default: automatic diff --git a/infrastructure/templates/public-api/ci/tasks/wait-for-endpoint-success.yml b/infrastructure/templates/public-api/ci/tasks/wait-for-endpoint-success.yml index 77e9454e7d..9b79efecbc 100644 --- a/infrastructure/templates/public-api/ci/tasks/wait-for-endpoint-success.yml +++ b/infrastructure/templates/public-api/ci/tasks/wait-for-endpoint-success.yml @@ -1,24 +1,18 @@ parameters: - - name: serviceConnection type: string - - name: displayName type: string default: Waiting for a successful response from endpoint - - name: accessTokenScope type: string default: null - - name: pollingDelaySeconds type: number default: 5 - - name: maxAttempts type: number default: 50 - - name: endpoint type: string @@ -60,4 +54,4 @@ steps: done echo "Timed out waiting for successful response." - exit 1 \ No newline at end of file + exit 1 diff --git a/infrastructure/templates/public-api/ci/tasks/wait-for-orchestrations-to-complete.yml b/infrastructure/templates/public-api/ci/tasks/wait-for-orchestrations-to-complete.yml index cf957664c6..435dc897c9 100644 --- a/infrastructure/templates/public-api/ci/tasks/wait-for-orchestrations-to-complete.yml +++ b/infrastructure/templates/public-api/ci/tasks/wait-for-orchestrations-to-complete.yml @@ -1,30 +1,22 @@ parameters: - - name: serviceConnection type: string - - name: displayName type: string default: Waiting for active orchestrations to complete - - name: condition type: string - - name: accessTokenScope type: string default: null - - name: pollingDelaySeconds type: number default: 5 - - name: maxAttempts type: number default: 50 - - name: endpoint type: string - - name: dependsOn type: object default: [] @@ -71,4 +63,4 @@ steps: done echo "Timed out waiting for active orchestrations to complete." - exit 1 \ No newline at end of file + exit 1 diff --git a/infrastructure/templates/public-api/main.bicep b/infrastructure/templates/public-api/main.bicep index 4b7f611bce..499e5b6cfb 100644 --- a/infrastructure/templates/public-api/main.bicep +++ b/infrastructure/templates/public-api/main.bicep @@ -10,7 +10,7 @@ param location string = resourceGroup().location @description('Public API Storage : Size of the file share in GB.') param publicApiDataFileShareQuota int = 1 -@description('Firewall rules for maintenance of the service by allowing key IP ranges access to resources.') +@description('Provides access to resources for specific IP address ranges used for service maintenance.') param maintenanceIpRanges IpRange[] = [] @description('Database : administrator login name.') @@ -189,7 +189,7 @@ module coreStorage 'application/shared/coreStorage.bicep' = { } module privateDnsZonesModule 'application/shared/privateDnsZones.bicep' = - if (deploySharedPrivateDnsZones || deployPsqlFlexibleServer || deployDataProcessor) { + if (deploySharedPrivateDnsZones) { name: 'privateDnsZonesApplicationModuleDeploy' params: { resourceNames: resourceNames diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningOrchestration.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningOrchestration.cs index 98e3330ec3..a2e55a20de 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningOrchestration.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningOrchestration.cs @@ -6,7 +6,7 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; -public static class LogRunningOrchestration +public static class LongRunningOrchestration { [Function(nameof(ProcessLongRunningOrchestration))] public static async Task ProcessLongRunningOrchestration( diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningTriggerFunction.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningTriggerFunction.cs index c37e5a8c6d..b57a6108b7 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningTriggerFunction.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningTriggerFunction.cs @@ -25,7 +25,7 @@ public async Task TriggerLongRunningOrchestration( httpRequest.GetRequestParamInt(paramName: "durationSeconds", 60); const string orchestratorName = - nameof(LogRunningOrchestration.ProcessLongRunningOrchestration); + nameof(LongRunningOrchestration.ProcessLongRunningOrchestration); var options = new StartOrchestrationOptions { InstanceId = instanceId.ToString() }; From 05a108580f7a357496da5a39bb2e1f3cbb6a4264 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Tue, 10 Dec 2024 16:41:41 +0000 Subject: [PATCH 11/12] EES-5446 - responded to various PR comments. Disabling long-running orchestration trigger by default. Code improvements and variable renaming. Tweaks to Bicep templates. --- .../public-api/publicApiDataProcessor.bicep | 43 ++++++++++++++++--- .../ci/jobs/deploy-data-processor.yml | 4 +- .../public-api/components/functionApp.bicep | 11 +++-- .../templates/public-api/main.bicep | 25 +++++------ .../Extensions/HttpRequestExtensions.cs | 4 +- .../Functions/LongRunningTriggerFunction.cs | 7 +-- .../Functions/StatusCheckFunction.cs | 13 +++--- 7 files changed, 68 insertions(+), 39 deletions(-) diff --git a/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep b/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep index 5891ab45d5..3abc099f17 100644 --- a/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep +++ b/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep @@ -6,9 +6,6 @@ param resourceNames ResourceNames @description('Specifies the location for all resources.') param location string -@description('Alert metric name prefix') -param metricsNamePrefix string - @description('The Application Insights key that is associated with this resource') param applicationInsightsKey string @@ -26,7 +23,7 @@ param devopsServicePrincipalId string param storageFirewallRules IpRange[] @description('The IP address ranges that can access the Data Processor Function App endpoints.') -param functionAppFirewallRules FirewallRule[] = [] +param functionAppFirewallRules FirewallRule[] @description('Whether to create or update Azure Monitor alerts during this deploy') param deployAlerts bool @@ -107,9 +104,6 @@ module dataProcessorFunctionAppModule '../../components/functionApp.bicep' = { } preWarmedInstanceCount: 1 healthCheckPath: '/api/HealthCheck' - appSettings: { - App__MetaInsertBatchSize: 1000 - } azureFileShares: [{ storageName: resourceNames.publicApi.publicApiFileShare storageAccountKey: publicApiStorageAccount.listKeys().keys[0].value @@ -122,6 +116,41 @@ module dataProcessorFunctionAppModule '../../components/functionApp.bicep' = { } } +module functionAppHealthAlert '../../components/alerts/sites/healthAlert.bicep' = if (deployAlerts) { + name: '${resourceNames.publicApi.dataProcessor}HealthDeploy' + params: { + resourceNames: [resourceNames.publicApi.dataProcessor] + alertsGroupName: resourceNames.existingResources.alertsGroup + tagValues: tagValues + } +} + +module storageAccountAvailabilityAlerts '../../components/alerts/storageAccounts/availabilityAlert.bicep' = if (deployAlerts) { + name: '${resourceNames.publicApi.dataProcessor}StorageAvailabilityDeploy' + params: { + resourceNames: [ + dataProcessorFunctionAppModule.outputs.managementStorageAccountName + dataProcessorFunctionAppModule.outputs.slot1StorageAccountName + dataProcessorFunctionAppModule.outputs.slot2StorageAccountName + ] + alertsGroupName: resourceNames.existingResources.alertsGroup + tagValues: tagValues + } +} + +module fileServiceAvailabilityAlerts '../../components/alerts/fileServices/availabilityAlert.bicep' = if (deployAlerts) { + name: '${resourceNames.publicApi.dataProcessor}FsAvailabilityDeploy' + params: { + resourceNames: [ + dataProcessorFunctionAppModule.outputs.managementStorageAccountName + dataProcessorFunctionAppModule.outputs.slot1StorageAccountName + dataProcessorFunctionAppModule.outputs.slot2StorageAccountName + ] + alertsGroupName: resourceNames.existingResources.alertsGroup + tagValues: tagValues + } +} + output managedIdentityName string = dataProcessorFunctionAppManagedIdentity.name output managedIdentityClientId string = dataProcessorFunctionAppManagedIdentity.properties.clientId output publicApiDataFileShareMountPath string = publicApiDataFileShareMountPath diff --git a/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml b/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml index 5124c1da6d..1ec275354e 100644 --- a/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml +++ b/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml @@ -43,9 +43,11 @@ jobs: --resource-group $(resourceGroupName) \ --slot staging \ --settings \ - "App__PrivateStorageConnectionString=@Microsoft.KeyVault(VaultName=$(keyVaultName); SecretName=$(coreStorageConnectionStringSecretKey))" \ + "App__MetaInsertBatchSize=1000" \ "App__EnableThemeDeletion=$(enableThemeDeletion)" \ + "App__PrivateStorageConnectionString=@Microsoft.KeyVault(VaultName=$(keyVaultName); SecretName=$(coreStorageConnectionStringSecretKey))" \ "AZURE_CLIENT_ID=$(dataProcessorFunctionAppManagedIdentityClientId)" \ + "AzureWebJobs.TriggerLongRunningOrchestration.Disabled=true" \ "DataFiles__BasePath=$(dataProcessorPublicApiDataFileShareMountPath)" az webapp config connection-string set \ diff --git a/infrastructure/templates/public-api/components/functionApp.bicep b/infrastructure/templates/public-api/components/functionApp.bicep index f77f780882..12d55ea2dc 100644 --- a/infrastructure/templates/public-api/components/functionApp.bicep +++ b/infrastructure/templates/public-api/components/functionApp.bicep @@ -1,4 +1,4 @@ -import { FirewallRule, IpRange, AzureFileshareMount, EntraIdAuthentication } from '../types.bicep' +import { FirewallRule, IpRange, AzureFileShareMount, EntraIdAuthentication } from '../types.bicep' @description('Specifies the location for all resources.') param location string @@ -37,7 +37,7 @@ param privateEndpointSubnetId string? param publicNetworkAccessEnabled bool = false @description('IP address ranges that are allowed to access the Function App endpoints. Dependent on "publicNetworkAccessEnabled" being true.') -param functionAppEndpointFirewallRules FirewallRule[] = [] +param functionAppFirewallRules FirewallRule[] = [] @description('An existing Managed Identity\'s Resource Id with which to associate this Function App') param userAssignedManagedIdentityParams { @@ -175,7 +175,7 @@ resource slot2FileShare 'Microsoft.Storage/storageAccounts/fileServices/shares@2 ] } -var firewallRules = [for (firewallRule, index) in functionAppEndpointFirewallRules: { +var firewallRules = [for (firewallRule, index) in functionAppFirewallRules: { name: firewallRule.name ipAddress: firewallRule.cidr action: 'Allow' @@ -202,7 +202,10 @@ var commonSiteProperties = { keyVaultReferenceIdentity: keyVaultReferenceIdentity publicNetworkAccess: publicNetworkAccessEnabled ? 'Enabled' : 'Disabled' ipSecurityRestrictions: publicNetworkAccessEnabled && length(firewallRules) > 0 ? firewallRules : null - ipSecurityRestrictionsDefaultAction: publicNetworkAccessEnabled && length(firewallRules) > 0 ? 'Deny' : 'Allow' + ipSecurityRestrictionsDefaultAction: 'Deny' + // TODO EES-5446 - this setting controls access to the deploy site for the Function App. + // This is currently the default value, but ideally we would lock this down to only be accessible + // by our runners and certain other whitelisted IP address ranges (e.g. trusted VPNs). scmIpSecurityRestrictions: [ { ipAddress: 'Any' diff --git a/infrastructure/templates/public-api/main.bicep b/infrastructure/templates/public-api/main.bicep index 499e5b6cfb..4426a92c6a 100644 --- a/infrastructure/templates/public-api/main.bicep +++ b/infrastructure/templates/public-api/main.bicep @@ -385,30 +385,22 @@ module appGatewayModule 'application/shared/appGateway.bicep' = if (deployContai } } -var adminSubnetFirewallRule = { - name: 'Admin App Service subnet range' - cidr: vNetModule.outputs.adminAppServiceSubnetCidr - tag: 'Default' - priority: 100 -} - module dataProcessorModule 'application/public-api/publicApiDataProcessor.bicep' = if (deployDataProcessor) { name: 'publicApiDataProcessorApplicationModuleDeploy' params: { location: location resourceNames: resourceNames - metricsNamePrefix: '${subscription}PublicDataProcessor' applicationInsightsKey: appInsightsModule.outputs.appInsightsKey dataProcessorAppRegistrationClientId: dataProcessorAppRegistrationClientId devopsServicePrincipalId: devopsServicePrincipalId storageFirewallRules: maintenanceIpRanges functionAppFirewallRules: union([ - adminSubnetFirewallRule - // TODO EES-5446 - reinstate when static IP range available for runner scale sets - // { - // name: 'Pipeline runner IP address range' - // cidr: pipelineRunnerCidr - // } + { + name: 'Admin App Service subnet range' + cidr: vNetModule.outputs.adminAppServiceSubnetCidr + tag: 'Default' + priority: 100 + } // TODO EES-5446 - remove service tag whitelisting when runner scale set IP range reinstated { cidr: 'AzureCloud' @@ -416,6 +408,11 @@ module dataProcessorModule 'application/public-api/publicApiDataProcessor.bicep' priority: 101 name: 'AzureCloud' } + // TODO EES-5446 - reinstate when static IP range available for runner scale sets + // { + // name: 'Pipeline runner IP address range' + // cidr: pipelineRunnerCidr + // } ], maintenanceFirewallRules) dataProcessorFunctionAppExists: dataProcessorFunctionAppExists deployAlerts: deployAlerts diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/HttpRequestExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/HttpRequestExtensions.cs index 3d95e32591..9e60dc9f13 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/HttpRequestExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/HttpRequestExtensions.cs @@ -111,7 +111,7 @@ public static bool GetRequestParamBool( string paramName, bool defaultValue) { - var paramValue = GetRequestParam(httpRequest, paramName, defaultValue + ""); + var paramValue = GetRequestParam(httpRequest, paramName, defaultValue.ToString()); return bool.Parse(paramValue); } @@ -120,7 +120,7 @@ public static int GetRequestParamInt( string paramName, int defaultValue) { - var paramValue = GetRequestParam(httpRequest, paramName, defaultValue + ""); + var paramValue = GetRequestParam(httpRequest, paramName, defaultValue.ToString()); return int.Parse(paramValue); } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningTriggerFunction.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningTriggerFunction.cs index b57a6108b7..5012da598e 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningTriggerFunction.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningTriggerFunction.cs @@ -22,7 +22,7 @@ public async Task TriggerLongRunningOrchestration( var instanceId = Guid.NewGuid(); var durationSeconds = - httpRequest.GetRequestParamInt(paramName: "durationSeconds", 60); + httpRequest.GetRequestParamInt("durationSeconds", defaultValue: 60); const string orchestratorName = nameof(LongRunningOrchestration.ProcessLongRunningOrchestration); @@ -37,10 +37,7 @@ public async Task TriggerLongRunningOrchestration( await client.ScheduleNewOrchestrationInstanceAsync( orchestratorName, - new LongRunningOrchestrationContext - { - DurationSeconds = durationSeconds - }, + new LongRunningOrchestrationContext { DurationSeconds = durationSeconds }, options, cancellationToken); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/StatusCheckFunction.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/StatusCheckFunction.cs index 877e663232..7a59add443 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/StatusCheckFunction.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/StatusCheckFunction.cs @@ -9,14 +9,14 @@ public class StatusCheckFunction { private static readonly OrchestrationQuery ActiveOrchestrationsQuery = new() { - Statuses = new List - { + Statuses = + [ OrchestrationRuntimeStatus.Pending, OrchestrationRuntimeStatus.Running - } + ] }; - [Function("StatusCheck")] + [Function(nameof(StatusCheck))] [Produces("application/json")] public static async Task StatusCheck( [HttpTrigger(AuthorizationLevel.Anonymous, "get")] @@ -27,7 +27,8 @@ public static async Task StatusCheck( { var activeOrchestrations = await client .GetAllInstancesAsync(filter: ActiveOrchestrationsQuery) - .ToListAsync(); - return new OkObjectResult(new { ActiveOrchestrations = activeOrchestrations.Count }); + .CountAsync(); + + return new OkObjectResult(new { ActiveOrchestrations = activeOrchestrations }); } } From a6a783207e0b8ab25a65504bfda8452b97952b8d Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Tue, 10 Dec 2024 16:52:44 +0000 Subject: [PATCH 12/12] EES-5446 - grouped long-running test functions into single class --- .../Functions/LongRunningFunctions.cs | 65 +++++++++++++++++++ .../Functions/LongRunningOrchestration.cs | 37 ----------- .../Functions/LongRunningTriggerFunction.cs | 46 ------------- 3 files changed, 65 insertions(+), 83 deletions(-) delete mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningOrchestration.cs delete mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningTriggerFunction.cs diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningFunctions.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningFunctions.cs index f17bc7a812..4835da3a1c 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningFunctions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningFunctions.cs @@ -1,12 +1,77 @@ using System.Diagnostics; +using GovUk.Education.ExploreEducationStatistics.Common.Extensions; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Extensions; using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; using Microsoft.Azure.Functions.Worker; +using Microsoft.DurableTask; +using Microsoft.DurableTask.Client; using Microsoft.Extensions.Logging; namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; public class LongRunningFunctions(ILogger logger) { + [Function(nameof(TriggerLongRunningOrchestration))] + public async Task TriggerLongRunningOrchestration( + [HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = nameof(TriggerLongRunningOrchestration))] + HttpRequest httpRequest, + [DurableClient] DurableTaskClient client, + CancellationToken cancellationToken) + { + var instanceId = Guid.NewGuid(); + + var durationSeconds = + httpRequest.GetRequestParamInt("durationSeconds", defaultValue: 60); + + const string orchestratorName = + nameof(ProcessLongRunningOrchestration); + + var options = new StartOrchestrationOptions { InstanceId = instanceId.ToString() }; + + logger.LogInformation( + "Scheduling '{OrchestratorName}' (InstanceId={InstanceId}, DurationSeconds={DurationSeconds}))", + orchestratorName, + instanceId, + durationSeconds); + + await client.ScheduleNewOrchestrationInstanceAsync( + orchestratorName, + new LongRunningOrchestrationContext { DurationSeconds = durationSeconds }, + options, + cancellationToken); + + return new OkResult(); + } + + [Function(nameof(ProcessLongRunningOrchestration))] + public static async Task ProcessLongRunningOrchestration( + [OrchestrationTrigger] TaskOrchestrationContext context, + LongRunningOrchestrationContext input) + { + var logger = context.CreateReplaySafeLogger(nameof(ProcessLongRunningOrchestration)); + + logger.LogInformation( + "Processing long-running orchestration (InstanceId={InstanceId}, DurationSeconds={DurationSeconds})", + context.InstanceId, + input.DurationSeconds); + + try + { + await context.CallActivity(nameof(LongRunningActivity), logger, input); + } + catch (Exception e) + { + logger.LogError(e, + "Activity failed with an exception (InstanceId={InstanceId}, DurationSeconds={DurationSeconds})", + context.InstanceId, + input.DurationSeconds); + + await context.CallActivity(ActivityNames.HandleProcessingFailure, logger, context.InstanceId); + } + } + [Function(nameof(LongRunningActivity))] public async Task LongRunningActivity( [ActivityTrigger] LongRunningOrchestrationContext input, diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningOrchestration.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningOrchestration.cs deleted file mode 100644 index a2e55a20de..0000000000 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningOrchestration.cs +++ /dev/null @@ -1,37 +0,0 @@ -using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Extensions; -using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; -using Microsoft.Azure.Functions.Worker; -using Microsoft.DurableTask; -using Microsoft.Extensions.Logging; - -namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; - -public static class LongRunningOrchestration -{ - [Function(nameof(ProcessLongRunningOrchestration))] - public static async Task ProcessLongRunningOrchestration( - [OrchestrationTrigger] TaskOrchestrationContext context, - LongRunningOrchestrationContext input) - { - var logger = context.CreateReplaySafeLogger(nameof(ProcessLongRunningOrchestration)); - - logger.LogInformation( - "Processing long-running orchestration (InstanceId={InstanceId}, DurationSeconds={DurationSeconds})", - context.InstanceId, - input.DurationSeconds); - - try - { - await context.CallActivity(nameof(LongRunningFunctions.LongRunningActivity), logger, input); - } - catch (Exception e) - { - logger.LogError(e, - "Activity failed with an exception (InstanceId={InstanceId}, DurationSeconds={DurationSeconds})", - context.InstanceId, - input.DurationSeconds); - - await context.CallActivity(ActivityNames.HandleProcessingFailure, logger, context.InstanceId); - } - } -} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningTriggerFunction.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningTriggerFunction.cs deleted file mode 100644 index 5012da598e..0000000000 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningTriggerFunction.cs +++ /dev/null @@ -1,46 +0,0 @@ -using GovUk.Education.ExploreEducationStatistics.Common.Extensions; -using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; -using Microsoft.AspNetCore.Http; -using Microsoft.AspNetCore.Mvc; -using Microsoft.Azure.Functions.Worker; -using Microsoft.DurableTask; -using Microsoft.DurableTask.Client; -using Microsoft.Extensions.Logging; - -namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; - -public class LongRunningTriggerFunction( - ILogger logger) -{ - [Function(nameof(TriggerLongRunningOrchestration))] - public async Task TriggerLongRunningOrchestration( - [HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = nameof(TriggerLongRunningOrchestration))] - HttpRequest httpRequest, - [DurableClient] DurableTaskClient client, - CancellationToken cancellationToken) - { - var instanceId = Guid.NewGuid(); - - var durationSeconds = - httpRequest.GetRequestParamInt("durationSeconds", defaultValue: 60); - - const string orchestratorName = - nameof(LongRunningOrchestration.ProcessLongRunningOrchestration); - - var options = new StartOrchestrationOptions { InstanceId = instanceId.ToString() }; - - logger.LogInformation( - "Scheduling '{OrchestratorName}' (InstanceId={InstanceId}, DurationSeconds={DurationSeconds}))", - orchestratorName, - instanceId, - durationSeconds); - - await client.ScheduleNewOrchestrationInstanceAsync( - orchestratorName, - new LongRunningOrchestrationContext { DurationSeconds = durationSeconds }, - options, - cancellationToken); - - return new OkResult(); - } -}