Merge pull request #5521 from dfe-analytical-services/dev

Merge Dev into Master

.config/dotnet-tools.json

@@ -6,7 +6,15 @@
       "version": "8.0.7",
       "commands": [
         "dotnet-ef"
-      ]
+      ],
+      "rollForward": false
+    },
+    "swashbuckle.aspnetcore.cli": {
+      "version": "7.1.0",
+      "commands": [
+        "swagger"
+      ],
+      "rollForward": false
     }
   }
 }

azure-pipelines-main.yml

@@ -11,9 +11,12 @@ variables:
   BuildConfiguration: Release
   IsBranchDeployable: ${{ containsValue(parameters.DeployBranches, variables['Build.SourceBranchName']) }}
   CI: true
+  DotNetVersion: '8.0.x'
   NodeVersion: '20.16.0'
   RubyVersion: '3.2'
   AcrServiceConnection: s101d-datahub-spn-ees-dfe-gov-uk-docker-managed-service-connection
+  PublicApiDir: GovUk.Education.ExploreEducationStatistics.Public.Data.Api
+  PublicApiDocsDir: explore-education-statistics-api-docs
 
 trigger:
   branches:
@@ -31,36 +34,28 @@ pr:
   - test
 
 jobs:
-  - job: Backend
-    pool: ees-ubuntu2204-xlarge
+  - job: BackendVerify
+    pool: ees-ubuntu2204-large
     workspace:
       clean: all
     steps:
       - task: UseDotNet@2
-        displayName: Install .NET 8.0 SDK
+        displayName: Install .NET $(DotNetVersion)
         inputs:
-          version: 8.0.x
+          version: $(DotNetVersion)
           performMultiLevelLookup: true
 
-      - task: DotNetCoreCLI@2
-        displayName: Build
-        inputs:
-          projects: |
-            **/GovUk.*/*csproj
-            !**/GovUk.Education.ExploreEducationStatistics.Admin/*csproj
-          arguments: --configuration $(BuildConfiguration)
-
       # TODO: Uncomment this step once formatter has been run globally.
       # This is because whitespace rules cannot be warnings or suggestions; they will always be errors
       # - task: DotNetCoreCLI@2
-      #   displayName:'Verify Formatting and Style
+      #   displayName: Verify formatting
       #   inputs:
       #     command: custom
       #     custom: format whitespace src/GovUk.Education.ExploreEducationStatistics.sln --verify-no-changes --severity error
       #     arguments: --verify-no-changes --verbosity diagnostic
 
       - task: DotNetCoreCLI@2
-        displayName: Verify Formatting and Style
+        displayName: Verify style
         inputs:
           command: custom
           custom: format
@@ -69,15 +64,45 @@ jobs:
           projects: src/GovUk.Education.ExploreEducationStatistics.sln
 
       - task: DotNetCoreCLI@2
-        displayName: Verify Formatting and Style
+        displayName: Verify analyzers
         inputs:
           command: custom
           custom: format
           ## TODO: Remove "--severity error" once work has been done to resolve build warnings (https://dfedigital.atlassian.net/browse/EES-4594).
           arguments: analyzers --verify-no-changes --verbosity diagnostic --severity error
           projects: src/GovUk.Education.ExploreEducationStatistics.sln
 
-      # TODO: Wrap these ^ three tasks up into a single `dotnet format` task once all 3 above TODOs are TO-DONE ;)
+      # TODO: Wrap the above three tasks up into a single `dotnet format` task once all TODOs are done
+
+      - task: DotNetCoreCLI@2
+        displayName: Build Public API
+        inputs:
+          projects: '**/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.csproj'
+          arguments: --configuration $(BuildConfiguration)
+
+      - task: Bash@3
+        displayName: Diff Public API OpenAPI docs
+        inputs:
+          targetType: inline
+          script: |
+            docker run --rm -t \
+              -v $(System.DefaultWorkingDirectory)/src/$(PublicApiDir)/bin/$(BuildConfiguration)/net8.0:/api \
+              -v $(System.DefaultWorkingDirectory)/src/$(PublicApiDocsDir):/api-docs \
+              -v $(System.DefaultWorkingDirectory)/ci/scripts:/scripts \
+              --entrypoint /bin/sh \
+              tufin/oasdiff \
+              /scripts/public-api-openapi-diff.sh
+
+  - job: BackendTest
+    pool: ees-ubuntu2204-xlarge
+    workspace:
+      clean: all
+    steps:
+      - task: UseDotNet@2
+        displayName: Install .NET $(DotNetVersion)
+        inputs:
+          version: $(DotNetVersion)
+          performMultiLevelLookup: true
 
       - task: DotNetCoreCLI@2
         displayName: Test
@@ -88,6 +113,16 @@ jobs:
             !**/GovUk.Education.ExploreEducationStatistics.Admin.Tests/*csproj
           arguments: --configuration $(BuildConfiguration)
 
+  - job: BackendPublish
+    pool:
+      vmImage: ubuntu-22.04
+    steps:
+      - task: UseDotNet@2
+        displayName: Install .NET $(DotNetVersion)
+        inputs:
+          version: $(DotNetVersion)
+          performMultiLevelLookup: true
+
       - task: DotNetCoreCLI@2
         displayName: Package Data API
         inputs:
@@ -97,8 +132,9 @@ jobs:
           arguments: --self-contained true -r win-x64 --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory)/data-api
           zipAfterPublish: True
 
-      - task: PublishPipelineArtifact@0
+      - task: PublishPipelineArtifact@1
         displayName: Publish Data API artifact
+        condition: and(succeeded(), eq(variables.IsBranchDeployable, true))
         inputs:
           artifactName: data-api
           targetPath: $(Build.ArtifactStagingDirectory)/data-api
@@ -112,8 +148,9 @@ jobs:
           arguments: --self-contained true -r win-x64 --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory)/content-api
           zipAfterPublish: True
 
-      - task: PublishPipelineArtifact@0
+      - task: PublishPipelineArtifact@1
         displayName: Publish Content API artifact
+        condition: and(succeeded(), eq(variables.IsBranchDeployable, true))
         inputs:
           artifactName: content-api
           targetPath: $(Build.ArtifactStagingDirectory)/content-api
@@ -126,7 +163,6 @@ jobs:
           projects: '**/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.csproj'
           arguments: -r linux-musl-x64 --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory)/public-api
           zipAfterPublish: False
-
       - task: Docker@2
         displayName: Build Public API Docker image
         condition: and(succeeded(), eq(variables.IsBranchDeployable, true))
@@ -160,8 +196,9 @@ jobs:
           arguments: --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory)/public-api-data-processor
           zipAfterPublish: True
 
-      - task: PublishPipelineArtifact@0
+      - task: PublishPipelineArtifact@1
         displayName: Publish Public API Data Processor artifact
+        condition: and(succeeded(), eq(variables.IsBranchDeployable, true))
         inputs:
           artifactName: public-api-data-processor
           targetPath: $(Build.ArtifactStagingDirectory)/public-api-data-processor
@@ -175,8 +212,9 @@ jobs:
           arguments: --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory)/notifier
           zipAfterPublish: True
 
-      - task: PublishPipelineArtifact@0
+      - task: PublishPipelineArtifact@1
         displayName: Publish Notifier artifact
+        condition: and(succeeded(), eq(variables.IsBranchDeployable, true))
         inputs:
           artifactName: notifier
           targetPath: $(Build.ArtifactStagingDirectory)/notifier
@@ -190,8 +228,9 @@ jobs:
           arguments: --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory)/publisher
           zipAfterPublish: True
 
-      - task: PublishPipelineArtifact@0
+      - task: PublishPipelineArtifact@1
         displayName: Publish Publisher artifact
+        condition: and(succeeded(), eq(variables.IsBranchDeployable, true))
         inputs:
           artifactName: publisher
           targetPath: $(Build.ArtifactStagingDirectory)/publisher
@@ -205,8 +244,9 @@ jobs:
           arguments: --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory)/processor
           zipAfterPublish: True
 
-      - task: PublishPipelineArtifact@0
+      - task: PublishPipelineArtifact@1
         displayName: Publish Processor artifact
+        condition: and(succeeded(), eq(variables.IsBranchDeployable, true))
         inputs:
           artifactName: processor
           targetPath: $(Build.ArtifactStagingDirectory)/processor
@@ -229,9 +269,9 @@ jobs:
           script: corepack enable
 
       - task: UseDotNet@2
-        displayName: Install .NET 8.0 SDK
+        displayName: Install .NET $(DotNetVersion)
         inputs:
-          version: 8.0.x
+          version: $(DotNetVersion)
           performMultiLevelLookup: true
 
       - task: DotNetCoreCLI@2
@@ -267,13 +307,15 @@ jobs:
 
       - task: DotNetCoreCLI@2
         displayName: Package Admin app
+        condition: and(succeeded(), eq(variables.IsBranchDeployable, true))
         inputs:
           command: publish
           publishWebProjects: false
           projects: '**/GovUk.Education.ExploreEducationStatistics.Admin.csproj'
           arguments: --self-contained true -r win-x64 --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory)
 
-      - task: PublishPipelineArtifact@0
+      - task: PublishPipelineArtifact@1
+        condition: and(succeeded(), eq(variables.IsBranchDeployable, true))
         displayName: Publish Admin artifact
         inputs:
           artifactName: admin
@@ -387,22 +429,44 @@ jobs:
         inputs:
           versionSpec: '>= $(RubyVersion)'
 
+      - task: Cache@2
+        displayName: Cache Gems
+        inputs:
+          key: 'gems | "$(Agent.OS)" | $(WorkingDirectory)/Gemfile.lock'
+          path: $(WorkingDirectory)/vendor
+          restoreKeys: |
+            gems | "$(Agent.OS)"
+            gems
+
+      - task: Bash@3
+        displayName: Set deployment config
+        inputs:
+          workingDirectory: $(WorkingDirectory)
+          targetType: inline
+          script: bundle config set deployment true
+
+      - task: Bash@3
+        displayName: Install Gems
+        inputs:
+          workingDirectory: $(WorkingDirectory)
+          targetType: inline
+          script: bundle install
+
       - task: Bash@3
         displayName: Build
-        env:
-          TECH_DOCS_API_URL: https://dev.statistics.api.education.gov.uk
         inputs:
           workingDirectory: $(WorkingDirectory)
           targetType: inline
-          script: |
-            bundle install
+          script: | 
             bundle exec middleman build
+            # Remove afterwards as we don't want it in the artifact
+            rm -rf build
 
       - task: PublishPipelineArtifact@1
-        displayName: Publish artifact
+        displayName: Publish Public API docs artifact
         inputs:
           artifactName: public-api-docs
-          targetPath: $(WorkingDirectory)/build
+          targetPath: $(System.DefaultWorkingDirectory)/src/$(PublicApiDocsDir)
 
   - job: MiscellaneousArtifacts
     pool:

ci/scripts/public-api-openapi-diff.sh

@@ -0,0 +1,60 @@
+#/usr/bin/env sh
+
+set -u
+
+# This script should be run from inside the oasdiff Docker container.
+# It checks for any undocumented changes in the public API.
+
+has_undocumented_changes=false
+
+for file in /api-docs/source/openapi-v*.json; do
+  filename=$(basename $file)
+
+  echo "Checking $filename"
+  oasdiff changelog --flatten-allof -o INFO $file /api/wwwroot/$filename
+
+  if [ $? != 0 ]; then
+    has_undocumented_changes=true
+
+    cat <<- EOF
+ERROR: There were changes that must be documented in the public API docs.
+
+To document these changes, you must update 'src/explore-education-statistics-api-docs/source/$filename'
+by copying the changes from the OpenAPI endpoint when running the public API locally.
+
+The OpenAPI endpoint locally is: http://localhost:5050/$filename
+
+You should ensure that these changes are expected. If there are error level changes reported above, this
+means there are breaking changes that are being introduced. If these changes are intentional, you should:
+
+  1. Create a new major API version in the public API project.
+  2. Create a new 'openapi-v*.json' for the major version in the public API docs.
+  3. Document the major version and its breaking changes in the public API docs (e.g. in a changelog).
+  4. Review any existing API documentation and update accordingly.
+  5. Deprecate the previous API version when appropriate.
+
+Consult the following DevOps wiki page for more information:
+https://dfe-gov-uk.visualstudio.com.mcas.ms/s101-Explore-Education-Statistics/_wiki/wikis/s101-Explore-Education-Statistics.wiki/9070
+EOF
+  fi
+done || true
+
+if $has_undocumented_changes; then
+  exit 1
+fi
+
+has_undocumented_version=false
+
+# Check there isn't an undocumented API version
+for file in /api/wwwroot/openapi-v*.json; do
+  filename=$(basename $file)
+
+  if [ ! -f "/api-docs/source/$filename" ]; then
+    has_undocumented_version=true
+    echo "ERROR: $filename is required in the public API docs"
+  fi
+done
+
+if $has_undocumented_version; then
+  exit 1
+fi

infrastructure/templates/public-api/abbreviations.bicep

@@ -5,11 +5,14 @@ var abbreviations = {
   appManagedEnvironments: 'cae'
   // TODO - remove the "-flexibleserver" suffix and change the suffix of our PSQL instance to "-01"
   dBforPostgreSQLServers: 'psql-flexibleserver'
+  // 'fs' is non-standard - it should be 'share'
+  fileShare: 'fs'
   // 'ai' is non-standard - it should be 'appi'
   insightsComponents: 'ai'
   managedIdentityUserAssignedIdentities: 'id'
   networkApplicationGateways: 'agw'
   operationalInsightsWorkspaces: 'log'
+  recoveryServicesVault: 'rsv'
   staticWebApps: 'stapp'
   // 'sa' is non-standard - it should be 'st'
   storageStorageAccounts: 'sa'