From 7f4a4818f5e4e5fad0b3279aa976db1c3dd1da53 Mon Sep 17 00:00:00 2001 From: Jagadish Murugan Date: Wed, 2 Oct 2024 17:12:09 -0700 Subject: [PATCH] changes for M2 integration (#201) * Users/jagamu/enable workload identity for aio (#1) * enable workloadidentity for AIO * check for apiserver readiness after apiserver restart * Users/jagamu/wi changes for k8s (#2) * ensure connected status * enable WI for k8s for AIO * use same timeout as Connect-AksEdgeArc * align tags with AKSEE PS (#3) * fix serviceaccountissuer for K8s (#4) * handle az cli cmd errors gracefully and cleanup logging (#5) * handle az cli cmd errors gracefully and cleanup logging * use Arc PublicPreview bits * add CustomLocationsOid parameter * script cleanup, integrate gateway feature (#6) * script cleanup, integrate gateway feature * integrate arc gateway * use connected.whl version supporting WLIF & Gateway together (#7) * initial changes for easier merge with main * use latest public preview version of connectedk8s * remove canary region - eastus2euap * use jagadishmurugan till the time the changes are merged to main * use latest public preview connectedk8s drop * proxy support * remove default no_proxy settings to avoid ACSA extension failure * replace fork to Azure --- tools/modules/AksEdgeDeploy/README.md | 6 +- .../modules/AksEdgeDeploy/aide-ucschema.json | 11 +- .../AksEdgeDeploy/aide-userconfig.puml | 7 +- .../AksEdgeQuickStartForAio.ps1 | 235 ++++++++++++++---- 4 files changed, 205 insertions(+), 54 deletions(-) diff --git a/tools/modules/AksEdgeDeploy/README.md b/tools/modules/AksEdgeDeploy/README.md index 8ced7c2..1a2896c 100644 --- a/tools/modules/AksEdgeDeploy/README.md +++ b/tools/modules/AksEdgeDeploy/README.md @@ -16,7 +16,7 @@ The `Start-AideWorkflow` function in the modole does the following: ```json { - "SchemaVersion": "1.1", + "SchemaVersion": "1.2", "Version": "1.0", "AksEdgeProduct" : "AKS Edge Essentials - K3s", "AksEdgeConfig": { @@ -50,7 +50,8 @@ The `Start-AideWorkflow` function in the modole does the following: "TenantId":"", "ResourceGroupName": "aksedge-rg", "ServicePrincipalName" : "aksedge-sp", - "Location" : "EastUS" + "Location" : "EastUS", + "EnableWorkloadIdentity": true } } ``` @@ -81,6 +82,7 @@ Find below the details of the supported parameters in the json file. || CustomLocationOID | Optional | GUID | ObjectID for the custom location resource provider | || `Auth`.ServicePrincipalId |Mandatory | GUID | Specify service principal appID to use| || `Auth`.Password |Mandatory| String | Specify the password (in clear) | +|| EnableWorkloadIdentity |Optional| Boolean | Enable secure workload access to azure resource | |InstallOptions| InstallPath | Optional | String | Path to install the product | || VhdxPath | Optional | String | Path to store the vhdx files | |VSwitch| Name | Optional | String | Name for the external switch, mandatory for ScalableCluster| diff --git a/tools/modules/AksEdgeDeploy/aide-ucschema.json b/tools/modules/AksEdgeDeploy/aide-ucschema.json index 84a97fa..2eb5eae 100644 --- a/tools/modules/AksEdgeDeploy/aide-ucschema.json +++ b/tools/modules/AksEdgeDeploy/aide-ucschema.json @@ -6,7 +6,7 @@ "properties": { "SchemaVersion": { "type": "string", - "default": "1.1", + "default": "1.2", "description": "Version of the schema/format of the json" }, "Version": { @@ -72,6 +72,15 @@ "description": "Password corresponding to ServicePrincipalId to onboard Arc-enabled Server/Kubernetes" } } + }, + "EnableWorkloadIdentity": { + "type": "boolean" + }, + "EnableKeyManagement": { + "type": "boolean" + }, + "GatewayResourceId": { + "type": "string" } } }, diff --git a/tools/modules/AksEdgeDeploy/aide-userconfig.puml b/tools/modules/AksEdgeDeploy/aide-userconfig.puml index 35ccc6e..3b3f874 100644 --- a/tools/modules/AksEdgeDeploy/aide-userconfig.puml +++ b/tools/modules/AksEdgeDeploy/aide-userconfig.puml @@ -21,7 +21,7 @@ jsonDiagram { #highlight "SchemaVersion" { - "SchemaVersion":"1.1", + "SchemaVersion":"1.2", "Version":"1.0", "AksEdgeProduct" : [ "AKS Edge Essentials - K8s", @@ -40,7 +40,10 @@ jsonDiagram { "Auth": { "ServicePrincipalId" : "GUID", "Password" : "String" - } + }, + "EnableWorkloadIdentity" : "Boolean", + "EnableKeyManagement" : "Boolean", + "GatewayResourceId": "String" } , "InstallOptions":{ diff --git a/tools/scripts/AksEdgeQuickStart/AksEdgeQuickStartForAio.ps1 b/tools/scripts/AksEdgeQuickStart/AksEdgeQuickStartForAio.ps1 index 1113d6e..03e7aad 100644 --- a/tools/scripts/AksEdgeQuickStart/AksEdgeQuickStartForAio.ps1 +++ b/tools/scripts/AksEdgeQuickStart/AksEdgeQuickStartForAio.ps1 @@ -13,45 +13,110 @@ param( [ValidateNotNullOrEmpty()] [String] $ClusterName, [String] $CustomLocationOid, - # k8s is not fully validated. Please refer AIO documentation for validated platforms [Switch] $UseK8s=$false, [string] $Tag ) #Requires -RunAsAdministrator +New-Variable -Name gAksEdgeQuickStartForAioVersion -Value "1.0.240904.1500" -Option Constant -ErrorAction SilentlyContinue + +# Specify only AIO supported regions +New-Variable -Option Constant -ErrorAction SilentlyContinue -Name arcLocations -Value @( + "eastus", "eastus2", "northeurope", "westeurope", "westus", "westus2", "westus3" +) + +function Wait-ApiServerReady +{ + $retries = 120 + for (; $retries -gt 0; $retries--) + { + $ret = & kubectl get --raw='/readyz' + if ($ret -eq "ok") + { + Write-Host "ApiServerReady!" + break + } + + Write-Host "WaitForApiServer - Retry..." + Start-Sleep -Seconds 1 + } + + if ($retries -eq 0) + { + exit -1 + } +} + +function Restart-ApiServer +{ +param( + [Parameter(Mandatory=$true)] + [string] $serviceAccountIssuer, + [Switch] $useK8s=$false +) + + Write-Host "serviceAccountIssuer = $serviceAccountIssuer" + + if ($useK8s) + { + Invoke-AksEdgeNodeCommand -command "sudo cat /etc/kubernetes/manifests/kube-apiserver.yaml | tee /home/aksedge-user/kube-apiserver.yaml | tee /home/aksedge-user/kube-apiserver.yaml.working > /dev/null" + Invoke-AksEdgeNodeCommand -command "sudo sed -i 's|service-account-issuer.*|service-account-issuer=$serviceAccountIssuer|' /home/aksedge-user/kube-apiserver.yaml" + Invoke-AksEdgeNodeCommand -command "sudo cp /home/aksedge-user/kube-apiserver.yaml /etc/kubernetes/manifests/kube-apiserver.yaml" + & kubectl delete pod -n kube-system -l component=kube-apiserver + } + else + { + Invoke-AksEdgeNodeCommand -command "sudo cat /var/.eflow/config/k3s/k3s-config.yml | tee /home/aksedge-user/k3s-config.yml | tee /home/aksedge-user/k3s-config.yml.working > /dev/null" + Invoke-AksEdgeNodeCommand -command "sudo sed -i 's|service-account-issuer.*|service-account-issuer=$serviceAccountIssuer|' /home/aksedge-user/k3s-config.yml" + Invoke-AksEdgeNodeCommand -command "sudo cp /home/aksedge-user/k3s-config.yml /var/.eflow/config/k3s/k3s-config.yml" + Invoke-AksEdgeNodeCommand -command "sudo systemctl restart k3s.service" + } + + Wait-ApiServerReady +} function Verify-ConnectedStatus { param( [Parameter(Mandatory=$true)] - [object] $arcArgs, + [string] $resourceGroup, [Parameter(Mandatory=$true)] - [string] $clusterName + [string] $clusterName, + [Parameter(Mandatory=$true)] + [string] $subscriptionId, + [Switch] $enableWorkloadIdentity=$false ) - $k8sConnectArgs = @("-g", $arcArgs.ResourceGroupName) - $k8sConnectArgs += @("-n", $clusterName) - $k8sConnectArgs += @("--subscription", $arcArgs.SubscriptionId) - - # 15 min timeout to check for Connected status - as recommended by Arc team $retries = 90 - $sleepDurationInSeconds = 10 for (; $retries -gt 0; $retries--) { - $connectedCluster = az connectedk8s show $k8sConnectArgs | ConvertFrom-Json + $connectedCluster = az connectedk8s show -g $resourceGroup -n $clusterName --subscription $subscriptionId | ConvertFrom-Json + + if ($enableWorkloadIdentity) + { + $agentState = $connectedCluster.arcAgentProfile.agentState + Write-Host "$retries, AgentState = $agentState" + } + + $connectivityStatus = $connectedCluster.ConnectivityStatus + Write-Host "$retries, connectivityStatus = $connectivityStatus" + if ($connectedCluster.ConnectivityStatus -eq "Connected") { - Write-Host "Cluster reached connected status" - break + if ((-Not $enableWorkloadIdentity) -Or ($connectedCluster.arcAgentProfile.agentState -eq "Succeeded")) + { + Write-Host "Cluster reached connected status" + break + } } Write-Host "Arc connection status is $($connectedCluster.ConnectivityStatus). Waiting for status to be connected..." - Start-Sleep -Seconds $sleepDurationInSeconds - } + Start-Sleep -Seconds 10 + } if ($retries -eq 0) { - throw "Arc Connection timed out!" - } + exit -1 + } } function New-ConnectedCluster @@ -61,11 +126,11 @@ param( [object] $arcArgs, [Parameter(Mandatory=$true)] [string] $clusterName, + [object] $proxyArgs, [Switch] $useK8s=$false ) Write-Host "New-ConnectedCluster" - $tags = @("SKU=AKSEdgeEssentials") $aksEdgeVersion = (Get-Module -Name AksEdge).Version.ToString() if ($aksEdgeVersion) { @@ -80,29 +145,87 @@ param( $tags += @("ClusterId=$clusterid") } + $errOut = $($retVal = & {az extension remove --name connectedk8s}) 2>&1 + if ($LASTEXITCODE -ne 0) + { + throw "Error removing extension connecktedk8s : $errOut" + } + + Push-Location $env:TEMP + $progressPreference = 'silentlyContinue' + Invoke-WebRequest -Uri "https://aka.ms/ArcK8sPrivateWhl" -OutFile .\connectedk8s-1.10.0-py2.py3-none-any.whl + $connectedK8sWhlFile = (Get-ChildItem . -Filter "connectedk8s-1.10.0-py2.py3-none-any.whl").FullName + $errOut = $($retVal = & {az extension add --source $connectedK8sWhlFile --allow-preview true -y}) 2>&1 + if ($LASTEXITCODE -ne 0) + { + throw "Error installing extension connectedk8s ($connectedK8sWhlFile) : $errOut" + } + Pop-Location + $k8sConnectArgs = @("-g", $arcArgs.ResourceGroupName) $k8sConnectArgs += @("-n", $clusterName) $k8sConnectArgs += @("-l", $arcArgs.Location) $k8sConnectArgs += @("--subscription", $arcArgs.SubscriptionId) $k8sConnectArgs += @("--tags", $tags) + $k8sConnectArgs += @("--disable-auto-upgrade") + if ($null -ne $proxyArgs) + { + if (-Not [string]::IsNullOrEmpty($proxyArgs.Http)) + { + $k8sConnectArgs += @("--proxy-http", $proxyArgs.Http) + } + if (-Not [string]::IsNullOrEmpty($proxyArgs.Https)) + { + $k8sConnectArgs += @("--proxy-https", $proxyArgs.Https) + } + if (-Not [string]::IsNullOrEmpty($proxyArgs.No)) + { + $k8sConnectArgs += @("--proxy-skip-range", $proxyArgs.No) + } + } - Write-Host "Connect cmd args - $k8sConnectArgs" + $tag = "1.20.1-preview" + $env:HELMREGISTRY="azurearcfork8s.azurecr.io/public/azurearck8s/canary/preview2/azure-arc-k8sagents:$tag" + if ($arcArgs.EnableWorkloadIdentity) + { + $k8sConnectArgs += @("--enable-oidc-issuer", "--enable-workload-identity") + } - $errOut = $($retVal = & {az connectedk8s connect $k8sConnectArgs}) 2>&1 - if ($LASTEXITCODE -ne 0) + if (-Not [string]::IsNullOrEmpty($arcArgs.GatewayResourceId)) { + $k8sConnectArgs += @("--gateway-resource-id", $arcArgs.GatewayResourceId) + } + + Write-Host "Connect cmd args - $k8sConnectArgs" + $errOut = $($retVal = & {az connectedk8s connect $k8sConnectArgs}) 2>&1 + if ($LASTEXITCODE -ne 0) + { throw "Arc Connection failed with error : $errOut" } - Verify-ConnectedStatus -arcArgs $arcArgs -clusterName $ClusterName -} + # For debugging + Write-Host "az connectedk8s out : $retVal" -New-Variable -Name gAksEdgeQuickStartForAioVersion -Value "1.0.240904.1500" -Option Constant -ErrorAction SilentlyContinue + Verify-ConnectedStatus -clusterName $ClusterName -resourcegroup $arcArgs.ResourceGroupName -subscriptionId $arcArgs.SubscriptionId -enableWorkloadIdentity:$arcArgs.EnableWorkloadIdentity -# Specify only AIO supported regions -New-Variable -Option Constant -ErrorAction SilentlyContinue -Name arcLocations -Value @( - "eastus", "eastus2", "northeurope", "westeurope", "westus", "westus2", "westus3" -) + if ($arcArgs.EnableWorkloadIdentity) + { + $errOut = $($obj = & {az connectedk8s show -g $arcArgs.ResourceGroupName -n $clusterName | ConvertFrom-Json}) 2>&1 + if ($null -eq $obj) + { + throw "Invalid, empty IssuerUrl!" + } + + $serviceAccountIssuer = $obj.oidcIssuerProfile.issuerUrl + if ([string]::IsNullOrEmpty($serviceAccountIssuer)) + { + throw "Invalid, empty IssuerUrl!" + } + + Write-Host "serviceAccountIssuer = $serviceAccountIssuer" + Restart-ApiServer -serviceAccountIssuer $serviceAccountIssuer -useK8s:$useK8s + } +} if (! [Environment]::Is64BitProcess) { Write-Host "Error: Run this in 64bit Powershell session" -ForegroundColor Red @@ -136,12 +259,16 @@ if ( $null -eq $azureLogin){ } # Ensure `connectedk8s` az cli extension is installed and up to date. -az extension add --upgrade --name connectedk8s -y +$errOut = $($retVal = & {az extension add --upgrade --name connectedk8s -y}) 2>&1 +if ($LASTEXITCODE -ne 0) +{ + throw "Error upgrading extension connecktedk8s : $errOut" +} $installDir = $((Get-Location).Path) $productName = "AKS Edge Essentials - K3s" $networkplugin = "flannel" -$productUrl = "https://download.microsoft.com/download/9/d/b/9db70435-27fc-4feb-8792-04444d585526/AksEdge-K3s-1.28.3-1.7.639.0.msi" +$productUrl = "https://download.microsoft.com/download/9/0/8/9089c6e0-bc8e-4318-b1e0-a045c29fc14d/AksEdge-K3s-1.29.6-1.8.202.0.msi" if ($UseK8s) { $productName ="AKS Edge Essentials - K8s" $networkplugin = "calico" @@ -164,6 +291,9 @@ $aideuserConfig = @" "ServicePrincipalName": "aksedge-sp", "Location": "$Location", "CustomLocationOID":"$CustomLocationOid", + "EnableWorkloadIdentity": true, + "EnableKeyManagement": true, + "GatewayResourceId": "", "Auth":{ "ServicePrincipalId":"", "Password":"" @@ -182,7 +312,12 @@ $aksedgeConfig = @" }, "Network": { "NetworkPlugin": "$networkplugin", - "InternetDisabled": false + "InternetDisabled": false, + "Proxy": { + "Http": null, + "Https": null, + "No": null + } }, "User": { "AcceptEula": true, @@ -249,10 +384,10 @@ $aideuserConfigJson = $aideuserConfig | ConvertFrom-Json $aksedgejson = (Get-ChildItem -Path "$workdir" -Filter aksedge-config.json -Recurse).FullName Set-Content -Path $aksedgejson -Value $aksedgeConfig -Force +$aksedgeConfigJson = $aksedgeConfig | ConvertFrom-Json $aksedgeShell = (Get-ChildItem -Path "$workdir" -Filter AksEdgeShell.ps1 -Recurse).FullName . $aksedgeShell - # Download, install and deploy AKS EE Write-Host "Step 2: Download, install and deploy AKS Edge Essentials" -ForegroundColor Cyan # invoke the workflow, the json file already updated above. @@ -269,8 +404,8 @@ if ($retval) { Write-Host "Step 3: Connect the cluster to Azure" -ForegroundColor Cyan # Set the azure subscription $errOut = $($retVal = & {az account set -s $SubscriptionId}) 2>&1 -if ($LASTEXITCODE -ne 0) -{ +if ($LASTEXITCODE -ne 0) +{ throw "Error setting Subscription ($SubscriptionId): $errOut" } @@ -279,13 +414,14 @@ $errOut = $($rgExists = & {az group show --resource-group $ResourceGroupName}) 2 if ($null -eq $rgExists) { Write-Host "Creating resource group: $ResourceGroupName" -ForegroundColor Cyan $errOut = $($retVal = & {az group create --location $Location --resource-group $ResourceGroupName --subscription $SubscriptionId}) 2>&1 - if ($LASTEXITCODE -ne 0) - { + if ($LASTEXITCODE -ne 0) + { throw "Error creating ResourceGroup ($ResourceGroupName): $errOut" } -} +} # Register the required resource providers +Write-Host "Registering the required resource providers for AIO" -ForegroundColor Cyan $resourceProviders = @( "Microsoft.ExtendedLocation", @@ -295,42 +431,43 @@ $resourceProviders = foreach($rp in $resourceProviders) { $errOut = $($obj = & {az provider show -n $rp | ConvertFrom-Json}) 2>&1 - if ($LASTEXITCODE -ne 0) - { + if ($LASTEXITCODE -ne 0) + { throw "Error querying provider $rp : $errOut" } if ($obj.registrationState -eq "Registered") { - continue + continue } - - $errOut = $($retVal = & {az provider register -n $rp}) 2>&1 - if ($LASTEXITCODE -ne 0) - { + + $errOut = $($retVal = & {az provider register -n $rp}) 2>&1 + if ($LASTEXITCODE -ne 0) + { throw "Error registering provider $rp : $errOut" - } + } } # Arc-enable the Kubernetes cluster Write-Host "Arc enable the kubernetes cluster $ClusterName" -ForegroundColor Cyan -New-ConnectedCluster -clusterName $ClusterName -arcArgs $aideuserConfigJson.Azure -useK8s:$UseK8s + +New-ConnectedCluster -clusterName $ClusterName -arcArgs $aideuserConfigJson.Azure -proxyArgs $aksedgeConfigJson.Network.Proxy -useK8s:$UseK8s # Enable custom location support on your cluster using az connectedk8s enable-features command -Write-Host "Associate Custom location with $ClusterName cluster" $objectId = $aideuserConfigJson.Azure.CustomLocationOID if ([string]::IsNullOrEmpty($objectId)) { + Write-Host "Associate Custom location with $ClusterName cluster" $customLocationsAppId = "bc313c14-388c-4e7d-a58e-70017303ee3b" $errOut = $($objectId = & {az ad sp show --id $customLocationsAppId --query id -o tsv}) 2>&1 if ($null -eq $objectId) { - throw "Error querying ObjectId for CustomLocationsAppId : $errOut" + throw "Error querying ObjectId for CustomLocationsAppId : $errOut" } } $errOut = $($retVal = & {az connectedk8s enable-features -n $ClusterName -g $ResourceGroupName --custom-locations-oid $objectId --features cluster-connect custom-locations}) 2>&1 -if ($LASTEXITCODE -ne 0) -{ +if ($LASTEXITCODE -ne 0) +{ throw "Error enabling feature CustomLocations : $errOut" }