Install Azure Key Vault gMSA plugin if configured

images/capi/ansible/windows/example.vars.yml

@@ -16,6 +16,7 @@ kubernetes_base_url: https://dl.k8s.io/release/v1.19.2/bin/windows/amd64
 cloudbase_init_url: https://github.com/cloudbase/cloudbase-init/releases/download/1.1.2/CloudbaseInitSetup_1_1_2_x64.msi
 wins_url: https://github.com/rancher/wins/releases/download/v0.0.4/wins.exe
 nssm_url: https://azurek8scishared.blob.core.windows.net/nssm/nssm.exe
+gmsa_keyvault_url: https://kubernetesartifacts.azureedge.net/ccgakvplugin/v1.1.4/binaries/windows-gmsa-ccgakvplugin-v1.1.4.zip
 
 runtime: docker-ee
 docker_ee_version: "19.03.12"

images/capi/ansible/windows/node_windows.yml

@@ -75,6 +75,8 @@
         name: runtimes
     - include_role:
         name: kubernetes
+    - include_role:
+        name: gmsa
     - include_role:
         name: load_additional_components
       when: load_additional_components | bool

images/capi/ansible/windows/roles/gmsa/defaults/main.yml

@@ -0,0 +1,16 @@
+# Copyright 2022 The Kubernetes Authors.
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+
+gmsa_keyvault: '{{ true if (gmsa_keyvault_url is defined) and (gmsa_keyvault_url | length > 0) else false }}'

images/capi/ansible/windows/roles/gmsa/files/install-gmsa-keyvault-plugin.ps1

@@ -0,0 +1,134 @@
+# Copyright 2022 The Kubernetes Authors.
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# script modified from https://github.com/Azure/AgentBaker/blob/8d5323f3b1a622d558e624e5a6b0963229f80b2a/staging/cse/windows/configfunc.ps1 under MIT
+$ErrorActionPreference = 'Stop'
+
+function Enable-Privilege {
+    param($Privilege)
+    $Definition = @'
+  using System;
+  using System.Runtime.InteropServices;
+  public class AdjPriv {
+    [DllImport("advapi32.dll", ExactSpelling = true, SetLastError = true)]
+    internal static extern bool AdjustTokenPrivileges(IntPtr htok, bool disall,
+      ref TokPriv1Luid newst, int len, IntPtr prev, IntPtr rele);
+    [DllImport("advapi32.dll", ExactSpelling = true, SetLastError = true)]
+    internal static extern bool OpenProcessToken(IntPtr h, int acc, ref IntPtr phtok);
+    [DllImport("advapi32.dll", SetLastError = true)]
+    internal static extern bool LookupPrivilegeValue(string host, string name,
+      ref long pluid);
+    [StructLayout(LayoutKind.Sequential, Pack = 1)]
+    internal struct TokPriv1Luid {
+      public int Count;
+      public long Luid;
+      public int Attr;
+    }
+    internal const int SE_PRIVILEGE_ENABLED = 0x00000002;
+    internal const int TOKEN_QUERY = 0x00000008;
+    internal const int TOKEN_ADJUST_PRIVILEGES = 0x00000020;
+    public static bool EnablePrivilege(long processHandle, string privilege) {
+      bool retVal;
+      TokPriv1Luid tp;
+      IntPtr hproc = new IntPtr(processHandle);
+      IntPtr htok = IntPtr.Zero;
+      retVal = OpenProcessToken(hproc, TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY,
+        ref htok);
+      tp.Count = 1;
+      tp.Luid = 0;
+      tp.Attr = SE_PRIVILEGE_ENABLED;
+      retVal = LookupPrivilegeValue(null, privilege, ref tp.Luid);
+      retVal = AdjustTokenPrivileges(htok, false, ref tp, 0, IntPtr.Zero,
+        IntPtr.Zero);
+      return retVal;
+    }
+  }
+'@
+    $ProcessHandle = (Get-Process -id $pid).Handle
+    $type = Add-Type $definition -PassThru
+    $type[0]::EnablePrivilege($processHandle, $Privilege)
+}
+
+function Aquire-Privilege {
+  param($Privilege)
+
+  write-output "Acquiring the $Privilege privilege"
+  $enablePrivilegeResponse=$false
+  for($i = 0; $i -lt 10; $i++) {
+      write-output "Retry $i : Trying to enable the $Privilege privilege"
+      $enablePrivilegeResponse = Enable-Privilege -Privilege "$Privilege" -ErrorAction 'Continue'
+      if ($enablePrivilegeResponse) {
+          break
+      }
+      Start-Sleep 1
+  }
+  if(!$enablePrivilegeResponse) {
+      write-output "Failed to enable the $Privilege privilege."
+      exit 1
+  }
+}
+
+# Enable the PowerShell privilege to set the registry permissions.
+Aquire-Privilege -Privilege "SeTakeOwnershipPrivilege"
+
+# Set the registry permissions.
+write-output "Setting GMSA plugin registry permissions"
+try {
+    $ccgKeyPath = "System\CurrentControlSet\Control\CCG\COMClasses"
+    $owner = [System.Security.Principal.NTAccount]"BUILTIN\Administrators"
+
+    $key = [Microsoft.Win32.Registry]::LocalMachine.OpenSubKey(
+        $ccgKeyPath,
+        [Microsoft.Win32.RegistryKeyPermissionCheck]::ReadWriteSubTree,
+        [System.Security.AccessControl.RegistryRights]::TakeOwnership)
+    $acl = $key.GetAccessControl()
+    $originalOwner = $acl.owner
+    $acl.SetOwner($owner)
+    $key.SetAccessControl($acl)
+
+    $key = [Microsoft.Win32.Registry]::LocalMachine.OpenSubKey(
+        $ccgKeyPath,
+        [Microsoft.Win32.RegistryKeyPermissionCheck]::ReadWriteSubTree,
+        [System.Security.AccessControl.RegistryRights]::ChangePermissions)
+    $acl = $key.GetAccessControl()
+    $rule = New-Object System.Security.AccessControl.RegistryAccessRule(
+        $owner,
+        [System.Security.AccessControl.RegistryRights]::FullControl,
+        [System.Security.AccessControl.AccessControlType]::Allow)
+    $acl.SetAccessRule($rule)
+    $key.SetAccessControl($acl)
+} catch {
+    write-output "Failed to set GMSA plugin registry permissions. $_"
+    exit 1
+}
+
+# Set the appropriate registry values.
+try {
+    write-output "Setting the appropriate GMSA plugin registry values"
+    reg.exe import "registerplugin.reg"
+} catch {
+    write-output "Failed to set GMSA plugin registry values. $_"
+    exit 1
+}
+
+write-output "Restore original access to registry key"
+$acl = $key.GetAccessControl()
+$acl.RemoveAccessRule($rule)
+$acl.SetOwner([System.Security.Principal.NTAccount]$originalowner)
+Aquire-Privilege -Privilege "SeRestorePrivilege"
+$key.SetAccessControl($acl)
+$key.close()
+
+
+write-output "Successfully installed the GMSA plugin"

images/capi/ansible/windows/roles/gmsa/tasks/gmsa_keyvault.yml

@@ -0,0 +1,64 @@
+# Copyright 2022 The Kubernetes Authors.
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+- name: Download gMSA Key Vault plugin
+  win_get_url:
+    url: '{{ gmsa_keyvault_url }}'
+    dest: '{{ tempdir.stdout | trim }}\windows-gmsa-ccgakvplugin.zip'
+  async: 1800
+  poll: 60
+  retries: 5
+  delay: 3
+  register: gmsadownload
+  until: gmsadownload is not failed
+
+- name: Unzip gMSA Key Vault Archive
+  win_unzip:
+    src: '{{ gmsadownload.dest }}'
+    dest: '{{ kubernetes_install_path }}'
+    recurse: no
+    delete_archive: yes
+
+- name: Copy gMSA Key Vault plugin to System32
+  win_shell: |
+    Move-Item -Force -Path {{ kubernetes_install_path }}\CCGAKVPlugin.dll -Destination {{ systemdrive.stdout | trim }}\Windows\System32\
+
+# This is done via a script because Ansible doesn't have the ability to take ownership of registry keys
+# The script enables the privilege for the process running and modifies the reg keys. Once process exits it no longer has privileges
+# See https://groups.google.com/g/ansible-project/c/5Bt7jgq6ZFA/m/_XJtVzmhBwAJ
+- name: Copy gMSA Key Vault installer file
+  win_copy:
+    src: install-gmsa-keyvault-plugin.ps1
+    dest: '{{ kubernetes_install_path }}'
+
+- name: Register gMSA Key Vault plugin
+  win_shell: |
+    {{ kubernetes_install_path }}\install-gmsa-keyvault-plugin.ps1
+
+- name: Install registry CCG logging manifest
+  win_shell: |
+    wevtutil.exe um {{ kubernetes_install_path }}\CCGEvents.man
+    wevtutil.exe im {{ kubernetes_install_path }}\CCGEvents.man
+
+- name: Install registry Key Vault plugin logging manifest
+  win_shell: |
+    wevtutil.exe um {{ kubernetes_install_path }}\CCGAKVPluginEvents.man
+    wevtutil.exe im {{ kubernetes_install_path }}\CCGAKVPluginEvents.man
+
+- name: Clean up gMSA install files
+  win_shell: |
+    Remove-Item {{ kubernetes_install_path }}\CCGEvents.man
+    Remove-Item {{ kubernetes_install_path }}\CCGAKVPluginEvents.man
+    Remove-Item {{ kubernetes_install_path }}\registerplugin.reg
+    Remove-Item {{ kubernetes_install_path }}\install-gmsa-keyvault-plugin.ps1

images/capi/ansible/windows/roles/gmsa/tasks/main.yml

@@ -0,0 +1,17 @@
+# Copyright 2022 The Kubernetes Authors.
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+
+- import_tasks: gmsa_keyvault.yml
+  when: gmsa_keyvault | bool

images/capi/packer/azure/packer-windows.json

@@ -117,7 +117,9 @@
         "--extra-vars",
         "{{user `ansible_extra_vars`}}",
         "--extra-vars",
-        "{{user `ansible_user_vars`}}"
+        "{{user `ansible_user_vars`}}",
+        "--extra-vars",
+        "gmsa_keyvault_url={{user `gmsa_keyvault_url`}}"
       ],
       "playbook_file": "ansible/windows/node_windows.yml",
       "type": "ansible",
@@ -194,6 +196,7 @@
     "cloudbase_plugins_unattend": "cloudbaseinit.plugins.common.mtu.MTUPlugin",
     "containerd_url": "",
     "containerd_version": null,
+    "gmsa_keyvault_url": "https://kubernetesartifacts.azureedge.net/ccgakvplugin/v1.1.4/binaries/windows-gmsa-ccgakvplugin-v1.1.4.zip",
     "ib_version": "{{env `IB_VERSION`}}",
     "image_offer": null,
     "image_publisher": null,

images/capi/packer/goss/goss-command.yaml

@@ -174,5 +174,31 @@ command:
     - True
     stderr: []
     timeout: 30000
+
+  # this could be moved to place for other providers if they want to install it
+  Key Vault gMSA binary is installed:
+    exec: powershell -command "Test-Path -Path  C:\Windows\System32\CCGAKVPlugin.dll"
+    exit-status: 0
+    stdout:
+    - "True"
+    timeout: 30000
+  Key Vault gMSA binary COM is registered:
+    exec: powershell -command "(Get-Item 'HKLM:SYSTEM\CurrentControlSet\Control\CCG\COMClasses\{CCC2A336-D7F3-4818-A213-272B7924213E}')  | Ft -autosize -wrap"
+    exit-status: 0
+    stdout:
+    - "CCC2A336-D7F3-4818-A213-272B7924213E"
+    timeout: 30000
+  Key Vault gMSA binary is registered:
+    exec: powershell -command "Get-ItemProperty -Path 'HKLM:SOFTWARE\CLASSES\CLSID\{CCC2A336-D7F3-4818-A213-272B7924213E}\InprocServer32\'"
+    exit-status: 0
+    stdout:
+    - "C:\\Windows\\System32\\CCGAKVPlugin.dll"
+    timeout: 30000
+  Key Vault gMSA CCG interface is registered:
+    exec: powershell -command "(Get-Item 'HKLM:SOFTWARE\Classes\Interface\{6ECDA518-2010-4437-8BC3-46E752B7B172}') | Ft -autosize -wrap"
+    exit-status: 0
+    stdout:
+    - "ICcgDomainAuthCredentials"
+    timeout: 30000
 {{end}}
 {{end}} #end windows