From 624f89e5cd892ea9fd42a4863cf2184103b90479 Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Thu, 17 Mar 2022 14:19:12 -0700 Subject: [PATCH] Install Azure Key Vault plugin if configured --- images/capi/ansible/windows/example.vars.yml | 1 + images/capi/ansible/windows/node_windows.yml | 2 + .../windows/roles/gmsa/defaults/main.yml | 16 +++ .../files/install-gmsa-keyvault-plugin.ps1 | 134 ++++++++++++++++++ .../roles/gmsa/tasks/gmsa_keyvault.yml | 64 +++++++++ .../ansible/windows/roles/gmsa/tasks/main.yml | 17 +++ images/capi/packer/azure/packer-windows.json | 5 +- images/capi/packer/goss/goss-command.yaml | 26 ++++ 8 files changed, 264 insertions(+), 1 deletion(-) create mode 100644 images/capi/ansible/windows/roles/gmsa/defaults/main.yml create mode 100644 images/capi/ansible/windows/roles/gmsa/files/install-gmsa-keyvault-plugin.ps1 create mode 100644 images/capi/ansible/windows/roles/gmsa/tasks/gmsa_keyvault.yml create mode 100644 images/capi/ansible/windows/roles/gmsa/tasks/main.yml diff --git a/images/capi/ansible/windows/example.vars.yml b/images/capi/ansible/windows/example.vars.yml index 996b259a11..7f4f3c0afc 100644 --- a/images/capi/ansible/windows/example.vars.yml +++ b/images/capi/ansible/windows/example.vars.yml @@ -16,6 +16,7 @@ kubernetes_base_url: https://dl.k8s.io/release/v1.19.2/bin/windows/amd64 cloudbase_init_url: https://github.com/cloudbase/cloudbase-init/releases/download/1.1.2/CloudbaseInitSetup_1_1_2_x64.msi wins_url: https://github.com/rancher/wins/releases/download/v0.0.4/wins.exe nssm_url: https://azurek8scishared.blob.core.windows.net/nssm/nssm.exe +gmsa_keyvault_url: https://acs-mirror.azureedge.net/ccgakvplugin/v1.1.4/binaries/windows-gmsa-ccgakvplugin-v1.1.4.zip runtime: docker-ee docker_ee_version: "19.03.12" diff --git a/images/capi/ansible/windows/node_windows.yml b/images/capi/ansible/windows/node_windows.yml index 0017b6121d..64718f0243 100644 --- a/images/capi/ansible/windows/node_windows.yml +++ b/images/capi/ansible/windows/node_windows.yml @@ -75,6 +75,8 @@ name: runtimes - include_role: name: kubernetes + - include_role: + name: gmsa - include_role: name: load_additional_components when: load_additional_components | bool diff --git a/images/capi/ansible/windows/roles/gmsa/defaults/main.yml b/images/capi/ansible/windows/roles/gmsa/defaults/main.yml new file mode 100644 index 0000000000..8e2393d23c --- /dev/null +++ b/images/capi/ansible/windows/roles/gmsa/defaults/main.yml @@ -0,0 +1,16 @@ +# Copyright 2022 The Kubernetes Authors. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +gmsa_keyvault: '{{ true if (gmsa_keyvault_url is defined) and (gmsa_keyvault_url | length > 0) else false }}' diff --git a/images/capi/ansible/windows/roles/gmsa/files/install-gmsa-keyvault-plugin.ps1 b/images/capi/ansible/windows/roles/gmsa/files/install-gmsa-keyvault-plugin.ps1 new file mode 100644 index 0000000000..a670b07d84 --- /dev/null +++ b/images/capi/ansible/windows/roles/gmsa/files/install-gmsa-keyvault-plugin.ps1 @@ -0,0 +1,134 @@ +# Copyright 2022 The Kubernetes Authors. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# script modified from https://github.com/Azure/AgentBaker/blob/8d5323f3b1a622d558e624e5a6b0963229f80b2a/staging/cse/windows/configfunc.ps1 under MIT +$ErrorActionPreference = 'Stop' + +function Enable-Privilege { + param($Privilege) + $Definition = @' + using System; + using System.Runtime.InteropServices; + public class AdjPriv { + [DllImport("advapi32.dll", ExactSpelling = true, SetLastError = true)] + internal static extern bool AdjustTokenPrivileges(IntPtr htok, bool disall, + ref TokPriv1Luid newst, int len, IntPtr prev, IntPtr rele); + [DllImport("advapi32.dll", ExactSpelling = true, SetLastError = true)] + internal static extern bool OpenProcessToken(IntPtr h, int acc, ref IntPtr phtok); + [DllImport("advapi32.dll", SetLastError = true)] + internal static extern bool LookupPrivilegeValue(string host, string name, + ref long pluid); + [StructLayout(LayoutKind.Sequential, Pack = 1)] + internal struct TokPriv1Luid { + public int Count; + public long Luid; + public int Attr; + } + internal const int SE_PRIVILEGE_ENABLED = 0x00000002; + internal const int TOKEN_QUERY = 0x00000008; + internal const int TOKEN_ADJUST_PRIVILEGES = 0x00000020; + public static bool EnablePrivilege(long processHandle, string privilege) { + bool retVal; + TokPriv1Luid tp; + IntPtr hproc = new IntPtr(processHandle); + IntPtr htok = IntPtr.Zero; + retVal = OpenProcessToken(hproc, TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, + ref htok); + tp.Count = 1; + tp.Luid = 0; + tp.Attr = SE_PRIVILEGE_ENABLED; + retVal = LookupPrivilegeValue(null, privilege, ref tp.Luid); + retVal = AdjustTokenPrivileges(htok, false, ref tp, 0, IntPtr.Zero, + IntPtr.Zero); + return retVal; + } + } +'@ + $ProcessHandle = (Get-Process -id $pid).Handle + $type = Add-Type $definition -PassThru + $type[0]::EnablePrivilege($processHandle, $Privilege) +} + +function Aquire-Privilege { + param($Privilege) + + write-output "Acquiring the $Privilege privilege" + $enablePrivilegeResponse=$false + for($i = 0; $i -lt 10; $i++) { + write-output "Retry $i : Trying to enable the $Privilege privilege" + $enablePrivilegeResponse = Enable-Privilege -Privilege "$Privilege" -ErrorAction 'Continue' + if ($enablePrivilegeResponse) { + break + } + Start-Sleep 1 + } + if(!$enablePrivilegeResponse) { + write-output "Failed to enable the $Privilege privilege." + exit 1 + } +} + +# Enable the PowerShell privilege to set the registry permissions. +Aquire-Privilege -Privilege "SeTakeOwnershipPrivilege" + +# Set the registry permissions. +write-output "Setting GMSA plugin registry permissions" +try { + $ccgKeyPath = "System\CurrentControlSet\Control\CCG\COMClasses" + $owner = [System.Security.Principal.NTAccount]"BUILTIN\Administrators" + + $key = [Microsoft.Win32.Registry]::LocalMachine.OpenSubKey( + $ccgKeyPath, + [Microsoft.Win32.RegistryKeyPermissionCheck]::ReadWriteSubTree, + [System.Security.AccessControl.RegistryRights]::TakeOwnership) + $acl = $key.GetAccessControl() + $originalOwner = $acl.owner + $acl.SetOwner($owner) + $key.SetAccessControl($acl) + + $key = [Microsoft.Win32.Registry]::LocalMachine.OpenSubKey( + $ccgKeyPath, + [Microsoft.Win32.RegistryKeyPermissionCheck]::ReadWriteSubTree, + [System.Security.AccessControl.RegistryRights]::ChangePermissions) + $acl = $key.GetAccessControl() + $rule = New-Object System.Security.AccessControl.RegistryAccessRule( + $owner, + [System.Security.AccessControl.RegistryRights]::FullControl, + [System.Security.AccessControl.AccessControlType]::Allow) + $acl.SetAccessRule($rule) + $key.SetAccessControl($acl) +} catch { + write-output "Failed to set GMSA plugin registry permissions. $_" + exit 1 +} + +# Set the appropriate registry values. +try { + write-output "Setting the appropriate GMSA plugin registry values" + reg.exe import "registerplugin.reg" +} catch { + write-output "Failed to set GMSA plugin registry values. $_" + exit 1 +} + +write-output "Restore original access to registry key" +$acl = $key.GetAccessControl() +$acl.RemoveAccessRule($rule) +$acl.SetOwner([System.Security.Principal.NTAccount]$originalowner) +Aquire-Privilege -Privilege "SeRestorePrivilege" +$key.SetAccessControl($acl) +$key.close() + + +write-output "Successfully installed the GMSA plugin" diff --git a/images/capi/ansible/windows/roles/gmsa/tasks/gmsa_keyvault.yml b/images/capi/ansible/windows/roles/gmsa/tasks/gmsa_keyvault.yml new file mode 100644 index 0000000000..cc12d9aa87 --- /dev/null +++ b/images/capi/ansible/windows/roles/gmsa/tasks/gmsa_keyvault.yml @@ -0,0 +1,64 @@ +# Copyright 2022 The Kubernetes Authors. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +- name: Download gMSA Key Vault plugin + win_get_url: + url: '{{ gmsa_keyvault_url }}' + dest: '{{ tempdir.stdout | trim }}\windows-gmsa-ccgakvplugin.zip' + async: 1800 + poll: 60 + retries: 5 + delay: 3 + register: gmsadownload + until: gmsadownload is not failed + +- name: Unzip gMSA Key Vault Archive + win_unzip: + src: '{{ gmsadownload.dest }}' + dest: '{{ kubernetes_install_path }}' + recurse: no + delete_archive: yes + +- name: Copy gMSA Key Vault plugin to System32 + win_shell: | + Move-Item -Force -Path {{ kubernetes_install_path }}\CCGAKVPlugin.dll -Destination {{ systemdrive.stdout | trim }}\Windows\System32\ + +# This is done via a script because Ansible doesn't have the ability to take ownership of registry keys +# The script enables the privilege for the process running and modifies the reg keys. Once process exits it no longer has privileges +# See https://groups.google.com/g/ansible-project/c/5Bt7jgq6ZFA/m/_XJtVzmhBwAJ +- name: Copy gMSA Key Vault installer file + win_copy: + src: install-gmsa-keyvault-plugin.ps1 + dest: '{{ kubernetes_install_path }}' + +- name: Register gMSA Key Vault plugin + win_shell: | + {{ kubernetes_install_path }}\install-gmsa-keyvault-plugin.ps1 + +- name: Install registry CCG logging manifest + win_shell: | + wevtutil.exe um {{ kubernetes_install_path }}\CCGEvents.man + wevtutil.exe im {{ kubernetes_install_path }}\CCGEvents.man + +- name: Install registry Key Vault plugin logging manifest + win_shell: | + wevtutil.exe um {{ kubernetes_install_path }}\CCGAKVPluginEvents.man + wevtutil.exe im {{ kubernetes_install_path }}\CCGAKVPluginEvents.man + +- name: Clean up gMSA install files + win_shell: | + Remove-Item {{ kubernetes_install_path }}\CCGEvents.man + Remove-Item {{ kubernetes_install_path }}\CCGAKVPluginEvents.man + Remove-Item {{ kubernetes_install_path }}\registerplugin.reg + Remove-Item {{ kubernetes_install_path }}\install-gmsa-keyvault-plugin.ps1 diff --git a/images/capi/ansible/windows/roles/gmsa/tasks/main.yml b/images/capi/ansible/windows/roles/gmsa/tasks/main.yml new file mode 100644 index 0000000000..2cfd9f5175 --- /dev/null +++ b/images/capi/ansible/windows/roles/gmsa/tasks/main.yml @@ -0,0 +1,17 @@ +# Copyright 2022 The Kubernetes Authors. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +- import_tasks: gmsa_keyvault.yml + when: gmsa_keyvault | bool diff --git a/images/capi/packer/azure/packer-windows.json b/images/capi/packer/azure/packer-windows.json index ca3d5957cb..65e64f01f6 100644 --- a/images/capi/packer/azure/packer-windows.json +++ b/images/capi/packer/azure/packer-windows.json @@ -117,7 +117,9 @@ "--extra-vars", "{{user `ansible_extra_vars`}}", "--extra-vars", - "{{user `ansible_user_vars`}}" + "{{user `ansible_user_vars`}}", + "--extra-vars", + "gmsa_keyvault_url={{user `gmsa_keyvault_url`}}" ], "playbook_file": "ansible/windows/node_windows.yml", "type": "ansible", @@ -194,6 +196,7 @@ "cloudbase_plugins_unattend": "cloudbaseinit.plugins.common.mtu.MTUPlugin", "containerd_url": "", "containerd_version": null, + "gmsa_keyvault_url": "https://kubernetesartifacts.azureedge.net/ccgakvplugin/v1.1.4/binaries/windows-gmsa-ccgakvplugin-v1.1.4.zip", "ib_version": "{{env `IB_VERSION`}}", "image_offer": null, "image_publisher": null, diff --git a/images/capi/packer/goss/goss-command.yaml b/images/capi/packer/goss/goss-command.yaml index 620c9513a2..e4fca5801b 100644 --- a/images/capi/packer/goss/goss-command.yaml +++ b/images/capi/packer/goss/goss-command.yaml @@ -174,5 +174,31 @@ command: - True stderr: [] timeout: 30000 + + # this could be moved to place for other providers if they want to install it + Key Vault gMSA binary is installed: + exec: powershell -command "Test-Path -Path C:\Windows\System32\CCGAKVPlugin.dll" + exit-status: 0 + stdout: + - "True" + timeout: 30000 + Key Vault gMSA binary COM is registered: + exec: powershell -command "(Get-Item 'HKLM:SYSTEM\CurrentControlSet\Control\CCG\COMClasses\{CCC2A336-D7F3-4818-A213-272B7924213E}') | Ft -autosize -wrap" + exit-status: 0 + stdout: + - "CCC2A336-D7F3-4818-A213-272B7924213E" + timeout: 30000 + Key Vault gMSA binary is registered: + exec: powershell -command "Get-ItemProperty -Path 'HKLM:SOFTWARE\CLASSES\CLSID\{CCC2A336-D7F3-4818-A213-272B7924213E}\InprocServer32\'" + exit-status: 0 + stdout: + - "C:\\Windows\\System32\\CCGAKVPlugin.dll" + timeout: 30000 + Key Vault gMSA CCG interface is registered: + exec: powershell -command "(Get-Item 'HKLM:SOFTWARE\Classes\Interface\{6ECDA518-2010-4437-8BC3-46E752B7B172}') | Ft -autosize -wrap" + exit-status: 0 + stdout: + - "ICcgDomainAuthCredentials" + timeout: 30000 {{end}} {{end}} #end windows