diff --git a/src/Arcus.Scripting.Tests.Integration/Arcus.Scripting.IntegrationAccount.tests.ps1 b/src/Arcus.Scripting.Tests.Integration/Arcus.Scripting.IntegrationAccount.tests.ps1
index 611cd033..1bae5300 100644
--- a/src/Arcus.Scripting.Tests.Integration/Arcus.Scripting.IntegrationAccount.tests.ps1
+++ b/src/Arcus.Scripting.Tests.Integration/Arcus.Scripting.IntegrationAccount.tests.ps1
@@ -704,7 +704,7 @@ InModuleScope Arcus.Scripting.IntegrationAccount {
                 $expectedCertificateName = $certificate.BaseName
                 $executionDateTime = (Get-Date).ToUniversalTime()
                 $subscriptionId = $config.Arcus.SubscriptionId
-                $keyName = "PrivateCertificateKey"
+                $keyName = "PrivateCertificateKey-$([System.Guid]::NewGuid())"
                 $keyVaultName = $config.Arcus.KeyVault.VaultName
                 $keyVaultId = "/subscriptions/$subscriptionId/resourceGroups/$resourceGroupName/providers/Microsoft.KeyVault/vaults/$keyVaultName"
 
@@ -737,7 +737,7 @@ InModuleScope Arcus.Scripting.IntegrationAccount {
                 $expectedCertificateName = $artifactsPrefix + $certificate.BaseName
                 $executionDateTime = (Get-Date).ToUniversalTime()
                 $subscriptionId = $config.Arcus.SubscriptionId
-                $keyName = 'PrivateCertificateKeyPrefix'
+                $keyName = "PrivateCertificateKeyPrefix-$([System.Guid]::NewGuid())"
                 $keyVaultName = $config.Arcus.KeyVault.VaultName
                 $keyVaultId = "/subscriptions/$subscriptionId/resourceGroups/$resourceGroupName/providers/Microsoft.KeyVault/vaults/$keyVaultName"
 
@@ -1094,4 +1094,4 @@ InModuleScope Arcus.Scripting.IntegrationAccount {
             }
         }
     }
-}
\ No newline at end of file
+}