diff --git a/pkg/azure/types.go b/pkg/azure/types.go
index 83cfbfec7..54ee4e22d 100644
--- a/pkg/azure/types.go
+++ b/pkg/azure/types.go
@@ -155,6 +155,11 @@ const (
 	SeedAnnotationUseFlowValueNew = "new"
 	// AnnotationEnableVolumeAttributesClass is the annotation to use on shoots to enable VolumeAttributesClasses
 	AnnotationEnableVolumeAttributesClass = "azure.provider.extensions.gardener.cloud/enable-volume-attributes-class"
+
+	// CCMServiceTagKey is the service key applied for public IP tags.
+	CCMServiceTagKey = "k8s-azure-service"
+	// CCMLegacyServiceTagKey is the legacy service key applied for public IP tags.
+	CCMLegacyServiceTagKey = "service"
 )
 
 // UsernamePrefix is a constant for the username prefix of components deployed by Azure.
diff --git a/pkg/controller/infrastructure/infraflow/ensurer.go b/pkg/controller/infrastructure/infraflow/ensurer.go
index 1590cea27..e9ffb3eb6 100644
--- a/pkg/controller/infrastructure/infraflow/ensurer.go
+++ b/pkg/controller/infrastructure/infraflow/ensurer.go
@@ -414,7 +414,8 @@ func (fctx *FlowContext) ensurePublicIps(ctx context.Context) error {
 	}
 	currentIPs = Filter(currentIPs, func(address *armnetwork.PublicIPAddress) bool {
 		// filter only these IpConfigs prefixed by the cluster name and that do not contain the CCM tags.
-		return fctx.adapter.HasShootPrefix(address.Name) && address.Tags["k8s-azure-service"] == nil
+		return fctx.adapter.HasShootPrefix(address.Name) &&
+			(address.Tags[azure.CCMServiceTagKey] == nil && address.Tags[azure.CCMLegacyServiceTagKey] == nil)
 	})
 	// obtain an indexed list of current IPs
 	nameToCurrentIps := ToMap(currentIPs, func(t *armnetwork.PublicIPAddress) string {
@@ -455,8 +456,9 @@ func (fctx *FlowContext) ensurePublicIps(ctx context.Context) error {
 		err := fctx.providerAccess.DeletePublicIP(ctx, fctx.adapter.ResourceGroupName(), ipName)
 		if err != nil {
 			joinError = errors.Join(joinError, err)
+		} else {
+			fctx.inventory.Delete(ip)
 		}
-		fctx.inventory.Delete(ip)
 	}
 
 	if joinError != nil {