diff --git a/pkg/provider/azure_backoff.go b/pkg/provider/azure_backoff.go
index 73aa33949c..90256825b0 100644
--- a/pkg/provider/azure_backoff.go
+++ b/pkg/provider/azure_backoff.go
@@ -336,6 +336,7 @@ func (az *Cloud) ListManagedLBs(service *v1.Service, nodes []*v1.Node, clusterNa
 
 	// return early if wantLb=false
 	if nodes == nil {
+		klog.V(4).Infof("ListManagedLBs: return all LBs in the resource group %s, including unmanaged LBs", az.getLoadBalancerResourceGroup())
 		return allLBs, nil
 	}
 
diff --git a/pkg/provider/azure_loadbalancer.go b/pkg/provider/azure_loadbalancer.go
index 0a00544c68..af29758029 100644
--- a/pkg/provider/azure_loadbalancer.go
+++ b/pkg/provider/azure_loadbalancer.go
@@ -462,9 +462,18 @@ func (az *Cloud) reconcileSharedLoadBalancer(service *v1.Service, clusterName st
 		return existingLBs, nil
 	}
 
+	// Skip if nodes is nil, which means the service is being deleted.
+	// When nodes is nil, all LBs included unmanaged LBs will be returned,
+	// if we don't skip this function, the unmanaged ones may be deleted later.
+	if nodes == nil {
+		klog.V(4).Infof("reconcileSharedLoadBalancer: returning early because the service %s is being deleted", service.Name)
+		return existingLBs, nil
+	}
+
 	lbNamesToBeDeleted := sets.NewString()
 	// delete unwanted LBs
 	for _, lb := range existingLBs {
+		klog.V(4).Infof("reconcileSharedLoadBalancer: checking LB %s", to.String(lb.Name))
 		// skip the internal or external primary load balancer
 		lbNamePrefix := strings.TrimSuffix(to.String(lb.Name), consts.InternalLoadBalancerNameSuffix)
 		if strings.EqualFold(lbNamePrefix, clusterName) {
diff --git a/pkg/provider/azure_loadbalancer_test.go b/pkg/provider/azure_loadbalancer_test.go
index 5644c66f54..7ec14dcb3c 100644
--- a/pkg/provider/azure_loadbalancer_test.go
+++ b/pkg/provider/azure_loadbalancer_test.go
@@ -4753,9 +4753,17 @@ func TestReconcileSharedLoadBalancer(t *testing.T) {
 			useBasicLB:        true,
 			expectedListCount: 1,
 		},
+		{
+			description:       "reconcileSharedLoadBalancer should do nothing if `nodes` is nil",
+			expectedListCount: 1,
+		},
 		{
 			description:     "reconcileSharedLoadBalancer should do nothing if the vmSet is not sharing the primary slb",
 			useMultipleSLBs: true,
+			nodes: []*v1.Node{
+				{ObjectMeta: metav1.ObjectMeta{Name: "kubernetes"}},
+				{ObjectMeta: metav1.ObjectMeta{Name: "vmss1"}},
+			},
 			existingLBs: []network.LoadBalancer{
 				{
 					Name: to.StringPtr("kubernetes"),
@@ -4791,8 +4799,8 @@ func TestReconcileSharedLoadBalancer(t *testing.T) {
 
 			cloud.NodePoolsWithoutDedicatedSLB = tc.vmSetsSharingPrimarySLB
 
+			cloud.LoadBalancerSku = consts.VMTypeStandard
 			if tc.useMultipleSLBs {
-				cloud.LoadBalancerSku = consts.VMTypeStandard
 				cloud.EnableMultipleStandardLoadBalancers = true
 			} else if tc.useBasicLB {
 				cloud.LoadBalancerSku = consts.LoadBalancerSkuBasic