From 3c2c65831e39c1f0ffd0fc93e6002eb701b7d884 Mon Sep 17 00:00:00 2001 From: Kiran Mova Date: Fri, 10 Apr 2020 20:40:04 +0530 Subject: [PATCH] fix(localpv-hostpath): pv deletion caused panic when node not found (#1662) After the PV is created and node affinity is set based on kubernetes.io/hostname label, either: - hostname label changed on the node or - the node is deleted from the cluster. This PR fixes the panic, by logging the details of the node hostname that no longer exists in the cluster. The code doesn't force delete the PV, considering that the node might intermittently be out of the cluster. If the user decides that node is not going to come back, then the user can go ahead with force delete of pv using `kubectl delete pv`. An alternate case is that node hostname has really changed. In this case, user will have to perform a manual migration of the localpv-hostpath PV to the new node using the following steps: - Scale down the deployment or sts using the PVC. - Save the PVC and PV yamls files. - Delete the PVC and PV - Modify the saved PV yaml will the node hostname and apply Note: when re-applying the yamls, the uuid of pv and pvc objects will change, so the metadata around self-uuid,etc needs to be cleared. - Modify the saved PVC yaml for stale references and re-apply. - Update the PV yaml with the uuid of the newly created PVC - Scale up the deployment. Signed-off-by: kmova (cherry picked from commit 728a00363ccb2e0f6a676b1b04358a7916364203) --- cmd/provisioner-localpv/app/provisioner_hostpath.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cmd/provisioner-localpv/app/provisioner_hostpath.go b/cmd/provisioner-localpv/app/provisioner_hostpath.go index 2f6da47a6d..35484b131d 100644 --- a/cmd/provisioner-localpv/app/provisioner_hostpath.go +++ b/cmd/provisioner-localpv/app/provisioner_hostpath.go @@ -134,8 +134,12 @@ func (p *Provisioner) GetNodeObjectFromHostName(hostName string) (*v1.Node, erro Limit: 1, } nodeList, err := p.kubeClient.CoreV1().Nodes().List(listOptions) - if err != nil { - return nil, errors.Errorf("Unable to get the Node with the NodeHostName") + if err != nil || len(nodeList.Items) == 0 { + // After the PV is created and node affinity is set + // based on kubernetes.io/hostname label, either: + // - hostname label changed on the node or + // - the node is deleted from the cluster. + return nil, errors.Errorf("Unable to get the Node with the NodeHostName [%s]", hostName) } return &nodeList.Items[0], nil