Skip to content

Commit

Permalink
[ws-manager] Backup on pod eviction (#4405)
Browse files Browse the repository at this point in the history
* [ws-manager] Backup on pod eviction

* Fixed changelog

* Update components/ws-manager/pkg/manager/monitor.go

Co-authored-by: Cornelius A. Ludmann <[email protected]>
Co-authored-by: Jan Koehnlein <[email protected]>
Co-authored-by: Cornelius A. Ludmann <[email protected]>
  • Loading branch information
3 people authored Jun 8, 2021
1 parent c645cb9 commit 715ef34
Show file tree
Hide file tree
Showing 5 changed files with 98 additions and 9 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ All notable changes to this project will be documented in this file.

## June 2021

- Improve backup stability when pods get evicted ([#4405](https://github.com/gitpod-io/gitpod/pull/4405))
- Fix text color in workspaces list for dark theme ([#4410](https://github.com/gitpod-io/gitpod/pull/4410))
- Better reflect incremental prebuilds in prebuilt workspace logs ([#4293](https://github.com/gitpod-io/gitpod/pull/4293))
- Run shellcheck against scripts ([#4280](https://github.com/gitpod-io/gitpod/pull/4280))
Expand Down
3 changes: 3 additions & 0 deletions components/ws-manager/pkg/manager/annotations.go
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,9 @@ const (

// disposalStatusAnnotation contains the status of the workspace disposal process
disposalStatusAnnotation = "gitpod.io/disposalStatus"

// hostIPAnnotation contains the IP of the node the pod ran on. We use this to remeber the IP in case the pod gets evicted.
hostIPAnnotation = "gitpod.io/hostIP"
)

// markWorkspaceAsReady adds annotations to a workspace pod
Expand Down
16 changes: 8 additions & 8 deletions components/ws-manager/pkg/manager/monitor.go
Original file line number Diff line number Diff line change
Expand Up @@ -326,14 +326,14 @@ func (m *Monitor) actOnPodEvent(ctx context.Context, status *api.WorkspaceStatus
}
}

if !wso.IsWorkspaceHeadless() {
span.LogKV("event", "removeTraceAnnotation")
// once a regular workspace is up and running, we'll remove the traceID information so that the parent span
// ends once the workspace has started
err := m.manager.markWorkspace(ctx, workspaceID, deleteMark(wsk8s.TraceIDAnnotation))
if err != nil {
log.WithError(err).Warn("was unable to remove traceID annotation from workspace")
}
// once a regular workspace is up and running, we'll remove the traceID information so that the parent span
// ends once the workspace has started.
//
// Also, in case the pod gets evicted we would not know the hostIP that pod ran on anymore.
// In preparation for those cases, we'll add it as an annotation.
err := m.manager.markWorkspace(ctx, workspaceID, deleteMark(wsk8s.TraceIDAnnotation), addMark(hostIPAnnotation, wso.HostIP()))
if err != nil {
log.WithError(err).Warn("was unable to remove traceID and/or add host IP annotation from/to workspace")
}
}

Expand Down
9 changes: 8 additions & 1 deletion components/ws-manager/pkg/manager/status.go
Original file line number Diff line number Diff line change
Expand Up @@ -136,9 +136,16 @@ func (wso *workspaceObjects) WasEverReady() (res bool) {

// HostIP returns the IP of the node this workspace is/was deployed to. If this workspace has never been deployed anywhere, HostIP returns an empty string.
func (wso *workspaceObjects) HostIP() string {
if wso.Pod != nil {
if wso.Pod == nil {
return ""
}
if wso.Pod.Status.HostIP != "" {
return wso.Pod.Status.HostIP
}
if res, ok := wso.Pod.Annotations[hostIPAnnotation]; ok {
return res
}

return ""
}

Expand Down
78 changes: 78 additions & 0 deletions components/ws-manager/pkg/manager/status_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,13 @@ import (
"testing"
"time"

v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

ctesting "github.com/gitpod-io/gitpod/common-go/testing"
"github.com/gitpod-io/gitpod/common-go/util"
"github.com/gitpod-io/gitpod/ws-manager/api"
"github.com/google/go-cmp/cmp"
)

func TestIsWorkspaceTimedout(t *testing.T) {
Expand Down Expand Up @@ -151,5 +153,81 @@ func BenchmarkGetStatus(b *testing.B) {
}
})
}
}

func TestGetHostIP(t *testing.T) {
tests := []struct {
Name string
WSO workspaceObjects
Expectation string
}{
{
Name: "no hostIP",
},
{
Name: "spec",
WSO: workspaceObjects{
Pod: &v1.Pod{
Status: v1.PodStatus{
HostIP: "foobar",
},
},
},
Expectation: "foobar",
},
{
Name: "annotation",
WSO: workspaceObjects{
Pod: &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
hostIPAnnotation: "from-anno",
},
},
Status: v1.PodStatus{
HostIP: "",
},
},
},
Expectation: "from-anno",
},
{
Name: "annotation pod no status",
WSO: workspaceObjects{
Pod: &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
hostIPAnnotation: "from-anno",
},
},
},
},
Expectation: "from-anno",
},
{
Name: "spec and annotation",
WSO: workspaceObjects{
Pod: &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
hostIPAnnotation: "from-anno",
},
},
Status: v1.PodStatus{
HostIP: "from-spec",
},
},
},
Expectation: "from-spec",
},
}

for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
act := test.WSO.HostIP()
if diff := cmp.Diff(test.Expectation, act); diff != "" {
t.Errorf("unexpected hostIP (-want +got):\n%s", diff)
}
})
}
}

0 comments on commit 715ef34

Please sign in to comment.