From 8fad9ca8407248cbf9a80d0a0f6c2f96ad6a0b29 Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Tue, 16 Mar 2021 17:50:12 -0700 Subject: [PATCH] Deprecate log plugins (#123) Signed-off-by: Haytham Abuelfutuh --- clients/go/coreutils/logs/cloudwatch.go | 45 -------------- clients/go/coreutils/logs/cloudwatch_test.go | 50 ---------------- clients/go/coreutils/logs/kubernetes.go | 26 --------- clients/go/coreutils/logs/kubernetes_test.go | 23 -------- clients/go/coreutils/logs/logs.go | 13 ----- clients/go/coreutils/logs/stackdriver.go | 40 ------------- clients/go/coreutils/logs/stackdriver_test.go | 58 ------------------- 7 files changed, 255 deletions(-) delete mode 100644 clients/go/coreutils/logs/cloudwatch.go delete mode 100644 clients/go/coreutils/logs/cloudwatch_test.go delete mode 100644 clients/go/coreutils/logs/kubernetes.go delete mode 100644 clients/go/coreutils/logs/kubernetes_test.go delete mode 100644 clients/go/coreutils/logs/logs.go delete mode 100644 clients/go/coreutils/logs/stackdriver.go delete mode 100644 clients/go/coreutils/logs/stackdriver_test.go diff --git a/clients/go/coreutils/logs/cloudwatch.go b/clients/go/coreutils/logs/cloudwatch.go deleted file mode 100644 index 75bca9c8ec..0000000000 --- a/clients/go/coreutils/logs/cloudwatch.go +++ /dev/null @@ -1,45 +0,0 @@ -package logs - -import ( - "fmt" - "strings" - - "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" -) - -type cloudwatchLogPlugin struct { - region string - groupName string -} - -func (s cloudwatchLogPlugin) GetTaskLog(podName, namespace, containerName, containerID, logName string) (core.TaskLog, error) { - - // Container IDs are prefixed with docker://, cri-o://, etc. which is stripped by fluentd before pushing to a log - // stream. Therefore, we must also strip the prefix. - // Also, container names are - stripDelimiter := "://" - if split := strings.Split(containerID, stripDelimiter); len(split) > 1 { - containerID = split[1] - } - - return core.TaskLog{ - Uri: fmt.Sprintf( - "https://console.aws.amazon.com/cloudwatch/home?region=%s#logEventViewer:group=%s;stream=var.log.containers.%s_%s_%s-%s.log", - s.region, - s.groupName, - podName, - namespace, - containerName, - containerID), - Name: logName, - MessageFormat: core.TaskLog_JSON, - }, nil -} - -// Deprecated: Please use NewTemplateLogPlugin from github.com/lyft/flyteplugins/go/tasks/pluginmachinery/tasklog instead. -func NewCloudwatchLogPlugin(region, groupName string) LogPlugin { - return &cloudwatchLogPlugin{ - region: region, - groupName: groupName, - } -} diff --git a/clients/go/coreutils/logs/cloudwatch_test.go b/clients/go/coreutils/logs/cloudwatch_test.go deleted file mode 100644 index 9896ca2171..0000000000 --- a/clients/go/coreutils/logs/cloudwatch_test.go +++ /dev/null @@ -1,50 +0,0 @@ -package logs - -import ( - "testing" - - "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" - "github.com/stretchr/testify/assert" -) - -func TestCloudwatchLogMaker_CriO(t *testing.T) { - p := NewCloudwatchLogPlugin("us-east-1", "/flyte-production/kubernetes") - tl, err := p.GetTaskLog( - "f-uuid-driver", - "flyteexamples-production", - "spark-kubernetes-driver", - "cri-o://abc", - "main_logs") - assert.NoError(t, err) - assert.Equal(t, tl.GetName(), "main_logs") - assert.Equal(t, tl.GetMessageFormat(), core.TaskLog_JSON) - assert.Equal(t, "https://console.aws.amazon.com/cloudwatch/home?region=us-east-1#logEventViewer:group=/flyte-production/kubernetes;stream=var.log.containers.f-uuid-driver_flyteexamples-production_spark-kubernetes-driver-abc.log", tl.Uri) -} - -func TestCloudwatchLogMaker_Docker(t *testing.T) { - p := NewCloudwatchLogPlugin("us-east-1", "/flyte-staging/kubernetes") - tl, err := p.GetTaskLog( - "f-uuid-driver", - "flyteexamples-staging", - "spark-kubernetes-driver", - "docker://abc123", - "main_logs") - assert.NoError(t, err) - assert.Equal(t, tl.GetName(), "main_logs") - assert.Equal(t, tl.GetMessageFormat(), core.TaskLog_JSON) - assert.Equal(t, "https://console.aws.amazon.com/cloudwatch/home?region=us-east-1#logEventViewer:group=/flyte-staging/kubernetes;stream=var.log.containers.f-uuid-driver_flyteexamples-staging_spark-kubernetes-driver-abc123.log", tl.Uri) -} - -func TestCloudwatchLogMaker_NoPrefix(t *testing.T) { - p := NewCloudwatchLogPlugin("us-east-1", "/flyte-staging/kubernetes") - tl, err := p.GetTaskLog( - "f-uuid-driver", - "flyteexamples-staging", - "spark-kubernetes-driver", - "123abc", - "main_logs") - assert.NoError(t, err) - assert.Equal(t, tl.GetName(), "main_logs") - assert.Equal(t, tl.GetMessageFormat(), core.TaskLog_JSON) - assert.Equal(t, "https://console.aws.amazon.com/cloudwatch/home?region=us-east-1#logEventViewer:group=/flyte-staging/kubernetes;stream=var.log.containers.f-uuid-driver_flyteexamples-staging_spark-kubernetes-driver-123abc.log", tl.Uri) -} diff --git a/clients/go/coreutils/logs/kubernetes.go b/clients/go/coreutils/logs/kubernetes.go deleted file mode 100644 index 5f0e1058a9..0000000000 --- a/clients/go/coreutils/logs/kubernetes.go +++ /dev/null @@ -1,26 +0,0 @@ -package logs - -import ( - "fmt" - - "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" -) - -type kubernetesLogPlugin struct { - k8sURL string -} - -func (s kubernetesLogPlugin) GetTaskLog(podName, namespace, containerName, containerID, logName string) (core.TaskLog, error) { - return core.TaskLog{ - Uri: fmt.Sprintf("%s/#!/log/%s/%s/pod?namespace=%s", s.k8sURL, namespace, podName, namespace), - Name: logName, - MessageFormat: core.TaskLog_UNKNOWN, - }, nil -} - -// Deprecated: Please use NewTemplateLogPlugin from github.com/lyft/flyteplugins/go/tasks/pluginmachinery/tasklog instead. -func NewKubernetesLogPlugin(k8sURL string) LogPlugin { - return &kubernetesLogPlugin{ - k8sURL: k8sURL, - } -} diff --git a/clients/go/coreutils/logs/kubernetes_test.go b/clients/go/coreutils/logs/kubernetes_test.go deleted file mode 100644 index 277c3c004c..0000000000 --- a/clients/go/coreutils/logs/kubernetes_test.go +++ /dev/null @@ -1,23 +0,0 @@ -package logs - -import ( - "testing" - - "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" - "github.com/stretchr/testify/assert" -) - -func TestKubernetesLogMaker(t *testing.T) { - p := NewKubernetesLogPlugin("https://dashboard.k8s.net") - tl, err := p.GetTaskLog( - "flyteexamples-development-task-name", - "flyteexamples-development", - "ignore", - "ignore", - "main_logs", - ) - assert.NoError(t, err) - assert.Equal(t, tl.GetName(), "main_logs") - assert.Equal(t, tl.GetMessageFormat(), core.TaskLog_UNKNOWN) - assert.Equal(t, "https://dashboard.k8s.net/#!/log/flyteexamples-development/flyteexamples-development-task-name/pod?namespace=flyteexamples-development", tl.Uri) -} diff --git a/clients/go/coreutils/logs/logs.go b/clients/go/coreutils/logs/logs.go deleted file mode 100644 index 8355aa216a..0000000000 --- a/clients/go/coreutils/logs/logs.go +++ /dev/null @@ -1,13 +0,0 @@ -// Contains interface definitions and helper functions for log plugins - -package logs - -import ( - "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" -) - -// Deprecated: Please use Plugin interface from github.com/lyft/flyteplugins/go/tasks/pluginmachinery/tasklog instead. -type LogPlugin interface { - // Generates a TaskLog object given necessary computation information - GetTaskLog(podName, namespace, containerName, containerID, logName string) (core.TaskLog, error) -} diff --git a/clients/go/coreutils/logs/stackdriver.go b/clients/go/coreutils/logs/stackdriver.go deleted file mode 100644 index 4c45416bab..0000000000 --- a/clients/go/coreutils/logs/stackdriver.go +++ /dev/null @@ -1,40 +0,0 @@ -package logs - -import ( - "fmt" - - "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" -) - -// TL;DR Log links in Stackdriver for configured GCP project and log Resource -// -// This is a simple stackdriver log plugin that creates a preformatted log link for a given project and logResource -// using resource.labels.pod_name as advancedFilter -type stackdriverLogPlugin struct { - // the name of the project in GCP that the logs are being published under - gcpProject string - // The Log resource name for which the logs are published under - logResource string -} - -func (s *stackdriverLogPlugin) GetTaskLog(podName, namespace, containerName, containerID, logName string) (core.TaskLog, error) { - return core.TaskLog{ - Uri: fmt.Sprintf( - "https://console.cloud.google.com/logs/viewer?project=%s&angularJsUrl=%%2Flogs%%2Fviewer%%3Fproject%%3D%s&resource=%s&advancedFilter=resource.labels.pod_name%%3D%s", - s.gcpProject, - s.gcpProject, - s.logResource, - podName, - ), - Name: logName, - MessageFormat: core.TaskLog_JSON, - }, nil -} - -// Deprecated: Please use NewTemplateLogPlugin from github.com/lyft/flyteplugins/go/tasks/pluginmachinery/tasklog instead. -func NewStackdriverLogPlugin(gcpProject, logResource string) LogPlugin { - return &stackdriverLogPlugin{ - gcpProject: gcpProject, - logResource: logResource, - } -} diff --git a/clients/go/coreutils/logs/stackdriver_test.go b/clients/go/coreutils/logs/stackdriver_test.go deleted file mode 100644 index 84b91f0004..0000000000 --- a/clients/go/coreutils/logs/stackdriver_test.go +++ /dev/null @@ -1,58 +0,0 @@ -package logs - -import ( - "reflect" - "testing" - - "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" -) - -func Test_stackdriverLogPlugin_GetTaskLog(t *testing.T) { - type fields struct { - gcpProject string - logResource string - } - type args struct { - podName string - namespace string - containerName string - containerID string - logName string - } - tests := []struct { - name string - fields fields - args args - want core.TaskLog - wantErr bool - }{ - { - "podName-proj1", - fields{gcpProject: "test-gcp-project", logResource: "aws_ec2_instance"}, - args{podName: "podName"}, - core.TaskLog{Uri: "https://console.cloud.google.com/logs/viewer?project=test-gcp-project&angularJsUrl=%2Flogs%2Fviewer%3Fproject%3Dtest-gcp-project&resource=aws_ec2_instance&advancedFilter=resource.labels.pod_name%3DpodName", MessageFormat: core.TaskLog_JSON}, false, - }, - { - "podName2-proj2", - fields{gcpProject: "proj2", logResource: "res1"}, - args{podName: "long-pod-name-xyyyx"}, - core.TaskLog{Uri: "https://console.cloud.google.com/logs/viewer?project=proj2&angularJsUrl=%2Flogs%2Fviewer%3Fproject%3Dproj2&resource=res1&advancedFilter=resource.labels.pod_name%3Dlong-pod-name-xyyyx", MessageFormat: core.TaskLog_JSON}, false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := &stackdriverLogPlugin{ - gcpProject: tt.fields.gcpProject, - logResource: tt.fields.logResource, - } - got, err := s.GetTaskLog(tt.args.podName, tt.args.namespace, tt.args.containerName, tt.args.containerID, tt.args.logName) - if (err != nil) != tt.wantErr { - t.Errorf("stackdriverLogPlugin.GetTaskLog() error = %v, wantErr %v", err, tt.wantErr) - return - } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("stackdriverLogPlugin.GetTaskLog() = %v, want %v", got, tt.want) - } - }) - } -}