Merge pull request flyteorg#44 from lyft/simpler-hive-plugin-init

Hive plugin: using primary cluster labels to execute queries and manage resource; simplify hive plugin init interface
eapolinario · Jan 13, 2020 · 88298ee · 88298ee
2 parents c64816d + 7bd4a45
commit 88298ee
Show file tree

Hide file tree

Showing 46 changed files with 513 additions and 387 deletions.
diff --git a/flyteplugins/Gopkg.lock b/flyteplugins/Gopkg.lock
diff --git a/flyteplugins/Makefile b/flyteplugins/Makefile
@@ -8,7 +8,7 @@ update_boilerplate:
 
 generate:
 	which pflags || (go get github.com/lyft/flytestdlib/cli/pflags)
-	which mockery || (go install github.com/lyft/flyteidl/vendor/github.com/vektra/mockery/cmd/mockery)
+	which mockery || (go install github.com/lyft/flyteplugins/vendor/github.com/vektra/mockery/cmd/mockery)
 	which enumer || (go get github.com/alvaroloes/enumer)
 	@go generate ./...
 

diff --git a/flyteplugins/go/tasks/logs/config.go b/flyteplugins/go/tasks/logs/config.go
@@ -30,4 +30,3 @@ func GetLogConfig() *LogConfig {
 func SetLogConfig(logConfig *LogConfig) error {
 	return logConfigSection.SetConfig(logConfig)
 }
-
diff --git a/flyteplugins/go/tasks/pluginmachinery/catalog/mocks/async_client.go b/flyteplugins/go/tasks/pluginmachinery/catalog/mocks/async_client.go
diff --git a/flyteplugins/go/tasks/pluginmachinery/catalog/mocks/client.go b/flyteplugins/go/tasks/pluginmachinery/catalog/mocks/client.go
diff --git a/flyteplugins/go/tasks/pluginmachinery/catalog/mocks/download_future.go b/flyteplugins/go/tasks/pluginmachinery/catalog/mocks/download_future.go
diff --git a/flyteplugins/go/tasks/pluginmachinery/catalog/mocks/download_response.go b/flyteplugins/go/tasks/pluginmachinery/catalog/mocks/download_response.go
diff --git a/flyteplugins/go/tasks/pluginmachinery/catalog/mocks/future.go b/flyteplugins/go/tasks/pluginmachinery/catalog/mocks/future.go
diff --git a/flyteplugins/go/tasks/pluginmachinery/catalog/mocks/upload_future.go b/flyteplugins/go/tasks/pluginmachinery/catalog/mocks/upload_future.go
diff --git a/flyteplugins/go/tasks/pluginmachinery/core/mocks/events_recorder.go b/flyteplugins/go/tasks/pluginmachinery/core/mocks/events_recorder.go
diff --git a/flyteplugins/go/tasks/pluginmachinery/core/mocks/kube_client.go b/flyteplugins/go/tasks/pluginmachinery/core/mocks/kube_client.go
diff --git a/flyteplugins/go/tasks/pluginmachinery/core/mocks/plugin.go b/flyteplugins/go/tasks/pluginmachinery/core/mocks/plugin.go
diff --git a/flyteplugins/go/tasks/pluginmachinery/core/mocks/resource_manager.go b/flyteplugins/go/tasks/pluginmachinery/core/mocks/resource_manager.go
diff --git a/flyteplugins/go/tasks/pluginmachinery/core/mocks/resource_registrar.go b/flyteplugins/go/tasks/pluginmachinery/core/mocks/resource_registrar.go
diff --git a/flyteplugins/go/tasks/pluginmachinery/core/mocks/secret_manager.go b/flyteplugins/go/tasks/pluginmachinery/core/mocks/secret_manager.go
diff --git a/flyteplugins/go/tasks/pluginmachinery/core/mocks/setup_context.go b/flyteplugins/go/tasks/pluginmachinery/core/mocks/setup_context.go
diff --git a/flyteplugins/go/tasks/pluginmachinery/core/mocks/task_execution_context.go b/flyteplugins/go/tasks/pluginmachinery/core/mocks/task_execution_context.go
diff --git a/flyteplugins/go/tasks/pluginmachinery/core/mocks/task_execution_id.go b/flyteplugins/go/tasks/pluginmachinery/core/mocks/task_execution_id.go
diff --git a/flyteplugins/go/tasks/pluginmachinery/core/mocks/task_execution_metadata.go b/flyteplugins/go/tasks/pluginmachinery/core/mocks/task_execution_metadata.go
diff --git a/flyteplugins/go/tasks/pluginmachinery/core/mocks/task_overrides.go b/flyteplugins/go/tasks/pluginmachinery/core/mocks/task_overrides.go
diff --git a/flyteplugins/go/tasks/pluginmachinery/core/mocks/task_reader.go b/flyteplugins/go/tasks/pluginmachinery/core/mocks/task_reader.go
diff --git a/flyteplugins/go/tasks/pluginmachinery/core/resource_manager.go b/flyteplugins/go/tasks/pluginmachinery/core/resource_manager.go
@@ -35,6 +35,7 @@ type ResourceRegistrar interface {
 
 // Resource Manager manages a single resource type, and each allocation is of size one
 type ResourceManager interface {
+	GetID() string
 	AllocateResource(ctx context.Context, namespace ResourceNamespace, allocationToken string) (AllocationStatus, error)
 	ReleaseResource(ctx context.Context, namespace ResourceNamespace, allocationToken string) error
 }
diff --git a/flyteplugins/go/tasks/pluginmachinery/flytek8s/config/config.go b/flyteplugins/go/tasks/pluginmachinery/flytek8s/config/config.go
@@ -1,7 +1,7 @@
 package config
 
 import (
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 
 	"github.com/lyft/flyteplugins/go/tasks/config"
 )
@@ -35,7 +35,7 @@ type K8sPluginConfig struct {
 	// Provide additional environment variable pairs that plugin authors will provide to containers
 	DefaultEnvVars map[string]string `json:"default-env-vars" pflag:"-,Additional environment variable that should be injected into every resource"`
 	// Provide additional environment variable pairs whose values resolve from the plugin's execution environment.
-	DefaultEnvVarsFromEnv map[string]string `json:"default-env-vars-from-env" pflag:",Additional environment variable that should be injected into every resource"`
+	DefaultEnvVarsFromEnv map[string]string `json:"default-env-vars-from-env" pflag:"-,Additional environment variable that should be injected into every resource"`
 	// Tolerations in the cluster that should be applied for a specific resource
 	// Currently we support simple resource based tolerations only
 	ResourceTolerations map[v1.ResourceName][]v1.Toleration `json:"resource-tolerations"  pflag:"-,Default tolerations to be applied for resource of type 'key'"`

diff --git a/flyteplugins/go/tasks/pluginmachinery/io/mocks/input_file_paths.go b/flyteplugins/go/tasks/pluginmachinery/io/mocks/input_file_paths.go
diff --git a/flyteplugins/go/tasks/pluginmachinery/io/mocks/input_reader.go b/flyteplugins/go/tasks/pluginmachinery/io/mocks/input_reader.go
Original file line number	Diff line number	Diff line change
Expand Up		@@ -30,4 +30,3 @@ func GetLogConfig() *LogConfig {
		func SetLogConfig(logConfig *LogConfig) error {
		return logConfigSection.SetConfig(logConfig)
		}