Migrate to go module and remove old Hive integration tests (#36)

migrate to go mod and temporarily remove some broken tests

Makefile

@@ -1,15 +1,11 @@
-export GO111MODULE=off
 export REPOSITORY=flyteplugins
 include boilerplate/lyft/golang_test_targets/Makefile
 
 .PHONY: update_boilerplate
 update_boilerplate:
 	@boilerplate/update.sh
 
-generate:
-	which pflags || (go get github.com/lyft/flytestdlib/cli/pflags)
-	which mockery || (go install github.com/lyft/flyteplugins/vendor/github.com/vektra/mockery/cmd/mockery)
-	which enumer || (go get github.com/alvaroloes/enumer)
+generate: download_tooling
 	@go generate ./...
 
 clean:

boilerplate/lyft/golang_support_tools/go.mod

@@ -0,0 +1,12 @@
+module github.com/lyft/boilerplate
+
+go 1.13
+
+require (
+	github.com/alvaroloes/enumer v1.1.2
+	github.com/golangci/golangci-lint v1.22.2
+	github.com/lyft/flytestdlib v0.2.31
+	github.com/vektra/mockery v0.0.0-20181123154057-e78b021dcbb5
+)
+
+replace github.com/vektra/mockery => github.com/enghabu/mockery v0.0.0-20191009061720-9d0c8670c2f0

boilerplate/lyft/golang_support_tools/tools.go

@@ -0,0 +1,10 @@
+// +build tools
+
+package tools
+
+import (
+	_ "github.com/golangci/golangci-lint/cmd/golangci-lint"
+	_ "github.com/lyft/flytestdlib/cli/pflags"
+	_ "github.com/vektra/mockery/cmd/mockery"
+	_ "github.com/alvaroloes/enumer"
+)

boilerplate/lyft/golang_test_targets/Makefile

@@ -3,26 +3,30 @@
 #
 # TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst
 
+
+.PHONY: download_tooling
+download_tooling: #download dependencies (including test deps) for the package
+	@boilerplate/lyft/golang_test_targets/download_tooling.sh
+
 .PHONY: lint
-lint: #lints the package for common code smells
-	which golangci-lint || GO111MODULE=on go install github.com/golangci/golangci-lint/cmd/golangci-lint
-	# Calling lint twice here is a hack. The first call seem to fail when internally calling `go list...`
-    # However, that call seem to have some effects (e.g. https://github.com/golang/go/issues/29452) which, for some
-    # reason, allows the subsequent calls to succeed.
-    # TODO: Evaluate whether this is still a problem after moving admin dependency system to go modules..
-#     GO111MODULE=off GL_DEBUG=linters_output,loader,env golangci-lint run --exclude deprecated -v || true
-	GO111MODULE=off GL_DEBUG=linters_output,loader,env golangci-lint run --deadline=5m --exclude deprecated -v
+lint: download_tooling #lints the package for common code smells
+	GL_DEBUG=linters_output,env golangci-lint run --deadline=5m --exclude deprecated -v
 
 # If code is failing goimports linter, this will fix.
 # skips 'vendor'
 .PHONY: goimports
 goimports:
 	@boilerplate/lyft/golang_test_targets/goimports
 
+.PHONY: mod_download
+mod_download: #download dependencies (including test deps) for the package
+	go mod download
+
 .PHONY: install
-install: #download dependencies (including test deps) for the package
-	which dep || (curl "https://raw.githubusercontent.com/golang/dep/master/install.sh" | sh)
-	dep ensure --vendor-only
+install: download_tooling mod_download
+
+.PHONY: show
+show: go list -m all
 
 .PHONY: test_unit
 test_unit:
@@ -38,4 +42,5 @@ test_unit_cover:
 
 .PHONY: test_unit_visual
 test_unit_visual:
-	go test ./... -coverprofile /tmp/cover.out -covermode=count; go tool cover -html=/tmp/cover.out
+	go test ./... -coverprofile /tmp/cover.out -covermode=count; go tool cover -html=/tmp/cover.out 
+

boilerplate/lyft/golang_test_targets/Readme.rst

@@ -1,7 +1,7 @@
 Golang Test Targets
 ~~~~~~~~~~~~~~~~~~~
 
-Provides an ``install`` make target that uses ``dep`` install golang dependencies.
+Provides an ``install`` make target that uses ``go mod`` to install golang dependencies.
 
 Provides a ``lint`` make target that uses golangci to lint your code.
 
@@ -17,7 +17,7 @@ Provides a ``test_benchmark`` target for benchmark tests.
 
 Add ``lyft/golang_test_targets`` to your ``boilerplate/update.cfg`` file.
 
-Make sure you're using ``dep`` for dependency management.
+Make sure you're using ``go mod`` for dependency management.
 
 Provide a ``.golangci`` configuration (the lint target requires it).
 

boilerplate/lyft/golang_test_targets/download_tooling.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# Everything in this file needs to be installed outside of current module
+# The reason we cannot turn off module entirely and install is that we need the replace statement in go.mod
+# because we are installing a mockery fork. Turning it off would result installing the original not the fork.
+# We also want to version all the other tools.  We also want to be able to run go mod tidy without removing the version
+# pins.  To facilitate this, we're maintaining two sets of go.mod/sum files - the second one only for tooling.  This is
+# the same approach that go 1.14 will take as well.
+# See:
+#   https://github.com/lyft/flyte/issues/129
+#   https://github.com/golang/go/issues/30515 for some background context
+#   https://github.com/go-modules-by-example/index/blob/5ec250b4b78114a55001bd7c9cb88f6e07270ea5/010_tools/README.md
+
+set -e
+
+# List of tools to go get
+# In the format of "<cli>:<package>" or ":<package>" if no cli
+tools=(
+  "github.com/vektra/mockery/cmd/mockery"
+  "github.com/lyft/flytestdlib/cli/pflags"
+  "github.com/golangci/golangci-lint/cmd/golangci-lint"
+  "github.com/alvaroloes/enumer"
+)
+
+tmp_dir=$(mktemp -d -t gotooling-XXX)
+echo "Using temp directory ${tmp_dir}"
+cp -R boilerplate/lyft/golang_support_tools/* $tmp_dir
+pushd "$tmp_dir"
+
+for tool in "${tools[@]}"
+do
+    echo "Installing ${tool}"
+    GO111MODULE=on go install $tool
+done
+
+popd

boilerplate/update.cfg

@@ -1,2 +1,3 @@
 lyft/golang_test_targets
 lyft/golangci_file
+lyft/golang_support_tools

boilerplate/update.sh

@@ -34,6 +34,7 @@ if [ -z "$REPOSITORY" ]; then
 fi
 
 while read directory; do
+  # TODO: Skip empty lines, whitespace only lines, and comment lines
   echo "***********************************************************************************"
   echo "$directory is configured in update.cfg."
   echo "-----------------------------------------------------------------------------------"

go.mod

@@ -0,0 +1,41 @@
+module github.com/lyft/flyteplugins
+
+go 1.13
+
+require (
+	github.com/GoogleCloudPlatform/spark-on-k8s-operator v0.1.3
+	github.com/aws/aws-sdk-go v1.28.9
+	github.com/coocood/freecache v1.1.0
+	github.com/go-test/deep v1.0.5
+	github.com/golang/protobuf v1.3.2
+	github.com/hashicorp/golang-lru v0.5.4
+	github.com/json-iterator/go v1.1.9 // indirect
+	github.com/lyft/flyteidl v0.17.0
+	github.com/lyft/flytestdlib v0.3.0
+	github.com/magiconair/properties v1.8.1
+	github.com/mitchellh/mapstructure v1.1.2
+	github.com/pkg/errors v0.9.1
+	github.com/prometheus/client_golang v1.3.0
+	github.com/spf13/pflag v1.0.5
+	github.com/stretchr/testify v1.4.0
+	golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa
+	google.golang.org/grpc v1.26.0
+	k8s.io/api v0.17.2
+	k8s.io/apimachinery v0.17.2
+	k8s.io/client-go v11.0.0+incompatible
+	k8s.io/klog v1.0.0 // indirect
+	k8s.io/utils v0.0.0-20200122174043-1e243dd1a584 // indirect
+	sigs.k8s.io/controller-runtime v0.4.0
+)
+
+// Pin the version of client-go to something that's compatible with katrogan's fork of api and apimachinery
+// Type the following
+//   replace k8s.io/client-go => k8s.io/client-go kubernetes-1.16.2
+// and it will be replaced with the 'sha' variant of the version
+
+replace (
+	github.com/GoogleCloudPlatform/spark-on-k8s-operator => github.com/lyft/spark-on-k8s-operator v0.1.3
+	k8s.io/api => github.com/lyft/api v0.0.0-20191031200350-b49a72c274e0
+	k8s.io/apimachinery => github.com/lyft/apimachinery v0.0.0-20191031200210-047e3ea32d7f
+	k8s.io/client-go => k8s.io/client-go v0.0.0-20191016111102-bec269661e48
+)