Update CI post migration (#95)

* Migrate off of travis

* move proto test to actions

* wip

* wip

* wip

* wip

* install git

* wip

* wip

* update script

* try uploading

* wip

* wip

* add the rest of the steps

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* Update pull_request.yml

* Update pull_request.yml

* Update pull_request.yml

* Ignore generated code

* Update pull_request.yml

* Update pull_request.yml

* wip

* regenerate

* fix workdir

* wip

* wip

* wip

* wip

* wip

* fixing lint

* lint

* Add NPM Publish Step

* cleanup

* Update setup.py

* update maintainer emails

flyteidl/.github/workflows/npmpublish.yaml

@@ -0,0 +1,20 @@
+name: Publish NPM Package
+
+on:
+  release:
+    types: [created]
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      # Setup .npmrc file to publish to npm
+      - uses: actions/setup-node@v1
+        with:
+          node-version: "12.x"
+          registry-url: "https://registry.npmjs.org"
+      - run: npm install
+      - run: npm publish
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

flyteidl/.github/workflows/pythonpublish.yaml

@@ -1,4 +1,4 @@
-name: Upload Python Package
+name: Upload PyPi Package
 
 on:
   release:
@@ -12,7 +12,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v1
         with:
-          python-version: '3.x'
+          python-version: "3.x"
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip

flyteidl/.github/workflows/verification.yml

@@ -0,0 +1,236 @@
+name: Verification Tests
+
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+
+jobs:
+  tests-lint:
+    name: Run tests and lint
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: "0"
+      - name: Unit Tests
+        uses: cedrickring/[email protected]
+        env:
+          GO111MODULE: "on"
+        with:
+          args: make install && make test_unit
+      - name: Lint
+        uses: cedrickring/[email protected]
+        env:
+          GO111MODULE: "on"
+        with:
+          args: make install && make lint
+  generate-protos:
+    runs-on: ubuntu-latest
+    name: Generate Protos
+    container:
+      image: lyft/protocgenerator:8167e11d3b3439373c2f033080a4b550078884a2
+      options: --cpus 1
+      volumes:
+        - /github/workspace:/defs
+      env:
+        REPO_BLOB_SHA: master
+        PROJECT_ANNOTATION_PREFIX: flyte.interface
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: "0"
+      - name: "Clean Generated"
+        run: rm -rf ./gen
+      # GO Protos
+      - name: Proto-Service-Go
+        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/service --with_gateway -l go --go_source_relative
+      - name: Proto-Admin-Go
+        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/admin --with_gateway -l go --go_source_relative --validate_out
+      - name: Proto-Core-Go
+        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/core --with_gateway -l go --go_source_relative --validate_out
+      - name: Proto-Event-Go
+        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/event --with_gateway -l go --go_source_relative --validate_out
+      - name: Proto-Plugins-Go
+        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/plugins -l go --go_source_relative --validate_out
+      - name: Proto-Datacatalog-Go
+        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/datacatalog -l go --go_source_relative --validate_out
+
+      # Python
+      - name: Proto-Service-Python
+        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/service -l python
+      - name: Proto-Admin-Python
+        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/admin -l python
+      - name: Proto-Core-Python
+        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/core -l python
+      - name: Proto-Event-Python
+        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/event -l python
+      - name: Proto-Plugins-Python
+        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/plugins -l python
+      - name: Proto-Datacatalog-Python
+        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/datacatalog -l python
+
+      # Cpp
+      - name: Proto-Service-Cpp
+        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/service -l cpp
+      - name: Proto-Admin-Cpp
+        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/admin -l cpp
+      - name: Proto-Core-Cpp
+        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/core -l cpp
+      - name: Proto-Event-Cpp
+        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/event -l cpp
+      - name: Proto-Plugins-Cpp
+        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/plugins -l cpp
+      - name: Proto-Datacatalog-Cpp
+        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/datacatalog -l cpp
+
+      # Java
+      - name: Proto-Service-Java
+        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/service -l java
+      - name: Proto-Admin-Java
+        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/admin -l java
+      - name: Proto-Core-Java
+        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/core -l java
+      - name: Proto-Event-Java
+        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/event -l java
+      - name: Proto-Plugins-Java
+        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/plugins -l java
+      - name: Proto-Datacatalog-Java
+        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/datacatalog -l java
+
+      # Docs
+      - name: Docs-Service
+        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/service -l protodoc
+      - name: Docs-Admin
+        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/admin -l protodoc
+      - name: Docs-Core
+        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/core -l protodoc
+      - name: Docs-Event
+        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/event -l protodoc
+      - name: Docs-Plugins
+        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/plugins -l protodoc
+      - name: Docs-Datacatalog
+        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/datacatalog -l protodoc
+
+      - uses: actions/upload-artifact@master
+        with:
+          name: generated
+          path: ./gen
+
+  generate-admin-swagger-code:
+    runs-on: ubuntu-latest
+    name: Generate Admin Swagger Code
+    needs: [generate-protos]
+    container:
+      image: lyft/protocgenerator:8167e11d3b3439373c2f033080a4b550078884a2
+      options: --cpus 1
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: "0"
+      - name: "Clean Generated"
+        run: rm -rf ./gen
+      - uses: actions/download-artifact@master
+        with:
+          name: generated
+          path: ./gen
+      - run: ls gen/pb-go/flyteidl/service/
+      - run: cp -R gen/pb-go/flyteidl/service/ ../
+      - run: ls ../service/
+      # Open API 2
+      - name: OpenAPI-Binary
+        run: go-bindata -pkg service -o ../service/openapi.go -prefix ../service/ -modtime 1562572800 ../service/admin.swagger.json
+      - run: rm -rf gen/pb-go/flyteidl/service
+      - run: mkdir -p gen/pb-go/flyteidl/service
+      - run: cp -R ../service gen/pb-go/flyteidl/
+      - run: ls gen/pb-go/flyteidl/
+      - run: ls gen/pb-go/flyteidl/service/
+      - uses: actions/upload-artifact@master
+        with:
+          name: generated
+          path: ./gen
+
+  generate-js-code:
+    runs-on: ubuntu-latest
+    name: Generate JS Code
+    needs: [generate-admin-swagger-code]
+    container:
+      image: schottra/docker-protobufjs:v0.0.2
+      options: --cpus 1
+      volumes:
+        - /github/workspace:/defs
+      env:
+        REPO_BLOB_SHA: master
+        PROJECT_ANNOTATION_PREFIX: flyte.interface
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: "0"
+      - name: "Clean Generated"
+        run: rm -rf ./gen
+      - uses: actions/download-artifact@master
+        with:
+          name: generated
+          path: ./gen
+      - run: mkdir -p /defs
+      - run: mkdir -p /gen
+      - run: cp -R . /defs
+      - run: node /code/generate-protobuf.js --module-name flyteidl -d protos/flyteidl/core  -d protos/flyteidl/event -d protos/flyteidl/admin -d protos/flyteidl/service  -- --root flyteidl -t static-module -w es6 --no-delimited --force-long --no-convert -p /defs/protos
+      - run: ls /defs/gen/
+      - uses: actions/upload-artifact@master
+        with:
+          name: generated
+          path: /defs/gen
+
+  generate-swagger-code:
+    runs-on: ubuntu-latest
+    name: Generate Swagger Code
+    needs: [generate-js-code]
+    container:
+      image: docker.io/lyft/swagger-codegen-cli:dc5ce6ec6d7d4d980fa882d6bd13a83cba3be3c3
+      options: --cpus 1
+      volumes:
+        - /github/workspace:/defs
+      env:
+        REPO_BLOB_SHA: master
+        PROJECT_ANNOTATION_PREFIX: flyte.interface
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: "0"
+      - name: "Clean Generated"
+        run: rm -rf ./gen
+      - uses: actions/download-artifact@master
+        with:
+          name: generated
+          path: ./gen
+      - run: java -jar /opt/swagger-codegen-cli/swagger-codegen-cli.jar generate -i ./gen/pb-go/flyteidl/service/admin.swagger.json -l go -o ./gen/pb-go/flyteidl/service/flyteadmin --additional-properties=packageName=flyteadmin
+      - run: java -jar /opt/swagger-codegen-cli/swagger-codegen-cli.jar generate -i ./gen/pb-go/flyteidl/service/admin.swagger.json -l python -o ./gen/pb_python/flyteidl/service/flyteadmin --additional-properties=packageName=flyteadmin
+      - uses: actions/upload-artifact@master
+        with:
+          name: generated
+          path: ./gen
+
+  test-proto-changes:
+    runs-on: ubuntu-latest
+    name: Test Proto Changes
+    needs: [generate-swagger-code]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: "0"
+      - name: "Clean Generated"
+        run: rm -rf ./gen
+      - uses: actions/download-artifact@master
+        with:
+          name: generated
+          path: ./gen
+      - name: Test
+        run: ./scripts/test_diff.sh

flyteidl/.golangci.yml

@@ -6,6 +6,7 @@
 run:
   skip-dirs:
     - pkg/client
+    - gen
 
 linters:
   disable-all: true

flyteidl/clients/go/admin/client.go

@@ -4,6 +4,11 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strings"
+	"sync"
+
 	"github.com/coreos/go-oidc"
 	grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
 	grpc_retry "github.com/grpc-ecosystem/go-grpc-middleware/retry"
@@ -14,10 +19,6 @@ import (
 	"golang.org/x/oauth2/clientcredentials"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
-	"io/ioutil"
-	"net/http"
-	"strings"
-	"sync"
 )
 
 var (
@@ -97,7 +98,7 @@ func getAuthenticationDialOption(ctx context.Context, cfg Config) (grpc.DialOpti
 	secret := strings.TrimSpace(string(secretBytes))
 
 	ccConfig := clientcredentials.Config{
-		ClientID:     cfg.ClientId,
+		ClientID:     cfg.ClientID,
 		ClientSecret: secret,
 		TokenURL:     tokenURL,
 		Scopes:       cfg.Scopes,
@@ -117,7 +118,7 @@ func NewAdminConnection(ctx context.Context, cfg Config) (*grpc.ClientConn, erro
 		creds := credentials.NewClientTLSFromCert(nil, "")
 		opts = append(opts, grpc.WithTransportCredentials(creds))
 		if cfg.UseAuth {
-			logger.Infof(ctx, "Instantiating a token source to authenticate against Admin, ID: %s", cfg.ClientId)
+			logger.Infof(ctx, "Instantiating a token source to authenticate against Admin, ID: %s", cfg.ClientID)
 			jwtDialOption, err := getAuthenticationDialOption(ctx, cfg)
 			if err != nil {
 				return nil, err

flyteidl/clients/go/admin/config.go

@@ -22,7 +22,7 @@ type Config struct {
 	// Auth can only be used if also running with a secure connection. If UseInsecureConnection is set to true, none
 	// of the following options will even be referenced.
 	UseAuth              bool     `json:"useAuth" pflag:",Whether or not to try to authenticate with options below"`
-	ClientId             string   `json:"clientId" pflag:",Client ID"`
+	ClientID             string   `json:"clientId" pflag:",Client ID"`
 	ClientSecretLocation string   `json:"clientSecretLocation" pflag:",File containing the client secret"`
 	Scopes               []string `json:"scopes" pflag:",List of scopes to request"`
 

flyteidl/clients/go/admin/token_source.go

@@ -1,7 +1,10 @@
 package admin
 
-import "golang.org/x/oauth2"
-import "context"
+import (
+	"context"
+
+	"golang.org/x/oauth2"
+)
 
 // This class is here because we cannot use the normal "github.com/grpc/grpc-go/credentials/oauth" package to satisfy
 // the credentials.PerRPCCredentials interface. This is because we want to be able to support a different 'header'

flyteidl/clients/go/admin/token_source_test.go

@@ -5,8 +5,8 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"golang.org/x/oauth2"
 )
-import "golang.org/x/oauth2"
 
 type DummyTestTokenSource struct {
 	oauth2.TokenSource