Implement DataProxy CreateUploadLocation API (flyteorg#373)

* Implement DataProxy CreateUploadLocation API Signed-off-by: Haytham Abuelfutuh <[email protected]> * Regenerate Signed-off-by: Haytham Abuelfutuh <[email protected]> * fix unit test Signed-off-by: Haytham Abuelfutuh <[email protected]> * goimports Signed-off-by: Haytham Abuelfutuh <[email protected]> * Use ShardedStorage Signed-off-by: Haytham Abuelfutuh <[email protected]> * Add unit tests Signed-off-by: Haytham Abuelfutuh <[email protected]> * Implement a plugin registry and injection pattern Signed-off-by: Haytham Abuelfutuh <[email protected]> * Migrate WorkflowExecutor to the new Plugin Registry Signed-off-by: Haytham Abuelfutuh <[email protected]> * Update flyteidl to lastest commit Signed-off-by: Haytham Abuelfutuh <[email protected]> * Update to latest stow and stdlib commits Signed-off-by: Haytham Abuelfutuh <[email protected]> * Using released versions Signed-off-by: Haytham Abuelfutuh <[email protected]> * Using released versions Signed-off-by: Haytham Abuelfutuh <[email protected]> * bump Signed-off-by: Haytham Abuelfutuh <[email protected]> * Trying Go 1.17 Signed-off-by: Haytham Abuelfutuh <[email protected]> * Update golang.org/x/tools Signed-off-by: Haytham Abuelfutuh <[email protected]> * bump Signed-off-by: Haytham Abuelfutuh <[email protected]> * Update x/tools in boilerplate Signed-off-by: Haytham Abuelfutuh <[email protected]> * Regenerate Signed-off-by: Haytham Abuelfutuh <[email protected]> * Update go mod Signed-off-by: Haytham Abuelfutuh <[email protected]> * Cleanup & lint Signed-off-by: Haytham Abuelfutuh <[email protected]> * Ineffective assignment Signed-off-by: Haytham Abuelfutuh <[email protected]> * Upgrade go for lint Signed-off-by: Haytham Abuelfutuh <[email protected]> * Upgrade all checks' versions to upgrade-go branch Signed-off-by: Haytham Abuelfutuh <[email protected]> * Update to merged changes Signed-off-by: Haytham Abuelfutuh <[email protected]> * Use latest flytestdlib Signed-off-by: Haytham Abuelfutuh <[email protected]> * Regenerate Signed-off-by: Haytham Abuelfutuh <[email protected]> * Remove authorizedUris from cmdline flags Signed-off-by: Haytham Abuelfutuh <[email protected]> * Code comments Signed-off-by: Haytham Abuelfutuh <[email protected]>
eapolinario · Mar 29, 2022 · 6a62741 · 6a62741
1 parent f532007
commit 6a62741
Show file tree

Hide file tree

Showing 30 changed files with 824 additions and 372 deletions.
diff --git a/flyteadmin/.github/workflows/checks.yml b/flyteadmin/.github/workflows/checks.yml
@@ -26,21 +26,21 @@ jobs:
 
   endtoend:
     name: End2End Test
-    needs: [docker-build]
+    needs: [ docker-build ]
     uses: flyteorg/flytetools/.github/workflows/end2end.yml@master
     with:
       priorities: "P0"
       cache_key: ${{ needs.docker-build.outputs.cache_key }}
 
   integration:
     name: Integration Test
-    needs: [docker-build]
+    needs: [ docker-build ]
     uses: flyteorg/flytetools/.github/workflows/integration.yml@master
     with:
       cache_key: ${{ needs.docker-build.outputs.cache_key }}
 
   generate:
-    name: Check Go Gennerate
+    name: Check Go Generate
     uses: flyteorg/flytetools/.github/workflows/go_generate.yml@master
 
   bump_version:

diff --git a/flyteadmin/Dockerfile b/flyteadmin/Dockerfile
@@ -3,7 +3,7 @@
 # 
 # TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst
 
-FROM golang:1.17.1-alpine3.14 as builder
+FROM golang:1.18-alpine3.15 as builder
 RUN apk add git openssh-client make curl
 
 # COPY only the go mod files for efficient caching
@@ -24,7 +24,7 @@ RUN make linux_compile
 ENV PATH="/artifacts:${PATH}"
 
 # This will eventually move to centurylink/ca-certs:latest for minimum possible image size
-FROM alpine:3.14
+FROM alpine:3.15
 LABEL org.opencontainers.image.source https://github.com/flyteorg/flyteadmin
 
 COPY --from=builder /artifacts /bin

diff --git a/flyteadmin/auth/config/config.go b/flyteadmin/auth/config/config.go
@@ -15,23 +15,30 @@ import (
 type SecretName = string
 
 const (
+	// SecretNameOIdCClientSecret defines the default OIdC client secret name to use.
 	// #nosec
-	// Default OIdC client secret name to use.
 	SecretNameOIdCClientSecret SecretName = "oidc_client_secret"
+
+	// SecretNameCookieHashKey defines the default cookie hash key secret name to use.
 	// #nosec
 	SecretNameCookieHashKey SecretName = "cookie_hash_key"
+
+	// SecretNameCookieBlockKey defines the default cookie block key secret name to use.
 	// #nosec
 	SecretNameCookieBlockKey SecretName = "cookie_block_key"
+
+	// SecretNameClaimSymmetricKey must be a base64 encoded secret of exactly 32 bytes
 	// #nosec
-	// Base64 encoded secret of exactly 32 bytes
 	SecretNameClaimSymmetricKey SecretName = "claim_symmetric_key"
+
+	// SecretNameTokenSigningRSAKey is the privateKey used to sign JWT tokens. The default strategy uses RS256 (RSA Signature with SHA-256)
 	// #nosec
-	// PrivateKey is used to sign JWT tokens. The default strategy uses RS256 (RSA Signature with SHA-256)
 	SecretNameTokenSigningRSAKey SecretName = "token_rsa_key.pem"
-	// #nosec
-	// PrivateKey that was used to sign old JWT tokens. The default strategy uses RS256 (RSA Signature with SHA-256)
+
+	// SecretNameOldTokenSigningRSAKey is the privateKey used to sign old JWT tokens. The default strategy uses RS256 (RSA Signature with SHA-256)
 	// This is used to support key rotation. When present, it'll only be used to validate incoming tokens. New tokens
 	// will not be issued using this key.
+	// #nosec
 	SecretNameOldTokenSigningRSAKey SecretName = "token_rsa_key_old.pem"
 )
 
@@ -139,7 +146,7 @@ type Config struct {
 	// when generating metadata endpoints and when validating audience and issuer claims. If no matching authorizedUri
 	// is found, it'll default to the first one. If not provided, the urls will be deduced based on the request url and
 	// the `secure` setting.
-	AuthorizedURIs []config.URL `json:"authorizedUris" pflag:",Optional: Defines the set of URIs that clients are allowed to visit the service on. If set, the system will attempt to match the incoming host to the first authorized URIs and use that (including the scheme) when generating metadata endpoints and when validating audience and issuer claims. If not provided, the urls will be deduced based on the request url and the 'secure' setting."`
+	AuthorizedURIs []config.URL `json:"authorizedUris" pflag:"-,Optional: Defines the set of URIs that clients are allowed to visit the service on. If set, the system will attempt to match the incoming host to the first authorized URIs and use that (including the scheme) when generating metadata endpoints and when validating audience and issuer claims. If not provided, the urls will be deduced based on the request url and the 'secure' setting."`
 
 	// UserAuth settings used to authenticate end users in web-browsers.
 	UserAuth UserAuthConfig `json:"userAuth" pflag:",Defines Auth options for users."`

diff --git a/flyteadmin/auth/config/config_flags.go b/flyteadmin/auth/config/config_flags.go
diff --git a/flyteadmin/auth/config/config_flags_test.go b/flyteadmin/auth/config/config_flags_test.go
diff --git a/flyteadmin/boilerplate/flyte/golang_support_tools/go.mod b/flyteadmin/boilerplate/flyte/golang_support_tools/go.mod
@@ -4,7 +4,7 @@ go 1.17
 
 require (
 	github.com/alvaroloes/enumer v1.1.2
-	github.com/flyteorg/flytestdlib v0.4.7
+	github.com/flyteorg/flytestdlib v0.4.16
 	github.com/golangci/golangci-lint v1.38.0
 	github.com/pseudomuto/protoc-gen-doc v1.4.1
 	github.com/vektra/mockery v0.0.0-20181123154057-e78b021dcbb5
@@ -14,7 +14,10 @@ require (
 	4d63.com/gochecknoglobals v0.0.0-20201008074935-acfc0b28355a // indirect
 	cloud.google.com/go v0.75.0 // indirect
 	cloud.google.com/go/storage v1.12.0 // indirect
-	github.com/Azure/azure-sdk-for-go v51.0.0+incompatible // indirect
+	github.com/Azure/azure-sdk-for-go v62.3.0+incompatible // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0 // indirect
 	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
 	github.com/Azure/go-autorest/autorest v0.11.17 // indirect
 	github.com/Azure/go-autorest/autorest/adal v0.9.10 // indirect
@@ -46,6 +49,7 @@ require (
 	github.com/esimonov/ifshort v1.0.1 // indirect
 	github.com/fatih/color v1.10.0 // indirect
 	github.com/fatih/structtag v1.2.0 // indirect
+	github.com/flyteorg/stow v0.3.1 // indirect
 	github.com/form3tech-oss/jwt-go v3.2.2+incompatible // indirect
 	github.com/fsnotify/fsnotify v1.4.9 // indirect
 	github.com/fzipp/gocyclo v0.3.1 // indirect
@@ -62,6 +66,7 @@ require (
 	github.com/go-xmlfmt/xmlfmt v0.0.0-20191208150333-d5b6f63a941b // indirect
 	github.com/gobwas/glob v0.2.3 // indirect
 	github.com/gofrs/flock v0.8.0 // indirect
+	github.com/gofrs/uuid v4.2.0+incompatible // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
 	github.com/golang/protobuf v1.4.3 // indirect
@@ -82,7 +87,6 @@ require (
 	github.com/gostaticanalysis/comment v1.4.1 // indirect
 	github.com/gostaticanalysis/forcetypeassert v0.0.0-20200621232751-01d4955beaa5 // indirect
 	github.com/gostaticanalysis/nilerr v0.1.1 // indirect
-	github.com/graymeta/stow v0.2.7 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/huandu/xstrings v1.0.0 // indirect
 	github.com/imdario/mergo v0.3.5 // indirect
@@ -135,7 +139,6 @@ require (
 	github.com/ryancurrah/gomodguard v1.2.0 // indirect
 	github.com/ryanrolds/sqlclosecheck v0.3.0 // indirect
 	github.com/sanposhiho/wastedassign v0.1.3 // indirect
-	github.com/satori/go.uuid v1.2.0 // indirect
 	github.com/securego/gosec/v2 v2.6.1 // indirect
 	github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c // indirect
 	github.com/sirupsen/logrus v1.8.0 // indirect
@@ -160,15 +163,15 @@ require (
 	github.com/ultraware/whitespace v0.0.4 // indirect
 	github.com/uudashr/gocognit v1.0.1 // indirect
 	go.opencensus.io v0.22.6 // indirect
-	golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad // indirect
+	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect
 	golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 // indirect
-	golang.org/x/mod v0.4.1 // indirect
-	golang.org/x/net v0.0.0-20210119194325-5f4716e94777 // indirect
+	golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect
+	golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f // indirect
 	golang.org/x/oauth2 v0.0.0-20210126194326-f9ce19ea3013 // indirect
-	golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c // indirect
-	golang.org/x/text v0.3.5 // indirect
+	golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 // indirect
+	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect
-	golang.org/x/tools v0.1.0 // indirect
+	golang.org/x/tools v0.1.10 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	google.golang.org/api v0.38.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect