Merge branch 'master' into feature/fast-cache

Signed-off-by: Daniel Rammer <[email protected]>

CODEOWNERS

@@ -1,3 +1,3 @@
 # These owners will be the default owners for everything in
 # the repo. Unless a later match takes precedence.
-*       @katrogan @wild-endeavor @anandswaminathan @EngHabu @kumare3 @pmahindrakar-oss
+*       @katrogan @wild-endeavor @EngHabu @kumare3 @pmahindrakar-oss @hamersaw @eapolinario

Dockerfile

@@ -3,7 +3,12 @@
 # 
 # TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst
 
-FROM golang:1.18-alpine3.15 as builder
+FROM --platform=${BUILDPLATFORM} golang:1.18-alpine3.15 as builder
+
+ARG TARGETARCH
+ENV GOARCH "${TARGETARCH}"
+ENV GOOS linux
+
 RUN apk add git openssh-client make curl
 
 # Create the artifacts directory

auth/auth_context.go

@@ -133,6 +133,11 @@ func NewAuthenticationContext(ctx context.Context, sm core.SecretManager, oauth2
 		Timeout: IdpConnectionTimeout,
 	}
 
+	if len(options.UserAuth.HTTPProxyURL.String()) > 0 {
+		logger.Infof(ctx, "HTTPProxy URL for OAuth2 is: %s", options.UserAuth.HTTPProxyURL.String())
+		httpClient.Transport = &http.Transport{Proxy: http.ProxyURL(&options.UserAuth.HTTPProxyURL.URL)}
+	}
+
 	// Construct an oidc Provider, which needs its own http Client.
 	oidcCtx := oidc.ClientContext(ctx, httpClient)
 	baseURL := options.UserAuth.OpenID.BaseURL.String()

auth/authzserver/claims_verifier.go

@@ -64,5 +64,5 @@ func verifyClaims(expectedAudience sets.String, claimsRaw map[string]interface{}
 		scopes.Insert(auth.ScopeAll)
 	}
 
-	return auth.NewIdentityContext(claims.Audience[foundAudIndex], claims.Subject, clientID, claims.IssuedAt, scopes, userInfo, claimsRaw), nil
+	return auth.NewIdentityContext(claims.Audience[foundAudIndex], claims.Subject, clientID, claims.IssuedAt, scopes, userInfo, claimsRaw)
 }

auth/authzserver/metadata_provider.go

@@ -90,6 +90,7 @@ func (s OAuth2MetadataProvider) GetPublicClientConfig(context.Context, *service.
 		RedirectUri:              s.cfg.AppAuth.ThirdParty.FlyteClientConfig.RedirectURI,
 		Scopes:                   s.cfg.AppAuth.ThirdParty.FlyteClientConfig.Scopes,
 		AuthorizationMetadataKey: s.cfg.GrpcAuthorizationHeader,
+		Audience:                 s.cfg.AppAuth.ThirdParty.FlyteClientConfig.Audience,
 	}, nil
 }
 

auth/authzserver/metadata_provider_test.go

@@ -24,6 +24,7 @@ func TestOAuth2MetadataProvider_FlyteClient(t *testing.T) {
 					ClientID:    "my-client",
 					RedirectURI: "client/",
 					Scopes:      []string{"all"},
+					Audience:    "http://dummyServer",
 				},
 			},
 		},
@@ -35,6 +36,7 @@ func TestOAuth2MetadataProvider_FlyteClient(t *testing.T) {
 	assert.Equal(t, "my-client", resp.ClientId)
 	assert.Equal(t, "client/", resp.RedirectUri)
 	assert.Equal(t, []string{"all"}, resp.Scopes)
+	assert.Equal(t, "http://dummyServer", resp.Audience)
 }
 
 func TestOAuth2MetadataProvider_OAuth2Metadata(t *testing.T) {

auth/config/config.go

@@ -215,6 +215,9 @@ type UserAuthConfig struct {
 	OpenID OpenIDOptions `json:"openId" pflag:",OpenID Configuration for User Auth"`
 	// Possibly add basicAuth & SAML/p support.
 
+	// HTTPProxyURL allows operators to access external OAuth2 servers using an external HTTP Proxy
+	HTTPProxyURL config.URL `json:"httpProxyURL" pflag:",OPTIONAL: HTTP Proxy to be used for OAuth requests."`
+
 	// Secret names, defaults are set in DefaultConfig variable above but are possible to override through configs.
 	CookieHashKeySecretName  string         `json:"cookieHashKeySecretName" pflag:",OPTIONAL: Secret name to use for cookie hash key."`
 	CookieBlockKeySecretName string         `json:"cookieBlockKeySecretName" pflag:",OPTIONAL: Secret name to use for cookie block key."`

auth/config/third_party_config.go

@@ -10,6 +10,7 @@ type FlyteClientConfig struct {
 	ClientID    string   `json:"clientId" pflag:",public identifier for the app which handles authorization for a Flyte deployment"`
 	RedirectURI string   `json:"redirectUri" pflag:",This is the callback uri registered with the app which handles authorization for a Flyte deployment"`
 	Scopes      []string `json:"scopes" pflag:",Recommended scopes for the client to request."`
+	Audience    string   `json:"audience" pflag:",Audience to use when initiating OAuth2 authorization requests."`
 }
 
 func (o ThirdPartyConfigOptions) IsEmpty() bool {

auth/create_secrets.go

@@ -0,0 +1,185 @@
+package auth
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"k8s.io/client-go/rest"
+
+	"github.com/flyteorg/flytestdlib/logger"
+	kubeErrors "k8s.io/apimachinery/pkg/api/errors"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/flyteorg/flyteadmin/pkg/config"
+	executioncluster "github.com/flyteorg/flyteadmin/pkg/executioncluster/impl"
+	"github.com/flyteorg/flyteadmin/pkg/executioncluster/interfaces"
+	"github.com/flyteorg/flyteadmin/pkg/runtime"
+	"github.com/flyteorg/flytestdlib/errors"
+	"github.com/flyteorg/flytestdlib/promutils"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+	"k8s.io/client-go/kubernetes"
+)
+
+const (
+	PodNamespaceEnvVar  = "POD_NAMESPACE"
+	podDefaultNamespace = "default"
+)
+
+var (
+	secretName       string
+	secretsLocalPath string
+	forceUpdate      bool
+)
+
+func GetCreateSecretsCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use: "create",
+		Long: `Creates a new secret (or noop if one exists unless --force is provided) using keys found in the provided path.
+If POD_NAMESPACE env var is set, the secret will be created in that namespace.
+`,
+		Example: `
+Create a secret using default name (flyte-admin-auth) in default namespace
+flyteadmin secret create --fromPath=/path/in/container
+
+Override an existing secret if one exists (reads secrets from default path /etc/secrets/):
+flyteadmin secret create --name "my-auth-secrets" --force
+`,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return persistSecrets(context.Background(), cmd.Flags())
+		},
+	}
+	cmd.Flags().StringVar(&secretName, "name", "flyte-admin-auth", "Chooses secret name to create/update")
+	cmd.Flags().StringVar(&secretsLocalPath, "fromPath", filepath.Join(string(os.PathSeparator), "etc", "secrets"), "Chooses secret name to create/update")
+	cmd.Flags().BoolVarP(&forceUpdate, "force", "f", false, "Whether to update the secret if one exists")
+
+	return cmd
+}
+
+func persistSecrets(ctx context.Context, _ *pflag.FlagSet) error {
+	configuration := runtime.NewConfigurationProvider()
+	scope := promutils.NewScope(configuration.ApplicationConfiguration().GetTopLevelConfig().MetricsScope)
+	initializationErrorCounter := scope.NewSubScope("secrets").MustNewCounter(
+		"flyteclient_initialization_error",
+		"count of errors encountered initializing a flyte client from kube config")
+
+	var listTargetsProvider interfaces.ListTargetsInterface
+	var err error
+	if len(configuration.ClusterConfiguration().GetClusterConfigs()) == 0 {
+		serverConfig := config.GetConfig()
+		listTargetsProvider, err = executioncluster.NewInCluster(initializationErrorCounter, serverConfig.KubeConfig, serverConfig.Master)
+	} else {
+		listTargetsProvider, err = executioncluster.NewListTargets(initializationErrorCounter, executioncluster.NewExecutionTargetProvider(), configuration.ClusterConfiguration())
+	}
+	if err != nil {
+		return err
+	}
+
+	targets := listTargetsProvider.GetValidTargets()
+	// Since we are targeting the cluster Admin is running in, this list should contain exactly one item
+	if len(targets) != 1 {
+		return fmt.Errorf("expected exactly 1 valid target cluster. Found [%v]", len(targets))
+	}
+	var clusterCfg rest.Config
+	for _, target := range targets {
+		// We've just ascertained targets contains exactly 1 item, so we can safely assume we'll assign the clusterCfg
+		// from that one item now.
+		clusterCfg = target.Config
+	}
+
+	kubeClient, err := kubernetes.NewForConfig(&clusterCfg)
+	if err != nil {
+		return errors.Wrapf("INIT", err, "Error building kubernetes clientset")
+	}
+
+	podNamespace, found := os.LookupEnv(PodNamespaceEnvVar)
+	if !found {
+		podNamespace = podDefaultNamespace
+	}
+
+	secretsData, err := buildK8sSecretData(ctx, secretsLocalPath)
+	if err != nil {
+		return errors.Wrapf("INIT", err, "Error building k8s secret's data field.")
+	}
+
+	secretsClient := kubeClient.CoreV1().Secrets(podNamespace)
+	newSecret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      secretName,
+			Namespace: podNamespace,
+		},
+		Type: corev1.SecretTypeOpaque,
+		Data: secretsData,
+	}
+
+	_, err = secretsClient.Create(ctx, newSecret, metav1.CreateOptions{})
+
+	if err != nil && kubeErrors.IsAlreadyExists(err) {
+		if forceUpdate {
+			logger.Infof(ctx, "A secret already exists with the same name. Attempting to update it.")
+			_, err = secretsClient.Update(ctx, newSecret, metav1.UpdateOptions{})
+		} else {
+			var existingSecret *corev1.Secret
+			existingSecret, err = secretsClient.Get(ctx, newSecret.Name, metav1.GetOptions{})
+			if err != nil {
+				logger.Infof(ctx, "Failed to retrieve existing secret. Error: %v", err)
+				return err
+			}
+
+			if existingSecret.Data == nil {
+				existingSecret.Data = map[string][]byte{}
+			}
+
+			needsUpdate := false
+			for key, val := range secretsData {
+				if _, found := existingSecret.Data[key]; !found {
+					existingSecret.Data[key] = val
+					needsUpdate = true
+				}
+			}
+
+			if needsUpdate {
+				_, err = secretsClient.Update(ctx, existingSecret, metav1.UpdateOptions{})
+				if err != nil && kubeErrors.IsConflict(err) {
+					logger.Infof(ctx, "Another instance of flyteadmin has updated the same secret. Ignoring this update")
+					err = nil
+				}
+			}
+		}
+
+		return err
+	}
+
+	return err
+}
+
+func buildK8sSecretData(_ context.Context, localPath string) (map[string][]byte, error) {
+	secretsData := make(map[string][]byte, 4)
+
+	err := filepath.Walk(localPath, func(path string, info os.FileInfo, err error) error {
+		if err != nil || info.IsDir() {
+			return nil
+		}
+
+		data, err := ioutil.ReadFile(path)
+		if err != nil {
+			return err
+		}
+
+		secretsData[strings.TrimPrefix(path, filepath.Dir(path)+string(filepath.Separator))] = data
+		return nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	return secretsData, nil
+}