Skip to content

Commit

Permalink
Try 2: Add retry for oauth metadata endpoint (#4262)
Browse files Browse the repository at this point in the history
* logic and commented out tests

Signed-off-by: squiishyy <[email protected]>

* make RetryOnSpecificErrorCodes, added functionality for calling sendhttprequestwithretry, need to work on config mappingn still, tests in

Signed-off-by: squiishyy <[email protected]>

* adding default config values

Signed-off-by: squiishyy <[email protected]>

* imports

Signed-off-by: squiishyy <[email protected]>

* updated tests to work

Signed-off-by: squiishyy <[email protected]>

* added test

Signed-off-by: squiishyy <[email protected]>

* spelling and info

Signed-off-by: squiishyy <[email protected]>

* working, removing comment

Signed-off-by: squiishyy <[email protected]>

* add back

Signed-off-by: squiishyy <[email protected]>

* code review

Signed-off-by: squiishyy <[email protected]>

* code review, refactoring to use retry.onerror

Signed-off-by: squiishyy <[email protected]>

* few code review comments, removing unused logic

Signed-off-by: squiishyy <[email protected]>

* retry on a wider range

Signed-off-by: squiishyy <[email protected]>

* comment

Signed-off-by: squiishyy <[email protected]>

* updated logs

Signed-off-by: squiishyy <[email protected]>

* lint

Signed-off-by: squiishyy <[email protected]>

* linting 2

Signed-off-by: squiishyy <[email protected]>

---------

Signed-off-by: squiishyy <[email protected]>
  • Loading branch information
squiishyy authored Oct 19, 2023
1 parent 5c0b7ae commit 9af6d84
Show file tree
Hide file tree
Showing 5 changed files with 167 additions and 2 deletions.
48 changes: 47 additions & 1 deletion flyteadmin/auth/authzserver/metadata_provider.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,22 @@ package authzserver

import (
"context"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"strings"
"time"

"google.golang.org/grpc/codes"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/util/retry"

"github.com/flyteorg/flyte/flyteadmin/auth"
authConfig "github.com/flyteorg/flyte/flyteadmin/auth/config"
flyteErrors "github.com/flyteorg/flyte/flyteadmin/pkg/errors"
"github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/service"
"github.com/flyteorg/flyte/flytestdlib/logger"
)

type OAuth2MetadataProvider struct {
Expand Down Expand Up @@ -72,7 +80,7 @@ func (s OAuth2MetadataProvider) GetOAuth2Metadata(ctx context.Context, r *servic
httpClient.Transport = transport
}

response, err := httpClient.Get(externalMetadataURL.String())
response, err := sendAndRetryHTTPRequest(ctx, httpClient, externalMetadataURL.String(), s.cfg.AppAuth.ExternalAuthServer.RetryAttempts, s.cfg.AppAuth.ExternalAuthServer.RetryDelay.Duration)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -107,3 +115,41 @@ func NewService(config *authConfig.Config) OAuth2MetadataProvider {
cfg: config,
}
}

func sendAndRetryHTTPRequest(ctx context.Context, client *http.Client, url string, retryAttempts int, retryDelay time.Duration) (*http.Response, error) {
var response *http.Response
var err error
totalAttempts := retryAttempts + 1 // Add one for initial http request attempt

backoff := wait.Backoff{
Duration: retryDelay,
Steps: totalAttempts,
}

retryableOauthMetadataError := flyteErrors.NewFlyteAdminError(codes.Internal, "Failed to get oauth metadata.")
err = retry.OnError(backoff,
func(err error) bool { // Determine if error is retryable
return err == retryableOauthMetadataError
}, func() error { // Send HTTP request
response, err = client.Get(url)
if err != nil {
logger.Errorf(ctx, "Failed to send oauth metadata HTTP request. Err: %v", err)
return err
}
if response != nil && response.StatusCode >= http.StatusUnauthorized && response.StatusCode <= http.StatusNetworkAuthenticationRequired {
logger.Errorf(ctx, "Failed to get oauth metadata, going to retry. StatusCode: %v Err: %v", response.StatusCode, err)
return retryableOauthMetadataError
}
return nil
})

if err != nil {
return nil, err
}

if response != nil && response.StatusCode != http.StatusOK {
return response, fmt.Errorf("failed to get oauth metadata with status code %v", response.StatusCode)
}

return response, nil
}
83 changes: 83 additions & 0 deletions flyteadmin/auth/authzserver/metadata_provider_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ import (
config2 "github.com/flyteorg/flyte/flytestdlib/config"
)

var oauthMetadataFailureErrorMessage = "Failed to get oauth metadata."

func TestOAuth2MetadataProvider_FlyteClient(t *testing.T) {
provider := NewService(&authConfig.Config{
AppAuth: authConfig.OAuth2Options{
Expand Down Expand Up @@ -111,3 +113,84 @@ func TestOAuth2MetadataProvider_OAuth2Metadata(t *testing.T) {
assert.Equal(t, "https://dev-14186422.okta.com", resp.Issuer)
})
}

func TestSendAndRetryHttpRequest(t *testing.T) {
t.Run("Retry into failure", func(t *testing.T) {
requestAttempts := 0
hf := func(w http.ResponseWriter, r *http.Request) {
switch strings.TrimSpace(r.URL.Path) {
case "/":
w.WriteHeader(500)
requestAttempts++
default:
http.NotFoundHandler().ServeHTTP(w, r)
}
}

server := httptest.NewServer(http.HandlerFunc(hf))
defer server.Close()
http.DefaultClient = server.Client()
retryAttempts := 5
totalAttempts := retryAttempts + 1 // 1 for the initial try

resp, err := sendAndRetryHttpRequest(context.Background(), server.Client(), server.URL, retryAttempts, 0 /* for testing */)
assert.Error(t, err)
assert.Equal(t, oauthMetadataFailureErrorMessage, err.Error())
assert.Nil(t, resp)
assert.Equal(t, totalAttempts, requestAttempts)
})

t.Run("Retry into success", func(t *testing.T) {
requestAttempts := 0
hf := func(w http.ResponseWriter, r *http.Request) {
switch strings.TrimSpace(r.URL.Path) {
case "/":
if requestAttempts > 2 {
w.WriteHeader(200)
} else {
requestAttempts++
w.WriteHeader(500)
}
default:
http.NotFoundHandler().ServeHTTP(w, r)
}
}

server := httptest.NewServer(http.HandlerFunc(hf))
defer server.Close()
http.DefaultClient = server.Client()
retryAttempts := 5
expectedRequestAttempts := 3

resp, err := sendAndRetryHttpRequest(context.Background(), server.Client(), server.URL, retryAttempts, 0 /* for testing */)
assert.NoError(t, err)
assert.NotNil(t, resp)
assert.Equal(t, 200, resp.StatusCode)
assert.Equal(t, expectedRequestAttempts, requestAttempts)
})

t.Run("Success", func(t *testing.T) {
requestAttempts := 0
hf := func(w http.ResponseWriter, r *http.Request) {
switch strings.TrimSpace(r.URL.Path) {
case "/":
w.WriteHeader(200)
default:
http.NotFoundHandler().ServeHTTP(w, r)
}
}

server := httptest.NewServer(http.HandlerFunc(hf))
defer server.Close()
http.DefaultClient = server.Client()
retryAttempts := 5
expectedRequestAttempts := 0

resp, err := sendAndRetryHttpRequest(context.Background(), server.Client(), server.URL, retryAttempts, 0 /* for testing */)
assert.NoError(t, err)
assert.NotNil(t, resp)
assert.Equal(t, 200, resp.StatusCode)
assert.Equal(t, expectedRequestAttempts, requestAttempts)
})

}
8 changes: 7 additions & 1 deletion flyteadmin/auth/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,10 @@ var (
},
},
AppAuth: OAuth2Options{
ExternalAuthServer: ExternalAuthorizationServer{
RetryAttempts: 5,
RetryDelay: config.Duration{Duration: 1 * time.Second},
},
AuthServerType: AuthorizationServerTypeSelf,
ThirdParty: ThirdPartyConfigOptions{
FlyteClientConfig: FlyteClientConfig{
Expand Down Expand Up @@ -191,7 +195,9 @@ type ExternalAuthorizationServer struct {
AllowedAudience []string `json:"allowedAudience" pflag:",Optional: A list of allowed audiences. If not provided, the audience is expected to be the public Uri of the service."`
MetadataEndpointURL config.URL `json:"metadataUrl" pflag:",Optional: If the server doesn't support /.well-known/oauth-authorization-server, you can set a custom metadata url here.'"`
// HTTPProxyURL allows operators to access external OAuth2 servers using an external HTTP Proxy
HTTPProxyURL config.URL `json:"httpProxyURL" pflag:",OPTIONAL: HTTP Proxy to be used for OAuth requests."`
HTTPProxyURL config.URL `json:"httpProxyURL" pflag:",OPTIONAL: HTTP Proxy to be used for OAuth requests."`
RetryAttempts int `json:"retryAttempts" pflag:", Optional: The number of attempted retries on a transient failure to get the OAuth metadata"`
RetryDelay config.Duration `json:"retryDelay" pflag:", Optional, Duration to wait between retries"`
}

// OAuth2Options defines settings for app auth.
Expand Down
2 changes: 2 additions & 0 deletions flyteadmin/auth/config/config_flags.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

28 changes: 28 additions & 0 deletions flyteadmin/auth/config/config_flags_test.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit 9af6d84

Please sign in to comment.