diff --git a/credentials/tls/certprovider/distributor.go b/credentials/tls/certprovider/distributor.go index d6feb3afeba1..fdb38a663fe8 100644 --- a/credentials/tls/certprovider/distributor.go +++ b/credentials/tls/certprovider/distributor.go @@ -42,7 +42,11 @@ type Distributor struct { km *KeyMaterial pErr error - ready *grpcsync.Event + // ready channel to unblock KeyMaterial() invocations blocked on + // availability of key material. + ready *grpcsync.Event + // done channel to notify provider implementations and unblock any + // KeyMaterial() calls, once the Distributor is closed. closed *grpcsync.Event } @@ -75,7 +79,7 @@ func (d *Distributor) Set(km *KeyMaterial, err error) { d.mu.Unlock() } -// KeyMaterial returns the most recent key material provided to the distributor. +// KeyMaterial returns the most recent key material provided to the Distributor. // If no key material was provided at the time of this call, it will block until // the deadline on the context expires or fresh key material arrives. func (d *Distributor) KeyMaterial(ctx context.Context) (*KeyMaterial, error) { diff --git a/credentials/tls/certprovider/distributor_test.go b/credentials/tls/certprovider/distributor_test.go index e6c41564c293..bec00e919bcf 100644 --- a/credentials/tls/certprovider/distributor_test.go +++ b/credentials/tls/certprovider/distributor_test.go @@ -1,3 +1,5 @@ +// +build go1.13 + /* * * Copyright 2020 gRPC authors. @@ -21,139 +23,90 @@ package certprovider import ( "context" "errors" - "fmt" - "reflect" - "sync" "testing" "time" ) var errProviderTestInternal = errors.New("provider internal error") +// TestDistributorEmpty tries to read key material from an empty distributor and +// expects the call to timeout. +func (s) TestDistributorEmpty(t *testing.T) { + dist := NewDistributor() + + // This call to KeyMaterial() should timeout because no key material has + // been set on the distributor as yet. + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + if err := readAndVerifyKeyMaterial(ctx, dist, nil); !errors.Is(err, context.DeadlineExceeded) { + t.Fatal(err) + } +} + // TestDistributor invokes the different methods on the Distributor type and // verifies the results. func (s) TestDistributor(t *testing.T) { dist := NewDistributor() // Read cert/key files from testdata. - km, err := loadKeyMaterials() - if err != nil { + km1 := loadKeyMaterials(t, "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem") + km2 := loadKeyMaterials(t, "x509/server2_cert.pem", "x509/server2_key.pem", "x509/client_ca_cert.pem") + + // Push key material into the distributor and make sure that a call to + // KeyMaterial() returns the expected key material, with both the local + // certs and root certs. + dist.Set(km1, nil) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := readAndVerifyKeyMaterial(ctx, dist, km1); err != nil { t.Fatal(err) } - // wantKM1 has both local and root certs. - wantKM1 := *km - // wantKM2 has only local certs. Roots are nil-ed out. - wantKM2 := *km - wantKM2.Roots = nil - - // Create a goroutines which work in lockstep with the rest of the test. - // This goroutine reads the key material from the distributor while the rest - // of the test sets it. - var wg sync.WaitGroup - wg.Add(1) - errCh := make(chan error) - proceedCh := make(chan struct{}) - go func() { - defer wg.Done() - - // The first call to KeyMaterial() should timeout because no key - // material has been set on the distributor as yet. - ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout/2) - defer cancel() - if _, err := dist.KeyMaterial(ctx); err != context.DeadlineExceeded { - errCh <- err - return - } - proceedCh <- struct{}{} - - // This call to KeyMaterial() should return the key material with both - // the local certs and the root certs. - ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout) - defer cancel() - gotKM, err := dist.KeyMaterial(ctx) - if err != nil { - errCh <- err - return - } - if !reflect.DeepEqual(gotKM, &wantKM1) { - errCh <- fmt.Errorf("provider.KeyMaterial() = %+v, want %+v", gotKM, wantKM1) - } - proceedCh <- struct{}{} - - // This call to KeyMaterial() should eventually return key material with - // only the local certs. - ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout) - defer cancel() - for { - gotKM, err := dist.KeyMaterial(ctx) - if err != nil { - errCh <- err - return - } - if reflect.DeepEqual(gotKM, &wantKM2) { - break - } - } - proceedCh <- struct{}{} - - // This call to KeyMaterial() should return nil key material and a - // non-nil error. - ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout) - defer cancel() - for { - gotKM, err := dist.KeyMaterial(ctx) - if gotKM == nil && err == errProviderTestInternal { - break - } - if err != nil { - // If we have gotten any error other than - // errProviderTestInternal, we should bail out. - errCh <- err - return - } - } - proceedCh <- struct{}{} - - // This call to KeyMaterial() should eventually return errProviderClosed - // error. - ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout) - defer cancel() - for { - if _, err := dist.KeyMaterial(ctx); err == errProviderClosed { - break - } - time.Sleep(100 * time.Millisecond) - } - }() - waitAndDo(t, proceedCh, errCh, func() { - dist.Set(&wantKM1, nil) - }) + // Push new key material into the distributor and make sure that a call to + // KeyMaterial() returns the expected key material, with only root certs. + dist.Set(km2, nil) + if err := readAndVerifyKeyMaterial(ctx, dist, km2); err != nil { + t.Fatal(err) + } - waitAndDo(t, proceedCh, errCh, func() { - dist.Set(&wantKM2, nil) - }) + // Push an error into the distributor and make sure that a call to + // KeyMaterial() returns that error and nil keyMaterial. + dist.Set(km2, errProviderTestInternal) + if gotKM, err := dist.KeyMaterial(ctx); gotKM != nil || !errors.Is(err, errProviderTestInternal) { + t.Fatalf("KeyMaterial() = {%v, %v}, want {nil, %v}", gotKM, err, errProviderTestInternal) + } - waitAndDo(t, proceedCh, errCh, func() { - dist.Set(&wantKM2, errProviderTestInternal) - }) + // Stop the distributor and KeyMaterial() should return errProviderClosed. + dist.Stop() + if km, err := dist.KeyMaterial(ctx); !errors.Is(err, errProviderClosed) { + t.Fatalf("KeyMaterial() = {%v, %v}, want {nil, %v}", km, err, errProviderClosed) + } +} - waitAndDo(t, proceedCh, errCh, func() { - dist.Stop() - }) +// TestDistributorConcurrency invokes methods on the distributor in parallel. It +// exercises that the scenario where a distributor's KeyMaterial() method is +// blocked waiting for keyMaterial, while the Set() method is called from +// another goroutine. It verifies that the KeyMaterial() method eventually +// returns with expected keyMaterial. +func (s) TestDistributorConcurrency(t *testing.T) { + dist := NewDistributor() -} + // Read cert/key files from testdata. + km := loadKeyMaterials(t, "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem") -func waitAndDo(t *testing.T, proceedCh chan struct{}, errCh chan error, do func()) { - t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() - timer := time.NewTimer(defaultTestTimeout) - select { - case <-timer.C: - t.Fatalf("test timed out when waiting for event from distributor") - case <-proceedCh: - do() - case err := <-errCh: + // Push key material into the distributor from a goroutine and read from + // here to verify that the distributor returns the expected keyMaterial. + go func() { + // Add a small sleep here to make sure that the call to KeyMaterial() + // happens before the call to Set(), thereby the former is blocked till + // the latter happens. + time.Sleep(100 * time.Microsecond) + dist.Set(km, nil) + }() + if err := readAndVerifyKeyMaterial(ctx, dist, km); err != nil { t.Fatal(err) } } diff --git a/credentials/tls/certprovider/provider.go b/credentials/tls/certprovider/provider.go index 58c93c1e731f..204e55686e91 100644 --- a/credentials/tls/certprovider/provider.go +++ b/credentials/tls/certprovider/provider.go @@ -16,7 +16,7 @@ * */ -// Package certprovider defines APIs for certificate providers in gRPC. +// Package certprovider defines APIs for Certificate Providers in gRPC. // // Experimental // @@ -36,18 +36,18 @@ var ( // closed. errProviderClosed = errors.New("provider instance is closed") - // m is a map from name to provider builder. + // m is a map from name to Provider builder. m = make(map[string]Builder) ) -// Register registers the provider builder, whose name as returned by its Name() +// Register registers the Provider builder, whose name as returned by its Name() // method will be used as the name registered with this builder. Registered // Builders are used by the Store to create Providers. func Register(b Builder) { m[b.Name()] = b } -// getBuilder returns the provider builder registered with the given name. +// getBuilder returns the Provider builder registered with the given name. // If no builder is registered with the provided name, nil will be returned. func getBuilder(name string) Builder { if b, ok := m[name]; ok { @@ -58,8 +58,9 @@ func getBuilder(name string) Builder { // Builder creates a Provider. type Builder interface { - // Build creates a new provider with the provided config. - Build(StableConfig) Provider + // Build creates a new Provider and initializes it with the given config and + // options combination. + Build(StableConfig, Options) Provider // ParseConfig converts config input in a format specific to individual // implementations and returns an implementation of the StableConfig @@ -72,9 +73,9 @@ type Builder interface { Name() string } -// StableConfig wraps the method to return a stable provider configuration. +// StableConfig wraps the method to return a stable Provider configuration. type StableConfig interface { - // Canonical returns provider config as an arbitrary byte slice. + // Canonical returns Provider config as an arbitrary byte slice. // Equivalent configurations must return the same output. Canonical() []byte } @@ -87,18 +88,30 @@ type StableConfig interface { // the latest secrets, and free to share any state between different // instantiations as they deem fit. type Provider interface { - // KeyMaterial returns the key material sourced by the provider. + // KeyMaterial returns the key material sourced by the Provider. // Callers are expected to use the returned value as read-only. KeyMaterial(ctx context.Context) (*KeyMaterial, error) - // Close cleans up resources allocated by the provider. + // Close cleans up resources allocated by the Provider. Close() } -// KeyMaterial wraps the certificates and keys returned by a provider instance. +// KeyMaterial wraps the certificates and keys returned by a Provider instance. type KeyMaterial struct { // Certs contains a slice of cert/key pairs used to prove local identity. Certs []tls.Certificate // Roots contains the set of trusted roots to validate the peer's identity. Roots *x509.CertPool } + +// Options contains configuration knobs passed to a Provider at creation time. +type Options struct { + // CertName holds the certificate name, whose key material is of interest to + // the caller. + CertName string + // WantRoot indicates if the caller is interested in the root certificate. + WantRoot bool + // WantIdentity indicates if the caller is interested in the identity + // certificate. + WantIdentity bool +} diff --git a/credentials/tls/certprovider/store.go b/credentials/tls/certprovider/store.go index 0d2b29b675b1..990e6ab06c34 100644 --- a/credentials/tls/certprovider/store.go +++ b/credentials/tls/certprovider/store.go @@ -38,6 +38,8 @@ type storeKey struct { name string // configuration of the certificate provider in string form. config string + // opts contains the certificate name and other keyMaterial options. + opts Options } // wrappedProvider wraps a provider instance with a reference count. @@ -57,17 +59,20 @@ type store struct { providers map[storeKey]*wrappedProvider } -// GetProvider returns a provider instance corresponding to name and config. -// name is the registered name of the provider and config is the -// provider-specific configuration. Implementations of the Builder interface -// should clearly document the type of configuration accepted by them. +// GetProvider returns a provider instance from which keyMaterial can be read. // -// If a provider exists for the (name+config) combination, its reference count -// is incremented before returning. If no provider exists for the (name+config) -// combination, a new one is created using the registered builder. If no -// registered builder is found, or the provider configuration is rejected by it, -// a non-nil error is returned. -func GetProvider(name string, config interface{}) (Provider, error) { +// name is the registered name of the provider, config is the provider-specific +// configuration, opts contains extra information that controls the keyMaterial +// returned by the provider. +// +// Implementations of the Builder interface should clearly document the type of +// configuration accepted by them. +// +// If a provider exists for passed arguments, its reference count is incremented +// before returning. If no provider exists for the passed arguments, a new one +// is created using the registered builder. If no registered builder is found, +// or the provider configuration is rejected by it, a non-nil error is returned. +func GetProvider(name string, config interface{}, opts Options) (Provider, error) { provStore.mu.Lock() defer provStore.mu.Unlock() @@ -83,13 +88,14 @@ func GetProvider(name string, config interface{}) (Provider, error) { sk := storeKey{ name: name, config: string(stableConfig.Canonical()), + opts: opts, } if wp, ok := provStore.providers[sk]; ok { wp.refCount++ return wp, nil } - provider := builder.Build(stableConfig) + provider := builder.Build(stableConfig, opts) if provider == nil { return nil, fmt.Errorf("certprovider.Build(%v) failed", sk) } diff --git a/credentials/tls/certprovider/store_test.go b/credentials/tls/certprovider/store_test.go index 33135d41c854..a116148945eb 100644 --- a/credentials/tls/certprovider/store_test.go +++ b/credentials/tls/certprovider/store_test.go @@ -1,3 +1,5 @@ +// +build go1.13 + /* * * Copyright 2020 gRPC authors. @@ -22,6 +24,7 @@ import ( "context" "crypto/tls" "crypto/x509" + "errors" "fmt" "io/ioutil" "reflect" @@ -29,6 +32,7 @@ import ( "time" "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/testutils" "google.golang.org/grpc/testdata" ) @@ -39,9 +43,19 @@ const ( defaultTestTimeout = 1 * time.Second ) +var fpb1, fpb2 *fakeProviderBuilder + func init() { - Register(&fakeProviderBuilder{name: fakeProvider1Name}) - Register(&fakeProviderBuilder{name: fakeProvider2Name}) + fpb1 = &fakeProviderBuilder{ + name: fakeProvider1Name, + providerChan: testutils.NewChannel(), + } + fpb2 = &fakeProviderBuilder{ + name: fakeProvider2Name, + providerChan: testutils.NewChannel(), + } + Register(fpb1) + Register(fpb2) } type s struct { @@ -55,25 +69,20 @@ func Test(t *testing.T) { // fakeProviderBuilder builds new instances of fakeProvider and interprets the // config provided to it as a string. type fakeProviderBuilder struct { - name string + name string + providerChan *testutils.Channel } -func (b *fakeProviderBuilder) Build(StableConfig) Provider { - ctx, cancel := context.WithCancel(context.Background()) - p := &fakeProvider{ - Distributor: NewDistributor(), - cancel: cancel, - done: make(chan struct{}), - kmCh: make(chan *KeyMaterial, 2), - } - go p.run(ctx) +func (b *fakeProviderBuilder) Build(StableConfig, Options) Provider { + p := &fakeProvider{Distributor: NewDistributor()} + b.providerChan.Send(p) return p } func (b *fakeProviderBuilder) ParseConfig(config interface{}) (StableConfig, error) { s, ok := config.(string) if !ok { - return nil, fmt.Errorf("provider %s received bad config %v", b.name, config) + return nil, fmt.Errorf("providerBuilder %s received config of type %T, want string", b.name, config) } return &fakeStableConfig{config: s}, nil } @@ -90,262 +99,309 @@ func (c *fakeStableConfig) Canonical() []byte { return []byte(c.config) } -// fakeProvider is an implementation of the Provider interface which embeds a -// Distributor and exposes two channels for the user: -// 1. to be notified when the provider is closed -// 2. to push new key material into the provider +// fakeProvider is an implementation of the Provider interface which provides a +// method for tests to invoke to push new key materials. type fakeProvider struct { *Distributor - - // Used to cancel the run goroutine when the provider is closed. - cancel context.CancelFunc - // This channel is closed when the provider is closed. Tests should block on - // this to make sure the provider is closed. - done chan struct{} - // Tests can push new key material on this channel, and the provider will - // return this on subsequent calls to KeyMaterial(). - kmCh chan *KeyMaterial } -func (p *fakeProvider) run(ctx context.Context) { - for { - select { - case <-ctx.Done(): - return - case km := <-p.kmCh: - p.Distributor.Set(km, nil) - } - } +// newKeyMaterial allows tests to push new key material to the fake provider +// which will be made available to users of this provider. +func (p *fakeProvider) newKeyMaterial(km *KeyMaterial, err error) { + p.Distributor.Set(km, err) } +// Close helps implement the Provider interface. func (p *fakeProvider) Close() { - p.cancel() p.Distributor.Stop() } // loadKeyMaterials is a helper to read cert/key files from testdata and convert -// them into a KeyMaterial struct. -func loadKeyMaterials() (*KeyMaterial, error) { - certs, err := tls.LoadX509KeyPair(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem")) +// them into a KeyMaterialReader struct. +func loadKeyMaterials(t *testing.T, cert, key, ca string) *KeyMaterial { + t.Helper() + + certs, err := tls.LoadX509KeyPair(testdata.Path(cert), testdata.Path(key)) if err != nil { - return nil, err + t.Fatalf("Failed to load keyPair: %v", err) } - pemData, err := ioutil.ReadFile(testdata.Path("x509/client_ca_cert.pem")) + pemData, err := ioutil.ReadFile(testdata.Path(ca)) if err != nil { - return nil, err + t.Fatal(err) } roots := x509.NewCertPool() roots.AppendCertsFromPEM(pemData) - return &KeyMaterial{Certs: []tls.Certificate{certs}, Roots: roots}, nil + return &KeyMaterial{Certs: []tls.Certificate{certs}, Roots: roots} } -func makeProvider(t *testing.T, name, config string) (Provider, *fakeProvider) { - t.Helper() +// kmReader wraps the KeyMaterial method exposed by Provider and Distributor +// implementations. Defining the interface here makes it possible to use the +// same helper from both provider and distributor tests. +type kmReader interface { + KeyMaterial(context.Context) (*KeyMaterial, error) +} - prov, err := GetProvider(name, config) +// readAndVerifyKeyMaterial attempts to read key material from the given +// provider and compares it against the expected key material. +func readAndVerifyKeyMaterial(ctx context.Context, kmr kmReader, wantKM *KeyMaterial) error { + gotKM, err := kmr.KeyMaterial(ctx) if err != nil { - t.Fatal(err) + return fmt.Errorf("KeyMaterial(ctx) failed: %w", err) } + return compareKeyMaterial(gotKM, wantKM) +} - // The store returns a wrappedProvider, which holds a reference to the - // actual provider, which in our case in the fakeProvider. - wp := prov.(*wrappedProvider) - fp := wp.Provider.(*fakeProvider) - return prov, fp +func compareKeyMaterial(got, want *KeyMaterial) error { + // TODO(easwars): Remove all references to reflect.DeepEqual and use + // cmp.Equal instead. Currently, the later panics because x509.Certificate + // type defines an Equal method, but does not check for nil. This has been + // fixed in + // https://github.com/golang/go/commit/89865f8ba64ccb27f439cce6daaa37c9aa38f351, + // but this is only available starting go1.14. So, once we remove support + // for go1.13, we can make the switch. + if !reflect.DeepEqual(got, want) { + return fmt.Errorf("provider.KeyMaterial() = %+v, want %+v", got, want) + } + return nil } -// TestStoreWithSingleProvider creates a single provider through the store and -// calls methods on it. -func (s) TestStoreWithSingleProvider(t *testing.T) { - prov, fp := makeProvider(t, fakeProvider1Name, fakeConfig) +// TestStoreSingleProvider creates a single provider through the store and calls +// methods on them. +func (s) TestStoreSingleProvider(t *testing.T) { + // Create a Provider through the store. + kmOpts := Options{CertName: "default"} + prov, err := GetProvider(fakeProvider1Name, fakeConfig, kmOpts) + if err != nil { + t.Fatalf("GetProvider(%s, %s, %v) failed: %v", fakeProvider1Name, fakeConfig, kmOpts, err) + } + defer prov.Close() - // Push key materials into the provider. - wantKM, err := loadKeyMaterials() + // Our fakeProviderBuilder pushes newly created providers on a channel. Grab + // the fake provider from that channel. + p, err := fpb1.providerChan.Receive() if err != nil { - t.Fatal(err) + t.Fatalf("Timeout when expecting certProvider %q to be created", fakeProvider1Name) } - fp.kmCh <- wantKM + fakeProv := p.(*fakeProvider) - // Get key materials from the provider and compare it to the ones we pushed - // above. + // Attempt to read from key material from the Provider returned by the + // store. This will fail because we have not pushed any key material into + // our fake provider. ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() - gotKM, err := prov.KeyMaterial(ctx) - if err != nil { - t.Fatalf("provider.KeyMaterial() = %v", err) + if err := readAndVerifyKeyMaterial(ctx, prov, nil); !errors.Is(err, context.DeadlineExceeded) { + t.Fatal(err) } - // TODO(easwars): Remove all references to reflect.DeepEqual and use - // cmp.Equal instead. Currently, the later panics because x509.Certificate - // type defines an Equal method, but does not check for nil. This has been - // fixed in - // https://github.com/golang/go/commit/89865f8ba64ccb27f439cce6daaa37c9aa38f351, - // but this is only available starting go1.14. So, once we remove support - // for go1.13, we can make the switch. - if !reflect.DeepEqual(gotKM, wantKM) { - t.Fatalf("provider.KeyMaterial() = %+v, want %+v", gotKM, wantKM) + + // Load key material from testdata directory, push it into out fakeProvider + // and attempt to read from the Provider returned by the store. + testKM1 := loadKeyMaterials(t, "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem") + fakeProv.newKeyMaterial(testKM1, nil) + ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := readAndVerifyKeyMaterial(ctx, prov, testKM1); err != nil { + t.Fatal(err) } - // Close the provider and retry the KeyMaterial() call, and expect it to - // fail with a known error. - prov.Close() - if _, err := prov.KeyMaterial(ctx); err != errProviderClosed { - t.Fatalf("provider.KeyMaterial() = %v, wantErr: %v", err, errProviderClosed) + // Push new key material and read from the Provider. This should returned + // updated key material. + testKM2 := loadKeyMaterials(t, "x509/server2_cert.pem", "x509/server2_key.pem", "x509/client_ca_cert.pem") + fakeProv.newKeyMaterial(testKM2, nil) + ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := readAndVerifyKeyMaterial(ctx, prov, testKM2); err != nil { + t.Fatal(err) } } -// TestStoreWithSingleProviderWithSharing creates multiple instances of the same -// type of provider through the store (and expects the store's sharing mechanism -// to kick in) and calls methods on it. -func (s) TestStoreWithSingleProviderWithSharing(t *testing.T) { - prov1, fp1 := makeProvider(t, fakeProvider1Name, fakeConfig) - prov2, _ := makeProvider(t, fakeProvider1Name, fakeConfig) +// TestStoreSingleProviderSameConfigDifferentOpts creates multiple providers of +// same type, for same configs but different keyMaterial options through the +// store (and expects the store's sharing mechanism to kick in) and calls +// methods on them. +func (s) TestStoreSingleProviderSameConfigDifferentOpts(t *testing.T) { + // Create three readers on the same fake provider. Two of these readers use + // certName `foo`, while the third one uses certName `bar`. + optsFoo := Options{CertName: "foo"} + optsBar := Options{CertName: "bar"} + provFoo1, err := GetProvider(fakeProvider1Name, fakeConfig, optsFoo) + if err != nil { + t.Fatalf("GetProvider(%s, %s, %v) failed: %v", fakeProvider1Name, fakeConfig, optsFoo, err) + } + defer provFoo1.Close() + provFoo2, err := GetProvider(fakeProvider1Name, fakeConfig, optsFoo) + if err != nil { + t.Fatalf("GetProvider(%s, %s, %v) failed: %v", fakeProvider1Name, fakeConfig, optsFoo, err) + } + defer provFoo2.Close() + // Our fakeProviderBuilder pushes newly created providers on a channel. + // Grab the fake provider for optsFoo. + p, err := fpb1.providerChan.Receive() + if err != nil { + t.Fatalf("Timeout when expecting certProvider %q to be created", fakeProvider1Name) + } + fakeProvFoo := p.(*fakeProvider) - // Push key materials into the fake provider1. - wantKM, err := loadKeyMaterials() + provBar1, err := GetProvider(fakeProvider1Name, fakeConfig, optsBar) if err != nil { - t.Fatal(err) + t.Fatalf("GetProvider(%s, %s, %v) failed: %v", fakeProvider1Name, fakeConfig, optsBar, err) } - fp1.kmCh <- wantKM + defer provBar1.Close() + // Grab the fake provider for optsBar. + p, err = fpb1.providerChan.Receive() + if err != nil { + t.Fatalf("Timeout when expecting certProvider %q to be created", fakeProvider1Name) + } + fakeProvBar := p.(*fakeProvider) - // Get key materials from the fake provider2 and compare it to the ones we - // pushed above. + // Push key material for optsFoo, and make sure the foo providers return + // appropriate key material and the bar provider times out. + fooKM := loadKeyMaterials(t, "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem") + fakeProvFoo.newKeyMaterial(fooKM, nil) ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() - gotKM, err := prov2.KeyMaterial(ctx) - if err != nil { - t.Fatalf("provider.KeyMaterial() = %v", err) + if err := readAndVerifyKeyMaterial(ctx, provFoo1, fooKM); err != nil { + t.Fatal(err) + } + if err := readAndVerifyKeyMaterial(ctx, provFoo2, fooKM); err != nil { + t.Fatal(err) } - if !reflect.DeepEqual(gotKM, wantKM) { - t.Fatalf("provider.KeyMaterial() = %+v, want %+v", gotKM, wantKM) + if err := readAndVerifyKeyMaterial(ctx, provBar1, nil); !errors.Is(err, context.DeadlineExceeded) { + t.Fatal(err) } - // Close the provider1 and retry the KeyMaterial() call on prov2, and expect - // it to succeed. - prov1.Close() - if _, err := prov2.KeyMaterial(ctx); err != nil { - t.Fatalf("provider.KeyMaterial() = %v", err) + // Push key material for optsBar, and make sure the bar provider returns + // appropriate key material. + barKM := loadKeyMaterials(t, "x509/server2_cert.pem", "x509/server2_key.pem", "x509/client_ca_cert.pem") + fakeProvBar.newKeyMaterial(barKM, nil) + ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := readAndVerifyKeyMaterial(ctx, provBar1, barKM); err != nil { + t.Fatal(err) } - prov2.Close() - if _, err := prov2.KeyMaterial(ctx); err != errProviderClosed { - t.Fatalf("provider.KeyMaterial() = %v, wantErr: %v", err, errProviderClosed) + // Make sure the above push of new key material does not affect foo readers. + if err := readAndVerifyKeyMaterial(ctx, provFoo1, fooKM); err != nil { + t.Fatal(err) } } -// TestStoreWithSingleProviderWithoutSharing creates multiple instances of the +// TestStoreSingleProviderDifferentConfigs creates multiple instances of the // same type of provider through the store with different configs. The store // would end up creating different provider instances for these and no sharing // would take place. -func (s) TestStoreWithSingleProviderWithoutSharing(t *testing.T) { - prov1, fp1 := makeProvider(t, fakeProvider1Name, fakeConfig+"1111") - prov2, fp2 := makeProvider(t, fakeProvider1Name, fakeConfig+"2222") - - // Push the same key materials into the two providers. - wantKM, err := loadKeyMaterials() +func (s) TestStoreSingleProviderDifferentConfigs(t *testing.T) { + // Create two providers of the same type, but with different configs. + opts := Options{CertName: "foo"} + cfg1 := fakeConfig + "1111" + cfg2 := fakeConfig + "2222" + prov1, err := GetProvider(fakeProvider1Name, cfg1, opts) if err != nil { - t.Fatal(err) + t.Fatalf("GetProvider(%s, %s, %v) failed: %v", fakeProvider1Name, cfg1, opts, err) } - fp1.kmCh <- wantKM - fp2.kmCh <- wantKM - - // Get key materials from the fake provider1 and compare it to the ones we - // pushed above. - ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) - defer cancel() - gotKM, err := prov1.KeyMaterial(ctx) + defer prov1.Close() + // Our fakeProviderBuilder pushes newly created providers on a channel. Grab + // the fake provider from that channel. + p1, err := fpb1.providerChan.Receive() if err != nil { - t.Fatalf("provider.KeyMaterial() = %v", err) - } - if !reflect.DeepEqual(gotKM, wantKM) { - t.Fatalf("provider.KeyMaterial() = %+v, want %+v", gotKM, wantKM) + t.Fatalf("Timeout when expecting certProvider %q to be created", fakeProvider1Name) } + fakeProv1 := p1.(*fakeProvider) - // Get key materials from the fake provider2 and compare it to the ones we - // pushed above. - gotKM, err = prov2.KeyMaterial(ctx) + prov2, err := GetProvider(fakeProvider1Name, cfg2, opts) if err != nil { - t.Fatalf("provider.KeyMaterial() = %v", err) + t.Fatalf("GetProvider(%s, %s, %v) failed: %v", fakeProvider1Name, cfg2, opts, err) } - if !reflect.DeepEqual(gotKM, wantKM) { - t.Fatalf("provider.KeyMaterial() = %+v, want %+v", gotKM, wantKM) + defer prov2.Close() + // Grab the second provider from the channel. + p2, err := fpb1.providerChan.Receive() + if err != nil { + t.Fatalf("Timeout when expecting certProvider %q to be created", fakeProvider1Name) } + fakeProv2 := p2.(*fakeProvider) - // Update the key materials used by provider1, and make sure provider2 is - // not affected. - newKM, err := loadKeyMaterials() - if err != nil { + // Push the same key material into both fake providers and verify that the + // providers returned by the store return the appropriate key material. + km1 := loadKeyMaterials(t, "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem") + fakeProv1.newKeyMaterial(km1, nil) + fakeProv2.newKeyMaterial(km1, nil) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := readAndVerifyKeyMaterial(ctx, prov1, km1); err != nil { + t.Fatal(err) + } + if err := readAndVerifyKeyMaterial(ctx, prov2, km1); err != nil { t.Fatal(err) } - newKM.Roots = nil - fp1.kmCh <- newKM - gotKM, err = prov2.KeyMaterial(ctx) - if err != nil { - t.Fatalf("provider.KeyMaterial() = %v", err) + // Push new key material into only one of the fake providers and and verify + // that the providers returned by the store return the appropriate key + // material. + km2 := loadKeyMaterials(t, "x509/server2_cert.pem", "x509/server2_key.pem", "x509/client_ca_cert.pem") + fakeProv2.newKeyMaterial(km2, nil) + ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := readAndVerifyKeyMaterial(ctx, prov1, km1); err != nil { + t.Fatal(err) } - if !reflect.DeepEqual(gotKM, wantKM) { - t.Fatalf("provider.KeyMaterial() = %+v, want %+v", gotKM, wantKM) + if err := readAndVerifyKeyMaterial(ctx, prov2, km2); err != nil { + t.Fatal(err) } - // Close the provider1 and retry the KeyMaterial() call on prov2, and expect - // it to succeed. + // Close one of the providers and verify that the other one is not affected. prov1.Close() - if _, err := prov2.KeyMaterial(ctx); err != nil { - t.Fatalf("provider.KeyMaterial() = %v", err) - } - - prov2.Close() - if _, err := prov2.KeyMaterial(ctx); err != errProviderClosed { - t.Fatalf("provider.KeyMaterial() = %v, wantErr: %v", err, errProviderClosed) + if err := readAndVerifyKeyMaterial(ctx, prov2, km2); err != nil { + t.Fatal(err) } } -// TestStoreWithMultipleProviders creates multiple providers of different types -// and make sure closing of one does not affect the other. -func (s) TestStoreWithMultipleProviders(t *testing.T) { - prov1, fp1 := makeProvider(t, fakeProvider1Name, fakeConfig) - prov2, fp2 := makeProvider(t, fakeProvider2Name, fakeConfig) - - // Push key materials into the fake providers. - wantKM, err := loadKeyMaterials() +// TestStoreMultipleProviders creates providers of different types and makes +// sure closing of one does not affect the other. +func (s) TestStoreMultipleProviders(t *testing.T) { + opts := Options{CertName: "foo"} + prov1, err := GetProvider(fakeProvider1Name, fakeConfig, opts) if err != nil { - t.Fatal(err) + t.Fatalf("GetProvider(%s, %s, %v) failed: %v", fakeProvider1Name, fakeConfig, opts, err) } - fp1.kmCh <- wantKM - fp2.kmCh <- wantKM - - // Get key materials from the fake provider1 and compare it to the ones we - // pushed above. - ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) - defer cancel() - gotKM, err := prov1.KeyMaterial(ctx) + defer prov1.Close() + // Our fakeProviderBuilder pushes newly created providers on a channel. Grab + // the fake provider from that channel. + p1, err := fpb1.providerChan.Receive() if err != nil { - t.Fatalf("provider.KeyMaterial() = %v", err) - } - if !reflect.DeepEqual(gotKM, wantKM) { - t.Fatalf("provider.KeyMaterial() = %+v, want %+v", gotKM, wantKM) + t.Fatalf("Timeout when expecting certProvider %q to be created", fakeProvider1Name) } + fakeProv1 := p1.(*fakeProvider) - // Get key materials from the fake provider2 and compare it to the ones we - // pushed above. - gotKM, err = prov2.KeyMaterial(ctx) + prov2, err := GetProvider(fakeProvider2Name, fakeConfig, opts) if err != nil { - t.Fatalf("provider.KeyMaterial() = %v", err) + t.Fatalf("GetProvider(%s, %s, %v) failed: %v", fakeProvider1Name, fakeConfig, opts, err) } - if !reflect.DeepEqual(gotKM, wantKM) { - t.Fatalf("provider.KeyMaterial() = %+v, want %+v", gotKM, wantKM) + defer prov2.Close() + // Grab the second provider from the channel. + p2, err := fpb2.providerChan.Receive() + if err != nil { + t.Fatalf("Timeout when expecting certProvider %q to be created", fakeProvider2Name) } + fakeProv2 := p2.(*fakeProvider) - // Close the provider1 and retry the KeyMaterial() call on prov2, and expect - // it to succeed. - prov1.Close() - if _, err := prov2.KeyMaterial(ctx); err != nil { - t.Fatalf("provider.KeyMaterial() = %v", err) + // Push the key material into both providers and verify that the + // readers return the appropriate key material. + km1 := loadKeyMaterials(t, "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem") + fakeProv1.newKeyMaterial(km1, nil) + km2 := loadKeyMaterials(t, "x509/server2_cert.pem", "x509/server2_key.pem", "x509/client_ca_cert.pem") + fakeProv2.newKeyMaterial(km2, nil) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := readAndVerifyKeyMaterial(ctx, prov1, km1); err != nil { + t.Fatal(err) + } + if err := readAndVerifyKeyMaterial(ctx, prov2, km2); err != nil { + t.Fatal(err) } - prov2.Close() - if _, err := prov2.KeyMaterial(ctx); err != errProviderClosed { - t.Fatalf("provider.KeyMaterial() = %v, wantErr: %v", err, errProviderClosed) + // Close one of the providers and verify that the other one is not affected. + prov1.Close() + if err := readAndVerifyKeyMaterial(ctx, prov2, km2); err != nil { + t.Fatal(err) } }