From 026e2bd26eb8a5dd9d7ef9ee0b9f88d064aad518 Mon Sep 17 00:00:00 2001 From: Ashwin Venkatesh Date: Tue, 22 Aug 2023 13:41:49 -0400 Subject: [PATCH] Update trust bundle into proxy-state-template --- agent/consul/server.go | 22 +++++++-- internal/mesh/exports.go | 10 ++++ .../mesh/internal/controllers/register.go | 22 +++++++++ .../internal/controllers/xds/controller.go | 31 ++++++++++-- .../controllers/xds/controller_test.go | 48 +++++++++++++++++-- .../internal/controllers/xds/mock_updater.go | 10 ++++ .../internal/controllers/xds/status/status.go | 9 ++++ 7 files changed, 139 insertions(+), 13 deletions(-) create mode 100644 internal/mesh/internal/controllers/register.go diff --git a/agent/consul/server.go b/agent/consul/server.go index 4346bb6131388..e440c6c27da9c 100644 --- a/agent/consul/server.go +++ b/agent/consul/server.go @@ -19,9 +19,8 @@ import ( "sync/atomic" "time" - "github.com/hashicorp/consul/internal/resource" - "github.com/armon/go-metrics" + "github.com/hashicorp/consul-net-rpc/net/rpc" "github.com/hashicorp/go-connlimit" "github.com/hashicorp/go-hclog" "github.com/hashicorp/go-memdb" @@ -38,8 +37,6 @@ import ( "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/reflection" - "github.com/hashicorp/consul-net-rpc/net/rpc" - "github.com/hashicorp/consul/acl" "github.com/hashicorp/consul/acl/resolver" "github.com/hashicorp/consul/agent/blockingquery" @@ -75,6 +72,8 @@ import ( "github.com/hashicorp/consul/agent/token" "github.com/hashicorp/consul/internal/catalog" "github.com/hashicorp/consul/internal/controller" + "github.com/hashicorp/consul/internal/mesh" + "github.com/hashicorp/consul/internal/resource" "github.com/hashicorp/consul/internal/resource/demo" "github.com/hashicorp/consul/internal/resource/reaper" raftstorage "github.com/hashicorp/consul/internal/storage/raft" @@ -82,6 +81,7 @@ import ( "github.com/hashicorp/consul/lib/routine" "github.com/hashicorp/consul/lib/stringslice" "github.com/hashicorp/consul/logging" + "github.com/hashicorp/consul/proto-public/pbmesh/v1alpha1/pbproxystate" "github.com/hashicorp/consul/proto-public/pbresource" "github.com/hashicorp/consul/proto/private/pbsubscribe" "github.com/hashicorp/consul/tlsutil" @@ -876,6 +876,20 @@ func NewServer(config *Config, flat Deps, externalGRPCServer *grpc.Server, incom func (s *Server) registerControllers(deps Deps) { if stringslice.Contains(deps.Experiments, catalogResourceExperimentName) { catalog.RegisterControllers(s.controllerManager, catalog.DefaultControllerDependencies()) + mesh.RegisterControllers(s.controllerManager, mesh.ControllerDependencies{ + TrustBundleFetcher: func() (*pbproxystate.TrustBundle, error) { + var bundle pbproxystate.TrustBundle + roots, err := s.getCARoots(nil, s.GetState()) + if err != nil { + return nil, err + } + bundle.TrustDomain = roots.TrustDomain + for _, root := range roots.Roots { + bundle.Roots = append(bundle.Roots, root.RootCert) + } + return &bundle, nil + }, + }) } reaper.RegisterControllers(s.controllerManager) diff --git a/internal/mesh/exports.go b/internal/mesh/exports.go index 2e40e05893078..6a6f97221f303 100644 --- a/internal/mesh/exports.go +++ b/internal/mesh/exports.go @@ -4,6 +4,8 @@ package mesh import ( + "github.com/hashicorp/consul/internal/controller" + "github.com/hashicorp/consul/internal/mesh/internal/controllers" "github.com/hashicorp/consul/internal/mesh/internal/types" "github.com/hashicorp/consul/internal/resource" ) @@ -57,3 +59,11 @@ var ( func RegisterTypes(r resource.Registry) { types.Register(r) } + +// RegisterControllers registers controllers for the mesh types with +// the given controller Manager. +func RegisterControllers(mgr *controller.Manager, deps ControllerDependencies) { + controllers.Register(mgr, deps) +} + +type ControllerDependencies = controllers.Dependencies diff --git a/internal/mesh/internal/controllers/register.go b/internal/mesh/internal/controllers/register.go new file mode 100644 index 0000000000000..b6b8ac855b5bb --- /dev/null +++ b/internal/mesh/internal/controllers/register.go @@ -0,0 +1,22 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package controllers + +import ( + "github.com/hashicorp/consul/internal/catalog" + "github.com/hashicorp/consul/internal/controller" + "github.com/hashicorp/consul/internal/mesh/internal/controllers/xds" + "github.com/hashicorp/consul/internal/mesh/internal/types" + "github.com/hashicorp/consul/internal/resource/mappers/bimapper" +) + +type Dependencies struct { + TrustBundleFetcher xds.TrustBundleFetcher +} + +func Register(mgr *controller.Manager, deps Dependencies) { + mapper := bimapper.New(types.ProxyStateTemplateType, catalog.ServiceEndpointsType) + // TODO: Pass in a "real" updater once proxy tracker work has completed. + mgr.Register(xds.Controller(mapper, nil, deps.TrustBundleFetcher)) +} diff --git a/internal/mesh/internal/controllers/xds/controller.go b/internal/mesh/internal/controllers/xds/controller.go index 5d1e5da7cc6cf..6bc49f9236ea6 100644 --- a/internal/mesh/internal/controllers/xds/controller.go +++ b/internal/mesh/internal/controllers/xds/controller.go @@ -16,9 +16,10 @@ import ( const ControllerName = "consul.io/xds-controller" -func Controller(mapper *bimapper.Mapper, updater ProxyUpdater) controller.Controller { - if mapper == nil || updater == nil { - panic("mapper and updater are required") +func Controller(mapper *bimapper.Mapper, updater ProxyUpdater, fetcher TrustBundleFetcher) controller.Controller { + //if mapper == nil || updater == nil || fetcher == nil { + if mapper == nil || fetcher == nil { + panic("mapper, updater and fetcher are required") } return controller.ForType(types.ProxyStateTemplateType). @@ -28,10 +29,13 @@ func Controller(mapper *bimapper.Mapper, updater ProxyUpdater) controller.Contro } type xdsReconciler struct { - bimapper *bimapper.Mapper - updater ProxyUpdater + bimapper *bimapper.Mapper + updater ProxyUpdater + fetchTrustBundle TrustBundleFetcher } +type TrustBundleFetcher func() (*pbproxystate.TrustBundle, error) + // ProxyUpdater is an interface that defines the ability to push proxy updates to the updater // and also check its connectivity to the server. type ProxyUpdater interface { @@ -78,6 +82,23 @@ func (r *xdsReconciler) Reconcile(ctx context.Context, rt controller.Runtime, re return err } + + // TODO: Fetch trust bundles for all peers when peering is supported. + trustBundle, err := r.fetchTrustBundle() + if err != nil { + rt.Logger.Error("error fetching root trust bundle", "error", err) + // Set the status. + statusCondition = status.ConditionRejectedTrustBundleFetchFailed(status.KeyFromID(req.ID)) + status.WriteStatusIfChanged(ctx, rt, pstResource, statusCondition) + return err + } + + if proxyStateTemplate.Template.ProxyState.TrustBundles == nil { + proxyStateTemplate.Template.ProxyState.TrustBundles = make(map[string]*pbproxystate.TrustBundle) + } + // TODO: Figure out the correct key for the default trust bundle. + proxyStateTemplate.Template.ProxyState.TrustBundles["local"] = trustBundle + if proxyStateTemplate.Template.ProxyState.Endpoints == nil { proxyStateTemplate.Template.ProxyState.Endpoints = make(map[string]*pbproxystate.Endpoints) } diff --git a/internal/mesh/internal/controllers/xds/controller_test.go b/internal/mesh/internal/controllers/xds/controller_test.go index e6df71c70eda8..8b5268f0e1a50 100644 --- a/internal/mesh/internal/controllers/xds/controller_test.go +++ b/internal/mesh/internal/controllers/xds/controller_test.go @@ -48,6 +48,8 @@ type xdsControllerTestSuite struct { fooBarService *pbresource.Resource expectedFooProxyStateEndpoints map[string]*pbproxystate.Endpoints expectedBarProxyStateEndpoints map[string]*pbproxystate.Endpoints + fetcher TrustBundleFetcher + expectedTrustBundle map[string]*pbproxystate.TrustBundle } func (suite *xdsControllerTestSuite) SetupTest() { @@ -55,16 +57,27 @@ func (suite *xdsControllerTestSuite) SetupTest() { resourceClient := svctest.RunResourceService(suite.T(), types.Register, catalog.RegisterTypes) suite.runtime = controller.Runtime{Client: resourceClient, Logger: testutil.Logger(suite.T())} suite.client = resourcetest.NewClient(resourceClient) + suite.fetcher = mockFetcher suite.mapper = bimapper.New(types.ProxyStateTemplateType, catalog.ServiceEndpointsType) suite.updater = NewMockUpdater() suite.ctl = &xdsReconciler{ - bimapper: suite.mapper, - updater: suite.updater, + bimapper: suite.mapper, + updater: suite.updater, + fetchTrustBundle: suite.fetcher, } } +func mockFetcher() (*pbproxystate.TrustBundle, error) { + var bundle pbproxystate.TrustBundle + bundle = pbproxystate.TrustBundle{ + TrustDomain: "some-trust-domain", + Roots: []string{"some-root", "some-other-root"}, + } + return &bundle, nil +} + // This test ensures when a ProxyState is deleted, it is no longer tracked in the mapper. func (suite *xdsControllerTestSuite) TestReconcile_NoProxyStateTemplate() { // Track the id of a non-existent ProxyStateTemplate. @@ -217,6 +230,25 @@ func (suite *xdsControllerTestSuite) TestReconcile_ProxyStateTemplateComputesEnd prototest.AssertDeepEqual(suite.T(), suite.expectedFooProxyStateEndpoints, actualEndpoints) } +func (suite *xdsControllerTestSuite) TestReconcile_ProxyStateTemplateSetsTrustBundles() { + // This test is a happy path creation test to make sure pbproxystate.Template.TrustBundles are created in the computed + // pbmesh.ProxyState from the TrustBundleFetcher. + suite.setupFooProxyStateTemplateAndEndpoints() + + // Run the reconcile. + err := suite.ctl.Reconcile(context.Background(), suite.runtime, controller.Request{ + ID: suite.fooProxyStateTemplate.Id, + }) + require.NoError(suite.T(), err) + + // Assert on the status. + suite.client.RequireStatusCondition(suite.T(), suite.fooProxyStateTemplate.Id, ControllerName, status.ConditionAccepted()) + + // Assert that the endpoints computed in the controller matches the expected endpoints. + actualTrustBundle := suite.updater.GetTrustBundle(suite.fooProxyStateTemplate.Id.Name) + prototest.AssertDeepEqual(suite.T(), suite.expectedTrustBundle, actualTrustBundle) +} + // This test is a happy path creation test that calls reconcile multiple times with a more complex setup. This // scenario is trickier to test in the controller test because the end computation of the xds controller is not // stored in the state store. So this test ensures that between multiple reconciles the correct ProxyStates are @@ -257,7 +289,7 @@ func (suite *xdsControllerTestSuite) TestReconcile_MultipleProxyStateTemplatesCo func (suite *xdsControllerTestSuite) TestController_ComputeAddUpdateEndpoints() { // Run the controller manager. mgr := controller.NewManager(suite.client, suite.runtime.Logger) - mgr.Register(Controller(suite.mapper, suite.updater)) + mgr.Register(Controller(suite.mapper, suite.updater, mockFetcher)) mgr.SetRaftLeader(true) go mgr.Run(suite.ctx) @@ -470,12 +502,20 @@ func (suite *xdsControllerTestSuite) setupFooProxyStateTemplateAndEndpoints() { }, }}, } + + expectedTrustBundle := map[string]*pbproxystate.TrustBundle{ + "local": { + TrustDomain: "some-trust-domain", + Roots: []string{"some-root", "some-other-root"}, + }, + } + suite.fooService = fooService suite.fooEndpoints = fooEndpoints suite.fooEndpointRefs = fooRequiredEndpoints suite.fooProxyStateTemplate = fooProxyStateTemplate suite.expectedFooProxyStateEndpoints = expectedFooProxyStateEndpoints - + suite.expectedTrustBundle = expectedTrustBundle } // Setup: diff --git a/internal/mesh/internal/controllers/xds/mock_updater.go b/internal/mesh/internal/controllers/xds/mock_updater.go index 33564062bc07c..fc27382d6ef34 100644 --- a/internal/mesh/internal/controllers/xds/mock_updater.go +++ b/internal/mesh/internal/controllers/xds/mock_updater.go @@ -79,6 +79,16 @@ func (p *mockUpdater) GetEndpoints(name string) map[string]*pbproxystate.Endpoin return nil } +func (p *mockUpdater) GetTrustBundle(name string) map[string]*pbproxystate.TrustBundle { + p.lock.Lock() + defer p.lock.Unlock() + ps, ok := p.latestPs[name] + if ok { + return ps.TrustBundles + } + return nil +} + func (p *mockUpdater) Set(name string, ps *pbmesh.ProxyState) { p.lock.Lock() defer p.lock.Unlock() diff --git a/internal/mesh/internal/controllers/xds/status/status.go b/internal/mesh/internal/controllers/xds/status/status.go index 706152329035e..796bec422250e 100644 --- a/internal/mesh/internal/controllers/xds/status/status.go +++ b/internal/mesh/internal/controllers/xds/status/status.go @@ -16,6 +16,7 @@ const ( StatusReasonEndpointNotRead = "ProxyStateEndpointReferenceReadError" StatusReasonCreatingProxyStateEndpointsFailed = "ProxyStateEndpointsNotComputed" StatusReasonPushChangeFailed = "ProxyStatePushChangeFailed" + StatusReasonTrustBundleFetchFailed = "ProxyStateTrustBundleFetchFailed" ) func KeyFromID(id *pbresource.ID) string { @@ -65,6 +66,14 @@ func ConditionRejectedPushChangeFailed(pstRef string) *pbresource.Condition { Message: fmt.Sprintf("failed to push change for proxy state template %q", pstRef), } } +func ConditionRejectedTrustBundleFetchFailed(pstRef string) *pbresource.Condition { + return &pbresource.Condition{ + Type: StatusConditionProxyStateAccepted, + State: pbresource.Condition_STATE_FALSE, + Reason: StatusReasonTrustBundleFetchFailed, + Message: fmt.Sprintf("failed to fetch trust bundle for proxy state template %q", pstRef), + } +} // WriteStatusIfChanged updates the ProxyStateTemplate status if it has changed. func WriteStatusIfChanged(ctx context.Context, rt controller.Runtime, res *pbresource.Resource, condition *pbresource.Condition) {