Skip to content

Commit

Permalink
Backport of [NET-5772] Make tcp external service registered on termin…
Browse files Browse the repository at this point in the history 
…ating gw reachable from peered cluster into release/1.18.x (#20946)

* backport of commit 7ed7f23

* backport of commit 9c8529b

* backport of commit 5753ee8

* backport of commit 17bc757

* backport of commit d0fa205

---------

Co-authored-by: Nathan Coleman <[email protected]>
  • Loading branch information
@hc-github-team-consul-core @nathancoleman
hc-github-team-consul-core and nathancoleman authored Apr 3, 2024
1 parent f1fdeba commit 0e62661
Show file tree
Hide file tree
Showing 4 changed files with 97 additions and 46 deletions.
3 changes: 3 additions & 0 deletions .changelog/19881.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
```release-note:bug
xds: Make TCP external service registered with terminating gateway reachable from peered cluster
```
70 changes: 46 additions & 24 deletions agent/xds/listeners.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import (
envoy_tcp_proxy_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/tcp_proxy/v3"
envoy_tls_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
envoy_type_v3 "github.com/envoyproxy/go-control-plane/envoy/type/v3"

"github.com/hashicorp/consul/agent/xds/config"
"github.com/hashicorp/consul/agent/xds/naming"
"github.com/hashicorp/consul/agent/xds/platform"
Expand Down Expand Up @@ -1178,29 +1179,9 @@ func createDownstreamTransportSocketForConnectTLS(cfgSnap *proxycfg.ConfigSnapsh
}

// Inject peering trust bundles if this service is exported to peered clusters.
if len(peerBundles) > 0 {
spiffeConfig, err := makeSpiffeValidatorConfig(
cfgSnap.Roots.TrustDomain,
cfgSnap.RootPEMs(),
peerBundles,
)
if err != nil {
return nil, err
}

typ, ok := tlsContext.ValidationContextType.(*envoy_tls_v3.CommonTlsContext_ValidationContext)
if !ok {
return nil, fmt.Errorf("unexpected type for TLS context validation: %T", tlsContext.ValidationContextType)
}

// makeCommonTLSFromLead injects the local trust domain's CA root certs as the TrustedCA.
// We nil it out here since the local roots are included in the SPIFFE validator config.
typ.ValidationContext.TrustedCa = nil
typ.ValidationContext.CustomValidatorConfig = &envoy_core_v3.TypedExtensionConfig{
// The typed config name is hard-coded because it is not available as a wellknown var in the control plane lib.
Name: "envoy.tls.cert_validator.spiffe",
TypedConfig: spiffeConfig,
}
err := injectSpiffeValidatorConfigForPeers(cfgSnap, tlsContext, peerBundles)
if err != nil {
return nil, err
}

return makeDownstreamTLSTransportSocket(&envoy_tls_v3.DownstreamTlsContext{
Expand All @@ -1209,6 +1190,32 @@ func createDownstreamTransportSocketForConnectTLS(cfgSnap *proxycfg.ConfigSnapsh
})
}

func injectSpiffeValidatorConfigForPeers(cfgSnap *proxycfg.ConfigSnapshot, tlsContext *envoy_tls_v3.CommonTlsContext, peerBundles []*pbpeering.PeeringTrustBundle) error {
if len(peerBundles) == 0 {
return nil
}

spiffeConfig, err := makeSpiffeValidatorConfig(cfgSnap.Roots.TrustDomain, cfgSnap.RootPEMs(), peerBundles)
if err != nil {
return err
}

typ, ok := tlsContext.ValidationContextType.(*envoy_tls_v3.CommonTlsContext_ValidationContext)
if !ok {
return fmt.Errorf("unexpected type for TLS context validation: %T", tlsContext.ValidationContextType)
}

// makeCommonTLSFromLead injects the local trust domain's CA root certs as the TrustedCA.
// We nil it out here since the local roots are included in the SPIFFE validator config.
typ.ValidationContext.TrustedCa = nil
typ.ValidationContext.CustomValidatorConfig = &envoy_core_v3.TypedExtensionConfig{
// The typed config name is hard-coded because it is not available as a wellknown var in the control plane lib.
Name: "envoy.tls.cert_validator.spiffe",
TypedConfig: spiffeConfig,
}
return nil
}

// SPIFFECertValidatorConfig is used to validate certificates from trust domains other than our own.
// With cluster peering we expect peered clusters to have independent certificate authorities.
// This means that we cannot use a single set of root CA certificates to validate client certificates for mTLS,
Expand Down Expand Up @@ -1752,6 +1759,15 @@ type terminatingGatewayFilterChainOpts struct {
}

func (s *ResourceGenerator) makeFilterChainTerminatingGateway(cfgSnap *proxycfg.ConfigSnapshot, tgtwyOpts terminatingGatewayFilterChainOpts) (*envoy_listener_v3.FilterChain, error) {
// We need to at least match the SNI and use the root PEMs from the local cluster; however, requests coming
// from peered clusters where the external service is exported to will have their own SNI and root PEMs.
sniMatches := []string{tgtwyOpts.cluster}
for _, bundle := range tgtwyOpts.peerTrustBundles {
svc := tgtwyOpts.service
sourceSNI := connect.PeeredServiceSNI(svc.Name, svc.NamespaceOrDefault(), svc.PartitionOrDefault(), bundle.PeerName, cfgSnap.Roots.TrustDomain)
sniMatches = append(sniMatches, sourceSNI)
}

tlsContext := &envoy_tls_v3.DownstreamTlsContext{
CommonTlsContext: makeCommonTLSContext(
cfgSnap.TerminatingGateway.ServiceLeaves[tgtwyOpts.service],
Expand All @@ -1760,13 +1776,19 @@ func (s *ResourceGenerator) makeFilterChainTerminatingGateway(cfgSnap *proxycfg.
),
RequireClientCertificate: &wrapperspb.BoolValue{Value: true},
}

err := injectSpiffeValidatorConfigForPeers(cfgSnap, tlsContext.CommonTlsContext, tgtwyOpts.peerTrustBundles)
if err != nil {
return nil, err
}

transportSocket, err := makeDownstreamTLSTransportSocket(tlsContext)
if err != nil {
return nil, err
}

filterChain := &envoy_listener_v3.FilterChain{
FilterChainMatch: makeSNIFilterChainMatch(tgtwyOpts.cluster),
FilterChainMatch: makeSNIFilterChainMatch(sniMatches...),
Filters: make([]*envoy_listener_v3.Filter, 0, 3),
TransportSocket: transportSocket,
}
Expand Down
28 changes: 10 additions & 18 deletions agent/xds/resources_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@ import (
"github.com/hashicorp/consul/agent/xds/testcommon"
"github.com/hashicorp/consul/agent/xdsv2"
"github.com/hashicorp/consul/envoyextensions/xdscommon"
"github.com/hashicorp/consul/proto/private/pbpeering"
"github.com/hashicorp/consul/sdk/testutil"
"github.com/hashicorp/consul/types"
)
Expand Down Expand Up @@ -2281,32 +2280,25 @@ func getTerminatingGatewayPeeringGoldenTestCases() []goldenTestCase {
{
name: "terminating-gateway-with-peer-trust-bundle",
create: func(t testinf.T) *proxycfg.ConfigSnapshot {
roots, _ := proxycfg.TestCerts(t)
bundles := proxycfg.TestPeerTrustBundles(t)
return proxycfg.TestConfigSnapshotTerminatingGateway(t, true, nil, []proxycfg.UpdateEvent{
{
CorrelationID: "peer-trust-bundle:web",
Result: &pbpeering.TrustBundleListByServiceResponse{
Bundles: []*pbpeering.PeeringTrustBundle{
{
TrustDomain: "foo.bar.gov",
PeerName: "dc2",
Partition: "default",
RootPEMs: []string{
roots.Roots[0].RootCert,
},
ExportedPartition: "default",
CreateIndex: 0,
ModifyIndex: 0,
},
},
},
Result: bundles,
},
{
CorrelationID: "service-intentions:web",
Result: structs.SimplifiedIntentions{
{
SourceName: "source",
SourcePeer: "dc2",
SourcePeer: bundles.Bundles[0].PeerName,
DestinationName: "web",
DestinationPartition: "default",
Action: structs.IntentionActionAllow,
},
{
SourceName: "source",
SourcePeer: bundles.Bundles[1].PeerName,
DestinationName: "web",
DestinationPartition: "default",
Action: structs.IntentionActionAllow,
Expand Down
42 changes: 38 additions & 4 deletions agent/xds/testdata/listeners/terminating-gateway-with-peer-trust-bundle.latest.golden
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,9 @@
{
"filterChainMatch": {
"serverNames": [
"web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
"web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"web.default.default.peer-a.external.11111111-2222-3333-4444-555555555555.consul",
"web.default.default.peer-b.external.11111111-2222-3333-4444-555555555555.consul"
]
},
"filters": [
Expand All @@ -184,7 +186,16 @@
"authenticated": {
"principalName": {
"safeRegex": {
"regex": "^spiffe://foo.bar.gov/ns/default/dc/[^/]+/svc/source$"
"regex": "^spiffe://1c053652-8512-4373-90cf-5a7f6263a994.consul/ns/default/dc/[^/]+/svc/source$"
}
}
}
},
{
"authenticated": {
"principalName": {
"safeRegex": {
"regex": "^spiffe://d89ac423-e95a-475d-94f2-1c557c57bf31.consul/ns/default/dc/[^/]+/svc/source$"
}
}
}
Expand Down Expand Up @@ -222,8 +233,31 @@
],
"tlsParams": {},
"validationContext": {
"trustedCa": {
"inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n"
"customValidatorConfig": {
"name": "envoy.tls.cert_validator.spiffe",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.SPIFFECertValidatorConfig",
"trustDomains": [
{
"name": "11111111-2222-3333-4444-555555555555.consul",
"trustBundle": {
"inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n"
}
},
{
"name": "1c053652-8512-4373-90cf-5a7f6263a994.consul",
"trustBundle": {
"inlineString": "-----BEGIN CERTIFICATE-----\nMIICczCCAdwCCQC3BLnEmLCrSjANBgkqhkiG9w0BAQsFADB+MQswCQYDVQQGEwJV\nUzELMAkGA1UECAwCQVoxEjAQBgNVBAcMCUZsYWdzdGFmZjEMMAoGA1UECgwDRm9v\nMRAwDgYDVQQLDAdleGFtcGxlMQ8wDQYDVQQDDAZwZWVyLWExHTAbBgkqhkiG9w0B\nCQEWDmZvb0BwZWVyLWEuY29tMB4XDTIyMDUyNjAxMDQ0NFoXDTIzMDUyNjAxMDQ0\nNFowfjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkFaMRIwEAYDVQQHDAlGbGFnc3Rh\nZmYxDDAKBgNVBAoMA0ZvbzEQMA4GA1UECwwHZXhhbXBsZTEPMA0GA1UEAwwGcGVl\nci1hMR0wGwYJKoZIhvcNAQkBFg5mb29AcGVlci1hLmNvbTCBnzANBgkqhkiG9w0B\nAQEFAAOBjQAwgYkCgYEA2zFYGTbXDAntT5pLTpZ2+VTiqx4J63VRJH1kdu11f0FV\nc2jl1pqCuYDbQXknDU0Pv1Q5y0+nSAihD2KqGS571r+vHQiPtKYPYRqPEe9FzAhR\n2KhWH6v/tk5DG1HqOjV9/zWRKB12gdFNZZqnw/e7NjLNq3wZ2UAwxXip5uJ8uwMC\nAwEAATANBgkqhkiG9w0BAQsFAAOBgQC/CJ9Syf4aL91wZizKTejwouRYoWv4gRAk\nyto45ZcNMHfJ0G2z+XAMl9ZbQsLgXmzAx4IM6y5Jckq8pKC4PEijCjlKTktLHlEy\n0ggmFxtNB1tid2NC8dOzcQ3l45+gDjDqdILhAvLDjlAIebdkqVqb2CfFNW/I2CQH\nZAuKN1aoKA==\n-----END CERTIFICATE-----\n"
}
},
{
"name": "d89ac423-e95a-475d-94f2-1c557c57bf31.consul",
"trustBundle": {
"inlineString": "-----BEGIN CERTIFICATE-----\nMIICcTCCAdoCCQDyGxC08cD0BDANBgkqhkiG9w0BAQsFADB9MQswCQYDVQQGEwJV\nUzELMAkGA1UECAwCQ0ExETAPBgNVBAcMCENhcmxzYmFkMQwwCgYDVQQKDANGb28x\nEDAOBgNVBAsMB2V4YW1wbGUxDzANBgNVBAMMBnBlZXItYjEdMBsGCSqGSIb3DQEJ\nARYOZm9vQHBlZXItYi5jb20wHhcNMjIwNTI2MDExNjE2WhcNMjMwNTI2MDExNjE2\nWjB9MQswCQYDVQQGEwJVUzELMAkGA1UECAwCQ0ExETAPBgNVBAcMCENhcmxzYmFk\nMQwwCgYDVQQKDANGb28xEDAOBgNVBAsMB2V4YW1wbGUxDzANBgNVBAMMBnBlZXIt\nYjEdMBsGCSqGSIb3DQEJARYOZm9vQHBlZXItYi5jb20wgZ8wDQYJKoZIhvcNAQEB\nBQADgY0AMIGJAoGBAL4i5erdZ5vKk3mzW9Qt6Wvw/WN/IpMDlL0a28wz9oDCtMLN\ncD/XQB9yT5jUwb2s4mD1lCDZtee8MHeD8zygICozufWVB+u2KvMaoA50T9GMQD0E\nz/0nz/Z703I4q13VHeTpltmEpYcfxw/7nJ3leKA34+Nj3zteJ70iqvD/TNBBAgMB\nAAEwDQYJKoZIhvcNAQELBQADgYEAbL04gicH+EIznDNhZJEb1guMBtBBJ8kujPyU\nao8xhlUuorDTLwhLpkKsOhD8619oSS8KynjEBichidQRkwxIaze0a2mrGT+tGBMf\npVz6UeCkqpde6bSJ/ozEe/2seQzKqYvRT1oUjLwYvY7OIh2DzYibOAxh6fewYAmU\n5j5qNLc=\n-----END CERTIFICATE-----\n"
}
}
]
}
}
}
},
Expand Down

0 comments on commit 0e62661

Please sign in to comment.