Skip to content

Commit

Permalink
Merge pull request #68 from numberly/v1-ing-secrets-sync
Browse files Browse the repository at this point in the history 
Add TLS certificates synchronization from ingresses' secrets
  • Loading branch information
@DewaldV
DewaldV authored Aug 17, 2023
2 parents a316893 + 8c901cf commit 7b34337
Show file tree
Hide file tree
Showing 11 changed files with 499 additions and 30 deletions.
9 changes: 9 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,12 +116,21 @@ spec:
servicePort: 80
```

## Dynamic TLS certificates synchronization from Kubernetes secrets

Downstream TLS certificates can be dynamically fetched and updated from Kubernetes secrets configured under ingresses' `spec.tls` by setting `syncSecrets` true in Yggdrasil configuration (false by default).

In this mode, only a single `certificate` may be specified in Yggdrasil configuration. It will be used for hosts with misconfigured or invalid secret.

**Note**: ECDSA >256 keys are not supported by envoy and will be discarded. See https://github.com/envoyproxy/envoy/issues/10855

## Configuration
Yggdrasil can be configured using a config file e.g:
```json
{
"nodeName": "foo",
"ingressClasses": ["multi-cluster", "multi-cluster-staging"],
"syncSecrets": false,
"certificates": [
{
"hosts": ["*.api.com"],
Expand Down
8 changes: 7 additions & 1 deletion cmd/root.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ type config struct {
IngressClass string `json:"ingressClass"`
NodeName string `json:"nodeName"`
Clusters []clusterConfig `json:"clusters"`
SyncSecrets bool `json:"syncSecrets"`
Certificates []envoy.Certificate `json:"certificates"`
TrustCA string `json:"trustCA"`
UpstreamPort uint32 `json:"upstreamPort"`
Expand Down Expand Up @@ -168,6 +169,10 @@ func main(*cobra.Command, []string) error {
log.SetLevel(log.DebugLevel)
}

if c.SyncSecrets && len(c.Certificates) > 1 {
return fmt.Errorf("only one certificate can be declared when syncSecrets is true")
}

clusterSources, err := createSources(c.Clusters)
if err != nil {
return fmt.Errorf("error creating sources: %s", err)
Expand Down Expand Up @@ -218,7 +223,7 @@ func main(*cobra.Command, []string) error {
c.Certificates[idx].Cert = string(certBytes)
c.Certificates[idx].Key = string(keyBytes)
}
aggregator := k8s.NewAggregator(sources, ctx)
aggregator := k8s.NewAggregator(sources, ctx, c.SyncSecrets)
configurator := envoy.NewKubernetesConfigurator(
viper.GetString("nodeName"),
c.Certificates,
Expand All @@ -233,6 +238,7 @@ func main(*cobra.Command, []string) error {
envoy.WithUseRemoteAddress(c.UseRemoteAddress),
envoy.WithHttpExtAuthzCluster(c.HttpExtAuthz),
envoy.WithHttpGrpcLogger(c.HttpGrpcLogger),
envoy.WithSyncSecrets(c.SyncSecrets),
envoy.WithDefaultRetryOn(viper.GetString("retryOn")),
envoy.WithAccessLog(c.AccessLogger),
)
Expand Down
71 changes: 62 additions & 9 deletions pkg/envoy/configurator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,10 @@ import (
route "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
tcache "github.com/envoyproxy/go-control-plane/pkg/cache/types"
cache "github.com/envoyproxy/go-control-plane/pkg/cache/v3"
"google.golang.org/protobuf/types/known/anypb"

"github.com/sirupsen/logrus"
"github.com/uswitch/yggdrasil/pkg/k8s"
"google.golang.org/protobuf/types/known/anypb"
v1 "k8s.io/api/core/v1"
)

type Certificate struct {
Expand Down Expand Up @@ -54,6 +55,7 @@ type AccessLogger struct {
type KubernetesConfigurator struct {
ingressClasses []string
nodeID string
syncSecrets bool
certificates []Certificate
trustCA string
upstreamPort uint32
Expand Down Expand Up @@ -83,12 +85,13 @@ func NewKubernetesConfigurator(nodeID string, certificates []Certificate, ca str
return c
}

// Generate creates a new snapshot
func (c *KubernetesConfigurator) Generate(ingresses []*k8s.Ingress) (cache.Snapshot, error) {
//Generate creates a new snapshot
func (c *KubernetesConfigurator) Generate(ingresses []*k8s.Ingress, secrets []*v1.Secret) (cache.Snapshot, error) {
c.Lock()
defer c.Unlock()

config := translateIngresses(validIngressFilter(classFilter(ingresses, c.ingressClasses)))
validIngresses := validIngressFilter(classFilter(ingresses, c.ingressClasses))
config := translateIngresses(validIngresses, c.syncSecrets, secrets)

vmatch, cmatch := config.equals(c.previousConfig)

Expand Down Expand Up @@ -121,7 +124,7 @@ func (c *KubernetesConfigurator) NodeID() string {

}

var errNoCertificateMatch = errors.New("No certificate match")
var errNoCertificateMatch = errors.New("no certificate match")

func compareHosts(pattern, host string) bool {
patternParts := strings.Split(pattern, ".")
Expand Down Expand Up @@ -161,7 +164,9 @@ func (c *KubernetesConfigurator) matchCertificateIndices(virtualHost *virtualHos
func (c *KubernetesConfigurator) generateListeners(config *envoyConfiguration) ([]tcache.Resource, error) {
var filterChains []*listener.FilterChain
var err error
if len(c.certificates) > 0 {
if c.syncSecrets {
filterChains, err = c.generateDynamicTLSFilterChains(config)
} else if len(c.certificates) > 0 {
filterChains, err = c.generateTLSFilterChains(config)
} else {
filterChains, err = c.generateHTTPFilterChain(config)
Expand All @@ -173,6 +178,54 @@ func (c *KubernetesConfigurator) generateListeners(config *envoyConfiguration) (
return []tcache.Resource{listener}, err
}

func (c *KubernetesConfigurator) generateDynamicTLSFilterChains(config *envoyConfiguration) ([]*listener.FilterChain, error) {
filterChains := []*listener.FilterChain{}

allVhosts := []*route.VirtualHost{}

for _, virtualHost := range config.VirtualHosts {
envoyVhost, err := makeVirtualHost(virtualHost, c.hostSelectionRetryAttempts, c.defaultRetryOn)
if err != nil {
return nil, err
}
allVhosts = append(allVhosts, envoyVhost)

if virtualHost.TlsCert == "" || virtualHost.TlsKey == "" {
if len(c.certificates) == 0 {
logrus.Warnf("skipping vhost because of no certificate: %s", virtualHost.Host)
} else {
logrus.Infof("using default certificate for %s", virtualHost.Host)
}
continue
}
certificate := Certificate{
Hosts: []string{virtualHost.Host},
Cert: virtualHost.TlsCert,
Key: virtualHost.TlsKey,
}
filterChain, err := c.makeFilterChain(certificate, []*route.VirtualHost{envoyVhost})
if err != nil {
logrus.Warnf("error making filter chain: %v", err)
}
filterChains = append(filterChains, &filterChain)
}

if len(c.certificates) == 1 {
defaultCert := Certificate{
Hosts: []string{"*"},
Cert: c.certificates[0].Cert,
Key: c.certificates[0].Key,
}
if defaultFC, err := c.makeFilterChain(defaultCert, allVhosts); err != nil {
logrus.Warnf("error making default filter chain: %v", err)
} else {
filterChains = append(filterChains, &defaultFC)
}
}

return filterChains, nil
}

func (c *KubernetesConfigurator) generateHTTPFilterChain(config *envoyConfiguration) ([]*listener.FilterChain, error) {
virtualHosts := []*route.VirtualHost{}
for _, virtualHost := range config.VirtualHosts {
Expand Down Expand Up @@ -209,7 +262,7 @@ func (c *KubernetesConfigurator) generateTLSFilterChains(config *envoyConfigurat
for _, virtualHost := range config.VirtualHosts {
certificateIndicies, err := c.matchCertificateIndices(virtualHost)
if err != nil {
log.Printf("Error matching certificate for '%s': %v", virtualHost.Host, err)
log.Printf("error matching certificate for '%s': %v", virtualHost.Host, err)
} else {
for _, idx := range certificateIndicies {
vhost, err := makeVirtualHost(virtualHost, c.hostSelectionRetryAttempts, c.defaultRetryOn)
Expand All @@ -231,7 +284,7 @@ func (c *KubernetesConfigurator) generateTLSFilterChains(config *envoyConfigurat

filterChain, err := c.makeFilterChain(certificate, virtualHosts)
if err != nil {
log.Printf("Error making filter chain: %v", err)
log.Printf("error making filter chain: %v", err)
}

filterChains = append(filterChains, &filterChain)
Expand Down
11 changes: 6 additions & 5 deletions pkg/envoy/configurator_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
hcm "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
tcache "github.com/envoyproxy/go-control-plane/pkg/cache/types"
"github.com/uswitch/yggdrasil/pkg/k8s"
v1 "k8s.io/api/core/v1"
)

func assertNumberOfVirtualHosts(t *testing.T, filterChain *listener.FilterChain, expected int) {
Expand Down Expand Up @@ -53,7 +54,7 @@ func TestGenerate(t *testing.T) {
{Hosts: []string{"*"}, Cert: "b", Key: "c"},
}, "d", []string{"bar"})

snapshot, _ := configurator.Generate(ingresses)
snapshot, _ := configurator.Generate(ingresses, []*v1.Secret{})

if len(snapshot.Resources[tcache.Listener].Items) != 1 {
t.Fatalf("Num listeners: %d", len(snapshot.Resources[tcache.Listener].Items))
Expand All @@ -74,7 +75,7 @@ func TestGenerateMultipleCerts(t *testing.T) {
{Hosts: []string{"*.internal.api.co.uk"}, Cert: "couk", Key: "couk"},
}, "d", []string{"bar"})

snapshot, err := configurator.Generate(ingresses)
snapshot, err := configurator.Generate(ingresses, []*v1.Secret{})
if err != nil {
t.Fatalf("Error generating snapshot %v", err)
}
Expand All @@ -99,7 +100,7 @@ func TestGenerateMultipleHosts(t *testing.T) {
{Hosts: []string{"*.internal.api.com", "*.internal.api.co.uk"}, Cert: "com", Key: "com"},
}, "d", []string{"bar"})

snapshot, err := configurator.Generate(ingresses)
snapshot, err := configurator.Generate(ingresses, []*v1.Secret{})
if err != nil {
t.Fatalf("Error generating snapshot %v", err)
}
Expand All @@ -124,7 +125,7 @@ func TestGenerateNoMatchingCert(t *testing.T) {
{Hosts: []string{"*.internal.api.com"}, Cert: "com", Key: "com"},
}, "d", []string{"bar"})

snapshot, err := configurator.Generate(ingresses)
snapshot, err := configurator.Generate(ingresses, []*v1.Secret{})
if err != nil {
t.Fatalf("Error generating snapshot %v", err)
}
Expand All @@ -146,7 +147,7 @@ func TestGenerateIntoTwoCerts(t *testing.T) {
{Hosts: []string{"*"}, Cert: "all", Key: "all"},
}, "d", []string{"bar"})

snapshot, err := configurator.Generate(ingresses)
snapshot, err := configurator.Generate(ingresses, []*v1.Secret{})
if err != nil {
t.Fatalf("Error generating snapshot %v", err)
}
Expand Down
97 changes: 95 additions & 2 deletions pkg/envoy/ingress_translator.go
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,18 @@
package envoy

import (
"crypto/ecdsa"
"crypto/x509"
"encoding/pem"
"fmt"
"regexp"
"sort"
"strings"
"time"

"github.com/sirupsen/logrus"
"github.com/uswitch/yggdrasil/pkg/k8s"
v1 "k8s.io/api/core/v1"
)

func sortCluster(clusters []*cluster) {
Expand Down Expand Up @@ -65,6 +71,8 @@ type virtualHost struct {
UpstreamCluster string
Timeout time.Duration
PerTryTimeout time.Duration
TlsKey string
TlsCert string
RetryOn string
}

Expand All @@ -77,6 +85,8 @@ func (v *virtualHost) Equals(other *virtualHost) bool {
v.Timeout == other.Timeout &&
v.UpstreamCluster == other.UpstreamCluster &&
v.PerTryTimeout == other.PerTryTimeout &&
v.TlsKey == other.TlsKey &&
v.TlsCert == other.TlsCert &&
v.RetryOn == other.RetryOn
}

Expand Down Expand Up @@ -208,6 +218,75 @@ func (ing *envoyIngress) addTimeout(timeout time.Duration) {
ing.vhost.PerTryTimeout = timeout
}

// hostMatch returns true if tlsHost and ruleHost match, with wildcard support
//
// *.a.b ruleHost accepts tlsHost *.a.b but not a.a.b or a.b or a.a.a.b
// a.a.b ruleHost accepts tlsHost a.a.b and *.a.b but not *.a.a.b
func hostMatch(ruleHost, tlsHost string) bool {
// TODO maybe cache the results for speedup
pattern := strings.ReplaceAll(strings.ReplaceAll(tlsHost, ".", "\\."), "*", "(?:\\*|[a-z0-9][a-z0-9-_]*)")
matched, err := regexp.MatchString("^"+pattern+"$", ruleHost)
if err != nil {
logrus.Errorf("error in ingress hostname comparison: %s", err.Error())
return false
}
return matched
}

// getHostTlsSecret returns the tls secret configured for a given ingress host
func getHostTlsSecret(ingress *k8s.Ingress, host string, secrets []*v1.Secret) (*v1.Secret, error) {
for _, tls := range ingress.TLS {
// TODO prefer a.a.b tls secret over *.a.b for host a.a.b when both are configured
if hostMatch(host, tls.Host) {
for _, secret := range secrets {
if secret.Namespace == ingress.Namespace &&
secret.Name == tls.SecretName {
return secret, nil
}
}
return nil, fmt.Errorf("secret %s/%s not found for host '%s'", ingress.Namespace, tls.SecretName, host)
}
}
return nil, fmt.Errorf("ingress %s/%s - %s has no tls secret configured", ingress.Namespace, ingress.Name, host)
}

// validateTlsSecret checks that the given secret holds valid tls certificate and key
func validateTlsSecret(secret *v1.Secret) (bool, error) {
tlsCert, certOk := secret.Data["tls.crt"]
tlsKey, keyOk := secret.Data["tls.key"]

if !certOk || !keyOk {
logrus.Infof("skipping certificate %s/%s: missing 'tls.crt' or 'tls.key'", secret.Namespace, secret.Name)
return false, nil
}
if len(tlsCert) == 0 || len(tlsKey) == 0 {
logrus.Infof("skipping certificate %s/%s: empty 'tls.crt' or 'tls.key'", secret.Namespace, secret.Name)
return false, nil
}

// discard P-384 EC private keys
// see https://github.com/envoyproxy/envoy/issues/10855
block, _ := pem.Decode(tlsCert)
if block == nil {
return false, fmt.Errorf("error parsing x509 certificate - no PEM block found")
}
x509crt, err := x509.ParseCertificate(block.Bytes)
if err != nil {
return false, fmt.Errorf("error parsing x509 certificate: %s", err.Error())
}
if x509crt.PublicKeyAlgorithm == x509.ECDSA {
ecdsaPub, ok := x509crt.PublicKey.(*ecdsa.PublicKey)
if !ok {
return false, fmt.Errorf("error in *ecdsa.PublicKey type assertion")
}
if ecdsaPub.Curve.Params().BitSize > 256 {
logrus.Infof("skipping ECDSA %s certificate %s/%s: only P-256 certificates are supported", ecdsaPub.Curve.Params().Name, secret.Namespace, secret.Name)
return false, nil
}
}
return true, nil
}

func (envoyIng *envoyIngress) addRetryOn(ingress *k8s.Ingress) {
if ingress.Annotations["yggdrasil.uswitch.com/retry-on"] != "" {
retryOn := ingress.Annotations["yggdrasil.uswitch.com/retry-on"]
Expand All @@ -219,7 +298,7 @@ func (envoyIng *envoyIngress) addRetryOn(ingress *k8s.Ingress) {
}
}

func translateIngresses(ingresses []*k8s.Ingress) *envoyConfiguration {
func translateIngresses(ingresses []*k8s.Ingress, syncSecrets bool, secrets []*v1.Secret) *envoyConfiguration {
cfg := &envoyConfiguration{}
envoyIngresses := map[string]*envoyIngress{}

Expand All @@ -232,7 +311,6 @@ func translateIngresses(ingresses []*k8s.Ingress) *envoyConfiguration {
}

envoyIngress := envoyIngresses[ruleHost]

envoyIngress.addUpstream(j)

if i.Annotations["yggdrasil.uswitch.com/healthcheck-path"] != "" {
Expand All @@ -245,7 +323,22 @@ func translateIngresses(ingresses []*k8s.Ingress) *envoyConfiguration {
envoyIngress.addTimeout(timeout)
}
}

envoyIngress.addRetryOn(i)

if syncSecrets && envoyIngress.vhost.TlsKey == "" && envoyIngress.vhost.TlsCert == "" {
if hostTlsSecret, err := getHostTlsSecret(i, ruleHost, secrets); err != nil {
logrus.Infof(err.Error())
} else {
valid, err := validateTlsSecret(hostTlsSecret)
if err != nil {
logrus.Warnf("secret %s/%s is not valid: %s", hostTlsSecret.Namespace, hostTlsSecret.Name, err.Error())
} else if valid {
envoyIngress.vhost.TlsKey = string(hostTlsSecret.Data["tls.key"])
envoyIngress.vhost.TlsCert = string(hostTlsSecret.Data["tls.crt"])
}
}
}
}
}
}
Expand Down
Loading

0 comments on commit 7b34337

Please sign in to comment.