From c7b5a3bd8027642f2be4802a8ea39bd1f031ffe1 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 27 Jun 2022 14:14:38 -0400 Subject: [PATCH 1/4] maintain: move server TLS funcs These function will be modified in the next commit. Moving them now so that changes are more obvious. --- internal/certs/certs.go | 108 ------------------- internal/server/server.go | 62 ----------- internal/server/server_test.go | 42 -------- internal/server/tls.go | 182 +++++++++++++++++++++++++++++++++ internal/server/tls_test.go | 54 ++++++++++ 5 files changed, 236 insertions(+), 212 deletions(-) create mode 100644 internal/server/tls.go create mode 100644 internal/server/tls_test.go diff --git a/internal/certs/certs.go b/internal/certs/certs.go index a819a5f677..5b51cf8ae0 100644 --- a/internal/certs/certs.go +++ b/internal/certs/certs.go @@ -5,7 +5,6 @@ import ( "crypto/rand" "crypto/rsa" "crypto/sha256" - "crypto/tls" "crypto/x509" "crypto/x509/pkix" "encoding/pem" @@ -14,10 +13,6 @@ import ( "net" "strings" "time" - - "golang.org/x/crypto/acme/autocert" - - "github.com/infrahq/infra/internal/logging" ) func GenerateCertificate(hosts []string, caCert *x509.Certificate, caKey crypto.PrivateKey) (certPEM []byte, keyPEM []byte, err error) { @@ -62,68 +57,6 @@ func GenerateCertificate(hosts []string, caCert *x509.Certificate, caKey crypto. return PEMEncodeCertificate(certBytes), keyBytes, nil } -func SelfSignedOrLetsEncryptCert(manager *autocert.Manager) func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) { - return func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) { - ctx := hello.Context() - cert, err := manager.GetCertificate(hello) - if err == nil { - return cert, nil - } - - serverName := hello.ServerName - - if serverName == "" { - serverName, _, err = net.SplitHostPort(hello.Conn.LocalAddr().String()) - if err != nil { - return nil, err - } - } - - certBytes, err := manager.Cache.Get(ctx, serverName+".crt") - if err != nil { - logging.Warnf("cert: %s", err) - } - - keyBytes, err := manager.Cache.Get(ctx, serverName+".key") - if err != nil { - logging.Warnf("key: %s", err) - } - - // if either cert or key is missing, create it - if certBytes == nil || keyBytes == nil { - ca, caPrivKey, err := newCA() - if err != nil { - return nil, err - } - - certBytes, keyBytes, err = GenerateCertificate([]string{serverName}, ca, caPrivKey) - if err != nil { - return nil, err - } - - if err := manager.Cache.Put(ctx, serverName+".crt", certBytes); err != nil { - return nil, err - } - - if err := manager.Cache.Put(ctx, serverName+".key", keyBytes); err != nil { - return nil, err - } - - logging.L.Info(). - Str("serverName", serverName). - Str("fingerprint", Fingerprint(pemDecode(certBytes))). - Msg("new server certificate") - } - - keypair, err := tls.X509KeyPair(certBytes, keyBytes) - if err != nil { - return nil, err - } - - return &keypair, nil - } -} - // Fingerprint returns a sha256 checksum of the certificate formatted as // hex pairs separated by colons. This is a common format used by browsers. // The bytes must be the ASN.1 DER form of the x509.Certificate. @@ -133,11 +66,6 @@ func Fingerprint(raw []byte) string { return strings.ToUpper(s) } -func pemDecode(raw []byte) []byte { - block, _ := pem.Decode(raw) - return block.Bytes -} - // PEMEncodeCertificate accepts the bytes of a x509 certificate in ASN.1 DER form // and returns a PEM encoded representation of that certificate. func PEMEncodeCertificate(raw []byte) []byte { @@ -149,39 +77,3 @@ func PEMEncodeCertificate(raw []byte) []byte { func pemEncodePrivateKey(raw []byte) []byte { return pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: raw}) } - -func newCA() (*x509.Certificate, *rsa.PrivateKey, error) { - // Generate a CA to sign self-signed certificates - serialNumber, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) - if err != nil { - return nil, nil, err - } - - caPrivKey, err := rsa.GenerateKey(rand.Reader, 4096) - if err != nil { - return nil, nil, err - } - - ca := &x509.Certificate{ - SerialNumber: serialNumber, - Subject: pkix.Name{ - Organization: []string{"Infra"}, - }, - NotBefore: time.Now().Add(-5 * time.Minute).UTC(), - NotAfter: time.Now().AddDate(0, 0, 365).UTC(), - KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageCertSign, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, - IsCA: true, - BasicConstraintsValid: true, - } - - caBytes, err := x509.CreateCertificate(rand.Reader, ca, ca, &caPrivKey.PublicKey, caPrivKey) - if err != nil { - return nil, nil, err - } - - // TODO: is there really no other way to get the Raw field populated? - ca, _ = x509.ParseCertificate(caBytes) - - return ca, caPrivKey, nil -} diff --git a/internal/server/server.go b/internal/server/server.go index f606fade1d..449dc766c7 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -2,8 +2,6 @@ package server import ( "context" - "crypto/tls" - "crypto/x509" "embed" "errors" "fmt" @@ -12,7 +10,6 @@ import ( "net" "net/http" "net/http/httputil" - "os" "strings" "time" @@ -20,12 +17,10 @@ import ( "github.com/gin-contrib/static" "github.com/gin-gonic/gin" "github.com/infrahq/secrets" - "golang.org/x/crypto/acme/autocert" "golang.org/x/sync/errgroup" "gorm.io/gorm" "github.com/infrahq/infra/internal" - "github.com/infrahq/infra/internal/certs" "github.com/infrahq/infra/internal/cmd/types" "github.com/infrahq/infra/internal/ginutil" "github.com/infrahq/infra/internal/logging" @@ -298,63 +293,6 @@ type routine struct { stop func() } -func tlsConfigFromOptions( - storage map[string]secrets.SecretStorage, - tlsCacheDir string, - opts TLSOptions, -) (*tls.Config, error) { - // TODO: print CA fingerprint when the client can trust that fingerprint - - if opts.Certificate != "" && opts.PrivateKey != "" { - roots, err := x509.SystemCertPool() - if err != nil { - logging.Warnf("failed to load TLS roots from system: %v", err) - roots = x509.NewCertPool() - } - - if opts.CA != "" { - if !roots.AppendCertsFromPEM([]byte(opts.CA)) { - logging.Warnf("failed to load TLS CA, invalid PEM") - } - } - - key, err := secrets.GetSecret(opts.PrivateKey, storage) - if err != nil { - return nil, fmt.Errorf("failed to load TLS private key: %w", err) - } - - cert, err := tls.X509KeyPair([]byte(opts.Certificate), []byte(key)) - if err != nil { - return nil, fmt.Errorf("failed to load TLS key pair: %w", err) - } - - return &tls.Config{ - MinVersion: tls.VersionTLS12, - // enable HTTP/2 - NextProtos: []string{"h2", "http/1.1"}, - Certificates: []tls.Certificate{cert}, - // enabled optional mTLS - ClientAuth: tls.VerifyClientCertIfGiven, - ClientCAs: roots, - }, nil - } - - if err := os.MkdirAll(tlsCacheDir, 0o700); err != nil { - return nil, fmt.Errorf("create tls cache: %w", err) - } - - manager := &autocert.Manager{ - Prompt: autocert.AcceptTOS, - Cache: autocert.DirCache(tlsCacheDir), - } - tlsConfig := manager.TLSConfig() - tlsConfig.MinVersion = tls.VersionTLS12 - // TODO: enabled optional mTLS when opts.CA is set - tlsConfig.GetCertificate = certs.SelfSignedOrLetsEncryptCert(manager) - - return tlsConfig, nil -} - func (s *Server) getDatabaseDriver() (gorm.Dialector, error) { postgres, err := s.getPostgresConnectionString() if err != nil { diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 074eef2662..6b359dc7de 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "crypto/tls" - "crypto/x509" "encoding/json" "io/ioutil" "net/http" @@ -20,10 +19,8 @@ import ( "github.com/rs/zerolog" "gotest.tools/v3/assert" is "gotest.tools/v3/assert/cmp" - "gotest.tools/v3/golden" "github.com/infrahq/infra/api" - "github.com/infrahq/infra/internal/cmd/types" "github.com/infrahq/infra/internal/logging" "github.com/infrahq/infra/internal/server/data" ) @@ -487,42 +484,3 @@ func TestServer_PersistSignupUser(t *testing.T) { // retry the authenticated endpoint checkAuthenticated() } - -func TestTLSConfigFromOptions(t *testing.T) { - storage := map[string]secrets.SecretStorage{ - "plaintext": &secrets.PlainSecretProvider{}, - "file": &secrets.FileSecretProvider{}, - } - - ca := golden.Get(t, "pki/ca.crt") - t.Run("user provided certificate", func(t *testing.T) { - opts := TLSOptions{ - CA: types.StringOrFile(ca), - Certificate: types.StringOrFile(golden.Get(t, "pki/localhost.crt")), - PrivateKey: "file:testdata/pki/localhost.key", - } - config, err := tlsConfigFromOptions(storage, t.TempDir(), opts) - assert.NilError(t, err) - - srv := httptest.NewUnstartedServer(noopHandler) - srv.TLS = config - srv.StartTLS() - t.Cleanup(srv.Close) - - roots := x509.NewCertPool() - roots.AppendCertsFromPEM(ca) - client := &http.Client{ - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{RootCAs: roots, MinVersion: tls.VersionTLS12}, - }, - } - - resp, err := client.Get(srv.URL) - assert.NilError(t, err) - assert.Equal(t, resp.StatusCode, http.StatusOK) - }) -} - -var noopHandler = http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusOK) -}) diff --git a/internal/server/tls.go b/internal/server/tls.go new file mode 100644 index 0000000000..3f78d5375f --- /dev/null +++ b/internal/server/tls.go @@ -0,0 +1,182 @@ +package server + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "net" + "os" + "time" + + "github.com/infrahq/secrets" + "golang.org/x/crypto/acme/autocert" + + "github.com/infrahq/infra/internal/certs" + "github.com/infrahq/infra/internal/logging" +) + +func tlsConfigFromOptions( + storage map[string]secrets.SecretStorage, + tlsCacheDir string, + opts TLSOptions, +) (*tls.Config, error) { + // TODO: print CA fingerprint when the client can trust that fingerprint + + if opts.Certificate != "" && opts.PrivateKey != "" { + roots, err := x509.SystemCertPool() + if err != nil { + logging.Warnf("failed to load TLS roots from system: %v", err) + roots = x509.NewCertPool() + } + + if opts.CA != "" { + if !roots.AppendCertsFromPEM([]byte(opts.CA)) { + logging.Warnf("failed to load TLS CA, invalid PEM") + } + } + + key, err := secrets.GetSecret(opts.PrivateKey, storage) + if err != nil { + return nil, fmt.Errorf("failed to load TLS private key: %w", err) + } + + cert, err := tls.X509KeyPair([]byte(opts.Certificate), []byte(key)) + if err != nil { + return nil, fmt.Errorf("failed to load TLS key pair: %w", err) + } + + return &tls.Config{ + MinVersion: tls.VersionTLS12, + // enable HTTP/2 + NextProtos: []string{"h2", "http/1.1"}, + Certificates: []tls.Certificate{cert}, + // enabled optional mTLS + ClientAuth: tls.VerifyClientCertIfGiven, + ClientCAs: roots, + }, nil + } + + if err := os.MkdirAll(tlsCacheDir, 0o700); err != nil { + return nil, fmt.Errorf("create tls cache: %w", err) + } + + manager := &autocert.Manager{ + Prompt: autocert.AcceptTOS, + Cache: autocert.DirCache(tlsCacheDir), + } + tlsConfig := manager.TLSConfig() + tlsConfig.MinVersion = tls.VersionTLS12 + // TODO: enabled optional mTLS when opts.CA is set + tlsConfig.GetCertificate = SelfSignedOrLetsEncryptCert(manager) + + return tlsConfig, nil +} + +func SelfSignedOrLetsEncryptCert(manager *autocert.Manager) func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) { + return func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) { + ctx := hello.Context() + cert, err := manager.GetCertificate(hello) + if err == nil { + return cert, nil + } + + serverName := hello.ServerName + + if serverName == "" { + serverName, _, err = net.SplitHostPort(hello.Conn.LocalAddr().String()) + if err != nil { + return nil, err + } + } + + certBytes, err := manager.Cache.Get(ctx, serverName+".crt") + if err != nil { + logging.Warnf("cert: %s", err) + } + + keyBytes, err := manager.Cache.Get(ctx, serverName+".key") + if err != nil { + logging.Warnf("key: %s", err) + } + + // if either cert or key is missing, create it + if certBytes == nil || keyBytes == nil { + ca, caPrivKey, err := newCA() + if err != nil { + return nil, err + } + + certBytes, keyBytes, err = certs.GenerateCertificate([]string{serverName}, ca, caPrivKey) + if err != nil { + return nil, err + } + + if err := manager.Cache.Put(ctx, serverName+".crt", certBytes); err != nil { + return nil, err + } + + if err := manager.Cache.Put(ctx, serverName+".key", keyBytes); err != nil { + return nil, err + } + + logging.L.Info(). + Str("Server name", serverName). + Str("SHA256 fingerprint", certs.Fingerprint(pemDecode(certBytes))). + Msg("new server certificate") + } + + keypair, err := tls.X509KeyPair(certBytes, keyBytes) + if err != nil { + return nil, err + } + + return &keypair, nil + } +} + +func pemDecode(raw []byte) []byte { + block, _ := pem.Decode(raw) + return block.Bytes +} + +// TODO: remove +func newCA() (*x509.Certificate, *rsa.PrivateKey, error) { + // Generate a CA to sign self-signed certificates + serialNumber, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + if err != nil { + return nil, nil, err + } + + caPrivKey, err := rsa.GenerateKey(rand.Reader, 4096) + if err != nil { + return nil, nil, err + } + + ca := &x509.Certificate{ + SerialNumber: serialNumber, + Subject: pkix.Name{ + Organization: []string{"Infra"}, + }, + NotBefore: time.Now().Add(-5 * time.Minute).UTC(), + NotAfter: time.Now().AddDate(0, 0, 365).UTC(), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageCertSign, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + IsCA: true, + BasicConstraintsValid: true, + } + + caBytes, err := x509.CreateCertificate(rand.Reader, ca, ca, &caPrivKey.PublicKey, caPrivKey) + if err != nil { + return nil, nil, err + } + + // TODO: is there really no other way to get the Raw field populated? + ca, _ = x509.ParseCertificate(caBytes) + + return ca, caPrivKey, nil +} diff --git a/internal/server/tls_test.go b/internal/server/tls_test.go new file mode 100644 index 0000000000..83b1c6157c --- /dev/null +++ b/internal/server/tls_test.go @@ -0,0 +1,54 @@ +package server + +import ( + "crypto/tls" + "crypto/x509" + "net/http" + "net/http/httptest" + "testing" + + "github.com/infrahq/secrets" + "gotest.tools/v3/assert" + "gotest.tools/v3/golden" + + "github.com/infrahq/infra/internal/cmd/types" +) + +func TestTLSConfigFromOptions(t *testing.T) { + storage := map[string]secrets.SecretStorage{ + "plaintext": &secrets.PlainSecretProvider{}, + "file": &secrets.FileSecretProvider{}, + } + + ca := golden.Get(t, "pki/ca.crt") + t.Run("user provided certificate", func(t *testing.T) { + opts := TLSOptions{ + CA: types.StringOrFile(ca), + Certificate: types.StringOrFile(golden.Get(t, "pki/localhost.crt")), + PrivateKey: "file:testdata/pki/localhost.key", + } + config, err := tlsConfigFromOptions(storage, t.TempDir(), opts) + assert.NilError(t, err) + + srv := httptest.NewUnstartedServer(noopHandler) + srv.TLS = config + srv.StartTLS() + t.Cleanup(srv.Close) + + roots := x509.NewCertPool() + roots.AppendCertsFromPEM(ca) + client := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{RootCAs: roots, MinVersion: tls.VersionTLS12}, + }, + } + + resp, err := client.Get(srv.URL) + assert.NilError(t, err) + assert.Equal(t, resp.StatusCode, http.StatusOK) + }) +} + +var noopHandler = http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) +}) From a6a8685ed70a0e088d0f7193ed67e5eee22d024c Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 23 Jun 2022 15:53:35 -0400 Subject: [PATCH 2/4] improve: generate server TLS cert from CA With this commit, a user has 3 options for configuring TLS: 1. set tls.ACME=true, to use ACME to create a publicly trusted certificate 2. set tls.ca and tls.caPrivateKey, which will be used to generate a TLS ceritifcate for any hostname the client requests 3. set tls.certificate and tls.privateKey to use that certificate for TLS --- internal/cmd/server_test.go | 2 + internal/server/server.go | 5 + internal/server/server_test.go | 11 ++ internal/server/tls.go | 203 +++++++++++++++++---------------- internal/server/tls_test.go | 35 ++++++ 5 files changed, 158 insertions(+), 98 deletions(-) diff --git a/internal/cmd/server_test.go b/internal/cmd/server_test.go index 107e773494..3d6a7962a0 100644 --- a/internal/cmd/server_test.go +++ b/internal/cmd/server_test.go @@ -142,6 +142,7 @@ tls: caPrivateKey: file:ca.key certificate: testdata/server.crt privateKey: file:server.key + ACME: true keys: - kind: vault @@ -223,6 +224,7 @@ users: CAPrivateKey: "file:ca.key", Certificate: "-----BEGIN CERTIFICATE-----\nnot a real server certificate\n-----END CERTIFICATE-----\n", PrivateKey: "file:server.key", + ACME: true, }, Keys: []server.KeyProvider{ diff --git a/internal/server/server.go b/internal/server/server.go index 449dc766c7..f3f0963a96 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -79,6 +79,11 @@ type TLSOptions struct { CAPrivateKey string Certificate types.StringOrFile PrivateKey string + + // ACME enables automated certificate manangement. When set to true a TLS + // certificate will be requested from Let's Encrypt, which will be cached + // in the TLSCache. + ACME bool } type Server struct { diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 6b359dc7de..d806978f5f 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -19,8 +19,10 @@ import ( "github.com/rs/zerolog" "gotest.tools/v3/assert" is "gotest.tools/v3/assert/cmp" + "gotest.tools/v3/golden" "github.com/infrahq/infra/api" + "github.com/infrahq/infra/internal/cmd/types" "github.com/infrahq/infra/internal/logging" "github.com/infrahq/infra/internal/server/data" ) @@ -119,7 +121,12 @@ func TestServer_Run(t *testing.T) { DBEncryptionKey: filepath.Join(dir, "sqlite3.db.key"), TLSCache: filepath.Join(dir, "tlscache"), DBFile: filepath.Join(dir, "sqlite3.db"), + TLS: TLSOptions{ + CA: types.StringOrFile(golden.Get(t, "pki/ca.crt")), + CAPrivateKey: string(golden.Get(t, "pki/ca.key")), + }, } + srv, err := New(opts) assert.NilError(t, err) @@ -199,6 +206,10 @@ func TestServer_Run_UIProxy(t *testing.T) { DBFile: filepath.Join(dir, "sqlite3.db"), UI: UIOptions{Enabled: true}, EnableSignup: true, + TLS: TLSOptions{ + CA: types.StringOrFile(golden.Get(t, "pki/ca.crt")), + CAPrivateKey: string(golden.Get(t, "pki/ca.key")), + }, } assert.NilError(t, opts.UI.ProxyURL.Set(uiSrv.URL)) diff --git a/internal/server/tls.go b/internal/server/tls.go index 3f78d5375f..4919e83de9 100644 --- a/internal/server/tls.go +++ b/internal/server/tls.go @@ -1,17 +1,14 @@ package server import ( - "crypto/rand" - "crypto/rsa" + "context" "crypto/tls" "crypto/x509" - "crypto/x509/pkix" "encoding/pem" "fmt" - "math/big" "net" "os" - "time" + "sync" "github.com/infrahq/secrets" "golang.org/x/crypto/acme/autocert" @@ -25,21 +22,48 @@ func tlsConfigFromOptions( tlsCacheDir string, opts TLSOptions, ) (*tls.Config, error) { - // TODO: print CA fingerprint when the client can trust that fingerprint + // TODO: how can we test this? + if opts.ACME { + if err := os.MkdirAll(tlsCacheDir, 0o700); err != nil { + return nil, fmt.Errorf("create tls cache: %w", err) + } - if opts.Certificate != "" && opts.PrivateKey != "" { - roots, err := x509.SystemCertPool() - if err != nil { - logging.Warnf("failed to load TLS roots from system: %v", err) - roots = x509.NewCertPool() + manager := &autocert.Manager{ + Prompt: autocert.AcceptTOS, + Cache: autocert.DirCache(tlsCacheDir), + // TODO: according to the docs HostPolicy should be set to prevent + // a DoS attack on certificate requests. + // See https://github.com/infrahq/infra/issues/2484 } + tlsConfig := manager.TLSConfig() + tlsConfig.MinVersion = tls.VersionTLS12 + return tlsConfig, nil + } - if opts.CA != "" { - if !roots.AppendCertsFromPEM([]byte(opts.CA)) { - logging.Warnf("failed to load TLS CA, invalid PEM") - } + // TODO: print CA fingerprint when the client can trust that fingerprint + + roots, err := x509.SystemCertPool() + if err != nil { + logging.Warnf("failed to load TLS roots from system: %v", err) + roots = x509.NewCertPool() + } + + if opts.CA != "" { + if !roots.AppendCertsFromPEM([]byte(opts.CA)) { + logging.Warnf("failed to load TLS CA, invalid PEM") } + } + cfg := &tls.Config{ + MinVersion: tls.VersionTLS12, + // enable HTTP/2 + NextProtos: []string{"h2", "http/1.1"}, + // enabled optional mTLS + ClientAuth: tls.VerifyClientCertIfGiven, + ClientCAs: roots, + } + + if opts.Certificate != "" && opts.PrivateKey != "" { key, err := secrets.GetSecret(opts.PrivateKey, storage) if err != nil { return nil, fmt.Errorf("failed to load TLS private key: %w", err) @@ -50,86 +74,98 @@ func tlsConfigFromOptions( return nil, fmt.Errorf("failed to load TLS key pair: %w", err) } - return &tls.Config{ - MinVersion: tls.VersionTLS12, - // enable HTTP/2 - NextProtos: []string{"h2", "http/1.1"}, - Certificates: []tls.Certificate{cert}, - // enabled optional mTLS - ClientAuth: tls.VerifyClientCertIfGiven, - ClientCAs: roots, - }, nil + cfg.Certificates = []tls.Certificate{cert} + return cfg, nil } - if err := os.MkdirAll(tlsCacheDir, 0o700); err != nil { - return nil, fmt.Errorf("create tls cache: %w", err) + if opts.CA == "" || opts.CAPrivateKey == "" { + return nil, fmt.Errorf("either a TLS certificate and key or a TLS CA and key is required") } - manager := &autocert.Manager{ - Prompt: autocert.AcceptTOS, - Cache: autocert.DirCache(tlsCacheDir), + key, err := secrets.GetSecret(opts.CAPrivateKey, storage) + if err != nil { + return nil, fmt.Errorf("failed to load TLS CA private key: %w", err) } - tlsConfig := manager.TLSConfig() - tlsConfig.MinVersion = tls.VersionTLS12 - // TODO: enabled optional mTLS when opts.CA is set - tlsConfig.GetCertificate = SelfSignedOrLetsEncryptCert(manager) - return tlsConfig, nil + ca := keyPair{cert: []byte(opts.CA), key: []byte(key)} + cfg.GetCertificate = getCertificate(autocert.DirCache(tlsCacheDir), ca) + return cfg, nil } -func SelfSignedOrLetsEncryptCert(manager *autocert.Manager) func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) { - return func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) { - ctx := hello.Context() - cert, err := manager.GetCertificate(hello) - if err == nil { - return cert, nil +type keyPair struct { + cert []byte + key []byte +} + +func getCertificate(cache autocert.Cache, ca keyPair) func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) { + var lock sync.RWMutex + + getKeyPair := func(ctx context.Context, serverName string) (cert, key []byte) { + certBytes, _ := cache.Get(ctx, serverName+".crt") + keyBytes, _ := cache.Get(ctx, serverName+".key") + if certBytes == nil || keyBytes == nil { + logging.Infof("no cached TLS cert for %v", serverName) } + return certBytes, keyBytes + } + return func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) { + ctx := hello.Context() serverName := hello.ServerName if serverName == "" { + var err error serverName, _, err = net.SplitHostPort(hello.Conn.LocalAddr().String()) if err != nil { return nil, err } } - certBytes, err := manager.Cache.Get(ctx, serverName+".crt") - if err != nil { - logging.Warnf("cert: %s", err) + lock.RLock() + certBytes, keyBytes := getKeyPair(ctx, serverName) + lock.RUnlock() + if certBytes != nil && keyBytes != nil { + return tlsCertFromKeyPair(certBytes, keyBytes) } - keyBytes, err := manager.Cache.Get(ctx, serverName+".key") - if err != nil { - logging.Warnf("key: %s", err) + lock.Lock() + // must check again after write lock is acquired + certBytes, keyBytes = getKeyPair(ctx, serverName) + defer lock.Unlock() + if certBytes != nil && keyBytes != nil { + return tlsCertFromKeyPair(certBytes, keyBytes) } // if either cert or key is missing, create it - if certBytes == nil || keyBytes == nil { - ca, caPrivKey, err := newCA() - if err != nil { - return nil, err - } + ca, err := tls.X509KeyPair(ca.cert, ca.key) + if err != nil { + return nil, err + } - certBytes, keyBytes, err = certs.GenerateCertificate([]string{serverName}, ca, caPrivKey) - if err != nil { - return nil, err - } + caCert, err := x509.ParseCertificate(ca.Certificate[0]) + if err != nil { + return nil, err + } - if err := manager.Cache.Put(ctx, serverName+".crt", certBytes); err != nil { - return nil, err - } + hosts := []string{"127.0.0.1", "::1", serverName} + certBytes, keyBytes, err = certs.GenerateCertificate(hosts, caCert, ca.PrivateKey) + if err != nil { + return nil, err + } - if err := manager.Cache.Put(ctx, serverName+".key", keyBytes); err != nil { - return nil, err - } + if err := cache.Put(ctx, serverName+".crt", certBytes); err != nil { + return nil, err + } - logging.L.Info(). - Str("Server name", serverName). - Str("SHA256 fingerprint", certs.Fingerprint(pemDecode(certBytes))). - Msg("new server certificate") + if err := cache.Put(ctx, serverName+".key", keyBytes); err != nil { + return nil, err } + logging.L.Info(). + Str("Server name", serverName). + Str("SHA256 fingerprint", certs.Fingerprint(pemDecode(certBytes))). + Msg("new server certificate") + keypair, err := tls.X509KeyPair(certBytes, keyBytes) if err != nil { return nil, err @@ -144,39 +180,10 @@ func pemDecode(raw []byte) []byte { return block.Bytes } -// TODO: remove -func newCA() (*x509.Certificate, *rsa.PrivateKey, error) { - // Generate a CA to sign self-signed certificates - serialNumber, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) - if err != nil { - return nil, nil, err - } - - caPrivKey, err := rsa.GenerateKey(rand.Reader, 4096) +func tlsCertFromKeyPair(cert, key []byte) (*tls.Certificate, error) { + keypair, err := tls.X509KeyPair(cert, key) if err != nil { - return nil, nil, err + return nil, err } - - ca := &x509.Certificate{ - SerialNumber: serialNumber, - Subject: pkix.Name{ - Organization: []string{"Infra"}, - }, - NotBefore: time.Now().Add(-5 * time.Minute).UTC(), - NotAfter: time.Now().AddDate(0, 0, 365).UTC(), - KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageCertSign, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, - IsCA: true, - BasicConstraintsValid: true, - } - - caBytes, err := x509.CreateCertificate(rand.Reader, ca, ca, &caPrivKey.PublicKey, caPrivKey) - if err != nil { - return nil, nil, err - } - - // TODO: is there really no other way to get the Raw field populated? - ca, _ = x509.ParseCertificate(caBytes) - - return ca, caPrivKey, nil + return &keypair, nil } diff --git a/internal/server/tls_test.go b/internal/server/tls_test.go index 83b1c6157c..4dd0beefd8 100644 --- a/internal/server/tls_test.go +++ b/internal/server/tls_test.go @@ -3,6 +3,7 @@ package server import ( "crypto/tls" "crypto/x509" + "net" "net/http" "net/http/httptest" "testing" @@ -47,6 +48,40 @@ func TestTLSConfigFromOptions(t *testing.T) { assert.NilError(t, err) assert.Equal(t, resp.StatusCode, http.StatusOK) }) + + t.Run("generate TLS cert from CA", func(t *testing.T) { + opts := TLSOptions{ + CA: types.StringOrFile(ca), + CAPrivateKey: "file:testdata/pki/ca.key", + } + config, err := tlsConfigFromOptions(storage, t.TempDir(), opts) + assert.NilError(t, err) + + l, err := net.Listen("tcp", "127.0.0.1:0") + assert.NilError(t, err) + + l = tls.NewListener(l, config) + srv := http.Server{Handler: noopHandler} + + go func() { + if err := srv.Serve(l); err != http.ErrServerClosed { + t.Log(err) + } + }() + t.Cleanup(func() { _ = srv.Close() }) + + roots := x509.NewCertPool() + roots.AppendCertsFromPEM(ca) + client := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{RootCAs: roots, MinVersion: tls.VersionTLS12}, + }, + } + + resp, err := client.Get("https://" + l.Addr().String()) + assert.NilError(t, err) + assert.Equal(t, resp.StatusCode, http.StatusOK) + }) } var noopHandler = http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { From 3b625ec86ee1821447cddb829d93b71bb376e84e Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 27 Jun 2022 14:25:51 -0400 Subject: [PATCH 3/4] maintain: update CLI tests for new TLS config The certificate is specified from opts instead of populating the cache. --- internal/cmd/list_test.go | 3 +-- internal/cmd/login_test.go | 33 +++++++++------------------------ internal/cmd/server_test.go | 5 +++++ internal/server/tls_test.go | 1 + 4 files changed, 16 insertions(+), 26 deletions(-) diff --git a/internal/cmd/list_test.go b/internal/cmd/list_test.go index f5471ad14b..11fdf1ac11 100644 --- a/internal/cmd/list_test.go +++ b/internal/cmd/list_test.go @@ -34,14 +34,13 @@ func TestListCmd(t *testing.T) { {User: "manygrants@example.com", Resource: "moon", Role: "inhabitant"}, }, } - opts.Addr = server.ListenerOptions{HTTPS: "127.0.0.1:0", HTTP: "127.0.0.1:0"} + setupServerTLSOptions(t, &opts) srv, err := server.New(opts) assert.NilError(t, err) ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) - setupCertManager(t, opts.TLSCache, srv.Addrs.HTTPS.String()) go func() { assert.Check(t, srv.Run(ctx)) }() diff --git a/internal/cmd/login_test.go b/internal/cmd/login_test.go index f5a04cdcf0..dcb3c43697 100644 --- a/internal/cmd/login_test.go +++ b/internal/cmd/login_test.go @@ -3,7 +3,6 @@ package cmd import ( "context" "encoding/pem" - "net" "os" "path/filepath" "testing" @@ -15,7 +14,6 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/hinshun/vt10x" - "golang.org/x/crypto/acme/autocert" "golang.org/x/sync/errgroup" "gotest.tools/v3/assert" "gotest.tools/v3/assert/opt" @@ -24,6 +22,7 @@ import ( "github.com/infrahq/infra/api" "github.com/infrahq/infra/internal/certs" + "github.com/infrahq/infra/internal/cmd/types" "github.com/infrahq/infra/internal/race" "github.com/infrahq/infra/internal/server" "github.com/infrahq/infra/uid" @@ -35,7 +34,7 @@ func TestLoginCmd_SetupAdminOnFirstLogin(t *testing.T) { dir := setupEnv(t) opts := defaultServerOptions(dir) - opts.Addr = server.ListenerOptions{HTTPS: "127.0.0.1:0", HTTP: "127.0.0.1:0"} + setupServerTLSOptions(t, &opts) srv, err := server.New(opts) assert.NilError(t, err) @@ -43,10 +42,6 @@ func TestLoginCmd_SetupAdminOnFirstLogin(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) - host, _, err := net.SplitHostPort(srv.Addrs.HTTPS.String()) - assert.NilError(t, err) - - setupCertManager(t, opts.TLSCache, host) go func() { assert.Check(t, srv.Run(ctx)) }() @@ -133,7 +128,7 @@ func TestLoginCmd_Options(t *testing.T) { dir := setupEnv(t) opts := defaultServerOptions(dir) - opts.Addr = server.ListenerOptions{HTTPS: "127.0.0.1:0", HTTP: "127.0.0.1:0"} + setupServerTLSOptions(t, &opts) adminAccessKey := "aaaaaaaaaa.bbbbbbbbbbbbbbbbbbbbbbbb" opts.Config.Users = []server.User{ { @@ -147,7 +142,6 @@ func TestLoginCmd_Options(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) - setupCertManager(t, opts.TLSCache, srv.Addrs.HTTPS.String()) go func() { assert.Check(t, srv.Run(ctx)) }() @@ -286,23 +280,18 @@ func setupEnv(t *testing.T) string { return dir } -// setupCertManager copies the static TLS cert and key into the cache that will -// be used by the server. This allows the server to skip generating a private key -// for both the CA and server certificate, which takes multiple seconds. -func setupCertManager(t *testing.T, dir string, serverName string) { +func setupServerTLSOptions(t *testing.T, opts *server.Options) { t.Helper() - ctx := context.Background() - cache := autocert.DirCache(dir) + + opts.Addr = server.ListenerOptions{HTTPS: "127.0.0.1:0", HTTP: "127.0.0.1:0"} key, err := os.ReadFile("testdata/pki/localhost.key") assert.NilError(t, err) - err = cache.Put(ctx, serverName+".key", key) - assert.NilError(t, err) + opts.TLS.PrivateKey = string(key) cert, err := os.ReadFile("testdata/pki/localhost.crt") assert.NilError(t, err) - err = cache.Put(ctx, serverName+".crt", cert) - assert.NilError(t, err) + opts.TLS.Certificate = types.StringOrFile(cert) } func TestLoginCmd_TLSVerify(t *testing.T) { @@ -314,21 +303,17 @@ func TestLoginCmd_TLSVerify(t *testing.T) { t.Setenv("KUBECONFIG", kubeConfigPath) opts := defaultServerOptions(dir) + setupServerTLSOptions(t, &opts) accessKey := "0000000001.adminadminadminadmin1234" opts.Users = []server.User{ {Name: "admin@example.com", AccessKey: accessKey}, } - opts.Addr = server.ListenerOptions{HTTPS: "127.0.0.1:0", HTTP: "127.0.0.1:0"} srv, err := server.New(opts) assert.NilError(t, err) ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) - host, _, err := net.SplitHostPort(srv.Addrs.HTTPS.String()) - assert.NilError(t, err) - - setupCertManager(t, opts.TLSCache, host) go func() { assert.Check(t, srv.Run(ctx)) }() diff --git a/internal/cmd/server_test.go b/internal/cmd/server_test.go index 3d6a7962a0..ff32ee7b4c 100644 --- a/internal/cmd/server_test.go +++ b/internal/cmd/server_test.go @@ -321,6 +321,11 @@ func TestServerCmd_WithSecretsConfig(t *testing.T) { http: "127.0.0.1:0" https: "127.0.0.1:0" metrics: "127.0.0.1:0" + + tls: + ca: some-ca + caPrivateKey: some-key + secrets: - kind: env name: base64env diff --git a/internal/server/tls_test.go b/internal/server/tls_test.go index 4dd0beefd8..5a28c1c0a6 100644 --- a/internal/server/tls_test.go +++ b/internal/server/tls_test.go @@ -64,6 +64,7 @@ func TestTLSConfigFromOptions(t *testing.T) { srv := http.Server{Handler: noopHandler} go func() { + // nolint:errorlint if err := srv.Serve(l); err != http.ErrServerClosed { t.Log(err) } From 4dc9598189c0f377546631eee60271f92c13fcb3 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 29 Jun 2022 17:00:21 -0400 Subject: [PATCH 4/4] improve: create infra-server-ca secret --- Makefile | 2 +- .../infra/templates/server/configmap.yaml | 6 ++++ .../infra/templates/server/deployment.yaml | 9 ++++++ .../charts/infra/templates/server/secret.yaml | 29 +++++++++++++++++++ helm/charts/infra/values.yaml | 13 +++++++++ internal/server/server.go | 2 +- 6 files changed, 59 insertions(+), 2 deletions(-) create mode 100644 helm/charts/infra/templates/server/secret.yaml diff --git a/Makefile b/Makefile index a877dd6076..bf342ad77d 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ test/update: go test ./internal/cmd -test.update-golden dev: - docker build . -t infrahq/infra:dev + docker buildx build . -t infrahq/infra:dev kubectl config use-context docker-desktop helm upgrade --install --wait \ --set global.image.pullPolicy=Never \ diff --git a/helm/charts/infra/templates/server/configmap.yaml b/helm/charts/infra/templates/server/configmap.yaml index 5efec29757..f457829856 100644 --- a/helm/charts/infra/templates/server/configmap.yaml +++ b/helm/charts/infra/templates/server/configmap.yaml @@ -36,6 +36,12 @@ data: {{- end }} {{- end }} +{{- if not .Values.server.config.tls }} + tls: + ca: "/var/run/secrets/infrahq.com/tls-ca/ca.crt" + caPrivateKey: "file:/var/run/secrets/infrahq.com/tls-ca/ca.key" +{{- end }} + providers: {{- .Values.server.additionalProviders | default list | concat .Values.server.config.providers | uniq | toYaml | nindent 6 }} diff --git a/helm/charts/infra/templates/server/deployment.yaml b/helm/charts/infra/templates/server/deployment.yaml index 1c95a7ee48..5f418864e3 100644 --- a/helm/charts/infra/templates/server/deployment.yaml +++ b/helm/charts/infra/templates/server/deployment.yaml @@ -48,6 +48,10 @@ spec: - name: conf mountPath: /etc/infrahq readOnly: true +{{- if (not .Values.server.config.tls) }} + - name: tls-ca + mountPath: /var/run/secrets/infrahq.com/tls-ca +{{- end }} {{- if .Values.server.persistence.enabled }} - name: data mountPath: /var/lib/infrahq/server @@ -89,6 +93,11 @@ spec: - name: conf configMap: name: {{ include "server.fullname" . }} +{{- if (not .Values.server.config.tls) }} + - name: tls-ca + secret: + secretName: {{ include "server.fullname" . }}-ca +{{- end }} {{- if .Values.server.persistence.enabled }} - name: data persistentVolumeClaim: diff --git a/helm/charts/infra/templates/server/secret.yaml b/helm/charts/infra/templates/server/secret.yaml new file mode 100644 index 0000000000..d27756ad19 --- /dev/null +++ b/helm/charts/infra/templates/server/secret.yaml @@ -0,0 +1,29 @@ + +{{- if include "server.enabled" . | eq "true" }} +{{- if (not .Values.server.config.tls) }} + +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "server.fullname" . }}-ca + labels: +{{- include "server.labels" . | nindent 4 }} +data: + +{{- $secret := lookup "v1" "Secret" .Release.Namespace (printf "%s-ca" (include "server.fullname" .)) -}} +{{- if $secret.data }} + ca.crt: | +{{- get $secret.data "ca.crt" | nindent 4 }} + ca.key: | +{{- get $secret.data "ca.key" | nindent 4 }} + +{{- else }} +{{- $ca := genCA "Infra Server" 3650 }} + ca.crt: | +{{- $ca.Cert | b64enc | nindent 4 }} + ca.key: | +{{- $ca.Key | b64enc | nindent 4 }} + +{{- end }}{{/* if secret.data */}} +{{- end }}{{/* if not tls */}} +{{- end }}{{/* if server.enabled */}} diff --git a/helm/charts/infra/values.yaml b/helm/charts/infra/values.yaml index 7c032fa888..68fef50aaf 100644 --- a/helm/charts/infra/values.yaml +++ b/helm/charts/infra/values.yaml @@ -531,6 +531,19 @@ server: # - name: dev@example.com # password: file:/var/run/secrets/dev@example.com + # TLS configuration for the API server. Defaults to generating a self-signed CA and + # generating certificates from that CA. + tls: {} + + # Configure a CA and private key using files + # ca: /path/to/ca.crt + # caPrivateKey: file:/path/to/ca.key + + # Configure a TLS certificate and private key using files + # certificate: /path/to/server.crt + # privateKey: file:/path/to/server.key + + ## Default connector configurations connector: diff --git a/internal/server/server.go b/internal/server/server.go index f3f0963a96..84ca0fc3b2 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -80,7 +80,7 @@ type TLSOptions struct { Certificate types.StringOrFile PrivateKey string - // ACME enables automated certificate manangement. When set to true a TLS + // ACME enables automated certificate management. When set to true a TLS // certificate will be requested from Let's Encrypt, which will be cached // in the TLSCache. ACME bool