add central certificate storage (s3/minio)

Makefile

@@ -34,6 +34,25 @@ docker-services: docker
 docker-update: docker
 	docker service update --image icecave/honeycomb:dev --force honeycomb
 
+.PHONY: docker-minio
+docker-minio: docker-services
+	-@mkdir -p artifacts/minio/data
+	-docker service rm minio
+	docker service create \
+		--name minio \
+		--constraint "node.role==manager" \
+		--mount "type=bind,target=/data,source=$(shell pwd)/artifacts/minio/data" \
+		--network public \
+		--env "MINIO_ACCESS_KEY=TESTACCESSKEY" \
+		--env "MINIO_SECRET_KEY=TESTSECRETKEY" \
+		--publish "9000:9000/tcp" \
+		minio/minio:latest \
+		server /data
+	docker service update \
+		--env-add "MINIO_ACCESS_KEY=TESTACCESSKEY" \
+		--env-add "MINIO_SECRET_KEY=TESTSECRETKEY" \
+		honeycomb
+
 MINIFY := artifacts/minify/bin/minify
 $(MINIFY):
 	-@mkdir -p "$(@D)"

cmd/config.go

@@ -28,6 +28,16 @@ type certificateConfig struct {
 	ServerCertificate string
 	ServerKey         string
 	CABundles         []string
+	Minio             minioStoreConfig
+}
+
+type minioStoreConfig struct {
+	Endpoint        string
+	Region          string
+	BucketName      string
+	AccessKeyID     string
+	SecretAccessKey string
+	SSL             bool
 }
 
 // GetConfigFromEnvironment creates Config object based on the shell environment.
@@ -46,6 +56,14 @@ func GetConfigFromEnvironment() *Config {
 				env("CA_PATH", "/app/etc/ca-bundle.pem,/run/secrets/ca-bundle.pem"),
 				",",
 			),
+			Minio: minioStoreConfig{
+				Endpoint:        env("MINIO_ENDPOINT", "minio:9000"),
+				Region:          env("MINIO_REGION", "us-east-1"),
+				BucketName:      env("MINIO_BUCKETNAME", "honeycomb"),
+				AccessKeyID:     env("MINIO_ACCESS_KEY", ""),
+				SecretAccessKey: env("MINIO_SECRET_KEY", ""),
+				SSL:             envBool("MINIO_SSL", false),
+			},
 		},
 		ProxyProtocol: envBool("PROXY_PROTOCOL", false),
 		CheckTimeout:  envDuration("CHECK_TIMEOUT", 500*time.Millisecond),

cmd/honeycomb/main.go

@@ -79,8 +79,14 @@ func main() {
 		logger.Fatalln(err)
 	}
 
+	primaryCertProvider := cert.MultiProvider{
+		primaryFileCertificateProvider(config, logger),
+	}
+	if config.Certificates.Minio.AccessKeyID != "" {
+		primaryCertProvider = append(primaryCertProvider, primaryMinioCertificateProvider(config, logger))
+	}
 	providerAdaptor := &cert.ProviderAdaptor{
-		PrimaryProvider:   primaryCertificateProvider(config, logger),
+		PrimaryProvider:   primaryCertProvider,
 		SecondaryProvider: secondaryCertProvider,
 	}
 
@@ -213,7 +219,27 @@ func loadDefaultCertificate(config *cmd.Config) (*tls.Certificate, error) {
 	return &cert, err
 }
 
-func primaryCertificateProvider(
+func primaryMinioCertificateProvider(
+	config *cmd.Config,
+	logger *log.Logger,
+) cert.Provider {
+	mc, err := cert.NewMinioProvider(
+		logger,
+		config.Certificates.Minio.Endpoint,
+		config.Certificates.Minio.Region,
+		config.Certificates.Minio.BucketName,
+		config.Certificates.Minio.AccessKeyID,
+		config.Certificates.Minio.SecretAccessKey,
+		config.Certificates.Minio.SSL,
+	)
+	if err != nil {
+		logger.Fatalln(err)
+	}
+
+	return mc
+}
+
+func primaryFileCertificateProvider(
 	config *cmd.Config,
 	logger *log.Logger,
 ) cert.Provider {

frontend/cert/adhoc_provider.go

@@ -44,6 +44,7 @@ func (provider *AdhocProvider) GetExistingCertificate(
 	serverName name.ServerName,
 ) (*tls.Certificate, error) {
 	cache, _ := provider.cache.Load().(certificateCache)
+
 	return provider.fetch(cache, serverName), nil
 }
 

frontend/cert/file_provider.go

@@ -4,7 +4,7 @@ import (
 	"context"
 	"crypto/tls"
 	"crypto/x509"
-	"errors"
+	"fmt"
 	"log"
 	"os"
 	"path"
@@ -15,8 +15,10 @@ import (
 	"github.com/icecave/honeycomb/name"
 )
 
-const certExtension = ".crt"
-const keyExtension = ".key"
+const (
+	certExtension = ".crt"
+	keyExtension  = ".key"
+)
 
 // FileProvider a certificate provider that reads certificates from a loader.
 type FileProvider struct {
@@ -37,7 +39,7 @@ func (p *FileProvider) GetCertificate(ctx context.Context, n name.ServerName) (*
 		return cert, err
 	}
 
-	return nil, errors.New("file provider can not generated certificates")
+	return nil, fmt.Errorf("file %w", ErrProviderGenerateUnsupported)
 }
 
 // GetExistingCertificate attempts to fetch an existing certificate for the
@@ -50,7 +52,7 @@ func (p *FileProvider) GetExistingCertificate(ctx context.Context, n name.Server
 	}
 
 	for _, filename := range p.resolveFilenames(n) {
-		cert, err := p.loadCertificate(ctx, n, filename)
+		cert, err := p.loadCertificate(n, filename)
 		if cert != nil || err != nil {
 			return cert, err
 		}
@@ -60,7 +62,6 @@ func (p *FileProvider) GetExistingCertificate(ctx context.Context, n name.Server
 }
 
 func (p *FileProvider) loadCertificate(
-	ctx context.Context,
 	n name.ServerName,
 	filename string,
 ) (*tls.Certificate, error) {

frontend/cert/minio_provider.go

@@ -0,0 +1,217 @@
+package cert
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/icecave/honeycomb/name"
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+)
+
+// MinioProvider a certificate provider that reads certificates from a loader.
+type MinioProvider struct {
+	Client     *minio.Client
+	BucketName string
+	CacheAge   time.Duration
+	Logger     *log.Logger
+
+	mutex sync.RWMutex
+	cache map[string]*minioCacheItem
+}
+
+type minioCacheItem struct {
+	Certificate *tls.Certificate
+	LastSeen    time.Time
+}
+
+// errMinioCertNotFound is returned when the certificate is not found or not correctly formed in minio.
+var errMinioCertNotFound = errors.New("certificate not found")
+
+// NewMinioProvider returns a MinioProvider preconfigured and setup for use as a Provider.
+func NewMinioProvider(
+	logger *log.Logger,
+	endpoint, region, bucketName, accessKeyID, secretAccessKey string,
+	useSSL bool,
+) (Provider, error) {
+	c, err := minio.New(endpoint, &minio.Options{
+		Creds:  credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
+		Secure: useSSL,
+		Region: region,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	if ok, err := c.BucketExists(context.Background(), bucketName); err == nil && !ok {
+		err := c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{
+			Region: region,
+		})
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return &MinioProvider{
+		Client:     c,
+		BucketName: bucketName,
+		Logger:     logger,
+	}, err
+}
+
+// GetCertificate attempts to fetch an existing certificate for the given
+// server name. If no such certificate exists, it generates one.
+func (p *MinioProvider) GetCertificate(ctx context.Context, n name.ServerName) (*tls.Certificate, error) {
+	cert, err := p.GetExistingCertificate(ctx, n)
+	if err != nil {
+		return nil, err
+	} else if cert != nil {
+		return cert, err
+	}
+
+	return nil, fmt.Errorf("minio %w", ErrProviderGenerateUnsupported)
+}
+
+// GetExistingCertificate attempts to fetch an existing certificate for the
+// given server name. It never generates new certificates. A non-nil error
+// indicates an error with the provider itself; otherwise, a nil certificate
+// indicates a failure to find an existing certificate.
+func (p *MinioProvider) GetExistingCertificate(ctx context.Context, n name.ServerName) (*tls.Certificate, error) {
+	// If cache has not expired, attempt to find in cache.
+	if !p.expiredInCache(n) {
+		if cert, ok := p.findInCache(n); ok {
+			return cert, nil
+		}
+	}
+
+	for _, objectName := range p.resolveObjectNames(n) {
+		// No cache (or expired), attempt to look up in redis
+		// but if redis is down or broken suddenly, we should reuse the
+		// cached certificate until it's replaced.
+		if cert, err := p.getMinioCertificate(ctx, objectName); err == nil {
+			p.writeToCache(n, cert)
+
+			return cert, nil
+		}
+	}
+
+	// fail through to getting it from the cache.
+	if cert, ok := p.findInCache(n); ok {
+		p.Logger.Printf("expired but falling through to cache for %s", n.Unicode)
+
+		return cert, nil
+	}
+
+	// and finally we just fail.
+	return nil, nil
+}
+
+func (p *MinioProvider) getMinioObject(ctx context.Context, objectName string) (*minio.Object, error) {
+	return p.Client.GetObject(
+		ctx,
+		p.BucketName,
+		fmt.Sprintf("%s.crt", objectName),
+		minio.GetObjectOptions{},
+	)
+}
+
+func (p *MinioProvider) getMinioCertificate(ctx context.Context, objectName string) (*tls.Certificate, error) {
+	var (
+		certObj, keyObj *minio.Object
+		err             error
+	)
+
+	if certObj, err = p.getMinioObject(ctx, fmt.Sprintf("%s.crt", objectName)); err != nil {
+		return nil, err
+	}
+
+	if keyObj, err = p.getMinioObject(ctx, fmt.Sprintf("%s.key", objectName)); err != nil {
+		return nil, err
+	}
+
+	certBuf := bytes.NewBuffer(nil)
+	if _, err = io.Copy(certBuf, certObj); err != nil {
+		return nil, err
+	}
+
+	keyBuf := bytes.NewBuffer(nil)
+	if _, err = io.Copy(keyBuf, keyObj); err != nil {
+		return nil, err
+	}
+
+	if cert, err := tls.X509KeyPair(certBuf.Bytes(), keyBuf.Bytes()); err == nil {
+		return &cert, nil
+	}
+
+	return nil, errMinioCertNotFound
+}
+
+func (p *MinioProvider) resolveObjectNames(
+	n name.ServerName,
+) (filenames []string) {
+	tail := n.Punycode
+	filenames = []string{tail}
+
+	for {
+		parts := strings.SplitN(tail, ".", 2)
+		if len(parts) == 1 {
+			return
+		}
+
+		tail = parts[1]
+		filenames = append(filenames, "_."+tail, tail)
+	}
+}
+
+func (p *MinioProvider) expiredInCache(n name.ServerName) bool {
+	p.mutex.RLock()
+	defer p.mutex.RUnlock()
+
+	if item, ok := p.cache[n.Unicode]; ok {
+		if p.CacheAge > 0 && item.LastSeen.Before(time.Now().Add(-1*p.CacheAge)) {
+			return true
+		}
+	}
+
+	return false
+}
+
+func (p *MinioProvider) findInCache(
+	n name.ServerName,
+) (*tls.Certificate, bool) {
+	p.mutex.RLock()
+	defer p.mutex.RUnlock()
+
+	if item, ok := p.cache[n.Unicode]; ok {
+		return item.Certificate, ok
+	}
+
+	return nil, false
+}
+
+func (p *MinioProvider) writeToCache(
+	n name.ServerName,
+	cert *tls.Certificate,
+) {
+	p.mutex.Lock()
+	defer p.mutex.Unlock()
+
+	if p.cache == nil {
+		p.cache = map[string]*minioCacheItem{}
+	}
+
+	item := &minioCacheItem{
+		Certificate: cert,
+		LastSeen:    time.Now(),
+	}
+
+	p.cache[n.Unicode] = item
+}